Project

General

Profile

1 73 aaronmk
# String manipulation
2
3 1295 aaronmk
import codecs
4 861 aaronmk
import re
5
6 1232 aaronmk
import util
7
8 4028 aaronmk
##### Sentinel values
9
10
class NonInternedStr(str):
11
    '''Each instance is unique and does not compare identical with `is`.'''
12
    pass
13
14
none_str = NonInternedStr()
15 4032 aaronmk
isspace_none_str = '\r\n\t\v' # clone-safe, but must be compared with ==
16 4028 aaronmk
17 1622 aaronmk
##### Parsing
18
19 4115 aaronmk
def raw_extra_len(str_):
20
    '''Calculates the difference between the character length and byte length of
21
    a string. These lengths will differ for Unicode strings containing
22
    multi-byte characters.'''
23
    return len(to_raw_str(str_))-len(str_)
24
25 3749 aaronmk
def concat(str0, str1, max_len):
26 4116 aaronmk
    '''Concatenates two strings, ensuring that the combined byte length is no
27
    greater than the provided length limit.
28
    Note that if the combined length is longer than max_len, the truncated
29
    length may sometimes be shorter than max_len if there are multi-byte
30
    characters after str0's truncation point. Thus, you *cannot determine if the
31
    string was truncated* by checking if the new length equals max_len.
32
    '''
33
    # Use raw_extra_len() because Unicode characters can be multi-byte, and
34
    # length limits often apply to the byte length, not the character length.
35
    max_len -= raw_extra_len(str0)+raw_extra_len(str1)
36 5299 aaronmk
    str0_new_len = max_len - len(str1)
37
    if str0_new_len < 0: return str1[:max_len] # str1 too long
38
    return str0[:str0_new_len]+str1
39 2932 aaronmk
40 5814 aaronmk
def join(sep, strs):
41
    '''Returns None if .join() returns ""'''
42
    return util.none_if(sep.join(strs), u'')
43
44 1622 aaronmk
def split(sep, str_):
45
    '''Returns [] if str_ == ""'''
46
    if str_ == '': return []
47
    else: return str_.split(sep)
48
49 4937 aaronmk
def remove_prefix(prefix, str_, removed_ref=None, require=False):
50 2221 aaronmk
    if removed_ref == None: removed_ref = [False]
51
52
    removed_ref[0] = str_.startswith(prefix)
53
    if removed_ref[0]: return str_[len(prefix):]
54 4937 aaronmk
    elif require: raise Exception(urepr(str_)+' needs '+urepr(prefix)+' prefix')
55 1622 aaronmk
    else: return str_
56
57 1680 aaronmk
def remove_prefixes(prefixes, str_):
58
    for prefix in prefixes: str_ = remove_prefix(prefix, str_)
59
    return str_
60
61
def with_prefixes(prefixes, str_): return (p+str_ for p in prefixes)
62
63 4937 aaronmk
def remove_suffix(suffix, str_, removed_ref=None, require=False):
64 2221 aaronmk
    if removed_ref == None: removed_ref = [False]
65
66
    removed_ref[0] = str_.endswith(suffix)
67
    if removed_ref[0]: return str_[:-len(suffix)]
68 4937 aaronmk
    elif require: raise Exception(urepr(str_)+' needs '+urepr(suffix)+' suffix')
69 1622 aaronmk
    else: return str_
70
71 5432 aaronmk
def find_any(haystack, needles):
72 1959 aaronmk
    for needle in needles:
73 5432 aaronmk
        if haystack.find(needle) >= 0: return needle
74
    return None
75 1959 aaronmk
76 1622 aaronmk
def overlaps(str0, str1): return str0.find(str1) >= 0 or str1.find(str0) >= 0
77
78 5147 aaronmk
def flip_map(map_):
79
    '''
80
    For use with replace_all(), replace_all_re().
81
    @param map_ [(from_, to), ...]
82
    @return [(to, from_), ...]
83
    '''
84
    return [(to, from_) for from_, to in map_]
85
86
def replace_all(map_, str_):
87
    '''
88
    @param map_ [(from_, to), ...]
89
    '''
90
    for from_, to in map_: str_ = str_.replace(from_, to)
91
    return str_
92
93
def replace_all_re(map_, str_):
94
    '''
95
    @param map_ [(from_, to), ...]
96
    '''
97
    for from_, to in map_: str_ = re.sub(from_, to, str_)
98
    return str_
99
100 2761 aaronmk
##### Escaping
101
102 5142 aaronmk
def esc_quotes(str_, quote='"', esc='\\', quote_esc=None):
103
    if quote_esc == None: quote_esc = esc+quote
104
105
    if esc != quote: str_ = str_.replace(esc, esc+esc)
106
    str_ = str_.replace(quote, quote_esc)
107
    return str_
108
109 5161 aaronmk
json_encode_map = [
110
    ('\n', r'\n'),
111
    ('\r', r'\r'),
112
]
113 5143 aaronmk
114 5161 aaronmk
def json_encode(str_):
115
    return replace_all(json_encode_map, esc_quotes(str_, '"'))
116
117 2761 aaronmk
def esc_for_mogrify(query):
118
    '''Escapes a query right before being passed to a mogrifying function.'''
119
    return query.replace('%', '%%')
120
121 5148 aaronmk
def regexp_repl_esc(str_): return lambda m: str_
122
123 1401 aaronmk
##### Unicode
124
125 2882 aaronmk
def to_raw_str(str_):
126
    if isinstance(str_, unicode): str_ = str_.encode('utf_8')
127
    return str_
128
129 1295 aaronmk
unicode_reader = codecs.getreader('utf_8')
130
131 73 aaronmk
def to_unicode(str_):
132
    if isinstance(str_, unicode): return str_
133
    encodings = ['utf_8', 'latin_1']
134
    for encoding in encodings:
135
        try: return unicode(str_, encoding)
136
        except UnicodeDecodeError, e: pass
137
    raise AssertionError(encoding+' is not a catch-all encoding')
138 340 aaronmk
139 3748 aaronmk
def ustr(value):
140 1232 aaronmk
    '''Like built-in str() but converts to unicode object'''
141 3748 aaronmk
    if util.is_str(value): str_ = value # already a string
142
    elif hasattr(value, '__str__'): str_ = value.__str__()
143
    else: str_ = str(value)
144
    return to_unicode(str_)
145 1232 aaronmk
146 3747 aaronmk
def urepr(value):
147
    '''Like built-in repr() but converts to unicode object'''
148
    if hasattr(value, '__repr__'): str_ = value.__repr__()
149
    else: str_ = repr(value)
150
    return to_unicode(str_)
151
152 2502 aaronmk
def repr_no_u(value):
153
    '''Like built-in repr() but removes the "u" in `u'...'`'''
154 3747 aaronmk
    return re.sub(r"^u(?=')", r'', urepr(value))
155 2502 aaronmk
156 1401 aaronmk
##### Line endings
157
158 1622 aaronmk
def extract_line_ending(line):
159
    '''@return tuple (contents, ending)'''
160
    contents = remove_suffix('\r', remove_suffix('\n', line))
161
    return (contents, line[len(contents):])
162 714 aaronmk
163 1622 aaronmk
def remove_line_ending(line): return extract_line_ending(line)[0]
164
165
def ensure_newl(str_): return remove_line_ending(str_)+'\n'
166
167 856 aaronmk
def is_multiline(str_):
168
    newl_idx = str_.find('\n')
169
    return newl_idx >= 0 and newl_idx != len(str_)-1 # has newline before end
170
171 860 aaronmk
def remove_extra_newl(str_):
172 856 aaronmk
    if is_multiline(str_): return str_
173 2480 aaronmk
    else: return str_.rstrip('\n')
174 856 aaronmk
175 714 aaronmk
def std_newl(str_): return str_.replace('\r\n', '\n').replace('\r', '\n')
176
177 3255 aaronmk
def join_lines(lines): return ''.join((l+'\n' for l in lines))
178
179 1401 aaronmk
##### Whitespace
180
181 714 aaronmk
def cleanup(str_): return std_newl(str_.strip())
182 861 aaronmk
183 1364 aaronmk
def single_space(str_): return re.sub(r' {2,}', r' ', str_.strip())
184
185 861 aaronmk
def one_line(str_): return re.sub(r'\n *', r' ', cleanup(str_))
186 1761 aaronmk
187
##### Control characters
188
189
def is_ctrl(char):
190
    '''Whether char is a (non-printable) control character'''
191
    return ord(char) < 32 and not char.isspace()
192
193
def strip_ctrl(str_):
194
    '''Strips (non-printable) control characters'''
195
    return ''.join(filter(lambda c: not is_ctrl(c), str_))
196 2471 aaronmk
197 3172 aaronmk
##### Text
198
199
def first_word(str_): return str_.partition(' ')[0]
200
201 2471 aaronmk
##### Formatting
202
203 3466 aaronmk
def indent(str_, level=1, indent_str='    '):
204
    indent_str *= level
205
    return ('\n'.join((indent_str+l for l in str_.rstrip().split('\n'))))+'\n'
206
207 2477 aaronmk
def as_tt(str_): return '@'+str_+'@'
208
209 2475 aaronmk
def as_code(str_, lang=None, multiline=True):
210 2471 aaronmk
    '''Wraps a string in Redmine tags to syntax-highlight it.'''
211 2480 aaronmk
    str_ = '\n'+str_.rstrip('\n')+'\n'
212 2471 aaronmk
    if lang != None: str_ = '<code class="'+lang+'">'+str_+'</code>'
213 2475 aaronmk
    if multiline: str_ = '<pre>'+str_+'</pre>'
214
    return str_
215 2477 aaronmk
216 2504 aaronmk
def as_inline_table(dict_, key_label='Output', value_label='Input', ustr=ustr):
217 2477 aaronmk
    '''Wraps a dict in Redmine tags to format it as a table.'''
218
    str_ = ''
219 2481 aaronmk
    def row(entry): return (': '.join(entry))+'\n'
220 2477 aaronmk
    str_ += row([key_label, value_label])
221 2481 aaronmk
    for entry in dict_.iteritems(): str_ += row([ustr(v) for v in entry])
222
    return '<pre>\n'+str_+'</pre>'
223 2482 aaronmk
224 2504 aaronmk
def as_table(dict_, key_label='Output', value_label='Input', ustr=ustr):
225 2482 aaronmk
    '''Wraps a dict in Redmine tags to format it as a table.'''
226
    str_ = ''
227
    def row(entry): return ('|'.join(['']+entry+['']))+'\n'# '' for outer border
228
    str_ += row([key_label, value_label])
229
    for entry in dict_.iteritems(): str_ += row([as_tt(ustr(v)) for v in entry])
230
    return '\n'+str_+' ' # space protects last \n so blank line ends table