Project

General

Profile

1
# String manipulation
2

    
3
import codecs
4
import re
5

    
6
import util
7

    
8
##### Sentinel values
9

    
10
class NonInternedStr(str):
11
    '''Each instance is unique and does not compare identical with `is`.'''
12
    pass
13

    
14
none_str = NonInternedStr()
15
isspace_none_str = '\r\n\t\v' # clone-safe, but must be compared with ==
16

    
17
##### Parsing
18

    
19
def raw_extra_len(str_):
20
    '''Calculates the difference between the character length and byte length of
21
    a string. These lengths will differ for Unicode strings containing
22
    multi-byte characters.'''
23
    return len(to_raw_str(str_))-len(str_)
24

    
25
def concat(str0, str1, max_len):
26
    '''Concatenates two strings, ensuring that the combined byte length is no
27
    greater than the provided length limit.
28
    Note that if the combined length is longer than max_len, the truncated
29
    length may sometimes be shorter than max_len if there are multi-byte
30
    characters after str0's truncation point. Thus, you *cannot determine if the
31
    string was truncated* by checking if the new length equals max_len.
32
    '''
33
    # Use raw_extra_len() because Unicode characters can be multi-byte, and
34
    # length limits often apply to the byte length, not the character length.
35
    max_len -= raw_extra_len(str0)+raw_extra_len(str1)
36
    str0_new_len = max_len - len(str1)
37
    if str0_new_len < 0: return str1[:max_len] # str1 too long
38
    return str0[:str0_new_len]+str1
39

    
40
def join(sep, strs):
41
    '''Returns None if .join() returns ""'''
42
    return util.none_if(sep.join(strs), u'')
43

    
44
def split(sep, str_):
45
    '''Returns [] if str_ == ""'''
46
    if str_ == '': return []
47
    else: return str_.split(sep)
48

    
49
def remove_prefix(prefix, str_, removed_ref=None, require=False):
50
    if removed_ref == None: removed_ref = [False]
51
    
52
    removed_ref[0] = str_.startswith(prefix)
53
    if removed_ref[0]: return str_[len(prefix):]
54
    elif require: raise Exception(urepr(str_)+' needs '+urepr(prefix)+' prefix')
55
    else: return str_
56

    
57
def remove_prefixes(prefixes, str_):
58
    for prefix in prefixes: str_ = remove_prefix(prefix, str_)
59
    return str_
60

    
61
def with_prefixes(prefixes, str_): return (p+str_ for p in prefixes)
62

    
63
def remove_suffix(suffix, str_, removed_ref=None, require=False):
64
    if removed_ref == None: removed_ref = [False]
65
    
66
    removed_ref[0] = str_.endswith(suffix)
67
    if removed_ref[0]: return str_[:-len(suffix)]
68
    elif require: raise Exception(urepr(str_)+' needs '+urepr(suffix)+' suffix')
69
    else: return str_
70

    
71
def find_any(haystack, needles):
72
    for needle in needles:
73
        if haystack.find(needle) >= 0: return needle
74
    return None
75

    
76
def overlaps(str0, str1): return str0.find(str1) >= 0 or str1.find(str0) >= 0
77

    
78
def flip_map(map_):
79
    '''
80
    For use with replace_all(), replace_all_re().
81
    @param map_ [(from_, to), ...]
82
    @return [(to, from_), ...]
83
    '''
84
    return [(to, from_) for from_, to in map_]
85

    
86
def replace_all(map_, str_):
87
    '''
88
    @param map_ [(from_, to), ...]
89
    '''
90
    for from_, to in map_: str_ = str_.replace(from_, to)
91
    return str_
92

    
93
def replace_all_re(map_, str_):
94
    '''
95
    @param map_ [(from_, to), ...]
96
    '''
97
    for from_, to in map_: str_ = re.sub(from_, to, str_)
98
    return str_
99

    
100
##### Escaping
101

    
102
def esc_quotes(str_, quote='"', esc='\\', quote_esc=None):
103
    if quote_esc == None: quote_esc = esc+quote
104
    
105
    if esc != quote: str_ = str_.replace(esc, esc+esc)
106
    str_ = str_.replace(quote, quote_esc)
107
    return str_
108

    
109
json_encode_map = [
110
    ('\n', r'\n'),
111
    ('\r', r'\r'),
112
]
113

    
114
def json_encode(str_):
115
    return replace_all(json_encode_map, esc_quotes(str_, '"'))
116

    
117
def esc_for_mogrify(query):
118
    '''Escapes a query right before being passed to a mogrifying function.'''
119
    return query.replace('%', '%%')
120

    
121
def regexp_repl_esc(str_): return lambda m: str_
122

    
123
##### Unicode
124

    
125
def to_raw_str(str_):
126
    if isinstance(str_, unicode): str_ = str_.encode('utf_8')
127
    return str_
128

    
129
unicode_reader = codecs.getreader('utf_8')
130

    
131
def to_unicode(str_):
132
    if isinstance(str_, unicode): return str_
133
    encodings = ['utf_8', 'latin_1']
134
    for encoding in encodings:
135
        try: return unicode(str_, encoding)
136
        except UnicodeDecodeError, e: pass
137
    raise AssertionError(encoding+' is not a catch-all encoding')
138

    
139
def ustr(value):
140
    '''Like built-in str() but converts to unicode object'''
141
    if util.is_str(value): str_ = value # already a string
142
    elif hasattr(value, '__str__'): str_ = value.__str__()
143
    else: str_ = str(value)
144
    return to_unicode(str_)
145

    
146
def urepr(value):
147
    '''Like built-in repr() but converts to unicode object'''
148
    if hasattr(value, '__repr__'): str_ = value.__repr__()
149
    else: str_ = repr(value)
150
    return to_unicode(str_)
151

    
152
def repr_no_u(value):
153
    '''Like built-in repr() but removes the "u" in `u'...'`'''
154
    return re.sub(r"^u(?=')", r'', urepr(value))
155

    
156
##### Line endings
157

    
158
def extract_line_ending(line):
159
    '''@return tuple (contents, ending)'''
160
    contents = remove_suffix('\r', remove_suffix('\n', line))
161
    return (contents, line[len(contents):])
162

    
163
def remove_line_ending(line): return extract_line_ending(line)[0]
164

    
165
def ensure_newl(str_): return remove_line_ending(str_)+'\n'
166

    
167
def is_multiline(str_):
168
    newl_idx = str_.find('\n')
169
    return newl_idx >= 0 and newl_idx != len(str_)-1 # has newline before end
170

    
171
def remove_extra_newl(str_):
172
    if is_multiline(str_): return str_
173
    else: return str_.rstrip('\n')
174

    
175
def std_newl(str_): return str_.replace('\r\n', '\n').replace('\r', '\n')
176

    
177
def join_lines(lines): return ''.join((l+'\n' for l in lines))
178

    
179
##### Whitespace
180

    
181
def cleanup(str_): return std_newl(str_.strip())
182

    
183
def single_space(str_): return re.sub(r' {2,}', r' ', str_.strip())
184

    
185
def one_line(str_): return re.sub(r'\n *', r' ', cleanup(str_))
186

    
187
##### Control characters
188

    
189
def is_ctrl(char):
190
    '''Whether char is a (non-printable) control character'''
191
    return ord(char) < 32 and not char.isspace()
192

    
193
def strip_ctrl(str_):
194
    '''Strips (non-printable) control characters'''
195
    return ''.join(filter(lambda c: not is_ctrl(c), str_))
196

    
197
##### Text
198

    
199
def first_word(str_): return str_.partition(' ')[0]
200

    
201
##### Formatting
202

    
203
def indent(str_, level=1, indent_str='    '):
204
    indent_str *= level
205
    return ('\n'.join((indent_str+l for l in str_.rstrip().split('\n'))))+'\n'
206

    
207
def as_tt(str_): return '@'+str_+'@'
208

    
209
def as_code(str_, lang=None, multiline=True):
210
    '''Wraps a string in Redmine tags to syntax-highlight it.'''
211
    str_ = '\n'+str_.rstrip('\n')+'\n'
212
    if lang != None: str_ = '<code class="'+lang+'">'+str_+'</code>'
213
    if multiline: str_ = '<pre>'+str_+'</pre>'
214
    return str_
215

    
216
def as_inline_table(dict_, key_label='Output', value_label='Input', ustr=ustr):
217
    '''Wraps a dict in Redmine tags to format it as a table.'''
218
    str_ = ''
219
    def row(entry): return (': '.join(entry))+'\n'
220
    str_ += row([key_label, value_label])
221
    for entry in dict_.iteritems(): str_ += row([ustr(v) for v in entry])
222
    return '<pre>\n'+str_+'</pre>'
223

    
224
def as_table(dict_, key_label='Output', value_label='Input', ustr=ustr):
225
    '''Wraps a dict in Redmine tags to format it as a table.'''
226
    str_ = ''
227
    def row(entry): return ('|'.join(['']+entry+['']))+'\n'# '' for outer border
228
    str_ += row([key_label, value_label])
229
    for entry in dict_.iteritems(): str_ += row([as_tt(ustr(v)) for v in entry])
230
    return '\n'+str_+' ' # space protects last \n so blank line ends table
(35-35/45)