/lib/strings.py - Annotate - BIEN 3 - NCEAS Projects

root/lib/strings.py @ 5190

-            aaronmk
+# String manipulation
-            aaronmk
+import codecs
-            aaronmk
+import re
-            aaronmk
+import util
-            aaronmk
+##### Sentinel values
 class NonInternedStr(str):
     '''Each instance is unique and does not compare identical with `is`.'''
     pass
 none_str = NonInternedStr()
-            aaronmk
+isspace_none_str = '\r\n\t\v' # clone-safe, but must be compared with ==
             aaronmk
-            aaronmk
+##### Parsing
-            aaronmk
+def raw_extra_len(str_):
     '''Calculates the difference between the character length and byte length of
     a string. These lengths will differ for Unicode strings containing
     multi-byte characters.'''
     return len(to_raw_str(str_))-len(str_)
-            aaronmk
+def concat(str0, str1, max_len):
-            aaronmk
+    '''Concatenates two strings, ensuring that the combined byte length is no
     greater than the provided length limit.
     Note that if the combined length is longer than max_len, the truncated
     length may sometimes be shorter than max_len if there are multi-byte
     characters after str0's truncation point. Thus, you *cannot determine if the
     string was truncated* by checking if the new length equals max_len.
     '''
     # Use raw_extra_len() because Unicode characters can be multi-byte, and
     # length limits often apply to the byte length, not the character length.
     max_len -= raw_extra_len(str0)+raw_extra_len(str1)
     return str0[:max_len-len(str1)]+str1
             aaronmk
-            aaronmk
+def split(sep, str_):
     '''Returns [] if str_ == ""'''
     if str_ == '': return []
     else: return str_.split(sep)
-            aaronmk
+def remove_prefix(prefix, str_, removed_ref=None, require=False):
-            aaronmk
+    if removed_ref == None: removed_ref = [False]
     removed_ref[0] = str_.startswith(prefix)
     if removed_ref[0]: return str_[len(prefix):]
-            aaronmk
+    elif require: raise Exception(urepr(str_)+' needs '+urepr(prefix)+' prefix')
-            aaronmk
+    else: return str_
-            aaronmk
+def remove_prefixes(prefixes, str_):
     for prefix in prefixes: str_ = remove_prefix(prefix, str_)
     return str_
 def with_prefixes(prefixes, str_): return (p+str_ for p in prefixes)
-            aaronmk
+def remove_suffix(suffix, str_, removed_ref=None, require=False):
-            aaronmk
+    if removed_ref == None: removed_ref = [False]
     removed_ref[0] = str_.endswith(suffix)
     if removed_ref[0]: return str_[:-len(suffix)]
-            aaronmk
+    elif require: raise Exception(urepr(str_)+' needs '+urepr(suffix)+' suffix')
-            aaronmk
+    else: return str_
-            aaronmk
+def contains_any(haystack, needles):
     for needle in needles:
         if haystack.find(needle) >= 0: return True
     return False
-            aaronmk
+def overlaps(str0, str1): return str0.find(str1) >= 0 or str1.find(str0) >= 0
-            aaronmk
+def flip_map(map_):
     '''
     For use with replace_all(), replace_all_re().
     @param map_ [(from_, to), ...]
     @return [(to, from_), ...]
     '''
     return [(to, from_) for from_, to in map_]
 def replace_all(map_, str_):
     '''
     @param map_ [(from_, to), ...]
     '''
     for from_, to in map_: str_ = str_.replace(from_, to)
     return str_
 def replace_all_re(map_, str_):
     '''
     @param map_ [(from_, to), ...]
     '''
     for from_, to in map_: str_ = re.sub(from_, to, str_)
     return str_
-            aaronmk
+##### Escaping
-            aaronmk
+def esc_quotes(str_, quote='"', esc='\\', quote_esc=None):
     if quote_esc == None: quote_esc = esc+quote
     if esc != quote: str_ = str_.replace(esc, esc+esc)
     str_ = str_.replace(quote, quote_esc)
     return str_
-            aaronmk
+json_encode_map = [
     ('\n', r'\n'),
     ('\r', r'\r'),
+]
             aaronmk
-            aaronmk
+def json_encode(str_):
     return replace_all(json_encode_map, esc_quotes(str_, '"'))
-            aaronmk
+def esc_for_mogrify(query):
     '''Escapes a query right before being passed to a mogrifying function.'''
     return query.replace('%', '%%')
-            aaronmk
+def regexp_repl_esc(str_): return lambda m: str_
-            aaronmk
+##### Unicode
-            aaronmk
+def to_raw_str(str_):
     if isinstance(str_, unicode): str_ = str_.encode('utf_8')
     return str_
-            aaronmk
+unicode_reader = codecs.getreader('utf_8')
-            aaronmk
+def to_unicode(str_):
     if isinstance(str_, unicode): return str_
     encodings = ['utf_8', 'latin_1']
     for encoding in encodings:
         try: return unicode(str_, encoding)
         except UnicodeDecodeError, e: pass
     raise AssertionError(encoding+' is not a catch-all encoding')
             aaronmk
-            aaronmk
+def ustr(value):
-            aaronmk
+    '''Like built-in str() but converts to unicode object'''
-            aaronmk
+    if util.is_str(value): str_ = value # already a string
     elif hasattr(value, '__str__'): str_ = value.__str__()
     else: str_ = str(value)
     return to_unicode(str_)
             aaronmk
-            aaronmk
+def urepr(value):
     '''Like built-in repr() but converts to unicode object'''
     if hasattr(value, '__repr__'): str_ = value.__repr__()
     else: str_ = repr(value)
     return to_unicode(str_)
-            aaronmk
+def repr_no_u(value):
     '''Like built-in repr() but removes the "u" in `u'...'`'''
-            aaronmk
+    return re.sub(r"^u(?=')", r'', urepr(value))
             aaronmk
-            aaronmk
+##### Line endings
-            aaronmk
+def extract_line_ending(line):
     '''@return tuple (contents, ending)'''
     contents = remove_suffix('\r', remove_suffix('\n', line))
     return (contents, line[len(contents):])
             aaronmk
-            aaronmk
+def remove_line_ending(line): return extract_line_ending(line)[0]
 def ensure_newl(str_): return remove_line_ending(str_)+'\n'
-            aaronmk
+def is_multiline(str_):
     newl_idx = str_.find('\n')
     return newl_idx >= 0 and newl_idx != len(str_)-1 # has newline before end
-            aaronmk
+def remove_extra_newl(str_):
-            aaronmk
+    if is_multiline(str_): return str_
-            aaronmk
+    else: return str_.rstrip('\n')
             aaronmk
-            aaronmk
+def std_newl(str_): return str_.replace('\r\n', '\n').replace('\r', '\n')
-            aaronmk
+def join_lines(lines): return ''.join((l+'\n' for l in lines))
-            aaronmk
+##### Whitespace
-            aaronmk
+def cleanup(str_): return std_newl(str_.strip())
             aaronmk
-            aaronmk
+def single_space(str_): return re.sub(r' {2,}', r' ', str_.strip())
-            aaronmk
+def one_line(str_): return re.sub(r'\n *', r' ', cleanup(str_))
             aaronmk
 ##### Control characters
 def is_ctrl(char):
     '''Whether char is a (non-printable) control character'''
     return ord(char) < 32 and not char.isspace()
 def strip_ctrl(str_):
     '''Strips (non-printable) control characters'''
     return ''.join(filter(lambda c: not is_ctrl(c), str_))
             aaronmk
-            aaronmk
+##### Text
 def first_word(str_): return str_.partition(' ')[0]
-            aaronmk
+##### Formatting
-            aaronmk
+def indent(str_, level=1, indent_str='    '):
     indent_str *= level
     return ('\n'.join((indent_str+l for l in str_.rstrip().split('\n'))))+'\n'
-            aaronmk
+def as_tt(str_): return '@'+str_+'@'
-            aaronmk
+def as_code(str_, lang=None, multiline=True):
-            aaronmk
+    '''Wraps a string in Redmine tags to syntax-highlight it.'''
-            aaronmk
+    str_ = '\n'+str_.rstrip('\n')+'\n'
-            aaronmk
+    if lang != None: str_ = '<code class="'+lang+'">'+str_+'</code>'
-            aaronmk
+    if multiline: str_ = '<pre>'+str_+'</pre>'
     return str_
             aaronmk
-            aaronmk
+def as_inline_table(dict_, key_label='Output', value_label='Input', ustr=ustr):
-            aaronmk
+    '''Wraps a dict in Redmine tags to format it as a table.'''
     str_ = ''
-            aaronmk
+    def row(entry): return (': '.join(entry))+'\n'
-            aaronmk
+    str_ += row([key_label, value_label])
-            aaronmk
+    for entry in dict_.iteritems(): str_ += row([ustr(v) for v in entry])
     return '<pre>\n'+str_+'</pre>'
             aaronmk
-            aaronmk
+def as_table(dict_, key_label='Output', value_label='Input', ustr=ustr):
-            aaronmk
+    '''Wraps a dict in Redmine tags to format it as a table.'''
     str_ = ''
     def row(entry): return ('|'.join(['']+entry+['']))+'\n'# '' for outer border
     str_ += row([key_label, value_label])
     for entry in dict_.iteritems(): str_ += row([as_tt(ustr(v)) for v in entry])
     return '\n'+str_+' ' # space protects last \n so blank line ends table

Project

General

Profile