Project

General

Profile

1 205 aaronmk
#!/usr/bin/env python
2
# Modifies a map spreadsheet A->B or any file using a replacements spreadsheet
3
# A->C or B->C
4
5
import csv
6 9785 aaronmk
import HTMLParser
7 1705 aaronmk
import os.path
8 205 aaronmk
import re
9
import sys
10
11 1705 aaronmk
sys.path.append(os.path.dirname(__file__)+"/../lib")
12
13
import maps
14 4357 aaronmk
import opts
15 9784 aaronmk
import strings
16 1705 aaronmk
17 10348 aaronmk
quote_re = '[\'"`]'
18 12746 aaronmk
excluded_prefix_re = (
19 12897 aaronmk
     '(?<![a-z] )' #not if it's a word in a sentence (uppercase SQL keywords OK)
20 12750 aaronmk
    +'(?<!-)' # not if it's part of a '-'-separated identifier
21 12749 aaronmk
    +'(?<!\*)' # don't double leading *
22
    )
23
excluded_suffix_re = (
24 12897 aaronmk
     '(?! [a-z])' # not if it's a word in a sentence (uppercase SQL keywords OK)
25 12750 aaronmk
    +'(?!-)' # not if it's part of a '-'-separated identifier
26 12749 aaronmk
    )
27 10348 aaronmk
28 9785 aaronmk
def unescape_html(str_): return HTMLParser.HTMLParser().unescape(str_)
29
30
def repl_unescape_html(match): return unescape_html(match.group(0))
31
32 205 aaronmk
def main():
33 4357 aaronmk
    env_names = []
34
    def usage_err():
35
        raise SystemExit('Usage: '+opts.env_usage(env_names, True)+' '
36
            +sys.argv[0]+' <map repl [col_num] [| '+sys.argv[0]
37
            +' repl_1 [col_num_1]]... >new_map')
38
39 205 aaronmk
    col_num = None
40
    try: col_num = sys.argv[2]
41 211 aaronmk
    except IndexError: pass
42 205 aaronmk
    if col_num != None: col_num = int(col_num) # 0-based
43 12914 aaronmk
    # whether all patterns are plain text
44
    # defaults to on if matching entire cells in a spreadsheet (w/ col_num)
45
    text = opts.env_flag('text', col_num != None, env_names)
46 205 aaronmk
47 12914 aaronmk
    try: _prog_name, repl_path = sys.argv[:2]
48
    except ValueError: usage_err()
49
50 205 aaronmk
    # Get replacements
51
    repls = []
52
    stream = open(repl_path, 'rb')
53
    reader = csv.reader(stream)
54 8740 aaronmk
    reader.next() # skip header
55 205 aaronmk
    for row in reader:
56 210 aaronmk
        in_, out = row[:2]
57 216 aaronmk
        if in_ != '':
58 10348 aaronmk
            is_word = re.match(r'^\w+$', in_)
59
            if text or is_word: # match as whole-word text (like SQL identifier)
60 13076 aaronmk
                # this can also be done in Postgres with expression substitution
61
                # (wiki.vegpath.org/Postgres_queries#expression-substitution)
62 13077 aaronmk
                # this is a generalization of lib/sql_gen.py map_expr() to work
63
                # on entire source files
64 10348 aaronmk
                in_str_re = re.escape(in_)
65
                q = quote_re
66
                in_ = '(?<='+q+')'+in_str_re+'(?='+q+')' # require quotes
67 12742 aaronmk
                if is_word: # also match with quotes optional
68 12748 aaronmk
                    # don't try to match word w/ suffix, because there are cases
69
                    # where a mapping adds a suffix which would cause the same
70
                    # replacement to be performed repeatedly
71 12754 aaronmk
                    in_word_re = r'\b'+in_str_re+r'\b'
72
73
                    # only use excluded_prefix_re/excluded_suffix_re in text
74
                    # mode (used in renaming columns in SQL scripts), to prevent
75
                    # the special coding for column renames from also affecting
76
                    # regular regexp/word replacements
77
                    if text: in_word_re = (excluded_prefix_re+in_word_re
78 12749 aaronmk
                        +excluded_suffix_re)
79 12754 aaronmk
80 10348 aaronmk
                    in_ = '(?:'+in_+'|'+in_word_re+')'
81 6713 aaronmk
            repls.append((r'(?m)'+in_, out))
82 205 aaronmk
    stream.close()
83
    def repl_all(str_):
84 9784 aaronmk
        str_ = strings.ustr(str_)
85 9785 aaronmk
        for repl, with_ in repls:
86
            if with_ == 'unescape_html()': with_ = repl_unescape_html
87
            str_ = re.sub(repl, with_, str_)
88 205 aaronmk
        return str_
89
90
    # Modify map or file
91
    if col_num != None:
92
        reader = csv.reader(sys.stdin)
93
        writer = csv.writer(sys.stdout)
94
        cols = reader.next()
95
        writer.writerow(cols)
96
        for row in reader:
97
            row[col_num] = repl_all(row[col_num])
98
            writer.writerow(row)
99 9784 aaronmk
    else: sys.stdout.write(strings.to_raw_str(repl_all(sys.stdin.read())))
100 205 aaronmk
101
main()