Project

General

Profile

1 53 aaronmk
#!/usr/bin/env python
2
# Maps one datasource to another, using a map spreadsheet if needed
3
# For outputting an XML file to a PostgreSQL database, use the general format of
4
# http://vegbank.org/vegdocs/xml/vegbank_example_ver1.0.2.xml
5
6
import os.path
7
import sys
8 299 aaronmk
import xml.dom.minidom as minidom
9 53 aaronmk
10 266 aaronmk
sys.path.append(os.path.dirname(__file__)+"/../lib")
11 53 aaronmk
12 344 aaronmk
import exc
13 64 aaronmk
import opts
14 281 aaronmk
import Parser
15 131 aaronmk
import sql
16 715 aaronmk
import strings
17 828 aaronmk
import term
18 310 aaronmk
import util
19 133 aaronmk
import xml_dom
20 86 aaronmk
import xml_func
21 53 aaronmk
22 84 aaronmk
def metadata_value(name):
23 164 aaronmk
    if type(name) == str and name.startswith(':'): return name[1:]
24 84 aaronmk
    else: return None
25
26 53 aaronmk
def main():
27 434 aaronmk
    ex_tracker = exc.ExTracker()
28
29 131 aaronmk
    env_names = []
30
    def usage_err():
31 662 aaronmk
        raise SystemExit('Usage: ' + opts.env_usage(env_names, True)
32
            +' [commit=1] [test=1] [verbose=1] [debug=1] '+sys.argv[0]
33
            +' [map_path...] [<input] [>output]')
34 838 aaronmk
35
    # Get db config from env vars
36
    db_config_names = ['engine', 'host', 'user', 'password', 'database']
37
    def get_db_config(prefix):
38
        return opts.get_env_vars(db_config_names, prefix, env_names)
39
    in_db_config = get_db_config('in')
40
    out_db_config = get_db_config('out')
41
    in_is_db = 'engine' in in_db_config
42
    out_is_db = 'engine' in out_db_config
43
44
    # Get other config from env vars
45
    end = util.cast(int, opts.get_env_var('n', None, env_names))
46
    start = util.cast(int, opts.get_env_var('start', '0', env_names))
47
    if end != None: end += start
48 400 aaronmk
    test = opts.env_flag('test')
49
    commit = not test and opts.env_flag('commit') # never commit in test mode
50 662 aaronmk
    debug = opts.env_flag('debug')
51
    verbose = debug or opts.env_flag('verbose')
52 131 aaronmk
53 838 aaronmk
    # Logging
54 662 aaronmk
    def log(msg, on=verbose):
55
        if on: sys.stderr.write(msg)
56 663 aaronmk
    def log_start(action, on=verbose): log(action+'...', on)
57
    def log_done(on=verbose): log('Done\n', on)
58 662 aaronmk
59 53 aaronmk
    # Parse args
60 510 aaronmk
    map_paths = sys.argv[1:]
61 512 aaronmk
    if map_paths == []:
62
        if in_is_db or not out_is_db: usage_err()
63
        else: map_paths = [None]
64 53 aaronmk
65 646 aaronmk
    def connect_db(db_config):
66 662 aaronmk
        log_start('Connecting to '+sql.db_config_str(db_config))
67 646 aaronmk
        db = sql.connect(db_config)
68 662 aaronmk
        log_done()
69 646 aaronmk
        return db
70
71 751 aaronmk
    out_is_xml_ref = [False]
72
73 838 aaronmk
    def process_input(root, row_ready, map_path):
74 512 aaronmk
        '''Inputs datasource to XML tree, mapping if needed'''
75
        # Load map header
76
        in_is_xpaths = True
77 751 aaronmk
        out_is_xpaths = True
78 512 aaronmk
        out_label = None
79
        if map_path != None:
80
            import copy
81
            import csv
82
83
            import xpath
84
85
            metadata = []
86
            mappings = []
87
            stream = open(map_path, 'rb')
88
            reader = csv.reader(stream)
89
            in_label, out_label = reader.next()[:2]
90
            def split_col_name(name):
91
                name, sep, root = name.partition(':')
92
                return name, sep != '', root
93
            in_label, in_is_xpaths, in_root = split_col_name(in_label)
94
            out_label, out_is_xpaths, out_root = split_col_name(out_label)
95
            has_types = out_root.startswith('/*s/') # outer elements are types
96
            for row in reader:
97
                in_, out = row[:2]
98
                if out != '':
99
                    if out_is_xpaths: out = xpath.parse(out_root+out)
100
                    mappings.append((in_, out))
101
            stream.close()
102
103
            root.ownerDocument.documentElement.tagName = out_label
104
        in_is_xml = in_is_xpaths and not in_is_db
105 751 aaronmk
        out_is_xml_ref[0] = out_is_xpaths and not out_is_db
106 56 aaronmk
107 512 aaronmk
        if in_is_xml:
108
            doc0 = minidom.parse(sys.stdin)
109
            if out_label == None: out_label = doc0.documentElement.tagName
110 53 aaronmk
111 838 aaronmk
        def process_rows(process_row, rows):
112
            '''Processes input rows
113
            @param process_row(in_row, i)
114 297 aaronmk
            '''
115 838 aaronmk
            i = -1 # in case for loop does not execute
116 314 aaronmk
            for i, row in enumerate(rows):
117 838 aaronmk
                if i < start: continue
118
                if end != None and i >= end: break
119
                process_row(row, i)
120
                row_ready(i, row)
121
            sys.stderr.write('Processed '+str(i-start+1)+' input rows\n')
122
123
        def map_rows(get_value, rows):
124
            '''Maps input rows
125
            @param get_value(in_, row):str
126
            '''
127
            def process_row(row, i):
128 316 aaronmk
                row_id = str(i)
129
                for in_, out in mappings:
130
                    value = metadata_value(in_)
131 662 aaronmk
                    if value == None:
132
                        log_start('Getting '+str(in_), debug)
133
                        value = get_value(in_, row)
134
                        log_done(debug)
135 715 aaronmk
                    if value != None: xpath.put_obj(root, out, row_id,
136
                        has_types, strings.cleanup(value))
137 838 aaronmk
            process_rows(process_row, rows)
138 297 aaronmk
139 310 aaronmk
        if map_path == None:
140
            iter_ = xml_dom.NodeElemIter(doc0.documentElement)
141
            util.skip(iter_, xml_dom.is_text) # skip metadata
142 838 aaronmk
            process_rows(lambda row, i: root.appendChild(row), iter_)
143 309 aaronmk
        elif in_is_db:
144 130 aaronmk
            assert in_is_xpaths
145 126 aaronmk
146 117 aaronmk
            import db_xml
147
148 161 aaronmk
            in_root_xml = xpath.path2xml(in_root)
149 164 aaronmk
            for i, mapping in enumerate(mappings):
150
                in_, out = mapping
151
                if metadata_value(in_) == None:
152 168 aaronmk
                    mappings[i] = (xpath.path2xml(in_root+'/'+in_), out)
153 126 aaronmk
154 646 aaronmk
            in_db = connect_db(in_db_config)
155 133 aaronmk
            in_pkeys = {}
156 297 aaronmk
            def get_value(in_, row):
157 167 aaronmk
                pkey, = row
158 297 aaronmk
                in_ = in_.cloneNode(True) # don't modify orig value!
159
                xml_dom.set_id(xpath.get(in_, in_root), pkey)
160
                value = sql.value_or_none(db_xml.get(in_db, in_, in_pkeys))
161
                if value != None: return str(value)
162
                else: return None
163 838 aaronmk
            map_rows(get_value, sql.rows(db_xml.get(in_db, in_root_xml,
164
                in_pkeys, end)))
165 117 aaronmk
            in_db.close()
166 161 aaronmk
        elif in_is_xml:
167 297 aaronmk
            def get_value(in_, row):
168
                node = xpath.get(row, in_)
169
                if node != None: return xml_dom.value(node)
170
                else: return None
171
            row0 = xpath.get(doc0.documentElement, in_root)
172 838 aaronmk
            map_rows(get_value, xml_dom.NodeElemIter(row0.parentNode))
173 56 aaronmk
        else: # input is CSV
174 133 aaronmk
            map_ = dict(mappings)
175 59 aaronmk
            reader = csv.reader(sys.stdin)
176 84 aaronmk
            cols = reader.next()
177 162 aaronmk
            col_idxs = dict([(value, idx) for idx, value in enumerate(cols)])
178 164 aaronmk
            for i, mapping in enumerate(mappings):
179
                in_, out = mapping
180
                if metadata_value(in_) == None:
181
                    try: mappings[i] = (col_idxs[in_], out)
182
                    except KeyError: pass
183 162 aaronmk
184 297 aaronmk
            def get_value(in_, row):
185
                value = row[in_]
186
                if value != '': return value
187
                else: return None
188 838 aaronmk
            map_rows(get_value, reader)
189 53 aaronmk
190 838 aaronmk
    def process_inputs(root, row_ready):
191
        for map_path in map_paths: process_input(root, row_ready, map_path)
192 512 aaronmk
193 53 aaronmk
    # Output XML tree
194 512 aaronmk
    doc = xml_dom.create_doc()
195 316 aaronmk
    root = doc.documentElement
196 130 aaronmk
    if out_is_db:
197 53 aaronmk
        import db_xml
198
199 646 aaronmk
        out_db = connect_db(out_db_config)
200 310 aaronmk
        out_pkeys = {}
201 53 aaronmk
        try:
202 400 aaronmk
            if test: sql.empty_db(out_db)
203 53 aaronmk
            row_ct_ref = [0]
204 449 aaronmk
205 838 aaronmk
            def row_ready(row_num, input_row):
206 452 aaronmk
                def on_error(e):
207 835 aaronmk
                    exc.add_msg(e, term.emph('row #:')+' '+str(row_num))
208 828 aaronmk
                    exc.add_msg(e, term.emph('input row:')+'\n'+str(input_row))
209
                    exc.add_msg(e, term.emph('output row:')+'\n'+str(root))
210 452 aaronmk
                    ex_tracker.track(e)
211
212 449 aaronmk
                xml_func.process(root, on_error)
213 442 aaronmk
                if not xml_dom.is_empty(root):
214
                    assert xml_dom.has_one_child(root)
215
                    try:
216
                        sql.with_savepoint(out_db, lambda: db_xml.put(out_db,
217 462 aaronmk
                            root.firstChild, out_pkeys, row_ct_ref, on_error))
218 442 aaronmk
                        if commit: out_db.commit()
219 449 aaronmk
                    except sql.DatabaseErrors, e: on_error(e)
220 316 aaronmk
                root.clear()
221 449 aaronmk
222 838 aaronmk
            process_inputs(root, row_ready)
223 717 aaronmk
            sys.stdout.write('Inserted '+str(row_ct_ref[0])+
224 460 aaronmk
                ' new rows into database\n')
225 53 aaronmk
        finally:
226 133 aaronmk
            out_db.rollback()
227
            out_db.close()
228 751 aaronmk
    else:
229 759 aaronmk
        def on_error(e): ex_tracker.track(e)
230 838 aaronmk
        def row_ready(row_num, input_row): pass
231
        process_inputs(root, row_ready)
232 759 aaronmk
        xml_func.process(root, on_error)
233 751 aaronmk
        if out_is_xml_ref[0]:
234
            doc.writexml(sys.stdout, **xml_dom.prettyxml_config)
235
        else: # output is CSV
236
            raise NotImplementedError('CSV output not supported yet')
237 53 aaronmk
238 846 aaronmk
if __name__ == '__main__':
239
    try: main()
240
    except Parser.SyntaxException, e: raise SystemExit(str(e))