Project

General

Profile

1 53 aaronmk
#!/usr/bin/env python
2
# Maps one datasource to another, using a map spreadsheet if needed
3
# For outputting an XML file to a PostgreSQL database, use the general format of
4
# http://vegbank.org/vegdocs/xml/vegbank_example_ver1.0.2.xml
5
6
import os.path
7
import sys
8 299 aaronmk
import xml.dom.minidom as minidom
9 53 aaronmk
10 266 aaronmk
sys.path.append(os.path.dirname(__file__)+"/../lib")
11 53 aaronmk
12 344 aaronmk
import exc
13 64 aaronmk
import opts
14 281 aaronmk
import Parser
15 131 aaronmk
import sql
16 310 aaronmk
import util
17 133 aaronmk
import xml_dom
18 86 aaronmk
import xml_func
19 53 aaronmk
20 84 aaronmk
def metadata_value(name):
21 164 aaronmk
    if type(name) == str and name.startswith(':'): return name[1:]
22 84 aaronmk
    else: return None
23
24 53 aaronmk
def main():
25 434 aaronmk
    ex_tracker = exc.ExTracker()
26
27 131 aaronmk
    env_names = []
28
    def usage_err():
29 400 aaronmk
        raise SystemExit('Usage: '+opts.env_usage(env_names, True)+' [commit=1]'
30
            ' [test=1] '+sys.argv[0]+' [map_path] [<input] [>output]')
31 146 aaronmk
    limit = opts.get_env_var('n', None, env_names)
32
    if limit != None: limit = int(limit)
33 400 aaronmk
    test = opts.env_flag('test')
34
    commit = not test and opts.env_flag('commit') # never commit in test mode
35 131 aaronmk
36 53 aaronmk
    # Get db config from env vars
37 131 aaronmk
    db_config_names = ['engine', 'host', 'user', 'password', 'database']
38 53 aaronmk
    def get_db_config(prefix):
39 64 aaronmk
        return opts.get_env_vars(db_config_names, prefix, env_names)
40 67 aaronmk
    in_db_config = get_db_config('in')
41
    out_db_config = get_db_config('out')
42 131 aaronmk
    in_is_db = 'engine' in in_db_config
43
    out_is_db = 'engine' in out_db_config
44 53 aaronmk
45
    # Parse args
46 73 aaronmk
    map_path = None
47 67 aaronmk
    try: _prog_name, map_path = sys.argv
48 53 aaronmk
    except ValueError:
49 338 aaronmk
        if in_is_db or not out_is_db: usage_err()
50 53 aaronmk
51 57 aaronmk
    # Load map header
52 130 aaronmk
    in_is_xpaths = True
53 318 aaronmk
    out_label = None
54 73 aaronmk
    if map_path != None:
55 56 aaronmk
        import copy
56
        import csv
57
58 53 aaronmk
        import xpath
59
60 133 aaronmk
        metadata = []
61 73 aaronmk
        mappings = []
62
        stream = open(map_path, 'rb')
63
        reader = csv.reader(stream)
64 161 aaronmk
        in_label, out_label = reader.next()[:2]
65 61 aaronmk
        def split_col_name(name):
66 72 aaronmk
            name, sep, root = name.partition(':')
67
            return name, sep != '', root
68 161 aaronmk
        in_label, in_is_xpaths, in_root = split_col_name(in_label)
69
        out_label, out_is_xpaths, out_root = split_col_name(out_label)
70 133 aaronmk
        assert out_is_xpaths # CSV output not supported yet
71 161 aaronmk
        has_types = out_root.startswith('/*s/') # outer elements are types
72 73 aaronmk
        for row in reader:
73
            in_, out = row[:2]
74
            if out != '':
75 432 aaronmk
                if out_is_xpaths: out = xpath.parse(out_root+out)
76 164 aaronmk
                mappings.append((in_, out))
77 73 aaronmk
        stream.close()
78 130 aaronmk
    in_is_xml = in_is_xpaths and not in_is_db
79 56 aaronmk
80 318 aaronmk
    if in_is_xml:
81
        doc0 = minidom.parse(sys.stdin)
82
        if out_label == None: out_label = doc0.documentElement.tagName
83 294 aaronmk
84 316 aaronmk
    def process_input(root, process_row):
85 309 aaronmk
        '''Inputs datasource to XML tree, mapping if needed'''
86
        def process_rows(get_value, rows):
87 297 aaronmk
            '''Processes input values
88
            @param get_value f(in_, row):str
89
            '''
90 314 aaronmk
            for i, row in enumerate(rows):
91 316 aaronmk
                if not (limit == None or i < limit): break
92
                row_id = str(i)
93
                for in_, out in mappings:
94
                    value = metadata_value(in_)
95
                    if value == None: value = get_value(in_, row)
96
                    if value != None:
97
                        xpath.put_obj(root, out, row_id, has_types, value)
98
                process_row()
99 297 aaronmk
100 310 aaronmk
        if map_path == None:
101
            iter_ = xml_dom.NodeElemIter(doc0.documentElement)
102
            util.skip(iter_, xml_dom.is_text) # skip metadata
103 317 aaronmk
            for child in iter_:
104
                root.appendChild(child)
105
                process_row()
106 309 aaronmk
        elif in_is_db:
107 130 aaronmk
            assert in_is_xpaths
108 126 aaronmk
109 117 aaronmk
            import db_xml
110
111 161 aaronmk
            in_root_xml = xpath.path2xml(in_root)
112 164 aaronmk
            for i, mapping in enumerate(mappings):
113
                in_, out = mapping
114
                if metadata_value(in_) == None:
115 168 aaronmk
                    mappings[i] = (xpath.path2xml(in_root+'/'+in_), out)
116 126 aaronmk
117 131 aaronmk
            in_db = sql.connect(in_db_config)
118 133 aaronmk
            in_pkeys = {}
119 297 aaronmk
            def get_value(in_, row):
120 167 aaronmk
                pkey, = row
121 297 aaronmk
                in_ = in_.cloneNode(True) # don't modify orig value!
122
                xml_dom.set_id(xpath.get(in_, in_root), pkey)
123
                value = sql.value_or_none(db_xml.get(in_db, in_, in_pkeys))
124
                if value != None: return str(value)
125
                else: return None
126 309 aaronmk
            process_rows(get_value, sql.rows(db_xml.get(in_db, in_root_xml,
127
                in_pkeys, limit)))
128 117 aaronmk
            in_db.close()
129 161 aaronmk
        elif in_is_xml:
130 297 aaronmk
            def get_value(in_, row):
131
                node = xpath.get(row, in_)
132
                if node != None: return xml_dom.value(node)
133
                else: return None
134
            row0 = xpath.get(doc0.documentElement, in_root)
135 309 aaronmk
            process_rows(get_value, xml_dom.NodeElemIter(row0.parentNode))
136 56 aaronmk
        else: # input is CSV
137 133 aaronmk
            map_ = dict(mappings)
138 59 aaronmk
            reader = csv.reader(sys.stdin)
139 84 aaronmk
            cols = reader.next()
140 162 aaronmk
            col_idxs = dict([(value, idx) for idx, value in enumerate(cols)])
141 164 aaronmk
            for i, mapping in enumerate(mappings):
142
                in_, out = mapping
143
                if metadata_value(in_) == None:
144
                    try: mappings[i] = (col_idxs[in_], out)
145
                    except KeyError: pass
146 162 aaronmk
147 297 aaronmk
            def get_value(in_, row):
148
                value = row[in_]
149
                if value != '': return value
150
                else: return None
151 309 aaronmk
            process_rows(get_value, reader)
152 53 aaronmk
153
    # Output XML tree
154 316 aaronmk
    doc = xml_dom.create_doc(out_label)
155
    root = doc.documentElement
156 130 aaronmk
    if out_is_db:
157 53 aaronmk
        from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE
158
        import db_xml
159
160 133 aaronmk
        out_db = sql.connect(out_db_config)
161
        out_db.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE)
162 310 aaronmk
        out_pkeys = {}
163 53 aaronmk
        try:
164 400 aaronmk
            if test: sql.empty_db(out_db)
165 53 aaronmk
            row_ct_ref = [0]
166 316 aaronmk
            def process_row():
167
                try: xml_func.process(root)
168 434 aaronmk
                except xml_func.SyntaxException, e: ex_tracker.track(e, False)
169 437 aaronmk
                assert xml_dom.has_one_child(root)
170
                child = root.firstChild
171
                try:
172
                    sql.with_savepoint(out_db, lambda: db_xml.put(out_db,
173
                        child, False, row_ct_ref, out_pkeys))
174
                    if commit: out_db.commit()
175
                except sql.DatabaseErrors, e: ex_tracker.track(e)
176 316 aaronmk
                root.clear()
177
            process_input(root, process_row)
178 53 aaronmk
            print 'Inserted '+str(row_ct_ref[0])+' rows'
179
        finally:
180 133 aaronmk
            out_db.rollback()
181
            out_db.close()
182 299 aaronmk
    else: # output is XML
183 316 aaronmk
        def process_row(): pass
184
        process_input(root, process_row)
185
        xml_func.process(root)
186
        doc.writexml(sys.stdout, **xml_dom.prettyxml_config)
187 53 aaronmk
188 133 aaronmk
try: main()
189 294 aaronmk
except Parser.SyntaxException, e: raise SystemExit(str(e))