Project

General

Profile

1 53 aaronmk
#!/usr/bin/env python
2
# Maps one datasource to another, using a map spreadsheet if needed
3
# For outputting an XML file to a PostgreSQL database, use the general format of
4
# http://vegbank.org/vegdocs/xml/vegbank_example_ver1.0.2.xml
5
6
import os.path
7
import sys
8
import xml.dom.minidom
9
10
sys.path.append(os.path.dirname(__file__)+"/lib")
11
12 64 aaronmk
import opts
13 133 aaronmk
from Parser import SyntaxException
14 131 aaronmk
import sql
15 133 aaronmk
import xml_dom
16 86 aaronmk
import xml_func
17 53 aaronmk
18 84 aaronmk
def metadata_value(name):
19
    if name.startswith(':'): return name[1:]
20
    else: return None
21
22 53 aaronmk
def main():
23 131 aaronmk
    env_names = []
24
    def usage_err():
25
        raise SystemExit('Usage: '+opts.env_usage(env_names, True)
26
            +' [commit=1] '+sys.argv[0]+' [map_path] [<input] [>output]')
27 135 aaronmk
    limit = int(opts.get_env_var('n', sys.maxint, env_names))
28
    commit = opts.env_flag('commit')
29 131 aaronmk
30 53 aaronmk
    # Get db config from env vars
31 131 aaronmk
    db_config_names = ['engine', 'host', 'user', 'password', 'database']
32 53 aaronmk
    def get_db_config(prefix):
33 64 aaronmk
        return opts.get_env_vars(db_config_names, prefix, env_names)
34 67 aaronmk
    in_db_config = get_db_config('in')
35
    out_db_config = get_db_config('out')
36 131 aaronmk
    in_is_db = 'engine' in in_db_config
37
    out_is_db = 'engine' in out_db_config
38 53 aaronmk
39
    # Parse args
40 73 aaronmk
    map_path = None
41 67 aaronmk
    try: _prog_name, map_path = sys.argv
42 53 aaronmk
    except ValueError:
43 135 aaronmk
        if in_is_db: usage_err()
44 53 aaronmk
45 57 aaronmk
    # Load map header
46 130 aaronmk
    in_is_xpaths = True
47 73 aaronmk
    if map_path != None:
48 56 aaronmk
        import copy
49
        import csv
50
51 53 aaronmk
        import xpath
52
53 133 aaronmk
        metadata = []
54 73 aaronmk
        mappings = []
55
        stream = open(map_path, 'rb')
56
        reader = csv.reader(stream)
57
        src, dest = reader.next()[:2]
58 61 aaronmk
        def split_col_name(name):
59 72 aaronmk
            name, sep, root = name.partition(':')
60
            return name, sep != '', root
61 130 aaronmk
        src, in_is_xpaths, src_root = split_col_name(src)
62
        dest, out_is_xpaths, dest_root = split_col_name(dest)
63 133 aaronmk
        assert out_is_xpaths # CSV output not supported yet
64 67 aaronmk
        has_types = dest_root.startswith('/*s/') # outer elements are types
65 73 aaronmk
        for row in reader:
66
            in_, out = row[:2]
67
            if out != '':
68 133 aaronmk
                value = metadata_value(in_)
69
                is_metadata = value != None
70
                if in_is_xpaths and not is_metadata:
71
                    in_ = xpath.parse(src_root+in_)
72
                if out_is_xpaths: out = xpath.parse(dest_root+out)
73
                if is_metadata: metadata.append((value, out))
74
                else: mappings.append((in_, out))
75 73 aaronmk
        stream.close()
76 130 aaronmk
    in_is_xml = in_is_xpaths and not in_is_db
77 56 aaronmk
78 57 aaronmk
    # Input datasource to XML tree, mapping if needed
79 116 aaronmk
    if in_is_xml: doc0 = xml.dom.minidom.parse(sys.stdin)
80 73 aaronmk
    if map_path != None:
81 133 aaronmk
        doc1 = xml_dom.create_doc(dest)
82 126 aaronmk
        if in_is_db:
83 130 aaronmk
            assert in_is_xpaths
84 126 aaronmk
85 117 aaronmk
            import db_xml
86
87 135 aaronmk
            src_root = xpath.parse(src_root)
88
            src_root_xml = xpath.path2xml(src_root)
89
            mappings = [(xpath.path2xml(in_), out) for in_, out in mappings]
90 126 aaronmk
91 131 aaronmk
            in_db = sql.connect(in_db_config)
92 133 aaronmk
            in_pkeys = {}
93 135 aaronmk
            for row_idx, row in enumerate(sql.rows(db_xml.get(in_db,
94
                src_root_xml, in_pkeys))):
95
                if not row_idx < limit: break
96
                row_id, = row
97
                row_id = str(row_id)
98
99 133 aaronmk
                def put_col(path, value):
100
                    xpath.put_obj(doc1, path, row_id, has_types, value)
101
                for value, out in metadata: put_col(out, value)
102
                for in_, out in mappings:
103 135 aaronmk
                    root = xpath.get(in_.ownerDocument, src_root)
104
                    xml_dom.replace(root, root.cloneNode(False))
105
                    xml_dom.set_id(root, row_id)
106
107
                    cur = db_xml.get(in_db, in_, in_pkeys)
108
                    try: value = sql.value(cur)
109
                    except StopIteration: continue
110
                    put_col(out, str(value))
111 117 aaronmk
            in_db.close()
112 133 aaronmk
        elif in_is_xml: raise SystemExit('XML input not supported yet')
113 56 aaronmk
        else: # input is CSV
114 133 aaronmk
            map_ = dict(mappings)
115 59 aaronmk
            reader = csv.reader(sys.stdin)
116 84 aaronmk
            cols = reader.next()
117 83 aaronmk
            for row_idx, row in enumerate(reader):
118 135 aaronmk
                if not row_idx < limit: break
119 56 aaronmk
                row_id = str(row_idx)
120 135 aaronmk
121 84 aaronmk
                def put_col(path, value):
122 116 aaronmk
                    xpath.put_obj(doc1, path, row_id, has_types, value)
123 84 aaronmk
                for value, out in metadata: put_col(out, value)
124
                for i, col in enumerate(cols):
125
                    if row[i] != '' and col in map_: put_col(map_[col], row[i])
126 116 aaronmk
        xml_func.process(doc1)
127
    else: doc1 = doc0
128 53 aaronmk
129
    # Output XML tree
130 130 aaronmk
    if out_is_db:
131 53 aaronmk
        from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE
132
        import db_xml
133
134 133 aaronmk
        out_db = sql.connect(out_db_config)
135
        out_db.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE)
136 53 aaronmk
        try:
137
            row_ct_ref = [0]
138 133 aaronmk
            db_xml.xml2db(out_db, doc1.documentElement, row_ct_ref)
139 53 aaronmk
            print 'Inserted '+str(row_ct_ref[0])+' rows'
140 133 aaronmk
            if commit: out_db.commit()
141 53 aaronmk
        finally:
142 133 aaronmk
            out_db.rollback()
143
            out_db.close()
144 135 aaronmk
    else: xml_dom.writexml(sys.stdout, doc1) # output is XML
145 53 aaronmk
146 133 aaronmk
try: main()
147
except SyntaxException, ex: raise SystemExit(str(ex))