Project

General

Profile

1 53 aaronmk
#!/usr/bin/env python
2
# Maps one datasource to another, using a map spreadsheet if needed
3
# For outputting an XML file to a PostgreSQL database, use the general format of
4
# http://vegbank.org/vegdocs/xml/vegbank_example_ver1.0.2.xml
5
6
import os
7
import os.path
8
import sys
9
import xml.dom.minidom
10
11
sys.path.append(os.path.dirname(__file__)+"/lib")
12
13 64 aaronmk
import opts
14 86 aaronmk
import xml_func
15 53 aaronmk
16 84 aaronmk
def metadata_value(name):
17
    if name.startswith(':'): return name[1:]
18
    else: return None
19
20 53 aaronmk
def main():
21
    # Get db config from env vars
22
    db_config_names = ['host', 'user', 'password', 'database']
23
    env_names = []
24
    def get_db_config(prefix):
25 64 aaronmk
        return opts.get_env_vars(db_config_names, prefix, env_names)
26 67 aaronmk
    in_db_config = get_db_config('in')
27
    out_db_config = get_db_config('out')
28
    in_is_db = in_db_config != None
29
    out_is_db = out_db_config != None
30 53 aaronmk
31
    # Parse args
32 73 aaronmk
    map_path = None
33 67 aaronmk
    try: _prog_name, map_path = sys.argv
34 53 aaronmk
    except ValueError:
35 73 aaronmk
        if in_is_db or not out_is_db: raise SystemExit('Usage: '
36
            +opts.env_usage(env_names, True)+' [commit=1] '+sys.argv[0]
37
            +' [map_path] [<input] [>output]')
38 64 aaronmk
    commit = opts.env_flag('commit')
39 53 aaronmk
40 57 aaronmk
    # Load map header
41
    in_is_xml = True
42 73 aaronmk
    if map_path != None:
43 56 aaronmk
        import copy
44
        import csv
45
46 75 aaronmk
        from Parser import SyntaxException
47 53 aaronmk
        import xpath
48
49 73 aaronmk
        mappings = []
50
        stream = open(map_path, 'rb')
51
        reader = csv.reader(stream)
52
        src, dest = reader.next()[:2]
53 61 aaronmk
        def split_col_name(name):
54 72 aaronmk
            name, sep, root = name.partition(':')
55
            return name, sep != '', root
56 67 aaronmk
        src, in_is_xml, src_root = split_col_name(src)
57
        dest, out_is_xml, dest_root = split_col_name(dest)
58 61 aaronmk
        assert out_is_xml
59 67 aaronmk
        has_types = dest_root.startswith('/*s/') # outer elements are types
60 73 aaronmk
        for row in reader:
61
            in_, out = row[:2]
62
            if out != '':
63 84 aaronmk
                try: mappings.append((in_, xpath.parse(dest_root+out)))
64 73 aaronmk
                except SyntaxException, ex: raise SystemExit(str(ex))
65
        stream.close()
66 56 aaronmk
67 57 aaronmk
    # Input datasource to XML tree, mapping if needed
68 116 aaronmk
    if in_is_xml: doc0 = xml.dom.minidom.parse(sys.stdin)
69 73 aaronmk
    if map_path != None:
70 116 aaronmk
        doc1 = xml.dom.minidom.getDOMImplementation().createDocument(None,
71 56 aaronmk
            dest, None)
72 118 aaronmk
        if in_is_xml: raise SystemExit('XML-XML mapping not supported yet')
73 117 aaronmk
        elif in_is_db:
74 118 aaronmk
            import psycopg2
75
76 117 aaronmk
            import db_xml
77
78
            in_db = psycopg2.connect(**in_db_config)
79
            for in_, out in mappings:
80
                value = metadata_value(in_)
81
                if value == None:
82 118 aaronmk
                    raise SystemExit('DB-XML mapping not supported yet')
83 117 aaronmk
            in_db.close()
84 56 aaronmk
        else: # input is CSV
85 84 aaronmk
            metadata = []
86
            map_ = {}
87
            for in_, out in mappings:
88
                value = metadata_value(in_)
89
                if value != None: metadata.append((value, out))
90
                else: map_[in_] = out
91
92 59 aaronmk
            reader = csv.reader(sys.stdin)
93 84 aaronmk
            cols = reader.next()
94 83 aaronmk
            for row_idx, row in enumerate(reader):
95 56 aaronmk
                row_id = str(row_idx)
96 84 aaronmk
                def put_col(path, value):
97 116 aaronmk
                    xpath.put_obj(doc1, path, row_id, has_types, value)
98 84 aaronmk
                for value, out in metadata: put_col(out, value)
99
                for i, col in enumerate(cols):
100
                    if row[i] != '' and col in map_: put_col(map_[col], row[i])
101 116 aaronmk
        xml_func.process(doc1)
102
    else: doc1 = doc0
103 53 aaronmk
104
    # Output XML tree
105 67 aaronmk
    if out_db_config != None: # output is database
106 53 aaronmk
        import psycopg2
107
        from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE
108
109
        import db_xml
110
111 67 aaronmk
        db = psycopg2.connect(**out_db_config)
112 53 aaronmk
        db.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE)
113
        try:
114
            row_ct_ref = [0]
115 116 aaronmk
            db_xml.xml2db(db, doc1.documentElement, row_ct_ref)
116 53 aaronmk
            print 'Inserted '+str(row_ct_ref[0])+' rows'
117
            if commit: db.commit()
118
        finally:
119
            db.rollback()
120
            db.close()
121 116 aaronmk
    else: doc1.writexml(sys.stdout, addindent='    ', newl='\n') # output is XML
122 53 aaronmk
123
main()