Project

General

Profile

1 53 aaronmk
#!/usr/bin/env python
2
# Maps one datasource to another, using a map spreadsheet if needed
3
# For outputting an XML file to a PostgreSQL database, use the general format of
4
# http://vegbank.org/vegdocs/xml/vegbank_example_ver1.0.2.xml
5
6
import os
7
import os.path
8
import sys
9
import xml.dom.minidom
10
11
sys.path.append(os.path.dirname(__file__)+"/lib")
12
13 64 aaronmk
import opts
14 53 aaronmk
15
def main():
16
    # Get db config from env vars
17
    db_config_names = ['host', 'user', 'password', 'database']
18
    env_names = []
19
    def get_db_config(prefix):
20 64 aaronmk
        return opts.get_env_vars(db_config_names, prefix, env_names)
21 67 aaronmk
    in_db_config = get_db_config('in')
22
    out_db_config = get_db_config('out')
23
    in_is_db = in_db_config != None
24
    out_is_db = out_db_config != None
25 58 aaronmk
    uses_map = in_is_db or not out_is_db
26 53 aaronmk
27
    # Parse args
28 67 aaronmk
    try: _prog_name, map_path = sys.argv
29 53 aaronmk
    except ValueError:
30 67 aaronmk
        if uses_map: raise SystemExit('Usage: '+opts.env_usage(env_names, True)
31
            +' [commit=1] '+sys.argv[0]+' [map_path] [<input] [>output]')
32 64 aaronmk
    commit = opts.env_flag('commit')
33 53 aaronmk
34 57 aaronmk
    # Load map header
35
    in_is_xml = True
36 56 aaronmk
    if uses_map:
37
        import copy
38
        import csv
39
40 53 aaronmk
        import xpath
41
42 57 aaronmk
        map_stream = open(map_path, 'rb')
43 59 aaronmk
        map_reader = csv.reader(map_stream)
44 57 aaronmk
        src, dest = map_reader.next()[:2]
45 61 aaronmk
        def split_col_name(name):
46 72 aaronmk
            name, sep, root = name.partition(':')
47
            return name, sep != '', root
48 67 aaronmk
        src, in_is_xml, src_root = split_col_name(src)
49
        dest, out_is_xml, dest_root = split_col_name(dest)
50 61 aaronmk
        assert out_is_xml
51 67 aaronmk
        has_types = dest_root.startswith('/*s/') # outer elements are types
52 56 aaronmk
53 57 aaronmk
    # Input datasource to XML tree, mapping if needed
54
    if in_is_xml: doc = xml.dom.minidom.parse(sys.stdin)
55 56 aaronmk
    if uses_map:
56 61 aaronmk
        from Parser import SyntaxException
57 56 aaronmk
        import xml_xpath
58 53 aaronmk
59 57 aaronmk
        map_ = {}
60
        for row in map_reader:
61
            in_, out = row[:2]
62
            if out != '':
63 67 aaronmk
                try: out = xpath.parse(dest_root+out)
64 61 aaronmk
                except SyntaxException, ex: raise SystemExit(str(ex))
65 57 aaronmk
                if in_is_xml: pass # TODO: process the mapping
66 58 aaronmk
                elif in_is_db: pass # TODO: process the mapping
67 57 aaronmk
                else: map_[in_] = out
68
        map_stream.close()
69
70 56 aaronmk
        out_doc = xml.dom.minidom.getDOMImplementation().createDocument(None,
71
            dest, None)
72 57 aaronmk
        if in_is_xml: raise Exception('XML-XML mapping not supported yet')
73 56 aaronmk
        else: # input is CSV
74 59 aaronmk
            reader = csv.reader(sys.stdin)
75 56 aaronmk
            fieldnames = reader.next()
76
            row_idx = 0
77
            for row in reader:
78
                row_id = str(row_idx)
79
                for idx, name in enumerate(fieldnames):
80
                    value = row[idx]
81
                    if value != '' and name in map_:
82
                        path = copy.deepcopy(map_[name]) # don't modify value!
83
                        xpath.set_id(path, row_id, has_types)
84
                        xpath.set_value(path, value)
85
                        xml_xpath.get(out_doc, path, True)
86
                row_idx += 1
87
        doc = out_doc
88 53 aaronmk
89
    # Output XML tree
90 67 aaronmk
    if out_db_config != None: # output is database
91 53 aaronmk
        import psycopg2
92
        from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE
93
94
        import db_xml
95
96 67 aaronmk
        db = psycopg2.connect(**out_db_config)
97 53 aaronmk
        db.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE)
98
        try:
99
            row_ct_ref = [0]
100
            db_xml.xml2db(db, doc.documentElement, row_ct_ref)
101
            print 'Inserted '+str(row_ct_ref[0])+' rows'
102
            if commit: db.commit()
103
        finally:
104
            db.rollback()
105
            db.close()
106
    else: doc.writexml(sys.stdout, addindent='    ', newl='\n') # output is XML
107
108
main()