Project

General

Profile

1
#!/usr/bin/env python
2
# Maps one datasource to another, using a map spreadsheet if needed
3
# For outputting an XML file to a PostgreSQL database, use the general format of
4
# http://vegbank.org/vegdocs/xml/vegbank_example_ver1.0.2.xml
5

    
6
import os
7
import os.path
8
import sys
9
import xml.dom.minidom
10

    
11
sys.path.append(os.path.dirname(__file__)+"/lib")
12

    
13
import opts
14

    
15
def main():
16
    # Get db config from env vars
17
    db_config_names = ['host', 'user', 'password', 'database']
18
    env_names = []
19
    def get_db_config(prefix):
20
        return opts.get_env_vars(db_config_names, prefix, env_names)
21
    in_db_config = get_db_config('in')
22
    out_db_config = get_db_config('out')
23
    in_is_db = in_db_config != None
24
    out_is_db = out_db_config != None
25
    
26
    # Parse args
27
    map_path = None
28
    try: _prog_name, map_path = sys.argv
29
    except ValueError:
30
        if in_is_db or not out_is_db: raise SystemExit('Usage: '
31
            +opts.env_usage(env_names, True)+' [commit=1] '+sys.argv[0]
32
            +' [map_path] [<input] [>output]')
33
    commit = opts.env_flag('commit')
34
    
35
    # Load map header
36
    in_is_xml = True
37
    if map_path != None:
38
        import copy
39
        import csv
40
        
41
        from Parser import SyntaxException
42
        import xpath
43
        
44
        mappings = []
45
        stream = open(map_path, 'rb')
46
        reader = csv.reader(stream)
47
        src, dest = reader.next()[:2]
48
        def split_col_name(name):
49
            name, sep, root = name.partition(':')
50
            return name, sep != '', root
51
        src, in_is_xml, src_root = split_col_name(src)
52
        dest, out_is_xml, dest_root = split_col_name(dest)
53
        assert out_is_xml
54
        has_types = dest_root.startswith('/*s/') # outer elements are types
55
        for row in reader:
56
            in_, out = row[:2]
57
            if out != '':
58
                try: out = xpath.parse(dest_root+out)
59
                except SyntaxException, ex: raise SystemExit(str(ex))
60
                mappings.append((in_, out))
61
        stream.close()
62
    
63
    # Input datasource to XML tree, mapping if needed
64
    if in_is_xml: doc = xml.dom.minidom.parse(sys.stdin)
65
    if map_path != None:
66
        out_doc = xml.dom.minidom.getDOMImplementation().createDocument(None,
67
            dest, None)
68
        if in_is_xml: raise Exception('XML-XML mapping not supported yet')
69
        elif in_is_db: raise Exception('DB-XML mapping not supported yet')
70
        else: # input is CSV
71
            map_ = dict(mappings)
72
            reader = csv.reader(sys.stdin)
73
            fieldnames = reader.next()
74
            row_idx = 0
75
            for row in reader:
76
                row_id = str(row_idx)
77
                for idx, name in enumerate(fieldnames):
78
                    value = row[idx]
79
                    if value != '' and name in map_:
80
                        path = copy.deepcopy(map_[name]) # don't modify value!
81
                        xpath.set_id(path, row_id, has_types)
82
                        xpath.set_value(path, value)
83
                        xpath.get(out_doc, path, True)
84
                row_idx += 1
85
        doc = out_doc
86
    
87
    # Output XML tree
88
    if out_db_config != None: # output is database
89
        import psycopg2
90
        from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE
91
        
92
        import db_xml
93
        
94
        db = psycopg2.connect(**out_db_config)
95
        db.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE)
96
        try:
97
            row_ct_ref = [0]
98
            db_xml.xml2db(db, doc.documentElement, row_ct_ref)
99
            print 'Inserted '+str(row_ct_ref[0])+' rows'
100
            if commit: db.commit()
101
        finally:
102
            db.rollback()
103
            db.close()
104
    else: doc.writexml(sys.stdout, addindent='    ', newl='\n') # output is XML
105

    
106
main()
(2-2/3)