Project

General

Profile

1
#!/usr/bin/env python
2
# Maps one datasource to another, using a map spreadsheet if needed
3
# For outputting an XML file to a PostgreSQL database, use the general format of
4
# http://vegbank.org/vegdocs/xml/vegbank_example_ver1.0.2.xml
5

    
6
import os.path
7
import sys
8
import traceback
9
import xml.dom.minidom as minidom
10

    
11
sys.path.append(os.path.dirname(__file__)+"/../lib")
12

    
13
import opts
14
import Parser
15
import sql
16
import xml_dom
17
import xml_func
18

    
19
def metadata_value(name):
20
    if type(name) == str and name.startswith(':'): return name[1:]
21
    else: return None
22

    
23
def main():
24
    env_names = []
25
    def usage_err():
26
        raise SystemExit('Usage: '+opts.env_usage(env_names, True)
27
            +' [commit=1] '+sys.argv[0]+' [map_path] [<input] [>output]')
28
    limit = opts.get_env_var('n', None, env_names)
29
    if limit != None: limit = int(limit)
30
    commit = opts.env_flag('commit')
31
    
32
    # Get db config from env vars
33
    db_config_names = ['engine', 'host', 'user', 'password', 'database']
34
    def get_db_config(prefix):
35
        return opts.get_env_vars(db_config_names, prefix, env_names)
36
    in_db_config = get_db_config('in')
37
    out_db_config = get_db_config('out')
38
    in_is_db = 'engine' in in_db_config
39
    out_is_db = 'engine' in out_db_config
40
    
41
    # Parse args
42
    map_path = None
43
    try: _prog_name, map_path = sys.argv
44
    except ValueError:
45
        if in_is_db: usage_err()
46
    
47
    # Load map header
48
    in_is_xpaths = True
49
    if map_path != None:
50
        import copy
51
        import csv
52
        
53
        import xpath
54
        
55
        metadata = []
56
        mappings = []
57
        stream = open(map_path, 'rb')
58
        reader = csv.reader(stream)
59
        in_label, out_label = reader.next()[:2]
60
        def split_col_name(name):
61
            name, sep, root = name.partition(':')
62
            return name, sep != '', root
63
        in_label, in_is_xpaths, in_root = split_col_name(in_label)
64
        out_label, out_is_xpaths, out_root = split_col_name(out_label)
65
        assert out_is_xpaths # CSV output not supported yet
66
        has_types = out_root.startswith('/*s/') # outer elements are types
67
        for row in reader:
68
            in_, out = row[:2]
69
            if out != '':
70
                if out_is_xpaths: out = out_root+out
71
                mappings.append((in_, out))
72
        stream.close()
73
    in_is_xml = in_is_xpaths and not in_is_db
74
    
75
    if in_is_xml: doc0 = minidom.parse(sys.stdin)
76
    
77
    def process_xml(use_row):
78
        '''Maps datasource to XML tree'''
79
        doc1 = xml_dom.create_doc(out_label)
80
        root = doc1.documentElement
81
        
82
        def process(get_value, rows):
83
            '''Processes input values
84
            @param get_value f(in_, row):str
85
            '''
86
            for i, row in enumerate(rows):
87
                if not (limit == None or i < limit): break
88
                row_id = str(i)
89
                for in_, out in mappings:
90
                    value = metadata_value(in_)
91
                    if value == None: value = get_value(in_, row)
92
                    if value != None:
93
                        try: xpath.put_obj(root, out, row_id, has_types, value)
94
                        except Exception: traceback.print_exc()
95
        
96
        if in_is_db:
97
            assert in_is_xpaths
98
            
99
            import db_xml
100
            
101
            in_root_xml = xpath.path2xml(in_root)
102
            for i, mapping in enumerate(mappings):
103
                in_, out = mapping
104
                if metadata_value(in_) == None:
105
                    mappings[i] = (xpath.path2xml(in_root+'/'+in_), out)
106
            
107
            in_db = sql.connect(in_db_config)
108
            in_pkeys = {}
109
            def get_value(in_, row):
110
                pkey, = row
111
                in_ = in_.cloneNode(True) # don't modify orig value!
112
                xml_dom.set_id(xpath.get(in_, in_root), pkey)
113
                value = sql.value_or_none(db_xml.get(in_db, in_, in_pkeys))
114
                if value != None: return str(value)
115
                else: return None
116
            process(get_value, sql.rows(db_xml.get(in_db, in_root_xml, in_pkeys,
117
                limit)))
118
            in_db.close()
119
        elif in_is_xml:
120
            def get_value(in_, row):
121
                node = xpath.get(row, in_)
122
                if node != None: return xml_dom.value(node)
123
                else: return None
124
            row0 = xpath.get(doc0.documentElement, in_root)
125
            process(get_value, xml_dom.NodeElemIter(row0.parentNode))
126
        else: # input is CSV
127
            map_ = dict(mappings)
128
            reader = csv.reader(sys.stdin)
129
            cols = reader.next()
130
            col_idxs = dict([(value, idx) for idx, value in enumerate(cols)])
131
            for i, mapping in enumerate(mappings):
132
                in_, out = mapping
133
                if metadata_value(in_) == None:
134
                    try: mappings[i] = (col_idxs[in_], out)
135
                    except KeyError: pass
136
            
137
            def get_value(in_, row):
138
                value = row[in_]
139
                if value != '': return value
140
                else: return None
141
            process(get_value, reader)
142
        xml_func.process(root)
143
        return doc1
144
    
145
    def get_xml(use_row):
146
        '''Inputs datasource to XML tree, mapping if needed'''
147
        if map_path != None: return process_xml(use_row)
148
        else: return doc0
149
    
150
    # Output XML tree
151
    if out_is_db:
152
        from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE
153
        import db_xml
154
        
155
        out_db = sql.connect(out_db_config)
156
        out_db.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE)
157
        try:
158
            row_ct_ref = [0]
159
            def use_row(root): db_xml.xml2db(out_db, root, commit, row_ct_ref)
160
            db_xml.xml2db(out_db, get_xml(use_row).documentElement, commit,
161
                row_ct_ref)
162
            print 'Inserted '+str(row_ct_ref[0])+' rows'
163
        finally:
164
            out_db.rollback()
165
            out_db.close()
166
    else: # output is XML
167
        output = sys.stdout
168
        config = xml_dom.prettyxml_config
169
        doc = xml_dom.create_doc(out_label)
170
        doc.write_opening(output, **config)
171
        def use_row(root): root.writexml(output, indent=1, **config)
172
        map(use_row, get_xml(use_row).firstChild)
173
        doc.write_closing(output, **config)
174

    
175
try: main()
176
except Parser.SyntaxException, e: raise SystemExit(str(e))
(7-7/14)