1
|
#!/usr/bin/env python
|
2
|
# Inner-joins two map spreadsheets A->B and B->C to A->C
|
3
|
# Multi-safe (supports an input appearing multiple times). If multiple outputs
|
4
|
# should be considered ambiguous, they can be discarded by setting only_one.
|
5
|
|
6
|
import csv
|
7
|
import operator
|
8
|
import os.path
|
9
|
import re
|
10
|
import sys
|
11
|
import warnings
|
12
|
|
13
|
sys.path.append(os.path.dirname(__file__)+"/../lib")
|
14
|
|
15
|
import maps
|
16
|
import opts
|
17
|
import util
|
18
|
|
19
|
def simplify(str_): return re.sub(r'[\W_]+', r'', str_.lower())
|
20
|
|
21
|
def main():
|
22
|
env_names = []
|
23
|
def usage_err():
|
24
|
raise SystemExit('Usage: '+opts.env_usage(env_names, True)+' '
|
25
|
+sys.argv[0]+' <map_0 map_1 [| '+sys.argv[0]+' map_2]... >out_map')
|
26
|
|
27
|
only_one = opts.env_flag('only_one', False, env_names)
|
28
|
# Uses only columns 0 and 1 of map_1
|
29
|
map_1_core_only = opts.env_flag('map_1_core_only', False, env_names)
|
30
|
# Turns off "No input mapping" errors
|
31
|
quiet = opts.env_flag('quiet', False, env_names)
|
32
|
try: _prog_name, map_1_path = sys.argv
|
33
|
except ValueError: usage_err()
|
34
|
|
35
|
# Get map 1
|
36
|
map_1 = {}
|
37
|
stream = open(map_1_path, 'rb')
|
38
|
reader = csv.reader(stream)
|
39
|
map_1_cols = reader.next()
|
40
|
for row in reader:
|
41
|
if map_1_core_only: row = row[:2]
|
42
|
if row[0] != '': map_1.setdefault(simplify(row[0]), []).append(row)
|
43
|
stream.close()
|
44
|
|
45
|
# Join map 1 to map 0
|
46
|
reader = csv.reader(sys.stdin)
|
47
|
writer = csv.writer(sys.stdout)
|
48
|
map_0_cols = reader.next()
|
49
|
if not maps.join_combinable(map_0_cols, map_1_cols):
|
50
|
raise SystemExit('Map error: Map 0 output column name "'+map_0_cols[1]
|
51
|
+'" doesn\'t match map 1 input column name "'+map_1_cols[0]+'"')
|
52
|
writer.writerow(maps.merge_mappings(map_0_cols, map_1_cols))
|
53
|
for row in reader:
|
54
|
def set_error(msg):
|
55
|
row[1] = ''
|
56
|
if not quiet:
|
57
|
warnings.warn(UserWarning(msg))
|
58
|
row[2] = '** '+msg+' ** '+util.list_setdefault(row, 2, '')
|
59
|
|
60
|
row_written = False
|
61
|
if row[1] != '':
|
62
|
out_orig = row[1] # used in "No join mapping" error msg
|
63
|
|
64
|
# Look for a match
|
65
|
out_rows = []
|
66
|
suffix = ''
|
67
|
while True:
|
68
|
try:
|
69
|
out_rows = map_1[simplify(row[1])]
|
70
|
break
|
71
|
except KeyError:
|
72
|
# Heuristically look for a match on a parent path.
|
73
|
# If this produces a syntactically invalid parent path (e.g.
|
74
|
# there is a / within []), it will be ignored since there
|
75
|
# wouldn't be a an entry in map_1.
|
76
|
row[1], sep, new_suffix = row[1].rpartition('/')
|
77
|
if sep == '': break
|
78
|
suffix = sep+new_suffix+suffix # prepend new suffix
|
79
|
|
80
|
# Write new mapping
|
81
|
is_empty = len(out_rows) == 1 and out_rows[0][1] == ''
|
82
|
if only_one and len(out_rows) > 1: # multiple outputs are ambiguous
|
83
|
set_error('Ambiguous mapping for '+row[1]) # discards mapping
|
84
|
elif out_rows and not is_empty: # found non-empty mapping(s)
|
85
|
for out_row in out_rows:
|
86
|
row_ = row[:] # don't modify row, since it will be reused
|
87
|
row_ = maps.merge_mappings(row_, out_row)
|
88
|
if row_[1] != '': row_[1] += suffix # don't modify out_row!
|
89
|
writer.writerow(row_)
|
90
|
row_written = True
|
91
|
else:
|
92
|
msg = 'No'
|
93
|
if is_empty: msg += ' non-empty'
|
94
|
msg += ' join mapping for '+out_orig
|
95
|
set_error(msg)
|
96
|
elif row[2] == '': # also no comment explaining why no input mapping
|
97
|
set_error('No input mapping for '+row[0])
|
98
|
|
99
|
if not row_written: writer.writerow(row)
|
100
|
|
101
|
main()
|