| 1 |
51
|
aaronmk
|
#!/usr/bin/env python
|
| 2 |
|
|
# Inner-joins two map spreadsheets A->B and B->C to A->C
|
| 3 |
3822
|
aaronmk
|
# Multi-safe (supports an input appearing multiple times). If multiple outputs
|
| 4 |
|
|
# should be considered ambiguous, they can be discarded by setting only_one.
|
| 5 |
4506
|
aaronmk
|
# Case- and punctuation-insensitive.
|
| 6 |
51
|
aaronmk
|
|
| 7 |
|
|
import csv
|
| 8 |
3766
|
aaronmk
|
import operator
|
| 9 |
734
|
aaronmk
|
import os.path
|
| 10 |
4498
|
aaronmk
|
import re
|
| 11 |
51
|
aaronmk
|
import sys
|
| 12 |
939
|
aaronmk
|
import warnings
|
| 13 |
51
|
aaronmk
|
|
| 14 |
734
|
aaronmk
|
sys.path.append(os.path.dirname(__file__)+"/../lib")
|
| 15 |
732
|
aaronmk
|
|
| 16 |
734
|
aaronmk
|
import maps
|
| 17 |
3822
|
aaronmk
|
import opts
|
| 18 |
1282
|
aaronmk
|
import util
|
| 19 |
734
|
aaronmk
|
|
| 20 |
51
|
aaronmk
|
def main():
|
| 21 |
3923
|
aaronmk
|
env_names = []
|
| 22 |
|
|
def usage_err():
|
| 23 |
|
|
raise SystemExit('Usage: '+opts.env_usage(env_names, True)+' '
|
| 24 |
|
|
+sys.argv[0]+' <map_0 map_1 [| '+sys.argv[0]+' map_2]... >out_map')
|
| 25 |
|
|
|
| 26 |
|
|
only_one = opts.env_flag('only_one', False, env_names)
|
| 27 |
3924
|
aaronmk
|
# Uses only columns 0 and 1 of map_1
|
| 28 |
|
|
map_1_core_only = opts.env_flag('map_1_core_only', False, env_names)
|
| 29 |
4628
|
aaronmk
|
# Passes through terms with no input mapping or no join mapping
|
| 30 |
|
|
passthru = opts.env_flag('passthru', False, env_names)
|
| 31 |
3923
|
aaronmk
|
# Turns off "No input mapping" errors
|
| 32 |
4628
|
aaronmk
|
quiet = passthru or opts.env_flag('quiet', False, env_names)
|
| 33 |
67
|
aaronmk
|
try: _prog_name, map_1_path = sys.argv
|
| 34 |
3923
|
aaronmk
|
except ValueError: usage_err()
|
| 35 |
51
|
aaronmk
|
|
| 36 |
|
|
# Get map 1
|
| 37 |
|
|
map_1 = {}
|
| 38 |
|
|
stream = open(map_1_path, 'rb')
|
| 39 |
59
|
aaronmk
|
reader = csv.reader(stream)
|
| 40 |
737
|
aaronmk
|
map_1_cols = reader.next()
|
| 41 |
51
|
aaronmk
|
for row in reader:
|
| 42 |
3924
|
aaronmk
|
if map_1_core_only: row = row[:2]
|
| 43 |
4500
|
aaronmk
|
if row[0] != '': map_1.setdefault(maps.simplify(row[0]), []).append(row)
|
| 44 |
51
|
aaronmk
|
stream.close()
|
| 45 |
|
|
|
| 46 |
|
|
# Join map 1 to map 0
|
| 47 |
59
|
aaronmk
|
reader = csv.reader(sys.stdin)
|
| 48 |
120
|
aaronmk
|
writer = csv.writer(sys.stdout)
|
| 49 |
737
|
aaronmk
|
map_0_cols = reader.next()
|
| 50 |
1766
|
aaronmk
|
if not maps.join_combinable(map_0_cols, map_1_cols):
|
| 51 |
|
|
raise SystemExit('Map error: Map 0 output column name "'+map_0_cols[1]
|
| 52 |
1355
|
aaronmk
|
+'" doesn\'t match map 1 input column name "'+map_1_cols[0]+'"')
|
| 53 |
737
|
aaronmk
|
writer.writerow(maps.merge_mappings(map_0_cols, map_1_cols))
|
| 54 |
51
|
aaronmk
|
for row in reader:
|
| 55 |
1739
|
aaronmk
|
def set_error(msg):
|
| 56 |
4628
|
aaronmk
|
if not passthru: row[1] = ''
|
| 57 |
|
|
elif row[1] == '': row[1] = row[0] # no input mapping
|
| 58 |
4156
|
aaronmk
|
if not quiet:
|
| 59 |
|
|
warnings.warn(UserWarning(msg))
|
| 60 |
|
|
row[2] = '** '+msg+' ** '+util.list_setdefault(row, 2, '')
|
| 61 |
1739
|
aaronmk
|
|
| 62 |
3777
|
aaronmk
|
row_written = False
|
| 63 |
730
|
aaronmk
|
if row[1] != '':
|
| 64 |
1283
|
aaronmk
|
out_orig = row[1] # used in "No join mapping" error msg
|
| 65 |
|
|
|
| 66 |
1170
|
aaronmk
|
# Look for a match
|
| 67 |
3766
|
aaronmk
|
out_rows = []
|
| 68 |
1170
|
aaronmk
|
suffix = ''
|
| 69 |
|
|
while True:
|
| 70 |
|
|
try:
|
| 71 |
4500
|
aaronmk
|
out_rows = map_1[maps.simplify(row[1])]
|
| 72 |
1170
|
aaronmk
|
break
|
| 73 |
|
|
except KeyError:
|
| 74 |
|
|
# Heuristically look for a match on a parent path.
|
| 75 |
|
|
# If this produces a syntactically invalid parent path (e.g.
|
| 76 |
|
|
# there is a / within []), it will be ignored since there
|
| 77 |
|
|
# wouldn't be a an entry in map_1.
|
| 78 |
|
|
row[1], sep, new_suffix = row[1].rpartition('/')
|
| 79 |
|
|
if sep == '': break
|
| 80 |
|
|
suffix = sep+new_suffix+suffix # prepend new suffix
|
| 81 |
|
|
|
| 82 |
4634
|
aaronmk
|
is_empty = len(out_rows) == 1 and out_rows[0][1] == ''
|
| 83 |
4637
|
aaronmk
|
if is_empty:
|
| 84 |
|
|
if passthru: out_rows[0][1] = row[1] # for maps.merge_mappings()
|
| 85 |
|
|
set_error('No non-empty join mapping for '+out_orig)
|
| 86 |
4634
|
aaronmk
|
|
| 87 |
1170
|
aaronmk
|
# Write new mapping
|
| 88 |
3822
|
aaronmk
|
if only_one and len(out_rows) > 1: # multiple outputs are ambiguous
|
| 89 |
|
|
set_error('Ambiguous mapping for '+row[1]) # discards mapping
|
| 90 |
4634
|
aaronmk
|
elif out_rows: # found mapping(s)
|
| 91 |
3766
|
aaronmk
|
for out_row in out_rows:
|
| 92 |
3777
|
aaronmk
|
row_ = row[:] # don't modify row, since it will be reused
|
| 93 |
|
|
row_ = maps.merge_mappings(row_, out_row)
|
| 94 |
|
|
if row_[1] != '': row_[1] += suffix # don't modify out_row!
|
| 95 |
|
|
writer.writerow(row_)
|
| 96 |
|
|
row_written = True
|
| 97 |
7405
|
aaronmk
|
elif row[1].startswith('/'): pass # leave row as is
|
| 98 |
4634
|
aaronmk
|
else: set_error('No join mapping for '+out_orig)
|
| 99 |
1739
|
aaronmk
|
elif row[2] == '': # also no comment explaining why no input mapping
|
| 100 |
4156
|
aaronmk
|
set_error('No input mapping for '+row[0])
|
| 101 |
1739
|
aaronmk
|
|
| 102 |
3777
|
aaronmk
|
if not row_written: writer.writerow(row)
|
| 103 |
51
|
aaronmk
|
|
| 104 |
|
|
main()
|
