#!/bin/bash

# Sets a password environment variable

test "$#" -ge 1 || \

{ echo "Usage: . $0 env_var_name [msg] (note the initial \".\")" >&2; exit 2;}

msg="$2"

test -n "$msg" || msg="$1"

if test -z "${!1+t}"; then # env var with name $1 is unset

    read -s -p "Enter $msg password: "; echo

    export "$1"="$REPLY"

fi

#!/usr/bin/env python

# Transforms a map spreadsheet to use a different root

import csv

import os.path

import re

import sys

sys.path.append(os.path.dirname(__file__)+"/../lib")

import opts

def all_not_none(list_):

    return reduce(lambda a, b: a and b, map(lambda e: e != None, list_))

def main():

    # Get config from env vars

    config_names = ['in_root', 'out_root']

    env_names = []

    configs = map(lambda col: opts.get_env_vars(config_names, col, env_names),

        ['left', 'right'])

    if not all_not_none(configs): raise SystemExit(

        'Usage: '+opts.env_usage(env_names)+' '+sys.argv[0]+' <in_map >out_map')

    # Transform map

    reader = csv.reader(sys.stdin)

    writer = csv.writer(sys.stdout)

    cols = reader.next()

    for i in xrange(len(configs)):

        config = configs[i]

        label, sep, config['root'] = cols[i].partition(':')

        cols[i] = label+sep+config['out_root']

        config['in_root_re'] = r'^'+re.escape(config['in_root'])+r'\b'

    writer.writerow(cols)

    for row in reader:

        for i in xrange(len(configs)): row[i] = configs[i]['root']+row[i]

        def sub(i):

            row[i], n = re.subn(configs[i]['in_root_re'], r'', row[i])

            return n > 0

        if sub(0):

            if not sub(1): raise SystemExit('Map error: Root "'

                +configs[1]['in_root']+'" is not contained in output mapping: '

                +row[1])

            writer.writerow(row)

main()

#!/usr/bin/env python

# Maps one datasource to another, using a map spreadsheet if needed

# For outputting an XML file to a PostgreSQL database, use the general format of

# http://vegbank.org/vegdocs/xml/vegbank_example_ver1.0.2.xml

import os.path

import sys

import xml.dom.minidom

sys.path.append(os.path.dirname(__file__)+"/../lib")

import opts

from Parser import SyntaxException

import sql

import xml_dom

import xml_func

def metadata_value(name):

    if type(name) == str and name.startswith(':'): return name[1:]

    else: return None

def main():

    env_names = []

    def usage_err():

        raise SystemExit('Usage: '+opts.env_usage(env_names, True)

            +' [commit=1] '+sys.argv[0]+' [map_path] [<input] [>output]')

    limit = opts.get_env_var('n', None, env_names)

    if limit != None: limit = int(limit)

    commit = opts.env_flag('commit')

    # Get db config from env vars

    db_config_names = ['engine', 'host', 'user', 'password', 'database']

    def get_db_config(prefix):

        return opts.get_env_vars(db_config_names, prefix, env_names)

    in_db_config = get_db_config('in')

    out_db_config = get_db_config('out')

    in_is_db = 'engine' in in_db_config

    out_is_db = 'engine' in out_db_config

    # Parse args

    map_path = None

    try: _prog_name, map_path = sys.argv

    except ValueError:

        if in_is_db: usage_err()

    # Load map header

    in_is_xpaths = True

    if map_path != None:

        import copy

        import csv

        import xpath

        metadata = []

        mappings = []

        stream = open(map_path, 'rb')

        reader = csv.reader(stream)

        in_label, out_label = reader.next()[:2]

        def split_col_name(name):

            name, sep, root = name.partition(':')

            return name, sep != '', root

        in_label, in_is_xpaths, in_root = split_col_name(in_label)

        out_label, out_is_xpaths, out_root = split_col_name(out_label)

        assert out_is_xpaths # CSV output not supported yet

        has_types = out_root.startswith('/*s/') # outer elements are types

        for row in reader:

            in_, out = row[:2]

            if out != '':

                if out_is_xpaths: out = out_root+out

                mappings.append((in_, out))

        stream.close()

    in_is_xml = in_is_xpaths and not in_is_db

    # Input datasource to XML tree, mapping if needed

    if in_is_xml:

        doc0 = xml.dom.minidom.parse(sys.stdin)

    if map_path != None:

        doc1 = xml_dom.create_doc(out_label)

        root = doc1.documentElement

        if in_is_db:

            assert in_is_xpaths

            import db_xml

            in_root_xml = xpath.path2xml(in_root)

            for i, mapping in enumerate(mappings):

                in_, out = mapping

                if metadata_value(in_) == None:

                    mappings[i] = (xpath.path2xml(in_root+'/'+in_), out)

            in_db = sql.connect(in_db_config)

            in_pkeys = {}

            for row_idx, row in enumerate(sql.rows(db_xml.get(in_db,

                in_root_xml, in_pkeys, limit))):

                row_id = str(row_idx)

                pkey, = row

                for in_, out in mappings:

                    value = metadata_value(in_)

                    if value == None:

                        in_ = in_.cloneNode(True) # don't modify orig value!

                        xml_dom.set_id(xpath.get(in_, in_root), pkey)

                        value = sql.value_or_none(db_xml.get(in_db, in_,

                            in_pkeys))

                    if value != None:

                        xpath.put_obj(root, out, row_id, has_types, str(value))

            in_db.close()

        elif in_is_xml:

            row = xpath.get(doc0.documentElement, in_root)

            for row_idx, row in enumerate(xml_dom.NodeElemIter(row.parentNode)):

                if not (limit == None or row_idx < limit): break

                row_id = str(row_idx)

                for in_, out in mappings:

                    value = metadata_value(in_)

                    if value == None:

                        node = xpath.get(row, in_)

                        if node != None: value = xml_dom.value(node)

                    if value != None:

                        xpath.put_obj(root, out, row_id, has_types, value)

        else: # input is CSV

            map_ = dict(mappings)

            reader = csv.reader(sys.stdin)

            cols = reader.next()

            col_idxs = dict([(value, idx) for idx, value in enumerate(cols)])

            for i, mapping in enumerate(mappings):

                in_, out = mapping

                if metadata_value(in_) == None:

                    try: mappings[i] = (col_idxs[in_], out)

                    except KeyError: pass

            for row_idx, row in enumerate(reader):

                if not (limit == None or row_idx < limit): break

                row_id = str(row_idx)

                for in_, out in mappings:

                    value = metadata_value(in_)

                    if value == None:

                        value = row[in_]

                        if value == '': value = None

                    if value != None:

                        xpath.put_obj(root, out, row_id, has_types, value)

        xml_func.process(root)

    else: doc1 = doc0

    # Output XML tree

    if out_is_db:

        from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE

        import db_xml

        out_db = sql.connect(out_db_config)

        out_db.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE)

        try:

            row_ct_ref = [0]

            db_xml.xml2db(out_db, doc1.documentElement, row_ct_ref)

            print 'Inserted '+str(row_ct_ref[0])+' rows'

            if commit: out_db.commit()

        finally:

            out_db.rollback()

            out_db.close()

    else: xml_dom.writexml(sys.stdout, doc1) # output is XML

try: main()

except SyntaxException, ex: raise SystemExit(str(ex))

#!/bin/sh

# Requires var bien_password

export out_engine=PostgreSQL out_host=localhost out_user=bien \

out_password="$bien_password" out_database=vegbien

#!/bin/sh

# Accesses vegbank db

selfDir="$(dirname -- "$0")"

bien_password="$(cat "$selfDir/bien_password")"

. "$selfDir/vegbien_dest"

export PGHOST="$out_host" PGUSER="$out_user" PGPASSWORD="$out_password"

psql --dbname="$out_database" "$@"

#!/usr/bin/env python

# Inner-joins two map spreadsheets A->B and B->C to A->C

import csv

import sys

def main():

    try: _prog_name, map_1_path = sys.argv

    except ValueError:

        raise SystemExit('Usage: '+sys.argv[0]+' <map_0 map_1 [| '+sys.argv[0]

            +' map_2]... >joined_map')

    # Get map 1

    map_1 = {}

    stream = open(map_1_path, 'rb')

    reader = csv.reader(stream)

    map_1_in, map_1_out = reader.next()[:2]

    for row in reader:

        if row[1] != '': map_1[row[0]] = row[1]

    stream.close()

    # Join map 1 to map 0

    reader = csv.reader(sys.stdin)

    writer = csv.writer(sys.stdout)

    cols = reader.next()

    if not cols[1] == map_1_in: raise SystemExit('Map error: '

        'Map 0 output column name doesn\'t match map 1 input column name')

    cols[1] = map_1_out

    writer.writerow(cols)

    for row in reader:

        try: row[1] = map_1[row[1]]

        except KeyError: continue # skip row

        writer.writerow(row)

main()

#!/usr/bin/env python

# Removes duplication from XPath expressions

# Filters one XPath per line from stdin to stdout

import re

import sys

def main():

    while True:

        line = sys.stdin.readline()

        if line == '': break

        # Forward * abbrs

        line = re.sub(r'(?i)(\w+)(?=\w*(?:->/[^/]*)?/\1\b)', r'*', line)

        # Backward * abbrs

        line = re.sub(r'(?i)((\w+)->/[^/]*/[^/]*\[)\2', r'\1*', line)

        sys.stdout.write(line)

main()

#!/usr/bin/env python

# Converts a map spreadsheet to human-readable (but machine unusable) form

# Usage: self <in_map >out_ma

import csv

import re

import sys

def sub_nested(regex, repl, str_):

    while True:

        str_, n = re.subn(regex, repl, str_)

        if n == 0: return str_

def cleanup(xpath):

    truncated = False

    # Remove attrs

    xpath = sub_nested(r':\[[^\[\]]*?\]', r'', xpath)

    # Remove lookahead assertions

    xpath = sub_nested(r'\((/[^\)]*?)\)(?=/)', r'\1', xpath)

    # Remove pointers

    xpath, n = re.subn(r'^.*->', r'', xpath)

    if n > 0: truncated = True

    # Remove part of path before first key list, XML function, or path end

    # Leave enough to include the table of a user-defined value

    xpath, n = re.subn(r'^(?:/(?!_)[\w*]+)*(?=(?:/(?!_)[\w*]+){2}(?:\[|/_|$))',

        r'', xpath)

    # Prepend / to show truncation

    if n > 0: truncated = True

    # Remove backward (child-to-parent) pointer's target ID attr

    xpath = re.sub(r'\[[\w*]+\]|(?<=\[)[\w*]+,', r'', xpath)

    # Remove negative keys

    xpath = re.sub(r',?!(?:[\w*]+/)*@?[\w*]+', r'', xpath)

    # Remove path before key

    xpath = re.sub(r'(?:[\w*]+/)*(@?[\w*]+)(?==)', r'\1', xpath)

    # Prepend / to show truncation

    if truncated: xpath = '/'+xpath

    return xpath

def main():

    # Convert map

    reader = csv.reader(sys.stdin)

    writer = csv.writer(sys.stdout)

    writer.writerow(reader.next())

    for row in reader:

        for i in xrange(2): row[i] = cleanup(row[i])

        writer.writerow(row)

main()

#!/bin/sh

# Sets correct permissions on shared bien files

selfDir="$(dirname -- "$0")" # dir of symlink $0, not this script itself

cd "$selfDir"

opts='--quiet --changes'

chgrp --recursive --no-dereference $opts bien .

find . -type d -exec chmod $opts g+s "{}" \; # only dirs should be setgid

for perm in r w x; do # extend user perms to group

    find . -not -type l -perm -u=$perm -exec chmod $opts g+$perm "{}" \;

        # don't chmod symlinks

done

#!/usr/bin/env python

# Modifies a map spreadsheet A->B or any file using a replacements spreadsheet

# A->C or B->C

import csv

import re

import sys

def main():

    try: _prog_name, repl_path = sys.argv[:2]

    except ValueError:

        raise SystemExit('Usage: '+sys.argv[0]+' <map repl [col_num] [| '+

            sys.argv[0]+' repl_1 [col_num_1]]... >new_map')

    col_num = None

    try: col_num = sys.argv[2]

    except IndexError: pass

    if col_num != None: col_num = int(col_num) # 0-based

    # Get replacements

    repls = []

    stream = open(repl_path, 'rb')

    reader = csv.reader(stream)

    repl_in, repl_out = reader.next()[:2]

    for row in reader:

        in_, out = row[:2]

        if in_ != '':

            if re.match(r'^\w+$', in_): in_ = r'\b'+in_+r'(?=(?:(?i)_id)?\b)'

                # match whole word

            repls.append((in_, out))

    stream.close()

    def repl_all(str_):

        for repl, with_ in repls: str_ = re.sub(repl, with_, str_)

        return str_

    # Modify map or file

    if col_num != None:

        reader = csv.reader(sys.stdin)

        writer = csv.writer(sys.stdout)

        cols = reader.next()

        label, sep, root = cols[col_num].partition(':')

        if label != repl_in: raise SystemExit('Map error: Map column '+

            str(col_num)+' label "'+label+'" doesn\'t match replacements input '

            'column label "'+repl_in+'"')

        cols[col_num] = repl_out+sep+repl_all(root)

        writer.writerow(cols)

        for row in reader:

            row[col_num] = repl_all(row[col_num])

            writer.writerow(row)

    else:

        while True:

            line = sys.stdin.readline()

            if line == '': break

            sys.stdout.write(repl_all(line))

main()

#!/bin/bash

# Inner-joins two map spreadsheets and sorts the output on the output column

selfDir="$(dirname -- "$0")"

"$selfDir/join" "$@"|"$selfDir/sort" 1 0

#!/usr/bin/env python

# Sorts a spreadsheet based on a column

import csv

import sys

def by_idx(list_, idxes): return [list_[i] for i in idxes]

def main():

    col_nums = sys.argv[1:]

    if col_nums == []:

        raise SystemExit('Usage: '+sys.argv[0]+' col_num... <map >sorted_map')

    col_nums = map(int, col_nums) # 0-based

    # Sort map

    reader = csv.reader(sys.stdin)

    cols = reader.next()

    def get_sort_by(row): return by_idx(row, col_nums)

    rows = list(reader)

    rows.sort(lambda *rows: cmp(*map(get_sort_by, rows)))

    writer = csv.writer(sys.stdout)

    writer.writerow(cols)

    map(writer.writerow, rows)

main()

#!/usr/bin/env python

# Combines two map spreadsheets A0->B and A1->C to A->B, with A0 overwriting A1

import csv

import sys

def main():

    try: _prog_name, map_1_path = sys.argv

    except ValueError:

        raise SystemExit('Usage: '+sys.argv[0]+' <map_0 map_1 [| '+sys.argv[0]

            +' map_2]... >union_map')

    map_ = {}

    def add_map(reader):

        for row in reader:

            if row[1] != '': map_[row[0]] = row[1:]

    # Get map 1

    stream = open(map_1_path, 'rb')

    reader = csv.reader(stream)

    map_1_cols = reader.next()

    add_map(reader)

    stream.close()

    # Add map 0 to map 1, overwriting existing entries

    reader = csv.reader(sys.stdin)

    map_0_cols = reader.next()

    if not map_0_cols[0] == map_1_cols[0]: raise SystemExit('Map error: '

        'Map 1 column 0 name doesn\'t match map 0 column 0 name')

    add_map(reader)

    # Write combined map

    writer = csv.writer(sys.stdout)

    writer.writerow(map_0_cols)

    for in_, rest in map_.iteritems(): writer.writerow([in_]+rest)

main()

#!/bin/sh

# Joins two map spreadsheets A->B0 and B0->B1 to A->B1, B1 a modification of B0

selfDir="$(dirname -- "$0")"

test "$#" -eq 2 || { echo "Usage: $0 map_0 map_1 >joined_map" >&2; exit 2;}

"$selfDir/join" <"$1" "$2"|"$selfDir/union" "$1"

test -n "$isNimoy" && . ../util/env_password mysql_password "your MySQL"

vegbienDest=../util/vegbien_dest

util := $(root)/util

	$(util)/join_sort <$< $(mappings)/VegX-VegBIEN.$(*F).csv >$@

psqlAsBien := ./util/psql_vegbien $(psqlOpts)

utilDir="$selfDir/util"

                . "$utilDir/env_password" mysql_password "your MySQL"

test _"$out_database" = _vegbien && . "$utilDir/vegbien_dest"

"$utilDir/map" "$@"

    ../util/join <"$map" "$1-$2.$type.csv"|\

    ../util/sort 1 0 >"${map/$1/$2}" # sort on output col

    ../util/review <"$in" >"for_review/$in"

repl = ../util/repl

sort = ../util/sort 1 0

chRoot = ../util/ch_root <$< >$@