#!/usr/bin/env python

# Maps one datasource to another, using a map spreadsheet if needed

# For outputting an XML file to a PostgreSQL database, use the general format of

# http://vegbank.org/vegdocs/xml/vegbank_example_ver1.0.2.xml

import os.path

import sys

import xml.dom.minidom as minidom

sys.path.append(os.path.dirname(__file__)+"/../lib")

import exc

import opts

import Parser

import sql

import strings

import util

import xml_dom

import xml_func

def metadata_value(name):

    if type(name) == str and name.startswith(':'): return name[1:]

    else: return None

def main():

    ex_tracker = exc.ExTracker()

    env_names = []

    def usage_err():

        raise SystemExit('Usage: ' + opts.env_usage(env_names, True)

            +' [commit=1] [test=1] [verbose=1] [debug=1] '+sys.argv[0]

            +' [map_path...] [<input] [>output]')

    limit = opts.get_env_var('n', None, env_names)

    if limit != None: limit = int(limit)

    test = opts.env_flag('test')

    commit = not test and opts.env_flag('commit') # never commit in test mode

    debug = opts.env_flag('debug')

    verbose = debug or opts.env_flag('verbose')

    def log(msg, on=verbose):

        if on: sys.stderr.write(msg)

    def log_start(action, on=verbose): log(action+'...', on)

    def log_done(on=verbose): log('Done\n', on)

    # Get db config from env vars

    db_config_names = ['engine', 'host', 'user', 'password', 'database']

    def get_db_config(prefix):

        return opts.get_env_vars(db_config_names, prefix, env_names)

    in_db_config = get_db_config('in')

    out_db_config = get_db_config('out')

    in_is_db = 'engine' in in_db_config

    out_is_db = 'engine' in out_db_config

    # Parse args

    map_paths = sys.argv[1:]

    if map_paths == []:

        if in_is_db or not out_is_db: usage_err()

        else: map_paths = [None]

    def connect_db(db_config):

        log_start('Connecting to '+sql.db_config_str(db_config))

        db = sql.connect(db_config)

        log_done()

        return db

    out_is_xml_ref = [False]

    def process_input(root, process_row, map_path):

        '''Inputs datasource to XML tree, mapping if needed'''

        # Load map header

        in_is_xpaths = True

        out_is_xpaths = True

        out_label = None

        if map_path != None:

            import copy

            import csv

            import xpath

            metadata = []

            mappings = []

            stream = open(map_path, 'rb')

            reader = csv.reader(stream)

            in_label, out_label = reader.next()[:2]

            def split_col_name(name):

                name, sep, root = name.partition(':')

                return name, sep != '', root

            in_label, in_is_xpaths, in_root = split_col_name(in_label)

            out_label, out_is_xpaths, out_root = split_col_name(out_label)

            has_types = out_root.startswith('/*s/') # outer elements are types

            for row in reader:

                in_, out = row[:2]

                if out != '':

                    if out_is_xpaths: out = xpath.parse(out_root+out)

                    mappings.append((in_, out))

            stream.close()

            root.ownerDocument.documentElement.tagName = out_label

        in_is_xml = in_is_xpaths and not in_is_db

        out_is_xml_ref[0] = out_is_xpaths and not out_is_db

        if in_is_xml:

            doc0 = minidom.parse(sys.stdin)

            if out_label == None: out_label = doc0.documentElement.tagName

        def process_rows(get_value, rows):

            '''Processes input values

            @param get_value f(in_, row):str

            '''

            for i, row in enumerate(rows):

                if not (limit == None or i < limit): break

                row_id = str(i)

                for in_, out in mappings:

                    value = metadata_value(in_)

                    if value == None:

                        log_start('Getting '+str(in_), debug)

                        value = get_value(in_, row)

                        log_done(debug)

                    if value != None: xpath.put_obj(root, out, row_id,

                        has_types, strings.cleanup(value))

                process_row(row)

            sys.stderr.write('Processed '+str(i+1)+' input rows\n')

        if map_path == None:

            iter_ = xml_dom.NodeElemIter(doc0.documentElement)

            util.skip(iter_, xml_dom.is_text) # skip metadata

            for child in iter_:

                root.appendChild(child)

                process_row(child)

        elif in_is_db:

            assert in_is_xpaths

            import db_xml

            in_root_xml = xpath.path2xml(in_root)

            for i, mapping in enumerate(mappings):

                in_, out = mapping

                if metadata_value(in_) == None:

                    mappings[i] = (xpath.path2xml(in_root+'/'+in_), out)

            in_db = connect_db(in_db_config)

            in_pkeys = {}

            def get_value(in_, row):

                pkey, = row

                in_ = in_.cloneNode(True) # don't modify orig value!

                xml_dom.set_id(xpath.get(in_, in_root), pkey)

                value = sql.value_or_none(db_xml.get(in_db, in_, in_pkeys))

                if value != None: return str(value)

                else: return None

            process_rows(get_value, sql.rows(db_xml.get(in_db, in_root_xml,

                in_pkeys, limit)))

            in_db.close()

        elif in_is_xml:

            def get_value(in_, row):

                node = xpath.get(row, in_)

                if node != None: return xml_dom.value(node)

                else: return None

            row0 = xpath.get(doc0.documentElement, in_root)

            process_rows(get_value, xml_dom.NodeElemIter(row0.parentNode))

        else: # input is CSV

            map_ = dict(mappings)

            reader = csv.reader(sys.stdin)

            cols = reader.next()

            col_idxs = dict([(value, idx) for idx, value in enumerate(cols)])

            for i, mapping in enumerate(mappings):

                in_, out = mapping

                if metadata_value(in_) == None:

                    try: mappings[i] = (col_idxs[in_], out)

                    except KeyError: pass

            def get_value(in_, row):

                value = row[in_]

                if value != '': return value

                else: return None

            process_rows(get_value, reader)

    def process_inputs(root, process_row):

        for map_path in map_paths: process_input(root, process_row, map_path)

    # Output XML tree

    doc = xml_dom.create_doc()

    root = doc.documentElement

    if out_is_db:

        import db_xml

        out_db = connect_db(out_db_config)

        out_pkeys = {}

        try:

            if test: sql.empty_db(out_db)

            row_ct_ref = [0]

            def process_row(input_row):

                def on_error(e):

                    exc.add_msg(e, 'output row:\n'+str(root))

                    exc.add_msg(e, 'input row:\n'+str(input_row))

                    ex_tracker.track(e)

                xml_func.process(root, on_error)

                if not xml_dom.is_empty(root):

                    assert xml_dom.has_one_child(root)

                    try:

                        sql.with_savepoint(out_db, lambda: db_xml.put(out_db,

                            root.firstChild, out_pkeys, row_ct_ref, on_error))

                        if commit: out_db.commit()

                    except sql.DatabaseErrors, e: on_error(e)

                root.clear()

            process_inputs(root, process_row)

            sys.stdout.write('Inserted '+str(row_ct_ref[0])+

                ' new rows into database\n')

        finally:

            out_db.rollback()

            out_db.close()

    else:

        def process_row(input_row): pass

        process_inputs(root, process_row)

        xml_func.process(root)

        if out_is_xml_ref[0]:

            doc.writexml(sys.stdout, **xml_dom.prettyxml_config)

        else: # output is CSV

            raise NotImplementedError('CSV output not supported yet')

try: main()

except Parser.SyntaxException, e: raise SystemExit(str(e))