#!/usr/bin/env python

# Loads a command's CSV output stream into a PostgreSQL table.

# The command may be run more than once.

import csv

import os.path

import re

import subprocess

import sys

sys.path.append(os.path.dirname(__file__)+"/../lib")

import csvs

import exc

import opts

import sql

import streams

import strings

import util

def main():

    # Usage

    env_names = []

    def usage_err():

        raise SystemExit('Usage: '+opts.env_usage(env_names)+' '+sys.argv[0]

            +' input_cmd [args...]')

    # Parse args

    input_cmd = sys.argv[1:]

    if input_cmd == []: usage_err()

    # Get config from env vars

    table = opts.get_env_var('table', None, env_names)

    schema = opts.get_env_var('schema', 'public', env_names)

    db_config = opts.get_env_vars(sql.db_config_names, None, env_names)

    debug = opts.env_flag('debug', False, env_names)

    if not (table != None and 'engine' in db_config): usage_err()

    # Connect to DB

    db = sql.connect(db_config)

    use_copy_from = [True]

    # Loads data into the table using the currently-selected approach.

    def load_():

        # Open input stream

        proc = subprocess.Popen(input_cmd, stdout=subprocess.PIPE, bufsize=-1)

        in_ = proc.stdout

        # Get format info

        info = csvs.stream_info(in_, parse_header=True)

        dialect = info.dialect

        if csvs.is_tsv(dialect): use_copy_from[0] = False

        cols = info.header

        for i, col in enumerate(cols): # replace empty column names

            if col == '': cols[i] = 'column_'+str(i)

        # Select schema and escape names

        def esc_name(name): return sql.esc_name(db, name)

        sql.run_query(db, 'SET search_path TO '+esc_name(schema))

        esc_table = esc_name(table)

        esc_cols = map(esc_name, cols)

        # Create CREATE TABLE statement

        pkey = esc_name(table+'_pkey')

        create_table = 'CREATE TABLE '+esc_table+' (\n'

        create_table += '    row_num serial NOT NULL PRIMARY KEY\n'

        for esc_col in esc_cols: create_table += '    , '+esc_col+' text\n'

        create_table += ');\n'

        if debug: sys.stderr.write(create_table)

        # Create table

        sql.run_query(db, create_table)

        # Create COPY FROM statement

        if use_copy_from[0]:

            cur = db.db.cursor()

            copy_from = ('COPY '+esc_table+' ('+(', '.join(esc_cols))

                +') FROM STDIN DELIMITER %(delimiter)s NULL %(null)s')

            assert not csvs.is_tsv(dialect)

            copy_from += ' CSV'

            if dialect.quoting != csv.QUOTE_NONE:

                copy_from += ' QUOTE %(quotechar)s'

                if dialect.doublequote: copy_from += ' ESCAPE %(quotechar)s'

            copy_from += ';\n'

            copy_from = cur.mogrify(copy_from, dict(delimiter=dialect.delimiter,

                null='', quotechar=dialect.quotechar))

            if debug: sys.stderr.write(copy_from)

        # Load the data

        line_in = streams.ProgressInputStream(in_, sys.stderr,

            'Processed %d row(s)', n=1000)

        try:

            if use_copy_from[0]:

                sys.stderr.write('Using COPY FROM\n')

                db.db.cursor().copy_expert(copy_from, line_in)

            else:

                sys.stderr.write('Using INSERT\n')

                cols_ct = len(cols)+1 # +1 for row_num

                for row in csvs.make_reader(line_in, dialect):

                    row = map(strings.to_unicode, row)

                    row.insert(0, sql.default) # leave space for autogen row_num

                    util.list_set_length(row, cols_ct) # truncate extra cols

                    sql.insert(db, table, row)

        finally:

            line_in.close() # also closes proc.stdout

            proc.wait()

        # Clean up the data

        sys.stderr.write('Cleaning up table\n')

        sql.cleanup_table(db, table, cols)

    load = lambda: sql.with_savepoint(db, load_)

    try: load()

    except sql.DatabaseErrors, e:

        if use_copy_from[0]: # first try

            exc.print_ex(e, plain=True)

            use_copy_from[0] = False

            load() # try again with different approach

        else: raise e

    db.db.commit()

main()