#!/usr/bin/env python

# Scrubs the taxonlabels in VegBIEN using TNRS.

import os.path

import StringIO

import sys

import time

sys.path.append(os.path.dirname(__file__)+"/../lib")

import csvs

import dates

import opts

import profiling

import sql

import sql_gen

import sql_io

import streams

import strings

import tnrs

tnrs_input = sql_gen.Table('tnrs_input_name')

tnrs_data = sql_gen.Table('tnrs')

def main():

    # Input

    env_names = []

    db_config = opts.get_env_vars(sql.db_config_names, None, env_names)

    verbosity = float(opts.get_env_var('verbosity', 3, env_names))

    if not 'engine' in db_config: raise SystemExit('Usage: '

        +opts.env_usage(env_names)+' '+sys.argv[0]+' 2>>log')

    def log(msg, level=1):

        '''Higher level -> more verbose'''

        if level <= verbosity:

            sys.stderr.write(strings.to_raw_str(msg.rstrip('\n')+'\n'))

    # Connect to DB

    db = sql.connect(db_config, log_debug=log)

    cumulative_tnrs_profiler = profiling.ItersProfiler(iter_text='name')

    # Iterate over unscrubbed verbatim taxonlabels

    while True:

        # Fetch next set

        cur = sql.select(db, tnrs_input, limit=tnrs.max_names, cacheable=False)

        this_ct = cur.rowcount

        log('Processing '+str(this_ct)+' taxonlabels')

        if this_ct == 0: break

        # otherwise, rows found

        names = list(sql.values(cur))

        def process():

            # Run TNRS

            log('Making TNRS request')

            cumulative_tnrs_profiler.start()

            try: stream = tnrs.tnrs_request(names)

            finally:

                cumulative_tnrs_profiler.stop(iter_ct=len(names))

                log('Cumulatively: '+cumulative_tnrs_profiler.msg())

            log('Storing TNRS response data')

            sql_io.append_csv(db, tnrs_data, *csvs.reader_and_header(stream))

        # start transaction *before* submitting data, so Time_submitted is

        # correctly set to the submission time rather than the insertion time.

        # these may differ by several minutes if TNRS is slow.

        sql.with_savepoint(db, process)

main()