#!/usr/bin/env python

# Scrubs the taxonlabels in VegBIEN using TNRS.

# Runs continuously until no new rows are added after max_pause.

import os.path

import StringIO

import sys

import time

sys.path.append(os.path.dirname(__file__)+"/../lib")

import csvs

import dates

import opts

import profiling

import sql

import sql_gen

import sql_io

import streams

import strings

import tnrs

# Config

pause = 2*60*60 # sec; = 2 hr

max_pause = 9*60*60 # sec; = 9 hr; must be >= max partition import time (1.5 hr)

assert pause <= max_pause

tnrs_input = sql_gen.Table('tnrs_input_name')

tnrs_data = sql_gen.Table('tnrs')

time_col_name = 'Time_submitted'

def main():

    # Input

    env_names = []

    db_config = opts.get_env_vars(sql.db_config_names, None, env_names)

    verbosity = float(opts.get_env_var('verbosity', 3, env_names))

    wait = opts.env_flag('wait', False, env_names)

    if not 'engine' in db_config: raise SystemExit('Usage: '

        +opts.env_usage(env_names)+' '+sys.argv[0]+' 2>>log')

    def log(msg, level=1):

        '''Higher level -> more verbose'''

        if level <= verbosity:

            sys.stderr.write(strings.to_raw_str(msg.rstrip('\n')+'\n'))

    # Connect to DB

    db = sql.connect(db_config, log_debug=log)

    tnrs_profiler = profiling.ItersProfiler(iter_text='name')

    # Iterate over unscrubbed verbatim taxonlabels

    total_pause = 0

    while True:

        # Fetch next set

        cur = sql.select(db, tnrs_input, limit=tnrs.max_names, cacheable=False)

        this_ct = cur.rowcount

        log('Processing '+str(this_ct)+' taxonlabels')

        if this_ct == 0:

            if not wait: break

            log('Waited '+str(total_pause)+' sec total')

            total_pause += pause

            if total_pause > max_pause: break

            log('Waiting '+str(pause)+' sec...')

            time.sleep(pause) # wait for more rows

            continue # try again

        # otherwise, rows found

        total_pause = 0

        names = list(sql.values(cur))

        def process():

            # Run TNRS

            log('Making TNRS request')

            now_str = str(dates.now())

            tnrs_profiler.start()

            try: stream = tnrs.repeated_tnrs_request(names)

            finally:

                tnrs_profiler.stop(iter_ct=this_ct)

                log('Cumulatively: '+tnrs_profiler.msg())

            log('Storing TNRS response data')

            reader, header = csvs.reader_and_header(stream)

            header.insert(0, time_col_name)

            reader = csvs.ColInsertFilter(reader, now_str)

            sql_io.append_csv(db, tnrs_data, reader, header)

        # start transaction *before* submitting data, so Time_submitted is

        # correctly set to the submission time rather than the insertion time.

        # these may differ by several minutes if TNRS is slow.

        sql.with_savepoint(db, process)

main()