#!/usr/bin/env python

# Scrubs the taxonpaths in VegBIEN using TNRS.

# Runs continuously until no new rows are added after max_pause.

import os.path

import StringIO

import sys

import time

sys.path.append(os.path.dirname(__file__)+"/../lib")

import csvs

import opts

import profiling

import sql

import sql_gen

import sql_io

import streams

import strings

import tnrs

# Config

pause = 60 # sec

max_pause = 2*60*60 # sec; = 2 hr; must be >= max import time of one partition

assert pause <= max_pause

tnrs_data = sql_gen.Table('tnrs')

def main():

    # Input

    env_names = []

    db_config = opts.get_env_vars(sql.db_config_names, None, env_names)

    verbosity = float(opts.get_env_var('verbosity', 3, env_names))

    if not 'engine' in db_config: raise SystemExit('Usage: '

        +opts.env_usage(env_names)+' '+sys.argv[0]+' 2>>log')

    def log(msg, level=1):

        '''Higher level -> more verbose'''

        if level <= verbosity:

            sys.stderr.write(strings.to_raw_str(msg.rstrip('\n')+'\n'))

    # Connect to DB

    db = sql.connect(db_config, log_debug=log)

    tnrs_profiler = profiling.ItersProfiler(iter_text='row')

    try:

        # Iterate over unscrubbed verbatim taxonpaths

        start = 0

        total_pause = 0

        while True:

            # Fetch next set

            cur = sql.select(db, 'taxonpath', ['taxonomicnamewithauthor'],

                [('canon_taxonpath_id', None)], limit=tnrs.max_names,

                start=start, cacheable=False)

            this_ct = cur.rowcount

            start += this_ct # advance start to fetch next set

            if this_ct == 0:

                total_pause += pause

                if total_pause > max_pause: break

                log('Waited '+str(total_pause)+' sec. Waiting...')

                time.sleep(pause) # wait for more rows

                continue # try again

            # otherwise, rows found

            total_pause = 0

            names = list(sql.values(cur))

            # Run TNRS

            log('Processing '+str(this_ct)+' taxonpaths')

            log('Making TNRS request')

            tnrs_profiler.start()

            try:

                try: stream = tnrs.repeated_tnrs_request(names)

                finally: tnrs_profiler.stop(iter_ct=this_ct)

            except tnrs.InvalidResponse: pass # skip set in case it caused error

            else:

                log('Storing TNRS response data')

                stream_info = csvs.stream_info(stream, parse_header=True)

                stream = streams.ProgressInputStream(stream, sys.stderr, n=1000)

                sql_io.append_csv(db, tnrs_data, stream_info, stream)

    finally:

        log(tnrs_profiler.msg())

main()