#!/usr/bin/env python

# Scrubs the taxonpaths in VegBIEN using TNRS.

# Runs continuously until no new rows are added after max_pause.

import os.path

import StringIO

import sys

import time

sys.path.append(os.path.dirname(__file__)+"/../lib")

import csvs

import opts

import profiling

import sql

import sql_gen

import sql_io

import streams

import strings

import tnrs

# Config

pause = 30*60 # sec; = 30 min

max_pause = 2*60*60 # sec; = 2 hr; must be >= max import time of one partition

assert pause <= max_pause

tnrs_data = sql_gen.Table('tnrs')

def main():

    # Input

    env_names = []

    db_config = opts.get_env_vars(sql.db_config_names, None, env_names)

    verbosity = float(opts.get_env_var('verbosity', 3, env_names))

    if not 'engine' in db_config: raise SystemExit('Usage: '

        +opts.env_usage(env_names)+' '+sys.argv[0]+' 2>>log')

    def log(msg, level=1):

        '''Higher level -> more verbose'''

        if level <= verbosity:

            sys.stderr.write(strings.to_raw_str(msg.rstrip('\n')+'\n'))

    # Connect to DB

    db = sql.connect(db_config, log_debug=log)

    tnrs_profiler = profiling.ItersProfiler(iter_text='name')

    # Iterate over unscrubbed verbatim taxonpaths

    total_pause = 0

    tables = ['taxonpath', sql_gen.Join('tnrs',

        {'Name_submitted': 'taxonomicnamewithauthor'}, sql_gen.filter_out)]

    while True:

        # Fetch next set

        cur = sql.select(db, tables, ['taxonomicnamewithauthor'],

            [('canon_taxonpath_id', None)], limit=tnrs.max_names,

            cacheable=False)

        this_ct = cur.rowcount

        if this_ct == 0:

            total_pause += pause

            if total_pause > max_pause: break

            log('Waited '+str(total_pause)+' sec. Waiting...')

            time.sleep(pause) # wait for more rows

            continue # try again

        # otherwise, rows found

        total_pause = 0

        names = list(sql.values(cur))

        # Run TNRS

        log('Processing '+str(this_ct)+' taxonpaths')

        log('Making TNRS request')

        tnrs_profiler.start()

        try:

            try: stream = tnrs.repeated_tnrs_request(names)

            finally:

                tnrs_profiler.stop(iter_ct=this_ct)

                log('Cumulatively: '+tnrs_profiler.msg())

        except tnrs.InvalidResponse: pass # skip set in case it caused error

        else:

            log('Storing TNRS response data')

            stream_info = csvs.stream_info(stream, parse_header=True)

            stream = streams.ProgressInputStream(stream, sys.stderr, n=1000)

            sql_io.append_csv(db, tnrs_data, stream_info, stream)

main()