#!/usr/bin/env python

# Scrubs the taxonlabels in VegBIEN using TNRS.

# runtime: 162 ms/name ("real 458m50.126s" for "169,539 name(s)" [1])

# [1] $ tail -c +12953807 ../inputs/.TNRS/tnrs/logs/tnrs.make.log.sql|head -15

# total runtime: 10 days ("Rows (counted) 5221748" (TNRS.tnrs @r9998)

# * 162 ms/name (above) * 1s/1000ms * 1h/3600s * 1day/24h = 9.79 days)

# to estimate total runtime:

# bin/psql_vegbien <<<'SELECT COUNT(*) FROM tnrs_input_name'

# # names from above * 1.5 multiplier for scrubbing accepted names

# (the test_taxonomic_names sample from Brad produces 8 accepted names for

# 15 input names)

# * ((# ms/name from log file * 1 sec/1000 ms) + (# sec to run

#   `SELECT * FROM "tnrs_input_name"` in log file / tnrs.max_names names/batch))

# * 1 hr / 3600 sec * 1 day / 24 hr = # days

import os.path

import sys

sys.path.append(os.path.dirname(__file__)+"/../lib")

import csvs

import opts

import profiling

import sql

import sql_gen

import sql_io

import strings

import tnrs

tnrs_input = sql_gen.Table('tnrs_input_name')

tnrs_batch = sql_gen.Table('batch')

tnrs_data = sql_gen.Table('tnrs')

def main():

    # Input

    env_names = []

    db_config = opts.get_env_vars(sql.db_config_names, None, env_names)

    verbosity = float(opts.get_env_var('verbosity', 3, env_names))

    if not 'engine' in db_config: raise SystemExit('Usage: '

        +opts.env_usage(env_names)+' '+sys.argv[0]+' 2>>log')

    def log(msg, level=1):

        '''Higher level -> more verbose'''

        if level <= verbosity:

            sys.stderr.write(strings.to_raw_str(msg.rstrip('\n')+'\n'))

    # Connect to DB

    db = sql.connect(db_config, log_debug=log)

    cumulative_tnrs_profiler = profiling.ItersProfiler(iter_text='name')

    # Iterate over unscrubbed verbatim taxonlabels

    while True:

        # Fetch next set

        cur = sql.select(db, tnrs_input, limit=tnrs.max_names, cacheable=False)

        this_ct = cur.rowcount

        log('Processing '+str(this_ct)+' taxonlabels')

        if this_ct == 0: break

        # otherwise, rows found

        names = list(sql.values(cur))

        def process():

            # Run TNRS

            log('Making TNRS request')

            stream = tnrs.tnrs_request(names,

                cumulative_profiler=cumulative_tnrs_profiler)

            log('Storing TNRS response data')

            sql.insert(db, tnrs_batch, []) # time_submitted is autopopulated

            sql_io.append_csv(db, tnrs_data, *csvs.reader_and_header(stream))

        # start transaction *before* submitting data, so Time_submitted is

        # correctly set to the submission time rather than the insertion time.

        # these may differ by several minutes if TNRS is slow.

        sql.with_savepoint(db, process)

main()