Project

General

Profile

1
#!/usr/bin/env python
2
# Scrubs the taxonlabels in VegBIEN using TNRS.
3

    
4
# runtime: 162 ms/name ("real 458m50.126s" for "169,539 name(s)" [1])
5
# [1] $ tail -c +12953807 ../inputs/.TNRS/tnrs/logs/tnrs.make.log.sql|head -15
6

    
7
# total runtime: 10 days ("Rows (counted) 5221748" (TNRS.tnrs @r9998)
8
# * 162 ms/name (above) * 1s/1000ms * 1h/3600s * 1day/24h = 9.79 days)
9

    
10
# to estimate total runtime:
11
# bin/psql_vegbien <<<'SELECT COUNT(*) FROM tnrs_input_name'
12
# # names from above * 1.5 multiplier for scrubbing accepted names
13
# (the test_taxonomic_names sample from Brad produces 8 accepted names for
14
# 15 input names)
15
# * ((# ms/name from log file * 1 sec/1000 ms) + (# sec to run
16
#   `SELECT * FROM "tnrs_input_name"` in log file / tnrs.max_names names/batch))
17
# * 1 hr / 3600 sec * 1 day / 24 hr = # days
18

    
19
import os.path
20
import sys
21

    
22
sys.path.append(os.path.dirname(__file__)+"/../lib")
23

    
24
import csvs
25
import opts
26
import profiling
27
import sql
28
import sql_gen
29
import sql_io
30
import strings
31
import tnrs
32

    
33
tnrs_input = sql_gen.Table('tnrs_input_name')
34
tnrs_data = sql_gen.Table('tnrs')
35

    
36
def main():
37
    # Input
38
    env_names = []
39
    db_config = opts.get_env_vars(sql.db_config_names, None, env_names)
40
    verbosity = float(opts.get_env_var('verbosity', 3, env_names))
41
    if not 'engine' in db_config: raise SystemExit('Usage: '
42
        +opts.env_usage(env_names)+' '+sys.argv[0]+' 2>>log')
43
    
44
    def log(msg, level=1):
45
        '''Higher level -> more verbose'''
46
        if level <= verbosity:
47
            sys.stderr.write(strings.to_raw_str(msg.rstrip('\n')+'\n'))
48
    
49
    # Connect to DB
50
    db = sql.connect(db_config, log_debug=log)
51
    
52
    cumulative_tnrs_profiler = profiling.ItersProfiler(iter_text='name')
53
    
54
    # Iterate over unscrubbed verbatim taxonlabels
55
    while True:
56
        # Fetch next set
57
        cur = sql.select(db, tnrs_input, limit=tnrs.max_names, cacheable=False)
58
        this_ct = cur.rowcount
59
        log('Processing '+str(this_ct)+' taxonlabels')
60
        if this_ct == 0: break
61
        # otherwise, rows found
62
        names = list(sql.values(cur))
63
        
64
        def process():
65
            # Run TNRS
66
            log('Making TNRS request')
67
            stream = tnrs.tnrs_request(names,
68
                cumulative_profiler=cumulative_tnrs_profiler)
69
            
70
            log('Storing TNRS response data')
71
            sql_io.append_csv(db, tnrs_data, *csvs.reader_and_header(stream))
72
        # start transaction *before* submitting data, so Time_submitted is
73
        # correctly set to the submission time rather than the insertion time.
74
        # these may differ by several minutes if TNRS is slow.
75
        sql.with_savepoint(db, process)
76

    
77
main()
(77-77/85)