Project

General

Profile

1
# TNRS
2

    
3
import re
4
import sys
5
import time
6
import urllib2
7

    
8
import streams
9

    
10
# Config
11
initial_pause = 0.35 # sec
12
pause_growth_factor = 1.3
13
max_pause = 600 # sec; = 10 min
14
assert initial_pause <= max_pause
15
max_taxons = 5000# according to http://tnrs.iplantcollaborative.org/TNRSapp.html
16
max_taxons = 500 # the maximum above crashes the TNRS server
17
max_tries = 5
18

    
19
# Protocol params
20
url_base = 'http://tnrs.iplantcollaborative.org/tnrsdemo/'
21
url = url_base+'search'
22
initial_headers = {
23
    'Content-Type': 'text/x-gwt-rpc; charset=utf-8',
24
    'X-GWT-Module-Base': url_base,
25
    'X-GWT-Permutation': '574AA16D15D917C7704646FD92AFF6B3',
26
}
27
submission_request_template = ('7|0|7|'+url_base+
28
'||org.iplantc.tnrs.demo.client.SearchService|doSearch|\
29
java.lang.String/2004016611|{"sources":"gcc,tropicos,usda", "names":"[taxons]"\
30
, "type":"matching", "taxonomic":"true", "classification":"tropicos", \
31
"match_to_rank":"true"}|0.05|1|2|3|4|2|5|5|6|7|')
32
submission_response_pattern = r'^//OK\[1,\["(\w+)"\],0,7\]$'
33
retrieval_request_template = ('7|0|15|'+url_base+
34
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService\
35
|getRemoteData|com.extjs.gxt.ui.client.data.PagingLoadConfig|\
36
java.lang.String/2004016611|com.extjs.gxt.ui.client.data.BasePagingLoadConfig/\
37
2011366567|com.extjs.gxt.ui.client.data.RpcMap/3441186752|sortField|sortDir|\
38
com.extjs.gxt.ui.client.Style$SortDir/640452531|offset|java.lang.Integer/\
39
3438268394|limit|{"email":"tnrs@lka5jjs.orv", "key":"[key]", \
40
"taxonomic_constraint":"false", "source_sorting":"false", "first":"false"}\
41
|1|2|3|4|2|5|6|7|0|1|8|4|9|0|10|11|0|12|13|0|14|13|100|15|')
42
retrieval_response_pattern = '^//OK\[.*?\["com.extjs.gxt.ui.client.data.\
43
BasePagingLoadResult/496878394","java.util.ArrayList/4159755760","org.iplantc.\
44
tnrs.demo.shared.BeanTNRSEntry/1039545748",".*"\],0,7\]$'
45
retrieval_response_info_pattern = r'(?ms).*^Set-Cookie: JSESSIONID=(\w+);'
46
download_request_template = ('7|0|6|'+url_base+
47
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
48
downloadRemoteResults|java.lang.String/2004016611|{"name":"tnrs_results.txt", \
49
"mode":"Best", "type":"Detailed", "encoding":"utf8", "dirty":"false", \
50
"sources":"false", "taxonomic":"false", "email":"tnrs@lka5jjs.orv", \
51
"key":"[key]"}|1|2|3|4|1|5|6|')
52
download_response_pattern = '^//OK\[1,\["(.*)"\],0,7\]$'
53
download_url_suffix = '&name=tnrs_results.txt&encoding=utf8'
54

    
55
class InvalidResponse(Exception): pass
56

    
57
def gwt_encode(str_): return re.sub(r'[^\w.() -]+', r' ', str_)
58

    
59
def parse_response(name, pattern, response):
60
    match = re.match(pattern, response)
61
    if not match: raise InvalidResponse('Invalid '+name+' response:\n'+response)
62
    return match.groups()
63

    
64
def tnrs_request(taxons, debug=False):
65
    assert len(taxons) <= max_taxons
66
    
67
    # Logging
68
    def debug_log(label, str_=''):
69
        if debug: sys.stderr.write('\n'+label+':\n'+str_+'\n')
70
    
71
    ## HTTP
72
    headers = initial_headers
73
    
74
    def do_request(request):
75
        debug_log('request', str(request))
76
        response = urllib2.urlopen(urllib2.Request(url, request, headers))
77
        response_str = streams.read_all(response)
78
        response_info = str(response.info())
79
        debug_log('response info', response_info)
80
        debug_log('response str', response_str)
81
        return response_str, response_info
82
    
83
    def do_repeated_request(request):
84
        pause = initial_pause
85
        total_pause = 0
86
        while True:
87
            total_pause += pause
88
            if total_pause > max_pause: raise # error is not temporary
89
            debug_log('total_pause', str(total_pause)+'s')
90
            time.sleep(pause) # wait for job to complete
91
            
92
            try: return do_request(request)
93
            except urllib2.HTTPError: pass # try again
94
            pause *= pause_growth_factor
95
    
96
    debug_log('Submit')
97
    request = submission_request_template.replace('[taxons]',
98
        r'\\n'.join(map(gwt_encode, taxons))) # double-escape \n
99
    response, response_info = do_request(request)
100
    key, = parse_response('submission', submission_response_pattern, response)
101
    debug_log('key', key)
102
    key_enc = gwt_encode(key)
103
    
104
    debug_log('Retrieve')
105
    request = retrieval_request_template.replace('[key]', key_enc)
106
    response, response_info = do_repeated_request(request)
107
    parse_response('retrieval', retrieval_response_pattern, response)
108
    session_id, = parse_response('retrieval info',
109
        retrieval_response_info_pattern, response_info)
110
    debug_log('session_id', session_id)
111
    headers['Cookie'] = 'JSESSIONID='+session_id
112
    
113
    # The output of the retrieve step is unusable because the array has
114
    # different lengths depending on the taxonomic ranks present in the provided
115
    # taxon name. The extra download step is therefore necessary.
116
    
117
    debug_log('Prepare download')
118
    request = download_request_template.replace('[key]', key_enc)
119
    response, response_info = do_request(request)
120
    csv_url, = parse_response('download', download_response_pattern, response)
121
    csv_url += download_url_suffix
122
    debug_log('csv_url', csv_url)
123
    
124
    debug_log('Download')
125
    response = urllib2.urlopen(urllib2.Request(csv_url))
126
    debug_log('response info', str(response.info()))
127
    return response
128

    
129
def repeated_tnrs_request(taxons, debug=False, **kw_args):
130
    for try_num in xrange(max_tries):
131
        try: return tnrs_request(taxons, debug, **kw_args)
132
        except InvalidResponse:
133
            debug = True # next time, output protocol info for debugging
134
            # try again
135
    raise # error is not temporary
(34-34/41)