1
|
# TNRS
|
2
|
|
3
|
import re
|
4
|
import sys
|
5
|
import time
|
6
|
import urllib2
|
7
|
|
8
|
import exc
|
9
|
import profiling
|
10
|
import streams
|
11
|
|
12
|
# Config
|
13
|
initial_pause = 0.35 # sec
|
14
|
pause_growth_factor = 1.3
|
15
|
max_pause = 30*60 # sec; = 30 min; TNRS sometimes freezes for ~10 min
|
16
|
assert initial_pause <= max_pause
|
17
|
max_names = 5000 # according to http://tnrs.iplantcollaborative.org/TNRSapp.html
|
18
|
max_names = 500 # the maximum above crashes the TNRS server
|
19
|
|
20
|
# Protocol params
|
21
|
url_base = 'http://tnrs.iplantcollaborative.org/tnrsdemo/'
|
22
|
url = url_base+'search'
|
23
|
initial_headers = {
|
24
|
'Content-Type': 'text/x-gwt-rpc; charset=utf-8',
|
25
|
'X-GWT-Module-Base': url_base,
|
26
|
'X-GWT-Permutation': '574AA16D15D917C7704646FD92AFF6B3',
|
27
|
}
|
28
|
submission_request_template = ('7|0|7|'+url_base+
|
29
|
'||org.iplantc.tnrs.demo.client.SearchService|doSearch|\
|
30
|
java.lang.String/2004016611|{"sources":"gcc,tropicos,usda", "names":"[names]"\
|
31
|
, "type":"matching", "taxonomic":"true", "classification":"tropicos", \
|
32
|
"match_to_rank":"true"}|0.05|1|2|3|4|2|5|5|6|7|')
|
33
|
submission_response_pattern = r'^//OK\[1,\["(\w+)"\],0,7\]$'
|
34
|
retrieval_request_template = ('7|0|15|'+url_base+
|
35
|
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService\
|
36
|
|getRemoteData|com.extjs.gxt.ui.client.data.PagingLoadConfig|\
|
37
|
java.lang.String/2004016611|com.extjs.gxt.ui.client.data.BasePagingLoadConfig/\
|
38
|
2011366567|com.extjs.gxt.ui.client.data.RpcMap/3441186752|sortField|sortDir|\
|
39
|
com.extjs.gxt.ui.client.Style$SortDir/640452531|offset|java.lang.Integer/\
|
40
|
3438268394|limit|{"email":"tnrs@lka5jjs.orv", "key":"[key]", \
|
41
|
"taxonomic_constraint":"false", "source_sorting":"false", "first":"false"}\
|
42
|
|1|2|3|4|2|5|6|7|0|1|8|4|9|0|10|11|0|12|13|0|14|13|100|15|')
|
43
|
retrieval_response_pattern = '^//OK\[.*?\["com.extjs.gxt.ui.client.data.\
|
44
|
BasePagingLoadResult/496878394","java.util.ArrayList/4159755760","org.iplantc.\
|
45
|
tnrs.demo.shared.BeanTNRSEntry/1039545748",".*"\],0,7\]$'
|
46
|
retrieval_response_info_pattern = r'(?ms).*^Set-Cookie: JSESSIONID=(\w+);'
|
47
|
download_request_template = ('7|0|6|'+url_base+
|
48
|
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
|
49
|
downloadRemoteResults|java.lang.String/2004016611|{"name":"tnrs_results.txt", \
|
50
|
"mode":"Best", "type":"Detailed", "encoding":"utf8", "dirty":"false", \
|
51
|
"sources":"false", "taxonomic":"false", "email":"tnrs@lka5jjs.orv", \
|
52
|
"key":"[key]"}|1|2|3|4|1|5|6|')
|
53
|
download_response_pattern = '^//OK\[1,\["(.*)"\],0,7\]$'
|
54
|
download_url_suffix = '&name=tnrs_results.txt&encoding=utf8'
|
55
|
|
56
|
class InvalidResponse(Exception): pass
|
57
|
|
58
|
def gwt_encode(str_): return re.sub(r'[^\w.() -]+', r' ', str_)
|
59
|
|
60
|
def parse_response(name, pattern, str_, response, response_info):
|
61
|
match = re.match(pattern, str_)
|
62
|
if not match:
|
63
|
raise InvalidResponse('Invalid '+name+' response:\n'+response_info+'\n'
|
64
|
+response)
|
65
|
return match.groups()
|
66
|
|
67
|
def tnrs_request(names, debug=False):
|
68
|
name_ct = len(names)
|
69
|
assert name_ct <= max_names
|
70
|
|
71
|
# Logging
|
72
|
def debug_log(label, str_=''):
|
73
|
if debug: sys.stderr.write('\n'+label+':\n'+str_+'\n')
|
74
|
|
75
|
## HTTP
|
76
|
headers = initial_headers.copy() # don't modify global constant!
|
77
|
|
78
|
def do_request(request):
|
79
|
debug_log('request', str(request))
|
80
|
response = urllib2.urlopen(urllib2.Request(url, request, headers))
|
81
|
response_str = streams.read_all(response)
|
82
|
response_info = str(response.info())
|
83
|
debug_log('response info', response_info)
|
84
|
debug_log('response str', response_str)
|
85
|
return response_str, response_info
|
86
|
|
87
|
def do_repeated_request(request):
|
88
|
pause = initial_pause
|
89
|
total_pause = 0
|
90
|
while True:
|
91
|
total_pause += pause
|
92
|
if total_pause > max_pause: raise # error is not temporary
|
93
|
debug_log('total_pause', str(total_pause)+'s')
|
94
|
time.sleep(pause) # wait for job to complete
|
95
|
|
96
|
try: return do_request(request)
|
97
|
except urllib2.HTTPError: pass # try again
|
98
|
pause *= pause_growth_factor
|
99
|
|
100
|
profiler = profiling.ItersProfiler(start_now=True, iter_text='name')
|
101
|
try:
|
102
|
debug_log('Submit')
|
103
|
request = submission_request_template.replace('[names]',
|
104
|
r'\\n'.join(map(gwt_encode, names))) # double-escape \n
|
105
|
response, response_info = do_request(request)
|
106
|
key, = parse_response('submission', submission_response_pattern, response,
|
107
|
response, response_info)
|
108
|
debug_log('key', key)
|
109
|
key_enc = gwt_encode(key)
|
110
|
|
111
|
debug_log('Retrieve')
|
112
|
request = retrieval_request_template.replace('[key]', key_enc)
|
113
|
response, response_info = do_repeated_request(request)
|
114
|
parse_response('retrieval', retrieval_response_pattern, response, response,
|
115
|
response_info)
|
116
|
session_id, = parse_response('retrieval info',
|
117
|
retrieval_response_info_pattern, response_info, response, response_info)
|
118
|
debug_log('session_id', session_id)
|
119
|
headers['Cookie'] = 'JSESSIONID='+session_id
|
120
|
|
121
|
# The output of the retrieve step is unusable because the array has
|
122
|
# different lengths depending on the taxonomic ranks present in the provided
|
123
|
# taxon name. The extra download step is therefore necessary.
|
124
|
|
125
|
debug_log('Prepare download')
|
126
|
request = download_request_template.replace('[key]', key_enc)
|
127
|
response, response_info = do_request(request)
|
128
|
csv_url, = parse_response('download', download_response_pattern, response,
|
129
|
response, response_info)
|
130
|
csv_url += download_url_suffix
|
131
|
debug_log('csv_url', csv_url)
|
132
|
|
133
|
debug_log('Download')
|
134
|
response = urllib2.urlopen(urllib2.Request(csv_url))
|
135
|
debug_log('response info', str(response.info()))
|
136
|
return response
|
137
|
finally:
|
138
|
profiler.stop(name_ct)
|
139
|
sys.stderr.write(profiler.msg()+'\n')
|
140
|
|
141
|
def repeated_tnrs_request(names, debug=False, **kw_args):
|
142
|
for try_num in xrange(2):
|
143
|
try: return tnrs_request(names, debug, **kw_args)
|
144
|
except InvalidResponse, e:
|
145
|
exc.print_ex(e, detail=False)
|
146
|
debug = True
|
147
|
# try again with debug turned on
|
148
|
raise # error is not temporary
|