1 |
5169
|
aaronmk
|
# TNRS
|
2 |
4990
|
aaronmk
|
|
3 |
|
|
import re
|
4 |
|
|
import sys
|
5 |
|
|
import time
|
6 |
|
|
import urllib2
|
7 |
|
|
|
8 |
5149
|
aaronmk
|
import csvs
|
9 |
5107
|
aaronmk
|
import exc
|
10 |
5120
|
aaronmk
|
import profiling
|
11 |
4990
|
aaronmk
|
import streams
|
12 |
5144
|
aaronmk
|
import strings
|
13 |
4990
|
aaronmk
|
|
14 |
|
|
# Config
|
15 |
|
|
initial_pause = 0.35 # sec
|
16 |
|
|
pause_growth_factor = 1.3
|
17 |
5125
|
aaronmk
|
max_pause = 30*60 # sec; = 30 min; TNRS sometimes freezes for ~10 min
|
18 |
4990
|
aaronmk
|
assert initial_pause <= max_pause
|
19 |
5121
|
aaronmk
|
max_names = 5000 # according to http://tnrs.iplantcollaborative.org/TNRSapp.html
|
20 |
|
|
max_names = 500 # the maximum above crashes the TNRS server
|
21 |
4990
|
aaronmk
|
|
22 |
|
|
# Protocol params
|
23 |
|
|
url_base = 'http://tnrs.iplantcollaborative.org/tnrsdemo/'
|
24 |
|
|
url = url_base+'search'
|
25 |
|
|
initial_headers = {
|
26 |
|
|
'Content-Type': 'text/x-gwt-rpc; charset=utf-8',
|
27 |
|
|
'X-GWT-Module-Base': url_base,
|
28 |
|
|
'X-GWT-Permutation': '574AA16D15D917C7704646FD92AFF6B3',
|
29 |
|
|
}
|
30 |
|
|
submission_request_template = ('7|0|7|'+url_base+
|
31 |
|
|
'||org.iplantc.tnrs.demo.client.SearchService|doSearch|\
|
32 |
5691
|
aaronmk
|
java.lang.String/2004016611|{"sources":"tropicos", "names":"[names]"\
|
33 |
4990
|
aaronmk
|
, "type":"matching", "taxonomic":"true", "classification":"tropicos", \
|
34 |
|
|
"match_to_rank":"true"}|0.05|1|2|3|4|2|5|5|6|7|')
|
35 |
|
|
submission_response_pattern = r'^//OK\[1,\["(\w+)"\],0,7\]$'
|
36 |
|
|
retrieval_request_template = ('7|0|15|'+url_base+
|
37 |
|
|
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService\
|
38 |
|
|
|getRemoteData|com.extjs.gxt.ui.client.data.PagingLoadConfig|\
|
39 |
|
|
java.lang.String/2004016611|com.extjs.gxt.ui.client.data.BasePagingLoadConfig/\
|
40 |
|
|
2011366567|com.extjs.gxt.ui.client.data.RpcMap/3441186752|sortField|sortDir|\
|
41 |
|
|
com.extjs.gxt.ui.client.Style$SortDir/640452531|offset|java.lang.Integer/\
|
42 |
|
|
3438268394|limit|{"email":"tnrs@lka5jjs.orv", "key":"[key]", \
|
43 |
5786
|
aaronmk
|
"taxonomic_constraint":"true", "source_sorting":"true", "first":"false"}\
|
44 |
4990
|
aaronmk
|
|1|2|3|4|2|5|6|7|0|1|8|4|9|0|10|11|0|12|13|0|14|13|100|15|')
|
45 |
|
|
retrieval_response_pattern = '^//OK\[.*?\["com.extjs.gxt.ui.client.data.\
|
46 |
|
|
BasePagingLoadResult/496878394","java.util.ArrayList/4159755760","org.iplantc.\
|
47 |
|
|
tnrs.demo.shared.BeanTNRSEntry/1039545748",".*"\],0,7\]$'
|
48 |
|
|
retrieval_response_info_pattern = r'(?ms).*^Set-Cookie: JSESSIONID=(\w+);'
|
49 |
|
|
download_request_template = ('7|0|6|'+url_base+
|
50 |
|
|
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
|
51 |
|
|
downloadRemoteResults|java.lang.String/2004016611|{"name":"tnrs_results.txt", \
|
52 |
|
|
"mode":"Best", "type":"Detailed", "encoding":"utf8", "dirty":"false", \
|
53 |
|
|
"sources":"false", "taxonomic":"false", "email":"tnrs@lka5jjs.orv", \
|
54 |
|
|
"key":"[key]"}|1|2|3|4|1|5|6|')
|
55 |
|
|
download_response_pattern = '^//OK\[1,\["(.*)"\],0,7\]$'
|
56 |
|
|
download_url_suffix = '&name=tnrs_results.txt&encoding=utf8'
|
57 |
|
|
|
58 |
5083
|
aaronmk
|
class InvalidResponse(Exception): pass
|
59 |
|
|
|
60 |
5144
|
aaronmk
|
def gwt_encode(str_):
|
61 |
|
|
return strings.esc_quotes(strings.json_encode(str_), '|', quote_esc='\!')
|
62 |
4990
|
aaronmk
|
|
63 |
5149
|
aaronmk
|
def make_spliced_decode_map(decode_map):
|
64 |
|
|
return [(r'(?: |(?<=\t)|^)'+re.escape(from_.strip())+r'(?: |(?=\t)|$)',
|
65 |
|
|
strings.regexp_repl_esc(to)) for from_, to in decode_map]
|
66 |
|
|
|
67 |
5154
|
aaronmk
|
padding = ' !pad ' # prepend to empty and whitespace-only strings
|
68 |
5149
|
aaronmk
|
encode_map = [
|
69 |
5167
|
aaronmk
|
('!', ' !exc '), # our escape char
|
70 |
|
|
('\t', ' !tab '), # TNRS replaces with " "
|
71 |
|
|
('\n', ' !nl '), # used to separate multiple names
|
72 |
|
|
('\r', ' !cr '), # used to separate multiple names
|
73 |
5168
|
aaronmk
|
('"', ' !quo '), # TNRS removes it when at the beginning or end
|
74 |
5167
|
aaronmk
|
('%', ' !pct '), # TNRS URL-decodes it in matched fields
|
75 |
5168
|
aaronmk
|
("'", ' !apo '), # TNRS removes it when at the beginning or end
|
76 |
5167
|
aaronmk
|
(';', ' !sem '), # changes TNRS response format
|
77 |
|
|
('\\', ' !bsl '), # TNRS removes it
|
78 |
5165
|
aaronmk
|
('_', ' !und '), # TNRS replaces with " "
|
79 |
5171
|
aaronmk
|
('', ' !sub '), # TNRS removes it
|
80 |
5169
|
aaronmk
|
('×', ' !mul '), # TNRS replaces with "x"
|
81 |
5149
|
aaronmk
|
]
|
82 |
|
|
decode_map = strings.flip_map(encode_map)
|
83 |
5154
|
aaronmk
|
decode_map.append((padding, ''))
|
84 |
5149
|
aaronmk
|
spliced_decode_map = make_spliced_decode_map(decode_map)
|
85 |
|
|
|
86 |
5154
|
aaronmk
|
def encode(str_):
|
87 |
|
|
str_ = strings.replace_all(encode_map, str_)
|
88 |
|
|
# Empty and whitespace-only strings are ignored by TNRS (no response row)
|
89 |
|
|
if str_.strip() == '': str_ = padding+str_
|
90 |
|
|
return str_
|
91 |
5149
|
aaronmk
|
|
92 |
|
|
def decode(str_): return strings.replace_all_re(spliced_decode_map, str_)
|
93 |
|
|
|
94 |
|
|
decode_for_tsv_map = make_spliced_decode_map([(from_, strings.replace_all(
|
95 |
|
|
csvs.tsv_encode_map, to)) for from_, to in decode_map])
|
96 |
|
|
|
97 |
|
|
def decode_for_tsv(str_):
|
98 |
|
|
return strings.replace_all_re(decode_for_tsv_map, str_)
|
99 |
|
|
|
100 |
|
|
class TnrsOutputStream(streams.FilterStream):
|
101 |
|
|
'''Decodes a TNRS response whose names were encoded with encode()'''
|
102 |
|
|
def __init__(self, stream):
|
103 |
|
|
streams.FilterStream.__init__(self, decode_for_tsv, stream)
|
104 |
|
|
|
105 |
5106
|
aaronmk
|
def parse_response(name, pattern, str_, response, response_info):
|
106 |
|
|
match = re.match(pattern, str_)
|
107 |
|
|
if not match:
|
108 |
|
|
raise InvalidResponse('Invalid '+name+' response:\n'+response_info+'\n'
|
109 |
|
|
+response)
|
110 |
4990
|
aaronmk
|
return match.groups()
|
111 |
|
|
|
112 |
5121
|
aaronmk
|
def tnrs_request(names, debug=False):
|
113 |
5127
|
aaronmk
|
'''
|
114 |
|
|
Note that names containing only whitespace characters (after gwt_encode())
|
115 |
|
|
are ignored by TNRS and do not receive a response row. Thus, you should
|
116 |
|
|
always match up the Name_submitted returned by TNRS with the actual
|
117 |
|
|
submitted name to determine the corresponding TNRS response row.
|
118 |
|
|
'''
|
119 |
5121
|
aaronmk
|
name_ct = len(names)
|
120 |
|
|
assert name_ct <= max_names
|
121 |
4990
|
aaronmk
|
|
122 |
|
|
# Logging
|
123 |
|
|
def debug_log(label, str_=''):
|
124 |
|
|
if debug: sys.stderr.write('\n'+label+':\n'+str_+'\n')
|
125 |
|
|
|
126 |
|
|
## HTTP
|
127 |
5119
|
aaronmk
|
headers = initial_headers.copy() # don't modify global constant!
|
128 |
4990
|
aaronmk
|
|
129 |
5005
|
aaronmk
|
def do_request(request):
|
130 |
4990
|
aaronmk
|
debug_log('request', str(request))
|
131 |
|
|
response = urllib2.urlopen(urllib2.Request(url, request, headers))
|
132 |
|
|
response_str = streams.read_all(response)
|
133 |
|
|
response_info = str(response.info())
|
134 |
|
|
debug_log('response info', response_info)
|
135 |
|
|
debug_log('response str', response_str)
|
136 |
|
|
return response_str, response_info
|
137 |
|
|
|
138 |
|
|
def do_repeated_request(request):
|
139 |
|
|
pause = initial_pause
|
140 |
|
|
total_pause = 0
|
141 |
|
|
while True:
|
142 |
|
|
total_pause += pause
|
143 |
|
|
if total_pause > max_pause: raise # error is not temporary
|
144 |
|
|
debug_log('total_pause', str(total_pause)+'s')
|
145 |
|
|
time.sleep(pause) # wait for job to complete
|
146 |
|
|
|
147 |
|
|
try: return do_request(request)
|
148 |
|
|
except urllib2.HTTPError: pass # try again
|
149 |
|
|
pause *= pause_growth_factor
|
150 |
|
|
|
151 |
5120
|
aaronmk
|
profiler = profiling.ItersProfiler(start_now=True, iter_text='name')
|
152 |
|
|
try:
|
153 |
|
|
debug_log('Submit')
|
154 |
5121
|
aaronmk
|
request = submission_request_template.replace('[names]',
|
155 |
5150
|
aaronmk
|
gwt_encode('\n'.join(map(encode, names))))
|
156 |
5120
|
aaronmk
|
response, response_info = do_request(request)
|
157 |
5151
|
aaronmk
|
key, = parse_response('submission', submission_response_pattern,
|
158 |
|
|
response, response, response_info)
|
159 |
5120
|
aaronmk
|
debug_log('key', key)
|
160 |
|
|
key_enc = gwt_encode(key)
|
161 |
|
|
|
162 |
|
|
debug_log('Retrieve')
|
163 |
|
|
request = retrieval_request_template.replace('[key]', key_enc)
|
164 |
|
|
response, response_info = do_repeated_request(request)
|
165 |
5151
|
aaronmk
|
parse_response('retrieval', retrieval_response_pattern, response,
|
166 |
|
|
response, response_info)
|
167 |
|
|
session_id, = parse_response('retrieval info',
|
168 |
|
|
retrieval_response_info_pattern, response_info, response,
|
169 |
5120
|
aaronmk
|
response_info)
|
170 |
|
|
debug_log('session_id', session_id)
|
171 |
|
|
headers['Cookie'] = 'JSESSIONID='+session_id
|
172 |
|
|
|
173 |
|
|
# The output of the retrieve step is unusable because the array has
|
174 |
5151
|
aaronmk
|
# different lengths depending on the taxonomic ranks present in the
|
175 |
|
|
# provided taxon name. The extra download step is therefore necessary.
|
176 |
5120
|
aaronmk
|
|
177 |
|
|
debug_log('Prepare download')
|
178 |
|
|
request = download_request_template.replace('[key]', key_enc)
|
179 |
|
|
response, response_info = do_request(request)
|
180 |
5151
|
aaronmk
|
csv_url, = parse_response('download', download_response_pattern,
|
181 |
|
|
response, response, response_info)
|
182 |
5120
|
aaronmk
|
csv_url += download_url_suffix
|
183 |
|
|
debug_log('csv_url', csv_url)
|
184 |
|
|
|
185 |
|
|
debug_log('Download')
|
186 |
|
|
response = urllib2.urlopen(urllib2.Request(csv_url))
|
187 |
|
|
debug_log('response info', str(response.info()))
|
188 |
5150
|
aaronmk
|
return TnrsOutputStream(response)
|
189 |
5120
|
aaronmk
|
finally:
|
190 |
5121
|
aaronmk
|
profiler.stop(name_ct)
|
191 |
5120
|
aaronmk
|
sys.stderr.write(profiler.msg()+'\n')
|
192 |
5088
|
aaronmk
|
|
193 |
5121
|
aaronmk
|
def repeated_tnrs_request(names, debug=False, **kw_args):
|
194 |
5108
|
aaronmk
|
for try_num in xrange(2):
|
195 |
5121
|
aaronmk
|
try: return tnrs_request(names, debug, **kw_args)
|
196 |
5160
|
aaronmk
|
except (urllib2.HTTPError, InvalidResponse), e:
|
197 |
5107
|
aaronmk
|
exc.print_ex(e, detail=False)
|
198 |
5108
|
aaronmk
|
debug = True
|
199 |
|
|
# try again with debug turned on
|
200 |
5088
|
aaronmk
|
raise # error is not temporary
|