| 1 |
4990
|
aaronmk
|
# TNRS
|
| 2 |
|
|
|
| 3 |
|
|
import re
|
| 4 |
|
|
import sys
|
| 5 |
|
|
import time
|
| 6 |
|
|
import urllib2
|
| 7 |
|
|
|
| 8 |
5107
|
aaronmk
|
import exc
|
| 9 |
5120
|
aaronmk
|
import profiling
|
| 10 |
4990
|
aaronmk
|
import streams
|
| 11 |
|
|
|
| 12 |
|
|
# Config
|
| 13 |
|
|
initial_pause = 0.35 # sec
|
| 14 |
|
|
pause_growth_factor = 1.3
|
| 15 |
|
|
max_pause = 600 # sec; = 10 min
|
| 16 |
|
|
assert initial_pause <= max_pause
|
| 17 |
5121
|
aaronmk
|
max_names = 5000 # according to http://tnrs.iplantcollaborative.org/TNRSapp.html
|
| 18 |
|
|
max_names = 500 # the maximum above crashes the TNRS server
|
| 19 |
4990
|
aaronmk
|
|
| 20 |
|
|
# Protocol params
|
| 21 |
|
|
url_base = 'http://tnrs.iplantcollaborative.org/tnrsdemo/'
|
| 22 |
|
|
url = url_base+'search'
|
| 23 |
|
|
initial_headers = {
|
| 24 |
|
|
'Content-Type': 'text/x-gwt-rpc; charset=utf-8',
|
| 25 |
|
|
'X-GWT-Module-Base': url_base,
|
| 26 |
|
|
'X-GWT-Permutation': '574AA16D15D917C7704646FD92AFF6B3',
|
| 27 |
|
|
}
|
| 28 |
|
|
submission_request_template = ('7|0|7|'+url_base+
|
| 29 |
|
|
'||org.iplantc.tnrs.demo.client.SearchService|doSearch|\
|
| 30 |
5121
|
aaronmk
|
java.lang.String/2004016611|{"sources":"gcc,tropicos,usda", "names":"[names]"\
|
| 31 |
4990
|
aaronmk
|
, "type":"matching", "taxonomic":"true", "classification":"tropicos", \
|
| 32 |
|
|
"match_to_rank":"true"}|0.05|1|2|3|4|2|5|5|6|7|')
|
| 33 |
|
|
submission_response_pattern = r'^//OK\[1,\["(\w+)"\],0,7\]$'
|
| 34 |
|
|
retrieval_request_template = ('7|0|15|'+url_base+
|
| 35 |
|
|
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService\
|
| 36 |
|
|
|getRemoteData|com.extjs.gxt.ui.client.data.PagingLoadConfig|\
|
| 37 |
|
|
java.lang.String/2004016611|com.extjs.gxt.ui.client.data.BasePagingLoadConfig/\
|
| 38 |
|
|
2011366567|com.extjs.gxt.ui.client.data.RpcMap/3441186752|sortField|sortDir|\
|
| 39 |
|
|
com.extjs.gxt.ui.client.Style$SortDir/640452531|offset|java.lang.Integer/\
|
| 40 |
|
|
3438268394|limit|{"email":"tnrs@lka5jjs.orv", "key":"[key]", \
|
| 41 |
|
|
"taxonomic_constraint":"false", "source_sorting":"false", "first":"false"}\
|
| 42 |
|
|
|1|2|3|4|2|5|6|7|0|1|8|4|9|0|10|11|0|12|13|0|14|13|100|15|')
|
| 43 |
|
|
retrieval_response_pattern = '^//OK\[.*?\["com.extjs.gxt.ui.client.data.\
|
| 44 |
|
|
BasePagingLoadResult/496878394","java.util.ArrayList/4159755760","org.iplantc.\
|
| 45 |
|
|
tnrs.demo.shared.BeanTNRSEntry/1039545748",".*"\],0,7\]$'
|
| 46 |
|
|
retrieval_response_info_pattern = r'(?ms).*^Set-Cookie: JSESSIONID=(\w+);'
|
| 47 |
|
|
download_request_template = ('7|0|6|'+url_base+
|
| 48 |
|
|
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
|
| 49 |
|
|
downloadRemoteResults|java.lang.String/2004016611|{"name":"tnrs_results.txt", \
|
| 50 |
|
|
"mode":"Best", "type":"Detailed", "encoding":"utf8", "dirty":"false", \
|
| 51 |
|
|
"sources":"false", "taxonomic":"false", "email":"tnrs@lka5jjs.orv", \
|
| 52 |
|
|
"key":"[key]"}|1|2|3|4|1|5|6|')
|
| 53 |
|
|
download_response_pattern = '^//OK\[1,\["(.*)"\],0,7\]$'
|
| 54 |
|
|
download_url_suffix = '&name=tnrs_results.txt&encoding=utf8'
|
| 55 |
|
|
|
| 56 |
5083
|
aaronmk
|
class InvalidResponse(Exception): pass
|
| 57 |
|
|
|
| 58 |
4990
|
aaronmk
|
def gwt_encode(str_): return re.sub(r'[^\w.() -]+', r' ', str_)
|
| 59 |
|
|
|
| 60 |
5106
|
aaronmk
|
def parse_response(name, pattern, str_, response, response_info):
|
| 61 |
|
|
match = re.match(pattern, str_)
|
| 62 |
|
|
if not match:
|
| 63 |
|
|
raise InvalidResponse('Invalid '+name+' response:\n'+response_info+'\n'
|
| 64 |
|
|
+response)
|
| 65 |
4990
|
aaronmk
|
return match.groups()
|
| 66 |
|
|
|
| 67 |
5121
|
aaronmk
|
def tnrs_request(names, debug=False):
|
| 68 |
|
|
name_ct = len(names)
|
| 69 |
|
|
assert name_ct <= max_names
|
| 70 |
4990
|
aaronmk
|
|
| 71 |
|
|
# Logging
|
| 72 |
|
|
def debug_log(label, str_=''):
|
| 73 |
|
|
if debug: sys.stderr.write('\n'+label+':\n'+str_+'\n')
|
| 74 |
|
|
|
| 75 |
|
|
## HTTP
|
| 76 |
5119
|
aaronmk
|
headers = initial_headers.copy() # don't modify global constant!
|
| 77 |
4990
|
aaronmk
|
|
| 78 |
5005
|
aaronmk
|
def do_request(request):
|
| 79 |
4990
|
aaronmk
|
debug_log('request', str(request))
|
| 80 |
|
|
response = urllib2.urlopen(urllib2.Request(url, request, headers))
|
| 81 |
|
|
response_str = streams.read_all(response)
|
| 82 |
|
|
response_info = str(response.info())
|
| 83 |
|
|
debug_log('response info', response_info)
|
| 84 |
|
|
debug_log('response str', response_str)
|
| 85 |
|
|
return response_str, response_info
|
| 86 |
|
|
|
| 87 |
|
|
def do_repeated_request(request):
|
| 88 |
|
|
pause = initial_pause
|
| 89 |
|
|
total_pause = 0
|
| 90 |
|
|
while True:
|
| 91 |
|
|
total_pause += pause
|
| 92 |
|
|
if total_pause > max_pause: raise # error is not temporary
|
| 93 |
|
|
debug_log('total_pause', str(total_pause)+'s')
|
| 94 |
|
|
time.sleep(pause) # wait for job to complete
|
| 95 |
|
|
|
| 96 |
|
|
try: return do_request(request)
|
| 97 |
|
|
except urllib2.HTTPError: pass # try again
|
| 98 |
|
|
pause *= pause_growth_factor
|
| 99 |
|
|
|
| 100 |
5120
|
aaronmk
|
profiler = profiling.ItersProfiler(start_now=True, iter_text='name')
|
| 101 |
|
|
try:
|
| 102 |
|
|
debug_log('Submit')
|
| 103 |
5121
|
aaronmk
|
request = submission_request_template.replace('[names]',
|
| 104 |
|
|
r'\\n'.join(map(gwt_encode, names))) # double-escape \n
|
| 105 |
5120
|
aaronmk
|
response, response_info = do_request(request)
|
| 106 |
|
|
key, = parse_response('submission', submission_response_pattern, response,
|
| 107 |
|
|
response, response_info)
|
| 108 |
|
|
debug_log('key', key)
|
| 109 |
|
|
key_enc = gwt_encode(key)
|
| 110 |
|
|
|
| 111 |
|
|
debug_log('Retrieve')
|
| 112 |
|
|
request = retrieval_request_template.replace('[key]', key_enc)
|
| 113 |
|
|
response, response_info = do_repeated_request(request)
|
| 114 |
|
|
parse_response('retrieval', retrieval_response_pattern, response, response,
|
| 115 |
|
|
response_info)
|
| 116 |
|
|
session_id, = parse_response('retrieval info',
|
| 117 |
|
|
retrieval_response_info_pattern, response_info, response, response_info)
|
| 118 |
|
|
debug_log('session_id', session_id)
|
| 119 |
|
|
headers['Cookie'] = 'JSESSIONID='+session_id
|
| 120 |
|
|
|
| 121 |
|
|
# The output of the retrieve step is unusable because the array has
|
| 122 |
|
|
# different lengths depending on the taxonomic ranks present in the provided
|
| 123 |
|
|
# taxon name. The extra download step is therefore necessary.
|
| 124 |
|
|
|
| 125 |
|
|
debug_log('Prepare download')
|
| 126 |
|
|
request = download_request_template.replace('[key]', key_enc)
|
| 127 |
|
|
response, response_info = do_request(request)
|
| 128 |
|
|
csv_url, = parse_response('download', download_response_pattern, response,
|
| 129 |
|
|
response, response_info)
|
| 130 |
|
|
csv_url += download_url_suffix
|
| 131 |
|
|
debug_log('csv_url', csv_url)
|
| 132 |
|
|
|
| 133 |
|
|
debug_log('Download')
|
| 134 |
|
|
response = urllib2.urlopen(urllib2.Request(csv_url))
|
| 135 |
|
|
debug_log('response info', str(response.info()))
|
| 136 |
|
|
return response
|
| 137 |
|
|
finally:
|
| 138 |
5121
|
aaronmk
|
profiler.stop(name_ct)
|
| 139 |
5120
|
aaronmk
|
sys.stderr.write(profiler.msg()+'\n')
|
| 140 |
5088
|
aaronmk
|
|
| 141 |
5121
|
aaronmk
|
def repeated_tnrs_request(names, debug=False, **kw_args):
|
| 142 |
5108
|
aaronmk
|
for try_num in xrange(2):
|
| 143 |
5121
|
aaronmk
|
try: return tnrs_request(names, debug, **kw_args)
|
| 144 |
5107
|
aaronmk
|
except InvalidResponse, e:
|
| 145 |
|
|
exc.print_ex(e, detail=False)
|
| 146 |
5108
|
aaronmk
|
debug = True
|
| 147 |
|
|
# try again with debug turned on
|
| 148 |
5088
|
aaronmk
|
raise # error is not temporary
|
