Project

General

Profile

1 5169 aaronmk
# TNRS
2 4990 aaronmk
3 13464 aaronmk
import os.path
4 4990 aaronmk
import re
5
import sys
6
import time
7
import urllib2
8
9 5149 aaronmk
import csvs
10 5107 aaronmk
import exc
11 5120 aaronmk
import profiling
12 4990 aaronmk
import streams
13 5144 aaronmk
import strings
14 4990 aaronmk
15
# Config
16
initial_pause = 0.35 # sec
17
pause_growth_factor = 1.3
18 5125 aaronmk
max_pause = 30*60 # sec; = 30 min; TNRS sometimes freezes for ~10 min
19 4990 aaronmk
assert initial_pause <= max_pause
20 13548 aaronmk
#max_names = 5000 #according to http://tnrs.iplantcollaborative.org/TNRSapp.html
21 13562 aaronmk
#max_names = 500 # the maximum above crashes the live TNRS server
22
max_names = 100000 # dev server can handle more names b/c no simultaneous users
23 4990 aaronmk
24
# Protocol params
25 13464 aaronmk
server = streams.file_get_contents(os.path.dirname(__file__)+"/tnrs.url")
26
#server = 'tnrs.iplantcollaborative.org' # live server
27 13462 aaronmk
url_base = 'http://'+server+'/tnrsdemo/'
28 4990 aaronmk
url = url_base+'search'
29
initial_headers = {
30
    'Content-Type': 'text/x-gwt-rpc; charset=utf-8',
31
    'X-GWT-Module-Base': url_base,
32
    'X-GWT-Permutation': '574AA16D15D917C7704646FD92AFF6B3',
33
}
34
submission_request_template = ('7|0|7|'+url_base+
35 9912 aaronmk
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
36 13544 aaronmk
doSearch|java.lang.String/2004016611|{"sources":"gcc,tpl,tropicos,usda", \
37 9912 aaronmk
"names":"[names]", "type":"matching", "taxonomic":"true", \
38
"classification":"tropicos", "match_to_rank":"true"}|0.05|1|2|3|4|2|5|5|6|7|')
39 4990 aaronmk
submission_response_pattern = r'^//OK\[1,\["(\w+)"\],0,7\]$'
40
retrieval_request_template = ('7|0|15|'+url_base+
41
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService\
42
|getRemoteData|com.extjs.gxt.ui.client.data.PagingLoadConfig|\
43
java.lang.String/2004016611|com.extjs.gxt.ui.client.data.BasePagingLoadConfig/\
44
2011366567|com.extjs.gxt.ui.client.data.RpcMap/3441186752|sortField|sortDir|\
45
com.extjs.gxt.ui.client.Style$SortDir/640452531|offset|java.lang.Integer/\
46
3438268394|limit|{"email":"tnrs@lka5jjs.orv", "key":"[key]", \
47 9909 aaronmk
"taxonomic_constraint":"true", "source_sorting":"false", "first":"false"}\
48 4990 aaronmk
|1|2|3|4|2|5|6|7|0|1|8|4|9|0|10|11|0|12|13|0|14|13|100|15|')
49 13436 aaronmk
	# taxonomic_constraint (Constrain by Higher Taxonomy): selects lower ranks
50
	#  only from within the matched higher ranks. must be turned on, to ensure
51
	#  that higher ranks are always authoritative.
52
	# source_sorting (Constrain by Source): always selects a match from the
53
	#  first source in the list, no matter how low the match score. must be
54
	#  turned off, to avoid worse matches being selected instead of better ones.
55 4990 aaronmk
retrieval_response_pattern = '^//OK\[.*?\["com.extjs.gxt.ui.client.data.\
56
BasePagingLoadResult/496878394","java.util.ArrayList/4159755760","org.iplantc.\
57
tnrs.demo.shared.BeanTNRSEntry/1039545748",".*"\],0,7\]$'
58
retrieval_response_info_pattern = r'(?ms).*^Set-Cookie: JSESSIONID=(\w+);'
59
download_request_template = ('7|0|6|'+url_base+
60
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
61
downloadRemoteResults|java.lang.String/2004016611|{"name":"tnrs_results.txt", \
62 13591 aaronmk
"mode":"All", "type":"Detailed", "encoding":"utf8", "dirty":"true", \
63 9911 aaronmk
"sources":"false", "taxonomic":"true", "email":"tnrs@lka5jjs.orv", \
64 4990 aaronmk
"key":"[key]"}|1|2|3|4|1|5|6|')
65
download_response_pattern = '^//OK\[1,\["(.*)"\],0,7\]$'
66
download_url_suffix = '&name=tnrs_results.txt&encoding=utf8'
67
68 5083 aaronmk
class InvalidResponse(Exception): pass
69
70 5144 aaronmk
def gwt_encode(str_):
71
    return strings.esc_quotes(strings.json_encode(str_), '|', quote_esc='\!')
72 4990 aaronmk
73 5149 aaronmk
def make_spliced_decode_map(decode_map):
74
    return [(r'(?: |(?<=\t)|^)'+re.escape(from_.strip())+r'(?: |(?=\t)|$)',
75
        strings.regexp_repl_esc(to)) for from_, to in decode_map]
76
77 5154 aaronmk
padding = ' !pad ' # prepend to empty and whitespace-only strings
78 5149 aaronmk
encode_map = [
79 5167 aaronmk
    ('!', ' !exc '), # our escape char
80
    ('\t', ' !tab '), # TNRS replaces with " "
81
    ('\n', ' !nl '), # used to separate multiple names
82
    ('\r', ' !cr '), # used to separate multiple names
83 5168 aaronmk
    ('"', ' !quo '), # TNRS removes it when at the beginning or end
84 5167 aaronmk
    ('%', ' !pct '), # TNRS URL-decodes it in matched fields
85 5168 aaronmk
    ("'", ' !apo '), # TNRS removes it when at the beginning or end
86 5167 aaronmk
    (';', ' !sem '), # changes TNRS response format
87
    ('\\', ' !bsl '), # TNRS removes it
88 5165 aaronmk
    ('_', ' !und '), # TNRS replaces with " "
89 5171 aaronmk
    ('­', ' !sub '), # TNRS removes it
90 5169 aaronmk
    ('×', ' !mul '), # TNRS replaces with "x"
91 5149 aaronmk
]
92
decode_map = strings.flip_map(encode_map)
93 5154 aaronmk
decode_map.append((padding, ''))
94 5149 aaronmk
spliced_decode_map = make_spliced_decode_map(decode_map)
95
96 5154 aaronmk
def encode(str_):
97
    str_ = strings.replace_all(encode_map, str_)
98
    # Empty and whitespace-only strings are ignored by TNRS (no response row)
99
    if str_.strip() == '': str_ = padding+str_
100
    return str_
101 5149 aaronmk
102
def decode(str_): return strings.replace_all_re(spliced_decode_map, str_)
103
104
decode_for_tsv_map = make_spliced_decode_map([(from_, strings.replace_all(
105
    csvs.tsv_encode_map, to)) for from_, to in decode_map])
106
107
def decode_for_tsv(str_):
108
    return strings.replace_all_re(decode_for_tsv_map, str_)
109
110
class TnrsOutputStream(streams.FilterStream):
111
    '''Decodes a TNRS response whose names were encoded with encode()'''
112
    def __init__(self, stream):
113
        streams.FilterStream.__init__(self, decode_for_tsv, stream)
114
115 5106 aaronmk
def parse_response(name, pattern, str_, response, response_info):
116
    match = re.match(pattern, str_)
117
    if not match:
118
        raise InvalidResponse('Invalid '+name+' response:\n'+response_info+'\n'
119
            +response)
120 4990 aaronmk
    return match.groups()
121
122 9525 aaronmk
def single_tnrs_request(names, debug=False, cumulative_profiler=None):
123 5127 aaronmk
    '''
124
    Note that names containing only whitespace characters (after gwt_encode())
125
    are ignored by TNRS and do not receive a response row. Thus, you should
126
    always match up the Name_submitted returned by TNRS with the actual
127
    submitted name to determine the corresponding TNRS response row.
128
    '''
129 5121 aaronmk
    name_ct = len(names)
130
    assert name_ct <= max_names
131 4990 aaronmk
132
    # Logging
133
    def debug_log(label, str_=''):
134
        if debug: sys.stderr.write('\n'+label+':\n'+str_+'\n')
135
136
    ## HTTP
137 5119 aaronmk
    headers = initial_headers.copy() # don't modify global constant!
138 4990 aaronmk
139 5005 aaronmk
    def do_request(request):
140 4990 aaronmk
        debug_log('request', str(request))
141
        response = urllib2.urlopen(urllib2.Request(url, request, headers))
142
        response_str = streams.read_all(response)
143
        response_info = str(response.info())
144
        debug_log('response info', response_info)
145
        debug_log('response str', response_str)
146
        return response_str, response_info
147
148
    def do_repeated_request(request):
149
        pause = initial_pause
150
        total_pause = 0
151
        while True:
152
            total_pause += pause
153
            if total_pause > max_pause: raise # error is not temporary
154
            debug_log('total_pause', str(total_pause)+'s')
155
            time.sleep(pause) # wait for job to complete
156
157
            try: return do_request(request)
158
            except urllib2.HTTPError: pass # try again
159
            pause *= pause_growth_factor
160
161 5120 aaronmk
    profiler = profiling.ItersProfiler(start_now=True, iter_text='name')
162
    try:
163
        debug_log('Submit')
164 5121 aaronmk
        request = submission_request_template.replace('[names]',
165 5150 aaronmk
            gwt_encode('\n'.join(map(encode, names))))
166 5120 aaronmk
        response, response_info = do_request(request)
167 5151 aaronmk
        key, = parse_response('submission', submission_response_pattern,
168
            response, response, response_info)
169 5120 aaronmk
        debug_log('key', key)
170
        key_enc = gwt_encode(key)
171
172
        debug_log('Retrieve')
173
        request = retrieval_request_template.replace('[key]', key_enc)
174
        response, response_info = do_repeated_request(request)
175 5151 aaronmk
        parse_response('retrieval', retrieval_response_pattern, response,
176
            response, response_info)
177
        session_id, = parse_response('retrieval info',
178
            retrieval_response_info_pattern, response_info, response,
179 5120 aaronmk
            response_info)
180
        debug_log('session_id', session_id)
181
        headers['Cookie'] = 'JSESSIONID='+session_id
182
183
        # The output of the retrieve step is unusable because the array has
184 5151 aaronmk
        # different lengths depending on the taxonomic ranks present in the
185
        # provided taxon name. The extra download step is therefore necessary.
186 5120 aaronmk
187
        debug_log('Prepare download')
188
        request = download_request_template.replace('[key]', key_enc)
189
        response, response_info = do_request(request)
190 5151 aaronmk
        csv_url, = parse_response('download', download_response_pattern,
191
            response, response, response_info)
192 5120 aaronmk
        csv_url += download_url_suffix
193
        debug_log('csv_url', csv_url)
194
195
        debug_log('Download')
196
        response = urllib2.urlopen(urllib2.Request(csv_url))
197
        debug_log('response info', str(response.info()))
198 5150 aaronmk
        return TnrsOutputStream(response)
199 5120 aaronmk
    finally:
200 5121 aaronmk
        profiler.stop(name_ct)
201 5120 aaronmk
        sys.stderr.write(profiler.msg()+'\n')
202 9525 aaronmk
203
        if cumulative_profiler != None:
204
            cumulative_profiler.add_subprofiler(profiler)
205
            sys.stderr.write('Cumulatively: '+cumulative_profiler.msg()+'\n')
206 5088 aaronmk
207 9520 aaronmk
def tnrs_request(names, debug=False, **kw_args):
208 5108 aaronmk
    for try_num in xrange(2):
209 9519 aaronmk
        try: return single_tnrs_request(names, debug, **kw_args)
210 5160 aaronmk
        except (urllib2.HTTPError, InvalidResponse), e:
211 5107 aaronmk
            exc.print_ex(e, detail=False)
212 5108 aaronmk
            debug = True
213
            # try again with debug turned on
214 5088 aaronmk
    raise # error is not temporary