Project

General

Profile

1
# TNRS
2

    
3
import json
4
import os.path
5
import re
6
import sys
7
import time
8
import urllib2
9

    
10
import csvs
11
import exc
12
import profiling
13
import streams
14
import strings
15

    
16
# Config
17
initial_pause = 0.35 # sec
18
pause_growth_factor = 1.3
19
max_pause = 30*60 # sec; = 30 min; TNRS sometimes freezes for ~10 min
20
assert initial_pause <= max_pause
21
#max_names = 5000 #according to http://tnrs.iplantcollaborative.org/TNRSapp.html
22
max_names = 500 # the maximum above crashes the live and dev TNRS servers
23

    
24
# Protocol params
25
server = streams.file_get_contents(os.path.dirname(__file__)+"/tnrs.url")
26
#server = 'tnrs.iplantcollaborative.org' # live server
27
url_base = 'http://'+server+'/tnrsdemo/'
28
url = url_base+'search'
29
initial_headers = {
30
    'Content-Type': 'text/x-gwt-rpc; charset=utf-8',
31
    'X-GWT-Module-Base': url_base,
32
    'X-GWT-Permutation': '574AA16D15D917C7704646FD92AFF6B3',
33
}
34
submission_request_template = ('7|0|7|'+url_base+
35
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
36
doSearch|java.lang.String/2004016611|{"sources":"gcc,tpl,tropicos,usda", \
37
"names":"[names]", "type":"matching", "taxonomic":"true", \
38
"classification":"tropicos", "match_to_rank":"true"}|0.05|1|2|3|4|2|5|5|6|7|')
39
submission_response_pattern = r'^//OK\[1,\["(\w+)"\],0,7\]$'
40
retrieval_request_template = ('7|0|15|'+url_base+
41
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService\
42
|getRemoteData|com.extjs.gxt.ui.client.data.PagingLoadConfig|\
43
java.lang.String/2004016611|com.extjs.gxt.ui.client.data.BasePagingLoadConfig/\
44
2011366567|com.extjs.gxt.ui.client.data.RpcMap/3441186752|sortField|sortDir|\
45
com.extjs.gxt.ui.client.Style$SortDir/640452531|offset|java.lang.Integer/\
46
3438268394|limit|{"email":"tnrs@lka5jjs.orv", "key":"[key]", \
47
"taxonomic_constraint":"true", "source_sorting":"true", "first":"false"}\
48
|1|2|3|4|2|5|6|7|0|1|8|4|9|0|10|11|0|12|13|0|14|13|100|15|')
49
	# taxonomic_constraint (Constrain by Higher Taxonomy): selects lower ranks
50
	#  only from within the matched higher ranks. must be turned on, to ensure
51
	#  that higher ranks are always authoritative.
52
	# source_sorting (Constrain by Source): always puts matches in the order of
53
	#  the sources, regardless of match score. should never be turned on,
54
	#  because it selects worse matches instead of better ones. *however*, since
55
	#  this is currently broken and always forced on, we turn it on so that the
56
	#  download settings reflect what TNRS actually used.
57
retrieval_response_pattern = '^//OK\[.*?\["com.extjs.gxt.ui.client.data.\
58
BasePagingLoadResult/496878394","java.util.ArrayList/4159755760","org.iplantc.\
59
tnrs.demo.shared.BeanTNRSEntry/1039545748",".*"\],0,7\]$'
60
retrieval_response_info_pattern = r'(?ms).*^Set-Cookie: JSESSIONID=(\w+);'
61
download_request_template = ('7|0|6|'+url_base+
62
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
63
downloadRemoteResults|java.lang.String/2004016611|{"name":"tnrs_results.txt", \
64
"mode":"All", "type":"Detailed", "encoding":"utf8", "dirty":"false", \
65
"sources":"false", "taxonomic":"true", "email":"tnrs@lka5jjs.orv", \
66
"key":"[key]"}|1|2|3|4|1|5|6|')
67
	# dirty:
68
	#  when on, sometimes marks multiple names as Selected. must be turned *off*
69
	#  so that only one name is marked as Selected. note that this actually used
70
	#  to be on in the web app (see r9910, 2013-6-18), but does not appear to be
71
	#  needed (the source_sorting bug alluded to in r9910 is not fixed by
72
	#  enabling the dirty setting).
73
download_response_pattern = '^//OK\[1,\["(.*)"\],0,7\]$'
74
download_url_suffix = '&name=tnrs_results.txt&encoding=utf8'
75
tsv_to_json_col_map = {
76
	'Name_matched_author': 'Canonical_author',
77
	'Accepted_name_species': 'Accepted_species',
78
	'Accepted_name_family': 'Accepted_family',
79
	'Selected': 'selected',
80
}
81

    
82
class InvalidResponse(Exception): pass
83

    
84
def gwt_encode(str_):
85
    # embedded strs within GWT sections additionally have " JSON-escaped
86
    return strings.esc_quotes(strings.json_encode(str_), '|', quote_esc='\!')
87

    
88
def gwt_decode(str_):
89
    # embedded strs within GWT sections additionally have " JSON-escaped
90
    return strings.json_decode(strings.unesc_quotes(str_, '|', quote_esc='\!'))
91

    
92
def make_spliced_decode_map(decode_map):
93
    return [(r'(?: |(?<=\t)|^)'+re.escape(from_.strip())+r'(?: |(?=\t)|$)',
94
        strings.regexp_repl_esc(to)) for from_, to in decode_map]
95

    
96
padding = ' !pad ' # prepend to empty and whitespace-only strings
97
encode_map = [
98
    ('!', ' !exc '), # our escape char
99
    ('\t', ' !tab '), # TNRS replaces with " "
100
    ('\n', ' !nl '), # used to separate multiple names
101
    ('\r', ' !cr '), # used to separate multiple names
102
    ('"', ' !quo '), # TNRS removes it when at the beginning or end
103
    ('%', ' !pct '), # TNRS URL-decodes it in matched fields
104
    ("'", ' !apo '), # TNRS removes it when at the beginning or end
105
    (';', ' !sem '), # changes TNRS response format
106
    ('\\', ' !bsl '), # TNRS removes it
107
    ('_', ' !und '), # TNRS replaces with " "
108
    ('­', ' !sub '), # TNRS removes it
109
    ('×', ' !mul '), # TNRS replaces with "x"
110
]
111
decode_map = strings.flip_map(encode_map)
112
decode_map.append((padding, ''))
113
spliced_decode_map = make_spliced_decode_map(decode_map)
114

    
115
def encode(str_):
116
    str_ = strings.replace_all(encode_map, str_)
117
    # Empty and whitespace-only strings are ignored by TNRS (no response row)
118
    if str_.strip() == '': str_ = padding+str_
119
    return str_
120

    
121
def decode(str_): return strings.replace_all_re(spliced_decode_map, str_)
122

    
123
decode_for_tsv_map = make_spliced_decode_map([(from_, strings.replace_all(
124
    csvs.tsv_encode_map, to)) for from_, to in decode_map])
125

    
126
def decode_for_tsv(str_):
127
    return strings.replace_all_re(decode_for_tsv_map, str_)
128

    
129
class TnrsOutputStream(streams.FilterStream):
130
    '''Decodes a TNRS response whose names were encoded with encode()'''
131
    def __init__(self, stream):
132
        streams.FilterStream.__init__(self, decode_for_tsv, stream)
133

    
134
def parse_response(name, pattern, str_, response, response_info):
135
    match = re.match(pattern, str_)
136
    if not match:
137
        raise InvalidResponse('Invalid '+name+' response:\n'+response_info+'\n'
138
            +response)
139
    return match.groups()
140

    
141
export_cols = None
142

    
143
def single_tnrs_request(names, debug=False, cumulative_profiler=None,
144
    use_tnrs_export=True):
145
    '''
146
    Note that names containing only whitespace characters (after gwt_encode())
147
    are ignored by TNRS and do not receive a response row. Thus, you should
148
    always match up the Name_submitted returned by TNRS with the actual
149
    submitted name to determine the corresponding TNRS response row.
150
    
151
    @param use_tnrs_export whether to use TNRS's TSV export feature, which
152
        currently returns incorrect selected matches (vegpath.org/issues/943)
153
    '''
154
    global export_cols
155
    
156
    name_ct = len(names)
157
    assert name_ct <= max_names
158
    assert name_ct >= 1 # with no names, TNRS hangs indefinitely
159
    
160
    # Logging
161
    def debug_log(label, str_=''):
162
        if debug: sys.stderr.write('\n'+label+':\n'+str_+'\n')
163
    
164
    if not use_tnrs_export and export_cols == None: # need export_cols
165
        debug_log('fetching export columns')
166
        export_cols = csvs.header(single_tnrs_request([''], debug,
167
            use_tnrs_export=True)) # need at least 1 name
168
        debug_log('TSV export columns', strings.urepr(export_cols))
169
        json_cols = [tsv_to_json_col_map.get(c, c) for c in export_cols]
170
        debug_log('JSON columns', strings.urepr(json_cols))
171
    
172
    ## HTTP
173
    headers = initial_headers.copy() # don't modify global constant!
174
    
175
    def do_request(request):
176
        request_obj = urllib2.Request(url, request, headers)
177
        debug_log('request URL', str(url))
178
        debug_log('request info', str(request_obj.header_items()))
179
        debug_log('request str', str(request_obj.get_data()))
180
        response = urllib2.urlopen(request_obj)
181
        response_str = streams.read_all(response)
182
        response_info = str(response.info())
183
        debug_log('response info', response_info)
184
        debug_log('response str', response_str)
185
        return response_str, response_info
186
    
187
    def do_repeated_request(request):
188
        pause = initial_pause
189
        total_pause = 0
190
        while True:
191
            total_pause += pause
192
            if total_pause > max_pause: raise # error is not temporary
193
            debug_log('total_pause', str(total_pause)+'s')
194
            time.sleep(pause) # wait for job to complete
195
            
196
            try: return do_request(request)
197
            except urllib2.HTTPError: pass # try again
198
            pause *= pause_growth_factor
199
    
200
    profiler = profiling.ItersProfiler(start_now=True, iter_text='name')
201
    try:
202
        debug_log('Submit')
203
        request = submission_request_template.replace('[names]',
204
            gwt_encode('\n'.join(map(encode, names))))
205
        response, response_info = do_request(request)
206
        key, = parse_response('submission', submission_response_pattern,
207
            response, response, response_info)
208
        debug_log('key', key)
209
        key_enc = gwt_encode(key)
210
        
211
        debug_log('Retrieve')
212
        request = retrieval_request_template.replace('[key]', key_enc)
213
        response, response_info = do_repeated_request(request)
214
        parse_response('retrieval', retrieval_response_pattern, response,
215
            response, response_info)
216
        session_id, = parse_response('retrieval info',
217
            retrieval_response_info_pattern, response_info, response,
218
            response_info)
219
        debug_log('session_id', session_id)
220
        headers['Cookie'] = 'JSESSIONID='+session_id
221
        
222
        # the output of the retrieve step is unusable because the array does not
223
        # contain all the columns, contains no column names, and has different
224
        # lengths depending on the taxonomic ranks present in the provided taxon
225
        # name. the extra download step is therefore necessary.
226
        
227
        if use_tnrs_export:
228
            debug_log('Prepare download')
229
            request = download_request_template.replace('[key]', key_enc)
230
            response, response_info = do_request(request)
231
            csv_url, = parse_response('download', download_response_pattern,
232
                response, response, response_info)
233
            csv_url += download_url_suffix
234
            debug_log('csv_url', csv_url)
235
            
236
            debug_log('Download')
237
            request_obj = urllib2.Request(csv_url)
238
            debug_log('request URL', str(csv_url))
239
            debug_log('request info', str(request_obj.header_items()))
240
            debug_log('request str', str(request_obj.get_data()))
241
            response = urllib2.urlopen(request_obj)
242
            response_info = str(response.info())
243
            debug_log('response info', response_info)
244
            output_stream = response
245
        else:
246
            debug_log('Expand')
247
            request = ('7|0|6|http://tnrs.iplantcollaborative.org/tnrsdemo/|\
248
1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
249
requestGroupMembers|java.lang.String/2004016611|{"group":"0", \
250
"email":"tnrs@lka5jjs.orv", "key":"[key]", "source_sorting":"false", \
251
"taxonomic_constraint":"false"}|1|2|3|4|1|5|6|'.replace('[key]', key_enc))
252
            response, response_info = do_request(request)
253
            data_gwt, = parse_response('expand', r'^//OK\[1,\["(.*)"\],0,7\]$',
254
                response, response, response_info)
255
            data_json = json.loads(gwt_decode(data_gwt))
256
            debug_log('JSON data', strings.urepr(data_json))
257
            output_stream = csvs.InputRewriter(
258
                csvs.RowNumFilter(
259
                    csvs.JsonReader(data_json['items'], json_cols[1:]),
260
                    json_cols[0]
261
                )
262
            )
263
        return TnrsOutputStream(output_stream)
264
    finally:
265
        profiler.stop(name_ct)
266
        sys.stderr.write(profiler.msg()+'\n')
267
        
268
        if cumulative_profiler != None:
269
            cumulative_profiler.add_subprofiler(profiler)
270
            sys.stderr.write('Cumulatively: '+cumulative_profiler.msg()+'\n')
271

    
272
def tnrs_request(names, debug=False, **kw_args):
273
    for try_num in xrange(2):
274
        try: return single_tnrs_request(names, debug, **kw_args)
275
        except (urllib2.HTTPError, InvalidResponse), e:
276
            exc.print_ex(e, detail=False)
277
            debug = True
278
            # try again with debug turned on
279
    raise # error is not temporary
(42-42/49)