Project

General

Profile

1
# TNRS
2

    
3
import re
4
import sys
5
import time
6
import urllib2
7

    
8
import csvs
9
import exc
10
import profiling
11
import streams
12
import strings
13

    
14
# Config
15
initial_pause = 0.35 # sec
16
pause_growth_factor = 1.3
17
max_pause = 30*60 # sec; = 30 min; TNRS sometimes freezes for ~10 min
18
assert initial_pause <= max_pause
19
max_names = 5000 # according to http://tnrs.iplantcollaborative.org/TNRSapp.html
20
max_names = 500 # the maximum above crashes the TNRS server
21

    
22
# Protocol params
23
url_base = 'http://tnrs.iplantcollaborative.org/tnrsdemo/'
24
url = url_base+'search'
25
initial_headers = {
26
    'Content-Type': 'text/x-gwt-rpc; charset=utf-8',
27
    'X-GWT-Module-Base': url_base,
28
    'X-GWT-Permutation': '574AA16D15D917C7704646FD92AFF6B3',
29
}
30
submission_request_template = ('7|0|7|'+url_base+
31
'||org.iplantc.tnrs.demo.client.SearchService|doSearch|\
32
java.lang.String/2004016611|{"sources":"tropicos", "names":"[names]"\
33
, "type":"matching", "taxonomic":"true", "classification":"tropicos", \
34
"match_to_rank":"true"}|0.05|1|2|3|4|2|5|5|6|7|')
35
submission_response_pattern = r'^//OK\[1,\["(\w+)"\],0,7\]$'
36
retrieval_request_template = ('7|0|15|'+url_base+
37
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService\
38
|getRemoteData|com.extjs.gxt.ui.client.data.PagingLoadConfig|\
39
java.lang.String/2004016611|com.extjs.gxt.ui.client.data.BasePagingLoadConfig/\
40
2011366567|com.extjs.gxt.ui.client.data.RpcMap/3441186752|sortField|sortDir|\
41
com.extjs.gxt.ui.client.Style$SortDir/640452531|offset|java.lang.Integer/\
42
3438268394|limit|{"email":"tnrs@lka5jjs.orv", "key":"[key]", \
43
"taxonomic_constraint":"true", "source_sorting":"true", "first":"false"}\
44
|1|2|3|4|2|5|6|7|0|1|8|4|9|0|10|11|0|12|13|0|14|13|100|15|')
45
retrieval_response_pattern = '^//OK\[.*?\["com.extjs.gxt.ui.client.data.\
46
BasePagingLoadResult/496878394","java.util.ArrayList/4159755760","org.iplantc.\
47
tnrs.demo.shared.BeanTNRSEntry/1039545748",".*"\],0,7\]$'
48
retrieval_response_info_pattern = r'(?ms).*^Set-Cookie: JSESSIONID=(\w+);'
49
download_request_template = ('7|0|6|'+url_base+
50
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
51
downloadRemoteResults|java.lang.String/2004016611|{"name":"tnrs_results.txt", \
52
"mode":"Best", "type":"Detailed", "encoding":"utf8", "dirty":"false", \
53
"sources":"false", "taxonomic":"false", "email":"tnrs@lka5jjs.orv", \
54
"key":"[key]"}|1|2|3|4|1|5|6|')
55
download_response_pattern = '^//OK\[1,\["(.*)"\],0,7\]$'
56
download_url_suffix = '&name=tnrs_results.txt&encoding=utf8'
57

    
58
class InvalidResponse(Exception): pass
59

    
60
def gwt_encode(str_):
61
    return strings.esc_quotes(strings.json_encode(str_), '|', quote_esc='\!')
62

    
63
def make_spliced_decode_map(decode_map):
64
    return [(r'(?: |(?<=\t)|^)'+re.escape(from_.strip())+r'(?: |(?=\t)|$)',
65
        strings.regexp_repl_esc(to)) for from_, to in decode_map]
66

    
67
padding = ' !pad ' # prepend to empty and whitespace-only strings
68
encode_map = [
69
    ('!', ' !exc '), # our escape char
70
    ('\t', ' !tab '), # TNRS replaces with " "
71
    ('\n', ' !nl '), # used to separate multiple names
72
    ('\r', ' !cr '), # used to separate multiple names
73
    ('"', ' !quo '), # TNRS removes it when at the beginning or end
74
    ('%', ' !pct '), # TNRS URL-decodes it in matched fields
75
    ("'", ' !apo '), # TNRS removes it when at the beginning or end
76
    (';', ' !sem '), # changes TNRS response format
77
    ('\\', ' !bsl '), # TNRS removes it
78
    ('_', ' !und '), # TNRS replaces with " "
79
    ('­', ' !sub '), # TNRS removes it
80
    ('×', ' !mul '), # TNRS replaces with "x"
81
]
82
decode_map = strings.flip_map(encode_map)
83
decode_map.append((padding, ''))
84
spliced_decode_map = make_spliced_decode_map(decode_map)
85

    
86
def encode(str_):
87
    str_ = strings.replace_all(encode_map, str_)
88
    # Empty and whitespace-only strings are ignored by TNRS (no response row)
89
    if str_.strip() == '': str_ = padding+str_
90
    return str_
91

    
92
def decode(str_): return strings.replace_all_re(spliced_decode_map, str_)
93

    
94
decode_for_tsv_map = make_spliced_decode_map([(from_, strings.replace_all(
95
    csvs.tsv_encode_map, to)) for from_, to in decode_map])
96

    
97
def decode_for_tsv(str_):
98
    return strings.replace_all_re(decode_for_tsv_map, str_)
99

    
100
class TnrsOutputStream(streams.FilterStream):
101
    '''Decodes a TNRS response whose names were encoded with encode()'''
102
    def __init__(self, stream):
103
        streams.FilterStream.__init__(self, decode_for_tsv, stream)
104

    
105
def parse_response(name, pattern, str_, response, response_info):
106
    match = re.match(pattern, str_)
107
    if not match:
108
        raise InvalidResponse('Invalid '+name+' response:\n'+response_info+'\n'
109
            +response)
110
    return match.groups()
111

    
112
def tnrs_request(names, debug=False):
113
    '''
114
    Note that names containing only whitespace characters (after gwt_encode())
115
    are ignored by TNRS and do not receive a response row. Thus, you should
116
    always match up the Name_submitted returned by TNRS with the actual
117
    submitted name to determine the corresponding TNRS response row.
118
    '''
119
    name_ct = len(names)
120
    assert name_ct <= max_names
121
    
122
    # Logging
123
    def debug_log(label, str_=''):
124
        if debug: sys.stderr.write('\n'+label+':\n'+str_+'\n')
125
    
126
    ## HTTP
127
    headers = initial_headers.copy() # don't modify global constant!
128
    
129
    def do_request(request):
130
        debug_log('request', str(request))
131
        response = urllib2.urlopen(urllib2.Request(url, request, headers))
132
        response_str = streams.read_all(response)
133
        response_info = str(response.info())
134
        debug_log('response info', response_info)
135
        debug_log('response str', response_str)
136
        return response_str, response_info
137
    
138
    def do_repeated_request(request):
139
        pause = initial_pause
140
        total_pause = 0
141
        while True:
142
            total_pause += pause
143
            if total_pause > max_pause: raise # error is not temporary
144
            debug_log('total_pause', str(total_pause)+'s')
145
            time.sleep(pause) # wait for job to complete
146
            
147
            try: return do_request(request)
148
            except urllib2.HTTPError: pass # try again
149
            pause *= pause_growth_factor
150
    
151
    profiler = profiling.ItersProfiler(start_now=True, iter_text='name')
152
    try:
153
        debug_log('Submit')
154
        request = submission_request_template.replace('[names]',
155
            gwt_encode('\n'.join(map(encode, names))))
156
        response, response_info = do_request(request)
157
        key, = parse_response('submission', submission_response_pattern,
158
            response, response, response_info)
159
        debug_log('key', key)
160
        key_enc = gwt_encode(key)
161
        
162
        debug_log('Retrieve')
163
        request = retrieval_request_template.replace('[key]', key_enc)
164
        response, response_info = do_repeated_request(request)
165
        parse_response('retrieval', retrieval_response_pattern, response,
166
            response, response_info)
167
        session_id, = parse_response('retrieval info',
168
            retrieval_response_info_pattern, response_info, response,
169
            response_info)
170
        debug_log('session_id', session_id)
171
        headers['Cookie'] = 'JSESSIONID='+session_id
172
        
173
        # The output of the retrieve step is unusable because the array has
174
        # different lengths depending on the taxonomic ranks present in the
175
        # provided taxon name. The extra download step is therefore necessary.
176
        
177
        debug_log('Prepare download')
178
        request = download_request_template.replace('[key]', key_enc)
179
        response, response_info = do_request(request)
180
        csv_url, = parse_response('download', download_response_pattern,
181
            response, response, response_info)
182
        csv_url += download_url_suffix
183
        debug_log('csv_url', csv_url)
184
        
185
        debug_log('Download')
186
        response = urllib2.urlopen(urllib2.Request(csv_url))
187
        debug_log('response info', str(response.info()))
188
        return TnrsOutputStream(response)
189
    finally:
190
        profiler.stop(name_ct)
191
        sys.stderr.write(profiler.msg()+'\n')
192

    
193
def repeated_tnrs_request(names, debug=False, **kw_args):
194
    for try_num in xrange(2):
195
        try: return tnrs_request(names, debug, **kw_args)
196
        except (urllib2.HTTPError, InvalidResponse), e:
197
            exc.print_ex(e, detail=False)
198
            debug = True
199
            # try again with debug turned on
200
    raise # error is not temporary
(38-38/45)