Project

General

Profile

1
# TNRS
2

    
3
import os.path
4
import re
5
import sys
6
import time
7
import urllib2
8

    
9
import csvs
10
import exc
11
import profiling
12
import streams
13
import strings
14

    
15
# Config
16
initial_pause = 0.35 # sec
17
pause_growth_factor = 1.3
18
max_pause = 30*60 # sec; = 30 min; TNRS sometimes freezes for ~10 min
19
assert initial_pause <= max_pause
20
#max_names = 5000 #according to http://tnrs.iplantcollaborative.org/TNRSapp.html
21
max_names = 500 # the maximum above crashes the live and dev TNRS servers
22

    
23
# Protocol params
24
server = streams.file_get_contents(os.path.dirname(__file__)+"/tnrs.url")
25
#server = 'tnrs.iplantcollaborative.org' # live server
26
url_base = 'http://'+server+'/tnrsdemo/'
27
url = url_base+'search'
28
initial_headers = {
29
    'Content-Type': 'text/x-gwt-rpc; charset=utf-8',
30
    'X-GWT-Module-Base': url_base,
31
    'X-GWT-Permutation': '574AA16D15D917C7704646FD92AFF6B3',
32
}
33
submission_request_template = ('7|0|7|'+url_base+
34
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
35
doSearch|java.lang.String/2004016611|{"sources":"gcc,tpl,tropicos,usda", \
36
"names":"[names]", "type":"matching", "taxonomic":"true", \
37
"classification":"tropicos", "match_to_rank":"true"}|0.05|1|2|3|4|2|5|5|6|7|')
38
submission_response_pattern = r'^//OK\[1,\["(\w+)"\],0,7\]$'
39
retrieval_request_template = ('7|0|15|'+url_base+
40
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService\
41
|getRemoteData|com.extjs.gxt.ui.client.data.PagingLoadConfig|\
42
java.lang.String/2004016611|com.extjs.gxt.ui.client.data.BasePagingLoadConfig/\
43
2011366567|com.extjs.gxt.ui.client.data.RpcMap/3441186752|sortField|sortDir|\
44
com.extjs.gxt.ui.client.Style$SortDir/640452531|offset|java.lang.Integer/\
45
3438268394|limit|{"email":"tnrs@lka5jjs.orv", "key":"[key]", \
46
"taxonomic_constraint":"true", "source_sorting":"true", "first":"false"}\
47
|1|2|3|4|2|5|6|7|0|1|8|4|9|0|10|11|0|12|13|0|14|13|100|15|')
48
	# taxonomic_constraint (Constrain by Higher Taxonomy): selects lower ranks
49
	#  only from within the matched higher ranks. must be turned on, to ensure
50
	#  that higher ranks are always authoritative.
51
	# source_sorting (Constrain by Source):
52
	#  has different behavior depending on the match mode:
53
	#  - in all-matches mode (`"mode":"All"` @ download_request_template):
54
	#    always puts matches in the order of the sources. although the 1st match
55
	#    won't necessarily be the best one, the 1st match *with Selected=true*
56
	#    will be (according to the source sort order). must be turned on, so
57
	#    that the selected matches are sorted by source.
58
	#  - in best-match mode (`"mode":"Best"` @ download_request_template):
59
	#    always selects a match from the first source in the list, no matter how
60
	#    low the match score. must be turned *off* (unlike in all-matches mode),
61
	#    to avoid worse matches being selected instead of better ones.
62
	#    *however*, since this is currently broken and always on, we turn it on
63
	#    so that the download settings reflect what TNRS actually used.
64
retrieval_response_pattern = '^//OK\[.*?\["com.extjs.gxt.ui.client.data.\
65
BasePagingLoadResult/496878394","java.util.ArrayList/4159755760","org.iplantc.\
66
tnrs.demo.shared.BeanTNRSEntry/1039545748",".*"\],0,7\]$'
67
retrieval_response_info_pattern = r'(?ms).*^Set-Cookie: JSESSIONID=(\w+);'
68
download_request_template = ('7|0|6|'+url_base+
69
'|1E87C78041CEFBF0992F46BDF84D7D60|org.iplantc.tnrs.demo.client.SearchService|\
70
downloadRemoteResults|java.lang.String/2004016611|{"name":"tnrs_results.txt", \
71
"mode":"All", "type":"Detailed", "encoding":"utf8", "dirty":"false", \
72
"sources":"false", "taxonomic":"true", "email":"tnrs@lka5jjs.orv", \
73
"key":"[key]"}|1|2|3|4|1|5|6|')
74
	# dirty:
75
	#  when on, sometimes marks multiple names as Selected. must be turned *off*
76
	#  so that only one name is marked as Selected.
77
download_response_pattern = '^//OK\[1,\["(.*)"\],0,7\]$'
78
download_url_suffix = '&name=tnrs_results.txt&encoding=utf8'
79

    
80
class InvalidResponse(Exception): pass
81

    
82
def gwt_encode(str_):
83
    return strings.esc_quotes(strings.json_encode(str_), '|', quote_esc='\!')
84

    
85
def make_spliced_decode_map(decode_map):
86
    return [(r'(?: |(?<=\t)|^)'+re.escape(from_.strip())+r'(?: |(?=\t)|$)',
87
        strings.regexp_repl_esc(to)) for from_, to in decode_map]
88

    
89
padding = ' !pad ' # prepend to empty and whitespace-only strings
90
encode_map = [
91
    ('!', ' !exc '), # our escape char
92
    ('\t', ' !tab '), # TNRS replaces with " "
93
    ('\n', ' !nl '), # used to separate multiple names
94
    ('\r', ' !cr '), # used to separate multiple names
95
    ('"', ' !quo '), # TNRS removes it when at the beginning or end
96
    ('%', ' !pct '), # TNRS URL-decodes it in matched fields
97
    ("'", ' !apo '), # TNRS removes it when at the beginning or end
98
    (';', ' !sem '), # changes TNRS response format
99
    ('\\', ' !bsl '), # TNRS removes it
100
    ('_', ' !und '), # TNRS replaces with " "
101
    ('­', ' !sub '), # TNRS removes it
102
    ('×', ' !mul '), # TNRS replaces with "x"
103
]
104
decode_map = strings.flip_map(encode_map)
105
decode_map.append((padding, ''))
106
spliced_decode_map = make_spliced_decode_map(decode_map)
107

    
108
def encode(str_):
109
    str_ = strings.replace_all(encode_map, str_)
110
    # Empty and whitespace-only strings are ignored by TNRS (no response row)
111
    if str_.strip() == '': str_ = padding+str_
112
    return str_
113

    
114
def decode(str_): return strings.replace_all_re(spliced_decode_map, str_)
115

    
116
decode_for_tsv_map = make_spliced_decode_map([(from_, strings.replace_all(
117
    csvs.tsv_encode_map, to)) for from_, to in decode_map])
118

    
119
def decode_for_tsv(str_):
120
    return strings.replace_all_re(decode_for_tsv_map, str_)
121

    
122
class TnrsOutputStream(streams.FilterStream):
123
    '''Decodes a TNRS response whose names were encoded with encode()'''
124
    def __init__(self, stream):
125
        streams.FilterStream.__init__(self, decode_for_tsv, stream)
126

    
127
def parse_response(name, pattern, str_, response, response_info):
128
    match = re.match(pattern, str_)
129
    if not match:
130
        raise InvalidResponse('Invalid '+name+' response:\n'+response_info+'\n'
131
            +response)
132
    return match.groups()
133

    
134
def single_tnrs_request(names, debug=False, cumulative_profiler=None):
135
    '''
136
    Note that names containing only whitespace characters (after gwt_encode())
137
    are ignored by TNRS and do not receive a response row. Thus, you should
138
    always match up the Name_submitted returned by TNRS with the actual
139
    submitted name to determine the corresponding TNRS response row.
140
    '''
141
    name_ct = len(names)
142
    assert name_ct <= max_names
143
    
144
    # Logging
145
    def debug_log(label, str_=''):
146
        if debug: sys.stderr.write('\n'+label+':\n'+str_+'\n')
147
    
148
    ## HTTP
149
    headers = initial_headers.copy() # don't modify global constant!
150
    
151
    def do_request(request):
152
        request_obj = urllib2.Request(url, request, headers)
153
        debug_log('request info', str(request_obj.header_items()))
154
        debug_log('request str', str(request_obj.get_data()))
155
        response = urllib2.urlopen(request_obj)
156
        response_str = streams.read_all(response)
157
        response_info = str(response.info())
158
        debug_log('response info', response_info)
159
        debug_log('response str', response_str)
160
        return response_str, response_info
161
    
162
    def do_repeated_request(request):
163
        pause = initial_pause
164
        total_pause = 0
165
        while True:
166
            total_pause += pause
167
            if total_pause > max_pause: raise # error is not temporary
168
            debug_log('total_pause', str(total_pause)+'s')
169
            time.sleep(pause) # wait for job to complete
170
            
171
            try: return do_request(request)
172
            except urllib2.HTTPError: pass # try again
173
            pause *= pause_growth_factor
174
    
175
    profiler = profiling.ItersProfiler(start_now=True, iter_text='name')
176
    try:
177
        debug_log('Submit')
178
        request = submission_request_template.replace('[names]',
179
            gwt_encode('\n'.join(map(encode, names))))
180
        response, response_info = do_request(request)
181
        key, = parse_response('submission', submission_response_pattern,
182
            response, response, response_info)
183
        debug_log('key', key)
184
        key_enc = gwt_encode(key)
185
        
186
        debug_log('Retrieve')
187
        request = retrieval_request_template.replace('[key]', key_enc)
188
        response, response_info = do_repeated_request(request)
189
        parse_response('retrieval', retrieval_response_pattern, response,
190
            response, response_info)
191
        session_id, = parse_response('retrieval info',
192
            retrieval_response_info_pattern, response_info, response,
193
            response_info)
194
        debug_log('session_id', session_id)
195
        headers['Cookie'] = 'JSESSIONID='+session_id
196
        
197
        # The output of the retrieve step is unusable because the array has
198
        # different lengths depending on the taxonomic ranks present in the
199
        # provided taxon name. The extra download step is therefore necessary.
200
        
201
        debug_log('Prepare download')
202
        request = download_request_template.replace('[key]', key_enc)
203
        response, response_info = do_request(request)
204
        csv_url, = parse_response('download', download_response_pattern,
205
            response, response, response_info)
206
        csv_url += download_url_suffix
207
        debug_log('csv_url', csv_url)
208
        
209
        debug_log('Download')
210
        response = urllib2.urlopen(urllib2.Request(csv_url))
211
        debug_log('response info', str(response.info()))
212
        return TnrsOutputStream(response)
213
    finally:
214
        profiler.stop(name_ct)
215
        sys.stderr.write(profiler.msg()+'\n')
216
        
217
        if cumulative_profiler != None:
218
            cumulative_profiler.add_subprofiler(profiler)
219
            sys.stderr.write('Cumulatively: '+cumulative_profiler.msg()+'\n')
220

    
221
def tnrs_request(names, debug=False, **kw_args):
222
    for try_num in xrange(2):
223
        try: return single_tnrs_request(names, debug, **kw_args)
224
        except (urllib2.HTTPError, InvalidResponse), e:
225
            exc.print_ex(e, detail=False)
226
            debug = True
227
            # try again with debug turned on
228
    raise # error is not temporary
(42-42/49)