1
|
# XML "function" nodes that transform their contents
|
2
|
|
3
|
import datetime
|
4
|
import re
|
5
|
import sre_constants
|
6
|
import warnings
|
7
|
|
8
|
import angles
|
9
|
import dates
|
10
|
import exc
|
11
|
import format
|
12
|
import maps
|
13
|
import sql
|
14
|
import sql_io
|
15
|
import strings
|
16
|
import term
|
17
|
import units
|
18
|
import util
|
19
|
import xml_dom
|
20
|
import xpath
|
21
|
|
22
|
##### Exceptions
|
23
|
|
24
|
class SyntaxError(exc.ExceptionWithCause):
|
25
|
def __init__(self, cause):
|
26
|
exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
|
27
|
cause)
|
28
|
|
29
|
class FormatException(exc.ExceptionWithCause):
|
30
|
def __init__(self, cause):
|
31
|
exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
|
32
|
|
33
|
##### Helper functions
|
34
|
|
35
|
def map_names(func, params):
|
36
|
return [(func(name), value) for name, value in params]
|
37
|
|
38
|
def variadic_args(node):
|
39
|
args = map_names(float, xml_dom.NodeEntryIter(node))
|
40
|
args.sort()
|
41
|
return [value for name, value in args]
|
42
|
|
43
|
def map_items(func, items):
|
44
|
return [(name, func(value)) for name, value in items]
|
45
|
|
46
|
def cast(type_, val):
|
47
|
'''Throws FormatException if can't cast'''
|
48
|
try: return type_(val)
|
49
|
except ValueError, e: raise FormatException(e)
|
50
|
|
51
|
def conv_items(type_, items):
|
52
|
return map_items(lambda val: cast(type_, val),
|
53
|
xml_dom.TextEntryOnlyIter(items))
|
54
|
|
55
|
def pop_value(items, name='value'):
|
56
|
'''@param name Name of value param, or None to accept any name'''
|
57
|
try: last = items.pop() # last entry contains value
|
58
|
except IndexError: return None # input is empty and no actions
|
59
|
if name != None and last[0] != name: return None # input is empty
|
60
|
return last[1]
|
61
|
|
62
|
def merge_tagged(root):
|
63
|
'''Merges siblings in root that are marked as mergeable.
|
64
|
Used to recombine pieces of nodes that were split apart in the mappings.
|
65
|
'''
|
66
|
for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
|
67
|
xml_dom.merge_by_name(root, name)
|
68
|
|
69
|
# Recurse
|
70
|
for child in xml_dom.NodeElemIter(root): merge_tagged(child)
|
71
|
|
72
|
funcs = {}
|
73
|
simplifying_funcs = {}
|
74
|
|
75
|
##### Public functions
|
76
|
|
77
|
var_name_prefix = '$'
|
78
|
|
79
|
def is_var_name(str_): return str_.startswith(var_name_prefix)
|
80
|
|
81
|
def is_var(node):
|
82
|
return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
|
83
|
|
84
|
def is_func_name(name):
|
85
|
return name.startswith('_') and name != '_' # '_' is default root node name
|
86
|
|
87
|
def is_func(node): return is_func_name(node.tagName)
|
88
|
|
89
|
def is_xml_func_name(name): return is_func_name(name) and name in funcs
|
90
|
|
91
|
def is_xml_func(node): return is_xml_func_name(node.tagName)
|
92
|
|
93
|
def passthru(node):
|
94
|
'''Passes through single child node. First prunes the node.'''
|
95
|
xml_dom.prune(node)
|
96
|
children = list(xml_dom.NodeEntryIter(node))
|
97
|
if len(children) == 1: xml_dom.replace(node, children[0][1])
|
98
|
|
99
|
def simplify(node):
|
100
|
'''Simplifies an XML tree.
|
101
|
* Merges nodes tagged as mergable
|
102
|
* Runs simplifying functions
|
103
|
'''
|
104
|
for child in xml_dom.NodeElemIter(node): simplify(child)
|
105
|
merge_tagged(node)
|
106
|
|
107
|
name = node.tagName
|
108
|
|
109
|
# Pass-through optimizations
|
110
|
if is_func_name(name):
|
111
|
try: func = simplifying_funcs[name]
|
112
|
except KeyError: pass
|
113
|
else: func(node)
|
114
|
# Pruning optimizations
|
115
|
else: # these should not run on functions because they would remove args
|
116
|
xml_dom.prune_children(node)
|
117
|
|
118
|
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
|
119
|
'''Evaluates the XML functions in an XML tree.
|
120
|
@param is_rel_func None|f(str) Tests if a name is a relational function.
|
121
|
* If != None: Non-relational functions are removed, or relational
|
122
|
functions are treated specially, depending on the db param (below).
|
123
|
@param db
|
124
|
* If None: Non-relational functions other than structural functions are
|
125
|
replaced with their last parameter (usually the value), not evaluated.
|
126
|
This is used in column-based mode to remove XML-only functions.
|
127
|
* If != None: Relational functions are evaluated directly. This is used
|
128
|
in row-based mode to combine relational and XML functions.
|
129
|
'''
|
130
|
has_rel_funcs = is_rel_func != None
|
131
|
assert db == None or has_rel_funcs # rel_funcs required if db set
|
132
|
|
133
|
for child in xml_dom.NodeElemIter(node):
|
134
|
process(child, on_error, is_rel_func, db)
|
135
|
merge_tagged(node)
|
136
|
|
137
|
name = node.tagName
|
138
|
if not is_func_name(name): return node # not any kind of function
|
139
|
|
140
|
row_mode = has_rel_funcs and db != None
|
141
|
column_mode = has_rel_funcs and db == None
|
142
|
func = funcs.get(name, None)
|
143
|
items = list(xml_dom.NodeTextEntryIter(node))
|
144
|
|
145
|
# Parse function
|
146
|
if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
|
147
|
# pass-through optimization for aggregating functions with one arg
|
148
|
value = items[0][1] # pass through first arg
|
149
|
elif row_mode and (is_rel_func(name) or func == None): # row-based mode
|
150
|
try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
|
151
|
except sql.DoesNotExistException: return # preserve unknown funcs
|
152
|
# possibly a built-in function of db_xml.put()
|
153
|
elif column_mode or func == None:
|
154
|
# local XML function can't be used or does not exist
|
155
|
if column_mode and is_rel_func(name): return # preserve relational funcs
|
156
|
# otherwise XML-only in column mode, or DB-only in XML output mode
|
157
|
value = pop_value(items, None) # just replace with last param
|
158
|
else: # local XML function
|
159
|
try: value = func(items, node)
|
160
|
except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
|
161
|
# Save in case another exception raised, overwriting sys.exc_info()
|
162
|
exc.add_traceback(e)
|
163
|
str_ = strings.ustr(node)
|
164
|
exc.add_msg(e, 'function:\n'+str_)
|
165
|
xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
|
166
|
'\n'+term.emph_multiline(str_)))
|
167
|
|
168
|
on_error(e)
|
169
|
return # in case on_error() returns
|
170
|
|
171
|
xml_dom.replace_with_text(node, value)
|
172
|
|
173
|
##### Simplifying functions
|
174
|
|
175
|
# Function names must start with _ to avoid collisions with real tags
|
176
|
# Functions take params (node) and have no return value
|
177
|
|
178
|
#### Logic
|
179
|
|
180
|
def _exists(node):
|
181
|
'''Returns whether its node is non-empty'''
|
182
|
xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
|
183
|
simplifying_funcs['_exists'] = _exists
|
184
|
|
185
|
def _if(node):
|
186
|
'''
|
187
|
*Must* be run to remove conditions that functions._if() can't handle.
|
188
|
Note: Can add `@name` attr to distinguish separate _if statements.
|
189
|
'''
|
190
|
params = dict(xml_dom.NodeEntryIter(node))
|
191
|
then = params.get('then', None)
|
192
|
cond = params.get('cond', None)
|
193
|
else_ = params.get('else', None)
|
194
|
|
195
|
if cond == None: xml_dom.replace(node, else_) # always False
|
196
|
elif then == else_: xml_dom.replace(node, then) # always same value
|
197
|
elif is_var(cond): pass # can't simplify variable conditions
|
198
|
elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
|
199
|
xml_dom.replace(node, then)
|
200
|
simplifying_funcs['_if'] = _if
|
201
|
|
202
|
#### Merging
|
203
|
|
204
|
simplifying_funcs['_alt'] = passthru
|
205
|
simplifying_funcs['_join'] = passthru
|
206
|
simplifying_funcs['_merge'] = passthru
|
207
|
simplifying_funcs['_name'] = passthru
|
208
|
|
209
|
def _first(node):
|
210
|
'''Chooses the first param (after sorting by numeric param name)'''
|
211
|
args = variadic_args(node)
|
212
|
try: first = args[0]
|
213
|
except IndexError: first = None
|
214
|
xml_dom.replace(node, first)
|
215
|
simplifying_funcs['_first'] = _first
|
216
|
|
217
|
##### XML functions
|
218
|
|
219
|
# Function names must start with _ to avoid collisions with real tags
|
220
|
# Functions take arguments (items, node)
|
221
|
|
222
|
#### Transforming values
|
223
|
|
224
|
def repl(repls, value):
|
225
|
'''Raises error if value not in map and no special '*' entry
|
226
|
@param repls dict repl:with
|
227
|
repl "*" means all other input values
|
228
|
with "*" means keep input value the same
|
229
|
with "" means ignore input value
|
230
|
'''
|
231
|
try: new_value = repls[value]
|
232
|
except KeyError, e:
|
233
|
# Save traceback right away in case another exception raised
|
234
|
fe = FormatException(e)
|
235
|
try: new_value = repls['*']
|
236
|
except KeyError: raise fe
|
237
|
if new_value == '*': new_value = value # '*' means keep input value the same
|
238
|
return new_value
|
239
|
|
240
|
def _map(items, node):
|
241
|
'''See repl()
|
242
|
@param items
|
243
|
<last_entry> Value
|
244
|
<other_entries> name=value Mappings. Special values: See repl() repls.
|
245
|
'''
|
246
|
value = pop_value(items)
|
247
|
if value == None: return None # input is empty
|
248
|
return util.none_if(repl(dict(items), value), u'') # empty value means None
|
249
|
funcs['_map'] = _map
|
250
|
|
251
|
def _replace(items, node):
|
252
|
items = conv_items(strings.ustr, items) # get *once* from iter, check types
|
253
|
value = pop_value(items)
|
254
|
if value == None: return None # input is empty
|
255
|
try:
|
256
|
for repl, with_ in items:
|
257
|
if re.match(r'^\w+$', repl):
|
258
|
repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
|
259
|
value = re.sub(repl, with_, value)
|
260
|
except sre_constants.error, e: raise SyntaxError(e)
|
261
|
return util.none_if(value.strip(), u'') # empty strings always mean None
|
262
|
funcs['_replace'] = _replace
|
263
|
|
264
|
#### Quantities
|
265
|
|
266
|
def _units(items, node):
|
267
|
value = pop_value(items)
|
268
|
if value == None: return None # input is empty
|
269
|
|
270
|
quantity = units.str2quantity(value)
|
271
|
try:
|
272
|
for action, units_ in items:
|
273
|
units_ = util.none_if(units_, u'')
|
274
|
if action == 'default': units.set_default_units(quantity, units_)
|
275
|
elif action == 'to':
|
276
|
try: quantity = units.convert(quantity, units_)
|
277
|
except ValueError, e: raise FormatException(e)
|
278
|
else: raise SyntaxError(ValueError('Invalid action: '+action))
|
279
|
except units.MissingUnitsException, e: raise FormatException(e)
|
280
|
return units.quantity2str(quantity)
|
281
|
funcs['_units'] = _units
|
282
|
|
283
|
def parse_range(str_, range_sep='-'):
|
284
|
default = (str_, None)
|
285
|
start, sep, end = str_.partition(range_sep)
|
286
|
if sep == '': return default # not a range
|
287
|
if start == '' and range_sep == '-': return default # negative number
|
288
|
return tuple(d.strip() for d in (start, end))
|
289
|
|
290
|
def _rangeStart(items, node):
|
291
|
items = dict(conv_items(strings.ustr, items))
|
292
|
try: value = items['value']
|
293
|
except KeyError: return None # input is empty
|
294
|
return parse_range(value)[0]
|
295
|
funcs['_rangeStart'] = _rangeStart
|
296
|
|
297
|
def _rangeEnd(items, node):
|
298
|
items = dict(conv_items(strings.ustr, items))
|
299
|
try: value = items['value']
|
300
|
except KeyError: return None # input is empty
|
301
|
return parse_range(value)[1]
|
302
|
funcs['_rangeEnd'] = _rangeEnd
|
303
|
|
304
|
def _range(items, node):
|
305
|
items = dict(conv_items(float, items))
|
306
|
from_ = items.get('from', None)
|
307
|
to = items.get('to', None)
|
308
|
if from_ == None or to == None: return None
|
309
|
return str(to - from_)
|
310
|
funcs['_range'] = _range
|
311
|
|
312
|
def _avg(items, node):
|
313
|
count = 0
|
314
|
sum_ = 0.
|
315
|
for name, value in conv_items(float, items):
|
316
|
count += 1
|
317
|
sum_ += value
|
318
|
if count == 0: return None # input is empty
|
319
|
else: return str(sum_/count)
|
320
|
funcs['_avg'] = _avg
|
321
|
|
322
|
class CvException(Exception):
|
323
|
def __init__(self):
|
324
|
Exception.__init__(self, 'CV (coefficient of variation) values are only'
|
325
|
' allowed for ratio scale data '
|
326
|
'(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
|
327
|
|
328
|
def _noCV(items, node):
|
329
|
items = list(conv_items(strings.ustr, items))
|
330
|
try: name, value = items.pop() # last entry contains value
|
331
|
except IndexError: return None # input is empty
|
332
|
if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
|
333
|
return value
|
334
|
funcs['_noCV'] = _noCV
|
335
|
|
336
|
#### Names
|
337
|
|
338
|
_name_parts_slices_items = [
|
339
|
('first', slice(None, 1)),
|
340
|
('middle', slice(1, -1)),
|
341
|
('last', slice(-1, None)),
|
342
|
]
|
343
|
name_parts_slices = dict(_name_parts_slices_items)
|
344
|
name_parts = [name for name, slice_ in _name_parts_slices_items]
|
345
|
|
346
|
def _name(items, node):
|
347
|
items = dict(list(conv_items(strings.ustr, items)))
|
348
|
parts = []
|
349
|
for part in name_parts:
|
350
|
if part in items: parts.append(items[part])
|
351
|
if not parts: return None # pass None values through; handle no name parts
|
352
|
return ' '.join(parts)
|
353
|
funcs['_name'] = _name
|
354
|
|
355
|
#### Angles
|
356
|
|
357
|
def _compass(items, node):
|
358
|
'''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
|
359
|
items = dict(conv_items(strings.ustr, items))
|
360
|
try: value = items['value']
|
361
|
except KeyError: return None # input is empty
|
362
|
|
363
|
if not value.isupper(): return value # pass through other coordinate formats
|
364
|
try: return util.cast(str, angles.compass2heading(value)) # ignore None
|
365
|
except KeyError, e: raise FormatException(e)
|
366
|
funcs['_compass'] = _compass
|