1 |
996
|
aaronmk
|
# XML "function" nodes that transform their contents
|
2 |
86
|
aaronmk
|
|
3 |
111
|
aaronmk
|
import datetime
|
4 |
5190
|
aaronmk
|
import operator
|
5 |
6398
|
aaronmk
|
import os
|
6 |
968
|
aaronmk
|
import re
|
7 |
1219
|
aaronmk
|
import sre_constants
|
8 |
2017
|
aaronmk
|
import warnings
|
9 |
111
|
aaronmk
|
|
10 |
1607
|
aaronmk
|
import angles
|
11 |
818
|
aaronmk
|
import dates
|
12 |
300
|
aaronmk
|
import exc
|
13 |
1580
|
aaronmk
|
import format
|
14 |
7008
|
aaronmk
|
import lists
|
15 |
917
|
aaronmk
|
import maps
|
16 |
6754
|
aaronmk
|
import scalar
|
17 |
3688
|
aaronmk
|
import sql
|
18 |
3077
|
aaronmk
|
import sql_io
|
19 |
1234
|
aaronmk
|
import strings
|
20 |
827
|
aaronmk
|
import term
|
21 |
1468
|
aaronmk
|
import units
|
22 |
1047
|
aaronmk
|
import util
|
23 |
86
|
aaronmk
|
import xml_dom
|
24 |
1321
|
aaronmk
|
import xpath
|
25 |
86
|
aaronmk
|
|
26 |
995
|
aaronmk
|
##### Exceptions
|
27 |
|
|
|
28 |
1612
|
aaronmk
|
class SyntaxError(exc.ExceptionWithCause):
|
29 |
797
|
aaronmk
|
def __init__(self, cause):
|
30 |
1611
|
aaronmk
|
exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
|
31 |
|
|
cause)
|
32 |
278
|
aaronmk
|
|
33 |
1613
|
aaronmk
|
class FormatException(exc.ExceptionWithCause):
|
34 |
|
|
def __init__(self, cause):
|
35 |
|
|
exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
|
36 |
843
|
aaronmk
|
|
37 |
1992
|
aaronmk
|
##### Helper functions
|
38 |
995
|
aaronmk
|
|
39 |
4334
|
aaronmk
|
def map_names(func, params):
|
40 |
|
|
return [(func(name), value) for name, value in params]
|
41 |
|
|
|
42 |
|
|
def variadic_args(node):
|
43 |
|
|
args = map_names(float, xml_dom.NodeEntryIter(node))
|
44 |
|
|
args.sort()
|
45 |
|
|
return [value for name, value in args]
|
46 |
|
|
|
47 |
1992
|
aaronmk
|
def map_items(func, items):
|
48 |
|
|
return [(name, func(value)) for name, value in items]
|
49 |
|
|
|
50 |
|
|
def cast(type_, val):
|
51 |
|
|
'''Throws FormatException if can't cast'''
|
52 |
|
|
try: return type_(val)
|
53 |
|
|
except ValueError, e: raise FormatException(e)
|
54 |
|
|
|
55 |
|
|
def conv_items(type_, items):
|
56 |
|
|
return map_items(lambda val: cast(type_, val),
|
57 |
|
|
xml_dom.TextEntryOnlyIter(items))
|
58 |
|
|
|
59 |
|
|
def pop_value(items, name='value'):
|
60 |
|
|
'''@param name Name of value param, or None to accept any name'''
|
61 |
|
|
try: last = items.pop() # last entry contains value
|
62 |
|
|
except IndexError: return None # input is empty and no actions
|
63 |
|
|
if name != None and last[0] != name: return None # input is empty
|
64 |
|
|
return last[1]
|
65 |
|
|
|
66 |
3335
|
aaronmk
|
def merge_tagged(root):
|
67 |
|
|
'''Merges siblings in root that are marked as mergeable.
|
68 |
|
|
Used to recombine pieces of nodes that were split apart in the mappings.
|
69 |
|
|
'''
|
70 |
|
|
for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
|
71 |
|
|
xml_dom.merge_by_name(root, name)
|
72 |
|
|
|
73 |
|
|
# Recurse
|
74 |
|
|
for child in xml_dom.NodeElemIter(root): merge_tagged(child)
|
75 |
|
|
|
76 |
995
|
aaronmk
|
funcs = {}
|
77 |
4236
|
aaronmk
|
simplifying_funcs = {}
|
78 |
995
|
aaronmk
|
|
79 |
1992
|
aaronmk
|
##### Public functions
|
80 |
|
|
|
81 |
4239
|
aaronmk
|
var_name_prefix = '$'
|
82 |
|
|
|
83 |
|
|
def is_var_name(str_): return str_.startswith(var_name_prefix)
|
84 |
|
|
|
85 |
|
|
def is_var(node):
|
86 |
|
|
return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
|
87 |
|
|
|
88 |
2112
|
aaronmk
|
def is_func_name(name):
|
89 |
|
|
return name.startswith('_') and name != '_' # '_' is default root node name
|
90 |
|
|
|
91 |
|
|
def is_func(node): return is_func_name(node.tagName)
|
92 |
|
|
|
93 |
|
|
def is_xml_func_name(name): return is_func_name(name) and name in funcs
|
94 |
|
|
|
95 |
|
|
def is_xml_func(node): return is_xml_func_name(node.tagName)
|
96 |
|
|
|
97 |
7006
|
aaronmk
|
def is_scalar(value):
|
98 |
|
|
return scalar.is_scalar(value) and not (util.is_str(value)
|
99 |
|
|
and is_var_name(value))
|
100 |
6754
|
aaronmk
|
|
101 |
4300
|
aaronmk
|
def passthru(node):
|
102 |
4302
|
aaronmk
|
'''Passes through single child node. First prunes the node.'''
|
103 |
4322
|
aaronmk
|
xml_dom.prune(node)
|
104 |
4300
|
aaronmk
|
children = list(xml_dom.NodeEntryIter(node))
|
105 |
|
|
if len(children) == 1: xml_dom.replace(node, children[0][1])
|
106 |
|
|
|
107 |
4041
|
aaronmk
|
def simplify(node):
|
108 |
4305
|
aaronmk
|
'''Simplifies an XML tree.
|
109 |
4041
|
aaronmk
|
* Merges nodes tagged as mergable
|
110 |
4236
|
aaronmk
|
* Runs simplifying functions
|
111 |
4041
|
aaronmk
|
'''
|
112 |
|
|
for child in xml_dom.NodeElemIter(node): simplify(child)
|
113 |
|
|
merge_tagged(node)
|
114 |
|
|
|
115 |
4227
|
aaronmk
|
name = node.tagName
|
116 |
4041
|
aaronmk
|
|
117 |
4078
|
aaronmk
|
# Pass-through optimizations
|
118 |
4228
|
aaronmk
|
if is_func_name(name):
|
119 |
4236
|
aaronmk
|
try: func = simplifying_funcs[name]
|
120 |
4756
|
aaronmk
|
except KeyError: xml_dom.prune_empty(node)
|
121 |
4236
|
aaronmk
|
else: func(node)
|
122 |
4229
|
aaronmk
|
# Pruning optimizations
|
123 |
|
|
else: # these should not run on functions because they would remove args
|
124 |
4319
|
aaronmk
|
xml_dom.prune_children(node)
|
125 |
4041
|
aaronmk
|
|
126 |
3660
|
aaronmk
|
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
|
127 |
2597
|
aaronmk
|
'''Evaluates the XML functions in an XML tree.
|
128 |
3424
|
aaronmk
|
@param is_rel_func None|f(str) Tests if a name is a relational function.
|
129 |
2602
|
aaronmk
|
* If != None: Non-relational functions are removed, or relational
|
130 |
|
|
functions are treated specially, depending on the db param (below).
|
131 |
|
|
@param db
|
132 |
|
|
* If None: Non-relational functions other than structural functions are
|
133 |
|
|
replaced with their last parameter (usually the value), not evaluated.
|
134 |
|
|
This is used in column-based mode to remove XML-only functions.
|
135 |
|
|
* If != None: Relational functions are evaluated directly. This is used
|
136 |
|
|
in row-based mode to combine relational and XML functions.
|
137 |
2597
|
aaronmk
|
'''
|
138 |
3424
|
aaronmk
|
has_rel_funcs = is_rel_func != None
|
139 |
2602
|
aaronmk
|
assert db == None or has_rel_funcs # rel_funcs required if db set
|
140 |
2597
|
aaronmk
|
|
141 |
3333
|
aaronmk
|
for child in xml_dom.NodeElemIter(node):
|
142 |
3424
|
aaronmk
|
process(child, on_error, is_rel_func, db)
|
143 |
3335
|
aaronmk
|
merge_tagged(node)
|
144 |
3333
|
aaronmk
|
|
145 |
995
|
aaronmk
|
name = node.tagName
|
146 |
3227
|
aaronmk
|
if not is_func_name(name): return node # not any kind of function
|
147 |
2602
|
aaronmk
|
|
148 |
|
|
row_mode = has_rel_funcs and db != None
|
149 |
|
|
column_mode = has_rel_funcs and db == None
|
150 |
3629
|
aaronmk
|
func = funcs.get(name, None)
|
151 |
3028
|
aaronmk
|
items = list(xml_dom.NodeTextEntryIter(node))
|
152 |
2602
|
aaronmk
|
|
153 |
3029
|
aaronmk
|
# Parse function
|
154 |
|
|
if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
|
155 |
|
|
# pass-through optimization for aggregating functions with one arg
|
156 |
|
|
value = items[0][1] # pass through first arg
|
157 |
3629
|
aaronmk
|
elif row_mode and (is_rel_func(name) or func == None): # row-based mode
|
158 |
6753
|
aaronmk
|
if items and reduce(operator.or_, (xml_dom.is_node(v)
|
159 |
|
|
for n, v in items)): return # preserve complex funcs
|
160 |
5720
|
aaronmk
|
# Evaluate using DB
|
161 |
|
|
try: value = sql_io.put(db, name, dict(items), on_error=on_error)
|
162 |
3688
|
aaronmk
|
except sql.DoesNotExistException: return # preserve unknown funcs
|
163 |
|
|
# possibly a built-in function of db_xml.put()
|
164 |
4024
|
aaronmk
|
elif column_mode or func == None:
|
165 |
3640
|
aaronmk
|
# local XML function can't be used or does not exist
|
166 |
|
|
if column_mode and is_rel_func(name): return # preserve relational funcs
|
167 |
|
|
# otherwise XML-only in column mode, or DB-only in XML output mode
|
168 |
|
|
value = pop_value(items, None) # just replace with last param
|
169 |
2602
|
aaronmk
|
else: # local XML function
|
170 |
3629
|
aaronmk
|
try: value = func(items, node)
|
171 |
1613
|
aaronmk
|
except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
|
172 |
1371
|
aaronmk
|
# Save in case another exception raised, overwriting sys.exc_info()
|
173 |
|
|
exc.add_traceback(e)
|
174 |
1562
|
aaronmk
|
str_ = strings.ustr(node)
|
175 |
995
|
aaronmk
|
exc.add_msg(e, 'function:\n'+str_)
|
176 |
1810
|
aaronmk
|
xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
|
177 |
|
|
'\n'+term.emph_multiline(str_)))
|
178 |
|
|
|
179 |
995
|
aaronmk
|
on_error(e)
|
180 |
2602
|
aaronmk
|
return # in case on_error() returns
|
181 |
3227
|
aaronmk
|
|
182 |
3333
|
aaronmk
|
xml_dom.replace_with_text(node, value)
|
183 |
995
|
aaronmk
|
|
184 |
4236
|
aaronmk
|
##### Simplifying functions
|
185 |
|
|
|
186 |
|
|
# Function names must start with _ to avoid collisions with real tags
|
187 |
|
|
# Functions take params (node) and have no return value
|
188 |
|
|
|
189 |
4237
|
aaronmk
|
#### Logic
|
190 |
|
|
|
191 |
7008
|
aaronmk
|
def _and(node):
|
192 |
|
|
values = [v for k, v in xml_dom.NodeTextEntryIter(node)]
|
193 |
|
|
|
194 |
|
|
if lists.and_(map(is_scalar, values)): # all constants
|
195 |
|
|
xml_dom.replace_with_text(node, lists.and_(values))
|
196 |
|
|
else: passthru(node)
|
197 |
|
|
simplifying_funcs['_and'] = _and
|
198 |
6431
|
aaronmk
|
|
199 |
7008
|
aaronmk
|
def _or(node):
|
200 |
|
|
values = [v for k, v in xml_dom.NodeTextEntryIter(node)]
|
201 |
|
|
|
202 |
|
|
if lists.and_(map(is_scalar, values)): # all constants
|
203 |
|
|
xml_dom.replace_with_text(node, lists.or_(values))
|
204 |
|
|
else: passthru(node)
|
205 |
|
|
simplifying_funcs['_or'] = _or
|
206 |
|
|
|
207 |
4237
|
aaronmk
|
def _exists(node):
|
208 |
|
|
'''Returns whether its node is non-empty'''
|
209 |
|
|
xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
|
210 |
|
|
simplifying_funcs['_exists'] = _exists
|
211 |
|
|
|
212 |
4240
|
aaronmk
|
def _if(node):
|
213 |
4477
|
aaronmk
|
'''
|
214 |
|
|
*Must* be run to remove conditions that functions._if() can't handle.
|
215 |
|
|
Note: Can add `@name` attr to distinguish separate _if statements.
|
216 |
|
|
'''
|
217 |
4240
|
aaronmk
|
params = dict(xml_dom.NodeEntryIter(node))
|
218 |
|
|
then = params.get('then', None)
|
219 |
|
|
cond = params.get('cond', None)
|
220 |
|
|
else_ = params.get('else', None)
|
221 |
|
|
|
222 |
|
|
if cond == None: xml_dom.replace(node, else_) # always False
|
223 |
|
|
elif then == else_: xml_dom.replace(node, then) # always same value
|
224 |
|
|
elif is_var(cond): pass # can't simplify variable conditions
|
225 |
|
|
elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
|
226 |
|
|
xml_dom.replace(node, then)
|
227 |
|
|
simplifying_funcs['_if'] = _if
|
228 |
|
|
|
229 |
6357
|
aaronmk
|
def _nullIf(node):
|
230 |
|
|
'''
|
231 |
|
|
*Must* be run to remove conditions that functions._nullIf() can't handle.
|
232 |
|
|
'''
|
233 |
|
|
params = dict(xml_dom.NodeEntryIter(node))
|
234 |
|
|
null = params.get('null', None)
|
235 |
|
|
value = params.get('value', None)
|
236 |
|
|
|
237 |
|
|
if value == None: xml_dom.prune_parent(node) # empty
|
238 |
|
|
elif null == None: xml_dom.replace(node, value) # nothing to null out
|
239 |
|
|
simplifying_funcs['_nullIf'] = _nullIf
|
240 |
|
|
|
241 |
6755
|
aaronmk
|
#### Comparison
|
242 |
|
|
|
243 |
|
|
def _eq(node):
|
244 |
|
|
params = dict(xml_dom.NodeTextEntryIter(node))
|
245 |
|
|
left = params.get('left', None)
|
246 |
|
|
right = params.get('right', None)
|
247 |
|
|
|
248 |
|
|
if is_scalar(left) and is_scalar(right): # constant
|
249 |
|
|
xml_dom.replace_with_text(node, left == right)
|
250 |
|
|
elif left == right: xml_dom.replace_with_text(node, True) # always True
|
251 |
|
|
simplifying_funcs['_eq'] = _eq
|
252 |
|
|
|
253 |
4303
|
aaronmk
|
#### Merging
|
254 |
|
|
|
255 |
|
|
simplifying_funcs['_alt'] = passthru
|
256 |
4326
|
aaronmk
|
simplifying_funcs['_join'] = passthru
|
257 |
5011
|
aaronmk
|
simplifying_funcs['_join_words'] = passthru
|
258 |
7141
|
aaronmk
|
simplifying_funcs['_merge_prefix'] = passthru
|
259 |
4303
|
aaronmk
|
simplifying_funcs['_merge'] = passthru
|
260 |
5409
|
aaronmk
|
simplifying_funcs['_min'] = passthru
|
261 |
|
|
simplifying_funcs['_max'] = passthru
|
262 |
7704
|
aaronmk
|
simplifying_funcs['_avg'] = passthru
|
263 |
4303
|
aaronmk
|
|
264 |
4335
|
aaronmk
|
def _first(node):
|
265 |
7001
|
aaronmk
|
'''Chooses the first non-empty param (sorting by numeric param name)'''
|
266 |
|
|
xml_dom.prune_children(node)
|
267 |
4335
|
aaronmk
|
args = variadic_args(node)
|
268 |
|
|
try: first = args[0]
|
269 |
|
|
except IndexError: first = None
|
270 |
|
|
xml_dom.replace(node, first)
|
271 |
|
|
simplifying_funcs['_first'] = _first
|
272 |
|
|
|
273 |
6398
|
aaronmk
|
#### Environment access
|
274 |
|
|
|
275 |
|
|
def _env(node):
|
276 |
6401
|
aaronmk
|
params = dict(xml_dom.NodeTextEntryIter(node))
|
277 |
6398
|
aaronmk
|
try: name = params['name']
|
278 |
|
|
except KeyError, e: raise SyntaxError(e)
|
279 |
|
|
|
280 |
6400
|
aaronmk
|
xml_dom.replace_with_text(node, os.environ[name])
|
281 |
6398
|
aaronmk
|
simplifying_funcs['_env'] = _env
|
282 |
|
|
|
283 |
1469
|
aaronmk
|
##### XML functions
|
284 |
995
|
aaronmk
|
|
285 |
|
|
# Function names must start with _ to avoid collisions with real tags
|
286 |
4144
|
aaronmk
|
# Functions take arguments (items, node)
|
287 |
995
|
aaronmk
|
|
288 |
1469
|
aaronmk
|
#### Transforming values
|
289 |
|
|
|
290 |
2016
|
aaronmk
|
def _replace(items, node):
|
291 |
1562
|
aaronmk
|
items = conv_items(strings.ustr, items) # get *once* from iter, check types
|
292 |
1581
|
aaronmk
|
value = pop_value(items)
|
293 |
|
|
if value == None: return None # input is empty
|
294 |
1219
|
aaronmk
|
try:
|
295 |
|
|
for repl, with_ in items:
|
296 |
|
|
if re.match(r'^\w+$', repl):
|
297 |
|
|
repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
|
298 |
|
|
value = re.sub(repl, with_, value)
|
299 |
1612
|
aaronmk
|
except sre_constants.error, e: raise SyntaxError(e)
|
300 |
1624
|
aaronmk
|
return util.none_if(value.strip(), u'') # empty strings always mean None
|
301 |
1219
|
aaronmk
|
funcs['_replace'] = _replace
|
302 |
|
|
|
303 |
1469
|
aaronmk
|
#### Quantities
|
304 |
|
|
|
305 |
2016
|
aaronmk
|
def _units(items, node):
|
306 |
1581
|
aaronmk
|
value = pop_value(items)
|
307 |
|
|
if value == None: return None # input is empty
|
308 |
1471
|
aaronmk
|
|
309 |
1581
|
aaronmk
|
quantity = units.str2quantity(value)
|
310 |
1471
|
aaronmk
|
try:
|
311 |
|
|
for action, units_ in items:
|
312 |
|
|
units_ = util.none_if(units_, u'')
|
313 |
|
|
if action == 'default': units.set_default_units(quantity, units_)
|
314 |
1567
|
aaronmk
|
elif action == 'to':
|
315 |
|
|
try: quantity = units.convert(quantity, units_)
|
316 |
1609
|
aaronmk
|
except ValueError, e: raise FormatException(e)
|
317 |
1612
|
aaronmk
|
else: raise SyntaxError(ValueError('Invalid action: '+action))
|
318 |
1609
|
aaronmk
|
except units.MissingUnitsException, e: raise FormatException(e)
|
319 |
1471
|
aaronmk
|
return units.quantity2str(quantity)
|
320 |
1225
|
aaronmk
|
funcs['_units'] = _units
|
321 |
|
|
|
322 |
2016
|
aaronmk
|
def _rangeStart(items, node):
|
323 |
1562
|
aaronmk
|
items = dict(conv_items(strings.ustr, items))
|
324 |
1399
|
aaronmk
|
try: value = items['value']
|
325 |
1406
|
aaronmk
|
except KeyError: return None # input is empty
|
326 |
7655
|
aaronmk
|
return units.parse_range(value)[0]
|
327 |
1399
|
aaronmk
|
funcs['_rangeStart'] = _rangeStart
|
328 |
|
|
|
329 |
2016
|
aaronmk
|
def _rangeEnd(items, node):
|
330 |
1562
|
aaronmk
|
items = dict(conv_items(strings.ustr, items))
|
331 |
1399
|
aaronmk
|
try: value = items['value']
|
332 |
1406
|
aaronmk
|
except KeyError: return None # input is empty
|
333 |
7655
|
aaronmk
|
return units.parse_range(value)[1]
|
334 |
1399
|
aaronmk
|
funcs['_rangeEnd'] = _rangeEnd
|
335 |
|
|
|
336 |
2016
|
aaronmk
|
def _range(items, node):
|
337 |
1472
|
aaronmk
|
items = dict(conv_items(float, items))
|
338 |
|
|
from_ = items.get('from', None)
|
339 |
|
|
to = items.get('to', None)
|
340 |
|
|
if from_ == None or to == None: return None
|
341 |
|
|
return str(to - from_)
|
342 |
|
|
funcs['_range'] = _range
|
343 |
|
|
|
344 |
2016
|
aaronmk
|
def _avg(items, node):
|
345 |
86
|
aaronmk
|
count = 0
|
346 |
|
|
sum_ = 0.
|
347 |
278
|
aaronmk
|
for name, value in conv_items(float, items):
|
348 |
86
|
aaronmk
|
count += 1
|
349 |
|
|
sum_ += value
|
350 |
1472
|
aaronmk
|
if count == 0: return None # input is empty
|
351 |
|
|
else: return str(sum_/count)
|
352 |
995
|
aaronmk
|
funcs['_avg'] = _avg
|
353 |
86
|
aaronmk
|
|
354 |
968
|
aaronmk
|
class CvException(Exception):
|
355 |
|
|
def __init__(self):
|
356 |
|
|
Exception.__init__(self, 'CV (coefficient of variation) values are only'
|
357 |
|
|
' allowed for ratio scale data '
|
358 |
|
|
'(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
|
359 |
|
|
|
360 |
2016
|
aaronmk
|
def _noCV(items, node):
|
361 |
3631
|
aaronmk
|
items = list(conv_items(strings.ustr, items))
|
362 |
3046
|
aaronmk
|
try: name, value = items.pop() # last entry contains value
|
363 |
|
|
except IndexError: return None # input is empty
|
364 |
1609
|
aaronmk
|
if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
|
365 |
968
|
aaronmk
|
return value
|
366 |
995
|
aaronmk
|
funcs['_noCV'] = _noCV
|
367 |
968
|
aaronmk
|
|
368 |
1607
|
aaronmk
|
#### Angles
|
369 |
|
|
|
370 |
2016
|
aaronmk
|
def _compass(items, node):
|
371 |
1607
|
aaronmk
|
'''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
|
372 |
|
|
items = dict(conv_items(strings.ustr, items))
|
373 |
|
|
try: value = items['value']
|
374 |
|
|
except KeyError: return None # input is empty
|
375 |
|
|
|
376 |
|
|
if not value.isupper(): return value # pass through other coordinate formats
|
377 |
|
|
try: return util.cast(str, angles.compass2heading(value)) # ignore None
|
378 |
|
|
except KeyError, e: raise FormatException(e)
|
379 |
|
|
funcs['_compass'] = _compass
|