Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 5190 aaronmk
import operator
5 6398 aaronmk
import os
6 968 aaronmk
import re
7 1219 aaronmk
import sre_constants
8 2017 aaronmk
import warnings
9 111 aaronmk
10 1607 aaronmk
import angles
11 818 aaronmk
import dates
12 300 aaronmk
import exc
13 1580 aaronmk
import format
14 917 aaronmk
import maps
15 6754 aaronmk
import scalar
16 3688 aaronmk
import sql
17 3077 aaronmk
import sql_io
18 1234 aaronmk
import strings
19 827 aaronmk
import term
20 1468 aaronmk
import units
21 1047 aaronmk
import util
22 86 aaronmk
import xml_dom
23 1321 aaronmk
import xpath
24 86 aaronmk
25 995 aaronmk
##### Exceptions
26
27 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
28 797 aaronmk
    def __init__(self, cause):
29 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
30
            cause)
31 278 aaronmk
32 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
33
    def __init__(self, cause):
34
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
35 843 aaronmk
36 1992 aaronmk
##### Helper functions
37 995 aaronmk
38 4334 aaronmk
def map_names(func, params):
39
    return [(func(name), value) for name, value in params]
40
41
def variadic_args(node):
42
    args = map_names(float, xml_dom.NodeEntryIter(node))
43
    args.sort()
44
    return [value for name, value in args]
45
46 1992 aaronmk
def map_items(func, items):
47
    return [(name, func(value)) for name, value in items]
48
49
def cast(type_, val):
50
    '''Throws FormatException if can't cast'''
51
    try: return type_(val)
52
    except ValueError, e: raise FormatException(e)
53
54
def conv_items(type_, items):
55
    return map_items(lambda val: cast(type_, val),
56
        xml_dom.TextEntryOnlyIter(items))
57
58
def pop_value(items, name='value'):
59
    '''@param name Name of value param, or None to accept any name'''
60
    try: last = items.pop() # last entry contains value
61
    except IndexError: return None # input is empty and no actions
62
    if name != None and last[0] != name: return None # input is empty
63
    return last[1]
64
65 3335 aaronmk
def merge_tagged(root):
66
    '''Merges siblings in root that are marked as mergeable.
67
    Used to recombine pieces of nodes that were split apart in the mappings.
68
    '''
69
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
70
        xml_dom.merge_by_name(root, name)
71
72
    # Recurse
73
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
74
75 995 aaronmk
funcs = {}
76 4236 aaronmk
simplifying_funcs = {}
77 995 aaronmk
78 1992 aaronmk
##### Public functions
79
80 4239 aaronmk
var_name_prefix = '$'
81
82
def is_var_name(str_): return str_.startswith(var_name_prefix)
83
84
def is_var(node):
85
    return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
86
87 2112 aaronmk
def is_func_name(name):
88
    return name.startswith('_') and name != '_' # '_' is default root node name
89
90
def is_func(node): return is_func_name(node.tagName)
91
92
def is_xml_func_name(name): return is_func_name(name) and name in funcs
93
94
def is_xml_func(node): return is_xml_func_name(node.tagName)
95
96 6754 aaronmk
def is_scalar(value): return scalar.is_scalar(value) and not is_var_name(value)
97
98 4300 aaronmk
def passthru(node):
99 4302 aaronmk
    '''Passes through single child node. First prunes the node.'''
100 4322 aaronmk
    xml_dom.prune(node)
101 4300 aaronmk
    children = list(xml_dom.NodeEntryIter(node))
102
    if len(children) == 1: xml_dom.replace(node, children[0][1])
103
104 4041 aaronmk
def simplify(node):
105 4305 aaronmk
    '''Simplifies an XML tree.
106 4041 aaronmk
    * Merges nodes tagged as mergable
107 4236 aaronmk
    * Runs simplifying functions
108 4041 aaronmk
    '''
109
    for child in xml_dom.NodeElemIter(node): simplify(child)
110
    merge_tagged(node)
111
112 4227 aaronmk
    name = node.tagName
113 4041 aaronmk
114 4078 aaronmk
    # Pass-through optimizations
115 4228 aaronmk
    if is_func_name(name):
116 4236 aaronmk
        try: func = simplifying_funcs[name]
117 4756 aaronmk
        except KeyError: xml_dom.prune_empty(node)
118 4236 aaronmk
        else: func(node)
119 4229 aaronmk
    # Pruning optimizations
120
    else: # these should not run on functions because they would remove args
121 4319 aaronmk
        xml_dom.prune_children(node)
122 4041 aaronmk
123 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
124 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
125 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
126 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
127
          functions are treated specially, depending on the db param (below).
128
    @param db
129
        * If None: Non-relational functions other than structural functions are
130
          replaced with their last parameter (usually the value), not evaluated.
131
          This is used in column-based mode to remove XML-only functions.
132
        * If != None: Relational functions are evaluated directly. This is used
133
          in row-based mode to combine relational and XML functions.
134 2597 aaronmk
    '''
135 3424 aaronmk
    has_rel_funcs = is_rel_func != None
136 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
137 2597 aaronmk
138 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
139 3424 aaronmk
        process(child, on_error, is_rel_func, db)
140 3335 aaronmk
    merge_tagged(node)
141 3333 aaronmk
142 995 aaronmk
    name = node.tagName
143 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
144 2602 aaronmk
145
    row_mode = has_rel_funcs and db != None
146
    column_mode = has_rel_funcs and db == None
147 3629 aaronmk
    func = funcs.get(name, None)
148 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
149 2602 aaronmk
150 3029 aaronmk
    # Parse function
151
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
152
        # pass-through optimization for aggregating functions with one arg
153
        value = items[0][1] # pass through first arg
154 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
155 6753 aaronmk
        if items and reduce(operator.or_, (xml_dom.is_node(v)
156
            for n, v in items)): return # preserve complex funcs
157 5720 aaronmk
        # Evaluate using DB
158
        try: value = sql_io.put(db, name, dict(items), on_error=on_error)
159 3688 aaronmk
        except sql.DoesNotExistException: return # preserve unknown funcs
160
            # possibly a built-in function of db_xml.put()
161 4024 aaronmk
    elif column_mode or func == None:
162 3640 aaronmk
        # local XML function can't be used or does not exist
163
        if column_mode and is_rel_func(name): return # preserve relational funcs
164
        # otherwise XML-only in column mode, or DB-only in XML output mode
165
        value = pop_value(items, None) # just replace with last param
166 2602 aaronmk
    else: # local XML function
167 3629 aaronmk
        try: value = func(items, node)
168 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
169 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
170
            exc.add_traceback(e)
171 1562 aaronmk
            str_ = strings.ustr(node)
172 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
173 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
174
                '\n'+term.emph_multiline(str_)))
175
176 995 aaronmk
            on_error(e)
177 2602 aaronmk
            return # in case on_error() returns
178 3227 aaronmk
179 3333 aaronmk
    xml_dom.replace_with_text(node, value)
180 995 aaronmk
181 4236 aaronmk
##### Simplifying functions
182
183
# Function names must start with _ to avoid collisions with real tags
184
# Functions take params (node) and have no return value
185
186 4237 aaronmk
#### Logic
187
188 6431 aaronmk
simplifying_funcs['_and'] = passthru
189
simplifying_funcs['_or'] = passthru
190
191 4237 aaronmk
def _exists(node):
192
    '''Returns whether its node is non-empty'''
193
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
194
simplifying_funcs['_exists'] = _exists
195
196 4240 aaronmk
def _if(node):
197 4477 aaronmk
    '''
198
    *Must* be run to remove conditions that functions._if() can't handle.
199
    Note: Can add `@name` attr to distinguish separate _if statements.
200
    '''
201 4240 aaronmk
    params = dict(xml_dom.NodeEntryIter(node))
202
    then = params.get('then', None)
203
    cond = params.get('cond', None)
204
    else_ = params.get('else', None)
205
206
    if cond == None: xml_dom.replace(node, else_) # always False
207
    elif then == else_: xml_dom.replace(node, then) # always same value
208
    elif is_var(cond): pass # can't simplify variable conditions
209
    elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
210
        xml_dom.replace(node, then)
211
simplifying_funcs['_if'] = _if
212
213 6357 aaronmk
def _nullIf(node):
214
    '''
215
    *Must* be run to remove conditions that functions._nullIf() can't handle.
216
    '''
217
    params = dict(xml_dom.NodeEntryIter(node))
218
    null = params.get('null', None)
219
    value = params.get('value', None)
220
221
    if value == None: xml_dom.prune_parent(node) # empty
222
    elif null == None: xml_dom.replace(node, value) # nothing to null out
223
simplifying_funcs['_nullIf'] = _nullIf
224
225 6755 aaronmk
#### Comparison
226
227
def _eq(node):
228
    params = dict(xml_dom.NodeTextEntryIter(node))
229
    left = params.get('left', None)
230
    right = params.get('right', None)
231
232
    if is_scalar(left) and is_scalar(right): # constant
233
        xml_dom.replace_with_text(node, left == right)
234
    elif left == right: xml_dom.replace_with_text(node, True) # always True
235
simplifying_funcs['_eq'] = _eq
236
237 4303 aaronmk
#### Merging
238
239
simplifying_funcs['_alt'] = passthru
240 4326 aaronmk
simplifying_funcs['_join'] = passthru
241 5011 aaronmk
simplifying_funcs['_join_words'] = passthru
242 4303 aaronmk
simplifying_funcs['_merge'] = passthru
243 5409 aaronmk
simplifying_funcs['_min'] = passthru
244
simplifying_funcs['_max'] = passthru
245 4303 aaronmk
246 4335 aaronmk
def _first(node):
247
    '''Chooses the first param (after sorting by numeric param name)'''
248
    args = variadic_args(node)
249
    try: first = args[0]
250
    except IndexError: first = None
251
    xml_dom.replace(node, first)
252
simplifying_funcs['_first'] = _first
253
254 6398 aaronmk
#### Environment access
255
256
def _env(node):
257 6401 aaronmk
    params = dict(xml_dom.NodeTextEntryIter(node))
258 6398 aaronmk
    try: name = params['name']
259
    except KeyError, e: raise SyntaxError(e)
260
261 6400 aaronmk
    xml_dom.replace_with_text(node, os.environ[name])
262 6398 aaronmk
simplifying_funcs['_env'] = _env
263
264 1469 aaronmk
##### XML functions
265 995 aaronmk
266
# Function names must start with _ to avoid collisions with real tags
267 4144 aaronmk
# Functions take arguments (items, node)
268 995 aaronmk
269 1469 aaronmk
#### Transforming values
270
271 2016 aaronmk
def _replace(items, node):
272 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
273 1581 aaronmk
    value = pop_value(items)
274
    if value == None: return None # input is empty
275 1219 aaronmk
    try:
276
        for repl, with_ in items:
277
            if re.match(r'^\w+$', repl):
278
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
279
            value = re.sub(repl, with_, value)
280 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
281 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
282 1219 aaronmk
funcs['_replace'] = _replace
283
284 1469 aaronmk
#### Quantities
285
286 2016 aaronmk
def _units(items, node):
287 1581 aaronmk
    value = pop_value(items)
288
    if value == None: return None # input is empty
289 1471 aaronmk
290 1581 aaronmk
    quantity = units.str2quantity(value)
291 1471 aaronmk
    try:
292
        for action, units_ in items:
293
            units_ = util.none_if(units_, u'')
294
            if action == 'default': units.set_default_units(quantity, units_)
295 1567 aaronmk
            elif action == 'to':
296
                try: quantity = units.convert(quantity, units_)
297 1609 aaronmk
                except ValueError, e: raise FormatException(e)
298 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
299 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
300 1471 aaronmk
    return units.quantity2str(quantity)
301 1225 aaronmk
funcs['_units'] = _units
302
303 1399 aaronmk
def parse_range(str_, range_sep='-'):
304
    default = (str_, None)
305
    start, sep, end = str_.partition(range_sep)
306
    if sep == '': return default # not a range
307 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
308 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
309
310 2016 aaronmk
def _rangeStart(items, node):
311 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
312 1399 aaronmk
    try: value = items['value']
313 1406 aaronmk
    except KeyError: return None # input is empty
314 1399 aaronmk
    return parse_range(value)[0]
315
funcs['_rangeStart'] = _rangeStart
316
317 2016 aaronmk
def _rangeEnd(items, node):
318 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
319 1399 aaronmk
    try: value = items['value']
320 1406 aaronmk
    except KeyError: return None # input is empty
321 1399 aaronmk
    return parse_range(value)[1]
322
funcs['_rangeEnd'] = _rangeEnd
323
324 2016 aaronmk
def _range(items, node):
325 1472 aaronmk
    items = dict(conv_items(float, items))
326
    from_ = items.get('from', None)
327
    to = items.get('to', None)
328
    if from_ == None or to == None: return None
329
    return str(to - from_)
330
funcs['_range'] = _range
331
332 2016 aaronmk
def _avg(items, node):
333 86 aaronmk
    count = 0
334
    sum_ = 0.
335 278 aaronmk
    for name, value in conv_items(float, items):
336 86 aaronmk
        count += 1
337
        sum_ += value
338 1472 aaronmk
    if count == 0: return None # input is empty
339
    else: return str(sum_/count)
340 995 aaronmk
funcs['_avg'] = _avg
341 86 aaronmk
342 968 aaronmk
class CvException(Exception):
343
    def __init__(self):
344
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
345
            ' allowed for ratio scale data '
346
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
347
348 2016 aaronmk
def _noCV(items, node):
349 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
350 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
351
    except IndexError: return None # input is empty
352 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
353 968 aaronmk
    return value
354 995 aaronmk
funcs['_noCV'] = _noCV
355 968 aaronmk
356 1607 aaronmk
#### Angles
357
358 2016 aaronmk
def _compass(items, node):
359 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
360
    items = dict(conv_items(strings.ustr, items))
361
    try: value = items['value']
362
    except KeyError: return None # input is empty
363
364
    if not value.isupper(): return value # pass through other coordinate formats
365
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
366
    except KeyError, e: raise FormatException(e)
367
funcs['_compass'] = _compass