Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 5190 aaronmk
import operator
5 968 aaronmk
import re
6 1219 aaronmk
import sre_constants
7 2017 aaronmk
import warnings
8 111 aaronmk
9 1607 aaronmk
import angles
10 818 aaronmk
import dates
11 300 aaronmk
import exc
12 1580 aaronmk
import format
13 917 aaronmk
import maps
14 3688 aaronmk
import sql
15 3077 aaronmk
import sql_io
16 1234 aaronmk
import strings
17 827 aaronmk
import term
18 1468 aaronmk
import units
19 1047 aaronmk
import util
20 86 aaronmk
import xml_dom
21 1321 aaronmk
import xpath
22 86 aaronmk
23 995 aaronmk
##### Exceptions
24
25 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
26 797 aaronmk
    def __init__(self, cause):
27 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
28
            cause)
29 278 aaronmk
30 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
31
    def __init__(self, cause):
32
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
33 843 aaronmk
34 1992 aaronmk
##### Helper functions
35 995 aaronmk
36 4334 aaronmk
def map_names(func, params):
37
    return [(func(name), value) for name, value in params]
38
39
def variadic_args(node):
40
    args = map_names(float, xml_dom.NodeEntryIter(node))
41
    args.sort()
42
    return [value for name, value in args]
43
44 1992 aaronmk
def map_items(func, items):
45
    return [(name, func(value)) for name, value in items]
46
47
def cast(type_, val):
48
    '''Throws FormatException if can't cast'''
49
    try: return type_(val)
50
    except ValueError, e: raise FormatException(e)
51
52
def conv_items(type_, items):
53
    return map_items(lambda val: cast(type_, val),
54
        xml_dom.TextEntryOnlyIter(items))
55
56
def pop_value(items, name='value'):
57
    '''@param name Name of value param, or None to accept any name'''
58
    try: last = items.pop() # last entry contains value
59
    except IndexError: return None # input is empty and no actions
60
    if name != None and last[0] != name: return None # input is empty
61
    return last[1]
62
63 3335 aaronmk
def merge_tagged(root):
64
    '''Merges siblings in root that are marked as mergeable.
65
    Used to recombine pieces of nodes that were split apart in the mappings.
66
    '''
67
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
68
        xml_dom.merge_by_name(root, name)
69
70
    # Recurse
71
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
72
73 995 aaronmk
funcs = {}
74 4236 aaronmk
simplifying_funcs = {}
75 995 aaronmk
76 1992 aaronmk
##### Public functions
77
78 4239 aaronmk
var_name_prefix = '$'
79
80
def is_var_name(str_): return str_.startswith(var_name_prefix)
81
82
def is_var(node):
83
    return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
84
85 2112 aaronmk
def is_func_name(name):
86
    return name.startswith('_') and name != '_' # '_' is default root node name
87
88
def is_func(node): return is_func_name(node.tagName)
89
90
def is_xml_func_name(name): return is_func_name(name) and name in funcs
91
92
def is_xml_func(node): return is_xml_func_name(node.tagName)
93
94 4300 aaronmk
def passthru(node):
95 4302 aaronmk
    '''Passes through single child node. First prunes the node.'''
96 4322 aaronmk
    xml_dom.prune(node)
97 4300 aaronmk
    children = list(xml_dom.NodeEntryIter(node))
98
    if len(children) == 1: xml_dom.replace(node, children[0][1])
99
100 4041 aaronmk
def simplify(node):
101 4305 aaronmk
    '''Simplifies an XML tree.
102 4041 aaronmk
    * Merges nodes tagged as mergable
103 4236 aaronmk
    * Runs simplifying functions
104 4041 aaronmk
    '''
105
    for child in xml_dom.NodeElemIter(node): simplify(child)
106
    merge_tagged(node)
107
108 4227 aaronmk
    name = node.tagName
109 4041 aaronmk
110 4078 aaronmk
    # Pass-through optimizations
111 4228 aaronmk
    if is_func_name(name):
112 4236 aaronmk
        try: func = simplifying_funcs[name]
113 4756 aaronmk
        except KeyError: xml_dom.prune_empty(node)
114 4236 aaronmk
        else: func(node)
115 4229 aaronmk
    # Pruning optimizations
116
    else: # these should not run on functions because they would remove args
117 4319 aaronmk
        xml_dom.prune_children(node)
118 4041 aaronmk
119 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
120 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
121 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
122 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
123
          functions are treated specially, depending on the db param (below).
124
    @param db
125
        * If None: Non-relational functions other than structural functions are
126
          replaced with their last parameter (usually the value), not evaluated.
127
          This is used in column-based mode to remove XML-only functions.
128
        * If != None: Relational functions are evaluated directly. This is used
129
          in row-based mode to combine relational and XML functions.
130 2597 aaronmk
    '''
131 3424 aaronmk
    has_rel_funcs = is_rel_func != None
132 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
133 2597 aaronmk
134 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
135 3424 aaronmk
        process(child, on_error, is_rel_func, db)
136 3335 aaronmk
    merge_tagged(node)
137 3333 aaronmk
138 995 aaronmk
    name = node.tagName
139 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
140 2602 aaronmk
141
    row_mode = has_rel_funcs and db != None
142
    column_mode = has_rel_funcs and db == None
143 3629 aaronmk
    func = funcs.get(name, None)
144 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
145 2602 aaronmk
146 3029 aaronmk
    # Parse function
147
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
148
        # pass-through optimization for aggregating functions with one arg
149
        value = items[0][1] # pass through first arg
150 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
151 5190 aaronmk
        if reduce(operator.or_, (xml_dom.is_node(v) for n, v in items)):
152
            return # preserve complex funcs
153 5720 aaronmk
        # Evaluate using DB
154
        try: value = sql_io.put(db, name, dict(items), on_error=on_error)
155 3688 aaronmk
        except sql.DoesNotExistException: return # preserve unknown funcs
156
            # possibly a built-in function of db_xml.put()
157 4024 aaronmk
    elif column_mode or func == None:
158 3640 aaronmk
        # local XML function can't be used or does not exist
159
        if column_mode and is_rel_func(name): return # preserve relational funcs
160
        # otherwise XML-only in column mode, or DB-only in XML output mode
161
        value = pop_value(items, None) # just replace with last param
162 2602 aaronmk
    else: # local XML function
163 3629 aaronmk
        try: value = func(items, node)
164 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
165 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
166
            exc.add_traceback(e)
167 1562 aaronmk
            str_ = strings.ustr(node)
168 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
169 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
170
                '\n'+term.emph_multiline(str_)))
171
172 995 aaronmk
            on_error(e)
173 2602 aaronmk
            return # in case on_error() returns
174 3227 aaronmk
175 3333 aaronmk
    xml_dom.replace_with_text(node, value)
176 995 aaronmk
177 4236 aaronmk
##### Simplifying functions
178
179
# Function names must start with _ to avoid collisions with real tags
180
# Functions take params (node) and have no return value
181
182 4237 aaronmk
#### Logic
183
184
def _exists(node):
185
    '''Returns whether its node is non-empty'''
186
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
187
simplifying_funcs['_exists'] = _exists
188
189 4240 aaronmk
def _if(node):
190 4477 aaronmk
    '''
191
    *Must* be run to remove conditions that functions._if() can't handle.
192
    Note: Can add `@name` attr to distinguish separate _if statements.
193
    '''
194 4240 aaronmk
    params = dict(xml_dom.NodeEntryIter(node))
195
    then = params.get('then', None)
196
    cond = params.get('cond', None)
197
    else_ = params.get('else', None)
198
199
    if cond == None: xml_dom.replace(node, else_) # always False
200
    elif then == else_: xml_dom.replace(node, then) # always same value
201
    elif is_var(cond): pass # can't simplify variable conditions
202
    elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
203
        xml_dom.replace(node, then)
204
simplifying_funcs['_if'] = _if
205
206 4303 aaronmk
#### Merging
207
208
simplifying_funcs['_alt'] = passthru
209 4326 aaronmk
simplifying_funcs['_join'] = passthru
210 5011 aaronmk
simplifying_funcs['_join_words'] = passthru
211 4303 aaronmk
simplifying_funcs['_merge'] = passthru
212 5409 aaronmk
simplifying_funcs['_min'] = passthru
213
simplifying_funcs['_max'] = passthru
214 4303 aaronmk
215 4335 aaronmk
def _first(node):
216
    '''Chooses the first param (after sorting by numeric param name)'''
217
    args = variadic_args(node)
218
    try: first = args[0]
219
    except IndexError: first = None
220
    xml_dom.replace(node, first)
221
simplifying_funcs['_first'] = _first
222
223 1469 aaronmk
##### XML functions
224 995 aaronmk
225
# Function names must start with _ to avoid collisions with real tags
226 4144 aaronmk
# Functions take arguments (items, node)
227 995 aaronmk
228 1469 aaronmk
#### Transforming values
229
230 1602 aaronmk
def repl(repls, value):
231 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
232 1602 aaronmk
    @param repls dict repl:with
233
        repl "*" means all other input values
234
        with "*" means keep input value the same
235
        with "" means ignore input value
236 1537 aaronmk
    '''
237 1602 aaronmk
    try: new_value = repls[value]
238 1304 aaronmk
    except KeyError, e:
239 1537 aaronmk
        # Save traceback right away in case another exception raised
240 2984 aaronmk
        fe = FormatException(e)
241 1602 aaronmk
        try: new_value = repls['*']
242 1609 aaronmk
        except KeyError: raise fe
243 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
244 1607 aaronmk
    return new_value
245 1602 aaronmk
246 2016 aaronmk
def _map(items, node):
247 1602 aaronmk
    '''See repl()
248
    @param items
249
        <last_entry> Value
250
        <other_entries> name=value Mappings. Special values: See repl() repls.
251
    '''
252
    value = pop_value(items)
253
    if value == None: return None # input is empty
254 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
255 1219 aaronmk
funcs['_map'] = _map
256
257 2016 aaronmk
def _replace(items, node):
258 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
259 1581 aaronmk
    value = pop_value(items)
260
    if value == None: return None # input is empty
261 1219 aaronmk
    try:
262
        for repl, with_ in items:
263
            if re.match(r'^\w+$', repl):
264
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
265
            value = re.sub(repl, with_, value)
266 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
267 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
268 1219 aaronmk
funcs['_replace'] = _replace
269
270 1469 aaronmk
#### Quantities
271
272 2016 aaronmk
def _units(items, node):
273 1581 aaronmk
    value = pop_value(items)
274
    if value == None: return None # input is empty
275 1471 aaronmk
276 1581 aaronmk
    quantity = units.str2quantity(value)
277 1471 aaronmk
    try:
278
        for action, units_ in items:
279
            units_ = util.none_if(units_, u'')
280
            if action == 'default': units.set_default_units(quantity, units_)
281 1567 aaronmk
            elif action == 'to':
282
                try: quantity = units.convert(quantity, units_)
283 1609 aaronmk
                except ValueError, e: raise FormatException(e)
284 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
285 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
286 1471 aaronmk
    return units.quantity2str(quantity)
287 1225 aaronmk
funcs['_units'] = _units
288
289 1399 aaronmk
def parse_range(str_, range_sep='-'):
290
    default = (str_, None)
291
    start, sep, end = str_.partition(range_sep)
292
    if sep == '': return default # not a range
293 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
294 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
295
296 2016 aaronmk
def _rangeStart(items, node):
297 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
298 1399 aaronmk
    try: value = items['value']
299 1406 aaronmk
    except KeyError: return None # input is empty
300 1399 aaronmk
    return parse_range(value)[0]
301
funcs['_rangeStart'] = _rangeStart
302
303 2016 aaronmk
def _rangeEnd(items, node):
304 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
305 1399 aaronmk
    try: value = items['value']
306 1406 aaronmk
    except KeyError: return None # input is empty
307 1399 aaronmk
    return parse_range(value)[1]
308
funcs['_rangeEnd'] = _rangeEnd
309
310 2016 aaronmk
def _range(items, node):
311 1472 aaronmk
    items = dict(conv_items(float, items))
312
    from_ = items.get('from', None)
313
    to = items.get('to', None)
314
    if from_ == None or to == None: return None
315
    return str(to - from_)
316
funcs['_range'] = _range
317
318 2016 aaronmk
def _avg(items, node):
319 86 aaronmk
    count = 0
320
    sum_ = 0.
321 278 aaronmk
    for name, value in conv_items(float, items):
322 86 aaronmk
        count += 1
323
        sum_ += value
324 1472 aaronmk
    if count == 0: return None # input is empty
325
    else: return str(sum_/count)
326 995 aaronmk
funcs['_avg'] = _avg
327 86 aaronmk
328 968 aaronmk
class CvException(Exception):
329
    def __init__(self):
330
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
331
            ' allowed for ratio scale data '
332
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
333
334 2016 aaronmk
def _noCV(items, node):
335 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
336 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
337
    except IndexError: return None # input is empty
338 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
339 968 aaronmk
    return value
340 995 aaronmk
funcs['_noCV'] = _noCV
341 968 aaronmk
342 1607 aaronmk
#### Angles
343
344 2016 aaronmk
def _compass(items, node):
345 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
346
    items = dict(conv_items(strings.ustr, items))
347
    try: value = items['value']
348
    except KeyError: return None # input is empty
349
350
    if not value.isupper(): return value # pass through other coordinate formats
351
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
352
    except KeyError, e: raise FormatException(e)
353
funcs['_compass'] = _compass