Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 4334 aaronmk
def map_names(func, params):
36
    return [(func(name), value) for name, value in params]
37
38
def variadic_args(node):
39
    args = map_names(float, xml_dom.NodeEntryIter(node))
40
    args.sort()
41
    return [value for name, value in args]
42
43 1992 aaronmk
def map_items(func, items):
44
    return [(name, func(value)) for name, value in items]
45
46
def cast(type_, val):
47
    '''Throws FormatException if can't cast'''
48
    try: return type_(val)
49
    except ValueError, e: raise FormatException(e)
50
51
def conv_items(type_, items):
52
    return map_items(lambda val: cast(type_, val),
53
        xml_dom.TextEntryOnlyIter(items))
54
55
def pop_value(items, name='value'):
56
    '''@param name Name of value param, or None to accept any name'''
57
    try: last = items.pop() # last entry contains value
58
    except IndexError: return None # input is empty and no actions
59
    if name != None and last[0] != name: return None # input is empty
60
    return last[1]
61
62 3335 aaronmk
def merge_tagged(root):
63
    '''Merges siblings in root that are marked as mergeable.
64
    Used to recombine pieces of nodes that were split apart in the mappings.
65
    '''
66
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
67
        xml_dom.merge_by_name(root, name)
68
69
    # Recurse
70
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
71
72 995 aaronmk
funcs = {}
73 4236 aaronmk
simplifying_funcs = {}
74 995 aaronmk
75 1992 aaronmk
##### Public functions
76
77 4239 aaronmk
var_name_prefix = '$'
78
79
def is_var_name(str_): return str_.startswith(var_name_prefix)
80
81
def is_var(node):
82
    return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
83
84 2112 aaronmk
def is_func_name(name):
85
    return name.startswith('_') and name != '_' # '_' is default root node name
86
87
def is_func(node): return is_func_name(node.tagName)
88
89
def is_xml_func_name(name): return is_func_name(name) and name in funcs
90
91
def is_xml_func(node): return is_xml_func_name(node.tagName)
92
93 4300 aaronmk
def passthru(node):
94 4302 aaronmk
    '''Passes through single child node. First prunes the node.'''
95 4322 aaronmk
    xml_dom.prune(node)
96 4300 aaronmk
    children = list(xml_dom.NodeEntryIter(node))
97
    if len(children) == 1: xml_dom.replace(node, children[0][1])
98
99 4041 aaronmk
def simplify(node):
100 4305 aaronmk
    '''Simplifies an XML tree.
101 4041 aaronmk
    * Merges nodes tagged as mergable
102 4236 aaronmk
    * Runs simplifying functions
103 4041 aaronmk
    '''
104
    for child in xml_dom.NodeElemIter(node): simplify(child)
105
    merge_tagged(node)
106
107 4227 aaronmk
    name = node.tagName
108 4041 aaronmk
109 4078 aaronmk
    # Pass-through optimizations
110 4228 aaronmk
    if is_func_name(name):
111 4236 aaronmk
        try: func = simplifying_funcs[name]
112 4756 aaronmk
        except KeyError: xml_dom.prune_empty(node)
113 4236 aaronmk
        else: func(node)
114 4229 aaronmk
    # Pruning optimizations
115
    else: # these should not run on functions because they would remove args
116 4319 aaronmk
        xml_dom.prune_children(node)
117 4041 aaronmk
118 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
119 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
120 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
121 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
122
          functions are treated specially, depending on the db param (below).
123
    @param db
124
        * If None: Non-relational functions other than structural functions are
125
          replaced with their last parameter (usually the value), not evaluated.
126
          This is used in column-based mode to remove XML-only functions.
127
        * If != None: Relational functions are evaluated directly. This is used
128
          in row-based mode to combine relational and XML functions.
129 2597 aaronmk
    '''
130 3424 aaronmk
    has_rel_funcs = is_rel_func != None
131 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
132 2597 aaronmk
133 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
134 3424 aaronmk
        process(child, on_error, is_rel_func, db)
135 3335 aaronmk
    merge_tagged(node)
136 3333 aaronmk
137 995 aaronmk
    name = node.tagName
138 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
139 2602 aaronmk
140
    row_mode = has_rel_funcs and db != None
141
    column_mode = has_rel_funcs and db == None
142 3629 aaronmk
    func = funcs.get(name, None)
143 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
144 2602 aaronmk
145 3029 aaronmk
    # Parse function
146
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
147
        # pass-through optimization for aggregating functions with one arg
148
        value = items[0][1] # pass through first arg
149 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
150 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
151
        except sql.DoesNotExistException: return # preserve unknown funcs
152
            # possibly a built-in function of db_xml.put()
153 4024 aaronmk
    elif column_mode or func == None:
154 3640 aaronmk
        # local XML function can't be used or does not exist
155
        if column_mode and is_rel_func(name): return # preserve relational funcs
156
        # otherwise XML-only in column mode, or DB-only in XML output mode
157
        value = pop_value(items, None) # just replace with last param
158 2602 aaronmk
    else: # local XML function
159 3629 aaronmk
        try: value = func(items, node)
160 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
161 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
162
            exc.add_traceback(e)
163 1562 aaronmk
            str_ = strings.ustr(node)
164 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
165 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
166
                '\n'+term.emph_multiline(str_)))
167
168 995 aaronmk
            on_error(e)
169 2602 aaronmk
            return # in case on_error() returns
170 3227 aaronmk
171 3333 aaronmk
    xml_dom.replace_with_text(node, value)
172 995 aaronmk
173 4236 aaronmk
##### Simplifying functions
174
175
# Function names must start with _ to avoid collisions with real tags
176
# Functions take params (node) and have no return value
177
178 4237 aaronmk
#### Logic
179
180
def _exists(node):
181
    '''Returns whether its node is non-empty'''
182
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
183
simplifying_funcs['_exists'] = _exists
184
185 4240 aaronmk
def _if(node):
186 4477 aaronmk
    '''
187
    *Must* be run to remove conditions that functions._if() can't handle.
188
    Note: Can add `@name` attr to distinguish separate _if statements.
189
    '''
190 4240 aaronmk
    params = dict(xml_dom.NodeEntryIter(node))
191
    then = params.get('then', None)
192
    cond = params.get('cond', None)
193
    else_ = params.get('else', None)
194
195
    if cond == None: xml_dom.replace(node, else_) # always False
196
    elif then == else_: xml_dom.replace(node, then) # always same value
197
    elif is_var(cond): pass # can't simplify variable conditions
198
    elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
199
        xml_dom.replace(node, then)
200
simplifying_funcs['_if'] = _if
201
202 4303 aaronmk
#### Merging
203
204
simplifying_funcs['_alt'] = passthru
205 4326 aaronmk
simplifying_funcs['_join'] = passthru
206 5011 aaronmk
simplifying_funcs['_join_words'] = passthru
207 4303 aaronmk
simplifying_funcs['_merge'] = passthru
208 4304 aaronmk
simplifying_funcs['_name'] = passthru
209 4303 aaronmk
210 4335 aaronmk
def _first(node):
211
    '''Chooses the first param (after sorting by numeric param name)'''
212
    args = variadic_args(node)
213
    try: first = args[0]
214
    except IndexError: first = None
215
    xml_dom.replace(node, first)
216
simplifying_funcs['_first'] = _first
217
218 1469 aaronmk
##### XML functions
219 995 aaronmk
220
# Function names must start with _ to avoid collisions with real tags
221 4144 aaronmk
# Functions take arguments (items, node)
222 995 aaronmk
223 1469 aaronmk
#### Transforming values
224
225 1602 aaronmk
def repl(repls, value):
226 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
227 1602 aaronmk
    @param repls dict repl:with
228
        repl "*" means all other input values
229
        with "*" means keep input value the same
230
        with "" means ignore input value
231 1537 aaronmk
    '''
232 1602 aaronmk
    try: new_value = repls[value]
233 1304 aaronmk
    except KeyError, e:
234 1537 aaronmk
        # Save traceback right away in case another exception raised
235 2984 aaronmk
        fe = FormatException(e)
236 1602 aaronmk
        try: new_value = repls['*']
237 1609 aaronmk
        except KeyError: raise fe
238 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
239 1607 aaronmk
    return new_value
240 1602 aaronmk
241 2016 aaronmk
def _map(items, node):
242 1602 aaronmk
    '''See repl()
243
    @param items
244
        <last_entry> Value
245
        <other_entries> name=value Mappings. Special values: See repl() repls.
246
    '''
247
    value = pop_value(items)
248
    if value == None: return None # input is empty
249 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
250 1219 aaronmk
funcs['_map'] = _map
251
252 2016 aaronmk
def _replace(items, node):
253 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
254 1581 aaronmk
    value = pop_value(items)
255
    if value == None: return None # input is empty
256 1219 aaronmk
    try:
257
        for repl, with_ in items:
258
            if re.match(r'^\w+$', repl):
259
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
260
            value = re.sub(repl, with_, value)
261 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
262 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
263 1219 aaronmk
funcs['_replace'] = _replace
264
265 1469 aaronmk
#### Quantities
266
267 2016 aaronmk
def _units(items, node):
268 1581 aaronmk
    value = pop_value(items)
269
    if value == None: return None # input is empty
270 1471 aaronmk
271 1581 aaronmk
    quantity = units.str2quantity(value)
272 1471 aaronmk
    try:
273
        for action, units_ in items:
274
            units_ = util.none_if(units_, u'')
275
            if action == 'default': units.set_default_units(quantity, units_)
276 1567 aaronmk
            elif action == 'to':
277
                try: quantity = units.convert(quantity, units_)
278 1609 aaronmk
                except ValueError, e: raise FormatException(e)
279 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
280 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
281 1471 aaronmk
    return units.quantity2str(quantity)
282 1225 aaronmk
funcs['_units'] = _units
283
284 1399 aaronmk
def parse_range(str_, range_sep='-'):
285
    default = (str_, None)
286
    start, sep, end = str_.partition(range_sep)
287
    if sep == '': return default # not a range
288 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
289 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
290
291 2016 aaronmk
def _rangeStart(items, node):
292 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
293 1399 aaronmk
    try: value = items['value']
294 1406 aaronmk
    except KeyError: return None # input is empty
295 1399 aaronmk
    return parse_range(value)[0]
296
funcs['_rangeStart'] = _rangeStart
297
298 2016 aaronmk
def _rangeEnd(items, node):
299 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
300 1399 aaronmk
    try: value = items['value']
301 1406 aaronmk
    except KeyError: return None # input is empty
302 1399 aaronmk
    return parse_range(value)[1]
303
funcs['_rangeEnd'] = _rangeEnd
304
305 2016 aaronmk
def _range(items, node):
306 1472 aaronmk
    items = dict(conv_items(float, items))
307
    from_ = items.get('from', None)
308
    to = items.get('to', None)
309
    if from_ == None or to == None: return None
310
    return str(to - from_)
311
funcs['_range'] = _range
312
313 2016 aaronmk
def _avg(items, node):
314 86 aaronmk
    count = 0
315
    sum_ = 0.
316 278 aaronmk
    for name, value in conv_items(float, items):
317 86 aaronmk
        count += 1
318
        sum_ += value
319 1472 aaronmk
    if count == 0: return None # input is empty
320
    else: return str(sum_/count)
321 995 aaronmk
funcs['_avg'] = _avg
322 86 aaronmk
323 968 aaronmk
class CvException(Exception):
324
    def __init__(self):
325
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
326
            ' allowed for ratio scale data '
327
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
328
329 2016 aaronmk
def _noCV(items, node):
330 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
331 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
332
    except IndexError: return None # input is empty
333 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
334 968 aaronmk
    return value
335 995 aaronmk
funcs['_noCV'] = _noCV
336 968 aaronmk
337 1469 aaronmk
#### Names
338
339 328 aaronmk
_name_parts_slices_items = [
340
    ('first', slice(None, 1)),
341
    ('middle', slice(1, -1)),
342
    ('last', slice(-1, None)),
343
]
344
name_parts_slices = dict(_name_parts_slices_items)
345
name_parts = [name for name, slice_ in _name_parts_slices_items]
346
347 2016 aaronmk
def _name(items, node):
348 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
349 102 aaronmk
    parts = []
350 328 aaronmk
    for part in name_parts:
351
        if part in items: parts.append(items[part])
352 4035 aaronmk
    if not parts: return None # pass None values through; handle no name parts
353 102 aaronmk
    return ' '.join(parts)
354 995 aaronmk
funcs['_name'] = _name
355 102 aaronmk
356 1607 aaronmk
#### Angles
357
358 2016 aaronmk
def _compass(items, node):
359 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
360
    items = dict(conv_items(strings.ustr, items))
361
    try: value = items['value']
362
    except KeyError: return None # input is empty
363
364
    if not value.isupper(): return value # pass through other coordinate formats
365
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
366
    except KeyError, e: raise FormatException(e)
367
funcs['_compass'] = _compass