Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 1992 aaronmk
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53
54 3335 aaronmk
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63
64 995 aaronmk
funcs = {}
65 4236 aaronmk
simplifying_funcs = {}
66 995 aaronmk
67 1992 aaronmk
##### Public functions
68
69 4239 aaronmk
var_name_prefix = '$'
70
71
def is_var_name(str_): return str_.startswith(var_name_prefix)
72
73
def is_var(node):
74
    return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
75
76 2112 aaronmk
def is_func_name(name):
77
    return name.startswith('_') and name != '_' # '_' is default root node name
78
79
def is_func(node): return is_func_name(node.tagName)
80
81
def is_xml_func_name(name): return is_func_name(name) and name in funcs
82
83
def is_xml_func(node): return is_xml_func_name(node.tagName)
84
85 4298 aaronmk
def prune(node):
86
    '''Removes empty children'''
87
    for child in xml_dom.NodeElemIter(node):
88
        if xml_dom.is_empty(child): xml_dom.remove(child)
89
90 4300 aaronmk
def passthru(node):
91
    '''Passes through single child node'''
92
    children = list(xml_dom.NodeEntryIter(node))
93
    if len(children) == 1: xml_dom.replace(node, children[0][1])
94
95 4041 aaronmk
def simplify(node):
96
    '''Simplifies the XML functions in an XML tree.
97
    * Merges nodes tagged as mergable
98 4236 aaronmk
    * Runs simplifying functions
99 4078 aaronmk
    * Applies pass-through optimizations for XML functions
100 4041 aaronmk
    '''
101
    for child in xml_dom.NodeElemIter(node): simplify(child)
102
    merge_tagged(node)
103
104 4227 aaronmk
    name = node.tagName
105 4041 aaronmk
106 4078 aaronmk
    # Pass-through optimizations
107 4228 aaronmk
    if is_func_name(name):
108 4236 aaronmk
        try: func = simplifying_funcs[name]
109
        except KeyError:
110
            children = list(xml_dom.NodeElemIter(node))
111
            if len(children) == 1: # single arg function
112
                func = name
113
                param_node = children[0]
114
                param = param_node.tagName
115
                value = param_node.firstChild
116
117
                is_agg_func = param.isdigit() or func == '_name'
118 4243 aaronmk
                if is_agg_func: xml_dom.replace(node, value)
119 4236 aaronmk
        else: func(node)
120 4229 aaronmk
    # Pruning optimizations
121
    else: # these should not run on functions because they would remove args
122 4299 aaronmk
        prune(node)
123 4041 aaronmk
124 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
125 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
126 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
127 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
128
          functions are treated specially, depending on the db param (below).
129
    @param db
130
        * If None: Non-relational functions other than structural functions are
131
          replaced with their last parameter (usually the value), not evaluated.
132
          This is used in column-based mode to remove XML-only functions.
133
        * If != None: Relational functions are evaluated directly. This is used
134
          in row-based mode to combine relational and XML functions.
135 2597 aaronmk
    '''
136 3424 aaronmk
    has_rel_funcs = is_rel_func != None
137 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
138 2597 aaronmk
139 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
140 3424 aaronmk
        process(child, on_error, is_rel_func, db)
141 3335 aaronmk
    merge_tagged(node)
142 3333 aaronmk
143 995 aaronmk
    name = node.tagName
144 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
145 2602 aaronmk
146
    row_mode = has_rel_funcs and db != None
147
    column_mode = has_rel_funcs and db == None
148 3629 aaronmk
    func = funcs.get(name, None)
149 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
150 2602 aaronmk
151 3029 aaronmk
    # Parse function
152
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
153
        # pass-through optimization for aggregating functions with one arg
154
        value = items[0][1] # pass through first arg
155 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
156 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
157
        except sql.DoesNotExistException: return # preserve unknown funcs
158
            # possibly a built-in function of db_xml.put()
159 4024 aaronmk
    elif column_mode or func == None:
160 3640 aaronmk
        # local XML function can't be used or does not exist
161
        if column_mode and is_rel_func(name): return # preserve relational funcs
162
        # otherwise XML-only in column mode, or DB-only in XML output mode
163
        value = pop_value(items, None) # just replace with last param
164 2602 aaronmk
    else: # local XML function
165 3629 aaronmk
        try: value = func(items, node)
166 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
167 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
168
            exc.add_traceback(e)
169 1562 aaronmk
            str_ = strings.ustr(node)
170 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
171 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
172
                '\n'+term.emph_multiline(str_)))
173
174 995 aaronmk
            on_error(e)
175 2602 aaronmk
            return # in case on_error() returns
176 3227 aaronmk
177 3333 aaronmk
    xml_dom.replace_with_text(node, value)
178 995 aaronmk
179 4236 aaronmk
##### Simplifying functions
180
181
# Function names must start with _ to avoid collisions with real tags
182
# Functions take params (node) and have no return value
183
184 4237 aaronmk
#### Logic
185
186
def _exists(node):
187
    '''Returns whether its node is non-empty'''
188
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
189
simplifying_funcs['_exists'] = _exists
190
191 4240 aaronmk
def _if(node):
192
    '''Can add `@name` attr to distinguish separate _if statements'''
193
    params = dict(xml_dom.NodeEntryIter(node))
194
    then = params.get('then', None)
195
    cond = params.get('cond', None)
196
    else_ = params.get('else', None)
197
198
    if cond == None: xml_dom.replace(node, else_) # always False
199
    elif then == else_: xml_dom.replace(node, then) # always same value
200
    elif is_var(cond): pass # can't simplify variable conditions
201
    elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
202
        xml_dom.replace(node, then)
203
simplifying_funcs['_if'] = _if
204
205 1469 aaronmk
##### XML functions
206 995 aaronmk
207
# Function names must start with _ to avoid collisions with real tags
208 4144 aaronmk
# Functions take arguments (items, node)
209 995 aaronmk
210 1469 aaronmk
#### Transforming values
211
212 1602 aaronmk
def repl(repls, value):
213 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
214 1602 aaronmk
    @param repls dict repl:with
215
        repl "*" means all other input values
216
        with "*" means keep input value the same
217
        with "" means ignore input value
218 1537 aaronmk
    '''
219 1602 aaronmk
    try: new_value = repls[value]
220 1304 aaronmk
    except KeyError, e:
221 1537 aaronmk
        # Save traceback right away in case another exception raised
222 2984 aaronmk
        fe = FormatException(e)
223 1602 aaronmk
        try: new_value = repls['*']
224 1609 aaronmk
        except KeyError: raise fe
225 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
226 1607 aaronmk
    return new_value
227 1602 aaronmk
228 2016 aaronmk
def _map(items, node):
229 1602 aaronmk
    '''See repl()
230
    @param items
231
        <last_entry> Value
232
        <other_entries> name=value Mappings. Special values: See repl() repls.
233
    '''
234
    value = pop_value(items)
235
    if value == None: return None # input is empty
236 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
237 1219 aaronmk
funcs['_map'] = _map
238
239 2016 aaronmk
def _replace(items, node):
240 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
241 1581 aaronmk
    value = pop_value(items)
242
    if value == None: return None # input is empty
243 1219 aaronmk
    try:
244
        for repl, with_ in items:
245
            if re.match(r'^\w+$', repl):
246
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
247
            value = re.sub(repl, with_, value)
248 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
249 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
250 1219 aaronmk
funcs['_replace'] = _replace
251
252 1469 aaronmk
#### Quantities
253
254 2016 aaronmk
def _units(items, node):
255 1581 aaronmk
    value = pop_value(items)
256
    if value == None: return None # input is empty
257 1471 aaronmk
258 1581 aaronmk
    quantity = units.str2quantity(value)
259 1471 aaronmk
    try:
260
        for action, units_ in items:
261
            units_ = util.none_if(units_, u'')
262
            if action == 'default': units.set_default_units(quantity, units_)
263 1567 aaronmk
            elif action == 'to':
264
                try: quantity = units.convert(quantity, units_)
265 1609 aaronmk
                except ValueError, e: raise FormatException(e)
266 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
267 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
268 1471 aaronmk
    return units.quantity2str(quantity)
269 1225 aaronmk
funcs['_units'] = _units
270
271 1399 aaronmk
def parse_range(str_, range_sep='-'):
272
    default = (str_, None)
273
    start, sep, end = str_.partition(range_sep)
274
    if sep == '': return default # not a range
275 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
276 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
277
278 2016 aaronmk
def _rangeStart(items, node):
279 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
280 1399 aaronmk
    try: value = items['value']
281 1406 aaronmk
    except KeyError: return None # input is empty
282 1399 aaronmk
    return parse_range(value)[0]
283
funcs['_rangeStart'] = _rangeStart
284
285 2016 aaronmk
def _rangeEnd(items, node):
286 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
287 1399 aaronmk
    try: value = items['value']
288 1406 aaronmk
    except KeyError: return None # input is empty
289 1399 aaronmk
    return parse_range(value)[1]
290
funcs['_rangeEnd'] = _rangeEnd
291
292 2016 aaronmk
def _range(items, node):
293 1472 aaronmk
    items = dict(conv_items(float, items))
294
    from_ = items.get('from', None)
295
    to = items.get('to', None)
296
    if from_ == None or to == None: return None
297
    return str(to - from_)
298
funcs['_range'] = _range
299
300 2016 aaronmk
def _avg(items, node):
301 86 aaronmk
    count = 0
302
    sum_ = 0.
303 278 aaronmk
    for name, value in conv_items(float, items):
304 86 aaronmk
        count += 1
305
        sum_ += value
306 1472 aaronmk
    if count == 0: return None # input is empty
307
    else: return str(sum_/count)
308 995 aaronmk
funcs['_avg'] = _avg
309 86 aaronmk
310 968 aaronmk
class CvException(Exception):
311
    def __init__(self):
312
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
313
            ' allowed for ratio scale data '
314
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
315
316 2016 aaronmk
def _noCV(items, node):
317 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
318 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
319
    except IndexError: return None # input is empty
320 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
321 968 aaronmk
    return value
322 995 aaronmk
funcs['_noCV'] = _noCV
323 968 aaronmk
324 1469 aaronmk
#### Names
325
326 328 aaronmk
_name_parts_slices_items = [
327
    ('first', slice(None, 1)),
328
    ('middle', slice(1, -1)),
329
    ('last', slice(-1, None)),
330
]
331
name_parts_slices = dict(_name_parts_slices_items)
332
name_parts = [name for name, slice_ in _name_parts_slices_items]
333
334 2016 aaronmk
def _name(items, node):
335 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
336 102 aaronmk
    parts = []
337 328 aaronmk
    for part in name_parts:
338
        if part in items: parts.append(items[part])
339 4035 aaronmk
    if not parts: return None # pass None values through; handle no name parts
340 102 aaronmk
    return ' '.join(parts)
341 995 aaronmk
funcs['_name'] = _name
342 102 aaronmk
343 1607 aaronmk
#### Angles
344
345 2016 aaronmk
def _compass(items, node):
346 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
347
    items = dict(conv_items(strings.ustr, items))
348
    try: value = items['value']
349
    except KeyError: return None # input is empty
350
351
    if not value.isupper(): return value # pass through other coordinate formats
352
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
353
    except KeyError, e: raise FormatException(e)
354
funcs['_compass'] = _compass