Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 1992 aaronmk
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53
54 3335 aaronmk
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63
64 995 aaronmk
funcs = {}
65 4236 aaronmk
simplifying_funcs = {}
66 995 aaronmk
67 1992 aaronmk
##### Public functions
68
69 4239 aaronmk
var_name_prefix = '$'
70
71
def is_var_name(str_): return str_.startswith(var_name_prefix)
72
73
def is_var(node):
74
    return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
75
76 2112 aaronmk
def is_func_name(name):
77
    return name.startswith('_') and name != '_' # '_' is default root node name
78
79
def is_func(node): return is_func_name(node.tagName)
80
81
def is_xml_func_name(name): return is_func_name(name) and name in funcs
82
83
def is_xml_func(node): return is_xml_func_name(node.tagName)
84
85 4298 aaronmk
def prune(node):
86
    '''Removes empty children'''
87
    for child in xml_dom.NodeElemIter(node):
88
        if xml_dom.is_empty(child): xml_dom.remove(child)
89
90 4300 aaronmk
def passthru(node):
91 4302 aaronmk
    '''Passes through single child node. First prunes the node.'''
92
    prune(node)
93 4300 aaronmk
    children = list(xml_dom.NodeEntryIter(node))
94
    if len(children) == 1: xml_dom.replace(node, children[0][1])
95
96 4041 aaronmk
def simplify(node):
97
    '''Simplifies the XML functions in an XML tree.
98
    * Merges nodes tagged as mergable
99 4236 aaronmk
    * Runs simplifying functions
100 4078 aaronmk
    * Applies pass-through optimizations for XML functions
101 4041 aaronmk
    '''
102
    for child in xml_dom.NodeElemIter(node): simplify(child)
103
    merge_tagged(node)
104
105 4227 aaronmk
    name = node.tagName
106 4041 aaronmk
107 4078 aaronmk
    # Pass-through optimizations
108 4228 aaronmk
    if is_func_name(name):
109 4236 aaronmk
        try: func = simplifying_funcs[name]
110
        except KeyError:
111
            children = list(xml_dom.NodeElemIter(node))
112
            if len(children) == 1: # single arg function
113
                func = name
114
                param_node = children[0]
115
                param = param_node.tagName
116
                value = param_node.firstChild
117
118
                is_agg_func = param.isdigit() or func == '_name'
119 4301 aaronmk
                if is_agg_func: passthru(node)
120 4236 aaronmk
        else: func(node)
121 4229 aaronmk
    # Pruning optimizations
122
    else: # these should not run on functions because they would remove args
123 4299 aaronmk
        prune(node)
124 4041 aaronmk
125 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
126 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
127 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
128 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
129
          functions are treated specially, depending on the db param (below).
130
    @param db
131
        * If None: Non-relational functions other than structural functions are
132
          replaced with their last parameter (usually the value), not evaluated.
133
          This is used in column-based mode to remove XML-only functions.
134
        * If != None: Relational functions are evaluated directly. This is used
135
          in row-based mode to combine relational and XML functions.
136 2597 aaronmk
    '''
137 3424 aaronmk
    has_rel_funcs = is_rel_func != None
138 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
139 2597 aaronmk
140 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
141 3424 aaronmk
        process(child, on_error, is_rel_func, db)
142 3335 aaronmk
    merge_tagged(node)
143 3333 aaronmk
144 995 aaronmk
    name = node.tagName
145 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
146 2602 aaronmk
147
    row_mode = has_rel_funcs and db != None
148
    column_mode = has_rel_funcs and db == None
149 3629 aaronmk
    func = funcs.get(name, None)
150 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
151 2602 aaronmk
152 3029 aaronmk
    # Parse function
153
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
154
        # pass-through optimization for aggregating functions with one arg
155
        value = items[0][1] # pass through first arg
156 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
157 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
158
        except sql.DoesNotExistException: return # preserve unknown funcs
159
            # possibly a built-in function of db_xml.put()
160 4024 aaronmk
    elif column_mode or func == None:
161 3640 aaronmk
        # local XML function can't be used or does not exist
162
        if column_mode and is_rel_func(name): return # preserve relational funcs
163
        # otherwise XML-only in column mode, or DB-only in XML output mode
164
        value = pop_value(items, None) # just replace with last param
165 2602 aaronmk
    else: # local XML function
166 3629 aaronmk
        try: value = func(items, node)
167 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
168 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
169
            exc.add_traceback(e)
170 1562 aaronmk
            str_ = strings.ustr(node)
171 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
172 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
173
                '\n'+term.emph_multiline(str_)))
174
175 995 aaronmk
            on_error(e)
176 2602 aaronmk
            return # in case on_error() returns
177 3227 aaronmk
178 3333 aaronmk
    xml_dom.replace_with_text(node, value)
179 995 aaronmk
180 4236 aaronmk
##### Simplifying functions
181
182
# Function names must start with _ to avoid collisions with real tags
183
# Functions take params (node) and have no return value
184
185 4237 aaronmk
#### Logic
186
187
def _exists(node):
188
    '''Returns whether its node is non-empty'''
189
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
190
simplifying_funcs['_exists'] = _exists
191
192 4240 aaronmk
def _if(node):
193
    '''Can add `@name` attr to distinguish separate _if statements'''
194
    params = dict(xml_dom.NodeEntryIter(node))
195
    then = params.get('then', None)
196
    cond = params.get('cond', None)
197
    else_ = params.get('else', None)
198
199
    if cond == None: xml_dom.replace(node, else_) # always False
200
    elif then == else_: xml_dom.replace(node, then) # always same value
201
    elif is_var(cond): pass # can't simplify variable conditions
202
    elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
203
        xml_dom.replace(node, then)
204
simplifying_funcs['_if'] = _if
205
206 4303 aaronmk
#### Merging
207
208
simplifying_funcs['_alt'] = passthru
209
simplifying_funcs['_merge'] = passthru
210 4304 aaronmk
simplifying_funcs['_name'] = passthru
211 4303 aaronmk
212 1469 aaronmk
##### XML functions
213 995 aaronmk
214
# Function names must start with _ to avoid collisions with real tags
215 4144 aaronmk
# Functions take arguments (items, node)
216 995 aaronmk
217 1469 aaronmk
#### Transforming values
218
219 1602 aaronmk
def repl(repls, value):
220 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
221 1602 aaronmk
    @param repls dict repl:with
222
        repl "*" means all other input values
223
        with "*" means keep input value the same
224
        with "" means ignore input value
225 1537 aaronmk
    '''
226 1602 aaronmk
    try: new_value = repls[value]
227 1304 aaronmk
    except KeyError, e:
228 1537 aaronmk
        # Save traceback right away in case another exception raised
229 2984 aaronmk
        fe = FormatException(e)
230 1602 aaronmk
        try: new_value = repls['*']
231 1609 aaronmk
        except KeyError: raise fe
232 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
233 1607 aaronmk
    return new_value
234 1602 aaronmk
235 2016 aaronmk
def _map(items, node):
236 1602 aaronmk
    '''See repl()
237
    @param items
238
        <last_entry> Value
239
        <other_entries> name=value Mappings. Special values: See repl() repls.
240
    '''
241
    value = pop_value(items)
242
    if value == None: return None # input is empty
243 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
244 1219 aaronmk
funcs['_map'] = _map
245
246 2016 aaronmk
def _replace(items, node):
247 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
248 1581 aaronmk
    value = pop_value(items)
249
    if value == None: return None # input is empty
250 1219 aaronmk
    try:
251
        for repl, with_ in items:
252
            if re.match(r'^\w+$', repl):
253
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
254
            value = re.sub(repl, with_, value)
255 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
256 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
257 1219 aaronmk
funcs['_replace'] = _replace
258
259 1469 aaronmk
#### Quantities
260
261 2016 aaronmk
def _units(items, node):
262 1581 aaronmk
    value = pop_value(items)
263
    if value == None: return None # input is empty
264 1471 aaronmk
265 1581 aaronmk
    quantity = units.str2quantity(value)
266 1471 aaronmk
    try:
267
        for action, units_ in items:
268
            units_ = util.none_if(units_, u'')
269
            if action == 'default': units.set_default_units(quantity, units_)
270 1567 aaronmk
            elif action == 'to':
271
                try: quantity = units.convert(quantity, units_)
272 1609 aaronmk
                except ValueError, e: raise FormatException(e)
273 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
274 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
275 1471 aaronmk
    return units.quantity2str(quantity)
276 1225 aaronmk
funcs['_units'] = _units
277
278 1399 aaronmk
def parse_range(str_, range_sep='-'):
279
    default = (str_, None)
280
    start, sep, end = str_.partition(range_sep)
281
    if sep == '': return default # not a range
282 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
283 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
284
285 2016 aaronmk
def _rangeStart(items, node):
286 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
287 1399 aaronmk
    try: value = items['value']
288 1406 aaronmk
    except KeyError: return None # input is empty
289 1399 aaronmk
    return parse_range(value)[0]
290
funcs['_rangeStart'] = _rangeStart
291
292 2016 aaronmk
def _rangeEnd(items, node):
293 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
294 1399 aaronmk
    try: value = items['value']
295 1406 aaronmk
    except KeyError: return None # input is empty
296 1399 aaronmk
    return parse_range(value)[1]
297
funcs['_rangeEnd'] = _rangeEnd
298
299 2016 aaronmk
def _range(items, node):
300 1472 aaronmk
    items = dict(conv_items(float, items))
301
    from_ = items.get('from', None)
302
    to = items.get('to', None)
303
    if from_ == None or to == None: return None
304
    return str(to - from_)
305
funcs['_range'] = _range
306
307 2016 aaronmk
def _avg(items, node):
308 86 aaronmk
    count = 0
309
    sum_ = 0.
310 278 aaronmk
    for name, value in conv_items(float, items):
311 86 aaronmk
        count += 1
312
        sum_ += value
313 1472 aaronmk
    if count == 0: return None # input is empty
314
    else: return str(sum_/count)
315 995 aaronmk
funcs['_avg'] = _avg
316 86 aaronmk
317 968 aaronmk
class CvException(Exception):
318
    def __init__(self):
319
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
320
            ' allowed for ratio scale data '
321
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
322
323 2016 aaronmk
def _noCV(items, node):
324 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
325 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
326
    except IndexError: return None # input is empty
327 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
328 968 aaronmk
    return value
329 995 aaronmk
funcs['_noCV'] = _noCV
330 968 aaronmk
331 1469 aaronmk
#### Names
332
333 328 aaronmk
_name_parts_slices_items = [
334
    ('first', slice(None, 1)),
335
    ('middle', slice(1, -1)),
336
    ('last', slice(-1, None)),
337
]
338
name_parts_slices = dict(_name_parts_slices_items)
339
name_parts = [name for name, slice_ in _name_parts_slices_items]
340
341 2016 aaronmk
def _name(items, node):
342 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
343 102 aaronmk
    parts = []
344 328 aaronmk
    for part in name_parts:
345
        if part in items: parts.append(items[part])
346 4035 aaronmk
    if not parts: return None # pass None values through; handle no name parts
347 102 aaronmk
    return ' '.join(parts)
348 995 aaronmk
funcs['_name'] = _name
349 102 aaronmk
350 1607 aaronmk
#### Angles
351
352 2016 aaronmk
def _compass(items, node):
353 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
354
    items = dict(conv_items(strings.ustr, items))
355
    try: value = items['value']
356
    except KeyError: return None # input is empty
357
358
    if not value.isupper(): return value # pass through other coordinate formats
359
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
360
    except KeyError, e: raise FormatException(e)
361
funcs['_compass'] = _compass