Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 1992 aaronmk
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53
54 3335 aaronmk
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63
64 995 aaronmk
funcs = {}
65 4236 aaronmk
simplifying_funcs = {}
66 995 aaronmk
67 1992 aaronmk
##### Public functions
68
69 4239 aaronmk
var_name_prefix = '$'
70
71
def is_var_name(str_): return str_.startswith(var_name_prefix)
72
73
def is_var(node):
74
    return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
75
76 2112 aaronmk
def is_func_name(name):
77
    return name.startswith('_') and name != '_' # '_' is default root node name
78
79
def is_func(node): return is_func_name(node.tagName)
80
81
def is_xml_func_name(name): return is_func_name(name) and name in funcs
82
83
def is_xml_func(node): return is_xml_func_name(node.tagName)
84
85 4298 aaronmk
def prune(node):
86
    '''Removes empty children'''
87
    for child in xml_dom.NodeElemIter(node):
88
        if xml_dom.is_empty(child): xml_dom.remove(child)
89
90 4300 aaronmk
def passthru(node):
91 4302 aaronmk
    '''Passes through single child node. First prunes the node.'''
92
    prune(node)
93 4300 aaronmk
    children = list(xml_dom.NodeEntryIter(node))
94
    if len(children) == 1: xml_dom.replace(node, children[0][1])
95
96 4041 aaronmk
def simplify(node):
97 4305 aaronmk
    '''Simplifies an XML tree.
98 4041 aaronmk
    * Merges nodes tagged as mergable
99 4236 aaronmk
    * Runs simplifying functions
100 4041 aaronmk
    '''
101
    for child in xml_dom.NodeElemIter(node): simplify(child)
102
    merge_tagged(node)
103
104 4227 aaronmk
    name = node.tagName
105 4041 aaronmk
106 4078 aaronmk
    # Pass-through optimizations
107 4228 aaronmk
    if is_func_name(name):
108 4236 aaronmk
        try: func = simplifying_funcs[name]
109 4305 aaronmk
        except KeyError: pass
110 4236 aaronmk
        else: func(node)
111 4229 aaronmk
    # Pruning optimizations
112
    else: # these should not run on functions because they would remove args
113 4299 aaronmk
        prune(node)
114 4041 aaronmk
115 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
116 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
117 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
118 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
119
          functions are treated specially, depending on the db param (below).
120
    @param db
121
        * If None: Non-relational functions other than structural functions are
122
          replaced with their last parameter (usually the value), not evaluated.
123
          This is used in column-based mode to remove XML-only functions.
124
        * If != None: Relational functions are evaluated directly. This is used
125
          in row-based mode to combine relational and XML functions.
126 2597 aaronmk
    '''
127 3424 aaronmk
    has_rel_funcs = is_rel_func != None
128 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
129 2597 aaronmk
130 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
131 3424 aaronmk
        process(child, on_error, is_rel_func, db)
132 3335 aaronmk
    merge_tagged(node)
133 3333 aaronmk
134 995 aaronmk
    name = node.tagName
135 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
136 2602 aaronmk
137
    row_mode = has_rel_funcs and db != None
138
    column_mode = has_rel_funcs and db == None
139 3629 aaronmk
    func = funcs.get(name, None)
140 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
141 2602 aaronmk
142 3029 aaronmk
    # Parse function
143
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
144
        # pass-through optimization for aggregating functions with one arg
145
        value = items[0][1] # pass through first arg
146 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
147 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
148
        except sql.DoesNotExistException: return # preserve unknown funcs
149
            # possibly a built-in function of db_xml.put()
150 4024 aaronmk
    elif column_mode or func == None:
151 3640 aaronmk
        # local XML function can't be used or does not exist
152
        if column_mode and is_rel_func(name): return # preserve relational funcs
153
        # otherwise XML-only in column mode, or DB-only in XML output mode
154
        value = pop_value(items, None) # just replace with last param
155 2602 aaronmk
    else: # local XML function
156 3629 aaronmk
        try: value = func(items, node)
157 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
158 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
159
            exc.add_traceback(e)
160 1562 aaronmk
            str_ = strings.ustr(node)
161 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
162 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
163
                '\n'+term.emph_multiline(str_)))
164
165 995 aaronmk
            on_error(e)
166 2602 aaronmk
            return # in case on_error() returns
167 3227 aaronmk
168 3333 aaronmk
    xml_dom.replace_with_text(node, value)
169 995 aaronmk
170 4236 aaronmk
##### Simplifying functions
171
172
# Function names must start with _ to avoid collisions with real tags
173
# Functions take params (node) and have no return value
174
175 4237 aaronmk
#### Logic
176
177
def _exists(node):
178
    '''Returns whether its node is non-empty'''
179
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
180
simplifying_funcs['_exists'] = _exists
181
182 4240 aaronmk
def _if(node):
183
    '''Can add `@name` attr to distinguish separate _if statements'''
184
    params = dict(xml_dom.NodeEntryIter(node))
185
    then = params.get('then', None)
186
    cond = params.get('cond', None)
187
    else_ = params.get('else', None)
188
189
    if cond == None: xml_dom.replace(node, else_) # always False
190
    elif then == else_: xml_dom.replace(node, then) # always same value
191
    elif is_var(cond): pass # can't simplify variable conditions
192
    elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
193
        xml_dom.replace(node, then)
194
simplifying_funcs['_if'] = _if
195
196 4303 aaronmk
#### Merging
197
198
simplifying_funcs['_alt'] = passthru
199
simplifying_funcs['_merge'] = passthru
200 4304 aaronmk
simplifying_funcs['_name'] = passthru
201 4303 aaronmk
202 1469 aaronmk
##### XML functions
203 995 aaronmk
204
# Function names must start with _ to avoid collisions with real tags
205 4144 aaronmk
# Functions take arguments (items, node)
206 995 aaronmk
207 1469 aaronmk
#### Transforming values
208
209 1602 aaronmk
def repl(repls, value):
210 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
211 1602 aaronmk
    @param repls dict repl:with
212
        repl "*" means all other input values
213
        with "*" means keep input value the same
214
        with "" means ignore input value
215 1537 aaronmk
    '''
216 1602 aaronmk
    try: new_value = repls[value]
217 1304 aaronmk
    except KeyError, e:
218 1537 aaronmk
        # Save traceback right away in case another exception raised
219 2984 aaronmk
        fe = FormatException(e)
220 1602 aaronmk
        try: new_value = repls['*']
221 1609 aaronmk
        except KeyError: raise fe
222 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
223 1607 aaronmk
    return new_value
224 1602 aaronmk
225 2016 aaronmk
def _map(items, node):
226 1602 aaronmk
    '''See repl()
227
    @param items
228
        <last_entry> Value
229
        <other_entries> name=value Mappings. Special values: See repl() repls.
230
    '''
231
    value = pop_value(items)
232
    if value == None: return None # input is empty
233 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
234 1219 aaronmk
funcs['_map'] = _map
235
236 2016 aaronmk
def _replace(items, node):
237 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
238 1581 aaronmk
    value = pop_value(items)
239
    if value == None: return None # input is empty
240 1219 aaronmk
    try:
241
        for repl, with_ in items:
242
            if re.match(r'^\w+$', repl):
243
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
244
            value = re.sub(repl, with_, value)
245 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
246 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
247 1219 aaronmk
funcs['_replace'] = _replace
248
249 1469 aaronmk
#### Quantities
250
251 2016 aaronmk
def _units(items, node):
252 1581 aaronmk
    value = pop_value(items)
253
    if value == None: return None # input is empty
254 1471 aaronmk
255 1581 aaronmk
    quantity = units.str2quantity(value)
256 1471 aaronmk
    try:
257
        for action, units_ in items:
258
            units_ = util.none_if(units_, u'')
259
            if action == 'default': units.set_default_units(quantity, units_)
260 1567 aaronmk
            elif action == 'to':
261
                try: quantity = units.convert(quantity, units_)
262 1609 aaronmk
                except ValueError, e: raise FormatException(e)
263 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
264 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
265 1471 aaronmk
    return units.quantity2str(quantity)
266 1225 aaronmk
funcs['_units'] = _units
267
268 1399 aaronmk
def parse_range(str_, range_sep='-'):
269
    default = (str_, None)
270
    start, sep, end = str_.partition(range_sep)
271
    if sep == '': return default # not a range
272 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
273 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
274
275 2016 aaronmk
def _rangeStart(items, node):
276 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
277 1399 aaronmk
    try: value = items['value']
278 1406 aaronmk
    except KeyError: return None # input is empty
279 1399 aaronmk
    return parse_range(value)[0]
280
funcs['_rangeStart'] = _rangeStart
281
282 2016 aaronmk
def _rangeEnd(items, node):
283 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
284 1399 aaronmk
    try: value = items['value']
285 1406 aaronmk
    except KeyError: return None # input is empty
286 1399 aaronmk
    return parse_range(value)[1]
287
funcs['_rangeEnd'] = _rangeEnd
288
289 2016 aaronmk
def _range(items, node):
290 1472 aaronmk
    items = dict(conv_items(float, items))
291
    from_ = items.get('from', None)
292
    to = items.get('to', None)
293
    if from_ == None or to == None: return None
294
    return str(to - from_)
295
funcs['_range'] = _range
296
297 2016 aaronmk
def _avg(items, node):
298 86 aaronmk
    count = 0
299
    sum_ = 0.
300 278 aaronmk
    for name, value in conv_items(float, items):
301 86 aaronmk
        count += 1
302
        sum_ += value
303 1472 aaronmk
    if count == 0: return None # input is empty
304
    else: return str(sum_/count)
305 995 aaronmk
funcs['_avg'] = _avg
306 86 aaronmk
307 968 aaronmk
class CvException(Exception):
308
    def __init__(self):
309
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
310
            ' allowed for ratio scale data '
311
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
312
313 2016 aaronmk
def _noCV(items, node):
314 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
315 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
316
    except IndexError: return None # input is empty
317 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
318 968 aaronmk
    return value
319 995 aaronmk
funcs['_noCV'] = _noCV
320 968 aaronmk
321 1469 aaronmk
#### Names
322
323 328 aaronmk
_name_parts_slices_items = [
324
    ('first', slice(None, 1)),
325
    ('middle', slice(1, -1)),
326
    ('last', slice(-1, None)),
327
]
328
name_parts_slices = dict(_name_parts_slices_items)
329
name_parts = [name for name, slice_ in _name_parts_slices_items]
330
331 2016 aaronmk
def _name(items, node):
332 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
333 102 aaronmk
    parts = []
334 328 aaronmk
    for part in name_parts:
335
        if part in items: parts.append(items[part])
336 4035 aaronmk
    if not parts: return None # pass None values through; handle no name parts
337 102 aaronmk
    return ' '.join(parts)
338 995 aaronmk
funcs['_name'] = _name
339 102 aaronmk
340 1607 aaronmk
#### Angles
341
342 2016 aaronmk
def _compass(items, node):
343 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
344
    items = dict(conv_items(strings.ustr, items))
345
    try: value = items['value']
346
    except KeyError: return None # input is empty
347
348
    if not value.isupper(): return value # pass through other coordinate formats
349
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
350
    except KeyError, e: raise FormatException(e)
351
funcs['_compass'] = _compass