Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3077 aaronmk
import sql_io
14 1234 aaronmk
import strings
15 827 aaronmk
import term
16 1468 aaronmk
import units
17 1047 aaronmk
import util
18 86 aaronmk
import xml_dom
19 1321 aaronmk
import xpath
20 86 aaronmk
21 995 aaronmk
##### Exceptions
22
23 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
24 797 aaronmk
    def __init__(self, cause):
25 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
26
            cause)
27 278 aaronmk
28 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
29
    def __init__(self, cause):
30
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
31 843 aaronmk
32 1992 aaronmk
##### Helper functions
33 995 aaronmk
34 1992 aaronmk
def map_items(func, items):
35
    return [(name, func(value)) for name, value in items]
36
37
def cast(type_, val):
38
    '''Throws FormatException if can't cast'''
39
    try: return type_(val)
40
    except ValueError, e: raise FormatException(e)
41
42
def conv_items(type_, items):
43
    return map_items(lambda val: cast(type_, val),
44
        xml_dom.TextEntryOnlyIter(items))
45
46
def pop_value(items, name='value'):
47
    '''@param name Name of value param, or None to accept any name'''
48
    try: last = items.pop() # last entry contains value
49
    except IndexError: return None # input is empty and no actions
50
    if name != None and last[0] != name: return None # input is empty
51
    return last[1]
52
53 3335 aaronmk
def merge_tagged(root):
54
    '''Merges siblings in root that are marked as mergeable.
55
    Used to recombine pieces of nodes that were split apart in the mappings.
56
    '''
57
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
58
        xml_dom.merge_by_name(root, name)
59
60
    # Recurse
61
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
62
63 995 aaronmk
funcs = {}
64
65 2557 aaronmk
structural_funcs = set()
66
67 1992 aaronmk
##### Public functions
68
69 2112 aaronmk
def is_func_name(name):
70
    return name.startswith('_') and name != '_' # '_' is default root node name
71
72
def is_func(node): return is_func_name(node.tagName)
73
74
def is_xml_func_name(name): return is_func_name(name) and name in funcs
75
76
def is_xml_func(node): return is_xml_func_name(node.tagName)
77
78 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
79 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
80 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
81 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
82
          functions are treated specially, depending on the db param (below).
83
    @param db
84
        * If None: Non-relational functions other than structural functions are
85
          replaced with their last parameter (usually the value), not evaluated.
86
          This is used in column-based mode to remove XML-only functions.
87
        * If != None: Relational functions are evaluated directly. This is used
88
          in row-based mode to combine relational and XML functions.
89 2597 aaronmk
    '''
90 3424 aaronmk
    has_rel_funcs = is_rel_func != None
91 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
92 2597 aaronmk
93 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
94 3424 aaronmk
        process(child, on_error, is_rel_func, db)
95 3335 aaronmk
    merge_tagged(node)
96 3333 aaronmk
97 995 aaronmk
    name = node.tagName
98 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
99 2602 aaronmk
100
    row_mode = has_rel_funcs and db != None
101
    column_mode = has_rel_funcs and db == None
102 3629 aaronmk
    func = funcs.get(name, None)
103 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
104 2602 aaronmk
105 3029 aaronmk
    # Parse function
106
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
107
        # pass-through optimization for aggregating functions with one arg
108
        value = items[0][1] # pass through first arg
109 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
110
        value = sql_io.put(db, name, dict(items)) # evaluate using DB
111 3640 aaronmk
    elif (column_mode and not name in structural_funcs) or func == None:
112
        # local XML function can't be used or does not exist
113
        if column_mode and is_rel_func(name): return # preserve relational funcs
114
        # otherwise XML-only in column mode, or DB-only in XML output mode
115
        value = pop_value(items, None) # just replace with last param
116 2602 aaronmk
    else: # local XML function
117 3629 aaronmk
        try: value = func(items, node)
118 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
119 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
120
            exc.add_traceback(e)
121 1562 aaronmk
            str_ = strings.ustr(node)
122 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
123 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
124
                '\n'+term.emph_multiline(str_)))
125
126 995 aaronmk
            on_error(e)
127 2602 aaronmk
            return # in case on_error() returns
128 3227 aaronmk
129 3333 aaronmk
    xml_dom.replace_with_text(node, value)
130 995 aaronmk
131 1469 aaronmk
##### XML functions
132 995 aaronmk
133
# Function names must start with _ to avoid collisions with real tags
134
# Functions take arguments (items)
135
136 2557 aaronmk
#### Structural
137 1469 aaronmk
138 2017 aaronmk
def _ignore(items, node):
139 994 aaronmk
    '''Used to "comment out" an XML subtree'''
140
    return None
141 995 aaronmk
funcs['_ignore'] = _ignore
142 2557 aaronmk
structural_funcs.add('_ignore')
143 994 aaronmk
144 2017 aaronmk
def _ref(items, node):
145
    '''Used to retrieve a value from another XML node
146
    @param items
147
        addr=<path> XPath to value, relative to the XML func's parent node
148
    '''
149
    items = dict(items)
150
    try: addr = items['addr']
151
    except KeyError, e: raise SyntaxError(e)
152
153
    value = xpath.get_value(node.parentNode, addr)
154
    if value == None:
155
        warnings.warn(UserWarning('_ref: XPath reference target missing: '
156
            +str(addr)))
157
    return value
158
funcs['_ref'] = _ref
159 2557 aaronmk
structural_funcs.add('_ref')
160 2017 aaronmk
161 1469 aaronmk
#### Conditionals
162
163 2016 aaronmk
def _eq(items, node):
164 1234 aaronmk
    items = dict(items)
165
    try:
166
        left = items['left']
167
        right = items['right']
168
    except KeyError: return '' # a value was None
169
    return util.bool2str(left == right)
170
funcs['_eq'] = _eq
171
172 2016 aaronmk
def _if(items, node):
173 1234 aaronmk
    items = dict(items)
174
    try:
175
        cond = items['cond']
176
        then = items['then']
177 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
178 1234 aaronmk
    else_ = items.get('else', None)
179 1562 aaronmk
    cond = bool(cast(strings.ustr, cond))
180 1234 aaronmk
    if cond: return then
181
    else: return else_
182
funcs['_if'] = _if
183
184 1469 aaronmk
#### Transforming values
185
186 2016 aaronmk
def _collapse(items, node):
187 2012 aaronmk
    '''Collapses a subtree if the "value" element in it is NULL'''
188
    items = dict(items)
189
    try: require = cast(strings.ustr, items['require'])
190
    except KeyError, e: raise SyntaxError(e)
191
    value = items.get('value', None)
192
193 2558 aaronmk
    if xpath.get_value(value, require, allow_rooted=False) == None: return None
194 2012 aaronmk
    else: return value
195
funcs['_collapse'] = _collapse
196
197 1602 aaronmk
def repl(repls, value):
198 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
199 1602 aaronmk
    @param repls dict repl:with
200
        repl "*" means all other input values
201
        with "*" means keep input value the same
202
        with "" means ignore input value
203 1537 aaronmk
    '''
204 1602 aaronmk
    try: new_value = repls[value]
205 1304 aaronmk
    except KeyError, e:
206 1537 aaronmk
        # Save traceback right away in case another exception raised
207 2984 aaronmk
        fe = FormatException(e)
208 1602 aaronmk
        try: new_value = repls['*']
209 1609 aaronmk
        except KeyError: raise fe
210 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
211 1607 aaronmk
    return new_value
212 1602 aaronmk
213 2016 aaronmk
def _map(items, node):
214 1602 aaronmk
    '''See repl()
215
    @param items
216
        <last_entry> Value
217
        <other_entries> name=value Mappings. Special values: See repl() repls.
218
    '''
219
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
220
    value = pop_value(items)
221
    if value == None: return None # input is empty
222 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
223 1219 aaronmk
funcs['_map'] = _map
224
225 2016 aaronmk
def _replace(items, node):
226 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
227 1581 aaronmk
    value = pop_value(items)
228
    if value == None: return None # input is empty
229 1219 aaronmk
    try:
230
        for repl, with_ in items:
231
            if re.match(r'^\w+$', repl):
232
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
233
            value = re.sub(repl, with_, value)
234 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
235 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
236 1219 aaronmk
funcs['_replace'] = _replace
237
238 1469 aaronmk
#### Quantities
239
240 2016 aaronmk
def _units(items, node):
241 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
242 1581 aaronmk
    value = pop_value(items)
243
    if value == None: return None # input is empty
244 1471 aaronmk
245 1581 aaronmk
    quantity = units.str2quantity(value)
246 1471 aaronmk
    try:
247
        for action, units_ in items:
248
            units_ = util.none_if(units_, u'')
249
            if action == 'default': units.set_default_units(quantity, units_)
250 1567 aaronmk
            elif action == 'to':
251
                try: quantity = units.convert(quantity, units_)
252 1609 aaronmk
                except ValueError, e: raise FormatException(e)
253 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
254 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
255 1471 aaronmk
    return units.quantity2str(quantity)
256 1225 aaronmk
funcs['_units'] = _units
257
258 1399 aaronmk
def parse_range(str_, range_sep='-'):
259
    default = (str_, None)
260
    start, sep, end = str_.partition(range_sep)
261
    if sep == '': return default # not a range
262 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
263 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
264
265 2016 aaronmk
def _rangeStart(items, node):
266 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
267 1399 aaronmk
    try: value = items['value']
268 1406 aaronmk
    except KeyError: return None # input is empty
269 1399 aaronmk
    return parse_range(value)[0]
270
funcs['_rangeStart'] = _rangeStart
271
272 2016 aaronmk
def _rangeEnd(items, node):
273 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
274 1399 aaronmk
    try: value = items['value']
275 1406 aaronmk
    except KeyError: return None # input is empty
276 1399 aaronmk
    return parse_range(value)[1]
277
funcs['_rangeEnd'] = _rangeEnd
278
279 2016 aaronmk
def _range(items, node):
280 1472 aaronmk
    items = dict(conv_items(float, items))
281
    from_ = items.get('from', None)
282
    to = items.get('to', None)
283
    if from_ == None or to == None: return None
284
    return str(to - from_)
285
funcs['_range'] = _range
286
287 2016 aaronmk
def _avg(items, node):
288 86 aaronmk
    count = 0
289
    sum_ = 0.
290 278 aaronmk
    for name, value in conv_items(float, items):
291 86 aaronmk
        count += 1
292
        sum_ += value
293 1472 aaronmk
    if count == 0: return None # input is empty
294
    else: return str(sum_/count)
295 995 aaronmk
funcs['_avg'] = _avg
296 86 aaronmk
297 968 aaronmk
class CvException(Exception):
298
    def __init__(self):
299
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
300
            ' allowed for ratio scale data '
301
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
302
303 2016 aaronmk
def _noCV(items, node):
304 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
305 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
306
    except IndexError: return None # input is empty
307 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
308 968 aaronmk
    return value
309 995 aaronmk
funcs['_noCV'] = _noCV
310 968 aaronmk
311 1469 aaronmk
#### Names
312
313 328 aaronmk
_name_parts_slices_items = [
314
    ('first', slice(None, 1)),
315
    ('middle', slice(1, -1)),
316
    ('last', slice(-1, None)),
317
]
318
name_parts_slices = dict(_name_parts_slices_items)
319
name_parts = [name for name, slice_ in _name_parts_slices_items]
320
321 2016 aaronmk
def _name(items, node):
322 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
323 102 aaronmk
    parts = []
324 328 aaronmk
    for part in name_parts:
325
        if part in items: parts.append(items[part])
326 102 aaronmk
    return ' '.join(parts)
327 995 aaronmk
funcs['_name'] = _name
328 102 aaronmk
329 1607 aaronmk
#### Angles
330
331 2016 aaronmk
def _compass(items, node):
332 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
333
    items = dict(conv_items(strings.ustr, items))
334
    try: value = items['value']
335
    except KeyError: return None # input is empty
336
337
    if not value.isupper(): return value # pass through other coordinate formats
338
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
339
    except KeyError, e: raise FormatException(e)
340
funcs['_compass'] = _compass
341
342 1469 aaronmk
#### Paths
343
344 2016 aaronmk
def _simplifyPath(items, node):
345 1321 aaronmk
    items = dict(items)
346
    try:
347 1562 aaronmk
        next = cast(strings.ustr, items['next'])
348
        require = cast(strings.ustr, items['require'])
349 1321 aaronmk
        root = items['path']
350 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
351 1321 aaronmk
352
    node = root
353
    while node != None:
354
        new_node = xpath.get_1(node, next, allow_rooted=False)
355 2558 aaronmk
        if xpath.get_value(node, require, allow_rooted=False) == None: # empty
356 1321 aaronmk
            xml_dom.replace(node, new_node) # remove current elem
357
            if node is root: root = new_node # also update root
358
        node = new_node
359
    return root
360
funcs['_simplifyPath'] = _simplifyPath