Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 1992 aaronmk
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53
54 3335 aaronmk
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63
64 995 aaronmk
funcs = {}
65
66 2557 aaronmk
structural_funcs = set()
67
68 1992 aaronmk
##### Public functions
69
70 2112 aaronmk
def is_func_name(name):
71
    return name.startswith('_') and name != '_' # '_' is default root node name
72
73
def is_func(node): return is_func_name(node.tagName)
74
75
def is_xml_func_name(name): return is_func_name(name) and name in funcs
76
77
def is_xml_func(node): return is_xml_func_name(node.tagName)
78
79 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
80 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
81 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
82 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
83
          functions are treated specially, depending on the db param (below).
84
    @param db
85
        * If None: Non-relational functions other than structural functions are
86
          replaced with their last parameter (usually the value), not evaluated.
87
          This is used in column-based mode to remove XML-only functions.
88
        * If != None: Relational functions are evaluated directly. This is used
89
          in row-based mode to combine relational and XML functions.
90 2597 aaronmk
    '''
91 3424 aaronmk
    has_rel_funcs = is_rel_func != None
92 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
93 2597 aaronmk
94 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
95 3424 aaronmk
        process(child, on_error, is_rel_func, db)
96 3335 aaronmk
    merge_tagged(node)
97 3333 aaronmk
98 995 aaronmk
    name = node.tagName
99 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
100 2602 aaronmk
101
    row_mode = has_rel_funcs and db != None
102
    column_mode = has_rel_funcs and db == None
103 3629 aaronmk
    func = funcs.get(name, None)
104 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
105 2602 aaronmk
106 3029 aaronmk
    # Parse function
107
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
108
        # pass-through optimization for aggregating functions with one arg
109
        value = items[0][1] # pass through first arg
110 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
111 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
112
        except sql.DoesNotExistException: return # preserve unknown funcs
113
            # possibly a built-in function of db_xml.put()
114 3640 aaronmk
    elif (column_mode and not name in structural_funcs) or func == None:
115
        # local XML function can't be used or does not exist
116
        if column_mode and is_rel_func(name): return # preserve relational funcs
117
        # otherwise XML-only in column mode, or DB-only in XML output mode
118
        value = pop_value(items, None) # just replace with last param
119 2602 aaronmk
    else: # local XML function
120 3629 aaronmk
        try: value = func(items, node)
121 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
122 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
123
            exc.add_traceback(e)
124 1562 aaronmk
            str_ = strings.ustr(node)
125 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
126 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
127
                '\n'+term.emph_multiline(str_)))
128
129 995 aaronmk
            on_error(e)
130 2602 aaronmk
            return # in case on_error() returns
131 3227 aaronmk
132 3333 aaronmk
    xml_dom.replace_with_text(node, value)
133 995 aaronmk
134 1469 aaronmk
##### XML functions
135 995 aaronmk
136
# Function names must start with _ to avoid collisions with real tags
137
# Functions take arguments (items)
138
139 2557 aaronmk
#### Structural
140 1469 aaronmk
141 2017 aaronmk
def _ignore(items, node):
142 994 aaronmk
    '''Used to "comment out" an XML subtree'''
143
    return None
144 995 aaronmk
funcs['_ignore'] = _ignore
145 2557 aaronmk
structural_funcs.add('_ignore')
146 994 aaronmk
147 2017 aaronmk
def _ref(items, node):
148
    '''Used to retrieve a value from another XML node
149
    @param items
150
        addr=<path> XPath to value, relative to the XML func's parent node
151
    '''
152
    items = dict(items)
153
    try: addr = items['addr']
154
    except KeyError, e: raise SyntaxError(e)
155
156
    value = xpath.get_value(node.parentNode, addr)
157
    if value == None:
158
        warnings.warn(UserWarning('_ref: XPath reference target missing: '
159
            +str(addr)))
160
    return value
161
funcs['_ref'] = _ref
162 2557 aaronmk
structural_funcs.add('_ref')
163 2017 aaronmk
164 1469 aaronmk
#### Conditionals
165
166 2016 aaronmk
def _eq(items, node):
167 1234 aaronmk
    items = dict(items)
168
    try:
169
        left = items['left']
170
        right = items['right']
171
    except KeyError: return '' # a value was None
172
    return util.bool2str(left == right)
173
funcs['_eq'] = _eq
174
175 2016 aaronmk
def _if(items, node):
176 1234 aaronmk
    items = dict(items)
177
    try:
178
        cond = items['cond']
179
        then = items['then']
180 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
181 1234 aaronmk
    else_ = items.get('else', None)
182 1562 aaronmk
    cond = bool(cast(strings.ustr, cond))
183 1234 aaronmk
    if cond: return then
184
    else: return else_
185
funcs['_if'] = _if
186
187 1469 aaronmk
#### Transforming values
188
189 2016 aaronmk
def _collapse(items, node):
190 2012 aaronmk
    '''Collapses a subtree if the "value" element in it is NULL'''
191
    items = dict(items)
192
    try: require = cast(strings.ustr, items['require'])
193
    except KeyError, e: raise SyntaxError(e)
194
    value = items.get('value', None)
195
196 2558 aaronmk
    if xpath.get_value(value, require, allow_rooted=False) == None: return None
197 2012 aaronmk
    else: return value
198
funcs['_collapse'] = _collapse
199
200 1602 aaronmk
def repl(repls, value):
201 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
202 1602 aaronmk
    @param repls dict repl:with
203
        repl "*" means all other input values
204
        with "*" means keep input value the same
205
        with "" means ignore input value
206 1537 aaronmk
    '''
207 1602 aaronmk
    try: new_value = repls[value]
208 1304 aaronmk
    except KeyError, e:
209 1537 aaronmk
        # Save traceback right away in case another exception raised
210 2984 aaronmk
        fe = FormatException(e)
211 1602 aaronmk
        try: new_value = repls['*']
212 1609 aaronmk
        except KeyError: raise fe
213 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
214 1607 aaronmk
    return new_value
215 1602 aaronmk
216 2016 aaronmk
def _map(items, node):
217 1602 aaronmk
    '''See repl()
218
    @param items
219
        <last_entry> Value
220
        <other_entries> name=value Mappings. Special values: See repl() repls.
221
    '''
222
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
223
    value = pop_value(items)
224
    if value == None: return None # input is empty
225 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
226 1219 aaronmk
funcs['_map'] = _map
227
228 2016 aaronmk
def _replace(items, node):
229 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
230 1581 aaronmk
    value = pop_value(items)
231
    if value == None: return None # input is empty
232 1219 aaronmk
    try:
233
        for repl, with_ in items:
234
            if re.match(r'^\w+$', repl):
235
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
236
            value = re.sub(repl, with_, value)
237 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
238 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
239 1219 aaronmk
funcs['_replace'] = _replace
240
241 1469 aaronmk
#### Quantities
242
243 2016 aaronmk
def _units(items, node):
244 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
245 1581 aaronmk
    value = pop_value(items)
246
    if value == None: return None # input is empty
247 1471 aaronmk
248 1581 aaronmk
    quantity = units.str2quantity(value)
249 1471 aaronmk
    try:
250
        for action, units_ in items:
251
            units_ = util.none_if(units_, u'')
252
            if action == 'default': units.set_default_units(quantity, units_)
253 1567 aaronmk
            elif action == 'to':
254
                try: quantity = units.convert(quantity, units_)
255 1609 aaronmk
                except ValueError, e: raise FormatException(e)
256 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
257 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
258 1471 aaronmk
    return units.quantity2str(quantity)
259 1225 aaronmk
funcs['_units'] = _units
260
261 1399 aaronmk
def parse_range(str_, range_sep='-'):
262
    default = (str_, None)
263
    start, sep, end = str_.partition(range_sep)
264
    if sep == '': return default # not a range
265 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
266 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
267
268 2016 aaronmk
def _rangeStart(items, node):
269 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
270 1399 aaronmk
    try: value = items['value']
271 1406 aaronmk
    except KeyError: return None # input is empty
272 1399 aaronmk
    return parse_range(value)[0]
273
funcs['_rangeStart'] = _rangeStart
274
275 2016 aaronmk
def _rangeEnd(items, node):
276 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
277 1399 aaronmk
    try: value = items['value']
278 1406 aaronmk
    except KeyError: return None # input is empty
279 1399 aaronmk
    return parse_range(value)[1]
280
funcs['_rangeEnd'] = _rangeEnd
281
282 2016 aaronmk
def _range(items, node):
283 1472 aaronmk
    items = dict(conv_items(float, items))
284
    from_ = items.get('from', None)
285
    to = items.get('to', None)
286
    if from_ == None or to == None: return None
287
    return str(to - from_)
288
funcs['_range'] = _range
289
290 2016 aaronmk
def _avg(items, node):
291 86 aaronmk
    count = 0
292
    sum_ = 0.
293 278 aaronmk
    for name, value in conv_items(float, items):
294 86 aaronmk
        count += 1
295
        sum_ += value
296 1472 aaronmk
    if count == 0: return None # input is empty
297
    else: return str(sum_/count)
298 995 aaronmk
funcs['_avg'] = _avg
299 86 aaronmk
300 968 aaronmk
class CvException(Exception):
301
    def __init__(self):
302
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
303
            ' allowed for ratio scale data '
304
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
305
306 2016 aaronmk
def _noCV(items, node):
307 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
308 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
309
    except IndexError: return None # input is empty
310 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
311 968 aaronmk
    return value
312 995 aaronmk
funcs['_noCV'] = _noCV
313 968 aaronmk
314 1469 aaronmk
#### Names
315
316 328 aaronmk
_name_parts_slices_items = [
317
    ('first', slice(None, 1)),
318
    ('middle', slice(1, -1)),
319
    ('last', slice(-1, None)),
320
]
321
name_parts_slices = dict(_name_parts_slices_items)
322
name_parts = [name for name, slice_ in _name_parts_slices_items]
323
324 2016 aaronmk
def _name(items, node):
325 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
326 102 aaronmk
    parts = []
327 328 aaronmk
    for part in name_parts:
328
        if part in items: parts.append(items[part])
329 102 aaronmk
    return ' '.join(parts)
330 995 aaronmk
funcs['_name'] = _name
331 102 aaronmk
332 1607 aaronmk
#### Angles
333
334 2016 aaronmk
def _compass(items, node):
335 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
336
    items = dict(conv_items(strings.ustr, items))
337
    try: value = items['value']
338
    except KeyError: return None # input is empty
339
340
    if not value.isupper(): return value # pass through other coordinate formats
341
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
342
    except KeyError, e: raise FormatException(e)
343
funcs['_compass'] = _compass
344
345 1469 aaronmk
#### Paths
346
347 2016 aaronmk
def _simplifyPath(items, node):
348 1321 aaronmk
    items = dict(items)
349
    try:
350 1562 aaronmk
        next = cast(strings.ustr, items['next'])
351
        require = cast(strings.ustr, items['require'])
352 1321 aaronmk
        root = items['path']
353 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
354 1321 aaronmk
355
    node = root
356
    while node != None:
357
        new_node = xpath.get_1(node, next, allow_rooted=False)
358 2558 aaronmk
        if xpath.get_value(node, require, allow_rooted=False) == None: # empty
359 1321 aaronmk
            xml_dom.replace(node, new_node) # remove current elem
360
            if node is root: root = new_node # also update root
361
        node = new_node
362
    return root
363
funcs['_simplifyPath'] = _simplifyPath