Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 2105 aaronmk
import sql
14 1234 aaronmk
import strings
15 827 aaronmk
import term
16 1468 aaronmk
import units
17 1047 aaronmk
import util
18 86 aaronmk
import xml_dom
19 1321 aaronmk
import xpath
20 86 aaronmk
21 995 aaronmk
##### Exceptions
22
23 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
24 797 aaronmk
    def __init__(self, cause):
25 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
26
            cause)
27 278 aaronmk
28 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
29
    def __init__(self, cause):
30
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
31 843 aaronmk
32 1992 aaronmk
##### Helper functions
33 995 aaronmk
34 1992 aaronmk
def map_items(func, items):
35
    return [(name, func(value)) for name, value in items]
36
37
def cast(type_, val):
38
    '''Throws FormatException if can't cast'''
39
    try: return type_(val)
40
    except ValueError, e: raise FormatException(e)
41
42
def conv_items(type_, items):
43
    return map_items(lambda val: cast(type_, val),
44
        xml_dom.TextEntryOnlyIter(items))
45
46
def pop_value(items, name='value'):
47
    '''@param name Name of value param, or None to accept any name'''
48
    try: last = items.pop() # last entry contains value
49
    except IndexError: return None # input is empty and no actions
50
    if name != None and last[0] != name: return None # input is empty
51
    return last[1]
52
53 995 aaronmk
funcs = {}
54
55 2557 aaronmk
structural_funcs = set()
56
57 1992 aaronmk
##### Public functions
58
59 2112 aaronmk
def is_func_name(name):
60
    return name.startswith('_') and name != '_' # '_' is default root node name
61
62
def is_func(node): return is_func_name(node.tagName)
63
64
def is_xml_func_name(name): return is_func_name(name) and name in funcs
65
66
def is_xml_func(node): return is_xml_func_name(node.tagName)
67
68 2602 aaronmk
def process(node, on_error=exc.raise_, rel_funcs=None, db=None):
69 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
70 2602 aaronmk
    @param rel_funcs None|set(str...) Relational functions
71 2597 aaronmk
        * container can be any iterable type
72 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
73
          functions are treated specially, depending on the db param (below).
74
    @param db
75
        * If None: Non-relational functions other than structural functions are
76
          replaced with their last parameter (usually the value), not evaluated.
77
          This is used in column-based mode to remove XML-only functions.
78
        * If != None: Relational functions are evaluated directly. This is used
79
          in row-based mode to combine relational and XML functions.
80 2597 aaronmk
    '''
81 2602 aaronmk
    has_rel_funcs = rel_funcs != None
82
    assert db == None or has_rel_funcs # rel_funcs required if db set
83 2597 aaronmk
84
    for child in xml_dom.NodeElemIter(node):
85 2602 aaronmk
        process(child, on_error, rel_funcs, db)
86
87 995 aaronmk
    name = node.tagName
88 2602 aaronmk
    if not is_func_name(name): return # not any kind of function
89
90
    # Change rel_funcs *after* processing child nodes, which needs orig value
91
    if not has_rel_funcs: rel_funcs = set()
92
    rel_funcs = set(rel_funcs)
93
94
    row_mode = has_rel_funcs and db != None
95
    column_mode = has_rel_funcs and db == None
96 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
97 2602 aaronmk
98 3029 aaronmk
    # Parse function
99
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
100
        # pass-through optimization for aggregating functions with one arg
101
        value = items[0][1] # pass through first arg
102
    elif row_mode and name in rel_funcs: # row-based mode: evaluate using DB
103 2602 aaronmk
        value = sql.put(db, name, dict(items))
104
    elif column_mode and not name in structural_funcs: # column-based mode
105
        if name in rel_funcs: return # preserve relational functions
106
        # otherwise XML-only, so just replace with last param
107 3028 aaronmk
        value = pop_value(items, None)
108 2602 aaronmk
    else: # local XML function
109
        try: value = funcs[name](items, node)
110 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
111 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
112
            exc.add_traceback(e)
113 1562 aaronmk
            str_ = strings.ustr(node)
114 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
115 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
116
                '\n'+term.emph_multiline(str_)))
117
118 995 aaronmk
            on_error(e)
119 2602 aaronmk
            return # in case on_error() returns
120
    xml_dom.replace_with_text(node, value)
121 995 aaronmk
122 1469 aaronmk
##### XML functions
123 995 aaronmk
124
# Function names must start with _ to avoid collisions with real tags
125
# Functions take arguments (items)
126
127 2557 aaronmk
#### Structural
128 1469 aaronmk
129 2017 aaronmk
def _ignore(items, node):
130 994 aaronmk
    '''Used to "comment out" an XML subtree'''
131
    return None
132 995 aaronmk
funcs['_ignore'] = _ignore
133 2557 aaronmk
structural_funcs.add('_ignore')
134 994 aaronmk
135 2017 aaronmk
def _ref(items, node):
136
    '''Used to retrieve a value from another XML node
137
    @param items
138
        addr=<path> XPath to value, relative to the XML func's parent node
139
    '''
140
    items = dict(items)
141
    try: addr = items['addr']
142
    except KeyError, e: raise SyntaxError(e)
143
144
    value = xpath.get_value(node.parentNode, addr)
145
    if value == None:
146
        warnings.warn(UserWarning('_ref: XPath reference target missing: '
147
            +str(addr)))
148
    return value
149
funcs['_ref'] = _ref
150 2557 aaronmk
structural_funcs.add('_ref')
151 2017 aaronmk
152 1469 aaronmk
#### Conditionals
153
154 2016 aaronmk
def _eq(items, node):
155 1234 aaronmk
    items = dict(items)
156
    try:
157
        left = items['left']
158
        right = items['right']
159
    except KeyError: return '' # a value was None
160
    return util.bool2str(left == right)
161
funcs['_eq'] = _eq
162
163 2016 aaronmk
def _if(items, node):
164 1234 aaronmk
    items = dict(items)
165
    try:
166
        cond = items['cond']
167
        then = items['then']
168 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
169 1234 aaronmk
    else_ = items.get('else', None)
170 1562 aaronmk
    cond = bool(cast(strings.ustr, cond))
171 1234 aaronmk
    if cond: return then
172
    else: return else_
173
funcs['_if'] = _if
174
175 1469 aaronmk
#### Combining values
176
177 2016 aaronmk
def _alt(items, node):
178 113 aaronmk
    items = list(items)
179
    items.sort()
180 1186 aaronmk
    try: return items[0][1] # value of lowest-numbered item
181 1609 aaronmk
    except IndexError: return None # input got removed by e.g. FormatException
182 995 aaronmk
funcs['_alt'] = _alt
183 113 aaronmk
184 2016 aaronmk
def _merge(items, node):
185 1234 aaronmk
    items = list(conv_items(strings.ustr, items))
186 1562 aaronmk
        # get *once* from iter, check types
187 917 aaronmk
    items.sort()
188
    return maps.merge_values(*[v for k, v in items])
189 995 aaronmk
funcs['_merge'] = _merge
190 917 aaronmk
191 2016 aaronmk
def _label(items, node):
192 1412 aaronmk
    items = dict(conv_items(strings.ustr, items))
193 1562 aaronmk
        # get *once* from iter, check types
194 2014 aaronmk
    value = items.get('value', None)
195
    if value == None: return None # input is empty
196
    try: label = items['label']
197 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
198 917 aaronmk
    return label+': '+value
199 995 aaronmk
funcs['_label'] = _label
200 917 aaronmk
201 1469 aaronmk
#### Transforming values
202
203 2016 aaronmk
def _collapse(items, node):
204 2012 aaronmk
    '''Collapses a subtree if the "value" element in it is NULL'''
205
    items = dict(items)
206
    try: require = cast(strings.ustr, items['require'])
207
    except KeyError, e: raise SyntaxError(e)
208
    value = items.get('value', None)
209
210 2558 aaronmk
    if xpath.get_value(value, require, allow_rooted=False) == None: return None
211 2012 aaronmk
    else: return value
212
funcs['_collapse'] = _collapse
213
214 1478 aaronmk
types_by_name = {None: strings.ustr, 'str': strings.ustr, 'float': float}
215 1477 aaronmk
216 2016 aaronmk
def _nullIf(items, node):
217 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
218 1477 aaronmk
    try: null = items['null']
219 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
220 1477 aaronmk
    value = items.get('value', None)
221 1219 aaronmk
    type_str = items.get('type', None)
222 1477 aaronmk
223
    try: type_ = types_by_name[type_str]
224 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
225 1477 aaronmk
    null = type_(null)
226
227
    try: return util.none_if(value, null)
228
    except ValueError: return value # value not convertible, so can't equal null
229 1047 aaronmk
funcs['_nullIf'] = _nullIf
230
231 1602 aaronmk
def repl(repls, value):
232 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
233 1602 aaronmk
    @param repls dict repl:with
234
        repl "*" means all other input values
235
        with "*" means keep input value the same
236
        with "" means ignore input value
237 1537 aaronmk
    '''
238 1602 aaronmk
    try: new_value = repls[value]
239 1304 aaronmk
    except KeyError, e:
240 1537 aaronmk
        # Save traceback right away in case another exception raised
241 2984 aaronmk
        fe = FormatException(e)
242 1602 aaronmk
        try: new_value = repls['*']
243 1609 aaronmk
        except KeyError: raise fe
244 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
245 1607 aaronmk
    return new_value
246 1602 aaronmk
247 2016 aaronmk
def _map(items, node):
248 1602 aaronmk
    '''See repl()
249
    @param items
250
        <last_entry> Value
251
        <other_entries> name=value Mappings. Special values: See repl() repls.
252
    '''
253
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
254
    value = pop_value(items)
255
    if value == None: return None # input is empty
256 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
257 1219 aaronmk
funcs['_map'] = _map
258
259 2016 aaronmk
def _replace(items, node):
260 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
261 1581 aaronmk
    value = pop_value(items)
262
    if value == None: return None # input is empty
263 1219 aaronmk
    try:
264
        for repl, with_ in items:
265
            if re.match(r'^\w+$', repl):
266
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
267
            value = re.sub(repl, with_, value)
268 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
269 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
270 1219 aaronmk
funcs['_replace'] = _replace
271
272 1469 aaronmk
#### Quantities
273
274 2016 aaronmk
def _units(items, node):
275 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
276 1581 aaronmk
    value = pop_value(items)
277
    if value == None: return None # input is empty
278 1471 aaronmk
279 1581 aaronmk
    quantity = units.str2quantity(value)
280 1471 aaronmk
    try:
281
        for action, units_ in items:
282
            units_ = util.none_if(units_, u'')
283
            if action == 'default': units.set_default_units(quantity, units_)
284 1567 aaronmk
            elif action == 'to':
285
                try: quantity = units.convert(quantity, units_)
286 1609 aaronmk
                except ValueError, e: raise FormatException(e)
287 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
288 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
289 1471 aaronmk
    return units.quantity2str(quantity)
290 1225 aaronmk
funcs['_units'] = _units
291
292 1399 aaronmk
def parse_range(str_, range_sep='-'):
293
    default = (str_, None)
294
    start, sep, end = str_.partition(range_sep)
295
    if sep == '': return default # not a range
296 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
297 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
298
299 2016 aaronmk
def _rangeStart(items, node):
300 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
301 1399 aaronmk
    try: value = items['value']
302 1406 aaronmk
    except KeyError: return None # input is empty
303 1399 aaronmk
    return parse_range(value)[0]
304
funcs['_rangeStart'] = _rangeStart
305
306 2016 aaronmk
def _rangeEnd(items, node):
307 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
308 1399 aaronmk
    try: value = items['value']
309 1406 aaronmk
    except KeyError: return None # input is empty
310 1399 aaronmk
    return parse_range(value)[1]
311
funcs['_rangeEnd'] = _rangeEnd
312
313 2016 aaronmk
def _range(items, node):
314 1472 aaronmk
    items = dict(conv_items(float, items))
315
    from_ = items.get('from', None)
316
    to = items.get('to', None)
317
    if from_ == None or to == None: return None
318
    return str(to - from_)
319
funcs['_range'] = _range
320
321 2016 aaronmk
def _avg(items, node):
322 86 aaronmk
    count = 0
323
    sum_ = 0.
324 278 aaronmk
    for name, value in conv_items(float, items):
325 86 aaronmk
        count += 1
326
        sum_ += value
327 1472 aaronmk
    if count == 0: return None # input is empty
328
    else: return str(sum_/count)
329 995 aaronmk
funcs['_avg'] = _avg
330 86 aaronmk
331 968 aaronmk
class CvException(Exception):
332
    def __init__(self):
333
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
334
            ' allowed for ratio scale data '
335
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
336
337 2016 aaronmk
def _noCV(items, node):
338 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
339
    except IndexError: return None # input is empty
340 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
341 968 aaronmk
    return value
342 995 aaronmk
funcs['_noCV'] = _noCV
343 968 aaronmk
344 1469 aaronmk
#### Dates
345
346 2016 aaronmk
def _date(items, node):
347 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
348
        # get *once* from iter, check types
349 1514 aaronmk
    try: str_ = items['date']
350 786 aaronmk
    except KeyError:
351 1515 aaronmk
        # Year is required
352
        try: items['year']
353 1309 aaronmk
        except KeyError, e:
354
            if items == {}: return None # entire date is empty
355 1609 aaronmk
            else: raise FormatException(e)
356 1515 aaronmk
357
        # Convert month name to number
358
        try: month = items['month']
359
        except KeyError: pass
360
        else:
361
            if not month.isdigit(): # month is name
362 1582 aaronmk
                try: items['month'] = str(dates.strtotime(month).month)
363 1609 aaronmk
                except ValueError, e: raise FormatException(e)
364 1515 aaronmk
365 1580 aaronmk
        items = dict(conv_items(format.str2int, items.iteritems()))
366 786 aaronmk
        items.setdefault('month', 1)
367
        items.setdefault('day', 1)
368 1535 aaronmk
369
        for try_num in xrange(2):
370
            try:
371
                date = datetime.date(**items)
372
                break
373
            except ValueError, e:
374 1609 aaronmk
                if try_num > 0: raise FormatException(e)
375 1536 aaronmk
                    # exception still raised after retry
376 1562 aaronmk
                msg = strings.ustr(e)
377 1535 aaronmk
                if msg == 'month must be in 1..12': # try swapping month and day
378
                    items['month'], items['day'] = items['day'], items['month']
379 1609 aaronmk
                else: raise FormatException(e)
380 786 aaronmk
    else:
381 324 aaronmk
        try: year = float(str_)
382
        except ValueError:
383 1264 aaronmk
            try: date = dates.strtotime(str_)
384 324 aaronmk
            except ImportError: return str_
385 1609 aaronmk
            except ValueError, e: raise FormatException(e)
386 324 aaronmk
        else: date = (datetime.date(int(year), 1, 1) +
387
            datetime.timedelta(round((year % 1.)*365)))
388 818 aaronmk
    try: return dates.strftime('%Y-%m-%d', date)
389 843 aaronmk
    except ValueError, e: raise FormatException(e)
390 995 aaronmk
funcs['_date'] = _date
391 86 aaronmk
392 2016 aaronmk
def _dateRangeStart(items, node):
393 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
394 1366 aaronmk
    try: value = items['value']
395 1406 aaronmk
    except KeyError: return None # input is empty
396 1366 aaronmk
    return dates.parse_date_range(value)[0]
397
funcs['_dateRangeStart'] = _dateRangeStart
398 1311 aaronmk
399 2016 aaronmk
def _dateRangeEnd(items, node):
400 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
401 1366 aaronmk
    try: value = items['value']
402 1406 aaronmk
    except KeyError: return None # input is empty
403 1366 aaronmk
    return dates.parse_date_range(value)[1]
404
funcs['_dateRangeEnd'] = _dateRangeEnd
405 1311 aaronmk
406 1469 aaronmk
#### Names
407
408 328 aaronmk
_name_parts_slices_items = [
409
    ('first', slice(None, 1)),
410
    ('middle', slice(1, -1)),
411
    ('last', slice(-1, None)),
412
]
413
name_parts_slices = dict(_name_parts_slices_items)
414
name_parts = [name for name, slice_ in _name_parts_slices_items]
415
416 2016 aaronmk
def _name(items, node):
417 89 aaronmk
    items = dict(items)
418 102 aaronmk
    parts = []
419 328 aaronmk
    for part in name_parts:
420
        if part in items: parts.append(items[part])
421 102 aaronmk
    return ' '.join(parts)
422 995 aaronmk
funcs['_name'] = _name
423 102 aaronmk
424 2016 aaronmk
def _namePart(items, node):
425 328 aaronmk
    out_items = []
426
    for part, value in items:
427
        try: slice_ = name_parts_slices[part]
428 1612 aaronmk
        except KeyError, e: raise SyntaxError(e)
429 1219 aaronmk
        out_items.append((part, ' '.join(value.split(' ')[slice_])))
430 2016 aaronmk
    return _name(out_items, node)
431 995 aaronmk
funcs['_namePart'] = _namePart
432 1321 aaronmk
433 1607 aaronmk
#### Angles
434
435 2016 aaronmk
def _compass(items, node):
436 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
437
    items = dict(conv_items(strings.ustr, items))
438
    try: value = items['value']
439
    except KeyError: return None # input is empty
440
441
    if not value.isupper(): return value # pass through other coordinate formats
442
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
443
    except KeyError, e: raise FormatException(e)
444
funcs['_compass'] = _compass
445
446 1469 aaronmk
#### Paths
447
448 2016 aaronmk
def _simplifyPath(items, node):
449 1321 aaronmk
    items = dict(items)
450
    try:
451 1562 aaronmk
        next = cast(strings.ustr, items['next'])
452
        require = cast(strings.ustr, items['require'])
453 1321 aaronmk
        root = items['path']
454 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
455 1321 aaronmk
456
    node = root
457
    while node != None:
458
        new_node = xpath.get_1(node, next, allow_rooted=False)
459 2558 aaronmk
        if xpath.get_value(node, require, allow_rooted=False) == None: # empty
460 1321 aaronmk
            xml_dom.replace(node, new_node) # remove current elem
461
            if node is root: root = new_node # also update root
462
        node = new_node
463
    return root
464
funcs['_simplifyPath'] = _simplifyPath