Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 2105 aaronmk
import sql
14 1234 aaronmk
import strings
15 827 aaronmk
import term
16 1468 aaronmk
import units
17 1047 aaronmk
import util
18 86 aaronmk
import xml_dom
19 1321 aaronmk
import xpath
20 86 aaronmk
21 995 aaronmk
##### Exceptions
22
23 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
24 797 aaronmk
    def __init__(self, cause):
25 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
26
            cause)
27 278 aaronmk
28 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
29
    def __init__(self, cause):
30
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
31 843 aaronmk
32 1992 aaronmk
##### Helper functions
33 995 aaronmk
34 1992 aaronmk
def map_items(func, items):
35
    return [(name, func(value)) for name, value in items]
36
37
def cast(type_, val):
38
    '''Throws FormatException if can't cast'''
39
    try: return type_(val)
40
    except ValueError, e: raise FormatException(e)
41
42
def conv_items(type_, items):
43
    return map_items(lambda val: cast(type_, val),
44
        xml_dom.TextEntryOnlyIter(items))
45
46
def pop_value(items, name='value'):
47
    '''@param name Name of value param, or None to accept any name'''
48
    try: last = items.pop() # last entry contains value
49
    except IndexError: return None # input is empty and no actions
50
    if name != None and last[0] != name: return None # input is empty
51
    return last[1]
52
53 995 aaronmk
funcs = {}
54
55 2557 aaronmk
structural_funcs = set()
56
57 1992 aaronmk
##### Public functions
58
59 2112 aaronmk
def is_func_name(name):
60
    return name.startswith('_') and name != '_' # '_' is default root node name
61
62
def is_func(node): return is_func_name(node.tagName)
63
64
def is_xml_func_name(name): return is_func_name(name) and name in funcs
65
66
def is_xml_func(node): return is_xml_func_name(node.tagName)
67
68 2602 aaronmk
def process(node, on_error=exc.raise_, rel_funcs=None, db=None):
69 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
70 2602 aaronmk
    @param rel_funcs None|set(str...) Relational functions
71 2597 aaronmk
        * container can be any iterable type
72 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
73
          functions are treated specially, depending on the db param (below).
74
    @param db
75
        * If None: Non-relational functions other than structural functions are
76
          replaced with their last parameter (usually the value), not evaluated.
77
          This is used in column-based mode to remove XML-only functions.
78
        * If != None: Relational functions are evaluated directly. This is used
79
          in row-based mode to combine relational and XML functions.
80 2597 aaronmk
    '''
81 2602 aaronmk
    has_rel_funcs = rel_funcs != None
82
    assert db == None or has_rel_funcs # rel_funcs required if db set
83 2597 aaronmk
84
    for child in xml_dom.NodeElemIter(node):
85 2602 aaronmk
        process(child, on_error, rel_funcs, db)
86
87 995 aaronmk
    name = node.tagName
88 2602 aaronmk
    if not is_func_name(name): return # not any kind of function
89
90
    # Change rel_funcs *after* processing child nodes, which needs orig value
91
    if not has_rel_funcs: rel_funcs = set()
92
    rel_funcs = set(rel_funcs)
93
94
    row_mode = has_rel_funcs and db != None
95
    column_mode = has_rel_funcs and db == None
96
    items = xml_dom.NodeTextEntryIter(node)
97
98
    if row_mode and name in rel_funcs: # row-based mode: evaluate using DB
99
        value = sql.put(db, name, dict(items))
100
    elif column_mode and not name in structural_funcs: # column-based mode
101
        if name in rel_funcs: return # preserve relational functions
102
        # otherwise XML-only, so just replace with last param
103
        value = pop_value(list(items), None)
104
    else: # local XML function
105
        try: value = funcs[name](items, node)
106 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
107 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
108
            exc.add_traceback(e)
109 1562 aaronmk
            str_ = strings.ustr(node)
110 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
111 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
112
                '\n'+term.emph_multiline(str_)))
113
114 995 aaronmk
            on_error(e)
115 2602 aaronmk
            return # in case on_error() returns
116
    xml_dom.replace_with_text(node, value)
117 995 aaronmk
118 1469 aaronmk
##### XML functions
119 995 aaronmk
120
# Function names must start with _ to avoid collisions with real tags
121
# Functions take arguments (items)
122
123 2557 aaronmk
#### Structural
124 1469 aaronmk
125 2017 aaronmk
def _ignore(items, node):
126 994 aaronmk
    '''Used to "comment out" an XML subtree'''
127
    return None
128 995 aaronmk
funcs['_ignore'] = _ignore
129 2557 aaronmk
structural_funcs.add('_ignore')
130 994 aaronmk
131 2017 aaronmk
def _ref(items, node):
132
    '''Used to retrieve a value from another XML node
133
    @param items
134
        addr=<path> XPath to value, relative to the XML func's parent node
135
    '''
136
    items = dict(items)
137
    try: addr = items['addr']
138
    except KeyError, e: raise SyntaxError(e)
139
140
    value = xpath.get_value(node.parentNode, addr)
141
    if value == None:
142
        warnings.warn(UserWarning('_ref: XPath reference target missing: '
143
            +str(addr)))
144
    return value
145
funcs['_ref'] = _ref
146 2557 aaronmk
structural_funcs.add('_ref')
147 2017 aaronmk
148 1469 aaronmk
#### Conditionals
149
150 2016 aaronmk
def _eq(items, node):
151 1234 aaronmk
    items = dict(items)
152
    try:
153
        left = items['left']
154
        right = items['right']
155
    except KeyError: return '' # a value was None
156
    return util.bool2str(left == right)
157
funcs['_eq'] = _eq
158
159 2016 aaronmk
def _if(items, node):
160 1234 aaronmk
    items = dict(items)
161
    try:
162
        cond = items['cond']
163
        then = items['then']
164 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
165 1234 aaronmk
    else_ = items.get('else', None)
166 1562 aaronmk
    cond = bool(cast(strings.ustr, cond))
167 1234 aaronmk
    if cond: return then
168
    else: return else_
169
funcs['_if'] = _if
170
171 1469 aaronmk
#### Combining values
172
173 2016 aaronmk
def _alt(items, node):
174 113 aaronmk
    items = list(items)
175
    items.sort()
176 1186 aaronmk
    try: return items[0][1] # value of lowest-numbered item
177 1609 aaronmk
    except IndexError: return None # input got removed by e.g. FormatException
178 995 aaronmk
funcs['_alt'] = _alt
179 113 aaronmk
180 2016 aaronmk
def _merge(items, node):
181 1234 aaronmk
    items = list(conv_items(strings.ustr, items))
182 1562 aaronmk
        # get *once* from iter, check types
183 917 aaronmk
    items.sort()
184
    return maps.merge_values(*[v for k, v in items])
185 995 aaronmk
funcs['_merge'] = _merge
186 917 aaronmk
187 2016 aaronmk
def _label(items, node):
188 1412 aaronmk
    items = dict(conv_items(strings.ustr, items))
189 1562 aaronmk
        # get *once* from iter, check types
190 2014 aaronmk
    value = items.get('value', None)
191
    if value == None: return None # input is empty
192
    try: label = items['label']
193 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
194 917 aaronmk
    return label+': '+value
195 995 aaronmk
funcs['_label'] = _label
196 917 aaronmk
197 1469 aaronmk
#### Transforming values
198
199 2016 aaronmk
def _collapse(items, node):
200 2012 aaronmk
    '''Collapses a subtree if the "value" element in it is NULL'''
201
    items = dict(items)
202
    try: require = cast(strings.ustr, items['require'])
203
    except KeyError, e: raise SyntaxError(e)
204
    value = items.get('value', None)
205
206 2558 aaronmk
    if xpath.get_value(value, require, allow_rooted=False) == None: return None
207 2012 aaronmk
    else: return value
208
funcs['_collapse'] = _collapse
209
210 1478 aaronmk
types_by_name = {None: strings.ustr, 'str': strings.ustr, 'float': float}
211 1477 aaronmk
212 2016 aaronmk
def _nullIf(items, node):
213 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
214 1477 aaronmk
    try: null = items['null']
215 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
216 1477 aaronmk
    value = items.get('value', None)
217 1219 aaronmk
    type_str = items.get('type', None)
218 1477 aaronmk
219
    try: type_ = types_by_name[type_str]
220 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
221 1477 aaronmk
    null = type_(null)
222
223
    try: return util.none_if(value, null)
224
    except ValueError: return value # value not convertible, so can't equal null
225 1047 aaronmk
funcs['_nullIf'] = _nullIf
226
227 1602 aaronmk
def repl(repls, value):
228 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
229 1602 aaronmk
    @param repls dict repl:with
230
        repl "*" means all other input values
231
        with "*" means keep input value the same
232
        with "" means ignore input value
233 1537 aaronmk
    '''
234 1602 aaronmk
    try: new_value = repls[value]
235 1304 aaronmk
    except KeyError, e:
236 1537 aaronmk
        # Save traceback right away in case another exception raised
237 2984 aaronmk
        fe = FormatException(e)
238 1602 aaronmk
        try: new_value = repls['*']
239 1609 aaronmk
        except KeyError: raise fe
240 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
241 1607 aaronmk
    return new_value
242 1602 aaronmk
243 2016 aaronmk
def _map(items, node):
244 1602 aaronmk
    '''See repl()
245
    @param items
246
        <last_entry> Value
247
        <other_entries> name=value Mappings. Special values: See repl() repls.
248
    '''
249
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
250
    value = pop_value(items)
251
    if value == None: return None # input is empty
252 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
253 1219 aaronmk
funcs['_map'] = _map
254
255 2016 aaronmk
def _replace(items, node):
256 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
257 1581 aaronmk
    value = pop_value(items)
258
    if value == None: return None # input is empty
259 1219 aaronmk
    try:
260
        for repl, with_ in items:
261
            if re.match(r'^\w+$', repl):
262
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
263
            value = re.sub(repl, with_, value)
264 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
265 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
266 1219 aaronmk
funcs['_replace'] = _replace
267
268 1469 aaronmk
#### Quantities
269
270 2016 aaronmk
def _units(items, node):
271 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
272 1581 aaronmk
    value = pop_value(items)
273
    if value == None: return None # input is empty
274 1471 aaronmk
275 1581 aaronmk
    quantity = units.str2quantity(value)
276 1471 aaronmk
    try:
277
        for action, units_ in items:
278
            units_ = util.none_if(units_, u'')
279
            if action == 'default': units.set_default_units(quantity, units_)
280 1567 aaronmk
            elif action == 'to':
281
                try: quantity = units.convert(quantity, units_)
282 1609 aaronmk
                except ValueError, e: raise FormatException(e)
283 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
284 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
285 1471 aaronmk
    return units.quantity2str(quantity)
286 1225 aaronmk
funcs['_units'] = _units
287
288 1399 aaronmk
def parse_range(str_, range_sep='-'):
289
    default = (str_, None)
290
    start, sep, end = str_.partition(range_sep)
291
    if sep == '': return default # not a range
292 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
293 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
294
295 2016 aaronmk
def _rangeStart(items, node):
296 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
297 1399 aaronmk
    try: value = items['value']
298 1406 aaronmk
    except KeyError: return None # input is empty
299 1399 aaronmk
    return parse_range(value)[0]
300
funcs['_rangeStart'] = _rangeStart
301
302 2016 aaronmk
def _rangeEnd(items, node):
303 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
304 1399 aaronmk
    try: value = items['value']
305 1406 aaronmk
    except KeyError: return None # input is empty
306 1399 aaronmk
    return parse_range(value)[1]
307
funcs['_rangeEnd'] = _rangeEnd
308
309 2016 aaronmk
def _range(items, node):
310 1472 aaronmk
    items = dict(conv_items(float, items))
311
    from_ = items.get('from', None)
312
    to = items.get('to', None)
313
    if from_ == None or to == None: return None
314
    return str(to - from_)
315
funcs['_range'] = _range
316
317 2016 aaronmk
def _avg(items, node):
318 86 aaronmk
    count = 0
319
    sum_ = 0.
320 278 aaronmk
    for name, value in conv_items(float, items):
321 86 aaronmk
        count += 1
322
        sum_ += value
323 1472 aaronmk
    if count == 0: return None # input is empty
324
    else: return str(sum_/count)
325 995 aaronmk
funcs['_avg'] = _avg
326 86 aaronmk
327 968 aaronmk
class CvException(Exception):
328
    def __init__(self):
329
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
330
            ' allowed for ratio scale data '
331
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
332
333 2016 aaronmk
def _noCV(items, node):
334 968 aaronmk
    try: name, value = items.next()
335
    except StopIteration: return None
336 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
337 968 aaronmk
    return value
338 995 aaronmk
funcs['_noCV'] = _noCV
339 968 aaronmk
340 1469 aaronmk
#### Dates
341
342 2016 aaronmk
def _date(items, node):
343 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
344
        # get *once* from iter, check types
345 1514 aaronmk
    try: str_ = items['date']
346 786 aaronmk
    except KeyError:
347 1515 aaronmk
        # Year is required
348
        try: items['year']
349 1309 aaronmk
        except KeyError, e:
350
            if items == {}: return None # entire date is empty
351 1609 aaronmk
            else: raise FormatException(e)
352 1515 aaronmk
353
        # Convert month name to number
354
        try: month = items['month']
355
        except KeyError: pass
356
        else:
357
            if not month.isdigit(): # month is name
358 1582 aaronmk
                try: items['month'] = str(dates.strtotime(month).month)
359 1609 aaronmk
                except ValueError, e: raise FormatException(e)
360 1515 aaronmk
361 1580 aaronmk
        items = dict(conv_items(format.str2int, items.iteritems()))
362 786 aaronmk
        items.setdefault('month', 1)
363
        items.setdefault('day', 1)
364 1535 aaronmk
365
        for try_num in xrange(2):
366
            try:
367
                date = datetime.date(**items)
368
                break
369
            except ValueError, e:
370 1609 aaronmk
                if try_num > 0: raise FormatException(e)
371 1536 aaronmk
                    # exception still raised after retry
372 1562 aaronmk
                msg = strings.ustr(e)
373 1535 aaronmk
                if msg == 'month must be in 1..12': # try swapping month and day
374
                    items['month'], items['day'] = items['day'], items['month']
375 1609 aaronmk
                else: raise FormatException(e)
376 786 aaronmk
    else:
377 324 aaronmk
        try: year = float(str_)
378
        except ValueError:
379 1264 aaronmk
            try: date = dates.strtotime(str_)
380 324 aaronmk
            except ImportError: return str_
381 1609 aaronmk
            except ValueError, e: raise FormatException(e)
382 324 aaronmk
        else: date = (datetime.date(int(year), 1, 1) +
383
            datetime.timedelta(round((year % 1.)*365)))
384 818 aaronmk
    try: return dates.strftime('%Y-%m-%d', date)
385 843 aaronmk
    except ValueError, e: raise FormatException(e)
386 995 aaronmk
funcs['_date'] = _date
387 86 aaronmk
388 2016 aaronmk
def _dateRangeStart(items, node):
389 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
390 1366 aaronmk
    try: value = items['value']
391 1406 aaronmk
    except KeyError: return None # input is empty
392 1366 aaronmk
    return dates.parse_date_range(value)[0]
393
funcs['_dateRangeStart'] = _dateRangeStart
394 1311 aaronmk
395 2016 aaronmk
def _dateRangeEnd(items, node):
396 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
397 1366 aaronmk
    try: value = items['value']
398 1406 aaronmk
    except KeyError: return None # input is empty
399 1366 aaronmk
    return dates.parse_date_range(value)[1]
400
funcs['_dateRangeEnd'] = _dateRangeEnd
401 1311 aaronmk
402 1469 aaronmk
#### Names
403
404 328 aaronmk
_name_parts_slices_items = [
405
    ('first', slice(None, 1)),
406
    ('middle', slice(1, -1)),
407
    ('last', slice(-1, None)),
408
]
409
name_parts_slices = dict(_name_parts_slices_items)
410
name_parts = [name for name, slice_ in _name_parts_slices_items]
411
412 2016 aaronmk
def _name(items, node):
413 89 aaronmk
    items = dict(items)
414 102 aaronmk
    parts = []
415 328 aaronmk
    for part in name_parts:
416
        if part in items: parts.append(items[part])
417 102 aaronmk
    return ' '.join(parts)
418 995 aaronmk
funcs['_name'] = _name
419 102 aaronmk
420 2016 aaronmk
def _namePart(items, node):
421 328 aaronmk
    out_items = []
422
    for part, value in items:
423
        try: slice_ = name_parts_slices[part]
424 1612 aaronmk
        except KeyError, e: raise SyntaxError(e)
425 1219 aaronmk
        out_items.append((part, ' '.join(value.split(' ')[slice_])))
426 2016 aaronmk
    return _name(out_items, node)
427 995 aaronmk
funcs['_namePart'] = _namePart
428 1321 aaronmk
429 1607 aaronmk
#### Angles
430
431 2016 aaronmk
def _compass(items, node):
432 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
433
    items = dict(conv_items(strings.ustr, items))
434
    try: value = items['value']
435
    except KeyError: return None # input is empty
436
437
    if not value.isupper(): return value # pass through other coordinate formats
438
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
439
    except KeyError, e: raise FormatException(e)
440
funcs['_compass'] = _compass
441
442 1469 aaronmk
#### Paths
443
444 2016 aaronmk
def _simplifyPath(items, node):
445 1321 aaronmk
    items = dict(items)
446
    try:
447 1562 aaronmk
        next = cast(strings.ustr, items['next'])
448
        require = cast(strings.ustr, items['require'])
449 1321 aaronmk
        root = items['path']
450 1612 aaronmk
    except KeyError, e: raise SyntaxError(e)
451 1321 aaronmk
452
    node = root
453
    while node != None:
454
        new_node = xpath.get_1(node, next, allow_rooted=False)
455 2558 aaronmk
        if xpath.get_value(node, require, allow_rooted=False) == None: # empty
456 1321 aaronmk
            xml_dom.replace(node, new_node) # remove current elem
457
            if node is root: root = new_node # also update root
458
        node = new_node
459
    return root
460
funcs['_simplifyPath'] = _simplifyPath