Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 1992 aaronmk
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53
54 3335 aaronmk
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63
64 995 aaronmk
funcs = {}
65 4236 aaronmk
simplifying_funcs = {}
66 995 aaronmk
67 1992 aaronmk
##### Public functions
68
69 4239 aaronmk
var_name_prefix = '$'
70
71
def is_var_name(str_): return str_.startswith(var_name_prefix)
72
73
def is_var(node):
74
    return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
75
76 2112 aaronmk
def is_func_name(name):
77
    return name.startswith('_') and name != '_' # '_' is default root node name
78
79
def is_func(node): return is_func_name(node.tagName)
80
81
def is_xml_func_name(name): return is_func_name(name) and name in funcs
82
83
def is_xml_func(node): return is_xml_func_name(node.tagName)
84
85 4300 aaronmk
def passthru(node):
86 4302 aaronmk
    '''Passes through single child node. First prunes the node.'''
87 4322 aaronmk
    xml_dom.prune(node)
88 4300 aaronmk
    children = list(xml_dom.NodeEntryIter(node))
89
    if len(children) == 1: xml_dom.replace(node, children[0][1])
90
91 4041 aaronmk
def simplify(node):
92 4305 aaronmk
    '''Simplifies an XML tree.
93 4041 aaronmk
    * Merges nodes tagged as mergable
94 4236 aaronmk
    * Runs simplifying functions
95 4041 aaronmk
    '''
96
    for child in xml_dom.NodeElemIter(node): simplify(child)
97
    merge_tagged(node)
98
99 4227 aaronmk
    name = node.tagName
100 4041 aaronmk
101 4078 aaronmk
    # Pass-through optimizations
102 4228 aaronmk
    if is_func_name(name):
103 4236 aaronmk
        try: func = simplifying_funcs[name]
104 4305 aaronmk
        except KeyError: pass
105 4236 aaronmk
        else: func(node)
106 4229 aaronmk
    # Pruning optimizations
107
    else: # these should not run on functions because they would remove args
108 4319 aaronmk
        xml_dom.prune_children(node)
109 4041 aaronmk
110 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
111 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
112 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
113 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
114
          functions are treated specially, depending on the db param (below).
115
    @param db
116
        * If None: Non-relational functions other than structural functions are
117
          replaced with their last parameter (usually the value), not evaluated.
118
          This is used in column-based mode to remove XML-only functions.
119
        * If != None: Relational functions are evaluated directly. This is used
120
          in row-based mode to combine relational and XML functions.
121 2597 aaronmk
    '''
122 3424 aaronmk
    has_rel_funcs = is_rel_func != None
123 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
124 2597 aaronmk
125 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
126 3424 aaronmk
        process(child, on_error, is_rel_func, db)
127 3335 aaronmk
    merge_tagged(node)
128 3333 aaronmk
129 995 aaronmk
    name = node.tagName
130 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
131 2602 aaronmk
132
    row_mode = has_rel_funcs and db != None
133
    column_mode = has_rel_funcs and db == None
134 3629 aaronmk
    func = funcs.get(name, None)
135 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
136 2602 aaronmk
137 3029 aaronmk
    # Parse function
138
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
139
        # pass-through optimization for aggregating functions with one arg
140
        value = items[0][1] # pass through first arg
141 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
142 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
143
        except sql.DoesNotExistException: return # preserve unknown funcs
144
            # possibly a built-in function of db_xml.put()
145 4024 aaronmk
    elif column_mode or func == None:
146 3640 aaronmk
        # local XML function can't be used or does not exist
147
        if column_mode and is_rel_func(name): return # preserve relational funcs
148
        # otherwise XML-only in column mode, or DB-only in XML output mode
149
        value = pop_value(items, None) # just replace with last param
150 2602 aaronmk
    else: # local XML function
151 3629 aaronmk
        try: value = func(items, node)
152 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
153 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
154
            exc.add_traceback(e)
155 1562 aaronmk
            str_ = strings.ustr(node)
156 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
157 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
158
                '\n'+term.emph_multiline(str_)))
159
160 995 aaronmk
            on_error(e)
161 2602 aaronmk
            return # in case on_error() returns
162 3227 aaronmk
163 3333 aaronmk
    xml_dom.replace_with_text(node, value)
164 995 aaronmk
165 4236 aaronmk
##### Simplifying functions
166
167
# Function names must start with _ to avoid collisions with real tags
168
# Functions take params (node) and have no return value
169
170 4237 aaronmk
#### Logic
171
172
def _exists(node):
173
    '''Returns whether its node is non-empty'''
174
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
175
simplifying_funcs['_exists'] = _exists
176
177 4240 aaronmk
def _if(node):
178
    '''Can add `@name` attr to distinguish separate _if statements'''
179
    params = dict(xml_dom.NodeEntryIter(node))
180
    then = params.get('then', None)
181
    cond = params.get('cond', None)
182
    else_ = params.get('else', None)
183
184
    if cond == None: xml_dom.replace(node, else_) # always False
185
    elif then == else_: xml_dom.replace(node, then) # always same value
186
    elif is_var(cond): pass # can't simplify variable conditions
187
    elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
188
        xml_dom.replace(node, then)
189
simplifying_funcs['_if'] = _if
190
191 4303 aaronmk
#### Merging
192
193
simplifying_funcs['_alt'] = passthru
194 4326 aaronmk
simplifying_funcs['_join'] = passthru
195 4303 aaronmk
simplifying_funcs['_merge'] = passthru
196 4304 aaronmk
simplifying_funcs['_name'] = passthru
197 4303 aaronmk
198 1469 aaronmk
##### XML functions
199 995 aaronmk
200
# Function names must start with _ to avoid collisions with real tags
201 4144 aaronmk
# Functions take arguments (items, node)
202 995 aaronmk
203 1469 aaronmk
#### Transforming values
204
205 1602 aaronmk
def repl(repls, value):
206 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
207 1602 aaronmk
    @param repls dict repl:with
208
        repl "*" means all other input values
209
        with "*" means keep input value the same
210
        with "" means ignore input value
211 1537 aaronmk
    '''
212 1602 aaronmk
    try: new_value = repls[value]
213 1304 aaronmk
    except KeyError, e:
214 1537 aaronmk
        # Save traceback right away in case another exception raised
215 2984 aaronmk
        fe = FormatException(e)
216 1602 aaronmk
        try: new_value = repls['*']
217 1609 aaronmk
        except KeyError: raise fe
218 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
219 1607 aaronmk
    return new_value
220 1602 aaronmk
221 2016 aaronmk
def _map(items, node):
222 1602 aaronmk
    '''See repl()
223
    @param items
224
        <last_entry> Value
225
        <other_entries> name=value Mappings. Special values: See repl() repls.
226
    '''
227
    value = pop_value(items)
228
    if value == None: return None # input is empty
229 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
230 1219 aaronmk
funcs['_map'] = _map
231
232 2016 aaronmk
def _replace(items, node):
233 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
234 1581 aaronmk
    value = pop_value(items)
235
    if value == None: return None # input is empty
236 1219 aaronmk
    try:
237
        for repl, with_ in items:
238
            if re.match(r'^\w+$', repl):
239
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
240
            value = re.sub(repl, with_, value)
241 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
242 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
243 1219 aaronmk
funcs['_replace'] = _replace
244
245 1469 aaronmk
#### Quantities
246
247 2016 aaronmk
def _units(items, node):
248 1581 aaronmk
    value = pop_value(items)
249
    if value == None: return None # input is empty
250 1471 aaronmk
251 1581 aaronmk
    quantity = units.str2quantity(value)
252 1471 aaronmk
    try:
253
        for action, units_ in items:
254
            units_ = util.none_if(units_, u'')
255
            if action == 'default': units.set_default_units(quantity, units_)
256 1567 aaronmk
            elif action == 'to':
257
                try: quantity = units.convert(quantity, units_)
258 1609 aaronmk
                except ValueError, e: raise FormatException(e)
259 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
260 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
261 1471 aaronmk
    return units.quantity2str(quantity)
262 1225 aaronmk
funcs['_units'] = _units
263
264 1399 aaronmk
def parse_range(str_, range_sep='-'):
265
    default = (str_, None)
266
    start, sep, end = str_.partition(range_sep)
267
    if sep == '': return default # not a range
268 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
269 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
270
271 2016 aaronmk
def _rangeStart(items, node):
272 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
273 1399 aaronmk
    try: value = items['value']
274 1406 aaronmk
    except KeyError: return None # input is empty
275 1399 aaronmk
    return parse_range(value)[0]
276
funcs['_rangeStart'] = _rangeStart
277
278 2016 aaronmk
def _rangeEnd(items, node):
279 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
280 1399 aaronmk
    try: value = items['value']
281 1406 aaronmk
    except KeyError: return None # input is empty
282 1399 aaronmk
    return parse_range(value)[1]
283
funcs['_rangeEnd'] = _rangeEnd
284
285 2016 aaronmk
def _range(items, node):
286 1472 aaronmk
    items = dict(conv_items(float, items))
287
    from_ = items.get('from', None)
288
    to = items.get('to', None)
289
    if from_ == None or to == None: return None
290
    return str(to - from_)
291
funcs['_range'] = _range
292
293 2016 aaronmk
def _avg(items, node):
294 86 aaronmk
    count = 0
295
    sum_ = 0.
296 278 aaronmk
    for name, value in conv_items(float, items):
297 86 aaronmk
        count += 1
298
        sum_ += value
299 1472 aaronmk
    if count == 0: return None # input is empty
300
    else: return str(sum_/count)
301 995 aaronmk
funcs['_avg'] = _avg
302 86 aaronmk
303 968 aaronmk
class CvException(Exception):
304
    def __init__(self):
305
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
306
            ' allowed for ratio scale data '
307
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
308
309 2016 aaronmk
def _noCV(items, node):
310 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
311 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
312
    except IndexError: return None # input is empty
313 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
314 968 aaronmk
    return value
315 995 aaronmk
funcs['_noCV'] = _noCV
316 968 aaronmk
317 1469 aaronmk
#### Names
318
319 328 aaronmk
_name_parts_slices_items = [
320
    ('first', slice(None, 1)),
321
    ('middle', slice(1, -1)),
322
    ('last', slice(-1, None)),
323
]
324
name_parts_slices = dict(_name_parts_slices_items)
325
name_parts = [name for name, slice_ in _name_parts_slices_items]
326
327 2016 aaronmk
def _name(items, node):
328 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
329 102 aaronmk
    parts = []
330 328 aaronmk
    for part in name_parts:
331
        if part in items: parts.append(items[part])
332 4035 aaronmk
    if not parts: return None # pass None values through; handle no name parts
333 102 aaronmk
    return ' '.join(parts)
334 995 aaronmk
funcs['_name'] = _name
335 102 aaronmk
336 1607 aaronmk
#### Angles
337
338 2016 aaronmk
def _compass(items, node):
339 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
340
    items = dict(conv_items(strings.ustr, items))
341
    try: value = items['value']
342
    except KeyError: return None # input is empty
343
344
    if not value.isupper(): return value # pass through other coordinate formats
345
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
346
    except KeyError, e: raise FormatException(e)
347
funcs['_compass'] = _compass