Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 1992 aaronmk
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53
54 3335 aaronmk
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63
64 995 aaronmk
funcs = {}
65 4236 aaronmk
simplifying_funcs = {}
66 995 aaronmk
67 1992 aaronmk
##### Public functions
68
69 4239 aaronmk
var_name_prefix = '$'
70
71
def is_var_name(str_): return str_.startswith(var_name_prefix)
72
73
def is_var(node):
74
    return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
75
76 2112 aaronmk
def is_func_name(name):
77
    return name.startswith('_') and name != '_' # '_' is default root node name
78
79
def is_func(node): return is_func_name(node.tagName)
80
81
def is_xml_func_name(name): return is_func_name(name) and name in funcs
82
83
def is_xml_func(node): return is_xml_func_name(node.tagName)
84
85 4041 aaronmk
def simplify(node):
86
    '''Simplifies the XML functions in an XML tree.
87
    * Merges nodes tagged as mergable
88 4236 aaronmk
    * Runs simplifying functions
89 4078 aaronmk
    * Applies pass-through optimizations for XML functions
90 4041 aaronmk
    '''
91
    for child in xml_dom.NodeElemIter(node): simplify(child)
92
    merge_tagged(node)
93
94 4227 aaronmk
    name = node.tagName
95 4041 aaronmk
96 4078 aaronmk
    # Pass-through optimizations
97 4228 aaronmk
    if is_func_name(name):
98 4236 aaronmk
        try: func = simplifying_funcs[name]
99
        except KeyError:
100
            children = list(xml_dom.NodeElemIter(node))
101
            if len(children) == 1: # single arg function
102
                func = name
103
                param_node = children[0]
104
                param = param_node.tagName
105
                value = param_node.firstChild
106
107
                is_agg_func = param.isdigit() or func == '_name'
108 4243 aaronmk
                if is_agg_func: xml_dom.replace(node, value)
109 4236 aaronmk
        else: func(node)
110 4229 aaronmk
    # Pruning optimizations
111
    else: # these should not run on functions because they would remove args
112
        # Remove empty children
113
        for child in xml_dom.NodeElemIter(node):
114
            if xml_dom.is_empty(child): xml_dom.remove(child)
115 4041 aaronmk
116 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
117 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
118 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
119 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
120
          functions are treated specially, depending on the db param (below).
121
    @param db
122
        * If None: Non-relational functions other than structural functions are
123
          replaced with their last parameter (usually the value), not evaluated.
124
          This is used in column-based mode to remove XML-only functions.
125
        * If != None: Relational functions are evaluated directly. This is used
126
          in row-based mode to combine relational and XML functions.
127 2597 aaronmk
    '''
128 3424 aaronmk
    has_rel_funcs = is_rel_func != None
129 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
130 2597 aaronmk
131 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
132 3424 aaronmk
        process(child, on_error, is_rel_func, db)
133 3335 aaronmk
    merge_tagged(node)
134 3333 aaronmk
135 995 aaronmk
    name = node.tagName
136 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
137 2602 aaronmk
138
    row_mode = has_rel_funcs and db != None
139
    column_mode = has_rel_funcs and db == None
140 3629 aaronmk
    func = funcs.get(name, None)
141 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
142 2602 aaronmk
143 3029 aaronmk
    # Parse function
144
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
145
        # pass-through optimization for aggregating functions with one arg
146
        value = items[0][1] # pass through first arg
147 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
148 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
149
        except sql.DoesNotExistException: return # preserve unknown funcs
150
            # possibly a built-in function of db_xml.put()
151 4024 aaronmk
    elif column_mode or func == None:
152 3640 aaronmk
        # local XML function can't be used or does not exist
153
        if column_mode and is_rel_func(name): return # preserve relational funcs
154
        # otherwise XML-only in column mode, or DB-only in XML output mode
155
        value = pop_value(items, None) # just replace with last param
156 2602 aaronmk
    else: # local XML function
157 3629 aaronmk
        try: value = func(items, node)
158 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
159 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
160
            exc.add_traceback(e)
161 1562 aaronmk
            str_ = strings.ustr(node)
162 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
163 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
164
                '\n'+term.emph_multiline(str_)))
165
166 995 aaronmk
            on_error(e)
167 2602 aaronmk
            return # in case on_error() returns
168 3227 aaronmk
169 3333 aaronmk
    xml_dom.replace_with_text(node, value)
170 995 aaronmk
171 4236 aaronmk
##### Simplifying functions
172
173
# Function names must start with _ to avoid collisions with real tags
174
# Functions take params (node) and have no return value
175
176 4237 aaronmk
#### Logic
177
178
def _exists(node):
179
    '''Returns whether its node is non-empty'''
180
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
181
simplifying_funcs['_exists'] = _exists
182
183 4240 aaronmk
def _if(node):
184
    '''Can add `@name` attr to distinguish separate _if statements'''
185
    params = dict(xml_dom.NodeEntryIter(node))
186
    then = params.get('then', None)
187
    cond = params.get('cond', None)
188
    else_ = params.get('else', None)
189
190
    if cond == None: xml_dom.replace(node, else_) # always False
191
    elif then == else_: xml_dom.replace(node, then) # always same value
192
    elif is_var(cond): pass # can't simplify variable conditions
193
    elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
194
        xml_dom.replace(node, then)
195
simplifying_funcs['_if'] = _if
196
197 1469 aaronmk
##### XML functions
198 995 aaronmk
199
# Function names must start with _ to avoid collisions with real tags
200 4144 aaronmk
# Functions take arguments (items, node)
201 995 aaronmk
202 1469 aaronmk
#### Transforming values
203
204 1602 aaronmk
def repl(repls, value):
205 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
206 1602 aaronmk
    @param repls dict repl:with
207
        repl "*" means all other input values
208
        with "*" means keep input value the same
209
        with "" means ignore input value
210 1537 aaronmk
    '''
211 1602 aaronmk
    try: new_value = repls[value]
212 1304 aaronmk
    except KeyError, e:
213 1537 aaronmk
        # Save traceback right away in case another exception raised
214 2984 aaronmk
        fe = FormatException(e)
215 1602 aaronmk
        try: new_value = repls['*']
216 1609 aaronmk
        except KeyError: raise fe
217 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
218 1607 aaronmk
    return new_value
219 1602 aaronmk
220 2016 aaronmk
def _map(items, node):
221 1602 aaronmk
    '''See repl()
222
    @param items
223
        <last_entry> Value
224
        <other_entries> name=value Mappings. Special values: See repl() repls.
225
    '''
226
    value = pop_value(items)
227
    if value == None: return None # input is empty
228 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
229 1219 aaronmk
funcs['_map'] = _map
230
231 2016 aaronmk
def _replace(items, node):
232 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
233 1581 aaronmk
    value = pop_value(items)
234
    if value == None: return None # input is empty
235 1219 aaronmk
    try:
236
        for repl, with_ in items:
237
            if re.match(r'^\w+$', repl):
238
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
239
            value = re.sub(repl, with_, value)
240 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
241 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
242 1219 aaronmk
funcs['_replace'] = _replace
243
244 1469 aaronmk
#### Quantities
245
246 2016 aaronmk
def _units(items, node):
247 1581 aaronmk
    value = pop_value(items)
248
    if value == None: return None # input is empty
249 1471 aaronmk
250 1581 aaronmk
    quantity = units.str2quantity(value)
251 1471 aaronmk
    try:
252
        for action, units_ in items:
253
            units_ = util.none_if(units_, u'')
254
            if action == 'default': units.set_default_units(quantity, units_)
255 1567 aaronmk
            elif action == 'to':
256
                try: quantity = units.convert(quantity, units_)
257 1609 aaronmk
                except ValueError, e: raise FormatException(e)
258 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
259 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
260 1471 aaronmk
    return units.quantity2str(quantity)
261 1225 aaronmk
funcs['_units'] = _units
262
263 1399 aaronmk
def parse_range(str_, range_sep='-'):
264
    default = (str_, None)
265
    start, sep, end = str_.partition(range_sep)
266
    if sep == '': return default # not a range
267 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
268 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
269
270 2016 aaronmk
def _rangeStart(items, node):
271 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
272 1399 aaronmk
    try: value = items['value']
273 1406 aaronmk
    except KeyError: return None # input is empty
274 1399 aaronmk
    return parse_range(value)[0]
275
funcs['_rangeStart'] = _rangeStart
276
277 2016 aaronmk
def _rangeEnd(items, node):
278 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
279 1399 aaronmk
    try: value = items['value']
280 1406 aaronmk
    except KeyError: return None # input is empty
281 1399 aaronmk
    return parse_range(value)[1]
282
funcs['_rangeEnd'] = _rangeEnd
283
284 2016 aaronmk
def _range(items, node):
285 1472 aaronmk
    items = dict(conv_items(float, items))
286
    from_ = items.get('from', None)
287
    to = items.get('to', None)
288
    if from_ == None or to == None: return None
289
    return str(to - from_)
290
funcs['_range'] = _range
291
292 2016 aaronmk
def _avg(items, node):
293 86 aaronmk
    count = 0
294
    sum_ = 0.
295 278 aaronmk
    for name, value in conv_items(float, items):
296 86 aaronmk
        count += 1
297
        sum_ += value
298 1472 aaronmk
    if count == 0: return None # input is empty
299
    else: return str(sum_/count)
300 995 aaronmk
funcs['_avg'] = _avg
301 86 aaronmk
302 968 aaronmk
class CvException(Exception):
303
    def __init__(self):
304
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
305
            ' allowed for ratio scale data '
306
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
307
308 2016 aaronmk
def _noCV(items, node):
309 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
310 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
311
    except IndexError: return None # input is empty
312 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
313 968 aaronmk
    return value
314 995 aaronmk
funcs['_noCV'] = _noCV
315 968 aaronmk
316 1469 aaronmk
#### Names
317
318 328 aaronmk
_name_parts_slices_items = [
319
    ('first', slice(None, 1)),
320
    ('middle', slice(1, -1)),
321
    ('last', slice(-1, None)),
322
]
323
name_parts_slices = dict(_name_parts_slices_items)
324
name_parts = [name for name, slice_ in _name_parts_slices_items]
325
326 2016 aaronmk
def _name(items, node):
327 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
328 102 aaronmk
    parts = []
329 328 aaronmk
    for part in name_parts:
330
        if part in items: parts.append(items[part])
331 4035 aaronmk
    if not parts: return None # pass None values through; handle no name parts
332 102 aaronmk
    return ' '.join(parts)
333 995 aaronmk
funcs['_name'] = _name
334 102 aaronmk
335 1607 aaronmk
#### Angles
336
337 2016 aaronmk
def _compass(items, node):
338 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
339
    items = dict(conv_items(strings.ustr, items))
340
    try: value = items['value']
341
    except KeyError: return None # input is empty
342
343
    if not value.isupper(): return value # pass through other coordinate formats
344
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
345
    except KeyError, e: raise FormatException(e)
346
funcs['_compass'] = _compass