Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 1992 aaronmk
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53
54 3335 aaronmk
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63
64 995 aaronmk
funcs = {}
65
66 1992 aaronmk
##### Public functions
67
68 2112 aaronmk
def is_func_name(name):
69
    return name.startswith('_') and name != '_' # '_' is default root node name
70
71
def is_func(node): return is_func_name(node.tagName)
72
73
def is_xml_func_name(name): return is_func_name(name) and name in funcs
74
75
def is_xml_func(node): return is_xml_func_name(node.tagName)
76
77 4041 aaronmk
def simplify(node):
78
    '''Simplifies the XML functions in an XML tree.
79
    * Merges nodes tagged as mergable
80 4078 aaronmk
    * Applies pass-through optimizations for XML functions
81 4041 aaronmk
    '''
82
    for child in xml_dom.NodeElemIter(node): simplify(child)
83
    merge_tagged(node)
84
85 4078 aaronmk
    func_name = node.tagName
86
    if not is_func_name(func_name): return # not a function
87 4041 aaronmk
88 4078 aaronmk
    params = list(xml_dom.NodeElemIter(node))
89 4041 aaronmk
90 4078 aaronmk
    # Pass-through optimizations
91
    if len(params) == 1: # single arg
92
        param = params[0]
93
        name = param.tagName
94
        value = param.firstChild
95
96
        is_agg_func = name.isdigit()
97
        is_if_no_cond = func_name = '_if' # no if condition means False
98
        if is_agg_func or (is_if_no_cond and name == 'else'):
99
            xml_dom.replace(node, value)
100
        elif is_if_no_cond and name == 'then': xml_dom.remove(node)
101 4041 aaronmk
102 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
103 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
104 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
105 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
106
          functions are treated specially, depending on the db param (below).
107
    @param db
108
        * If None: Non-relational functions other than structural functions are
109
          replaced with their last parameter (usually the value), not evaluated.
110
          This is used in column-based mode to remove XML-only functions.
111
        * If != None: Relational functions are evaluated directly. This is used
112
          in row-based mode to combine relational and XML functions.
113 2597 aaronmk
    '''
114 3424 aaronmk
    has_rel_funcs = is_rel_func != None
115 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
116 2597 aaronmk
117 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
118 3424 aaronmk
        process(child, on_error, is_rel_func, db)
119 3335 aaronmk
    merge_tagged(node)
120 3333 aaronmk
121 995 aaronmk
    name = node.tagName
122 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
123 2602 aaronmk
124
    row_mode = has_rel_funcs and db != None
125
    column_mode = has_rel_funcs and db == None
126 3629 aaronmk
    func = funcs.get(name, None)
127 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
128 2602 aaronmk
129 3029 aaronmk
    # Parse function
130
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
131
        # pass-through optimization for aggregating functions with one arg
132
        value = items[0][1] # pass through first arg
133 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
134 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
135
        except sql.DoesNotExistException: return # preserve unknown funcs
136
            # possibly a built-in function of db_xml.put()
137 4024 aaronmk
    elif column_mode or func == None:
138 3640 aaronmk
        # local XML function can't be used or does not exist
139
        if column_mode and is_rel_func(name): return # preserve relational funcs
140
        # otherwise XML-only in column mode, or DB-only in XML output mode
141
        value = pop_value(items, None) # just replace with last param
142 2602 aaronmk
    else: # local XML function
143 3629 aaronmk
        try: value = func(items, node)
144 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
145 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
146
            exc.add_traceback(e)
147 1562 aaronmk
            str_ = strings.ustr(node)
148 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
149 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
150
                '\n'+term.emph_multiline(str_)))
151
152 995 aaronmk
            on_error(e)
153 2602 aaronmk
            return # in case on_error() returns
154 3227 aaronmk
155 3333 aaronmk
    xml_dom.replace_with_text(node, value)
156 995 aaronmk
157 1469 aaronmk
##### XML functions
158 995 aaronmk
159
# Function names must start with _ to avoid collisions with real tags
160
# Functions take arguments (items)
161
162 1469 aaronmk
#### Conditionals
163
164 2016 aaronmk
def _eq(items, node):
165 1234 aaronmk
    items = dict(items)
166
    try:
167
        left = items['left']
168
        right = items['right']
169 3798 aaronmk
    except KeyError: return '' # a value was not mapped to
170 1234 aaronmk
    return util.bool2str(left == right)
171
funcs['_eq'] = _eq
172
173 2016 aaronmk
def _if(items, node):
174 3940 aaronmk
    '''Can add `@name` attr to distinguish separate _if statements'''
175 1234 aaronmk
    items = dict(items)
176 3920 aaronmk
    then = items.get('then', None)
177 3801 aaronmk
    cond = items.get('cond', None)
178 1234 aaronmk
    else_ = items.get('else', None)
179 3801 aaronmk
180
    cond = cond != None and bool(cast(strings.ustr, cond)) # None == False
181 1234 aaronmk
    if cond: return then
182
    else: return else_
183
funcs['_if'] = _if
184
185 1469 aaronmk
#### Transforming values
186
187 2016 aaronmk
def _collapse(items, node):
188 2012 aaronmk
    '''Collapses a subtree if the "value" element in it is NULL'''
189
    items = dict(items)
190
    try: require = cast(strings.ustr, items['require'])
191
    except KeyError, e: raise SyntaxError(e)
192
    value = items.get('value', None)
193
194 2558 aaronmk
    if xpath.get_value(value, require, allow_rooted=False) == None: return None
195 2012 aaronmk
    else: return value
196
funcs['_collapse'] = _collapse
197
198 1602 aaronmk
def repl(repls, value):
199 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
200 1602 aaronmk
    @param repls dict repl:with
201
        repl "*" means all other input values
202
        with "*" means keep input value the same
203
        with "" means ignore input value
204 1537 aaronmk
    '''
205 1602 aaronmk
    try: new_value = repls[value]
206 1304 aaronmk
    except KeyError, e:
207 1537 aaronmk
        # Save traceback right away in case another exception raised
208 2984 aaronmk
        fe = FormatException(e)
209 1602 aaronmk
        try: new_value = repls['*']
210 1609 aaronmk
        except KeyError: raise fe
211 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
212 1607 aaronmk
    return new_value
213 1602 aaronmk
214 2016 aaronmk
def _map(items, node):
215 1602 aaronmk
    '''See repl()
216
    @param items
217
        <last_entry> Value
218
        <other_entries> name=value Mappings. Special values: See repl() repls.
219
    '''
220
    value = pop_value(items)
221
    if value == None: return None # input is empty
222 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
223 1219 aaronmk
funcs['_map'] = _map
224
225 2016 aaronmk
def _replace(items, node):
226 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
227 1581 aaronmk
    value = pop_value(items)
228
    if value == None: return None # input is empty
229 1219 aaronmk
    try:
230
        for repl, with_ in items:
231
            if re.match(r'^\w+$', repl):
232
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
233
            value = re.sub(repl, with_, value)
234 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
235 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
236 1219 aaronmk
funcs['_replace'] = _replace
237
238 1469 aaronmk
#### Quantities
239
240 2016 aaronmk
def _units(items, node):
241 1581 aaronmk
    value = pop_value(items)
242
    if value == None: return None # input is empty
243 1471 aaronmk
244 1581 aaronmk
    quantity = units.str2quantity(value)
245 1471 aaronmk
    try:
246
        for action, units_ in items:
247
            units_ = util.none_if(units_, u'')
248
            if action == 'default': units.set_default_units(quantity, units_)
249 1567 aaronmk
            elif action == 'to':
250
                try: quantity = units.convert(quantity, units_)
251 1609 aaronmk
                except ValueError, e: raise FormatException(e)
252 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
253 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
254 1471 aaronmk
    return units.quantity2str(quantity)
255 1225 aaronmk
funcs['_units'] = _units
256
257 1399 aaronmk
def parse_range(str_, range_sep='-'):
258
    default = (str_, None)
259
    start, sep, end = str_.partition(range_sep)
260
    if sep == '': return default # not a range
261 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
262 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
263
264 2016 aaronmk
def _rangeStart(items, node):
265 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
266 1399 aaronmk
    try: value = items['value']
267 1406 aaronmk
    except KeyError: return None # input is empty
268 1399 aaronmk
    return parse_range(value)[0]
269
funcs['_rangeStart'] = _rangeStart
270
271 2016 aaronmk
def _rangeEnd(items, node):
272 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
273 1399 aaronmk
    try: value = items['value']
274 1406 aaronmk
    except KeyError: return None # input is empty
275 1399 aaronmk
    return parse_range(value)[1]
276
funcs['_rangeEnd'] = _rangeEnd
277
278 2016 aaronmk
def _range(items, node):
279 1472 aaronmk
    items = dict(conv_items(float, items))
280
    from_ = items.get('from', None)
281
    to = items.get('to', None)
282
    if from_ == None or to == None: return None
283
    return str(to - from_)
284
funcs['_range'] = _range
285
286 2016 aaronmk
def _avg(items, node):
287 86 aaronmk
    count = 0
288
    sum_ = 0.
289 278 aaronmk
    for name, value in conv_items(float, items):
290 86 aaronmk
        count += 1
291
        sum_ += value
292 1472 aaronmk
    if count == 0: return None # input is empty
293
    else: return str(sum_/count)
294 995 aaronmk
funcs['_avg'] = _avg
295 86 aaronmk
296 968 aaronmk
class CvException(Exception):
297
    def __init__(self):
298
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
299
            ' allowed for ratio scale data '
300
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
301
302 2016 aaronmk
def _noCV(items, node):
303 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
304 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
305
    except IndexError: return None # input is empty
306 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
307 968 aaronmk
    return value
308 995 aaronmk
funcs['_noCV'] = _noCV
309 968 aaronmk
310 1469 aaronmk
#### Names
311
312 328 aaronmk
_name_parts_slices_items = [
313
    ('first', slice(None, 1)),
314
    ('middle', slice(1, -1)),
315
    ('last', slice(-1, None)),
316
]
317
name_parts_slices = dict(_name_parts_slices_items)
318
name_parts = [name for name, slice_ in _name_parts_slices_items]
319
320 2016 aaronmk
def _name(items, node):
321 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
322 102 aaronmk
    parts = []
323 328 aaronmk
    for part in name_parts:
324
        if part in items: parts.append(items[part])
325 4035 aaronmk
    if not parts: return None # pass None values through; handle no name parts
326 102 aaronmk
    return ' '.join(parts)
327 995 aaronmk
funcs['_name'] = _name
328 102 aaronmk
329 1607 aaronmk
#### Angles
330
331 2016 aaronmk
def _compass(items, node):
332 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
333
    items = dict(conv_items(strings.ustr, items))
334
    try: value = items['value']
335
    except KeyError: return None # input is empty
336
337
    if not value.isupper(): return value # pass through other coordinate formats
338
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
339
    except KeyError, e: raise FormatException(e)
340
funcs['_compass'] = _compass