Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 1992 aaronmk
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53
54 3335 aaronmk
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63
64 995 aaronmk
funcs = {}
65
66 1992 aaronmk
##### Public functions
67
68 2112 aaronmk
def is_func_name(name):
69
    return name.startswith('_') and name != '_' # '_' is default root node name
70
71
def is_func(node): return is_func_name(node.tagName)
72
73
def is_xml_func_name(name): return is_func_name(name) and name in funcs
74
75
def is_xml_func(node): return is_xml_func_name(node.tagName)
76
77 4041 aaronmk
def simplify(node):
78
    '''Simplifies the XML functions in an XML tree.
79
    * Merges nodes tagged as mergable
80
    * Applies a pass-through optimization for aggregating functions with one arg
81
    '''
82
    for child in xml_dom.NodeElemIter(node): simplify(child)
83
    merge_tagged(node)
84
85
    name = node.tagName
86
    if not is_func_name(name): return # not a function
87
88
    items = list(xml_dom.NodeElemIter(node))
89
90
    if len(items) == 1 and items[0].tagName.isdigit(): # single numeric param
91
        # pass-through optimization for aggregating functions with one arg
92
        xml_dom.replace(node, items[0].firstChild) # use first arg's value
93
94 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
95 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
96 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
97 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
98
          functions are treated specially, depending on the db param (below).
99
    @param db
100
        * If None: Non-relational functions other than structural functions are
101
          replaced with their last parameter (usually the value), not evaluated.
102
          This is used in column-based mode to remove XML-only functions.
103
        * If != None: Relational functions are evaluated directly. This is used
104
          in row-based mode to combine relational and XML functions.
105 2597 aaronmk
    '''
106 3424 aaronmk
    has_rel_funcs = is_rel_func != None
107 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
108 2597 aaronmk
109 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
110 3424 aaronmk
        process(child, on_error, is_rel_func, db)
111 3335 aaronmk
    merge_tagged(node)
112 3333 aaronmk
113 995 aaronmk
    name = node.tagName
114 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
115 2602 aaronmk
116
    row_mode = has_rel_funcs and db != None
117
    column_mode = has_rel_funcs and db == None
118 3629 aaronmk
    func = funcs.get(name, None)
119 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
120 2602 aaronmk
121 3029 aaronmk
    # Parse function
122
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
123
        # pass-through optimization for aggregating functions with one arg
124
        value = items[0][1] # pass through first arg
125 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
126 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
127
        except sql.DoesNotExistException: return # preserve unknown funcs
128
            # possibly a built-in function of db_xml.put()
129 4024 aaronmk
    elif column_mode or func == None:
130 3640 aaronmk
        # local XML function can't be used or does not exist
131
        if column_mode and is_rel_func(name): return # preserve relational funcs
132
        # otherwise XML-only in column mode, or DB-only in XML output mode
133
        value = pop_value(items, None) # just replace with last param
134 2602 aaronmk
    else: # local XML function
135 3629 aaronmk
        try: value = func(items, node)
136 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
137 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
138
            exc.add_traceback(e)
139 1562 aaronmk
            str_ = strings.ustr(node)
140 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
141 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
142
                '\n'+term.emph_multiline(str_)))
143
144 995 aaronmk
            on_error(e)
145 2602 aaronmk
            return # in case on_error() returns
146 3227 aaronmk
147 3333 aaronmk
    xml_dom.replace_with_text(node, value)
148 995 aaronmk
149 1469 aaronmk
##### XML functions
150 995 aaronmk
151
# Function names must start with _ to avoid collisions with real tags
152
# Functions take arguments (items)
153
154 1469 aaronmk
#### Conditionals
155
156 2016 aaronmk
def _eq(items, node):
157 1234 aaronmk
    items = dict(items)
158
    try:
159
        left = items['left']
160
        right = items['right']
161 3798 aaronmk
    except KeyError: return '' # a value was not mapped to
162 1234 aaronmk
    return util.bool2str(left == right)
163
funcs['_eq'] = _eq
164
165 2016 aaronmk
def _if(items, node):
166 3940 aaronmk
    '''Can add `@name` attr to distinguish separate _if statements'''
167 1234 aaronmk
    items = dict(items)
168 3920 aaronmk
    then = items.get('then', None)
169 3801 aaronmk
    cond = items.get('cond', None)
170 1234 aaronmk
    else_ = items.get('else', None)
171 3801 aaronmk
172
    cond = cond != None and bool(cast(strings.ustr, cond)) # None == False
173 1234 aaronmk
    if cond: return then
174
    else: return else_
175
funcs['_if'] = _if
176
177 1469 aaronmk
#### Transforming values
178
179 2016 aaronmk
def _collapse(items, node):
180 2012 aaronmk
    '''Collapses a subtree if the "value" element in it is NULL'''
181
    items = dict(items)
182
    try: require = cast(strings.ustr, items['require'])
183
    except KeyError, e: raise SyntaxError(e)
184
    value = items.get('value', None)
185
186 2558 aaronmk
    if xpath.get_value(value, require, allow_rooted=False) == None: return None
187 2012 aaronmk
    else: return value
188
funcs['_collapse'] = _collapse
189
190 1602 aaronmk
def repl(repls, value):
191 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
192 1602 aaronmk
    @param repls dict repl:with
193
        repl "*" means all other input values
194
        with "*" means keep input value the same
195
        with "" means ignore input value
196 1537 aaronmk
    '''
197 1602 aaronmk
    try: new_value = repls[value]
198 1304 aaronmk
    except KeyError, e:
199 1537 aaronmk
        # Save traceback right away in case another exception raised
200 2984 aaronmk
        fe = FormatException(e)
201 1602 aaronmk
        try: new_value = repls['*']
202 1609 aaronmk
        except KeyError: raise fe
203 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
204 1607 aaronmk
    return new_value
205 1602 aaronmk
206 2016 aaronmk
def _map(items, node):
207 1602 aaronmk
    '''See repl()
208
    @param items
209
        <last_entry> Value
210
        <other_entries> name=value Mappings. Special values: See repl() repls.
211
    '''
212
    value = pop_value(items)
213
    if value == None: return None # input is empty
214 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
215 1219 aaronmk
funcs['_map'] = _map
216
217 2016 aaronmk
def _replace(items, node):
218 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
219 1581 aaronmk
    value = pop_value(items)
220
    if value == None: return None # input is empty
221 1219 aaronmk
    try:
222
        for repl, with_ in items:
223
            if re.match(r'^\w+$', repl):
224
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
225
            value = re.sub(repl, with_, value)
226 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
227 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
228 1219 aaronmk
funcs['_replace'] = _replace
229
230 1469 aaronmk
#### Quantities
231
232 2016 aaronmk
def _units(items, node):
233 1581 aaronmk
    value = pop_value(items)
234
    if value == None: return None # input is empty
235 1471 aaronmk
236 1581 aaronmk
    quantity = units.str2quantity(value)
237 1471 aaronmk
    try:
238
        for action, units_ in items:
239
            units_ = util.none_if(units_, u'')
240
            if action == 'default': units.set_default_units(quantity, units_)
241 1567 aaronmk
            elif action == 'to':
242
                try: quantity = units.convert(quantity, units_)
243 1609 aaronmk
                except ValueError, e: raise FormatException(e)
244 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
245 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
246 1471 aaronmk
    return units.quantity2str(quantity)
247 1225 aaronmk
funcs['_units'] = _units
248
249 1399 aaronmk
def parse_range(str_, range_sep='-'):
250
    default = (str_, None)
251
    start, sep, end = str_.partition(range_sep)
252
    if sep == '': return default # not a range
253 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
254 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
255
256 2016 aaronmk
def _rangeStart(items, node):
257 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
258 1399 aaronmk
    try: value = items['value']
259 1406 aaronmk
    except KeyError: return None # input is empty
260 1399 aaronmk
    return parse_range(value)[0]
261
funcs['_rangeStart'] = _rangeStart
262
263 2016 aaronmk
def _rangeEnd(items, node):
264 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
265 1399 aaronmk
    try: value = items['value']
266 1406 aaronmk
    except KeyError: return None # input is empty
267 1399 aaronmk
    return parse_range(value)[1]
268
funcs['_rangeEnd'] = _rangeEnd
269
270 2016 aaronmk
def _range(items, node):
271 1472 aaronmk
    items = dict(conv_items(float, items))
272
    from_ = items.get('from', None)
273
    to = items.get('to', None)
274
    if from_ == None or to == None: return None
275
    return str(to - from_)
276
funcs['_range'] = _range
277
278 2016 aaronmk
def _avg(items, node):
279 86 aaronmk
    count = 0
280
    sum_ = 0.
281 278 aaronmk
    for name, value in conv_items(float, items):
282 86 aaronmk
        count += 1
283
        sum_ += value
284 1472 aaronmk
    if count == 0: return None # input is empty
285
    else: return str(sum_/count)
286 995 aaronmk
funcs['_avg'] = _avg
287 86 aaronmk
288 968 aaronmk
class CvException(Exception):
289
    def __init__(self):
290
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
291
            ' allowed for ratio scale data '
292
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
293
294 2016 aaronmk
def _noCV(items, node):
295 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
296 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
297
    except IndexError: return None # input is empty
298 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
299 968 aaronmk
    return value
300 995 aaronmk
funcs['_noCV'] = _noCV
301 968 aaronmk
302 1469 aaronmk
#### Names
303
304 328 aaronmk
_name_parts_slices_items = [
305
    ('first', slice(None, 1)),
306
    ('middle', slice(1, -1)),
307
    ('last', slice(-1, None)),
308
]
309
name_parts_slices = dict(_name_parts_slices_items)
310
name_parts = [name for name, slice_ in _name_parts_slices_items]
311
312 2016 aaronmk
def _name(items, node):
313 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
314 102 aaronmk
    parts = []
315 328 aaronmk
    for part in name_parts:
316
        if part in items: parts.append(items[part])
317 4035 aaronmk
    if not parts: return None # pass None values through; handle no name parts
318 102 aaronmk
    return ' '.join(parts)
319 995 aaronmk
funcs['_name'] = _name
320 102 aaronmk
321 1607 aaronmk
#### Angles
322
323 2016 aaronmk
def _compass(items, node):
324 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
325
    items = dict(conv_items(strings.ustr, items))
326
    try: value = items['value']
327
    except KeyError: return None # input is empty
328
329
    if not value.isupper(): return value # pass through other coordinate formats
330
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
331
    except KeyError, e: raise FormatException(e)
332
funcs['_compass'] = _compass