Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 1992 aaronmk
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53
54 3335 aaronmk
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63
64 995 aaronmk
funcs = {}
65
66 1992 aaronmk
##### Public functions
67
68 2112 aaronmk
def is_func_name(name):
69
    return name.startswith('_') and name != '_' # '_' is default root node name
70
71
def is_func(node): return is_func_name(node.tagName)
72
73
def is_xml_func_name(name): return is_func_name(name) and name in funcs
74
75
def is_xml_func(node): return is_xml_func_name(node.tagName)
76
77 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
78 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
79 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
80 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
81
          functions are treated specially, depending on the db param (below).
82
    @param db
83
        * If None: Non-relational functions other than structural functions are
84
          replaced with their last parameter (usually the value), not evaluated.
85
          This is used in column-based mode to remove XML-only functions.
86
        * If != None: Relational functions are evaluated directly. This is used
87
          in row-based mode to combine relational and XML functions.
88 2597 aaronmk
    '''
89 3424 aaronmk
    has_rel_funcs = is_rel_func != None
90 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
91 2597 aaronmk
92 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
93 3424 aaronmk
        process(child, on_error, is_rel_func, db)
94 3335 aaronmk
    merge_tagged(node)
95 3333 aaronmk
96 995 aaronmk
    name = node.tagName
97 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
98 2602 aaronmk
99
    row_mode = has_rel_funcs and db != None
100
    column_mode = has_rel_funcs and db == None
101 3629 aaronmk
    func = funcs.get(name, None)
102 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
103 2602 aaronmk
104 3029 aaronmk
    # Parse function
105
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
106
        # pass-through optimization for aggregating functions with one arg
107
        value = items[0][1] # pass through first arg
108 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
109 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
110
        except sql.DoesNotExistException: return # preserve unknown funcs
111
            # possibly a built-in function of db_xml.put()
112 4024 aaronmk
    elif column_mode or func == None:
113 3640 aaronmk
        # local XML function can't be used or does not exist
114
        if column_mode and is_rel_func(name): return # preserve relational funcs
115
        # otherwise XML-only in column mode, or DB-only in XML output mode
116
        value = pop_value(items, None) # just replace with last param
117 2602 aaronmk
    else: # local XML function
118 3629 aaronmk
        try: value = func(items, node)
119 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
120 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
121
            exc.add_traceback(e)
122 1562 aaronmk
            str_ = strings.ustr(node)
123 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
124 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
125
                '\n'+term.emph_multiline(str_)))
126
127 995 aaronmk
            on_error(e)
128 2602 aaronmk
            return # in case on_error() returns
129 3227 aaronmk
130 3333 aaronmk
    xml_dom.replace_with_text(node, value)
131 995 aaronmk
132 1469 aaronmk
##### XML functions
133 995 aaronmk
134
# Function names must start with _ to avoid collisions with real tags
135
# Functions take arguments (items)
136
137 1469 aaronmk
#### Conditionals
138
139 2016 aaronmk
def _eq(items, node):
140 1234 aaronmk
    items = dict(items)
141
    try:
142
        left = items['left']
143
        right = items['right']
144 3798 aaronmk
    except KeyError: return '' # a value was not mapped to
145 1234 aaronmk
    return util.bool2str(left == right)
146
funcs['_eq'] = _eq
147
148 2016 aaronmk
def _if(items, node):
149 3940 aaronmk
    '''Can add `@name` attr to distinguish separate _if statements'''
150 1234 aaronmk
    items = dict(items)
151 3920 aaronmk
    then = items.get('then', None)
152 3801 aaronmk
    cond = items.get('cond', None)
153 1234 aaronmk
    else_ = items.get('else', None)
154 3801 aaronmk
155
    cond = cond != None and bool(cast(strings.ustr, cond)) # None == False
156 1234 aaronmk
    if cond: return then
157
    else: return else_
158
funcs['_if'] = _if
159
160 1469 aaronmk
#### Transforming values
161
162 2016 aaronmk
def _collapse(items, node):
163 2012 aaronmk
    '''Collapses a subtree if the "value" element in it is NULL'''
164
    items = dict(items)
165
    try: require = cast(strings.ustr, items['require'])
166
    except KeyError, e: raise SyntaxError(e)
167
    value = items.get('value', None)
168
169 2558 aaronmk
    if xpath.get_value(value, require, allow_rooted=False) == None: return None
170 2012 aaronmk
    else: return value
171
funcs['_collapse'] = _collapse
172
173 1602 aaronmk
def repl(repls, value):
174 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
175 1602 aaronmk
    @param repls dict repl:with
176
        repl "*" means all other input values
177
        with "*" means keep input value the same
178
        with "" means ignore input value
179 1537 aaronmk
    '''
180 1602 aaronmk
    try: new_value = repls[value]
181 1304 aaronmk
    except KeyError, e:
182 1537 aaronmk
        # Save traceback right away in case another exception raised
183 2984 aaronmk
        fe = FormatException(e)
184 1602 aaronmk
        try: new_value = repls['*']
185 1609 aaronmk
        except KeyError: raise fe
186 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
187 1607 aaronmk
    return new_value
188 1602 aaronmk
189 2016 aaronmk
def _map(items, node):
190 1602 aaronmk
    '''See repl()
191
    @param items
192
        <last_entry> Value
193
        <other_entries> name=value Mappings. Special values: See repl() repls.
194
    '''
195
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
196
    value = pop_value(items)
197
    if value == None: return None # input is empty
198 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
199 1219 aaronmk
funcs['_map'] = _map
200
201 2016 aaronmk
def _replace(items, node):
202 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
203 1581 aaronmk
    value = pop_value(items)
204
    if value == None: return None # input is empty
205 1219 aaronmk
    try:
206
        for repl, with_ in items:
207
            if re.match(r'^\w+$', repl):
208
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
209
            value = re.sub(repl, with_, value)
210 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
211 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
212 1219 aaronmk
funcs['_replace'] = _replace
213
214 1469 aaronmk
#### Quantities
215
216 2016 aaronmk
def _units(items, node):
217 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
218 1581 aaronmk
    value = pop_value(items)
219
    if value == None: return None # input is empty
220 1471 aaronmk
221 1581 aaronmk
    quantity = units.str2quantity(value)
222 1471 aaronmk
    try:
223
        for action, units_ in items:
224
            units_ = util.none_if(units_, u'')
225
            if action == 'default': units.set_default_units(quantity, units_)
226 1567 aaronmk
            elif action == 'to':
227
                try: quantity = units.convert(quantity, units_)
228 1609 aaronmk
                except ValueError, e: raise FormatException(e)
229 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
230 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
231 1471 aaronmk
    return units.quantity2str(quantity)
232 1225 aaronmk
funcs['_units'] = _units
233
234 1399 aaronmk
def parse_range(str_, range_sep='-'):
235
    default = (str_, None)
236
    start, sep, end = str_.partition(range_sep)
237
    if sep == '': return default # not a range
238 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
239 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
240
241 2016 aaronmk
def _rangeStart(items, node):
242 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
243 1399 aaronmk
    try: value = items['value']
244 1406 aaronmk
    except KeyError: return None # input is empty
245 1399 aaronmk
    return parse_range(value)[0]
246
funcs['_rangeStart'] = _rangeStart
247
248 2016 aaronmk
def _rangeEnd(items, node):
249 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
250 1399 aaronmk
    try: value = items['value']
251 1406 aaronmk
    except KeyError: return None # input is empty
252 1399 aaronmk
    return parse_range(value)[1]
253
funcs['_rangeEnd'] = _rangeEnd
254
255 2016 aaronmk
def _range(items, node):
256 1472 aaronmk
    items = dict(conv_items(float, items))
257
    from_ = items.get('from', None)
258
    to = items.get('to', None)
259
    if from_ == None or to == None: return None
260
    return str(to - from_)
261
funcs['_range'] = _range
262
263 2016 aaronmk
def _avg(items, node):
264 86 aaronmk
    count = 0
265
    sum_ = 0.
266 278 aaronmk
    for name, value in conv_items(float, items):
267 86 aaronmk
        count += 1
268
        sum_ += value
269 1472 aaronmk
    if count == 0: return None # input is empty
270
    else: return str(sum_/count)
271 995 aaronmk
funcs['_avg'] = _avg
272 86 aaronmk
273 968 aaronmk
class CvException(Exception):
274
    def __init__(self):
275
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
276
            ' allowed for ratio scale data '
277
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
278
279 2016 aaronmk
def _noCV(items, node):
280 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
281 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
282
    except IndexError: return None # input is empty
283 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
284 968 aaronmk
    return value
285 995 aaronmk
funcs['_noCV'] = _noCV
286 968 aaronmk
287 1469 aaronmk
#### Names
288
289 328 aaronmk
_name_parts_slices_items = [
290
    ('first', slice(None, 1)),
291
    ('middle', slice(1, -1)),
292
    ('last', slice(-1, None)),
293
]
294
name_parts_slices = dict(_name_parts_slices_items)
295
name_parts = [name for name, slice_ in _name_parts_slices_items]
296
297 2016 aaronmk
def _name(items, node):
298 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
299 102 aaronmk
    parts = []
300 328 aaronmk
    for part in name_parts:
301
        if part in items: parts.append(items[part])
302 102 aaronmk
    return ' '.join(parts)
303 995 aaronmk
funcs['_name'] = _name
304 102 aaronmk
305 1607 aaronmk
#### Angles
306
307 2016 aaronmk
def _compass(items, node):
308 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
309
    items = dict(conv_items(strings.ustr, items))
310
    try: value = items['value']
311
    except KeyError: return None # input is empty
312
313
    if not value.isupper(): return value # pass through other coordinate formats
314
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
315
    except KeyError, e: raise FormatException(e)
316
funcs['_compass'] = _compass