Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 1992 aaronmk
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53
54 3335 aaronmk
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63
64 995 aaronmk
funcs = {}
65 4236 aaronmk
simplifying_funcs = {}
66 995 aaronmk
67 1992 aaronmk
##### Public functions
68
69 2112 aaronmk
def is_func_name(name):
70
    return name.startswith('_') and name != '_' # '_' is default root node name
71
72
def is_func(node): return is_func_name(node.tagName)
73
74
def is_xml_func_name(name): return is_func_name(name) and name in funcs
75
76
def is_xml_func(node): return is_xml_func_name(node.tagName)
77
78 4041 aaronmk
def simplify(node):
79
    '''Simplifies the XML functions in an XML tree.
80
    * Merges nodes tagged as mergable
81 4236 aaronmk
    * Runs simplifying functions
82 4078 aaronmk
    * Applies pass-through optimizations for XML functions
83 4041 aaronmk
    '''
84
    for child in xml_dom.NodeElemIter(node): simplify(child)
85
    merge_tagged(node)
86
87 4227 aaronmk
    name = node.tagName
88 4041 aaronmk
89 4078 aaronmk
    # Pass-through optimizations
90 4228 aaronmk
    if is_func_name(name):
91 4236 aaronmk
        try: func = simplifying_funcs[name]
92
        except KeyError:
93
            children = list(xml_dom.NodeElemIter(node))
94
            if len(children) == 1: # single arg function
95
                func = name
96
                param_node = children[0]
97
                param = param_node.tagName
98
                value = param_node.firstChild
99
100
                is_agg_func = param.isdigit() or func == '_name'
101
                if is_agg_func or (func == '_if' and param == 'else'):
102
                    xml_dom.replace(node, value)
103
                elif func == '_if': xml_dom.remove(node) # only cond or then arg
104
        else: func(node)
105 4229 aaronmk
    # Pruning optimizations
106
    else: # these should not run on functions because they would remove args
107
        # Remove empty children
108
        for child in xml_dom.NodeElemIter(node):
109
            if xml_dom.is_empty(child): xml_dom.remove(child)
110 4041 aaronmk
111 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
112 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
113 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
114 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
115
          functions are treated specially, depending on the db param (below).
116
    @param db
117
        * If None: Non-relational functions other than structural functions are
118
          replaced with their last parameter (usually the value), not evaluated.
119
          This is used in column-based mode to remove XML-only functions.
120
        * If != None: Relational functions are evaluated directly. This is used
121
          in row-based mode to combine relational and XML functions.
122 2597 aaronmk
    '''
123 3424 aaronmk
    has_rel_funcs = is_rel_func != None
124 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
125 2597 aaronmk
126 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
127 3424 aaronmk
        process(child, on_error, is_rel_func, db)
128 3335 aaronmk
    merge_tagged(node)
129 3333 aaronmk
130 995 aaronmk
    name = node.tagName
131 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
132 2602 aaronmk
133
    row_mode = has_rel_funcs and db != None
134
    column_mode = has_rel_funcs and db == None
135 3629 aaronmk
    func = funcs.get(name, None)
136 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
137 2602 aaronmk
138 3029 aaronmk
    # Parse function
139
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
140
        # pass-through optimization for aggregating functions with one arg
141
        value = items[0][1] # pass through first arg
142 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
143 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
144
        except sql.DoesNotExistException: return # preserve unknown funcs
145
            # possibly a built-in function of db_xml.put()
146 4024 aaronmk
    elif column_mode or func == None:
147 3640 aaronmk
        # local XML function can't be used or does not exist
148
        if column_mode and is_rel_func(name): return # preserve relational funcs
149
        # otherwise XML-only in column mode, or DB-only in XML output mode
150
        value = pop_value(items, None) # just replace with last param
151 2602 aaronmk
    else: # local XML function
152 3629 aaronmk
        try: value = func(items, node)
153 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
154 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
155
            exc.add_traceback(e)
156 1562 aaronmk
            str_ = strings.ustr(node)
157 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
158 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
159
                '\n'+term.emph_multiline(str_)))
160
161 995 aaronmk
            on_error(e)
162 2602 aaronmk
            return # in case on_error() returns
163 3227 aaronmk
164 3333 aaronmk
    xml_dom.replace_with_text(node, value)
165 995 aaronmk
166 4236 aaronmk
##### Simplifying functions
167
168
# Function names must start with _ to avoid collisions with real tags
169
# Functions take params (node) and have no return value
170
171 4237 aaronmk
#### Logic
172
173
def _exists(node):
174
    '''Returns whether its node is non-empty'''
175
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
176
simplifying_funcs['_exists'] = _exists
177
178 1469 aaronmk
##### XML functions
179 995 aaronmk
180
# Function names must start with _ to avoid collisions with real tags
181 4144 aaronmk
# Functions take arguments (items, node)
182 995 aaronmk
183 1469 aaronmk
#### Transforming values
184
185 1602 aaronmk
def repl(repls, value):
186 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
187 1602 aaronmk
    @param repls dict repl:with
188
        repl "*" means all other input values
189
        with "*" means keep input value the same
190
        with "" means ignore input value
191 1537 aaronmk
    '''
192 1602 aaronmk
    try: new_value = repls[value]
193 1304 aaronmk
    except KeyError, e:
194 1537 aaronmk
        # Save traceback right away in case another exception raised
195 2984 aaronmk
        fe = FormatException(e)
196 1602 aaronmk
        try: new_value = repls['*']
197 1609 aaronmk
        except KeyError: raise fe
198 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
199 1607 aaronmk
    return new_value
200 1602 aaronmk
201 2016 aaronmk
def _map(items, node):
202 1602 aaronmk
    '''See repl()
203
    @param items
204
        <last_entry> Value
205
        <other_entries> name=value Mappings. Special values: See repl() repls.
206
    '''
207
    value = pop_value(items)
208
    if value == None: return None # input is empty
209 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
210 1219 aaronmk
funcs['_map'] = _map
211
212 2016 aaronmk
def _replace(items, node):
213 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
214 1581 aaronmk
    value = pop_value(items)
215
    if value == None: return None # input is empty
216 1219 aaronmk
    try:
217
        for repl, with_ in items:
218
            if re.match(r'^\w+$', repl):
219
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
220
            value = re.sub(repl, with_, value)
221 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
222 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
223 1219 aaronmk
funcs['_replace'] = _replace
224
225 1469 aaronmk
#### Quantities
226
227 2016 aaronmk
def _units(items, node):
228 1581 aaronmk
    value = pop_value(items)
229
    if value == None: return None # input is empty
230 1471 aaronmk
231 1581 aaronmk
    quantity = units.str2quantity(value)
232 1471 aaronmk
    try:
233
        for action, units_ in items:
234
            units_ = util.none_if(units_, u'')
235
            if action == 'default': units.set_default_units(quantity, units_)
236 1567 aaronmk
            elif action == 'to':
237
                try: quantity = units.convert(quantity, units_)
238 1609 aaronmk
                except ValueError, e: raise FormatException(e)
239 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
240 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
241 1471 aaronmk
    return units.quantity2str(quantity)
242 1225 aaronmk
funcs['_units'] = _units
243
244 1399 aaronmk
def parse_range(str_, range_sep='-'):
245
    default = (str_, None)
246
    start, sep, end = str_.partition(range_sep)
247
    if sep == '': return default # not a range
248 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
249 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
250
251 2016 aaronmk
def _rangeStart(items, node):
252 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
253 1399 aaronmk
    try: value = items['value']
254 1406 aaronmk
    except KeyError: return None # input is empty
255 1399 aaronmk
    return parse_range(value)[0]
256
funcs['_rangeStart'] = _rangeStart
257
258 2016 aaronmk
def _rangeEnd(items, node):
259 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
260 1399 aaronmk
    try: value = items['value']
261 1406 aaronmk
    except KeyError: return None # input is empty
262 1399 aaronmk
    return parse_range(value)[1]
263
funcs['_rangeEnd'] = _rangeEnd
264
265 2016 aaronmk
def _range(items, node):
266 1472 aaronmk
    items = dict(conv_items(float, items))
267
    from_ = items.get('from', None)
268
    to = items.get('to', None)
269
    if from_ == None or to == None: return None
270
    return str(to - from_)
271
funcs['_range'] = _range
272
273 2016 aaronmk
def _avg(items, node):
274 86 aaronmk
    count = 0
275
    sum_ = 0.
276 278 aaronmk
    for name, value in conv_items(float, items):
277 86 aaronmk
        count += 1
278
        sum_ += value
279 1472 aaronmk
    if count == 0: return None # input is empty
280
    else: return str(sum_/count)
281 995 aaronmk
funcs['_avg'] = _avg
282 86 aaronmk
283 968 aaronmk
class CvException(Exception):
284
    def __init__(self):
285
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
286
            ' allowed for ratio scale data '
287
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
288
289 2016 aaronmk
def _noCV(items, node):
290 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
291 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
292
    except IndexError: return None # input is empty
293 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
294 968 aaronmk
    return value
295 995 aaronmk
funcs['_noCV'] = _noCV
296 968 aaronmk
297 1469 aaronmk
#### Names
298
299 328 aaronmk
_name_parts_slices_items = [
300
    ('first', slice(None, 1)),
301
    ('middle', slice(1, -1)),
302
    ('last', slice(-1, None)),
303
]
304
name_parts_slices = dict(_name_parts_slices_items)
305
name_parts = [name for name, slice_ in _name_parts_slices_items]
306
307 2016 aaronmk
def _name(items, node):
308 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
309 102 aaronmk
    parts = []
310 328 aaronmk
    for part in name_parts:
311
        if part in items: parts.append(items[part])
312 4035 aaronmk
    if not parts: return None # pass None values through; handle no name parts
313 102 aaronmk
    return ' '.join(parts)
314 995 aaronmk
funcs['_name'] = _name
315 102 aaronmk
316 1607 aaronmk
#### Angles
317
318 2016 aaronmk
def _compass(items, node):
319 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
320
    items = dict(conv_items(strings.ustr, items))
321
    try: value = items['value']
322
    except KeyError: return None # input is empty
323
324
    if not value.isupper(): return value # pass through other coordinate formats
325
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
326
    except KeyError, e: raise FormatException(e)
327
funcs['_compass'] = _compass