Project

General

Profile

1 996 aaronmk
# XML "function" nodes that transform their contents
2 86 aaronmk
3 111 aaronmk
import datetime
4 968 aaronmk
import re
5 1219 aaronmk
import sre_constants
6 2017 aaronmk
import warnings
7 111 aaronmk
8 1607 aaronmk
import angles
9 818 aaronmk
import dates
10 300 aaronmk
import exc
11 1580 aaronmk
import format
12 917 aaronmk
import maps
13 3688 aaronmk
import sql
14 3077 aaronmk
import sql_io
15 1234 aaronmk
import strings
16 827 aaronmk
import term
17 1468 aaronmk
import units
18 1047 aaronmk
import util
19 86 aaronmk
import xml_dom
20 1321 aaronmk
import xpath
21 86 aaronmk
22 995 aaronmk
##### Exceptions
23
24 1612 aaronmk
class SyntaxError(exc.ExceptionWithCause):
25 797 aaronmk
    def __init__(self, cause):
26 1611 aaronmk
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28 278 aaronmk
29 1613 aaronmk
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32 843 aaronmk
33 1992 aaronmk
##### Helper functions
34 995 aaronmk
35 1992 aaronmk
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53
54 3335 aaronmk
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63
64 995 aaronmk
funcs = {}
65
66 1992 aaronmk
##### Public functions
67
68 2112 aaronmk
def is_func_name(name):
69
    return name.startswith('_') and name != '_' # '_' is default root node name
70
71
def is_func(node): return is_func_name(node.tagName)
72
73
def is_xml_func_name(name): return is_func_name(name) and name in funcs
74
75
def is_xml_func(node): return is_xml_func_name(node.tagName)
76
77 4041 aaronmk
def simplify(node):
78
    '''Simplifies the XML functions in an XML tree.
79
    * Merges nodes tagged as mergable
80 4078 aaronmk
    * Applies pass-through optimizations for XML functions
81 4041 aaronmk
    '''
82
    for child in xml_dom.NodeElemIter(node): simplify(child)
83
    merge_tagged(node)
84
85 4227 aaronmk
    name = node.tagName
86 4041 aaronmk
87 4078 aaronmk
    # Pass-through optimizations
88 4228 aaronmk
    if is_func_name(name):
89
        children = list(xml_dom.NodeElemIter(node))
90
        if len(children) == 1: # single arg function
91
            func = name
92
            param_node = children[0]
93
            param = param_node.tagName
94
            value = param_node.firstChild
95
96
            is_agg_func = param.isdigit() or func == '_name'
97 4230 aaronmk
            if is_agg_func or (func == '_if' and param == 'else'):
98 4228 aaronmk
                xml_dom.replace(node, value)
99 4230 aaronmk
            elif func == '_if': xml_dom.remove(node) # only a cond or then arg
100 4229 aaronmk
    # Pruning optimizations
101
    else: # these should not run on functions because they would remove args
102
        # Remove empty children
103
        for child in xml_dom.NodeElemIter(node):
104
            if xml_dom.is_empty(child): xml_dom.remove(child)
105 4041 aaronmk
106 3660 aaronmk
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
107 2597 aaronmk
    '''Evaluates the XML functions in an XML tree.
108 3424 aaronmk
    @param is_rel_func None|f(str) Tests if a name is a relational function.
109 2602 aaronmk
        * If != None: Non-relational functions are removed, or relational
110
          functions are treated specially, depending on the db param (below).
111
    @param db
112
        * If None: Non-relational functions other than structural functions are
113
          replaced with their last parameter (usually the value), not evaluated.
114
          This is used in column-based mode to remove XML-only functions.
115
        * If != None: Relational functions are evaluated directly. This is used
116
          in row-based mode to combine relational and XML functions.
117 2597 aaronmk
    '''
118 3424 aaronmk
    has_rel_funcs = is_rel_func != None
119 2602 aaronmk
    assert db == None or has_rel_funcs # rel_funcs required if db set
120 2597 aaronmk
121 3333 aaronmk
    for child in xml_dom.NodeElemIter(node):
122 3424 aaronmk
        process(child, on_error, is_rel_func, db)
123 3335 aaronmk
    merge_tagged(node)
124 3333 aaronmk
125 995 aaronmk
    name = node.tagName
126 3227 aaronmk
    if not is_func_name(name): return node # not any kind of function
127 2602 aaronmk
128
    row_mode = has_rel_funcs and db != None
129
    column_mode = has_rel_funcs and db == None
130 3629 aaronmk
    func = funcs.get(name, None)
131 3028 aaronmk
    items = list(xml_dom.NodeTextEntryIter(node))
132 2602 aaronmk
133 3029 aaronmk
    # Parse function
134
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
135
        # pass-through optimization for aggregating functions with one arg
136
        value = items[0][1] # pass through first arg
137 3629 aaronmk
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
138 3688 aaronmk
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
139
        except sql.DoesNotExistException: return # preserve unknown funcs
140
            # possibly a built-in function of db_xml.put()
141 4024 aaronmk
    elif column_mode or func == None:
142 3640 aaronmk
        # local XML function can't be used or does not exist
143
        if column_mode and is_rel_func(name): return # preserve relational funcs
144
        # otherwise XML-only in column mode, or DB-only in XML output mode
145
        value = pop_value(items, None) # just replace with last param
146 2602 aaronmk
    else: # local XML function
147 3629 aaronmk
        try: value = func(items, node)
148 1613 aaronmk
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
149 1371 aaronmk
            # Save in case another exception raised, overwriting sys.exc_info()
150
            exc.add_traceback(e)
151 1562 aaronmk
            str_ = strings.ustr(node)
152 995 aaronmk
            exc.add_msg(e, 'function:\n'+str_)
153 1810 aaronmk
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
154
                '\n'+term.emph_multiline(str_)))
155
156 995 aaronmk
            on_error(e)
157 2602 aaronmk
            return # in case on_error() returns
158 3227 aaronmk
159 3333 aaronmk
    xml_dom.replace_with_text(node, value)
160 995 aaronmk
161 1469 aaronmk
##### XML functions
162 995 aaronmk
163
# Function names must start with _ to avoid collisions with real tags
164 4144 aaronmk
# Functions take arguments (items, node)
165 995 aaronmk
166 1469 aaronmk
#### Transforming values
167
168 1602 aaronmk
def repl(repls, value):
169 1537 aaronmk
    '''Raises error if value not in map and no special '*' entry
170 1602 aaronmk
    @param repls dict repl:with
171
        repl "*" means all other input values
172
        with "*" means keep input value the same
173
        with "" means ignore input value
174 1537 aaronmk
    '''
175 1602 aaronmk
    try: new_value = repls[value]
176 1304 aaronmk
    except KeyError, e:
177 1537 aaronmk
        # Save traceback right away in case another exception raised
178 2984 aaronmk
        fe = FormatException(e)
179 1602 aaronmk
        try: new_value = repls['*']
180 1609 aaronmk
        except KeyError: raise fe
181 1537 aaronmk
    if new_value == '*': new_value = value # '*' means keep input value the same
182 1607 aaronmk
    return new_value
183 1602 aaronmk
184 2016 aaronmk
def _map(items, node):
185 1602 aaronmk
    '''See repl()
186
    @param items
187
        <last_entry> Value
188
        <other_entries> name=value Mappings. Special values: See repl() repls.
189
    '''
190
    value = pop_value(items)
191
    if value == None: return None # input is empty
192 1607 aaronmk
    return util.none_if(repl(dict(items), value), u'') # empty value means None
193 1219 aaronmk
funcs['_map'] = _map
194
195 2016 aaronmk
def _replace(items, node):
196 1562 aaronmk
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
197 1581 aaronmk
    value = pop_value(items)
198
    if value == None: return None # input is empty
199 1219 aaronmk
    try:
200
        for repl, with_ in items:
201
            if re.match(r'^\w+$', repl):
202
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
203
            value = re.sub(repl, with_, value)
204 1612 aaronmk
    except sre_constants.error, e: raise SyntaxError(e)
205 1624 aaronmk
    return util.none_if(value.strip(), u'') # empty strings always mean None
206 1219 aaronmk
funcs['_replace'] = _replace
207
208 1469 aaronmk
#### Quantities
209
210 2016 aaronmk
def _units(items, node):
211 1581 aaronmk
    value = pop_value(items)
212
    if value == None: return None # input is empty
213 1471 aaronmk
214 1581 aaronmk
    quantity = units.str2quantity(value)
215 1471 aaronmk
    try:
216
        for action, units_ in items:
217
            units_ = util.none_if(units_, u'')
218
            if action == 'default': units.set_default_units(quantity, units_)
219 1567 aaronmk
            elif action == 'to':
220
                try: quantity = units.convert(quantity, units_)
221 1609 aaronmk
                except ValueError, e: raise FormatException(e)
222 1612 aaronmk
            else: raise SyntaxError(ValueError('Invalid action: '+action))
223 1609 aaronmk
    except units.MissingUnitsException, e: raise FormatException(e)
224 1471 aaronmk
    return units.quantity2str(quantity)
225 1225 aaronmk
funcs['_units'] = _units
226
227 1399 aaronmk
def parse_range(str_, range_sep='-'):
228
    default = (str_, None)
229
    start, sep, end = str_.partition(range_sep)
230
    if sep == '': return default # not a range
231 1427 aaronmk
    if start == '' and range_sep == '-': return default # negative number
232 1399 aaronmk
    return tuple(d.strip() for d in (start, end))
233
234 2016 aaronmk
def _rangeStart(items, node):
235 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
236 1399 aaronmk
    try: value = items['value']
237 1406 aaronmk
    except KeyError: return None # input is empty
238 1399 aaronmk
    return parse_range(value)[0]
239
funcs['_rangeStart'] = _rangeStart
240
241 2016 aaronmk
def _rangeEnd(items, node):
242 1562 aaronmk
    items = dict(conv_items(strings.ustr, items))
243 1399 aaronmk
    try: value = items['value']
244 1406 aaronmk
    except KeyError: return None # input is empty
245 1399 aaronmk
    return parse_range(value)[1]
246
funcs['_rangeEnd'] = _rangeEnd
247
248 2016 aaronmk
def _range(items, node):
249 1472 aaronmk
    items = dict(conv_items(float, items))
250
    from_ = items.get('from', None)
251
    to = items.get('to', None)
252
    if from_ == None or to == None: return None
253
    return str(to - from_)
254
funcs['_range'] = _range
255
256 2016 aaronmk
def _avg(items, node):
257 86 aaronmk
    count = 0
258
    sum_ = 0.
259 278 aaronmk
    for name, value in conv_items(float, items):
260 86 aaronmk
        count += 1
261
        sum_ += value
262 1472 aaronmk
    if count == 0: return None # input is empty
263
    else: return str(sum_/count)
264 995 aaronmk
funcs['_avg'] = _avg
265 86 aaronmk
266 968 aaronmk
class CvException(Exception):
267
    def __init__(self):
268
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
269
            ' allowed for ratio scale data '
270
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
271
272 2016 aaronmk
def _noCV(items, node):
273 3631 aaronmk
    items = list(conv_items(strings.ustr, items))
274 3046 aaronmk
    try: name, value = items.pop() # last entry contains value
275
    except IndexError: return None # input is empty
276 1609 aaronmk
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
277 968 aaronmk
    return value
278 995 aaronmk
funcs['_noCV'] = _noCV
279 968 aaronmk
280 1469 aaronmk
#### Names
281
282 328 aaronmk
_name_parts_slices_items = [
283
    ('first', slice(None, 1)),
284
    ('middle', slice(1, -1)),
285
    ('last', slice(-1, None)),
286
]
287
name_parts_slices = dict(_name_parts_slices_items)
288
name_parts = [name for name, slice_ in _name_parts_slices_items]
289
290 2016 aaronmk
def _name(items, node):
291 3631 aaronmk
    items = dict(list(conv_items(strings.ustr, items)))
292 102 aaronmk
    parts = []
293 328 aaronmk
    for part in name_parts:
294
        if part in items: parts.append(items[part])
295 4035 aaronmk
    if not parts: return None # pass None values through; handle no name parts
296 102 aaronmk
    return ' '.join(parts)
297 995 aaronmk
funcs['_name'] = _name
298 102 aaronmk
299 1607 aaronmk
#### Angles
300
301 2016 aaronmk
def _compass(items, node):
302 1607 aaronmk
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
303
    items = dict(conv_items(strings.ustr, items))
304
    try: value = items['value']
305
    except KeyError: return None # input is empty
306
307
    if not value.isupper(): return value # pass through other coordinate formats
308
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
309
    except KeyError, e: raise FormatException(e)
310
funcs['_compass'] = _compass