Project

General

Profile

1
# XML "function" nodes that transform their contents
2

    
3
import datetime
4
import operator
5
import re
6
import sre_constants
7
import warnings
8

    
9
import angles
10
import dates
11
import exc
12
import format
13
import maps
14
import sql
15
import sql_io
16
import strings
17
import term
18
import units
19
import util
20
import xml_dom
21
import xpath
22

    
23
##### Exceptions
24

    
25
class SyntaxError(exc.ExceptionWithCause):
26
    def __init__(self, cause):
27
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
28
            cause)
29

    
30
class FormatException(exc.ExceptionWithCause):
31
    def __init__(self, cause):
32
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
33

    
34
##### Helper functions
35

    
36
def map_names(func, params):
37
    return [(func(name), value) for name, value in params]
38

    
39
def variadic_args(node):
40
    args = map_names(float, xml_dom.NodeEntryIter(node))
41
    args.sort()
42
    return [value for name, value in args]
43

    
44
def map_items(func, items):
45
    return [(name, func(value)) for name, value in items]
46

    
47
def cast(type_, val):
48
    '''Throws FormatException if can't cast'''
49
    try: return type_(val)
50
    except ValueError, e: raise FormatException(e)
51

    
52
def conv_items(type_, items):
53
    return map_items(lambda val: cast(type_, val),
54
        xml_dom.TextEntryOnlyIter(items))
55

    
56
def pop_value(items, name='value'):
57
    '''@param name Name of value param, or None to accept any name'''
58
    try: last = items.pop() # last entry contains value
59
    except IndexError: return None # input is empty and no actions
60
    if name != None and last[0] != name: return None # input is empty
61
    return last[1]
62

    
63
def merge_tagged(root):
64
    '''Merges siblings in root that are marked as mergeable.
65
    Used to recombine pieces of nodes that were split apart in the mappings.
66
    '''
67
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
68
        xml_dom.merge_by_name(root, name)
69
    
70
    # Recurse
71
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
72

    
73
funcs = {}
74
simplifying_funcs = {}
75

    
76
##### Public functions
77

    
78
var_name_prefix = '$'
79

    
80
def is_var_name(str_): return str_.startswith(var_name_prefix)
81

    
82
def is_var(node):
83
    return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
84

    
85
def is_func_name(name):
86
    return name.startswith('_') and name != '_' # '_' is default root node name
87

    
88
def is_func(node): return is_func_name(node.tagName)
89

    
90
def is_xml_func_name(name): return is_func_name(name) and name in funcs
91

    
92
def is_xml_func(node): return is_xml_func_name(node.tagName)
93

    
94
def passthru(node):
95
    '''Passes through single child node. First prunes the node.'''
96
    xml_dom.prune(node)
97
    children = list(xml_dom.NodeEntryIter(node))
98
    if len(children) == 1: xml_dom.replace(node, children[0][1])
99

    
100
def simplify(node):
101
    '''Simplifies an XML tree.
102
    * Merges nodes tagged as mergable
103
    * Runs simplifying functions
104
    '''
105
    for child in xml_dom.NodeElemIter(node): simplify(child)
106
    merge_tagged(node)
107
    
108
    name = node.tagName
109
    
110
    # Pass-through optimizations
111
    if is_func_name(name):
112
        try: func = simplifying_funcs[name]
113
        except KeyError: xml_dom.prune_empty(node)
114
        else: func(node)
115
    # Pruning optimizations
116
    else: # these should not run on functions because they would remove args
117
        xml_dom.prune_children(node)
118

    
119
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
120
    '''Evaluates the XML functions in an XML tree.
121
    @param is_rel_func None|f(str) Tests if a name is a relational function.
122
        * If != None: Non-relational functions are removed, or relational
123
          functions are treated specially, depending on the db param (below).
124
    @param db
125
        * If None: Non-relational functions other than structural functions are
126
          replaced with their last parameter (usually the value), not evaluated.
127
          This is used in column-based mode to remove XML-only functions.
128
        * If != None: Relational functions are evaluated directly. This is used
129
          in row-based mode to combine relational and XML functions.
130
    '''
131
    has_rel_funcs = is_rel_func != None
132
    assert db == None or has_rel_funcs # rel_funcs required if db set
133
    
134
    for child in xml_dom.NodeElemIter(node):
135
        process(child, on_error, is_rel_func, db)
136
    merge_tagged(node)
137
    
138
    name = node.tagName
139
    if not is_func_name(name): return node # not any kind of function
140
    
141
    row_mode = has_rel_funcs and db != None
142
    column_mode = has_rel_funcs and db == None
143
    func = funcs.get(name, None)
144
    items = list(xml_dom.NodeTextEntryIter(node))
145
    
146
    # Parse function
147
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
148
        # pass-through optimization for aggregating functions with one arg
149
        value = items[0][1] # pass through first arg
150
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
151
        if reduce(operator.or_, (xml_dom.is_node(v) for n, v in items)):
152
            return # preserve complex funcs
153
        # Evaluate using DB
154
        try: value = sql_io.put(db, name, dict(items), on_error=on_error)
155
        except sql.DoesNotExistException: return # preserve unknown funcs
156
            # possibly a built-in function of db_xml.put()
157
    elif column_mode or func == None:
158
        # local XML function can't be used or does not exist
159
        if column_mode and is_rel_func(name): return # preserve relational funcs
160
        # otherwise XML-only in column mode, or DB-only in XML output mode
161
        value = pop_value(items, None) # just replace with last param
162
    else: # local XML function
163
        try: value = func(items, node)
164
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
165
            # Save in case another exception raised, overwriting sys.exc_info()
166
            exc.add_traceback(e)
167
            str_ = strings.ustr(node)
168
            exc.add_msg(e, 'function:\n'+str_)
169
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
170
                '\n'+term.emph_multiline(str_)))
171
                
172
            on_error(e)
173
            return # in case on_error() returns
174
    
175
    xml_dom.replace_with_text(node, value)
176

    
177
##### Simplifying functions
178

    
179
# Function names must start with _ to avoid collisions with real tags
180
# Functions take params (node) and have no return value
181

    
182
#### Logic
183

    
184
def _exists(node):
185
    '''Returns whether its node is non-empty'''
186
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
187
simplifying_funcs['_exists'] = _exists
188

    
189
def _if(node):
190
    '''
191
    *Must* be run to remove conditions that functions._if() can't handle.
192
    Note: Can add `@name` attr to distinguish separate _if statements.
193
    '''
194
    params = dict(xml_dom.NodeEntryIter(node))
195
    then = params.get('then', None)
196
    cond = params.get('cond', None)
197
    else_ = params.get('else', None)
198
    
199
    if cond == None: xml_dom.replace(node, else_) # always False
200
    elif then == else_: xml_dom.replace(node, then) # always same value
201
    elif is_var(cond): pass # can't simplify variable conditions
202
    elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
203
        xml_dom.replace(node, then)
204
simplifying_funcs['_if'] = _if
205

    
206
#### Merging
207

    
208
simplifying_funcs['_alt'] = passthru
209
simplifying_funcs['_join'] = passthru
210
simplifying_funcs['_join_words'] = passthru
211
simplifying_funcs['_merge'] = passthru
212
simplifying_funcs['_min'] = passthru
213
simplifying_funcs['_max'] = passthru
214

    
215
def _first(node):
216
    '''Chooses the first param (after sorting by numeric param name)'''
217
    args = variadic_args(node)
218
    try: first = args[0]
219
    except IndexError: first = None
220
    xml_dom.replace(node, first)
221
simplifying_funcs['_first'] = _first
222

    
223
##### XML functions
224

    
225
# Function names must start with _ to avoid collisions with real tags
226
# Functions take arguments (items, node)
227

    
228
#### Transforming values
229

    
230
def repl(repls, value):
231
    '''Raises error if value not in map and no special '*' entry
232
    @param repls dict repl:with
233
        repl "*" means all other input values
234
        with "*" means keep input value the same
235
        with "" means ignore input value
236
    '''
237
    try: new_value = repls[value]
238
    except KeyError, e:
239
        # Save traceback right away in case another exception raised
240
        fe = FormatException(e)
241
        try: new_value = repls['*']
242
        except KeyError: raise fe
243
    if new_value == '*': new_value = value # '*' means keep input value the same
244
    return new_value
245

    
246
def _map(items, node):
247
    '''See repl()
248
    @param items
249
        <last_entry> Value
250
        <other_entries> name=value Mappings. Special values: See repl() repls.
251
    '''
252
    value = pop_value(items)
253
    if value == None: return None # input is empty
254
    return util.none_if(repl(dict(items), value), u'') # empty value means None
255
funcs['_map'] = _map
256

    
257
def _replace(items, node):
258
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
259
    value = pop_value(items)
260
    if value == None: return None # input is empty
261
    try:
262
        for repl, with_ in items:
263
            if re.match(r'^\w+$', repl):
264
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
265
            value = re.sub(repl, with_, value)
266
    except sre_constants.error, e: raise SyntaxError(e)
267
    return util.none_if(value.strip(), u'') # empty strings always mean None
268
funcs['_replace'] = _replace
269

    
270
#### Quantities
271

    
272
def _units(items, node):
273
    value = pop_value(items)
274
    if value == None: return None # input is empty
275
    
276
    quantity = units.str2quantity(value)
277
    try:
278
        for action, units_ in items:
279
            units_ = util.none_if(units_, u'')
280
            if action == 'default': units.set_default_units(quantity, units_)
281
            elif action == 'to':
282
                try: quantity = units.convert(quantity, units_)
283
                except ValueError, e: raise FormatException(e)
284
            else: raise SyntaxError(ValueError('Invalid action: '+action))
285
    except units.MissingUnitsException, e: raise FormatException(e)
286
    return units.quantity2str(quantity)
287
funcs['_units'] = _units
288

    
289
def parse_range(str_, range_sep='-'):
290
    default = (str_, None)
291
    start, sep, end = str_.partition(range_sep)
292
    if sep == '': return default # not a range
293
    if start == '' and range_sep == '-': return default # negative number
294
    return tuple(d.strip() for d in (start, end))
295

    
296
def _rangeStart(items, node):
297
    items = dict(conv_items(strings.ustr, items))
298
    try: value = items['value']
299
    except KeyError: return None # input is empty
300
    return parse_range(value)[0]
301
funcs['_rangeStart'] = _rangeStart
302

    
303
def _rangeEnd(items, node):
304
    items = dict(conv_items(strings.ustr, items))
305
    try: value = items['value']
306
    except KeyError: return None # input is empty
307
    return parse_range(value)[1]
308
funcs['_rangeEnd'] = _rangeEnd
309

    
310
def _range(items, node):
311
    items = dict(conv_items(float, items))
312
    from_ = items.get('from', None)
313
    to = items.get('to', None)
314
    if from_ == None or to == None: return None
315
    return str(to - from_)
316
funcs['_range'] = _range
317

    
318
def _avg(items, node):
319
    count = 0
320
    sum_ = 0.
321
    for name, value in conv_items(float, items):
322
        count += 1
323
        sum_ += value
324
    if count == 0: return None # input is empty
325
    else: return str(sum_/count)
326
funcs['_avg'] = _avg
327

    
328
class CvException(Exception):
329
    def __init__(self):
330
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
331
            ' allowed for ratio scale data '
332
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
333

    
334
def _noCV(items, node):
335
    items = list(conv_items(strings.ustr, items))
336
    try: name, value = items.pop() # last entry contains value
337
    except IndexError: return None # input is empty
338
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
339
    return value
340
funcs['_noCV'] = _noCV
341

    
342
#### Angles
343

    
344
def _compass(items, node):
345
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
346
    items = dict(conv_items(strings.ustr, items))
347
    try: value = items['value']
348
    except KeyError: return None # input is empty
349
    
350
    if not value.isupper(): return value # pass through other coordinate formats
351
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
352
    except KeyError, e: raise FormatException(e)
353
funcs['_compass'] = _compass
(39-39/42)