Project

General

Profile

1
# XML "function" nodes that transform their contents
2

    
3
import datetime
4
import operator
5
import os
6
import re
7
import sre_constants
8
import warnings
9

    
10
import angles
11
import dates
12
import exc
13
import format
14
import maps
15
import scalar
16
import sql
17
import sql_io
18
import strings
19
import term
20
import units
21
import util
22
import xml_dom
23
import xpath
24

    
25
##### Exceptions
26

    
27
class SyntaxError(exc.ExceptionWithCause):
28
    def __init__(self, cause):
29
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
30
            cause)
31

    
32
class FormatException(exc.ExceptionWithCause):
33
    def __init__(self, cause):
34
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
35

    
36
##### Helper functions
37

    
38
def map_names(func, params):
39
    return [(func(name), value) for name, value in params]
40

    
41
def variadic_args(node):
42
    args = map_names(float, xml_dom.NodeEntryIter(node))
43
    args.sort()
44
    return [value for name, value in args]
45

    
46
def map_items(func, items):
47
    return [(name, func(value)) for name, value in items]
48

    
49
def cast(type_, val):
50
    '''Throws FormatException if can't cast'''
51
    try: return type_(val)
52
    except ValueError, e: raise FormatException(e)
53

    
54
def conv_items(type_, items):
55
    return map_items(lambda val: cast(type_, val),
56
        xml_dom.TextEntryOnlyIter(items))
57

    
58
def pop_value(items, name='value'):
59
    '''@param name Name of value param, or None to accept any name'''
60
    try: last = items.pop() # last entry contains value
61
    except IndexError: return None # input is empty and no actions
62
    if name != None and last[0] != name: return None # input is empty
63
    return last[1]
64

    
65
def merge_tagged(root):
66
    '''Merges siblings in root that are marked as mergeable.
67
    Used to recombine pieces of nodes that were split apart in the mappings.
68
    '''
69
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
70
        xml_dom.merge_by_name(root, name)
71
    
72
    # Recurse
73
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
74

    
75
funcs = {}
76
simplifying_funcs = {}
77

    
78
##### Public functions
79

    
80
var_name_prefix = '$'
81

    
82
def is_var_name(str_): return str_.startswith(var_name_prefix)
83

    
84
def is_var(node):
85
    return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
86

    
87
def is_func_name(name):
88
    return name.startswith('_') and name != '_' # '_' is default root node name
89

    
90
def is_func(node): return is_func_name(node.tagName)
91

    
92
def is_xml_func_name(name): return is_func_name(name) and name in funcs
93

    
94
def is_xml_func(node): return is_xml_func_name(node.tagName)
95

    
96
def is_scalar(value): return scalar.is_scalar(value) and not is_var_name(value)
97

    
98
def passthru(node):
99
    '''Passes through single child node. First prunes the node.'''
100
    xml_dom.prune(node)
101
    children = list(xml_dom.NodeEntryIter(node))
102
    if len(children) == 1: xml_dom.replace(node, children[0][1])
103

    
104
def simplify(node):
105
    '''Simplifies an XML tree.
106
    * Merges nodes tagged as mergable
107
    * Runs simplifying functions
108
    '''
109
    for child in xml_dom.NodeElemIter(node): simplify(child)
110
    merge_tagged(node)
111
    
112
    name = node.tagName
113
    
114
    # Pass-through optimizations
115
    if is_func_name(name):
116
        try: func = simplifying_funcs[name]
117
        except KeyError: xml_dom.prune_empty(node)
118
        else: func(node)
119
    # Pruning optimizations
120
    else: # these should not run on functions because they would remove args
121
        xml_dom.prune_children(node)
122

    
123
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
124
    '''Evaluates the XML functions in an XML tree.
125
    @param is_rel_func None|f(str) Tests if a name is a relational function.
126
        * If != None: Non-relational functions are removed, or relational
127
          functions are treated specially, depending on the db param (below).
128
    @param db
129
        * If None: Non-relational functions other than structural functions are
130
          replaced with their last parameter (usually the value), not evaluated.
131
          This is used in column-based mode to remove XML-only functions.
132
        * If != None: Relational functions are evaluated directly. This is used
133
          in row-based mode to combine relational and XML functions.
134
    '''
135
    has_rel_funcs = is_rel_func != None
136
    assert db == None or has_rel_funcs # rel_funcs required if db set
137
    
138
    for child in xml_dom.NodeElemIter(node):
139
        process(child, on_error, is_rel_func, db)
140
    merge_tagged(node)
141
    
142
    name = node.tagName
143
    if not is_func_name(name): return node # not any kind of function
144
    
145
    row_mode = has_rel_funcs and db != None
146
    column_mode = has_rel_funcs and db == None
147
    func = funcs.get(name, None)
148
    items = list(xml_dom.NodeTextEntryIter(node))
149
    
150
    # Parse function
151
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
152
        # pass-through optimization for aggregating functions with one arg
153
        value = items[0][1] # pass through first arg
154
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
155
        if items and reduce(operator.or_, (xml_dom.is_node(v)
156
            for n, v in items)): return # preserve complex funcs
157
        # Evaluate using DB
158
        try: value = sql_io.put(db, name, dict(items), on_error=on_error)
159
        except sql.DoesNotExistException: return # preserve unknown funcs
160
            # possibly a built-in function of db_xml.put()
161
    elif column_mode or func == None:
162
        # local XML function can't be used or does not exist
163
        if column_mode and is_rel_func(name): return # preserve relational funcs
164
        # otherwise XML-only in column mode, or DB-only in XML output mode
165
        value = pop_value(items, None) # just replace with last param
166
    else: # local XML function
167
        try: value = func(items, node)
168
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
169
            # Save in case another exception raised, overwriting sys.exc_info()
170
            exc.add_traceback(e)
171
            str_ = strings.ustr(node)
172
            exc.add_msg(e, 'function:\n'+str_)
173
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
174
                '\n'+term.emph_multiline(str_)))
175
                
176
            on_error(e)
177
            return # in case on_error() returns
178
    
179
    xml_dom.replace_with_text(node, value)
180

    
181
##### Simplifying functions
182

    
183
# Function names must start with _ to avoid collisions with real tags
184
# Functions take params (node) and have no return value
185

    
186
#### Logic
187

    
188
simplifying_funcs['_and'] = passthru
189
simplifying_funcs['_or'] = passthru
190

    
191
def _exists(node):
192
    '''Returns whether its node is non-empty'''
193
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
194
simplifying_funcs['_exists'] = _exists
195

    
196
def _if(node):
197
    '''
198
    *Must* be run to remove conditions that functions._if() can't handle.
199
    Note: Can add `@name` attr to distinguish separate _if statements.
200
    '''
201
    params = dict(xml_dom.NodeEntryIter(node))
202
    then = params.get('then', None)
203
    cond = params.get('cond', None)
204
    else_ = params.get('else', None)
205
    
206
    if cond == None: xml_dom.replace(node, else_) # always False
207
    elif then == else_: xml_dom.replace(node, then) # always same value
208
    elif is_var(cond): pass # can't simplify variable conditions
209
    elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
210
        xml_dom.replace(node, then)
211
simplifying_funcs['_if'] = _if
212

    
213
def _nullIf(node):
214
    '''
215
    *Must* be run to remove conditions that functions._nullIf() can't handle.
216
    '''
217
    params = dict(xml_dom.NodeEntryIter(node))
218
    null = params.get('null', None)
219
    value = params.get('value', None)
220
    
221
    if value == None: xml_dom.prune_parent(node) # empty
222
    elif null == None: xml_dom.replace(node, value) # nothing to null out
223
simplifying_funcs['_nullIf'] = _nullIf
224

    
225
#### Comparison
226

    
227
def _eq(node):
228
    params = dict(xml_dom.NodeTextEntryIter(node))
229
    left = params.get('left', None)
230
    right = params.get('right', None)
231
    
232
    if is_scalar(left) and is_scalar(right): # constant
233
        xml_dom.replace_with_text(node, left == right)
234
    elif left == right: xml_dom.replace_with_text(node, True) # always True
235
simplifying_funcs['_eq'] = _eq
236

    
237
#### Merging
238

    
239
simplifying_funcs['_alt'] = passthru
240
simplifying_funcs['_join'] = passthru
241
simplifying_funcs['_join_words'] = passthru
242
simplifying_funcs['_merge'] = passthru
243
simplifying_funcs['_min'] = passthru
244
simplifying_funcs['_max'] = passthru
245

    
246
def _first(node):
247
    '''Chooses the first param (after sorting by numeric param name)'''
248
    args = variadic_args(node)
249
    try: first = args[0]
250
    except IndexError: first = None
251
    xml_dom.replace(node, first)
252
simplifying_funcs['_first'] = _first
253

    
254
#### Environment access
255

    
256
def _env(node):
257
    params = dict(xml_dom.NodeTextEntryIter(node))
258
    try: name = params['name']
259
    except KeyError, e: raise SyntaxError(e)
260
    
261
    xml_dom.replace_with_text(node, os.environ[name])
262
simplifying_funcs['_env'] = _env
263

    
264
##### XML functions
265

    
266
# Function names must start with _ to avoid collisions with real tags
267
# Functions take arguments (items, node)
268

    
269
#### Transforming values
270

    
271
def _replace(items, node):
272
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
273
    value = pop_value(items)
274
    if value == None: return None # input is empty
275
    try:
276
        for repl, with_ in items:
277
            if re.match(r'^\w+$', repl):
278
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
279
            value = re.sub(repl, with_, value)
280
    except sre_constants.error, e: raise SyntaxError(e)
281
    return util.none_if(value.strip(), u'') # empty strings always mean None
282
funcs['_replace'] = _replace
283

    
284
#### Quantities
285

    
286
def _units(items, node):
287
    value = pop_value(items)
288
    if value == None: return None # input is empty
289
    
290
    quantity = units.str2quantity(value)
291
    try:
292
        for action, units_ in items:
293
            units_ = util.none_if(units_, u'')
294
            if action == 'default': units.set_default_units(quantity, units_)
295
            elif action == 'to':
296
                try: quantity = units.convert(quantity, units_)
297
                except ValueError, e: raise FormatException(e)
298
            else: raise SyntaxError(ValueError('Invalid action: '+action))
299
    except units.MissingUnitsException, e: raise FormatException(e)
300
    return units.quantity2str(quantity)
301
funcs['_units'] = _units
302

    
303
def parse_range(str_, range_sep='-'):
304
    default = (str_, None)
305
    start, sep, end = str_.partition(range_sep)
306
    if sep == '': return default # not a range
307
    if start == '' and range_sep == '-': return default # negative number
308
    return tuple(d.strip() for d in (start, end))
309

    
310
def _rangeStart(items, node):
311
    items = dict(conv_items(strings.ustr, items))
312
    try: value = items['value']
313
    except KeyError: return None # input is empty
314
    return parse_range(value)[0]
315
funcs['_rangeStart'] = _rangeStart
316

    
317
def _rangeEnd(items, node):
318
    items = dict(conv_items(strings.ustr, items))
319
    try: value = items['value']
320
    except KeyError: return None # input is empty
321
    return parse_range(value)[1]
322
funcs['_rangeEnd'] = _rangeEnd
323

    
324
def _range(items, node):
325
    items = dict(conv_items(float, items))
326
    from_ = items.get('from', None)
327
    to = items.get('to', None)
328
    if from_ == None or to == None: return None
329
    return str(to - from_)
330
funcs['_range'] = _range
331

    
332
def _avg(items, node):
333
    count = 0
334
    sum_ = 0.
335
    for name, value in conv_items(float, items):
336
        count += 1
337
        sum_ += value
338
    if count == 0: return None # input is empty
339
    else: return str(sum_/count)
340
funcs['_avg'] = _avg
341

    
342
class CvException(Exception):
343
    def __init__(self):
344
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
345
            ' allowed for ratio scale data '
346
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
347

    
348
def _noCV(items, node):
349
    items = list(conv_items(strings.ustr, items))
350
    try: name, value = items.pop() # last entry contains value
351
    except IndexError: return None # input is empty
352
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
353
    return value
354
funcs['_noCV'] = _noCV
355

    
356
#### Angles
357

    
358
def _compass(items, node):
359
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
360
    items = dict(conv_items(strings.ustr, items))
361
    try: value = items['value']
362
    except KeyError: return None # input is empty
363
    
364
    if not value.isupper(): return value # pass through other coordinate formats
365
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
366
    except KeyError, e: raise FormatException(e)
367
funcs['_compass'] = _compass
(42-42/45)