Project

General

Profile

1
# XML "function" nodes that transform their contents
2

    
3
import datetime
4
import re
5
import sre_constants
6
import warnings
7

    
8
import angles
9
import dates
10
import exc
11
import format
12
import maps
13
import sql
14
import sql_io
15
import strings
16
import term
17
import units
18
import util
19
import xml_dom
20
import xpath
21

    
22
##### Exceptions
23

    
24
class SyntaxError(exc.ExceptionWithCause):
25
    def __init__(self, cause):
26
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28

    
29
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32

    
33
##### Helper functions
34

    
35
def map_names(func, params):
36
    return [(func(name), value) for name, value in params]
37

    
38
def variadic_args(node):
39
    args = map_names(float, xml_dom.NodeEntryIter(node))
40
    args.sort()
41
    return [value for name, value in args]
42

    
43
def map_items(func, items):
44
    return [(name, func(value)) for name, value in items]
45

    
46
def cast(type_, val):
47
    '''Throws FormatException if can't cast'''
48
    try: return type_(val)
49
    except ValueError, e: raise FormatException(e)
50

    
51
def conv_items(type_, items):
52
    return map_items(lambda val: cast(type_, val),
53
        xml_dom.TextEntryOnlyIter(items))
54

    
55
def pop_value(items, name='value'):
56
    '''@param name Name of value param, or None to accept any name'''
57
    try: last = items.pop() # last entry contains value
58
    except IndexError: return None # input is empty and no actions
59
    if name != None and last[0] != name: return None # input is empty
60
    return last[1]
61

    
62
def merge_tagged(root):
63
    '''Merges siblings in root that are marked as mergeable.
64
    Used to recombine pieces of nodes that were split apart in the mappings.
65
    '''
66
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
67
        xml_dom.merge_by_name(root, name)
68
    
69
    # Recurse
70
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
71

    
72
funcs = {}
73
simplifying_funcs = {}
74

    
75
##### Public functions
76

    
77
var_name_prefix = '$'
78

    
79
def is_var_name(str_): return str_.startswith(var_name_prefix)
80

    
81
def is_var(node):
82
    return xml_dom.is_text_node(node) and is_var_name(xml_dom.value(node))
83

    
84
def is_func_name(name):
85
    return name.startswith('_') and name != '_' # '_' is default root node name
86

    
87
def is_func(node): return is_func_name(node.tagName)
88

    
89
def is_xml_func_name(name): return is_func_name(name) and name in funcs
90

    
91
def is_xml_func(node): return is_xml_func_name(node.tagName)
92

    
93
def passthru(node):
94
    '''Passes through single child node. First prunes the node.'''
95
    xml_dom.prune(node)
96
    children = list(xml_dom.NodeEntryIter(node))
97
    if len(children) == 1: xml_dom.replace(node, children[0][1])
98

    
99
def simplify(node):
100
    '''Simplifies an XML tree.
101
    * Merges nodes tagged as mergable
102
    * Runs simplifying functions
103
    '''
104
    for child in xml_dom.NodeElemIter(node): simplify(child)
105
    merge_tagged(node)
106
    
107
    name = node.tagName
108
    
109
    # Pass-through optimizations
110
    if is_func_name(name):
111
        try: func = simplifying_funcs[name]
112
        except KeyError: xml_dom.prune_empty(node)
113
        else: func(node)
114
    # Pruning optimizations
115
    else: # these should not run on functions because they would remove args
116
        xml_dom.prune_children(node)
117

    
118
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
119
    '''Evaluates the XML functions in an XML tree.
120
    @param is_rel_func None|f(str) Tests if a name is a relational function.
121
        * If != None: Non-relational functions are removed, or relational
122
          functions are treated specially, depending on the db param (below).
123
    @param db
124
        * If None: Non-relational functions other than structural functions are
125
          replaced with their last parameter (usually the value), not evaluated.
126
          This is used in column-based mode to remove XML-only functions.
127
        * If != None: Relational functions are evaluated directly. This is used
128
          in row-based mode to combine relational and XML functions.
129
    '''
130
    has_rel_funcs = is_rel_func != None
131
    assert db == None or has_rel_funcs # rel_funcs required if db set
132
    
133
    for child in xml_dom.NodeElemIter(node):
134
        process(child, on_error, is_rel_func, db)
135
    merge_tagged(node)
136
    
137
    name = node.tagName
138
    if not is_func_name(name): return node # not any kind of function
139
    
140
    row_mode = has_rel_funcs and db != None
141
    column_mode = has_rel_funcs and db == None
142
    func = funcs.get(name, None)
143
    items = list(xml_dom.NodeTextEntryIter(node))
144
    
145
    # Parse function
146
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
147
        # pass-through optimization for aggregating functions with one arg
148
        value = items[0][1] # pass through first arg
149
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
150
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
151
        except sql.DoesNotExistException: return # preserve unknown funcs
152
            # possibly a built-in function of db_xml.put()
153
    elif column_mode or func == None:
154
        # local XML function can't be used or does not exist
155
        if column_mode and is_rel_func(name): return # preserve relational funcs
156
        # otherwise XML-only in column mode, or DB-only in XML output mode
157
        value = pop_value(items, None) # just replace with last param
158
    else: # local XML function
159
        try: value = func(items, node)
160
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
161
            # Save in case another exception raised, overwriting sys.exc_info()
162
            exc.add_traceback(e)
163
            str_ = strings.ustr(node)
164
            exc.add_msg(e, 'function:\n'+str_)
165
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
166
                '\n'+term.emph_multiline(str_)))
167
                
168
            on_error(e)
169
            return # in case on_error() returns
170
    
171
    xml_dom.replace_with_text(node, value)
172

    
173
##### Simplifying functions
174

    
175
# Function names must start with _ to avoid collisions with real tags
176
# Functions take params (node) and have no return value
177

    
178
#### Logic
179

    
180
def _exists(node):
181
    '''Returns whether its node is non-empty'''
182
    xml_dom.replace_with_text(node, not xml_dom.is_empty(node))
183
simplifying_funcs['_exists'] = _exists
184

    
185
def _if(node):
186
    '''
187
    *Must* be run to remove conditions that functions._if() can't handle.
188
    Note: Can add `@name` attr to distinguish separate _if statements.
189
    '''
190
    params = dict(xml_dom.NodeEntryIter(node))
191
    then = params.get('then', None)
192
    cond = params.get('cond', None)
193
    else_ = params.get('else', None)
194
    
195
    if cond == None: xml_dom.replace(node, else_) # always False
196
    elif then == else_: xml_dom.replace(node, then) # always same value
197
    elif is_var(cond): pass # can't simplify variable conditions
198
    elif xml_dom.is_text_node(cond) and bool(xml_dom.value(cond)): # always True
199
        xml_dom.replace(node, then)
200
simplifying_funcs['_if'] = _if
201

    
202
#### Merging
203

    
204
simplifying_funcs['_alt'] = passthru
205
simplifying_funcs['_join'] = passthru
206
simplifying_funcs['_merge'] = passthru
207
simplifying_funcs['_name'] = passthru
208

    
209
def _first(node):
210
    '''Chooses the first param (after sorting by numeric param name)'''
211
    args = variadic_args(node)
212
    try: first = args[0]
213
    except IndexError: first = None
214
    xml_dom.replace(node, first)
215
simplifying_funcs['_first'] = _first
216

    
217
##### XML functions
218

    
219
# Function names must start with _ to avoid collisions with real tags
220
# Functions take arguments (items, node)
221

    
222
#### Transforming values
223

    
224
def repl(repls, value):
225
    '''Raises error if value not in map and no special '*' entry
226
    @param repls dict repl:with
227
        repl "*" means all other input values
228
        with "*" means keep input value the same
229
        with "" means ignore input value
230
    '''
231
    try: new_value = repls[value]
232
    except KeyError, e:
233
        # Save traceback right away in case another exception raised
234
        fe = FormatException(e)
235
        try: new_value = repls['*']
236
        except KeyError: raise fe
237
    if new_value == '*': new_value = value # '*' means keep input value the same
238
    return new_value
239

    
240
def _map(items, node):
241
    '''See repl()
242
    @param items
243
        <last_entry> Value
244
        <other_entries> name=value Mappings. Special values: See repl() repls.
245
    '''
246
    value = pop_value(items)
247
    if value == None: return None # input is empty
248
    return util.none_if(repl(dict(items), value), u'') # empty value means None
249
funcs['_map'] = _map
250

    
251
def _replace(items, node):
252
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
253
    value = pop_value(items)
254
    if value == None: return None # input is empty
255
    try:
256
        for repl, with_ in items:
257
            if re.match(r'^\w+$', repl):
258
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
259
            value = re.sub(repl, with_, value)
260
    except sre_constants.error, e: raise SyntaxError(e)
261
    return util.none_if(value.strip(), u'') # empty strings always mean None
262
funcs['_replace'] = _replace
263

    
264
#### Quantities
265

    
266
def _units(items, node):
267
    value = pop_value(items)
268
    if value == None: return None # input is empty
269
    
270
    quantity = units.str2quantity(value)
271
    try:
272
        for action, units_ in items:
273
            units_ = util.none_if(units_, u'')
274
            if action == 'default': units.set_default_units(quantity, units_)
275
            elif action == 'to':
276
                try: quantity = units.convert(quantity, units_)
277
                except ValueError, e: raise FormatException(e)
278
            else: raise SyntaxError(ValueError('Invalid action: '+action))
279
    except units.MissingUnitsException, e: raise FormatException(e)
280
    return units.quantity2str(quantity)
281
funcs['_units'] = _units
282

    
283
def parse_range(str_, range_sep='-'):
284
    default = (str_, None)
285
    start, sep, end = str_.partition(range_sep)
286
    if sep == '': return default # not a range
287
    if start == '' and range_sep == '-': return default # negative number
288
    return tuple(d.strip() for d in (start, end))
289

    
290
def _rangeStart(items, node):
291
    items = dict(conv_items(strings.ustr, items))
292
    try: value = items['value']
293
    except KeyError: return None # input is empty
294
    return parse_range(value)[0]
295
funcs['_rangeStart'] = _rangeStart
296

    
297
def _rangeEnd(items, node):
298
    items = dict(conv_items(strings.ustr, items))
299
    try: value = items['value']
300
    except KeyError: return None # input is empty
301
    return parse_range(value)[1]
302
funcs['_rangeEnd'] = _rangeEnd
303

    
304
def _range(items, node):
305
    items = dict(conv_items(float, items))
306
    from_ = items.get('from', None)
307
    to = items.get('to', None)
308
    if from_ == None or to == None: return None
309
    return str(to - from_)
310
funcs['_range'] = _range
311

    
312
def _avg(items, node):
313
    count = 0
314
    sum_ = 0.
315
    for name, value in conv_items(float, items):
316
        count += 1
317
        sum_ += value
318
    if count == 0: return None # input is empty
319
    else: return str(sum_/count)
320
funcs['_avg'] = _avg
321

    
322
class CvException(Exception):
323
    def __init__(self):
324
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
325
            ' allowed for ratio scale data '
326
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
327

    
328
def _noCV(items, node):
329
    items = list(conv_items(strings.ustr, items))
330
    try: name, value = items.pop() # last entry contains value
331
    except IndexError: return None # input is empty
332
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
333
    return value
334
funcs['_noCV'] = _noCV
335

    
336
#### Names
337

    
338
_name_parts_slices_items = [
339
    ('first', slice(None, 1)),
340
    ('middle', slice(1, -1)),
341
    ('last', slice(-1, None)),
342
]
343
name_parts_slices = dict(_name_parts_slices_items)
344
name_parts = [name for name, slice_ in _name_parts_slices_items]
345

    
346
def _name(items, node):
347
    items = dict(list(conv_items(strings.ustr, items)))
348
    parts = []
349
    for part in name_parts:
350
        if part in items: parts.append(items[part])
351
    if not parts: return None # pass None values through; handle no name parts
352
    return ' '.join(parts)
353
funcs['_name'] = _name
354

    
355
#### Angles
356

    
357
def _compass(items, node):
358
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
359
    items = dict(conv_items(strings.ustr, items))
360
    try: value = items['value']
361
    except KeyError: return None # input is empty
362
    
363
    if not value.isupper(): return value # pass through other coordinate formats
364
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
365
    except KeyError, e: raise FormatException(e)
366
funcs['_compass'] = _compass
(37-37/40)