Project

General

Profile

1
# XML "function" nodes that transform their contents
2

    
3
import datetime
4
import re
5
import sre_constants
6
import warnings
7

    
8
import angles
9
import dates
10
import exc
11
import format
12
import maps
13
import sql
14
import sql_io
15
import strings
16
import term
17
import units
18
import util
19
import xml_dom
20
import xpath
21

    
22
##### Exceptions
23

    
24
class SyntaxError(exc.ExceptionWithCause):
25
    def __init__(self, cause):
26
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28

    
29
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32

    
33
##### Helper functions
34

    
35
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37

    
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42

    
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46

    
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53

    
54
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
    
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63

    
64
funcs = {}
65

    
66
##### Public functions
67

    
68
def is_func_name(name):
69
    return name.startswith('_') and name != '_' # '_' is default root node name
70

    
71
def is_func(node): return is_func_name(node.tagName)
72

    
73
def is_xml_func_name(name): return is_func_name(name) and name in funcs
74

    
75
def is_xml_func(node): return is_xml_func_name(node.tagName)
76

    
77
def simplify(node):
78
    '''Simplifies the XML functions in an XML tree.
79
    * Merges nodes tagged as mergable
80
    * Applies pass-through optimizations for XML functions
81
    '''
82
    for child in xml_dom.NodeElemIter(node): simplify(child)
83
    merge_tagged(node)
84
    
85
    func_name = node.tagName
86
    if not is_func_name(func_name): return # not a function
87
    
88
    params = list(xml_dom.NodeElemIter(node))
89
    
90
    # Pass-through optimizations
91
    if len(params) == 1: # single arg
92
        param = params[0]
93
        name = param.tagName
94
        value = param.firstChild
95
        
96
        is_agg_func = name.isdigit()
97
        is_if_no_cond = func_name = '_if' # no if condition means False
98
        if is_agg_func or (is_if_no_cond and name == 'else'):
99
            xml_dom.replace(node, value)
100
        elif is_if_no_cond and name == 'then': xml_dom.remove(node)
101

    
102
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
103
    '''Evaluates the XML functions in an XML tree.
104
    @param is_rel_func None|f(str) Tests if a name is a relational function.
105
        * If != None: Non-relational functions are removed, or relational
106
          functions are treated specially, depending on the db param (below).
107
    @param db
108
        * If None: Non-relational functions other than structural functions are
109
          replaced with their last parameter (usually the value), not evaluated.
110
          This is used in column-based mode to remove XML-only functions.
111
        * If != None: Relational functions are evaluated directly. This is used
112
          in row-based mode to combine relational and XML functions.
113
    '''
114
    has_rel_funcs = is_rel_func != None
115
    assert db == None or has_rel_funcs # rel_funcs required if db set
116
    
117
    for child in xml_dom.NodeElemIter(node):
118
        process(child, on_error, is_rel_func, db)
119
    merge_tagged(node)
120
    
121
    name = node.tagName
122
    if not is_func_name(name): return node # not any kind of function
123
    
124
    row_mode = has_rel_funcs and db != None
125
    column_mode = has_rel_funcs and db == None
126
    func = funcs.get(name, None)
127
    items = list(xml_dom.NodeTextEntryIter(node))
128
    
129
    # Parse function
130
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
131
        # pass-through optimization for aggregating functions with one arg
132
        value = items[0][1] # pass through first arg
133
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
134
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
135
        except sql.DoesNotExistException: return # preserve unknown funcs
136
            # possibly a built-in function of db_xml.put()
137
    elif column_mode or func == None:
138
        # local XML function can't be used or does not exist
139
        if column_mode and is_rel_func(name): return # preserve relational funcs
140
        # otherwise XML-only in column mode, or DB-only in XML output mode
141
        value = pop_value(items, None) # just replace with last param
142
    else: # local XML function
143
        try: value = func(items, node)
144
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
145
            # Save in case another exception raised, overwriting sys.exc_info()
146
            exc.add_traceback(e)
147
            str_ = strings.ustr(node)
148
            exc.add_msg(e, 'function:\n'+str_)
149
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
150
                '\n'+term.emph_multiline(str_)))
151
                
152
            on_error(e)
153
            return # in case on_error() returns
154
    
155
    xml_dom.replace_with_text(node, value)
156

    
157
##### XML functions
158

    
159
# Function names must start with _ to avoid collisions with real tags
160
# Functions take arguments (items)
161

    
162
#### Conditionals
163

    
164
def _eq(items, node):
165
    items = dict(items)
166
    try:
167
        left = items['left']
168
        right = items['right']
169
    except KeyError: return '' # a value was not mapped to
170
    return util.bool2str(left == right)
171
funcs['_eq'] = _eq
172

    
173
def _if(items, node):
174
    '''Can add `@name` attr to distinguish separate _if statements'''
175
    items = dict(items)
176
    then = items.get('then', None)
177
    cond = items.get('cond', None)
178
    else_ = items.get('else', None)
179
    
180
    cond = cond != None and bool(cast(strings.ustr, cond)) # None == False
181
    if cond: return then
182
    else: return else_
183
funcs['_if'] = _if
184

    
185
#### Transforming values
186

    
187
def _collapse(items, node):
188
    '''Collapses a subtree if the "value" element in it is NULL'''
189
    items = dict(items)
190
    try: require = cast(strings.ustr, items['require'])
191
    except KeyError, e: raise SyntaxError(e)
192
    value = items.get('value', None)
193
    
194
    if xpath.get_value(value, require, allow_rooted=False) == None: return None
195
    else: return value
196
funcs['_collapse'] = _collapse
197

    
198
def repl(repls, value):
199
    '''Raises error if value not in map and no special '*' entry
200
    @param repls dict repl:with
201
        repl "*" means all other input values
202
        with "*" means keep input value the same
203
        with "" means ignore input value
204
    '''
205
    try: new_value = repls[value]
206
    except KeyError, e:
207
        # Save traceback right away in case another exception raised
208
        fe = FormatException(e)
209
        try: new_value = repls['*']
210
        except KeyError: raise fe
211
    if new_value == '*': new_value = value # '*' means keep input value the same
212
    return new_value
213

    
214
def _map(items, node):
215
    '''See repl()
216
    @param items
217
        <last_entry> Value
218
        <other_entries> name=value Mappings. Special values: See repl() repls.
219
    '''
220
    value = pop_value(items)
221
    if value == None: return None # input is empty
222
    return util.none_if(repl(dict(items), value), u'') # empty value means None
223
funcs['_map'] = _map
224

    
225
def _replace(items, node):
226
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
227
    value = pop_value(items)
228
    if value == None: return None # input is empty
229
    try:
230
        for repl, with_ in items:
231
            if re.match(r'^\w+$', repl):
232
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
233
            value = re.sub(repl, with_, value)
234
    except sre_constants.error, e: raise SyntaxError(e)
235
    return util.none_if(value.strip(), u'') # empty strings always mean None
236
funcs['_replace'] = _replace
237

    
238
#### Quantities
239

    
240
def _units(items, node):
241
    value = pop_value(items)
242
    if value == None: return None # input is empty
243
    
244
    quantity = units.str2quantity(value)
245
    try:
246
        for action, units_ in items:
247
            units_ = util.none_if(units_, u'')
248
            if action == 'default': units.set_default_units(quantity, units_)
249
            elif action == 'to':
250
                try: quantity = units.convert(quantity, units_)
251
                except ValueError, e: raise FormatException(e)
252
            else: raise SyntaxError(ValueError('Invalid action: '+action))
253
    except units.MissingUnitsException, e: raise FormatException(e)
254
    return units.quantity2str(quantity)
255
funcs['_units'] = _units
256

    
257
def parse_range(str_, range_sep='-'):
258
    default = (str_, None)
259
    start, sep, end = str_.partition(range_sep)
260
    if sep == '': return default # not a range
261
    if start == '' and range_sep == '-': return default # negative number
262
    return tuple(d.strip() for d in (start, end))
263

    
264
def _rangeStart(items, node):
265
    items = dict(conv_items(strings.ustr, items))
266
    try: value = items['value']
267
    except KeyError: return None # input is empty
268
    return parse_range(value)[0]
269
funcs['_rangeStart'] = _rangeStart
270

    
271
def _rangeEnd(items, node):
272
    items = dict(conv_items(strings.ustr, items))
273
    try: value = items['value']
274
    except KeyError: return None # input is empty
275
    return parse_range(value)[1]
276
funcs['_rangeEnd'] = _rangeEnd
277

    
278
def _range(items, node):
279
    items = dict(conv_items(float, items))
280
    from_ = items.get('from', None)
281
    to = items.get('to', None)
282
    if from_ == None or to == None: return None
283
    return str(to - from_)
284
funcs['_range'] = _range
285

    
286
def _avg(items, node):
287
    count = 0
288
    sum_ = 0.
289
    for name, value in conv_items(float, items):
290
        count += 1
291
        sum_ += value
292
    if count == 0: return None # input is empty
293
    else: return str(sum_/count)
294
funcs['_avg'] = _avg
295

    
296
class CvException(Exception):
297
    def __init__(self):
298
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
299
            ' allowed for ratio scale data '
300
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
301

    
302
def _noCV(items, node):
303
    items = list(conv_items(strings.ustr, items))
304
    try: name, value = items.pop() # last entry contains value
305
    except IndexError: return None # input is empty
306
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
307
    return value
308
funcs['_noCV'] = _noCV
309

    
310
#### Names
311

    
312
_name_parts_slices_items = [
313
    ('first', slice(None, 1)),
314
    ('middle', slice(1, -1)),
315
    ('last', slice(-1, None)),
316
]
317
name_parts_slices = dict(_name_parts_slices_items)
318
name_parts = [name for name, slice_ in _name_parts_slices_items]
319

    
320
def _name(items, node):
321
    items = dict(list(conv_items(strings.ustr, items)))
322
    parts = []
323
    for part in name_parts:
324
        if part in items: parts.append(items[part])
325
    if not parts: return None # pass None values through; handle no name parts
326
    return ' '.join(parts)
327
funcs['_name'] = _name
328

    
329
#### Angles
330

    
331
def _compass(items, node):
332
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
333
    items = dict(conv_items(strings.ustr, items))
334
    try: value = items['value']
335
    except KeyError: return None # input is empty
336
    
337
    if not value.isupper(): return value # pass through other coordinate formats
338
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
339
    except KeyError, e: raise FormatException(e)
340
funcs['_compass'] = _compass
(37-37/40)