Project

General

Profile

1
# XML "function" nodes that transform their contents
2

    
3
import datetime
4
import re
5
import sre_constants
6
import warnings
7

    
8
import angles
9
import dates
10
import exc
11
import format
12
import maps
13
import sql
14
import sql_io
15
import strings
16
import term
17
import units
18
import util
19
import xml_dom
20
import xpath
21

    
22
##### Exceptions
23

    
24
class SyntaxError(exc.ExceptionWithCause):
25
    def __init__(self, cause):
26
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28

    
29
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32

    
33
##### Helper functions
34

    
35
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37

    
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42

    
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46

    
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53

    
54
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
    
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63

    
64
funcs = {}
65

    
66
##### Public functions
67

    
68
def is_func_name(name):
69
    return name.startswith('_') and name != '_' # '_' is default root node name
70

    
71
def is_func(node): return is_func_name(node.tagName)
72

    
73
def is_xml_func_name(name): return is_func_name(name) and name in funcs
74

    
75
def is_xml_func(node): return is_xml_func_name(node.tagName)
76

    
77
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
78
    '''Evaluates the XML functions in an XML tree.
79
    @param is_rel_func None|f(str) Tests if a name is a relational function.
80
        * If != None: Non-relational functions are removed, or relational
81
          functions are treated specially, depending on the db param (below).
82
    @param db
83
        * If None: Non-relational functions other than structural functions are
84
          replaced with their last parameter (usually the value), not evaluated.
85
          This is used in column-based mode to remove XML-only functions.
86
        * If != None: Relational functions are evaluated directly. This is used
87
          in row-based mode to combine relational and XML functions.
88
    '''
89
    has_rel_funcs = is_rel_func != None
90
    assert db == None or has_rel_funcs # rel_funcs required if db set
91
    
92
    for child in xml_dom.NodeElemIter(node):
93
        process(child, on_error, is_rel_func, db)
94
    merge_tagged(node)
95
    
96
    name = node.tagName
97
    if not is_func_name(name): return node # not any kind of function
98
    
99
    row_mode = has_rel_funcs and db != None
100
    column_mode = has_rel_funcs and db == None
101
    func = funcs.get(name, None)
102
    items = list(xml_dom.NodeTextEntryIter(node))
103
    
104
    # Parse function
105
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
106
        # pass-through optimization for aggregating functions with one arg
107
        value = items[0][1] # pass through first arg
108
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
109
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
110
        except sql.DoesNotExistException: return # preserve unknown funcs
111
            # possibly a built-in function of db_xml.put()
112
    elif column_mode or func == None:
113
        # local XML function can't be used or does not exist
114
        if column_mode and is_rel_func(name): return # preserve relational funcs
115
        # otherwise XML-only in column mode, or DB-only in XML output mode
116
        value = pop_value(items, None) # just replace with last param
117
    else: # local XML function
118
        try: value = func(items, node)
119
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
120
            # Save in case another exception raised, overwriting sys.exc_info()
121
            exc.add_traceback(e)
122
            str_ = strings.ustr(node)
123
            exc.add_msg(e, 'function:\n'+str_)
124
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
125
                '\n'+term.emph_multiline(str_)))
126
                
127
            on_error(e)
128
            return # in case on_error() returns
129
    
130
    xml_dom.replace_with_text(node, value)
131

    
132
##### XML functions
133

    
134
# Function names must start with _ to avoid collisions with real tags
135
# Functions take arguments (items)
136

    
137
#### Conditionals
138

    
139
def _eq(items, node):
140
    items = dict(items)
141
    try:
142
        left = items['left']
143
        right = items['right']
144
    except KeyError: return '' # a value was not mapped to
145
    return util.bool2str(left == right)
146
funcs['_eq'] = _eq
147

    
148
def _if(items, node):
149
    '''Can add `@name` attr to distinguish separate _if statements'''
150
    items = dict(items)
151
    then = items.get('then', None)
152
    cond = items.get('cond', None)
153
    else_ = items.get('else', None)
154
    
155
    cond = cond != None and bool(cast(strings.ustr, cond)) # None == False
156
    if cond: return then
157
    else: return else_
158
funcs['_if'] = _if
159

    
160
#### Transforming values
161

    
162
def _collapse(items, node):
163
    '''Collapses a subtree if the "value" element in it is NULL'''
164
    items = dict(items)
165
    try: require = cast(strings.ustr, items['require'])
166
    except KeyError, e: raise SyntaxError(e)
167
    value = items.get('value', None)
168
    
169
    if xpath.get_value(value, require, allow_rooted=False) == None: return None
170
    else: return value
171
funcs['_collapse'] = _collapse
172

    
173
def repl(repls, value):
174
    '''Raises error if value not in map and no special '*' entry
175
    @param repls dict repl:with
176
        repl "*" means all other input values
177
        with "*" means keep input value the same
178
        with "" means ignore input value
179
    '''
180
    try: new_value = repls[value]
181
    except KeyError, e:
182
        # Save traceback right away in case another exception raised
183
        fe = FormatException(e)
184
        try: new_value = repls['*']
185
        except KeyError: raise fe
186
    if new_value == '*': new_value = value # '*' means keep input value the same
187
    return new_value
188

    
189
def _map(items, node):
190
    '''See repl()
191
    @param items
192
        <last_entry> Value
193
        <other_entries> name=value Mappings. Special values: See repl() repls.
194
    '''
195
    value = pop_value(items)
196
    if value == None: return None # input is empty
197
    return util.none_if(repl(dict(items), value), u'') # empty value means None
198
funcs['_map'] = _map
199

    
200
def _replace(items, node):
201
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
202
    value = pop_value(items)
203
    if value == None: return None # input is empty
204
    try:
205
        for repl, with_ in items:
206
            if re.match(r'^\w+$', repl):
207
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
208
            value = re.sub(repl, with_, value)
209
    except sre_constants.error, e: raise SyntaxError(e)
210
    return util.none_if(value.strip(), u'') # empty strings always mean None
211
funcs['_replace'] = _replace
212

    
213
#### Quantities
214

    
215
def _units(items, node):
216
    value = pop_value(items)
217
    if value == None: return None # input is empty
218
    
219
    quantity = units.str2quantity(value)
220
    try:
221
        for action, units_ in items:
222
            units_ = util.none_if(units_, u'')
223
            if action == 'default': units.set_default_units(quantity, units_)
224
            elif action == 'to':
225
                try: quantity = units.convert(quantity, units_)
226
                except ValueError, e: raise FormatException(e)
227
            else: raise SyntaxError(ValueError('Invalid action: '+action))
228
    except units.MissingUnitsException, e: raise FormatException(e)
229
    return units.quantity2str(quantity)
230
funcs['_units'] = _units
231

    
232
def parse_range(str_, range_sep='-'):
233
    default = (str_, None)
234
    start, sep, end = str_.partition(range_sep)
235
    if sep == '': return default # not a range
236
    if start == '' and range_sep == '-': return default # negative number
237
    return tuple(d.strip() for d in (start, end))
238

    
239
def _rangeStart(items, node):
240
    items = dict(conv_items(strings.ustr, items))
241
    try: value = items['value']
242
    except KeyError: return None # input is empty
243
    return parse_range(value)[0]
244
funcs['_rangeStart'] = _rangeStart
245

    
246
def _rangeEnd(items, node):
247
    items = dict(conv_items(strings.ustr, items))
248
    try: value = items['value']
249
    except KeyError: return None # input is empty
250
    return parse_range(value)[1]
251
funcs['_rangeEnd'] = _rangeEnd
252

    
253
def _range(items, node):
254
    items = dict(conv_items(float, items))
255
    from_ = items.get('from', None)
256
    to = items.get('to', None)
257
    if from_ == None or to == None: return None
258
    return str(to - from_)
259
funcs['_range'] = _range
260

    
261
def _avg(items, node):
262
    count = 0
263
    sum_ = 0.
264
    for name, value in conv_items(float, items):
265
        count += 1
266
        sum_ += value
267
    if count == 0: return None # input is empty
268
    else: return str(sum_/count)
269
funcs['_avg'] = _avg
270

    
271
class CvException(Exception):
272
    def __init__(self):
273
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
274
            ' allowed for ratio scale data '
275
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
276

    
277
def _noCV(items, node):
278
    items = list(conv_items(strings.ustr, items))
279
    try: name, value = items.pop() # last entry contains value
280
    except IndexError: return None # input is empty
281
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
282
    return value
283
funcs['_noCV'] = _noCV
284

    
285
#### Names
286

    
287
_name_parts_slices_items = [
288
    ('first', slice(None, 1)),
289
    ('middle', slice(1, -1)),
290
    ('last', slice(-1, None)),
291
]
292
name_parts_slices = dict(_name_parts_slices_items)
293
name_parts = [name for name, slice_ in _name_parts_slices_items]
294

    
295
def _name(items, node):
296
    items = dict(list(conv_items(strings.ustr, items)))
297
    parts = []
298
    for part in name_parts:
299
        if part in items: parts.append(items[part])
300
    if not parts: return None # pass None values through; handle no name parts
301
    return ' '.join(parts)
302
funcs['_name'] = _name
303

    
304
#### Angles
305

    
306
def _compass(items, node):
307
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
308
    items = dict(conv_items(strings.ustr, items))
309
    try: value = items['value']
310
    except KeyError: return None # input is empty
311
    
312
    if not value.isupper(): return value # pass through other coordinate formats
313
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
314
    except KeyError, e: raise FormatException(e)
315
funcs['_compass'] = _compass
(37-37/40)