Project

General

Profile

1
# XML "function" nodes that transform their contents
2

    
3
import datetime
4
import re
5
import sre_constants
6
import warnings
7

    
8
import angles
9
import dates
10
import exc
11
import format
12
import maps
13
import sql
14
import sql_io
15
import strings
16
import term
17
import units
18
import util
19
import xml_dom
20
import xpath
21

    
22
##### Exceptions
23

    
24
class SyntaxError(exc.ExceptionWithCause):
25
    def __init__(self, cause):
26
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
27
            cause)
28

    
29
class FormatException(exc.ExceptionWithCause):
30
    def __init__(self, cause):
31
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
32

    
33
##### Helper functions
34

    
35
def map_items(func, items):
36
    return [(name, func(value)) for name, value in items]
37

    
38
def cast(type_, val):
39
    '''Throws FormatException if can't cast'''
40
    try: return type_(val)
41
    except ValueError, e: raise FormatException(e)
42

    
43
def conv_items(type_, items):
44
    return map_items(lambda val: cast(type_, val),
45
        xml_dom.TextEntryOnlyIter(items))
46

    
47
def pop_value(items, name='value'):
48
    '''@param name Name of value param, or None to accept any name'''
49
    try: last = items.pop() # last entry contains value
50
    except IndexError: return None # input is empty and no actions
51
    if name != None and last[0] != name: return None # input is empty
52
    return last[1]
53

    
54
def merge_tagged(root):
55
    '''Merges siblings in root that are marked as mergeable.
56
    Used to recombine pieces of nodes that were split apart in the mappings.
57
    '''
58
    for name in set((c.tagName for c in xpath.get(root, '*[@merge=1]'))):
59
        xml_dom.merge_by_name(root, name)
60
    
61
    # Recurse
62
    for child in xml_dom.NodeElemIter(root): merge_tagged(child)
63

    
64
funcs = {}
65

    
66
##### Public functions
67

    
68
def is_func_name(name):
69
    return name.startswith('_') and name != '_' # '_' is default root node name
70

    
71
def is_func(node): return is_func_name(node.tagName)
72

    
73
def is_xml_func_name(name): return is_func_name(name) and name in funcs
74

    
75
def is_xml_func(node): return is_xml_func_name(node.tagName)
76

    
77
def simplify(node):
78
    '''Simplifies the XML functions in an XML tree.
79
    * Merges nodes tagged as mergable
80
    * Applies a pass-through optimization for aggregating functions with one arg
81
    '''
82
    for child in xml_dom.NodeElemIter(node): simplify(child)
83
    merge_tagged(node)
84
    
85
    name = node.tagName
86
    if not is_func_name(name): return # not a function
87
    
88
    items = list(xml_dom.NodeElemIter(node))
89
    
90
    if len(items) == 1 and items[0].tagName.isdigit(): # single numeric param
91
        # pass-through optimization for aggregating functions with one arg
92
        xml_dom.replace(node, items[0].firstChild) # use first arg's value
93

    
94
def process(node, on_error=exc.reraise, is_rel_func=None, db=None):
95
    '''Evaluates the XML functions in an XML tree.
96
    @param is_rel_func None|f(str) Tests if a name is a relational function.
97
        * If != None: Non-relational functions are removed, or relational
98
          functions are treated specially, depending on the db param (below).
99
    @param db
100
        * If None: Non-relational functions other than structural functions are
101
          replaced with their last parameter (usually the value), not evaluated.
102
          This is used in column-based mode to remove XML-only functions.
103
        * If != None: Relational functions are evaluated directly. This is used
104
          in row-based mode to combine relational and XML functions.
105
    '''
106
    has_rel_funcs = is_rel_func != None
107
    assert db == None or has_rel_funcs # rel_funcs required if db set
108
    
109
    for child in xml_dom.NodeElemIter(node):
110
        process(child, on_error, is_rel_func, db)
111
    merge_tagged(node)
112
    
113
    name = node.tagName
114
    if not is_func_name(name): return node # not any kind of function
115
    
116
    row_mode = has_rel_funcs and db != None
117
    column_mode = has_rel_funcs and db == None
118
    func = funcs.get(name, None)
119
    items = list(xml_dom.NodeTextEntryIter(node))
120
    
121
    # Parse function
122
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
123
        # pass-through optimization for aggregating functions with one arg
124
        value = items[0][1] # pass through first arg
125
    elif row_mode and (is_rel_func(name) or func == None): # row-based mode
126
        try: value = sql_io.put(db, name, dict(items)) # evaluate using DB
127
        except sql.DoesNotExistException: return # preserve unknown funcs
128
            # possibly a built-in function of db_xml.put()
129
    elif column_mode or func == None:
130
        # local XML function can't be used or does not exist
131
        if column_mode and is_rel_func(name): return # preserve relational funcs
132
        # otherwise XML-only in column mode, or DB-only in XML output mode
133
        value = pop_value(items, None) # just replace with last param
134
    else: # local XML function
135
        try: value = func(items, node)
136
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
137
            # Save in case another exception raised, overwriting sys.exc_info()
138
            exc.add_traceback(e)
139
            str_ = strings.ustr(node)
140
            exc.add_msg(e, 'function:\n'+str_)
141
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
142
                '\n'+term.emph_multiline(str_)))
143
                
144
            on_error(e)
145
            return # in case on_error() returns
146
    
147
    xml_dom.replace_with_text(node, value)
148

    
149
##### XML functions
150

    
151
# Function names must start with _ to avoid collisions with real tags
152
# Functions take arguments (items)
153

    
154
#### Conditionals
155

    
156
def _eq(items, node):
157
    items = dict(items)
158
    try:
159
        left = items['left']
160
        right = items['right']
161
    except KeyError: return '' # a value was not mapped to
162
    return util.bool2str(left == right)
163
funcs['_eq'] = _eq
164

    
165
def _if(items, node):
166
    '''Can add `@name` attr to distinguish separate _if statements'''
167
    items = dict(items)
168
    then = items.get('then', None)
169
    cond = items.get('cond', None)
170
    else_ = items.get('else', None)
171
    
172
    cond = cond != None and bool(cast(strings.ustr, cond)) # None == False
173
    if cond: return then
174
    else: return else_
175
funcs['_if'] = _if
176

    
177
#### Transforming values
178

    
179
def _collapse(items, node):
180
    '''Collapses a subtree if the "value" element in it is NULL'''
181
    items = dict(items)
182
    try: require = cast(strings.ustr, items['require'])
183
    except KeyError, e: raise SyntaxError(e)
184
    value = items.get('value', None)
185
    
186
    if xpath.get_value(value, require, allow_rooted=False) == None: return None
187
    else: return value
188
funcs['_collapse'] = _collapse
189

    
190
def repl(repls, value):
191
    '''Raises error if value not in map and no special '*' entry
192
    @param repls dict repl:with
193
        repl "*" means all other input values
194
        with "*" means keep input value the same
195
        with "" means ignore input value
196
    '''
197
    try: new_value = repls[value]
198
    except KeyError, e:
199
        # Save traceback right away in case another exception raised
200
        fe = FormatException(e)
201
        try: new_value = repls['*']
202
        except KeyError: raise fe
203
    if new_value == '*': new_value = value # '*' means keep input value the same
204
    return new_value
205

    
206
def _map(items, node):
207
    '''See repl()
208
    @param items
209
        <last_entry> Value
210
        <other_entries> name=value Mappings. Special values: See repl() repls.
211
    '''
212
    value = pop_value(items)
213
    if value == None: return None # input is empty
214
    return util.none_if(repl(dict(items), value), u'') # empty value means None
215
funcs['_map'] = _map
216

    
217
def _replace(items, node):
218
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
219
    value = pop_value(items)
220
    if value == None: return None # input is empty
221
    try:
222
        for repl, with_ in items:
223
            if re.match(r'^\w+$', repl):
224
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
225
            value = re.sub(repl, with_, value)
226
    except sre_constants.error, e: raise SyntaxError(e)
227
    return util.none_if(value.strip(), u'') # empty strings always mean None
228
funcs['_replace'] = _replace
229

    
230
#### Quantities
231

    
232
def _units(items, node):
233
    value = pop_value(items)
234
    if value == None: return None # input is empty
235
    
236
    quantity = units.str2quantity(value)
237
    try:
238
        for action, units_ in items:
239
            units_ = util.none_if(units_, u'')
240
            if action == 'default': units.set_default_units(quantity, units_)
241
            elif action == 'to':
242
                try: quantity = units.convert(quantity, units_)
243
                except ValueError, e: raise FormatException(e)
244
            else: raise SyntaxError(ValueError('Invalid action: '+action))
245
    except units.MissingUnitsException, e: raise FormatException(e)
246
    return units.quantity2str(quantity)
247
funcs['_units'] = _units
248

    
249
def parse_range(str_, range_sep='-'):
250
    default = (str_, None)
251
    start, sep, end = str_.partition(range_sep)
252
    if sep == '': return default # not a range
253
    if start == '' and range_sep == '-': return default # negative number
254
    return tuple(d.strip() for d in (start, end))
255

    
256
def _rangeStart(items, node):
257
    items = dict(conv_items(strings.ustr, items))
258
    try: value = items['value']
259
    except KeyError: return None # input is empty
260
    return parse_range(value)[0]
261
funcs['_rangeStart'] = _rangeStart
262

    
263
def _rangeEnd(items, node):
264
    items = dict(conv_items(strings.ustr, items))
265
    try: value = items['value']
266
    except KeyError: return None # input is empty
267
    return parse_range(value)[1]
268
funcs['_rangeEnd'] = _rangeEnd
269

    
270
def _range(items, node):
271
    items = dict(conv_items(float, items))
272
    from_ = items.get('from', None)
273
    to = items.get('to', None)
274
    if from_ == None or to == None: return None
275
    return str(to - from_)
276
funcs['_range'] = _range
277

    
278
def _avg(items, node):
279
    count = 0
280
    sum_ = 0.
281
    for name, value in conv_items(float, items):
282
        count += 1
283
        sum_ += value
284
    if count == 0: return None # input is empty
285
    else: return str(sum_/count)
286
funcs['_avg'] = _avg
287

    
288
class CvException(Exception):
289
    def __init__(self):
290
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
291
            ' allowed for ratio scale data '
292
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
293

    
294
def _noCV(items, node):
295
    items = list(conv_items(strings.ustr, items))
296
    try: name, value = items.pop() # last entry contains value
297
    except IndexError: return None # input is empty
298
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
299
    return value
300
funcs['_noCV'] = _noCV
301

    
302
#### Names
303

    
304
_name_parts_slices_items = [
305
    ('first', slice(None, 1)),
306
    ('middle', slice(1, -1)),
307
    ('last', slice(-1, None)),
308
]
309
name_parts_slices = dict(_name_parts_slices_items)
310
name_parts = [name for name, slice_ in _name_parts_slices_items]
311

    
312
def _name(items, node):
313
    items = dict(list(conv_items(strings.ustr, items)))
314
    parts = []
315
    for part in name_parts:
316
        if part in items: parts.append(items[part])
317
    if not parts: return None # pass None values through; handle no name parts
318
    return ' '.join(parts)
319
funcs['_name'] = _name
320

    
321
#### Angles
322

    
323
def _compass(items, node):
324
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
325
    items = dict(conv_items(strings.ustr, items))
326
    try: value = items['value']
327
    except KeyError: return None # input is empty
328
    
329
    if not value.isupper(): return value # pass through other coordinate formats
330
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
331
    except KeyError, e: raise FormatException(e)
332
funcs['_compass'] = _compass
(37-37/40)