Project

General

Profile

1
# XML "function" nodes that transform their contents
2

    
3
import datetime
4
import re
5
import sre_constants
6
import warnings
7

    
8
import angles
9
import dates
10
import exc
11
import format
12
import maps
13
import sql
14
import strings
15
import term
16
import units
17
import util
18
import xml_dom
19
import xpath
20

    
21
##### Exceptions
22

    
23
class SyntaxError(exc.ExceptionWithCause):
24
    def __init__(self, cause):
25
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
26
            cause)
27

    
28
class FormatException(exc.ExceptionWithCause):
29
    def __init__(self, cause):
30
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
31

    
32
##### Helper functions
33

    
34
def map_items(func, items):
35
    return [(name, func(value)) for name, value in items]
36

    
37
def cast(type_, val):
38
    '''Throws FormatException if can't cast'''
39
    try: return type_(val)
40
    except ValueError, e: raise FormatException(e)
41

    
42
def conv_items(type_, items):
43
    return map_items(lambda val: cast(type_, val),
44
        xml_dom.TextEntryOnlyIter(items))
45

    
46
def pop_value(items, name='value'):
47
    '''@param name Name of value param, or None to accept any name'''
48
    try: last = items.pop() # last entry contains value
49
    except IndexError: return None # input is empty and no actions
50
    if name != None and last[0] != name: return None # input is empty
51
    return last[1]
52

    
53
funcs = {}
54

    
55
structural_funcs = set()
56

    
57
##### Public functions
58

    
59
def is_func_name(name):
60
    return name.startswith('_') and name != '_' # '_' is default root node name
61

    
62
def is_func(node): return is_func_name(node.tagName)
63

    
64
def is_xml_func_name(name): return is_func_name(name) and name in funcs
65

    
66
def is_xml_func(node): return is_xml_func_name(node.tagName)
67

    
68
def process(node, on_error=exc.raise_, rel_funcs=None, db=None):
69
    '''Evaluates the XML functions in an XML tree.
70
    @param rel_funcs None|set(str...) Relational functions
71
        * container can be any iterable type
72
        * If != None: Non-relational functions are removed, or relational
73
          functions are treated specially, depending on the db param (below).
74
    @param db
75
        * If None: Non-relational functions other than structural functions are
76
          replaced with their last parameter (usually the value), not evaluated.
77
          This is used in column-based mode to remove XML-only functions.
78
        * If != None: Relational functions are evaluated directly. This is used
79
          in row-based mode to combine relational and XML functions.
80
    '''
81
    has_rel_funcs = rel_funcs != None
82
    assert db == None or has_rel_funcs # rel_funcs required if db set
83
    
84
    for child in xml_dom.NodeElemIter(node):
85
        process(child, on_error, rel_funcs, db)
86
    
87
    name = node.tagName
88
    if not is_func_name(name): return # not any kind of function
89
    
90
    # Change rel_funcs *after* processing child nodes, which needs orig value
91
    if not has_rel_funcs: rel_funcs = set()
92
    rel_funcs = set(rel_funcs)
93
    
94
    row_mode = has_rel_funcs and db != None
95
    column_mode = has_rel_funcs and db == None
96
    items = list(xml_dom.NodeTextEntryIter(node))
97
    
98
    # Parse function
99
    if len(items) == 1 and items[0][0].isdigit(): # has single numeric param
100
        # pass-through optimization for aggregating functions with one arg
101
        value = items[0][1] # pass through first arg
102
    elif row_mode and name in rel_funcs: # row-based mode: evaluate using DB
103
        value = sql.put(db, name, dict(items))
104
    elif column_mode and not name in structural_funcs: # column-based mode
105
        if name in rel_funcs: return # preserve relational functions
106
        # otherwise XML-only, so just replace with last param
107
        value = pop_value(items, None)
108
    else: # local XML function
109
        try: value = funcs[name](items, node)
110
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
111
            # Save in case another exception raised, overwriting sys.exc_info()
112
            exc.add_traceback(e)
113
            str_ = strings.ustr(node)
114
            exc.add_msg(e, 'function:\n'+str_)
115
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
116
                '\n'+term.emph_multiline(str_)))
117
                
118
            on_error(e)
119
            return # in case on_error() returns
120
    xml_dom.replace_with_text(node, value)
121

    
122
##### XML functions
123

    
124
# Function names must start with _ to avoid collisions with real tags
125
# Functions take arguments (items)
126

    
127
#### Structural
128

    
129
def _ignore(items, node):
130
    '''Used to "comment out" an XML subtree'''
131
    return None
132
funcs['_ignore'] = _ignore
133
structural_funcs.add('_ignore')
134

    
135
def _ref(items, node):
136
    '''Used to retrieve a value from another XML node
137
    @param items
138
        addr=<path> XPath to value, relative to the XML func's parent node
139
    '''
140
    items = dict(items)
141
    try: addr = items['addr']
142
    except KeyError, e: raise SyntaxError(e)
143
    
144
    value = xpath.get_value(node.parentNode, addr)
145
    if value == None:
146
        warnings.warn(UserWarning('_ref: XPath reference target missing: '
147
            +str(addr)))
148
    return value
149
funcs['_ref'] = _ref
150
structural_funcs.add('_ref')
151

    
152
#### Conditionals
153

    
154
def _eq(items, node):
155
    items = dict(items)
156
    try:
157
        left = items['left']
158
        right = items['right']
159
    except KeyError: return '' # a value was None
160
    return util.bool2str(left == right)
161
funcs['_eq'] = _eq
162

    
163
def _if(items, node):
164
    items = dict(items)
165
    try:
166
        cond = items['cond']
167
        then = items['then']
168
    except KeyError, e: raise SyntaxError(e)
169
    else_ = items.get('else', None)
170
    cond = bool(cast(strings.ustr, cond))
171
    if cond: return then
172
    else: return else_
173
funcs['_if'] = _if
174

    
175
#### Combining values
176

    
177
def _alt(items, node):
178
    items = list(items)
179
    items.sort()
180
    try: return items[0][1] # value of lowest-numbered item
181
    except IndexError: return None # input got removed by e.g. FormatException
182
funcs['_alt'] = _alt
183

    
184
def _merge(items, node):
185
    items = list(conv_items(strings.ustr, items))
186
        # get *once* from iter, check types
187
    items.sort()
188
    return maps.merge_values(*[v for k, v in items])
189
funcs['_merge'] = _merge
190

    
191
def _label(items, node):
192
    items = dict(conv_items(strings.ustr, items))
193
        # get *once* from iter, check types
194
    value = items.get('value', None)
195
    if value == None: return None # input is empty
196
    try: label = items['label']
197
    except KeyError, e: raise SyntaxError(e)
198
    return label+': '+value
199
funcs['_label'] = _label
200

    
201
#### Transforming values
202

    
203
def _collapse(items, node):
204
    '''Collapses a subtree if the "value" element in it is NULL'''
205
    items = dict(items)
206
    try: require = cast(strings.ustr, items['require'])
207
    except KeyError, e: raise SyntaxError(e)
208
    value = items.get('value', None)
209
    
210
    if xpath.get_value(value, require, allow_rooted=False) == None: return None
211
    else: return value
212
funcs['_collapse'] = _collapse
213

    
214
types_by_name = {None: strings.ustr, 'str': strings.ustr, 'float': float}
215

    
216
def _nullIf(items, node):
217
    items = dict(conv_items(strings.ustr, items))
218
    try: null = items['null']
219
    except KeyError, e: raise SyntaxError(e)
220
    value = items.get('value', None)
221
    type_str = items.get('type', None)
222
    
223
    try: type_ = types_by_name[type_str]
224
    except KeyError, e: raise SyntaxError(e)
225
    null = type_(null)
226
    
227
    try: return util.none_if(value, null)
228
    except ValueError: return value # value not convertible, so can't equal null
229
funcs['_nullIf'] = _nullIf
230

    
231
def repl(repls, value):
232
    '''Raises error if value not in map and no special '*' entry
233
    @param repls dict repl:with
234
        repl "*" means all other input values
235
        with "*" means keep input value the same
236
        with "" means ignore input value
237
    '''
238
    try: new_value = repls[value]
239
    except KeyError, e:
240
        # Save traceback right away in case another exception raised
241
        fe = FormatException(e)
242
        try: new_value = repls['*']
243
        except KeyError: raise fe
244
    if new_value == '*': new_value = value # '*' means keep input value the same
245
    return new_value
246

    
247
def _map(items, node):
248
    '''See repl()
249
    @param items
250
        <last_entry> Value
251
        <other_entries> name=value Mappings. Special values: See repl() repls.
252
    '''
253
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
254
    value = pop_value(items)
255
    if value == None: return None # input is empty
256
    return util.none_if(repl(dict(items), value), u'') # empty value means None
257
funcs['_map'] = _map
258

    
259
def _replace(items, node):
260
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
261
    value = pop_value(items)
262
    if value == None: return None # input is empty
263
    try:
264
        for repl, with_ in items:
265
            if re.match(r'^\w+$', repl):
266
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
267
            value = re.sub(repl, with_, value)
268
    except sre_constants.error, e: raise SyntaxError(e)
269
    return util.none_if(value.strip(), u'') # empty strings always mean None
270
funcs['_replace'] = _replace
271

    
272
#### Quantities
273

    
274
def _units(items, node):
275
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
276
    value = pop_value(items)
277
    if value == None: return None # input is empty
278
    
279
    quantity = units.str2quantity(value)
280
    try:
281
        for action, units_ in items:
282
            units_ = util.none_if(units_, u'')
283
            if action == 'default': units.set_default_units(quantity, units_)
284
            elif action == 'to':
285
                try: quantity = units.convert(quantity, units_)
286
                except ValueError, e: raise FormatException(e)
287
            else: raise SyntaxError(ValueError('Invalid action: '+action))
288
    except units.MissingUnitsException, e: raise FormatException(e)
289
    return units.quantity2str(quantity)
290
funcs['_units'] = _units
291

    
292
def parse_range(str_, range_sep='-'):
293
    default = (str_, None)
294
    start, sep, end = str_.partition(range_sep)
295
    if sep == '': return default # not a range
296
    if start == '' and range_sep == '-': return default # negative number
297
    return tuple(d.strip() for d in (start, end))
298

    
299
def _rangeStart(items, node):
300
    items = dict(conv_items(strings.ustr, items))
301
    try: value = items['value']
302
    except KeyError: return None # input is empty
303
    return parse_range(value)[0]
304
funcs['_rangeStart'] = _rangeStart
305

    
306
def _rangeEnd(items, node):
307
    items = dict(conv_items(strings.ustr, items))
308
    try: value = items['value']
309
    except KeyError: return None # input is empty
310
    return parse_range(value)[1]
311
funcs['_rangeEnd'] = _rangeEnd
312

    
313
def _range(items, node):
314
    items = dict(conv_items(float, items))
315
    from_ = items.get('from', None)
316
    to = items.get('to', None)
317
    if from_ == None or to == None: return None
318
    return str(to - from_)
319
funcs['_range'] = _range
320

    
321
def _avg(items, node):
322
    count = 0
323
    sum_ = 0.
324
    for name, value in conv_items(float, items):
325
        count += 1
326
        sum_ += value
327
    if count == 0: return None # input is empty
328
    else: return str(sum_/count)
329
funcs['_avg'] = _avg
330

    
331
class CvException(Exception):
332
    def __init__(self):
333
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
334
            ' allowed for ratio scale data '
335
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
336

    
337
def _noCV(items, node):
338
    try: name, value = items.next()
339
    except StopIteration: return None
340
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
341
    return value
342
funcs['_noCV'] = _noCV
343

    
344
#### Dates
345

    
346
def _date(items, node):
347
    items = dict(conv_items(strings.ustr, items))
348
        # get *once* from iter, check types
349
    try: str_ = items['date']
350
    except KeyError:
351
        # Year is required
352
        try: items['year']
353
        except KeyError, e:
354
            if items == {}: return None # entire date is empty
355
            else: raise FormatException(e)
356
        
357
        # Convert month name to number
358
        try: month = items['month']
359
        except KeyError: pass
360
        else:
361
            if not month.isdigit(): # month is name
362
                try: items['month'] = str(dates.strtotime(month).month)
363
                except ValueError, e: raise FormatException(e)
364
        
365
        items = dict(conv_items(format.str2int, items.iteritems()))
366
        items.setdefault('month', 1)
367
        items.setdefault('day', 1)
368
        
369
        for try_num in xrange(2):
370
            try:
371
                date = datetime.date(**items)
372
                break
373
            except ValueError, e:
374
                if try_num > 0: raise FormatException(e)
375
                    # exception still raised after retry
376
                msg = strings.ustr(e)
377
                if msg == 'month must be in 1..12': # try swapping month and day
378
                    items['month'], items['day'] = items['day'], items['month']
379
                else: raise FormatException(e)
380
    else:
381
        try: year = float(str_)
382
        except ValueError:
383
            try: date = dates.strtotime(str_)
384
            except ImportError: return str_
385
            except ValueError, e: raise FormatException(e)
386
        else: date = (datetime.date(int(year), 1, 1) +
387
            datetime.timedelta(round((year % 1.)*365)))
388
    try: return dates.strftime('%Y-%m-%d', date)
389
    except ValueError, e: raise FormatException(e)
390
funcs['_date'] = _date
391

    
392
def _dateRangeStart(items, node):
393
    items = dict(conv_items(strings.ustr, items))
394
    try: value = items['value']
395
    except KeyError: return None # input is empty
396
    return dates.parse_date_range(value)[0]
397
funcs['_dateRangeStart'] = _dateRangeStart
398

    
399
def _dateRangeEnd(items, node):
400
    items = dict(conv_items(strings.ustr, items))
401
    try: value = items['value']
402
    except KeyError: return None # input is empty
403
    return dates.parse_date_range(value)[1]
404
funcs['_dateRangeEnd'] = _dateRangeEnd
405

    
406
#### Names
407

    
408
_name_parts_slices_items = [
409
    ('first', slice(None, 1)),
410
    ('middle', slice(1, -1)),
411
    ('last', slice(-1, None)),
412
]
413
name_parts_slices = dict(_name_parts_slices_items)
414
name_parts = [name for name, slice_ in _name_parts_slices_items]
415

    
416
def _name(items, node):
417
    items = dict(items)
418
    parts = []
419
    for part in name_parts:
420
        if part in items: parts.append(items[part])
421
    return ' '.join(parts)
422
funcs['_name'] = _name
423

    
424
def _namePart(items, node):
425
    out_items = []
426
    for part, value in items:
427
        try: slice_ = name_parts_slices[part]
428
        except KeyError, e: raise SyntaxError(e)
429
        out_items.append((part, ' '.join(value.split(' ')[slice_])))
430
    return _name(out_items, node)
431
funcs['_namePart'] = _namePart
432

    
433
#### Angles
434

    
435
def _compass(items, node):
436
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
437
    items = dict(conv_items(strings.ustr, items))
438
    try: value = items['value']
439
    except KeyError: return None # input is empty
440
    
441
    if not value.isupper(): return value # pass through other coordinate formats
442
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
443
    except KeyError, e: raise FormatException(e)
444
funcs['_compass'] = _compass
445

    
446
#### Paths
447

    
448
def _simplifyPath(items, node):
449
    items = dict(items)
450
    try:
451
        next = cast(strings.ustr, items['next'])
452
        require = cast(strings.ustr, items['require'])
453
        root = items['path']
454
    except KeyError, e: raise SyntaxError(e)
455
    
456
    node = root
457
    while node != None:
458
        new_node = xpath.get_1(node, next, allow_rooted=False)
459
        if xpath.get_value(node, require, allow_rooted=False) == None: # empty
460
            xml_dom.replace(node, new_node) # remove current elem
461
            if node is root: root = new_node # also update root
462
        node = new_node
463
    return root
464
funcs['_simplifyPath'] = _simplifyPath
(33-33/36)