Project

General

Profile

1
# XML "function" nodes that transform their contents
2

    
3
import datetime
4
import re
5
import sre_constants
6
import warnings
7

    
8
import angles
9
import dates
10
import exc
11
import format
12
import maps
13
import sql
14
import strings
15
import term
16
import units
17
import util
18
import xml_dom
19
import xpath
20

    
21
##### Exceptions
22

    
23
class SyntaxError(exc.ExceptionWithCause):
24
    def __init__(self, cause):
25
        exc.ExceptionWithCause.__init__(self, 'Invalid XML function syntax',
26
            cause)
27

    
28
class FormatException(exc.ExceptionWithCause):
29
    def __init__(self, cause):
30
        exc.ExceptionWithCause.__init__(self, 'Invalid input value', cause)
31

    
32
##### Helper functions
33

    
34
def map_items(func, items):
35
    return [(name, func(value)) for name, value in items]
36

    
37
def cast(type_, val):
38
    '''Throws FormatException if can't cast'''
39
    try: return type_(val)
40
    except ValueError, e: raise FormatException(e)
41

    
42
def conv_items(type_, items):
43
    return map_items(lambda val: cast(type_, val),
44
        xml_dom.TextEntryOnlyIter(items))
45

    
46
def pop_value(items, name='value'):
47
    '''@param name Name of value param, or None to accept any name'''
48
    try: last = items.pop() # last entry contains value
49
    except IndexError: return None # input is empty and no actions
50
    if name != None and last[0] != name: return None # input is empty
51
    return last[1]
52

    
53
funcs = {}
54

    
55
##### Public functions
56

    
57
def is_func_name(name):
58
    return name.startswith('_') and name != '_' # '_' is default root node name
59

    
60
def is_func(node): return is_func_name(node.tagName)
61

    
62
def is_xml_func_name(name): return is_func_name(name) and name in funcs
63

    
64
def is_xml_func(node): return is_xml_func_name(node.tagName)
65

    
66
def process(node, on_error=exc.raise_, db=None):
67
    for child in xml_dom.NodeElemIter(node): process(child, on_error)
68
    name = node.tagName
69
    if is_func_name(name):
70
        try:
71
            items = xml_dom.NodeTextEntryIter(node)
72
            try: func = funcs[name]
73
            except KeyError:
74
                if db != None: # DB with relational functions available
75
                    value = sql.put(db, name, dict(items))
76
                else: value = pop_value(list(items)) # pass value through
77
            else: value = func(items, node) # local XML function
78
            
79
            xml_dom.replace_with_text(node, value)
80
        except Exception, e: # also catch non-wrapped exceptions (XML func bugs)
81
            # Save in case another exception raised, overwriting sys.exc_info()
82
            exc.add_traceback(e)
83
            str_ = strings.ustr(node)
84
            exc.add_msg(e, 'function:\n'+str_)
85
            xml_dom.replace(node, xml_dom.mk_comment(node.ownerDocument,
86
                '\n'+term.emph_multiline(str_)))
87
                
88
            on_error(e)
89

    
90
def strip(node):
91
    '''Replaces every XML function with its last parameter (which is usually its
92
    value), except for _ignore, which is removed completely'''
93
    for child in xml_dom.NodeElemIter(node): strip(child)
94
    name = node.tagName
95
    if is_xml_func_name(name):
96
        if name == '_ignore': value = None
97
        else: value = pop_value(list(xml_dom.NodeTextEntryIter(node)), None)
98
        xml_dom.replace_with_text(node, value)
99

    
100
##### XML functions
101

    
102
# Function names must start with _ to avoid collisions with real tags
103
# Functions take arguments (items)
104

    
105
#### General
106

    
107
def _ignore(items, node):
108
    '''Used to "comment out" an XML subtree'''
109
    return None
110
funcs['_ignore'] = _ignore
111

    
112
def _ref(items, node):
113
    '''Used to retrieve a value from another XML node
114
    @param items
115
        addr=<path> XPath to value, relative to the XML func's parent node
116
    '''
117
    items = dict(items)
118
    try: addr = items['addr']
119
    except KeyError, e: raise SyntaxError(e)
120
    
121
    value = xpath.get_value(node.parentNode, addr)
122
    if value == None:
123
        warnings.warn(UserWarning('_ref: XPath reference target missing: '
124
            +str(addr)))
125
    return value
126
funcs['_ref'] = _ref
127

    
128
#### Conditionals
129

    
130
def _eq(items, node):
131
    items = dict(items)
132
    try:
133
        left = items['left']
134
        right = items['right']
135
    except KeyError: return '' # a value was None
136
    return util.bool2str(left == right)
137
funcs['_eq'] = _eq
138

    
139
def _if(items, node):
140
    items = dict(items)
141
    try:
142
        cond = items['cond']
143
        then = items['then']
144
    except KeyError, e: raise SyntaxError(e)
145
    else_ = items.get('else', None)
146
    cond = bool(cast(strings.ustr, cond))
147
    if cond: return then
148
    else: return else_
149
funcs['_if'] = _if
150

    
151
#### Combining values
152

    
153
def _alt(items, node):
154
    items = list(items)
155
    items.sort()
156
    try: return items[0][1] # value of lowest-numbered item
157
    except IndexError: return None # input got removed by e.g. FormatException
158
funcs['_alt'] = _alt
159

    
160
def _merge(items, node):
161
    items = list(conv_items(strings.ustr, items))
162
        # get *once* from iter, check types
163
    items.sort()
164
    return maps.merge_values(*[v for k, v in items])
165
funcs['_merge'] = _merge
166

    
167
def _label(items, node):
168
    items = dict(conv_items(strings.ustr, items))
169
        # get *once* from iter, check types
170
    value = items.get('value', None)
171
    if value == None: return None # input is empty
172
    try: label = items['label']
173
    except KeyError, e: raise SyntaxError(e)
174
    return label+': '+value
175
funcs['_label'] = _label
176

    
177
#### Transforming values
178

    
179
def _collapse(items, node):
180
    '''Collapses a subtree if the "value" element in it is NULL'''
181
    items = dict(items)
182
    try: require = cast(strings.ustr, items['require'])
183
    except KeyError, e: raise SyntaxError(e)
184
    value = items.get('value', None)
185
    
186
    required_node = xpath.get_1(value, require, allow_rooted=False)
187
    if required_node == None or xml_dom.is_empty(required_node): return None
188
    else: return value
189
funcs['_collapse'] = _collapse
190

    
191
types_by_name = {None: strings.ustr, 'str': strings.ustr, 'float': float}
192

    
193
def _nullIf(items, node):
194
    items = dict(conv_items(strings.ustr, items))
195
    try: null = items['null']
196
    except KeyError, e: raise SyntaxError(e)
197
    value = items.get('value', None)
198
    type_str = items.get('type', None)
199
    
200
    try: type_ = types_by_name[type_str]
201
    except KeyError, e: raise SyntaxError(e)
202
    null = type_(null)
203
    
204
    try: return util.none_if(value, null)
205
    except ValueError: return value # value not convertible, so can't equal null
206
funcs['_nullIf'] = _nullIf
207

    
208
def repl(repls, value):
209
    '''Raises error if value not in map and no special '*' entry
210
    @param repls dict repl:with
211
        repl "*" means all other input values
212
        with "*" means keep input value the same
213
        with "" means ignore input value
214
    '''
215
    try: new_value = repls[value]
216
    except KeyError, e:
217
        # Save traceback right away in case another exception raised
218
        fe = FormatException(e) 
219
        try: new_value = repls['*']
220
        except KeyError: raise fe
221
    if new_value == '*': new_value = value # '*' means keep input value the same
222
    return new_value
223

    
224
def _map(items, node):
225
    '''See repl()
226
    @param items
227
        <last_entry> Value
228
        <other_entries> name=value Mappings. Special values: See repl() repls.
229
    '''
230
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
231
    value = pop_value(items)
232
    if value == None: return None # input is empty
233
    return util.none_if(repl(dict(items), value), u'') # empty value means None
234
funcs['_map'] = _map
235

    
236
def _replace(items, node):
237
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
238
    value = pop_value(items)
239
    if value == None: return None # input is empty
240
    try:
241
        for repl, with_ in items:
242
            if re.match(r'^\w+$', repl):
243
                repl = r'(?<![^\W_])'+repl+r'(?![^\W_])' # match whole word
244
            value = re.sub(repl, with_, value)
245
    except sre_constants.error, e: raise SyntaxError(e)
246
    return util.none_if(value.strip(), u'') # empty strings always mean None
247
funcs['_replace'] = _replace
248

    
249
#### Quantities
250

    
251
def _units(items, node):
252
    items = conv_items(strings.ustr, items) # get *once* from iter, check types
253
    value = pop_value(items)
254
    if value == None: return None # input is empty
255
    
256
    quantity = units.str2quantity(value)
257
    try:
258
        for action, units_ in items:
259
            units_ = util.none_if(units_, u'')
260
            if action == 'default': units.set_default_units(quantity, units_)
261
            elif action == 'to':
262
                try: quantity = units.convert(quantity, units_)
263
                except ValueError, e: raise FormatException(e)
264
            else: raise SyntaxError(ValueError('Invalid action: '+action))
265
    except units.MissingUnitsException, e: raise FormatException(e)
266
    return units.quantity2str(quantity)
267
funcs['_units'] = _units
268

    
269
def parse_range(str_, range_sep='-'):
270
    default = (str_, None)
271
    start, sep, end = str_.partition(range_sep)
272
    if sep == '': return default # not a range
273
    if start == '' and range_sep == '-': return default # negative number
274
    return tuple(d.strip() for d in (start, end))
275

    
276
def _rangeStart(items, node):
277
    items = dict(conv_items(strings.ustr, items))
278
    try: value = items['value']
279
    except KeyError: return None # input is empty
280
    return parse_range(value)[0]
281
funcs['_rangeStart'] = _rangeStart
282

    
283
def _rangeEnd(items, node):
284
    items = dict(conv_items(strings.ustr, items))
285
    try: value = items['value']
286
    except KeyError: return None # input is empty
287
    return parse_range(value)[1]
288
funcs['_rangeEnd'] = _rangeEnd
289

    
290
def _range(items, node):
291
    items = dict(conv_items(float, items))
292
    from_ = items.get('from', None)
293
    to = items.get('to', None)
294
    if from_ == None or to == None: return None
295
    return str(to - from_)
296
funcs['_range'] = _range
297

    
298
def _avg(items, node):
299
    count = 0
300
    sum_ = 0.
301
    for name, value in conv_items(float, items):
302
        count += 1
303
        sum_ += value
304
    if count == 0: return None # input is empty
305
    else: return str(sum_/count)
306
funcs['_avg'] = _avg
307

    
308
class CvException(Exception):
309
    def __init__(self):
310
        Exception.__init__(self, 'CV (coefficient of variation) values are only'
311
            ' allowed for ratio scale data '
312
            '(see <http://en.wikipedia.org/wiki/Coefficient_of_variation>)')
313

    
314
def _noCV(items, node):
315
    try: name, value = items.next()
316
    except StopIteration: return None
317
    if re.match('^(?i)CV *\d+$', value): raise FormatException(CvException())
318
    return value
319
funcs['_noCV'] = _noCV
320

    
321
#### Dates
322

    
323
def _date(items, node):
324
    items = dict(conv_items(strings.ustr, items))
325
        # get *once* from iter, check types
326
    try: str_ = items['date']
327
    except KeyError:
328
        # Year is required
329
        try: items['year']
330
        except KeyError, e:
331
            if items == {}: return None # entire date is empty
332
            else: raise FormatException(e)
333
        
334
        # Convert month name to number
335
        try: month = items['month']
336
        except KeyError: pass
337
        else:
338
            if not month.isdigit(): # month is name
339
                try: items['month'] = str(dates.strtotime(month).month)
340
                except ValueError, e: raise FormatException(e)
341
        
342
        items = dict(conv_items(format.str2int, items.iteritems()))
343
        items.setdefault('month', 1)
344
        items.setdefault('day', 1)
345
        
346
        for try_num in xrange(2):
347
            try:
348
                date = datetime.date(**items)
349
                break
350
            except ValueError, e:
351
                if try_num > 0: raise FormatException(e)
352
                    # exception still raised after retry
353
                msg = strings.ustr(e)
354
                if msg == 'month must be in 1..12': # try swapping month and day
355
                    items['month'], items['day'] = items['day'], items['month']
356
                else: raise FormatException(e)
357
    else:
358
        try: year = float(str_)
359
        except ValueError:
360
            try: date = dates.strtotime(str_)
361
            except ImportError: return str_
362
            except ValueError, e: raise FormatException(e)
363
        else: date = (datetime.date(int(year), 1, 1) +
364
            datetime.timedelta(round((year % 1.)*365)))
365
    try: return dates.strftime('%Y-%m-%d', date)
366
    except ValueError, e: raise FormatException(e)
367
funcs['_date'] = _date
368

    
369
def _dateRangeStart(items, node):
370
    items = dict(conv_items(strings.ustr, items))
371
    try: value = items['value']
372
    except KeyError: return None # input is empty
373
    return dates.parse_date_range(value)[0]
374
funcs['_dateRangeStart'] = _dateRangeStart
375

    
376
def _dateRangeEnd(items, node):
377
    items = dict(conv_items(strings.ustr, items))
378
    try: value = items['value']
379
    except KeyError: return None # input is empty
380
    return dates.parse_date_range(value)[1]
381
funcs['_dateRangeEnd'] = _dateRangeEnd
382

    
383
#### Names
384

    
385
_name_parts_slices_items = [
386
    ('first', slice(None, 1)),
387
    ('middle', slice(1, -1)),
388
    ('last', slice(-1, None)),
389
]
390
name_parts_slices = dict(_name_parts_slices_items)
391
name_parts = [name for name, slice_ in _name_parts_slices_items]
392

    
393
def _name(items, node):
394
    items = dict(items)
395
    parts = []
396
    for part in name_parts:
397
        if part in items: parts.append(items[part])
398
    return ' '.join(parts)
399
funcs['_name'] = _name
400

    
401
def _namePart(items, node):
402
    out_items = []
403
    for part, value in items:
404
        try: slice_ = name_parts_slices[part]
405
        except KeyError, e: raise SyntaxError(e)
406
        out_items.append((part, ' '.join(value.split(' ')[slice_])))
407
    return _name(out_items, node)
408
funcs['_namePart'] = _namePart
409

    
410
#### Angles
411

    
412
def _compass(items, node):
413
    '''Converts a compass direction (N, NE, NNE, etc.) into a degree heading'''
414
    items = dict(conv_items(strings.ustr, items))
415
    try: value = items['value']
416
    except KeyError: return None # input is empty
417
    
418
    if not value.isupper(): return value # pass through other coordinate formats
419
    try: return util.cast(str, angles.compass2heading(value)) # ignore None
420
    except KeyError, e: raise FormatException(e)
421
funcs['_compass'] = _compass
422

    
423
#### Paths
424

    
425
def _simplifyPath(items, node):
426
    items = dict(items)
427
    try:
428
        next = cast(strings.ustr, items['next'])
429
        require = cast(strings.ustr, items['require'])
430
        root = items['path']
431
    except KeyError, e: raise SyntaxError(e)
432
    
433
    node = root
434
    while node != None:
435
        new_node = xpath.get_1(node, next, allow_rooted=False)
436
        required_node = xpath.get_1(node, require, allow_rooted=False)
437
        if required_node == None or xml_dom.is_empty(required_node):# empty elem
438
            xml_dom.replace(node, new_node) # remove current elem
439
            if node is root: root = new_node # also update root
440
        node = new_node
441
    return root
442
funcs['_simplifyPath'] = _simplifyPath
(31-31/34)