Revision 1901
Added by Aaron Marcuse-Kubitza over 12 years ago
| lib/sql.py | ||
|---|---|---|
| 2 | 2 |
|
| 3 | 3 |
import copy |
| 4 | 4 |
import re |
| 5 |
import sys |
|
| 6 | 5 |
import warnings |
| 7 | 6 |
|
| 8 | 7 |
import exc |
| ... | ... | |
| 77 | 76 |
|
| 78 | 77 |
def _query_lookup(query, params): return (query, util.cast(tuple, params)) |
| 79 | 78 |
|
| 79 |
log_debug_none = lambda msg: None |
|
| 80 |
|
|
| 80 | 81 |
class DbConn: |
| 81 |
def __init__(self, db_config, serializable=True, debug=False):
|
|
| 82 |
def __init__(self, db_config, serializable=True, log_debug=log_debug_none):
|
|
| 82 | 83 |
self.db_config = db_config |
| 83 | 84 |
self.serializable = serializable |
| 84 |
self.debug = debug
|
|
| 85 |
self.log_debug = log_debug
|
|
| 85 | 86 |
|
| 86 | 87 |
self.__db = None |
| 87 | 88 |
self.pkeys = {}
|
| ... | ... | |
| 161 | 162 |
except Exception, e: |
| 162 | 163 |
_add_cursor_info(e, cur) |
| 163 | 164 |
raise |
| 164 |
if self.debug:
|
|
| 165 |
sys.stderr.write(strings.one_line(get_cur_query(cur))+'\n')
|
|
| 165 |
if self.log_debug != log_debug_none: # only compute msg if needed
|
|
| 166 |
self.log_debug(strings.one_line(get_cur_query(cur)))
|
|
| 166 | 167 |
return cur |
| 167 | 168 |
else: return self.CacheCursor(actual_query, result) |
| 168 | 169 |
|
| bin/map | ||
|---|---|---|
| 95 | 95 |
|
| 96 | 96 |
# Logging |
| 97 | 97 |
def log(msg, on=verbose): |
| 98 |
if on: sys.stderr.write(msg) |
|
| 99 |
def log_start(action, on=verbose): log(action+'...\n', on) |
|
| 98 |
if on: sys.stderr.write(msg+'\n') |
|
| 99 |
if debug: log_debug = lambda msg: log(msg, debug) |
|
| 100 |
else: log_debug = sql.log_debug_none |
|
| 100 | 101 |
|
| 101 | 102 |
# Parse args |
| 102 | 103 |
map_paths = sys.argv[1:] |
| ... | ... | |
| 105 | 106 |
else: map_paths = [None] |
| 106 | 107 |
|
| 107 | 108 |
def connect_db(db_config): |
| 108 |
log_start('Connecting to '+sql.db_config_str(db_config))
|
|
| 109 |
return sql.connect(db_config, debug=debug)
|
|
| 109 |
log('Connecting to '+sql.db_config_str(db_config))
|
|
| 110 |
return sql.connect(db_config, log_debug=log_debug)
|
|
| 110 | 111 |
|
| 111 | 112 |
if end != None: end_str = str(end-1) |
| 112 | 113 |
else: end_str = 'end' |
| 113 |
log_start('Processing input rows '+str(start)+'-'+end_str)
|
|
| 114 |
log('Processing input rows '+str(start)+'-'+end_str)
|
|
| 114 | 115 |
|
| 115 | 116 |
ex_tracker = exc.ExPercentTracker(iter_text='row') |
| 116 | 117 |
profiler = profiling.ItersProfiler(start_now=True, iter_text='row') |
| 117 | 118 |
|
| 118 | 119 |
# Parallel processing |
| 119 | 120 |
pool = parallel.MultiProducerPool(cpus) |
| 120 |
log_start('Using '+str(pool.process_ct)+' parallel CPUs')
|
|
| 121 |
log('Using '+str(pool.process_ct)+' parallel CPUs')
|
|
| 121 | 122 |
|
| 122 | 123 |
doc = xml_dom.create_doc() |
| 123 | 124 |
root = doc.documentElement |
| ... | ... | |
| 202 | 203 |
for in_, out in mappings: |
| 203 | 204 |
value = metadata_value(in_) |
| 204 | 205 |
if value == None: |
| 205 |
log_start('Getting '+str(in_), debug)
|
|
| 206 |
log_debug('Getting '+str(in_))
|
|
| 206 | 207 |
value = cleanup(get_value(in_, row)) |
| 207 | 208 |
if value != None: |
| 208 |
log_start('Putting '+str(out), debug)
|
|
| 209 |
log_debug('Putting '+str(out))
|
|
| 209 | 210 |
xpath.put_obj(root, out, row_id, has_types, value) |
| 210 | 211 |
return process_rows(process_row, rows, **kw_args) |
| 211 | 212 |
|
Also available in: Unified diff
sql.py: DbConn: Allow creator to provide a log function to call on debug messages, instead of using stderr directly