Revision 2758
Added by Aaron Marcuse-Kubitza over 12 years ago
| lib/sql.py | ||
|---|---|---|
| 757 | 757 |
return mk_insert_select(db, errors_table, name_only_cols, |
| 758 | 758 |
*mk_select(db, joins, select_cols, order_by=None))[0] |
| 759 | 759 |
|
| 760 |
def track_data_error(db, errors_table, *args, **kw_args): |
|
| 761 |
''' |
|
| 762 |
@param errors_table If None, does nothing. |
|
| 763 |
''' |
|
| 764 |
if errors_table == None: return |
|
| 765 |
run_query(db, mk_track_data_error(db, errors_table, *args, **kw_args), |
|
| 766 |
cacheable=True, log_level=4) |
|
| 767 |
|
|
| 760 | 768 |
def cast(db, type_, col, errors_table=None): |
| 761 | 769 |
'''Casts an (unrenamed) column or value. |
| 762 | 770 |
If a column, converts any errors to warnings. |
| ... | ... | |
| 1191 | 1199 |
log_debug('Returning NULL for all rows')
|
| 1192 | 1200 |
limit_ref[0] = 0 # just create an empty pkeys table |
| 1193 | 1201 |
|
| 1194 |
def ignore(in_col, value): |
|
| 1202 |
def ignore(in_col, value, e): |
|
| 1203 |
track_data_error(db, errors_table_, in_col.srcs, value, e.cause.pgcode, |
|
| 1204 |
e.cause.pgerror) |
|
| 1205 |
|
|
| 1195 | 1206 |
in_col_str = strings.as_tt(repr(in_col)) |
| 1196 | 1207 |
db.log_debug('Adding index on '+in_col_str+' to enable fast filtering',
|
| 1197 | 1208 |
level=2.5) |
| ... | ... | |
| 1199 | 1210 |
|
| 1200 | 1211 |
log_debug('Ignoring rows with '+in_col_str+' = '
|
| 1201 | 1212 |
+strings.as_tt(repr(value))) |
| 1202 |
def remove_rows(in_col, value): |
|
| 1203 |
ignore(in_col, value) |
|
| 1213 |
def remove_rows(in_col, value, e):
|
|
| 1214 |
ignore(in_col, value, e)
|
|
| 1204 | 1215 |
cond = (in_col, sql_gen.CompareCond(value, '!=')) |
| 1205 | 1216 |
assert cond not in conds # avoid infinite loops |
| 1206 | 1217 |
conds.add(cond) |
| 1207 |
def invalid2null(in_col, value): |
|
| 1208 |
ignore(in_col, value) |
|
| 1218 |
def invalid2null(in_col, value, e):
|
|
| 1219 |
ignore(in_col, value, e)
|
|
| 1209 | 1220 |
update(db, in_table, [(in_col, None)], |
| 1210 | 1221 |
sql_gen.ColValueCond(in_col, value)) |
| 1211 | 1222 |
|
| ... | ... | |
| 1258 | 1269 |
except KeyError: |
| 1259 | 1270 |
log_debug('Missing mapping for NOT NULL column '+out_col)
|
| 1260 | 1271 |
remove_all_rows() |
| 1261 |
else: remove_rows(in_col, None) |
|
| 1272 |
else: remove_rows(in_col, None, e)
|
|
| 1262 | 1273 |
except FunctionValueException, e: |
| 1263 | 1274 |
log_exc(e) |
| 1264 | 1275 |
|
| 1265 | 1276 |
func_name = e.name |
| 1266 | 1277 |
value = e.value |
| 1267 | 1278 |
for out_col, in_col in mapping.iteritems(): |
| 1268 |
invalid2null(sql_gen.unwrap_func_call(in_col, func_name), value) |
|
| 1279 |
in_col = sql_gen.unwrap_func_call(in_col, func_name) |
|
| 1280 |
invalid2null(in_col, value, e) |
|
| 1269 | 1281 |
except MissingCastException, e: |
| 1270 | 1282 |
log_exc(e) |
| 1271 | 1283 |
|
Also available in: Unified diff
sql.py: Added track_data_error(). put_table(): ignore(): Take extra e param for the exception. Use track_data_error() to store the invalid value in the errors table.