Revision 3294
Added by Aaron Marcuse-Kubitza over 12 years ago
| lib/sql_io.py | ||
|---|---|---|
| 339 | 339 |
log_debug('Ignoring all rows')
|
| 340 | 340 |
limit_ref[0] = 0 # just create an empty pkeys table |
| 341 | 341 |
|
| 342 |
not_null_cols = set() |
|
| 342 | 343 |
def ignore(in_col, value, e): |
| 343 | 344 |
track_data_error(db, errors_table_, in_col.srcs, value, |
| 344 | 345 |
e.cause.pgcode, e.cause.pgerror) |
| 345 | 346 |
log_debug('Ignoring rows with '+strings.as_tt(repr(in_col))+' = '
|
| 346 | 347 |
+strings.as_tt(repr(value))) |
| 347 |
|
|
| 348 |
not_null_cols = set() |
|
| 349 |
def remove_rows(in_col, value, e): |
|
| 350 |
ignore(in_col, value, e) |
|
| 351 |
for table in insert_in_tables: # must delete from all copies |
|
| 352 |
sql.add_index(db, in_col, table) # enable fast filtering |
|
| 353 |
filter_ = sql_gen.ColValueCond(in_col, value) |
|
| 354 |
sql.delete(db, table, filter_) |
|
| 355 |
if value == None: not_null_cols.add(in_col) |
|
| 356 |
|
|
| 357 |
def invalid2null(in_col, value, e): |
|
| 358 |
if in_col in not_null_cols: remove_rows(in_col, value, e) |
|
| 348 |
|
|
| 349 |
if value != None and in_col not in not_null_cols: |
|
| 350 |
# Try just mapping the value to NULL |
|
| 351 |
sql.update(db, insert_in_table, [(in_col, None)], |
|
| 352 |
sql_gen.ColValueCond(in_col, value)) |
|
| 359 | 353 |
else: |
| 360 |
ignore(in_col, value, e) |
|
| 361 |
sql.update(db, in_table, [(in_col, None)], |
|
| 362 |
sql_gen.ColValueCond(in_col, value)) |
|
| 354 |
for table in insert_in_tables: # must delete from all copies |
|
| 355 |
sql.add_index(db, in_col, table) # enable fast filtering |
|
| 356 |
filter_ = sql_gen.ColValueCond(in_col, value) |
|
| 357 |
sql.delete(db, table, filter_) |
|
| 358 |
if value == None: not_null_cols.add(in_col) |
|
| 363 | 359 |
|
| 364 | 360 |
def insert_pkeys_table(which): |
| 365 | 361 |
return sql_gen.Table(sql_gen.concat(in_table.name, |
| ... | ... | |
| 422 | 418 |
except sql.InvalidValueException, e: |
| 423 | 419 |
log_exc(e) |
| 424 | 420 |
|
| 425 |
invalid2null(in_col, e.value, e)
|
|
| 421 |
ignore(in_col, e.value, e)
|
|
| 426 | 422 |
except sql.DuplicateKeyException, e: |
| 427 | 423 |
log_exc(e) |
| 428 | 424 |
|
| ... | ... | |
| 448 | 444 |
except KeyError: |
| 449 | 445 |
log_debug('Missing mapping for NOT NULL column '+out_col)
|
| 450 | 446 |
remove_all_rows() |
| 451 |
else: remove_rows(in_col, None, e)
|
|
| 447 |
else: ignore(in_col, None, e)
|
|
| 452 | 448 |
except sql.DatabaseErrors, e: |
| 453 | 449 |
log_exc(e) |
| 454 | 450 |
|
Also available in: Unified diff
sql_io.py: put_table(): Merged remove_rows() and invalid2null() into one ignore() function that chooses the action (map to NULL or delete) depending on the value and whether NULLs have been filtered out of the column