Project

General

Profile

« Previous | Next » 

Revision 3293

Added by Aaron Marcuse-Kubitza over 12 years ago

sql_io.py: put_table(): remove_rows(): Delete the rows containing the invalid value instead of filtering them out of each select, so that the filtering can be profiled separately from the insertion. This also requires deleting rows with invalid non-NULL values instead of mapping them to NULL if NULLs have already been filtered out of the column in question.

View differences:

lib/sql_io.py
345 345 
        log_debug('Ignoring rows with '+strings.as_tt(repr(in_col))+' = '
346 346 
            +strings.as_tt(repr(value)))
347 347
348 
    not_null_cols = set()
348 349 
    def remove_rows(in_col, value, e):
349 350 
        ignore(in_col, value, e)
350 
        cond = (in_col, sql_gen.CompareCond(value, '!='))
351 
        assert cond not in conds # avoid infinite loops
352 
        conds.add(cond)
351 
        for table in insert_in_tables: # must delete from all copies
352 
            sql.add_index(db, in_col, table) # enable fast filtering
353 
            filter_ = sql_gen.ColValueCond(in_col, value)
354 
            sql.delete(db, table, filter_)
355 
        if value == None: not_null_cols.add(in_col)
353 356
354 357 
    def invalid2null(in_col, value, e):
355 
        ignore(in_col, value, e)
356 
        sql.update(db, in_table, [(in_col, None)],
357 
            sql_gen.ColValueCond(in_col, value))
358 
        if in_col in not_null_cols: remove_rows(in_col, value, e)
359 
        else:
360 
            ignore(in_col, value, e)
361 
            sql.update(db, in_table, [(in_col, None)],
362 
                sql_gen.ColValueCond(in_col, value))
358 363
359 364 
    def insert_pkeys_table(which):
360 365 
        return sql_gen.Table(sql_gen.concat(in_table.name,

Also available in: Unified diff