Project

General

Profile

« Previous | Next » 

Revision 5584

sql_io.py: append_csv(): Removed no longer used INSERT mode, since all callers now use the default COPY FROM

View differences:

lib/sql_io.py
227 227
row_num_col_def.name = 'row_num'
228 228
row_num_col_def.type = 'integer'
229 229

  
230
def append_csv(db, table, stream_info, stream, use_copy_from=True):
230
def append_csv(db, table, stream_info, stream):
231 231
    assert sql.table_col_names(db, table) == stream_info.header
232 232
    
233 233
    def log(msg, level=1): db.log_debug(msg, level)
234 234
    
235 235
    dialect = stream_info.dialect
236
    if use_copy_from:
237
        log('Using COPY FROM')
238
        
239
        # Wrap in standardizing stream
240
        cols_ct = len(stream_info.header)
241
        stream = csvs.InputRewriter(csvs.ColCtFilter(csvs.make_reader(stream,
242
            dialect), cols_ct))
243
        dialect = stream.dialect # use default dialect
244
        
245
        # Create COPY FROM statement
246
        copy_from = ('COPY '+table.to_str(db)+' FROM STDIN DELIMITER '
247
            +db.esc_value(dialect.delimiter)+' NULL '+db.esc_value(''))
248
        assert not csvs.is_tsv(dialect)
249
        copy_from += ' CSV'
250
        if dialect.quoting != csv.QUOTE_NONE:
251
            quote_str = db.esc_value(dialect.quotechar)
252
            copy_from += ' QUOTE '+quote_str
253
            if dialect.doublequote: copy_from += ' ESCAPE '+quote_str
254
        copy_from += ';\n'
255
        
256
        log(copy_from, level=2)
257
        try: db.db.cursor().copy_expert(copy_from, stream)
258
        except Exception, e: sql.parse_exception(db, e, recover=True)
259
    else:
260
        log('Using INSERT')
261
        cols_ct = len(stream_info.header)
262
        row_ct = 0
263
        inserted_row_ct = 0
264
        for row in csvs.make_reader(stream, dialect):
265
            row_ct += 1
266
            row = map(strings.to_unicode, row)
267
            util.list_set_length(row, cols_ct) # truncate extra cols
268
            if sql.insert(db, table, row, ignore=True, cacheable=False,
269
                log_level=5) != None:
270
                inserted_row_ct += 1
271
        log('Inserted '+str(inserted_row_ct)+' of '+str(row_ct)+' rows')
236
    
237
    # Wrap in standardizing stream
238
    cols_ct = len(stream_info.header)
239
    stream = csvs.InputRewriter(csvs.ColCtFilter(csvs.make_reader(stream,
240
        dialect), cols_ct))
241
    dialect = stream.dialect # use default dialect
242
    
243
    # Create COPY FROM statement
244
    copy_from = ('COPY '+table.to_str(db)+' FROM STDIN DELIMITER '
245
        +db.esc_value(dialect.delimiter)+' NULL '+db.esc_value(''))
246
    assert not csvs.is_tsv(dialect)
247
    copy_from += ' CSV'
248
    if dialect.quoting != csv.QUOTE_NONE:
249
        quote_str = db.esc_value(dialect.quotechar)
250
        copy_from += ' QUOTE '+quote_str
251
        if dialect.doublequote: copy_from += ' ESCAPE '+quote_str
252
    copy_from += ';\n'
253
    
254
    log(copy_from, level=2)
255
    try: db.db.cursor().copy_expert(copy_from, stream)
256
    except Exception, e: sql.parse_exception(db, e, recover=True)
272 257

  
273 258
def import_csv(db, table, stream):
274 259
    def log(msg, level=1): db.log_debug(msg, level)

Also available in: Unified diff