Revision 5584
Added by Aaron Marcuse-Kubitza about 12 years ago
lib/sql_io.py | ||
---|---|---|
227 | 227 |
row_num_col_def.name = 'row_num' |
228 | 228 |
row_num_col_def.type = 'integer' |
229 | 229 |
|
230 |
def append_csv(db, table, stream_info, stream, use_copy_from=True):
|
|
230 |
def append_csv(db, table, stream_info, stream): |
|
231 | 231 |
assert sql.table_col_names(db, table) == stream_info.header |
232 | 232 |
|
233 | 233 |
def log(msg, level=1): db.log_debug(msg, level) |
234 | 234 |
|
235 | 235 |
dialect = stream_info.dialect |
236 |
if use_copy_from: |
|
237 |
log('Using COPY FROM') |
|
238 |
|
|
239 |
# Wrap in standardizing stream |
|
240 |
cols_ct = len(stream_info.header) |
|
241 |
stream = csvs.InputRewriter(csvs.ColCtFilter(csvs.make_reader(stream, |
|
242 |
dialect), cols_ct)) |
|
243 |
dialect = stream.dialect # use default dialect |
|
244 |
|
|
245 |
# Create COPY FROM statement |
|
246 |
copy_from = ('COPY '+table.to_str(db)+' FROM STDIN DELIMITER ' |
|
247 |
+db.esc_value(dialect.delimiter)+' NULL '+db.esc_value('')) |
|
248 |
assert not csvs.is_tsv(dialect) |
|
249 |
copy_from += ' CSV' |
|
250 |
if dialect.quoting != csv.QUOTE_NONE: |
|
251 |
quote_str = db.esc_value(dialect.quotechar) |
|
252 |
copy_from += ' QUOTE '+quote_str |
|
253 |
if dialect.doublequote: copy_from += ' ESCAPE '+quote_str |
|
254 |
copy_from += ';\n' |
|
255 |
|
|
256 |
log(copy_from, level=2) |
|
257 |
try: db.db.cursor().copy_expert(copy_from, stream) |
|
258 |
except Exception, e: sql.parse_exception(db, e, recover=True) |
|
259 |
else: |
|
260 |
log('Using INSERT') |
|
261 |
cols_ct = len(stream_info.header) |
|
262 |
row_ct = 0 |
|
263 |
inserted_row_ct = 0 |
|
264 |
for row in csvs.make_reader(stream, dialect): |
|
265 |
row_ct += 1 |
|
266 |
row = map(strings.to_unicode, row) |
|
267 |
util.list_set_length(row, cols_ct) # truncate extra cols |
|
268 |
if sql.insert(db, table, row, ignore=True, cacheable=False, |
|
269 |
log_level=5) != None: |
|
270 |
inserted_row_ct += 1 |
|
271 |
log('Inserted '+str(inserted_row_ct)+' of '+str(row_ct)+' rows') |
|
236 |
|
|
237 |
# Wrap in standardizing stream |
|
238 |
cols_ct = len(stream_info.header) |
|
239 |
stream = csvs.InputRewriter(csvs.ColCtFilter(csvs.make_reader(stream, |
|
240 |
dialect), cols_ct)) |
|
241 |
dialect = stream.dialect # use default dialect |
|
242 |
|
|
243 |
# Create COPY FROM statement |
|
244 |
copy_from = ('COPY '+table.to_str(db)+' FROM STDIN DELIMITER ' |
|
245 |
+db.esc_value(dialect.delimiter)+' NULL '+db.esc_value('')) |
|
246 |
assert not csvs.is_tsv(dialect) |
|
247 |
copy_from += ' CSV' |
|
248 |
if dialect.quoting != csv.QUOTE_NONE: |
|
249 |
quote_str = db.esc_value(dialect.quotechar) |
|
250 |
copy_from += ' QUOTE '+quote_str |
|
251 |
if dialect.doublequote: copy_from += ' ESCAPE '+quote_str |
|
252 |
copy_from += ';\n' |
|
253 |
|
|
254 |
log(copy_from, level=2) |
|
255 |
try: db.db.cursor().copy_expert(copy_from, stream) |
|
256 |
except Exception, e: sql.parse_exception(db, e, recover=True) |
|
272 | 257 |
|
273 | 258 |
def import_csv(db, table, stream): |
274 | 259 |
def log(msg, level=1): db.log_debug(msg, level) |
Also available in: Unified diff
sql_io.py: append_csv(): Removed no longer used INSERT mode, since all callers now use the default COPY FROM