Revision 5590
Added by Aaron Marcuse-Kubitza about 12 years ago
| sql_io.py | ||
|---|---|---|
| 229 | 229 |
row_num_col_def.name = 'row_num' |
| 230 | 230 |
row_num_col_def.type = 'integer' |
| 231 | 231 |
|
| 232 |
def append_csv(db, table, stream_info, stream):
|
|
| 233 |
assert sql.table_col_names(db, table) == stream_info.header
|
|
| 232 |
def append_csv(db, table, reader, header):
|
|
| 233 |
assert sql.table_col_names(db, table) == header |
|
| 234 | 234 |
|
| 235 | 235 |
def log(msg, level=1): db.log_debug(msg, level) |
| 236 | 236 |
|
| 237 |
dialect = stream_info.dialect |
|
| 238 |
|
|
| 239 | 237 |
# Wrap in standardizing stream |
| 240 |
cols_ct = len(stream_info.header)
|
|
| 238 |
cols_ct = len(header) |
|
| 241 | 239 |
stream = csvs.InputRewriter(streams.ProgressInputStream(csvs.StreamFilter( |
| 242 |
csvs.ColCtFilter(csvs.make_reader(stream, dialect), cols_ct)),
|
|
| 243 |
sys.stderr, msg='Read %d row(s)', n=1000))
|
|
| 240 |
csvs.ColCtFilter(reader, cols_ct)), sys.stderr, msg='Read %d row(s)',
|
|
| 241 |
n=1000)) |
|
| 244 | 242 |
dialect = stream.dialect # use default dialect |
| 245 | 243 |
|
| 246 | 244 |
# Create COPY FROM statement |
| ... | ... | |
| 262 | 260 |
def log(msg, level=1): db.log_debug(msg, level) |
| 263 | 261 |
|
| 264 | 262 |
# Get format info |
| 265 |
info = csvs.stream_info(stream, parse_header=True)
|
|
| 266 |
col_names = map(strings.to_unicode, info.header)
|
|
| 263 |
reader, header = csvs.reader_and_header(stream)
|
|
| 264 |
col_names = map(strings.to_unicode, header) |
|
| 267 | 265 |
for i, col in enumerate(col_names): # replace empty column names |
| 268 | 266 |
if col == '': col_names[i] = 'column_'+str(i) |
| 269 | 267 |
|
| ... | ... | |
| 283 | 281 |
sql.truncate(db, table) |
| 284 | 282 |
|
| 285 | 283 |
# Load the data |
| 286 |
def load(): append_csv(db, table, info, stream)
|
|
| 284 |
def load(): append_csv(db, table, reader, header)
|
|
| 287 | 285 |
sql.with_savepoint(db, load) |
| 288 | 286 |
|
| 289 | 287 |
cleanup_table(db, table) |
Also available in: Unified diff
sql_io.py: append_csv(): Take a reader and header rather than a stream_info and stream to allow callers to use the simpler csvs.reader_and_header() function. This also allows callers to pass in a wrapped CSV reader for filtering, etc.