| 229 |
229 |
row_num_col_def.name = 'row_num'
|
| 230 |
230 |
row_num_col_def.type = 'integer'
|
| 231 |
231 |
|
| 232 |
|
def append_csv(db, table, stream_info, stream):
|
| 233 |
|
assert sql.table_col_names(db, table) == stream_info.header
|
|
232 |
def append_csv(db, table, reader, header):
|
|
233 |
assert sql.table_col_names(db, table) == header
|
| 234 |
234 |
|
| 235 |
235 |
def log(msg, level=1): db.log_debug(msg, level)
|
| 236 |
236 |
|
| 237 |
|
dialect = stream_info.dialect
|
| 238 |
|
|
| 239 |
237 |
# Wrap in standardizing stream
|
| 240 |
|
cols_ct = len(stream_info.header)
|
|
238 |
cols_ct = len(header)
|
| 241 |
239 |
stream = csvs.InputRewriter(streams.ProgressInputStream(csvs.StreamFilter(
|
| 242 |
|
csvs.ColCtFilter(csvs.make_reader(stream, dialect), cols_ct)),
|
| 243 |
|
sys.stderr, msg='Read %d row(s)', n=1000))
|
|
240 |
csvs.ColCtFilter(reader, cols_ct)), sys.stderr, msg='Read %d row(s)',
|
|
241 |
n=1000))
|
| 244 |
242 |
dialect = stream.dialect # use default dialect
|
| 245 |
243 |
|
| 246 |
244 |
# Create COPY FROM statement
|
| ... | ... | |
| 262 |
260 |
def log(msg, level=1): db.log_debug(msg, level)
|
| 263 |
261 |
|
| 264 |
262 |
# Get format info
|
| 265 |
|
info = csvs.stream_info(stream, parse_header=True)
|
| 266 |
|
col_names = map(strings.to_unicode, info.header)
|
|
263 |
reader, header = csvs.reader_and_header(stream)
|
|
264 |
col_names = map(strings.to_unicode, header)
|
| 267 |
265 |
for i, col in enumerate(col_names): # replace empty column names
|
| 268 |
266 |
if col == '': col_names[i] = 'column_'+str(i)
|
| 269 |
267 |
|
| ... | ... | |
| 283 |
281 |
sql.truncate(db, table)
|
| 284 |
282 |
|
| 285 |
283 |
# Load the data
|
| 286 |
|
def load(): append_csv(db, table, info, stream)
|
|
284 |
def load(): append_csv(db, table, reader, header)
|
| 287 |
285 |
sql.with_savepoint(db, load)
|
| 288 |
286 |
|
| 289 |
287 |
cleanup_table(db, table)
|
sql_io.py: append_csv(): Take a reader and header rather than a stream_info and stream to allow callers to use the simpler csvs.reader_and_header() function. This also allows callers to pass in a wrapped CSV reader for filtering, etc.