Revision 3081
Added by Aaron Marcuse-Kubitza over 12 years ago
| lib/sql.py | ||
|---|---|---|
| 1197 | 1197 |
def empty_db(db, schema='public', **kw_args): |
| 1198 | 1198 |
'''For kw_args, see tables()''' |
| 1199 | 1199 |
for table in tables(db, schema, **kw_args): truncate(db, table, schema) |
| 1200 |
|
|
| 1201 |
##### Database management |
|
| 1202 |
|
|
| 1203 |
##### Data cleanup |
|
| 1204 |
|
|
| 1205 |
def cleanup_table(db, table, cols): |
|
| 1206 |
table = sql_gen.as_Table(table) |
|
| 1207 |
cols = map(sql_gen.as_Col, cols) |
|
| 1208 |
|
|
| 1209 |
expr = ('nullif(nullif(trim(both from %s), '+db.esc_value('')+'), '
|
|
| 1210 |
+db.esc_value(r'\N')+')') |
|
| 1211 |
changes = [(v, sql_gen.CustomCode(expr % v.to_str(db))) |
|
| 1212 |
for v in cols] |
|
| 1213 |
|
|
| 1214 |
update(db, table, changes, in_place=True) |
|
| lib/sql_io.py | ||
|---|---|---|
| 7 | 7 |
import strings |
| 8 | 8 |
import util |
| 9 | 9 |
|
| 10 |
##### Data cleanup |
|
| 11 |
|
|
| 12 |
def cleanup_table(db, table, cols): |
|
| 13 |
table = sql_gen.as_Table(table) |
|
| 14 |
cols = map(sql_gen.as_Col, cols) |
|
| 15 |
|
|
| 16 |
expr = ('nullif(nullif(trim(both from %s), '+db.esc_value('')+'), '
|
|
| 17 |
+db.esc_value(r'\N')+')') |
|
| 18 |
changes = [(v, sql_gen.CustomCode(expr % v.to_str(db))) |
|
| 19 |
for v in cols] |
|
| 20 |
|
|
| 21 |
sql.update(db, table, changes, in_place=True) |
|
| 22 |
|
|
| 10 | 23 |
##### Error tracking |
| 11 | 24 |
|
| 12 | 25 |
def track_data_error(db, errors_table, cols, value, error_code, error): |
| bin/csv2db | ||
|---|---|---|
| 124 | 124 |
sql.with_savepoint(db, load_) |
| 125 | 125 |
|
| 126 | 126 |
log('Cleaning up table')
|
| 127 |
sql.cleanup_table(db, table, col_names) |
|
| 127 |
sql_io.cleanup_table(db, table, col_names)
|
|
| 128 | 128 |
|
| 129 | 129 |
log('Adding indexes')
|
| 130 | 130 |
for col in typed_cols[1:]: # exclude pkey |
Also available in: Unified diff
Moved Data cleanup from sql.py to sql_io.py