| 1 |
1942
|
aaronmk
|
#!/usr/bin/env python
|
| 2 |
|
|
# Loads a command's CSV output stream into a PostgreSQL table.
|
| 3 |
4446
|
aaronmk
|
# When no command is specified, just cleans up the specified table.
|
| 4 |
1942
|
aaronmk
|
# The command may be run more than once.
|
| 5 |
|
|
|
| 6 |
|
|
import os.path
|
| 7 |
|
|
import subprocess
|
| 8 |
|
|
import sys
|
| 9 |
|
|
|
| 10 |
|
|
sys.path.append(os.path.dirname(__file__)+"/../lib")
|
| 11 |
|
|
|
| 12 |
|
|
import exc
|
| 13 |
|
|
import opts
|
| 14 |
|
|
import sql
|
| 15 |
3080
|
aaronmk
|
import sql_io
|
| 16 |
2680
|
aaronmk
|
import sql_gen
|
| 17 |
1942
|
aaronmk
|
import streams
|
| 18 |
1963
|
aaronmk
|
import strings
|
| 19 |
1965
|
aaronmk
|
import util
|
| 20 |
1942
|
aaronmk
|
|
| 21 |
|
|
def main():
|
| 22 |
|
|
# Usage
|
| 23 |
|
|
env_names = []
|
| 24 |
|
|
def usage_err():
|
| 25 |
|
|
raise SystemExit('Usage: '+opts.env_usage(env_names)+' '+sys.argv[0]
|
| 26 |
4446
|
aaronmk
|
+' [input_cmd args...]')
|
| 27 |
1942
|
aaronmk
|
|
| 28 |
|
|
# Parse args
|
| 29 |
|
|
input_cmd = sys.argv[1:]
|
| 30 |
|
|
|
| 31 |
|
|
# Get config from env vars
|
| 32 |
|
|
table = opts.get_env_var('table', None, env_names)
|
| 33 |
|
|
schema = opts.get_env_var('schema', 'public', env_names)
|
| 34 |
|
|
db_config = opts.get_env_vars(sql.db_config_names, None, env_names)
|
| 35 |
3271
|
aaronmk
|
verbosity = util.cast(float, opts.get_env_var('verbosity', 3, env_names))
|
| 36 |
1942
|
aaronmk
|
|
| 37 |
4446
|
aaronmk
|
if not (table != None and 'engine' in db_config): usage_err()
|
| 38 |
2890
|
aaronmk
|
|
| 39 |
1942
|
aaronmk
|
# Connect to DB
|
| 40 |
2680
|
aaronmk
|
def log(msg, level=1):
|
| 41 |
|
|
'''Higher level -> more verbose'''
|
| 42 |
3610
|
aaronmk
|
if level <= verbosity:
|
| 43 |
|
|
sys.stderr.write(strings.to_raw_str(msg.rstrip('\n')+'\n'))
|
| 44 |
2680
|
aaronmk
|
db = sql.connect(db_config, log_debug=log)
|
| 45 |
1942
|
aaronmk
|
|
| 46 |
3138
|
aaronmk
|
table = sql_gen.Table(table, schema)
|
| 47 |
|
|
|
| 48 |
4996
|
aaronmk
|
use_copy_from = True
|
| 49 |
1963
|
aaronmk
|
|
| 50 |
|
|
# Loads data into the table using the currently-selected approach.
|
| 51 |
2680
|
aaronmk
|
def load():
|
| 52 |
1942
|
aaronmk
|
# Open input stream
|
| 53 |
|
|
proc = subprocess.Popen(input_cmd, stdout=subprocess.PIPE, bufsize=-1)
|
| 54 |
|
|
in_ = proc.stdout
|
| 55 |
4996
|
aaronmk
|
line_in = streams.ProgressInputStream(in_, sys.stderr, n=1000)
|
| 56 |
1942
|
aaronmk
|
|
| 57 |
4996
|
aaronmk
|
# Import data
|
| 58 |
5027
|
aaronmk
|
try: sql_io.import_csv(db, table, line_in, use_copy_from)
|
| 59 |
4996
|
aaronmk
|
finally:
|
| 60 |
|
|
line_in.close() # also closes proc.stdout
|
| 61 |
|
|
proc.wait()
|
| 62 |
1942
|
aaronmk
|
|
| 63 |
4446
|
aaronmk
|
if input_cmd != []:
|
| 64 |
|
|
try: load()
|
| 65 |
|
|
except sql.DatabaseErrors, e:
|
| 66 |
4996
|
aaronmk
|
if use_copy_from: # first try
|
| 67 |
4446
|
aaronmk
|
exc.print_ex(e, plain=True)
|
| 68 |
4996
|
aaronmk
|
use_copy_from = False
|
| 69 |
4446
|
aaronmk
|
load() # try again with different approach
|
| 70 |
|
|
else: raise
|
| 71 |
4996
|
aaronmk
|
else: sql_io.cleanup_table(db, table)
|
| 72 |
1942
|
aaronmk
|
|
| 73 |
|
|
main()
|
