| 1 |
8291
|
aaronmk
|
#!/bin/bash -e
|
| 2 |
8273
|
aaronmk
|
|
| 3 |
|
|
if false; then #### run script template:
|
| 4 |
8291
|
aaronmk
|
#!/bin/bash -e
|
| 5 |
8273
|
aaronmk
|
. "$(dirname "${BASH_SOURCE[0]}")"/path/to/table.run
|
| 6 |
8161
|
aaronmk
|
|
| 7 |
9039
|
aaronmk
|
func_override import__table_run
|
| 8 |
9074
|
aaronmk
|
import()
|
| 9 |
8273
|
aaronmk
|
{
|
| 10 |
8881
|
aaronmk
|
echo_func
|
| 11 |
8273
|
aaronmk
|
before_import_cmds
|
| 12 |
9681
|
aaronmk
|
self_make import__table_run "$@"
|
| 13 |
8273
|
aaronmk
|
after_import_cmds
|
| 14 |
|
|
}
|
| 15 |
|
|
fi ####
|
| 16 |
8213
|
aaronmk
|
|
| 17 |
8296
|
aaronmk
|
. "$(dirname "${BASH_SOURCE[0]}")"/import.run
|
| 18 |
9854
|
aaronmk
|
.rel subdir.run
|
| 19 |
|
|
.rel ../sh/db_make.sh
|
| 20 |
8273
|
aaronmk
|
|
| 21 |
8705
|
aaronmk
|
if self_not_included; then
|
| 22 |
|
|
|
| 23 |
11602
|
aaronmk
|
##### utils
|
| 24 |
|
|
|
| 25 |
9585
|
aaronmk
|
: "${table=$subdir}"; export table
|
| 26 |
|
|
|
| 27 |
|
|
table_make() { subdir_make "$@"; }
|
| 28 |
|
|
|
| 29 |
9935
|
aaronmk
|
install_log_rel=logs/install.log.sql
|
| 30 |
|
|
|
| 31 |
9934
|
aaronmk
|
table_make_install()
|
| 32 |
|
|
{
|
| 33 |
11769
|
aaronmk
|
echo_func
|
| 34 |
|
|
if ! remaking && pg_table_exists; then return 0; fi
|
| 35 |
|
|
local install_log="${install_log-$top_dir/$install_log_rel}"
|
| 36 |
9934
|
aaronmk
|
local verbosity_min= # install logs require default verbosity
|
| 37 |
11770
|
aaronmk
|
if ! remaking && test -e "$install_log"; then local_export noclobber=1; fi
|
| 38 |
|
|
table_make "$@"
|
| 39 |
9934
|
aaronmk
|
}
|
| 40 |
|
|
|
| 41 |
9936
|
aaronmk
|
datasrc_make_install()
|
| 42 |
|
|
{
|
| 43 |
|
|
echo_func
|
| 44 |
|
|
subdir=. install_log="$top_dir/../$install_log_rel" table_make_install "$@"
|
| 45 |
|
|
}
|
| 46 |
|
|
|
| 47 |
10248
|
aaronmk
|
postprocess_sql="$top_dir"/postprocess.sql
|
| 48 |
|
|
|
| 49 |
10249
|
aaronmk
|
is_view="$(test -e "$postprocess_sql" \
|
| 50 |
|
|
&& grep -F force_update_view "$postprocess_sql" >/dev/null; exit2bool)"
|
| 51 |
|
|
|
| 52 |
10368
|
aaronmk
|
map_table="\"~$table.map\"" # sort after other tables to avoid clutter
|
| 53 |
|
|
|
| 54 |
11602
|
aaronmk
|
##### targets
|
| 55 |
|
|
|
| 56 |
|
|
load_data()
|
| 57 |
|
|
{
|
| 58 |
|
|
echo_func; set_make_vars
|
| 59 |
|
|
if remaking; then in_top_dir rm -f header.csv; fi # remake it
|
| 60 |
|
|
datasrc_make_install schema
|
| 61 |
|
|
is_view="$is_view" table_make_install ${_remake:+re}install # just the table
|
| 62 |
|
|
}
|
| 63 |
|
|
|
| 64 |
|
|
trim_table()
|
| 65 |
|
|
{
|
| 66 |
|
|
echo_func; set_make_vars
|
| 67 |
|
|
if test "$is_view"; then return 0; fi # do not modify view columns
|
| 68 |
|
|
psql <<EOF
|
| 69 |
|
|
SELECT util.trim('"$table"', '$map_table')
|
| 70 |
|
|
EOF
|
| 71 |
|
|
}
|
| 72 |
|
|
|
| 73 |
10369
|
aaronmk
|
reset_col_names()
|
| 74 |
|
|
{
|
| 75 |
|
|
echo_func; set_make_vars
|
| 76 |
|
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
| 77 |
|
|
psql <<EOF
|
| 78 |
|
|
SELECT util.reset_col_names('"$table"', '$map_table')
|
| 79 |
|
|
EOF
|
| 80 |
|
|
}
|
| 81 |
|
|
|
| 82 |
9074
|
aaronmk
|
map_table()
|
| 83 |
10210
|
aaronmk
|
# note that collisions may prevent all renames from being made at once.
|
| 84 |
10211
|
aaronmk
|
# if this is the case, run map_table repeatedly until no more renames are made:
|
| 85 |
|
|
# $ while true; do .../run map_table; done
|
| 86 |
10210
|
aaronmk
|
# collisions may result if the staging table gets messed up (e.g. due to missing
|
| 87 |
|
|
# input columns in map.csv).
|
| 88 |
8161
|
aaronmk
|
{
|
| 89 |
9838
|
aaronmk
|
echo_func; set_make_vars
|
| 90 |
10250
|
aaronmk
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
| 91 |
|
|
|
| 92 |
10370
|
aaronmk
|
if remaking; then reset_col_names; fi
|
| 93 |
9386
|
aaronmk
|
table_make map.csv
|
| 94 |
8214
|
aaronmk
|
psql <<EOF
|
| 95 |
10156
|
aaronmk
|
SELECT util.reset_map_table('$map_table');
|
| 96 |
|
|
ALTER TABLE $map_table DISABLE TRIGGER map_filter_insert;
|
| 97 |
|
|
\copy $map_table FROM 'map.csv' CSV HEADER;
|
| 98 |
10366
|
aaronmk
|
SELECT util.set_col_names_with_metadata('"$table"', '$map_table');
|
| 99 |
8161
|
aaronmk
|
EOF
|
| 100 |
|
|
}
|
| 101 |
8198
|
aaronmk
|
|
| 102 |
11913
|
aaronmk
|
test_() { echo_func; set_make_vars; piped_cmd yes|table_make test; }
|
| 103 |
|
|
|
| 104 |
12691
|
aaronmk
|
mappings()
|
| 105 |
8245
|
aaronmk
|
{
|
| 106 |
9838
|
aaronmk
|
echo_func; set_make_vars
|
| 107 |
9047
|
aaronmk
|
public_schema_exists || return 0
|
| 108 |
11000
|
aaronmk
|
if remaking; then in_top_dir rm -f map.csv; fi # remake it
|
| 109 |
11913
|
aaronmk
|
test_
|
| 110 |
8245
|
aaronmk
|
}
|
| 111 |
8273
|
aaronmk
|
|
| 112 |
10273
|
aaronmk
|
custom_postprocess() # overridable
|
| 113 |
8273
|
aaronmk
|
{
|
| 114 |
9838
|
aaronmk
|
echo_func; set_make_vars
|
| 115 |
10248
|
aaronmk
|
local file="$postprocess_sql"
|
| 116 |
8273
|
aaronmk
|
if test -e "$file"; then psql "$@"; fi
|
| 117 |
|
|
}
|
| 118 |
|
|
|
| 119 |
11602
|
aaronmk
|
mk_derived()
|
| 120 |
|
|
{ echo_func; set_make_vars; "$root_dir"/schemas/VegCore/mk_derived; }
|
| 121 |
9392
|
aaronmk
|
|
| 122 |
10274
|
aaronmk
|
postprocess()
|
| 123 |
8273
|
aaronmk
|
{
|
| 124 |
9838
|
aaronmk
|
echo_func; set_make_vars
|
| 125 |
10275
|
aaronmk
|
kw_params can_test; local can_test="${can_test-1}"
|
| 126 |
10372
|
aaronmk
|
if remaking; then trim_table; fi
|
| 127 |
8282
|
aaronmk
|
map_table
|
| 128 |
10273
|
aaronmk
|
custom_postprocess
|
| 129 |
10171
|
aaronmk
|
mk_derived
|
| 130 |
12691
|
aaronmk
|
if test "$can_test"; then mappings; fi
|
| 131 |
8273
|
aaronmk
|
}
|
| 132 |
8705
|
aaronmk
|
|
| 133 |
10274
|
aaronmk
|
import()
|
| 134 |
|
|
{
|
| 135 |
|
|
echo_func; set_make_vars
|
| 136 |
|
|
self_make load_data
|
| 137 |
|
|
postprocess
|
| 138 |
|
|
}
|
| 139 |
|
|
|
| 140 |
8705
|
aaronmk
|
fi
|
