1 |
8291
|
aaronmk
|
#!/bin/bash -e
|
2 |
8273
|
aaronmk
|
|
3 |
|
|
if false; then #### run script template:
|
4 |
8291
|
aaronmk
|
#!/bin/bash -e
|
5 |
8273
|
aaronmk
|
. "$(dirname "${BASH_SOURCE[0]}")"/path/to/table.run
|
6 |
8161
|
aaronmk
|
|
7 |
9039
|
aaronmk
|
func_override import__table_run
|
8 |
9074
|
aaronmk
|
import()
|
9 |
8273
|
aaronmk
|
{
|
10 |
8881
|
aaronmk
|
echo_func
|
11 |
8273
|
aaronmk
|
before_import_cmds
|
12 |
9681
|
aaronmk
|
self_make import__table_run "$@"
|
13 |
8273
|
aaronmk
|
after_import_cmds
|
14 |
|
|
}
|
15 |
|
|
fi ####
|
16 |
8213
|
aaronmk
|
|
17 |
8296
|
aaronmk
|
. "$(dirname "${BASH_SOURCE[0]}")"/import.run
|
18 |
9854
|
aaronmk
|
.rel subdir.run
|
19 |
|
|
.rel ../sh/db_make.sh
|
20 |
8273
|
aaronmk
|
|
21 |
8705
|
aaronmk
|
if self_not_included; then
|
22 |
|
|
|
23 |
9585
|
aaronmk
|
: "${table=$subdir}"; export table
|
24 |
|
|
|
25 |
|
|
table_make() { subdir_make "$@"; }
|
26 |
|
|
|
27 |
9935
|
aaronmk
|
install_log_rel=logs/install.log.sql
|
28 |
|
|
|
29 |
9934
|
aaronmk
|
table_make_install()
|
30 |
|
|
{
|
31 |
9935
|
aaronmk
|
echo_func; local install_log="${install_log-$top_dir/$install_log_rel}"
|
32 |
9934
|
aaronmk
|
local verbosity_min= # install logs require default verbosity
|
33 |
|
|
set -- table_make "$@"
|
34 |
|
|
if remaking || test ! -e "$install_log"; then "$@" # OK to clobber log
|
35 |
|
|
else benign_error=1 noclobber=1 "$@" || true
|
36 |
|
|
fi
|
37 |
|
|
}
|
38 |
|
|
|
39 |
9936
|
aaronmk
|
datasrc_make_install()
|
40 |
|
|
{
|
41 |
|
|
echo_func
|
42 |
|
|
subdir=. install_log="$top_dir/../$install_log_rel" table_make_install "$@"
|
43 |
|
|
}
|
44 |
|
|
|
45 |
10248
|
aaronmk
|
postprocess_sql="$top_dir"/postprocess.sql
|
46 |
|
|
|
47 |
10249
|
aaronmk
|
is_view="$(test -e "$postprocess_sql" \
|
48 |
|
|
&& grep -F force_update_view "$postprocess_sql" >/dev/null; exit2bool)"
|
49 |
|
|
|
50 |
10368
|
aaronmk
|
map_table="\"~$table.map\"" # sort after other tables to avoid clutter
|
51 |
|
|
|
52 |
10369
|
aaronmk
|
reset_col_names()
|
53 |
|
|
{
|
54 |
|
|
echo_func; set_make_vars
|
55 |
|
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
56 |
|
|
psql <<EOF
|
57 |
|
|
SELECT util.reset_col_names('"$table"', '$map_table')
|
58 |
|
|
EOF
|
59 |
|
|
}
|
60 |
|
|
|
61 |
9074
|
aaronmk
|
map_table()
|
62 |
10210
|
aaronmk
|
# note that collisions may prevent all renames from being made at once.
|
63 |
10211
|
aaronmk
|
# if this is the case, run map_table repeatedly until no more renames are made:
|
64 |
|
|
# $ while true; do .../run map_table; done
|
65 |
10210
|
aaronmk
|
# collisions may result if the staging table gets messed up (e.g. due to missing
|
66 |
|
|
# input columns in map.csv).
|
67 |
8161
|
aaronmk
|
{
|
68 |
9838
|
aaronmk
|
echo_func; set_make_vars
|
69 |
10250
|
aaronmk
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
70 |
|
|
|
71 |
10370
|
aaronmk
|
if remaking; then reset_col_names; fi
|
72 |
9386
|
aaronmk
|
table_make map.csv
|
73 |
8214
|
aaronmk
|
psql <<EOF
|
74 |
10156
|
aaronmk
|
SELECT util.reset_map_table('$map_table');
|
75 |
|
|
ALTER TABLE $map_table DISABLE TRIGGER map_filter_insert;
|
76 |
|
|
\copy $map_table FROM 'map.csv' CSV HEADER;
|
77 |
10366
|
aaronmk
|
SELECT util.set_col_names_with_metadata('"$table"', '$map_table');
|
78 |
8161
|
aaronmk
|
EOF
|
79 |
|
|
}
|
80 |
8198
|
aaronmk
|
|
81 |
9838
|
aaronmk
|
mk_derived()
|
82 |
|
|
{ echo_func; set_make_vars; "$root_dir"/schemas/VegCore/mk_derived; }
|
83 |
8245
|
aaronmk
|
|
84 |
9074
|
aaronmk
|
remake_VegBIEN_mappings()
|
85 |
8245
|
aaronmk
|
{
|
86 |
9838
|
aaronmk
|
echo_func; set_make_vars
|
87 |
9047
|
aaronmk
|
public_schema_exists || return 0
|
88 |
11000
|
aaronmk
|
if remaking; then in_top_dir rm -f map.csv; fi # remake it
|
89 |
9967
|
aaronmk
|
piped_cmd yes|table_make test
|
90 |
8245
|
aaronmk
|
}
|
91 |
8273
|
aaronmk
|
|
92 |
10273
|
aaronmk
|
custom_postprocess() # overridable
|
93 |
8273
|
aaronmk
|
{
|
94 |
9838
|
aaronmk
|
echo_func; set_make_vars
|
95 |
10248
|
aaronmk
|
local file="$postprocess_sql"
|
96 |
8273
|
aaronmk
|
if test -e "$file"; then psql "$@"; fi
|
97 |
|
|
}
|
98 |
|
|
|
99 |
9836
|
aaronmk
|
load_data()
|
100 |
9392
|
aaronmk
|
{
|
101 |
9838
|
aaronmk
|
echo_func; set_make_vars
|
102 |
9937
|
aaronmk
|
datasrc_make_install schema
|
103 |
9934
|
aaronmk
|
table_make_install ${_remake:+re}install # just the table
|
104 |
9392
|
aaronmk
|
}
|
105 |
|
|
|
106 |
10371
|
aaronmk
|
trim_table()
|
107 |
|
|
{
|
108 |
|
|
echo_func; set_make_vars
|
109 |
|
|
if test "$is_view"; then return 0; fi # do not modify view columns
|
110 |
|
|
psql <<EOF
|
111 |
|
|
SELECT util.trim('"$table"', '$map_table')
|
112 |
|
|
EOF
|
113 |
|
|
}
|
114 |
|
|
|
115 |
10274
|
aaronmk
|
postprocess()
|
116 |
8273
|
aaronmk
|
{
|
117 |
9838
|
aaronmk
|
echo_func; set_make_vars
|
118 |
10275
|
aaronmk
|
kw_params can_test; local can_test="${can_test-1}"
|
119 |
10372
|
aaronmk
|
if remaking; then trim_table; fi
|
120 |
8282
|
aaronmk
|
map_table
|
121 |
10273
|
aaronmk
|
custom_postprocess
|
122 |
10171
|
aaronmk
|
mk_derived
|
123 |
10367
|
aaronmk
|
if test "$can_test"; then remake_VegBIEN_mappings; fi
|
124 |
8273
|
aaronmk
|
}
|
125 |
8705
|
aaronmk
|
|
126 |
10274
|
aaronmk
|
import()
|
127 |
|
|
{
|
128 |
|
|
echo_func; set_make_vars
|
129 |
|
|
self_make load_data
|
130 |
|
|
postprocess
|
131 |
|
|
}
|
132 |
|
|
|
133 |
8705
|
aaronmk
|
fi
|