1 |
8291
|
aaronmk
|
#!/bin/bash -e
|
2 |
8273
|
aaronmk
|
|
3 |
|
|
if false; then #### run script template:
|
4 |
8291
|
aaronmk
|
#!/bin/bash -e
|
5 |
8273
|
aaronmk
|
. "$(dirname "${BASH_SOURCE[0]}")"/path/to/table.run
|
6 |
8161
|
aaronmk
|
|
7 |
9039
|
aaronmk
|
func_override import__table_run
|
8 |
9074
|
aaronmk
|
import()
|
9 |
8273
|
aaronmk
|
{
|
10 |
8881
|
aaronmk
|
echo_func
|
11 |
8273
|
aaronmk
|
before_import_cmds
|
12 |
9681
|
aaronmk
|
self_make import__table_run "$@"
|
13 |
8273
|
aaronmk
|
after_import_cmds
|
14 |
|
|
}
|
15 |
|
|
fi ####
|
16 |
8213
|
aaronmk
|
|
17 |
8296
|
aaronmk
|
. "$(dirname "${BASH_SOURCE[0]}")"/import.run
|
18 |
9854
|
aaronmk
|
.rel subdir.run
|
19 |
|
|
.rel ../sh/db_make.sh
|
20 |
8273
|
aaronmk
|
|
21 |
8705
|
aaronmk
|
if self_not_included; then
|
22 |
|
|
|
23 |
11602
|
aaronmk
|
##### utils
|
24 |
|
|
|
25 |
9585
|
aaronmk
|
: "${table=$subdir}"; export table
|
26 |
|
|
|
27 |
|
|
table_make() { subdir_make "$@"; }
|
28 |
|
|
|
29 |
9935
|
aaronmk
|
install_log_rel=logs/install.log.sql
|
30 |
|
|
|
31 |
9934
|
aaronmk
|
table_make_install()
|
32 |
|
|
{
|
33 |
11769
|
aaronmk
|
echo_func
|
34 |
12864
|
aaronmk
|
if ! remaking && pg_table_exists; then skip_table; return 0; fi
|
35 |
11769
|
aaronmk
|
local install_log="${install_log-$top_dir/$install_log_rel}"
|
36 |
9934
|
aaronmk
|
local verbosity_min= # install logs require default verbosity
|
37 |
11770
|
aaronmk
|
if ! remaking && test -e "$install_log"; then local_export noclobber=1; fi
|
38 |
|
|
table_make "$@"
|
39 |
9934
|
aaronmk
|
}
|
40 |
|
|
|
41 |
10248
|
aaronmk
|
postprocess_sql="$top_dir"/postprocess.sql
|
42 |
|
|
|
43 |
10249
|
aaronmk
|
is_view="$(test -e "$postprocess_sql" \
|
44 |
|
|
&& grep -F force_update_view "$postprocess_sql" >/dev/null; exit2bool)"
|
45 |
|
|
|
46 |
10368
|
aaronmk
|
map_table="\"~$table.map\"" # sort after other tables to avoid clutter
|
47 |
|
|
|
48 |
11602
|
aaronmk
|
##### targets
|
49 |
|
|
|
50 |
|
|
load_data()
|
51 |
|
|
{
|
52 |
12779
|
aaronmk
|
begin_target
|
53 |
11602
|
aaronmk
|
if remaking; then in_top_dir rm -f header.csv; fi # remake it
|
54 |
12850
|
aaronmk
|
"$top_dir/../run" schema/make
|
55 |
11602
|
aaronmk
|
is_view="$is_view" table_make_install ${_remake:+re}install # just the table
|
56 |
|
|
}
|
57 |
|
|
|
58 |
|
|
trim_table()
|
59 |
|
|
{
|
60 |
12779
|
aaronmk
|
begin_target
|
61 |
11602
|
aaronmk
|
if test "$is_view"; then return 0; fi # do not modify view columns
|
62 |
|
|
psql <<EOF
|
63 |
12847
|
aaronmk
|
SELECT util.trim('"$table"', '$map_table');
|
64 |
11602
|
aaronmk
|
EOF
|
65 |
|
|
}
|
66 |
|
|
|
67 |
10369
|
aaronmk
|
reset_col_names()
|
68 |
|
|
{
|
69 |
12779
|
aaronmk
|
begin_target
|
70 |
10369
|
aaronmk
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
71 |
|
|
psql <<EOF
|
72 |
12847
|
aaronmk
|
SELECT util.reset_col_names('"$table"', '$map_table');
|
73 |
10369
|
aaronmk
|
EOF
|
74 |
|
|
}
|
75 |
|
|
|
76 |
9074
|
aaronmk
|
map_table()
|
77 |
10210
|
aaronmk
|
# note that collisions may prevent all renames from being made at once.
|
78 |
10211
|
aaronmk
|
# if this is the case, run map_table repeatedly until no more renames are made:
|
79 |
|
|
# $ while true; do .../run map_table; done
|
80 |
10210
|
aaronmk
|
# collisions may result if the staging table gets messed up (e.g. due to missing
|
81 |
|
|
# input columns in map.csv).
|
82 |
8161
|
aaronmk
|
{
|
83 |
12779
|
aaronmk
|
begin_target
|
84 |
10250
|
aaronmk
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
85 |
|
|
|
86 |
10370
|
aaronmk
|
if remaking; then reset_col_names; fi
|
87 |
9386
|
aaronmk
|
table_make map.csv
|
88 |
8214
|
aaronmk
|
psql <<EOF
|
89 |
10156
|
aaronmk
|
SELECT util.reset_map_table('$map_table');
|
90 |
|
|
ALTER TABLE $map_table DISABLE TRIGGER map_filter_insert;
|
91 |
|
|
\copy $map_table FROM 'map.csv' CSV HEADER;
|
92 |
10366
|
aaronmk
|
SELECT util.set_col_names_with_metadata('"$table"', '$map_table');
|
93 |
8161
|
aaronmk
|
EOF
|
94 |
|
|
}
|
95 |
8198
|
aaronmk
|
|
96 |
12779
|
aaronmk
|
test_() { begin_target; piped_cmd yes|table_make test; }
|
97 |
11913
|
aaronmk
|
|
98 |
12691
|
aaronmk
|
mappings()
|
99 |
8245
|
aaronmk
|
{
|
100 |
12779
|
aaronmk
|
begin_target
|
101 |
12804
|
aaronmk
|
public_schema_exists || return 0
|
102 |
11000
|
aaronmk
|
if remaking; then in_top_dir rm -f map.csv; fi # remake it
|
103 |
11913
|
aaronmk
|
test_
|
104 |
8245
|
aaronmk
|
}
|
105 |
8273
|
aaronmk
|
|
106 |
10273
|
aaronmk
|
custom_postprocess() # overridable
|
107 |
8273
|
aaronmk
|
{
|
108 |
12779
|
aaronmk
|
begin_target
|
109 |
10248
|
aaronmk
|
local file="$postprocess_sql"
|
110 |
12743
|
aaronmk
|
if test -e "$file"; then
|
111 |
|
|
table_make "$(basename "$file")" # apply renames to SQL statements
|
112 |
|
|
psql "$@"
|
113 |
|
|
fi
|
114 |
8273
|
aaronmk
|
}
|
115 |
|
|
|
116 |
11602
|
aaronmk
|
mk_derived()
|
117 |
12779
|
aaronmk
|
{ begin_target; "$root_dir"/schemas/VegCore/mk_derived; }
|
118 |
9392
|
aaronmk
|
|
119 |
10274
|
aaronmk
|
postprocess()
|
120 |
8273
|
aaronmk
|
{
|
121 |
12779
|
aaronmk
|
begin_target
|
122 |
10275
|
aaronmk
|
kw_params can_test; local can_test="${can_test-1}"
|
123 |
10372
|
aaronmk
|
if remaking; then trim_table; fi
|
124 |
8282
|
aaronmk
|
map_table
|
125 |
10273
|
aaronmk
|
custom_postprocess
|
126 |
10171
|
aaronmk
|
mk_derived
|
127 |
12691
|
aaronmk
|
if test "$can_test"; then mappings; fi
|
128 |
8273
|
aaronmk
|
}
|
129 |
8705
|
aaronmk
|
|
130 |
10274
|
aaronmk
|
import()
|
131 |
|
|
{
|
132 |
12779
|
aaronmk
|
begin_target
|
133 |
10274
|
aaronmk
|
self_make load_data
|
134 |
|
|
postprocess
|
135 |
|
|
}
|
136 |
|
|
|
137 |
8705
|
aaronmk
|
fi
|