1 |
8291
|
aaronmk
|
#!/bin/bash -e
|
2 |
8273
|
aaronmk
|
|
3 |
|
|
if false; then #### run script template:
|
4 |
8291
|
aaronmk
|
#!/bin/bash -e
|
5 |
8273
|
aaronmk
|
. "$(dirname "${BASH_SOURCE[0]}")"/path/to/table.run
|
6 |
8161
|
aaronmk
|
|
7 |
9039
|
aaronmk
|
func_override import__table_run
|
8 |
9074
|
aaronmk
|
import()
|
9 |
8273
|
aaronmk
|
{
|
10 |
12968
|
aaronmk
|
begin_target
|
11 |
8273
|
aaronmk
|
before_import_cmds
|
12 |
12967
|
aaronmk
|
with_rm import__table_run "$@"
|
13 |
8273
|
aaronmk
|
after_import_cmds
|
14 |
|
|
}
|
15 |
|
|
fi ####
|
16 |
8213
|
aaronmk
|
|
17 |
8296
|
aaronmk
|
. "$(dirname "${BASH_SOURCE[0]}")"/import.run
|
18 |
9854
|
aaronmk
|
.rel subdir.run
|
19 |
|
|
.rel ../sh/db_make.sh
|
20 |
8273
|
aaronmk
|
|
21 |
8705
|
aaronmk
|
if self_not_included; then
|
22 |
|
|
|
23 |
11602
|
aaronmk
|
##### utils
|
24 |
|
|
|
25 |
9585
|
aaronmk
|
: "${table=$subdir}"; export table
|
26 |
|
|
|
27 |
|
|
table_make() { subdir_make "$@"; }
|
28 |
|
|
|
29 |
9935
|
aaronmk
|
install_log_rel=logs/install.log.sql
|
30 |
|
|
|
31 |
9934
|
aaronmk
|
table_make_install()
|
32 |
|
|
{
|
33 |
11769
|
aaronmk
|
echo_func
|
34 |
14762
|
aaronmk
|
if ! remaking && pg_table_exists; then skip_table; return 0; fi
|
35 |
11769
|
aaronmk
|
local install_log="${install_log-$top_dir/$install_log_rel}"
|
36 |
9934
|
aaronmk
|
local verbosity_min= # install logs require default verbosity
|
37 |
11770
|
aaronmk
|
if ! remaking && test -e "$install_log"; then local_export noclobber=1; fi
|
38 |
|
|
table_make "$@"
|
39 |
9934
|
aaronmk
|
}
|
40 |
|
|
|
41 |
10248
|
aaronmk
|
postprocess_sql="$top_dir"/postprocess.sql
|
42 |
|
|
|
43 |
10249
|
aaronmk
|
is_view="$(test -e "$postprocess_sql" \
|
44 |
|
|
&& grep -F force_update_view "$postprocess_sql" >/dev/null; exit2bool)"
|
45 |
|
|
|
46 |
10368
|
aaronmk
|
map_table="\"~$table.map\"" # sort after other tables to avoid clutter
|
47 |
|
|
|
48 |
14773
|
aaronmk
|
srcs=($(output_data=1 table_make list_srcs))
|
49 |
14775
|
aaronmk
|
srcs=("${srcs[@]/#/$datasrc_dir/}") # empty str at start (/#) -> $datasrc_dir/
|
50 |
14773
|
aaronmk
|
_1st_src="$(echo1 "${srcs[@]}")"
|
51 |
14767
|
aaronmk
|
|
52 |
14774
|
aaronmk
|
header() { echo_func; : "${1:?}"; head -1 "$1"; }
|
53 |
|
|
|
54 |
14776
|
aaronmk
|
header_txt="$top_dir"/header.txt
|
55 |
|
|
|
56 |
11602
|
aaronmk
|
##### targets
|
57 |
|
|
|
58 |
14774
|
aaronmk
|
header.txt() { begin_target; to_target header "$_1st_src"; }
|
59 |
14768
|
aaronmk
|
|
60 |
14776
|
aaronmk
|
check_headers()
|
61 |
|
|
{
|
62 |
|
|
begin_target
|
63 |
|
|
header.txt
|
64 |
|
|
local header="$(header "$header_txt")"
|
65 |
|
|
for file in "${srcs[@]}"; do
|
66 |
|
|
test "$(echo_run header "$file")" = "$header" \
|
67 |
|
|
|| die "$file: header mismatched"
|
68 |
|
|
done
|
69 |
|
|
}
|
70 |
|
|
|
71 |
11602
|
aaronmk
|
load_data()
|
72 |
|
|
{
|
73 |
12779
|
aaronmk
|
begin_target
|
74 |
11602
|
aaronmk
|
if remaking; then in_top_dir rm -f header.csv; fi # remake it
|
75 |
12878
|
aaronmk
|
make --directory="$top_dir/.." schema # works w/ old-style srcs w/o ../run
|
76 |
|
|
#"$top_dir/../run" schema/make # uncomment once everything is new-style
|
77 |
11602
|
aaronmk
|
is_view="$is_view" table_make_install ${_remake:+re}install # just the table
|
78 |
|
|
}
|
79 |
|
|
|
80 |
|
|
trim_table()
|
81 |
|
|
{
|
82 |
12779
|
aaronmk
|
begin_target
|
83 |
11602
|
aaronmk
|
if test "$is_view"; then return 0; fi # do not modify view columns
|
84 |
|
|
psql <<EOF
|
85 |
12847
|
aaronmk
|
SELECT util.trim('"$table"', '$map_table');
|
86 |
11602
|
aaronmk
|
EOF
|
87 |
|
|
}
|
88 |
|
|
|
89 |
10369
|
aaronmk
|
reset_col_names()
|
90 |
|
|
{
|
91 |
12779
|
aaronmk
|
begin_target
|
92 |
10369
|
aaronmk
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
93 |
|
|
psql <<EOF
|
94 |
12847
|
aaronmk
|
SELECT util.reset_col_names('"$table"', '$map_table');
|
95 |
10369
|
aaronmk
|
EOF
|
96 |
|
|
}
|
97 |
|
|
|
98 |
9074
|
aaronmk
|
map_table()
|
99 |
10210
|
aaronmk
|
# note that collisions may prevent all renames from being made at once.
|
100 |
10211
|
aaronmk
|
# if this is the case, run map_table repeatedly until no more renames are made:
|
101 |
|
|
# $ while true; do .../run map_table; done
|
102 |
10210
|
aaronmk
|
# collisions may result if the staging table gets messed up (e.g. due to missing
|
103 |
|
|
# input columns in map.csv).
|
104 |
8161
|
aaronmk
|
{
|
105 |
12779
|
aaronmk
|
begin_target
|
106 |
10250
|
aaronmk
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
107 |
|
|
|
108 |
10370
|
aaronmk
|
if remaking; then reset_col_names; fi
|
109 |
9386
|
aaronmk
|
table_make map.csv
|
110 |
8214
|
aaronmk
|
psql <<EOF
|
111 |
10156
|
aaronmk
|
SELECT util.reset_map_table('$map_table');
|
112 |
|
|
ALTER TABLE $map_table DISABLE TRIGGER map_filter_insert;
|
113 |
|
|
\copy $map_table FROM 'map.csv' CSV HEADER;
|
114 |
10366
|
aaronmk
|
SELECT util.set_col_names_with_metadata('"$table"', '$map_table');
|
115 |
8161
|
aaronmk
|
EOF
|
116 |
|
|
}
|
117 |
8198
|
aaronmk
|
|
118 |
12779
|
aaronmk
|
test_() { begin_target; piped_cmd yes|table_make test; }
|
119 |
11913
|
aaronmk
|
|
120 |
12691
|
aaronmk
|
mappings()
|
121 |
8245
|
aaronmk
|
{
|
122 |
12779
|
aaronmk
|
begin_target
|
123 |
13344
|
aaronmk
|
benign_error=1 public_schema_exists || return 0
|
124 |
11000
|
aaronmk
|
if remaking; then in_top_dir rm -f map.csv; fi # remake it
|
125 |
11913
|
aaronmk
|
test_
|
126 |
8245
|
aaronmk
|
}
|
127 |
8273
|
aaronmk
|
|
128 |
10273
|
aaronmk
|
custom_postprocess() # overridable
|
129 |
8273
|
aaronmk
|
{
|
130 |
12779
|
aaronmk
|
begin_target
|
131 |
10248
|
aaronmk
|
local file="$postprocess_sql"
|
132 |
12743
|
aaronmk
|
if test -e "$file"; then
|
133 |
|
|
table_make "$(basename "$file")" # apply renames to SQL statements
|
134 |
|
|
psql "$@"
|
135 |
|
|
fi
|
136 |
8273
|
aaronmk
|
}
|
137 |
|
|
|
138 |
11602
|
aaronmk
|
mk_derived()
|
139 |
12779
|
aaronmk
|
{ begin_target; "$root_dir"/schemas/VegCore/mk_derived; }
|
140 |
9392
|
aaronmk
|
|
141 |
10274
|
aaronmk
|
postprocess()
|
142 |
8273
|
aaronmk
|
{
|
143 |
12779
|
aaronmk
|
begin_target
|
144 |
10275
|
aaronmk
|
kw_params can_test; local can_test="${can_test-1}"
|
145 |
10372
|
aaronmk
|
if remaking; then trim_table; fi
|
146 |
8282
|
aaronmk
|
map_table
|
147 |
10273
|
aaronmk
|
custom_postprocess
|
148 |
10171
|
aaronmk
|
mk_derived
|
149 |
12691
|
aaronmk
|
if test "$can_test"; then mappings; fi
|
150 |
8273
|
aaronmk
|
}
|
151 |
8705
|
aaronmk
|
|
152 |
10274
|
aaronmk
|
import()
|
153 |
|
|
{
|
154 |
12779
|
aaronmk
|
begin_target
|
155 |
12967
|
aaronmk
|
with_rm load_data
|
156 |
10274
|
aaronmk
|
postprocess
|
157 |
|
|
}
|
158 |
|
|
|
159 |
8705
|
aaronmk
|
fi
|