1
|
#!/bin/bash -e
|
2
|
|
3
|
if false; then #### run script template:
|
4
|
#!/bin/bash -e
|
5
|
. "$(dirname "${BASH_SOURCE[0]}")"/path/to/table.run
|
6
|
|
7
|
func_override import__table_run
|
8
|
import()
|
9
|
{
|
10
|
begin_target
|
11
|
before_import_cmds
|
12
|
with_rm import__table_run "$@"
|
13
|
after_import_cmds
|
14
|
}
|
15
|
fi ####
|
16
|
|
17
|
. "$(dirname "${BASH_SOURCE[0]}")"/import.run
|
18
|
.rel subdir.run
|
19
|
.rel ../sh/db_make.sh
|
20
|
|
21
|
if self_not_included; then
|
22
|
|
23
|
##### utils
|
24
|
|
25
|
: "${table=$subdir}"; export table
|
26
|
|
27
|
table_make() { subdir_make "$@"; }
|
28
|
|
29
|
install_log_rel=logs/install.log.sql
|
30
|
|
31
|
table_make_install()
|
32
|
{
|
33
|
echo_func
|
34
|
if ! remaking && pg_table_exists; then skip_table; return 0; fi
|
35
|
local install_log="${install_log-$top_dir/$install_log_rel}"
|
36
|
local verbosity_min= # install logs require default verbosity
|
37
|
if ! remaking && test -e "$install_log"; then local_export noclobber=1; fi
|
38
|
table_make "$@"
|
39
|
}
|
40
|
|
41
|
postprocess_sql="$top_dir"/postprocess.sql
|
42
|
|
43
|
is_view="$(test -e "$postprocess_sql" \
|
44
|
&& grep -F force_update_view "$postprocess_sql" >/dev/null; exit2bool)"
|
45
|
|
46
|
map_table="\"~$table.map\"" # sort after other tables to avoid clutter
|
47
|
|
48
|
srcs=($(output_data=1 table_make list_srcs))
|
49
|
srcs=("${srcs[@]/#/$datasrc_dir/}") # empty str at start (/#) -> $datasrc_dir/
|
50
|
_1st_src="$(echo1 "${srcs[@]}")"
|
51
|
|
52
|
header() { echo_func; : "${1:?}"; head -1 "$1"; }
|
53
|
|
54
|
##### targets
|
55
|
|
56
|
header.txt() { begin_target; to_target header "$_1st_src"; }
|
57
|
|
58
|
load_data()
|
59
|
{
|
60
|
begin_target
|
61
|
if remaking; then in_top_dir rm -f header.csv; fi # remake it
|
62
|
make --directory="$top_dir/.." schema # works w/ old-style srcs w/o ../run
|
63
|
#"$top_dir/../run" schema/make # uncomment once everything is new-style
|
64
|
is_view="$is_view" table_make_install ${_remake:+re}install # just the table
|
65
|
}
|
66
|
|
67
|
trim_table()
|
68
|
{
|
69
|
begin_target
|
70
|
if test "$is_view"; then return 0; fi # do not modify view columns
|
71
|
psql <<EOF
|
72
|
SELECT util.trim('"$table"', '$map_table');
|
73
|
EOF
|
74
|
}
|
75
|
|
76
|
reset_col_names()
|
77
|
{
|
78
|
begin_target
|
79
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
80
|
psql <<EOF
|
81
|
SELECT util.reset_col_names('"$table"', '$map_table');
|
82
|
EOF
|
83
|
}
|
84
|
|
85
|
map_table()
|
86
|
# note that collisions may prevent all renames from being made at once.
|
87
|
# if this is the case, run map_table repeatedly until no more renames are made:
|
88
|
# $ while true; do .../run map_table; done
|
89
|
# collisions may result if the staging table gets messed up (e.g. due to missing
|
90
|
# input columns in map.csv).
|
91
|
{
|
92
|
begin_target
|
93
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
94
|
|
95
|
if remaking; then reset_col_names; fi
|
96
|
table_make map.csv
|
97
|
psql <<EOF
|
98
|
SELECT util.reset_map_table('$map_table');
|
99
|
ALTER TABLE $map_table DISABLE TRIGGER map_filter_insert;
|
100
|
\copy $map_table FROM 'map.csv' CSV HEADER;
|
101
|
SELECT util.set_col_names_with_metadata('"$table"', '$map_table');
|
102
|
EOF
|
103
|
}
|
104
|
|
105
|
test_() { begin_target; piped_cmd yes|table_make test; }
|
106
|
|
107
|
mappings()
|
108
|
{
|
109
|
begin_target
|
110
|
benign_error=1 public_schema_exists || return 0
|
111
|
if remaking; then in_top_dir rm -f map.csv; fi # remake it
|
112
|
test_
|
113
|
}
|
114
|
|
115
|
custom_postprocess() # overridable
|
116
|
{
|
117
|
begin_target
|
118
|
local file="$postprocess_sql"
|
119
|
if test -e "$file"; then
|
120
|
table_make "$(basename "$file")" # apply renames to SQL statements
|
121
|
psql "$@"
|
122
|
fi
|
123
|
}
|
124
|
|
125
|
mk_derived()
|
126
|
{ begin_target; "$root_dir"/schemas/VegCore/mk_derived; }
|
127
|
|
128
|
postprocess()
|
129
|
{
|
130
|
begin_target
|
131
|
kw_params can_test; local can_test="${can_test-1}"
|
132
|
if remaking; then trim_table; fi
|
133
|
map_table
|
134
|
custom_postprocess
|
135
|
mk_derived
|
136
|
if test "$can_test"; then mappings; fi
|
137
|
}
|
138
|
|
139
|
import()
|
140
|
{
|
141
|
begin_target
|
142
|
with_rm load_data
|
143
|
postprocess
|
144
|
}
|
145
|
|
146
|
fi
|