|
1
|
#!/bin/bash -e
|
|
2
|
|
|
3
|
if false; then #### run script template:
|
|
4
|
#!/bin/bash -e
|
|
5
|
. "$(dirname "${BASH_SOURCE[0]}")"/path/to/table.run
|
|
6
|
|
|
7
|
func_override import__table_run
|
|
8
|
import()
|
|
9
|
{
|
|
10
|
echo_func
|
|
11
|
before_import_cmds
|
|
12
|
self_make import__table_run "$@"
|
|
13
|
after_import_cmds
|
|
14
|
}
|
|
15
|
fi ####
|
|
16
|
|
|
17
|
. "$(dirname "${BASH_SOURCE[0]}")"/import.run
|
|
18
|
.rel subdir.run
|
|
19
|
.rel ../sh/db_make.sh
|
|
20
|
|
|
21
|
if self_not_included; then
|
|
22
|
|
|
23
|
##### utils
|
|
24
|
|
|
25
|
: "${table=$subdir}"; export table
|
|
26
|
|
|
27
|
table_make() { subdir_make "$@"; }
|
|
28
|
|
|
29
|
install_log_rel=logs/install.log.sql
|
|
30
|
|
|
31
|
table_make_install()
|
|
32
|
{
|
|
33
|
echo_func
|
|
34
|
if ! remaking && pg_table_exists; then return 0; fi
|
|
35
|
local install_log="${install_log-$top_dir/$install_log_rel}"
|
|
36
|
local verbosity_min= # install logs require default verbosity
|
|
37
|
if ! remaking && test -e "$install_log"; then local_export noclobber=1; fi
|
|
38
|
table_make "$@"
|
|
39
|
}
|
|
40
|
|
|
41
|
datasrc_make_install()
|
|
42
|
{
|
|
43
|
echo_func
|
|
44
|
subdir=. install_log="$top_dir/../$install_log_rel" table_make_install "$@"
|
|
45
|
}
|
|
46
|
|
|
47
|
postprocess_sql="$top_dir"/postprocess.sql
|
|
48
|
|
|
49
|
is_view="$(test -e "$postprocess_sql" \
|
|
50
|
&& grep -F force_update_view "$postprocess_sql" >/dev/null; exit2bool)"
|
|
51
|
|
|
52
|
map_table="\"~$table.map\"" # sort after other tables to avoid clutter
|
|
53
|
|
|
54
|
##### targets
|
|
55
|
|
|
56
|
load_data()
|
|
57
|
{
|
|
58
|
echo_func; set_make_vars
|
|
59
|
if remaking; then in_top_dir rm -f header.csv; fi # remake it
|
|
60
|
datasrc_make_install schema
|
|
61
|
is_view="$is_view" table_make_install ${_remake:+re}install # just the table
|
|
62
|
}
|
|
63
|
|
|
64
|
trim_table()
|
|
65
|
{
|
|
66
|
echo_func; set_make_vars
|
|
67
|
if test "$is_view"; then return 0; fi # do not modify view columns
|
|
68
|
psql <<EOF
|
|
69
|
SELECT util.trim('"$table"', '$map_table')
|
|
70
|
EOF
|
|
71
|
}
|
|
72
|
|
|
73
|
reset_col_names()
|
|
74
|
{
|
|
75
|
echo_func; set_make_vars
|
|
76
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
|
77
|
psql <<EOF
|
|
78
|
SELECT util.reset_col_names('"$table"', '$map_table')
|
|
79
|
EOF
|
|
80
|
}
|
|
81
|
|
|
82
|
map_table()
|
|
83
|
# note that collisions may prevent all renames from being made at once.
|
|
84
|
# if this is the case, run map_table repeatedly until no more renames are made:
|
|
85
|
# $ while true; do .../run map_table; done
|
|
86
|
# collisions may result if the staging table gets messed up (e.g. due to missing
|
|
87
|
# input columns in map.csv).
|
|
88
|
{
|
|
89
|
echo_func; set_make_vars
|
|
90
|
if test "$is_view"; then return 0; fi # do not rename view columns
|
|
91
|
|
|
92
|
if remaking; then reset_col_names; fi
|
|
93
|
table_make map.csv
|
|
94
|
psql <<EOF
|
|
95
|
SELECT util.reset_map_table('$map_table');
|
|
96
|
ALTER TABLE $map_table DISABLE TRIGGER map_filter_insert;
|
|
97
|
\copy $map_table FROM 'map.csv' CSV HEADER;
|
|
98
|
SELECT util.set_col_names_with_metadata('"$table"', '$map_table');
|
|
99
|
EOF
|
|
100
|
}
|
|
101
|
|
|
102
|
test_() { echo_func; set_make_vars; piped_cmd yes|table_make test; }
|
|
103
|
|
|
104
|
mappings()
|
|
105
|
{
|
|
106
|
echo_func; set_make_vars
|
|
107
|
public_schema_exists || return 0
|
|
108
|
if remaking; then in_top_dir rm -f map.csv; fi # remake it
|
|
109
|
test_
|
|
110
|
}
|
|
111
|
|
|
112
|
custom_postprocess() # overridable
|
|
113
|
{
|
|
114
|
echo_func; set_make_vars
|
|
115
|
local file="$postprocess_sql"
|
|
116
|
if test -e "$file"; then psql "$@"; fi
|
|
117
|
}
|
|
118
|
|
|
119
|
mk_derived()
|
|
120
|
{ echo_func; set_make_vars; "$root_dir"/schemas/VegCore/mk_derived; }
|
|
121
|
|
|
122
|
postprocess()
|
|
123
|
{
|
|
124
|
echo_func; set_make_vars
|
|
125
|
kw_params can_test; local can_test="${can_test-1}"
|
|
126
|
if remaking; then trim_table; fi
|
|
127
|
map_table
|
|
128
|
custom_postprocess
|
|
129
|
mk_derived
|
|
130
|
if test "$can_test"; then mappings; fi
|
|
131
|
}
|
|
132
|
|
|
133
|
import()
|
|
134
|
{
|
|
135
|
echo_func; set_make_vars
|
|
136
|
self_make load_data
|
|
137
|
postprocess
|
|
138
|
}
|
|
139
|
|
|
140
|
fi
|
