Project

General

Profile

1 8291 aaronmk
#!/bin/bash -e
2 8273 aaronmk
3
if false; then #### run script template:
4 8291 aaronmk
#!/bin/bash -e
5 8273 aaronmk
. "$(dirname "${BASH_SOURCE[0]}")"/path/to/table.run
6 8161 aaronmk
7 9039 aaronmk
func_override import__table_run
8 9074 aaronmk
import()
9 8273 aaronmk
{
10 12968 aaronmk
	begin_target
11 8273 aaronmk
	before_import_cmds
12 12967 aaronmk
	with_rm import__table_run "$@"
13 8273 aaronmk
	after_import_cmds
14
}
15
fi ####
16 8213 aaronmk
17 8296 aaronmk
. "$(dirname "${BASH_SOURCE[0]}")"/import.run
18 9854 aaronmk
.rel subdir.run
19
.rel ../sh/db_make.sh
20 8273 aaronmk
21 8705 aaronmk
if self_not_included; then
22
23 11602 aaronmk
##### utils
24
25 9585 aaronmk
: "${table=$subdir}"; export table
26
27
table_make() { subdir_make "$@"; }
28
29 9935 aaronmk
install_log_rel=logs/install.log.sql
30
31 9934 aaronmk
table_make_install()
32
{
33 11769 aaronmk
	echo_func
34 14762 aaronmk
	if ! remaking && pg_table_exists; then skip_table; return 0; fi
35 11769 aaronmk
	local install_log="${install_log-$top_dir/$install_log_rel}"
36 9934 aaronmk
	local verbosity_min= # install logs require default verbosity
37 11770 aaronmk
	if ! remaking && test -e "$install_log"; then local_export noclobber=1; fi
38
	table_make "$@"
39 9934 aaronmk
}
40
41 10248 aaronmk
postprocess_sql="$top_dir"/postprocess.sql
42
43 10249 aaronmk
is_view="$(test -e "$postprocess_sql" \
44
&& grep -F force_update_view "$postprocess_sql" >/dev/null; exit2bool)"
45
46 10368 aaronmk
map_table="\"~$table.map\"" # sort after other tables to avoid clutter
47
48 14773 aaronmk
srcs=($(output_data=1 table_make list_srcs))
49 14775 aaronmk
srcs=("${srcs[@]/#/$datasrc_dir/}") # empty str at start (/#) -> $datasrc_dir/
50 14773 aaronmk
_1st_src="$(echo1 "${srcs[@]}")"
51 14767 aaronmk
52 14774 aaronmk
header() { echo_func; : "${1:?}"; head -1 "$1"; }
53
54 14776 aaronmk
header_txt="$top_dir"/header.txt
55
56 11602 aaronmk
##### targets
57
58 14774 aaronmk
header.txt() { begin_target; to_target header "$_1st_src"; }
59 14768 aaronmk
60 14776 aaronmk
check_headers()
61
{
62
	begin_target
63
	header.txt
64
	local header="$(header "$header_txt")"
65
	for file in "${srcs[@]}"; do
66
		test "$(echo_run header "$file")" = "$header" \
67
|| die "$file: header mismatched"
68
	done
69
}
70
71 11602 aaronmk
load_data()
72
{
73 12779 aaronmk
	begin_target
74 11602 aaronmk
	if remaking; then in_top_dir rm -f header.csv; fi # remake it
75 12878 aaronmk
	make --directory="$top_dir/.." schema # works w/ old-style srcs w/o ../run
76
	#"$top_dir/../run" schema/make # uncomment once everything is new-style
77 11602 aaronmk
	is_view="$is_view" table_make_install ${_remake:+re}install # just the table
78
}
79
80
trim_table()
81
{
82 12779 aaronmk
	begin_target
83 11602 aaronmk
	if test "$is_view"; then return 0; fi # do not modify view columns
84
	psql <<EOF
85 12847 aaronmk
SELECT util.trim('"$table"', '$map_table');
86 11602 aaronmk
EOF
87
}
88
89 10369 aaronmk
reset_col_names()
90
{
91 12779 aaronmk
	begin_target
92 10369 aaronmk
	if test "$is_view"; then return 0; fi # do not rename view columns
93
	psql <<EOF
94 12847 aaronmk
SELECT util.reset_col_names('"$table"', '$map_table');
95 10369 aaronmk
EOF
96
}
97
98 9074 aaronmk
map_table()
99 10210 aaronmk
# note that collisions may prevent all renames from being made at once.
100 10211 aaronmk
# if this is the case, run map_table repeatedly until no more renames are made:
101
# $ while true; do .../run map_table; done
102 10210 aaronmk
# collisions may result if the staging table gets messed up (e.g. due to missing
103
# input columns in map.csv).
104 8161 aaronmk
{
105 12779 aaronmk
	begin_target
106 10250 aaronmk
	if test "$is_view"; then return 0; fi # do not rename view columns
107
108 10370 aaronmk
	if remaking; then reset_col_names; fi
109 9386 aaronmk
	table_make map.csv
110 8214 aaronmk
	psql <<EOF
111 10156 aaronmk
SELECT util.reset_map_table('$map_table');
112
ALTER TABLE $map_table DISABLE TRIGGER map_filter_insert;
113
\copy $map_table FROM 'map.csv' CSV HEADER;
114 10366 aaronmk
SELECT util.set_col_names_with_metadata('"$table"', '$map_table');
115 8161 aaronmk
EOF
116
}
117 8198 aaronmk
118 12779 aaronmk
test_() { begin_target; piped_cmd yes|table_make test; }
119 11913 aaronmk
120 12691 aaronmk
mappings()
121 8245 aaronmk
{
122 12779 aaronmk
	begin_target
123 13344 aaronmk
	benign_error=1 public_schema_exists || return 0
124 11000 aaronmk
	if remaking; then in_top_dir rm -f map.csv; fi # remake it
125 11913 aaronmk
	test_
126 8245 aaronmk
}
127 8273 aaronmk
128 10273 aaronmk
custom_postprocess() # overridable
129 8273 aaronmk
{
130 12779 aaronmk
	begin_target
131 10248 aaronmk
	local file="$postprocess_sql"
132 12743 aaronmk
	if test -e "$file"; then
133
		table_make "$(basename "$file")" # apply renames to SQL statements
134
		psql "$@"
135
	fi
136 8273 aaronmk
}
137
138 11602 aaronmk
mk_derived()
139 12779 aaronmk
{ begin_target; "$root_dir"/schemas/VegCore/mk_derived; }
140 9392 aaronmk
141 10274 aaronmk
postprocess()
142 8273 aaronmk
{
143 12779 aaronmk
	begin_target
144 10275 aaronmk
	kw_params can_test; local can_test="${can_test-1}"
145 10372 aaronmk
	if remaking; then trim_table; fi
146 8282 aaronmk
	map_table
147 10273 aaronmk
	custom_postprocess
148 10171 aaronmk
	mk_derived
149 12691 aaronmk
	if test "$can_test"; then mappings; fi
150 8273 aaronmk
}
151 8705 aaronmk
152 10274 aaronmk
import()
153
{
154 12779 aaronmk
	begin_target
155 12967 aaronmk
	with_rm load_data
156 10274 aaronmk
	postprocess
157
}
158
159 8705 aaronmk
fi