Project

General

Profile

1 8291 aaronmk
#!/bin/bash -e
2 8952 aaronmk
set -o errexit # in case caller did not have -e in #! line
3
4 8957 aaronmk
function extern () { (exec "$@") || return; }
5 8917 aaronmk
6 9002 aaronmk
isset () { test -n "${!1+isset}"; }
7
8 8716 aaronmk
realpath () { readlink -f -- "$1"; }
9 8703 aaronmk
10 8917 aaronmk
include_guard_var () { realpath "$1"|"extern" sed 's/[^a-zA-Z0-9_]/_/g'; }
11 8716 aaronmk
12 8703 aaronmk
self_not_included () # usage: if self_not_included; then ... fi
13
{
14 8971 aaronmk
	test $# -ge 1 || set -- "${BASH_SOURCE[1]}"
15 8703 aaronmk
	local include_guard="$(include_guard_var "$1")"
16 8793 aaronmk
	alias self_being_included=false
17 9003 aaronmk
	! isset "$include_guard" && \
18 8793 aaronmk
	{ eval "$include_guard"=1; alias self_being_included=true; }
19 8703 aaronmk
}
20
21 8793 aaronmk
# to load newly-defined aliases for use in functions in the same file:
22
## fi # load new aliases
23
## if self_being_included; then
24
# this is needed because aliases defined inside an if statement are not
25
# available inside that if statement
26
27 8704 aaronmk
if self_not_included "${BASH_SOURCE[0]}"; then
28
29 8709 aaronmk
shopt -s expand_aliases
30
31 8889 aaronmk
unalias () { builtin unalias "$@" 2>&- || true; } # no error if undefined
32
33 8886 aaronmk
#### exceptions
34
35 8976 aaronmk
alias rethrow='return "$e"'
36
alias rethrow_subshell='exit "$e"'
37
38 8978 aaronmk
fi # load new aliases
39
if self_being_included; then
40
41
# usage: save_e cmd... || use $e (or $?)
42
function save_e () { "$@"; e=$?; rethrow; } # rethrow because e= clears $?
43
alias save_e='declare e; save_e ' # last space alias-expands next word
44
45 8981 aaronmk
function log_e () # usage: log_e cmd... [ || use $e (or $?) ]
46
{
47 8982 aaronmk
	if ! "save_e" "$@"; then
48 8981 aaronmk
		echo "! $*" >&2
49
		echo "! command exited with error $e" >&2
50
		rethrow
51
	fi
52
}
53 8982 aaronmk
alias log_e='declare e; log_e ' # last space alias-expands next word
54 8981 aaronmk
55 8886 aaronmk
# usage: try cmd...; ignore status; if catch status; then ...; fi; end_try
56
57 8980 aaronmk
function try () { "save_e" "$@" || true; }
58
alias try='declare e; try ' # last space alias-expands next word
59 8886 aaronmk
60
catch () { test "$e" -eq "$1"; e=0; }
61
62
ignore () { catch "$@" || true; }
63
64 8977 aaronmk
alias end_try='rethrow'
65
alias end_try_subshell='rethrow_subshell'
66 8886 aaronmk
67 8899 aaronmk
#### integers
68
69
let () { builtin let "$@" || true; }
70
	# "If the last ARG evaluates to 0, let returns 1" (`help let`)
71
72 9004 aaronmk
bool2int () { save_e test -z "$1"; echo "$e"; } # empty->0; non-empty->1
73
74 8854 aaronmk
#### arrays
75 8694 aaronmk
76 8833 aaronmk
join () { local IFS="$delim"; echo "$*"; } # usage: delim=... join elems...
77 8816 aaronmk
78 8691 aaronmk
reverse () # usage: array=($(reverse args...))
79
{
80
	local i
81
	for (( i=$#; i >= 1; i-- )); do printf '%q ' "${!i}"; done
82
}
83
84 8854 aaronmk
#### verbose output
85
86 8973 aaronmk
# usage: (stdout2stderr; cmd...) || return
87
# `|| return` needed on Mac because of bug where -e doesn't apply to ()
88 8914 aaronmk
stdout2stderr () { exec >&2; }
89
90 9006 aaronmk
# set verbosity
91 9005 aaronmk
if isset verbose; then : "${verbosity:=$(bool2int "$verbose")}"; fi
92 9006 aaronmk
: "${verbosity=3}" # default
93
: "${verbosity:=0}" # ensure non-empty
94 9007 aaronmk
declare -i verbosity # ensure integer
95 9000 aaronmk
export verbosity # propagate the verbosity to invoked commands
96 8710 aaronmk
97 8898 aaronmk
can_log () { test "$verbosity" -gt 0; } # verbosity=0 turns off all logging
98 8895 aaronmk
99 8920 aaronmk
: "${log_indent=  }"
100 8919 aaronmk
101 8900 aaronmk
# usage: in func:      inc_log_level; ...
102
#        outside func: inc_log_level; ...; dec_log_level
103 8919 aaronmk
alias inc_log_level='declare verbosity="$verbosity" PS4="$log_indent$PS4"
104
let verbosity--'
105
alias dec_log_level='declare verbosity="$verbosity" PS4="${PS4#$log_indent}"
106
let verbosity++'
107 8900 aaronmk
108 8995 aaronmk
fi # load new aliases
109
if self_being_included; then
110
111 8994 aaronmk
# usage: (limit_stderr; cmd...) || return
112 8973 aaronmk
# `|| return` needed on Mac because of bug where -e doesn't apply to ()
113 8995 aaronmk
limit_stderr () { inc_log_level; if ! can_log; then exec 2>/dev/null; fi; }
114 8904 aaronmk
115 8994 aaronmk
limit_stderr_cmd () # usage: [stdout2stderr=1] limit_stderr_cmd cmd...
116 8915 aaronmk
{
117 9009 aaronmk
	case "$1" in echo_run) shift;; esac
118 8996 aaronmk
	echo_cmd "$@"
119 8994 aaronmk
	(limit_stderr
120 8915 aaronmk
		if test -n "$stdout2stderr"; then stdout2stderr; fi
121
		"$@"
122 8955 aaronmk
	) || return
123 8915 aaronmk
}
124 9008 aaronmk
alias limit_stderr_cmd='limit_stderr_cmd ' # last space alias-expands next word
125 8911 aaronmk
126 8897 aaronmk
echo_cmd () { if can_log; then echo "$PS4$*" >&2; fi; }
127 8272 aaronmk
128 8278 aaronmk
echo_run () { echo_cmd "$@"; "$@"; }
129
130 8984 aaronmk
# auto-echo common external commands
131
for cmd in rm; do alias "$cmd=echo_run $cmd"; done; unset cmd
132
133 8994 aaronmk
# limits stderr of (and echoes) an external command
134 8996 aaronmk
alias limit_stderr_extern='limit_stderr_cmd "extern" '
135 8983 aaronmk
	# last space alias-expands next word
136 8906 aaronmk
137 8994 aaronmk
# echo and limit stderr of all external commands
138
alias extern="limit_stderr_extern " # last space alias-expands next word
139 8907 aaronmk
140 8929 aaronmk
alias self='extern "$FUNCNAME"' # usage: wrapper () { self ...; }
141
142 8908 aaronmk
# commands that are always external
143
for cmd in env; do alias "$cmd=extern $cmd"; done; unset cmd
144 8872 aaronmk
145 8646 aaronmk
canon_rel_path ()
146
{
147
	local path="$1"
148 8716 aaronmk
	path="$(realpath "$path")" # canonicalize
149 8773 aaronmk
	path="${path#$(pwd -P)/}" # remove any shared prefix with the current dir
150 8646 aaronmk
	echo "$path"
151
}
152
153 8909 aaronmk
fi # load new aliases
154 8884 aaronmk
if self_being_included; then
155
156 8909 aaronmk
function echo_func ()
157 8463 aaronmk
{
158 8901 aaronmk
	inc_log_level
159 8647 aaronmk
	local script="$(canon_rel_path "${BASH_SOURCE[1]}")"
160
	echo_cmd "$script:${BASH_LINENO[0]}" "${FUNCNAME[1]}" "$@"
161 8463 aaronmk
}
162 8885 aaronmk
163 8997 aaronmk
# usage: cmd1 | { pipe_delay; cmd2; }
164
alias pipe_delay='sleep 0.1' # s; display after leading output of cmd1
165
166
fi # load new aliases
167
if self_being_included; then
168
169 8702 aaronmk
echo_stdin () # usage: input|echo_stdin|cmd
170
{
171 8901 aaronmk
	inc_log_level
172 8897 aaronmk
	if can_log; then
173 8998 aaronmk
		pipe_delay
174 8897 aaronmk
		echo ----- >&2
175
		tee -a /dev/stderr;
176
		echo ----- >&2
177
	else cat
178
	fi
179 8702 aaronmk
}
180 8275 aaronmk
181 8856 aaronmk
echo_vars () # usage: echo_vars var...
182 8901 aaronmk
{
183 8922 aaronmk
	inc_log_level; inc_log_level
184 8921 aaronmk
	if can_log; then
185
		local var
186
		for var in "${@%%=*}"; do { echo -n "$PS4"; declare -p "$var";} >&2;done
187
	fi
188 8901 aaronmk
}
189 8641 aaronmk
190
echo_export ()
191
{
192 8700 aaronmk
	builtin export "$@"
193 8641 aaronmk
	echo_vars "$@"
194
}
195
196 8713 aaronmk
if test "$verbosity" -ge 2; then
197 8711 aaronmk
	alias export="echo_export" # automatically echo env vars when they are set
198
fi
199 8642 aaronmk
200 8272 aaronmk
usage () { echo "Usage: $1" >&2; (exit 2); }
201
202 8873 aaronmk
fi # load new aliases
203
if self_being_included; then
204
205 8871 aaronmk
#### strings
206
207
sed_ere_flag="$(test "$(uname)" = Darwin && echo E || echo r)"
208
209 8930 aaronmk
sed () { self -"$sed_ere_flag" "$@";}
210 8871 aaronmk
211 8854 aaronmk
#### vars
212
213
set_var () { eval "$1"'="$2"'; }
214
215 8857 aaronmk
set_inv () { set_var no_"$1" "$(test -n "${!1}" || echo 1)"; }
216
217 8859 aaronmk
# usage: local var=...; local_inv
218 8888 aaronmk
alias local_inv='declare "no_$var=$(test -n "${!var}" || echo 1)"'
219 8859 aaronmk
220 8863 aaronmk
# usage: local prefix=..._; import_vars
221
alias import_vars="$(cat <<'EOF'
222
: "${prefix:?}"
223
local src_var dest_var
224
for src_var in $(eval echo '${!'$prefix'*}'); do
225
	dest_var="${src_var#$prefix}"
226
	local "$dest_var=${!src_var}"; echo_vars "$dest_var"
227
done
228
EOF
229
)"
230
231 8854 aaronmk
#### commands
232
233 8934 aaronmk
top_script="$0" # outermost script
234
top_dir="$(dirname "$top_script")"
235 8272 aaronmk
236 8986 aaronmk
# auto-removes a command's output file on error (like make's .DELETE_ON_ERROR)
237
function to_file () # usage: stdout=... to_file cmd...
238
{ : "${stdout?}"; log_e "$@" >"$stdout" || { rm "$stdout"; rethrow; }; }
239
alias to_file='to_file ' # last space alias-expands next word
240
241 8465 aaronmk
run_args_cmd () # runs the command line args command
242 8272 aaronmk
{
243 8971 aaronmk
	test $? -eq 0 || return
244 8693 aaronmk
	eval set -- "$(reverse "${BASH_ARGV[@]}")"
245 8971 aaronmk
	test $# -ge 1 || set -- all
246 8648 aaronmk
	echo_cmd "$(canon_rel_path "$0")" "$@"; "$@"
247 8272 aaronmk
}
248
249 8284 aaronmk
fwd () # usage: subdirs=(...); fwd "$FUNCNAME" "$@"
250 8272 aaronmk
{
251 8881 aaronmk
	echo_func
252 8284 aaronmk
	: "${subdirs?}"
253
254 8272 aaronmk
	for subdir in "${subdirs[@]}"; do
255 8280 aaronmk
		"$(dirname "${BASH_SOURCE[1]}")"/"$subdir"/run "$@"
256 8272 aaronmk
	done
257
}
258
259 8854 aaronmk
#### make
260
261 8881 aaronmk
# usage: target_filename/command () { echo_func; set_make_vars; ...; }
262 8880 aaronmk
alias set_make_vars="$(cat <<'EOF'
263
local command="${FUNCNAME##*/}"; echo_vars command
264
local target_filename="${FUNCNAME%/*}"; echo_vars target_filename
265
local target="$top_dir/$target_filename"; echo_vars target
266
EOF
267
)"
268
269 8989 aaronmk
# usage: set_make_vars; check_target_exists
270
alias check_target_exists='! test -e "$target" || return 0'
271
272 8987 aaronmk
# usage: set_make_vars; to_target cmd...
273
alias to_target='stdout="$target" to_file ' # last space alias-expands next word
274
275 8959 aaronmk
make () { echo_func; stdout2stderr=1 self "$@"; }
276 8272 aaronmk
277 8276 aaronmk
if false; then ## usage:
278 8652 aaronmk
inline_make 3<<'EOF'
279 8276 aaronmk
target:
280
	$(self_dir)/cmd >$@
281
EOF
282
# target will be run automatically because it's first in the makefile
283
fi ##
284
inline_make ()
285
{
286 8881 aaronmk
	echo_func
287 8640 aaronmk
	local self="$(readlink -f "${BASH_SOURCE[1]}")"
288
	local self_dir="$(dirname "$self")"
289 8645 aaronmk
	export self self_dir
290 8640 aaronmk
291 8651 aaronmk
	make --makefile=<((
292 8652 aaronmk
		cat /dev/fd/3
293 8651 aaronmk
		echo -n "
294 8276 aaronmk
.SUFFIXES: # turn off built-in suffix rules
295
.SECONDARY: # don't automatically delete intermediate files
296
.DELETE_ON_ERROR: # delete target if recipe fails
297 8651 aaronmk
"
298
	)|echo_stdin) "$@"
299 8276 aaronmk
}
300 8658 aaronmk
301 8854 aaronmk
#### compression
302
303
### zip
304
305 8887 aaronmk
zip ()
306
{
307 8930 aaronmk
	stdout2stderr=1 try self "$@"
308 8887 aaronmk
	ignore 12 # "zip has nothing to do" (`man zip`)
309
	end_try
310
}
311
312 8930 aaronmk
unzip () { stdout2stderr=1 self "$@"; }
313 8902 aaronmk
314 8858 aaronmk
set_inv force
315 8974 aaronmk
alias zip_newer='zip ${no_force:+-u }'
316
alias unzip_newer='unzip ${no_force:+-u }-o'
317 8858 aaronmk
	# -o is safe because -u only extracts newer files
318 8704 aaronmk
319 8966 aaronmk
#### URLs
320
321
localize_url () { test _"$1" = _"$(hostname -f)" || echo "$1"; }
322
323 8775 aaronmk
#### databases
324
325 8868 aaronmk
# using prefixed connection vars
326 8975 aaronmk
alias use_local='declare prefix=local_; import_vars'
327
alias use_remote='declare prefix=remote_; import_vars'
328
alias use_local_remote='use_local; use_remote'
329 8864 aaronmk
330 8775 aaronmk
quote='"'
331
332
esc_name () { echo "$quote${1//$quote/$quote$quote}$quote"; }
333
334
mk_esc_name () { set_var "$1"_esc "$(esc_name "${!1}")"; }
335
336 8975 aaronmk
alias mk_schema_esc='declare schema_esc; mk_esc_name schema'
337
alias mk_table_esc='declare table_esc; mk_esc_name table'
338 8796 aaronmk
339
fi # load new aliases
340
if self_being_included; then
341
342
log_sql () { test "$verbosity" -ge 2; }
343
344
### MySQL
345
346 8862 aaronmk
# auto-adds connection/login opts when specified
347 8865 aaronmk
mysql_cmd () # usage: mysql* () { ...; mysql_cmd "$@"; }
348 8860 aaronmk
{
349 8881 aaronmk
	echo_func
350 8967 aaronmk
	local ssh_server="$(localize_url "$ssh_server")"
351 8968 aaronmk
	local server="$(localize_url "$server")"
352 8869 aaronmk
	if test -n "$ssh_server"; then
353
		local ssh_dest="${ssh_dest-${ssh_user:+$ssh_user@}$ssh_server}"
354
	fi
355 8875 aaronmk
	if test -n "$schema"; then local database="${database-$schema}"; fi
356 8869 aaronmk
357 8860 aaronmk
	local var=ssh_dest; local_inv
358 8910 aaronmk
	extern ${ssh_dest:+ssh "$ssh_dest" }"${FUNCNAME[1]}" \
359 8879 aaronmk
${server:+ --host="$server" }${user:+--user="$user" } --password\
360
${password+="$password"} ${database:+--databases "$database" --tables } "$@"
361 8860 aaronmk
}
362
363 8959 aaronmk
mysql () { echo_func; mysql_cmd --verbose "$@"; }
364 8775 aaronmk
365
mysql_ANSI ()
366
{
367 8881 aaronmk
	echo_func
368 8775 aaronmk
	(echo "SET sql_mode = 'ANSI';"; cat)|mysql "$@"
369
}
370
371 8866 aaronmk
mysqldump () # usage: [schema=1 | data=1] mysqldump db [table...]
372
{
373 8881 aaronmk
	echo_func
374 8866 aaronmk
	mysql_cmd --quick --lock-tables=false --set-charset \
375
${postgres_compat:+--compatible=postgresql --add-locks=false }\
376
${schema:+--no-data }${data:+--no-create-info }"$@"
377
}
378
379 8867 aaronmk
mysqldump_diffable ()
380
{
381 8881 aaronmk
	echo_func
382 8867 aaronmk
	mysqldump "$@"|sed 's/^(-- Dump completed).*$/\1/'
383
}
384
385 8796 aaronmk
### PostgreSQL
386
387
pg_copy_to ()
388
{
389 8881 aaronmk
	echo_func
390 8805 aaronmk
	if test -z "$source"; then
391
		: "${table:?}"; mk_table_esc
392 8834 aaronmk
		if test -z "$limit"; then local source="$table_esc"
393
		else local source="(SELECT * FROM $table_esc LIMIT $limit)"
394 8805 aaronmk
		fi
395
	fi
396 8834 aaronmk
	local pg_copy_format="${pg_copy_format-CSV HEADER}"
397 8805 aaronmk
398 8807 aaronmk
	psql "$@" <<<"COPY $source TO STDOUT $pg_copy_format;"
399 8796 aaronmk
}
400
401
pg_header ()
402
{
403 8881 aaronmk
	echo_func
404 8806 aaronmk
	local pg_copy_format="CSV HEADER" limit=0
405 8807 aaronmk
	pg_copy_to "$@"|echo_stdin
406 8796 aaronmk
}
407
408
pg_export_table_no_header ()
409
{
410 8881 aaronmk
	echo_func
411 8796 aaronmk
	local pg_copy_format="CSV"
412 8807 aaronmk
	pg_copy_to "$@"
413 8796 aaronmk
}
414
415
pg_export_table_to_dir_no_header ()
416
{
417 8881 aaronmk
	echo_func
418 8796 aaronmk
	local table="$1"; shift; mk_table_esc
419 8807 aaronmk
	local cols="$(pg_header)"
420 8796 aaronmk
	pg_export_table_no_header "$@" >"$exports_dir/$table.no_header.cols=$cols.csv"
421
}
422
423 8704 aaronmk
fi