Revision 8881
Added by Aaron Marcuse-Kubitza over 11 years ago
| inputs/GBIF/table.run | ||
|---|---|---|
| 5 | 5 |
|
| 6 | 6 |
MySQL_export () |
| 7 | 7 |
{
|
| 8 |
echo_func "$@"
|
|
| 8 |
echo_func |
|
| 9 | 9 |
inline_make 3<<'EOF' |
| 10 | 10 |
table.tsv: |
| 11 | 11 |
$(self_dir)/MySQL_export >$@ |
| ... | ... | |
| 14 | 14 |
|
| 15 | 15 |
import () |
| 16 | 16 |
{
|
| 17 |
echo_func "$@"
|
|
| 17 |
echo_func |
|
| 18 | 18 |
MySQL_export |
| 19 | 19 |
"$(dirname "${BASH_SOURCE[0]}")"/../../lib/runscripts/table.run "$FUNCNAME" "$@"
|
| 20 | 20 |
} |
| inputs/GBIF/raw_occurrence_record/run | ||
|---|---|---|
| 5 | 5 |
|
| 6 | 6 |
MySQL_export () |
| 7 | 7 |
{
|
| 8 |
echo_func "$@"
|
|
| 8 |
echo_func |
|
| 9 | 9 |
"$(dirname "${BASH_SOURCE[0]}")"/../table.run "$FUNCNAME" "$@"
|
| 10 | 10 |
} |
| 11 | 11 |
|
| inputs/GBIF/_MySQL/run | ||
|---|---|---|
| 5 | 5 |
|
| 6 | 6 |
import () |
| 7 | 7 |
{
|
| 8 |
echo_func "$@"
|
|
| 8 |
echo_func |
|
| 9 | 9 |
make |
| 10 | 10 |
} |
| 11 | 11 |
|
| inputs/GBIF/run | ||
|---|---|---|
| 7 | 7 |
|
| 8 | 8 |
import () |
| 9 | 9 |
{
|
| 10 |
echo_func "$@"
|
|
| 10 |
echo_func |
|
| 11 | 11 |
fwd "$FUNCNAME" "$@" |
| 12 | 12 |
} |
| 13 | 13 |
|
| schemas/VegCore/VegCore.ERD.mwb.run | ||
|---|---|---|
| 10 | 10 |
|
| 11 | 11 |
export_ () |
| 12 | 12 |
{
|
| 13 |
echo_func "$@"
|
|
| 13 |
echo_func |
|
| 14 | 14 |
(cd "$top_dir" |
| 15 | 15 |
unzip_newer "$archive_filename" document.mwb.xml) |
| 16 | 16 |
} |
| 17 | 17 |
|
| 18 | 18 |
import () |
| 19 | 19 |
{
|
| 20 |
echo_func "$@"
|
|
| 20 |
echo_func |
|
| 21 | 21 |
(cd "$top_dir" |
| 22 | 22 |
zip_newer "$archive_filename" document.mwb.xml) |
| 23 | 23 |
} |
| 24 | 24 |
|
| 25 | 25 |
sync () |
| 26 | 26 |
{
|
| 27 |
echo_func "$@"
|
|
| 27 |
echo_func |
|
| 28 | 28 |
export_ "$@" |
| 29 | 29 |
import "$@" |
| 30 | 30 |
} |
| 31 | 31 |
|
| 32 | 32 |
all () |
| 33 | 33 |
{
|
| 34 |
echo_func "$@"
|
|
| 34 |
echo_func |
|
| 35 | 35 |
sync |
| 36 | 36 |
} |
| 37 | 37 |
|
| run | ||
|---|---|---|
| 9 | 9 |
|
| 10 | 10 |
export_ () |
| 11 | 11 |
{
|
| 12 |
echo_func "$@"
|
|
| 12 |
echo_func |
|
| 13 | 13 |
pg_export_table_to_dir_no_header geoscrub_input "$@" |
| 14 | 14 |
} |
| 15 | 15 |
|
| lib/runscripts/util.run | ||
|---|---|---|
| 10 | 10 |
|
| 11 | 11 |
cmd () |
| 12 | 12 |
{
|
| 13 |
echo_func "$@"
|
|
| 13 |
echo_func |
|
| 14 | 14 |
"$(dirname "${BASH_SOURCE[0]}")"/path_relative_to_self
|
| 15 | 15 |
"$(dirname "${BASH_SOURCE[1]}")"/path_relative_to_caller
|
| 16 | 16 |
"$top_dir"/path_relative_to_outermost_script |
| lib/runscripts/table.run | ||
|---|---|---|
| 6 | 6 |
|
| 7 | 7 |
import () |
| 8 | 8 |
{
|
| 9 |
echo_func "$@"
|
|
| 9 |
echo_func |
|
| 10 | 10 |
before_import_cmds |
| 11 | 11 |
"$(dirname "${BASH_SOURCE[0]}")"/path/to/table.run "$FUNCNAME" "$@"
|
| 12 | 12 |
after_import_cmds |
| ... | ... | |
| 26 | 26 |
|
| 27 | 27 |
input_make () |
| 28 | 28 |
{
|
| 29 |
echo_func "$@"
|
|
| 29 |
echo_func |
|
| 30 | 30 |
env make --directory="$top_dir"/.. "${@/#/$table/}"
|
| 31 | 31 |
} |
| 32 | 32 |
|
| 33 | 33 |
map_table () |
| 34 | 34 |
{
|
| 35 |
echo_func "$@"
|
|
| 35 |
echo_func |
|
| 36 | 36 |
input_make map.csv |
| 37 | 37 |
psql <<EOF |
| 38 | 38 |
SELECT util.reset_map_table('pg_temp.map');
|
| ... | ... | |
| 43 | 43 |
|
| 44 | 44 |
mk_derived () |
| 45 | 45 |
{
|
| 46 |
echo_func "$@"
|
|
| 46 |
echo_func |
|
| 47 | 47 |
"$root_dir"/schemas/VegCore/mk_derived |
| 48 | 48 |
} |
| 49 | 49 |
|
| 50 | 50 |
remake_VegBIEN_mappings () |
| 51 | 51 |
{
|
| 52 |
echo_func "$@"
|
|
| 52 |
echo_func |
|
| 53 | 53 |
echo_run rm header.csv map.csv # remake them |
| 54 | 54 |
yes|input_make test |
| 55 | 55 |
} |
| 56 | 56 |
|
| 57 | 57 |
postprocess () # overridable |
| 58 | 58 |
{
|
| 59 |
echo_func "$@"
|
|
| 59 |
echo_func |
|
| 60 | 60 |
local file="$top_dir"/postprocess.sql |
| 61 | 61 |
if test -e "$file"; then psql "$@"; fi |
| 62 | 62 |
} |
| 63 | 63 |
|
| 64 | 64 |
import () |
| 65 | 65 |
{
|
| 66 |
echo_func "$@"
|
|
| 66 |
echo_func |
|
| 67 | 67 |
map_table |
| 68 | 68 |
postprocess |
| 69 | 69 |
mk_derived |
| lib/runscripts/import.run | ||
|---|---|---|
| 6 | 6 |
|
| 7 | 7 |
import () |
| 8 | 8 |
{
|
| 9 |
echo_func "$@"
|
|
| 9 |
echo_func |
|
| 10 | 10 |
import_cmds |
| 11 | 11 |
} |
| 12 | 12 |
fi #### |
| ... | ... | |
| 17 | 17 |
|
| 18 | 18 |
import () |
| 19 | 19 |
{
|
| 20 |
echo_func "$@"
|
|
| 20 |
echo_func |
|
| 21 | 21 |
} |
| 22 | 22 |
|
| 23 | 23 |
export_ () |
| 24 | 24 |
{
|
| 25 |
echo_func "$@"
|
|
| 25 |
echo_func |
|
| 26 | 26 |
} |
| 27 | 27 |
|
| 28 | 28 |
all () |
| 29 | 29 |
{
|
| 30 |
echo_func "$@"
|
|
| 30 |
echo_func |
|
| 31 | 31 |
import |
| 32 | 32 |
export_ |
| 33 | 33 |
} |
| lib/runscripts/local.run | ||
|---|---|---|
| 11 | 11 |
|
| 12 | 12 |
mysql () |
| 13 | 13 |
{
|
| 14 |
echo_func "$@"
|
|
| 14 |
echo_func |
|
| 15 | 15 |
echo_run mysql_bien --database="$schema" --verbose "$@" |
| 16 | 16 |
} |
| 17 | 17 |
|
| 18 | 18 |
psql () # usage: [file=...] [dir=...] self |
| 19 | 19 |
{
|
| 20 |
echo_func "$@"
|
|
| 20 |
echo_func |
|
| 21 | 21 |
if test -n "$file"; then |
| 22 | 22 |
set -- --file "$file" "$@" |
| 23 | 23 |
local dir="${dir:-$(dirname "$file")}"
|
| lib/util.sh | ||
|---|---|---|
| 52 | 52 |
echo "$path" |
| 53 | 53 |
} |
| 54 | 54 |
|
| 55 |
# usage: echo_func "$@" |
|
| 56 | 55 |
echo_func () |
| 57 | 56 |
{
|
| 58 | 57 |
local script="$(canon_rel_path "${BASH_SOURCE[1]}")"
|
| 59 | 58 |
echo_cmd "$script:${BASH_LINENO[0]}" "${FUNCNAME[1]}" "$@"
|
| 60 | 59 |
} |
| 60 |
alias echo_func='"echo_func" "$@"' # usage: func () { echo_func; ...; }
|
|
| 61 | 61 |
|
| 62 | 62 |
echo_stdin () # usage: input|echo_stdin|cmd |
| 63 | 63 |
{
|
| ... | ... | |
| 124 | 124 |
|
| 125 | 125 |
fwd () # usage: subdirs=(...); fwd "$FUNCNAME" "$@" |
| 126 | 126 |
{
|
| 127 |
echo_func "$@"
|
|
| 127 |
echo_func |
|
| 128 | 128 |
: "${subdirs?}"
|
| 129 | 129 |
|
| 130 | 130 |
for subdir in "${subdirs[@]}"; do
|
| ... | ... | |
| 134 | 134 |
|
| 135 | 135 |
#### make |
| 136 | 136 |
|
| 137 |
# usage: target_filename/command () { echo_func "$@"; set_make_vars; ...; }
|
|
| 137 |
# usage: target_filename/command () { echo_func; set_make_vars; ...; }
|
|
| 138 | 138 |
alias set_make_vars="$(cat <<'EOF' |
| 139 | 139 |
local command="${FUNCNAME##*/}"; echo_vars command
|
| 140 | 140 |
local target_filename="${FUNCNAME%/*}"; echo_vars target_filename
|
| ... | ... | |
| 144 | 144 |
|
| 145 | 145 |
make () |
| 146 | 146 |
{
|
| 147 |
echo_func "$@"
|
|
| 147 |
echo_func |
|
| 148 | 148 |
env make --directory="$top_dir" "$@" |
| 149 | 149 |
} |
| 150 | 150 |
|
| ... | ... | |
| 157 | 157 |
fi ## |
| 158 | 158 |
inline_make () |
| 159 | 159 |
{
|
| 160 |
echo_func "$@"
|
|
| 160 |
echo_func |
|
| 161 | 161 |
local self="$(readlink -f "${BASH_SOURCE[1]}")"
|
| 162 | 162 |
local self_dir="$(dirname "$self")" |
| 163 | 163 |
export self self_dir |
| ... | ... | |
| 209 | 209 |
# auto-adds connection/login opts when specified |
| 210 | 210 |
mysql_cmd () # usage: mysql* () { ...; mysql_cmd "$@"; }
|
| 211 | 211 |
{
|
| 212 |
echo_func "$@"
|
|
| 212 |
echo_func |
|
| 213 | 213 |
if test _"$ssh_server" = _"$(hostname -f)"; then local ssh_server=; fi |
| 214 | 214 |
if test -n "$ssh_server"; then |
| 215 | 215 |
local ssh_dest="${ssh_dest-${ssh_user:+$ssh_user@}$ssh_server}"
|
| ... | ... | |
| 224 | 224 |
|
| 225 | 225 |
mysql () |
| 226 | 226 |
{
|
| 227 |
echo_func "$@"
|
|
| 227 |
echo_func |
|
| 228 | 228 |
mysql_cmd --verbose "$@" |
| 229 | 229 |
} |
| 230 | 230 |
|
| 231 | 231 |
mysql_ANSI () |
| 232 | 232 |
{
|
| 233 |
echo_func "$@"
|
|
| 233 |
echo_func |
|
| 234 | 234 |
(echo "SET sql_mode = 'ANSI';"; cat)|mysql "$@" |
| 235 | 235 |
} |
| 236 | 236 |
|
| 237 | 237 |
mysqldump () # usage: [schema=1 | data=1] mysqldump db [table...] |
| 238 | 238 |
{
|
| 239 |
echo_func "$@"
|
|
| 239 |
echo_func |
|
| 240 | 240 |
mysql_cmd --quick --lock-tables=false --set-charset \ |
| 241 | 241 |
${postgres_compat:+--compatible=postgresql --add-locks=false }\
|
| 242 | 242 |
${schema:+--no-data }${data:+--no-create-info }"$@"
|
| ... | ... | |
| 244 | 244 |
|
| 245 | 245 |
mysqldump_diffable () |
| 246 | 246 |
{
|
| 247 |
echo_func "$@"
|
|
| 247 |
echo_func |
|
| 248 | 248 |
mysqldump "$@"|sed 's/^(-- Dump completed).*$/\1/' |
| 249 | 249 |
} |
| 250 | 250 |
|
| ... | ... | |
| 252 | 252 |
|
| 253 | 253 |
pg_copy_to () |
| 254 | 254 |
{
|
| 255 |
echo_func "$@"
|
|
| 255 |
echo_func |
|
| 256 | 256 |
if test -z "$source"; then |
| 257 | 257 |
: "${table:?}"; mk_table_esc
|
| 258 | 258 |
if test -z "$limit"; then local source="$table_esc" |
| ... | ... | |
| 266 | 266 |
|
| 267 | 267 |
pg_header () |
| 268 | 268 |
{
|
| 269 |
echo_func "$@"
|
|
| 269 |
echo_func |
|
| 270 | 270 |
local pg_copy_format="CSV HEADER" limit=0 |
| 271 | 271 |
pg_copy_to "$@"|echo_stdin |
| 272 | 272 |
} |
| 273 | 273 |
|
| 274 | 274 |
pg_export_table_no_header () |
| 275 | 275 |
{
|
| 276 |
echo_func "$@"
|
|
| 276 |
echo_func |
|
| 277 | 277 |
local pg_copy_format="CSV" |
| 278 | 278 |
pg_copy_to "$@" |
| 279 | 279 |
} |
| 280 | 280 |
|
| 281 | 281 |
pg_export_table_to_dir_no_header () |
| 282 | 282 |
{
|
| 283 |
echo_func "$@"
|
|
| 283 |
echo_func |
|
| 284 | 284 |
local table="$1"; shift; mk_table_esc |
| 285 | 285 |
local cols="$(pg_header)" |
| 286 | 286 |
pg_export_table_no_header "$@" >"$exports_dir/$table.no_header.cols=$cols.csv" |
| web/links/index.htm.run | ||
|---|---|---|
| 8 | 8 |
|
| 9 | 9 |
clean_up () |
| 10 | 10 |
{
|
| 11 |
echo_func "$@"
|
|
| 11 |
echo_func |
|
| 12 | 12 |
echo_run in_place "$linked_file" repl \ |
| 13 | 13 |
"$root_dir"/lib/Firefox_bookmarks.reformat.csv |
| 14 | 14 |
} |
| 15 | 15 |
|
| 16 | 16 |
all () |
| 17 | 17 |
{
|
| 18 |
echo_func "$@"
|
|
| 18 |
echo_func |
|
| 19 | 19 |
clean_up |
| 20 | 20 |
} |
| 21 | 21 |
|
Also available in: Unified diff
*{.sh,run}: changed echo_func to an alias that includes the "$
", so that callers don't need to specify the "$" manually. although the original echo_func function is still there, callers need to switch over to the alias at the same time as it's defined because otherwise the "$@" would be doubled, since echo_func refers to the alias instead.