herbaria_filter.ih.csv_/make()

{

	use_local

	limit= query='SELECT DISTINCT "specimenHolderInstitutions" FROM herbaria.ih ORDER BY "specimenHolderInstitutions"' \

ih_herbarium/make() # stores IH herbaria

{

	use_local_remote

	local table="$target_stem"

plant_fraction/make() # runtime: 1 hr ("10603 rows affected (53 min 44.68 sec)")

{

	use_local_remote

	if remaking; then table=plant_fraction table_esc= mk_drop|mysql; fi

plant_fraction/seal()

{

	use_local_remote

	table=plant_fraction table_esc= mysql_seal_table

plant_fraction_for_herbaria_filter/make()

{

	plant_fraction/make

	use_local_remote

herbaria_filter/make() # runtime: ~0 ("0m2.235s"): CSV import, index scans only

{

	use_local_remote

	local table="$target_stem"

herbaria_filter/seal()

{

	use_local_remote

	table=herbaria_filter table_esc= mysql_seal_table

table/make() # makes the filter view

{

	herbaria_filter/make

	use_local_remote

# to view runtime when using `screen`:

# press Ctrl-A [ , use arrow keys/Ctrl-B/Ctrl-F, and then Esc to leave copy mode

{

	table/make

	self_make table.tsv/make__src_table_run

func_override ^.preamble.sql/make__resume_import_sh

^.preamble.sql/make()

{

	top_file="$dumpfile" self_make ^.preamble.sql/make__resume_import_sh

}

refresh() # usage: .../data.sql.run refresh

{

	"$top_dir"/../test_taxonomic_names/test_scrub

	remake=1 export_ # update data.sql with the changes

}

*.htm/make()

{

	local letter_range="${letter_range-[A-Z]}"

	local rank="${table%_homonyms}"

	curl --output '#1.htm' "http://www.marine.csiro.au/mirrorsearch/ir_search.list_homonyms?hlevel=$rank&start_letter=$letter_range&restrict_by=pln&output_style=delim_rows"

*.txt/make()

{

	*.htm/make

	for in in "$top_dir"/*.htm; do

		stdout="${in/%.htm/.txt}" to_file grep -vE '^<' "$in" # strip tags

func_override import__table_run

import()

{

	*.txt/make

	self_make import__table_run "$@"

}

refresh() # usage: on *vegbiendev*: .../run refresh

{

	local remake=1 # run all commands below in this mode

	datasrc_make vegbank.schema.sql

	datasrc_make vegbank.data.sql # runtime: 3.5 min (":17:38" - ":14:16" =3:22)

../VegCore.my.sql/make()

{

	database=VegCore create_db=1 mysqldump_local --skip-add-drop-table \

>"$target"

}

../VegCore.pg.sql/make()

{

	sql_ascii=1 my2pg <"${target%.pg.sql}.my.sql" >"$target"

}

cleanup()

{

	in_place "$top_file" $sed_cmd \

-e 's/&/\&/g

# undo double-encodings' \

header.htm/make()

{

	stdout="$target" to_file sed \

-e 's/( alt="[^:"]*): ([^"]*)(")/\1\3 title="\2"/g' \

"$top_file"

all() # runtime: 1 min ("0m56.053s")

{

	if ! isset password; then

		local_export password; read -rsp 'Enter your MySQL password: ' password

taxon_trait/make() # runtime: ~0 ("real 0m0.411s") @ r10826

{

	local order_by=(scientificName measurementType)

	pg_export_table_to_dir "$target_filename"

}

import()

{

	if remaking; then datasrc_make uninstall; fi

	prep_try; if ! pg_schema_exists; then # uses $schema

		rethrow

export_() # usage: rm=1 .../file.run export_

{

	create_schema= to_top_file pg_dump_local \

--exclude-table='*.*\y"Source"\y*' "$@"

}

load_data()

{

	if remaking; then in_top_dir rm -f header.csv; fi # remake it

	datasrc_make_install schema

	is_view="$is_view" table_make_install ${_remake:+re}install # just the table

trim_table()

{

	if test "$is_view"; then return 0; fi # do not modify view columns

	psql <<EOF

SELECT util.trim('"$table"', '$map_table')

reset_col_names()

{

	if test "$is_view"; then return 0; fi # do not rename view columns

	psql <<EOF

SELECT util.reset_col_names('"$table"', '$map_table')

# collisions may result if the staging table gets messed up (e.g. due to missing

# input columns in map.csv).

{

	if test "$is_view"; then return 0; fi # do not rename view columns

	if remaking; then reset_col_names; fi

EOF

}

mappings()

{

	prep_try; public_schema_exists || { rethrow; return 0; }; rethrow

	if remaking; then in_top_dir rm -f map.csv; fi # remake it

	test_

custom_postprocess() # overridable

{

	local file="$postprocess_sql"

	if test -e "$file"; then

		table_make "$(basename "$file")" # apply renames to SQL statements

}

mk_derived()

postprocess()

{

	kw_params can_test; local can_test="${can_test-1}"

	if remaking; then trim_table; fi

	map_table

import()

{

	self_make load_data

	postprocess

}

table.tsv.md5/make()

{

	table_make "$target_filename"

}

table.tsv.gz/upload()

{

	live=1 sync_upload "$target"

}

table.tsv.gz/make()

{

	table_make "$target_filename"

	if test ! "$test"; then table.tsv.gz/upload; fi

}

table.tsv/make() # usage: [remake=1] [limit=...] .../run table.tsv/make

{

	(set_large_table; to_target mysql_export_local) || return

	remake=1 bg_cmd "$target_filename".gz/make; sleep 6 # wait for log msgs

	remake=1 "$target_filename".md5/make

archive_prev_versions()

{

	(mv_glob table.*.tsv "$top_dir"/_archive/)

}

func_override import__table_run

import()

{

	export_mysql

	archive_prev_versions

	self_make import__table_run "$@"

datasrc_make() # runs make in datasrc dir

# requires Makefile in datasrc dir with `include ../input.Makefile`

{

	if remaking; then set -- "${@/%/-remake}"; fi #replace empty str at end (/%)

	make --directory="$top_dir" "$@"

}

export_()

{

	table="$extract_view" to_top_file pg_export

}

^.preamble.sql/make()

{

	local target="$preamble_file"; check_target_exists

	to_target sql_preamble <"$top_file"

}

pg_export_table_to_dir()

{

	local table="$1"; shift; mk_table_esc

	stdout="$exports_dir/$table.csv" to_file pg_export "$@"

}