$ aaronmk@frenzy:~/Dropbox/svn inputs/VegBank/run + inputs/VegBank/run | + main @lib/runscripts/util.run:151 | | + all @lib/runscripts/install.run:23 | | | + install @lib/runscripts/import.run:29 | | | | + import @lib/runscripts/datasrc_dir.run:29 | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | + declare -x PGHOST="localhost" | | | | | | | + declare -x PGUSER="bien" | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.281 ms CREATE TEMP TABLE "VegBank".t () Time: 0.463 ms psql:/dev/fd/40:4: ERROR: cannot create temporary relation in non-temporary schema real 0m0.115s user 0m0.035s sys 0m0.038s | | | | | + inputs/VegBank/Source/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/Source/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.201 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.271 ms 0.05 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.662s user 0m0.377s sys 0m0.200s real 0m0.168s user 0m0.022s sys 0m0.023s : : : : : : : : : table Source already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank Source/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile Source/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place Source/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place Source/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place Source/map.csv ../../bin/fix_line_endings touch Source/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile Source/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile Source/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 Source/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >Source/unmapped_terms.csv ../../bin/autoremove Source/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile Source/new_terms.csv /usr/bin/make --makefile=../input.Makefile Source/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 Source/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 Source/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >Source/new_terms.csv; exit 0 ../../bin/autoremove Source/new_terms.csv real 0m3.419s user 0m2.261s sys 0m0.917s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.214 ms \cd inputs/VegBank/Source \set schema "VegBank" \set table "Source" \set table_str '''"Source"''' SET search_path TO "VegBank"; Time: 0.103 ms SELECT util.reset_map_table('"~Source.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~Source.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~Source.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~Source.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 662.608 ms ALTER TABLE "~Source.map" DISABLE TRIGGER map_filter_insert; Time: 0.536 ms \copy "~Source.map" FROM 'map.csv' CSV HEADER; Time: 0.720 ms SELECT util.set_col_names_with_metadata('"Source"', '"~Source.map"'); psql:/dev/fd/40:12: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; ALTER TABLE "Source" ADD COLUMN "sourceType" text NOT NULL DEFAULT 'aggregator'; COMMENT ON COLUMN "Source"."sourceType" IS ' constant '; psql:/dev/fd/40:12: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; ALTER TABLE "Source" ADD COLUMN "observationGranularity" text NOT NULL DEFAULT 'plot'; COMMENT ON COLUMN "Source"."observationGranularity" IS ' constant '; set_col_names_with_metadata ----------------------------- (1 row) Time: 106.993 ms real 0m0.836s user 0m0.029s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd schemas/VegCore \set schema "VegBank" \set table "Source" \set table_str '''"Source"''' SET search_path TO "VegBank"; Time: 0.100 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 34.405 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('("""Source""",cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 30.862 ms real 0m0.125s user 0m0.027s sys 0m0.026s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.186 ms \cd inputs/VegBank/Source \set schema "public" \set table "Source" \set table_str '''"Source"''' SET search_path TO "public"; Time: 0.148 ms CREATE TEMP TABLE "public".t () Time: 0.332 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.081s user 0m0.035s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank Source/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile Source/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "Source/VegBIEN.csv" ln: Source/VegBIEN.csv: File exists make: [Source/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/Source/test.xml... >Source/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=Source out_database=vegbien source=VegBank.new ../../map Source/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 1 input rows Took 0:00:01.196747 sec/1 row(s) = 1.20 s/row Encountered 0 error(s)/1 row(s) = 0.0% + diff --unified=2 --ignore-space-change Source/test.xml.ref Source/test.xml real 0m3.531s user 0m0.978s sys 0m0.424s real 0m11.393s user 0m4.559s sys 0m2.570s | | | | | + inputs/VegBank/project/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/project/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.194 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.222 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.618s user 0m0.385s sys 0m0.199s real 0m0.239s user 0m0.023s sys 0m0.024s : : : : : : : : : table project already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank project/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile project/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place project/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place project/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place project/map.csv ../../bin/fix_line_endings touch project/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile project/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile project/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 project/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >project/unmapped_terms.csv ../../bin/autoremove project/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile project/new_terms.csv /usr/bin/make --makefile=../input.Makefile project/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 project/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 project/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >project/new_terms.csv; exit 0 ../../bin/autoremove project/new_terms.csv real 0m2.996s user 0m2.274s sys 0m0.899s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.246 ms \cd inputs/VegBank/project \set schema "VegBank" \set table "project" \set table_str '''"project"''' SET search_path TO "VegBank"; Time: 0.107 ms SELECT util.reset_map_table('"~project.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~project.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~project.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~project.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 159.135 ms ALTER TABLE "~project.map" DISABLE TRIGGER map_filter_insert; Time: 0.359 ms \copy "~project.map" FROM 'map.csv' CSV HEADER; Time: 0.564 ms SELECT util.set_col_names_with_metadata('"project"', '"~project.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.460 ms real 0m0.227s user 0m0.028s sys 0m0.028s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.189 ms \cd schemas/VegCore \set schema "VegBank" \set table "project" \set table_str '''"project"''' SET search_path TO "VegBank"; Time: 0.091 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 30.788 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(project,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.285 ms real 0m0.096s user 0m0.027s sys 0m0.027s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.185 ms \cd inputs/VegBank/project \set schema "public" \set table "project" \set table_str '''"project"''' SET search_path TO "public"; Time: 0.096 ms CREATE TEMP TABLE "public".t () Time: 0.289 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.080s user 0m0.035s sys 0m0.037s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank project/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile project/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "project/VegBIEN.csv" ln: project/VegBIEN.csv: File exists make: [project/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/project/test.xml... >project/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=project out_database=vegbien source=VegBank.new ../../map project/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.640686 sec/2 row(s) = 320. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change project/test.xml.ref project/test.xml real 0m1.984s user 0m1.020s sys 0m0.388s real 0m8.758s user 0m4.632s sys 0m2.522s | | | | | + inputs/VegBank/plot/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/plot/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.188 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.219 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.628s user 0m0.398s sys 0m0.205s real 0m0.158s user 0m0.022s sys 0m0.023s : : : : : : : : : table plot already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place plot/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place plot/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place plot/map.csv ../../bin/fix_line_endings touch plot/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile plot/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile plot/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 plot/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >plot/unmapped_terms.csv ../../bin/autoremove plot/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile plot/new_terms.csv /usr/bin/make --makefile=../input.Makefile plot/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 plot/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 plot/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >plot/new_terms.csv; exit 0 ../../bin/autoremove plot/new_terms.csv real 0m2.962s user 0m2.248s sys 0m0.890s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.206 ms \cd inputs/VegBank/plot \set schema "VegBank" \set table "plot" \set table_str '''"plot"''' SET search_path TO "VegBank"; Time: 0.113 ms SELECT util.reset_map_table('"~plot.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~plot.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~plot.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~plot.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 113.048 ms ALTER TABLE "~plot.map" DISABLE TRIGGER map_filter_insert; Time: 0.327 ms \copy "~plot.map" FROM 'map.csv' CSV HEADER; Time: 1.311 ms SELECT util.set_col_names_with_metadata('"plot"', '"~plot.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.735 ms real 0m0.183s user 0m0.029s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place plot/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.968s user 0m0.716s sys 0m0.215s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.198 ms \cd inputs/VegBank/plot \set schema "VegBank" \set table "plot" \set table_str '''"plot"''' SET search_path TO "VegBank"; Time: 0.113 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.224 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("locationID")$$); -- runtime: 0.2 s ("Time: 238.474 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "plot" ADD PRIMARY KEY ("locationID") create_if_not_exists ---------------------- (1 row) Time: 19.890 ms -- remove institutions that we have direct data for -- CVS DELETE FROM :table WHERE "locationName" IN (SELECT "locationName" FROM "CVS".plot_) -- removes 2323 of 7079 CVS plots (75340 - 73017) ; -- runtime: 1 min ("48999.044 ms") @starscream Time: 392.541 ms -- map_*() derived cols -- runtime: 2 s @starscream CREATE OR REPLACE FUNCTION map_access_rights(value integer) RETURNS text AS $BODY$ /* "closed List of Values: Value Description Sorting 0 Public 1 1 1 km radius (nearest 0.01 degree) 2 2 10 km radius (nearest 0.1 degree) 3 3 100 km radius (nearest degree) 4 4 Location embargo 5 5 Public embargo on data 6 6 Full embargo on data 7 " (http://vegbank.org/vegbank/views/dba_fielddescription_detail.jsp?view=detail&wparam=1581&entity=dba_fielddescription) */ SELECT util._map('0=>public, 1=>public, 2=>public, 3=>public, 4=>public, 5=>"data by request", 6=>private', $1::text) $BODY$ LANGUAGE sql IMMUTABLE COST 100; Time: 11.921 ms SELECT mk_derived_col((:table_str, 'accessRights'), $$map_access_rights("accessRights_verbatim")$$); -- runtime: 1 s ("Time: 1167.547 ms") @starscream psql:/dev/fd/40:46: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(map_access_rights("accessRights_verbatim")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.28 rows=1 width=4) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:46: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN "accessRights" text; ALTER TABLE plot ALTER COLUMN "accessRights" TYPE text USING map_access_rights("accessRights_verbatim"); mk_derived_col ---------------- (1 row) Time: 5.208 ms CREATE OR REPLACE FUNCTION map_fuzzing(value integer) RETURNS double precision AS $BODY$ /* "closed List of Values: Value Description Sorting 0 Public 1 1 1 km radius (nearest 0.01 degree) 2 2 10 km radius (nearest 0.1 degree) 3 3 100 km radius (nearest degree) 4 4 Location embargo 5 5 Public embargo on data 6 6 Full embargo on data 7 " (http://vegbank.org/vegbank/views/dba_fielddescription_detail.jsp?view=detail&wparam=1581&entity=dba_fielddescription) */ SELECT util._km_to_m(util._map('0=>NULL, 1=>1, 2=>10, 3=>100, 4=>NULL, 5=>NULL, 6=>NULL', $1::text)::double precision) $BODY$ LANGUAGE sql IMMUTABLE COST 100; Time: 36.424 ms SELECT mk_derived_col((:table_str, 'coordinateUncertaintyInMeters__from_fuzzing'), $$map_fuzzing("accessRights_verbatim")$$); -- runtime: 1 s ("Time: 1064.189 ms") @starscream psql:/dev/fd/40:68: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(map_fuzzing("accessRights_verbatim")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.29 rows=1 width=4) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:68: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN "coordinateUncertaintyInMeters__from_fuzzing" double precision; ALTER TABLE plot ALTER COLUMN "coordinateUncertaintyInMeters__from_fuzzing" TYPE double precision USING map_fuzzing("accessRights_verbatim"); mk_derived_col ---------------- (1 row) Time: 4.091 ms SELECT mk_derived_col((:table_str, 'coordinateUncertaintyInMeters'), $$GREATEST("coordinateUncertaintyInMeters__from_fuzzing", "coordinateUncertaintyInMeters__from_coords")$$) ; -- runtime: 0.5 s ("Time: 586.453 ms") @starscream psql:/dev/fd/40:73: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(GREATEST("coordinateUncertaintyInMeters__from_fuzzing", "coordinateUncertaintyInMeters__from_coords")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.02 rows=1 width=16) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:73: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN "coordinateUncertaintyInMeters" double precision; ALTER TABLE plot ALTER COLUMN "coordinateUncertaintyInMeters" TYPE double precision USING GREATEST("coordinateUncertaintyInMeters__from_fuzzing", "coordinateUncertaintyInMeters__from_coords"); mk_derived_col ---------------- (1 row) Time: 3.771 ms SELECT mk_derived_col((:table_str, 'locality'), $$_join("directions_to_place", "site_name")$$) ; -- runtime: 4 s ("Time: 3651.355 ms") @starscream psql:/dev/fd/40:77: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join("directions_to_place", "site_name")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=450) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:77: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN locality text; ALTER TABLE plot ALTER COLUMN locality TYPE text USING _join("directions_to_place", "site_name"); mk_derived_col ---------------- (1 row) Time: 3.846 ms real 0m0.562s user 0m0.028s sys 0m0.028s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.195 ms \cd schemas/VegCore \set schema "VegBank" \set table "plot" \set table_str '''"plot"''' SET search_path TO "VegBank"; Time: 0.093 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 28.130 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(plot,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.824 ms real 0m0.094s user 0m0.027s sys 0m0.027s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.200 ms \cd inputs/VegBank/plot \set schema "public" \set table "plot" \set table_str '''"plot"''' SET search_path TO "public"; Time: 0.086 ms CREATE TEMP TABLE "public".t () Time: 0.295 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.080s user 0m0.034s sys 0m0.037s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "plot/VegBIEN.csv" ln: plot/VegBIEN.csv: File exists make: [plot/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/plot/test.xml... >plot/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=plot out_database=vegbien source=VegBank.new ../../map plot/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:00.809591 sec/2 row(s) = 405. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change plot/test.xml.ref plot/test.xml real 0m2.132s user 0m1.198s sys 0m0.396s real 0m10.696s user 0m5.605s sys 0m2.846s | | | | | + inputs/VegBank/plot_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/plot_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.191 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.203 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.604s user 0m0.376s sys 0m0.194s real 0m0.075s user 0m0.021s sys 0m0.022s : : : : : : : : : table plot_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place plot_/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place plot_/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place plot_/map.csv ../../bin/fix_line_endings touch plot_/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile plot_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile plot_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 plot_/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >plot_/unmapped_terms.csv ../../bin/autoremove plot_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile plot_/new_terms.csv /usr/bin/make --makefile=../input.Makefile plot_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 plot_/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 plot_/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >plot_/new_terms.csv; exit 0 ../../bin/autoremove plot_/new_terms.csv real 0m2.976s user 0m2.253s sys 0m0.896s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.202 ms \cd inputs/VegBank/plot_ \set schema "VegBank" \set table "plot_" \set table_str '''"plot_"''' SET search_path TO "VegBank"; Time: 0.148 ms SELECT util.reset_map_table('"~plot_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~plot_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~plot_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~plot_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 160.879 ms ALTER TABLE "~plot_.map" DISABLE TRIGGER map_filter_insert; Time: 0.436 ms \copy "~plot_.map" FROM 'map.csv' CSV HEADER; Time: 1.645 ms SELECT util.set_col_names_with_metadata('"plot_"', '"~plot_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 15.802 ms real 0m0.242s user 0m0.028s sys 0m0.028s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.198 ms \cd schemas/VegCore \set schema "VegBank" \set table "plot_" \set table_str '''"plot_"''' SET search_path TO "VegBank"; Time: 0.087 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.131 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(plot_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.621 ms real 0m0.082s user 0m0.027s sys 0m0.027s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.191 ms \cd inputs/VegBank/plot_ \set schema "public" \set table "plot_" \set table_str '''"plot_"''' SET search_path TO "public"; Time: 0.080 ms CREATE TEMP TABLE "public".t () Time: 0.286 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.079s user 0m0.034s sys 0m0.037s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "plot_/VegBIEN.csv" ln: plot_/VegBIEN.csv: File exists make: [plot_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/plot_/test.xml... >plot_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=plot_ out_database=vegbien source=VegBank.new ../../map plot_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:00.649181 sec/2 row(s) = 325. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change plot_/test.xml.ref plot_/test.xml real 0m1.971s user 0m1.200s sys 0m0.395s real 0m8.700s user 0m4.775s sys 0m2.513s | | | | | + inputs/VegBank/observation_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/observation_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.191 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.246 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.639s user 0m0.397s sys 0m0.206s real 0m0.128s user 0m0.022s sys 0m0.022s : : : : : : : : : table observation_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place observation_/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place observation_/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place observation_/map.csv ../../bin/fix_line_endings touch observation_/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile observation_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile observation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 observation_/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >observation_/unmapped_terms.csv ../../bin/autoremove observation_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile observation_/new_terms.csv /usr/bin/make --makefile=../input.Makefile observation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 observation_/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 observation_/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >observation_/new_terms.csv; exit 0 ../../bin/autoremove observation_/new_terms.csv real 0m2.970s user 0m2.256s sys 0m0.890s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.205 ms \cd inputs/VegBank/observation_ \set schema "VegBank" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "VegBank"; Time: 0.096 ms SELECT util.reset_map_table('"~observation_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~observation_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~observation_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~observation_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 165.542 ms ALTER TABLE "~observation_.map" DISABLE TRIGGER map_filter_insert; Time: 0.311 ms \copy "~observation_.map" FROM 'map.csv' CSV HEADER; Time: 3.498 ms SELECT util.set_col_names_with_metadata('"observation_"', '"~observation_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 4.178 ms real 0m0.237s user 0m0.028s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place observation_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.889s user 0m0.640s sys 0m0.214s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.202 ms \cd inputs/VegBank/observation_ \set schema "VegBank" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "VegBank"; Time: 0.105 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 34.985 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("eventID")$$); -- runtime: ~0 ("Time: 1.838 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "observation_" ADD PRIMARY KEY ("eventID") create_if_not_exists ---------------------- (1 row) Time: 19.796 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("locationID")$$); -- runtime: ~0 ("158.792 ms") @starscream psql:/dev/fd/40:13: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "observation___parent" ON "observation_" ("locationID") create_if_not_exists ---------------------- (1 row) Time: 11.857 ms real 0m0.131s user 0m0.028s sys 0m0.029s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.210 ms \cd schemas/VegCore \set schema "VegBank" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "VegBank"; Time: 0.097 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.163 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(observation_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.995 ms real 0m0.084s user 0m0.027s sys 0m0.027s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.207 ms \cd inputs/VegBank/observation_ \set schema "public" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "public"; Time: 0.093 ms CREATE TEMP TABLE "public".t () Time: 0.312 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.080s user 0m0.034s sys 0m0.037s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "observation_/VegBIEN.csv" ln: observation_/VegBIEN.csv: File exists make: [observation_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/observation_/test.xml... >observation_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=observation_ out_database=vegbien source=VegBank.new ../../map observation_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:00.741053 sec/2 row(s) = 371. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change observation_/test.xml.ref observation_/test.xml real 0m2.069s user 0m1.142s sys 0m0.398s real 0m10.208s user 0m5.486s sys 0m2.859s | | | | | + inputs/VegBank/stratumtype/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stratumtype/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.194 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.266 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.604s user 0m0.376s sys 0m0.195s real 0m0.162s user 0m0.022s sys 0m0.022s : : : : : : : : : table stratumtype already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratumtype/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratumtype/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place stratumtype/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place stratumtype/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place stratumtype/map.csv ../../bin/fix_line_endings touch stratumtype/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile stratumtype/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile stratumtype/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 stratumtype/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >stratumtype/unmapped_terms.csv ../../bin/autoremove stratumtype/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile stratumtype/new_terms.csv /usr/bin/make --makefile=../input.Makefile stratumtype/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 stratumtype/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 stratumtype/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >stratumtype/new_terms.csv; exit 0 ../../bin/autoremove stratumtype/new_terms.csv real 0m2.973s user 0m2.246s sys 0m0.891s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.290 ms \cd inputs/VegBank/stratumtype \set schema "VegBank" \set table "stratumtype" \set table_str '''"stratumtype"''' SET search_path TO "VegBank"; Time: 0.134 ms SELECT util.reset_map_table('"~stratumtype.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stratumtype.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stratumtype.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stratumtype.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 141.180 ms ALTER TABLE "~stratumtype.map" DISABLE TRIGGER map_filter_insert; Time: 0.430 ms \copy "~stratumtype.map" FROM 'map.csv' CSV HEADER; Time: 0.622 ms SELECT util.set_col_names_with_metadata('"stratumtype"', '"~stratumtype.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.401 ms real 0m0.209s user 0m0.029s sys 0m0.028s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.201 ms \cd schemas/VegCore \set schema "VegBank" \set table "stratumtype" \set table_str '''"stratumtype"''' SET search_path TO "VegBank"; Time: 0.102 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.021 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stratumtype,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.659 ms real 0m0.084s user 0m0.027s sys 0m0.026s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.201 ms \cd inputs/VegBank/stratumtype \set schema "public" \set table "stratumtype" \set table_str '''"stratumtype"''' SET search_path TO "public"; Time: 0.105 ms CREATE TEMP TABLE "public".t () Time: 0.341 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.085s user 0m0.036s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratumtype/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratumtype/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stratumtype/VegBIEN.csv" ln: stratumtype/VegBIEN.csv: File exists make: [stratumtype/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stratumtype/test.xml... >stratumtype/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stratumtype out_database=vegbien source=VegBank.new ../../map stratumtype/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.496883 sec/2 row(s) = 248. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stratumtype/test.xml.ref stratumtype/test.xml real 0m1.816s user 0m1.012s sys 0m0.387s real 0m8.528s user 0m4.591s sys 0m2.516s | | | | | + inputs/VegBank/stratum/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stratum/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.190 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.299 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.604s user 0m0.376s sys 0m0.194s real 0m0.123s user 0m0.021s sys 0m0.022s : : : : : : : : : table stratum already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratum/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratum/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place stratum/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place stratum/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place stratum/map.csv ../../bin/fix_line_endings touch stratum/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile stratum/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile stratum/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 stratum/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >stratum/unmapped_terms.csv ../../bin/autoremove stratum/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile stratum/new_terms.csv /usr/bin/make --makefile=../input.Makefile stratum/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 stratum/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 stratum/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >stratum/new_terms.csv; exit 0 ../../bin/autoremove stratum/new_terms.csv real 0m2.965s user 0m2.253s sys 0m0.888s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.200 ms \cd inputs/VegBank/stratum \set schema "VegBank" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "VegBank"; Time: 0.110 ms SELECT util.reset_map_table('"~stratum.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stratum.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stratum.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stratum.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 95.170 ms ALTER TABLE "~stratum.map" DISABLE TRIGGER map_filter_insert; Time: 0.632 ms \copy "~stratum.map" FROM 'map.csv' CSV HEADER; Time: 0.733 ms SELECT util.set_col_names_with_metadata('"stratum"', '"~stratum.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.850 ms real 0m0.165s user 0m0.029s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratum/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place stratum/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.888s user 0m0.638s sys 0m0.214s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.202 ms \cd inputs/VegBank/stratum \set schema "VegBank" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "VegBank"; Time: 0.105 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.202 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("*stratum_id")$$); -- runtime: ~0 ("Time: 2.594 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "stratum" ADD PRIMARY KEY ("*stratum_id") create_if_not_exists ---------------------- (1 row) Time: 19.134 ms real 0m0.102s user 0m0.028s sys 0m0.030s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.243 ms \cd schemas/VegCore \set schema "VegBank" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "VegBank"; Time: 0.145 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.210 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stratum,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.617 ms real 0m0.083s user 0m0.026s sys 0m0.027s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.202 ms \cd inputs/VegBank/stratum \set schema "public" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "public"; Time: 0.086 ms CREATE TEMP TABLE "public".t () Time: 0.290 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.080s user 0m0.035s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratum/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratum/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stratum/VegBIEN.csv" ln: stratum/VegBIEN.csv: File exists make: [stratum/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stratum/test.xml... >stratum/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stratum out_database=vegbien source=VegBank.new ../../map stratum/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/map:296: UserWarning: Map warning: No mappings or no column name matches. Are you importing the correct input table? warnings.warn(UserWarning('Map warning: No mappings or no ' Processed 2 input rows Took 0:00:00.335846 sec/2 row(s) = 168. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stratum/test.xml.ref stratum/test.xml real 0m1.672s user 0m0.960s sys 0m0.391s real 0m9.690s user 0m5.284s sys 0m2.851s | | | | | + inputs/VegBank/plantconcept_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/plantconcept_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.197 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.216 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.607s user 0m0.377s sys 0m0.196s real 0m0.078s user 0m0.021s sys 0m0.022s : : : : : : : : : table plantconcept_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plantconcept_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plantconcept_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place plantconcept_/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place plantconcept_/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place plantconcept_/map.csv ../../bin/fix_line_endings touch plantconcept_/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile plantconcept_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile plantconcept_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 plantconcept_/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >plantconcept_/unmapped_terms.csv ../../bin/autoremove plantconcept_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile plantconcept_/new_terms.csv /usr/bin/make --makefile=../input.Makefile plantconcept_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 plantconcept_/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 plantconcept_/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >plantconcept_/new_terms.csv; exit 0 ../../bin/autoremove plantconcept_/new_terms.csv real 0m2.967s user 0m2.251s sys 0m0.893s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.205 ms \cd inputs/VegBank/plantconcept_ \set schema "VegBank" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "VegBank"; Time: 0.096 ms SELECT util.reset_map_table('"~plantconcept_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~plantconcept_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~plantconcept_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~plantconcept_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 80.929 ms ALTER TABLE "~plantconcept_.map" DISABLE TRIGGER map_filter_insert; Time: 0.309 ms \copy "~plantconcept_.map" FROM 'map.csv' CSV HEADER; Time: 0.855 ms SELECT util.set_col_names_with_metadata('"plantconcept_"', '"~plantconcept_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.598 ms real 0m0.151s user 0m0.029s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plantconcept_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place plantconcept_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.889s user 0m0.636s sys 0m0.215s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.217 ms \cd inputs/VegBank/plantconcept_ \set schema "VegBank" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "VegBank"; Time: 0.140 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 20.083 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("plantconcept_id")$$); -- runtime: 1 s ("Time: 675.125 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "plantconcept_" ADD PRIMARY KEY ("plantconcept_id") create_if_not_exists ---------------------- (1 row) Time: 21.223 ms real 0m0.109s user 0m0.029s sys 0m0.030s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.211 ms \cd schemas/VegCore \set schema "VegBank" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "VegBank"; Time: 0.090 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 17.837 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(plantconcept_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.696 ms real 0m0.084s user 0m0.027s sys 0m0.026s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.192 ms \cd inputs/VegBank/plantconcept_ \set schema "public" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "public"; Time: 0.080 ms CREATE TEMP TABLE "public".t () Time: 0.295 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.080s user 0m0.035s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plantconcept_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plantconcept_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "plantconcept_/VegBIEN.csv" ln: plantconcept_/VegBIEN.csv: File exists make: [plantconcept_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/plantconcept_/test.xml... >plantconcept_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=plantconcept_ out_database=vegbien source=VegBank.new ../../map plantconcept_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:01.033992 sec/2 row(s) = 517. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change plantconcept_/test.xml.ref plantconcept_/test.xml real 0m2.361s user 0m1.328s sys 0m0.404s real 0m10.369s user 0m5.650s sys 0m2.865s | | | | | + inputs/VegBank/taxonobservation_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/taxonobservation_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.196 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.251 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.606s user 0m0.377s sys 0m0.196s real 0m0.110s user 0m0.021s sys 0m0.021s : : : : : : : : : table taxonobservation_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxonobservation_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile taxonobservation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place taxonobservation_/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place taxonobservation_/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place taxonobservation_/map.csv ../../bin/fix_line_endings touch taxonobservation_/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile taxonobservation_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile taxonobservation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 taxonobservation_/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >taxonobservation_/unmapped_terms.csv ../../bin/autoremove taxonobservation_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile taxonobservation_/new_terms.csv /usr/bin/make --makefile=../input.Makefile taxonobservation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 taxonobservation_/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 taxonobservation_/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >taxonobservation_/new_terms.csv; exit 0 ../../bin/autoremove taxonobservation_/new_terms.csv real 0m2.965s user 0m2.251s sys 0m0.889s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.198 ms \cd inputs/VegBank/taxonobservation_ \set schema "VegBank" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "VegBank"; Time: 0.108 ms SELECT util.reset_map_table('"~taxonobservation_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~taxonobservation_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~taxonobservation_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~taxonobservation_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 156.641 ms ALTER TABLE "~taxonobservation_.map" DISABLE TRIGGER map_filter_insert; Time: 0.453 ms \copy "~taxonobservation_.map" FROM 'map.csv' CSV HEADER; Time: 2.545 ms SELECT util.set_col_names_with_metadata('"taxonobservation_"', '"~taxonobservation_.map"'); psql:/dev/fd/40:12: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; ALTER TABLE taxonobservation_ ADD COLUMN "observationGranularity" text NOT NULL DEFAULT 'plot'; COMMENT ON COLUMN taxonobservation_."observationGranularity" IS ' constant '; set_col_names_with_metadata ----------------------------- (1 row) Time: 19.517 ms real 0m0.244s user 0m0.029s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxonobservation_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place taxonobservation_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.893s user 0m0.643s sys 0m0.215s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.203 ms \cd inputs/VegBank/taxonobservation_ \set schema "VegBank" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "VegBank"; Time: 0.087 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 17.990 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("eventID")$$); -- runtime: 2 s ("2416.676 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "taxonobservation___parent" ON "taxonobservation_" ("eventID") create_if_not_exists ---------------------- (1 row) Time: 3.505 ms SELECT mk_derived_col((:table_str, 'identifiedBy'), $$_join_words("identifiedBy__first", "identifiedBy__middle", "identifiedBy__last")$$) ; -- runtime: 30 s ("29290.701 ms") @starscream psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join_words("identifiedBy__first", "identifiedBy__middle", "identifiedBy__last")) FROM (SELECT (NULL::taxonobservation_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=96) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE taxonobservation_ ADD COLUMN "identifiedBy" text; ALTER TABLE taxonobservation_ ALTER COLUMN "identifiedBy" TYPE text USING _join_words("identifiedBy__first", "identifiedBy__middle", "identifiedBy__last"); mk_derived_col ---------------- (1 row) Time: 5.359 ms real 0m0.092s user 0m0.028s sys 0m0.029s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.202 ms \cd schemas/VegCore \set schema "VegBank" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "VegBank"; Time: 0.089 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.295 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(taxonobservation_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.709 ms real 0m0.083s user 0m0.027s sys 0m0.026s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.187 ms \cd inputs/VegBank/taxonobservation_ \set schema "public" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "public"; Time: 0.091 ms CREATE TEMP TABLE "public".t () Time: 0.279 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.079s user 0m0.034s sys 0m0.037s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxonobservation_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile taxonobservation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "taxonobservation_/VegBIEN.csv" ln: taxonobservation_/VegBIEN.csv: File exists make: [taxonobservation_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/taxonobservation_/test.xml... >taxonobservation_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=taxonobservation_ out_database=vegbien source=VegBank.new ../../map taxonobservation_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:01.778163 sec/2 row(s) = 889. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change taxonobservation_/test.xml.ref taxonobservation_/test.xml real 0m3.107s user 0m1.510s sys 0m0.411s real 0m11.181s user 0m5.833s sys 0m2.860s | | | | | + inputs/VegBank/stemcount_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stemcount_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.192 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.211 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.613s user 0m0.377s sys 0m0.198s real 0m0.095s user 0m0.022s sys 0m0.022s : : : : : : : : : table stemcount_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemcount_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemcount_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place stemcount_/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place stemcount_/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place stemcount_/map.csv ../../bin/fix_line_endings touch stemcount_/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile stemcount_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile stemcount_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 stemcount_/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >stemcount_/unmapped_terms.csv ../../bin/autoremove stemcount_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile stemcount_/new_terms.csv /usr/bin/make --makefile=../input.Makefile stemcount_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 stemcount_/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 stemcount_/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >stemcount_/new_terms.csv; exit 0 ../../bin/autoremove stemcount_/new_terms.csv real 0m2.982s user 0m2.248s sys 0m0.895s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.193 ms \cd inputs/VegBank/stemcount_ \set schema "VegBank" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "VegBank"; Time: 0.111 ms SELECT util.reset_map_table('"~stemcount_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stemcount_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stemcount_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stemcount_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 90.340 ms ALTER TABLE "~stemcount_.map" DISABLE TRIGGER map_filter_insert; Time: 0.392 ms \copy "~stemcount_.map" FROM 'map.csv' CSV HEADER; Time: 0.794 ms SELECT util.set_col_names_with_metadata('"stemcount_"', '"~stemcount_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.598 ms real 0m0.160s user 0m0.028s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemcount_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place stemcount_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.904s user 0m0.652s sys 0m0.215s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.202 ms \cd inputs/VegBank/stemcount_ \set schema "VegBank" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "VegBank"; Time: 0.099 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.101 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("taxonOccurrenceID__overall_plot")$$); -- runtime: 2 s ("Time: 1766.315 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "stemcount___parent" ON "stemcount_" ("taxonOccurrenceID__overall_plot") create_if_not_exists ---------------------- (1 row) Time: 30.900 ms SELECT mk_derived_col((:table_str, 'aggregateOrganismObservationID'), $$_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)$$) ; -- runtime: 15 s ("Time: 15805.876 ms") @starscream psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)) FROM (SELECT (NULL::stemcount_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.28 rows=1 width=8) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE stemcount_ ADD COLUMN "aggregateOrganismObservationID" text; ALTER TABLE stemcount_ ALTER COLUMN "aggregateOrganismObservationID" TYPE text USING _join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text); mk_derived_col ---------------- (1 row) Time: 5.148 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("aggregateOrganismObservationID")$$); -- runtime: 20 s ("Time: 20194.932 ms") @starscream psql:/dev/fd/40:19: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "stemcount_" ADD PRIMARY KEY ("aggregateOrganismObservationID") create_if_not_exists ---------------------- (1 row) Time: 2.964 ms real 0m0.122s user 0m0.028s sys 0m0.030s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.202 ms \cd schemas/VegCore \set schema "VegBank" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "VegBank"; Time: 0.086 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.530 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stemcount_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.592 ms real 0m0.081s user 0m0.026s sys 0m0.026s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.246 ms \cd inputs/VegBank/stemcount_ \set schema "public" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "public"; Time: 0.082 ms CREATE TEMP TABLE "public".t () Time: 0.291 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.080s user 0m0.035s sys 0m0.037s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemcount_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemcount_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stemcount_/VegBIEN.csv" ln: stemcount_/VegBIEN.csv: File exists make: [stemcount_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stemcount_/test.xml... >stemcount_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stemcount_ out_database=vegbien source=VegBank.new ../../map stemcount_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.532673 sec/2 row(s) = 266. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stemcount_/test.xml.ref stemcount_/test.xml real 0m1.848s user 0m1.062s sys 0m0.390s real 0m9.910s user 0m5.395s sys 0m2.857s | | | | | + inputs/VegBank/stemlocation_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stemlocation_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.190 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.219 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.608s user 0m0.376s sys 0m0.196s real 0m0.127s user 0m0.022s sys 0m0.022s : : : : : : : : : table stemlocation_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemlocation_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemlocation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place stemlocation_/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place stemlocation_/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place stemlocation_/map.csv ../../bin/fix_line_endings touch stemlocation_/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile stemlocation_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile stemlocation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 stemlocation_/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >stemlocation_/unmapped_terms.csv ../../bin/autoremove stemlocation_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile stemlocation_/new_terms.csv /usr/bin/make --makefile=../input.Makefile stemlocation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 stemlocation_/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 stemlocation_/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >stemlocation_/new_terms.csv; exit 0 ../../bin/autoremove stemlocation_/new_terms.csv real 0m2.984s user 0m2.244s sys 0m0.889s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.204 ms \cd inputs/VegBank/stemlocation_ \set schema "VegBank" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "VegBank"; Time: 0.111 ms SELECT util.reset_map_table('"~stemlocation_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stemlocation_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stemlocation_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stemlocation_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 66.845 ms ALTER TABLE "~stemlocation_.map" DISABLE TRIGGER map_filter_insert; Time: 0.357 ms \copy "~stemlocation_.map" FROM 'map.csv' CSV HEADER; Time: 0.728 ms SELECT util.set_col_names_with_metadata('"stemlocation_"', '"~stemlocation_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.518 ms real 0m0.135s user 0m0.028s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemlocation_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place stemlocation_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.904s user 0m0.655s sys 0m0.215s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.205 ms \cd inputs/VegBank/stemlocation_ \set schema "VegBank" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "VegBank"; Time: 0.110 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 17.944 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("stemID")$$); -- runtime: 0.1 s ("Time: 74.164 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "stemlocation_" ADD PRIMARY KEY ("stemID") create_if_not_exists ---------------------- (1 row) Time: 12.675 ms SELECT mk_derived_col((:table_str, 'individualID'), $$"stemID"$$) ; -- runtime: ~0 ("Time: 1.562 ms") @starscream psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof("stemID") FROM (SELECT (NULL::stemlocation_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.02 rows=1 width=4) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE stemlocation_ ADD COLUMN "individualID" integer; ALTER TABLE stemlocation_ ALTER COLUMN "individualID" TYPE integer USING "stemID"; mk_derived_col ---------------- (1 row) Time: 4.938 ms SELECT mk_derived_col((:table_str, 'aggregateOrganismObservationID'), $$_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)$$) ; -- runtime: 0.5 s ("Time: 441.059 ms") @starscream psql:/dev/fd/40:18: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)) FROM (SELECT (NULL::stemlocation_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.28 rows=1 width=8) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:18: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE stemlocation_ ADD COLUMN "aggregateOrganismObservationID" text; ALTER TABLE stemlocation_ ALTER COLUMN "aggregateOrganismObservationID" TYPE text USING _join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text); mk_derived_col ---------------- (1 row) Time: 4.237 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("aggregateOrganismObservationID")$$); -- runtime: 0.1 s ("Time: 131.907 ms") @starscream psql:/dev/fd/40:21: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "stemlocation___parent" ON "stemlocation_" ("aggregateOrganismObservationID") create_if_not_exists ---------------------- (1 row) Time: 2.355 ms real 0m0.107s user 0m0.028s sys 0m0.029s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.355 ms \cd schemas/VegCore \set schema "VegBank" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "VegBank"; Time: 0.088 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 17.969 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stemlocation_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.610 ms real 0m0.081s user 0m0.026s sys 0m0.026s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.187 ms \cd inputs/VegBank/stemlocation_ \set schema "public" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "public"; Time: 0.082 ms CREATE TEMP TABLE "public".t () Time: 0.292 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.080s user 0m0.035s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemlocation_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemlocation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stemlocation_/VegBIEN.csv" ln: stemlocation_/VegBIEN.csv: File exists make: [stemlocation_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stemlocation_/test.xml... >stemlocation_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stemlocation_ out_database=vegbien source=VegBank.new ../../map stemlocation_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.537182 sec/2 row(s) = 269. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stemlocation_/test.xml.ref stemlocation_/test.xml real 0m1.853s user 0m1.060s sys 0m0.389s real 0m9.881s user 0m5.395s sys 0m2.850s | | | | | + inputs/VegBank/taxon_observation.**/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxon_observation.**/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place taxon_observation.**/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.890s user 0m0.639s sys 0m0.213s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.243 ms \cd inputs/VegBank/taxon_observation.** \set schema "VegBank" \set table "taxon_observation.**" \set table_str '''"taxon_observation.**"''' SET search_path TO "VegBank"; Time: 0.098 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 17.844 ms SELECT force_update_view(:table_str, $$ /* no row_num because left-join to stemcount_, stemlocation_ adds rows to each taxonobservation_ */ SELECT "identificationID" AS sort_col, * , NULLIF(concat_ws('; ', "taxonOccurrenceID__overall_plot"::text, "aggregateOrganismObservationID"), '') AS "taxonOccurrenceID" FROM taxonobservation_ NATURAL LEFT JOIN stemcount_ NATURAL LEFT JOIN stemlocation_ NATURAL LEFT JOIN stratum NATURAL LEFT JOIN stratumtype NATURAL LEFT JOIN observation_ NATURAL LEFT JOIN plot_ NATURAL LEFT JOIN project $$); force_update_view ------------------- (1 row) Time: 161.548 ms SELECT mk_subset_by_row_num_func(:table_str); mk_subset_by_row_num_func --------------------------- (1 row) Time: 10.074 ms real 0m0.250s user 0m0.027s sys 0m0.027s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.249 ms \cd schemas/VegCore \set schema "VegBank" \set table "taxon_observation.**" \set table_str '''"taxon_observation.**"''' SET search_path TO "VegBank"; Time: 0.093 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 17.780 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('("""taxon_observation.**""",cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 7.553 ms real 0m0.083s user 0m0.025s sys 0m0.026s | | | | | | | | + builtin cd -P inputs/VegBank/taxon_observation.** | | | | | | | | | + rm -f header.csv | | | | | | | | + mappings__table_run @lib/sh/util.sh:299 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.249 ms \cd inputs/VegBank/taxon_observation.** \set schema "public" \set table "taxon_observation.**" \set table_str '''"taxon_observation.**"''' SET search_path TO "public"; Time: 0.113 ms CREATE TEMP TABLE "public".t () Time: 0.307 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.085s user 0m0.036s sys 0m0.039s | | | | | | | | | + builtin cd -P inputs/VegBank/taxon_observation.** | | | | | | | | | | + rm -f map.csv | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxon_observation.**/test make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make --makefile=../input.Makefile "taxon_observation.**/header.csv" set -o pipefail; \ echo 'COPY (SELECT * FROM "VegBank"."taxon_observation.**" LIMIT 0) TO STDOUT CSV HEADER;'|\ env no_search_path=1 ../../bin/psql_script_vegbien >taxon_observation.**/header.csv (. ../../mappings/root.sh; env datasrc=VegBank ../../bin/src_map taxon_observation.**/map.csv) /usr/bin/make --makefile=../input.Makefile taxon_observation.**/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place taxon_observation.**/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place taxon_observation.**/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place taxon_observation.**/map.csv ../../bin/fix_line_endings touch taxon_observation.**/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile taxon_observation.**/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile taxon_observation.**/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 taxon_observation.**/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >taxon_observation.**/unmapped_terms.csv ../../bin/autoremove taxon_observation.**/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile taxon_observation.**/new_terms.csv /usr/bin/make --makefile=../input.Makefile taxon_observation.**/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 taxon_observation.**/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 taxon_observation.**/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >taxon_observation.**/new_terms.csv; exit 0 ../../bin/autoremove taxon_observation.**/new_terms.csv ln -s "../../../mappings/VegCore-VegBIEN.csv" "taxon_observation.**/VegBIEN.csv" ln: taxon_observation.**/VegBIEN.csv: File exists make: [taxon_observation.**/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/taxon_observation.**/test.xml... >taxon_observation.**/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=taxon_observation.** out_database=vegbien source=VegBank.new ../../map taxon_observation.**/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:01:09.708992 sec/2 row(s) = 34.9 s/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change 'taxon_observation.**/test.xml.ref' 'taxon_observation.**/test.xml' real 1m13.716s user 0m3.957s sys 0m1.134s real 1m17.229s user 0m5.477s sys 0m2.300s | | | | | + inputs/VegBank/^taxon_observation.**.sample/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/^taxon_observation.**.sample/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.191 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.204 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.606s user 0m0.376s sys 0m0.195s real 0m0.074s user 0m0.021s sys 0m0.022s : : : : : : : : : table ^taxon_observation.**.sample already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank ^taxon_observation.**.sample/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile ^taxon_observation.**.sample/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place ^taxon_observation.**.sample/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place ^taxon_observation.**.sample/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place ^taxon_observation.**.sample/map.csv ../../bin/fix_line_endings touch ^taxon_observation.**.sample/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile ^taxon_observation.**.sample/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile ^taxon_observation.**.sample/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 ^taxon_observation.**.sample/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >^taxon_observation.**.sample/unmapped_terms.csv ../../bin/autoremove ^taxon_observation.**.sample/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile ^taxon_observation.**.sample/new_terms.csv /usr/bin/make --makefile=../input.Makefile ^taxon_observation.**.sample/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 ^taxon_observation.**.sample/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 ^taxon_observation.**.sample/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >^taxon_observation.**.sample/new_terms.csv; exit 0 ../../bin/autoremove ^taxon_observation.**.sample/new_terms.csv real 0m2.985s user 0m2.264s sys 0m0.902s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.194 ms \cd inputs/VegBank/^taxon_observation.**.sample \set schema "VegBank" \set table "^taxon_observation.**.sample" \set table_str '''"^taxon_observation.**.sample"''' SET search_path TO "VegBank"; Time: 0.107 ms SELECT util.reset_map_table('"~^taxon_observation.**.sample.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~^taxon_observation.**.sample.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~^taxon_observation.**.sample.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~^taxon_observation.**.sample.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 133.936 ms ALTER TABLE "~^taxon_observation.**.sample.map" DISABLE TRIGGER map_filter_insert; Time: 0.424 ms \copy "~^taxon_observation.**.sample.map" FROM 'map.csv' CSV HEADER; Time: 2.112 ms SELECT util.set_col_names_with_metadata('"^taxon_observation.**.sample"', '"~^taxon_observation.**.sample.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.801 ms real 0m0.204s user 0m0.028s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.200 ms \cd schemas/VegCore \set schema "VegBank" \set table "^taxon_observation.**.sample" \set table_str '''"^taxon_observation.**.sample"''' SET search_path TO "VegBank"; Time: 0.096 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.261 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('("""^taxon_observation.**.sample""",cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 21.017 ms real 0m0.101s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.198 ms \cd inputs/VegBank/^taxon_observation.**.sample \set schema "public" \set table "^taxon_observation.**.sample" \set table_str '''"^taxon_observation.**.sample"''' SET search_path TO "public"; Time: 0.099 ms CREATE TEMP TABLE "public".t () Time: 0.293 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.083s user 0m0.036s sys 0m0.039s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank ^taxon_observation.**.sample/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile ^taxon_observation.**.sample/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "^taxon_observation.**.sample/VegBIEN.csv" ln: ^taxon_observation.**.sample/VegBIEN.csv: File exists make: [^taxon_observation.**.sample/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/^taxon_observation.**.sample/test.xml... >^taxon_observation.**.sample/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=^taxon_observation.**.sample out_database=vegbien source=VegBank.new ../../map ^taxon_observation.**.sample/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:01.699880 sec/2 row(s) = 850. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change '^taxon_observation.**.sample/test.xml.ref' '^taxon_observation.**.sample/test.xml' real 0m3.058s user 0m1.991s sys 0m0.426s real 0m9.799s user 0m5.591s sys 0m2.571s | | | | | + inputs/VegBank/projectcontributor_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/projectcontributor_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.222 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.223 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.617s user 0m0.382s sys 0m0.200s real 0m0.077s user 0m0.023s sys 0m0.024s : : : : : : : : : table projectcontributor_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank projectcontributor_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile projectcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place projectcontributor_/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place projectcontributor_/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place projectcontributor_/map.csv ../../bin/fix_line_endings touch projectcontributor_/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile projectcontributor_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile projectcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 projectcontributor_/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >projectcontributor_/unmapped_terms.csv ../../bin/autoremove projectcontributor_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile projectcontributor_/new_terms.csv /usr/bin/make --makefile=../input.Makefile projectcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 projectcontributor_/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 projectcontributor_/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >projectcontributor_/new_terms.csv; exit 0 ../../bin/autoremove projectcontributor_/new_terms.csv real 0m3.029s user 0m2.292s sys 0m0.914s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd inputs/VegBank/projectcontributor_ \set schema "VegBank" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "VegBank"; Time: 0.099 ms SELECT util.reset_map_table('"~projectcontributor_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~projectcontributor_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~projectcontributor_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~projectcontributor_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 141.027 ms ALTER TABLE "~projectcontributor_.map" DISABLE TRIGGER map_filter_insert; Time: 0.407 ms \copy "~projectcontributor_.map" FROM 'map.csv' CSV HEADER; Time: 0.860 ms SELECT util.set_col_names_with_metadata('"projectcontributor_"', '"~projectcontributor_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.380 ms real 0m0.210s user 0m0.029s sys 0m0.028s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank projectcontributor_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place projectcontributor_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.916s user 0m0.642s sys 0m0.221s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.201 ms \cd inputs/VegBank/projectcontributor_ \set schema "VegBank" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "VegBank"; Time: 0.101 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.000 ms SELECT mk_derived_col((:table_str, 'project_participant'), $$_join_words(project_participant_name_first::text, project_participant_name_middle::text, project_participant_name_last::text)$$) ; -- runtime: ~0 ("2.887 ms") @starscream psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join_words(project_participant_name_first::text, project_participant_name_middle::text, project_participant_name_last::text)) FROM (SELECT (NULL::projectcontributor_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=96) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE projectcontributor_ ADD COLUMN project_participant text; ALTER TABLE projectcontributor_ ALTER COLUMN project_participant TYPE text USING _join_words(project_participant_name_first::text, project_participant_name_middle::text, project_participant_name_last::text); mk_derived_col ---------------- (1 row) Time: 17.570 ms real 0m0.100s user 0m0.028s sys 0m0.029s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.238 ms \cd schemas/VegCore \set schema "VegBank" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "VegBank"; Time: 0.087 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 17.914 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(projectcontributor_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.597 ms real 0m0.081s user 0m0.026s sys 0m0.026s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.200 ms \cd inputs/VegBank/projectcontributor_ \set schema "public" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "public"; Time: 0.100 ms CREATE TEMP TABLE "public".t () Time: 0.292 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.080s user 0m0.035s sys 0m0.037s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank projectcontributor_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile projectcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "projectcontributor_/VegBIEN.csv" ln: projectcontributor_/VegBIEN.csv: File exists make: [projectcontributor_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/projectcontributor_/test.xml... >projectcontributor_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=projectcontributor_ out_database=vegbien source=VegBank.new ../../map projectcontributor_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.454602 sec/2 row(s) = 227. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change projectcontributor_/test.xml.ref projectcontributor_/test.xml real 0m1.772s user 0m1.016s sys 0m0.388s real 0m9.920s user 0m5.391s sys 0m2.884s | | | | | + inputs/VegBank/observationcontributor_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/observationcontributor_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.190 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.208 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.608s user 0m0.376s sys 0m0.197s real 0m0.065s user 0m0.022s sys 0m0.023s : : : : : : : : : table observationcontributor_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observationcontributor_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observationcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place observationcontributor_/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place observationcontributor_/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place observationcontributor_/map.csv ../../bin/fix_line_endings touch observationcontributor_/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile observationcontributor_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile observationcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 observationcontributor_/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >observationcontributor_/unmapped_terms.csv ../../bin/autoremove observationcontributor_/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile observationcontributor_/new_terms.csv /usr/bin/make --makefile=../input.Makefile observationcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 observationcontributor_/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 observationcontributor_/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >observationcontributor_/new_terms.csv; exit 0 ../../bin/autoremove observationcontributor_/new_terms.csv real 0m2.977s user 0m2.248s sys 0m0.898s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.196 ms \cd inputs/VegBank/observationcontributor_ \set schema "VegBank" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "VegBank"; Time: 0.097 ms SELECT util.reset_map_table('"~observationcontributor_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~observationcontributor_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~observationcontributor_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~observationcontributor_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 88.072 ms ALTER TABLE "~observationcontributor_.map" DISABLE TRIGGER map_filter_insert; Time: 0.329 ms \copy "~observationcontributor_.map" FROM 'map.csv' CSV HEADER; Time: 0.772 ms SELECT util.set_col_names_with_metadata('"observationcontributor_"', '"~observationcontributor_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.529 ms real 0m0.157s user 0m0.028s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observationcontributor_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place observationcontributor_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.891s user 0m0.637s sys 0m0.217s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd inputs/VegBank/observationcontributor_ \set schema "VegBank" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "VegBank"; Time: 0.113 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.095 ms SELECT mk_derived_col((:table_str, 'event__participant'), $$_join_words(event__participant__name__first::text, event__participant__name__middle::text, event__participant__name__last::text)$$) ; -- runtime: 1 s ("Time: 1265.039 ms") @starscream psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join_words(event__participant__name__first::text, event__participant__name__middle::text, event__participant__name__last::text)) FROM (SELECT (NULL::observationcontributor_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=96) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE observationcontributor_ ADD COLUMN event__participant text; ALTER TABLE observationcontributor_ ALTER COLUMN event__participant TYPE text USING _join_words(event__participant__name__first::text, event__participant__name__middle::text, event__participant__name__last::text); mk_derived_col ---------------- (1 row) Time: 24.110 ms real 0m0.107s user 0m0.028s sys 0m0.029s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.196 ms \cd schemas/VegCore \set schema "VegBank" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "VegBank"; Time: 0.099 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.087 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(observationcontributor_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.714 ms real 0m0.084s user 0m0.027s sys 0m0.027s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.187 ms \cd inputs/VegBank/observationcontributor_ \set schema "public" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "public"; Time: 0.101 ms CREATE TEMP TABLE "public".t () Time: 0.284 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.082s user 0m0.035s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observationcontributor_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observationcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "observationcontributor_/VegBIEN.csv" ln: observationcontributor_/VegBIEN.csv: File exists make: [observationcontributor_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/observationcontributor_/test.xml... >observationcontributor_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=observationcontributor_ out_database=vegbien source=VegBank.new ../../map observationcontributor_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.478303 sec/2 row(s) = 239. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change observationcontributor_/test.xml.ref observationcontributor_/test.xml real 0m1.829s user 0m1.051s sys 0m0.402s real 0m9.858s user 0m5.374s sys 0m2.871s | | | | | + inputs/VegBank/observation__community/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/observation__community/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.193 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.203 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.606s user 0m0.376s sys 0m0.196s real 0m0.072s user 0m0.022s sys 0m0.022s : : : : : : : : : table observation__community already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation__community/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation__community/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place observation__community/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place observation__community/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place observation__community/map.csv ../../bin/fix_line_endings touch observation__community/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile observation__community/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile observation__community/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 observation__community/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >observation__community/unmapped_terms.csv ../../bin/autoremove observation__community/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile observation__community/new_terms.csv /usr/bin/make --makefile=../input.Makefile observation__community/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 observation__community/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 observation__community/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >observation__community/new_terms.csv; exit 0 ../../bin/autoremove observation__community/new_terms.csv real 0m2.974s user 0m2.258s sys 0m0.893s | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.212 ms \cd inputs/VegBank/observation__community \set schema "VegBank" \set table "observation__community" \set table_str '''"observation__community"''' SET search_path TO "VegBank"; Time: 0.108 ms SELECT util.reset_map_table('"~observation__community.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~observation__community.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~observation__community.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~observation__community.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 89.326 ms ALTER TABLE "~observation__community.map" DISABLE TRIGGER map_filter_insert; Time: 0.326 ms \copy "~observation__community.map" FROM 'map.csv' CSV HEADER; Time: 1.189 ms SELECT util.set_col_names_with_metadata('"observation__community"', '"~observation__community.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.624 ms real 0m0.158s user 0m0.029s sys 0m0.028s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.205 ms \cd schemas/VegCore \set schema "VegBank" \set table "observation__community" \set table_str '''"observation__community"''' SET search_path TO "VegBank"; Time: 0.094 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.428 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(observation__community,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.765 ms real 0m0.083s user 0m0.026s sys 0m0.026s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:419 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:410 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:403 | | | | | | | | | | | | + declare -x PGHOST="localhost" | | | | | | | | | | | | + declare -x PGUSER="bien" | | | | | | | | | | | | + declare -x PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.195 ms \cd inputs/VegBank/observation__community \set schema "public" \set table "observation__community" \set table_str '''"observation__community"''' SET search_path TO "public"; Time: 0.086 ms CREATE TEMP TABLE "public".t () Time: 0.295 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.079s user 0m0.034s sys 0m0.036s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation__community/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation__community/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "observation__community/VegBIEN.csv" ln: observation__community/VegBIEN.csv: File exists make: [observation__community/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/observation__community/test.xml... >observation__community/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=observation__community out_database=vegbien source=VegBank.new ../../map observation__community/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.490957 sec/2 row(s) = 245. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change observation__community/test.xml.ref observation__community/test.xml real 0m1.812s user 0m1.036s sys 0m0.390s real 0m8.456s user 0m4.623s sys 0m2.505s | | | | + export_ @lib/runscripts/import.run:24 real 4m18.692s user 1m39.248s sys 0m58.715s $ Wed 12:25:55 aaronmk@frenzy:~/Dropbox/svn