inputs/VegBank/run + inputs/VegBank/run | + main @lib/runscripts/util.run:56 -> 152 | | + all @lib/runscripts/util.run:152 -> lib/runscripts/install.run:23 | | | + install @lib/runscripts/install.run:26 -> lib/runscripts/import.run:29 | | | | + import @lib/runscripts/import.run:32 -> lib/runscripts/datasrc_dir.run:29 | | | | | + pg_schema_exists @lib/runscripts/datasrc_dir.run:33 -> lib/sh/db.sh:412 | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.198 ms CREATE TEMP TABLE "VegBank".t () Time: 0.310 ms psql:/dev/fd/40:4: ERROR: cannot create temporary relation in non-temporary schema real 0m0.263s user 0m0.033s sys 0m0.039s | | | | | + inputs/VegBank/Source/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/Source/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.221 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.298 ms 0.06 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m1.194s user 0m0.380s sys 0m0.215s real 0m0.286s user 0m0.023s sys 0m0.028s : : : : : : : : : table Source already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank Source/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile Source/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.037s user 0m0.609s sys 0m0.322s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.229 ms \cd inputs/VegBank/Source \set schema "VegBank" \set table "Source" \set table_str '''"Source"''' SET search_path TO "VegBank"; Time: 0.110 ms SELECT util.reset_map_table('"~Source.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~Source.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~Source.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~Source.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 400.250 ms ALTER TABLE "~Source.map" DISABLE TRIGGER map_filter_insert; Time: 10.437 ms \copy "~Source.map" FROM 'map.csv' CSV HEADER; Time: 28.305 ms SELECT util.set_col_names_with_metadata('"Source"', '"~Source.map"'); psql:/dev/fd/40:12: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; ALTER TABLE "Source" ADD COLUMN "sourceType" text NOT NULL DEFAULT 'aggregator'; COMMENT ON COLUMN "Source"."sourceType" IS ' constant '; psql:/dev/fd/40:12: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; ALTER TABLE "Source" ADD COLUMN "observationGranularity" text NOT NULL DEFAULT 'plot'; COMMENT ON COLUMN "Source"."observationGranularity" IS ' constant '; set_col_names_with_metadata ----------------------------- (1 row) Time: 73.537 ms real 0m0.579s user 0m0.028s sys 0m0.031s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.218 ms \cd schemas/VegCore \set schema "VegBank" \set table "Source" \set table_str '''"Source"''' SET search_path TO "VegBank"; Time: 0.138 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 74.067 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('("""Source""",cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 39.100 ms real 0m0.177s user 0m0.027s sys 0m0.029s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.209 ms \cd inputs/VegBank/Source \set schema "public" \set table "Source" \set table_str '''"Source"''' SET search_path TO "public"; Time: 0.099 ms CREATE TEMP TABLE "public".t () Time: 0.314 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.084s user 0m0.034s sys 0m0.040s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank Source/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile Source/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "Source/VegBIEN.csv" ln: Source/VegBIEN.csv: File exists make: [Source/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/Source/test.xml... >Source/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=Source out_database=vegbien source=VegBank.new ../../map Source/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 1 input rows Took 0:00:01.208019 sec/1 row(s) = 1.21 s/row Encountered 0 error(s)/1 row(s) = 0.0% + diff --unified=2 --ignore-space-change Source/test.xml.ref Source/test.xml real 0m3.545s user 0m0.943s sys 0m0.441s real 0m9.509s user 0m2.844s sys 0m2.020s | | | | | + inputs/VegBank/project/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/project/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.230 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.227 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.662s user 0m0.380s sys 0m0.208s real 0m0.277s user 0m0.023s sys 0m0.025s : : : : : : : : : table project already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank project/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile project/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.034s user 0m0.606s sys 0m0.320s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.207 ms \cd inputs/VegBank/project \set schema "VegBank" \set table "project" \set table_str '''"project"''' SET search_path TO "VegBank"; Time: 0.130 ms SELECT util.reset_map_table('"~project.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~project.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~project.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~project.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 259.999 ms ALTER TABLE "~project.map" DISABLE TRIGGER map_filter_insert; Time: 0.580 ms \copy "~project.map" FROM 'map.csv' CSV HEADER; Time: 28.715 ms SELECT util.set_col_names_with_metadata('"project"', '"~project.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.849 ms real 0m0.361s user 0m0.029s sys 0m0.032s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.206 ms \cd schemas/VegCore \set schema "VegBank" \set table "project" \set table_str '''"project"''' SET search_path TO "VegBank"; Time: 0.110 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 29.693 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(project,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 24.058 ms real 0m0.121s user 0m0.027s sys 0m0.029s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.189 ms \cd inputs/VegBank/project \set schema "public" \set table "project" \set table_str '''"project"''' SET search_path TO "public"; Time: 0.164 ms CREATE TEMP TABLE "public".t () Time: 0.331 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.083s user 0m0.035s sys 0m0.040s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank project/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile project/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "project/VegBIEN.csv" ln: project/VegBIEN.csv: File exists make: [project/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/project/test.xml... >project/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=project out_database=vegbien source=VegBank.new ../../map project/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.592433 sec/2 row(s) = 296. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change project/test.xml.ref project/test.xml real 0m2.019s user 0m0.979s sys 0m0.412s real 0m7.232s user 0m2.883s sys 0m1.985s | | | | | + inputs/VegBank/plot/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/plot/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.297 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.219 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.647s user 0m0.378s sys 0m0.211s real 0m0.211s user 0m0.023s sys 0m0.024s : : : : : : : : : table plot already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.018s user 0m0.613s sys 0m0.324s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.201 ms \cd inputs/VegBank/plot \set schema "VegBank" \set table "plot" \set table_str '''"plot"''' SET search_path TO "VegBank"; Time: 0.114 ms SELECT util.reset_map_table('"~plot.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~plot.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~plot.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~plot.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 128.432 ms ALTER TABLE "~plot.map" DISABLE TRIGGER map_filter_insert; Time: 0.409 ms \copy "~plot.map" FROM 'map.csv' CSV HEADER; Time: 20.394 ms SELECT util.set_col_names_with_metadata('"plot"', '"~plot.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.736 ms real 0m0.220s user 0m0.028s sys 0m0.032s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place plot/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m1.079s user 0m0.710s sys 0m0.230s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.197 ms \cd inputs/VegBank/plot \set schema "VegBank" \set table "plot" \set table_str '''"plot"''' SET search_path TO "VegBank"; Time: 0.112 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 66.426 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("locationID")$$); -- runtime: 0.2 s ("Time: 238.474 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "plot" ADD PRIMARY KEY ("locationID") create_if_not_exists ---------------------- (1 row) Time: 64.808 ms -- remove institutions that we have direct data for -- CVS DELETE FROM :table WHERE "locationName" IN (SELECT "locationName" FROM "CVS".plot_) -- removes 2323 of 7079 CVS plots (75340 - 73017) ; -- runtime: 1 min ("48999.044 ms") @starscream Time: 611.562 ms -- map_*() derived cols -- runtime: 2 s @starscream CREATE OR REPLACE FUNCTION map_access_rights(value integer) RETURNS text AS $BODY$ /* "closed List of Values: Value Description Sorting 0 Public 1 1 1 km radius (nearest 0.01 degree) 2 2 10 km radius (nearest 0.1 degree) 3 3 100 km radius (nearest degree) 4 4 Location embargo 5 5 Public embargo on data 6 6 Full embargo on data 7 " (http://vegbank.org/vegbank/views/dba_fielddescription_detail.jsp?view=detail&wparam=1581&entity=dba_fielddescription) */ SELECT util._map('0=>public, 1=>public, 2=>public, 3=>public, 4=>public, 5=>"data by request", 6=>private', $1::text) $BODY$ LANGUAGE sql IMMUTABLE COST 100; Time: 94.104 ms SELECT mk_derived_col((:table_str, 'accessRights'), $$map_access_rights("accessRights_verbatim")$$); -- runtime: 1 s ("Time: 1167.547 ms") @starscream psql:/dev/fd/40:46: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(map_access_rights("accessRights_verbatim")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.28 rows=1 width=4) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:46: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN "accessRights" text; ALTER TABLE plot ALTER COLUMN "accessRights" TYPE text USING map_access_rights("accessRights_verbatim"); mk_derived_col ---------------- (1 row) Time: 5.993 ms CREATE OR REPLACE FUNCTION map_fuzzing(value integer) RETURNS double precision AS $BODY$ /* "closed List of Values: Value Description Sorting 0 Public 1 1 1 km radius (nearest 0.01 degree) 2 2 10 km radius (nearest 0.1 degree) 3 3 100 km radius (nearest degree) 4 4 Location embargo 5 5 Public embargo on data 6 6 Full embargo on data 7 " (http://vegbank.org/vegbank/views/dba_fielddescription_detail.jsp?view=detail&wparam=1581&entity=dba_fielddescription) */ SELECT util._km_to_m(util._map('0=>NULL, 1=>1, 2=>10, 3=>100, 4=>NULL, 5=>NULL, 6=>NULL', $1::text)::double precision) $BODY$ LANGUAGE sql IMMUTABLE COST 100; Time: 59.635 ms SELECT mk_derived_col((:table_str, 'coordinateUncertaintyInMeters__from_fuzzing'), $$map_fuzzing("accessRights_verbatim")$$); -- runtime: 1 s ("Time: 1064.189 ms") @starscream psql:/dev/fd/40:68: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(map_fuzzing("accessRights_verbatim")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.29 rows=1 width=4) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:68: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN "coordinateUncertaintyInMeters__from_fuzzing" double precision; ALTER TABLE plot ALTER COLUMN "coordinateUncertaintyInMeters__from_fuzzing" TYPE double precision USING map_fuzzing("accessRights_verbatim"); mk_derived_col ---------------- (1 row) Time: 61.974 ms SELECT mk_derived_col((:table_str, 'coordinateUncertaintyInMeters'), $$GREATEST("coordinateUncertaintyInMeters__from_fuzzing", "coordinateUncertaintyInMeters__from_coords")$$) ; -- runtime: 0.5 s ("Time: 586.453 ms") @starscream psql:/dev/fd/40:73: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(GREATEST("coordinateUncertaintyInMeters__from_fuzzing", "coordinateUncertaintyInMeters__from_coords")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.02 rows=1 width=16) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:73: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN "coordinateUncertaintyInMeters" double precision; ALTER TABLE plot ALTER COLUMN "coordinateUncertaintyInMeters" TYPE double precision USING GREATEST("coordinateUncertaintyInMeters__from_fuzzing", "coordinateUncertaintyInMeters__from_coords"); mk_derived_col ---------------- (1 row) Time: 3.832 ms SELECT mk_derived_col((:table_str, 'locality'), $$_join("directions_to_place", "site_name")$$) ; -- runtime: 4 s ("Time: 3651.355 ms") @starscream psql:/dev/fd/40:77: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join("directions_to_place", "site_name")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=450) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:77: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN locality text; ALTER TABLE plot ALTER COLUMN locality TYPE text USING _join("directions_to_place", "site_name"); mk_derived_col ---------------- (1 row) Time: 4.469 ms real 0m1.043s user 0m0.029s sys 0m0.033s | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.200 ms \cd schemas/VegCore \set schema "VegBank" \set table "plot" \set table_str '''"plot"''' SET search_path TO "VegBank"; Time: 0.111 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.621 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(plot,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.585 ms real 0m0.086s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.225 ms \cd inputs/VegBank/plot \set schema "public" \set table "plot" \set table_str '''"plot"''' SET search_path TO "public"; Time: 0.129 ms CREATE TEMP TABLE "public".t () Time: 0.387 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.082s user 0m0.034s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "plot/VegBIEN.csv" ln: plot/VegBIEN.csv: File exists make: [plot/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/plot/test.xml... >plot/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=plot out_database=vegbien source=VegBank.new ../../map plot/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:01.108461 sec/2 row(s) = 554. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change plot/test.xml.ref plot/test.xml real 0m2.521s user 0m1.184s sys 0m0.416s real 0m9.930s user 0m3.897s sys 0m2.340s | | | | | + inputs/VegBank/plot_/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/plot_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.192 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.222 ms 0.06 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.648s user 0m0.379s sys 0m0.210s real 0m0.163s user 0m0.023s sys 0m0.026s : : : : : : : : : table plot_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.054s user 0m0.611s sys 0m0.324s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.196 ms \cd inputs/VegBank/plot_ \set schema "VegBank" \set table "plot_" \set table_str '''"plot_"''' SET search_path TO "VegBank"; Time: 0.108 ms SELECT util.reset_map_table('"~plot_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~plot_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~plot_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~plot_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 167.528 ms ALTER TABLE "~plot_.map" DISABLE TRIGGER map_filter_insert; Time: 0.329 ms \copy "~plot_.map" FROM 'map.csv' CSV HEADER; Time: 34.162 ms SELECT util.set_col_names_with_metadata('"plot_"', '"~plot_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 43.171 ms real 0m0.316s user 0m0.028s sys 0m0.032s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.225 ms \cd schemas/VegCore \set schema "VegBank" \set table "plot_" \set table_str '''"plot_"''' SET search_path TO "VegBank"; Time: 0.169 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 41.557 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(plot_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 21.620 ms real 0m0.127s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.196 ms \cd inputs/VegBank/plot_ \set schema "public" \set table "plot_" \set table_str '''"plot_"''' SET search_path TO "public"; Time: 0.110 ms CREATE TEMP TABLE "public".t () Time: 0.299 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.084s user 0m0.034s sys 0m0.040s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "plot_/VegBIEN.csv" ln: plot_/VegBIEN.csv: File exists make: [plot_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/plot_/test.xml... >plot_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=plot_ out_database=vegbien source=VegBank.new ../../map plot_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:00.750278 sec/2 row(s) = 375. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change plot_/test.xml.ref plot_/test.xml real 0m2.184s user 0m1.194s sys 0m0.415s real 0m7.243s user 0m3.102s sys 0m1.992s | | | | | + inputs/VegBank/observation_/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/observation_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.322 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.300 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.693s user 0m0.383s sys 0m0.217s real 0m0.156s user 0m0.023s sys 0m0.024s : : : : : : : : : table observation_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.007s user 0m0.608s sys 0m0.317s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.195 ms \cd inputs/VegBank/observation_ \set schema "VegBank" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "VegBank"; Time: 0.100 ms SELECT util.reset_map_table('"~observation_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~observation_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~observation_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~observation_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 189.852 ms ALTER TABLE "~observation_.map" DISABLE TRIGGER map_filter_insert; Time: 0.422 ms \copy "~observation_.map" FROM 'map.csv' CSV HEADER; Time: 36.974 ms SELECT util.set_col_names_with_metadata('"observation_"', '"~observation_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 4.192 ms real 0m0.327s user 0m0.029s sys 0m0.034s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place observation_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.923s user 0m0.634s sys 0m0.224s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.209 ms \cd inputs/VegBank/observation_ \set schema "VegBank" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "VegBank"; Time: 0.107 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 28.923 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("eventID")$$); -- runtime: ~0 ("Time: 1.838 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "observation_" ADD PRIMARY KEY ("eventID") create_if_not_exists ---------------------- (1 row) Time: 22.360 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("locationID")$$); -- runtime: ~0 ("158.792 ms") @starscream psql:/dev/fd/40:13: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "observation___parent" ON "observation_" ("locationID") create_if_not_exists ---------------------- (1 row) Time: 53.761 ms real 0m0.171s user 0m0.029s sys 0m0.031s | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.208 ms \cd schemas/VegCore \set schema "VegBank" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "VegBank"; Time: 0.109 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.714 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(observation_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 33.452 ms real 0m0.115s user 0m0.026s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.195 ms \cd inputs/VegBank/observation_ \set schema "public" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "public"; Time: 0.101 ms CREATE TEMP TABLE "public".t () Time: 0.305 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.082s user 0m0.034s sys 0m0.039s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "observation_/VegBIEN.csv" ln: observation_/VegBIEN.csv: File exists make: [observation_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/observation_/test.xml... >observation_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=observation_ out_database=vegbien source=VegBank.new ../../map observation_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:00.798079 sec/2 row(s) = 399. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change observation_/test.xml.ref observation_/test.xml real 0m2.208s user 0m1.126s sys 0m0.418s real 0m8.701s user 0m3.756s sys 0m2.323s | | | | | + inputs/VegBank/stratumtype/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stratumtype/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.194 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.237 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.646s user 0m0.376s sys 0m0.208s real 0m0.193s user 0m0.023s sys 0m0.025s : : : : : : : : : table stratumtype already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratumtype/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratumtype/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.019s user 0m0.611s sys 0m0.321s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.189 ms \cd inputs/VegBank/stratumtype \set schema "VegBank" \set table "stratumtype" \set table_str '''"stratumtype"''' SET search_path TO "VegBank"; Time: 0.097 ms SELECT util.reset_map_table('"~stratumtype.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stratumtype.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stratumtype.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stratumtype.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 75.291 ms ALTER TABLE "~stratumtype.map" DISABLE TRIGGER map_filter_insert; Time: 0.448 ms \copy "~stratumtype.map" FROM 'map.csv' CSV HEADER; Time: 27.573 ms SELECT util.set_col_names_with_metadata('"stratumtype"', '"~stratumtype.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.377 ms real 0m0.173s user 0m0.028s sys 0m0.031s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.200 ms \cd schemas/VegCore \set schema "VegBank" \set table "stratumtype" \set table_str '''"stratumtype"''' SET search_path TO "VegBank"; Time: 0.094 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.755 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stratumtype,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 31.569 ms real 0m0.137s user 0m0.027s sys 0m0.030s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.188 ms \cd inputs/VegBank/stratumtype \set schema "public" \set table "stratumtype" \set table_str '''"stratumtype"''' SET search_path TO "public"; Time: 0.169 ms CREATE TEMP TABLE "public".t () Time: 0.306 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.084s user 0m0.035s sys 0m0.041s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratumtype/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratumtype/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stratumtype/VegBIEN.csv" ln: stratumtype/VegBIEN.csv: File exists make: [stratumtype/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stratumtype/test.xml... >stratumtype/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stratumtype out_database=vegbien source=VegBank.new ../../map stratumtype/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.630596 sec/2 row(s) = 315. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stratumtype/test.xml.ref stratumtype/test.xml real 0m2.053s user 0m0.980s sys 0m0.409s real 0m6.947s user 0m2.880s sys 0m1.973s | | | | | + inputs/VegBank/stratum/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stratum/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.185 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.221 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.673s user 0m0.378s sys 0m0.208s real 0m0.095s user 0m0.024s sys 0m0.028s : : : : : : : : : table stratum already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratum/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratum/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.001s user 0m0.603s sys 0m0.315s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.236 ms \cd inputs/VegBank/stratum \set schema "VegBank" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "VegBank"; Time: 0.091 ms SELECT util.reset_map_table('"~stratum.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stratum.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stratum.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stratum.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 94.756 ms ALTER TABLE "~stratum.map" DISABLE TRIGGER map_filter_insert; Time: 0.333 ms \copy "~stratum.map" FROM 'map.csv' CSV HEADER; Time: 0.917 ms SELECT util.set_col_names_with_metadata('"stratum"', '"~stratum.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.546 ms real 0m0.166s user 0m0.028s sys 0m0.031s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratum/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place stratum/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.921s user 0m0.633s sys 0m0.226s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.202 ms \cd inputs/VegBank/stratum \set schema "VegBank" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "VegBank"; Time: 0.087 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.301 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("*stratum_id")$$); -- runtime: ~0 ("Time: 2.594 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "stratum" ADD PRIMARY KEY ("*stratum_id") create_if_not_exists ---------------------- (1 row) Time: 40.286 ms real 0m0.126s user 0m0.028s sys 0m0.031s | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.324 ms \cd schemas/VegCore \set schema "VegBank" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "VegBank"; Time: 0.142 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 20.719 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stratum,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 23.826 ms real 0m0.114s user 0m0.027s sys 0m0.029s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd inputs/VegBank/stratum \set schema "public" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "public"; Time: 0.106 ms CREATE TEMP TABLE "public".t () Time: 0.296 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.083s user 0m0.034s sys 0m0.039s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratum/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratum/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stratum/VegBIEN.csv" ln: stratum/VegBIEN.csv: File exists make: [stratum/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stratum/test.xml... >stratum/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stratum out_database=vegbien source=VegBank.new ../../map stratum/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/map:296: UserWarning: Map warning: No mappings or no column name matches. Are you importing the correct input table? warnings.warn(UserWarning('Map warning: No mappings or no ' Processed 2 input rows Took 0:00:00.332640 sec/2 row(s) = 166. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stratum/test.xml.ref stratum/test.xml real 0m2.059s user 0m0.926s sys 0m0.403s real 0m8.387s user 0m3.546s sys 0m2.299s | | | | | + inputs/VegBank/plantconcept_/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/plantconcept_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.837 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.320 ms 0.04 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.658s user 0m0.395s sys 0m0.213s real 0m0.110s user 0m0.023s sys 0m0.027s : : : : : : : : : table plantconcept_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plantconcept_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plantconcept_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.104s user 0m0.630s sys 0m0.332s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.248 ms \cd inputs/VegBank/plantconcept_ \set schema "VegBank" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "VegBank"; Time: 0.128 ms SELECT util.reset_map_table('"~plantconcept_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~plantconcept_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~plantconcept_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~plantconcept_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 135.867 ms ALTER TABLE "~plantconcept_.map" DISABLE TRIGGER map_filter_insert; Time: 0.558 ms \copy "~plantconcept_.map" FROM 'map.csv' CSV HEADER; Time: 24.049 ms SELECT util.set_col_names_with_metadata('"plantconcept_"', '"~plantconcept_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.599 ms real 0m0.231s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plantconcept_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place plantconcept_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m1.027s user 0m0.669s sys 0m0.242s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.243 ms \cd inputs/VegBank/plantconcept_ \set schema "VegBank" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "VegBank"; Time: 0.127 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 27.309 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("plantconcept_id")$$); -- runtime: 1 s ("Time: 675.125 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "plantconcept_" ADD PRIMARY KEY ("plantconcept_id") create_if_not_exists ---------------------- (1 row) Time: 17.664 ms real 0m0.114s user 0m0.029s sys 0m0.031s | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.242 ms \cd schemas/VegCore \set schema "VegBank" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "VegBank"; Time: 0.120 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.383 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(plantconcept_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 27.570 ms real 0m0.111s user 0m0.027s sys 0m0.029s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.203 ms \cd inputs/VegBank/plantconcept_ \set schema "public" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "public"; Time: 0.103 ms CREATE TEMP TABLE "public".t () Time: 0.295 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.108s user 0m0.034s sys 0m0.041s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plantconcept_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plantconcept_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "plantconcept_/VegBIEN.csv" ln: plantconcept_/VegBIEN.csv: File exists make: [plantconcept_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/plantconcept_/test.xml... >plantconcept_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=plantconcept_ out_database=vegbien source=VegBank.new ../../map plantconcept_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.891129 sec/2 row(s) = 446. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change plantconcept_/test.xml.ref plantconcept_/test.xml real 0m2.275s user 0m1.298s sys 0m0.420s real 0m8.986s user 0m4.001s sys 0m2.375s | | | | | + inputs/VegBank/taxonobservation_/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/taxonobservation_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.220 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.244 ms 0.04 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.649s user 0m0.375s sys 0m0.207s real 0m0.223s user 0m0.023s sys 0m0.025s : : : : : : : : : table taxonobservation_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxonobservation_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile taxonobservation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.995s user 0m0.608s sys 0m0.318s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.185 ms \cd inputs/VegBank/taxonobservation_ \set schema "VegBank" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "VegBank"; Time: 0.110 ms SELECT util.reset_map_table('"~taxonobservation_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~taxonobservation_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~taxonobservation_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~taxonobservation_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 225.349 ms ALTER TABLE "~taxonobservation_.map" DISABLE TRIGGER map_filter_insert; Time: 0.437 ms \copy "~taxonobservation_.map" FROM 'map.csv' CSV HEADER; Time: 20.333 ms SELECT util.set_col_names_with_metadata('"taxonobservation_"', '"~taxonobservation_.map"'); psql:/dev/fd/40:12: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; ALTER TABLE taxonobservation_ ADD COLUMN "observationGranularity" text NOT NULL DEFAULT 'plot'; COMMENT ON COLUMN taxonobservation_."observationGranularity" IS ' constant '; set_col_names_with_metadata ----------------------------- (1 row) Time: 20.692 ms real 0m0.331s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxonobservation_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place taxonobservation_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.907s user 0m0.643s sys 0m0.227s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.189 ms \cd inputs/VegBank/taxonobservation_ \set schema "VegBank" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "VegBank"; Time: 0.104 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 27.291 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("eventID")$$); -- runtime: 2 s ("2416.676 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "taxonobservation___parent" ON "taxonobservation_" ("eventID") create_if_not_exists ---------------------- (1 row) Time: 5.434 ms SELECT mk_derived_col((:table_str, 'identifiedBy'), $$_join_words("identifiedBy__first", "identifiedBy__middle", "identifiedBy__last")$$) ; -- runtime: 30 s ("29290.701 ms") @starscream psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join_words("identifiedBy__first", "identifiedBy__middle", "identifiedBy__last")) FROM (SELECT (NULL::taxonobservation_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=96) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE taxonobservation_ ADD COLUMN "identifiedBy" text; ALTER TABLE taxonobservation_ ALTER COLUMN "identifiedBy" TYPE text USING _join_words("identifiedBy__first", "identifiedBy__middle", "identifiedBy__last"); mk_derived_col ---------------- (1 row) Time: 30.199 ms real 0m0.129s user 0m0.028s sys 0m0.031s | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.197 ms \cd schemas/VegCore \set schema "VegBank" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "VegBank"; Time: 0.106 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 60.669 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(taxonobservation_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.589 ms real 0m0.154s user 0m0.027s sys 0m0.030s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.209 ms \cd inputs/VegBank/taxonobservation_ \set schema "public" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "public"; Time: 0.101 ms CREATE TEMP TABLE "public".t () Time: 0.331 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.082s user 0m0.034s sys 0m0.039s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxonobservation_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile taxonobservation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "taxonobservation_/VegBIEN.csv" ln: taxonobservation_/VegBIEN.csv: File exists make: [taxonobservation_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/taxonobservation_/test.xml... >taxonobservation_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=taxonobservation_ out_database=vegbien source=VegBank.new ../../map taxonobservation_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:01.954426 sec/2 row(s) = 977. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change taxonobservation_/test.xml.ref taxonobservation_/test.xml real 0m3.385s user 0m1.506s sys 0m0.428s real 0m9.806s user 0m4.136s sys 0m2.324s | | | | | + inputs/VegBank/stemcount_/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stemcount_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.205 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.224 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.890s user 0m0.387s sys 0m0.214s real 0m0.096s user 0m0.023s sys 0m0.024s : : : : : : : : : table stemcount_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemcount_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemcount_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.008s user 0m0.602s sys 0m0.318s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.189 ms \cd inputs/VegBank/stemcount_ \set schema "VegBank" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "VegBank"; Time: 0.110 ms SELECT util.reset_map_table('"~stemcount_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stemcount_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stemcount_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stemcount_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 96.683 ms ALTER TABLE "~stemcount_.map" DISABLE TRIGGER map_filter_insert; Time: 0.376 ms \copy "~stemcount_.map" FROM 'map.csv' CSV HEADER; Time: 33.622 ms SELECT util.set_col_names_with_metadata('"stemcount_"', '"~stemcount_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.670 ms real 0m0.199s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemcount_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place stemcount_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.972s user 0m0.650s sys 0m0.235s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.197 ms \cd inputs/VegBank/stemcount_ \set schema "VegBank" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "VegBank"; Time: 0.089 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 21.263 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("taxonOccurrenceID__overall_plot")$$); -- runtime: 2 s ("Time: 1766.315 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "stemcount___parent" ON "stemcount_" ("taxonOccurrenceID__overall_plot") create_if_not_exists ---------------------- (1 row) Time: 22.944 ms SELECT mk_derived_col((:table_str, 'aggregateOrganismObservationID'), $$_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)$$) ; -- runtime: 15 s ("Time: 15805.876 ms") @starscream psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)) FROM (SELECT (NULL::stemcount_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.28 rows=1 width=8) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE stemcount_ ADD COLUMN "aggregateOrganismObservationID" text; ALTER TABLE stemcount_ ALTER COLUMN "aggregateOrganismObservationID" TYPE text USING _join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text); mk_derived_col ---------------- (1 row) Time: 5.585 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("aggregateOrganismObservationID")$$); -- runtime: 20 s ("Time: 20194.932 ms") @starscream psql:/dev/fd/40:19: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "stemcount_" ADD PRIMARY KEY ("aggregateOrganismObservationID") create_if_not_exists ---------------------- (1 row) Time: 2.479 ms real 0m0.118s user 0m0.028s sys 0m0.030s | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd schemas/VegCore \set schema "VegBank" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "VegBank"; Time: 0.111 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.692 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stemcount_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.963 ms real 0m0.086s user 0m0.027s sys 0m0.029s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.241 ms \cd inputs/VegBank/stemcount_ \set schema "public" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "public"; Time: 0.100 ms CREATE TEMP TABLE "public".t () Time: 0.301 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.085s user 0m0.034s sys 0m0.041s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemcount_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemcount_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stemcount_/VegBIEN.csv" ln: stemcount_/VegBIEN.csv: File exists make: [stemcount_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stemcount_/test.xml... >stemcount_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stemcount_ out_database=vegbien source=VegBank.new ../../map stemcount_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.595703 sec/2 row(s) = 298. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stemcount_/test.xml.ref stemcount_/test.xml real 0m2.017s user 0m1.050s sys 0m0.411s real 0m8.613s user 0m3.697s sys 0m2.328s | | | | | + inputs/VegBank/stemlocation_/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stemlocation_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.191 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.235 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.619s user 0m0.381s sys 0m0.206s real 0m0.173s user 0m0.023s sys 0m0.027s : : : : : : : : : table stemlocation_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemlocation_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemlocation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.034s user 0m0.608s sys 0m0.320s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.198 ms \cd inputs/VegBank/stemlocation_ \set schema "VegBank" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "VegBank"; Time: 0.111 ms SELECT util.reset_map_table('"~stemlocation_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stemlocation_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stemlocation_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stemlocation_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 60.060 ms ALTER TABLE "~stemlocation_.map" DISABLE TRIGGER map_filter_insert; Time: 0.412 ms \copy "~stemlocation_.map" FROM 'map.csv' CSV HEADER; Time: 31.306 ms SELECT util.set_col_names_with_metadata('"stemlocation_"', '"~stemlocation_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.483 ms real 0m0.160s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemlocation_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place stemlocation_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.955s user 0m0.644s sys 0m0.226s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd inputs/VegBank/stemlocation_ \set schema "VegBank" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "VegBank"; Time: 0.098 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 27.172 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("stemID")$$); -- runtime: 0.1 s ("Time: 74.164 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "stemlocation_" ADD PRIMARY KEY ("stemID") create_if_not_exists ---------------------- (1 row) Time: 34.201 ms SELECT mk_derived_col((:table_str, 'individualID'), $$"stemID"$$) ; -- runtime: ~0 ("Time: 1.562 ms") @starscream psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof("stemID") FROM (SELECT (NULL::stemlocation_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.02 rows=1 width=4) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE stemlocation_ ADD COLUMN "individualID" integer; ALTER TABLE stemlocation_ ALTER COLUMN "individualID" TYPE integer USING "stemID"; mk_derived_col ---------------- (1 row) Time: 14.671 ms SELECT mk_derived_col((:table_str, 'aggregateOrganismObservationID'), $$_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)$$) ; -- runtime: 0.5 s ("Time: 441.059 ms") @starscream psql:/dev/fd/40:18: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)) FROM (SELECT (NULL::stemlocation_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.28 rows=1 width=8) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:18: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE stemlocation_ ADD COLUMN "aggregateOrganismObservationID" text; ALTER TABLE stemlocation_ ALTER COLUMN "aggregateOrganismObservationID" TYPE text USING _join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text); mk_derived_col ---------------- (1 row) Time: 4.245 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("aggregateOrganismObservationID")$$); -- runtime: 0.1 s ("Time: 131.907 ms") @starscream psql:/dev/fd/40:21: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "stemlocation___parent" ON "stemlocation_" ("aggregateOrganismObservationID") create_if_not_exists ---------------------- (1 row) Time: 2.527 ms real 0m0.149s user 0m0.028s sys 0m0.030s | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd schemas/VegCore \set schema "VegBank" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "VegBank"; Time: 0.113 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.020 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stemlocation_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.551 ms real 0m0.088s user 0m0.027s sys 0m0.030s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.207 ms \cd inputs/VegBank/stemlocation_ \set schema "public" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "public"; Time: 0.090 ms CREATE TEMP TABLE "public".t () Time: 0.294 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.109s user 0m0.034s sys 0m0.041s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemlocation_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemlocation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stemlocation_/VegBIEN.csv" ln: stemlocation_/VegBIEN.csv: File exists make: [stemlocation_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stemlocation_/test.xml... >stemlocation_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stemlocation_ out_database=vegbien source=VegBank.new ../../map stemlocation_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.588020 sec/2 row(s) = 294. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stemlocation_/test.xml.ref stemlocation_/test.xml real 0m1.973s user 0m1.043s sys 0m0.413s real 0m8.327s user 0m3.678s sys 0m2.314s | | | | | + inputs/VegBank/taxon_observation.**/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxon_observation.**/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place taxon_observation.**/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.939s user 0m0.652s sys 0m0.226s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.204 ms \cd inputs/VegBank/taxon_observation.** \set schema "VegBank" \set table "taxon_observation.**" \set table_str '''"taxon_observation.**"''' SET search_path TO "VegBank"; Time: 0.099 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 23.279 ms SELECT force_update_view(:table_str, $$ /* no row_num because left-join to stemcount_, stemlocation_ adds rows to each taxonobservation_ */ SELECT "identificationID" AS sort_col, * , NULLIF(concat_ws('; ', "taxonOccurrenceID__overall_plot"::text, "aggregateOrganismObservationID"), '') AS "taxonOccurrenceID" FROM taxonobservation_ NATURAL LEFT JOIN stemcount_ NATURAL LEFT JOIN stemlocation_ NATURAL LEFT JOIN stratum NATURAL LEFT JOIN stratumtype NATURAL LEFT JOIN observation_ NATURAL LEFT JOIN plot_ NATURAL LEFT JOIN project $$); force_update_view ------------------- (1 row) Time: 305.520 ms SELECT mk_subset_by_row_num_func(:table_str); mk_subset_by_row_num_func --------------------------- (1 row) Time: 66.592 ms real 0m0.460s user 0m0.028s sys 0m0.030s | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.210 ms \cd schemas/VegCore \set schema "VegBank" \set table "taxon_observation.**" \set table_str '''"taxon_observation.**"''' SET search_path TO "VegBank"; Time: 0.134 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 20.093 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('("""taxon_observation.**""",cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 8.267 ms real 0m0.093s user 0m0.027s sys 0m0.029s | | | | | | | | + builtin cd -P inputs/VegBank/taxon_observation.** | | | | | | | | | + rm -f header.csv | | | | | | | | + mappings__table_run @lib/runscripts/view.run:22 -> lib/sh/util.sh:419 | | | | | | | | | + public_schema_exists @lib/sh/util.sh:435 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.327 ms \cd inputs/VegBank/taxon_observation.** \set schema "public" \set table "taxon_observation.**" \set table_str '''"taxon_observation.**"''' SET search_path TO "public"; Time: 0.233 ms CREATE TEMP TABLE "public".t () Time: 0.400 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.099s user 0m0.035s sys 0m0.041s | | | | | | | | | + builtin cd -P inputs/VegBank/taxon_observation.** | | | | | | | | | | + rm -f map.csv | | | | | | | | | + test_ @lib/sh/util.sh:439 -> lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxon_observation.**/test make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make --makefile=../input.Makefile "taxon_observation.**/header.csv" set -o pipefail; \ echo 'COPY (SELECT * FROM "VegBank"."taxon_observation.**" LIMIT 0) TO STDOUT CSV HEADER;'|\ env no_search_path=1 ../../bin/psql_script_vegbien >taxon_observation.**/header.csv (. ../../mappings/root.sh; env datasrc=VegBank ../../bin/src_map taxon_observation.**/map.csv) /usr/bin/make --makefile=../input.Makefile taxon_observation.**/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place taxon_observation.**/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place taxon_observation.**/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place taxon_observation.**/map.csv ../../bin/fix_line_endings touch taxon_observation.**/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile taxon_observation.**/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile taxon_observation.**/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 taxon_observation.**/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >taxon_observation.**/unmapped_terms.csv ../../bin/autoremove taxon_observation.**/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile taxon_observation.**/new_terms.csv /usr/bin/make --makefile=../input.Makefile taxon_observation.**/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 taxon_observation.**/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 taxon_observation.**/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >taxon_observation.**/new_terms.csv; exit 0 ../../bin/autoremove taxon_observation.**/new_terms.csv ln -s "../../../mappings/VegCore-VegBIEN.csv" "taxon_observation.**/VegBIEN.csv" ln: taxon_observation.**/VegBIEN.csv: File exists make: [taxon_observation.**/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/taxon_observation.**/test.xml... >taxon_observation.**/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=taxon_observation.** out_database=vegbien source=VegBank.new ../../map taxon_observation.**/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:01:17.629221 sec/2 row(s) = 38.8 s/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change 'taxon_observation.**/test.xml.ref' 'taxon_observation.**/test.xml' real 1m22.502s user 0m4.030s sys 0m1.233s real 1m26.557s user 0m5.528s sys 0m2.409s | | | | | + inputs/VegBank/^taxon_observation.**.sample/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/^taxon_observation.**.sample/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.225 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.449 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.678s user 0m0.379s sys 0m0.216s real 0m0.080s user 0m0.023s sys 0m0.025s : : : : : : : : : table ^taxon_observation.**.sample already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank ^taxon_observation.**.sample/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile ^taxon_observation.**.sample/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.052s user 0m0.611s sys 0m0.323s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.204 ms \cd inputs/VegBank/^taxon_observation.**.sample \set schema "VegBank" \set table "^taxon_observation.**.sample" \set table_str '''"^taxon_observation.**.sample"''' SET search_path TO "VegBank"; Time: 0.096 ms SELECT util.reset_map_table('"~^taxon_observation.**.sample.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~^taxon_observation.**.sample.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~^taxon_observation.**.sample.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~^taxon_observation.**.sample.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 68.508 ms ALTER TABLE "~^taxon_observation.**.sample.map" DISABLE TRIGGER map_filter_insert; Time: 0.344 ms \copy "~^taxon_observation.**.sample.map" FROM 'map.csv' CSV HEADER; Time: 23.264 ms SELECT util.set_col_names_with_metadata('"^taxon_observation.**.sample"', '"~^taxon_observation.**.sample.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.992 ms real 0m0.189s user 0m0.028s sys 0m0.032s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.194 ms \cd schemas/VegCore \set schema "VegBank" \set table "^taxon_observation.**.sample" \set table_str '''"^taxon_observation.**.sample"''' SET search_path TO "VegBank"; Time: 0.105 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 44.008 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('("""^taxon_observation.**.sample""",cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.712 ms real 0m0.111s user 0m0.026s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.239 ms \cd inputs/VegBank/^taxon_observation.**.sample \set schema "public" \set table "^taxon_observation.**.sample" \set table_str '''"^taxon_observation.**.sample"''' SET search_path TO "public"; Time: 0.106 ms CREATE TEMP TABLE "public".t () Time: 0.293 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.089s user 0m0.035s sys 0m0.042s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank ^taxon_observation.**.sample/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile ^taxon_observation.**.sample/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "^taxon_observation.**.sample/VegBIEN.csv" ln: ^taxon_observation.**.sample/VegBIEN.csv: File exists make: [^taxon_observation.**.sample/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/^taxon_observation.**.sample/test.xml... >^taxon_observation.**.sample/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=^taxon_observation.**.sample out_database=vegbien source=VegBank.new ../../map ^taxon_observation.**.sample/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:01.805353 sec/2 row(s) = 903. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change '^taxon_observation.**.sample/test.xml.ref' '^taxon_observation.**.sample/test.xml' real 0m3.210s user 0m1.987s sys 0m0.444s real 0m8.160s user 0m3.889s sys 0m2.025s | | | | | + inputs/VegBank/projectcontributor_/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/projectcontributor_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.247 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.300 ms 0.04 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.651s user 0m0.379s sys 0m0.212s real 0m0.076s user 0m0.023s sys 0m0.026s : : : : : : : : : table projectcontributor_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank projectcontributor_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile projectcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.003s user 0m0.612s sys 0m0.323s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.197 ms \cd inputs/VegBank/projectcontributor_ \set schema "VegBank" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "VegBank"; Time: 0.093 ms SELECT util.reset_map_table('"~projectcontributor_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~projectcontributor_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~projectcontributor_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~projectcontributor_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 97.182 ms ALTER TABLE "~projectcontributor_.map" DISABLE TRIGGER map_filter_insert; Time: 0.453 ms \copy "~projectcontributor_.map" FROM 'map.csv' CSV HEADER; Time: 31.307 ms SELECT util.set_col_names_with_metadata('"projectcontributor_"', '"~projectcontributor_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.526 ms real 0m0.199s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank projectcontributor_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place projectcontributor_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.901s user 0m0.636s sys 0m0.227s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.208 ms \cd inputs/VegBank/projectcontributor_ \set schema "VegBank" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "VegBank"; Time: 0.166 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.560 ms SELECT mk_derived_col((:table_str, 'project_participant'), $$_join_words(project_participant_name_first::text, project_participant_name_middle::text, project_participant_name_last::text)$$) ; -- runtime: ~0 ("2.887 ms") @starscream psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join_words(project_participant_name_first::text, project_participant_name_middle::text, project_participant_name_last::text)) FROM (SELECT (NULL::projectcontributor_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=96) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE projectcontributor_ ADD COLUMN project_participant text; ALTER TABLE projectcontributor_ ALTER COLUMN project_participant TYPE text USING _join_words(project_participant_name_first::text, project_participant_name_middle::text, project_participant_name_last::text); mk_derived_col ---------------- (1 row) Time: 61.463 ms real 0m0.172s user 0m0.028s sys 0m0.032s | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.206 ms \cd schemas/VegCore \set schema "VegBank" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "VegBank"; Time: 0.117 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.245 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(projectcontributor_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.723 ms real 0m0.089s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.200 ms \cd inputs/VegBank/projectcontributor_ \set schema "public" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "public"; Time: 0.121 ms CREATE TEMP TABLE "public".t () Time: 0.338 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.087s user 0m0.035s sys 0m0.041s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank projectcontributor_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile projectcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "projectcontributor_/VegBIEN.csv" ln: projectcontributor_/VegBIEN.csv: File exists make: [projectcontributor_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/projectcontributor_/test.xml... >projectcontributor_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=projectcontributor_ out_database=vegbien source=VegBank.new ../../map projectcontributor_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.717974 sec/2 row(s) = 359. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change projectcontributor_/test.xml.ref projectcontributor_/test.xml real 0m2.097s user 0m0.993s sys 0m0.414s real 0m8.428s user 0m3.624s sys 0m2.330s | | | | | + inputs/VegBank/observationcontributor_/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/observationcontributor_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.191 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.229 ms 0.06 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.650s user 0m0.379s sys 0m0.210s real 0m0.119s user 0m0.023s sys 0m0.027s : : : : : : : : : table observationcontributor_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observationcontributor_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observationcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.026s user 0m0.600s sys 0m0.319s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.192 ms \cd inputs/VegBank/observationcontributor_ \set schema "VegBank" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "VegBank"; Time: 0.109 ms SELECT util.reset_map_table('"~observationcontributor_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~observationcontributor_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~observationcontributor_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~observationcontributor_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 76.633 ms ALTER TABLE "~observationcontributor_.map" DISABLE TRIGGER map_filter_insert; Time: 0.480 ms \copy "~observationcontributor_.map" FROM 'map.csv' CSV HEADER; Time: 29.199 ms SELECT util.set_col_names_with_metadata('"observationcontributor_"', '"~observationcontributor_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.594 ms real 0m0.178s user 0m0.028s sys 0m0.031s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observationcontributor_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place observationcontributor_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.906s user 0m0.633s sys 0m0.227s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.209 ms \cd inputs/VegBank/observationcontributor_ \set schema "VegBank" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "VegBank"; Time: 0.101 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.514 ms SELECT mk_derived_col((:table_str, 'event__participant'), $$_join_words(event__participant__name__first::text, event__participant__name__middle::text, event__participant__name__last::text)$$) ; -- runtime: 1 s ("Time: 1265.039 ms") @starscream psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join_words(event__participant__name__first::text, event__participant__name__middle::text, event__participant__name__last::text)) FROM (SELECT (NULL::observationcontributor_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=96) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE observationcontributor_ ADD COLUMN event__participant text; ALTER TABLE observationcontributor_ ALTER COLUMN event__participant TYPE text USING _join_words(event__participant__name__first::text, event__participant__name__middle::text, event__participant__name__last::text); mk_derived_col ---------------- (1 row) Time: 22.885 ms real 0m0.107s user 0m0.028s sys 0m0.031s | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.191 ms \cd schemas/VegCore \set schema "VegBank" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "VegBank"; Time: 0.105 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.402 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(observationcontributor_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.472 ms real 0m0.090s user 0m0.028s sys 0m0.030s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.243 ms \cd inputs/VegBank/observationcontributor_ \set schema "public" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "public"; Time: 0.125 ms CREATE TEMP TABLE "public".t () Time: 0.375 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.100s user 0m0.035s sys 0m0.042s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observationcontributor_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observationcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "observationcontributor_/VegBIEN.csv" ln: observationcontributor_/VegBIEN.csv: File exists make: [observationcontributor_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/observationcontributor_/test.xml... >observationcontributor_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=observationcontributor_ out_database=vegbien source=VegBank.new ../../map observationcontributor_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.619080 sec/2 row(s) = 310. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change observationcontributor_/test.xml.ref observationcontributor_/test.xml real 0m2.018s user 0m1.013s sys 0m0.409s real 0m8.227s user 0m3.622s sys 0m2.306s | | | | | + inputs/VegBank/observation__community/run import | | | | | | + import @lib/runscripts/util.run:56 -> lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:134 -> 50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/observation__community/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.204 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.218 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.639s user 0m0.373s sys 0m0.209s real 0m0.110s user 0m0.023s sys 0m0.028s : : : : : : : : : table observation__community already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:135 -> 120 | | | | | | | | + map_table @lib/runscripts/table.run:125 -> 77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation__community/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation__community/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.031s user 0m0.609s sys 0m0.321s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.197 ms \cd inputs/VegBank/observation__community \set schema "VegBank" \set table "observation__community" \set table_str '''"observation__community"''' SET search_path TO "VegBank"; Time: 0.090 ms SELECT util.reset_map_table('"~observation__community.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~observation__community.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~observation__community.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~observation__community.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 99.713 ms ALTER TABLE "~observation__community.map" DISABLE TRIGGER map_filter_insert; Time: 0.351 ms \copy "~observation__community.map" FROM 'map.csv' CSV HEADER; Time: 28.801 ms SELECT util.set_col_names_with_metadata('"observation__community"', '"~observation__community.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.585 ms real 0m0.199s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:126 -> 107 | | | | | | | | + mk_derived @lib/runscripts/table.run:127 -> 117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.195 ms \cd schemas/VegCore \set schema "VegBank" \set table "observation__community" \set table_str '''"observation__community"''' SET search_path TO "VegBank"; Time: 0.085 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.264 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(observation__community,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.658 ms real 0m0.086s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:128 -> 99 | | | | | | | | | + public_schema_exists @lib/runscripts/table.run:102 -> lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:421 -> 412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/util.sh:560 -> lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd inputs/VegBank/observation__community \set schema "public" \set table "observation__community" \set table_str '''"observation__community"''' SET search_path TO "public"; Time: 0.093 ms CREATE TEMP TABLE "public".t () Time: 0.294 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.110s user 0m0.035s sys 0m0.042s | | | | | | | | | + test_ @lib/runscripts/table.run:104 -> 97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation__community/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation__community/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "observation__community/VegBIEN.csv" ln: observation__community/VegBIEN.csv: File exists make: [observation__community/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/observation__community/test.xml... >observation__community/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=observation__community out_database=vegbien source=VegBank.new ../../map observation__community/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.561314 sec/2 row(s) = 281. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change observation__community/test.xml.ref observation__community/test.xml real 0m1.958s user 0m1.015s sys 0m0.415s real 0m6.907s user 0m2.913s sys 0m1.996s | | | | + export_ @lib/runscripts/import.run:33 -> 24 real 4m5.293s user 1m11.690s sys 0m49.292s $ Wed 23:29:14 aaronmk@frenzy:~/Dropbox/svn