inputs/VegBank/run + inputs/VegBank/run | + main @lib/runscripts/util.run:151 | | + all @lib/runscripts/install.run:23 | | | + install @lib/runscripts/import.run:29 | | | | + import @lib/runscripts/datasrc_dir.run:29 | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.238 ms CREATE TEMP TABLE "VegBank".t () Time: 0.320 ms psql:/dev/fd/40:4: ERROR: cannot create temporary relation in non-temporary schema real 0m0.367s user 0m0.035s sys 0m0.041s | | | | | + inputs/VegBank/Source/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/Source/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.231 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.278 ms 0.06 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.958s user 0m0.381s sys 0m0.213s real 0m0.055s user 0m0.022s sys 0m0.025s : : : : : : : : : table Source already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank Source/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile Source/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.989s user 0m0.632s sys 0m0.316s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.217 ms \cd inputs/VegBank/Source \set schema "VegBank" \set table "Source" \set table_str '''"Source"''' SET search_path TO "VegBank"; Time: 0.086 ms SELECT util.reset_map_table('"~Source.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~Source.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~Source.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~Source.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 182.045 ms ALTER TABLE "~Source.map" DISABLE TRIGGER map_filter_insert; Time: 0.431 ms \copy "~Source.map" FROM 'map.csv' CSV HEADER; Time: 0.640 ms SELECT util.set_col_names_with_metadata('"Source"', '"~Source.map"'); psql:/dev/fd/40:12: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; ALTER TABLE "Source" ADD COLUMN "sourceType" text NOT NULL DEFAULT 'aggregator'; COMMENT ON COLUMN "Source"."sourceType" IS ' constant '; psql:/dev/fd/40:12: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; ALTER TABLE "Source" ADD COLUMN "observationGranularity" text NOT NULL DEFAULT 'plot'; COMMENT ON COLUMN "Source"."observationGranularity" IS ' constant '; set_col_names_with_metadata ----------------------------- (1 row) Time: 39.246 ms real 0m0.287s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.202 ms \cd schemas/VegCore \set schema "VegBank" \set table "Source" \set table_str '''"Source"''' SET search_path TO "VegBank"; Time: 0.147 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.554 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('("""Source""",cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.537 ms real 0m0.086s user 0m0.027s sys 0m0.029s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.206 ms \cd inputs/VegBank/Source \set schema "public" \set table "Source" \set table_str '''"Source"''' SET search_path TO "public"; Time: 0.099 ms CREATE TEMP TABLE "public".t () Time: 0.296 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.081s user 0m0.035s sys 0m0.039s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank Source/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile Source/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "Source/VegBIEN.csv" ln: Source/VegBIEN.csv: File exists make: [Source/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/Source/test.xml... >Source/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=Source out_database=vegbien source=VegBank.new ../../map Source/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 1 input rows Took 0:00:00.382582 sec/1 row(s) = 383. ms/row Encountered 0 error(s)/1 row(s) = 0.0% + diff --unified=2 --ignore-space-change Source/test.xml.ref Source/test.xml real 0m1.925s user 0m0.964s sys 0m0.399s real 0m7.177s user 0m2.891s sys 0m1.958s | | | | | + inputs/VegBank/project/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/project/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.262 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.233 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.652s user 0m0.388s sys 0m0.209s real 0m0.061s user 0m0.024s sys 0m0.027s : : : : : : : : : table project already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank project/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile project/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.989s user 0m0.627s sys 0m0.316s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.215 ms \cd inputs/VegBank/project \set schema "VegBank" \set table "project" \set table_str '''"project"''' SET search_path TO "VegBank"; Time: 0.085 ms SELECT util.reset_map_table('"~project.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~project.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~project.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~project.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 126.000 ms ALTER TABLE "~project.map" DISABLE TRIGGER map_filter_insert; Time: 0.385 ms \copy "~project.map" FROM 'map.csv' CSV HEADER; Time: 0.688 ms SELECT util.set_col_names_with_metadata('"project"', '"~project.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.456 ms real 0m0.195s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.203 ms \cd schemas/VegCore \set schema "VegBank" \set table "project" \set table_str '''"project"''' SET search_path TO "VegBank"; Time: 0.082 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.628 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(project,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.587 ms real 0m0.085s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.212 ms \cd inputs/VegBank/project \set schema "public" \set table "project" \set table_str '''"project"''' SET search_path TO "public"; Time: 0.115 ms CREATE TEMP TABLE "public".t () Time: 0.323 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.086s user 0m0.036s sys 0m0.040s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank project/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile project/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "project/VegBIEN.csv" ln: project/VegBIEN.csv: File exists make: [project/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/project/test.xml... >project/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=project out_database=vegbien source=VegBank.new ../../map project/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.379552 sec/2 row(s) = 190. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change project/test.xml.ref project/test.xml real 0m1.720s user 0m1.018s sys 0m0.400s real 0m6.428s user 0m2.948s sys 0m1.959s | | | | | + inputs/VegBank/plot/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/plot/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.271 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.227 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.622s user 0m0.391s sys 0m0.209s real 0m0.056s user 0m0.023s sys 0m0.025s : : : : : : : : : table plot already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.985s user 0m0.625s sys 0m0.319s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.217 ms \cd inputs/VegBank/plot \set schema "VegBank" \set table "plot" \set table_str '''"plot"''' SET search_path TO "VegBank"; Time: 0.086 ms SELECT util.reset_map_table('"~plot.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~plot.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~plot.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~plot.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 56.224 ms ALTER TABLE "~plot.map" DISABLE TRIGGER map_filter_insert; Time: 0.453 ms \copy "~plot.map" FROM 'map.csv' CSV HEADER; Time: 1.431 ms SELECT util.set_col_names_with_metadata('"plot"', '"~plot.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.966 ms real 0m0.126s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place plot/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m1.023s user 0m0.715s sys 0m0.227s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.276 ms \cd inputs/VegBank/plot \set schema "VegBank" \set table "plot" \set table_str '''"plot"''' SET search_path TO "VegBank"; Time: 0.086 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.286 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("locationID")$$); -- runtime: 0.2 s ("Time: 238.474 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "plot" ADD PRIMARY KEY ("locationID") create_if_not_exists ---------------------- (1 row) Time: 3.575 ms -- remove institutions that we have direct data for -- CVS DELETE FROM :table WHERE "locationName" IN (SELECT "locationName" FROM "CVS".plot_) -- removes 2323 of 7079 CVS plots (75340 - 73017) ; -- runtime: 1 min ("48999.044 ms") @starscream Time: 39.655 ms -- map_*() derived cols -- runtime: 2 s @starscream CREATE OR REPLACE FUNCTION map_access_rights(value integer) RETURNS text AS $BODY$ /* "closed List of Values: Value Description Sorting 0 Public 1 1 1 km radius (nearest 0.01 degree) 2 2 10 km radius (nearest 0.1 degree) 3 3 100 km radius (nearest degree) 4 4 Location embargo 5 5 Public embargo on data 6 6 Full embargo on data 7 " (http://vegbank.org/vegbank/views/dba_fielddescription_detail.jsp?view=detail&wparam=1581&entity=dba_fielddescription) */ SELECT util._map('0=>public, 1=>public, 2=>public, 3=>public, 4=>public, 5=>"data by request", 6=>private', $1::text) $BODY$ LANGUAGE sql IMMUTABLE COST 100; Time: 0.789 ms SELECT mk_derived_col((:table_str, 'accessRights'), $$map_access_rights("accessRights_verbatim")$$); -- runtime: 1 s ("Time: 1167.547 ms") @starscream psql:/dev/fd/40:46: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(map_access_rights("accessRights_verbatim")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.28 rows=1 width=4) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:46: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN "accessRights" text; ALTER TABLE plot ALTER COLUMN "accessRights" TYPE text USING map_access_rights("accessRights_verbatim"); mk_derived_col ---------------- (1 row) Time: 5.084 ms CREATE OR REPLACE FUNCTION map_fuzzing(value integer) RETURNS double precision AS $BODY$ /* "closed List of Values: Value Description Sorting 0 Public 1 1 1 km radius (nearest 0.01 degree) 2 2 10 km radius (nearest 0.1 degree) 3 3 100 km radius (nearest degree) 4 4 Location embargo 5 5 Public embargo on data 6 6 Full embargo on data 7 " (http://vegbank.org/vegbank/views/dba_fielddescription_detail.jsp?view=detail&wparam=1581&entity=dba_fielddescription) */ SELECT util._km_to_m(util._map('0=>NULL, 1=>1, 2=>10, 3=>100, 4=>NULL, 5=>NULL, 6=>NULL', $1::text)::double precision) $BODY$ LANGUAGE sql IMMUTABLE COST 100; Time: 0.372 ms SELECT mk_derived_col((:table_str, 'coordinateUncertaintyInMeters__from_fuzzing'), $$map_fuzzing("accessRights_verbatim")$$); -- runtime: 1 s ("Time: 1064.189 ms") @starscream psql:/dev/fd/40:68: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(map_fuzzing("accessRights_verbatim")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.29 rows=1 width=4) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:68: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN "coordinateUncertaintyInMeters__from_fuzzing" double precision; ALTER TABLE plot ALTER COLUMN "coordinateUncertaintyInMeters__from_fuzzing" TYPE double precision USING map_fuzzing("accessRights_verbatim"); mk_derived_col ---------------- (1 row) Time: 4.356 ms SELECT mk_derived_col((:table_str, 'coordinateUncertaintyInMeters'), $$GREATEST("coordinateUncertaintyInMeters__from_fuzzing", "coordinateUncertaintyInMeters__from_coords")$$) ; -- runtime: 0.5 s ("Time: 586.453 ms") @starscream psql:/dev/fd/40:73: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(GREATEST("coordinateUncertaintyInMeters__from_fuzzing", "coordinateUncertaintyInMeters__from_coords")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.02 rows=1 width=16) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:73: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN "coordinateUncertaintyInMeters" double precision; ALTER TABLE plot ALTER COLUMN "coordinateUncertaintyInMeters" TYPE double precision USING GREATEST("coordinateUncertaintyInMeters__from_fuzzing", "coordinateUncertaintyInMeters__from_coords"); mk_derived_col ---------------- (1 row) Time: 3.821 ms SELECT mk_derived_col((:table_str, 'locality'), $$_join("directions_to_place", "site_name")$$) ; -- runtime: 4 s ("Time: 3651.355 ms") @starscream psql:/dev/fd/40:77: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join("directions_to_place", "site_name")) FROM (SELECT (NULL::plot).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=450) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:77: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE plot ADD COLUMN locality text; ALTER TABLE plot ALTER COLUMN locality TYPE text USING _join("directions_to_place", "site_name"); mk_derived_col ---------------- (1 row) Time: 4.151 ms real 0m0.143s user 0m0.027s sys 0m0.029s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.205 ms \cd schemas/VegCore \set schema "VegBank" \set table "plot" \set table_str '''"plot"''' SET search_path TO "VegBank"; Time: 0.100 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.638 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(plot,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.597 ms real 0m0.085s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.218 ms \cd inputs/VegBank/plot \set schema "public" \set table "plot" \set table_str '''"plot"''' SET search_path TO "public"; Time: 0.089 ms CREATE TEMP TABLE "public".t () Time: 0.338 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.088s user 0m0.036s sys 0m0.043s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "plot/VegBIEN.csv" ln: plot/VegBIEN.csv: File exists make: [plot/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/plot/test.xml... >plot/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=plot out_database=vegbien source=VegBank.new ../../map plot/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:00.637779 sec/2 row(s) = 319. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change plot/test.xml.ref plot/test.xml real 0m1.979s user 0m1.225s sys 0m0.411s real 0m8.123s user 0m3.962s sys 0m2.315s | | | | | + inputs/VegBank/plot_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/plot_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.195 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.201 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.774s user 0m0.388s sys 0m0.211s real 0m0.054s user 0m0.023s sys 0m0.025s : : : : : : : : : table plot_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.985s user 0m0.625s sys 0m0.320s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd inputs/VegBank/plot_ \set schema "VegBank" \set table "plot_" \set table_str '''"plot_"''' SET search_path TO "VegBank"; Time: 0.096 ms SELECT util.reset_map_table('"~plot_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~plot_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~plot_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~plot_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 149.798 ms ALTER TABLE "~plot_.map" DISABLE TRIGGER map_filter_insert; Time: 0.390 ms \copy "~plot_.map" FROM 'map.csv' CSV HEADER; Time: 1.720 ms SELECT util.set_col_names_with_metadata('"plot_"', '"~plot_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.683 ms real 0m0.219s user 0m0.028s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.263 ms \cd schemas/VegCore \set schema "VegBank" \set table "plot_" \set table_str '''"plot_"''' SET search_path TO "VegBank"; Time: 0.092 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.318 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(plot_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.370 ms real 0m0.086s user 0m0.027s sys 0m0.029s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.217 ms \cd inputs/VegBank/plot_ \set schema "public" \set table "plot_" \set table_str '''"plot_"''' SET search_path TO "public"; Time: 0.089 ms CREATE TEMP TABLE "public".t () Time: 0.323 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.082s user 0m0.035s sys 0m0.039s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plot_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plot_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "plot_/VegBIEN.csv" ln: plot_/VegBIEN.csv: File exists make: [plot_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/plot_/test.xml... >plot_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=plot_ out_database=vegbien source=VegBank.new ../../map plot_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:00.662433 sec/2 row(s) = 331. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change plot_/test.xml.ref plot_/test.xml real 0m2.008s user 0m1.235s sys 0m0.414s real 0m6.905s user 0m3.162s sys 0m1.984s | | | | | + inputs/VegBank/observation_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/observation_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.217 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.243 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.615s user 0m0.381s sys 0m0.203s real 0m0.054s user 0m0.023s sys 0m0.024s : : : : : : : : : table observation_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.977s user 0m0.611s sys 0m0.311s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.255 ms \cd inputs/VegBank/observation_ \set schema "VegBank" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "VegBank"; Time: 0.094 ms SELECT util.reset_map_table('"~observation_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~observation_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~observation_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~observation_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 51.857 ms ALTER TABLE "~observation_.map" DISABLE TRIGGER map_filter_insert; Time: 0.446 ms \copy "~observation_.map" FROM 'map.csv' CSV HEADER; Time: 3.647 ms SELECT util.set_col_names_with_metadata('"observation_"', '"~observation_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.979 ms real 0m0.123s user 0m0.027s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place observation_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.892s user 0m0.636s sys 0m0.219s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.204 ms \cd inputs/VegBank/observation_ \set schema "VegBank" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "VegBank"; Time: 0.101 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.763 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("eventID")$$); -- runtime: ~0 ("Time: 1.838 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "observation_" ADD PRIMARY KEY ("eventID") create_if_not_exists ---------------------- (1 row) Time: 3.626 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("locationID")$$); -- runtime: ~0 ("158.792 ms") @starscream psql:/dev/fd/40:13: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "observation___parent" ON "observation_" ("locationID") create_if_not_exists ---------------------- (1 row) Time: 2.682 ms real 0m0.091s user 0m0.028s sys 0m0.029s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.267 ms \cd schemas/VegCore \set schema "VegBank" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "VegBank"; Time: 0.091 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.262 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(observation_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.590 ms real 0m0.087s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.256 ms \cd inputs/VegBank/observation_ \set schema "public" \set table "observation_" \set table_str '''"observation_"''' SET search_path TO "public"; Time: 0.104 ms CREATE TEMP TABLE "public".t () Time: 0.294 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.083s user 0m0.035s sys 0m0.039s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "observation_/VegBIEN.csv" ln: observation_/VegBIEN.csv: File exists make: [observation_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/observation_/test.xml... >observation_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=observation_ out_database=vegbien source=VegBank.new ../../map observation_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:00.580399 sec/2 row(s) = 290. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change observation_/test.xml.ref observation_/test.xml real 0m1.931s user 0m1.169s sys 0m0.410s real 0m7.852s user 0m3.805s sys 0m2.283s | | | | | + inputs/VegBank/stratumtype/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stratumtype/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.211 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.278 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.622s user 0m0.391s sys 0m0.208s real 0m0.056s user 0m0.023s sys 0m0.025s : : : : : : : : : table stratumtype already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratumtype/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratumtype/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.040s user 0m0.621s sys 0m0.313s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.215 ms \cd inputs/VegBank/stratumtype \set schema "VegBank" \set table "stratumtype" \set table_str '''"stratumtype"''' SET search_path TO "VegBank"; Time: 0.115 ms SELECT util.reset_map_table('"~stratumtype.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stratumtype.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stratumtype.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stratumtype.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 35.343 ms ALTER TABLE "~stratumtype.map" DISABLE TRIGGER map_filter_insert; Time: 0.330 ms \copy "~stratumtype.map" FROM 'map.csv' CSV HEADER; Time: 0.528 ms SELECT util.set_col_names_with_metadata('"stratumtype"', '"~stratumtype.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.572 ms real 0m0.142s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.198 ms \cd schemas/VegCore \set schema "VegBank" \set table "stratumtype" \set table_str '''"stratumtype"''' SET search_path TO "VegBank"; Time: 0.087 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.413 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stratumtype,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.653 ms real 0m0.085s user 0m0.027s sys 0m0.027s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd inputs/VegBank/stratumtype \set schema "public" \set table "stratumtype" \set table_str '''"stratumtype"''' SET search_path TO "public"; Time: 0.092 ms CREATE TEMP TABLE "public".t () Time: 0.290 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.082s user 0m0.035s sys 0m0.040s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratumtype/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratumtype/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stratumtype/VegBIEN.csv" ln: stratumtype/VegBIEN.csv: File exists make: [stratumtype/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stratumtype/test.xml... >stratumtype/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stratumtype out_database=vegbien source=VegBank.new ../../map stratumtype/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.381264 sec/2 row(s) = 191. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stratumtype/test.xml.ref stratumtype/test.xml real 0m1.756s user 0m1.015s sys 0m0.403s real 0m6.480s user 0m2.936s sys 0m1.951s | | | | | + inputs/VegBank/stratum/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stratum/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.203 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.266 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.622s user 0m0.391s sys 0m0.210s real 0m0.056s user 0m0.023s sys 0m0.025s : : : : : : : : : table stratum already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratum/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratum/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.208s user 0m0.616s sys 0m0.312s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd inputs/VegBank/stratum \set schema "VegBank" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "VegBank"; Time: 0.081 ms SELECT util.reset_map_table('"~stratum.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stratum.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stratum.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stratum.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 64.578 ms ALTER TABLE "~stratum.map" DISABLE TRIGGER map_filter_insert; Time: 0.313 ms \copy "~stratum.map" FROM 'map.csv' CSV HEADER; Time: 30.883 ms SELECT util.set_col_names_with_metadata('"stratum"', '"~stratum.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.496 ms real 0m0.160s user 0m0.027s sys 0m0.027s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratum/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place stratum/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.909s user 0m0.647s sys 0m0.223s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.200 ms \cd inputs/VegBank/stratum \set schema "VegBank" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "VegBank"; Time: 0.098 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.754 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("*stratum_id")$$); -- runtime: ~0 ("Time: 2.594 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "stratum" ADD PRIMARY KEY ("*stratum_id") create_if_not_exists ---------------------- (1 row) Time: 4.229 ms real 0m0.088s user 0m0.028s sys 0m0.030s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.203 ms \cd schemas/VegCore \set schema "VegBank" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "VegBank"; Time: 0.095 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.109 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stratum,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 5.274 ms real 0m0.086s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.214 ms \cd inputs/VegBank/stratum \set schema "public" \set table "stratum" \set table_str '''"stratum"''' SET search_path TO "public"; Time: 0.113 ms CREATE TEMP TABLE "public".t () Time: 0.311 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.083s user 0m0.035s sys 0m0.039s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stratum/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stratum/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stratum/VegBIEN.csv" ln: stratum/VegBIEN.csv: File exists make: [stratum/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stratum/test.xml... >stratum/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stratum out_database=vegbien source=VegBank.new ../../map stratum/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/map:296: UserWarning: Map warning: No mappings or no column name matches. Are you importing the correct input table? warnings.warn(UserWarning('Map warning: No mappings or no ' Processed 2 input rows Took 0:00:00.294181 sec/2 row(s) = 147. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stratum/test.xml.ref stratum/test.xml real 0m1.809s user 0m0.966s sys 0m0.399s real 0m8.001s user 0m3.623s sys 0m2.279s | | | | | + inputs/VegBank/plantconcept_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/plantconcept_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.200 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.208 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.625s user 0m0.393s sys 0m0.209s real 0m0.058s user 0m0.023s sys 0m0.026s : : : : : : : : : table plantconcept_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plantconcept_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plantconcept_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.984s user 0m0.626s sys 0m0.316s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.264 ms \cd inputs/VegBank/plantconcept_ \set schema "VegBank" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "VegBank"; Time: 0.085 ms SELECT util.reset_map_table('"~plantconcept_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~plantconcept_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~plantconcept_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~plantconcept_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 47.005 ms ALTER TABLE "~plantconcept_.map" DISABLE TRIGGER map_filter_insert; Time: 0.335 ms \copy "~plantconcept_.map" FROM 'map.csv' CSV HEADER; Time: 28.027 ms SELECT util.set_col_names_with_metadata('"plantconcept_"', '"~plantconcept_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 4.097 ms real 0m0.145s user 0m0.028s sys 0m0.031s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plantconcept_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place plantconcept_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.902s user 0m0.644s sys 0m0.222s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.261 ms \cd inputs/VegBank/plantconcept_ \set schema "VegBank" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "VegBank"; Time: 0.090 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.393 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("plantconcept_id")$$); -- runtime: 1 s ("Time: 675.125 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "plantconcept_" ADD PRIMARY KEY ("plantconcept_id") create_if_not_exists ---------------------- (1 row) Time: 3.494 ms real 0m0.087s user 0m0.028s sys 0m0.030s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.194 ms \cd schemas/VegCore \set schema "VegBank" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "VegBank"; Time: 0.091 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.365 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(plantconcept_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.475 ms real 0m0.086s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.242 ms \cd inputs/VegBank/plantconcept_ \set schema "public" \set table "plantconcept_" \set table_str '''"plantconcept_"''' SET search_path TO "public"; Time: 0.086 ms CREATE TEMP TABLE "public".t () Time: 0.353 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.081s user 0m0.034s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank plantconcept_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile plantconcept_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "plantconcept_/VegBIEN.csv" ln: plantconcept_/VegBIEN.csv: File exists make: [plantconcept_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/plantconcept_/test.xml... >plantconcept_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=plantconcept_ out_database=vegbien source=VegBank.new ../../map plantconcept_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.782893 sec/2 row(s) = 391. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change plantconcept_/test.xml.ref plantconcept_/test.xml real 0m2.167s user 0m1.340s sys 0m0.419s real 0m8.122s user 0m4.009s sys 0m2.309s | | | | | + inputs/VegBank/taxonobservation_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/taxonobservation_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.197 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.252 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.648s user 0m0.390s sys 0m0.210s real 0m0.056s user 0m0.023s sys 0m0.025s : : : : : : : : : table taxonobservation_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxonobservation_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile taxonobservation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.990s user 0m0.628s sys 0m0.319s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.192 ms \cd inputs/VegBank/taxonobservation_ \set schema "VegBank" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "VegBank"; Time: 0.095 ms SELECT util.reset_map_table('"~taxonobservation_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~taxonobservation_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~taxonobservation_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~taxonobservation_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 54.468 ms ALTER TABLE "~taxonobservation_.map" DISABLE TRIGGER map_filter_insert; Time: 0.335 ms \copy "~taxonobservation_.map" FROM 'map.csv' CSV HEADER; Time: 43.374 ms SELECT util.set_col_names_with_metadata('"taxonobservation_"', '"~taxonobservation_.map"'); psql:/dev/fd/40:12: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; ALTER TABLE taxonobservation_ ADD COLUMN "observationGranularity" text NOT NULL DEFAULT 'plot'; COMMENT ON COLUMN taxonobservation_."observationGranularity" IS ' constant '; set_col_names_with_metadata ----------------------------- (1 row) Time: 7.136 ms real 0m0.169s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxonobservation_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place taxonobservation_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.912s user 0m0.656s sys 0m0.224s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.212 ms \cd inputs/VegBank/taxonobservation_ \set schema "VegBank" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "VegBank"; Time: 0.092 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.637 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("eventID")$$); -- runtime: 2 s ("2416.676 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "taxonobservation___parent" ON "taxonobservation_" ("eventID") create_if_not_exists ---------------------- (1 row) Time: 3.658 ms SELECT mk_derived_col((:table_str, 'identifiedBy'), $$_join_words("identifiedBy__first", "identifiedBy__middle", "identifiedBy__last")$$) ; -- runtime: 30 s ("29290.701 ms") @starscream psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join_words("identifiedBy__first", "identifiedBy__middle", "identifiedBy__last")) FROM (SELECT (NULL::taxonobservation_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=96) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE taxonobservation_ ADD COLUMN "identifiedBy" text; ALTER TABLE taxonobservation_ ALTER COLUMN "identifiedBy" TYPE text USING _join_words("identifiedBy__first", "identifiedBy__middle", "identifiedBy__last"); mk_derived_col ---------------- (1 row) Time: 5.398 ms real 0m0.094s user 0m0.028s sys 0m0.030s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.252 ms \cd schemas/VegCore \set schema "VegBank" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "VegBank"; Time: 0.090 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.871 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(taxonobservation_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 35.042 ms real 0m0.116s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.247 ms \cd inputs/VegBank/taxonobservation_ \set schema "public" \set table "taxonobservation_" \set table_str '''"taxonobservation_"''' SET search_path TO "public"; Time: 0.093 ms CREATE TEMP TABLE "public".t () Time: 0.299 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.082s user 0m0.034s sys 0m0.039s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxonobservation_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile taxonobservation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "taxonobservation_/VegBIEN.csv" ln: taxonobservation_/VegBIEN.csv: File exists make: [taxonobservation_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/taxonobservation_/test.xml... >taxonobservation_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=taxonobservation_ out_database=vegbien source=VegBank.new ../../map taxonobservation_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:01.397512 sec/2 row(s) = 699. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change taxonobservation_/test.xml.ref taxonobservation_/test.xml real 0m2.808s user 0m1.529s sys 0m0.421s real 0m8.874s user 0m4.213s sys 0m2.327s | | | | | + inputs/VegBank/stemcount_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stemcount_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.201 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.209 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.647s user 0m0.393s sys 0m0.209s real 0m0.054s user 0m0.023s sys 0m0.024s : : : : : : : : : table stemcount_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemcount_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemcount_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.996s user 0m0.634s sys 0m0.321s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.253 ms \cd inputs/VegBank/stemcount_ \set schema "VegBank" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "VegBank"; Time: 0.145 ms SELECT util.reset_map_table('"~stemcount_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stemcount_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stemcount_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stemcount_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 111.474 ms ALTER TABLE "~stemcount_.map" DISABLE TRIGGER map_filter_insert; Time: 0.333 ms \copy "~stemcount_.map" FROM 'map.csv' CSV HEADER; Time: 44.278 ms SELECT util.set_col_names_with_metadata('"stemcount_"', '"~stemcount_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.721 ms real 0m0.222s user 0m0.028s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemcount_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place stemcount_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.919s user 0m0.654s sys 0m0.227s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.201 ms \cd inputs/VegBank/stemcount_ \set schema "VegBank" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "VegBank"; Time: 0.091 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.377 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("taxonOccurrenceID__overall_plot")$$); -- runtime: 2 s ("Time: 1766.315 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "stemcount___parent" ON "stemcount_" ("taxonOccurrenceID__overall_plot") create_if_not_exists ---------------------- (1 row) Time: 3.516 ms SELECT mk_derived_col((:table_str, 'aggregateOrganismObservationID'), $$_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)$$) ; -- runtime: 15 s ("Time: 15805.876 ms") @starscream psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)) FROM (SELECT (NULL::stemcount_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.28 rows=1 width=8) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:16: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE stemcount_ ADD COLUMN "aggregateOrganismObservationID" text; ALTER TABLE stemcount_ ALTER COLUMN "aggregateOrganismObservationID" TYPE text USING _join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text); mk_derived_col ---------------- (1 row) Time: 5.039 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("aggregateOrganismObservationID")$$); -- runtime: 20 s ("Time: 20194.932 ms") @starscream psql:/dev/fd/40:19: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "stemcount_" ADD PRIMARY KEY ("aggregateOrganismObservationID") create_if_not_exists ---------------------- (1 row) Time: 2.456 ms real 0m0.095s user 0m0.028s sys 0m0.030s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.192 ms \cd schemas/VegCore \set schema "VegBank" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "VegBank"; Time: 0.149 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.566 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stemcount_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.701 ms real 0m0.085s user 0m0.027s sys 0m0.027s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.200 ms \cd inputs/VegBank/stemcount_ \set schema "public" \set table "stemcount_" \set table_str '''"stemcount_"''' SET search_path TO "public"; Time: 0.091 ms CREATE TEMP TABLE "public".t () Time: 0.299 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.082s user 0m0.034s sys 0m0.039s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemcount_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemcount_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stemcount_/VegBIEN.csv" ln: stemcount_/VegBIEN.csv: File exists make: [stemcount_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stemcount_/test.xml... >stemcount_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stemcount_ out_database=vegbien source=VegBank.new ../../map stemcount_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.470015 sec/2 row(s) = 235. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stemcount_/test.xml.ref stemcount_/test.xml real 0m1.839s user 0m1.097s sys 0m0.407s real 0m7.948s user 0m3.783s sys 0m2.301s | | | | | + inputs/VegBank/stemlocation_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/stemlocation_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.200 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.210 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.620s user 0m0.382s sys 0m0.204s real 0m0.056s user 0m0.023s sys 0m0.025s : : : : : : : : : table stemlocation_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemlocation_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemlocation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.990s user 0m0.629s sys 0m0.322s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.196 ms \cd inputs/VegBank/stemlocation_ \set schema "VegBank" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "VegBank"; Time: 0.083 ms SELECT util.reset_map_table('"~stemlocation_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~stemlocation_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~stemlocation_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~stemlocation_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 45.949 ms ALTER TABLE "~stemlocation_.map" DISABLE TRIGGER map_filter_insert; Time: 0.363 ms \copy "~stemlocation_.map" FROM 'map.csv' CSV HEADER; Time: 12.929 ms SELECT util.set_col_names_with_metadata('"stemlocation_"', '"~stemlocation_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.562 ms real 0m0.129s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemlocation_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place stemlocation_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.921s user 0m0.667s sys 0m0.223s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.193 ms \cd inputs/VegBank/stemlocation_ \set schema "VegBank" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "VegBank"; Time: 0.095 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.262 ms SELECT create_if_not_exists($$ALTER TABLE $$||:table_str||$$ ADD PRIMARY KEY ("stemID")$$); -- runtime: 0.1 s ("Time: 74.164 ms") @starscream psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE "stemlocation_" ADD PRIMARY KEY ("stemID") create_if_not_exists ---------------------- (1 row) Time: 3.625 ms SELECT mk_derived_col((:table_str, 'individualID'), $$"stemID"$$) ; -- runtime: ~0 ("Time: 1.562 ms") @starscream psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof("stemID") FROM (SELECT (NULL::stemlocation_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.02 rows=1 width=4) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE stemlocation_ ADD COLUMN "individualID" integer; ALTER TABLE stemlocation_ ALTER COLUMN "individualID" TYPE integer USING "stemID"; mk_derived_col ---------------- (1 row) Time: 4.739 ms SELECT mk_derived_col((:table_str, 'aggregateOrganismObservationID'), $$_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)$$) ; -- runtime: 0.5 s ("Time: 441.059 ms") @starscream psql:/dev/fd/40:18: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text)) FROM (SELECT (NULL::stemlocation_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.28 rows=1 width=8) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:18: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE stemlocation_ ADD COLUMN "aggregateOrganismObservationID" text; ALTER TABLE stemlocation_ ALTER COLUMN "aggregateOrganismObservationID" TYPE text USING _join("aggregateOrganismObservationID__taxon_presence"::text, "aggregateOrganismObservationID__individual_count"::text); mk_derived_col ---------------- (1 row) Time: 4.073 ms SELECT create_if_not_exists($$CREATE INDEX $$||:table_str||%'__parent'||$$ ON $$||:table_str||$$ ("aggregateOrganismObservationID")$$); -- runtime: 0.1 s ("Time: 131.907 ms") @starscream psql:/dev/fd/40:21: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; CREATE INDEX "stemlocation___parent" ON "stemlocation_" ("aggregateOrganismObservationID") create_if_not_exists ---------------------- (1 row) Time: 2.353 ms real 0m0.100s user 0m0.029s sys 0m0.031s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.206 ms \cd schemas/VegCore \set schema "VegBank" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "VegBank"; Time: 0.089 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.426 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(stemlocation_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.495 ms real 0m0.085s user 0m0.027s sys 0m0.029s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.197 ms \cd inputs/VegBank/stemlocation_ \set schema "public" \set table "stemlocation_" \set table_str '''"stemlocation_"''' SET search_path TO "public"; Time: 0.081 ms CREATE TEMP TABLE "public".t () Time: 0.304 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.081s user 0m0.034s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank stemlocation_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile stemlocation_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "stemlocation_/VegBIEN.csv" ln: stemlocation_/VegBIEN.csv: File exists make: [stemlocation_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/stemlocation_/test.xml... >stemlocation_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=stemlocation_ out_database=vegbien source=VegBank.new ../../map stemlocation_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.478416 sec/2 row(s) = 239. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change stemlocation_/test.xml.ref stemlocation_/test.xml real 0m1.837s user 0m1.085s sys 0m0.408s real 0m7.840s user 0m3.777s sys 0m2.309s | | | | | + inputs/VegBank/taxon_observation.**/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxon_observation.**/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place taxon_observation.**/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.948s user 0m0.659s sys 0m0.221s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.268 ms \cd inputs/VegBank/taxon_observation.** \set schema "VegBank" \set table "taxon_observation.**" \set table_str '''"taxon_observation.**"''' SET search_path TO "VegBank"; Time: 0.095 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.121 ms SELECT force_update_view(:table_str, $$ /* no row_num because left-join to stemcount_, stemlocation_ adds rows to each taxonobservation_ */ SELECT "identificationID" AS sort_col, * , NULLIF(concat_ws('; ', "taxonOccurrenceID__overall_plot"::text, "aggregateOrganismObservationID"), '') AS "taxonOccurrenceID" FROM taxonobservation_ NATURAL LEFT JOIN stemcount_ NATURAL LEFT JOIN stemlocation_ NATURAL LEFT JOIN stratum NATURAL LEFT JOIN stratumtype NATURAL LEFT JOIN observation_ NATURAL LEFT JOIN plot_ NATURAL LEFT JOIN project $$); force_update_view ------------------- (1 row) Time: 76.379 ms SELECT mk_subset_by_row_num_func(:table_str); mk_subset_by_row_num_func --------------------------- (1 row) Time: 6.060 ms real 0m0.165s user 0m0.028s sys 0m0.029s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.251 ms \cd schemas/VegCore \set schema "VegBank" \set table "taxon_observation.**" \set table_str '''"taxon_observation.**"''' SET search_path TO "VegBank"; Time: 0.083 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.817 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('("""taxon_observation.**""",cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 7.919 ms real 0m0.088s user 0m0.026s sys 0m0.028s | | | | | | | | + builtin cd -P inputs/VegBank/taxon_observation.** | | | | | | | | | + rm -f header.csv | | | | | | | | + mappings__table_run @lib/sh/util.sh:299 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.199 ms \cd inputs/VegBank/taxon_observation.** \set schema "public" \set table "taxon_observation.**" \set table_str '''"taxon_observation.**"''' SET search_path TO "public"; Time: 0.101 ms CREATE TEMP TABLE "public".t () Time: 0.298 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.085s user 0m0.036s sys 0m0.041s | | | | | | | | | + builtin cd -P inputs/VegBank/taxon_observation.** | | | | | | | | | | + rm -f map.csv | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank taxon_observation.**/test make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make --makefile=../input.Makefile "taxon_observation.**/header.csv" set -o pipefail; \ echo 'COPY (SELECT * FROM "VegBank"."taxon_observation.**" LIMIT 0) TO STDOUT CSV HEADER;'|\ env no_search_path=1 ../../bin/psql_script_vegbien >taxon_observation.**/header.csv (. ../../mappings/root.sh; env datasrc=VegBank ../../bin/src_map taxon_observation.**/map.csv) /usr/bin/make --makefile=../input.Makefile taxon_observation.**/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ../../bin/in_place taxon_observation.**/map.csv ../../bin/canon 1 ../../mappings/VegCore.vocab.csv ../../bin/in_place taxon_observation.**/map.csv ../../bin/translate_ci 1 ../../mappings/VegCore.thesaurus.csv ../../bin/in_place taxon_observation.**/map.csv ../../bin/fix_line_endings touch taxon_observation.**/.map.csv.last_cleanup /usr/bin/make --makefile=../input.Makefile taxon_observation.**/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile taxon_observation.**/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 taxon_observation.**/map.csv|../../bin/cols 1|../../bin/filter_out_ci 0 ../../mappings/VegCore-VegBIEN.csv >taxon_observation.**/unmapped_terms.csv ../../bin/autoremove taxon_observation.**/unmapped_terms.csv /usr/bin/make --makefile=../input.Makefile taxon_observation.**/new_terms.csv /usr/bin/make --makefile=../input.Makefile taxon_observation.**/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[5]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[5]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[6]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[4]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[4]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[5]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. tail -n +2 taxon_observation.**/map.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.vocab.csv|../../bin/filter_out_ci 0 ../../mappings/VegCore.thesaurus.csv |../../bin/filter_out_ci 0 taxon_observation.**/unmapped_terms.csv |grep -vE -e '^"?:' -e 'UNUSED' >taxon_observation.**/new_terms.csv; exit 0 ../../bin/autoremove taxon_observation.**/new_terms.csv ln -s "../../../mappings/VegCore-VegBIEN.csv" "taxon_observation.**/VegBIEN.csv" ln: taxon_observation.**/VegBIEN.csv: File exists make: [taxon_observation.**/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/taxon_observation.**/test.xml... >taxon_observation.**/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=taxon_observation.** out_database=vegbien source=VegBank.new ../../map taxon_observation.**/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:18.335771 sec/2 row(s) = 9.17 s/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change 'taxon_observation.**/test.xml.ref' 'taxon_observation.**/test.xml' real 0m22.830s user 0m3.978s sys 0m1.174s real 0m26.324s user 0m5.503s sys 0m2.364s | | | | | + inputs/VegBank/^taxon_observation.**.sample/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/^taxon_observation.**.sample/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.259 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.255 ms 0.16 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.783s user 0m0.382s sys 0m0.206s real 0m0.079s user 0m0.022s sys 0m0.024s : : : : : : : : : table ^taxon_observation.**.sample already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank ^taxon_observation.**.sample/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile ^taxon_observation.**.sample/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.983s user 0m0.625s sys 0m0.319s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.225 ms \cd inputs/VegBank/^taxon_observation.**.sample \set schema "VegBank" \set table "^taxon_observation.**.sample" \set table_str '''"^taxon_observation.**.sample"''' SET search_path TO "VegBank"; Time: 0.088 ms SELECT util.reset_map_table('"~^taxon_observation.**.sample.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~^taxon_observation.**.sample.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~^taxon_observation.**.sample.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~^taxon_observation.**.sample.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 73.722 ms ALTER TABLE "~^taxon_observation.**.sample.map" DISABLE TRIGGER map_filter_insert; Time: 0.490 ms \copy "~^taxon_observation.**.sample.map" FROM 'map.csv' CSV HEADER; Time: 31.185 ms SELECT util.set_col_names_with_metadata('"^taxon_observation.**.sample"', '"~^taxon_observation.**.sample.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 27.213 ms real 0m0.197s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.218 ms \cd schemas/VegCore \set schema "VegBank" \set table "^taxon_observation.**.sample" \set table_str '''"^taxon_observation.**.sample"''' SET search_path TO "VegBank"; Time: 0.089 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.823 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('("""^taxon_observation.**.sample""",cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.719 ms real 0m0.085s user 0m0.027s sys 0m0.028s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.202 ms \cd inputs/VegBank/^taxon_observation.**.sample \set schema "public" \set table "^taxon_observation.**.sample" \set table_str '''"^taxon_observation.**.sample"''' SET search_path TO "public"; Time: 0.085 ms CREATE TEMP TABLE "public".t () Time: 0.298 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.081s user 0m0.034s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank ^taxon_observation.**.sample/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile ^taxon_observation.**.sample/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "^taxon_observation.**.sample/VegBIEN.csv" ln: ^taxon_observation.**.sample/VegBIEN.csv: File exists make: [^taxon_observation.**.sample/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/^taxon_observation.**.sample/test.xml... >^taxon_observation.**.sample/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=^taxon_observation.**.sample out_database=vegbien source=VegBank.new ../../map ^taxon_observation.**.sample/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien Processed 2 input rows Took 0:00:01.755708 sec/2 row(s) = 878. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change '^taxon_observation.**.sample/test.xml.ref' '^taxon_observation.**.sample/test.xml' real 0m3.390s user 0m2.014s sys 0m0.446s real 0m8.225s user 0m3.934s sys 0m1.995s | | | | | + inputs/VegBank/projectcontributor_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/projectcontributor_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.232 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.299 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.622s user 0m0.391s sys 0m0.206s real 0m0.055s user 0m0.023s sys 0m0.025s : : : : : : : : : table projectcontributor_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank projectcontributor_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile projectcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.987s user 0m0.628s sys 0m0.319s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.201 ms \cd inputs/VegBank/projectcontributor_ \set schema "VegBank" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "VegBank"; Time: 0.094 ms SELECT util.reset_map_table('"~projectcontributor_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~projectcontributor_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~projectcontributor_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~projectcontributor_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 68.467 ms ALTER TABLE "~projectcontributor_.map" DISABLE TRIGGER map_filter_insert; Time: 0.378 ms \copy "~projectcontributor_.map" FROM 'map.csv' CSV HEADER; Time: 20.103 ms SELECT util.set_col_names_with_metadata('"projectcontributor_"', '"~projectcontributor_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.447 ms real 0m0.157s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank projectcontributor_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place projectcontributor_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.920s user 0m0.643s sys 0m0.223s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.261 ms \cd inputs/VegBank/projectcontributor_ \set schema "VegBank" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "VegBank"; Time: 0.100 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.272 ms SELECT mk_derived_col((:table_str, 'project_participant'), $$_join_words(project_participant_name_first::text, project_participant_name_middle::text, project_participant_name_last::text)$$) ; -- runtime: ~0 ("2.887 ms") @starscream psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join_words(project_participant_name_first::text, project_participant_name_middle::text, project_participant_name_last::text)) FROM (SELECT (NULL::projectcontributor_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=96) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE projectcontributor_ ADD COLUMN project_participant text; ALTER TABLE projectcontributor_ ALTER COLUMN project_participant TYPE text USING _join_words(project_participant_name_first::text, project_participant_name_middle::text, project_participant_name_last::text); mk_derived_col ---------------- (1 row) Time: 6.068 ms real 0m0.087s user 0m0.027s sys 0m0.028s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.204 ms \cd schemas/VegCore \set schema "VegBank" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "VegBank"; Time: 0.089 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.693 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(projectcontributor_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 5.254 ms real 0m0.085s user 0m0.026s sys 0m0.027s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.233 ms \cd inputs/VegBank/projectcontributor_ \set schema "public" \set table "projectcontributor_" \set table_str '''"projectcontributor_"''' SET search_path TO "public"; Time: 0.128 ms CREATE TEMP TABLE "public".t () Time: 0.320 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.082s user 0m0.034s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank projectcontributor_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile projectcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "projectcontributor_/VegBIEN.csv" ln: projectcontributor_/VegBIEN.csv: File exists make: [projectcontributor_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/projectcontributor_/test.xml... >projectcontributor_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=projectcontributor_ out_database=vegbien source=VegBank.new ../../map projectcontributor_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.395493 sec/2 row(s) = 198. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change projectcontributor_/test.xml.ref projectcontributor_/test.xml real 0m1.782s user 0m1.037s sys 0m0.408s real 0m7.827s user 0m3.708s sys 0m2.304s | | | | | + inputs/VegBank/observationcontributor_/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/observationcontributor_/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.202 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.200 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.673s user 0m0.390s sys 0m0.205s real 0m0.052s user 0m0.022s sys 0m0.023s : : : : : : : : : table observationcontributor_ already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observationcontributor_/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observationcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m1.041s user 0m0.624s sys 0m0.315s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.245 ms \cd inputs/VegBank/observationcontributor_ \set schema "VegBank" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "VegBank"; Time: 0.101 ms SELECT util.reset_map_table('"~observationcontributor_.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~observationcontributor_.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~observationcontributor_.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~observationcontributor_.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 84.819 ms ALTER TABLE "~observationcontributor_.map" DISABLE TRIGGER map_filter_insert; Time: 0.386 ms \copy "~observationcontributor_.map" FROM 'map.csv' CSV HEADER; Time: 21.185 ms SELECT util.set_col_names_with_metadata('"observationcontributor_"', '"~observationcontributor_.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.339 ms real 0m0.173s user 0m0.028s sys 0m0.029s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observationcontributor_/postprocess.sql make: `import_order.txt' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[2]: `VegCore.thesaurus.csv' is up to date. ../../bin/in_place observationcontributor_/postprocess.sql env text=1 ../../bin/repl ../../mappings/VegCore.thesaurus.csv real 0m0.905s user 0m0.649s sys 0m0.222s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.204 ms \cd inputs/VegBank/observationcontributor_ \set schema "VegBank" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "VegBank"; Time: 0.083 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 19.066 ms SELECT mk_derived_col((:table_str, 'event__participant'), $$_join_words(event__participant__name__first::text, event__participant__name__middle::text, event__participant__name__last::text)$$) ; -- runtime: 1 s ("Time: 1265.039 ms") @starscream psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT pg_typeof(_join_words(event__participant__name__first::text, event__participant__name__middle::text, event__participant__name__last::text)) FROM (SELECT (NULL::observationcontributor_).*) _s EXPLAIN: Subquery Scan on _s (cost=0.00..0.27 rows=1 width=96) -> Result (cost=0.00..0.01 rows=1 width=0) psql:/dev/fd/40:14: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; ALTER TABLE observationcontributor_ ADD COLUMN event__participant text; ALTER TABLE observationcontributor_ ALTER COLUMN event__participant TYPE text USING _join_words(event__participant__name__first::text, event__participant__name__middle::text, event__participant__name__last::text); mk_derived_col ---------------- (1 row) Time: 6.067 ms real 0m0.112s user 0m0.028s sys 0m0.030s | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.219 ms \cd schemas/VegCore \set schema "VegBank" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "VegBank"; Time: 0.082 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 20.049 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(observationcontributor_,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.489 ms real 0m0.087s user 0m0.027s sys 0m0.029s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.207 ms \cd inputs/VegBank/observationcontributor_ \set schema "public" \set table "observationcontributor_" \set table_str '''"observationcontributor_"''' SET search_path TO "public"; Time: 0.102 ms CREATE TEMP TABLE "public".t () Time: 0.295 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.082s user 0m0.035s sys 0m0.039s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observationcontributor_/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observationcontributor_/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "observationcontributor_/VegBIEN.csv" ln: observationcontributor_/VegBIEN.csv: File exists make: [observationcontributor_/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/observationcontributor_/test.xml... >observationcontributor_/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=observationcontributor_ out_database=vegbien source=VegBank.new ../../map observationcontributor_/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.413368 sec/2 row(s) = 207. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change observationcontributor_/test.xml.ref observationcontributor_/test.xml real 0m1.784s user 0m1.055s sys 0m0.408s real 0m7.986s user 0m3.724s sys 0m2.288s | | | | | + inputs/VegBank/observation__community/run import | | | | | | + import @lib/runscripts/table.run:131 | | | | | | | + load_data @lib/runscripts/table.run:50 | | | | | | | | + /usr/bin/make --directory=inputs/VegBank/observation__community/.. schema make: `import_order.txt' is up to date. echo 'CREATE SCHEMA "VegBank";'|env no_search_path=1 ../../bin/psql_verbose_vegbien \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.213 ms CREATE SCHEMA "VegBank"; ERROR: schema "VegBank" already exists Time: 0.205 ms 0.03 real 0.01 user 0.01 sys make: [schema] Error 3 (ignored) real 0m0.622s user 0m0.393s sys 0m0.208s real 0m0.054s user 0m0.023s sys 0m0.025s : : : : : : : : : table observation__community already exists, skipping : : : : : : : : : to force-remake, prepend `rm=1` to the command | | | | | | | + postprocess @lib/runscripts/table.run:120 | | | | | | | | + map_table @lib/runscripts/table.run:77 | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation__community/map.csv make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation__community/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. real 0m0.984s user 0m0.626s sys 0m0.317s | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.214 ms \cd inputs/VegBank/observation__community \set schema "VegBank" \set table "observation__community" \set table_str '''"observation__community"''' SET search_path TO "VegBank"; Time: 0.092 ms SELECT util.reset_map_table('"~observation__community.map"'); psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; DROP TABLE IF EXISTS "~observation__community.map" psql:/dev/fd/40:9: NOTICE: SET /*LOCAL*/ search_path TO "VegBank"; CREATE TABLE "~observation__community.map" ( LIKE util.map INCLUDING ALL ); CREATE TRIGGER map_filter_insert BEFORE INSERT ON "~observation__community.map" FOR EACH ROW EXECUTE PROCEDURE util.map_filter_insert(); reset_map_table ----------------- (1 row) Time: 38.865 ms ALTER TABLE "~observation__community.map" DISABLE TRIGGER map_filter_insert; Time: 0.364 ms \copy "~observation__community.map" FROM 'map.csv' CSV HEADER; Time: 3.716 ms SELECT util.set_col_names_with_metadata('"observation__community"', '"~observation__community.map"'); set_col_names_with_metadata ----------------------------- (1 row) Time: 3.675 ms real 0m0.112s user 0m0.028s sys 0m0.030s | | | | | | | | + custom_postprocess @lib/runscripts/table.run:107 | | | | | | | | + mk_derived @lib/runscripts/table.run:117 | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.233 ms \cd schemas/VegCore \set schema "VegBank" \set table "observation__community" \set table_str '''"observation__community"''' SET search_path TO "VegBank"; Time: 0.137 ms SELECT util.search_path_append('util'); psql:/dev/fd/40:9: NOTICE: SET search_path TO "VegBank", util search_path_append -------------------- (1 row) Time: 18.965 ms SELECT try_mk_derived_col((:table_str, 'cultivated'), $$NOT "oldGrowth"$$); psql:/dev/fd/40:11: NOTICE: SET /*LOCAL*/ search_path TO "VegBank", util; SELECT util.mk_derived_col('(observation__community,cultivated)', 'NOT "oldGrowth"') try_mk_derived_col -------------------- (1 row) Time: 4.661 ms real 0m0.084s user 0m0.026s sys 0m0.027s | | | | | | | | + mappings @lib/runscripts/table.run:99 | | | | | | | | | + public_schema_exists @lib/sh/db.sh:421 | | | | | | | | | | + pg_schema_exists @lib/sh/db.sh:412 | | | | | | | | | | | + benign_does_not_exist_error psql @lib/sh/db.sh:405 | | | | | | | | | | | | + declare -x PGHOST="localhost" PGUSER="bien" PGDATABASE="vegbien" | | | | | | | | | | | | + psql --file /dev/fd/40 --set ON_ERROR_STOP=1 --quiet --echo-all --echo-hidden 40<&0 0<&20 41>&1 \set VERBOSITY terse \timing on SET client_min_messages = NOTICE; Time: 0.326 ms \cd inputs/VegBank/observation__community \set schema "public" \set table "observation__community" \set table_str '''"observation__community"''' SET search_path TO "public"; Time: 0.151 ms CREATE TEMP TABLE "public".t () Time: 0.302 ms psql:/dev/fd/40:9: ERROR: cannot create temporary relation in non-temporary schema real 0m0.080s user 0m0.035s sys 0m0.038s | | | | | | | | | + test_ @lib/runscripts/table.run:97 | | | | | | | | | | + /usr/bin/make --directory=inputs/VegBank observation__community/test make: `import_order.txt' is up to date. /usr/bin/make --makefile=../input.Makefile observation__community/.map.csv.last_cleanup /usr/bin/make "mappings/VegCore.vocab.csv" --directory=../.. /usr/bin/make VegCore.vocab.csv --directory=mappings make[3]: `VegCore.vocab.csv' is up to date. /usr/bin/make "mappings/VegCore.thesaurus.csv" --directory=../.. /usr/bin/make VegCore.thesaurus.csv --directory=mappings make[3]: `VegCore.thesaurus.csv' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[4]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. /usr/bin/make "mappings/VegCore-VegBIEN.csv" --directory=../.. /usr/bin/make VegCore-VegBIEN.csv --directory=mappings /usr/bin/make .VegCore-VegBIEN.csv.last_cleanup make[3]: `.VegCore-VegBIEN.csv.last_cleanup' is up to date. ln -s "../../../mappings/VegCore-VegBIEN.csv" "observation__community/VegBIEN.csv" ln: observation__community/VegBIEN.csv: File exists make: [observation__community/VegBIEN.csv] Error 1 (ignored) Testing /Users/aaronmk/Dropbox/docs/BIEN/svn/inputs/VegBank/observation__community/test.xml... >observation__community/test.xml env test=1 n=2 by_col= env in_database=vegbien in_schema=VegBank in_table=observation__community out_database=vegbien source=VegBank.new ../../map observation__community/VegBIEN.csv Processing input rows 0-1 Using 1 parallel CPUs Connecting to PostgreSQL database vegbien Connecting to PostgreSQL database vegbien ../../bin/../lib/sql_io.py:422: UserWarning: Inserting empty table row(s) warnings.warn(UserWarning('Inserting empty table row(s)')) Processed 2 input rows Took 0:00:00.431317 sec/2 row(s) = 216. ms/row Encountered 0 error(s)/2 row(s) = 0.0% + diff --unified=2 --ignore-space-change observation__community/test.xml.ref observation__community/test.xml real 0m1.803s user 0m1.060s sys 0m0.411s real 0m6.411s user 0m3.000s sys 0m1.970s | | | | + export_ @lib/runscripts/import.run:24 real 2m56.045s user 1m14.953s sys 0m51.230s $ Wed 16:55:21 aaronmk@frenzy:~/Dropbox/svn