##### Configuration

# Command line

continue ?=

debug ?=

full_import ?=

import_source ?= 1

log ?= $(if $(test),,1)

noclobber ?=

profile ?=

quiet ?=

reverify ?= 1

schema_only ?=

use_staged ?= $(by_col)

exts ?= csv tsv tab txt dat dmp xml

test_n ?= 2

# Paths

datasrc := $(patsubst .%,%,$(notdir $(realpath .)))

bin := $(root)/bin

# Make

SHELL := /bin/bash

selfMake = $(MAKE) --makefile=../input.Makefile

subMake = $(MAKE) $(@:$(root)/%=%) --directory=$(root)

+_ = $(+:_%=)

# used to filter the output of embedded $(shell make ...) invocations

addBeforeExt = $(basename $(2))$(1)$(suffix $(2))

# Terminal

termCols := $(shell tput cols)

# Commands

MKDIR = mkdir -p

diff = diff --unified=2

diffIgnoreSpace = $(diff) --ignore-space-change

diffVerbose = $(if $(verbose),diff --side-by-side --left-column\

# BIEN commands

sortFilenames = $(shell $(bin)/sort_filenames $(1))

selfMap = $(bin)/cols 0 0

psqlAsBien := $(bin)/psql_verbose_vegbien

psqlNoSearchPath := env no_search_path=1 $(psqlAsBien)

# Usage: ($(inDatasrc); cat $(file))|$(psqlCmd)

inDatasrc := echo 'SET search_path TO "$(datasrc)";'

# SVN

setSvnIgnore = svn propset svn:ignore $(2) $(1)

define addDirWithIgnore

$(setSvnIgnore)

endef

##### Environment

##### General targets

all: _always maps ;

clean: _always

	$(RM) $(all)

remake: _always clean

# Only remake if doesn't exist. This prevents unintentional remaking when the

%/: %/map.csv _always ;

%/: % _always ;

%: %.make

	$(make_script)

.PRECIOUS: % # save partial outputs of aborted src make scripts

##### Tables discovery

sortFile := import_order.txt

noImportFile := _no_import

dontImport = $(wildcard $(noImportFile))$(wildcard $(1)/$(noImportFile))$(if\

$(import_source),,$(filter Source,$(1)))

ifeq ($(sort_file_updated),)# keep $(sortFile) up-to-date

endif

sort_file_tables := $(if $(wildcard $(sortFile)),$(shell cat $(sortFile)))

    # $(shell) replaces "\n" with " "

tables := $(sort_file_tables)

allSubdirs := $(call wildcard/,*/)

has_visible_files = $(call wildcard/,$(1)/*)

ifeq ($(tables),)# none specified in sort file

list_tables: _always # use `make -s` to avoid echoing commands

$(sortFile): _always

	$(if $(filter-out $(sort_file_tables),$(tables)),$(selfMake) -s list_tables >$@)

##### SVN

svnFilesGlob:= */{,$(noImportFile),{,.}{data,map,VegBIEN}.csv{,.*},*header.*,*.sql,test.xml.ref}

svnFilesGlob := {map.csv,*{schema,grants,~}*.sql,{,*/}{*.make,*terms.csv},$(svnFilesGlob)}

_svnFilesGlob := {_MySQL/{,*schema*.sql,*.make},{,*/}{*{run,Makefile},*.{md5,url,pdf},*README.TXT}}

svnFiles = $(filter-out _% logs/% %.data.sql,$(call wildcard/,$(svnFilesGlob)))\

$(call wildcard/,$(_svnFilesGlob))

add: _always $(if $(call dontImport,.),,add!) ;

# To update all inputs with these settings: make inputs/add

add!: _always Source/add $(allTables:%=%/add)

	$(call setSvnIgnore,.,'*')

	$(call addDirWithIgnore,logs,$$'*.gz\n*.log.sql\n*.trace')

	$(call addDirWithIgnore,verify,$$'*.out\n*.csv\n*.xls\n*.xlsx')

	$(call addFile,import_order.txt)

	$(if $(wildcard _MySQL/),$(call addDirWithIgnore,_MySQL,'*'))

	$(if $(wildcard _src/),$(call addDirWithIgnore,_src,'*'))

	$(if $(wildcard _archive/),$(call addDirWithIgnore,_archive,'*'))

	$(call add*,$(svnFiles))

# Adds a new table subdir

	$(call addDirWithIgnore,$*/logs,$$'*.gz\n*.log.sql\n*.trace')

##### Existing maps discovery

anyMap := %/map.csv %/VegBIEN.csv %/unmapped_terms.csv %/new_terms.csv

exts := $(call ci,$(exts))

extsFilter := $(addprefix %.,$(exts))

anyTest = $*/test.%

srcsOnly = $(filter-out $(anyMap) $(anyTest) %/logs,$(call dataOnly,$(1)))

srcDict := map.csv

vocab := $(mappings)/VegCore.vocab.csv

coreMap := $(mappings)/VegCore-VegBIEN.csv

viaMaps := $(tables:%=%/map.csv)

autogenMaps := $(subst map.,VegBIEN.,$(viaMaps))

directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\

$(wildcard */VegBIEN.csv))

##### Sources

srcs = $(call sortFilenames,$(call srcsOnly,$(wildcard $*/*)))

nonHeaderSrcs = $(filter-out %/header.csv,$(srcs))

isXml = $(filter %.xml,$(nonHeaderSrcs))

    # true if $(srcs) non-empty and contains no *.xml

catSrcs = $(bin)/cat$(if $(nonXml),_csv) $(nonHeaderSrcs)

withCatSrcs = $(catSrcs:$(bin)/%=$(bin)/with_%) --

# Run with `make -s` to avoid echoing make commands

cat: $(importTables:%=%/cat) _always ;

%/cat: _always

	$(catSrcs)

##### Input data retrieval

##### Staging tables installation

srcTable := %.src

dbExportsWildcard = $(sort $(patsubst _MySQL/%.make,%,$(wildcard\

endif

dbExports := $(filter-out grants.sql,$(dbExports))

dbExports := $(strip $(dbExports))# += adds extra whitespace

allInstalls := $(if $(dbExports),sql) $(allTables)

install: _always schema $(allInstalls:%=%/install) ;

uninstall: _always confirm_rm_schema rm_schema ;

# rm_schema will also drop all staging tables

confirm_rm_schema: _always

schema: _always

	-echo 'CREATE SCHEMA "$(datasrc)";'|$(psqlNoSearchPath)

# ignore errors if schema exists

rm_schema: _always

	echo 'DROP SCHEMA IF EXISTS "$(datasrc)" CASCADE;'|$(psqlNoSearchPath)

installLog := logs/install.log.sql

log_dir = $(1)logs

has_log_dir = $(call and,$(wildcard $(log_dir)),$(call not,$(noclobber)))

logInstall = $(if $(has_log_dir),$(if $(quiet),$(2)$(1)$(installLog)\

logInstallRoot = $(call logInstall,,>)

logInstall* = $(call logInstall,$*/,>)

# Must come before %/install to override it

sql/install: $(dbExports)

	($(inDatasrc); cat $+|pg_dump_limit; $(if $(wildcard\

$(bin)/psql_$(if $(debug),verbose,script)_vegbien --set=schema='"$(datasrc)"' \

# $debug option runs the *.sql import verbosely, to display which statements are

cleanup = $(if $(wildcard $*/cleanup.sql),($(inDatasrc); cat $*/cleanup.sql)\

|"time" $(psqlNoSearchPath) --echo-all --set=table='"$*"' $(logInstall*Add),\

(export schema=$(datasrc) table=$*; . $(bin)/vegbien_dest; unset schemas; \

$(bin)/csv2db) $(logInstall*Add))

%/header.csv:

	echo 'COPY (SELECT * FROM "$(datasrc)"."$*" LIMIT 0) TO STDOUT CSV HEADER;'|\

exportHeader = $(selfMake) $*/header.csv

# Don't try to edit a view. Must come before %/install to override it.

%.sql/run: _always

%/postprocess.sql: %/map.csv

	$(if $(wildcard $*/run),$(bin)/in_place $@ env text=1 $(bin)/repl $<)

%/postprocess: _always

%/map_table: _always

# For staging tables which are derived by joining together other staging tables.

%/install %/header.csv: %/create.sql _always

	($(inDatasrc); echo 'CREATE TABLE "$*" AS'; cat $<; echo ';')|"time" \

$(psqlNoSearchPath) --echo-all --set=schema='"$(datasrc)"' --set=table='"$*"' \

$(logInstall*)

	$(selfMake) $*/postprocess

	$(exportHeader)

	$(cleanup)

	$(selfMake) $*/map_table

ifneq ($(noclobber),)

%/install: _always

	$(if $(isCsv),$(import_install_))

	$(selfMake) $*/postprocess

	-$(exportHeader)

	-$(if $(isCsv),,$(cleanup))

	$(selfMake) $*/map_table

# ignore errors if table does not exist (non-data dir)

define import_install_

(. $(bin)/vegbien_dest; unset schemas; "time" nice -n +5\

env schema=$(datasrc) table=$* $(bin)/csv2db $(catSrcs) $(logInstall*))

$(if $(filter $(srcTable),$*),($(inDatasrc);\

$(psqlNoSearchPath) --echo-all --set=table='"$*"' $(logInstall*Add))

endef

# table-scope src table's row_num col to allow joining it with other tables

%/uninstall: _always

	echo 'DROP TABLE IF EXISTS "$(datasrc)"."$*" CASCADE;'|$(psqlNoSearchPath)

postprocess: _always $(allTables:%=%/postprocess) ;

cleanup: _always $(tables:%=%/cleanup) ;

# WARNING: This removes any index comments, due to a PostgreSQL bug.

%/cleanup: _always

##### Maps building

# Maps to (try to) build are added to this

maps :=

srcRoot = $(mappings)/root.sh

mkSrcMap = (. $(srcRoot); env datasrc=$(datasrc) $(bin)/src_map <$*/header.csv\

translate = $(if $(wildcard $(1)),$(bin)/in_place $< $(bin)/translate_ci 1 $(1))

$(srcDict):# empty target in case it doesn't exist

# Via maps cleanup

ifneq ($(filter %/.map.csv.last_cleanup,$(MAKECMDGOALS)),)

%/.map.csv.last_cleanup: %/map.csv $(vocab) $(thesaurus) $(coreMap) $(srcDict)

	$(call translate,$(srcDict))

	$(bin)/in_place $< $(bin)/canon 1 $(vocab)

	$(call translate,$(thesaurus))

	$(bin)/in_place $< $(bin)/fix_line_endings

	touch $@

	+$(selfMake) $(<:%/map.csv=%/unmapped_terms.csv)

.PRECIOUS: %/.map.csv.last_cleanup

else

%/map.csv: _always

	$(if $(wildcard $@),,$(mk_map_csv))

	+$(selfMake) $(@:%/map.csv=%/.map.csv.last_cleanup)

define mk_map_csv

endif

%/VegBIEN.csv: %/map.csv $(coreMap)

	$(if $(wildcard $*/run),-ln -s ../$(coreMap) $@\

maps += $(autogenMaps)

maps: $(maps) _always ;

##### Maps validation

# `tail -n +2`: Remove header before running filter_out_ci because filter_out_ci

%/unmapped_terms.csv: %/map.csv $(coreMap)

	tail -n +2 $<|$(bin)/cols 1|$(bin)/filter_out_ci 0 $(coreMap) >$@

%/new_terms.csv: %/map.csv $(vocab) $(thesaurus) %/unmapped_terms.csv

	$(newTerms)

$(thesaurus) $(if $(wildcard $(word 4,$+)),|$(bin)/filter_out_ci 0 $(word 4,$+))\

|grep -vE -e '^"?:' -e 'UNUSED' >$@; exit 0 # grep exits nonzero if no match

termsSubdirs := $(tables)

include $(root)/lib/mappings.Makefile

##### External dependencies

.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error

##### Mapping

+maps = $(filter %/map.csv %/VegBIEN.csv $(mappings)/%,$(+_))

map2db = env in_database=vegbien in_schema=$(datasrc) in_table=$*\

out_database=vegbien source=$(datasrc).new $(root)/map $(+maps)

##### Import to VegBIEN

import: $(importTables:%=%/import) _always ;

import_publish: _always import publish ; # removes temp suffix when done

rm: _always

	echo "SELECT datasource_rm('$(datasrc)');"|"time" $(psqlAsBien)

reimport: _always rm import_publish ; # replace the previous import

profileTest = $(if $(profile),$(if $(test),1))

log_ = $*/logs/$(if $(n),n=$(n).,)$(version).log.sql

# Run with `make -s` to avoid echoing make commands

%/log_file: _always

trace = $(log_:.log.sql=.trace)

restart_row = $(shell set -x; grep -F Partition: $(log_)|tail -1|$(sed)\

import = -$(if $(profileTest),$(profileOnly),(set -x; date; "time" env commit=1\

$(if $(profile),profile_to=$(trace)) $(if $(continue),start=$(restart_row))\

# don't abort on import errors, which often relate to invalid input data

import? = $(if $(call and,$(full_import),$(call dontImport,.)),,$(import))

%/import: %/VegBIEN.csv _always

	$(import?)

# default:

%/import: _always ;

#### Taxonomic scrubbing

scrub: _always

	$(selfMake) $(root)/scrub & echo $$!

# using & (background process) ignores TNRS errors, so that TNRS bugs do not

# Removes TNRS taxondeterminations

rescrub: _always unscrub scrub ;

import_scrub: _always import scrub ;

reimport_scrub: _always reimport scrub ;

%/import_scrub: _always %/import scrub ;

##### Log files from import

logs := $(wildcard */logs/*.log.sql */logs/*.trace)

rm_logs: _always

	$(RM) $(logs)

##### Verification of import

verifyTables := $(patsubst verify/%.ref,%,$(wildcard verify/*.ref))

verify: $(verifyTables:%=%/verify) _always ;

%/verify: verify/%.ref verify/%.out _always

	-$(diffVerbose) $(+_)

# don't abort on verification errors, which are expected during development

# default:

%/verify: verify/%.out _always

	$(if $(shell test -e $< && echo t),cat $<)

# don't run if verify/%.out's default do-nothing action was used

# can't use $(wildcard) because it won't recheck file after verify/%.out is run

psqlExport := "time" $(psqlNoSearchPath) --no-align --field-separator=$$'\t'\

--pset=footer=off --pset=null=NULL

# Note that using $(inDatasrc) will not work with datasources whose tables are

verify/%.out: $(mappings)/verify.%.sql _always

	$(verify)

.PRECIOUS: verify/%.out # save partial output in case of error to help debugging

# default:

verify/%.out: _always ;

all += $(wildcard verify/*.out)

%.ref: %.ref.sql

	($(inDatasrc); cat $<)|$(psqlExport) >$@

.PRECIOUS: %.ref # there must always be a .ref for the make rules to work

##### Editing import

rename/%: _always

%/publish: _always # usage: make inputs/src/src.version/publish

publish: _always $(datasrc).new/publish ; # usage: make inputs/src/publish

##### Testing

testRefOutput = $(subst .by_col,,$(1))

testRef = $(testRefOutput).ref

# `rm $@`: Remove outputs of successful tests to reduce clutter

# `$(foreach use_staged...)`: Run with use_staged=1

define runTest

@echo "Testing $(abspath $@)..."

>$@ env test=1 n=$(test_n) $(1) $(foreach use_staged,1,$(map2db))

$(if $(hasOwnRef),,-)@(set -x; $(diffIgnoreSpace) $(call testRef,$@) $@) 2>&1\

$(if $(hasOwnRef),\

{\

echo $(emph)"Note: The preceding failed test is compared to another test's\

exit $$e;}

endef

tests :=

%/test: %/test.xml $(if $(by_col),%/test.by_col.xml) _always ;

# Requires staging tables. To create them, run `make inputs/<datasrc>/install`.

%/test.xml: %/VegBIEN.csv _always

	$(if $(nonXml),$(call runTest,by_col=))

tests += %/test.xml

%/test.by_col.xml: %/VegBIEN.csv _always

	$(if $(nonXml),$(call runTest,by_col=1))

tests += %/test.by_col.xml

endif

testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))

.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted

test: _always $(testOutputs) ;

all += $(wildcard %/test*.xml)

# Accepts a test output: make <test_output_path>-ok

%-ok: _always

	mv $* $(call testRef,$*)

accept-all: _always

	+yes|$(selfMake) test

##### Documentation

steps = $(selfMake) -s $*/import test=1 by_col=1 verbosity=2 n=100\

2>&1|$(bin)/debug2redmine >$@

%/logs/steps.by_col.log.sql: _always

	+$(steps)