##### Configuration

# Command line

log ?= $(if $(test),,1)

profile ?=

quiet ?=

reverify ?= 1

use_staged ?= $(by_col)

tablesSort ?= 0.plots 1.organisms 2.stems specimens

exts ?= csv tsv txt xml

test_n ?= 2

# Paths

# Make

SHELL := /bin/bash

selfMake = $(MAKE) --makefile=../input.Makefile

subMake = $(MAKE) $(@:$(root)/%=%) --directory=$(root)

+_ = $(+:_%=)

addBeforeExt = $(basename $(2))$(1)$(suffix $(2))

# OS

# System

# Terminal

termCols := $(shell tput cols)

esc := '['

# Commands

MKDIR = mkdir -p

CP = cp -p

diff = diff --unified=2

diffIgnoreSpace = $(diff) --ignore-space-change

diffVerbose = $(if $(verbose),diff --side-by-side --left-column\

# BIEN commands

selfMap = $(bin)/cols 0 0

psqlOpts := --set ON_ERROR_STOP=1 --quiet

psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)

# SVN

setSvnIgnore = svn propset svn:ignore $(2) $(1)

define addDirWithIgnore

$(setSvnIgnore)

endef

##### General targets

all: _always maps ;

clean: _always

	$(RM) $(all)

remake: _always clean

make_script = ./$< >$@

%/: % _always ;

# Must come before `%: %.make` to override it

	(set -x; $(make_script)) 2>>$<.log.sql

.PRECIOUS: src/% # save partial outputs of aborted src make scripts

##### SVN

add: _always

	$(call addDirWithIgnore,src,'*')

##### Existing maps discovery

srcMaps := $(wildcard src/*/src.csv)

srcMap := src/%/src.csv

anyTest = src/$*/test.%

srcsOnly = $(filter-out $(anyMap) $(anyTest) src/%/logs,$(1))

via := Veg+

coreMap := $(mappings)/$(via)-VegBIEN.csv

viaMaps := $(wildcard $(tablesSort:%=src/%/map.csv))

autogenMaps := $(subst map.,VegBIEN.,$(viaMaps))

directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\

$(wildcard src/*/VegBIEN.csv))

##### Sources

srcs = $(shell $(bin)/sort_filenames $(call srcsOnly,$(wildcard src/$*/*)))

isCsv = $(if $(srcs),$(if $(filter %.xml,$(srcs)),,1))

# Usage: `make {--silent|-s} inputs/<datasrc>/cat` (don't echo make commands)

cat: $(addprefix cat-,$(tables)) _always ;

##### Staging tables installation

install: _always schema $(addprefix install-,$(tables)) ;

uninstall: _always rm_schema ;

schema: _always

	-echo 'CREATE SCHEMA "$(datasrc)";'|$(psqlAsBien)

rm_schema: _always

	echo 'DROP SCHEMA IF EXISTS "$(datasrc)" CASCADE;'|$(psqlAsBien)

install-%: _always

	$(if $(isCsv),$(import_install_))

import_install_ = (prefix=; . $(bin)/vegbien_dest; "time" nice -n +5\

$(if $(log),$(if $(quiet),&>,2>&1|tee )src/$*/logs/install.log.sql))

##### Maps building

# Maps to (try to) build are added to one of these

maps :=

createOnlyMaps :=

srcRoot = $(mappings)/roots/$*.sh

# Src maps cleanup

ifneq ($(filter src/%/.map.csv.last_cleanup,$(MAKECMDGOALS)),)

	$(bin)/in_place $< $(bin)/cols +

	touch $@

# Autogen src maps with known table names

src/%/src.csv: _always

	$(if $(wildcard $@),,$(if $(isCsv),$(mkSrcMap)))

	+$(if $(isCsv),$(selfMake) $(@:src/%/src.csv=src/%/.map.csv.last_cleanup))

# only build if doesn't exist

# only build if CSV srcs exist for that table name

endif

# Try all table names

createOnlyMaps += $(tablesSort:%=src/%/src.csv)

# Must come before $(root)/% to override it

$(coreSelfMap): _always

	-+$(subMake)

# ignore errors if $(coreSelfMap) does not exist

joinSrcMap = $(if $(hasSrc),$(bin)/in_place $@ $(bin)/intersect $(word 3,$+) 0)

makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\

env ignore=1 $(bin)/union <$(wordlist 1,2,$+)|,<$< )$(bin)/sort_map >$@

# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs

# Must come before src/%/map.csv to override it

	$(makeFullCsv)

	$(joinSrcMap)

maps += $(patsubst src/%/map.csv,src/%/map.full.csv,$(viaMaps))

# Via maps cleanup

ifneq ($(filter src/%/.map.csv.last_cleanup,$(MAKECMDGOALS)),)

	$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1 2

src/%/map.csv: src/%/src.csv _always

	$(if $(wildcard $@),,$(if $(wildcard $<),$(CP) $< $@))

	+$(selfMake) $(@:src/%/map.csv=src/%/.map.csv.last_cleanup)

endif

src/%/VegBIEN.csv: src/%/map.full.csv $(coreMap)

	$(bin)/join <$+|$(bin)/sort_map >$@

maps: $(createOnlyMaps) $(maps) _always ;

##### Maps validation

missingMappingsCmd = +$(selfMake) remake 2>&1\

|$(SED) -n 's/^.*No $(*2Space) mapping for (.*)$$/\1/p'|$(SED) 's/\/_.*//'\

$(if $(filter non-empty_join,$*),|$(bin)/ucase_first 0)|sort|uniq

include $(root)/lib/mappings.Makefile

##### External dependencies

.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error

##### Mapping

+maps = $(filter src/%/map.csv src/%/VegBIEN.csv $(mappings)/%,$(+_))

map = $(if $(srcs),$(withCatSrcs) $(root)/map $(+maps),\

map2db = $(if $(use_staged),env in_database=vegbien in_schema=$(datasrc)\

##### Import to VegBIEN

profileTest = $(if $(profile),$(if $(test),1))

log_ = src/$*/logs/$(if $(n),n=$(n).,)$(date).log.sql

trace = $(log_:.log.sql=.trace)

import = -$(if $(profileTest),$(profileOnly),(set -x; "time" env commit=1\

$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\

$(if $(n),,&>$(log_)))$(if $(log),$(if $(n), 2>&1|tee -a $(log_))))

# don't abort on import errors, which often relate to invalid input data

import: $(addprefix import-,$(tables)) _always ;

import-%: src/%/VegBIEN.csv _always

	$(import)

# default:

##### Log files from import

logs := $(wildcard src/*/logs/*.log.sql src/*/logs/*.trace)

rm_logs: _always

	$(RM) $(logs)

##### Verification of import

verify: $(addprefix verify-,$(tables)) _always ;

verify-%: verify/%.ref verify/%.out _always

	-$(diffVerbose) $(+_)

# don't abort on verification errors, which are expected during development

# default:

verify-%: verify/%.out _always

	$(if $(shell test -e $< && echo t),cat $<)

# don't run if verify/%.out's default do-nothing action was used

# can't use $(wildcard) because it won't recheck file after verify/%.out is run

verify = $(if $(reverify),"time" $(psqlAsBien) --set=datasource="'$(datasrc)'"\

verify/%.out: $(mappings)/verify.%.sql _always

	$(verify)

# default:

verify/%.out: _always ;

all += $(wildcard verify/*.out)

##### Editing import

rotate: _always

	echo "UPDATE party SET organizationname = organizationname||'.$(date)'\

WHERE organizationname = '$(datasrc)';"|$(psqlAsBien)

rm: _always

	echo "DELETE FROM party WHERE organizationname = '$(datasrc)';"|\

##### Testing

testRefOutput = $(subst .2-step,,$(subst .staging,,$(1)))

define runTest

@echo "Testing $(abspath $@)..."

>$@ env test=1 n=$(test_n) $(1)

@(set -x; $(diffIgnoreSpace) $(call testRef,$@) $@) 2>&1 || { e=$$?;\

$(if $(wildcard $(call testRef,$@)),,cat $@;)\

$(if $(hasOwnRef),\

{\

echo $(emph)"Note: The preceding failed test is compared to another test's\

exit $$e;}

endef

test2Db = $(call runTest,$(map2db))

tests :=

src/%/test.xml: src/%/VegBIEN.csv _always

	$(test2Db)

tests += src/%/test.xml

testStaged2Db = $(foreach use_staged,1,$(test2Db))

src/%/test.staging.xml: src/%/VegBIEN.csv _always

	-$(if $(isCsv),$(testStaged2Db),cp -p $(@:.staging.xml=.xml) $@)

# Don't abort tester if only staging test fails, in case staging table missing

tests += src/%/test.staging.xml

testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))

.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted

test: _always $(testOutputs) ;

all += $(wildcard src/%/test*.xml)

# Accepts a test output: make <test_output_path>-ok

%-ok: _always

	$(CP) $* $(call testRef,$*)

accept-all: _always

	+yes|$(selfMake) test

##### Documentation

steps = $(selfMake) -s import-$* test=1 by_col=1 verbosity=2 n=100\

2>&1|$(bin)/debug2redmine >$@

src/%/logs/steps.by_col.log.sql: _always

	+$(steps)