selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))

root := $(selfDir_uZPPqC)..

include $(root)/lib/common.Makefile

##### Configuration

# Command line

log ?= $(if $(test),,1)

profile ?=

quiet ?=

reverify ?= 1

use_staged ?= $(by_col)

# Makefile

tablesSort ?= plots organisms stems specimens

exts ?= csv tsv txt xml

test_n ?= 2

##### Vars/functions

# Paths

datasrc := $(notdir $(realpath .))

bin := $(root)/bin

mappings := $(root)/mappings

# Make

SHELL := /bin/bash

selfMake = $(MAKE) --makefile=../input.Makefile

subMake = $(MAKE) $(@:$(root)/%=%) --directory=$(root)

+_ = $(+:_%=)

addBeforeExt = $(basename $(2))$(1)$(suffix $(2))

# OS

os := $(shell uname)

forOs = $(patsubst %,%-$(filter Linux Darwin,$(os)),$(1))

# Formatting

SED = sed -$(if $(filter Darwin,$(os)),E,r)

# System

date = $(shell date +"%Y-%m-%d-%H-%M-%S")

# Terminal

termCols := $(shell tput cols)

esc := '['

reset := $(esc)'0m'

emph := $(esc)'7m '

endEmph := ' '$(reset)

# Commands

MKDIR = mkdir -p

mkdir = $(MKDIR) $(@D)

CP = cp -p

diff = diff --unified=2

diffIgnoreSpace = $(diff) --ignore-space-change

diffVerbose = $(if $(verbose),diff --side-by-side --left-column\

--width=$(termCols),$(diff))

# BIEN commands

selfMap = $(bin)/cols 0 0

psqlOpts := --set ON_ERROR_STOP=1 --quiet

psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)

# SVN

addDir = $(if $(wildcard $(1)/),svn add --depth=empty $(1),svn mkdir $(1))

setSvnIgnore = svn propset svn:ignore $(2) $(1)

define addDirWithIgnore

$(addDir)

$(setSvnIgnore)

endef

##### General targets

all: _always maps ;

clean: _always

	$(RM) $(all)

remake: _always clean

	+$(selfMake)

# re-run make so that cache of existing files is reset

make_script = ./$< >$@

%/: % _always ;

# Must come before `%: %.make` to override it

src/%: src/%.make _always

	(set -x; $(make_script)) 2>>$<.log.sql

.PRECIOUS: src/% # save partial outputs of aborted src make scripts

%: %.make _always

	$(make_script)

##### SVN

add: _always

	$(call setSvnIgnore,.,$$'')

	$(call addDirWithIgnore,src,$$'*')

	$(call addDirWithIgnore,maps,$$'.~*')

	$(call addDirWithIgnore,import,$$'*')

	$(call addDirWithIgnore,test,$$'*.out\n*.xml')

	$(call addDirWithIgnore,verify,$$'*.out')

##### Existing maps discovery

inputFiles := $(wildcard $(exts:%=src/*.%))

ifeq ($(inputFiles),)

dbExport := $(firstword $(wildcard src/db.*.sql))

endif

srcMaps := $(wildcard maps/src.*.csv)

srcRoots := $(srcMaps:maps/src.%.csv=$(mappings)/roots/%.sh)

srcMap := maps/src.%.csv

fullViaMap := maps/%.full.csv

directMap := maps/VegBIEN.%.csv

allViaMaps := $(filter-out $(srcMap) $(fullViaMap) $(directMap),\

$(wildcard maps/*.csv))

via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))

# Look up via in src maps' roots

ifeq ($(via),)

ifneq ($(srcRoots),)

via := $(firstword $(sort\

$(shell $(SED) -n 's/^export out_root="([0-9A-Za-z_-]+).*$$/\1/p' $(srcRoots))))

endif

endif

coreMap := $(mappings)/$(via)-VegBIEN.%.csv

coreSelfMap := $(mappings)/$(via).self.%.csv

viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))

viaMaps += $(filter-out $(viaMaps),$(srcMaps:maps/src.%=maps/$(via).%))

viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))

autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))

directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\

$(wildcard maps/VegBIEN.*.csv))

tables := $(directMaps:maps/VegBIEN.%.csv=%)

##### Sources

srcs = $(shell $(bin)/sort_filenames $(wildcard $(exts:%=src/*.$*.%)))

isCsv = $(if $(srcs),$(if $(filter %.xml,$(srcs)),,1))

    # true if $(srcs) non-empty and contains no *.xml

catSrcs = $(bin)/cat$(if $(isCsv),_csv) $(srcs)

withCatSrcs = $(catSrcs:$(bin)/%=$(bin)/with_%) --

# Usage: `make {--silent|-s} inputs/<datasrc>/cat` (don't echo make commands)

cat: $(addprefix cat-,$(tables)) _always ;

cat-%: _always

	$(catSrcs)

##### Installation

install: _always src/install import/install ;

uninstall: _always import/uninstall src/uninstall ;

reinstall: _always uninstall install ;

##### Staging tables

import/install: _always import/schema $(addprefix import/install-,$(tables)) ;

import/uninstall: _always import/rm_schema ;

# rm_schema will also drop all staging tables

import/schema: _always

	-echo 'CREATE SCHEMA "$(datasrc)";'|$(psqlAsBien)

# ignore errors if schema exists

import/rm_schema: _always

	echo 'DROP SCHEMA IF EXISTS "$(datasrc)" CASCADE;'|$(psqlAsBien)

import/install-%: _always

	$(if $(isCsv),$(import_install_))

import_install_ = (prefix=; . $(bin)/vegbien_dest; "time" nice -n +5\

env schema=$(datasrc) table=$* $(bin)/csv2db $(catSrcs)\

$(if $(log),$(if $(quiet),&>,2>&1|tee )import/install-$*.log.sql))

##### Maps building

# Maps to (try to) build are added to one of these

maps :=

createOnlyMaps :=

srcRoot = $(mappings)/roots/$*.sh

mkSrcMap = $(catSrcs)|(. $(srcRoot); env datasrc=$(datasrc) $(bin)/src_map >$@)

# Src maps cleanup

ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)

maps/.src.%.csv.last_cleanup: maps/src.%.csv

	$(bin)/in_place $< $(bin)/cols '*'

	touch $@

else

# Autogen src maps with known table names

maps/src.%.csv: _always

	$(if $(wildcard $@),,$(if $(isCsv),$(mkSrcMap)))

	+$(if $(isCsv),$(selfMake) $(@:maps/%=maps/.%.last_cleanup))

# only build if doesn't exist

# only build if CSV srcs exist for that table name

endif

# Try all table names

createOnlyMaps += $(tablesSort:%=maps/src.%.csv)

# Must come before $(root)/% to override it

$(coreSelfMap): _always

	-+$(subMake)

# ignore errors if $(coreSelfMap) does not exist

joinSrcMap = $(if $(hasSrc),$(bin)/in_place $@ $(bin)/intersect $(word 3,$+) 0)

hasSrc = $(shell test -s $(word 3,$+) && echo t)

makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\

env ignore=1 $(bin)/union <$(wordlist 1,2,$+)|,<$< )$(bin)/sort_map >$@

# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs

# Must come before maps/$(via).%.csv to override it

maps/$(via).%.full.csv: maps/$(via).%.csv $(coreSelfMap) $(srcMap)

	$(makeFullCsv)

	$(joinSrcMap)

maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))

# Via maps cleanup

ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)

maps/.$(via).%.csv.last_cleanup: maps/$(via).%.csv $(coreMap)

	$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1 2

	touch $@

# Include comment column so commented mappings are never removed

else

maps/$(via).%.csv: maps/src.%.csv _always

	$(if $(wildcard $@),,$(if $(wildcard $<),$(CP) $< $@))

	+$(selfMake) $(@:maps/%=maps/.%.last_cleanup)

endif

maps/VegBIEN.%.csv: maps/$(via).%.full.csv $(coreMap)

	$(bin)/join <$+|$(bin)/sort_map >$@

maps += $(autogenMaps)

maps: $(createOnlyMaps) $(maps) _always ;

all += $(maps)

##### Maps validation

missing_mappings: _always missing_join_mappings missing_input_mappings ;

missing_%_mappings: _always # stem is one of join|input

	@echo $(emph)"Missing $* mappings:"$(endEmph)

	@+$(selfMake) remake 2>&1\

|$(SED) -n 's/^.*No.* $* mapping for ([0-9A-Za-z_-]+).*$$/\1/p'\

$(if $(filter join,$*),|$(bin)/ucase_first 0)|sort|uniq

##### External dependencies

$(root)/%: _always

	+$(subMake)

.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error

##### Mapping

+maps = $(filter maps/% $(mappings)/%,$(+_))

<in = $(firstword $(filter-out $(+maps),$(+_)))

map = $(if $(<in),<$(<in),\

$(if $(srcs),$(withCatSrcs),\

$(if $(mapEnv),env $(mapEnv),\

$(error No input file src/*.$*.{$(exts)}))))\

$(root)/map $(+maps)

map2db = $(if $(use_staged),env in_database=vegbien in_schema=$(datasrc)\

in_table=$*) env out_database=vegbien $(map)

##### Import to VegBIEN

ifneq ($(dbExport)$(inputFiles),)

profileTest = $(if $(profile),$(if $(test),1))

profileOnly = -env profile_to=/dev/fd/3 $(map2db) 3>&1 1>&2|\

$(bin)/profile_stats /dev/fd/0

log_ = import/$*$(if $(n),.n=$(n),).$(date).log.sql

trace = $(log_:.log.sql=.trace)

import = -$(if $(profileTest),$(profileOnly),(set -x; "time" env commit=1\

$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\

$(if $(n),,&>$(log_)))$(if $(log),$(if $(n), 2>&1|tee -a $(log_))))

# don't abort on import errors, which often relate to invalid input data

import: $(addprefix import-,$(tables)) _always ;

import-%: maps/VegBIEN.%.csv _always

	$(import)

# default:

import-%: _always ;

else

import: _always ;

endif

##### Log files from import

logs := $(wildcard import/*.log.sql import/*.trace)

rm_logs: _always

	$(RM) $(logs)

##### Verification of import

verify: $(addprefix verify-,$(tables)) _always ;

verify-%: verify/%.ref verify/%.out _always

	-$(diffVerbose) $(+_)

# don't abort on verification errors, which are expected during development

# default:

verify-%: verify/%.out _always

	$(if $(shell test -e $< && echo t),cat $<)

# don't run if verify/%.out's default do-nothing action was used

# can't use $(wildcard) because it won't recheck file after verify/%.out is run

verify = $(if $(reverify),"time" $(psqlAsBien) --set=datasource="'$(datasrc)'"\

--no-align --field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@)

verify/%.out: $(mappings)/verify.%.sql _always

	$(verify)

# default:

verify/%.out: _always ;

all += $(wildcard verify/*.out)

ifneq ($(dbExport),)

%.ref: %.ref.sql

	$(dbAsBien) $(db) <$< >$@

endif

##### Editing import

import/rotate: _always

	echo "UPDATE party SET organizationname = organizationname||'.$(date)'\

WHERE organizationname = '$(datasrc)';"|$(psqlAsBien)

import/rm: _always

	echo "DELETE FROM party WHERE organizationname = '$(datasrc)';"|\

$(psqlAsBien)

##### Testing

testRefOutput = $(subst .2-step,,$(subst .staging,,$(1)))

testRef = $(testRefOutput).ref

hasOwnRef = $(filter $@,$(call testRefOutput,$@))

# filter returns non-empty if they are equal

define runTest

@echo "Testing $(abspath $@)..."

>$@ env test=1 n=$(test_n) $(1)

@(set -x; $(diffIgnoreSpace) $(call testRef,$@) $@) 2>&1 || { e=$$?;\

$(if $(wildcard $(call testRef,$@)),,cat $@;)\

$(if $(hasOwnRef),\

{\

read -p $(emph)'Accept new test output? (y/n)'$(endEmph) REPLY;\

if test "$$REPLY" = y; then\

(set -x; $(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile);\

exit 0;\

fi;\

};,\

echo $(emph)"Note: The preceding failed test is compared to another test's\

output"$(endEmph);\

echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\

)\

exit $$e;}

endef

test2Db = $(call runTest,$(map2db))

tests :=

test/import.%.xml: maps/VegBIEN.%.csv _always

	$(test2Db)

tests += test/import.%.xml

testStaged2Db = $(foreach use_staged,1,$(test2Db))

    # run with use_staged=1

# Requires staging tables. To create them, run `make inputs/<datasrc>/install`.

test/import.%.staging.xml: maps/VegBIEN.%.csv _always

	-$(if $(isCsv),$(testStaged2Db),cp -p $(@:.staging.xml=.xml) $@)

# Don't abort tester if only staging test fails, in case staging table missing

# Non-flat-file inputs fall back to mimicking a successful test

tests += test/import.%.staging.xml

testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))

.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted

test: _always $(testOutputs) ;

all += $(filter-out %.ref,$(wildcard test/*))

# Accepts a test output: make <test_output_path>-ok

%-ok: _always

	$(CP) $* $(call testRef,$*)

test/accept-all: _always

	+yes|$(selfMake) test

##### Documentation

steps = $(selfMake) -s import test=1 by_col=1 verbosity=2 n=100\

2>&1|$(bin)/debug2redmine >$@

import/steps.by_col.sql: _always

	+$(steps)

##### Input-type-specific

# Each input type needs var $(mapEnv) and targets src/install, src/uninstall

#### DB export

ifneq ($(dbExport),)

dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbExport)))))

db := $(datasrc)

### Installation

src/install: _always db ;

src/uninstall: _always rm_db ;

### DB-engine-specific

# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db

ifeq ($(dbEngineExt),my)

dbEngine := MySQL

bienPassword := $(shell cat $(root)/config/bien_password)

mysqlAs = mysql --user=$(1) --password='$(bienPassword)'

mysqlAsRoot := $(call mysqlAs,root)

dbAsBien := $(call mysqlAs,bien)

dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))

define createDb

echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)

-$(mysqlAsRoot) --database=$(db) <$<

endef

# ignore errors in db import so that GRANT will still be run

db: $(dbExport) _always

	$(if $(dbExists),,$(createDb))

	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)

rm_db: _always

	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)

	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)

# ignore errors if grant not defined

## Unrecognized DB engine

else

$(error The DB filename $(dbExport) must be db.my.sql)

endif

### Other input types

else

src/install: _always ;

src/uninstall: _always ;

endif

#### DB connection info

ifneq ($(dbEngine),)

# Must come after dbEngine is set

mapEnv := in_engine=$(dbEngine) in_database=$(db)

endif