log ?= $(if $(test),,1)

profile ?=

exts ?= csv tsv txt xml

test_n ?= 2

selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))

# Make

SHELL := /bin/bash

selfMake = $(MAKE) --makefile=../input.Makefile

subMake = $(MAKE) $(@F) --directory=$(@D)

+_ = $(+:_%=)

addBeforeExt = $(basename $(2))$(1)$(suffix $(2))

# System

# Terminal

termCols := $(shell tput cols)

esc := '['

# Commands

MKDIR = mkdir -p

CP = cp -p

diff = diff --unified=2

# Paths

datasrc := $(notdir $(realpath .))

root := $(selfDir_uZPPqC)..

bin := $(root)/bin

mappings := $(root)/mappings

psqlOpts := --set ON_ERROR_STOP=1 --quiet

psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)

##### General targets

all: _always maps ;

.SUFFIXES: # turn off built-in suffix rules

.DELETE_ON_ERROR: # delete target if recipe fails

_always:

clean: _always

	$(RM) $(all)

%.out: %.make _always

	./$* >$@

##### SVN

svn_props: _always

	svn propset svn:ignore $$'*.log\n*.trace\nsrc*' .

	$(if $(wildcard maps/),svn propset svn:ignore $$'.~*' maps)

	$(if $(wildcard src/),svn propset svn:ignore $$'*' src)

	$(if $(wildcard verify/),svn propset svn:ignore $$'*.out' verify)

##### Installation

reinstall: _always uninstall install ;

##### Maps

srcMap := maps/src.%.csv

fullViaMap := maps/%.full.csv

directMap := maps/VegBIEN.%.csv

$(wildcard maps/*.csv))

via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))

coreMap := $(mappings)/$(via)-VegBIEN.%.csv

selfMap := $(mappings)/$(via).self.%.csv

noEmptyMap := $(mappings)/$(via)-VegBIEN.%.no_empty.csv

viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))

viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))

autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))

tables := $(directMaps:maps/VegBIEN.%.csv=%)

srcMaps := $(filter-out maps/src.join.%.csv,$(wildcard maps/src.*.csv))

# Must come before $(root)/% to override it

# Via maps cleanup

maps :=

maps/VegBIEN.%.csv: maps/$(via).%.csv $(coreMap)

	$(join) <$+ >$@

maps += $(autogenMaps)

makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\

env ignore=1 $(bin)/union <$+|$(bin)/sort_map >$@,$(CP) $< $@)

maps/$(via).%.full.csv: maps/$(via).%.csv $(selfMap)

	$(makeFullCsv)

maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))

maps/src.join.%.csv: maps/src.%.csv maps/$(via).%.full.csv $(noEmptyMap)

maps: $(maps) _always ;

##### External dependencies

##### Mapping

dbExport := $(firstword $(wildcard src/db.*.sql))

inputFiles := $(wildcard $(exts:%=src/*.%))

+maps = $(filter maps/% $(mappings)/%,$(+_))

<in = $(firstword $(filter-out $(+maps),$(+_)))

map2db = env out_database=vegbien $(map)

##### Import to VegBIEN

ifneq ($(dbExport)$(inputFiles),)

log_ = $(@:-all=)$(if $(n),.n=$(n),).$(date).log

import = -(set -x; "time" env commit=1 verbose=1\

$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\

# don't abort on import errors, which often relate to invalid input data

import: $(addprefix import-,$(tables)) _always ;

# default:

else

##### Log files from import

logs := $(wildcard *.log *.trace)

rm_logs: _always

	$(RM) $(logs)

##### Verification of import

verify: $(addprefix verify-,$(tables)) _always ;

verify-%: verify/%.ref verify/%.out _always

	-$(diffVerbose) $(+_)

# don't abort on verification errors, which are expected during development

# default:

verify-%: verify/%.out _always

	$(if $(shell test -e $< && echo t),cat $<)

# don't run if verify/%.out's default do-nothing action was used

# can't use $(wildcard) because it won't recheck file after verify/%.out is run

define verify

"time" $(psqlAsBien) --set=datasource="'$(datasrc)'" --no-align\

--field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@

verify/%.out: $(mappings)/verify.%.sql _always

	$(verify)

# default:

verify/%.out: _always ;

all += $(wildcard verify/*.out)

ifneq ($(dbExport),)

%.ref: %.ref.sql

##### Testing

ifneq ($(wildcard test/),)

hasOwnRef = $(filter-out %.2-step.xml,$@)

testRef = $(1:.2-step.xml=.xml).ref

define runTest

@echo "Testing $(abspath $@)..."

>$@ env test=1 n=$(test_n) $(1)

@(set -x; $(diff) $(call testRef,$@) $@) 2>&1 || { e=$$?;\

$(if $(wildcard $(call testRef,$@)),,cat $@;)\

$(if $(hasOwnRef),\

exit $$e;}

endef

test2File = $(call runTest,$(map))

tests :=

test/$(via).%.xml: maps/$(via).%.full.csv _always

	$(test2File)

tests += test/$(via).%.xml

test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always

	$(test2File)

tests += test/VegBIEN.%.xml

test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always

	-$(test2File)

# Don't abort tester if only 2-step test fails, as it's often finicky

tests += test/VegBIEN.%.2-step.xml

test/import.%.out: maps/VegBIEN.%.csv _always

	$(call runTest,$(map2db))

tests += test/import.%.out

testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))

.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted

test: _always $(testOutputs) ;

# Accepts a test output: make <test_output_path>-ok

%-ok: _always

	$(CP) $* $(call testRef,$*)

else

##### Input-type-specific

# Each input type needs var $(mapEnv) and targets install, uninstall

#### DB export

ifneq ($(dbExport),)

dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbExport)))))

db := $(datasrc)

### Installation

install: _always db ;

uninstall: _always rm_db ;

### DB-engine-specific

# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db

ifeq ($(dbEngineExt),my)

dbEngine := MySQL

bienPassword := $(shell cat $(root)/config/bien_password)

dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))

echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)

-$(mysqlAsRoot) --database=$(db) <$<

db: $(dbExport) _always

	$(if $(dbExists),,$(createDb))

	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)

rm_db: _always

	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)

# ignore errors if grant not defined

## Unrecognized DB engine

$(error The DB filename $(dbExport) must be db.my.sql)

endif

### Other input types

else

endif