Project

General

Profile

1
##### Configuration
2

    
3
nolog ?= $(test)
4
exts ?= csv xml
5
test_n ?= 2
6
tablesSort ?= plots organisms stems
7

    
8
##### Vars/functions
9

    
10
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
11

    
12
# Make
13
SHELL := /bin/bash
14
subMake = $(MAKE) $(@F) --directory=$(@D)
15
+_ = $(+:_%=)
16
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
17

    
18
# System
19
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
20

    
21
# Terminal
22
esc := '['
23
reset := $(esc)'0m'
24
emph := $(esc)'7m '
25
endEmph := ' '$(reset)
26

    
27
# Commands
28
CP = cp -p
29
DIFF = diff --unified=2
30

    
31
# Paths
32
root := $(selfDir_uZPPqC)..
33
mappings := $(root)/mappings
34
psqlOpts := --set ON_ERROR_STOP=1 --quiet
35
psqlAsBien := $(root)/bin/psql_vegbien $(psqlOpts)
36

    
37
##### General targets
38

    
39
all: _always maps ;
40

    
41
.SUFFIXES:
42

    
43
_always:
44
.PHONY: _always
45

    
46
clean: _always
47
	$(RM) $(all)
48

    
49
remake: _always clean all ;
50

    
51
%.out: %.make _always
52
	./$* >$@
53
.PRECIOUS: %.out
54

    
55
$(root)/%: _always
56
	+$(subMake)
57

    
58
##### SVN
59

    
60
svn_props: _always
61
	svn propset svn:ignore $$'.~*\n*.log\n*.out\nsrc*' .
62
	$(if $(wildcard maps/),svn propset svn:ignore $$'.~*' maps)
63
	$(if $(wildcard test/),svn propset svn:ignore $$'*.out\n*.xml' test)
64

    
65
##### Installation
66

    
67
reinstall: _always uninstall install ;
68

    
69
##### Maps
70

    
71
allViaMaps := $(filter-out maps/VegBIEN.%.csv,$(wildcard maps/*.csv))
72
via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))
73

    
74
coreMap := $(mappings)/$(via)-VegBIEN.%.csv
75

    
76
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
77
viaMaps += $(filter-out $(viaMaps),$(wildcard maps/$(via).*.csv))
78

    
79
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
80
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
81
$(wildcard maps/VegBIEN.*.csv))
82
tables := $(directMaps:maps/VegBIEN.%.csv=%)
83

    
84
maps: $(directMaps) _always ;
85

    
86
.PRECIOUS: $(allViaMaps) $(directMaps)
87

    
88
all += $(autogenMaps)
89

    
90
maps/VegBIEN.%.csv: maps/$(via).%.csv $(coreMap)
91
	$(root)/bin/join_sort <$+ >$@
92
.PRECIOUS: maps/VegBIEN.%.csv $(coreMap)
93

    
94
##### Mapping
95

    
96
dbFile := $(firstword $(wildcard src/db.*.sql))
97
inputFiles := $(wildcard src/*.csv src/*.xml)
98

    
99
+maps = $(filter maps/% $(mappings)/%,$(+_))
100
<in = $(firstword $(filter-out $(+maps),$(+_)) $(wildcard $(exts:%=src/*.$*.%)))
101
map = $(if $(<in),<$(<in) ,$(if $(mapEnv),env $(mapEnv) ,$(error\
102
No input file src/*.{$(tables)}.{$(exts)} )))$(root)/map $(+maps)
103
map2db = env out_database=vegbien $(map)
104

    
105
##### Import to VegBIEN
106

    
107
ifneq ($(dbFile)$(inputFiles),)
108

    
109
import: _always import-all verify ;
110

    
111
log = $(@:-all=)$(if $(n),.n=$(n),).$(date).log
112
import = -(set -x; "time" env commit=1 verbose=1 $(map2db)) $(if $(nolog),,\
113
$(if $(n),,>>$(log))) 2>&1$(if $(nolog),,$(if $(n),|tee -a $(log)))
114
# ignore import errors, which are often benign (e.g. invalid date format)
115

    
116
import-all: $(addprefix import-,$(tables)) _always ;
117

    
118
import-%: maps/VegBIEN.%.csv _always
119
	$(import)
120
# default:
121
import-%: _always ;
122

    
123
else
124
import: _always ;
125
endif
126

    
127
##### Log files from import
128

    
129
logs := $(wildcard *.log)
130

    
131
rm_logs: _always
132
	$(RM) $(logs)
133

    
134
##### Verification of import
135

    
136
ifneq ($(wildcard verify.ref*),)
137

    
138
verify: verify.ref verify.out _always
139
	$(DIFF) $(+_)
140

    
141
all += verify.out
142

    
143
%.out: %.sql _always
144
	$(out_cmd)
145
.PRECIOUS: %.out
146
out_cmd = $(psqlAsBien) --no-align --field-separator='	' --pset=footer=off\
147
--pset=null=NULL <$< >$@
148

    
149
ifneq ($(dbFile),)
150
%.ref: %.ref.sql
151
	$(dbAsBien) $(db) <$< >$@
152
.PRECIOUS: %.ref
153
endif
154

    
155
else
156
verify: _always ;
157
endif
158

    
159
##### Testing
160

    
161
ifneq ($(wildcard test/),)
162

    
163
hasOwnRef = $(filter-out %.2-step.xml,$@)
164
testRef = $(1:.2-step.xml=.xml).ref
165

    
166
define runTest
167
@echo "Testing $(abspath $@)..."
168
>$@ env test=1 n=$(test_n) $(1)
169
@(set -x; $(DIFF) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
170
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
171
$(if $(hasOwnRef),\
172
echo $(emph)"To accept new test output:"$(endEmph);\
173
echo "$(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile";\
174
,\
175
echo $(emph)"Note: The preceding failed test is compared to another test's\
176
output"$(endEmph);\
177
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
178
)\
179
exit $$e;}
180
endef
181

    
182
test2File = $(call runTest,$(map))
183

    
184
tests :=
185

    
186
test/$(via).%.xml: maps/$(via).%.csv _always
187
	$(test2File)
188
tests += test/$(via).%.xml
189

    
190
test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always
191
	$(test2File)
192
tests += test/VegBIEN.%.xml
193

    
194
test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always
195
	-$(test2File)
196
# Don't abort tester if only 2-step test fails, as it's often finicky
197
tests += test/VegBIEN.%.2-step.xml
198

    
199
test/import.%.out: maps/VegBIEN.%.csv _always
200
	$(call runTest,$(map2db))
201
tests += test/import.%.out
202

    
203
.PRECIOUS: $(tests)
204

    
205
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
206

    
207
test: _always $(testOutputs) ;
208

    
209
all += $(testOutputs)
210

    
211
# Accepts a test output: make <test_output_path>-ok
212
%-ok: _always
213
	$(CP) $* $(call testRef,$*)
214

    
215
else
216
test: _always ;
217
endif
218

    
219
##### Input-type-specific
220

    
221
# Each input type needs var $(mapEnv) and targets install, uninstall
222

    
223
ifneq ($(dbFile),)
224

    
225
dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbFile)))))
226
db := $(notdir $(realpath .))
227

    
228
#### Installation
229

    
230
install: _always db ;
231

    
232
uninstall: _always rm_db ;
233

    
234
#### DB-engine-specific
235

    
236
# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db
237

    
238
ifeq ($(dbEngineExt),my)
239

    
240
dbEngine := MySQL
241

    
242
bienPassword := $(shell cat $(root)/config/bien_password)
243
mysqlAs = mysql --user=$(1) --password='$(bienPassword)'
244
mysqlAsRoot := $(call mysqlAs,root)
245
dbAsBien := $(call mysqlAs,bien)
246

    
247
db: $(dbFile) _always
248
	echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)
249
	-$(mysqlAsRoot) --database=$(db) <$<
250
	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)
251
# ignore errors in db import so that GRANT will still be run
252

    
253
rm_db: _always
254
	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)
255
	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)
256
# ignore errors if grant not defined
257

    
258
### Unrecognized DB engine
259

    
260
else
261
$(error The DB filename $(dbFile) must be db.my.sql)
262
endif
263

    
264
# Must come after dbEngine is set
265
mapEnv := in_engine=$(dbEngine) in_database=$(db)
266

    
267
#### Unrecognized input type
268

    
269
else
270
install: _always ;
271
uninstall: _always ;
272
endif
(2-2/2)