Project

General

Profile

1
##### Configuration
2

    
3
log ?= $(if $(test),,1)
4
profile ?=
5
reverify ?= 1
6
exts ?= csv tsv txt xml
7
test_n ?= 2
8
tablesSort ?= plots organisms stems specimens
9

    
10
##### Vars/functions
11

    
12
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
13

    
14
# Paths
15
datasrc := $(notdir $(realpath .))
16
root := $(selfDir_uZPPqC)..
17
bin := $(root)/bin
18
mappings := $(root)/mappings
19

    
20
# Make
21
SHELL := /bin/bash
22
selfMake = $(MAKE) --makefile=../input.Makefile
23
subMake = $(MAKE) $(@:$(root)/%=%) --directory=$(root)
24
+_ = $(+:_%=)
25
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
26

    
27
# OS
28
os := $(shell uname)
29
forOs = $(patsubst %,%-$(filter Linux Darwin,$(os)),$(1))
30

    
31
# Formatting
32
SED = sed -$(if $(filter Darwin,$(os)),E,r)
33

    
34
# System
35
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
36

    
37
# Terminal
38
termCols := $(shell tput cols)
39
esc := '['
40
reset := $(esc)'0m'
41
emph := $(esc)'7m '
42
endEmph := ' '$(reset)
43

    
44
# Commands
45
MKDIR = mkdir -p
46
mkdir = $(MKDIR) $(@D)
47
CP = cp -p
48
diff = diff --unified=2
49
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
50
--width=$(termCols),$(diff))
51

    
52
# Commands
53
selfMap = $(bin)/cols 0 0
54
psqlOpts := --set ON_ERROR_STOP=1 --quiet
55
psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)
56

    
57
# SVN
58
addDir = $(if $(wildcard $(1)/),svn add --depth=empty $(1),svn mkdir $(1))
59
setSvnIgnore = svn propset svn:ignore $(2) $(1)
60
define addDirWithIgnore
61
$(addDir)
62
$(setSvnIgnore)
63
endef
64

    
65
##### General targets
66

    
67
all: _always maps ;
68

    
69
.SUFFIXES: # turn off built-in suffix rules
70
.SECONDARY: # don't automatically delete intermediate files
71
.DELETE_ON_ERROR: # delete target if recipe fails
72

    
73
_always:
74
.PHONY: _always
75

    
76
clean: _always
77
	$(RM) $(all)
78

    
79
remake: _always clean
80
	+$(selfMake)
81
# re-run make so that cache of existing files is reset
82

    
83
make_script = ./$< >$@
84

    
85
%/: % _always ;
86

    
87
# Must come before `%: %.make` to override it
88
src/%: src/%.make _always
89
	(set -x; $(make_script)) 2>>$<.log
90
.PRECIOUS: src/% # save partial outputs of aborted src make scripts
91

    
92
%: %.make _always
93
	$(make_script)
94

    
95
##### SVN
96

    
97
add: _always
98
	$(call setSvnIgnore,.,$$'')
99
	$(call addDirWithIgnore,src,$$'*')
100
	$(call addDirWithIgnore,maps,$$'.~*')
101
	$(call addDirWithIgnore,import,$$'*')
102
	$(call addDirWithIgnore,test,$$'*.out\n*.xml')
103
	$(call addDirWithIgnore,verify,$$'*.out')
104

    
105
##### Installation
106

    
107
reinstall: _always uninstall install ;
108

    
109
##### Existing maps discovery
110

    
111
srcMap := maps/src.%.csv
112
fullViaMap := maps/%.full.csv
113
directMap := maps/VegBIEN.%.csv
114
allViaMaps := $(filter-out $(srcMap) $(fullViaMap) $(directMap),\
115
$(wildcard maps/*.csv))
116
via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))
117

    
118
coreMap := $(mappings)/$(via)-VegBIEN.%.csv
119
coreSelfMap := $(mappings)/$(via).self.%.csv
120

    
121
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
122
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))
123

    
124
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
125
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
126
$(wildcard maps/VegBIEN.*.csv))
127
tables := $(directMaps:maps/VegBIEN.%.csv=%)
128

    
129
##### Sources
130

    
131
srcs = $(shell $(bin)/sort_filenames $(wildcard $(exts:%=src/*.$*.%)))
132
isXml = $(filter %.xml,$(firstword $(srcs)))
133
catSrcs = $(bin)/cat$(if $(isXml),,_csv) $(srcs)
134
withCatSrcs = $(catSrcs:$(bin)/%=$(bin)/with_%) --
135

    
136
cat: $(addprefix cat-,$(tables)) _always ;
137

    
138
cat-%: _always
139
	$(catSrcs)
140

    
141
##### Maps building
142

    
143
# Maps to (try to) build are added to one of these
144
maps :=
145
createOnlyMaps :=
146

    
147
srcMaps := $(wildcard maps/src.*.csv)
148

    
149
mkSrcMap = env datasrc=$(datasrc) $(withCatSrcs) $(bin)/src_map >$@
150

    
151
# Autogen src maps with known table names
152
maps/src.%.csv: # no prereqs so only built if doesn't exist
153
	$(if $(srcs),$(if $(isXml),,$(mkSrcMap)))
154
# only build if CSV srcs exist for that table name
155
# Try all table names
156
createOnlyMaps += $(tablesSort:%=maps/src.%.csv)
157

    
158
# Must come before $(root)/% to override it
159
$(coreSelfMap): _always
160
	-+$(subMake)
161
# ignore errors if $(coreSelfMap) does not exist
162

    
163
# Via maps cleanup
164
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
165
maps/.$(via).%.csv.last_cleanup: maps/$(via).%.csv $(coreMap)
166
	$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1
167
	touch $@
168
else
169
$(viaMaps): _always
170
	$(selfMake) $(@:maps/%=maps/.%.last_cleanup)
171
endif
172

    
173
joinSrcMap = $(if $(wildcard maps/src.$*.csv),$(bin)/in_place $@\
174
$(bin)/intersect maps/src.$*.csv 0)
175

    
176
makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\
177
env ignore=1 $(bin)/union <$+|,<$< )$(bin)/sort_map >$@
178
# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs
179

    
180
maps/$(via).%.full.csv: maps/$(via).%.csv $(coreSelfMap)
181
	$(makeFullCsv)
182
	$(joinSrcMap)
183
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
184

    
185
maps/VegBIEN.%.csv: maps/$(via).%.full.csv $(coreMap)
186
	$(bin)/join <$+|$(bin)/sort_map >$@
187
maps += $(autogenMaps)
188

    
189
maps: $(createOnlyMaps) $(maps) _always ;
190

    
191
all += $(maps)
192

    
193
##### Maps validation
194

    
195
missing_mappings: _always missing_join_mappings missing_input_mappings ;
196

    
197
missing_%_mappings: _always # stem is one of join|input
198
	@echo $(emph)"Missing $* mappings:"$(endEmph)
199
	@+$(selfMake) remake 2>&1\
200
|$(SED) -n 's/^.*No.* $* mapping for ([0-9A-Za-z_-]+).*$$/\1/p'\
201
$(if $(filter join,$*),|$(bin)/ucase_first 0)|sort|uniq
202

    
203
##### External dependencies
204

    
205
$(root)/%: _always
206
	+$(subMake)
207
.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error
208

    
209
##### Mapping
210

    
211
dbExport := $(firstword $(wildcard src/db.*.sql))
212
inputFiles := $(wildcard $(exts:%=src/*.%))
213

    
214
+maps = $(filter maps/% $(mappings)/%,$(+_))
215
<in = $(firstword $(filter-out $(+maps),$(+_)))
216
map = $(if $(<in),<$(<in),\
217
$(if $(srcs),$(withCatSrcs),\
218
$(if $(mapEnv),env $(mapEnv),\
219
$(error No input file src/*.$*.{$(exts)}))))\
220
$(root)/map $(+maps)
221
map2db = env out_database=vegbien $(map)
222

    
223
##### Import to VegBIEN
224

    
225
ifneq ($(dbExport)$(inputFiles),)
226

    
227
log_ = import/$*$(if $(n),.n=$(n),).$(date).log
228
trace = $(log_:.log=.trace)
229
import = -(set -x; "time" env commit=1\
230
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
231
$(if $(n),,>>$(log_))) 2>&1$(if $(log),$(if $(n),|tee -a $(log_)))
232
# don't abort on import errors, which often relate to invalid input data
233

    
234
import: $(addprefix import-,$(tables)) _always ;
235

    
236
import-%: maps/VegBIEN.%.csv _always
237
	$(import)
238
# default:
239
import-%: _always ;
240

    
241
else
242
import: _always ;
243
endif
244

    
245
##### Log files from import
246

    
247
logs := $(wildcard import/*.log import/*.trace)
248

    
249
rm_logs: _always
250
	$(RM) $(logs)
251

    
252
##### Verification of import
253

    
254
verify: $(addprefix verify-,$(tables)) _always ;
255

    
256
verify-%: verify/%.ref verify/%.out _always
257
	-$(diffVerbose) $(+_)
258
# don't abort on verification errors, which are expected during development
259
# default:
260
verify-%: verify/%.out _always
261
	$(if $(shell test -e $< && echo t),cat $<)
262
# don't run if verify/%.out's default do-nothing action was used
263
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
264

    
265
verify = $(if $(reverify),"time" $(psqlAsBien) --set=datasource="'$(datasrc)'"\
266
--no-align --field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@)
267

    
268
verify/%.out: $(mappings)/verify.%.sql _always
269
	$(verify)
270
# default:
271
verify/%.out: _always ;
272

    
273
all += $(wildcard verify/*.out)
274

    
275
ifneq ($(dbExport),)
276
%.ref: %.ref.sql
277
	$(dbAsBien) $(db) <$< >$@
278
endif
279

    
280
##### Editing import
281

    
282
import/rotate: _always
283
	echo "UPDATE party SET organizationname = organizationname||'.$(date)'\
284
WHERE organizationname = '$(db)';"|$(psqlAsBien)
285

    
286
import/rm: _always
287
	echo "DELETE FROM party WHERE organizationname = '$(db)';"|$(psqlAsBien)
288

    
289
##### Testing
290

    
291
hasOwnRef = $(filter-out %.2-step.xml,$@)
292
testRef = $(1:.2-step.xml=.xml).ref
293

    
294
define runTest
295
@echo "Testing $(abspath $@)..."
296
>$@ env test=1 n=$(test_n) $(1)
297
@(set -x; $(diff) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
298
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
299
$(if $(hasOwnRef),\
300
echo $(emph)"To accept new test output:"$(endEmph);\
301
echo "$(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile";\
302
,\
303
echo $(emph)"Note: The preceding failed test is compared to another test's\
304
output"$(endEmph);\
305
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
306
)\
307
exit $$e;}
308
endef
309

    
310
test2File = $(call runTest,$(map))
311

    
312
tests :=
313

    
314
test/$(via).%.xml: maps/$(via).%.full.csv _always
315
	$(test2File)
316
tests += test/$(via).%.xml
317

    
318
test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always
319
	$(test2File)
320
tests += test/VegBIEN.%.xml
321

    
322
test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always
323
	-$(test2File)
324
# Don't abort tester if only 2-step test fails, as it's often finicky
325
tests += test/VegBIEN.%.2-step.xml
326

    
327
test/import.%.out: maps/VegBIEN.%.csv _always
328
	$(call runTest,$(map2db))
329
tests += test/import.%.out
330

    
331
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
332

    
333
.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted
334

    
335
test: _always $(testOutputs) ;
336

    
337
all += $(testOutputs)
338

    
339
# Accepts a test output: make <test_output_path>-ok
340
%-ok: _always
341
	$(CP) $* $(call testRef,$*)
342

    
343
##### Input-type-specific
344

    
345
# Each input type needs var $(mapEnv) and targets install, uninstall
346

    
347
#### DB export
348

    
349
ifneq ($(dbExport),)
350

    
351
dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbExport)))))
352
db := $(datasrc)
353

    
354
### Installation
355

    
356
install: _always db ;
357

    
358
uninstall: _always rm_db ;
359

    
360
### DB-engine-specific
361

    
362
# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db
363

    
364
ifeq ($(dbEngineExt),my)
365

    
366
dbEngine := MySQL
367

    
368
bienPassword := $(shell cat $(root)/config/bien_password)
369
mysqlAs = mysql --user=$(1) --password='$(bienPassword)'
370
mysqlAsRoot := $(call mysqlAs,root)
371
dbAsBien := $(call mysqlAs,bien)
372

    
373
dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))
374

    
375
define createDb
376
echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)
377
-$(mysqlAsRoot) --database=$(db) <$<
378
endef
379
# ignore errors in db import so that GRANT will still be run
380

    
381
db: $(dbExport) _always
382
	$(if $(dbExists),,$(createDb))
383
	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)
384

    
385
rm_db: _always
386
	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)
387
	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)
388
# ignore errors if grant not defined
389

    
390
## Unrecognized DB engine
391

    
392
else
393
$(error The DB filename $(dbExport) must be db.my.sql)
394
endif
395

    
396
### Other input types
397

    
398
else
399

    
400
install: _always ;
401
uninstall: _always ;
402

    
403
endif
404

    
405
#### DB connection info
406

    
407
ifneq ($(dbEngine),)
408
# Must come after dbEngine is set
409
mapEnv := in_engine=$(dbEngine) in_database=$(db)
410
endif
(2-2/2)