Project

General

Profile

1
##### Configuration
2

    
3
# Command line
4
debug ?=
5
log ?= $(if $(test),,1)
6
profile ?=
7
quiet ?=
8
reverify ?= 1
9
use_staged ?= $(by_col)
10

    
11
# Makefile
12
tablesSort ?= plots organisms stems specimens
13
exts ?= csv tsv txt xml
14
test_n ?= 2
15

    
16
##### Vars/functions
17

    
18
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
19

    
20
# Paths
21
datasrc := $(notdir $(realpath .))
22
root := $(selfDir_uZPPqC)..
23
bin := $(root)/bin
24
mappings := $(root)/mappings
25

    
26
# Make
27
SHELL := /bin/bash
28
selfMake = $(MAKE) --makefile=../input.Makefile
29
subMake = $(MAKE) $(@:$(root)/%=%) --directory=$(root)
30
+_ = $(+:_%=)
31
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
32

    
33
# OS
34
os := $(shell uname)
35
forOs = $(patsubst %,%-$(filter Linux Darwin,$(os)),$(1))
36

    
37
# Formatting
38
SED = sed -$(if $(filter Darwin,$(os)),E,r)
39

    
40
# System
41
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
42

    
43
# Terminal
44
termCols := $(shell tput cols)
45
esc := '['
46
reset := $(esc)'0m'
47
emph := $(esc)'7m '
48
endEmph := ' '$(reset)
49

    
50
# Commands
51
MKDIR = mkdir -p
52
mkdir = $(MKDIR) $(@D)
53
CP = cp -p
54
diff = diff --unified=2
55
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
56
--width=$(termCols),$(diff))
57

    
58
# BIEN commands
59
selfMap = $(bin)/cols 0 0
60
psqlOpts := --set ON_ERROR_STOP=1 --quiet
61
psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)
62

    
63
# SVN
64
addDir = $(if $(wildcard $(1)/),svn add --depth=empty $(1),svn mkdir $(1))
65
setSvnIgnore = svn propset svn:ignore $(2) $(1)
66
define addDirWithIgnore
67
$(addDir)
68
$(setSvnIgnore)
69
endef
70

    
71
##### General targets
72

    
73
all: _always maps ;
74

    
75
.SUFFIXES: # turn off built-in suffix rules
76
.SECONDARY: # don't automatically delete intermediate files
77
.DELETE_ON_ERROR: # delete target if recipe fails
78

    
79
_always:
80
.PHONY: _always
81

    
82
clean: _always
83
	$(RM) $(all)
84

    
85
remake: _always clean
86
	+$(selfMake)
87
# re-run make so that cache of existing files is reset
88

    
89
make_script = ./$< >$@
90

    
91
%/: % _always ;
92

    
93
# Must come before `%: %.make` to override it
94
src/%: src/%.make _always
95
	(set -x; $(make_script)) 2>>$<.log
96
.PRECIOUS: src/% # save partial outputs of aborted src make scripts
97

    
98
%: %.make _always
99
	$(make_script)
100

    
101
##### SVN
102

    
103
add: _always
104
	$(call setSvnIgnore,.,$$'')
105
	$(call addDirWithIgnore,src,$$'*')
106
	$(call addDirWithIgnore,maps,$$'.~*')
107
	$(call addDirWithIgnore,import,$$'*')
108
	$(call addDirWithIgnore,test,$$'*.out\n*.xml')
109
	$(call addDirWithIgnore,verify,$$'*.out')
110

    
111
##### Existing maps discovery
112

    
113
inputFiles := $(wildcard $(exts:%=src/*.%))
114
ifeq ($(inputFiles),)
115
dbExport := $(firstword $(wildcard src/db.*.sql))
116
endif
117

    
118
srcMap := maps/src.%.csv
119
fullViaMap := maps/%.full.csv
120
directMap := maps/VegBIEN.%.csv
121
allViaMaps := $(filter-out $(srcMap) $(fullViaMap) $(directMap),\
122
$(wildcard maps/*.csv))
123
via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))
124

    
125
coreMap := $(mappings)/$(via)-VegBIEN.%.csv
126
coreSelfMap := $(mappings)/$(via).self.%.csv
127

    
128
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
129
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))
130

    
131
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
132
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
133
$(wildcard maps/VegBIEN.*.csv))
134
tables := $(directMaps:maps/VegBIEN.%.csv=%)
135

    
136
##### Sources
137

    
138
srcs = $(shell $(bin)/sort_filenames $(wildcard $(exts:%=src/*.$*.%)))
139
isCsv = $(if $(srcs),$(if $(filter %.xml,$(srcs)),,1))
140
    # true if $(srcs) non-empty and contains no *.xml
141
catSrcs = $(bin)/cat$(if $(isCsv),_csv) $(srcs)
142
withCatSrcs = $(catSrcs:$(bin)/%=$(bin)/with_%) --
143

    
144
# Usage: `make {--silent|-s} inputs/<datasrc>/cat` (don't echo make commands)
145
cat: $(addprefix cat-,$(tables)) _always ;
146

    
147
cat-%: _always
148
	$(catSrcs)
149

    
150
##### Installation
151

    
152
install: _always src/install import/install ;
153

    
154
uninstall: _always import/uninstall src/uninstall ;
155

    
156
reinstall: _always uninstall install ;
157

    
158
##### Staging tables
159

    
160
import/install: _always import/schema $(addprefix import/install-,$(tables)) ;
161

    
162
import/uninstall: _always import/rm_schema ;
163
# rm_schema will also drop all staging tables
164

    
165
import/schema: _always
166
	-echo 'CREATE SCHEMA "$(datasrc)";'|$(psqlAsBien)
167
# ignore errors if schema exists
168

    
169
import/rm_schema: _always
170
	echo 'DROP SCHEMA IF EXISTS "$(datasrc)" CASCADE;'|$(psqlAsBien)
171

    
172
import/install-%: _always
173
	$(if $(isCsv),$(import_install_))
174
import_install_ = (prefix=; . $(bin)/vegbien_dest; "time" env schema=$(datasrc)\
175
table=$* $(bin)/csv2db $(catSrcs) $(if $(quiet),&>,2>&1|tee )import/install.log)
176

    
177
##### Maps building
178

    
179
# Maps to (try to) build are added to one of these
180
maps :=
181
createOnlyMaps :=
182

    
183
srcMaps := $(wildcard maps/src.*.csv)
184

    
185
mkSrcMap = $(catSrcs)|env datasrc=$(datasrc) $(bin)/src_map >$@
186

    
187
# Autogen src maps with known table names
188
maps/src.%.csv: # no prereqs so only built if doesn't exist
189
	$(if $(isCsv),$(mkSrcMap))
190
# only build if CSV srcs exist for that table name
191
# Try all table names
192
createOnlyMaps += $(tablesSort:%=maps/src.%.csv)
193

    
194
# Must come before $(root)/% to override it
195
$(coreSelfMap): _always
196
	-+$(subMake)
197
# ignore errors if $(coreSelfMap) does not exist
198

    
199
# Via maps cleanup
200
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
201
maps/.$(via).%.csv.last_cleanup: maps/$(via).%.csv $(coreMap)
202
	$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1
203
	touch $@
204
else
205
$(viaMaps): _always
206
	$(selfMake) $(@:maps/%=maps/.%.last_cleanup)
207
endif
208

    
209
joinSrcMap = $(if $(wildcard maps/src.$*.csv),$(bin)/in_place $@\
210
$(bin)/intersect maps/src.$*.csv 0)
211

    
212
makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\
213
env ignore=1 $(bin)/union <$+|,<$< )$(bin)/sort_map >$@
214
# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs
215

    
216
maps/$(via).%.full.csv: maps/$(via).%.csv $(coreSelfMap)
217
	$(makeFullCsv)
218
	$(joinSrcMap)
219
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
220

    
221
maps/VegBIEN.%.csv: maps/$(via).%.full.csv $(coreMap)
222
	$(bin)/join <$+|$(bin)/sort_map >$@
223
maps += $(autogenMaps)
224

    
225
maps: $(createOnlyMaps) $(maps) _always ;
226

    
227
all += $(maps)
228

    
229
##### Maps validation
230

    
231
missing_mappings: _always missing_join_mappings missing_input_mappings ;
232

    
233
missing_%_mappings: _always # stem is one of join|input
234
	@echo $(emph)"Missing $* mappings:"$(endEmph)
235
	@+$(selfMake) remake 2>&1\
236
|$(SED) -n 's/^.*No.* $* mapping for ([0-9A-Za-z_-]+).*$$/\1/p'\
237
$(if $(filter join,$*),|$(bin)/ucase_first 0)|sort|uniq
238

    
239
##### External dependencies
240

    
241
$(root)/%: _always
242
	+$(subMake)
243
.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error
244

    
245
##### Mapping
246

    
247
+maps = $(filter maps/% $(mappings)/%,$(+_))
248
<in = $(firstword $(filter-out $(+maps),$(+_)))
249
map = $(if $(<in),<$(<in),\
250
$(if $(use_staged),env in_database=vegbien in_schema=$(datasrc) in_table=$*,\
251
$(if $(srcs),$(withCatSrcs),\
252
$(if $(mapEnv),env $(mapEnv),\
253
$(error No input file src/*.$*.{$(exts)})))))\
254
$(root)/map $(+maps)
255
map2db = env out_database=vegbien $(map)
256

    
257
##### Import to VegBIEN
258

    
259
ifneq ($(dbExport)$(inputFiles),)
260

    
261
profileTest = $(if $(profile),$(if $(test),1))
262
profileOnly = -env profile_to=/dev/fd/3 $(map2db) 3>&1 1>&2|\
263
$(bin)/profile_stats /dev/fd/0
264

    
265
log_ = import/$*$(if $(n),.n=$(n),).$(date).log
266
trace = $(log_:.log=.trace)
267
import = -$(if $(profileTest),$(profileOnly),(set -x; "time" env commit=1\
268
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
269
$(if $(n),,>>$(log_))) 2>&1$(if $(log),$(if $(n),|tee -a $(log_))))
270
# don't abort on import errors, which often relate to invalid input data
271

    
272
import: $(addprefix import-,$(tables)) _always ;
273

    
274
import-%: maps/VegBIEN.%.csv _always
275
	$(import)
276
# default:
277
import-%: _always ;
278

    
279
else
280
import: _always ;
281
endif
282

    
283
##### Log files from import
284

    
285
logs := $(wildcard import/*.log import/*.trace)
286

    
287
rm_logs: _always
288
	$(RM) $(logs)
289

    
290
##### Verification of import
291

    
292
verify: $(addprefix verify-,$(tables)) _always ;
293

    
294
verify-%: verify/%.ref verify/%.out _always
295
	-$(diffVerbose) $(+_)
296
# don't abort on verification errors, which are expected during development
297
# default:
298
verify-%: verify/%.out _always
299
	$(if $(shell test -e $< && echo t),cat $<)
300
# don't run if verify/%.out's default do-nothing action was used
301
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
302

    
303
verify = $(if $(reverify),"time" $(psqlAsBien) --set=datasource="'$(datasrc)'"\
304
--no-align --field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@)
305

    
306
verify/%.out: $(mappings)/verify.%.sql _always
307
	$(verify)
308
# default:
309
verify/%.out: _always ;
310

    
311
all += $(wildcard verify/*.out)
312

    
313
ifneq ($(dbExport),)
314
%.ref: %.ref.sql
315
	$(dbAsBien) $(db) <$< >$@
316
endif
317

    
318
##### Editing import
319

    
320
import/rotate: _always
321
	echo "UPDATE party SET organizationname = organizationname||'.$(date)'\
322
WHERE organizationname = '$(datasrc)';"|$(psqlAsBien)
323

    
324
import/rm: _always
325
	echo "DELETE FROM party WHERE organizationname = '$(datasrc)';"|\
326
$(psqlAsBien)
327

    
328
##### Testing
329

    
330
testRefOutput = $(subst .2-step,,$(subst .staging,,$(1)))
331
testRef = $(testRefOutput).ref
332
hasOwnRef = $(filter $@,$(call testRefOutput,$@))
333
# filter returns non-empty if they are equal
334

    
335
define runTest
336
@echo "Testing $(abspath $@)..."
337
>$@ env test=1 n=$(test_n) $(1)
338
@(set -x; $(diff) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
339
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
340
$(if $(hasOwnRef),\
341
echo $(emph)"To accept new test output:"$(endEmph);\
342
echo "$(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile";\
343
,\
344
echo $(emph)"Note: The preceding failed test is compared to another test's\
345
output"$(endEmph);\
346
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
347
)\
348
exit $$e;}
349
endef
350

    
351
test2File = $(call runTest,$(map))
352
test2Db = $(call runTest,$(map2db))
353

    
354
tests :=
355

    
356
test/$(via).%.xml: maps/$(via).%.full.csv _always
357
	$(test2File)
358
tests += test/$(via).%.xml
359

    
360
test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always
361
	$(test2File)
362
tests += test/VegBIEN.%.xml
363

    
364
test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always
365
	-$(test2File)
366
# Don't abort tester if only 2-step test fails, as it's often finicky
367
tests += test/VegBIEN.%.2-step.xml
368

    
369
test/import.%.out: maps/VegBIEN.%.csv _always
370
	$(test2Db)
371
tests += test/import.%.out
372

    
373
testStaged2Db = $(foreach use_staged,1,$(test2Db))
374
    # run with use_staged=1
375

    
376
# Requires staging tables. To create them, run `make inputs/<datasrc>/install`.
377
test/import.%.staging.out: maps/VegBIEN.%.csv _always
378
	-$(if $(isCsv),$(testStaged2Db),cp -p $(@:.staging.out=.out) $@)
379
# Don't abort tester if only staging test fails, in case staging table missing
380
# Non-flat-file inputs fall back to mimicking a successful test
381
tests += test/import.%.staging.out
382

    
383
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
384

    
385
.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted
386

    
387
test: _always $(testOutputs) ;
388

    
389
all += $(filter-out %.ref,$(wildcard test/*))
390

    
391
# Accepts a test output: make <test_output_path>-ok
392
%-ok: _always
393
	$(CP) $* $(call testRef,$*)
394

    
395
##### Input-type-specific
396

    
397
# Each input type needs var $(mapEnv) and targets src/install, src/uninstall
398

    
399
#### DB export
400

    
401
ifneq ($(dbExport),)
402

    
403
dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbExport)))))
404
db := $(datasrc)
405

    
406
### Installation
407

    
408
src/install: _always db ;
409

    
410
src/uninstall: _always rm_db ;
411

    
412
### DB-engine-specific
413

    
414
# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db
415

    
416
ifeq ($(dbEngineExt),my)
417

    
418
dbEngine := MySQL
419

    
420
bienPassword := $(shell cat $(root)/config/bien_password)
421
mysqlAs = mysql --user=$(1) --password='$(bienPassword)'
422
mysqlAsRoot := $(call mysqlAs,root)
423
dbAsBien := $(call mysqlAs,bien)
424

    
425
dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))
426

    
427
define createDb
428
echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)
429
-$(mysqlAsRoot) --database=$(db) <$<
430
endef
431
# ignore errors in db import so that GRANT will still be run
432

    
433
db: $(dbExport) _always
434
	$(if $(dbExists),,$(createDb))
435
	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)
436

    
437
rm_db: _always
438
	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)
439
	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)
440
# ignore errors if grant not defined
441

    
442
## Unrecognized DB engine
443

    
444
else
445
$(error The DB filename $(dbExport) must be db.my.sql)
446
endif
447

    
448
### Other input types
449

    
450
else
451

    
452
src/install: _always ;
453
src/uninstall: _always ;
454

    
455
endif
456

    
457
#### DB connection info
458

    
459
ifneq ($(dbEngine),)
460
# Must come after dbEngine is set
461
mapEnv := in_engine=$(dbEngine) in_database=$(db)
462
endif
(2-2/2)