Project

General

Profile

1
##### Configuration
2

    
3
# Command line
4
log ?= $(if $(test),,1)
5
profile ?=
6
quiet ?=
7
reverify ?= 1
8
use_staged ?= $(by_col)
9

    
10
# Makefile
11
tablesSort ?= plots organisms stems specimens
12
exts ?= csv tsv txt xml
13
test_n ?= 2
14

    
15
##### Vars/functions
16

    
17
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
18

    
19
# Paths
20
datasrc := $(notdir $(realpath .))
21
root := $(selfDir_uZPPqC)..
22
bin := $(root)/bin
23
mappings := $(root)/mappings
24

    
25
# Make
26
SHELL := /bin/bash
27
selfMake = $(MAKE) --makefile=../input.Makefile
28
subMake = $(MAKE) $(@:$(root)/%=%) --directory=$(root)
29
+_ = $(+:_%=)
30
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
31

    
32
# OS
33
os := $(shell uname)
34
forOs = $(patsubst %,%-$(filter Linux Darwin,$(os)),$(1))
35

    
36
# Formatting
37
SED = sed -$(if $(filter Darwin,$(os)),E,r)
38

    
39
# System
40
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
41

    
42
# Terminal
43
termCols := $(shell tput cols)
44
esc := '['
45
reset := $(esc)'0m'
46
emph := $(esc)'7m '
47
endEmph := ' '$(reset)
48

    
49
# Commands
50
MKDIR = mkdir -p
51
mkdir = $(MKDIR) $(@D)
52
CP = cp -p
53
diff = diff --unified=2
54
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
55
--width=$(termCols),$(diff))
56

    
57
# BIEN commands
58
selfMap = $(bin)/cols 0 0
59
psqlOpts := --set ON_ERROR_STOP=1 --quiet
60
psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)
61

    
62
# SVN
63
addDir = $(if $(wildcard $(1)/),svn add --depth=empty $(1),svn mkdir $(1))
64
setSvnIgnore = svn propset svn:ignore $(2) $(1)
65
define addDirWithIgnore
66
$(addDir)
67
$(setSvnIgnore)
68
endef
69

    
70
##### General targets
71

    
72
all: _always maps ;
73

    
74
.SUFFIXES: # turn off built-in suffix rules
75
.SECONDARY: # don't automatically delete intermediate files
76
.DELETE_ON_ERROR: # delete target if recipe fails
77

    
78
_always:
79
.PHONY: _always
80

    
81
clean: _always
82
	$(RM) $(all)
83

    
84
remake: _always clean
85
	+$(selfMake)
86
# re-run make so that cache of existing files is reset
87

    
88
make_script = ./$< >$@
89

    
90
%/: % _always ;
91

    
92
# Must come before `%: %.make` to override it
93
src/%: src/%.make _always
94
	(set -x; $(make_script)) 2>>$<.log
95
.PRECIOUS: src/% # save partial outputs of aborted src make scripts
96

    
97
%: %.make _always
98
	$(make_script)
99

    
100
##### SVN
101

    
102
add: _always
103
	$(call setSvnIgnore,.,$$'')
104
	$(call addDirWithIgnore,src,$$'*')
105
	$(call addDirWithIgnore,maps,$$'.~*')
106
	$(call addDirWithIgnore,import,$$'*')
107
	$(call addDirWithIgnore,test,$$'*.out\n*.xml')
108
	$(call addDirWithIgnore,verify,$$'*.out')
109

    
110
##### Existing maps discovery
111

    
112
inputFiles := $(wildcard $(exts:%=src/*.%))
113
ifeq ($(inputFiles),)
114
dbExport := $(firstword $(wildcard src/db.*.sql))
115
endif
116

    
117
srcMap := maps/src.%.csv
118
fullViaMap := maps/%.full.csv
119
directMap := maps/VegBIEN.%.csv
120
allViaMaps := $(filter-out $(srcMap) $(fullViaMap) $(directMap),\
121
$(wildcard maps/*.csv))
122
via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))
123

    
124
coreMap := $(mappings)/$(via)-VegBIEN.%.csv
125
coreSelfMap := $(mappings)/$(via).self.%.csv
126

    
127
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
128
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))
129

    
130
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
131
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
132
$(wildcard maps/VegBIEN.*.csv))
133
tables := $(directMaps:maps/VegBIEN.%.csv=%)
134

    
135
##### Sources
136

    
137
srcs = $(shell $(bin)/sort_filenames $(wildcard $(exts:%=src/*.$*.%)))
138
isCsv = $(if $(srcs),$(if $(filter %.xml,$(srcs)),,1))
139
    # true if $(srcs) non-empty and contains no *.xml
140
catSrcs = $(bin)/cat$(if $(isCsv),_csv) $(srcs)
141
withCatSrcs = $(catSrcs:$(bin)/%=$(bin)/with_%) --
142

    
143
# Usage: `make {--silent|-s} inputs/<datasrc>/cat` (don't echo make commands)
144
cat: $(addprefix cat-,$(tables)) _always ;
145

    
146
cat-%: _always
147
	$(catSrcs)
148

    
149
##### Installation
150

    
151
install: _always src/install import/install ;
152

    
153
uninstall: _always import/uninstall src/uninstall ;
154

    
155
reinstall: _always uninstall install ;
156

    
157
##### Staging tables
158

    
159
import/install: _always import/schema $(addprefix import/install-,$(tables)) ;
160

    
161
import/uninstall: _always import/rm_schema ;
162
# rm_schema will also drop all staging tables
163

    
164
import/schema: _always
165
	-echo 'CREATE SCHEMA "$(datasrc)";'|$(psqlAsBien)
166
# ignore errors if schema exists
167

    
168
import/rm_schema: _always
169
	echo 'DROP SCHEMA IF EXISTS "$(datasrc)" CASCADE;'|$(psqlAsBien)
170

    
171
import/install-%: _always
172
	$(if $(isCsv),$(import_install_))
173
import_install_ = (prefix=; . $(bin)/vegbien_dest; "time" env schema=$(datasrc)\
174
table=$* $(bin)/csv2db $(catSrcs) $(if $(quiet),&>,2>&1|tee )import/install.log)
175

    
176
##### Maps building
177

    
178
# Maps to (try to) build are added to one of these
179
maps :=
180
createOnlyMaps :=
181

    
182
srcMaps := $(wildcard maps/src.*.csv)
183

    
184
mkSrcMap = $(catSrcs)|env datasrc=$(datasrc) $(bin)/src_map >$@
185

    
186
# Autogen src maps with known table names
187
maps/src.%.csv: # no prereqs so only built if doesn't exist
188
	$(if $(isCsv),$(mkSrcMap))
189
# only build if CSV srcs exist for that table name
190
# Try all table names
191
createOnlyMaps += $(tablesSort:%=maps/src.%.csv)
192

    
193
# Must come before $(root)/% to override it
194
$(coreSelfMap): _always
195
	-+$(subMake)
196
# ignore errors if $(coreSelfMap) does not exist
197

    
198
# Via maps cleanup
199
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
200
maps/.$(via).%.csv.last_cleanup: maps/$(via).%.csv $(coreMap)
201
	$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1 2
202
	touch $@
203
# Include comment column so commented mappings are never removed
204
else
205
$(viaMaps): _always
206
	$(selfMake) $(@:maps/%=maps/.%.last_cleanup)
207
endif
208

    
209
joinSrcMap = $(if $(wildcard maps/src.$*.csv),$(bin)/in_place $@\
210
$(bin)/intersect maps/src.$*.csv 0)
211

    
212
makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\
213
env ignore=1 $(bin)/union <$+|,<$< )$(bin)/sort_map >$@
214
# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs
215

    
216
maps/$(via).%.full.csv: maps/$(via).%.csv $(coreSelfMap)
217
	$(makeFullCsv)
218
	$(joinSrcMap)
219
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
220

    
221
maps/VegBIEN.%.csv: maps/$(via).%.full.csv $(coreMap)
222
	$(bin)/join <$+|$(bin)/sort_map >$@
223
maps += $(autogenMaps)
224

    
225
maps: $(createOnlyMaps) $(maps) _always ;
226

    
227
all += $(maps)
228

    
229
##### Maps validation
230

    
231
missing_mappings: _always missing_join_mappings missing_input_mappings ;
232

    
233
missing_%_mappings: _always # stem is one of join|input
234
	@echo $(emph)"Missing $* mappings:"$(endEmph)
235
	@+$(selfMake) remake 2>&1\
236
|$(SED) -n 's/^.*No.* $* mapping for ([0-9A-Za-z_-]+).*$$/\1/p'\
237
$(if $(filter join,$*),|$(bin)/ucase_first 0)|sort|uniq
238

    
239
##### External dependencies
240

    
241
$(root)/%: _always
242
	+$(subMake)
243
.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error
244

    
245
##### Mapping
246

    
247
+maps = $(filter maps/% $(mappings)/%,$(+_))
248
<in = $(firstword $(filter-out $(+maps),$(+_)))
249
map = $(if $(<in),<$(<in),\
250
$(if $(use_staged),env in_database=vegbien in_schema=$(datasrc) in_table=$*,\
251
$(if $(srcs),$(withCatSrcs),\
252
$(if $(mapEnv),env $(mapEnv),\
253
$(error No input file src/*.$*.{$(exts)})))))\
254
$(root)/map $(+maps)
255
map2db = env out_database=vegbien $(map)
256

    
257
##### Import to VegBIEN
258

    
259
ifneq ($(dbExport)$(inputFiles),)
260

    
261
profileTest = $(if $(profile),$(if $(test),1))
262
profileOnly = -env profile_to=/dev/fd/3 $(map2db) 3>&1 1>&2|\
263
$(bin)/profile_stats /dev/fd/0
264

    
265
log_ = import/$*$(if $(n),.n=$(n),).$(date).log
266
trace = $(log_:.log=.trace)
267
import = -$(if $(profileTest),$(profileOnly),(set -x; "time" env commit=1\
268
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
269
$(if $(n),,>>$(log_))) 2>&1$(if $(log),$(if $(n),|tee -a $(log_))))
270
# don't abort on import errors, which often relate to invalid input data
271

    
272
import: $(addprefix import-,$(tables)) _always ;
273

    
274
import-%: maps/VegBIEN.%.csv _always
275
	$(import)
276
# default:
277
import-%: _always ;
278

    
279
else
280
import: _always ;
281
endif
282

    
283
##### Log files from import
284

    
285
logs := $(wildcard import/*.log import/*.trace)
286

    
287
rm_logs: _always
288
	$(RM) $(logs)
289

    
290
##### Verification of import
291

    
292
verify: $(addprefix verify-,$(tables)) _always ;
293

    
294
verify-%: verify/%.ref verify/%.out _always
295
	-$(diffVerbose) $(+_)
296
# don't abort on verification errors, which are expected during development
297
# default:
298
verify-%: verify/%.out _always
299
	$(if $(shell test -e $< && echo t),cat $<)
300
# don't run if verify/%.out's default do-nothing action was used
301
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
302

    
303
verify = $(if $(reverify),"time" $(psqlAsBien) --set=datasource="'$(datasrc)'"\
304
--no-align --field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@)
305

    
306
verify/%.out: $(mappings)/verify.%.sql _always
307
	$(verify)
308
# default:
309
verify/%.out: _always ;
310

    
311
all += $(wildcard verify/*.out)
312

    
313
ifneq ($(dbExport),)
314
%.ref: %.ref.sql
315
	$(dbAsBien) $(db) <$< >$@
316
endif
317

    
318
##### Editing import
319

    
320
import/rotate: _always
321
	echo "UPDATE party SET organizationname = organizationname||'.$(date)'\
322
WHERE organizationname = '$(datasrc)';"|$(psqlAsBien)
323

    
324
import/rm: _always
325
	echo "DELETE FROM party WHERE organizationname = '$(datasrc)';"|\
326
$(psqlAsBien)
327

    
328
##### Testing
329

    
330
testRefOutput = $(subst .2-step,,$(subst .staging,,$(1)))
331
testRef = $(testRefOutput).ref
332
hasOwnRef = $(filter $@,$(call testRefOutput,$@))
333
# filter returns non-empty if they are equal
334

    
335
define runTest
336
@echo "Testing $(abspath $@)..."
337
>$@ env test=1 n=$(test_n) $(1)
338
@(set -x; $(diff) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
339
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
340
$(if $(hasOwnRef),\
341
echo $(emph)"To accept new test output:"$(endEmph);\
342
echo "$(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile";\
343
,\
344
echo $(emph)"Note: The preceding failed test is compared to another test's\
345
output"$(endEmph);\
346
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
347
)\
348
exit $$e;}
349
endef
350

    
351
test2File = $(call runTest,$(map))
352
test2Db = $(call runTest,$(map2db))
353

    
354
tests :=
355

    
356
test/$(via).%.xml: maps/$(via).%.full.csv _always
357
	$(test2File)
358
tests += test/$(via).%.xml
359

    
360
test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always
361
	$(test2File)
362
tests += test/VegBIEN.%.xml
363

    
364
test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always
365
	-$(test2File)
366
# Don't abort tester if only 2-step test fails, as it's often finicky
367
tests += test/VegBIEN.%.2-step.xml
368

    
369
test/import.%.out: maps/VegBIEN.%.csv _always
370
	$(test2Db)
371
tests += test/import.%.out
372

    
373
testStaged2Db = $(foreach use_staged,1,$(test2Db))
374
    # run with use_staged=1
375

    
376
# Requires staging tables. To create them, run `make inputs/<datasrc>/install`.
377
test/import.%.staging.out: maps/VegBIEN.%.csv _always
378
	-$(if $(isCsv),$(testStaged2Db),cp -p $(@:.staging.out=.out) $@)
379
# Don't abort tester if only staging test fails, in case staging table missing
380
# Non-flat-file inputs fall back to mimicking a successful test
381
tests += test/import.%.staging.out
382

    
383
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
384

    
385
.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted
386

    
387
test: _always $(testOutputs) ;
388

    
389
all += $(filter-out %.ref,$(wildcard test/*))
390

    
391
# Accepts a test output: make <test_output_path>-ok
392
%-ok: _always
393
	$(CP) $* $(call testRef,$*)
394

    
395
##### Input-type-specific
396

    
397
# Each input type needs var $(mapEnv) and targets src/install, src/uninstall
398

    
399
#### DB export
400

    
401
ifneq ($(dbExport),)
402

    
403
dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbExport)))))
404
db := $(datasrc)
405

    
406
### Installation
407

    
408
src/install: _always db ;
409

    
410
src/uninstall: _always rm_db ;
411

    
412
### DB-engine-specific
413

    
414
# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db
415

    
416
ifeq ($(dbEngineExt),my)
417

    
418
dbEngine := MySQL
419

    
420
bienPassword := $(shell cat $(root)/config/bien_password)
421
mysqlAs = mysql --user=$(1) --password='$(bienPassword)'
422
mysqlAsRoot := $(call mysqlAs,root)
423
dbAsBien := $(call mysqlAs,bien)
424

    
425
dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))
426

    
427
define createDb
428
echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)
429
-$(mysqlAsRoot) --database=$(db) <$<
430
endef
431
# ignore errors in db import so that GRANT will still be run
432

    
433
db: $(dbExport) _always
434
	$(if $(dbExists),,$(createDb))
435
	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)
436

    
437
rm_db: _always
438
	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)
439
	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)
440
# ignore errors if grant not defined
441

    
442
## Unrecognized DB engine
443

    
444
else
445
$(error The DB filename $(dbExport) must be db.my.sql)
446
endif
447

    
448
### Other input types
449

    
450
else
451

    
452
src/install: _always ;
453
src/uninstall: _always ;
454

    
455
endif
456

    
457
#### DB connection info
458

    
459
ifneq ($(dbEngine),)
460
# Must come after dbEngine is set
461
mapEnv := in_engine=$(dbEngine) in_database=$(db)
462
endif
(2-2/2)