Project

General

Profile

1
##### Configuration
2

    
3
log ?= $(if $(test),,1)
4
profile ?=
5
reverify ?= 1
6
exts ?= csv tsv txt xml
7
test_n ?= 2
8
tablesSort ?= plots organisms stems specimens
9

    
10
##### Vars/functions
11

    
12
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
13

    
14
# Paths
15
datasrc := $(notdir $(realpath .))
16
root := $(selfDir_uZPPqC)..
17
bin := $(root)/bin
18
mappings := $(root)/mappings
19

    
20
# Make
21
SHELL := /bin/bash
22
selfMake = $(MAKE) --makefile=../input.Makefile
23
subMake = $(MAKE) $(@:$(root)/%=%) --directory=$(root)
24
+_ = $(+:_%=)
25
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
26

    
27
# OS
28
os := $(shell uname)
29
forOs = $(patsubst %,%-$(filter Linux Darwin,$(os)),$(1))
30

    
31
# Formatting
32
SED = sed -$(if $(filter Darwin,$(os)),E,r)
33

    
34
# System
35
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
36

    
37
# Terminal
38
termCols := $(shell tput cols)
39
esc := '['
40
reset := $(esc)'0m'
41
emph := $(esc)'7m '
42
endEmph := ' '$(reset)
43

    
44
# Commands
45
MKDIR = mkdir -p
46
mkdir = $(MKDIR) $(@D)
47
CP = cp -p
48
diff = diff --unified=2
49
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
50
--width=$(termCols),$(diff))
51

    
52
# BIEN commands
53
selfMap = $(bin)/cols 0 0
54
psqlOpts := --set ON_ERROR_STOP=1 --quiet
55
psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)
56

    
57
# SVN
58
addDir = $(if $(wildcard $(1)/),svn add --depth=empty $(1),svn mkdir $(1))
59
setSvnIgnore = svn propset svn:ignore $(2) $(1)
60
define addDirWithIgnore
61
$(addDir)
62
$(setSvnIgnore)
63
endef
64

    
65
##### General targets
66

    
67
all: _always maps ;
68

    
69
.SUFFIXES: # turn off built-in suffix rules
70
.SECONDARY: # don't automatically delete intermediate files
71
.DELETE_ON_ERROR: # delete target if recipe fails
72

    
73
_always:
74
.PHONY: _always
75

    
76
clean: _always
77
	$(RM) $(all)
78

    
79
remake: _always clean
80
	+$(selfMake)
81
# re-run make so that cache of existing files is reset
82

    
83
make_script = ./$< >$@
84

    
85
%/: % _always ;
86

    
87
# Must come before `%: %.make` to override it
88
src/%: src/%.make _always
89
	(set -x; $(make_script)) 2>>$<.log
90
.PRECIOUS: src/% # save partial outputs of aborted src make scripts
91

    
92
%: %.make _always
93
	$(make_script)
94

    
95
##### SVN
96

    
97
add: _always
98
	$(call setSvnIgnore,.,$$'')
99
	$(call addDirWithIgnore,src,$$'*')
100
	$(call addDirWithIgnore,maps,$$'.~*')
101
	$(call addDirWithIgnore,import,$$'*')
102
	$(call addDirWithIgnore,test,$$'*.out\n*.xml')
103
	$(call addDirWithIgnore,verify,$$'*.out')
104

    
105
##### Installation
106

    
107
install: _always src/install import/install ;
108

    
109
uninstall: _always import/uninstall src/uninstall ;
110

    
111
reinstall: _always uninstall install ;
112

    
113
##### Staging tables
114

    
115
import/install: _always
116
	-echo "CREATE SCHEMA $(datasrc);"|$(psqlAsBien)
117
# ignore errors if schema exists
118

    
119
import/uninstall: _always
120
	echo "DROP SCHEMA IF EXISTS $(datasrc) CASCADE;"|$(psqlAsBien)
121

    
122
##### Existing maps discovery
123

    
124
srcMap := maps/src.%.csv
125
fullViaMap := maps/%.full.csv
126
directMap := maps/VegBIEN.%.csv
127
allViaMaps := $(filter-out $(srcMap) $(fullViaMap) $(directMap),\
128
$(wildcard maps/*.csv))
129
via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))
130

    
131
coreMap := $(mappings)/$(via)-VegBIEN.%.csv
132
coreSelfMap := $(mappings)/$(via).self.%.csv
133

    
134
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
135
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))
136

    
137
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
138
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
139
$(wildcard maps/VegBIEN.*.csv))
140
tables := $(directMaps:maps/VegBIEN.%.csv=%)
141

    
142
##### Sources
143

    
144
srcs = $(shell $(bin)/sort_filenames $(wildcard $(exts:%=src/*.$*.%)))
145
isXml = $(filter %.xml,$(firstword $(srcs)))
146
catSrcs = $(bin)/cat$(if $(isXml),,_csv) $(srcs)
147
withCatSrcs = $(catSrcs:$(bin)/%=$(bin)/with_%) --
148

    
149
# Usage: `make {--silent|-s} cat` (removes echoed make commands)
150
cat: $(addprefix cat-,$(tables)) _always ;
151

    
152
cat-%: _always
153
	$(catSrcs)
154

    
155
##### Maps building
156

    
157
# Maps to (try to) build are added to one of these
158
maps :=
159
createOnlyMaps :=
160

    
161
srcMaps := $(wildcard maps/src.*.csv)
162

    
163
mkSrcMap = env datasrc=$(datasrc) $(withCatSrcs) $(bin)/src_map >$@
164

    
165
# Autogen src maps with known table names
166
maps/src.%.csv: # no prereqs so only built if doesn't exist
167
	$(if $(srcs),$(if $(isXml),,$(mkSrcMap)))
168
# only build if CSV srcs exist for that table name
169
# Try all table names
170
createOnlyMaps += $(tablesSort:%=maps/src.%.csv)
171

    
172
# Must come before $(root)/% to override it
173
$(coreSelfMap): _always
174
	-+$(subMake)
175
# ignore errors if $(coreSelfMap) does not exist
176

    
177
# Via maps cleanup
178
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
179
maps/.$(via).%.csv.last_cleanup: maps/$(via).%.csv $(coreMap)
180
	$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1
181
	touch $@
182
else
183
$(viaMaps): _always
184
	$(selfMake) $(@:maps/%=maps/.%.last_cleanup)
185
endif
186

    
187
joinSrcMap = $(if $(wildcard maps/src.$*.csv),$(bin)/in_place $@\
188
$(bin)/intersect maps/src.$*.csv 0)
189

    
190
makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\
191
env ignore=1 $(bin)/union <$+|,<$< )$(bin)/sort_map >$@
192
# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs
193

    
194
maps/$(via).%.full.csv: maps/$(via).%.csv $(coreSelfMap)
195
	$(makeFullCsv)
196
	$(joinSrcMap)
197
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
198

    
199
maps/VegBIEN.%.csv: maps/$(via).%.full.csv $(coreMap)
200
	$(bin)/join <$+|$(bin)/sort_map >$@
201
maps += $(autogenMaps)
202

    
203
maps: $(createOnlyMaps) $(maps) _always ;
204

    
205
all += $(maps)
206

    
207
##### Maps validation
208

    
209
missing_mappings: _always missing_join_mappings missing_input_mappings ;
210

    
211
missing_%_mappings: _always # stem is one of join|input
212
	@echo $(emph)"Missing $* mappings:"$(endEmph)
213
	@+$(selfMake) remake 2>&1\
214
|$(SED) -n 's/^.*No.* $* mapping for ([0-9A-Za-z_-]+).*$$/\1/p'\
215
$(if $(filter join,$*),|$(bin)/ucase_first 0)|sort|uniq
216

    
217
##### External dependencies
218

    
219
$(root)/%: _always
220
	+$(subMake)
221
.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error
222

    
223
##### Mapping
224

    
225
dbExport := $(firstword $(wildcard src/db.*.sql))
226
inputFiles := $(wildcard $(exts:%=src/*.%))
227

    
228
+maps = $(filter maps/% $(mappings)/%,$(+_))
229
<in = $(firstword $(filter-out $(+maps),$(+_)))
230
map = $(if $(<in),<$(<in),\
231
$(if $(srcs),$(withCatSrcs),\
232
$(if $(mapEnv),env $(mapEnv),\
233
$(error No input file src/*.$*.{$(exts)}))))\
234
$(root)/map $(+maps)
235
map2db = env out_database=vegbien $(map)
236

    
237
##### Import to VegBIEN
238

    
239
ifneq ($(dbExport)$(inputFiles),)
240

    
241
profileTest = $(if $(profile),$(if $(test),1))
242
profileOnly = -env profile_to=/dev/fd/3 $(map2db) 3>&1 1>&2|\
243
$(bin)/profile_stats /dev/fd/0
244

    
245
log_ = import/$*$(if $(n),.n=$(n),).$(date).log
246
trace = $(log_:.log=.trace)
247
import = -$(if $(profileTest),$(profileOnly),(set -x; "time" env commit=1\
248
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
249
$(if $(n),,>>$(log_))) 2>&1$(if $(log),$(if $(n),|tee -a $(log_))))
250
# don't abort on import errors, which often relate to invalid input data
251

    
252
import: $(addprefix import-,$(tables)) _always ;
253

    
254
import-%: maps/VegBIEN.%.csv _always
255
	$(import)
256
# default:
257
import-%: _always ;
258

    
259
else
260
import: _always ;
261
endif
262

    
263
##### Log files from import
264

    
265
logs := $(wildcard import/*.log import/*.trace)
266

    
267
rm_logs: _always
268
	$(RM) $(logs)
269

    
270
##### Verification of import
271

    
272
verify: $(addprefix verify-,$(tables)) _always ;
273

    
274
verify-%: verify/%.ref verify/%.out _always
275
	-$(diffVerbose) $(+_)
276
# don't abort on verification errors, which are expected during development
277
# default:
278
verify-%: verify/%.out _always
279
	$(if $(shell test -e $< && echo t),cat $<)
280
# don't run if verify/%.out's default do-nothing action was used
281
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
282

    
283
verify = $(if $(reverify),"time" $(psqlAsBien) --set=datasource="'$(datasrc)'"\
284
--no-align --field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@)
285

    
286
verify/%.out: $(mappings)/verify.%.sql _always
287
	$(verify)
288
# default:
289
verify/%.out: _always ;
290

    
291
all += $(wildcard verify/*.out)
292

    
293
ifneq ($(dbExport),)
294
%.ref: %.ref.sql
295
	$(dbAsBien) $(db) <$< >$@
296
endif
297

    
298
##### Editing import
299

    
300
import/rotate: _always
301
	echo "UPDATE party SET organizationname = organizationname||'.$(date)'\
302
WHERE organizationname = '$(datasrc)';"|$(psqlAsBien)
303

    
304
import/rm: _always
305
	echo "DELETE FROM party WHERE organizationname = '$(datasrc)';"|\
306
$(psqlAsBien)
307

    
308
##### Testing
309

    
310
hasOwnRef = $(filter-out %.2-step.xml,$@)
311
testRef = $(1:.2-step.xml=.xml).ref
312

    
313
define runTest
314
@echo "Testing $(abspath $@)..."
315
>$@ env test=1 n=$(test_n) $(1)
316
@(set -x; $(diff) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
317
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
318
$(if $(hasOwnRef),\
319
echo $(emph)"To accept new test output:"$(endEmph);\
320
echo "$(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile";\
321
,\
322
echo $(emph)"Note: The preceding failed test is compared to another test's\
323
output"$(endEmph);\
324
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
325
)\
326
exit $$e;}
327
endef
328

    
329
test2File = $(call runTest,$(map))
330

    
331
tests :=
332

    
333
test/$(via).%.xml: maps/$(via).%.full.csv _always
334
	$(test2File)
335
tests += test/$(via).%.xml
336

    
337
test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always
338
	$(test2File)
339
tests += test/VegBIEN.%.xml
340

    
341
test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always
342
	-$(test2File)
343
# Don't abort tester if only 2-step test fails, as it's often finicky
344
tests += test/VegBIEN.%.2-step.xml
345

    
346
test/import.%.out: maps/VegBIEN.%.csv _always
347
	$(call runTest,$(map2db))
348
tests += test/import.%.out
349

    
350
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
351

    
352
.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted
353

    
354
test: _always $(testOutputs) ;
355

    
356
all += $(testOutputs)
357

    
358
# Accepts a test output: make <test_output_path>-ok
359
%-ok: _always
360
	$(CP) $* $(call testRef,$*)
361

    
362
##### Input-type-specific
363

    
364
# Each input type needs var $(mapEnv) and targets src/install, src/uninstall
365

    
366
#### DB export
367

    
368
ifneq ($(dbExport),)
369

    
370
dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbExport)))))
371
db := $(datasrc)
372

    
373
### Installation
374

    
375
src/install: _always db ;
376

    
377
src/uninstall: _always rm_db ;
378

    
379
### DB-engine-specific
380

    
381
# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db
382

    
383
ifeq ($(dbEngineExt),my)
384

    
385
dbEngine := MySQL
386

    
387
bienPassword := $(shell cat $(root)/config/bien_password)
388
mysqlAs = mysql --user=$(1) --password='$(bienPassword)'
389
mysqlAsRoot := $(call mysqlAs,root)
390
dbAsBien := $(call mysqlAs,bien)
391

    
392
dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))
393

    
394
define createDb
395
echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)
396
-$(mysqlAsRoot) --database=$(db) <$<
397
endef
398
# ignore errors in db import so that GRANT will still be run
399

    
400
db: $(dbExport) _always
401
	$(if $(dbExists),,$(createDb))
402
	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)
403

    
404
rm_db: _always
405
	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)
406
	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)
407
# ignore errors if grant not defined
408

    
409
## Unrecognized DB engine
410

    
411
else
412
$(error The DB filename $(dbExport) must be db.my.sql)
413
endif
414

    
415
### Other input types
416

    
417
else
418

    
419
src/install: _always ;
420
src/uninstall: _always ;
421

    
422
endif
423

    
424
#### DB connection info
425

    
426
ifneq ($(dbEngine),)
427
# Must come after dbEngine is set
428
mapEnv := in_engine=$(dbEngine) in_database=$(db)
429
endif
(2-2/2)