Project

General

Profile

1
##### Configuration
2

    
3
log ?= $(if $(test),,1)
4
profile ?=
5
reverify ?= 1
6
exts ?= csv tsv txt xml
7
test_n ?= 2
8
tablesSort ?= plots organisms stems specimens
9

    
10
##### Vars/functions
11

    
12
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
13

    
14
# Paths
15
datasrc := $(notdir $(realpath .))
16
root := $(selfDir_uZPPqC)..
17
bin := $(root)/bin
18
mappings := $(root)/mappings
19

    
20
# Make
21
SHELL := /bin/bash
22
selfMake = $(MAKE) --makefile=../input.Makefile
23
subMake = $(MAKE) $(@:$(root)/%=%) --directory=$(root)
24
+_ = $(+:_%=)
25
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
26

    
27
# OS
28
os := $(shell uname)
29
forOs = $(patsubst %,%-$(filter Linux Darwin,$(os)),$(1))
30

    
31
# Formatting
32
SED = sed -$(if $(filter Darwin,$(os)),E,r)
33

    
34
# System
35
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
36

    
37
# Terminal
38
termCols := $(shell tput cols)
39
esc := '['
40
reset := $(esc)'0m'
41
emph := $(esc)'7m '
42
endEmph := ' '$(reset)
43

    
44
# Commands
45
MKDIR = mkdir -p
46
mkdir = $(MKDIR) $(@D)
47
CP = cp -p
48
diff = diff --unified=2
49
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
50
--width=$(termCols),$(diff))
51

    
52
# Commands
53
selfMap = $(bin)/cols 0 0
54
psqlOpts := --set ON_ERROR_STOP=1 --quiet
55
psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)
56

    
57
# SVN
58
addDir = $(if $(wildcard $(1)/),svn add --depth=empty $(1),svn mkdir $(1))
59
setSvnIgnore = svn propset svn:ignore $(2) $(1)
60
define addDirWithIgnore
61
$(addDir)
62
$(setSvnIgnore)
63
endef
64

    
65
##### General targets
66

    
67
all: _always maps ;
68

    
69
.SUFFIXES: # turn off built-in suffix rules
70
.SECONDARY: # don't automatically delete intermediate files
71
.DELETE_ON_ERROR: # delete target if recipe fails
72

    
73
_always:
74
.PHONY: _always
75

    
76
clean: _always
77
	$(RM) $(all)
78

    
79
remake: _always clean
80
	+$(selfMake)
81
# re-run make so that cache of existing files is reset
82

    
83
make_script = ./$< >$@
84

    
85
%/: % _always ;
86

    
87
# Must come before `%: %.make` to override it
88
src/%: src/%.make _always
89
	(set -x; $(make_script)) 2>>$<.log
90
.PRECIOUS: src/% # save partial outputs of aborted src make scripts
91

    
92
%: %.make _always
93
	$(make_script)
94

    
95
##### SVN
96

    
97
add: _always
98
	$(call setSvnIgnore,.,$$'')
99
	$(call addDirWithIgnore,src,$$'*')
100
	$(call addDirWithIgnore,maps,$$'.~*')
101
	$(call addDirWithIgnore,import,$$'*')
102
	$(call addDirWithIgnore,test,$$'*.out\n*.xml')
103
	$(call addDirWithIgnore,verify,$$'*.out')
104

    
105
##### Installation
106

    
107
reinstall: _always uninstall install ;
108

    
109
##### Existing maps discovery
110

    
111
srcMap := maps/src.%.csv
112
fullViaMap := maps/%.full.csv
113
directMap := maps/VegBIEN.%.csv
114
allViaMaps := $(filter-out $(srcMap) $(fullViaMap) $(directMap),\
115
$(wildcard maps/*.csv))
116
via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))
117

    
118
coreMap := $(mappings)/$(via)-VegBIEN.%.csv
119
coreSelfMap := $(mappings)/$(via).self.%.csv
120

    
121
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
122
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))
123

    
124
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
125
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
126
$(wildcard maps/VegBIEN.*.csv))
127
tables := $(directMaps:maps/VegBIEN.%.csv=%)
128

    
129
##### Sources
130

    
131
srcs = $(shell $(bin)/sort_filenames $(wildcard $(exts:%=src/*.$*.%)))
132
isXml = $(filter %.xml,$(firstword $(srcs)))
133
catSrcs = $(bin)/cat$(if $(isXml),,_csv) $(srcs)
134
withCatSrcs = $(catSrcs:$(bin)/%=$(bin)/with_%) --
135

    
136
# Usage: `make {--silent|-s} cat` (removes echoed make commands)
137
cat: $(addprefix cat-,$(tables)) _always ;
138

    
139
cat-%: _always
140
	$(catSrcs)
141

    
142
##### Maps building
143

    
144
# Maps to (try to) build are added to one of these
145
maps :=
146
createOnlyMaps :=
147

    
148
srcMaps := $(wildcard maps/src.*.csv)
149

    
150
mkSrcMap = env datasrc=$(datasrc) $(withCatSrcs) $(bin)/src_map >$@
151

    
152
# Autogen src maps with known table names
153
maps/src.%.csv: # no prereqs so only built if doesn't exist
154
	$(if $(srcs),$(if $(isXml),,$(mkSrcMap)))
155
# only build if CSV srcs exist for that table name
156
# Try all table names
157
createOnlyMaps += $(tablesSort:%=maps/src.%.csv)
158

    
159
# Must come before $(root)/% to override it
160
$(coreSelfMap): _always
161
	-+$(subMake)
162
# ignore errors if $(coreSelfMap) does not exist
163

    
164
# Via maps cleanup
165
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
166
maps/.$(via).%.csv.last_cleanup: maps/$(via).%.csv $(coreMap)
167
	$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1
168
	touch $@
169
else
170
$(viaMaps): _always
171
	$(selfMake) $(@:maps/%=maps/.%.last_cleanup)
172
endif
173

    
174
joinSrcMap = $(if $(wildcard maps/src.$*.csv),$(bin)/in_place $@\
175
$(bin)/intersect maps/src.$*.csv 0)
176

    
177
makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\
178
env ignore=1 $(bin)/union <$+|,<$< )$(bin)/sort_map >$@
179
# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs
180

    
181
maps/$(via).%.full.csv: maps/$(via).%.csv $(coreSelfMap)
182
	$(makeFullCsv)
183
	$(joinSrcMap)
184
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
185

    
186
maps/VegBIEN.%.csv: maps/$(via).%.full.csv $(coreMap)
187
	$(bin)/join <$+|$(bin)/sort_map >$@
188
maps += $(autogenMaps)
189

    
190
maps: $(createOnlyMaps) $(maps) _always ;
191

    
192
all += $(maps)
193

    
194
##### Maps validation
195

    
196
missing_mappings: _always missing_join_mappings missing_input_mappings ;
197

    
198
missing_%_mappings: _always # stem is one of join|input
199
	@echo $(emph)"Missing $* mappings:"$(endEmph)
200
	@+$(selfMake) remake 2>&1\
201
|$(SED) -n 's/^.*No.* $* mapping for ([0-9A-Za-z_-]+).*$$/\1/p'\
202
$(if $(filter join,$*),|$(bin)/ucase_first 0)|sort|uniq
203

    
204
##### External dependencies
205

    
206
$(root)/%: _always
207
	+$(subMake)
208
.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error
209

    
210
##### Mapping
211

    
212
dbExport := $(firstword $(wildcard src/db.*.sql))
213
inputFiles := $(wildcard $(exts:%=src/*.%))
214

    
215
+maps = $(filter maps/% $(mappings)/%,$(+_))
216
<in = $(firstword $(filter-out $(+maps),$(+_)))
217
map = $(if $(<in),<$(<in),\
218
$(if $(srcs),$(withCatSrcs),\
219
$(if $(mapEnv),env $(mapEnv),\
220
$(error No input file src/*.$*.{$(exts)}))))\
221
$(root)/map $(+maps)
222
map2db = env out_database=vegbien $(map)
223

    
224
##### Import to VegBIEN
225

    
226
ifneq ($(dbExport)$(inputFiles),)
227

    
228
profileTest = $(if $(profile),$(if $(test),1))
229
profileOnly = -env profile_to=/dev/fd/3 $(map2db) 3>&1 1>&2|\
230
$(bin)/profile_stats /dev/fd/0
231

    
232
log_ = import/$*$(if $(n),.n=$(n),).$(date).log
233
trace = $(log_:.log=.trace)
234
import = -$(if $(profileTest),$(profileOnly),(set -x; "time" env commit=1\
235
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
236
$(if $(n),,>>$(log_))) 2>&1$(if $(log),$(if $(n),|tee -a $(log_))))
237
# don't abort on import errors, which often relate to invalid input data
238

    
239
import: $(addprefix import-,$(tables)) _always ;
240

    
241
import-%: maps/VegBIEN.%.csv _always
242
	$(import)
243
# default:
244
import-%: _always ;
245

    
246
else
247
import: _always ;
248
endif
249

    
250
##### Log files from import
251

    
252
logs := $(wildcard import/*.log import/*.trace)
253

    
254
rm_logs: _always
255
	$(RM) $(logs)
256

    
257
##### Verification of import
258

    
259
verify: $(addprefix verify-,$(tables)) _always ;
260

    
261
verify-%: verify/%.ref verify/%.out _always
262
	-$(diffVerbose) $(+_)
263
# don't abort on verification errors, which are expected during development
264
# default:
265
verify-%: verify/%.out _always
266
	$(if $(shell test -e $< && echo t),cat $<)
267
# don't run if verify/%.out's default do-nothing action was used
268
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
269

    
270
verify = $(if $(reverify),"time" $(psqlAsBien) --set=datasource="'$(datasrc)'"\
271
--no-align --field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@)
272

    
273
verify/%.out: $(mappings)/verify.%.sql _always
274
	$(verify)
275
# default:
276
verify/%.out: _always ;
277

    
278
all += $(wildcard verify/*.out)
279

    
280
ifneq ($(dbExport),)
281
%.ref: %.ref.sql
282
	$(dbAsBien) $(db) <$< >$@
283
endif
284

    
285
##### Editing import
286

    
287
import/rotate: _always
288
	echo "UPDATE party SET organizationname = organizationname||'.$(date)'\
289
WHERE organizationname = '$(datasrc)';"|$(psqlAsBien)
290

    
291
import/rm: _always
292
	echo "DELETE FROM party WHERE organizationname = '$(datasrc)';"|\
293
$(psqlAsBien)
294

    
295
##### Testing
296

    
297
hasOwnRef = $(filter-out %.2-step.xml,$@)
298
testRef = $(1:.2-step.xml=.xml).ref
299

    
300
define runTest
301
@echo "Testing $(abspath $@)..."
302
>$@ env test=1 n=$(test_n) $(1)
303
@(set -x; $(diff) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
304
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
305
$(if $(hasOwnRef),\
306
echo $(emph)"To accept new test output:"$(endEmph);\
307
echo "$(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile";\
308
,\
309
echo $(emph)"Note: The preceding failed test is compared to another test's\
310
output"$(endEmph);\
311
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
312
)\
313
exit $$e;}
314
endef
315

    
316
test2File = $(call runTest,$(map))
317

    
318
tests :=
319

    
320
test/$(via).%.xml: maps/$(via).%.full.csv _always
321
	$(test2File)
322
tests += test/$(via).%.xml
323

    
324
test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always
325
	$(test2File)
326
tests += test/VegBIEN.%.xml
327

    
328
test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always
329
	-$(test2File)
330
# Don't abort tester if only 2-step test fails, as it's often finicky
331
tests += test/VegBIEN.%.2-step.xml
332

    
333
test/import.%.out: maps/VegBIEN.%.csv _always
334
	$(call runTest,$(map2db))
335
tests += test/import.%.out
336

    
337
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
338

    
339
.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted
340

    
341
test: _always $(testOutputs) ;
342

    
343
all += $(testOutputs)
344

    
345
# Accepts a test output: make <test_output_path>-ok
346
%-ok: _always
347
	$(CP) $* $(call testRef,$*)
348

    
349
##### Input-type-specific
350

    
351
# Each input type needs var $(mapEnv) and targets install, uninstall
352

    
353
#### DB export
354

    
355
ifneq ($(dbExport),)
356

    
357
dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbExport)))))
358
db := $(datasrc)
359

    
360
### Installation
361

    
362
install: _always db ;
363

    
364
uninstall: _always rm_db ;
365

    
366
### DB-engine-specific
367

    
368
# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db
369

    
370
ifeq ($(dbEngineExt),my)
371

    
372
dbEngine := MySQL
373

    
374
bienPassword := $(shell cat $(root)/config/bien_password)
375
mysqlAs = mysql --user=$(1) --password='$(bienPassword)'
376
mysqlAsRoot := $(call mysqlAs,root)
377
dbAsBien := $(call mysqlAs,bien)
378

    
379
dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))
380

    
381
define createDb
382
echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)
383
-$(mysqlAsRoot) --database=$(db) <$<
384
endef
385
# ignore errors in db import so that GRANT will still be run
386

    
387
db: $(dbExport) _always
388
	$(if $(dbExists),,$(createDb))
389
	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)
390

    
391
rm_db: _always
392
	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)
393
	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)
394
# ignore errors if grant not defined
395

    
396
## Unrecognized DB engine
397

    
398
else
399
$(error The DB filename $(dbExport) must be db.my.sql)
400
endif
401

    
402
### Other input types
403

    
404
else
405

    
406
install: _always ;
407
uninstall: _always ;
408

    
409
endif
410

    
411
#### DB connection info
412

    
413
ifneq ($(dbEngine),)
414
# Must come after dbEngine is set
415
mapEnv := in_engine=$(dbEngine) in_database=$(db)
416
endif
(2-2/2)