Project

General

Profile

1
##### Configuration
2

    
3
log ?= $(if $(test),,1)
4
profile ?=
5
reverify ?= 1
6
exts ?= csv tsv txt xml
7
test_n ?= 2
8
tablesSort ?= plots organisms stems specimens
9

    
10
##### Vars/functions
11

    
12
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
13

    
14
# Make
15
SHELL := /bin/bash
16
selfMake = $(MAKE) --makefile=../input.Makefile
17
subMake = $(MAKE) $(@F) --directory=$(@D)
18
+_ = $(+:_%=)
19
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
20

    
21
# OS
22
os := $(shell uname)
23
forOs = $(patsubst %,%-$(filter Linux Darwin,$(os)),$(1))
24

    
25
# Formatting
26
SED = sed -$(if $(filter Darwin,$(os)),E,r)
27

    
28
# System
29
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
30

    
31
# Terminal
32
termCols := $(shell tput cols)
33
esc := '['
34
reset := $(esc)'0m'
35
emph := $(esc)'7m '
36
endEmph := ' '$(reset)
37

    
38
# Commands
39
MKDIR = mkdir -p
40
mkdir = $(MKDIR) $(@D)
41
CP = cp -p
42
diff = diff --unified=2
43
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
44
--width=$(termCols),$(diff))
45

    
46
# Paths
47
datasrc := $(notdir $(realpath .))
48
root := $(selfDir_uZPPqC)..
49
bin := $(root)/bin
50
mappings := $(root)/mappings
51

    
52
# Commands
53
selfMap = $(bin)/cols 0 0
54
psqlOpts := --set ON_ERROR_STOP=1 --quiet
55
psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)
56

    
57
# SVN
58
addDir = $(if $(wildcard $(1)/),svn add --depth=empty $(1),svn mkdir $(1))
59
setSvnIgnore = svn propset svn:ignore $(2) $(1)
60
define addDirWithIgnore
61
$(addDir)
62
$(setSvnIgnore)
63
endef
64

    
65
##### General targets
66

    
67
all: _always maps ;
68

    
69
.SUFFIXES: # turn off built-in suffix rules
70
.SECONDARY: # don't automatically delete intermediate files
71
.DELETE_ON_ERROR: # delete target if recipe fails
72

    
73
_always:
74
.PHONY: _always
75

    
76
clean: _always
77
	$(RM) $(all)
78

    
79
remake: _always clean
80
	+$(selfMake)
81
# re-run make so that cache of existing files is reset
82

    
83
make_script = ./$< >$@
84

    
85
%/: % _always ;
86

    
87
# Must come before `%: %.make` to override it
88
src/%: src/%.make _always
89
	(set -x; $(make_script)) 2>>$<.log
90
.PRECIOUS: src/% # save partial outputs of aborted src make scripts
91

    
92
%: %.make _always
93
	$(make_script)
94

    
95
##### SVN
96

    
97
add: _always
98
	$(call setSvnIgnore,.,$$'')
99
	$(call addDirWithIgnore,src,$$'*')
100
	$(call addDirWithIgnore,maps,$$'.~*')
101
	$(call addDirWithIgnore,import,$$'*')
102
	$(call addDirWithIgnore,test,$$'*.out\n*.xml')
103
	$(call addDirWithIgnore,verify,$$'*.out')
104

    
105
##### Installation
106

    
107
reinstall: _always uninstall install ;
108

    
109
##### Existing maps discovery
110

    
111
srcMap := maps/src.%.csv
112
fullViaMap := maps/%.full.csv
113
directMap := maps/VegBIEN.%.csv
114
allViaMaps := $(filter-out $(srcMap) $(fullViaMap) $(directMap),\
115
$(wildcard maps/*.csv))
116
via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))
117

    
118
coreMap := $(mappings)/$(via)-VegBIEN.%.csv
119
coreSelfMap := $(mappings)/$(via).self.%.csv
120
noEmptyMap := $(mappings)/$(via)-VegBIEN.%.no_empty.csv
121

    
122
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
123
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))
124

    
125
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
126
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
127
$(wildcard maps/VegBIEN.*.csv))
128
tables := $(directMaps:maps/VegBIEN.%.csv=%)
129

    
130
##### Sources
131

    
132
srcs = $(shell $(bin)/sort_filenames $(wildcard $(exts:%=src/*.$*.%)))
133
isXml = $(filter %.xml,$(firstword $(srcs)))
134
catSrcs = $(bin)/cat$(if $(isXml),,_csv) $(srcs)
135
withCatSrcs = $(catSrcs:$(bin)/%=$(bin)/with_%) --
136

    
137
cat: $(addprefix cat-,$(tables)) _always ;
138

    
139
cat-%: _always
140
	$(catSrcs)
141

    
142
##### Maps building
143

    
144
# Maps to (try to) build are added to one of these
145
maps :=
146
createOnlyMaps :=
147

    
148
srcMaps := $(wildcard maps/src.*.csv)
149

    
150
mkSrcMap = env datasrc=$(datasrc) $(withCatSrcs) $(bin)/src_map >$@
151

    
152
# Autogen src maps with known table names
153
maps/src.%.csv: # no prereqs so only built if doesn't exist
154
	$(if $(srcs),$(if $(isXml),,$(mkSrcMap)))
155
# only build if CSV srcs exist for that table name
156
# Try all table names
157
createOnlyMaps += $(tablesSort:%=maps/src.%.csv)
158

    
159
# Must come before $(root)/% to override it
160
$(coreSelfMap): _always
161
	-+$(subMake)
162
# ignore errors if $(coreSelfMap) does not exist
163

    
164
# Via maps cleanup
165
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
166
maps/.$(via).%.csv.last_cleanup: maps/$(via).%.csv $(coreMap)
167
	$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1
168
	touch $@
169
else
170
$(viaMaps): _always
171
	$(selfMake) $(@:maps/%=maps/.%.last_cleanup)
172
endif
173

    
174
joinSrcMap = $(if $(wildcard maps/src.$*.csv),$(bin)/in_place $@\
175
$(bin)/intersect maps/src.$*.csv 0)
176

    
177
makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\
178
env ignore=1 $(bin)/union <$+|$(bin)/sort_map >$@,$(CP) $< $@)
179
# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs
180

    
181
maps/$(via).%.full.csv: maps/$(via).%.csv $(coreSelfMap)
182
	$(makeFullCsv)
183
	$(joinSrcMap)
184
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
185

    
186
maps/VegBIEN.%.csv: maps/$(via).%.full.csv $(coreMap)
187
	$(bin)/join <$+|$(bin)/sort_map >$@
188
maps += $(autogenMaps)
189

    
190
maps: $(createOnlyMaps) $(maps) _always ;
191

    
192
all += $(maps)
193

    
194
##### Maps validation
195

    
196
missing_mappings: _always missing_join_mappings missing_input_mappings ;
197

    
198
missing_%_mappings: _always # stem is one of join|input
199
	@echo $(emph)"Missing $* mappings:"$(endEmph)
200
	@+$(selfMake) remake 2>&1\
201
|$(SED) -n 's/^.*No $* mapping for ([0-9A-Za-z_-]+).*$$/\1/p'\
202
$(if $(filter join,$*),|$(bin)/ucase_first 0)|sort|uniq
203

    
204
##### External dependencies
205

    
206
$(root)/%: _always
207
	+$(subMake)
208
.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error
209

    
210
##### Mapping
211

    
212
dbExport := $(firstword $(wildcard src/db.*.sql))
213
inputFiles := $(wildcard $(exts:%=src/*.%))
214

    
215
+maps = $(filter maps/% $(mappings)/%,$(+_))
216
<in = $(firstword $(filter-out $(+maps),$(+_)))
217
map = $(if $(<in),<$(<in),\
218
$(if $(srcs),$(withCatSrcs),\
219
$(if $(mapEnv),env $(mapEnv),\
220
$(error No input file src/*.$*.{$(exts)}))))\
221
$(root)/map $(+maps)
222
map2db = env out_database=vegbien $(map)
223

    
224
##### Import to VegBIEN
225

    
226
ifneq ($(dbExport)$(inputFiles),)
227

    
228
log_ = import/$*$(if $(n),.n=$(n),).$(date).log
229
trace = $(log_:.log=.trace)
230
import = -(set -x; "time" env commit=1\
231
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
232
$(if $(n),,>>$(log_))) 2>&1$(if $(log),$(if $(n),|tee -a $(log_)))
233
# don't abort on import errors, which often relate to invalid input data
234

    
235
import: $(addprefix import-,$(tables)) _always ;
236

    
237
import-%: maps/VegBIEN.%.csv _always
238
	$(import)
239
# default:
240
import-%: _always ;
241

    
242
else
243
import: _always ;
244
endif
245

    
246
##### Log files from import
247

    
248
logs := $(wildcard import/*.log import/*.trace)
249

    
250
rm_logs: _always
251
	$(RM) $(logs)
252

    
253
##### Verification of import
254

    
255
verify: $(addprefix verify-,$(tables)) _always ;
256

    
257
verify-%: verify/%.ref verify/%.out _always
258
	-$(diffVerbose) $(+_)
259
# don't abort on verification errors, which are expected during development
260
# default:
261
verify-%: verify/%.out _always
262
	$(if $(shell test -e $< && echo t),cat $<)
263
# don't run if verify/%.out's default do-nothing action was used
264
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
265

    
266
verify = $(if $(reverify),"time" $(psqlAsBien) --set=datasource="'$(datasrc)'"\
267
--no-align --field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@)
268

    
269
verify/%.out: $(mappings)/verify.%.sql _always
270
	$(verify)
271
# default:
272
verify/%.out: _always ;
273

    
274
all += $(wildcard verify/*.out)
275

    
276
ifneq ($(dbExport),)
277
%.ref: %.ref.sql
278
	$(dbAsBien) $(db) <$< >$@
279
endif
280

    
281
##### Editing import
282

    
283
import/rotate: _always
284
	echo "UPDATE party SET organizationname = organizationname||'.$(date)'\
285
WHERE organizationname = '$(db)';"|$(psqlAsBien)
286

    
287
import/rm: _always
288
	echo "DELETE FROM party WHERE organizationname = '$(db)';"|$(psqlAsBien)
289

    
290
##### Testing
291

    
292
hasOwnRef = $(filter-out %.2-step.xml,$@)
293
testRef = $(1:.2-step.xml=.xml).ref
294

    
295
define runTest
296
@echo "Testing $(abspath $@)..."
297
>$@ env test=1 n=$(test_n) $(1)
298
@(set -x; $(diff) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
299
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
300
$(if $(hasOwnRef),\
301
echo $(emph)"To accept new test output:"$(endEmph);\
302
echo "$(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile";\
303
,\
304
echo $(emph)"Note: The preceding failed test is compared to another test's\
305
output"$(endEmph);\
306
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
307
)\
308
exit $$e;}
309
endef
310

    
311
test2File = $(call runTest,$(map))
312

    
313
tests :=
314

    
315
test/$(via).%.xml: maps/$(via).%.full.csv _always
316
	$(test2File)
317
tests += test/$(via).%.xml
318

    
319
test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always
320
	$(test2File)
321
tests += test/VegBIEN.%.xml
322

    
323
test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always
324
	-$(test2File)
325
# Don't abort tester if only 2-step test fails, as it's often finicky
326
tests += test/VegBIEN.%.2-step.xml
327

    
328
test/import.%.out: maps/VegBIEN.%.csv _always
329
	$(call runTest,$(map2db))
330
tests += test/import.%.out
331

    
332
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
333

    
334
.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted
335

    
336
test: _always $(testOutputs) ;
337

    
338
all += $(testOutputs)
339

    
340
# Accepts a test output: make <test_output_path>-ok
341
%-ok: _always
342
	$(CP) $* $(call testRef,$*)
343

    
344
##### Input-type-specific
345

    
346
# Each input type needs var $(mapEnv) and targets install, uninstall
347

    
348
#### DB export
349

    
350
ifneq ($(dbExport),)
351

    
352
dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbExport)))))
353
db := $(datasrc)
354

    
355
### Installation
356

    
357
install: _always db ;
358

    
359
uninstall: _always rm_db ;
360

    
361
### DB-engine-specific
362

    
363
# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db
364

    
365
ifeq ($(dbEngineExt),my)
366

    
367
dbEngine := MySQL
368

    
369
bienPassword := $(shell cat $(root)/config/bien_password)
370
mysqlAs = mysql --user=$(1) --password='$(bienPassword)'
371
mysqlAsRoot := $(call mysqlAs,root)
372
dbAsBien := $(call mysqlAs,bien)
373

    
374
dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))
375

    
376
define createDb
377
echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)
378
-$(mysqlAsRoot) --database=$(db) <$<
379
endef
380
# ignore errors in db import so that GRANT will still be run
381

    
382
db: $(dbExport) _always
383
	$(if $(dbExists),,$(createDb))
384
	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)
385

    
386
rm_db: _always
387
	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)
388
	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)
389
# ignore errors if grant not defined
390

    
391
## Unrecognized DB engine
392

    
393
else
394
$(error The DB filename $(dbExport) must be db.my.sql)
395
endif
396

    
397
### Other input types
398

    
399
else
400

    
401
install: _always ;
402
uninstall: _always ;
403

    
404
endif
405

    
406
#### DB connection info
407

    
408
ifneq ($(dbEngine),)
409
# Must come after dbEngine is set
410
mapEnv := in_engine=$(dbEngine) in_database=$(db)
411
endif
(2-2/2)