Project

General

Profile

1
##### Configuration
2

    
3
log ?= $(if $(test),,1)
4
profile ?=
5
exts ?= csv tsv txt xml
6
test_n ?= 2
7
tablesSort ?= plots organisms stems
8

    
9
##### Vars/functions
10

    
11
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
12

    
13
# Make
14
SHELL := /bin/bash
15
selfMake = $(MAKE) --makefile=../input.Makefile
16
subMake = $(MAKE) $(@F) --directory=$(@D)
17
+_ = $(+:_%=)
18
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
19

    
20
# System
21
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
22

    
23
# Terminal
24
termCols := $(shell tput cols)
25
esc := '['
26
reset := $(esc)'0m'
27
emph := $(esc)'7m '
28
endEmph := ' '$(reset)
29

    
30
# Commands
31
MKDIR = mkdir -p
32
mkdir = $(MKDIR) $(@D)
33
CP = cp -p
34
diff = diff --unified=2
35
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
36
--width=$(termCols),$(diff))
37

    
38
# Paths
39
datasrc := $(notdir $(realpath .))
40
root := $(selfDir_uZPPqC)..
41
bin := $(root)/bin
42
mappings := $(root)/mappings
43

    
44
# Commands
45
join = $(bin)/join_union_sort
46
selfMap = $(bin)/cols 0 0
47
psqlOpts := --set ON_ERROR_STOP=1 --quiet
48
psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)
49

    
50
##### General targets
51

    
52
all: _always maps ;
53

    
54
.SUFFIXES: # turn off built-in suffix rules
55
.SECONDARY: # don't automatically delete intermediate files
56
.DELETE_ON_ERROR: # delete target if recipe fails
57

    
58
_always:
59
.PHONY: _always
60

    
61
clean: _always
62
	$(RM) $(all)
63

    
64
%.out: %.make _always
65
	./$* >$@
66

    
67
##### SVN
68

    
69
svn_props: _always
70
	svn propset svn:ignore $$'*.log\n*.trace\nsrc*' .
71
	$(if $(wildcard maps/),svn propset svn:ignore $$'.~*' maps)
72
	$(if $(wildcard src/),svn propset svn:ignore $$'*' src)
73
	$(if $(wildcard test/),svn propset svn:ignore $$'*.out\n*.xml' test)
74
	$(if $(wildcard verify/),svn propset svn:ignore $$'*.out' verify)
75

    
76
##### Installation
77

    
78
reinstall: _always uninstall install ;
79

    
80
##### Maps
81

    
82
srcMap := maps/src.%.csv
83
fullViaMap := maps/%.full.csv
84
directMap := maps/VegBIEN.%.csv
85
allViaMaps := $(filter-out $(srcMap) $(fullViaMap) $(directMap),\
86
$(wildcard maps/*.csv))
87
via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))
88

    
89
coreMap := $(mappings)/$(via)-VegBIEN.%.csv
90
coreSelfMap := $(mappings)/$(via).self.%.csv
91
noEmptyMap := $(mappings)/$(via)-VegBIEN.%.no_empty.csv
92

    
93
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
94
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))
95

    
96
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
97
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
98
$(wildcard maps/VegBIEN.*.csv))
99
tables := $(directMaps:maps/VegBIEN.%.csv=%)
100

    
101
srcMaps := $(filter-out maps/src.join.%.csv,$(wildcard maps/src.*.csv))
102

    
103
# Must come before $(root)/% to override it
104
$(coreSelfMap): _always
105
	-+$(subMake)
106
# ignore errors if $(coreSelfMap) does not exist
107

    
108
# Via maps cleanup
109
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
110
maps/.$(via).%.csv.last_cleanup: maps/$(via).%.csv $(coreSelfMap)
111
	$(bin)/in_place $< $(bin)/subtract $(word 2,$+) 0 1
112
	touch $@
113
# default:
114
maps/.$(via).%.csv.last_cleanup: ;
115
else
116
$(viaMaps): _always
117
	$(selfMake) $(@:maps/%=maps/.%.last_cleanup)
118
endif
119

    
120
maps :=
121

    
122
joinSrcMap = $(if $(wildcard maps/src.$*.csv),$(bin)/in_place $@\
123
$(bin)/intersect maps/src.$*.csv 0)
124

    
125
makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\
126
env ignore=1 $(bin)/union <$+|$(bin)/sort_map >$@,$(CP) $< $@)
127
# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs
128

    
129
maps/$(via).%.full.csv: maps/$(via).%.csv $(coreSelfMap)
130
	$(makeFullCsv)
131
	$(joinSrcMap)
132
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
133

    
134
maps/VegBIEN.%.csv: maps/$(via).%.full.csv $(coreMap)
135
	$(bin)/join <$+|$(bin)/sort_map >$@
136
maps += $(autogenMaps)
137

    
138
maps: $(maps) _always ;
139

    
140
all += $(maps)
141

    
142
##### External dependencies
143

    
144
$(root)/%: _always
145
	+$(subMake)
146

    
147
##### Mapping
148

    
149
dbExport := $(firstword $(wildcard src/db.*.sql))
150
inputFiles := $(wildcard $(exts:%=src/*.%))
151

    
152
+maps = $(filter maps/% $(mappings)/%,$(+_))
153
<in = $(firstword $(filter-out $(+maps),$(+_)))
154
srcs = $(sort $(wildcard $(exts:%=src/*.$*.%)))
155
map = $(if $(<in),<$(<in),\
156
$(if $(srcs),$(bin)/with_cat_csv $(srcs) --,\
157
$(if $(mapEnv),env $(mapEnv),\
158
$(error No input file src/*.$*.{$(exts)}))))\
159
$(root)/map $(+maps)
160
map2db = env out_database=vegbien $(map)
161

    
162
##### Import to VegBIEN
163

    
164
ifneq ($(dbExport)$(inputFiles),)
165

    
166
log_ = $(@:-all=)$(if $(n),.n=$(n),).$(date).log
167
trace = $(log_:.log=.trace)
168
import = -(set -x; "time" env commit=1 verbose=1\
169
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
170
$(if $(n),,>>$(log_))) 2>&1$(if $(log),$(if $(n),|tee -a $(log_)))
171
# don't abort on import errors, which often relate to invalid input data
172

    
173
import: $(addprefix import-,$(tables)) _always ;
174

    
175
import-%: maps/VegBIEN.%.csv _always
176
	$(import)
177
# default:
178
import-%: _always ;
179

    
180
else
181
import: _always ;
182
endif
183

    
184
##### Log files from import
185

    
186
logs := $(wildcard *.log *.trace)
187

    
188
rm_logs: _always
189
	$(RM) $(logs)
190

    
191
##### Verification of import
192

    
193
verify: $(addprefix verify-,$(tables)) _always ;
194

    
195
verify-%: verify/%.ref verify/%.out _always
196
	-$(diffVerbose) $(+_)
197
# don't abort on verification errors, which are expected during development
198
# default:
199
verify-%: verify/%.out _always
200
	$(if $(shell test -e $< && echo t),cat $<)
201
# don't run if verify/%.out's default do-nothing action was used
202
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
203

    
204
define verify
205
$(mkdir)
206
"time" $(psqlAsBien) --set=datasource="'$(datasrc)'" --no-align\
207
--field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@
208
endef
209

    
210
verify/%.out: $(mappings)/verify.%.sql _always
211
	$(verify)
212
# default:
213
verify/%.out: _always ;
214

    
215
all += $(wildcard verify/*.out)
216

    
217
ifneq ($(dbExport),)
218
%.ref: %.ref.sql
219
	$(dbAsBien) $(db) <$< >$@
220
endif
221

    
222
##### Testing
223

    
224
ifneq ($(wildcard test/),)
225

    
226
hasOwnRef = $(filter-out %.2-step.xml,$@)
227
testRef = $(1:.2-step.xml=.xml).ref
228

    
229
define runTest
230
@echo "Testing $(abspath $@)..."
231
>$@ env test=1 n=$(test_n) $(1)
232
@(set -x; $(diff) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
233
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
234
$(if $(hasOwnRef),\
235
echo $(emph)"To accept new test output:"$(endEmph);\
236
echo "$(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile";\
237
,\
238
echo $(emph)"Note: The preceding failed test is compared to another test's\
239
output"$(endEmph);\
240
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
241
)\
242
exit $$e;}
243
endef
244

    
245
test2File = $(call runTest,$(map))
246

    
247
tests :=
248

    
249
test/$(via).%.xml: maps/$(via).%.full.csv _always
250
	$(test2File)
251
tests += test/$(via).%.xml
252

    
253
test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always
254
	$(test2File)
255
tests += test/VegBIEN.%.xml
256

    
257
test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always
258
	-$(test2File)
259
# Don't abort tester if only 2-step test fails, as it's often finicky
260
tests += test/VegBIEN.%.2-step.xml
261

    
262
test/import.%.out: maps/VegBIEN.%.csv _always
263
	$(call runTest,$(map2db))
264
tests += test/import.%.out
265

    
266
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
267

    
268
.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted
269

    
270
test: _always $(testOutputs) ;
271

    
272
all += $(testOutputs)
273

    
274
# Accepts a test output: make <test_output_path>-ok
275
%-ok: _always
276
	$(CP) $* $(call testRef,$*)
277

    
278
else
279
test: _always ;
280
endif
281

    
282
##### Input-type-specific
283

    
284
# Each input type needs var $(mapEnv) and targets install, uninstall
285

    
286
#### DB export
287

    
288
ifneq ($(dbExport),)
289

    
290
dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbExport)))))
291
db := $(datasrc)
292

    
293
### Installation
294

    
295
install: _always db ;
296

    
297
uninstall: _always rm_db ;
298

    
299
### DB-engine-specific
300

    
301
# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db
302

    
303
ifeq ($(dbEngineExt),my)
304

    
305
dbEngine := MySQL
306

    
307
bienPassword := $(shell cat $(root)/config/bien_password)
308
mysqlAs = mysql --user=$(1) --password='$(bienPassword)'
309
mysqlAsRoot := $(call mysqlAs,root)
310
dbAsBien := $(call mysqlAs,bien)
311

    
312
dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))
313

    
314
define createDb
315
echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)
316
-$(mysqlAsRoot) --database=$(db) <$<
317
endef
318
# ignore errors in db import so that GRANT will still be run
319

    
320
db: $(dbExport) _always
321
	$(if $(dbExists),,$(createDb))
322
	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)
323

    
324
rm_db: _always
325
	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)
326
	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)
327
# ignore errors if grant not defined
328

    
329
## Unrecognized DB engine
330

    
331
else
332
$(error The DB filename $(dbExport) must be db.my.sql)
333
endif
334

    
335
### Other input types
336

    
337
else
338

    
339
install: _always ;
340
uninstall: _always ;
341

    
342
endif
343

    
344
#### DB connection info
345

    
346
ifneq ($(dbEngine),)
347
# Must come after dbEngine is set
348
mapEnv := in_engine=$(dbEngine) in_database=$(db)
349
endif
(2-2/2)