Project

General

Profile

1
##### Configuration
2

    
3
log ?= $(if $(test),,1)
4
profile ?=
5
reverify ?= 1
6
exts ?= csv tsv txt xml
7
test_n ?= 2
8
tablesSort ?= plots organisms stems
9

    
10
##### Vars/functions
11

    
12
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
13

    
14
# Make
15
SHELL := /bin/bash
16
selfMake = $(MAKE) --makefile=../input.Makefile
17
subMake = $(MAKE) $(@F) --directory=$(@D)
18
+_ = $(+:_%=)
19
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
20

    
21
# OS
22
os := $(shell uname)
23
forOs = $(patsubst %,%-$(filter Linux Darwin,$(os)),$(1))
24

    
25
# Formatting
26
SED = sed -$(if $(filter Darwin,$(os)),E,r)
27

    
28
# System
29
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
30

    
31
# Terminal
32
termCols := $(shell tput cols)
33
esc := '['
34
reset := $(esc)'0m'
35
emph := $(esc)'7m '
36
endEmph := ' '$(reset)
37

    
38
# Commands
39
MKDIR = mkdir -p
40
mkdir = $(MKDIR) $(@D)
41
CP = cp -p
42
diff = diff --unified=2
43
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
44
--width=$(termCols),$(diff))
45

    
46
# Paths
47
datasrc := $(notdir $(realpath .))
48
root := $(selfDir_uZPPqC)..
49
bin := $(root)/bin
50
mappings := $(root)/mappings
51

    
52
# Commands
53
selfMap = $(bin)/cols 0 0
54
psqlOpts := --set ON_ERROR_STOP=1 --quiet
55
psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)
56

    
57
# SVN
58
addDir = $(if $(wildcard $(1)/),svn add --depth=empty $(1),svn mkdir $(1))
59
setSvnIgnore = svn propset svn:ignore $(2) $(1)
60
define addDirWithIgnore
61
$(addDir)
62
$(setSvnIgnore)
63
endef
64

    
65
##### General targets
66

    
67
all: _always maps ;
68

    
69
.SUFFIXES: # turn off built-in suffix rules
70
.SECONDARY: # don't automatically delete intermediate files
71
.DELETE_ON_ERROR: # delete target if recipe fails
72

    
73
_always:
74
.PHONY: _always
75

    
76
clean: _always
77
	$(RM) $(all)
78

    
79
remake: _always clean
80
	+$(selfMake)
81
# re-run make so that cache of existing files is reset
82

    
83
make_script = ./$< >$@
84

    
85
%/: % _always ;
86

    
87
# Must come before `%: %.make` to override it
88
src/%: src/%.make _always
89
	(set -x; $(make_script)) 2>>$<.log
90
.PRECIOUS: src/% # save partial outputs of aborted src make scripts
91

    
92
%: %.make _always
93
	$(make_script)
94

    
95
##### SVN
96

    
97
add: _always
98
	$(call setSvnIgnore,.,$$'')
99
	$(call addDirWithIgnore,src,$$'*')
100
	$(call addDirWithIgnore,maps,$$'.~*')
101
	$(call addDirWithIgnore,import,$$'*')
102
	$(call addDirWithIgnore,test,$$'*.out\n*.xml')
103
	$(call addDirWithIgnore,verify,$$'*.out')
104

    
105
##### Installation
106

    
107
reinstall: _always uninstall install ;
108

    
109
##### Maps
110

    
111
srcMap := maps/src.%.csv
112
fullViaMap := maps/%.full.csv
113
directMap := maps/VegBIEN.%.csv
114
allViaMaps := $(filter-out $(srcMap) $(fullViaMap) $(directMap),\
115
$(wildcard maps/*.csv))
116
via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))
117

    
118
coreMap := $(mappings)/$(via)-VegBIEN.%.csv
119
coreSelfMap := $(mappings)/$(via).self.%.csv
120
noEmptyMap := $(mappings)/$(via)-VegBIEN.%.no_empty.csv
121

    
122
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
123
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))
124

    
125
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
126
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
127
$(wildcard maps/VegBIEN.*.csv))
128
tables := $(directMaps:maps/VegBIEN.%.csv=%)
129

    
130
srcMaps := $(filter-out maps/src.join.%.csv,$(wildcard maps/src.*.csv))
131

    
132
# Must come before $(root)/% to override it
133
$(coreSelfMap): _always
134
	-+$(subMake)
135
# ignore errors if $(coreSelfMap) does not exist
136

    
137
# Via maps cleanup
138
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
139
maps/.$(via).%.csv.last_cleanup: maps/$(via).%.csv $(coreSelfMap)
140
	$(bin)/in_place $< $(bin)/subtract $(word 2,$+) 0 1
141
	touch $@
142
# default:
143
maps/.$(via).%.csv.last_cleanup: ;
144
else
145
$(viaMaps): _always
146
	$(selfMake) $(@:maps/%=maps/.%.last_cleanup)
147
endif
148

    
149
maps :=
150

    
151
joinSrcMap = $(if $(wildcard maps/src.$*.csv),$(bin)/in_place $@\
152
$(bin)/intersect maps/src.$*.csv 0)
153

    
154
makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\
155
env ignore=1 $(bin)/union <$+|$(bin)/sort_map >$@,$(CP) $< $@)
156
# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs
157

    
158
maps/$(via).%.full.csv: maps/$(via).%.csv $(coreSelfMap)
159
	$(makeFullCsv)
160
	$(joinSrcMap)
161
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
162

    
163
maps/VegBIEN.%.csv: maps/$(via).%.full.csv $(coreMap)
164
	$(bin)/join <$+|$(bin)/sort_map >$@
165
maps += $(autogenMaps)
166

    
167
maps: $(maps) _always ;
168

    
169
all += $(maps)
170

    
171
##### Maps validation
172

    
173
missing_mappings: _always missing_join_mappings missing_input_mappings ;
174

    
175
missing_%_mappings: _always # stem is one of join|input
176
	@echo $(emph)"Missing $* mappings:"$(endEmph)
177
	@+$(selfMake) remake 2>&1\
178
|$(SED) -n 's/^.*No $* mapping for ([0-9A-Za-z_-]+).*$$/\1/p'\
179
$(if $(filter join,$*),|$(bin)/ucase_first 0)|sort|uniq
180

    
181
##### External dependencies
182

    
183
$(root)/%: _always
184
	+$(subMake)
185
.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error
186

    
187
##### Mapping
188

    
189
dbExport := $(firstword $(wildcard src/db.*.sql))
190
inputFiles := $(wildcard $(exts:%=src/*.%))
191

    
192
+maps = $(filter maps/% $(mappings)/%,$(+_))
193
<in = $(firstword $(filter-out $(+maps),$(+_)))
194
srcs = $(shell $(bin)/sort_filenames $(wildcard $(exts:%=src/*.$*.%)))
195
isXml = $(filter %.xml,$(firstword $(srcs)))
196
map = $(if $(<in),<$(<in),\
197
$(if $(srcs),$(bin)/with_cat$(if $(isXml),,_csv) $(srcs) --,\
198
$(if $(mapEnv),env $(mapEnv),\
199
$(error No input file src/*.$*.{$(exts)}))))\
200
$(root)/map $(+maps)
201
map2db = env out_database=vegbien $(map)
202

    
203
##### Import to VegBIEN
204

    
205
ifneq ($(dbExport)$(inputFiles),)
206

    
207
log_ = import/$*$(if $(n),.n=$(n),).$(date).log
208
trace = $(log_:.log=.trace)
209
import = -(set -x; "time" env commit=1\
210
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
211
$(if $(n),,>>$(log_))) 2>&1$(if $(log),$(if $(n),|tee -a $(log_)))
212
# don't abort on import errors, which often relate to invalid input data
213

    
214
import: $(addprefix import-,$(tables)) _always ;
215

    
216
import-%: maps/VegBIEN.%.csv _always
217
	$(import)
218
# default:
219
import-%: _always ;
220

    
221
else
222
import: _always ;
223
endif
224

    
225
##### Log files from import
226

    
227
logs := $(wildcard import/*.log import/*.trace)
228

    
229
rm_logs: _always
230
	$(RM) $(logs)
231

    
232
##### Verification of import
233

    
234
verify: $(addprefix verify-,$(tables)) _always ;
235

    
236
verify-%: verify/%.ref verify/%.out _always
237
	-$(diffVerbose) $(+_)
238
# don't abort on verification errors, which are expected during development
239
# default:
240
verify-%: verify/%.out _always
241
	$(if $(shell test -e $< && echo t),cat $<)
242
# don't run if verify/%.out's default do-nothing action was used
243
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
244

    
245
verify = $(if $(reverify),"time" $(psqlAsBien) --set=datasource="'$(datasrc)'"\
246
--no-align --field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@)
247

    
248
verify/%.out: $(mappings)/verify.%.sql _always
249
	$(verify)
250
# default:
251
verify/%.out: _always ;
252

    
253
all += $(wildcard verify/*.out)
254

    
255
ifneq ($(dbExport),)
256
%.ref: %.ref.sql
257
	$(dbAsBien) $(db) <$< >$@
258
endif
259

    
260
##### Editing import
261

    
262
import/rotate: _always
263
	echo "UPDATE party SET organizationname = organizationname||'.$(date)'\
264
WHERE organizationname = '$(db)';"|$(psqlAsBien)
265

    
266
import/rm: _always
267
	echo "DELETE FROM party WHERE organizationname = '$(db)';"|$(psqlAsBien)
268

    
269
##### Testing
270

    
271
hasOwnRef = $(filter-out %.2-step.xml,$@)
272
testRef = $(1:.2-step.xml=.xml).ref
273

    
274
define runTest
275
@echo "Testing $(abspath $@)..."
276
>$@ env test=1 n=$(test_n) $(1)
277
@(set -x; $(diff) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
278
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
279
$(if $(hasOwnRef),\
280
echo $(emph)"To accept new test output:"$(endEmph);\
281
echo "$(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile";\
282
,\
283
echo $(emph)"Note: The preceding failed test is compared to another test's\
284
output"$(endEmph);\
285
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
286
)\
287
exit $$e;}
288
endef
289

    
290
test2File = $(call runTest,$(map))
291

    
292
tests :=
293

    
294
test/$(via).%.xml: maps/$(via).%.full.csv _always
295
	$(test2File)
296
tests += test/$(via).%.xml
297

    
298
test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always
299
	$(test2File)
300
tests += test/VegBIEN.%.xml
301

    
302
test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always
303
	-$(test2File)
304
# Don't abort tester if only 2-step test fails, as it's often finicky
305
tests += test/VegBIEN.%.2-step.xml
306

    
307
test/import.%.out: maps/VegBIEN.%.csv _always
308
	$(call runTest,$(map2db))
309
tests += test/import.%.out
310

    
311
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
312

    
313
.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted
314

    
315
test: _always $(testOutputs) ;
316

    
317
all += $(testOutputs)
318

    
319
# Accepts a test output: make <test_output_path>-ok
320
%-ok: _always
321
	$(CP) $* $(call testRef,$*)
322

    
323
##### Input-type-specific
324

    
325
# Each input type needs var $(mapEnv) and targets install, uninstall
326

    
327
#### DB export
328

    
329
ifneq ($(dbExport),)
330

    
331
dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbExport)))))
332
db := $(datasrc)
333

    
334
### Installation
335

    
336
install: _always db ;
337

    
338
uninstall: _always rm_db ;
339

    
340
### DB-engine-specific
341

    
342
# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db
343

    
344
ifeq ($(dbEngineExt),my)
345

    
346
dbEngine := MySQL
347

    
348
bienPassword := $(shell cat $(root)/config/bien_password)
349
mysqlAs = mysql --user=$(1) --password='$(bienPassword)'
350
mysqlAsRoot := $(call mysqlAs,root)
351
dbAsBien := $(call mysqlAs,bien)
352

    
353
dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))
354

    
355
define createDb
356
echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)
357
-$(mysqlAsRoot) --database=$(db) <$<
358
endef
359
# ignore errors in db import so that GRANT will still be run
360

    
361
db: $(dbExport) _always
362
	$(if $(dbExists),,$(createDb))
363
	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)
364

    
365
rm_db: _always
366
	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)
367
	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)
368
# ignore errors if grant not defined
369

    
370
## Unrecognized DB engine
371

    
372
else
373
$(error The DB filename $(dbExport) must be db.my.sql)
374
endif
375

    
376
### Other input types
377

    
378
else
379

    
380
install: _always ;
381
uninstall: _always ;
382

    
383
endif
384

    
385
#### DB connection info
386

    
387
ifneq ($(dbEngine),)
388
# Must come after dbEngine is set
389
mapEnv := in_engine=$(dbEngine) in_database=$(db)
390
endif
(2-2/2)