Project

General

Profile

1 783 aaronmk
##### Configuration
2
3 1921 aaronmk
# Command line
4 1190 aaronmk
log ?= $(if $(test),,1)
5 1257 aaronmk
profile ?=
6 1983 aaronmk
quiet ?=
7 1620 aaronmk
reverify ?= 1
8 1990 aaronmk
use_staged ?= $(by_col)
9 1921 aaronmk
10
# Makefile
11
tablesSort ?= plots organisms stems specimens
12 1386 aaronmk
exts ?= csv tsv txt xml
13 783 aaronmk
test_n ?= 2
14
15
##### Vars/functions
16
17 640 aaronmk
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
18 250 aaronmk
19 1821 aaronmk
# Paths
20
datasrc := $(notdir $(realpath .))
21
root := $(selfDir_uZPPqC)..
22
bin := $(root)/bin
23
mappings := $(root)/mappings
24
25 368 aaronmk
# Make
26 640 aaronmk
SHELL := /bin/bash
27 1509 aaronmk
selfMake = $(MAKE) --makefile=../input.Makefile
28 1821 aaronmk
subMake = $(MAKE) $(@:$(root)/%=%) --directory=$(root)
29 404 aaronmk
+_ = $(+:_%=)
30 368 aaronmk
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
31
32 1742 aaronmk
# OS
33
os := $(shell uname)
34
forOs = $(patsubst %,%-$(filter Linux Darwin,$(os)),$(1))
35
36
# Formatting
37
SED = sed -$(if $(filter Darwin,$(os)),E,r)
38
39 790 aaronmk
# System
40
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
41
42 640 aaronmk
# Terminal
43 1184 aaronmk
termCols := $(shell tput cols)
44 640 aaronmk
esc := '['
45
reset := $(esc)'0m'
46
emph := $(esc)'7m '
47
endEmph := ' '$(reset)
48
49 368 aaronmk
# Commands
50 1245 aaronmk
MKDIR = mkdir -p
51
mkdir = $(MKDIR) $(@D)
52 395 aaronmk
CP = cp -p
53 1184 aaronmk
diff = diff --unified=2
54
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
55
--width=$(termCols),$(diff))
56 368 aaronmk
57 1918 aaronmk
# BIEN commands
58 1524 aaronmk
selfMap = $(bin)/cols 0 0
59 1081 aaronmk
psqlOpts := --set ON_ERROR_STOP=1 --quiet
60 1289 aaronmk
psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)
61 353 aaronmk
62 1594 aaronmk
# SVN
63
addDir = $(if $(wildcard $(1)/),svn add --depth=empty $(1),svn mkdir $(1))
64 1771 aaronmk
setSvnIgnore = svn propset svn:ignore $(2) $(1)
65 1594 aaronmk
define addDirWithIgnore
66
$(addDir)
67 1771 aaronmk
$(setSvnIgnore)
68 1594 aaronmk
endef
69
70 783 aaronmk
##### General targets
71 250 aaronmk
72 418 aaronmk
all: _always maps ;
73 247 aaronmk
74 1260 aaronmk
.SUFFIXES: # turn off built-in suffix rules
75
.SECONDARY: # don't automatically delete intermediate files
76 1354 aaronmk
.DELETE_ON_ERROR: # delete target if recipe fails
77 247 aaronmk
78 383 aaronmk
_always:
79
.PHONY: _always
80 250 aaronmk
81 383 aaronmk
clean: _always
82 256 aaronmk
	$(RM) $(all)
83
84 1742 aaronmk
remake: _always clean
85
	+$(selfMake)
86
# re-run make so that cache of existing files is reset
87
88 1627 aaronmk
make_script = ./$< >$@
89 368 aaronmk
90 1604 aaronmk
%/: % _always ;
91
92 1627 aaronmk
# Must come before `%: %.make` to override it
93
src/%: src/%.make _always
94 1663 aaronmk
	(set -x; $(make_script)) 2>>$<.log
95 1627 aaronmk
.PRECIOUS: src/% # save partial outputs of aborted src make scripts
96
97
%: %.make _always
98
	$(make_script)
99
100 783 aaronmk
##### SVN
101 250 aaronmk
102 1594 aaronmk
add: _always
103 1771 aaronmk
	$(call setSvnIgnore,.,$$'')
104 1594 aaronmk
	$(call addDirWithIgnore,src,$$'*')
105
	$(call addDirWithIgnore,maps,$$'.~*')
106
	$(call addDirWithIgnore,import,$$'*')
107
	$(call addDirWithIgnore,test,$$'*.out\n*.xml')
108
	$(call addDirWithIgnore,verify,$$'*.out')
109 766 aaronmk
110 1969 aaronmk
##### Existing maps discovery
111 1955 aaronmk
112
inputFiles := $(wildcard $(exts:%=src/*.%))
113
ifeq ($(inputFiles),)
114
dbExport := $(firstword $(wildcard src/db.*.sql))
115
endif
116
117 1289 aaronmk
srcMap := maps/src.%.csv
118 1139 aaronmk
fullViaMap := maps/%.full.csv
119 1289 aaronmk
directMap := maps/VegBIEN.%.csv
120
allViaMaps := $(filter-out $(srcMap) $(fullViaMap) $(directMap),\
121 1139 aaronmk
$(wildcard maps/*.csv))
122 728 aaronmk
via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))
123 783 aaronmk
124 775 aaronmk
coreMap := $(mappings)/$(via)-VegBIEN.%.csv
125 1524 aaronmk
coreSelfMap := $(mappings)/$(via).self.%.csv
126 783 aaronmk
127 749 aaronmk
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
128 1139 aaronmk
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))
129 783 aaronmk
130 728 aaronmk
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
131
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
132
$(wildcard maps/VegBIEN.*.csv))
133 772 aaronmk
tables := $(directMaps:maps/VegBIEN.%.csv=%)
134 728 aaronmk
135 1969 aaronmk
##### Sources
136
137
srcs = $(shell $(bin)/sort_filenames $(wildcard $(exts:%=src/*.$*.%)))
138
isCsv = $(if $(srcs),$(if $(filter %.xml,$(srcs)),,1))
139
    # true if $(srcs) non-empty and contains no *.xml
140
catSrcs = $(bin)/cat$(if $(isCsv),_csv) $(srcs)
141
withCatSrcs = $(catSrcs:$(bin)/%=$(bin)/with_%) --
142
143 1986 aaronmk
# Usage: `make {--silent|-s} inputs/<datasrc>/cat` (don't echo make commands)
144 1969 aaronmk
cat: $(addprefix cat-,$(tables)) _always ;
145
146
cat-%: _always
147
	$(catSrcs)
148
149 1921 aaronmk
##### Installation
150
151
install: _always src/install import/install ;
152
153
uninstall: _always import/uninstall src/uninstall ;
154
155
reinstall: _always uninstall install ;
156
157
##### Staging tables
158
159
import/install: _always import/schema $(addprefix import/install-,$(tables)) ;
160
161
import/uninstall: _always import/rm_schema ;
162
# rm_schema will also drop all staging tables
163
164
import/schema: _always
165
	-echo 'CREATE SCHEMA "$(datasrc)";'|$(psqlAsBien)
166
# ignore errors if schema exists
167
168
import/rm_schema: _always
169
	echo 'DROP SCHEMA IF EXISTS "$(datasrc)" CASCADE;'|$(psqlAsBien)
170
171
import/install-%: _always
172 1955 aaronmk
	$(if $(isCsv),$(import_install_))
173
import_install_ = (prefix=; . $(bin)/vegbien_dest; "time" env schema=$(datasrc)\
174 1971 aaronmk
table=$* $(bin)/csv2db $(catSrcs) $(if $(quiet),&>,2>&1|tee )import/install.log)
175 1921 aaronmk
176 1777 aaronmk
##### Maps building
177
178 1789 aaronmk
# Maps to (try to) build are added to one of these
179 1779 aaronmk
maps :=
180 1789 aaronmk
createOnlyMaps :=
181 1779 aaronmk
182 1775 aaronmk
srcMaps := $(wildcard maps/src.*.csv)
183 1289 aaronmk
184 1931 aaronmk
mkSrcMap = $(catSrcs)|env datasrc=$(datasrc) $(bin)/src_map >$@
185 1779 aaronmk
186
# Autogen src maps with known table names
187
maps/src.%.csv: # no prereqs so only built if doesn't exist
188 1955 aaronmk
	$(if $(isCsv),$(mkSrcMap))
189 1779 aaronmk
# only build if CSV srcs exist for that table name
190
# Try all table names
191 1789 aaronmk
createOnlyMaps += $(tablesSort:%=maps/src.%.csv)
192 1779 aaronmk
193 1273 aaronmk
# Must come before $(root)/% to override it
194 1524 aaronmk
$(coreSelfMap): _always
195 1273 aaronmk
	-+$(subMake)
196 1524 aaronmk
# ignore errors if $(coreSelfMap) does not exist
197 1273 aaronmk
198 1509 aaronmk
# Via maps cleanup
199
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
200 1787 aaronmk
maps/.$(via).%.csv.last_cleanup: maps/$(via).%.csv $(coreMap)
201 2022 aaronmk
	$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1 2
202 1509 aaronmk
	touch $@
203 2022 aaronmk
# Include comment column so commented mappings are never removed
204 1509 aaronmk
else
205
$(viaMaps): _always
206
	$(selfMake) $(@:maps/%=maps/.%.last_cleanup)
207
endif
208 1273 aaronmk
209 1526 aaronmk
joinSrcMap = $(if $(wildcard maps/src.$*.csv),$(bin)/in_place $@\
210
$(bin)/intersect maps/src.$*.csv 0)
211
212 1273 aaronmk
makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\
213 1817 aaronmk
env ignore=1 $(bin)/union <$+|,<$< )$(bin)/sort_map >$@
214 1524 aaronmk
# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs
215 1273 aaronmk
216 1524 aaronmk
maps/$(via).%.full.csv: maps/$(via).%.csv $(coreSelfMap)
217 1273 aaronmk
	$(makeFullCsv)
218 1526 aaronmk
	$(joinSrcMap)
219 1139 aaronmk
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
220
221 1530 aaronmk
maps/VegBIEN.%.csv: maps/$(via).%.full.csv $(coreMap)
222
	$(bin)/join <$+|$(bin)/sort_map >$@
223
maps += $(autogenMaps)
224
225 1789 aaronmk
maps: $(createOnlyMaps) $(maps) _always ;
226 1139 aaronmk
227
all += $(maps)
228
229 1742 aaronmk
##### Maps validation
230
231
missing_mappings: _always missing_join_mappings missing_input_mappings ;
232
233
missing_%_mappings: _always # stem is one of join|input
234
	@echo $(emph)"Missing $* mappings:"$(endEmph)
235
	@+$(selfMake) remake 2>&1\
236 1800 aaronmk
|$(SED) -n 's/^.*No.* $* mapping for ([0-9A-Za-z_-]+).*$$/\1/p'\
237 1742 aaronmk
$(if $(filter join,$*),|$(bin)/ucase_first 0)|sort|uniq
238
239 1509 aaronmk
##### External dependencies
240
241
$(root)/%: _always
242
	+$(subMake)
243 1628 aaronmk
.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error
244 1509 aaronmk
245 783 aaronmk
##### Mapping
246 368 aaronmk
247 775 aaronmk
+maps = $(filter maps/% $(mappings)/%,$(+_))
248 1451 aaronmk
<in = $(firstword $(filter-out $(+maps),$(+_)))
249
map = $(if $(<in),<$(<in),\
250 1983 aaronmk
$(if $(use_staged),env in_database=vegbien in_schema=$(datasrc) in_table=$*,\
251 1777 aaronmk
$(if $(srcs),$(withCatSrcs),\
252 1451 aaronmk
$(if $(mapEnv),env $(mapEnv),\
253 1983 aaronmk
$(error No input file src/*.$*.{$(exts)})))))\
254 1451 aaronmk
$(root)/map $(+maps)
255 772 aaronmk
map2db = env out_database=vegbien $(map)
256 368 aaronmk
257 783 aaronmk
##### Import to VegBIEN
258 772 aaronmk
259 1378 aaronmk
ifneq ($(dbExport)$(inputFiles),)
260 772 aaronmk
261 1853 aaronmk
profileTest = $(if $(profile),$(if $(test),1))
262
profileOnly = -env profile_to=/dev/fd/3 $(map2db) 3>&1 1>&2|\
263
$(bin)/profile_stats /dev/fd/0
264
265 1594 aaronmk
log_ = import/$*$(if $(n),.n=$(n),).$(date).log
266 1190 aaronmk
trace = $(log_:.log=.trace)
267 1853 aaronmk
import = -$(if $(profileTest),$(profileOnly),(set -x; "time" env commit=1\
268 1190 aaronmk
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
269 1853 aaronmk
$(if $(n),,>>$(log_))) 2>&1$(if $(log),$(if $(n),|tee -a $(log_))))
270 1088 aaronmk
# don't abort on import errors, which often relate to invalid input data
271 624 aaronmk
272 1193 aaronmk
import: $(addprefix import-,$(tables)) _always ;
273 718 aaronmk
274
import-%: maps/VegBIEN.%.csv _always
275
	$(import)
276 772 aaronmk
# default:
277
import-%: _always ;
278 718 aaronmk
279 772 aaronmk
else
280
import: _always ;
281
endif
282
283 790 aaronmk
##### Log files from import
284
285 1594 aaronmk
logs := $(wildcard import/*.log import/*.trace)
286 339 aaronmk
287 383 aaronmk
rm_logs: _always
288 339 aaronmk
	$(RM) $(logs)
289
290 783 aaronmk
##### Verification of import
291 264 aaronmk
292 1194 aaronmk
verify: $(addprefix verify-,$(tables)) _always ;
293
294 1199 aaronmk
verify-%: verify/%.ref verify/%.out _always
295 1184 aaronmk
	-$(diffVerbose) $(+_)
296 1088 aaronmk
# don't abort on verification errors, which are expected during development
297 1194 aaronmk
# default:
298 1199 aaronmk
verify-%: verify/%.out _always
299 1200 aaronmk
	$(if $(shell test -e $< && echo t),cat $<)
300 1199 aaronmk
# don't run if verify/%.out's default do-nothing action was used
301 1200 aaronmk
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
302 368 aaronmk
303 1620 aaronmk
verify = $(if $(reverify),"time" $(psqlAsBien) --set=datasource="'$(datasrc)'"\
304
--no-align --field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@)
305 514 aaronmk
306 1199 aaronmk
verify/%.out: $(mappings)/verify.%.sql _always
307 1192 aaronmk
	$(verify)
308 1194 aaronmk
# default:
309 1199 aaronmk
verify/%.out: _always ;
310 369 aaronmk
311 1199 aaronmk
all += $(wildcard verify/*.out)
312 1192 aaronmk
313 1378 aaronmk
ifneq ($(dbExport),)
314 1082 aaronmk
%.ref: %.ref.sql
315
	$(dbAsBien) $(db) <$< >$@
316
endif
317
318 1667 aaronmk
##### Editing import
319
320
import/rotate: _always
321 1668 aaronmk
	echo "UPDATE party SET organizationname = organizationname||'.$(date)'\
322 1854 aaronmk
WHERE organizationname = '$(datasrc)';"|$(psqlAsBien)
323 1667 aaronmk
324
import/rm: _always
325 1854 aaronmk
	echo "DELETE FROM party WHERE organizationname = '$(datasrc)';"|\
326
$(psqlAsBien)
327 1667 aaronmk
328 783 aaronmk
##### Testing
329 368 aaronmk
330 1986 aaronmk
testRefOutput = $(subst .2-step,,$(subst .staging,,$(1)))
331
testRef = $(testRefOutput).ref
332
hasOwnRef = $(filter $@,$(call testRefOutput,$@))
333
# filter returns non-empty if they are equal
334 630 aaronmk
335 897 aaronmk
define runTest
336 776 aaronmk
@echo "Testing $(abspath $@)..."
337 991 aaronmk
>$@ env test=1 n=$(test_n) $(1)
338 1184 aaronmk
@(set -x; $(diff) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
339 780 aaronmk
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
340 777 aaronmk
$(if $(hasOwnRef),\
341
echo $(emph)"To accept new test output:"$(endEmph);\
342
echo "$(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile";\
343
,\
344
echo $(emph)"Note: The preceding failed test is compared to another test's\
345
output"$(endEmph);\
346
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
347
)\
348 652 aaronmk
exit $$e;}
349 628 aaronmk
endef
350
351 897 aaronmk
test2File = $(call runTest,$(map))
352 1986 aaronmk
test2Db = $(call runTest,$(map2db))
353 775 aaronmk
354 876 aaronmk
tests :=
355
356 1139 aaronmk
test/$(via).%.xml: maps/$(via).%.full.csv _always
357 775 aaronmk
	$(test2File)
358 876 aaronmk
tests += test/$(via).%.xml
359 629 aaronmk
360 876 aaronmk
test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always
361 775 aaronmk
	$(test2File)
362 876 aaronmk
tests += test/VegBIEN.%.xml
363 628 aaronmk
364 884 aaronmk
test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always
365 775 aaronmk
	-$(test2File)
366 722 aaronmk
# Don't abort tester if only 2-step test fails, as it's often finicky
367 876 aaronmk
tests += test/VegBIEN.%.2-step.xml
368 630 aaronmk
369 876 aaronmk
test/import.%.out: maps/VegBIEN.%.csv _always
370 1986 aaronmk
	$(test2Db)
371 876 aaronmk
tests += test/import.%.out
372 627 aaronmk
373 1987 aaronmk
testStaged2Db = $(foreach use_staged,1,$(test2Db))
374
    # run with use_staged=1
375 1986 aaronmk
376
# Requires staging tables. To create them, run `make inputs/<datasrc>/install`.
377
test/import.%.staging.out: maps/VegBIEN.%.csv _always
378 1988 aaronmk
	-$(if $(isCsv),$(testStaged2Db),cp -p $(@:.staging.out=.out) $@)
379
# Don't abort tester if only staging test fails, in case staging table missing
380
# Non-flat-file inputs fall back to mimicking a successful test
381 1986 aaronmk
tests += test/import.%.staging.out
382
383 876 aaronmk
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
384 724 aaronmk
385 1361 aaronmk
.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted
386
387 876 aaronmk
test: _always $(testOutputs) ;
388
389 1987 aaronmk
all += $(filter-out %.ref,$(wildcard test/*))
390 876 aaronmk
391 634 aaronmk
# Accepts a test output: make <test_output_path>-ok
392 626 aaronmk
%-ok: _always
393 630 aaronmk
	$(CP) $* $(call testRef,$*)
394 502 aaronmk
395 783 aaronmk
##### Input-type-specific
396 368 aaronmk
397 1917 aaronmk
# Each input type needs var $(mapEnv) and targets src/install, src/uninstall
398 624 aaronmk
399 1379 aaronmk
#### DB export
400
401 1378 aaronmk
ifneq ($(dbExport),)
402 256 aaronmk
403 1378 aaronmk
dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbExport)))))
404 1196 aaronmk
db := $(datasrc)
405 622 aaronmk
406 1379 aaronmk
### Installation
407 256 aaronmk
408 1917 aaronmk
src/install: _always db ;
409 339 aaronmk
410 1917 aaronmk
src/uninstall: _always rm_db ;
411 368 aaronmk
412 1379 aaronmk
### DB-engine-specific
413 621 aaronmk
414 624 aaronmk
# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db
415 256 aaronmk
416 622 aaronmk
ifeq ($(dbEngineExt),my)
417 621 aaronmk
418 622 aaronmk
dbEngine := MySQL
419
420 621 aaronmk
bienPassword := $(shell cat $(root)/config/bien_password)
421
mysqlAs = mysql --user=$(1) --password='$(bienPassword)'
422
mysqlAsRoot := $(call mysqlAs,root)
423
dbAsBien := $(call mysqlAs,bien)
424
425 1097 aaronmk
dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))
426
427
define createDb
428 1098 aaronmk
echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)
429 1097 aaronmk
-$(mysqlAsRoot) --database=$(db) <$<
430
endef
431
# ignore errors in db import so that GRANT will still be run
432
433 1378 aaronmk
db: $(dbExport) _always
434 1097 aaronmk
	$(if $(dbExists),,$(createDb))
435 368 aaronmk
	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)
436 250 aaronmk
437 383 aaronmk
rm_db: _always
438 368 aaronmk
	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)
439
	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)
440 365 aaronmk
# ignore errors if grant not defined
441 250 aaronmk
442 1379 aaronmk
## Unrecognized DB engine
443 368 aaronmk
444
else
445 1378 aaronmk
$(error The DB filename $(dbExport) must be db.my.sql)
446 368 aaronmk
endif
447
448 1379 aaronmk
### Other input types
449 622 aaronmk
450 1379 aaronmk
else
451 624 aaronmk
452 1917 aaronmk
src/install: _always ;
453
src/uninstall: _always ;
454 1379 aaronmk
455 256 aaronmk
endif
456 1379 aaronmk
457
#### DB connection info
458
459
ifneq ($(dbEngine),)
460
# Must come after dbEngine is set
461
mapEnv := in_engine=$(dbEngine) in_database=$(db)
462
endif