Project

General

Profile

1 783 aaronmk
##### Configuration
2
3 1190 aaronmk
log ?= $(if $(test),,1)
4
profile ?= $(log)
5 783 aaronmk
exts ?= csv xml
6
test_n ?= 2
7
tablesSort ?= plots organisms stems
8
9
##### Vars/functions
10
11 640 aaronmk
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
12 250 aaronmk
13 368 aaronmk
# Make
14 640 aaronmk
SHELL := /bin/bash
15 415 aaronmk
subMake = $(MAKE) $(@F) --directory=$(@D)
16 404 aaronmk
+_ = $(+:_%=)
17 368 aaronmk
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
18
19 790 aaronmk
# System
20
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
21
22 640 aaronmk
# Terminal
23 1184 aaronmk
termCols := $(shell tput cols)
24 640 aaronmk
esc := '['
25
reset := $(esc)'0m'
26
emph := $(esc)'7m '
27
endEmph := ' '$(reset)
28
29 368 aaronmk
# Commands
30 395 aaronmk
CP = cp -p
31 1184 aaronmk
diff = diff --unified=2
32
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
33
--width=$(termCols),$(diff))
34 368 aaronmk
35 783 aaronmk
# Paths
36 640 aaronmk
root := $(selfDir_uZPPqC)..
37 775 aaronmk
mappings := $(root)/mappings
38 1196 aaronmk
datasrc := $(notdir $(realpath .))
39 1081 aaronmk
psqlOpts := --set ON_ERROR_STOP=1 --quiet
40
psqlAsBien := $(root)/bin/psql_vegbien $(psqlOpts)
41 353 aaronmk
42 783 aaronmk
##### General targets
43 250 aaronmk
44 418 aaronmk
all: _always maps ;
45 247 aaronmk
46
.SUFFIXES:
47
48 383 aaronmk
_always:
49
.PHONY: _always
50 250 aaronmk
51 383 aaronmk
clean: _always
52 256 aaronmk
	$(RM) $(all)
53
54 423 aaronmk
remake: _always clean all ;
55
56 515 aaronmk
%.out: %.make _always
57 368 aaronmk
	./$* >$@
58
.PRECIOUS: %.out
59
60 415 aaronmk
$(root)/%: _always
61
	+$(subMake)
62
63 783 aaronmk
##### SVN
64 250 aaronmk
65 766 aaronmk
svn_props: _always
66 1201 aaronmk
	svn propset svn:ignore $$'*.log\n*.trace\nsrc*' .
67 951 aaronmk
	$(if $(wildcard maps/),svn propset svn:ignore $$'.~*' maps)
68 1199 aaronmk
	$(if $(wildcard verify/),svn propset svn:ignore $$'*.out' verify)
69 766 aaronmk
	$(if $(wildcard test/),svn propset svn:ignore $$'*.out\n*.xml' test)
70
71 783 aaronmk
##### Installation
72 766 aaronmk
73 418 aaronmk
reinstall: _always uninstall install ;
74 264 aaronmk
75 783 aaronmk
##### Maps
76 264 aaronmk
77 1139 aaronmk
fullViaMap := maps/%.full.csv
78
allViaMaps := $(filter-out maps/VegBIEN.%.csv $(fullViaMap),\
79
$(wildcard maps/*.csv))
80 728 aaronmk
via := $(firstword $(sort $(basename $(basename $(notdir $(allViaMaps))))))
81 783 aaronmk
82 775 aaronmk
coreMap := $(mappings)/$(via)-VegBIEN.%.csv
83 1139 aaronmk
selfMap := $(mappings)/$(via).self.%.csv
84 783 aaronmk
85 749 aaronmk
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
86 1139 aaronmk
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))
87 783 aaronmk
88 728 aaronmk
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
89
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
90
$(wildcard maps/VegBIEN.*.csv))
91 772 aaronmk
tables := $(directMaps:maps/VegBIEN.%.csv=%)
92 728 aaronmk
93
.PRECIOUS: $(allViaMaps) $(directMaps)
94 368 aaronmk
95 1139 aaronmk
maps :=
96 728 aaronmk
97 775 aaronmk
maps/VegBIEN.%.csv: maps/$(via).%.csv $(coreMap)
98 1126 aaronmk
	$(root)/bin/join_union_sort <$+ >$@
99 775 aaronmk
.PRECIOUS: maps/VegBIEN.%.csv $(coreMap)
100 264 aaronmk
101 1139 aaronmk
maps += $(autogenMaps)
102
103
maps/$(via).%.full.csv: maps/$(via).%.csv $(selfMap)
104
	env ignore=1 $(root)/bin/union <$+|$(root)/bin/sort_map >$@
105
maps/$(via).%.full.csv: maps/$(via).%.csv # fallback if no self map
106 1194 aaronmk
	$(CP) $< $@
107 1139 aaronmk
.PRECIOUS: maps/$(via).%.full.csv maps/$(via).%.csv $(selfMap)
108
109
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
110
111
maps: $(maps) _always ;
112
113
all += $(maps)
114
115 783 aaronmk
##### Mapping
116 368 aaronmk
117 728 aaronmk
dbFile := $(firstword $(wildcard src/db.*.sql))
118 772 aaronmk
inputFiles := $(wildcard src/*.csv src/*.xml)
119 728 aaronmk
120 775 aaronmk
+maps = $(filter maps/% $(mappings)/%,$(+_))
121
<in = $(firstword $(filter-out $(+maps),$(+_)) $(wildcard $(exts:%=src/*.$*.%)))
122 958 aaronmk
map = $(if $(<in),<$(<in) ,$(if $(mapEnv),env $(mapEnv) ,$(error\
123 1139 aaronmk
No input file src/*.$*.{$(exts)} )))$(root)/map $(+maps)
124 772 aaronmk
map2db = env out_database=vegbien $(map)
125 368 aaronmk
126 783 aaronmk
##### Import to VegBIEN
127 772 aaronmk
128
ifneq ($(dbFile)$(inputFiles),)
129
130 1190 aaronmk
log_ = $(@:-all=)$(if $(n),.n=$(n),).$(date).log
131
trace = $(log_:.log=.trace)
132 1188 aaronmk
import = -(set -x; "time" env commit=1 verbose=1\
133 1190 aaronmk
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
134
$(if $(n),,>>$(log_))) 2>&1$(if $(log),$(if $(n),|tee -a $(log_)))
135 1088 aaronmk
# don't abort on import errors, which often relate to invalid input data
136 624 aaronmk
137 1193 aaronmk
import: $(addprefix import-,$(tables)) _always ;
138 718 aaronmk
139
import-%: maps/VegBIEN.%.csv _always
140
	$(import)
141 772 aaronmk
# default:
142
import-%: _always ;
143 718 aaronmk
144 772 aaronmk
else
145
import: _always ;
146
endif
147
148 790 aaronmk
##### Log files from import
149
150 1188 aaronmk
logs := $(wildcard *.log *.trace)
151 339 aaronmk
152 383 aaronmk
rm_logs: _always
153 339 aaronmk
	$(RM) $(logs)
154
155 783 aaronmk
##### Verification of import
156 264 aaronmk
157 1194 aaronmk
verify: $(addprefix verify-,$(tables)) _always ;
158
159 1199 aaronmk
verify-%: verify/%.ref verify/%.out _always
160 1184 aaronmk
	-$(diffVerbose) $(+_)
161 1088 aaronmk
# don't abort on verification errors, which are expected during development
162 1194 aaronmk
# default:
163 1199 aaronmk
verify-%: verify/%.out _always
164 1200 aaronmk
	$(if $(shell test -e $< && echo t),cat $<)
165 1199 aaronmk
# don't run if verify/%.out's default do-nothing action was used
166 1200 aaronmk
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
167 368 aaronmk
168 1196 aaronmk
verify = $(psqlAsBien) --set=datasource="'$(datasrc)'" --no-align\
169 1192 aaronmk
--field-separator='	' --pset=footer=off --pset=null=NULL <$< >$@
170 514 aaronmk
171 1199 aaronmk
verify/%.out: $(mappings)/verify.%.sql _always
172 1192 aaronmk
	$(verify)
173 1199 aaronmk
.PRECIOUS: verify/%.out
174 1194 aaronmk
# default:
175 1199 aaronmk
verify/%.out: _always ;
176 369 aaronmk
177 1199 aaronmk
all += $(wildcard verify/*.out)
178 1192 aaronmk
179 1082 aaronmk
ifneq ($(dbFile),)
180
%.ref: %.ref.sql
181
	$(dbAsBien) $(db) <$< >$@
182
.PRECIOUS: %.ref
183
endif
184
185 783 aaronmk
##### Testing
186 368 aaronmk
187 624 aaronmk
ifneq ($(wildcard test/),)
188
189 723 aaronmk
hasOwnRef = $(filter-out %.2-step.xml,$@)
190 630 aaronmk
testRef = $(1:.2-step.xml=.xml).ref
191
192 897 aaronmk
define runTest
193 776 aaronmk
@echo "Testing $(abspath $@)..."
194 991 aaronmk
>$@ env test=1 n=$(test_n) $(1)
195 1184 aaronmk
@(set -x; $(diff) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
196 780 aaronmk
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
197 777 aaronmk
$(if $(hasOwnRef),\
198
echo $(emph)"To accept new test output:"$(endEmph);\
199
echo "$(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile";\
200
,\
201
echo $(emph)"Note: The preceding failed test is compared to another test's\
202
output"$(endEmph);\
203
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
204
)\
205 652 aaronmk
exit $$e;}
206 628 aaronmk
endef
207
208 897 aaronmk
test2File = $(call runTest,$(map))
209 775 aaronmk
210 876 aaronmk
tests :=
211
212 1139 aaronmk
test/$(via).%.xml: maps/$(via).%.full.csv _always
213 775 aaronmk
	$(test2File)
214 876 aaronmk
tests += test/$(via).%.xml
215 629 aaronmk
216 876 aaronmk
test/VegBIEN.%.xml: maps/VegBIEN.%.csv _always
217 775 aaronmk
	$(test2File)
218 876 aaronmk
tests += test/VegBIEN.%.xml
219 628 aaronmk
220 884 aaronmk
test/VegBIEN.%.2-step.xml: test/$(via).%.xml $(coreMap) _always
221 775 aaronmk
	-$(test2File)
222 722 aaronmk
# Don't abort tester if only 2-step test fails, as it's often finicky
223 876 aaronmk
tests += test/VegBIEN.%.2-step.xml
224 630 aaronmk
225 876 aaronmk
test/import.%.out: maps/VegBIEN.%.csv _always
226 897 aaronmk
	$(call runTest,$(map2db))
227 876 aaronmk
tests += test/import.%.out
228 627 aaronmk
229 724 aaronmk
.PRECIOUS: $(tests)
230
231 876 aaronmk
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
232 724 aaronmk
233 876 aaronmk
test: _always $(testOutputs) ;
234
235
all += $(testOutputs)
236
237 634 aaronmk
# Accepts a test output: make <test_output_path>-ok
238 626 aaronmk
%-ok: _always
239 630 aaronmk
	$(CP) $* $(call testRef,$*)
240 502 aaronmk
241 624 aaronmk
else
242
test: _always ;
243
endif
244
245 783 aaronmk
##### Input-type-specific
246 368 aaronmk
247 624 aaronmk
# Each input type needs var $(mapEnv) and targets install, uninstall
248
249 622 aaronmk
ifneq ($(dbFile),)
250 256 aaronmk
251 635 aaronmk
dbEngineExt := $(subst .,,$(suffix $(basename $(notdir $(dbFile)))))
252 1196 aaronmk
db := $(datasrc)
253 622 aaronmk
254 783 aaronmk
#### Installation
255 256 aaronmk
256 621 aaronmk
install: _always db ;
257 339 aaronmk
258 621 aaronmk
uninstall: _always rm_db ;
259 368 aaronmk
260 783 aaronmk
#### DB-engine-specific
261 621 aaronmk
262 624 aaronmk
# Each DB engine needs vars $(dbEngine), $(dbAsBien) and targets db, rm_db
263 256 aaronmk
264 622 aaronmk
ifeq ($(dbEngineExt),my)
265 621 aaronmk
266 622 aaronmk
dbEngine := MySQL
267
268 621 aaronmk
bienPassword := $(shell cat $(root)/config/bien_password)
269
mysqlAs = mysql --user=$(1) --password='$(bienPassword)'
270
mysqlAsRoot := $(call mysqlAs,root)
271
dbAsBien := $(call mysqlAs,bien)
272
273 1097 aaronmk
dbExists = $(shell echo "SHOW DATABASES LIKE '$(db)';"|$(mysqlAsRoot))
274
275
define createDb
276 1098 aaronmk
echo "CREATE DATABASE $(db) DEFAULT CHARACTER SET latin1;"|$(mysqlAsRoot)
277 1097 aaronmk
-$(mysqlAsRoot) --database=$(db) <$<
278
endef
279
# ignore errors in db import so that GRANT will still be run
280
281 622 aaronmk
db: $(dbFile) _always
282 1097 aaronmk
	$(if $(dbExists),,$(createDb))
283 368 aaronmk
	echo "GRANT SELECT ON $(db).* TO 'bien'@'localhost';"|$(mysqlAsRoot)
284 250 aaronmk
285 383 aaronmk
rm_db: _always
286 368 aaronmk
	-echo "REVOKE ALL ON $(db).* FROM 'bien'@'localhost';"|$(mysqlAsRoot)
287
	echo "DROP DATABASE IF EXISTS $(db);"|$(mysqlAsRoot)
288 365 aaronmk
# ignore errors if grant not defined
289 250 aaronmk
290 783 aaronmk
### Unrecognized DB engine
291 368 aaronmk
292
else
293 993 aaronmk
$(error The DB filename $(dbFile) must be db.my.sql)
294 368 aaronmk
endif
295
296 622 aaronmk
# Must come after dbEngine is set
297
mapEnv := in_engine=$(dbEngine) in_database=$(db)
298
299 783 aaronmk
#### Unrecognized input type
300 624 aaronmk
301 256 aaronmk
else
302 624 aaronmk
install: _always ;
303
uninstall: _always ;
304 256 aaronmk
endif