Project

General

Profile

1 3761 aaronmk
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
2
root := $(selfDir_uZPPqC)..
3
include $(root)/lib/common.Makefile
4
5
6 783 aaronmk
##### Configuration
7
8 1921 aaronmk
# Command line
9 1190 aaronmk
log ?= $(if $(test),,1)
10 1257 aaronmk
profile ?=
11 1983 aaronmk
quiet ?=
12 1620 aaronmk
reverify ?= 1
13 1990 aaronmk
use_staged ?= $(by_col)
14 1921 aaronmk
15
# Makefile
16
tablesSort ?= plots organisms stems specimens
17 1386 aaronmk
exts ?= csv tsv txt xml
18 783 aaronmk
test_n ?= 2
19
20
##### Vars/functions
21
22 1821 aaronmk
# Paths
23
datasrc := $(notdir $(realpath .))
24
bin := $(root)/bin
25
mappings := $(root)/mappings
26
27 368 aaronmk
# Make
28 640 aaronmk
SHELL := /bin/bash
29 1509 aaronmk
selfMake = $(MAKE) --makefile=../input.Makefile
30 1821 aaronmk
subMake = $(MAKE) $(@:$(root)/%=%) --directory=$(root)
31 404 aaronmk
+_ = $(+:_%=)
32 368 aaronmk
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
33
34 1742 aaronmk
# OS
35
os := $(shell uname)
36
forOs = $(patsubst %,%-$(filter Linux Darwin,$(os)),$(1))
37
38
# Formatting
39
SED = sed -$(if $(filter Darwin,$(os)),E,r)
40
41 790 aaronmk
# System
42
date = $(shell date +"%Y-%m-%d-%H-%M-%S")
43
44 640 aaronmk
# Terminal
45 1184 aaronmk
termCols := $(shell tput cols)
46 640 aaronmk
esc := '['
47
reset := $(esc)'0m'
48
emph := $(esc)'7m '
49
endEmph := ' '$(reset)
50
51 368 aaronmk
# Commands
52 1245 aaronmk
MKDIR = mkdir -p
53
mkdir = $(MKDIR) $(@D)
54 395 aaronmk
CP = cp -p
55 1184 aaronmk
diff = diff --unified=2
56 3721 aaronmk
diffIgnoreSpace = $(diff) --ignore-space-change
57 1184 aaronmk
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
58
--width=$(termCols),$(diff))
59 368 aaronmk
60 1918 aaronmk
# BIEN commands
61 1524 aaronmk
selfMap = $(bin)/cols 0 0
62 1081 aaronmk
psqlOpts := --set ON_ERROR_STOP=1 --quiet
63 1289 aaronmk
psqlAsBien := $(bin)/psql_vegbien $(psqlOpts)
64 353 aaronmk
65 1594 aaronmk
# SVN
66
addDir = $(if $(wildcard $(1)/),svn add --depth=empty $(1),svn mkdir $(1))
67 1771 aaronmk
setSvnIgnore = svn propset svn:ignore $(2) $(1)
68 1594 aaronmk
define addDirWithIgnore
69
$(addDir)
70 1771 aaronmk
$(setSvnIgnore)
71 1594 aaronmk
endef
72
73 783 aaronmk
##### General targets
74 250 aaronmk
75 418 aaronmk
all: _always maps ;
76 247 aaronmk
77 383 aaronmk
clean: _always
78 256 aaronmk
	$(RM) $(all)
79
80 1742 aaronmk
remake: _always clean
81
	+$(selfMake)
82
# re-run make so that cache of existing files is reset
83
84 1627 aaronmk
make_script = ./$< >$@
85 368 aaronmk
86 1604 aaronmk
%/: % _always ;
87
88 1627 aaronmk
# Must come before `%: %.make` to override it
89
src/%: src/%.make _always
90 3317 aaronmk
	(set -x; $(make_script)) 2>>$<.log.sql
91 1627 aaronmk
.PRECIOUS: src/% # save partial outputs of aborted src make scripts
92
93
%: %.make _always
94
	$(make_script)
95
96 783 aaronmk
##### SVN
97 250 aaronmk
98 1594 aaronmk
add: _always
99 1771 aaronmk
	$(call setSvnIgnore,.,$$'')
100 1594 aaronmk
	$(call addDirWithIgnore,src,$$'*')
101
	$(call addDirWithIgnore,maps,$$'.~*')
102
	$(call addDirWithIgnore,import,$$'*')
103
	$(call addDirWithIgnore,test,$$'*.out\n*.xml')
104
	$(call addDirWithIgnore,verify,$$'*.out')
105 766 aaronmk
106 1969 aaronmk
##### Existing maps discovery
107 1955 aaronmk
108
inputFiles := $(wildcard $(exts:%=src/*.%))
109
110 3574 aaronmk
srcMaps := $(wildcard maps/src.*.csv)
111
112 1289 aaronmk
srcMap := maps/src.%.csv
113 1139 aaronmk
fullViaMap := maps/%.full.csv
114 1289 aaronmk
directMap := maps/VegBIEN.%.csv
115 4095 aaronmk
via := Veg+
116 783 aaronmk
117 4098 aaronmk
coreMap := $(mappings)/$(via)-VegBIEN.csv
118
coreSelfMap := $(mappings)/$(via).self.csv
119 783 aaronmk
120 749 aaronmk
viaMaps := $(wildcard $(tablesSort:%=maps/$(via).%.csv))
121 3574 aaronmk
viaMaps += $(filter-out $(viaMaps),$(srcMaps:maps/src.%=maps/$(via).%))
122 1139 aaronmk
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/$(via).*.csv))
123 783 aaronmk
124 728 aaronmk
autogenMaps := $(subst $(via).,VegBIEN.,$(viaMaps))
125
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
126
$(wildcard maps/VegBIEN.*.csv))
127 772 aaronmk
tables := $(directMaps:maps/VegBIEN.%.csv=%)
128 728 aaronmk
129 1969 aaronmk
##### Sources
130
131
srcs = $(shell $(bin)/sort_filenames $(wildcard $(exts:%=src/*.$*.%)))
132
isCsv = $(if $(srcs),$(if $(filter %.xml,$(srcs)),,1))
133
    # true if $(srcs) non-empty and contains no *.xml
134
catSrcs = $(bin)/cat$(if $(isCsv),_csv) $(srcs)
135
withCatSrcs = $(catSrcs:$(bin)/%=$(bin)/with_%) --
136
137 1986 aaronmk
# Usage: `make {--silent|-s} inputs/<datasrc>/cat` (don't echo make commands)
138 1969 aaronmk
cat: $(addprefix cat-,$(tables)) _always ;
139
140
cat-%: _always
141
	$(catSrcs)
142
143 1921 aaronmk
##### Installation
144
145 4094 aaronmk
install: _always import/install ;
146 1921 aaronmk
147 4094 aaronmk
uninstall: _always import/uninstall ;
148 1921 aaronmk
149
reinstall: _always uninstall install ;
150
151
##### Staging tables
152
153
import/install: _always import/schema $(addprefix import/install-,$(tables)) ;
154
155
import/uninstall: _always import/rm_schema ;
156
# rm_schema will also drop all staging tables
157
158
import/schema: _always
159
	-echo 'CREATE SCHEMA "$(datasrc)";'|$(psqlAsBien)
160
# ignore errors if schema exists
161
162
import/rm_schema: _always
163
	echo 'DROP SCHEMA IF EXISTS "$(datasrc)" CASCADE;'|$(psqlAsBien)
164
165
import/install-%: _always
166 1955 aaronmk
	$(if $(isCsv),$(import_install_))
167 3476 aaronmk
import_install_ = (prefix=; . $(bin)/vegbien_dest; "time" nice -n +5\
168
env schema=$(datasrc) table=$* $(bin)/csv2db $(catSrcs)\
169 3317 aaronmk
$(if $(log),$(if $(quiet),&>,2>&1|tee )import/install-$*.log.sql))
170 1921 aaronmk
171 1777 aaronmk
##### Maps building
172
173 1789 aaronmk
# Maps to (try to) build are added to one of these
174 1779 aaronmk
maps :=
175 1789 aaronmk
createOnlyMaps :=
176 1779 aaronmk
177 3569 aaronmk
srcRoot = $(mappings)/roots/$*.sh
178
mkSrcMap = $(catSrcs)|(. $(srcRoot); env datasrc=$(datasrc) $(bin)/src_map >$@)
179 1779 aaronmk
180 3567 aaronmk
# Src maps cleanup
181
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
182
maps/.src.%.csv.last_cleanup: maps/src.%.csv
183 3814 aaronmk
	$(bin)/in_place $< $(bin)/cols +
184 3567 aaronmk
	touch $@
185
else
186 1779 aaronmk
# Autogen src maps with known table names
187 3567 aaronmk
maps/src.%.csv: _always
188
	$(if $(wildcard $@),,$(if $(isCsv),$(mkSrcMap)))
189
	+$(if $(isCsv),$(selfMake) $(@:maps/%=maps/.%.last_cleanup))
190
# only build if doesn't exist
191 1779 aaronmk
# only build if CSV srcs exist for that table name
192 3567 aaronmk
endif
193
194 1779 aaronmk
# Try all table names
195 1789 aaronmk
createOnlyMaps += $(tablesSort:%=maps/src.%.csv)
196 1779 aaronmk
197 1273 aaronmk
# Must come before $(root)/% to override it
198 1524 aaronmk
$(coreSelfMap): _always
199 1273 aaronmk
	-+$(subMake)
200 1524 aaronmk
# ignore errors if $(coreSelfMap) does not exist
201 1273 aaronmk
202 3561 aaronmk
joinSrcMap = $(if $(hasSrc),$(bin)/in_place $@ $(bin)/intersect $(word 3,$+) 0)
203
hasSrc = $(shell test -s $(word 3,$+) && echo t)
204 1526 aaronmk
205 1273 aaronmk
makeFullCsv = $(if $(shell test -e $(word 2,$+) && echo t),\
206 3561 aaronmk
env ignore=1 $(bin)/union <$(wordlist 1,2,$+)|,<$< )$(bin)/sort_map >$@
207 1524 aaronmk
# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs
208 1273 aaronmk
209 3572 aaronmk
# Must come before maps/$(via).%.csv to override it
210 3561 aaronmk
maps/$(via).%.full.csv: maps/$(via).%.csv $(coreSelfMap) $(srcMap)
211 1273 aaronmk
	$(makeFullCsv)
212 1526 aaronmk
	$(joinSrcMap)
213 1139 aaronmk
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
214
215 3572 aaronmk
# Via maps cleanup
216
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
217
maps/.$(via).%.csv.last_cleanup: maps/$(via).%.csv $(coreMap)
218
	$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1 2
219
	touch $@
220
# Include comment column so commented mappings are never removed
221
else
222 3574 aaronmk
maps/$(via).%.csv: maps/src.%.csv _always
223
	$(if $(wildcard $@),,$(if $(wildcard $<),$(CP) $< $@))
224 3572 aaronmk
	+$(selfMake) $(@:maps/%=maps/.%.last_cleanup)
225
endif
226
227 1530 aaronmk
maps/VegBIEN.%.csv: maps/$(via).%.full.csv $(coreMap)
228
	$(bin)/join <$+|$(bin)/sort_map >$@
229
maps += $(autogenMaps)
230
231 1789 aaronmk
maps: $(createOnlyMaps) $(maps) _always ;
232 1139 aaronmk
233
all += $(maps)
234
235 1742 aaronmk
##### Maps validation
236
237 3764 aaronmk
missingMappingsCmd = +$(selfMake) remake 2>&1\
238 3821 aaronmk
|$(SED) -n 's/^.*No $(*2Space) mapping for (.*)$$/\1/p'|$(SED) 's/\/_.*//'\
239 3764 aaronmk
$(if $(filter non-empty_join,$*),|$(bin)/ucase_first 0)|sort|uniq
240 1742 aaronmk
241 3764 aaronmk
include $(root)/lib/mappings.Makefile
242 1742 aaronmk
243 1509 aaronmk
##### External dependencies
244
245
$(root)/%: _always
246
	+$(subMake)
247 1628 aaronmk
.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error
248 1509 aaronmk
249 783 aaronmk
##### Mapping
250 368 aaronmk
251 775 aaronmk
+maps = $(filter maps/% $(mappings)/%,$(+_))
252 1451 aaronmk
<in = $(firstword $(filter-out $(+maps),$(+_)))
253
map = $(if $(<in),<$(<in),\
254 1777 aaronmk
$(if $(srcs),$(withCatSrcs),\
255 1451 aaronmk
$(if $(mapEnv),env $(mapEnv),\
256 3387 aaronmk
$(error No input file src/*.$*.{$(exts)}))))\
257 1451 aaronmk
$(root)/map $(+maps)
258 3387 aaronmk
map2db = $(if $(use_staged),env in_database=vegbien in_schema=$(datasrc)\
259
in_table=$*) env out_database=vegbien $(map)
260 368 aaronmk
261 783 aaronmk
##### Import to VegBIEN
262 772 aaronmk
263 4094 aaronmk
ifneq ($(inputFiles),)
264 772 aaronmk
265 1853 aaronmk
profileTest = $(if $(profile),$(if $(test),1))
266
profileOnly = -env profile_to=/dev/fd/3 $(map2db) 3>&1 1>&2|\
267
$(bin)/profile_stats /dev/fd/0
268
269 3317 aaronmk
log_ = import/$*$(if $(n),.n=$(n),).$(date).log.sql
270
trace = $(log_:.log.sql=.trace)
271 1853 aaronmk
import = -$(if $(profileTest),$(profileOnly),(set -x; "time" env commit=1\
272 1190 aaronmk
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
273 3616 aaronmk
$(if $(n),,&>$(log_)))$(if $(log),$(if $(n), 2>&1|tee -a $(log_))))
274 1088 aaronmk
# don't abort on import errors, which often relate to invalid input data
275 624 aaronmk
276 1193 aaronmk
import: $(addprefix import-,$(tables)) _always ;
277 718 aaronmk
278
import-%: maps/VegBIEN.%.csv _always
279
	$(import)
280 772 aaronmk
# default:
281
import-%: _always ;
282 718 aaronmk
283 772 aaronmk
else
284
import: _always ;
285
endif
286
287 790 aaronmk
##### Log files from import
288
289 3317 aaronmk
logs := $(wildcard import/*.log.sql import/*.trace)
290 339 aaronmk
291 383 aaronmk
rm_logs: _always
292 339 aaronmk
	$(RM) $(logs)
293
294 783 aaronmk
##### Verification of import
295 264 aaronmk
296 1194 aaronmk
verify: $(addprefix verify-,$(tables)) _always ;
297
298 1199 aaronmk
verify-%: verify/%.ref verify/%.out _always
299 1184 aaronmk
	-$(diffVerbose) $(+_)
300 1088 aaronmk
# don't abort on verification errors, which are expected during development
301 1194 aaronmk
# default:
302 1199 aaronmk
verify-%: verify/%.out _always
303 1200 aaronmk
	$(if $(shell test -e $< && echo t),cat $<)
304 1199 aaronmk
# don't run if verify/%.out's default do-nothing action was used
305 1200 aaronmk
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
306 368 aaronmk
307 1620 aaronmk
verify = $(if $(reverify),"time" $(psqlAsBien) --set=datasource="'$(datasrc)'"\
308
--no-align --field-separator=$$'\t' --pset=footer=off --pset=null=NULL <$< >$@)
309 514 aaronmk
310 1199 aaronmk
verify/%.out: $(mappings)/verify.%.sql _always
311 1192 aaronmk
	$(verify)
312 1194 aaronmk
# default:
313 1199 aaronmk
verify/%.out: _always ;
314 369 aaronmk
315 1199 aaronmk
all += $(wildcard verify/*.out)
316 1192 aaronmk
317 1667 aaronmk
##### Editing import
318
319
import/rotate: _always
320 1668 aaronmk
	echo "UPDATE party SET organizationname = organizationname||'.$(date)'\
321 1854 aaronmk
WHERE organizationname = '$(datasrc)';"|$(psqlAsBien)
322 1667 aaronmk
323
import/rm: _always
324 1854 aaronmk
	echo "DELETE FROM party WHERE organizationname = '$(datasrc)';"|\
325
$(psqlAsBien)
326 1667 aaronmk
327 783 aaronmk
##### Testing
328 368 aaronmk
329 1986 aaronmk
testRefOutput = $(subst .2-step,,$(subst .staging,,$(1)))
330
testRef = $(testRefOutput).ref
331
hasOwnRef = $(filter $@,$(call testRefOutput,$@))
332
# filter returns non-empty if they are equal
333 630 aaronmk
334 897 aaronmk
define runTest
335 776 aaronmk
@echo "Testing $(abspath $@)..."
336 991 aaronmk
>$@ env test=1 n=$(test_n) $(1)
337 3721 aaronmk
@(set -x; $(diffIgnoreSpace) $(call testRef,$@) $@) 2>&1 || { e=$$?;\
338 780 aaronmk
$(if $(wildcard $(call testRef,$@)),,cat $@;)\
339 777 aaronmk
$(if $(hasOwnRef),\
340 3179 aaronmk
{\
341
read -p $(emph)'Accept new test output? (y/n)'$(endEmph) REPLY;\
342
if test "$$REPLY" = y; then\
343
(set -x; $(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile);\
344
exit 0;\
345
fi;\
346
};,\
347 777 aaronmk
echo $(emph)"Note: The preceding failed test is compared to another test's\
348
output"$(endEmph);\
349
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
350
)\
351 652 aaronmk
exit $$e;}
352 628 aaronmk
endef
353
354 1986 aaronmk
test2Db = $(call runTest,$(map2db))
355 775 aaronmk
356 876 aaronmk
tests :=
357
358 3642 aaronmk
test/import.%.xml: maps/VegBIEN.%.csv _always
359 1986 aaronmk
	$(test2Db)
360 3642 aaronmk
tests += test/import.%.xml
361 627 aaronmk
362 1987 aaronmk
testStaged2Db = $(foreach use_staged,1,$(test2Db))
363
    # run with use_staged=1
364 1986 aaronmk
365
# Requires staging tables. To create them, run `make inputs/<datasrc>/install`.
366 3642 aaronmk
test/import.%.staging.xml: maps/VegBIEN.%.csv _always
367
	-$(if $(isCsv),$(testStaged2Db),cp -p $(@:.staging.xml=.xml) $@)
368 1988 aaronmk
# Don't abort tester if only staging test fails, in case staging table missing
369
# Non-flat-file inputs fall back to mimicking a successful test
370 3642 aaronmk
tests += test/import.%.staging.xml
371 1986 aaronmk
372 876 aaronmk
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))
373 724 aaronmk
374 1361 aaronmk
.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted
375
376 876 aaronmk
test: _always $(testOutputs) ;
377
378 1987 aaronmk
all += $(filter-out %.ref,$(wildcard test/*))
379 876 aaronmk
380 634 aaronmk
# Accepts a test output: make <test_output_path>-ok
381 626 aaronmk
%-ok: _always
382 630 aaronmk
	$(CP) $* $(call testRef,$*)
383 502 aaronmk
384 3579 aaronmk
test/accept-all: _always
385
	+yes|$(selfMake) test
386
387 3133 aaronmk
##### Documentation
388
389 3590 aaronmk
steps = $(selfMake) -s import test=1 by_col=1 verbosity=2 n=100\
390 3385 aaronmk
2>&1|$(bin)/debug2redmine >$@
391 3188 aaronmk
392 3133 aaronmk
import/steps.by_col.sql: _always
393 3188 aaronmk
	+$(steps)