Project

General

Profile

1
selfDir_uZPPqC := $(dir $(lastword $(MAKEFILE_LIST)))
2
root := $(selfDir_uZPPqC)..
3
include $(root)/lib/common.Makefile
4

    
5

    
6
##### Configuration
7

    
8
# Command line
9
import_source ?= 1
10
log ?= $(if $(test),,1)
11
profile ?=
12
quiet ?=
13
reverify ?= 1
14
schema_only ?=
15
use_staged ?= $(by_col)
16

    
17
# Makefile
18
exts ?= csv tsv txt dmp xml
19
test_n ?= 2
20

    
21
##### Vars/functions
22

    
23
# Paths
24
datasrc := $(patsubst .%,%,$(notdir $(realpath .)))
25
bin := $(root)/bin
26
mappings := $(root)/mappings
27

    
28
# Make
29
SHELL := /bin/bash
30
selfMake = $(MAKE) --makefile=../input.Makefile
31
subMake = $(MAKE) $(@:$(root)/%=%) --directory=$(root)
32
+_ = $(+:_%=)
33
addBeforeExt = $(basename $(2))$(1)$(suffix $(2))
34

    
35
# Terminal
36
termCols := $(shell tput cols)
37
esc := '['
38
reset := $(esc)'0m'
39
emph := $(esc)'7m '
40
endEmph := ' '$(reset)
41

    
42
# User interaction
43

    
44
confirm = $(if $(shell read -p $(emph)"$(1)"$(endEmph)$$'$(if\
45
$(2),\n$(2))\nContinue? (y/n) ' REPLY; test "$$REPLY" = y && echo t),,\
46
$(error Aborting))
47

    
48
# Commands
49
MKDIR = mkdir -p
50
mkdir = $(MKDIR) $(@D)
51
CP = cp -p
52
diff = diff --unified=2
53
diffIgnoreSpace = $(diff) --ignore-space-change
54
diffVerbose = $(if $(verbose),diff --side-by-side --left-column\
55
--width=$(termCols),$(diff))
56

    
57
# BIEN commands
58
sortFilenames = $(shell $(bin)/sort_filenames $(1))
59
selfMap = $(bin)/cols 0 0
60
psqlAsBien := $(bin)/psql_script_vegbien
61
psqlNoSearchPath := env no_search_path=1 $(psqlAsBien)
62
# Usage: ($(inDatasrc); cat $(file))|$(psqlCmd)
63
inDatasrc := echo 'SET search_path TO "$(datasrc)";'
64

    
65
# SVN
66
addDir = $(if $(wildcard $(1)/),svn add --depth=empty $(1),svn mkdir $(1))
67
setSvnIgnore = svn propset svn:ignore $(2) $(1)
68
define addDirWithIgnore
69
$(addDir)
70
$(setSvnIgnore)
71
endef
72

    
73
##### Environment
74

    
75
export PATH := $(bin):$(PATH)
76

    
77
##### General targets
78

    
79
all: _always maps ;
80

    
81
clean: _always
82
	$(RM) $(all)
83

    
84
remake: _always clean
85
	+$(selfMake)
86
# re-run make so that cache of existing files is reset
87

    
88
# Only remake if doesn't exist. This prevents unintentional remaking when the
89
# make script is newly checked out from svn (which sets the mod time to now) but
90
# the output is synced externally.
91
# Can't remove prereq to do this, because it determines when the rule applies.
92
make_script = $(if $(wildcard $@),,"time" ./$< >$@)
93

    
94
%/: %/map.csv _always ;
95

    
96
%/: % _always ;
97

    
98
%: %.make
99
	$(make_script)
100
.PRECIOUS: % # save partial outputs of aborted src make scripts
101

    
102
##### Tables discovery
103

    
104
sortFile := import_order.txt
105
noImportFile := _no_import
106

    
107
dontImport = $(wildcard $(1)/$(noImportFile))$(if\
108
$(import_source),,$(filter Source,$(1)))
109

    
110
tables := $(if $(wildcard $(sortFile)),$(shell cat $(sortFile)))
111
    # $(shell) replaces "\n" with " "
112
allSubdirs := $(call wildcard/,*/)
113
allTables := $(call sortFilenames,$(filter-out _% verify logs,$(allSubdirs:%/=%)))
114
joinedTables := $(filter-out $(tables),$(allTables))
115
allTables := $(strip $(joinedTables) $(tables))# move joined tables to beginning
116
ifeq ($(tables),)# none specified in sort file
117
tables := $(allTables)
118
endif
119
importTables := $(foreach table,$(tables),$(if\
120
$(call dontImport,$(table)),,$(table)))
121

    
122
##### SVN
123

    
124
svnFilesGlob:= */{,$(noImportFile),{,.}{map,*terms,VegBIEN}.csv{,.*},*header.*,*.sql,test.xml.ref}
125
svnFilesGlob := {map.csv,*{schema,~}*.sql,{,*/}*.make,$(svnFilesGlob)}
126
_svnFilesGlob := {_MySQL/{,*schema*.sql,*.make},_src/*.{url,pdf}}
127
svnFiles = $(filter-out _% logs/%,$(call wildcard/,$(svnFilesGlob)))\
128
$(call wildcard/,$(_svnFilesGlob))
129

    
130
# To update all inputs with these settings: make inputs/add
131
add: _always Source/add Source/map.csv $(allTables:%=%/add)
132
	$(call setSvnIgnore,.,'*')
133
	$(call addDirWithIgnore,logs,$$'*.log.sql\n*.trace')
134
	$(call addDirWithIgnore,verify,$$'*.out\n*.csv\n*.xls\n*.xlsx')
135
	$(call addFile,import_order.txt)
136
	$(if $(wildcard _MySQL/),$(call addDirWithIgnore,_MySQL,'*'))
137
	$(if $(wildcard _src/),$(call addDirWithIgnore,_src,'*'))
138
	$(if $(wildcard _archive/),$(call addDirWithIgnore,_archive,'*'))
139
	$(call add*,$(svnFiles))
140

    
141
# Adds a new table subdir
142
%/add: _always
143
	$(call addDirWithIgnore,$*,'*')
144
	$(call addDirWithIgnore,$*/logs,$$'*.log.sql\n*.trace')
145

    
146
##### Existing maps discovery
147

    
148
anyMap := %/map.csv %/VegBIEN.csv %/unmapped_terms.csv %/new_terms.csv
149

    
150
extsFilter := $(addprefix %.,$(exts))
151
dataOnly = $(filter $(extsFilter),$(1))
152

    
153
anyTest = $*/test.%
154
srcsOnly = $(filter-out $(anyMap) $(anyTest) %/logs,$(call dataOnly,$(1)))
155

    
156
srcDict := map.csv
157

    
158
vocab := $(mappings)/VegCore.csv
159
coreMap := $(mappings)/VegCore-VegBIEN.csv
160
dict := $(mappings)/Veg+-VegCore.csv
161

    
162
viaMaps := $(importTables:%=%/map.csv)
163

    
164
autogenMaps := $(subst map.,VegBIEN.,$(viaMaps))
165
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\
166
$(wildcard */VegBIEN.csv))
167

    
168
##### Sources
169

    
170
srcs = $(call sortFilenames,$(call srcsOnly,$(wildcard $*/*)))
171
nonHeaderSrcs = $(filter-out %/header.csv,$(srcs))
172
isRef = $(if $(nonHeaderSrcs),,1)
173
    # empty subdir, so references an already-installed staging table
174
isXml = $(filter %.xml,$(nonHeaderSrcs))
175
nonXml = $(if $(isXml),,1)
176
isCsv = $(if $(nonHeaderSrcs),$(if $(isXml),,1))
177
    # true if $(srcs) non-empty and contains no *.xml
178
catSrcs = $(bin)/cat$(if $(nonXml),_csv) $(srcs)
179
withCatSrcs = $(catSrcs:$(bin)/%=$(bin)/with_%) --
180

    
181
# Usage: `make {--silent|-s} inputs/<datasrc>/cat` (don't echo make commands)
182
cat: $(importTables:%=%/cat) _always ;
183

    
184
%/cat: _always
185
	$(catSrcs)
186

    
187
##### Input data retrieval
188

    
189
# Must come before `%.sql: _MySQL/%.sql` to override it
190
%.sql: %.sql.make
191
	$(make_script)
192

    
193
# The export must be created with:
194
# `--compatible=postgresql --add-locks=false --set-charset --no-create-info`
195
# Must come before `%.sql: _MySQL/%.sql` to override it
196
%.data.sql: _MySQL/%.data.sql
197
	$(if $(wildcard $@),,$(bin)/my2pg.data <$< >$@)
198

    
199
# The export must be created with:
200
# `--compatible=postgresql --add-locks=false --set-charset`
201
# Add `--no-data` to create a schema-only export.
202
%.sql: _MySQL/%.sql
203
	$(if $(wildcard $@),,$(bin)/my2pg <$< >$@)
204

    
205
##### Staging tables installation
206

    
207
srcTable := %.src
208

    
209
dbExportsWildcard = $(sort $(patsubst _MySQL/%.make,%,$(wildcard\
210
$(1) _MySQL/$(1).make)))
211

    
212
dbExports := $(call dbExportsWildcard,*schema*.sql)# schemas first
213
ifeq ($(schema_only),) # add rest of .sql files
214
dbExports += $(filter-out $(dbExports),$(call dbExportsWildcard,*.sql))
215
endif
216
dbExports := $(strip $(dbExports))# += adds extra whitespace
217
allInstalls := $(if $(dbExports),sql) $(filter-out Source,$(allTables))
218

    
219
install: _always schema $(allInstalls:%=%/install) ;
220

    
221
uninstall: _always confirm_rm_schema rm_schema ;
222
# rm_schema will also drop all staging tables
223

    
224
reinstall: _always uninstall install ;
225

    
226
confirm_rm_schema: _always
227
	$(if $(filter TNRS,$(datasrc)),$(call confirm,WARNING: This will delete the\
228
TNRS cache!,To save it: make backups/TNRS.backup-remake))
229

    
230
schema: _always
231
	-echo 'CREATE SCHEMA "$(datasrc)";'|$(psqlNoSearchPath)
232
# ignore errors if schema exists
233

    
234
rm_schema: _always
235
	echo 'DROP SCHEMA IF EXISTS "$(datasrc)" CASCADE;'|$(psqlNoSearchPath)
236

    
237
installLog := logs/install.log.sql
238

    
239
logInstall = $(if $(log),$(if $(quiet),$(2)$(1)$(installLog) 2>&1,2>&1|tee $(3)\
240
$(1)$(installLog)))
241
logInstallRoot = $(call logInstall,,>)
242
logInstall* = $(call logInstall,$*/,>)
243
logInstall*Add = $(call logInstall,$*/,>>,-a)# append to log
244

    
245
# Must come before %/install to override it
246
sql/install: $(dbExports)
247
	($(inDatasrc); cat $+|pg_dump_limit)|"time" $(psqlNoSearchPath) \
248
--set=schema='"$(datasrc)"' $(logInstallRoot)
249

    
250
cleanup = $(if $(wildcard $*/cleanup.sql),($(inDatasrc); cat $*/cleanup.sql)\
251
|"time" $(psqlNoSearchPath) --echo-all --set=table='"$*"' $(logInstall*Add),\
252
(prefix=; . $(bin)/vegbien_dest; unset schemas; env schema=$(datasrc) table=$*\
253
$(bin)/csv2db) $(logInstall*Add))
254

    
255
define exportHeader
256
$(cleanup)
257
echo 'SELECT * FROM "$(datasrc)"."$*" LIMIT 0;'|$(psqlNoSearchPath) \
258
--no-align --field-separator=, --pset=footer=off >$*/header.csv
259
endef
260

    
261
# For staging tables which are derived by joining together other staging tables.
262
%/install %/header.csv: %/create.sql _always
263
	($(inDatasrc); echo 'CREATE TABLE "$*" AS'; cat $<; echo ';')|"time" \
264
$(psqlNoSearchPath) --echo-all --set=schema='"$(datasrc)"' --set=table='"$*"' \
265
$(logInstall*)
266
	$(exportHeader)
267
.PRECIOUS: %/header.csv
268

    
269
%/install: _always
270
	$(if $(isRef),$(exportHeader),$(if $(nonXml),$(import_install_)))
271
	$(if $(wildcard $*/postprocess.sql),($(inDatasrc); cat $*/postprocess.sql;)\
272
|"time" $(psqlNoSearchPath) --echo-all --set=table='"$*"' $(logInstall*Add))
273
define import_install_
274
(prefix=; . $(bin)/vegbien_dest; unset schemas; "time" nice -n +5\
275
env schema=$(datasrc) table=$* $(bin)/csv2db $(catSrcs) $(logInstall*))
276
$(if $(filter $(srcTable),$*),($(inDatasrc);\
277
echo 'ALTER TABLE "$(datasrc)"."$*" RENAME row_num TO "$*.row_num";')|"time"\
278
$(psqlNoSearchPath) --echo-all --set=table='"$*"' $(logInstall*Add))
279
endef
280
# table-scope src table's row_num col to allow joining it with other tables
281

    
282
%/uninstall: _always
283
	echo 'DROP TABLE IF EXISTS "$(datasrc)"."$*" CASCADE;'|$(psqlNoSearchPath)
284

    
285
%/reinstall: _always %/uninstall %/install ;
286

    
287
cleanup: _always $(tables:%=%/cleanup) ;
288

    
289
# WARNING: This removes any index comments, due to a PostgreSQL bug.
290
# This occurs because ALTER TABLE recreates the index but not its comment.
291
%/cleanup: _always
292
	$(cleanup)
293

    
294
##### Maps building
295

    
296
# Maps to (try to) build are added to this
297
maps :=
298

    
299
srcRoot = $(mappings)/root.sh
300
mkSrcMap = $(catSrcs)|(. $(srcRoot); env datasrc=$(datasrc) $(bin)/src_map >$@)
301

    
302
define translate
303
$(bin)/in_place $< $(bin)/canon 1 $(1)
304
$(bin)/in_place $< $(bin)/translate 1 $(1)
305
endef
306
translate? = $(if $(wildcard $(1)),$(translate))
307

    
308
$(srcDict):# empty target in case it doesn't exist
309

    
310
# Via maps cleanup
311
ifneq ($(filter %/.map.csv.last_cleanup,$(MAKECMDGOALS)),)
312
%/.map.csv.last_cleanup: %/map.csv $(vocab) $(dict) $(coreMap) $(srcDict)
313
	$(call translate?,$(srcDict))
314
	$(bin)/in_place $< $(bin)/canon 1 $(vocab)
315
	$(call translate?,$(dict))
316
	touch $@
317
	+$(selfMake) $(<:%/map.csv=%/unmapped_terms.csv)
318
	+$(selfMake) $(<:%/map.csv=%/new_terms.csv)
319
.PRECIOUS: %/.map.csv.last_cleanup
320
else
321
%/map.csv: _always
322
	$(if $(wildcard $@),,$(if $(nonXml),$(mkSrcMap)))
323
	+$(selfMake) $(@:%/map.csv=%/.map.csv.last_cleanup)
324
.PRECIOUS: %/map.csv
325
endif
326

    
327
%/VegBIEN.csv: %/map.csv $(coreMap)
328
	<$< $(bin)/cat_cols 1 2|$(bin)/join $(coreMap)|$(bin)/sort_map >$@
329
maps += $(autogenMaps)
330

    
331
maps: $(maps) _always ;
332

    
333
all += $(maps)
334

    
335
##### Maps validation
336

    
337
# `tail -n +2`: Remove header before running filter_out_ci because filter_out_ci
338
# only removes the header if it matches the vocabulary's header.
339

    
340
%/unmapped_terms.csv: %/map.csv $(coreMap)
341
	tail -n +2 $<|$(bin)/cols 1|$(bin)/filter_out_ci 0 $(coreMap) >$@
342
	$(bin)/autoremove $@
343

    
344
%/new_terms.csv: %/map.csv $(vocab) $(dict) %/unmapped_terms.csv
345
	$(newTerms)
346
	$(bin)/autoremove $@
347
newTerms = tail -n +2 $<|$(bin)/filter_out_ci 0 $(vocab)|$(bin)/filter_out_ci 0\
348
$(dict) $(if $(wildcard $(word 4,$+)),|$(bin)/filter_out_ci 0 $(word 4,$+))\
349
|grep -vE '^"?:' >$@; exit 0# because grep exits nonzero if no match
350

    
351
termsSubdirs := $(importTables)
352

    
353
include $(root)/lib/mappings.Makefile
354

    
355
##### External dependencies
356

    
357
$(root)/%: _always
358
	+$(subMake)
359
.PRECIOUS: $(root)/% # let ext. dir's Makefile decide whether to delete on error
360

    
361
##### Mapping
362

    
363
+maps = $(filter %/map.csv %/VegBIEN.csv $(mappings)/%,$(+_))
364
map2db = env in_database=vegbien in_schema=$(datasrc) in_table=$*\
365
out_database=vegbien $(root)/map $(+maps)
366

    
367
##### Import to VegBIEN
368

    
369
profileTest = $(if $(profile),$(if $(test),1))
370
profileOnly = -env profile_to=/dev/fd/3 $(map2db) 3>&1 1>&2|\
371
$(bin)/profile_stats /dev/fd/0
372

    
373
log_ = $*/logs/$(if $(n),n=$(n).,)$(version).log.sql
374
trace = $(log_:.log.sql=.trace)
375
import = -$(if $(profileTest),$(profileOnly),(set -x; "time" env commit=1\
376
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\
377
$(if $(n),,&>$(log_)))$(if $(log),$(if $(n), 2>&1|tee -a $(log_))))
378
# don't abort on import errors, which often relate to invalid input data
379

    
380
import: $(importTables:%=%/import) _always ;
381

    
382
%/import: %/VegBIEN.csv _always
383
	$(import)
384
# default:
385
%/import: _always ;
386

    
387
##### Log files from import
388

    
389
logs := $(wildcard */logs/*.log.sql */logs/*.trace)
390

    
391
rm_logs: _always
392
	$(RM) $(logs)
393

    
394
##### Verification of import
395

    
396
verifyTables := $(patsubst verify/%.ref,%,$(wildcard verify/*.ref))
397

    
398
verify: $(verifyTables:%=%/verify) _always ;
399

    
400
%/verify: verify/%.ref verify/%.out _always
401
	-$(diffVerbose) $(+_)
402
# don't abort on verification errors, which are expected during development
403
# default:
404
%/verify: verify/%.out _always
405
	$(if $(shell test -e $< && echo t),cat $<)
406
# don't run if verify/%.out's default do-nothing action was used
407
# can't use $(wildcard) because it won't recheck file after verify/%.out is run
408

    
409
psqlExport := "time" $(psqlNoSearchPath) --no-align --field-separator=$$'\t'\
410
--pset=footer=off --pset=null=NULL
411
# Note that using $(inDatasrc) will not work with datasources whose tables are
412
# the same name as VegBIEN tables (likely only VegBank), because the datasource
413
# is first in the search_path.
414
verify = $(if $(reverify),($(inDatasrc); cat $<)|$(psqlExport)\
415
--set=datasource="'$(datasrc)'" >$@)
416

    
417
verify/%.out: $(mappings)/verify.%.sql _always
418
	$(verify)
419
.PRECIOUS: verify/%.out # save partial output in case of error to help debugging
420
# default:
421
verify/%.out: _always ;
422

    
423
all += $(wildcard verify/*.out)
424

    
425
%.ref: %.ref.sql
426
	($(inDatasrc); cat $<)|$(psqlExport) >$@
427
.PRECIOUS: %.ref # there must always be a .ref for the make rules to work
428

    
429
##### Editing import
430

    
431
rotate: _always
432
	echo "UPDATE source SET shortname = shortname||'.$(version)'\
433
WHERE shortname = '$(datasrc)';"|$(psqlAsBien)
434

    
435
rm: _always
436
	echo "DELETE FROM source WHERE shortname = '$(datasrc)';"|$(psqlAsBien)
437

    
438
##### Testing
439

    
440
testRefOutput = $(subst .by_col,,$(1))
441
testRef = $(testRefOutput).ref
442
hasOwnRef = $(filter $@,$(call testRefOutput,$@))
443
# filter returns non-empty if they are equal
444

    
445
# `rm $@`: Remove outputs of successful tests to reduce clutter
446
# `$(foreach use_staged...)`: Run with use_staged=1
447
define runTest
448
@echo "Testing $(abspath $@)..."
449
>$@ env test=1 n=$(test_n) $(1) $(foreach use_staged,1,$(map2db))
450
$(if $(hasOwnRef),,-)@(set -x; $(diffIgnoreSpace) $(call testRef,$@) $@) 2>&1\
451
&& rm $@ || { e=$$?; $(if $(wildcard $(call testRef,$@)),,cat $@;)\
452
$(if $(hasOwnRef),\
453
{\
454
read -p $(emph)'Accept new test output? (y/n)'$(endEmph) REPLY;\
455
if test "$$REPLY" = y; then\
456
(set -x; $(MAKE) $@-ok --directory=$(realpath .) --makefile=../input.Makefile);\
457
exit 0;\
458
fi;\
459
};,\
460
echo $(emph)"Note: The preceding failed test is compared to another test's\
461
output"$(endEmph);\
462
echo $(emph)"When it fails, this always indicates a bug"$(endEmph);\
463
)\
464
exit $$e;}
465
endef
466

    
467
tests :=
468

    
469
%/test: %/test.xml _always ;
470

    
471
# Requires staging tables. To create them, run `make inputs/<datasrc>/install`.
472
# Non-flat-file inputs fall back to mimicking a successful test
473
%/test.xml: %/VegBIEN.csv _always
474
	$(if $(nonXml),$(call runTest,by_col=))
475
tests += %/test.xml
476

    
477
%/test.by_col.xml: %/VegBIEN.csv _always
478
	$(if $(nonXml),$(call runTest,by_col=1))
479

    
480
# Only run column-based tests if column-based mode enabled, because these tests
481
# are much slower than the row-based tests for small numbers of rows
482
ifneq ($(by_col),)
483
tests += %/test.by_col.xml
484
endif
485

    
486
testOutputs := $(foreach test,$(tests),$(importTables:%=$(test)))
487

    
488
.PRECIOUS: $(testOutputs) # save outputs of failed tests so they can be accepted
489

    
490
test: _always $(testOutputs) ;
491

    
492
all += $(wildcard %/test*.xml)
493

    
494
# Accepts a test output: make <test_output_path>-ok
495
%-ok: _always
496
	mv $* $(call testRef,$*)
497

    
498
accept-all: _always
499
	+yes|$(selfMake) test
500

    
501
##### Documentation
502

    
503
steps = $(selfMake) -s $*/import test=1 by_col=1 verbosity=2 n=100\
504
2>&1|$(bin)/debug2redmine >$@
505

    
506
%/logs/steps.by_col.log.sql: _always
507
	+$(steps)
(4-4/4)