Revision 4118
Added by Aaron Marcuse-Kubitza over 12 years ago
| input.Makefile | ||
|---|---|---|
| 105 | 105 |
|
| 106 | 106 |
##### Existing maps discovery |
| 107 | 107 |
|
| 108 |
inputFiles := $(wildcard $(tablesSort:%=src/%/*))
|
|
| 108 |
srcMaps := $(wildcard src/*/src.csv)
|
|
| 109 | 109 |
|
| 110 |
srcMaps := $(wildcard maps/src.*.csv) |
|
| 110 |
srcMap := src/%/src.csv |
|
| 111 |
viaMap := src/%/map.csv |
|
| 112 |
fullViaMap := src/%/map.full.csv |
|
| 113 |
directMap := src/%/VegBIEN.csv |
|
| 114 |
anyMap := $(srcMap) $(viaMap) $(fullViaMap) $(directMap) |
|
| 115 |
srcsOnly = $(filter-out $(anyMap),$(1)) |
|
| 111 | 116 |
|
| 112 |
srcMap := maps/src.%.csv |
|
| 113 |
fullViaMap := maps/%.full.csv |
|
| 114 |
directMap := maps/VegBIEN.%.csv |
|
| 115 | 117 |
via := Veg+ |
| 116 | 118 |
|
| 117 | 119 |
coreMap := $(mappings)/$(via)-VegBIEN.csv |
| 118 | 120 |
coreSelfMap := $(mappings)/$(via).self.csv |
| 119 | 121 |
|
| 120 |
viaMaps := $(wildcard $(tablesSort:%=maps/map.%.csv))
|
|
| 121 |
viaMaps += $(filter-out $(viaMaps),$(srcMaps:maps/src.%=maps/map.%))
|
|
| 122 |
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard maps/map.*.csv))
|
|
| 122 |
viaMaps := $(wildcard $(tablesSort:%=src/%/map.csv))
|
|
| 123 |
viaMaps += $(filter-out $(viaMaps),$(srcMaps:src/%/src.csv=src/%/map.csv))
|
|
| 124 |
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard src/*/map.csv))
|
|
| 123 | 125 |
|
| 126 |
inputFiles := $(call srcsOnly,$(wildcard $(tablesSort:%=src/%/*))) |
|
| 127 |
|
|
| 124 | 128 |
autogenMaps := $(subst map.,VegBIEN.,$(viaMaps)) |
| 125 | 129 |
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\ |
| 126 |
$(wildcard maps/VegBIEN.*.csv))
|
|
| 127 |
tables := $(directMaps:maps/VegBIEN.%.csv=%)
|
|
| 130 |
$(wildcard src/*/VegBIEN.csv))
|
|
| 131 |
tables := $(directMaps:src/%/VegBIEN.csv=%)
|
|
| 128 | 132 |
|
| 129 | 133 |
##### Sources |
| 130 | 134 |
|
| 131 |
srcs = $(shell $(bin)/sort_filenames $(wildcard src/$*/*))
|
|
| 135 |
srcs = $(shell $(bin)/sort_filenames $(call srcsOnly,$(wildcard src/$*/*)))
|
|
| 132 | 136 |
isCsv = $(if $(srcs),$(if $(filter %.xml,$(srcs)),,1)) |
| 133 | 137 |
# true if $(srcs) non-empty and contains no *.xml |
| 134 | 138 |
catSrcs = $(bin)/cat$(if $(isCsv),_csv) $(srcs) |
| ... | ... | |
| 178 | 182 |
mkSrcMap = $(catSrcs)|(. $(srcRoot); env datasrc=$(datasrc) $(bin)/src_map >$@) |
| 179 | 183 |
|
| 180 | 184 |
# Src maps cleanup |
| 181 |
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
|
|
| 182 |
maps/.src.%.csv.last_cleanup: maps/src.%.csv
|
|
| 185 |
ifneq ($(filter src/%/.map.csv.last_cleanup,$(MAKECMDGOALS)),)
|
|
| 186 |
src/%/.src.csv.last_cleanup: src/%/src.csv
|
|
| 183 | 187 |
$(bin)/in_place $< $(bin)/cols + |
| 184 | 188 |
touch $@ |
| 185 | 189 |
else |
| 186 | 190 |
# Autogen src maps with known table names |
| 187 |
maps/src.%.csv: _always
|
|
| 191 |
src/%/src.csv: _always
|
|
| 188 | 192 |
$(if $(wildcard $@),,$(if $(isCsv),$(mkSrcMap))) |
| 189 |
+$(if $(isCsv),$(selfMake) $(@:maps/%=maps/.%.last_cleanup))
|
|
| 193 |
+$(if $(isCsv),$(selfMake) $(@:src/%/src.csv=src/%/.map.csv.last_cleanup))
|
|
| 190 | 194 |
# only build if doesn't exist |
| 191 | 195 |
# only build if CSV srcs exist for that table name |
| 192 | 196 |
endif |
| 193 | 197 |
|
| 194 | 198 |
# Try all table names |
| 195 |
createOnlyMaps += $(tablesSort:%=maps/src.%.csv)
|
|
| 199 |
createOnlyMaps += $(tablesSort:%=src/%/src.csv)
|
|
| 196 | 200 |
|
| 197 | 201 |
# Must come before $(root)/% to override it |
| 198 | 202 |
$(coreSelfMap): _always |
| ... | ... | |
| 206 | 210 |
env ignore=1 $(bin)/union <$(wordlist 1,2,$+)|,<$< )$(bin)/sort_map >$@ |
| 207 | 211 |
# can't use $(wildcard) because it won't recheck file after $(coreSelfMap) runs |
| 208 | 212 |
|
| 209 |
# Must come before maps/map.%.csv to override it
|
|
| 210 |
maps/map.%.full.csv: maps/map.%.csv $(coreSelfMap) $(srcMap)
|
|
| 213 |
# Must come before src/%/map.csv to override it
|
|
| 214 |
src/%/map.full.csv: src/%/map.csv $(coreSelfMap) $(srcMap)
|
|
| 211 | 215 |
$(makeFullCsv) |
| 212 | 216 |
$(joinSrcMap) |
| 213 |
maps += $(patsubst maps/%.csv,maps/%.full.csv,$(viaMaps))
|
|
| 217 |
maps += $(patsubst src/%/map.csv,src/%/map.full.csv,$(viaMaps))
|
|
| 214 | 218 |
|
| 215 | 219 |
# Via maps cleanup |
| 216 |
ifneq ($(filter maps/.%.last_cleanup,$(MAKECMDGOALS)),)
|
|
| 217 |
maps/.map.%.csv.last_cleanup: maps/map.%.csv $(coreMap)
|
|
| 220 |
ifneq ($(filter src/%/.map.csv.last_cleanup,$(MAKECMDGOALS)),)
|
|
| 221 |
src/%/.map.csv.last_cleanup: src/%/map.csv $(coreMap)
|
|
| 218 | 222 |
$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1 2 |
| 219 | 223 |
touch $@ |
| 220 | 224 |
# Include comment column so commented mappings are never removed |
| 221 | 225 |
else |
| 222 |
maps/map.%.csv: maps/src.%.csv _always
|
|
| 226 |
src/%/map.csv: src/%/src.csv _always
|
|
| 223 | 227 |
$(if $(wildcard $@),,$(if $(wildcard $<),$(CP) $< $@)) |
| 224 |
+$(selfMake) $(@:maps/%=maps/.%.last_cleanup)
|
|
| 228 |
+$(selfMake) $(@:src/%/map.csv=src/%/.map.csv.last_cleanup)
|
|
| 225 | 229 |
endif |
| 226 | 230 |
|
| 227 |
maps/VegBIEN.%.csv: maps/map.%.full.csv $(coreMap)
|
|
| 231 |
src/%/VegBIEN.csv: src/%/map.full.csv $(coreMap)
|
|
| 228 | 232 |
$(bin)/join <$+|$(bin)/sort_map >$@ |
| 229 | 233 |
maps += $(autogenMaps) |
| 230 | 234 |
|
| ... | ... | |
| 248 | 252 |
|
| 249 | 253 |
##### Mapping |
| 250 | 254 |
|
| 251 |
+maps = $(filter maps/% $(mappings)/%,$(+_))
|
|
| 255 |
+maps = $(filter src/%/map.csv src/%/VegBIEN.csv $(mappings)/%,$(+_))
|
|
| 252 | 256 |
map = $(if $(srcs),$(withCatSrcs) $(root)/map $(+maps),\ |
| 253 | 257 |
$(shell echo Warning: No input file src/$*/*.{$(exts)} >&2)false)
|
| 254 | 258 |
# need false to run some command, when prefixed by `env ...` below |
| ... | ... | |
| 272 | 276 |
|
| 273 | 277 |
import: $(addprefix import-,$(tables)) _always ; |
| 274 | 278 |
|
| 275 |
import-%: maps/VegBIEN.%.csv _always
|
|
| 279 |
import-%: src/%/VegBIEN.csv _always
|
|
| 276 | 280 |
$(import) |
| 277 | 281 |
# default: |
| 278 | 282 |
import-%: _always ; |
| ... | ... | |
| 352 | 356 |
|
| 353 | 357 |
tests := |
| 354 | 358 |
|
| 355 |
test/import.%.xml: maps/VegBIEN.%.csv _always
|
|
| 359 |
test/import.%.xml: src/%/VegBIEN.csv _always
|
|
| 356 | 360 |
$(test2Db) |
| 357 | 361 |
tests += test/import.%.xml |
| 358 | 362 |
|
| ... | ... | |
| 360 | 364 |
# run with use_staged=1 |
| 361 | 365 |
|
| 362 | 366 |
# Requires staging tables. To create them, run `make inputs/<datasrc>/install`. |
| 363 |
test/import.%.staging.xml: maps/VegBIEN.%.csv _always
|
|
| 367 |
test/import.%.staging.xml: src/%/VegBIEN.csv _always
|
|
| 364 | 368 |
-$(if $(isCsv),$(testStaged2Db),cp -p $(@:.staging.xml=.xml) $@) |
| 365 | 369 |
# Don't abort tester if only staging test fails, in case staging table missing |
| 366 | 370 |
# Non-flat-file inputs fall back to mimicking a successful test |
Also available in: Unified diff
inputs: Moved maps into subfolders, using the steps at <https://projects.nceas.ucsb.edu/nceas/projects/bien/wiki/VegCSV_subfolders#Move-maps-into-subfolders>