Project

General

Profile

« Previous | Next » 

Revision 4182

Added by Aaron Marcuse-Kubitza over 12 years ago

inputs: Move src subdir into main dir, using the steps at <https://projects.nceas.ucsb.edu/nceas/projects/bien/wiki/VegCSV_subfolders#Move-src-subdir-into-main-dir>

View differences:

input.Makefile
86 86 
%/: % _always ;
87 87 

  		




  88
  88
  
    
# Must come before `%: %.make` to override it


  		




  89
  
  
    
src/%: src/%.make _always


  		




  
  89
  
    
%: %.make _always


  		




  90
  90
  
    
	(set -x; $(make_script)) 2>>$<.log.sql


  		




  91
  
  
    
.PRECIOUS: src/% # save partial outputs of aborted src make scripts


  		




  
  91
  
    
.PRECIOUS: % # save partial outputs of aborted src make scripts


  		




  92
  92
  
    

  		




  93
  93
  
    
%: %.make _always


  		




  94
  94
  
    
	$(make_script)


  		




  ......




  96
  96
  
    
##### SVN


  		




  97
  97
  
    

  		




  98
  98
  
    
add: _always


  		




  99
  
  
    
	$(call addDirWithIgnore,src,'*')


  		




  100
  
  
    
	$(call addDirWithIgnore,src/specimens,'*')


  		




  101
  
  
    
	$(call addDirWithIgnore,src/specimens/logs,$$'*.log.sql\n*.trace')


  		




  
  99
  
    
	$(call setSvnIgnore,.,'*')


  		




  
  100
  
    
	$(call addDirWithIgnore,specimens,'*')


  		




  
  101
  
    
	$(call addDirWithIgnore,specimens/logs,$$'*.log.sql\n*.trace')


  		




  102
  102
  
    
	$(call addDirWithIgnore,verify,'*.out')


  		




  103
  103
  
    

  		




  104
  104
  
    
##### Existing maps discovery


  		




  105
  105
  
    

  		




  106
  
  
    
sortFile := src/import_order.txt


  		




  
  106
  
    
sortFile := import_order.txt


  		




  107
  107
  
    

  		




  108
  108
  
    
ifneq ($(wildcard $(sortFile)),)


  		




  109
  109
  
    
tables := $(shell cat $(sortFile))# $(shell) replaces "\n" with " "


  		




  110
  110
  
    
else


  		




  111
  
  
    
allSubdirs := $(call wildcard/,src/*/)


  		




  112
  
  
    
tables := $(call sortFilenames,$(filter-out _%,$(allSubdirs:src/%/=%)))


  		




  
  111
  
    
allSubdirs := $(call wildcard/,*/)


  		




  
  112
  
    
tables := $(call sortFilenames,$(filter-out _% src verify,$(allSubdirs:%/=%)))


  		




  113
  113
  
    
endif


  		




  114
  114
  
    

  		




  115
  
  
    
srcMaps := $(wildcard src/*/src.csv)


  		




  
  115
  
    
srcMaps := $(wildcard */src.csv)


  		




  116
  116
  
    

  		




  117
  
  
    
srcMap := src/%/src.csv


  		




  118
  
  
    
viaMap := src/%/map.csv


  		




  119
  
  
    
fullViaMap := src/%/map.full.csv


  		




  120
  
  
    
directMap := src/%/VegBIEN.csv


  		




  
  117
  
    
srcMap := %/src.csv


  		




  
  118
  
    
viaMap := %/map.csv


  		




  
  119
  
    
fullViaMap := %/map.full.csv


  		




  
  120
  
    
directMap := %/VegBIEN.csv


  		




  121
  121
  
    
anyMap := $(srcMap) $(viaMap) $(fullViaMap) $(directMap)


  		




  122
  122
  
    

  		




  123
  
  
    
anyTest = src/$*/test.%


  		




  124
  
  
    
srcsOnly = $(filter-out $(anyMap) $(anyTest) src/%/logs,$(1))


  		




  
  123
  
    
anyTest = $*/test.%


  		




  
  124
  
    
srcsOnly = $(filter-out $(anyMap) $(anyTest) %/logs,$(1))


  		




  125
  125
  
    

  		




  126
  126
  
    
via := Veg+


  		




  127
  127
  
    

  		




  128
  128
  
    
coreMap := $(mappings)/$(via)-VegBIEN.csv


  		




  129
  129
  
    
coreSelfMap := $(mappings)/$(via).self.csv


  		




  130
  130
  
    

  		




  131
  
  
    
viaMaps := $(wildcard $(tables:%=src/%/map.csv))


  		




  132
  
  
    
viaMaps += $(filter-out $(viaMaps),$(srcMaps:src/%/src.csv=src/%/map.csv))


  		




  133
  
  
    
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard src/*/map.csv))


  		




  
  131
  
    
viaMaps := $(wildcard $(tables:%=%/map.csv))


  		




  
  132
  
    
viaMaps += $(filter-out $(viaMaps),$(srcMaps:%/src.csv=%/map.csv))


  		




  
  133
  
    
viaMaps += $(filter-out $(viaMaps) $(fullViaMap),$(wildcard */map.csv))


  		




  134
  134
  
    

  		




  135
  135
  
    
autogenMaps := $(subst map.,VegBIEN.,$(viaMaps))


  		




  136
  136
  
    
directMaps := $(autogenMaps) $(filter-out $(autogenMaps),\


  		




  137
  
  
    
$(wildcard src/*/VegBIEN.csv))


  		




  
  137
  
    
$(wildcard */VegBIEN.csv))


  		




  138
  138
  
    

  		




  139
  139
  
    
##### Sources


  		




  140
  140
  
    

  		




  141
  
  
    
srcs = $(call sortFilenames,$(call srcsOnly,$(wildcard src/$*/*)))


  		




  
  141
  
    
srcs = $(call sortFilenames,$(call srcsOnly,$(wildcard $*/*)))


  		




  142
  142
  
    
isCsv = $(if $(srcs),$(if $(filter %.xml,$(srcs)),,1))


  		




  143
  143
  
    
    # true if $(srcs) non-empty and contains no *.xml


  		




  144
  144
  
    
catSrcs = $(bin)/cat$(if $(isCsv),_csv) $(srcs)


  		




  ......




  170
  170
  
    
	$(if $(isCsv),$(import_install_))


  		




  171
  171
  
    
import_install_ = (prefix=; . $(bin)/vegbien_dest; "time" nice -n +5\


  		




  172
  172
  
    
env schema=$(datasrc) table=$* $(bin)/csv2db $(catSrcs)\


  		




  173
  
  
    
$(if $(log),$(if $(quiet),&>,2>&1|tee )src/$*/logs/install.log.sql))


  		




  
  173
  
    
$(if $(log),$(if $(quiet),&>,2>&1|tee )$*/logs/install.log.sql))


  		




  174
  174
  
    

  		




  175
  175
  
    
##### Maps building


  		




  176
  176
  
    

  		




  ......




  182
  182
  
    
mkSrcMap = $(catSrcs)|(. $(srcRoot); env datasrc=$(datasrc) $(bin)/src_map >$@)


  		




  183
  183
  
    

  		




  184
  184
  
    
# Src maps cleanup


  		




  185
  
  
    
ifneq ($(filter src/%/.src.csv.last_cleanup,$(MAKECMDGOALS)),)


  		




  186
  
  
    
src/%/.src.csv.last_cleanup: src/%/src.csv


  		




  
  185
  
    
ifneq ($(filter %/.src.csv.last_cleanup,$(MAKECMDGOALS)),)


  		




  
  186
  
    
%/.src.csv.last_cleanup: %/src.csv


  		




  187
  187
  
    
	$(bin)/in_place $< $(bin)/cols +


  		




  188
  188
  
    
	touch $@


  		




  189
  189
  
    
else


  		




  190
  190
  
    
# Autogen src maps with known table names


  		




  191
  
  
    
src/%/src.csv: _always


  		




  
  191
  
    
%/src.csv: _always


  		




  192
  192
  
    
	$(if $(wildcard $@),,$(if $(isCsv),$(mkSrcMap)))


  		




  193
  
  
    
	+$(if $(isCsv),$(selfMake) $(@:src/%/src.csv=src/%/.src.csv.last_cleanup))


  		




  
  193
  
    
	+$(if $(isCsv),$(selfMake) $(@:%/src.csv=%/.src.csv.last_cleanup))


  		




  194
  194
  
    
# only build if doesn't exist


  		




  195
  195
  
    
# only build if CSV srcs exist for that table name


  		




  196
  196
  
    
endif


  		




  197
  197
  
    

  		




  198
  
  
    
createOnlyMaps += $(tables:%=src/%/src.csv)


  		




  
  198
  
    
createOnlyMaps += $(tables:%=%/src.csv)


  		




  199
  199
  
    

  		




  200
  200
  
    
# Must come before $(root)/% to override it


  		




  201
  201
  
    
$(coreSelfMap): _always


  		




  ......




  203
  203
  
    
# ignore errors if $(coreSelfMap) does not exist


  		




  204
  204
  
    

  		




  205
  205
  
    
# Via maps cleanup


  		




  206
  
  
    
ifneq ($(filter src/%/.map.csv.last_cleanup,$(MAKECMDGOALS)),)


  		




  207
  
  
    
src/%/.map.csv.last_cleanup: src/%/map.csv $(coreMap)


  		




  
  206
  
    
ifneq ($(filter %/.map.csv.last_cleanup,$(MAKECMDGOALS)),)


  		




  
  207
  
    
%/.map.csv.last_cleanup: %/map.csv $(coreMap)


  		




  208
  208
  
    
	$(bin)/in_place $< env ignore=1 $(bin)/subtract $(word 2,$+) 0 1 2


  		




  209
  209
  
    
	touch $@


  		




  210
  210
  
    
# Include comment column so commented mappings are never removed


  		




  211
  211
  
    
else


  		




  212
  
  
    
src/%/map.csv: src/%/src.csv _always


  		




  
  212
  
    
%/map.csv: %/src.csv _always


  		




  213
  213
  
    
	$(if $(wildcard $@),,$(if $(wildcard $<),$(CP) $< $@))


  		




  214
  
  
    
	+$(selfMake) $(@:src/%/map.csv=src/%/.map.csv.last_cleanup)


  		




  
  214
  
    
	+$(selfMake) $(@:%/map.csv=%/.map.csv.last_cleanup)


  		




  215
  215
  
    
endif


  		




  216
  216
  
    

  		




  217
  
  
    
srcMap* = src/$*/src.csv


  		




  
  217
  
    
srcMap* = $*/src.csv


  		




  218
  218
  
    
joinSrcMap = $(if $(hasSrc),$(bin)/in_place $@ $(bin)/intersect $(srcMap*) 0)


  		




  219
  219
  
    
hasSrc = $(shell test -s $(word 3,$+) && echo t)


  		




  220
  220
  
    

  		




  221
  221
  
    
makeFullCsv = env ignore=1 $(bin)/union <$(wordlist 1,2,$+)|$(bin)/sort_map >$@


  		




  222
  222
  
    

  		




  223
  
  
    
src/%/map.full.csv: src/%/map.csv $(coreSelfMap)


  		




  
  223
  
    
%/map.full.csv: %/map.csv $(coreSelfMap)


  		




  224
  224
  
    
	$(makeFullCsv)


  		




  225
  225
  
    
	$(joinSrcMap)


  		




  226
  
  
    
maps += $(patsubst src/%/map.csv,src/%/map.full.csv,$(viaMaps))


  		




  
  226
  
    
maps += $(patsubst %/map.csv,%/map.full.csv,$(viaMaps))


  		




  227
  227
  
    

  		




  228
  
  
    
src/%/VegBIEN.csv: src/%/map.full.csv $(coreMap)


  		




  
  228
  
    
%/VegBIEN.csv: %/map.full.csv $(coreMap)


  		




  229
  229
  
    
	$(bin)/join <$+|$(bin)/sort_map >$@


  		




  230
  230
  
    
maps += $(autogenMaps)


  		




  231
  231
  
    

  		




  ......




  249
  249
  
    

  		




  250
  250
  
    
##### Mapping


  		




  251
  251
  
    

  		




  252
  
  
    
+maps = $(filter src/%/map.csv src/%/VegBIEN.csv $(mappings)/%,$(+_))


  		




  
  252
  
    
+maps = $(filter %/map.csv %/VegBIEN.csv $(mappings)/%,$(+_))


  		




  253
  253
  
    
map = $(if $(srcs),$(withCatSrcs) $(root)/map $(+maps),\


  		




  254
  
  
    
$(shell echo Warning: No input file src/$*/*.{$(exts)} >&2)false)


  		




  
  254
  
    
$(shell echo Warning: No input file $*/*.{$(exts)} >&2)false)


  		




  255
  255
  
    
# need false to run some command, when prefixed by `env ...` below


  		




  256
  256
  
    
map2db = $(if $(use_staged),env in_database=vegbien in_schema=$(datasrc)\


  		




  257
  257
  
    
in_table=$*) env out_database=vegbien $(map)


  		




  ......




  262
  262
  
    
profileOnly = -env profile_to=/dev/fd/3 $(map2db) 3>&1 1>&2|\


  		




  263
  263
  
    
$(bin)/profile_stats /dev/fd/0


  		




  264
  264
  
    

  		




  265
  
  
    
log_ = src/$*/logs/$(if $(n),n=$(n).,)$(date).log.sql


  		




  
  265
  
    
log_ = $*/logs/$(if $(n),n=$(n).,)$(date).log.sql


  		




  266
  266
  
    
trace = $(log_:.log.sql=.trace)


  		




  267
  267
  
    
import = -$(if $(profileTest),$(profileOnly),(set -x; "time" env commit=1\


  		




  268
  268
  
    
$(if $(profile),profile_to=$(trace)) $(map2db)) $(if $(log),\


  		




  ......




  271
  271
  
    

  		




  272
  272
  
    
import: $(addprefix import-,$(tables)) _always ;


  		




  273
  273
  
    

  		




  274
  
  
    
import-%: src/%/VegBIEN.csv _always


  		




  
  274
  
    
import-%: %/VegBIEN.csv _always


  		




  275
  275
  
    
	$(import)


  		




  276
  276
  
    
# default:


  		




  277
  277
  
    
import-%: _always ;


  		




  278
  278
  
    

  		




  279
  279
  
    
##### Log files from import


  		




  280
  280
  
    

  		




  281
  
  
    
logs := $(wildcard src/*/logs/*.log.sql src/*/logs/*.trace)


  		




  
  281
  
    
logs := $(wildcard */logs/*.log.sql */logs/*.trace)


  		




  282
  282
  
    

  		




  283
  283
  
    
rm_logs: _always


  		




  284
  284
  
    
	$(RM) $(logs)


  		




  ......




  347
  347
  
    

  		




  348
  348
  
    
tests :=


  		




  349
  349
  
    

  		




  350
  
  
    
src/%/test.xml: src/%/VegBIEN.csv _always


  		




  
  350
  
    
%/test.xml: %/VegBIEN.csv _always


  		




  351
  351
  
    
	$(test2Db)


  		




  352
  
  
    
tests += src/%/test.xml


  		




  
  352
  
    
tests += %/test.xml


  		




  353
  353
  
    

  		




  354
  354
  
    
testStaged2Db = $(foreach use_staged,1,$(test2Db))


  		




  355
  355
  
    
    # run with use_staged=1


  		




  356
  356
  
    

  		




  357
  357
  
    
# Requires staging tables. To create them, run `make inputs/<datasrc>/install`.


  		




  358
  
  
    
src/%/test.staging.xml: src/%/VegBIEN.csv _always


  		




  
  358
  
    
%/test.staging.xml: %/VegBIEN.csv _always


  		




  359
  359
  
    
	-$(if $(isCsv),$(testStaged2Db),cp -p $(@:.staging.xml=.xml) $@)


  		




  360
  360
  
    
# Don't abort tester if only staging test fails, in case staging table missing


  		




  361
  361
  
    
# Non-flat-file inputs fall back to mimicking a successful test


  		




  362
  
  
    
tests += src/%/test.staging.xml


  		




  
  362
  
    
tests += %/test.staging.xml


  		




  363
  363
  
    

  		




  364
  364
  
    
testOutputs := $(foreach test,$(tests),$(tables:%=$(test)))


  		




  365
  365
  
    

  		




  ......




  367
  367
  
    

  		




  368
  368
  
    
test: _always $(testOutputs) ;


  		




  369
  369
  
    

  		




  370
  
  
    
all += $(wildcard src/%/test*.xml)


  		




  
  370
  
    
all += $(wildcard %/test*.xml)


  		




  371
  371
  
    

  		




  372
  372
  
    
# Accepts a test output: make <test_output_path>-ok


  		




  373
  373
  
    
%-ok: _always


  		




  ......




  381
  381
  
    
steps = $(selfMake) -s import-$* test=1 by_col=1 verbosity=2 n=100\


  		




  382
  382
  
    
2>&1|$(bin)/debug2redmine >$@


  		




  383
  383
  
    

  		




  384
  
  
    
src/%/logs/steps.by_col.log.sql: _always


  		




  
  384
  
    
%/logs/steps.by_col.log.sql: _always


  		




  385
  385
  
    
	+$(steps)


  		












Also available in:  Unified diff