Revision 77ed568c
Added by Jim Regetz over 12 years ago
- ID 77ed568c231ba784945c920039ec74668ddd2ac6
| shared/extra/organisms-content-migration.sh | ||
|---|---|---|
| 31 | 31 |
mkdir $LAYERS/code/climate |
| 32 | 32 |
mkdir $LAYERS/code/terrain |
| 33 | 33 |
mkdir $LAYERS/code/land-cover |
| 34 |
# create (temporary) home for other cruft that i'm not quite ready to delete |
|
| 35 |
mkdir $LAYERS/cruft |
|
| 36 | 34 |
# create (temporary?) home for everything else |
| 37 | 35 |
mkdir $LAYERS/experimental |
| 38 | 36 |
mkdir $LAYERS/experimental/terrain |
| 39 | 37 |
mkdir $LAYERS/experimental/land-cover |
| 38 |
mkdir $LAYERS/experimental/cruft |
|
| 40 | 39 |
|
| 41 | 40 |
#======================================================================= |
| 42 | 41 |
# carry out file migration, reorganization, and cleanup |
| ... | ... | |
| 150 | 149 |
# |
| 151 | 150 |
|
| 152 | 151 |
# remove gdal-generated metadata files |
| 153 |
rm $ORGANISMS/DEM/asterGdem2/*.aux.xml |
|
| 154 |
rm $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/*.aux.xml |
|
| 155 |
rm $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/Mosaiced/N59to60/*.aux.xml |
|
| 152 |
rm $ORGANISMS/DEM/asterGdem2/*.tif.aux.xml
|
|
| 153 |
rm $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/*.tif.aux.xml
|
|
| 154 |
rm $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/Mosaiced/N59to60/*.tif.aux.xml
|
|
| 156 | 155 |
|
| 157 | 156 |
# first flatten out directories |
| 158 | 157 |
mv $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/Mosaiced/N59to60 \ |
| ... | ... | |
| 264 | 263 |
# |
| 265 | 264 |
|
| 266 | 265 |
# remove gdal-generated metadata files |
| 267 |
rm $ORGANISMS/DEM/GlobalProduct/*.aux.xml |
|
| 266 |
rm $ORGANISMS/DEM/GlobalProduct/*.tif.aux.xml
|
|
| 268 | 267 |
# migrate to ~layers |
| 269 | 268 |
mv $ORGANISMS/DEM/GlobalProduct $LAYERS/data/terrain/dem-fused |
| 270 | 269 |
|
| ... | ... | |
| 306 | 305 |
rm $ORGANISMS/DEM/asterGdem/N59to81_E60to99/USGS_ErosDEM_N59to81E60to99/E060N90_Clipped |
| 307 | 306 |
|
| 308 | 307 |
# remove gdal-generated metadata file |
| 309 |
rm $ORGANISMS/DEM/usgsGTOPO30/e020n90/*.aux.xml |
|
| 308 |
rm $ORGANISMS/DEM/usgsGTOPO30/e020n90/*.DEM.aux.xml
|
|
| 310 | 309 |
# remove Mac OS X file metadata cruft |
| 311 | 310 |
find $ORGANISMS/DEM/usgsGTOPO30 -name ".DS_Store" -exec rm {} \;
|
| 312 | 311 |
find $ORGANISMS/DEM/usgsGTOPO30 -name "._*" -exec rm {} \;
|
| ... | ... | |
| 330 | 329 |
# |
| 331 | 330 |
|
| 332 | 331 |
# remove gdal-generated metadata files |
| 333 |
rm $ORGANISMS/GMTED2010/*.aux.xml |
|
| 332 |
rm $ORGANISMS/GMTED2010/*.tif.aux.xml
|
|
| 334 | 333 |
# remove ArcMap(?) schema lock file |
| 335 | 334 |
rm $ORGANISMS/GMTED2010/GMTED2010_Spatial_Metadata/GMTED2010_Spatial_Metadata.shp.IGSKMNCNWK00726.6284.8844.sr.lock |
| 336 | 335 |
# migrate original (I think) downloaded data (OR only) |
| ... | ... | |
| 363 | 362 |
# Nunokawa content |
| 364 | 363 |
# |
| 365 | 364 |
|
| 366 |
# TODO: deal with data |
|
| 367 |
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_82N.tif # Int16 59N-82N GDEM2 elevation |
|
| 368 |
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_below.tif # Int16 59N-60N GDEM2 elevation |
|
| 369 |
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_above.tif # Int16 60N-61N GDEM2 elevation |
|
| 370 |
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_straddle.tif # Int16 59N-61N GDEM2 elevation (GDEM2 above, GDEM2 below) |
|
| 371 |
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_straddle_a.tif # Flt32 59N-61N GDEM2-based aspect |
|
| 372 |
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_straddle_s.tif # Flt32 59N-61N GDEM2-based slope |
|
| 373 |
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_below_blendgau.tif # Flt32 59N-60N GDEM2/SRTM blended elevation |
|
| 374 |
|
|
| 375 |
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_straddle.tif # Int16 59N-61N GDEM2/SRTM unblended elevation (GDEM2 above, SRTM below) |
|
| 376 |
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_straddle_a.tif # Flt32 59N-61N unblended-based aspect |
|
| 377 |
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_straddle_s.tif # Flt32 59N-61N unblended-based aspect |
|
| 378 |
|
|
| 379 |
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_blendgau.tif # Int16 59N-61N GDEM2/SRTM blended elevation (GDEM2 above, blended below) |
|
| 380 |
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_blendgau_a.tif # Flt32 59N-61N blend-based aspect |
|
| 381 |
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_blendgau_s.tif # Flt32 59N-61N blend-based aspect |
|
| 382 |
|
|
| 383 |
$ORGANISMS/DEM/Yuni/Data/srtm/srtm_*_below.tif # Int16 59N-60N SRTM elevation |
|
| 384 |
$ORGANISMS/DEM/Yuni/Data/srtm/srtm_*_below_a.tif # Flt32 59N-60N SRTM-based aspect |
|
| 385 |
$ORGANISMS/DEM/Yuni/Data/srtm/srtm_*_below_below_s.tif # Flt32 59N-60N SRTM-based slope |
|
| 386 |
|
|
| 387 |
$ORGANISMS/DEM/Yuni/Data/aster2/aster.vrt # VRT - GDEM2 from ~59N-82N (with 1/2-pixel offset) |
|
| 388 |
$ORGANISMS/DEM/Yuni/Data/srtm/srtm.vrt # VRT - SRTM from ~55N-60N (with 1/2-pixel offset) |
|
| 365 |
# regetz notes about nunokawa boundary datasets |
|
| 366 |
# aster2/aster2_*_82N.tif # Int16 59N-82N GDEM2 elevation |
|
| 367 |
# aster2/aster2_*_below.tif # Int16 59N-60N GDEM2 elevation |
|
| 368 |
# aster2/aster2_*_above.tif # Int16 60N-61N GDEM2 elevation |
|
| 369 |
# aster2/aster2_*_straddle.tif # Int16 59N-61N GDEM2 elevation (GDEM2 above, GDEM2 below) |
|
| 370 |
# aster2/aster2_*_straddle_a.tif # Flt32 59N-61N GDEM2-based aspect |
|
| 371 |
# aster2/aster2_*_straddle_s.tif # Flt32 59N-61N GDEM2-based slope |
|
| 372 |
# aster2/aster2_*_below_blendgau.tif # Flt32 59N-60N GDEM2/SRTM blended elevation |
|
| 373 |
# aster2/fused_*_straddle.tif # Int16 59N-61N GDEM2/SRTM unblended elevation (GDEM2 above, SRTM below) |
|
| 374 |
# aster2/fused_*_straddle_a.tif # Flt32 59N-61N unblended-based aspect |
|
| 375 |
# aster2/fused_*_straddle_s.tif # Flt32 59N-61N unblended-based aspect |
|
| 376 |
# aster2/fused_*_blendgau.tif # Int16 59N-61N GDEM2/SRTM blended elevation (GDEM2 above, blended below) |
|
| 377 |
# aster2/fused_*_blendgau_a.tif # Flt32 59N-61N blend-based aspect |
|
| 378 |
# aster2/fused_*_blendgau_s.tif # Flt32 59N-61N blend-based aspect |
|
| 379 |
# srtm/srtm_*_below.tif # Int16 59N-60N SRTM elevation |
|
| 380 |
# srtm/srtm_*_below_a.tif # Flt32 59N-60N SRTM-based aspect |
|
| 381 |
# srtm/srtm_*_below_below_s.tif # Flt32 59N-60N SRTM-based slope |
|
| 382 |
# aster2/aster.vrt # VRT - GDEM2 from ~59N-82N (with 1/2-pixel offset) |
|
| 383 |
# srtm/srtm.vrt # VRT - SRTM from ~55N-60N (with 1/2-pixel offset) |
|
| 389 | 384 |
|
| 390 | 385 |
# remove duplicate file |
| 391 | 386 |
rm $ORGANISMS/DEM/Yuni/Data/aster2/fused_w180w141 |
| ... | ... | |
| 399 | 394 |
rm $ORGANISMS/DEM/Yuni/scripts/toProduceData/gaussian.r~ |
| 400 | 395 |
rm $ORGANISMS/DEM/Yuni/scripts/toProduceData/mkVrt_Tiff.r~ |
| 401 | 396 |
rm $ORGANISMS/DEM/Yuni/scripts/toProduceData/slope_aspect.r~ |
| 402 |
# migrate scripts files |
|
| 397 |
|
|
| 398 |
# migrate scripts |
|
| 403 | 399 |
mv $ORGANISMS/DEM/Yuni/scripts \ |
| 404 | 400 |
$LAYERS/code/terrain/nunokawa-scripts |
| 405 | 401 |
# migrate documents |
| ... | ... | |
| 409 | 405 |
$LAYERS/documentation/terrain/nunokawa-documents/ |
| 410 | 406 |
mv -i $ORGANISMS/DEM/Yuni/metadata.txt \ |
| 411 | 407 |
$LAYERS/documentation/terrain/nunokawa-documents/ |
| 408 |
# migrate remaining Data contents to 60N boundary dir |
|
| 409 |
mv -i $ORGANISMS/DEM/Yuni/Data \ |
|
| 410 |
$LAYERS/experimental/terrain/north-60 |
|
| 411 |
|
|
| 412 |
# remove now-empty directories |
|
| 413 |
rmdir $ORGANISMS/DEM/Yuni |
|
| 414 |
rmdir $ORGANISMS/DEM |
|
| 412 | 415 |
|
| 413 | 416 |
# |
| 414 | 417 |
# Remaining DEM content |
| ... | ... | |
| 419 | 422 |
|
| 420 | 423 |
|
| 421 | 424 |
rm $ORGANISMS/DEM/checklog |
| 422 |
mv $ORGANISMS/DEM/checkBadAsterGDEMFiles.sh $LAYERS/cruft/ |
|
| 423 |
mv $ORGANISMS/DEM/CheckGDEMLog.txt $LAYERS/cruft/ |
|
| 425 |
mv $ORGANISMS/DEM/checkBadAsterGDEMFiles.sh $LAYERS/experimental/cruft/
|
|
| 426 |
mv $ORGANISMS/DEM/CheckGDEMLog.txt $LAYERS/experimental/cruft/
|
|
| 424 | 427 |
|
| 425 | 428 |
|
| 426 | 429 |
#=========# |
| ... | ... | |
| 651 | 654 |
|
| 652 | 655 |
# move last few things... |
| 653 | 656 |
mv -i $ORGANISMS/MODIS_LST_Oregon/hdf.txt \ |
| 654 |
$LAYERS/cruft/clim-MOD11A1.004-OR-orig-hdf.txt |
|
| 657 |
$LAYERS/experimental/cruft/clim-MOD11A1.004-OR-orig-hdf.txt
|
|
| 655 | 658 |
mv -i $ORGANISMS/MODIS_LST_Oregon/SDS_PctFills.txt \ |
| 656 |
$LAYERS/cruft/clim-MOD11A1.004-OR-orig-SDS_PctFills.txt |
|
| 659 |
$LAYERS/experimental/cruft/clim-MOD11A1.004-OR-orig-SDS_PctFills.txt
|
|
| 657 | 660 |
|
| 658 | 661 |
# remove all the now-empty directories... |
| 659 | 662 |
# tree $ORGANISMS/MODIS_LST_Oregon |
| ... | ... | |
| 829 | 832 |
mv $ORGANISMS/README.txt $LAYERS/documentation/organisms-homedir-readme.txt |
| 830 | 833 |
mv $ORGANISMS/CHANGES.txt $LAYERS/documentation/organisms-homedir-changes.txt |
| 831 | 834 |
# move old file extension summary |
| 832 |
mv -i $ORGANISMS/file-extension-summary.txt $LAYERS/cruft/ |
|
| 835 |
mv -i $ORGANISMS/file-extension-summary.txt $LAYERS/experimental/cruft/
|
|
| 833 | 836 |
|
| 834 | 837 |
# |
| 835 | 838 |
# Desktop |
| 836 | 839 |
# |
| 837 | 840 |
|
| 838 | 841 |
# move MRT installer to cruft |
| 839 |
mv $ORGANISMS/Desktop/MRT_download_Linux $LAYERS/cruft/ |
|
| 842 |
mv $ORGANISMS/Desktop/MRT_download_Linux $LAYERS/experimental/cruft/
|
|
| 840 | 843 |
# remove orphaned bil metadata file |
| 841 | 844 |
rm $ORGANISMS/Desktop/ASTER_Test.bil.aux.xml |
| 842 | 845 |
# remove empty Desktop dir |
| ... | ... | |
| 847 | 850 |
# |
| 848 | 851 |
|
| 849 | 852 |
# move pyhdf package to cruft |
| 850 |
mv $ORGANISMS/pyhdf $LAYERS/cruft/ |
|
| 853 |
mv $ORGANISMS/pyhdf $LAYERS/experimental/cruft/
|
|
| 851 | 854 |
|
| 852 | 855 |
# |
| 853 | 856 |
# GIS/GDD |
| ... | ... | |
| 927 | 930 |
# Oregon |
| 928 | 931 |
# |
| 929 | 932 |
|
| 930 |
# TODO |
|
| 933 |
# for now just migrate to experimental area |
|
| 934 |
mv $ORGANISMS/Oregon $LAYERS/experimental/oregon |
|
| 931 | 935 |
|
| 932 | 936 |
# |
| 933 | 937 |
# steph |
| ... | ... | |
| 965 | 969 |
# now migrate code into git repository clone |
| 966 | 970 |
#======================================================================= |
| 967 | 971 |
|
| 972 |
export REPO="." |
|
| 973 |
|
|
| 968 | 974 |
# nunokawa terrain scripts |
| 969 | 975 |
mkdir terrain/research/gtopo30 |
| 970 |
mv -i code/terrain/nunokawa-scripts/toProduceData/clipUSGS.r terrain/research/gtopo30/ |
|
| 971 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/check.r terrain/tests/ |
|
| 972 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/meanElv_OnlyN59.r terrain/research/north-60/ |
|
| 976 |
mv -i code/terrain/nunokawa-scripts/toProduceData/clipUSGS.r \ |
|
| 977 |
$REPO/terrain/research/gtopo30/ |
|
| 978 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/check.r \ |
|
| 979 |
$REPO/terrain/tests/ |
|
| 980 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/meanElv_OnlyN59.r |
|
| 981 |
$REPO/terrain/research/north-60/ |
|
| 973 | 982 |
rm code/terrain/nunokawa-scripts/toAnalyzeData/meanElv.r |
| 974 | 983 |
rm code/terrain/nunokawa-scripts/toAnalyzeData/Deltas.r~ |
| 975 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/Deltas.r terrain/research/north-60/ |
|
| 984 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/Deltas.r \ |
|
| 985 |
$REPO/terrain/research/north-60/ |
|
| 976 | 986 |
rm code/terrain/nunokawa-scripts/toAnalyzeData/rmse_cor.r~ |
| 977 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/rmse_cor.r terrain/research/north-60/ |
|
| 987 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/rmse_cor.r \ |
|
| 988 |
$REPO/terrain/research/north-60/ |
|
| 978 | 989 |
rm code/terrain/nunokawa-scripts/toAnalyzeData/slope.r~ |
| 979 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/{aspect,slope}.r terrain/research/north-60/
|
|
| 980 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/negativeTable.r terrain/research/north-60/ |
|
| 981 |
mv -i code/terrain/nunokawa-scripts/toProduceData/*.{r,r~} terrain/research/north-60/
|
|
| 990 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/{aspect,slope}.r
|
|
| 991 |
$REPO/terrain/research/north-60/ |
|
| 992 |
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/negativeTable.r \ |
|
| 993 |
$REPO/terrain/research/north-60/ |
|
| 994 |
mv -i code/terrain/nunokawa-scripts/toProduceData/*.{r,r~} \
|
|
| 995 |
$REPO/terrain/research/north-60/ |
|
| 982 | 996 |
|
| 983 | 997 |
# robinson terrain scripts |
| 984 |
mv -i code/terrain/DEM_ProcessingScripts/Aster_CheckMosaicedTilesExtents.py terrain/tests/ |
|
| 985 |
mv -i code/terrain/DEM_ProcessingScripts/CheckPixelValuesAtOverlapZones.txt terrain/tests/ |
|
| 986 |
mv -i code/terrain/DEM_ProcessingScripts/Aster\&SRTM_* terrain/tests/ |
|
| 987 |
mv -i code/terrain/DEM_ProcessingScripts/Gaussian_Blend.r terrain/procedures/ |
|
| 988 |
mv -i terrain/tests/Aster\&SRTM_* terrain/procedures/ |
|
| 989 |
mv -i code/terrain/DEM_ProcessingScripts/SRTM_ClipToN59to60.txt terrain/procedures/ |
|
| 990 |
mv -i code/terrain/DEM_ProcessingScripts/Mosaicing_AllTiles_East\&WestHemispheres.txt terrain/procedures/ |
|
| 991 |
mv -i code/terrain/DEM_ProcessingScripts/AsterMosaicingScripts terrain/procedures/ |
|
| 998 |
mv -i code/terrain/DEM_ProcessingScripts/Aster_CheckMosaicedTilesExtents.py \ |
|
| 999 |
$REPO/terrain/tests/ |
|
| 1000 |
mv -i code/terrain/DEM_ProcessingScripts/CheckPixelValuesAtOverlapZones.txt \ |
|
| 1001 |
$REPO/terrain/tests/ |
|
| 1002 |
mv -i code/terrain/DEM_ProcessingScripts/Aster\&SRTM_* \ |
|
| 1003 |
$REPO/terrain/tests/ |
|
| 1004 |
mv -i code/terrain/DEM_ProcessingScripts/Gaussian_Blend.r \ |
|
| 1005 |
$REPO/terrain/procedures/ |
|
| 1006 |
mv -i terrain/tests/Aster\&SRTM_* \ |
|
| 1007 |
$REPO/terrain/procedures/ |
|
| 1008 |
mv -i code/terrain/DEM_ProcessingScripts/SRTM_ClipToN59to60.txt \ |
|
| 1009 |
$REPO/terrain/procedures/ |
|
| 1010 |
mv -i code/terrain/DEM_ProcessingScripts/Mosaicing_AllTiles_East\&WestHemispheres.txt \ |
|
| 1011 |
$REPO/terrain/procedures/ |
|
| 1012 |
mv -i code/terrain/DEM_ProcessingScripts/AsterMosaicingScripts \ |
|
| 1013 |
$REPO/terrain/procedures/ |
|
| 992 | 1014 |
|
| 993 | 1015 |
# robinson land-cover check |
| 994 |
mv -i code/land-cover/CheckForNoDataValues_LandCoverFiles.txt land-cover/tests/ |
|
| 1016 |
mv -i code/land-cover/CheckForNoDataValues_LandCoverFiles.txt \ |
|
| 1017 |
$REPO/land-cover/tests/ |
|
| 995 | 1018 |
|
| 996 | 1019 |
# robinson/donoghue Oregon MODIS LST processing code |
| 997 | 1020 |
mkdir climate/research/oregon |
| 998 | 1021 |
mkdir climate/research/oregon/modis-lst |
| 999 |
mv -i code/climate/modis-lst-oregon/*.{r,R,py} climate/research/oregon/modis-lst/
|
|
| 1022 |
mv -i code/climate/modis-lst-oregon/*.{r,R,py} \
|
|
| 1023 |
$REPO/climate/research/oregon/modis-lst/ |
|
| 1000 | 1024 |
|
| 1001 | 1025 |
# misc old (authorless) climate code |
| 1002 |
mv -i code/climate/cru_3.0_data_extract.r climate/procedures/ |
|
| 1003 |
mv -i code/climate/gdd-worldclim-tmean-map-algebra.txt climate/extra/ |
|
| 1026 |
mv -i code/climate/cru_3.0_data_extract.r \ |
|
| 1027 |
$REPO/climate/procedures/ |
|
| 1028 |
mv -i code/climate/gdd-worldclim-tmean-map-algebra.txt \ |
|
| 1029 |
$REPO/climate/extra/ |
|
| 1030 |
|
|
| 1031 |
mv -i $LAYERS/code/terrain/R_files/AsterCheck_demAndnum.r \ |
|
| 1032 |
$REPO/terrain/test/ |
|
| 1033 |
mv -i $LAYERS/code/terrain/R_files/PctNoLand.r \ |
|
| 1034 |
$REPO/terrain/research/gtopo30/ |
|
| 1035 |
|
|
| 1036 |
# set all file permissions to 644 before committing |
|
| 1037 |
find $REPO -type f -exec ^Cmod 644 {} \;
|
|
| 1004 | 1038 |
|
| 1005 |
# TODO: set all file permissions to 644 before committing |
|
Also available in: Unified diff
finished organisms->layers content migration