Project

General

Profile

« Previous | Next » 

Revision 77ed568c

Added by Jim Regetz almost 13 years ago

  • ID 77ed568c231ba784945c920039ec74668ddd2ac6

finished organisms->layers content migration

View differences:

shared/extra/organisms-content-migration.sh
31 31
mkdir $LAYERS/code/climate
32 32
mkdir $LAYERS/code/terrain
33 33
mkdir $LAYERS/code/land-cover
34
# create (temporary) home for other cruft that i'm not quite ready to delete
35
mkdir $LAYERS/cruft
36 34
# create (temporary?) home for everything else
37 35
mkdir $LAYERS/experimental
38 36
mkdir $LAYERS/experimental/terrain
39 37
mkdir $LAYERS/experimental/land-cover
38
mkdir $LAYERS/experimental/cruft
40 39

  
41 40
#=======================================================================
42 41
# carry out file migration, reorganization, and cleanup
......
150 149
#
151 150

  
152 151
# remove gdal-generated metadata files
153
rm $ORGANISMS/DEM/asterGdem2/*.aux.xml
154
rm $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/*.aux.xml
155
rm $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/Mosaiced/N59to60/*.aux.xml
152
rm $ORGANISMS/DEM/asterGdem2/*.tif.aux.xml
153
rm $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/*.tif.aux.xml
154
rm $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/Mosaiced/N59to60/*.tif.aux.xml
156 155

  
157 156
# first flatten out directories
158 157
mv $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/Mosaiced/N59to60 \
......
264 263
#
265 264

  
266 265
# remove gdal-generated metadata files
267
rm $ORGANISMS/DEM/GlobalProduct/*.aux.xml
266
rm $ORGANISMS/DEM/GlobalProduct/*.tif.aux.xml
268 267
# migrate to ~layers
269 268
mv $ORGANISMS/DEM/GlobalProduct $LAYERS/data/terrain/dem-fused
270 269

  
......
306 305
rm $ORGANISMS/DEM/asterGdem/N59to81_E60to99/USGS_ErosDEM_N59to81E60to99/E060N90_Clipped
307 306

  
308 307
# remove gdal-generated metadata file
309
rm $ORGANISMS/DEM/usgsGTOPO30/e020n90/*.aux.xml
308
rm $ORGANISMS/DEM/usgsGTOPO30/e020n90/*.DEM.aux.xml
310 309
# remove Mac OS X file metadata cruft
311 310
find $ORGANISMS/DEM/usgsGTOPO30 -name ".DS_Store" -exec rm {} \;
312 311
find $ORGANISMS/DEM/usgsGTOPO30 -name "._*" -exec rm {} \;
......
330 329
#
331 330

  
332 331
# remove gdal-generated metadata files
333
rm $ORGANISMS/GMTED2010/*.aux.xml
332
rm $ORGANISMS/GMTED2010/*.tif.aux.xml
334 333
# remove ArcMap(?) schema lock file
335 334
rm $ORGANISMS/GMTED2010/GMTED2010_Spatial_Metadata/GMTED2010_Spatial_Metadata.shp.IGSKMNCNWK00726.6284.8844.sr.lock
336 335
# migrate original (I think) downloaded data (OR only)
......
363 362
# Nunokawa content
364 363
#
365 364

  
366
# TODO: deal with data
367
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_82N.tif             # Int16 59N-82N GDEM2 elevation
368
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_below.tif           # Int16 59N-60N GDEM2 elevation
369
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_above.tif           # Int16 60N-61N GDEM2 elevation
370
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_straddle.tif        # Int16 59N-61N GDEM2 elevation (GDEM2 above, GDEM2 below)
371
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_straddle_a.tif      # Flt32 59N-61N GDEM2-based aspect
372
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_straddle_s.tif      # Flt32 59N-61N GDEM2-based slope
373
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_below_blendgau.tif  # Flt32 59N-60N GDEM2/SRTM blended elevation
374

  
375
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_straddle.tif         # Int16 59N-61N GDEM2/SRTM unblended elevation (GDEM2 above, SRTM below)
376
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_straddle_a.tif       # Flt32 59N-61N unblended-based aspect
377
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_straddle_s.tif       # Flt32 59N-61N unblended-based aspect
378

  
379
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_blendgau.tif         # Int16 59N-61N GDEM2/SRTM blended elevation (GDEM2 above, blended below)
380
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_blendgau_a.tif       # Flt32 59N-61N blend-based aspect
381
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_blendgau_s.tif       # Flt32 59N-61N blend-based aspect
382

  
383
$ORGANISMS/DEM/Yuni/Data/srtm/srtm_*_below.tif               # Int16 59N-60N SRTM elevation
384
$ORGANISMS/DEM/Yuni/Data/srtm/srtm_*_below_a.tif             # Flt32 59N-60N SRTM-based aspect
385
$ORGANISMS/DEM/Yuni/Data/srtm/srtm_*_below_below_s.tif       # Flt32 59N-60N SRTM-based slope
386

  
387
$ORGANISMS/DEM/Yuni/Data/aster2/aster.vrt                    # VRT - GDEM2 from ~59N-82N (with 1/2-pixel offset)
388
$ORGANISMS/DEM/Yuni/Data/srtm/srtm.vrt                       # VRT - SRTM from ~55N-60N (with 1/2-pixel offset)
365
# regetz notes about nunokawa boundary datasets
366
# aster2/aster2_*_82N.tif             # Int16 59N-82N GDEM2 elevation
367
# aster2/aster2_*_below.tif           # Int16 59N-60N GDEM2 elevation
368
# aster2/aster2_*_above.tif           # Int16 60N-61N GDEM2 elevation
369
# aster2/aster2_*_straddle.tif        # Int16 59N-61N GDEM2 elevation (GDEM2 above, GDEM2 below)
370
# aster2/aster2_*_straddle_a.tif      # Flt32 59N-61N GDEM2-based aspect
371
# aster2/aster2_*_straddle_s.tif      # Flt32 59N-61N GDEM2-based slope
372
# aster2/aster2_*_below_blendgau.tif  # Flt32 59N-60N GDEM2/SRTM blended elevation
373
# aster2/fused_*_straddle.tif         # Int16 59N-61N GDEM2/SRTM unblended elevation (GDEM2 above, SRTM below)
374
# aster2/fused_*_straddle_a.tif       # Flt32 59N-61N unblended-based aspect
375
# aster2/fused_*_straddle_s.tif       # Flt32 59N-61N unblended-based aspect
376
# aster2/fused_*_blendgau.tif         # Int16 59N-61N GDEM2/SRTM blended elevation (GDEM2 above, blended below)
377
# aster2/fused_*_blendgau_a.tif       # Flt32 59N-61N blend-based aspect
378
# aster2/fused_*_blendgau_s.tif       # Flt32 59N-61N blend-based aspect
379
# srtm/srtm_*_below.tif               # Int16 59N-60N SRTM elevation
380
# srtm/srtm_*_below_a.tif             # Flt32 59N-60N SRTM-based aspect
381
# srtm/srtm_*_below_below_s.tif       # Flt32 59N-60N SRTM-based slope
382
# aster2/aster.vrt                    # VRT - GDEM2 from ~59N-82N (with 1/2-pixel offset)
383
# srtm/srtm.vrt                       # VRT - SRTM from ~55N-60N (with 1/2-pixel offset)
389 384

  
390 385
# remove duplicate file
391 386
rm $ORGANISMS/DEM/Yuni/Data/aster2/fused_w180w141
......
399 394
rm $ORGANISMS/DEM/Yuni/scripts/toProduceData/gaussian.r~
400 395
rm $ORGANISMS/DEM/Yuni/scripts/toProduceData/mkVrt_Tiff.r~
401 396
rm $ORGANISMS/DEM/Yuni/scripts/toProduceData/slope_aspect.r~
402
# migrate scripts files
397

  
398
# migrate scripts
403 399
mv $ORGANISMS/DEM/Yuni/scripts \
404 400
   $LAYERS/code/terrain/nunokawa-scripts
405 401
# migrate documents
......
409 405
      $LAYERS/documentation/terrain/nunokawa-documents/
410 406
mv -i $ORGANISMS/DEM/Yuni/metadata.txt \
411 407
      $LAYERS/documentation/terrain/nunokawa-documents/
408
# migrate remaining Data contents to 60N boundary dir
409
mv -i $ORGANISMS/DEM/Yuni/Data \
410
      $LAYERS/experimental/terrain/north-60
411

  
412
# remove now-empty directories
413
rmdir $ORGANISMS/DEM/Yuni
414
rmdir $ORGANISMS/DEM
412 415

  
413 416
#
414 417
# Remaining DEM content
......
419 422

  
420 423

  
421 424
rm $ORGANISMS/DEM/checklog
422
mv $ORGANISMS/DEM/checkBadAsterGDEMFiles.sh $LAYERS/cruft/
423
mv $ORGANISMS/DEM/CheckGDEMLog.txt $LAYERS/cruft/
425
mv $ORGANISMS/DEM/checkBadAsterGDEMFiles.sh $LAYERS/experimental/cruft/
426
mv $ORGANISMS/DEM/CheckGDEMLog.txt $LAYERS/experimental/cruft/
424 427

  
425 428

  
426 429
#=========#
......
651 654

  
652 655
# move last few things...
653 656
mv -i $ORGANISMS/MODIS_LST_Oregon/hdf.txt \
654
      $LAYERS/cruft/clim-MOD11A1.004-OR-orig-hdf.txt
657
      $LAYERS/experimental/cruft/clim-MOD11A1.004-OR-orig-hdf.txt
655 658
mv -i $ORGANISMS/MODIS_LST_Oregon/SDS_PctFills.txt \
656
      $LAYERS/cruft/clim-MOD11A1.004-OR-orig-SDS_PctFills.txt
659
      $LAYERS/experimental/cruft/clim-MOD11A1.004-OR-orig-SDS_PctFills.txt
657 660

  
658 661
# remove all the now-empty directories...
659 662
# tree $ORGANISMS/MODIS_LST_Oregon
......
829 832
mv $ORGANISMS/README.txt $LAYERS/documentation/organisms-homedir-readme.txt
830 833
mv $ORGANISMS/CHANGES.txt $LAYERS/documentation/organisms-homedir-changes.txt
831 834
# move old file extension summary
832
mv -i $ORGANISMS/file-extension-summary.txt $LAYERS/cruft/
835
mv -i $ORGANISMS/file-extension-summary.txt $LAYERS/experimental/cruft/
833 836

  
834 837
#
835 838
# Desktop
836 839
#
837 840

  
838 841
# move MRT installer to cruft
839
mv $ORGANISMS/Desktop/MRT_download_Linux $LAYERS/cruft/
842
mv $ORGANISMS/Desktop/MRT_download_Linux $LAYERS/experimental/cruft/
840 843
# remove orphaned bil metadata file
841 844
rm $ORGANISMS/Desktop/ASTER_Test.bil.aux.xml
842 845
# remove empty Desktop dir
......
847 850
#
848 851

  
849 852
# move pyhdf package to cruft
850
mv $ORGANISMS/pyhdf $LAYERS/cruft/
853
mv $ORGANISMS/pyhdf $LAYERS/experimental/cruft/
851 854

  
852 855
#
853 856
# GIS/GDD
......
927 930
# Oregon
928 931
#
929 932

  
930
# TODO
933
# for now just migrate to experimental area
934
mv $ORGANISMS/Oregon $LAYERS/experimental/oregon
931 935

  
932 936
#
933 937
# steph
......
965 969
# now migrate code into git repository clone
966 970
#=======================================================================
967 971

  
972
export REPO="."
973

  
968 974
# nunokawa terrain scripts
969 975
mkdir terrain/research/gtopo30
970
mv -i code/terrain/nunokawa-scripts/toProduceData/clipUSGS.r terrain/research/gtopo30/
971
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/check.r terrain/tests/
972
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/meanElv_OnlyN59.r terrain/research/north-60/
976
mv -i code/terrain/nunokawa-scripts/toProduceData/clipUSGS.r \
977
      $REPO/terrain/research/gtopo30/
978
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/check.r \
979
      $REPO/terrain/tests/
980
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/meanElv_OnlyN59.r
981
      $REPO/terrain/research/north-60/
973 982
rm code/terrain/nunokawa-scripts/toAnalyzeData/meanElv.r 
974 983
rm code/terrain/nunokawa-scripts/toAnalyzeData/Deltas.r~
975
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/Deltas.r terrain/research/north-60/
984
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/Deltas.r \
985
      $REPO/terrain/research/north-60/
976 986
rm code/terrain/nunokawa-scripts/toAnalyzeData/rmse_cor.r~ 
977
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/rmse_cor.r terrain/research/north-60/
987
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/rmse_cor.r \
988
      $REPO/terrain/research/north-60/
978 989
rm code/terrain/nunokawa-scripts/toAnalyzeData/slope.r~
979
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/{aspect,slope}.r terrain/research/north-60/
980
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/negativeTable.r terrain/research/north-60/
981
mv -i code/terrain/nunokawa-scripts/toProduceData/*.{r,r~} terrain/research/north-60/
990
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/{aspect,slope}.r
991
      $REPO/terrain/research/north-60/
992
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/negativeTable.r \
993
      $REPO/terrain/research/north-60/
994
mv -i code/terrain/nunokawa-scripts/toProduceData/*.{r,r~} \
995
      $REPO/terrain/research/north-60/
982 996

  
983 997
# robinson terrain scripts
984
mv -i code/terrain/DEM_ProcessingScripts/Aster_CheckMosaicedTilesExtents.py terrain/tests/
985
mv -i code/terrain/DEM_ProcessingScripts/CheckPixelValuesAtOverlapZones.txt terrain/tests/
986
mv -i code/terrain/DEM_ProcessingScripts/Aster\&SRTM_* terrain/tests/
987
mv -i code/terrain/DEM_ProcessingScripts/Gaussian_Blend.r terrain/procedures/
988
mv -i terrain/tests/Aster\&SRTM_* terrain/procedures/
989
mv -i code/terrain/DEM_ProcessingScripts/SRTM_ClipToN59to60.txt terrain/procedures/
990
mv -i code/terrain/DEM_ProcessingScripts/Mosaicing_AllTiles_East\&WestHemispheres.txt terrain/procedures/
991
mv -i code/terrain/DEM_ProcessingScripts/AsterMosaicingScripts terrain/procedures/
998
mv -i code/terrain/DEM_ProcessingScripts/Aster_CheckMosaicedTilesExtents.py \
999
      $REPO/terrain/tests/
1000
mv -i code/terrain/DEM_ProcessingScripts/CheckPixelValuesAtOverlapZones.txt \
1001
      $REPO/terrain/tests/
1002
mv -i code/terrain/DEM_ProcessingScripts/Aster\&SRTM_* \
1003
      $REPO/terrain/tests/
1004
mv -i code/terrain/DEM_ProcessingScripts/Gaussian_Blend.r \
1005
      $REPO/terrain/procedures/
1006
mv -i terrain/tests/Aster\&SRTM_* \
1007
      $REPO/terrain/procedures/
1008
mv -i code/terrain/DEM_ProcessingScripts/SRTM_ClipToN59to60.txt \
1009
      $REPO/terrain/procedures/
1010
mv -i code/terrain/DEM_ProcessingScripts/Mosaicing_AllTiles_East\&WestHemispheres.txt \
1011
      $REPO/terrain/procedures/
1012
mv -i code/terrain/DEM_ProcessingScripts/AsterMosaicingScripts \
1013
      $REPO/terrain/procedures/
992 1014

  
993 1015
# robinson land-cover check
994
mv -i code/land-cover/CheckForNoDataValues_LandCoverFiles.txt land-cover/tests/
1016
mv -i code/land-cover/CheckForNoDataValues_LandCoverFiles.txt \
1017
      $REPO/land-cover/tests/
995 1018

  
996 1019
# robinson/donoghue Oregon MODIS LST processing code
997 1020
mkdir climate/research/oregon
998 1021
mkdir climate/research/oregon/modis-lst
999
mv -i code/climate/modis-lst-oregon/*.{r,R,py} climate/research/oregon/modis-lst/
1022
mv -i code/climate/modis-lst-oregon/*.{r,R,py} \
1023
      $REPO/climate/research/oregon/modis-lst/
1000 1024

  
1001 1025
# misc old (authorless) climate code
1002
mv -i code/climate/cru_3.0_data_extract.r climate/procedures/
1003
mv -i code/climate/gdd-worldclim-tmean-map-algebra.txt climate/extra/
1026
mv -i code/climate/cru_3.0_data_extract.r \
1027
      $REPO/climate/procedures/
1028
mv -i code/climate/gdd-worldclim-tmean-map-algebra.txt \
1029
      $REPO/climate/extra/
1030

  
1031
mv -i $LAYERS/code/terrain/R_files/AsterCheck_demAndnum.r \
1032
      $REPO/terrain/test/
1033
mv -i $LAYERS/code/terrain/R_files/PctNoLand.r \
1034
      $REPO/terrain/research/gtopo30/
1035

  
1036
# set all file permissions to 644 before committing
1037
find $REPO -type f -exec ^Cmod 644 {} \;
1004 1038

  
1005
# TODO: set all file permissions to 644 before committing
terrain/research/gtopo30/PctNoLand.r
1
library(raster)
2

  
3
#Get list of tile names and add full pathname to beginning
4
Tiles <- list.files("DEM/asterGdem/N59to81_W20toE19", pattern="^ASTGTM.*_dem.tif$")
5
Tiles<- paste("DEM/asterGdem/N59to81_W20toE19/", Tiles, sep="")
6

  
7
# datavalues: 0= water, -9999= nodata
8

  
9
#Create new vectors for name of tile and accompanying % of cells= ocean and non-landmass
10
#   IF % OCEAN AND % NON-LAND ARE VERY DIFFERENT USE % NON-LAND FOR QC CHECK, IF THEY
11
#   ARE THE SAME, % OCEAN IS FASTER TO CALCULATE SO USE THIS
12

  
13
l<- length (Tiles)
14
Names<- c(1:l)*NA
15
PctNotLand<- c(1:l) * NA
16
PctOcean<- c(1:l)*NA
17

  
18
#Fill vectors with % of each tiles where pixel value  = 0 (water) OR 0 & -9999 (water or nodata)
19
for (i in 1:l){
20
    Names[i]<-sapply(strsplit(Tiles[i], '[/]'), '[[', 4)
21
    PctOcean[i]<- count(raster(Tiles[i]),0)/ncell(raster(Tiles[i]))
22
    PctNotLand[i]<- (count(raster(Tiles[i]),0)+count(raster(Tiles[i]),-9999))/ncell(raster(Tiles[i]))
23
}
24

  
25
data.frame (Names, PctOcean,PctNotLand)
26

  
27
#Use the above calculation to calculate % landmass for each tile and compare total over all tiles
28
#  in folder to % landmass of USGS coverage for same area.
29

  
30
# Requires aggregating aster tiles to match pixel size of USGS coverage:
31

  
32
LandAgg<- c(1:l) * NA
33
Counts_Agg<- c(1:l)*NA
34
for (i in 1:l){
35
  Counts_Agg[i]<- count(aggregate(raster(Tiles[i]),30),0)+count(aggregate(raster(Tiles[i]),30),-9999)
36
  LandAgg[i]<-  ncell(aggregate(raster(Tiles[i]),30))-Counts_Agg[i]
37
}
38

  
39
a<-sum(LandAgg)
40

  
41

  
42

  
43
#Calculate % landmass in USGS coverage and compare:
44

  
45
AstRast<- raster("DEM/asterGdem/N59to81_W20toE19/w020n90/W020N90_clipped.dem")
46
b<-ncell(AstRast)-cellStats(AstRast, "countNA") #Total # cells- cells with no value
47
1-(b/a)  #Difference= 4.49%
48
#------------------------------------------------------------------
49
#after first try, anytime PctNotLand was >= 99%, so too was PctOcean and visa versa
50
#    All subsequent runs will only calc. PctOcean to save time
51

  
52
Tiles <- list.files("DEM/asterGdem/N59to81_E20to59", pattern="^ASTGTM.*_dem.tif$")
53
Tiles<- paste("DEM/asterGdem/N59to81_E20to59/", Tiles, sep="")
54

  
55
l<- length (Tiles)
56
Names<- c(1:l)*NA
57
PctNotLand<- c(1:l) * NA
58
PctOcean<- c(1:l)*NA
59
for (i in 1:l){
60
    Names[i]<-sapply(strsplit(Tiles[i], '[/]'), '[[', 4)
61
    PctOcean[i]<- count(raster(Tiles[i]),0)/ncell(raster(Tiles[i]))
62
}
63

  
64
data.frame (Names, PctOcean)
65

  
66
LandAgg<- c(1:l) * NA
67
Counts_Agg<- c(1:l)*NA
68
for (i in 1:l){
69
  Counts_Agg[i]<- count(aggregate(raster(Tiles[i]),30),0)+count(aggregate(raster(Tiles[i]),30),-9999)
70
  LandAgg[i]<-  ncell(aggregate(raster(Tiles[i]),30))-Counts_Agg[i]
71
    print( paste(round(100*i/l),"%",sep=""), quote=FALSE )
72
}
73

  
74
a<-sum(LandAgg)
75
AstRast<- raster("DEM/asterGdem/N59to81_E20to59/e020n90/E020N90_clipped.dem")
76
b<-ncell(AstRast)-cellStats(AstRast, "countNA") #Total # cells- cells with no value
77
1-(b/a) #Difference= 2.21%
78
#-----------------------------------------------------------------------
79
Tiles <- list.files("DEM/asterGdem/N59to81_E60to99", pattern="^ASTGTM.*_dem.tif$")
80
Tiles<- paste("DEM/asterGdem/N59to81_E60to99/", Tiles, sep="")
81

  
82
l<- length (Tiles)
83
Names<- c(1:l)*NA
84
PctNotLand<- c(1:l) * NA
85
PctOcean<- c(1:l)*NA
86
for (i in 1:l){
87
    Names[i]<-sapply(strsplit(Tiles[i], '[/]'), '[[', 4)
88
    PctOcean[i]<- count(raster(Tiles[i]),0)/ncell(raster(Tiles[i]))
89
}
90

  
91
data.frame (Names, PctOcean)
92

  
93
LandAgg<- c(1:l) * NA
94
Counts_Agg<- c(1:l)*NA
95
for (i in 1:l){
96
  Counts_Agg[i]<- count(aggregate(raster(Tiles[i]),30),0)+count(aggregate(raster(Tiles[i]),30),-9999)
97
  LandAgg[i]<-  ncell(aggregate(raster(Tiles[i]),30))-Counts_Agg[i]
98
}
99

  
100
a<-sum(LandAgg)
101
AstRast<- raster("DEM/asterGdem/N59to81_E60to99/USGS_ErosDEM_N59to81E60to99/E060N90_clipped.dem")
102
b<-ncell(AstRast)-cellStats(AstRast, "countNA") #Total # cells- cells with no value
103
1-(b/a)  # Datasets differ by 2.27%
104

  
105
cellStats(AstRast,max)
106
cellStats(AstRast,min)
107
count(AstRast, NA)
108
#-----------------------------------------------------------------------
109
Tiles <- list.files("DEM/asterGdem/N59to81_E100to139", pattern="^ASTGTM.*_dem.tif$")
110
Tiles<- paste("DEM/asterGdem/N59to81_E100to139/", Tiles, sep="")
111

  
112
l<- length (Tiles)
113
Names<- c(1:l)*NA
114
PctNotLand<- c(1:l) * NA
115
PctOcean<- c(1:l)*NA
116
for (i in 1:l){
117
    Names[i]<-sapply(strsplit(Tiles[i], '[/]'), '[[', 4)
118
    PctOcean[i]<- count(raster(Tiles[i]),0)/ncell(raster(Tiles[i]))
119
}
120

  
121
data.frame (Names, PctOcean)
122

  
123
LandAgg<- c(1:l) * NA
124
Counts_Agg<- c(1:l)*NA
125
for (i in 1:l){
126
  Counts_Agg[i]<- count(aggregate(raster(Tiles[i]),30),0)+count(aggregate(raster(Tiles[i]),30),-9999)
127
  LandAgg[i]<-  ncell(aggregate(raster(Tiles[i]),30))-Counts_Agg[i]
128
    print( paste(round(100*i/l),"%",sep=""), quote=FALSE )
129
}
130

  
131
a<-sum(LandAgg)
132
AstRast<- raster("DEM/asterGdem/N59to81_E100to139/e100n90/E100N90_clipped.dem")
133
b<-ncell(AstRast)-cellStats(AstRast, "countNA") #Total # cells- cells with no value
134
1-(b/a)  #Difference= 3.1%
135
#-----------------------------------------------------------------------
136
Tiles <- list.files("DEM/asterGdem/N59to81_E140to179", pattern="^ASTGTM.*_dem.tif$")
137
Tiles<- paste("DEM/asterGdem/N59to81_E140to179/", Tiles, sep="")
138

  
139
l<- length (Tiles)
140
Names<- c(1:l)*NA
141
PctNotLand<- c(1:l) * NA
142
PctOcean<- c(1:l)*NA
143
for (i in 1:l){
144
    Names[i]<-sapply(strsplit(Tiles[i], '[/]'), '[[', 4)
145
    PctOcean[i]<- count(raster(Tiles[i]),0)/ncell(raster(Tiles[i]))
146
}
147

  
148
data.frame (Names, PctOcean)
149

  
150
LandAgg<- c(1:l) * NA
151
Counts_Agg<- c(1:l)*NA
152
for (i in 1:l){
153
  Counts_Agg[i]<- count(aggregate(raster(Tiles[i]),30),0)+count(aggregate(raster(Tiles[i]),30),-9999)
154
  LandAgg[i]<-  ncell(aggregate(raster(Tiles[i]),30))-Counts_Agg[i]
155
    print( paste(round(100*i/l),"%",sep=""), quote=FALSE )
156
}
157

  
158
a<-sum(LandAgg)
159
AstRast<- raster("DEM/asterGdem/N59to81_E140to179/e140n90/E140N90_clipped.dem")
160
b<-ncell(AstRast)-cellStats(AstRast, "countNA") #Total # cells- cells with no value
161
1-(b/a)   #Difference= 3.4%
162
#-----------------------------------------------------------------------
163
Tiles <- list.files("DEM/asterGdem/N59to81_W60to21", pattern="^ASTGTM.*_dem.tif$")
164
Tiles<- paste("DEM/asterGdem/N59to81_W60to21/", Tiles, sep="")
165

  
166
l<- length (Tiles)
167
Names<- c(1:l)*NA
168
PctNotLand<- c(1:l) * NA
169
PctOcean<- c(1:l)*NA
170
for (i in 1:l){
171
    Names[i]<-sapply(strsplit(Tiles[i], '[/]'), '[[', 4)
172
    PctOcean[i]<- count(raster(Tiles[i]),0)/ncell(raster(Tiles[i]))
173
}
174

  
175
data.frame (Names, PctOcean)
176

  
177
LandAgg<- c(1:l) * NA
178
Counts_Agg<- c(1:l)*NA
179
for (i in 1:l){
180
  Counts_Agg[i]<- count(aggregate(raster(Tiles[i]),30),0)+count(aggregate(raster(Tiles[i]),30),-9999)
181
  LandAgg[i]<-  ncell(aggregate(raster(Tiles[i]),30))-Counts_Agg[i]
182
    print( paste(round(100*i/l),"%",sep=""), quote=FALSE )
183
}
184

  
185
a<-sum(LandAgg)
186
AstRast<- raster("DEM/asterGdem/N59to81_W60to21/w060n90/W060N90_clipped.dem")
187
b<-ncell(AstRast)-cellStats(AstRast, "countNA") #Total # cells- cells with no value
188
1-(b/a)  #Difference= 3.03%
189
#-----------------------------------------------------------------------
190
Tiles <- list.files("DEM/asterGdem/N59to81_W180to141", pattern="^ASTGTM.*_dem.tif$")
191
Tiles<- paste("DEM/asterGdem/N59to81_W180to141/", Tiles, sep="")
192

  
193
l<- length (Tiles)
194
Names<- c(1:l)*NA
195
PctNotLand<- c(1:l) * NA
196
PctOcean<- c(1:l)*NA
197
for (i in 1:l){
198
    Names[i]<-sapply(strsplit(Tiles[i], '[/]'), '[[', 4)
199
    PctOcean[i]<- count(raster(Tiles[i]),0)/ncell(raster(Tiles[i]))
200
}
201

  
202
data.frame (Names, PctOcean)
203

  
204
LandAgg<- c(1:l) * NA
205
Counts_Agg<- c(1:l)*NA
206
for (i in 1:l){
207
  Counts_Agg[i]<- count(aggregate(raster(Tiles[i]),30),0)+count(aggregate(raster(Tiles[i]),30),-9999)
208
  LandAgg[i]<-  ncell(aggregate(raster(Tiles[i]),30))-Counts_Agg[i]
209
     print( paste(round(100*i/l),"%",sep=""), quote=FALSE )
210
}
211

  
212
a<-sum(LandAgg)
213
AstRast<- raster("DEM/asterGdem/N59to81_W180to141/w180n90/W180N90_clipped.dem")
214
b<-ncell(AstRast)-cellStats(AstRast, "countNA") #Total # cells- cells with no value
215
1-(b/a) #Difference= 2.87%
216
#-----------------------------------------------------------------------
217
Tiles <- list.files("DEM/asterGdem/N59to81_W140to101", pattern="^ASTGTM.*_dem.tif$")
218
Tiles<- paste("DEM/asterGdem/N59to81_W140to101/", Tiles, sep="")
219

  
220
l<- length (Tiles)
221
Names<- c(1:l)*NA
222
PctNotLand<- c(1:l) * NA
223
PctOcean<- c(1:l)*NA
224
for (i in 1:l){
225
    Names[i]<-sapply(strsplit(Tiles[i], '[/]'), '[[', 4)
226
    PctOcean[i]<- count(raster(Tiles[i]),0)/ncell(raster(Tiles[i]))
227
}
228

  
229
data.frame (Names, PctOcean)
230

  
231
LandAgg<- c(1:l) * NA
232
Counts_Agg<- c(1:l)*NA
233
for (i in 1:l){
234
  Counts_Agg[i]<- count(aggregate(raster(Tiles[i]),30),0)+count(aggregate(raster(Tiles[i]),30),-9999)
235
  LandAgg[i]<-  ncell(aggregate(raster(Tiles[i]),30))-Counts_Agg[i]
236
    print( paste(round(100*i/l),"%",sep=""), quote=FALSE )
237
}
238

  
239
a<-sum(LandAgg)
240
AstRast<- raster("DEM/asterGdem/N59to81_W140to101/w140n90/W140N90_clipped.dem")
241
b<-ncell(AstRast)-cellStats(AstRast, "countNA") #Total # cells- cells with no value
242
1-(b/a) #Difference= 3.84%
243
#-----------------------------------------------------------------------
244
Tiles <- list.files("DEM/asterGdem/N59to81_W100to61", pattern="^ASTGTM.*_dem.tif$")
245
Tiles<- paste("DEM/asterGdem/N59to81_W100to61/", Tiles, sep="")
246

  
247
l<- length (Tiles)
248
Names<- c(1:l)*NA
249
PctNotLand<- c(1:l) * NA
250
PctOcean<- c(1:l)*NA
251
for (i in 1:l){
252
    Names[i]<-sapply(strsplit(Tiles[i], '[/]'), '[[', 4)
253
    PctOcean[i]<- count(raster(Tiles[i]),0)/ncell(raster(Tiles[i]))
254
}
255

  
256
data.frame (Names, PctOcean)
257

  
258
LandAgg<- c(1:l) * NA
259
Counts_Agg<- c(1:l)*NA
260
for (i in 1:l){
261
  Counts_Agg[i]<- count(aggregate(raster(Tiles[i]),30),0)+count(aggregate(raster(Tiles[i]),30),-9999)
262
  LandAgg[i]<-  ncell(aggregate(raster(Tiles[i]),30))-Counts_Agg[i]
263
    print( paste(round(100*i/l),"%",sep=""), quote=FALSE )
264
}
265

  
266
a<-sum(LandAgg)
267
AstRast<- raster("DEM/asterGdem/N59to81_W100to61/w100n90/W100N90_clipped.dem")
268
b<-ncell(AstRast)-cellStats(AstRast, "countNA") #Total # cells- cells with no value
269
1-(b/a) #Difference= 7.21% (lot's of coastline in this set of tiles!)
270
#-------------------------------------------------------------------
271
c<- raster("DEM/asterGdem/N59to81_E60to99/USGS_ErosDEM_N59to81E60to99/E060N90.DEM")
272
a<- raster("DEM/asterGdem/N59to81_E60to99/USGS_ErosDEM_N59to81E60to99/E060N90_clipped.dem")
273
b<- raster(nrow=82800, ncol=144000)
274
d<- raster("DEM/asterGdem/N59to81_E60to99/ASTGTM_N59E060_dem.tif")
275
s<- resample(a,b,method='ngb')
276
disaggregate(a,30)
277
e<-aggregate(d,30)
278

  
279
count(e, -9999)
280
count (e,0)
terrain/tests/AsterCheck_demAndnum.r
1
library(rgdal)
2
aster.dir <- "DEM/asterGdem"
3

  
4
#Check that path=correct:
5
a<- GDALinfo("DEM/asterGdem/ASTGTM_N59E069_dem.tif", 
6
    silent=TRUE,returnRAT=TRUE)
7

  
8
# for a given file, return TRUE if file name matches the internally
9
# specified lower left corner, otherwise return string with those
10
# coordinates
11
check <- function(tilename, path=".", silent=TRUE) {
12
    # build expected filename
13
    origin <- round(GDALinfo(file.path(path, tilename),
14
        silent=silent)[c("ll.x", "ll.y")])
15
    ly <- origin["ll.y"]
16
    y <- sprintf("%s%02d", if (ly>=0) "N" else "S", abs(ly))
17
    lx <- origin["ll.x"]
18
    x <- sprintf("%s%03d", if (lx>=0) "E" else "W", abs(lx))
19
    expected.name <- paste("ASTGTM_", y, x, "_dem.tif", sep="")
20
    # compare to actual filename
21
    if (tilename==expected.name) {
22
        TRUE
23
    } else {
24
        paste(y, x)
25
    }
26
}
27

  
28
aster.tiles <- list.files(aster.dir, pattern="^ASTGTM.*_dem.tif$") 
29
titlecheck <- sapply(aster.tiles, check, path=aster.dir)
30

  
31
BadAster_df<- data.frame(expected=titlecheck[titlecheck!="TRUE"])
32

  
33
# ASTGTM_N59E069_dem.tif  N63 E109
34
#ASTGTM_N63E113_dem.tif  N69 E107
35
# ASTGTM_N63E117_dem.tif	N69 E113
36
#ASTGTM_N64E098_dem.tif	N70 E117
37
#ASTGTM_N65E104_dem.tif	N73 E084
38
#	ASTGTM_N65E111_dem.tif	N73 E098
39
#	ASTGTM_N65E117_dem.tif	N66 E130
40

  
41
KnownBadcheck <- function(badtilename, path="DEM/asterGdem/incorrectTilesJuly2011", silent=TRUE) {
42
    # build expected filename
43
    origin <- round(GDALinfo(file.path(path, badtilename),
44
        silent=silent)[c("ll.x", "ll.y")])
45
    ly <- origin["ll.y"]
46
    y <- sprintf("%s%02d", if (ly>=0) "N" else "S", abs(ly))
47
    lx <- origin["ll.x"]
48
    x <- sprintf("%s%03d", if (lx>=0) "E" else "W", abs(lx))
49
    Badexpected.name <- paste("ASTGTM_", y, x, "_dem.tif", sep="")
50
    # compare to actual filename
51
    if (badtilename==Badexpected.name) {
52
        TRUE
53
    } else {
54
        paste(y, x)
55
    }
56
}
57

  
58
bad.tiles <- list.files("DEM/asterGdem/incorrectTilesJuly2011", pattern="^ASTGTM.*_dem.tif$") 
59
badtitlecheck <- sapply(bad.tiles, KnownBadcheck, path="DEM/asterGdem/incorrectTilesJuly2011")
60

  
61
BadTitle_df<- data.frame(expected=badtitlecheck[badtitlecheck!="TRUE"])
62

  
63
#----------------------------------------------------------------------------------------------------------------------
64
#Check num.tif files
65
b<- GDALinfo("DEM/asterGdem/ASTGTM_N49E000_num.tif", 
66
silent=TRUE,returnRAT=TRUE)
67
b
68

  
69
Numcheck <- function(Numtilename, path=".", silent=TRUE) {
70
    # build expected filename
71
    origin <- round(GDALinfo(file.path(path, Numtilename),
72
        silent=silent)[c("ll.x", "ll.y")])
73
    ly <- origin["ll.y"]
74
    y <- sprintf("%s%02d", if (ly>=0) "N" else "S", abs(ly))
75
    lx <- origin["ll.x"]
76
    x <- sprintf("%s%03d", if (lx>=0) "E" else "W", abs(lx))
77
    Numexpected.name <- paste("ASTGTM_", y, x, "_num.tif", sep="")
78
    # compare to actual filename
79
    if (Numtilename==Numexpected.name) {
80
        TRUE
81
    } else {
82
        paste(y, x)
83
    }
84
}
85

  
86
Num.tiles <- list.files("DEM/asterGdem/NewTiles", pattern="^ASTGTM.*_num.tif$") 
87
Numtitlecheck <- sapply(Num.tiles, Numcheck, path="DEM/asterGdem/NewTiles")
88

  
89
NumTitle_df<- data.frame(expected=Numtitlecheck[Numtitlecheck!="TRUE"])
90

  
91
#Located one bad num.tif file
92
N49E007_NumRead<- GDALinfo("DEM/asterGdem/ASTGTM_N49E007_num.tif", 
93
silent=TRUE,returnRAT=TRUE)
94

  
95
GDALinfo("DEM/asterGdem/ASTGTM_N59E069_dem.tif", 
96
silent=TRUE,returnRAT=TRUE)
97

  
98
#--------------------------------------------------------------------------------
99
#Check new tiles uploaded to replace faulty tiles
100

  
101
Newcheck <- function(Newtilename, path=".", silent=TRUE) {
102
    # build expected filename
103
    origin <- round(GDALinfo(file.path(path, Newtilename),
104
        silent=silent)[c("ll.x", "ll.y")])
105
    ly <- origin["ll.y"]
106
    y <- sprintf("%s%02d", if (ly>=0) "N" else "S", abs(ly))
107
    lx <- origin["ll.x"]
108
    x <- sprintf("%s%03d", if (lx>=0) "E" else "W", abs(lx))
109
    Newexpected.name <- paste("ASTGTM_", y, x, "_dem.tif", sep="")
110
    # compare to actual filename
111
    if (Newtilename==Newexpected.name) {
112
        TRUE
113
    } else {
114
        paste(y, x)
115
    }
116
}
117

  
118
New.tiles <- list.files("DEM/asterGdem/NewTiles", pattern="^ASTGTM.*_dem.tif$") 
119
Newtitlecheck <- sapply(New.tiles, Newcheck, path="DEM/asterGdem/NewTiles")
120

  
121
NewAster_df<- data.frame(Newexpected=Newtitlecheck[Newtitlecheck!="TRUE"])
122

  
123
#Yeah, no mistakes in first 4!
124

  
125
g<- list.files("DEM/asterGdem/N59to81_E60to99", pattern="^ASTGTM.*_dem.tif$")
126
h<- list.files("DEM/asterGdem/N59to81_E60to99/Non_DEM's", pattern="^ASTGTM.*_num.tif$")
127

  
128
GDALinfo("DEM/asterGdem/N59to81_E60to99/ASTGTM_N64E098_dem.tif", 
129
silent=TRUE,returnRAT=TRUE)
130

  
131
#--------------------------------------------------------------------------
132
#new folder check
133
nfcheck <- function(nftilename, path=".", silent=TRUE) {
134
    # build expected filename
135
    origin <- round(GDALinfo(file.path(path, nftilename),
136
        silent=silent)[c("ll.x", "ll.y")])
137
    ly <- origin["ll.y"]
138
    y <- sprintf("%s%02d", if (ly>=0) "N" else "S", abs(ly))
139
    lx <- origin["ll.x"]
140
    x <- sprintf("%s%03d", if (lx>=0) "E" else "W", abs(lx))
141
    nfexpected.name <- paste("ASTGTM_", y, x, "_dem.tif", sep="")
142
    # compare to actual filename
143
    if (nftilename==nfexpected.name) {
144
        TRUE
145
    } else {
146
        paste(y, x)
147
    }
148
}
149

  
150
nf.tiles <- list.files("DEM/asterGdem/N59to81_E60to99", pattern="^ASTGTM.*_dem.tif$") 
151
nftitlecheck <- sapply(nf.tiles, nfcheck, path="DEM/asterGdem/N59to81_E60to99")
152

  
153
nfAster_df<- data.frame(nfexpected=nftitlecheck[nftitlecheck!="TRUE"])
154

  
155

  
156
Numcheck <- function(Numtilename, path=".", silent=TRUE) {
157
    # build expected filename
158
    origin <- round(GDALinfo(file.path(path, Numtilename),
159
        silent=silent)[c("ll.x", "ll.y")])
160
    ly <- origin["ll.y"]
161
    y <- sprintf("%s%02d", if (ly>=0) "N" else "S", abs(ly))
162
    lx <- origin["ll.x"]
163
    x <- sprintf("%s%03d", if (lx>=0) "E" else "W", abs(lx))
164
    Numexpected.name <- paste("ASTGTM_", y, x, "_num.tif", sep="")
165
    # compare to actual filename
166
    if (Numtilename==Numexpected.name) {
167
        TRUE
168
    } else {
169
        paste(y, x)
170
    }
171
}

Also available in: Unified diff