Project

General

Profile

« Previous | Next » 

Revision ddd9a810

Added by Adam Wilson over 11 years ago

Updated script to process MOD35 Processing Path using HEG to interpolate sensor zenith observations. Also working on figures for MOD35 C5 vs C6 comparison in MOD35C5_Evaluation script

View differences:

climate/procedures/MOD35_ExtractProcessPath.r
5 5
library(multicore)
6 6
library(raster)
7 7
library(spgrass6)
8

  
8
library(rgeos)
9 9
##download 3 days of modis swath data:
10 10

  
11 11
url="ftp://ladsweb.nascom.nasa.gov/allData/51/MOD35_L2/2012/"
......
18 18
## get MODLAND tile to serve as base
19 19
#system("wget http://e4ftl01.cr.usgs.gov/MOLT/MOD13A3.005/2000.02.01/MOD13A3.A2000032.h00v08.005.2006271174446.hdf")
20 20
#t=raster(paste("HDF4_EOS:EOS_GRID:\"",getwd(),"/MOD13A3.A2000032.h00v08.005.2006271174446.hdf\":MOD_Grid_monthly_1km_VI:1 km monthly NDVI",sep=""))
21
t=raster(paste("../MOD17/Npp_1km_C5.1_mean_00_to_06.tif",sep=""))
21
t=raster(paste("../MOD17/MOD17A3_Science_NPP_mean_00_12.tif",sep=""))
22
projection(t)
22 23

  
23 24
## make global extent
24 25
pmodis="+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs"
25 26

  
26 27
glb=t
27
values(glb)=NA
28
#values(glb)=NA
28 29
glb=extend(glb,extent(-180,180,-90,90))
29 30

  
30 31
#glb=raster(glb,crs="+proj=longlat +datum=WGS84",nrows=42500,ncols=85000)
......
38 39

  
39 40
#### Grid and mosaic the swath data
40 41

  
41
  stitch="sudo MRTDATADIR=\"/usr/local/heg/data\" PGSHOME=/usr/local/heg/TOOLKIT_MTD PWD=/home/adamw /usr/local/heg/bin/swtif"
42
stitch="sudo MRTDATADIR=\"/usr/local/heg/data\" PGSHOME=/usr/local/heg/TOOLKIT_MTD PWD=/home/adamw /usr/local/heg/bin/swtif"
42 43

  
43 44
files=paste(getwd(),"/",list.files("swath",pattern="hdf$",full=T),sep="")
44 45

  
45 46
## vars to process
46 47
vars=as.data.frame(matrix(c(
47 48
  "Cloud_Mask",              "CM",
48
#  "Quality_Assurance",       "QA"),
49
  byrow=T,ncol=2,dimnames=list(1:2,c("variable","varid"))),stringsAsFactors=F)
50

  
51
## establish sudo priveleges to run swtif
52
system("sudo ls")
53

  
54
mclapply(files,function(file){
55
  
56
  tempfile=paste(tempdir(),"/",basename(file),sep="")
57
#  tempfile2=paste("gridded/",sub("hdf","tif",basename(tempfile)),sep="")
58
  tempfile2=paste(tempdir(),"/",sub("hdf","tif",basename(tempfile)),sep="")
59

  
60
  outfile=paste("gridded2/",sub("hdf","tif",basename(tempfile)),sep="")
61

  
62
  if(file.exists(outfile)) return(c(file,0))
63
  ## First write the parameter file (careful, heg is very finicky!)
64
  hdr=paste("NUM_RUNS = ",length(vars$varid),"|MULTI_BAND_HDFEOS:",length(vars$varid),sep="")
65
  grp=paste("
49
  "Sensor_Azimuth",       "ZA",
50
  "Sensor_Zenith",        "SZ"),
51
  byrow=T,ncol=2,dimnames=list(1:3,c("variable","varid"))),stringsAsFactors=F)
52

  
53
## global bounding box
54
   gbb=cbind(c(-180,-180,180,180,-180),c(-85,85,85,-85,-85))
55
   gpp = SpatialPolygons(list(Polygons(list(Polygon(gbb)),1)))
56
   proj4string(gpp)=projection(glb)
57

  
58

  
59
getpath<- function(file){  
60
   setwd(tempdir())
61
   bfile=sub(".hdf","",basename(file))
62
   tempfile_path=paste(tempdir(),"/path_",basename(file),sep="")  #gridded path
63
   tempfile_sz=paste(tempdir(),"/sz_",basename(file),sep="")  # gridded sensor zenith
64
   tempfile2_path=paste(tempdir(),"/",bfile,".tif",sep="")  #gridded/masked/processed path
65
   outfile=paste("~/acrobates/adamw/projects/interp/data/modis/mod35/gridded/",bfile,".tif",sep="")  #final file
66
   if(file.exists(outfile)) return(c(file,0))
67
   ## get bounding coordinates
68
   glat=as.numeric(do.call(c,strsplit(sub("GRINGPOINTLATITUDE=","",system(paste("gdalinfo ",file," | grep GRINGPOINTLATITUDE"),intern=T)),split=",")))
69
   glon=as.numeric(do.call(c,strsplit(sub("GRINGPOINTLONGITUDE=","",system(paste("gdalinfo ",file," | grep GRINGPOINTLONGITUDE"),intern=T)),split=",")))
70
   bb=cbind(c(glon,glon[1]),c(glat,glat[1]))
71
   pp = SpatialPolygons(list(Polygons(list(Polygon(bb)),1)))
72
   proj4string(pp)=projection(glb)
73
   ppc=gIntersection(pp,gpp)
74
   ppc=gBuffer(ppc,width=0.3)  #buffer a little to remove gaps between images
75
   ## First write the parameter file (careful, heg is very finicky!)
76
   hdr=paste("NUM_RUNS = ",length(vars$varid),"|MULTI_BAND_HDFEOS:",length(vars$varid),sep="")
77
   grp=paste("
66 78
BEGIN
67 79
INPUT_FILENAME=",file,"
68 80
OBJECT_NAME=mod35
......
70 82
BAND_NUMBER = ",1:length(vars$varid),"
71 83
OUTPUT_PIXEL_SIZE_X=0.008333333
72 84
OUTPUT_PIXEL_SIZE_Y=0.008333333
73
SPATIAL_SUBSET_UL_CORNER = ( 90 -180 )
74
SPATIAL_SUBSET_LR_CORNER = ( -90 180 )
85
SPATIAL_SUBSET_UL_CORNER = ( ",bbox(ppc)[2,2]," ",bbox(ppc)[1,1]," )
86
SPATIAL_SUBSET_LR_CORNER = ( ",bbox(ppc)[2,1]," ",bbox(ppc)[1,2]," )
75 87
OUTPUT_OBJECT_NAME = mod35|
76 88
RESAMPLING_TYPE =NN
77 89
OUTPUT_PROJECTION_TYPE = GEO
......
80 92
# projection parameters from http://landweb.nascom.nasa.gov/cgi-bin/QA_WWW/newPage.cgi?fileName=sn_gctp
81 93
ELLIPSOID_CODE = WGS84
82 94
OUTPUT_TYPE = HDFEOS
83
OUTPUT_FILENAME= ",tempfile,"
95
OUTPUT_FILENAME= ",tempfile_path,"
84 96
END
85 97

  
86 98
",sep="")
87 99

  
88
  ## if any remnants from previous runs remain, delete them
89
   if(length(list.files(tempdir(),pattern=basename(file)))>0)
90
    file.remove(list.files(tempdir(),pattern=basename(file),full=T))
91
  ## write it to a file
92
 cat( c(hdr,grp)   , file=paste(tempdir(),"/",basename(file),"_MODparms.txt",sep=""))
93

  
94
  ## now run the swath2grid tool
95
  ## write the gridded file
96
  print(paste("Starting",file))
97
  system(paste("(",stitch," -p ",tempdir(),"/",basename(file),"_MODparms.txt -d ; echo $$)",sep=""),intern=T,ignore.stderr=F)
98
  ## convert to land path
99
  d=raster(paste("HDF4_EOS:EOS_GRID:\"",tempfile,"\":mod35:Cloud_Mask_0",sep=""))
100
#  za=raster(paste("HDF4_EOS:EOS_GRID:\"",tempfile,"\":mod35:Quality_Assurance_1",sep=""))
101
  ## add projection information - ellipsoid should be wgs84
102
  projection(d)=projection(glb)
103
  extent(d)=alignExtent(d,extent(glb))
104
                                        #  d=readAll(d)
105
  getlc=function(x) {(x%/%2^6) %% 2^2}
106
  calc(d,fun=getlc,filename=outfile,options=c("COMPRESS=LZW", "LEVEL=9","PREDICTOR=2"),datatype="INT1U",overwrite=T)
107

  
108
  ### warp it to align all pixels
109
  system(paste("gdalwarp -overwrite -tap -tr 0.008333333 0.008333333 -co COMPRESS=LZW -co LEVEL=9 -co PREDICTOR=2 -s_srs \"",projection(glb),"\" ",tempfile2," ",outfile,sep=""))
110

  
111
#  getqa=function(x) {(x%/%2^1) %% 2^3}
112
#  za=calc(za,fun=getqa)#,filename=outfile,options=c("COMPRESS=LZW", "LEVEL=9","PREDICTOR=2"),datatype="INT1U",overwrite=T)
113
  ## resample to line up with glb so we can use mosaic later
114
## delete temporary files
115
  file.remove(tempfile,tempfile2)
116
  return(c(file,1))
117
})
100
   ## if any remnants from previous runs remain, delete them
101
   if(length(list.files(tempdir(),pattern=bfile)>0))
102
     file.remove(list.files(tempdir(),pattern=bfile,full=T))
103
   ## write it to a file
104
   cat( c(hdr,grp)   , file=paste(tempdir(),"/",basename(file),"_MODparms.txt",sep=""))
105
   ## now run the swath2grid tool
106
   ## write the gridded file
107
   print(paste("Starting",file))
108
   system(paste("(",stitch," -p ",tempdir(),"/",basename(file),"_MODparms.txt -d)",sep=""),intern=F)#,ignore.stderr=F)
109
##############  Now run the 5km summary
110
   ## First write the parameter file (careful, heg is very finicky!)
111
   hdr=paste("NUM_RUNS = ",nrow(vars[-1,]),"|MULTI_BAND_HDFEOS:",nrow(vars[-1,]),sep="")
112
   grp=paste("
113
BEGIN
114
INPUT_FILENAME=",file,"
115
OBJECT_NAME=mod35
116
FIELD_NAME=",vars$variable[-1],"|
117
BAND_NUMBER = ",1,"
118
OUTPUT_PIXEL_SIZE_X=0.008333333
119
#0.0416666
120
OUTPUT_PIXEL_SIZE_Y=0.008333333
121
#0.0416666
122
SPATIAL_SUBSET_UL_CORNER = ( ",bbox(ppc)[2,2]," ",bbox(ppc)[1,1]," )
123
SPATIAL_SUBSET_LR_CORNER = ( ",bbox(ppc)[2,1]," ",bbox(ppc)[1,2]," )
124
#OUTPUT_OBJECT_NAME = mod35|
125
RESAMPLING_TYPE =NN
126
OUTPUT_PROJECTION_TYPE = GEO
127
OUTPUT_PROJECTION_PARAMETERS = ( 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 )
128
#OUTPUT_PROJECTION_PARAMETERS = ( 6371007.181 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 )
129
# projection parameters from http://landweb.nascom.nasa.gov/cgi-bin/QA_WWW/newPage.cgi?fileName=sn_gctp
130
ELLIPSOID_CODE = WGS84
131
OUTPUT_TYPE = HDFEOS
132
OUTPUT_FILENAME= ",tempfile_sz,"
133
END
134

  
135
",sep="")
136

  
137
   ## write it to a file
138
   cat( c(hdr,grp)   , file=paste(tempdir(),"/",basename(file),"_MODparms_angle.txt",sep=""))
139
   
140
   ## now run the swath2grid tool
141
   ## write the gridded file
142
   print(paste("Starting",file))
143
   system(paste("(",stitch," -p ",tempdir(),"/",basename(file),"_MODparms_angle.txt -d)",sep=""),intern=F,ignore.stderr=F)
144
####### import to R for processing
145
   if(!file.exists(tempfile_path)) {
146
     file.remove(list.files(tempdir(),pattern=bfile,full=T))
147
     return(c(file,0))
148
   }
149
   ## convert to land path
150
   d=raster(paste("HDF4_EOS:EOS_GRID:\"",tempfile_path,"\":mod35:Cloud_Mask_0",sep=""))
151
   sz=raster(paste("HDF4_EOS:EOS_GRID:\"",tempfile_sz,"\":mod35:Sensor_Zenith",sep=""))
152
   NAvalue(sz)=0
153
   ## resample sensor angles to 1km grid and mask paths with angles >=30
154
#   sz2=resample(sz,d,method="ngb",file=sub("hdf","tif",tempfile_sz),overwrite=T)
155
   getlc=function(x,y) {ifelse(y==0|y>40,NA,((x%/%2^6) %% 2^2))}
156
   path=  overlay(d,sz,fun=getlc,filename=tempfile2_path,options=c("COMPRESS=LZW", "LEVEL=9","PREDICTOR=2"),datatype="INT1U",overwrite=T)
157
### warp them to align all pixels
158
   system(paste("gdalwarp -overwrite -srcnodata 255 -dstnodata 255 -tap -tr 0.008333333 0.008333333 -co COMPRESS=LZW -co ZLEVEL=9 -co PREDICTOR=2 -s_srs \"",projection(t),"\" ",tempfile2_path," ",outfile,sep=""))
159
   ## delete temporary files
160
   file.remove(list.files(tempdir(),pattern=bfile,full=T))
161
   return(c(file,1))
162
 }
163

  
164

  
165
## establish sudo priveleges to run swtif
166
system("sudo ls"); mclapply(files,getpath,mc.cores=10)
118 167

  
119 168
## check gdal can read all of them
120 169
gfiles=list.files("gridded",pattern="tif$",full=T)
......
130 179

  
131 180
file.remove(gfiles[check==0])
132 181

  
133
#  origin(raster(gfiles[5]))
134
  
135
  
136
  system(paste("gdalinfo ",gfiles[1]," | grep Origin"))
137
  system(paste("gdalinfo ",gfiles[2]," | grep Origin"))
138
  system(paste("gdalinfo ",gfiles[3]," | grep Origin"))
139

  
140
system(paste("gdalwarp -tap -tr 0.008333333 0.008333333 -s_srs \"",projection(glb),"\" ",gfiles[1]," test1.tif"))
141
system(paste("gdalwarp -tap -tr 0.008333333 0.008333333 -s_srs \"",projection(glb),"\" ",gfiles[2]," test2.tif"))
142
  system(paste("gdalinfo test1.tif | grep Origin"))
143
  system(paste("gdalinfo test2.tif | grep Origin"))
144
system(paste("pkmosaic ",paste("-i",gfiles[1:2],collapse=" ")," -o MOD35_path2.tif  -m 6 -v -t 255"))
145
system(paste("pkmosaic ",paste("-i",c("test1.tif","test2.tif"),collapse=" ")," -o MOD35_path2.tif  -m 6 -v -max 10 -ot Byte"))
182
## use new gdal
183
system(paste("/usr/local/gdal-1.10.0/bin/gdalwarp -wm 900 -overwrite -co COMPRESS=LZW -co PREDICTOR=2 -multi -r mode gridded/*.tif MOD35_path_gdalwarp.tif"))
146 184

  
147 185

  
186
#  origin(raster(gfiles[5]))
187
  
148 188
  ## try with pktools
149 189
  ## global
150
system(paste("pkmosaic -co COMPRESS=LZW -co PREDICTOR=2 ",paste("-i",list.files("gridded",full=T,pattern="tif$"),collapse=" ")," -o MOD35_path_pkmosaic.tif  -m 6 -v -t 255 -t 0 &"))
190
system(paste("pkmosaic -co COMPRESS=LZW -co PREDICTOR=2 ",paste("-i",list.files("gridded",full=T,pattern="tif$"),collapse=" ")," -o MOD35_path_pkmosaic_max.tif  -m 2 -v -t 255 -t 0 &"))
151 191
#bb="-ulx -180 -uly 90 -lrx 180 -lry -90"
152 192
#bb="-ulx -180 -uly 90 -lrx 170 -lry 80"
153 193
bb="-ulx -72 -uly 11 -lrx -59 -lry -1"
154 194

  
155 195

  
156 196
#expand.grid(x=seq(-180,170,by=10),y=seq(-90,80))
157
  
158
system(paste("pkmosaic ",bb," -co COMPRESS=LZW -co PREDICTOR=2 ",paste("-i",list.files("gridded",full=T,pattern="tif$"),collapse=" ")," -o h11v08_path_pkmosaic.tif  -m 6 -v -t 255 -t 0 &"))
197
gf2=  grep("2012009[.]03",gfiles,value=T)
198
system(paste("pkmosaic ",bb," -co COMPRESS=LZW -co PREDICTOR=2 ",paste("-i",gf2,collapse=" ")," -o h11v08_path_pkmosaic.tif -ot Byte -m 7 -v -t 255"))
159 199

  
160 200
                                        #  bounding box?  
161 201

  
......
193 233
  unlink_.gislock()
194 234
  system(paste("rm -frR ",tf,sep=""))
195 235

  
196

  
197 236
#########################
198 237

  
199 238

  

Also available in: Unified diff