Revision ddd9a810
Added by Adam Wilson over 11 years ago
climate/procedures/MOD35_ExtractProcessPath.r | ||
---|---|---|
5 | 5 |
library(multicore) |
6 | 6 |
library(raster) |
7 | 7 |
library(spgrass6) |
8 |
|
|
8 |
library(rgeos) |
|
9 | 9 |
##download 3 days of modis swath data: |
10 | 10 |
|
11 | 11 |
url="ftp://ladsweb.nascom.nasa.gov/allData/51/MOD35_L2/2012/" |
... | ... | |
18 | 18 |
## get MODLAND tile to serve as base |
19 | 19 |
#system("wget http://e4ftl01.cr.usgs.gov/MOLT/MOD13A3.005/2000.02.01/MOD13A3.A2000032.h00v08.005.2006271174446.hdf") |
20 | 20 |
#t=raster(paste("HDF4_EOS:EOS_GRID:\"",getwd(),"/MOD13A3.A2000032.h00v08.005.2006271174446.hdf\":MOD_Grid_monthly_1km_VI:1 km monthly NDVI",sep="")) |
21 |
t=raster(paste("../MOD17/Npp_1km_C5.1_mean_00_to_06.tif",sep="")) |
|
21 |
t=raster(paste("../MOD17/MOD17A3_Science_NPP_mean_00_12.tif",sep="")) |
|
22 |
projection(t) |
|
22 | 23 |
|
23 | 24 |
## make global extent |
24 | 25 |
pmodis="+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs" |
25 | 26 |
|
26 | 27 |
glb=t |
27 |
values(glb)=NA |
|
28 |
#values(glb)=NA
|
|
28 | 29 |
glb=extend(glb,extent(-180,180,-90,90)) |
29 | 30 |
|
30 | 31 |
#glb=raster(glb,crs="+proj=longlat +datum=WGS84",nrows=42500,ncols=85000) |
... | ... | |
38 | 39 |
|
39 | 40 |
#### Grid and mosaic the swath data |
40 | 41 |
|
41 |
stitch="sudo MRTDATADIR=\"/usr/local/heg/data\" PGSHOME=/usr/local/heg/TOOLKIT_MTD PWD=/home/adamw /usr/local/heg/bin/swtif"
|
|
42 |
stitch="sudo MRTDATADIR=\"/usr/local/heg/data\" PGSHOME=/usr/local/heg/TOOLKIT_MTD PWD=/home/adamw /usr/local/heg/bin/swtif" |
|
42 | 43 |
|
43 | 44 |
files=paste(getwd(),"/",list.files("swath",pattern="hdf$",full=T),sep="") |
44 | 45 |
|
45 | 46 |
## vars to process |
46 | 47 |
vars=as.data.frame(matrix(c( |
47 | 48 |
"Cloud_Mask", "CM", |
48 |
# "Quality_Assurance", "QA"), |
|
49 |
byrow=T,ncol=2,dimnames=list(1:2,c("variable","varid"))),stringsAsFactors=F) |
|
50 |
|
|
51 |
## establish sudo priveleges to run swtif |
|
52 |
system("sudo ls") |
|
53 |
|
|
54 |
mclapply(files,function(file){ |
|
55 |
|
|
56 |
tempfile=paste(tempdir(),"/",basename(file),sep="") |
|
57 |
# tempfile2=paste("gridded/",sub("hdf","tif",basename(tempfile)),sep="") |
|
58 |
tempfile2=paste(tempdir(),"/",sub("hdf","tif",basename(tempfile)),sep="") |
|
59 |
|
|
60 |
outfile=paste("gridded2/",sub("hdf","tif",basename(tempfile)),sep="") |
|
61 |
|
|
62 |
if(file.exists(outfile)) return(c(file,0)) |
|
63 |
## First write the parameter file (careful, heg is very finicky!) |
|
64 |
hdr=paste("NUM_RUNS = ",length(vars$varid),"|MULTI_BAND_HDFEOS:",length(vars$varid),sep="") |
|
65 |
grp=paste(" |
|
49 |
"Sensor_Azimuth", "ZA", |
|
50 |
"Sensor_Zenith", "SZ"), |
|
51 |
byrow=T,ncol=2,dimnames=list(1:3,c("variable","varid"))),stringsAsFactors=F) |
|
52 |
|
|
53 |
## global bounding box |
|
54 |
gbb=cbind(c(-180,-180,180,180,-180),c(-85,85,85,-85,-85)) |
|
55 |
gpp = SpatialPolygons(list(Polygons(list(Polygon(gbb)),1))) |
|
56 |
proj4string(gpp)=projection(glb) |
|
57 |
|
|
58 |
|
|
59 |
getpath<- function(file){ |
|
60 |
setwd(tempdir()) |
|
61 |
bfile=sub(".hdf","",basename(file)) |
|
62 |
tempfile_path=paste(tempdir(),"/path_",basename(file),sep="") #gridded path |
|
63 |
tempfile_sz=paste(tempdir(),"/sz_",basename(file),sep="") # gridded sensor zenith |
|
64 |
tempfile2_path=paste(tempdir(),"/",bfile,".tif",sep="") #gridded/masked/processed path |
|
65 |
outfile=paste("~/acrobates/adamw/projects/interp/data/modis/mod35/gridded/",bfile,".tif",sep="") #final file |
|
66 |
if(file.exists(outfile)) return(c(file,0)) |
|
67 |
## get bounding coordinates |
|
68 |
glat=as.numeric(do.call(c,strsplit(sub("GRINGPOINTLATITUDE=","",system(paste("gdalinfo ",file," | grep GRINGPOINTLATITUDE"),intern=T)),split=","))) |
|
69 |
glon=as.numeric(do.call(c,strsplit(sub("GRINGPOINTLONGITUDE=","",system(paste("gdalinfo ",file," | grep GRINGPOINTLONGITUDE"),intern=T)),split=","))) |
|
70 |
bb=cbind(c(glon,glon[1]),c(glat,glat[1])) |
|
71 |
pp = SpatialPolygons(list(Polygons(list(Polygon(bb)),1))) |
|
72 |
proj4string(pp)=projection(glb) |
|
73 |
ppc=gIntersection(pp,gpp) |
|
74 |
ppc=gBuffer(ppc,width=0.3) #buffer a little to remove gaps between images |
|
75 |
## First write the parameter file (careful, heg is very finicky!) |
|
76 |
hdr=paste("NUM_RUNS = ",length(vars$varid),"|MULTI_BAND_HDFEOS:",length(vars$varid),sep="") |
|
77 |
grp=paste(" |
|
66 | 78 |
BEGIN |
67 | 79 |
INPUT_FILENAME=",file," |
68 | 80 |
OBJECT_NAME=mod35 |
... | ... | |
70 | 82 |
BAND_NUMBER = ",1:length(vars$varid)," |
71 | 83 |
OUTPUT_PIXEL_SIZE_X=0.008333333 |
72 | 84 |
OUTPUT_PIXEL_SIZE_Y=0.008333333 |
73 |
SPATIAL_SUBSET_UL_CORNER = ( 90 -180 )
|
|
74 |
SPATIAL_SUBSET_LR_CORNER = ( -90 180 )
|
|
85 |
SPATIAL_SUBSET_UL_CORNER = ( ",bbox(ppc)[2,2]," ",bbox(ppc)[1,1]," )
|
|
86 |
SPATIAL_SUBSET_LR_CORNER = ( ",bbox(ppc)[2,1]," ",bbox(ppc)[1,2]," )
|
|
75 | 87 |
OUTPUT_OBJECT_NAME = mod35| |
76 | 88 |
RESAMPLING_TYPE =NN |
77 | 89 |
OUTPUT_PROJECTION_TYPE = GEO |
... | ... | |
80 | 92 |
# projection parameters from http://landweb.nascom.nasa.gov/cgi-bin/QA_WWW/newPage.cgi?fileName=sn_gctp |
81 | 93 |
ELLIPSOID_CODE = WGS84 |
82 | 94 |
OUTPUT_TYPE = HDFEOS |
83 |
OUTPUT_FILENAME= ",tempfile," |
|
95 |
OUTPUT_FILENAME= ",tempfile_path,"
|
|
84 | 96 |
END |
85 | 97 |
|
86 | 98 |
",sep="") |
87 | 99 |
|
88 |
## if any remnants from previous runs remain, delete them |
|
89 |
if(length(list.files(tempdir(),pattern=basename(file)))>0) |
|
90 |
file.remove(list.files(tempdir(),pattern=basename(file),full=T)) |
|
91 |
## write it to a file |
|
92 |
cat( c(hdr,grp) , file=paste(tempdir(),"/",basename(file),"_MODparms.txt",sep="")) |
|
93 |
|
|
94 |
## now run the swath2grid tool |
|
95 |
## write the gridded file |
|
96 |
print(paste("Starting",file)) |
|
97 |
system(paste("(",stitch," -p ",tempdir(),"/",basename(file),"_MODparms.txt -d ; echo $$)",sep=""),intern=T,ignore.stderr=F) |
|
98 |
## convert to land path |
|
99 |
d=raster(paste("HDF4_EOS:EOS_GRID:\"",tempfile,"\":mod35:Cloud_Mask_0",sep="")) |
|
100 |
# za=raster(paste("HDF4_EOS:EOS_GRID:\"",tempfile,"\":mod35:Quality_Assurance_1",sep="")) |
|
101 |
## add projection information - ellipsoid should be wgs84 |
|
102 |
projection(d)=projection(glb) |
|
103 |
extent(d)=alignExtent(d,extent(glb)) |
|
104 |
# d=readAll(d) |
|
105 |
getlc=function(x) {(x%/%2^6) %% 2^2} |
|
106 |
calc(d,fun=getlc,filename=outfile,options=c("COMPRESS=LZW", "LEVEL=9","PREDICTOR=2"),datatype="INT1U",overwrite=T) |
|
107 |
|
|
108 |
### warp it to align all pixels |
|
109 |
system(paste("gdalwarp -overwrite -tap -tr 0.008333333 0.008333333 -co COMPRESS=LZW -co LEVEL=9 -co PREDICTOR=2 -s_srs \"",projection(glb),"\" ",tempfile2," ",outfile,sep="")) |
|
110 |
|
|
111 |
# getqa=function(x) {(x%/%2^1) %% 2^3} |
|
112 |
# za=calc(za,fun=getqa)#,filename=outfile,options=c("COMPRESS=LZW", "LEVEL=9","PREDICTOR=2"),datatype="INT1U",overwrite=T) |
|
113 |
## resample to line up with glb so we can use mosaic later |
|
114 |
## delete temporary files |
|
115 |
file.remove(tempfile,tempfile2) |
|
116 |
return(c(file,1)) |
|
117 |
}) |
|
100 |
## if any remnants from previous runs remain, delete them |
|
101 |
if(length(list.files(tempdir(),pattern=bfile)>0)) |
|
102 |
file.remove(list.files(tempdir(),pattern=bfile,full=T)) |
|
103 |
## write it to a file |
|
104 |
cat( c(hdr,grp) , file=paste(tempdir(),"/",basename(file),"_MODparms.txt",sep="")) |
|
105 |
## now run the swath2grid tool |
|
106 |
## write the gridded file |
|
107 |
print(paste("Starting",file)) |
|
108 |
system(paste("(",stitch," -p ",tempdir(),"/",basename(file),"_MODparms.txt -d)",sep=""),intern=F)#,ignore.stderr=F) |
|
109 |
############## Now run the 5km summary |
|
110 |
## First write the parameter file (careful, heg is very finicky!) |
|
111 |
hdr=paste("NUM_RUNS = ",nrow(vars[-1,]),"|MULTI_BAND_HDFEOS:",nrow(vars[-1,]),sep="") |
|
112 |
grp=paste(" |
|
113 |
BEGIN |
|
114 |
INPUT_FILENAME=",file," |
|
115 |
OBJECT_NAME=mod35 |
|
116 |
FIELD_NAME=",vars$variable[-1],"| |
|
117 |
BAND_NUMBER = ",1," |
|
118 |
OUTPUT_PIXEL_SIZE_X=0.008333333 |
|
119 |
#0.0416666 |
|
120 |
OUTPUT_PIXEL_SIZE_Y=0.008333333 |
|
121 |
#0.0416666 |
|
122 |
SPATIAL_SUBSET_UL_CORNER = ( ",bbox(ppc)[2,2]," ",bbox(ppc)[1,1]," ) |
|
123 |
SPATIAL_SUBSET_LR_CORNER = ( ",bbox(ppc)[2,1]," ",bbox(ppc)[1,2]," ) |
|
124 |
#OUTPUT_OBJECT_NAME = mod35| |
|
125 |
RESAMPLING_TYPE =NN |
|
126 |
OUTPUT_PROJECTION_TYPE = GEO |
|
127 |
OUTPUT_PROJECTION_PARAMETERS = ( 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ) |
|
128 |
#OUTPUT_PROJECTION_PARAMETERS = ( 6371007.181 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ) |
|
129 |
# projection parameters from http://landweb.nascom.nasa.gov/cgi-bin/QA_WWW/newPage.cgi?fileName=sn_gctp |
|
130 |
ELLIPSOID_CODE = WGS84 |
|
131 |
OUTPUT_TYPE = HDFEOS |
|
132 |
OUTPUT_FILENAME= ",tempfile_sz," |
|
133 |
END |
|
134 |
|
|
135 |
",sep="") |
|
136 |
|
|
137 |
## write it to a file |
|
138 |
cat( c(hdr,grp) , file=paste(tempdir(),"/",basename(file),"_MODparms_angle.txt",sep="")) |
|
139 |
|
|
140 |
## now run the swath2grid tool |
|
141 |
## write the gridded file |
|
142 |
print(paste("Starting",file)) |
|
143 |
system(paste("(",stitch," -p ",tempdir(),"/",basename(file),"_MODparms_angle.txt -d)",sep=""),intern=F,ignore.stderr=F) |
|
144 |
####### import to R for processing |
|
145 |
if(!file.exists(tempfile_path)) { |
|
146 |
file.remove(list.files(tempdir(),pattern=bfile,full=T)) |
|
147 |
return(c(file,0)) |
|
148 |
} |
|
149 |
## convert to land path |
|
150 |
d=raster(paste("HDF4_EOS:EOS_GRID:\"",tempfile_path,"\":mod35:Cloud_Mask_0",sep="")) |
|
151 |
sz=raster(paste("HDF4_EOS:EOS_GRID:\"",tempfile_sz,"\":mod35:Sensor_Zenith",sep="")) |
|
152 |
NAvalue(sz)=0 |
|
153 |
## resample sensor angles to 1km grid and mask paths with angles >=30 |
|
154 |
# sz2=resample(sz,d,method="ngb",file=sub("hdf","tif",tempfile_sz),overwrite=T) |
|
155 |
getlc=function(x,y) {ifelse(y==0|y>40,NA,((x%/%2^6) %% 2^2))} |
|
156 |
path= overlay(d,sz,fun=getlc,filename=tempfile2_path,options=c("COMPRESS=LZW", "LEVEL=9","PREDICTOR=2"),datatype="INT1U",overwrite=T) |
|
157 |
### warp them to align all pixels |
|
158 |
system(paste("gdalwarp -overwrite -srcnodata 255 -dstnodata 255 -tap -tr 0.008333333 0.008333333 -co COMPRESS=LZW -co ZLEVEL=9 -co PREDICTOR=2 -s_srs \"",projection(t),"\" ",tempfile2_path," ",outfile,sep="")) |
|
159 |
## delete temporary files |
|
160 |
file.remove(list.files(tempdir(),pattern=bfile,full=T)) |
|
161 |
return(c(file,1)) |
|
162 |
} |
|
163 |
|
|
164 |
|
|
165 |
## establish sudo priveleges to run swtif |
|
166 |
system("sudo ls"); mclapply(files,getpath,mc.cores=10) |
|
118 | 167 |
|
119 | 168 |
## check gdal can read all of them |
120 | 169 |
gfiles=list.files("gridded",pattern="tif$",full=T) |
... | ... | |
130 | 179 |
|
131 | 180 |
file.remove(gfiles[check==0]) |
132 | 181 |
|
133 |
# origin(raster(gfiles[5])) |
|
134 |
|
|
135 |
|
|
136 |
system(paste("gdalinfo ",gfiles[1]," | grep Origin")) |
|
137 |
system(paste("gdalinfo ",gfiles[2]," | grep Origin")) |
|
138 |
system(paste("gdalinfo ",gfiles[3]," | grep Origin")) |
|
139 |
|
|
140 |
system(paste("gdalwarp -tap -tr 0.008333333 0.008333333 -s_srs \"",projection(glb),"\" ",gfiles[1]," test1.tif")) |
|
141 |
system(paste("gdalwarp -tap -tr 0.008333333 0.008333333 -s_srs \"",projection(glb),"\" ",gfiles[2]," test2.tif")) |
|
142 |
system(paste("gdalinfo test1.tif | grep Origin")) |
|
143 |
system(paste("gdalinfo test2.tif | grep Origin")) |
|
144 |
system(paste("pkmosaic ",paste("-i",gfiles[1:2],collapse=" ")," -o MOD35_path2.tif -m 6 -v -t 255")) |
|
145 |
system(paste("pkmosaic ",paste("-i",c("test1.tif","test2.tif"),collapse=" ")," -o MOD35_path2.tif -m 6 -v -max 10 -ot Byte")) |
|
182 |
## use new gdal |
|
183 |
system(paste("/usr/local/gdal-1.10.0/bin/gdalwarp -wm 900 -overwrite -co COMPRESS=LZW -co PREDICTOR=2 -multi -r mode gridded/*.tif MOD35_path_gdalwarp.tif")) |
|
146 | 184 |
|
147 | 185 |
|
186 |
# origin(raster(gfiles[5])) |
|
187 |
|
|
148 | 188 |
## try with pktools |
149 | 189 |
## global |
150 |
system(paste("pkmosaic -co COMPRESS=LZW -co PREDICTOR=2 ",paste("-i",list.files("gridded",full=T,pattern="tif$"),collapse=" ")," -o MOD35_path_pkmosaic.tif -m 6 -v -t 255 -t 0 &"))
|
|
190 |
system(paste("pkmosaic -co COMPRESS=LZW -co PREDICTOR=2 ",paste("-i",list.files("gridded",full=T,pattern="tif$"),collapse=" ")," -o MOD35_path_pkmosaic_max.tif -m 2 -v -t 255 -t 0 &"))
|
|
151 | 191 |
#bb="-ulx -180 -uly 90 -lrx 180 -lry -90" |
152 | 192 |
#bb="-ulx -180 -uly 90 -lrx 170 -lry 80" |
153 | 193 |
bb="-ulx -72 -uly 11 -lrx -59 -lry -1" |
154 | 194 |
|
155 | 195 |
|
156 | 196 |
#expand.grid(x=seq(-180,170,by=10),y=seq(-90,80)) |
157 |
|
|
158 |
system(paste("pkmosaic ",bb," -co COMPRESS=LZW -co PREDICTOR=2 ",paste("-i",list.files("gridded",full=T,pattern="tif$"),collapse=" ")," -o h11v08_path_pkmosaic.tif -m 6 -v -t 255 -t 0 &"))
|
|
197 |
gf2= grep("2012009[.]03",gfiles,value=T) |
|
198 |
system(paste("pkmosaic ",bb," -co COMPRESS=LZW -co PREDICTOR=2 ",paste("-i",gf2,collapse=" ")," -o h11v08_path_pkmosaic.tif -ot Byte -m 7 -v -t 255"))
|
|
159 | 199 |
|
160 | 200 |
# bounding box? |
161 | 201 |
|
... | ... | |
193 | 233 |
unlink_.gislock() |
194 | 234 |
system(paste("rm -frR ",tf,sep="")) |
195 | 235 |
|
196 |
|
|
197 | 236 |
######################### |
198 | 237 |
|
199 | 238 |
|
Also available in: Unified diff
Updated script to process MOD35 Processing Path using HEG to interpolate sensor zenith observations. Also working on figures for MOD35 C5 vs C6 comparison in MOD35C5_Evaluation script