Revision 65cc18e7
Added by Adam Wilson over 11 years ago
climate/procedures/MOD06_L2_process.r | ||
---|---|---|
11 | 11 |
|
12 | 12 |
## import commandline arguments |
13 | 13 |
library(getopt) |
14 |
## load libraries |
|
15 |
require(reshape) |
|
16 |
require(geosphere) |
|
17 |
require(raster) |
|
18 |
require(rgdal) |
|
19 |
require(spgrass6) |
|
20 |
require(RSQLite) |
|
21 |
|
|
14 | 22 |
## get options |
15 | 23 |
opta <- getopt(matrix(c( |
16 | 24 |
'date', 'd', 1, 'character', |
... | ... | |
27 | 35 |
|
28 | 36 |
|
29 | 37 |
## default date and tile to play with |
30 |
date="20030301"
|
|
38 |
date="20030102"
|
|
31 | 39 |
tile="h11v08" |
32 | 40 |
platform="pleiades" |
33 | 41 |
verbose=T |
... | ... | |
57 | 65 |
} |
58 | 66 |
|
59 | 67 |
if(platform="litoria"){ #if running on local server, use different paths |
60 |
## location of MOD06 files |
|
61 |
datadir="/nobackupp4/datapool/modis/MOD06_L2.005/" |
|
68 |
## specify working directory |
|
69 |
setwd("~/acrobates/projects/interp") |
|
70 |
gisBase="/usr/lib/grass64" |
|
71 |
## location of MOD06 files |
|
72 |
datadir="~/acrobates/projects/interp/data/modis/Venezuela/MOD06" |
|
62 | 73 |
## path to some executables |
63 |
ncopath="/nasa/sles11/nco/4.0.8/gcc/mpt/bin/"
|
|
74 |
ncopath="" |
|
64 | 75 |
swtifpath="sudo MRTDATADIR=\"/usr/local/heg/data\" PGSHOME=/usr/local/heg/TOOLKIT_MTD PWD=/home/adamw /usr/local/heg/bin/swtif" |
65 | 76 |
## path to swath database |
66 |
db="/nobackupp4/pvotava/DB/export/swath_geo.sql.sqlite3.db" |
|
67 |
## specify working directory |
|
68 |
setwd("/nobackupp1/awilso10/mod06") |
|
69 |
gisBase="/usr/lib/grass64" |
|
77 |
db="/home/adamw/acrobates/projects/interp/data/modis/mod06/swath_geo.sql.sqlite3.db" |
|
70 | 78 |
## get grid file |
71 | 79 |
td=raster(paste("~/acrobates/projects/interp/data/modis/mod06/summary/MOD06_",tile,".nc",sep=""),varname="CER") |
72 | 80 |
projection(td)="+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs +datum=WGS84 +ellps=WGS84 " |
... | ... | |
76 | 84 |
### print some status messages |
77 | 85 |
if(verbose) print(paste("Processing tile",tile," for date",date)) |
78 | 86 |
|
79 |
## load libraries |
|
80 |
require(reshape) |
|
81 |
require(geosphere) |
|
82 |
require(raster) |
|
83 |
require(rgdal) |
|
84 |
require(spgrass6) |
|
85 |
require(RSQLite) |
|
86 |
|
|
87 |
|
|
88 | 87 |
## load tile information and get bounding box |
89 | 88 |
load(file="modlandTiles.Rdata") |
90 | 89 |
tile_bb=tb[tb$tile==tile,] ## identify tile of interest |
... | ... | |
207 | 206 |
tf=paste(tempdir(),"/grass", Sys.getpid(),"/", sep="") #temporar |
208 | 207 |
if(!file.exists(tf)) dir.create(tf) |
209 | 208 |
## create output directory if needed |
210 |
if(!file.exists(dirname(ncfile))) dir.create(dirname(ncfile)) |
|
209 |
if(!file.exists(dirname(ncfile))) dir.create(dirname(ncfile),recursive=T)
|
|
211 | 210 |
|
212 | 211 |
## set up temporary grass instance for this PID |
213 | 212 |
if(verbose) print(paste("Set up temporary grass session in",tf)) |
214 |
initGRASS(gisBase=gisBase,gisDbase=tf,SG=td,override=T,location="mod06",mapset="PERMANENT",home=tf,pid=Sys.getpid())
|
|
213 |
initGRASS(gisBase=gisBase,gisDbase=tf,SG=as(td,"SpatialGridDataFrame"),override=T,location="mod06",mapset="PERMANENT",home=tf,pid=Sys.getpid())
|
|
215 | 214 |
system(paste("g.proj -c proj4=\"",projection(td),"\"",sep=""),ignore.stdout=T,ignore.stderr=T) |
216 | 215 |
|
217 | 216 |
## Define region by importing one MOD11A1 raster. |
218 | 217 |
print("Import one MOD11A1 raster to define grid") |
219 | 218 |
if(platform=="pleiades") execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",gridfile,"\":MODIS_Grid_Daily_1km_LST:Night_view_angl",sep=""), |
220 | 219 |
output="modisgrid",flags=c("quiet","overwrite","o")) |
221 |
if(platform=="litoria") execGRASS("r.in.gdal",input=paste("NETCDF:\"/home/adamw/acrobates/projects/interp/data/modis/mod06/summary/MOD06_",tile,".nc\":CER",sep=""),output="modisgrid",flags=c("quiet","overwrite","o"))
|
|
220 |
if(platform=="litoria") execGRASS("r.in.gdal",input=paste("NETCDF:\"/home/adamw/acrobates/projects/interp/data/modis/mod06/summary/MOD06_",tile,".nc\":CER",sep=""),output="modisgrid",flags=c("overwrite","o")) |
|
222 | 221 |
|
223 |
system("g.region rast=modisgrid save=roi --overwrite",ignore.stdout=T,ignore.stderr=T) |
|
222 |
system("g.region rast=modisgrid save=roi --overwrite",ignore.stdout=F,ignore.stderr=F) |
|
223 |
system("g.region rast=modisgrid.1 save=roi --overwrite",ignore.stdout=F,ignore.stderr=F) |
|
224 | 224 |
|
225 | 225 |
## Identify which files to process |
226 | 226 |
tfs=basename(swaths) |
227 | 227 |
## drop swaths that did not produce an output file (typically due to not overlapping the ROI) |
228 | 228 |
tfs=tfs[tfs%in%list.files(tempdir())] |
229 | 229 |
nfs=length(tfs) |
230 |
if(verbose) print(nfs,"swaths available for processing")
|
|
230 |
if(verbose) print(paste(nfs,"swaths available for processing"))
|
|
231 | 231 |
|
232 | 232 |
## loop through scenes and process QA flags |
233 | 233 |
for(i in 1:nfs){ |
... | ... | |
239 | 239 |
system(paste("r.mapcalc <<EOF |
240 | 240 |
CM_cloud_",i," = ((CM1_",i," / 2^0) % 2) == 1 && ((CM1_",i," / 2^1) % 2^2) == 0 |
241 | 241 |
CM_clear_",i," = ((CM1_",i," / 2^0) % 2) == 1 && ((CM1_",i," / 2^1) % 2^2) > 2 |
242 |
CM_path_",i," = ((CM1_",i," / 2^6) % 2^2) |
|
243 |
CM_cloud2_",i," = ((CM1_",i," / 2^1) % 2^2) |
|
244 |
|
|
242 | 245 |
EOF",sep="")) |
243 | 246 |
|
244 |
## QA
|
|
247 |
## QA |
|
245 | 248 |
execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",file,"\":mod06:Quality_Assurance_1km_0",sep=""), |
246 | 249 |
output=paste("QA_",i,sep=""),flags=c("overwrite","o")) ; print("") |
247 | 250 |
## QA_CER |
... | ... | |
263 | 266 |
execGRASS("r.null",map=paste("COT_",i,sep=""),setnull="-9999") |
264 | 267 |
## keep only positive COT values where quality is 'useful' and '>= good' & scale to real units |
265 | 268 |
system(paste("r.mapcalc \"COT_",i,"=if(QA_COT_",i,"&&QA_COT2_",i,"&&QA_COT3_",i,"&&COT_",i,">=0,COT_",i,",null())\"",sep="")) |
266 |
## set COT to 0 in clear-sky pixels |
|
267 |
system(paste("r.mapcalc \"COT2_",i,"=if(CM_clear_",i,"==0,COT_",i,",0)\"",sep=""))
|
|
269 |
## set null COT to 0 in clear-sky pixels
|
|
270 |
system(paste("r.mapcalc \"COT2_",i,"=if(isnull(COT_",i,")&&CM_clear_",i,"==1,0,COT_",i,")\"",sep=""))
|
|
268 | 271 |
|
269 | 272 |
## Effective radius ## |
270 | 273 |
execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",file,"\":mod06:Cloud_Effective_Radius",sep=""), |
... | ... | |
274 | 277 |
execGRASS("r.null",map=paste("CER_",i,sep=""),setnull="-9999") |
275 | 278 |
## keep only positive CER values where quality is 'useful' and '>= good' & scale to real units |
276 | 279 |
system(paste("r.mapcalc \"CER_",i,"=if(QA_CER_",i,"&&QA_CER2_",i,"&&CER_",i,">=0,CER_",i,",null())\"",sep="")) |
277 |
## set CER to 0 in clear-sky pixels |
|
278 |
system(paste("r.mapcalc \"CER2_",i,"=if(CM_clear_",i,"==0,CER_",i,",0)\"",sep=""))
|
|
280 |
## set null CER to 0 in clear-sky pixels
|
|
281 |
system(paste("r.mapcalc \"CER2_",i,"=if(isnull(CER_",i,")&&CM_clear_",i,"==1,0,CER_",i,")\"",sep=""))
|
|
279 | 282 |
|
280 | 283 |
## Cloud Water Path |
281 | 284 |
# execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",file,"\":mod06:Cloud_Water_Path",sep=""), |
282 | 285 |
# output=paste("CWP_",i,sep=""),title="cloud_water_path", |
283 | 286 |
# flags=c("overwrite","o")) ; print("") |
284 | 287 |
# execGRASS("r.null",map=paste("CWP_",i,sep=""),setnull="-9999") |
285 |
## keep only positive CER values where quality is 'useful' and 'very good' & scale to real units |
|
288 |
# system(paste("r.mapcalc \"CWP_",i,"=if(CWP_",i,"<0,null(),CWP_",i,")\"",sep="")) |
|
289 |
## keep only positive CER values where quality is 'useful' and 'very good' & scale to real units |
|
286 | 290 |
# system(paste("r.mapcalc \"CWP_",i,"=if(QA_CWP_",i,"&&QA_CWP2_",i,"&&CWP_",i,">=0,CWP_",i,",null())\"",sep="")) |
287 | 291 |
## set CER to 0 in clear-sky pixels |
288 | 292 |
# system(paste("r.mapcalc \"CWP2_",i,"=if(CM_clear_",i,"==0,CWP_",i,",0)\"",sep="")) |
... | ... | |
292 | 296 |
#### Now generate daily averages (or maximum in case of cloud flag) |
293 | 297 |
|
294 | 298 |
system(paste("r.mapcalc <<EOF |
295 |
COT_denom=",paste("!isnull(COT2_",1:nfs,")",sep="",collapse="+"),"
|
|
296 |
COT_numer=",paste("if(isnull(COT2_",1:nfs,"),0,COT2_",1:nfs,")",sep="",collapse="+"),"
|
|
299 |
COT_numer=",paste("if(isnull(COT_",1:nfs,"),0,COT_",1:nfs,")",sep="",collapse="+"),"
|
|
300 |
COT_denom=",paste("!isnull(COT_",1:nfs,")",sep="",collapse="+"),"
|
|
297 | 301 |
COT_daily=int(COT_numer/COT_denom) |
298 |
CER_denom=",paste("!isnull(CER2_",1:nfs,")",sep="",collapse="+")," |
|
299 |
CER_numer=",paste("if(isnull(CER2_",1:nfs,"),0,CER2_",1:nfs,")",sep="",collapse="+")," |
|
302 |
COT2_numer=",paste("if(isnull(COT2_",1:nfs,"),0,COT2_",1:nfs,")",sep="",collapse="+")," |
|
303 |
COT2_denom=",paste("!isnull(COT2_",1:nfs,")",sep="",collapse="+")," |
|
304 |
COT2_daily=int(COT2_numer/COT2_denom) |
|
305 |
CER_numer=",paste("if(isnull(CER_",1:nfs,"),0,CER_",1:nfs,")",sep="",collapse="+")," |
|
306 |
CER_denom=",paste("!isnull(CER_",1:nfs,")",sep="",collapse="+")," |
|
300 | 307 |
CER_daily=int(CER_numer/CER_denom) |
301 |
CLD_daily=int((max(",paste("if(isnull(CM_cloud_",1:nfs,"),0,CM_cloud_",1:nfs,")",sep="",collapse=","),"))*100) |
|
308 |
CER2_numer=",paste("if(isnull(CER2_",1:nfs,"),0,CER2_",1:nfs,")",sep="",collapse="+")," |
|
309 |
CER2_denom=",paste("!isnull(CER2_",1:nfs,")",sep="",collapse="+")," |
|
310 |
CER2_daily=int(CER2_numer/CER2_denom) |
|
311 |
CLD_daily=int((max(",paste("if(isnull(CM_cloud_",1:nfs,"),-9999,CM_cloud_",1:nfs,")",sep="",collapse=","),"))) |
|
302 | 312 |
EOF",sep="")) |
313 |
execGRASS("r.null",map="CLD_daily",setnull="-9999") |
|
303 | 314 |
|
304 | 315 |
|
305 | 316 |
### Write the files to a netcdf file |
306 | 317 |
## create image group to facilitate export as multiband netcdf |
307 |
execGRASS("i.group",group="mod06",input=c("CER_daily","COT_daily","CLD_daily")) ; print("")
|
|
318 |
execGRASS("i.group",group="mod06",input=c("CER_daily","CER2_daily","COT_daily","COT2_daily","CLD_daily")) ; print("")
|
|
308 | 319 |
|
309 | 320 |
if(file.exists(ncfile)) file.remove(ncfile) #if it exists already, delete it |
310 | 321 |
execGRASS("r.out.gdal",input="mod06",output=ncfile,type="Int16",nodata=-32768,flags=c("quiet"), |
... | ... | |
328 | 339 |
system(paste("ncgen -o ",tempdir(),"/time.nc ",tempdir(),"/time.cdl",sep="")) |
329 | 340 |
system(paste(ncopath,"ncks -A ",tempdir(),"/time.nc ",ncfile,sep="")) |
330 | 341 |
## add other attributes |
331 |
system(paste(ncopath,"ncrename -v Band1,CER -v Band2,COT -v Band3,CLD ",ncfile,sep=""))
|
|
342 |
system(paste(ncopath,"ncrename -v Band1,CER -v Band2,CER2 -v Band3,COT -v Band4,COT2 -v Band5,CLD ",ncfile,sep=""))
|
|
332 | 343 |
system(paste(ncopath,"ncatted -a scale_factor,CER,o,d,0.01 -a units,CER,o,c,\"micron\" -a missing_value,CER,o,d,-32768 -a long_name,CER,o,c,\"Cloud Particle Effective Radius\" ",ncfile,sep="")) |
333 |
system(paste(ncopath,"ncatted -a scale_factor,COT,o,d,0.01 -a units,COT,o,c,\"none\" -a missing_value,COT,o,d,-32768 -a long_name,COT,o,c,\"Cloud Optical Thickness\" ",ncfile,sep="")) |
|
344 |
system(paste(ncopath,"ncatted -a scale_factor,CER2,o,d,0.01 -a units,CER2,o,c,\"micron\" -a missing_value,CER2,o,d,-32768 -a long_name,CER2,o,c,\"Cloud Particle Effective Radius with clear sky set to zero\" ",ncfile,sep="")) |
|
345 |
|
|
346 |
system(paste(ncopath,"ncatted -a scale_factor,COT,o,d,0.01 -a units,COT,o,c,\"none\" -a missing_value,COT,o,d,-32768 -a long_name,COT,o,c,\"Cloud Optical Thickness\" ",ncfile,sep="")) |
|
347 |
system(paste(ncopath,"ncatted -a scale_factor,COT2,o,d,0.01 -a units,COT2,o,c,\"none\" -a missing_value,COT2,o,d,-32768 -a long_name,COT2,o,c,\"Cloud Optical Thickness with clear sky set to zero\" ",ncfile,sep="")) |
|
334 | 348 |
system(paste(ncopath,"ncatted -a scale_factor,CLD,o,d,0.01 -a units,CLD,o,c,\"none\" -a missing_value,CLD,o,d,-32768 -a long_name,CLD,o,c,\"Cloud Mask\" ",ncfile,sep="")) |
349 |
# system(paste(ncopath,"ncatted -a sourcecode,global,o,c,",script," ",ncfile,sep="")) |
|
335 | 350 |
|
336 | 351 |
|
337 | 352 |
### delete the temporary files |
Also available in: Unified diff
Adjusted handling of cloud flag to 'hopefully' improve missing data problem in MOD06 climatology