Project

General

Profile

« Previous | Next » 

Revision 65cc18e7

Added by Adam Wilson over 11 years ago

Adjusted handling of cloud flag to 'hopefully' improve missing data problem in MOD06 climatology

View differences:

climate/procedures/MOD06_L2_process.r
11 11

  
12 12
## import commandline arguments
13 13
library(getopt)
14
## load libraries
15
require(reshape)
16
require(geosphere)
17
require(raster)
18
require(rgdal)
19
require(spgrass6)
20
require(RSQLite)
21

  
14 22
## get options
15 23
opta <- getopt(matrix(c(
16 24
                        'date', 'd', 1, 'character',
......
27 35

  
28 36

  
29 37
## default date and tile to play with
30
date="20030301"
38
date="20030102"
31 39
tile="h11v08"
32 40
platform="pleiades" 
33 41
verbose=T
......
57 65
}
58 66

  
59 67
if(platform="litoria"){  #if running on local server, use different paths
60
  ## location of MOD06 files
61
  datadir="/nobackupp4/datapool/modis/MOD06_L2.005/"
68
  ## specify working directory
69
  setwd("~/acrobates/projects/interp")
70
  gisBase="/usr/lib/grass64"
71
   ## location of MOD06 files
72
  datadir="~/acrobates/projects/interp/data/modis/Venezuela/MOD06"
62 73
  ## path to some executables
63
  ncopath="/nasa/sles11/nco/4.0.8/gcc/mpt/bin/"
74
  ncopath=""
64 75
  swtifpath="sudo MRTDATADIR=\"/usr/local/heg/data\" PGSHOME=/usr/local/heg/TOOLKIT_MTD PWD=/home/adamw /usr/local/heg/bin/swtif"
65 76
  ## path to swath database
66
  db="/nobackupp4/pvotava/DB/export/swath_geo.sql.sqlite3.db"
67
  ## specify working directory
68
  setwd("/nobackupp1/awilso10/mod06")
69
  gisBase="/usr/lib/grass64"
77
  db="/home/adamw/acrobates/projects/interp/data/modis/mod06/swath_geo.sql.sqlite3.db"
70 78
  ## get grid file
71 79
  td=raster(paste("~/acrobates/projects/interp/data/modis/mod06/summary/MOD06_",tile,".nc",sep=""),varname="CER")
72 80
  projection(td)="+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs +datum=WGS84 +ellps=WGS84 "
......
76 84
### print some status messages
77 85
if(verbose) print(paste("Processing tile",tile," for date",date))
78 86

  
79
## load libraries
80
require(reshape)
81
require(geosphere)
82
require(raster)
83
require(rgdal)
84
require(spgrass6)
85
require(RSQLite)
86

  
87

  
88 87
## load tile information and get bounding box
89 88
load(file="modlandTiles.Rdata")
90 89
tile_bb=tb[tb$tile==tile,] ## identify tile of interest
......
207 206
  tf=paste(tempdir(),"/grass", Sys.getpid(),"/", sep="")  #temporar
208 207
  if(!file.exists(tf)) dir.create(tf)
209 208
  ## create output directory if needed
210
  if(!file.exists(dirname(ncfile))) dir.create(dirname(ncfile))
209
  if(!file.exists(dirname(ncfile))) dir.create(dirname(ncfile),recursive=T)
211 210
  
212 211
  ## set up temporary grass instance for this PID
213 212
  if(verbose) print(paste("Set up temporary grass session in",tf))
214
  initGRASS(gisBase=gisBase,gisDbase=tf,SG=td,override=T,location="mod06",mapset="PERMANENT",home=tf,pid=Sys.getpid())
213
  initGRASS(gisBase=gisBase,gisDbase=tf,SG=as(td,"SpatialGridDataFrame"),override=T,location="mod06",mapset="PERMANENT",home=tf,pid=Sys.getpid())
215 214
  system(paste("g.proj -c proj4=\"",projection(td),"\"",sep=""),ignore.stdout=T,ignore.stderr=T)
216 215

  
217 216
  ## Define region by importing one MOD11A1 raster.
218 217
  print("Import one MOD11A1 raster to define grid")
219 218
  if(platform=="pleiades") execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",gridfile,"\":MODIS_Grid_Daily_1km_LST:Night_view_angl",sep=""),
220 219
            output="modisgrid",flags=c("quiet","overwrite","o"))
221
   if(platform=="litoria") execGRASS("r.in.gdal",input=paste("NETCDF:\"/home/adamw/acrobates/projects/interp/data/modis/mod06/summary/MOD06_",tile,".nc\":CER",sep=""),output="modisgrid",flags=c("quiet","overwrite","o"))
220
   if(platform=="litoria") execGRASS("r.in.gdal",input=paste("NETCDF:\"/home/adamw/acrobates/projects/interp/data/modis/mod06/summary/MOD06_",tile,".nc\":CER",sep=""),output="modisgrid",flags=c("overwrite","o"))
222 221
  
223
system("g.region rast=modisgrid save=roi --overwrite",ignore.stdout=T,ignore.stderr=T)
222
system("g.region rast=modisgrid save=roi --overwrite",ignore.stdout=F,ignore.stderr=F)
223
system("g.region rast=modisgrid.1 save=roi --overwrite",ignore.stdout=F,ignore.stderr=F)
224 224

  
225 225
## Identify which files to process
226 226
tfs=basename(swaths)
227 227
## drop swaths that did not produce an output file (typically due to not overlapping the ROI)
228 228
tfs=tfs[tfs%in%list.files(tempdir())]
229 229
nfs=length(tfs)
230
if(verbose) print(nfs,"swaths available for processing")
230
if(verbose) print(paste(nfs,"swaths available for processing"))
231 231

  
232 232
## loop through scenes and process QA flags
233 233
  for(i in 1:nfs){
......
239 239
    system(paste("r.mapcalc <<EOF
240 240
                CM_cloud_",i," =  ((CM1_",i," / 2^0) % 2) == 1  &&  ((CM1_",i," / 2^1) % 2^2) == 0 
241 241
                CM_clear_",i," =  ((CM1_",i," / 2^0) % 2) == 1  &&  ((CM1_",i," / 2^1) % 2^2) >  2 
242
                CM_path_",i," =   ((CM1_",i," / 2^6) % 2^2) 
243
                CM_cloud2_",i," = ((CM1_",i," / 2^1) % 2^2) 
244

  
242 245
EOF",sep=""))
243 246

  
244
     ## QA
247
    ## QA
245 248
     execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",file,"\":mod06:Quality_Assurance_1km_0",sep=""),
246 249
             output=paste("QA_",i,sep=""),flags=c("overwrite","o")) ; print("")
247 250
   ## QA_CER
......
263 266
   execGRASS("r.null",map=paste("COT_",i,sep=""),setnull="-9999")
264 267
   ## keep only positive COT values where quality is 'useful' and '>= good' & scale to real units
265 268
   system(paste("r.mapcalc \"COT_",i,"=if(QA_COT_",i,"&&QA_COT2_",i,"&&QA_COT3_",i,"&&COT_",i,">=0,COT_",i,",null())\"",sep=""))   
266
   ## set COT to 0 in clear-sky pixels
267
   system(paste("r.mapcalc \"COT2_",i,"=if(CM_clear_",i,"==0,COT_",i,",0)\"",sep=""))   
269
   ## set null COT to 0 in clear-sky pixels
270
   system(paste("r.mapcalc \"COT2_",i,"=if(isnull(COT_",i,")&&CM_clear_",i,"==1,0,COT_",i,")\"",sep=""))   
268 271
   
269 272
   ## Effective radius ##
270 273
   execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",file,"\":mod06:Cloud_Effective_Radius",sep=""),
......
274 277
   execGRASS("r.null",map=paste("CER_",i,sep=""),setnull="-9999")
275 278
   ## keep only positive CER values where quality is 'useful' and '>= good' & scale to real units
276 279
   system(paste("r.mapcalc \"CER_",i,"=if(QA_CER_",i,"&&QA_CER2_",i,"&&CER_",i,">=0,CER_",i,",null())\"",sep=""))   
277
   ## set CER to 0 in clear-sky pixels
278
   system(paste("r.mapcalc \"CER2_",i,"=if(CM_clear_",i,"==0,CER_",i,",0)\"",sep=""))   
280
   ## set null CER to 0 in clear-sky pixels
281
   system(paste("r.mapcalc \"CER2_",i,"=if(isnull(CER_",i,")&&CM_clear_",i,"==1,0,CER_",i,")\"",sep=""))   
279 282

  
280 283
   ## Cloud Water Path
281 284
#   execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",file,"\":mod06:Cloud_Water_Path",sep=""),
282 285
#            output=paste("CWP_",i,sep=""),title="cloud_water_path",
283 286
#            flags=c("overwrite","o")) ; print("")
284 287
#   execGRASS("r.null",map=paste("CWP_",i,sep=""),setnull="-9999")
285
   ## keep only positive CER values where quality is 'useful' and 'very good' & scale to real units
288
#   system(paste("r.mapcalc \"CWP_",i,"=if(CWP_",i,"<0,null(),CWP_",i,")\"",sep=""))   
289
     ## keep only positive CER values where quality is 'useful' and 'very good' & scale to real units
286 290
#   system(paste("r.mapcalc \"CWP_",i,"=if(QA_CWP_",i,"&&QA_CWP2_",i,"&&CWP_",i,">=0,CWP_",i,",null())\"",sep=""))   
287 291
   ## set CER to 0 in clear-sky pixels
288 292
#   system(paste("r.mapcalc \"CWP2_",i,"=if(CM_clear_",i,"==0,CWP_",i,",0)\"",sep=""))   
......
292 296
#### Now generate daily averages (or maximum in case of cloud flag)
293 297
  
294 298
  system(paste("r.mapcalc <<EOF
295
         COT_denom=",paste("!isnull(COT2_",1:nfs,")",sep="",collapse="+"),"
296
         COT_numer=",paste("if(isnull(COT2_",1:nfs,"),0,COT2_",1:nfs,")",sep="",collapse="+"),"
299
         COT_numer=",paste("if(isnull(COT_",1:nfs,"),0,COT_",1:nfs,")",sep="",collapse="+"),"
300
         COT_denom=",paste("!isnull(COT_",1:nfs,")",sep="",collapse="+"),"
297 301
         COT_daily=int(COT_numer/COT_denom)
298
         CER_denom=",paste("!isnull(CER2_",1:nfs,")",sep="",collapse="+"),"
299
         CER_numer=",paste("if(isnull(CER2_",1:nfs,"),0,CER2_",1:nfs,")",sep="",collapse="+"),"
302
         COT2_numer=",paste("if(isnull(COT2_",1:nfs,"),0,COT2_",1:nfs,")",sep="",collapse="+"),"
303
         COT2_denom=",paste("!isnull(COT2_",1:nfs,")",sep="",collapse="+"),"
304
         COT2_daily=int(COT2_numer/COT2_denom)
305
         CER_numer=",paste("if(isnull(CER_",1:nfs,"),0,CER_",1:nfs,")",sep="",collapse="+"),"
306
         CER_denom=",paste("!isnull(CER_",1:nfs,")",sep="",collapse="+"),"
300 307
         CER_daily=int(CER_numer/CER_denom)
301
         CLD_daily=int((max(",paste("if(isnull(CM_cloud_",1:nfs,"),0,CM_cloud_",1:nfs,")",sep="",collapse=","),"))*100) 
308
         CER2_numer=",paste("if(isnull(CER2_",1:nfs,"),0,CER2_",1:nfs,")",sep="",collapse="+"),"
309
         CER2_denom=",paste("!isnull(CER2_",1:nfs,")",sep="",collapse="+"),"
310
         CER2_daily=int(CER2_numer/CER2_denom)
311
         CLD_daily=int((max(",paste("if(isnull(CM_cloud_",1:nfs,"),-9999,CM_cloud_",1:nfs,")",sep="",collapse=","),"))) 
302 312
EOF",sep=""))
313
   execGRASS("r.null",map="CLD_daily",setnull="-9999")
303 314

  
304 315

  
305 316
  ### Write the files to a netcdf file
306 317
  ## create image group to facilitate export as multiband netcdf
307
    execGRASS("i.group",group="mod06",input=c("CER_daily","COT_daily","CLD_daily")) ; print("")
318
    execGRASS("i.group",group="mod06",input=c("CER_daily","CER2_daily","COT_daily","COT2_daily","CLD_daily")) ; print("")
308 319
   
309 320
  if(file.exists(ncfile)) file.remove(ncfile)  #if it exists already, delete it
310 321
  execGRASS("r.out.gdal",input="mod06",output=ncfile,type="Int16",nodata=-32768,flags=c("quiet"),
......
328 339
system(paste("ncgen -o ",tempdir(),"/time.nc ",tempdir(),"/time.cdl",sep=""))
329 340
system(paste(ncopath,"ncks -A ",tempdir(),"/time.nc ",ncfile,sep=""))
330 341
## add other attributes
331
  system(paste(ncopath,"ncrename -v Band1,CER -v Band2,COT -v Band3,CLD ",ncfile,sep=""))
342
  system(paste(ncopath,"ncrename -v Band1,CER -v Band2,CER2 -v Band3,COT -v Band4,COT2 -v Band5,CLD ",ncfile,sep=""))
332 343
  system(paste(ncopath,"ncatted -a scale_factor,CER,o,d,0.01 -a units,CER,o,c,\"micron\" -a missing_value,CER,o,d,-32768 -a long_name,CER,o,c,\"Cloud Particle Effective Radius\" ",ncfile,sep=""))
333
  system(paste(ncopath,"ncatted -a scale_factor,COT,o,d,0.01 -a units,COT,o,c,\"none\" -a missing_value,COT,o,d,-32768 -a long_name,COT,o,c,\"Cloud Optical Thickness\" ",ncfile,sep=""))
344
  system(paste(ncopath,"ncatted -a scale_factor,CER2,o,d,0.01 -a units,CER2,o,c,\"micron\" -a missing_value,CER2,o,d,-32768 -a long_name,CER2,o,c,\"Cloud Particle Effective Radius with clear sky set to zero\" ",ncfile,sep=""))
345

  
346
system(paste(ncopath,"ncatted -a scale_factor,COT,o,d,0.01 -a units,COT,o,c,\"none\" -a missing_value,COT,o,d,-32768 -a long_name,COT,o,c,\"Cloud Optical Thickness\" ",ncfile,sep=""))
347
system(paste(ncopath,"ncatted -a scale_factor,COT2,o,d,0.01 -a units,COT2,o,c,\"none\" -a missing_value,COT2,o,d,-32768 -a long_name,COT2,o,c,\"Cloud Optical Thickness with clear sky set to zero\" ",ncfile,sep=""))
334 348
  system(paste(ncopath,"ncatted -a scale_factor,CLD,o,d,0.01 -a units,CLD,o,c,\"none\" -a missing_value,CLD,o,d,-32768 -a long_name,CLD,o,c,\"Cloud Mask\" ",ncfile,sep=""))
349
#  system(paste(ncopath,"ncatted -a sourcecode,global,o,c,",script," ",ncfile,sep=""))
335 350
   
336 351
  
337 352
### delete the temporary files 

Also available in: Unified diff