1
|
###################################################################################
|
2
|
### R code to aquire and process MOD06_L2 cloud data from the MODIS platform
|
3
|
|
4
|
|
5
|
# Redirect all warnings to stderr()
|
6
|
#options(warn = -1)
|
7
|
#write("2) write() to stderr", stderr())
|
8
|
#write("2) write() to stdout", stdout())
|
9
|
#warning("2) warning()")
|
10
|
|
11
|
|
12
|
## import commandline arguments
|
13
|
library(getopt)
|
14
|
## get options
|
15
|
opta <- getopt(matrix(c(
|
16
|
'date', 'd', 1, 'character',
|
17
|
'tile', 't', 1, 'character',
|
18
|
'verbose','v',1,'logical',
|
19
|
'help', 'h', 0, 'logical'
|
20
|
), ncol=4, byrow=TRUE))
|
21
|
if ( !is.null(opta$help) )
|
22
|
{
|
23
|
prg <- commandArgs()[1];
|
24
|
cat(paste("Usage: ", prg, " --date | -d <file> :: The date to process\n", sep=""));
|
25
|
q(status=1);
|
26
|
}
|
27
|
|
28
|
|
29
|
date=opta$date #date="20101225"
|
30
|
tile=opta$tile #tile="h11v08"
|
31
|
verbose=opta$verbose #print out extensive information for debugging?
|
32
|
outdir=paste("daily/",tile,"/",sep="") #directory for separate daily files
|
33
|
|
34
|
## permanent storage on lou:
|
35
|
ldir=paste("lfe1.nas.nasa.gov:/u/awilso10/mod06/daily/",tile,sep="")
|
36
|
|
37
|
print(paste("Processing tile",tile," for date",date))
|
38
|
|
39
|
## load libraries
|
40
|
require(reshape)
|
41
|
require(geosphere)
|
42
|
require(raster)
|
43
|
library(rgdal)
|
44
|
require(spgrass6)
|
45
|
#require(RSQLite)
|
46
|
|
47
|
|
48
|
## specify some working directories
|
49
|
setwd("/nobackupp1/awilso10/mod06")
|
50
|
|
51
|
## load ancillary data
|
52
|
load(file="allfiles.Rdata")
|
53
|
|
54
|
## load tile information
|
55
|
load(file="modlandTiles.Rdata")
|
56
|
|
57
|
## vars to process
|
58
|
vars=as.data.frame(matrix(c(
|
59
|
"Cloud_Effective_Radius", "CER",
|
60
|
"Cloud_Effective_Radius_Uncertainty", "CERU",
|
61
|
"Cloud_Optical_Thickness", "COT",
|
62
|
"Cloud_Optical_Thickness_Uncertainty", "COTU",
|
63
|
"Cloud_Water_Path", "CWP",
|
64
|
"Cloud_Water_Path_Uncertainty", "CWPU",
|
65
|
"Cloud_Phase_Optical_Properties", "CPOP",
|
66
|
"Cloud_Multi_Layer_Flag", "CMLF",
|
67
|
"Cloud_Mask_1km", "CM1",
|
68
|
"Quality_Assurance_1km", "QA"),
|
69
|
byrow=T,ncol=2,dimnames=list(1:10,c("variable","varid"))),stringsAsFactors=F)
|
70
|
|
71
|
|
72
|
############################################################################
|
73
|
############################################################################
|
74
|
### Define functions to process a particular date-tile
|
75
|
|
76
|
#### get swaths
|
77
|
#getswaths<-function(tile,db="/nobackupp4/pvotava/DB/export/swath_geo.sql.sqlite3.db"){
|
78
|
# db=dbConnect(
|
79
|
# }
|
80
|
|
81
|
swath2grid=function(file,vars,upleft,lowright){
|
82
|
## Function to generate hegtool parameter file for multi-band HDF-EOS file
|
83
|
print(paste("Starting file",basename(file)))
|
84
|
outfile=paste(tempdir(),"/",basename(file),sep="")
|
85
|
## First write the parameter file (careful, heg is very finicky!)
|
86
|
hdr=paste("NUM_RUNS = ",length(vars$varid),"|MULTI_BAND_HDFEOS:",length(vars$varid),sep="")
|
87
|
grp=paste("
|
88
|
BEGIN
|
89
|
INPUT_FILENAME=",file,"
|
90
|
OBJECT_NAME=mod06
|
91
|
FIELD_NAME=",vars$variable,"|
|
92
|
BAND_NUMBER = 1
|
93
|
OUTPUT_PIXEL_SIZE_X=1000
|
94
|
OUTPUT_PIXEL_SIZE_Y=1000
|
95
|
SPATIAL_SUBSET_UL_CORNER = ( ",upleft," )
|
96
|
SPATIAL_SUBSET_LR_CORNER = ( ",lowright," )
|
97
|
#RESAMPLING_TYPE =",ifelse(grepl("Flag|Mask|Quality",vars),"NN","CUBIC"),"
|
98
|
RESAMPLING_TYPE =NN
|
99
|
OUTPUT_PROJECTION_TYPE = SIN
|
100
|
OUTPUT_PROJECTION_PARAMETERS = ( 6371007.181 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 )
|
101
|
# projection parameters from http://landweb.nascom.nasa.gov/cgi-bin/QA_WWW/newPage.cgi?fileName=sn_gctp
|
102
|
ELLIPSOID_CODE = WGS84
|
103
|
OUTPUT_TYPE = HDFEOS
|
104
|
OUTPUT_FILENAME = ",outfile,"
|
105
|
END
|
106
|
|
107
|
",sep="")
|
108
|
|
109
|
## if any remnants from previous runs remain, delete them
|
110
|
if(length(list.files(tempdir(),pattern=basename(file)))>0)
|
111
|
file.remove(list.files(tempdir(),pattern=basename(file),full=T))
|
112
|
## write it to a file
|
113
|
cat(c(hdr,grp) , file=paste(tempdir(),"/",basename(file),"_MODparms.txt",sep=""))
|
114
|
## now run the swath2grid tool
|
115
|
## write the gridded file and save the log including the pid of the parent process
|
116
|
# log=system(paste("( /nobackupp1/awilso10/software/heg/bin/swtif -p ",tempdir(),"/",basename(file),"_MODparms.txt -d ; echo $$)",sep=""),intern=T)
|
117
|
log=system(paste("( /nobackupp1/awilso10/software/heg/bin/swtif -p ",tempdir(),"/",basename(file),"_MODparms.txt -d ; echo $$)",sep=""),intern=T,ignore.stderr=T)
|
118
|
log=system(paste("(sudo MRTDATADIR=/usr/local/heg/data PGSHOME=/usr/local/heg/TOOLKIT_MTD PWD=/home/adamw /usr/local/heg/bin/swtif -p ",tempdir(),"/",basename(file),"_MODparms.txt -d )",sep=""))
|
119
|
|
120
|
## clean up temporary files in working directory
|
121
|
file.remove(list.files(pattern=
|
122
|
paste("filetable.temp_",
|
123
|
as.numeric(log[length(log)]):(as.numeric(log[length(log)])+3),sep="",collapse="|"))) #Look for files with PID within 3 of parent process
|
124
|
if(verbose) print(log)
|
125
|
print(paste("Finished ", file))
|
126
|
}
|
127
|
|
128
|
|
129
|
##############################################################
|
130
|
### Import to GRASS for processing
|
131
|
|
132
|
## function to convert binary to decimal to assist in identifying correct values
|
133
|
## this is helpful when defining QA handling below, but isn't used in processing
|
134
|
## b2d=function(x) sum(x * 2^(rev(seq_along(x)) - 1)) #http://tolstoy.newcastle.edu.au/R/e2/help/07/02/10596.html
|
135
|
## for example:
|
136
|
## b2d(c(T,T))
|
137
|
|
138
|
## set Grass to overwrite
|
139
|
Sys.setenv(GRASS_OVERWRITE=1)
|
140
|
Sys.setenv(DEBUG=1)
|
141
|
Sys.setenv(GRASS_GUI="txt")
|
142
|
|
143
|
### Function to extract various SDSs from a single gridded HDF file and use QA data to throw out 'bad' observations
|
144
|
loadcloud<-function(date,fs,ncfile){
|
145
|
tf=paste(tempdir(),"/grass", Sys.getpid(),"/", sep="")
|
146
|
if(!file.exists(tf))dir.create(tf)
|
147
|
|
148
|
print(paste("Set up temporary grass session in",tf))
|
149
|
|
150
|
## set up temporary grass instance for this PID
|
151
|
initGRASS(gisBase="/u/armichae/pr/grass-6.4.2/",gisDbase=tf,SG=td,override=T,location="mod06",mapset="PERMANENT",home=tf,pid=Sys.getpid())
|
152
|
initGRASS(gisBase="/usr/lib/grass64/",SG=td,gisDbase=tf,override=T,location="mod06",mapset="PERMANENT",home=tf,pid=Sys.getpid())
|
153
|
system(paste("g.proj -c proj4=\"",projection(td),"\"",sep=""),ignore.stdout=T,ignore.stderr=T)
|
154
|
|
155
|
## Define region by importing one MOD11A1 raster.
|
156
|
print("Import one MOD11A1 raster to define grid")
|
157
|
execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",gridfile,"\":MODIS_Grid_Daily_1km_LST:Night_view_angl",sep=""), output="modisgrid",flags=c("quiet","overwrite","o"))
|
158
|
execGRASS("r.in.gdal",input=paste("NETCDF:\"",gridfile,"\":CER",sep=""), output="modisgrid",flags=c("o"))
|
159
|
system("g.region rast=modisgrid.1 save=roi --overwrite",ignore.stdout=T,ignore.stderr=T)
|
160
|
## Identify which files to process
|
161
|
tfs=fs$file[fs$dateid==date]
|
162
|
## drop swaths that did not produce an output file (typically due to not overlapping the ROI)
|
163
|
tfs=tfs[tfs%in%list.files(tempdir(),pattern="hdf$")]
|
164
|
nfs=length(tfs)
|
165
|
|
166
|
## loop through scenes and process QA flags
|
167
|
for(i in 1:nfs){
|
168
|
file=paste(tempdir(),"/",tfs[i],sep="")
|
169
|
## Cloud Mask
|
170
|
execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",file,"\":mod06:Cloud_Mask_1km_0",sep=""),
|
171
|
output=paste("CM1_",i,sep=""),flags=c("overwrite","o")) ; print("")
|
172
|
system(paste("r.mapcalc <<EOF
|
173
|
CM_determined_",i," = ((CM1_",i," / 2^0) % 2)
|
174
|
CM_state_",i," = ((CM1_",i," / 2^1) % 2^2)
|
175
|
EOF",sep=""))
|
176
|
|
177
|
## extract cloudy and 'confidently clear' pixels
|
178
|
system(paste("r.mapcalc <<EOF
|
179
|
CM_cloud_",i," = (((CM1_",i," / 2^0) % 2) == 1) && (((CM1_",i," / 2^1) % 2^2) == 0 )
|
180
|
CM_clear_",i," = ((CM1_",i," / 2^0) % 2) == 1 && ( ((CM1_",i," / 2^1) % 2^2) > 2 )
|
181
|
EOF",sep=""))
|
182
|
## QA
|
183
|
execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",file,"\":mod06:Quality_Assurance_1km_0",sep=""),
|
184
|
output=paste("QA_",i,sep=""),flags=c("overwrite","o")) ; print("")
|
185
|
## QA_CER
|
186
|
system(paste("r.mapcalc <<EOF
|
187
|
QA_COT_",i,"= ((QA_",i," / 2^0) % 2^1 )==1
|
188
|
QA_COT2_",i,"= ((QA_",i," / 2^1) % 2^2 )>=2
|
189
|
QA_COT3_",i,"= ((QA_",i," / 2^3) % 2^2 )==0
|
190
|
QA_CER_",i,"= ((QA_",i," / 2^5) % 2^1 )==1
|
191
|
QA_CER2_",i,"= ((QA_",i," / 2^6) % 2^2 )>=2
|
192
|
EOF",sep=""))
|
193
|
# QA_CWP_",i,"= ((QA_",i," / 2^8) % 2^1 )==1
|
194
|
# QA_CWP2_",i,"= ((QA_",i," / 2^9) % 2^2 )==3
|
195
|
|
196
|
## Optical Thickness
|
197
|
execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",file,"\":mod06:Cloud_Optical_Thickness",sep=""),
|
198
|
output=paste("COT_",i,sep=""),
|
199
|
title="cloud_effective_radius",
|
200
|
flags=c("overwrite","o")) ; print("")
|
201
|
execGRASS("r.null",map=paste("COT_",i,sep=""),setnull="-9999")
|
202
|
## keep only positive COT values where quality is 'useful' and 'very good' & scale to real units
|
203
|
system(paste("r.mapcalc \"COT_",i,"=if(QA_COT_",i,"&&QA_COT2_",i,"&&QA_COT3_",i,"&&COT_",i,">=0,COT_",i,"*0.009999999776482582,null())\"",sep=""))
|
204
|
## set COT to 0 in clear-sky pixels
|
205
|
system(paste("r.mapcalc \"COT2_",i,"=if(CM_clear_",i,"==0,COT_",i,",0)\"",sep=""))
|
206
|
|
207
|
## Effective radius ##
|
208
|
execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",file,"\":mod06:Cloud_Effective_Radius",sep=""),
|
209
|
output=paste("CER_",i,sep=""),
|
210
|
title="cloud_effective_radius",
|
211
|
flags=c("overwrite","o")) ; print("")
|
212
|
execGRASS("r.null",map=paste("CER_",i,sep=""),setnull="-9999")
|
213
|
## keep only positive CER values where quality is 'useful' and 'very good' & scale to real units
|
214
|
system(paste("r.mapcalc \"CER_",i,"=if(QA_CER_",i,"&&QA_CER2_",i,"&&CER_",i,">=0,CER_",i,"*0.009999999776482582,null())\"",sep=""))
|
215
|
## set CER to 0 in clear-sky pixels
|
216
|
system(paste("r.mapcalc \"CER2_",i,"=if(CM_clear_",i,"==0,CER_",i,",0)\"",sep=""))
|
217
|
|
218
|
## Cloud Water Path
|
219
|
# execGRASS("r.in.gdal",input=paste("HDF4_EOS:EOS_GRID:\"",file,"\":mod06:Cloud_Water_Path",sep=""),
|
220
|
# output=paste("CWP_",i,sep=""),title="cloud_water_path",
|
221
|
# flags=c("overwrite","o")) ; print("")
|
222
|
# execGRASS("r.null",map=paste("CWP_",i,sep=""),setnull="-9999")
|
223
|
## keep only positive CER values where quality is 'useful' and 'very good' & scale to real units
|
224
|
# system(paste("r.mapcalc \"CWP_",i,"=if(QA_CWP_",i,"&&QA_CWP2_",i,"&&CWP_",i,">=0,CWP_",i,"*0.009999999776482582,null())\"",sep=""))
|
225
|
## set CER to 0 in clear-sky pixels
|
226
|
# system(paste("r.mapcalc \"CWP2_",i,"=if(CM_clear_",i,"==0,CWP_",i,",0)\"",sep=""))
|
227
|
|
228
|
} #end loop through sub daily files
|
229
|
|
230
|
#### Now generate daily averages (or maximum in case of cloud flag)
|
231
|
|
232
|
system(paste("r.mapcalc <<EOF
|
233
|
COT_denom=",paste("!isnull(COT2_",1:nfs,")",sep="",collapse="+"),"
|
234
|
COT_numer=",paste("if(isnull(COT2_",1:nfs,"),0,COT2_",1:nfs,")",sep="",collapse="+"),"
|
235
|
COT_daily=int((COT_numer/COT_denom)*100)
|
236
|
CER_denom=",paste("!isnull(CER2_",1:nfs,")",sep="",collapse="+"),"
|
237
|
CER_numer=",paste("if(isnull(CER2_",1:nfs,"),0,CER2_",1:nfs,")",sep="",collapse="+"),"
|
238
|
CER_daily=int(100*(CER_numer/CER_denom))
|
239
|
CLD_daily=int((max(",paste("if(isnull(CM_cloud_",1:nfs,"),0,CM_cloud_",1:nfs,")",sep="",collapse=","),"))*100)
|
240
|
EOF",sep=""))
|
241
|
|
242
|
|
243
|
### Write the files to a netcdf file
|
244
|
## create image group to facilitate export as multiband netcdf
|
245
|
execGRASS("i.group",group="mod06",input=c("CER_daily","COT_daily","CLD_daily")) ; print("")
|
246
|
|
247
|
if(file.exists(ncfile)) file.remove(ncfile) #if it exists already, delete it
|
248
|
execGRASS("r.out.gdal",input="mod06",output=ncfile,type="Int16",nodata=-32768,flags=c("quiet"),
|
249
|
# createopt=c("FORMAT=NC4","ZLEVEL=5","COMPRESS=DEFLATE","WRITE_GDAL_TAGS=YES","WRITE_LONLAT=NO"),format="netCDF") #for compressed netcdf
|
250
|
createopt=c("FORMAT=NC","WRITE_GDAL_TAGS=YES","WRITE_LONLAT=NO"),format="netCDF")
|
251
|
|
252
|
ncopath="/nasa/sles11/nco/4.0.8/gcc/mpt/bin/"
|
253
|
system(paste(ncopath,"ncecat -O -u time ",ncfile," ",ncfile,sep=""))
|
254
|
## create temporary nc file with time information to append to MOD06 data
|
255
|
cat(paste("
|
256
|
netcdf time {
|
257
|
dimensions:
|
258
|
time = 1 ;
|
259
|
variables:
|
260
|
int time(time) ;
|
261
|
time:units = \"days since 2000-01-01 00:00:00\" ;
|
262
|
time:calendar = \"gregorian\";
|
263
|
time:long_name = \"time of observation\";
|
264
|
data:
|
265
|
time=",as.integer(fs$date[fs$dateid==date][1]-as.Date("2000-01-01")),";
|
266
|
}"),file=paste(tempdir(),"/time.cdl",sep=""))
|
267
|
system(paste("ncgen -o ",tempdir(),"/time.nc ",tempdir(),"/time.cdl",sep=""))
|
268
|
system(paste(ncopath,"ncks -A ",tempdir(),"/time.nc ",ncfile,sep=""))
|
269
|
## add other attributes
|
270
|
system(paste(ncopath,"ncrename -v Band1,CER -v Band2,COT -v Band3,CLD ",ncfile,sep=""))
|
271
|
system(paste(ncopath,"ncatted -a scale_factor,CER,o,d,0.01 -a units,CER,o,c,\"micron\" -a missing_value,CER,o,d,-32768 -a long_name,CER,o,c,\"Cloud Particle Effective Radius\" ",ncfile,sep=""))
|
272
|
system(paste(ncopath,"ncatted -a scale_factor,COT,o,d,0.01 -a units,COT,o,c,\"none\" -a missing_value,COT,o,d,-32768 -a long_name,COT,o,c,\"Cloud Optical Thickness\" ",ncfile,sep=""))
|
273
|
system(paste(ncopath,"ncatted -a scale_factor,CLD,o,d,0.01 -a units,CLD,o,c,\"none\" -a missing_value,CLD,o,d,-32768 -a long_name,CLD,o,c,\"Cloud Mask\" ",ncfile,sep=""))
|
274
|
|
275
|
|
276
|
### delete the temporary files
|
277
|
unlink_.gislock()
|
278
|
system("clean_temp")
|
279
|
system(paste("rm -frR ",tf,sep=""))
|
280
|
}
|
281
|
|
282
|
|
283
|
###########################################
|
284
|
### Define a wrapper function that will call the two functions above (gridding and QA-handling) for a single tile-date
|
285
|
|
286
|
mod06<-function(date,tile){
|
287
|
print(paste("Processing date ",date," for tile",tile))
|
288
|
#####################################################
|
289
|
## Run the gridding procedure
|
290
|
tile_bb=tb[tb$tile==tile,] ## identify tile of interest
|
291
|
|
292
|
# if(venezuela) tile_bb$lat_max=11.0780999176 #increase northern boundary for KT
|
293
|
|
294
|
lapply(fs$path[fs$dateid==date],swath2grid,vars=vars,upleft=paste(tile_bb$lat_max,tile_bb$lon_min),lowright=paste(tile_bb$lat_min,tile_bb$lon_max))
|
295
|
|
296
|
## confirm at least one file for this date is present
|
297
|
outfiles=paste(tempdir(),"/",basename(fs$path[fs$dateid==date]),sep="")
|
298
|
if(!any(file.exists(outfiles))) {
|
299
|
print(paste("######################################## No gridded files for region exist for tile",tile," on date",date))
|
300
|
q("no",status=0)
|
301
|
}
|
302
|
|
303
|
#####################################################
|
304
|
## Process the gridded files
|
305
|
|
306
|
## run themod06 processing for this date
|
307
|
ncfile=paste(outdir,"/MOD06_",tile,"_",date,".nc",sep="") #this is the 'final' daily output file
|
308
|
loadcloud(date,fs=fs,ncfile=ncfile)
|
309
|
|
310
|
## Confirm that the file has the correct attributes, otherwise delete it
|
311
|
ntime=as.numeric(system(paste("cdo -s ntime ",ncfile),intern=T))
|
312
|
npar=as.numeric(system(paste("cdo -s npar ",ncfile),intern=T))
|
313
|
if(!ntime==1&npar>0) {
|
314
|
print(paste("File for tile ",tile," and date ",date," was not outputted correctly, deleting... "))
|
315
|
file.remove(ncfile)
|
316
|
}
|
317
|
|
318
|
## push file to lou and then delete it from pfe
|
319
|
# system(paste("scp ",ncfile," ",ldir,sep="") )
|
320
|
# file.remove(ncfile)
|
321
|
|
322
|
## print out some info
|
323
|
print(paste(" ################################################################### Finished ",date,"
|
324
|
################################################################"))
|
325
|
}
|
326
|
|
327
|
## test it
|
328
|
##date=notdone[1]
|
329
|
mod06(date,tile)
|
330
|
|
331
|
## run it for all dates - Use this if running on a workstation/server (otherwise use qsub)
|
332
|
#mclapply(notdone,mod06,tile,mc.cores=ncores) # use ncores/2 because system() commands can add second process for each spawned R
|
333
|
#foreach(i=notdone[1:3],.packages=(.packages())) %dopar% mod06(i,tile)
|
334
|
#foreach(i=1:20) %dopar% print(i)
|
335
|
|
336
|
## delete old files
|
337
|
#system("cleartemp")
|
338
|
|
339
|
q("no",status=0)
|