Project

General

Profile

Download (11.1 KB) Statistics
| Branch: | Revision:
1 3682f238 Adam M. Wilson @ pfe
#### Script to facilitate processing of MOD06 data
2
3
setwd("/nobackupp1/awilso10/mod06")
4 b6e392da Adam M. Wilson @ pfe
5 0bddce86 Adam M. Wilson @ pfe
library(rgdal)
6
library(raster)
7 3682f238 Adam M. Wilson @ pfe
8 9d52d7e0 Adam M. Wilson @ pfe
## get MODLAND tile information
9
tb=read.table("http://landweb.nascom.nasa.gov/developers/sn_tiles/sn_bound_10deg.txt",skip=6,nrows=648,header=T)
10
tb$tile=paste("h",sprintf("%02d",tb$ih),"v",sprintf("%02d",tb$iv),sep="")
11 807fa48c Adam M. Wilson @ pfe
save(tb,file="modlandTiles.Rdata")
12 3682f238 Adam M. Wilson @ pfe
13 b6e392da Adam M. Wilson @ pfe
## delete temporary log file that can grow to GB
14
system("rm /nobackupp1/awilso10/software/heg/TOOLKIT_MTD/runtime/LogStatus")
15
16
tile="h11v08"  # Venezuela
17
tile="h11v07"  # Venezuela coast
18
tile="h09v04"  # Oregon
19
20 2c238837 Adam M. Wilson @ pfe
outdir="2_daily"  #directory for separate daily files
21 b6e392da Adam M. Wilson @ pfe
outdir=paste("daily/",tile,"/",sep="")  #directory for separate daily files
22
       if(!file.exists(outdir)) dir.create(outdir)
23
outdir2="summary" #directory for combined daily files and summarized files
24
       if(!file.exists(outdir2)) dir.create(outdir2)
25 2c238837 Adam M. Wilson @ pfe
26
  ## load a MOD11A1 file to define grid
27
gridfile=list.files("/nobackupp4/datapool/modis/MOD11A1.005/2006.01.27/",pattern=paste(tile,".*hdf$",sep=""),full=T)[1]
28
  td=readGDAL(paste("HDF4_EOS:EOS_GRID:\"",gridfile,"\":MODIS_Grid_Daily_1km_LST:Night_view_angl",sep=""))
29
  projection(td)="+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs +datum=WGS84 +ellps=WGS84 "
30
31
32
### get list of files to process
33
datadir="/nobackupp4/datapool/modis/MOD06_L2.005/"
34 b6e392da Adam M. Wilson @ pfe
datadir="/nobackupp1/awilso10/mod06/data"   #for data downloaded from 
35 2c238837 Adam M. Wilson @ pfe
36
fs=data.frame(path=list.files(datadir,full=T,recursive=T,pattern="hdf"),stringsAsFactors=F)
37
fs$file=basename(fs$path)
38
fs$date=as.Date(substr(fs$file,11,17),"%Y%j")
39
fs$month=format(fs$date,"%m")
40
fs$year=format(fs$date,"%Y")
41
fs$time=substr(fs$file,19,22)
42
fs$datetime=as.POSIXct(strptime(paste(substr(fs$file,11,17),substr(fs$file,19,22)), '%Y%j %H%M'))
43
fs$dateid=format(fs$date,"%Y%m%d")
44
fs$path=as.character(fs$path)
45
fs$file=as.character(fs$file)
46
47
## get all unique dates
48
alldates=unique(fs$dateid)
49
50
51
#### Generate submission file
52
## identify which have been completed
53 b6e392da Adam M. Wilson @ pfe
#fdone=system(paste("ssh -q lou2 \"ls ",pdir," | grep \"nc$\"\"",sep=""),intern=T)
54
fdone=list.files(outdir,pattern="nc$")
55
done=alldates%in%substr(fdone,14,21)
56 2c238837 Adam M. Wilson @ pfe
57 b6e392da Adam M. Wilson @ pfe
if(exists("fdly")){  #update using table from below
58
  done[alldates%in%fdly$dateid[is.na(fdly$npar)]]=F
59
}
60 07f76f37 Adam M. Wilson @ pfe
61 b6e392da Adam M. Wilson @ pfe
table(done)
62
notdone=alldates[!done]  #these are the dates that still need to be processed
63 07f76f37 Adam M. Wilson @ pfe
64 2c238837 Adam M. Wilson @ pfe
script="/u/awilso10/environmental-layers/climate/procedures/MOD06_L2_process.r"
65 b6e392da Adam M. Wilson @ pfe
66
write.table(paste("--verbose ",script," --date ",notdone," --verbose T --tile \"",tile,"\"",sep=""),file=paste(tile,"_notdone.txt",sep=""),row.names=F,col.names=F,quote=F)
67 2c238837 Adam M. Wilson @ pfe
68
save(fs,alldates,gridfile,td,file="allfiles.Rdata")
69
70 b6e392da Adam M. Wilson @ pfe
### qsub script
71 07f76f37 Adam M. Wilson @ pfe
cat(paste("
72
#PBS -S /bin/bash
73 b6e392da Adam M. Wilson @ pfe
#PBS -l select=40:ncpus=8:mpiprocs=8
74 07f76f37 Adam M. Wilson @ pfe
##PBS -l select=2:ncpus=4:mpiprocs=4
75 b6e392da Adam M. Wilson @ pfe
#PBS -l walltime=2:00:00
76 07f76f37 Adam M. Wilson @ pfe
#PBS -j n
77
#PBS -m be
78
#PBS -N mod06
79
#PBS -q devel
80
#PBS -V
81
82 b6e392da Adam M. Wilson @ pfe
CORES=320
83 07f76f37 Adam M. Wilson @ pfe
HDIR=/u/armichae/pr/
84
  source $HDIR/etc/environ.sh
85
  source /u/awilso10/.bashrc
86
IDIR=/nobackupp1/awilso10/mod06/
87
##WORKLIST=$HDIR/var/run/pxrRgrs/work.txt
88 b6e392da Adam M. Wilson @ pfe
WORKLIST=$IDIR/",tile,"_notdone.txt
89 07f76f37 Adam M. Wilson @ pfe
EXE=Rscript
90
LOGSTDOUT=$IDIR/log/log.stdout
91
LOGSTDERR=$IDIR/log/log.stderr
92
mpiexec -np $CORES pxargs -a $WORKLIST -p $EXE -v -v -v --work-analyze 1> $LOGSTDOUT 2> $LOGSTDERR
93 b6e392da Adam M. Wilson @ pfe
",sep=""),file=paste(tile,"_mod06_qsub",sep=""))
94 827b4b87 Adam M. Wilson @ pfe
95 9d52d7e0 Adam M. Wilson @ pfe
96
### Check the file
97 b6e392da Adam M. Wilson @ pfe
system(paste("cat ",tile,"_mod06_qsub",sep=""))
98 807fa48c Adam M. Wilson @ pfe
#system("cat ~/environmental-layers/climate/procedures/MOD06_L2_process.r")
99
100 92fd8a10 Adam M. Wilson @ pfe
## check queue status
101
system("/u/scicon/tools/bin/node_stats.sh")
102 0bddce86 Adam M. Wilson @ pfe
system("/u/scicon/tools/bin/qtop.pl 492352")
103 92fd8a10 Adam M. Wilson @ pfe
104 807fa48c Adam M. Wilson @ pfe
## Submit it (and keep the pid)!
105 b6e392da Adam M. Wilson @ pfe
system(paste("qsub ",tile,"_mod06_qsub",sep=""))
106 9d52d7e0 Adam M. Wilson @ pfe
107 807fa48c Adam M. Wilson @ pfe
## work in interactive mode
108 827b4b87 Adam M. Wilson @ pfe
# system("qsub -I -l walltime=2:00:00 -lselect=2:ncpus=16:model=san -q devel")
109
# mpirun -np 1 -r ssh R --no-save
110 9d52d7e0 Adam M. Wilson @ pfe
111
## check progress
112
system("qstat -u awilso10")
113 07f76f37 Adam M. Wilson @ pfe
system(paste("/u/scicon/tools/bin/qps ",568835))
114 807fa48c Adam M. Wilson @ pfe
system(paste("qstat -t -x",pid))
115 9d52d7e0 Adam M. Wilson @ pfe
116
117 2c238837 Adam M. Wilson @ pfe
####################################
118
119 b6e392da Adam M. Wilson @ pfe
  ## move to permanent storage on lou?
120
#  system(paste("scp -r ",ncfile," ",pdir,sep=""))
121
file.remove(
122
list.files(pattern="filetable[.]temp|GetAttrtemp|core[.]|MCFWrite[.]temp")
123
)
124 2c238837 Adam M. Wilson @ pfe
125
################################################################################
126
## now generate the climatologies
127
fdly=data.frame(
128
  path=list.files(outdir,pattern="nc$",full=T),
129
  file=list.files(outdir,pattern="nc$"))
130 b6e392da Adam M. Wilson @ pfe
  fdly$dateid=substr(fdly$file,14,21)
131
  fdly$date=as.Date(substr(fdly$file,14,21),"%Y%m%d")
132 07f76f37 Adam M. Wilson @ pfe
  fdly$month=format(fdly$date,"%m")
133
  fdly$year=format(fdly$date,"%Y")
134 b6e392da Adam M. Wilson @ pfe
nrow(fdly)
135 2c238837 Adam M. Wilson @ pfe
136
## check validity (via npar and ntime) of nc files
137
for(i in 1:nrow(fdly)){
138 b6e392da Adam M. Wilson @ pfe
  fdly$ntime[i]<-as.numeric(system(paste("cdo -s ntime ",fdly$path[i]),intern=T))
139 07f76f37 Adam M. Wilson @ pfe
  fdly$npar[i]<-as.numeric(system(paste("cdo -s npar ",fdly$path[i]),intern=T))
140 b6e392da Adam M. Wilson @ pfe
  fdly$fyear[i]<-as.numeric(system(paste("cdo -s showyear ",fdly$path[i]),intern=T))
141
  fdly$fmonth[i]<-as.numeric(system(paste("cdo -s showmon ",fdly$path[i]),intern=T))
142
  fdly$fvar[i]<-system(paste("cdo -s showvar ",fdly$path[i]),intern=T)
143
  print(paste(i," out of ",nrow(fdly)," for year ",  fdly$fyear[i]))
144
}
145
146
### table of problematic files
147
table(is.na(fdly$npar))
148
table(fdly$fmonth)
149
table(fdly$fvar)
150
151
fdly[is.na(fdly$npar),]
152
153
## delete files that fail check?
154
delete=T
155
if(delete) {
156
  drop=is.na(fdly$npar)|fdly$fvar!=" CER COT CLD"
157
  file.remove(as.character(fdly$path[drop]))
158
  fdly=fdly[!drop,]
159 2c238837 Adam M. Wilson @ pfe
}
160
161
## Combine all days within years into a single file (can't mergetime all days at once because this opens too many files)
162 b6e392da Adam M. Wilson @ pfe
ncopath="/nasa/sles11/nco/4.0.8/gcc/mpt/bin/"
163
164
library(multicore)
165
166
## create temporary directory to put intermediate files (will be deleted when R quits)
167 2c238837 Adam M. Wilson @ pfe
tsdir=paste(tempdir(),"/summary",sep="")
168
dir.create(tsdir)
169
170 b6e392da Adam M. Wilson @ pfe
## create a file of daily scenes for each year
171
#mclapply(unique(fdly$year),function(y){
172
#  system(paste("cdo -O mergetime ",paste(fdly$path[!is.na(fdly$npar)&fdly$year==y],collapse=" ")," ",tsdir,"/MOD06_",tile,"_",y,"_daily.nc",sep=""))
173
#  print(paste("Finished merging daily files for year",y))
174
#},mc.cores=5)
175
176
177
## merge all daily files to create a single file with all dates
178
  system(paste(ncopath,"ncrcat -O ",outdir,"/*nc ",outdir2,"/MOD06_",tile,"_daily.nc",sep=""))
179
180
##############################
181
## Combine the year-by-year files into a single daily file in the summary directory (for archiving)
182
#system(paste("cdo -O mergetime ",paste(list.files(tsdir,full=T,pattern="daily[.]nc$"),collapse=" ")," ",outdir2,"/MOD06_",tile,"_daily.nc",sep=""))
183
system(paste(ncopath,"ncatted ",
184
" -a units,time,o,c,\"days since 2000-1-1 0:0:0\" ",
185
" -a title,global,o,c,\"MODIS Cloud Product (MOD06) Daily Timeseries\" ",
186
" -a institution,global,o,c,\"Yale University\" ",
187
" -a source,global,o,c,\"MODIS Cloud Product (MOD06)\" ",
188
" -a comment,global,o,c,\"Compiled by Adam M. Wilson (adam.wilson@yale.edu)\" ",
189
outdir2,"/MOD06_",tile,"_daily.nc",sep=""))
190
191
#system(paste("cdo -O monmean ",outdir2,"/MOD06_",tile,"_daily.nc ",tsdir,"/MOD06_",tile,"_monmean.nc",sep=""))
192
193
#############################
194
##  Generate the Climatologies
195
myear=as.integer(max(fdly$year))  #this year will be used in all dates of monthly climatologies (and day will = 15)
196
197
## Monthly means
198
system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -ymonmean ",outdir2,"/MOD06_",tile,"_daily.nc ",tsdir,"/MOD06_",tile,"_ymonmean.nc",sep=""))
199
200
## Monthly standard deviation
201
system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -ymonstd ",outdir2,"/MOD06_",tile,"_daily.nc ",tsdir,"/MOD06_",tile,"_ymonstd.nc",sep=""))
202
system(paste(ncopath,"ncrename -v CER,CER_sd -v CLD,CLD_sd -v COT,COT_sd ",tsdir,"/MOD06_",tile,"_ymonstd.nc",sep=""))
203
system(paste(ncopath,"ncatted ",
204
" -a long_name,CER_sd,o,c,\"Cloud Particle Effective Radius (standard deviation of daily observations)\" ",
205
" -a long_name,CLD_sd,o,c,\"Cloud Mask (standard deviation of daily observations)\" ",
206
" -a long_name,COT_sd,o,c,\"Cloud Optical Thickness (standard deviation of daily observations)\" ",
207
tsdir,"/MOD06_",tile,"_ymonstd.nc",sep=""))
208
209
## cer > 20
210
system(paste("cdo -O  sorttimestamp -setyear,",myear," -setday,15 -ymonmean -gtc,20 -selvar,CER ",outdir2,"/MOD06_",tile,"_daily.nc ",tsdir,"/MOD06_",tile,"_ymoncer20.nc",sep=""))
211
system(paste(ncopath,"ncrename -v CER,CER20 ",tsdir,"/MOD06_",tile,"_ymoncer20.nc",sep=""))
212
system(paste(ncopath,"ncatted ",
213
" -a long_name,CER20,o,c,\"Proportion of Days with Cloud Particle Effective Radius > 20um\" ",
214
" -a units,CER20,o,c,\"Proportion\" ",
215
tsdir,"/MOD06_",tile,"_ymoncer20.nc",sep=""))
216
217
## number of observations
218
system(paste("cdo -O sorttimestamp  -setyear,",myear," -setday,15 -nint -mulc,100 -ymonmean -eqc,9999 -setmisstoc,9999   -selvar,CER,CLD ",outdir2,"/MOD06_",tile,"_daily.nc ",tsdir,"/MOD06_",tile,"_ymonmiss.nc",sep=""))
219
system(paste(ncopath,"ncrename -v CER,CER_pmiss -v CLD,CLD_pmiss ",tsdir,"/MOD06_",tile,"_ymonmiss.nc",sep=""))
220
system(paste(ncopath,"ncatted ",
221
" -a long_name,CER_pmiss,o,c,\"Proportion of Days with missing data for CER\" ",
222
" -a long_name,CLD_pmiss,o,c,\"Proportion of Days with missing data for CLD\" ",
223
" -a scale_factor,CER_pmiss,o,d,0.01 ",
224
" -a units,CER_pmiss,o,c,\"Proportion\" ",
225
" -a scale_factor,CLD_pmiss,o,d,0.01 ",
226
" -a units,CLD_pmiss,o,c,\"Proportion\" ",
227
tsdir,"/MOD06_",tile,"_ymonmiss.nc",sep=""))
228
229
## append variables to a single file
230
system(paste(ncopath,"ncks -O ",tsdir,"/MOD06_",tile,"_ymonmean.nc  ",tsdir,"/MOD06_",tile,"_ymon.nc",sep=""))
231
system(paste(ncopath,"ncks -A ",tsdir,"/MOD06_",tile,"_ymonstd.nc  ",tsdir,"/MOD06_",tile,"_ymon.nc",sep=""))
232
system(paste(ncopath,"ncks -A ",tsdir,"/MOD06_",tile,"_ymoncer20.nc  ",tsdir,"/MOD06_",tile,"_ymon.nc",sep=""))
233
system(paste(ncopath,"ncks -A ",tsdir,"/MOD06_",tile,"_ymonmiss.nc  ",tsdir,"/MOD06_",tile,"_ymon.nc",sep=""))
234
235
## append sinusoidal grid from one of input files as CDO doesn't transfer all attributes
236
system(paste(ncopath,"ncks -A -v sinusoidal ",list.files(tsdir,full=T,pattern="daily[.]nc$")[1],"  ",tsdir,"/MOD06_",tile,"_ymon.nc",sep=""))
237
238
## invert latitude so it plays nicely with gdal
239
system(paste(ncopath,"ncpdq -O -a -y ",tsdir,"/MOD06_",tile,"_ymon.nc ",outdir2,"/MOD06_",tile,".nc",sep=""))
240
241
## update attributes
242
system(paste(ncopath,"ncatted ",
243
#" -a standard_parallel,sinusoidal,o,c,\"0\" ",
244
#" -a longitude_of_central_meridian,sinusoidal,o,c,\"0\" ",
245
#" -a latitude_of_central_meridian,sinusoidal,o,c,\"0\" ",
246
" -a units,time,o,c,\"days since 2000-1-1 0:0:0\" ",
247
" -a title,global,o,c,\"MODIS Cloud Product (MOD06) Climatology\" ",
248
" -a institution,global,o,c,\"Yale University\" ",
249
" -a source,global,o,c,\"MODIS Cloud Product (MOD06)\" ",
250
" -a comment,global,o,c,\"Compiled by Adam M. Wilson (adam.wilson@yale.edu)\" ",
251
outdir2,"/MOD06_",tile,".nc",sep=""))
252
253 2c238837 Adam M. Wilson @ pfe
254
print("Finished!   %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
255
## quit R
256 07f76f37 Adam M. Wilson @ pfe
#q("no")
257 2c238837 Adam M. Wilson @ pfe
 
258
259
#################################################################
260 9d52d7e0 Adam M. Wilson @ pfe
261 807fa48c Adam M. Wilson @ pfe
### copy the files back to Yale
262 b6e392da Adam M. Wilson @ pfe
system(paste("scp ",outdir2,"/MOD06_",tile,".nc adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod06/summary",sep=""))
263 3959e686 Adam M. Wilson @ pfe
264
system("scp  /tmp/Rtmp6I6tFn/MOD06_L2.A2000061.1615.051.2010273184629.hdf adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/Venezuela")
265
system("scp 2_daily/MOD06_20000410.nc adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/Venezuela")
266 cf724805 Adam M. Wilson @ pfe
267
268 2c238837 Adam M. Wilson @ pfe
269
270