Revision f9d40a14
Added by Adam Wilson almost 12 years ago
| climate/procedures/Pleiades.R | ||
|---|---|---|
| 14 | 14 |
system("rm /nobackupp1/awilso10/software/heg/TOOLKIT_MTD/runtime/LogStatus")
|
| 15 | 15 |
|
| 16 | 16 |
tile="h11v08" # Venezuela |
| 17 |
tile="h11v07" # Venezuela coast |
|
| 18 |
tile="h09v04" # Oregon |
|
| 19 |
|
|
| 20 |
outdir="2_daily" #directory for separate daily files |
|
| 21 |
outdir=paste("daily/",tile,"/",sep="") #directory for separate daily files
|
|
| 22 |
if(!file.exists(outdir)) dir.create(outdir) |
|
| 23 |
outdir2="summary" #directory for combined daily files and summarized files |
|
| 24 |
if(!file.exists(outdir2)) dir.create(outdir2) |
|
| 25 |
|
|
| 26 |
## load a MOD11A1 file to define grid |
|
| 27 |
gridfile=list.files("/nobackupp4/datapool/modis/MOD11A1.005/2006.01.27/",pattern=paste(tile,".*hdf$",sep=""),full=T)[1]
|
|
| 28 |
td=readGDAL(paste("HDF4_EOS:EOS_GRID:\"",gridfile,"\":MODIS_Grid_Daily_1km_LST:Night_view_angl",sep=""))
|
|
| 29 |
projection(td)="+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs +datum=WGS84 +ellps=WGS84 " |
|
| 30 |
|
|
| 17 |
#tile="h11v07" # Venezuela coast |
|
| 18 |
#tile="h09v04" # Oregon |
|
| 19 |
tile="h21v09" #Kenya |
|
| 31 | 20 |
|
| 32 | 21 |
### get list of files to process |
| 33 | 22 |
datadir="/nobackupp4/datapool/modis/MOD06_L2.005/" |
| 34 |
datadir="/nobackupp1/awilso10/mod06/data" #for data downloaded from |
|
| 35 |
|
|
| 36 |
fs=data.frame(path=list.files(datadir,full=T,recursive=T,pattern="hdf"),stringsAsFactors=F) |
|
| 37 |
fs$file=basename(fs$path) |
|
| 38 |
fs$date=as.Date(substr(fs$file,11,17),"%Y%j") |
|
| 39 |
fs$month=format(fs$date,"%m") |
|
| 40 |
fs$year=format(fs$date,"%Y") |
|
| 41 |
fs$time=substr(fs$file,19,22) |
|
| 42 |
fs$datetime=as.POSIXct(strptime(paste(substr(fs$file,11,17),substr(fs$file,19,22)), '%Y%j %H%M')) |
|
| 43 |
fs$dateid=format(fs$date,"%Y%m%d") |
|
| 44 |
fs$path=as.character(fs$path) |
|
| 45 |
fs$file=as.character(fs$file) |
|
| 23 |
#datadir="/nobackupp1/awilso10/mod06/data" #for data downloaded from |
|
| 24 |
|
|
| 25 |
outdir=paste("daily/",tile,sep="")
|
|
| 26 |
|
|
| 27 |
##find swaths in region from sqlite database for the specified date/tile |
|
| 28 |
## path to swath database |
|
| 29 |
db="/nobackupp4/pvotava/DB/export/swath_geo.sql.sqlite3.db" |
|
| 30 |
con=dbConnect("SQLite", dbname = db)
|
|
| 31 |
fs=dbGetQuery(con,paste("SELECT * from swath_geo
|
|
| 32 |
WHERE east>=",tile_bb$lon_min," AND |
|
| 33 |
west<=",tile_bb$lon_max," AND |
|
| 34 |
north>=",tile_bb$lat_min," AND |
|
| 35 |
south<=",tile_bb$lat_max) |
|
| 36 |
) |
|
| 37 |
con=dbDisconnect(con) |
|
| 38 |
fs$id=substr(fs$id,7,19) |
|
| 39 |
|
|
| 40 |
### Identify which swaths are available in the datapool |
|
| 41 |
swaths=data.frame(path=list.files(datadir,pattern=paste("hdf$"),recursive=T,full=T),stringsAsFactors=F) #all swaths in data pool
|
|
| 42 |
swaths$id=substr(basename(swaths$path),10,22) |
|
| 43 |
fs$exists=fs$id%in%swaths$id |
|
| 44 |
fs$path=swaths$path[match(fs$id,swaths$id)] |
|
| 45 |
|
|
| 46 |
if(verbose) print(paste("###############",nrow(fs)," swath IDs recieved from database"))
|
|
| 46 | 47 |
|
| 47 |
## get all unique dates |
|
| 48 |
alldates=unique(fs$dateid) |
|
| 49 | 48 |
|
| 49 |
## get all unique dates |
|
| 50 |
fs$dateid=format(as.Date(paste(fs$year,fs$day,sep=""),"%Y%j"),"%Y%m%d") |
|
| 51 |
alldates=unique(fs$dateid[fs$exists]) |
|
| 50 | 52 |
|
| 51 | 53 |
#### Generate submission file |
| 52 | 54 |
## identify which have been completed |
| 53 |
#fdone=system(paste("ssh -q lou2 \"ls ",pdir," | grep \"nc$\"\"",sep=""),intern=T)
|
|
| 54 | 55 |
fdone=list.files(outdir,pattern="nc$") |
| 55 | 56 |
done=alldates%in%substr(fdone,14,21) |
| 56 | 57 |
|
| 57 |
if(exists("fdly")){ #update using table from below
|
|
| 58 |
done[alldates%in%fdly$dateid[is.na(fdly$npar)]]=F |
|
| 59 |
} |
|
| 58 |
### report on what has already been processed |
|
| 59 |
print(paste(table(done)[2]," out of",length(alldates), |
|
| 60 |
"(",round(100*table(done)[2]/length(alldates),1),
|
|
| 61 |
"%) dates for tile",tile, |
|
| 62 |
"have been processed. Breakdown by year of completed days:")) |
|
| 63 |
print(table(substr(alldates[done],1,4))) |
|
| 64 |
|
|
| 65 |
#updatedone=F #update the "done" list using the |
|
| 66 |
#if(updatedone&exists("fdly")){ #update using table from below
|
|
| 67 |
# done[alldates%in%fdly$dateid[fdly$drop]]=F |
|
| 68 |
#} |
|
| 60 | 69 |
|
| 61 |
table(done) |
|
| 62 |
notdone=alldates[!done] #these are the dates that still need to be processed |
|
| 70 |
## Identify which dates still need to be processed |
|
| 71 |
## This vector will be used to tell mpiexec which days to include |
|
| 72 |
notdone=alldates[!done] |
|
| 63 | 73 |
|
| 64 | 74 |
script="/u/awilso10/environmental-layers/climate/procedures/MOD06_L2_process.r" |
| 75 |
climatescript="/u/awilso10/environmental-layers/climate/procedures/MOD06_Climatology.r" |
|
| 65 | 76 |
|
| 77 |
## write the table processed by mpiexec |
|
| 66 | 78 |
write.table(paste("--verbose ",script," --date ",notdone," --verbose T --tile \"",tile,"\"",sep=""),file=paste(tile,"_notdone.txt",sep=""),row.names=F,col.names=F,quote=F)
|
| 67 | 79 |
|
| 68 |
save(fs,alldates,gridfile,td,file="allfiles.Rdata") |
|
| 69 |
|
|
| 70 | 80 |
### qsub script |
| 71 | 81 |
cat(paste("
|
| 72 | 82 |
#PBS -S /bin/bash |
| 73 |
#PBS -l select=40:ncpus=8:mpiprocs=8
|
|
| 83 |
#PBS -l select=50:ncpus=8:mpiprocs=8
|
|
| 74 | 84 |
##PBS -l select=2:ncpus=4:mpiprocs=4 |
| 75 | 85 |
#PBS -l walltime=2:00:00 |
| 76 | 86 |
#PBS -j n |
| ... | ... | |
| 79 | 89 |
#PBS -q devel |
| 80 | 90 |
#PBS -V |
| 81 | 91 |
|
| 82 |
CORES=320
|
|
| 92 |
CORES=400
|
|
| 83 | 93 |
HDIR=/u/armichae/pr/ |
| 84 | 94 |
source $HDIR/etc/environ.sh |
| 85 | 95 |
source /u/awilso10/.bashrc |
| ... | ... | |
| 87 | 97 |
##WORKLIST=$HDIR/var/run/pxrRgrs/work.txt |
| 88 | 98 |
WORKLIST=$IDIR/",tile,"_notdone.txt |
| 89 | 99 |
EXE=Rscript |
| 90 |
LOGSTDOUT=$IDIR/log/log.stdout |
|
| 91 |
LOGSTDERR=$IDIR/log/log.stderr |
|
| 100 |
LOGSTDOUT=$IDIR/log/",tile,"_stdout |
|
| 101 |
LOGSTDERR=$IDIR/log/",tile,"_stderr |
|
| 102 |
### use mpiexec to parallelize across days |
|
| 92 | 103 |
mpiexec -np $CORES pxargs -a $WORKLIST -p $EXE -v -v -v --work-analyze 1> $LOGSTDOUT 2> $LOGSTDERR |
| 104 |
### Now process the climatologies |
|
| 105 |
Rscript --verbose ",climatescript," --verbose T --tile \"",tile,"\" |
|
| 93 | 106 |
",sep=""),file=paste(tile,"_mod06_qsub",sep="")) |
| 94 | 107 |
|
| 95 | 108 |
|
| 96 | 109 |
### Check the file |
| 97 | 110 |
system(paste("cat ",tile,"_mod06_qsub",sep=""))
|
| 98 |
#system("cat ~/environmental-layers/climate/procedures/MOD06_L2_process.r")
|
|
| 99 |
|
|
| 100 |
## check queue status |
|
| 101 |
system("/u/scicon/tools/bin/node_stats.sh")
|
|
| 102 |
system("/u/scicon/tools/bin/qtop.pl 492352")
|
|
| 103 | 111 |
|
| 104 | 112 |
## Submit it (and keep the pid)! |
| 105 | 113 |
system(paste("qsub ",tile,"_mod06_qsub",sep=""))
|
| ... | ... | |
| 110 | 118 |
|
| 111 | 119 |
## check progress |
| 112 | 120 |
system("qstat -u awilso10")
|
| 113 |
system(paste("/u/scicon/tools/bin/qps ",568835))
|
|
| 114 |
system(paste("qstat -t -x",pid))
|
|
| 115 |
|
|
| 116 |
|
|
| 117 |
#################################### |
|
| 118 |
|
|
| 119 |
## move to permanent storage on lou? |
|
| 120 |
# system(paste("scp -r ",ncfile," ",pdir,sep=""))
|
|
| 121 |
file.remove( |
|
| 122 |
list.files(pattern="filetable[.]temp|GetAttrtemp|core[.]|MCFWrite[.]temp") |
|
| 123 |
) |
|
| 124 |
|
|
| 125 |
################################################################################ |
|
| 126 |
## now generate the climatologies |
|
| 127 |
fdly=data.frame( |
|
| 128 |
path=list.files(outdir,pattern="nc$",full=T), |
|
| 129 |
file=list.files(outdir,pattern="nc$")) |
|
| 130 |
fdly$dateid=substr(fdly$file,14,21) |
|
| 131 |
fdly$date=as.Date(substr(fdly$file,14,21),"%Y%m%d") |
|
| 132 |
fdly$month=format(fdly$date,"%m") |
|
| 133 |
fdly$year=format(fdly$date,"%Y") |
|
| 134 |
nrow(fdly) |
|
| 135 |
|
|
| 136 |
## check validity (via npar and ntime) of nc files |
|
| 137 |
for(i in 1:nrow(fdly)){
|
|
| 138 |
fdly$ntime[i]<-as.numeric(system(paste("cdo -s ntime ",fdly$path[i]),intern=T))
|
|
| 139 |
fdly$npar[i]<-as.numeric(system(paste("cdo -s npar ",fdly$path[i]),intern=T))
|
|
| 140 |
fdly$fyear[i]<-as.numeric(system(paste("cdo -s showyear ",fdly$path[i]),intern=T))
|
|
| 141 |
fdly$fmonth[i]<-as.numeric(system(paste("cdo -s showmon ",fdly$path[i]),intern=T))
|
|
| 142 |
fdly$fvar[i]<-system(paste("cdo -s showvar ",fdly$path[i]),intern=T)
|
|
| 143 |
print(paste(i," out of ",nrow(fdly)," for year ", fdly$fyear[i])) |
|
| 144 |
} |
|
| 145 |
|
|
| 146 |
### table of problematic files |
|
| 147 |
table(is.na(fdly$npar)) |
|
| 148 |
table(fdly$fmonth) |
|
| 149 |
table(fdly$fvar) |
|
| 150 |
|
|
| 151 |
fdly[is.na(fdly$npar),] |
|
| 152 |
|
|
| 153 |
## delete files that fail check? |
|
| 154 |
delete=T |
|
| 155 |
if(delete) {
|
|
| 156 |
drop=is.na(fdly$npar)|fdly$fvar!=" CER COT CLD" |
|
| 157 |
file.remove(as.character(fdly$path[drop])) |
|
| 158 |
fdly=fdly[!drop,] |
|
| 159 |
} |
|
| 160 |
|
|
| 161 |
## Combine all days within years into a single file (can't mergetime all days at once because this opens too many files) |
|
| 162 |
ncopath="/nasa/sles11/nco/4.0.8/gcc/mpt/bin/" |
|
| 163 |
|
|
| 164 |
library(multicore) |
|
| 165 |
|
|
| 166 |
## create temporary directory to put intermediate files (will be deleted when R quits) |
|
| 167 |
tsdir=paste(tempdir(),"/summary",sep="") |
|
| 168 |
dir.create(tsdir) |
|
| 169 |
|
|
| 170 |
## create a file of daily scenes for each year |
|
| 171 |
#mclapply(unique(fdly$year),function(y){
|
|
| 172 |
# system(paste("cdo -O mergetime ",paste(fdly$path[!is.na(fdly$npar)&fdly$year==y],collapse=" ")," ",tsdir,"/MOD06_",tile,"_",y,"_daily.nc",sep=""))
|
|
| 173 |
# print(paste("Finished merging daily files for year",y))
|
|
| 174 |
#},mc.cores=5) |
|
| 175 |
|
|
| 176 |
|
|
| 177 |
## merge all daily files to create a single file with all dates |
|
| 178 |
system(paste(ncopath,"ncrcat -O ",outdir,"/*nc ",outdir2,"/MOD06_",tile,"_daily.nc",sep="")) |
|
| 179 |
|
|
| 180 |
############################## |
|
| 181 |
## Combine the year-by-year files into a single daily file in the summary directory (for archiving) |
|
| 182 |
#system(paste("cdo -O mergetime ",paste(list.files(tsdir,full=T,pattern="daily[.]nc$"),collapse=" ")," ",outdir2,"/MOD06_",tile,"_daily.nc",sep=""))
|
|
| 183 |
system(paste(ncopath,"ncatted ", |
|
| 184 |
" -a units,time,o,c,\"days since 2000-1-1 0:0:0\" ", |
|
| 185 |
" -a title,global,o,c,\"MODIS Cloud Product (MOD06) Daily Timeseries\" ", |
|
| 186 |
" -a institution,global,o,c,\"Yale University\" ", |
|
| 187 |
" -a source,global,o,c,\"MODIS Cloud Product (MOD06)\" ", |
|
| 188 |
" -a comment,global,o,c,\"Compiled by Adam M. Wilson (adam.wilson@yale.edu)\" ", |
|
| 189 |
outdir2,"/MOD06_",tile,"_daily.nc",sep="")) |
|
| 190 |
|
|
| 191 |
#system(paste("cdo -O monmean ",outdir2,"/MOD06_",tile,"_daily.nc ",tsdir,"/MOD06_",tile,"_monmean.nc",sep=""))
|
|
| 192 |
|
|
| 193 |
############################# |
|
| 194 |
## Generate the Climatologies |
|
| 195 |
myear=as.integer(max(fdly$year)) #this year will be used in all dates of monthly climatologies (and day will = 15) |
|
| 196 |
|
|
| 197 |
## Monthly means |
|
| 198 |
system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -ymonmean ",outdir2,"/MOD06_",tile,"_daily.nc ",tsdir,"/MOD06_",tile,"_ymonmean.nc",sep=""))
|
|
| 199 |
|
|
| 200 |
## Monthly standard deviation |
|
| 201 |
system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -ymonstd ",outdir2,"/MOD06_",tile,"_daily.nc ",tsdir,"/MOD06_",tile,"_ymonstd.nc",sep=""))
|
|
| 202 |
system(paste(ncopath,"ncrename -v CER,CER_sd -v CLD,CLD_sd -v COT,COT_sd ",tsdir,"/MOD06_",tile,"_ymonstd.nc",sep="")) |
|
| 203 |
system(paste(ncopath,"ncatted ", |
|
| 204 |
" -a long_name,CER_sd,o,c,\"Cloud Particle Effective Radius (standard deviation of daily observations)\" ", |
|
| 205 |
" -a long_name,CLD_sd,o,c,\"Cloud Mask (standard deviation of daily observations)\" ", |
|
| 206 |
" -a long_name,COT_sd,o,c,\"Cloud Optical Thickness (standard deviation of daily observations)\" ", |
|
| 207 |
tsdir,"/MOD06_",tile,"_ymonstd.nc",sep="")) |
|
| 208 |
|
|
| 209 |
## cer > 20 |
|
| 210 |
system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -ymonmean -gtc,20 -selvar,CER ",outdir2,"/MOD06_",tile,"_daily.nc ",tsdir,"/MOD06_",tile,"_ymoncer20.nc",sep=""))
|
|
| 211 |
system(paste(ncopath,"ncrename -v CER,CER20 ",tsdir,"/MOD06_",tile,"_ymoncer20.nc",sep="")) |
|
| 212 |
system(paste(ncopath,"ncatted ", |
|
| 213 |
" -a long_name,CER20,o,c,\"Proportion of Days with Cloud Particle Effective Radius > 20um\" ", |
|
| 214 |
" -a units,CER20,o,c,\"Proportion\" ", |
|
| 215 |
tsdir,"/MOD06_",tile,"_ymoncer20.nc",sep="")) |
|
| 216 |
|
|
| 217 |
## number of observations |
|
| 218 |
system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -nint -mulc,100 -ymonmean -eqc,9999 -setmisstoc,9999 -selvar,CER,CLD ",outdir2,"/MOD06_",tile,"_daily.nc ",tsdir,"/MOD06_",tile,"_ymonmiss.nc",sep=""))
|
|
| 219 |
system(paste(ncopath,"ncrename -v CER,CER_pmiss -v CLD,CLD_pmiss ",tsdir,"/MOD06_",tile,"_ymonmiss.nc",sep="")) |
|
| 220 |
system(paste(ncopath,"ncatted ", |
|
| 221 |
" -a long_name,CER_pmiss,o,c,\"Proportion of Days with missing data for CER\" ", |
|
| 222 |
" -a long_name,CLD_pmiss,o,c,\"Proportion of Days with missing data for CLD\" ", |
|
| 223 |
" -a scale_factor,CER_pmiss,o,d,0.01 ", |
|
| 224 |
" -a units,CER_pmiss,o,c,\"Proportion\" ", |
|
| 225 |
" -a scale_factor,CLD_pmiss,o,d,0.01 ", |
|
| 226 |
" -a units,CLD_pmiss,o,c,\"Proportion\" ", |
|
| 227 |
tsdir,"/MOD06_",tile,"_ymonmiss.nc",sep="")) |
|
| 228 |
|
|
| 229 |
## append variables to a single file |
|
| 230 |
system(paste(ncopath,"ncks -O ",tsdir,"/MOD06_",tile,"_ymonmean.nc ",tsdir,"/MOD06_",tile,"_ymon.nc",sep="")) |
|
| 231 |
system(paste(ncopath,"ncks -A ",tsdir,"/MOD06_",tile,"_ymonstd.nc ",tsdir,"/MOD06_",tile,"_ymon.nc",sep="")) |
|
| 232 |
system(paste(ncopath,"ncks -A ",tsdir,"/MOD06_",tile,"_ymoncer20.nc ",tsdir,"/MOD06_",tile,"_ymon.nc",sep="")) |
|
| 233 |
system(paste(ncopath,"ncks -A ",tsdir,"/MOD06_",tile,"_ymonmiss.nc ",tsdir,"/MOD06_",tile,"_ymon.nc",sep="")) |
|
| 234 |
|
|
| 235 |
## append sinusoidal grid from one of input files as CDO doesn't transfer all attributes |
|
| 236 |
system(paste(ncopath,"ncks -A -v sinusoidal ",list.files(tsdir,full=T,pattern="daily[.]nc$")[1]," ",tsdir,"/MOD06_",tile,"_ymon.nc",sep="")) |
|
| 237 |
|
|
| 238 |
## invert latitude so it plays nicely with gdal |
|
| 239 |
system(paste(ncopath,"ncpdq -O -a -y ",tsdir,"/MOD06_",tile,"_ymon.nc ",outdir2,"/MOD06_",tile,".nc",sep="")) |
|
| 240 |
|
|
| 241 |
## update attributes |
|
| 242 |
system(paste(ncopath,"ncatted ", |
|
| 243 |
#" -a standard_parallel,sinusoidal,o,c,\"0\" ", |
|
| 244 |
#" -a longitude_of_central_meridian,sinusoidal,o,c,\"0\" ", |
|
| 245 |
#" -a latitude_of_central_meridian,sinusoidal,o,c,\"0\" ", |
|
| 246 |
" -a units,time,o,c,\"days since 2000-1-1 0:0:0\" ", |
|
| 247 |
" -a title,global,o,c,\"MODIS Cloud Product (MOD06) Climatology\" ", |
|
| 248 |
" -a institution,global,o,c,\"Yale University\" ", |
|
| 249 |
" -a source,global,o,c,\"MODIS Cloud Product (MOD06)\" ", |
|
| 250 |
" -a comment,global,o,c,\"Compiled by Adam M. Wilson (adam.wilson@yale.edu)\" ", |
|
| 251 |
outdir2,"/MOD06_",tile,".nc",sep="")) |
|
| 252 |
|
|
| 253 |
|
|
| 254 |
print("Finished! %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
|
|
| 255 |
## quit R |
|
| 256 |
#q("no")
|
|
| 257 |
|
|
| 258 | 121 |
|
| 259 |
################################################################# |
|
| 260 | 122 |
|
| 123 |
################################################################# |
|
| 261 | 124 |
### copy the files back to Yale |
| 262 |
system(paste("scp ",outdir2,"/MOD06_",tile,".nc adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod06/summary",sep=""))
|
|
| 125 |
summarydir="summary" |
|
| 126 |
|
|
| 127 |
|
|
| 263 | 128 |
|
| 264 |
system("scp /tmp/Rtmp6I6tFn/MOD06_L2.A2000061.1615.051.2010273184629.hdf adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/Venezuela")
|
|
| 265 |
system("scp 2_daily/MOD06_20000410.nc adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/Venezuela")
|
|
| 129 |
system(paste("scp ",summarydir,"/MOD06_",tile,".nc adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod06/summary",sep=""))
|
|
| 266 | 130 |
|
| 131 |
system(paste("scp ",tsdir,"/MOD06_",tile,"*.nc adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod06/summary",sep=""))
|
|
| 132 |
system(paste("scp ",paste(fs$path[40421:40422],collapse=" ")," adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod06/swaths",sep=""))
|
|
| 267 | 133 |
|
| 268 | 134 |
|
| 269 | 135 |
|
Also available in: Unified diff
Adding Climatology script to process daily files to climatologies