Revision f27b378d
Added by Adam Wilson over 11 years ago
| climate/procedures/Pleiades.R | ||
|---|---|---|
| 1 | 1 |
#### Script to facilitate processing of MOD06 data |
| 2 |
|
|
| 2 |
|
|
| 3 | 3 |
setwd("/nobackupp1/awilso10/mod06")
|
| 4 | 4 |
|
| 5 | 5 |
library(rgdal) |
| ... | ... | |
| 10 | 10 |
verbose=T |
| 11 | 11 |
|
| 12 | 12 |
## get MODLAND tile information |
| 13 |
txbb=read.table("http://landweb.nascom.nasa.gov/developers/sn_tiles/sn_bound_10deg.txt",skip=6,nrows=648,header=T)
|
|
| 13 |
tb=read.table("http://landweb.nascom.nasa.gov/developers/sn_tiles/sn_bound_10deg.txt",skip=6,nrows=648,header=T)
|
|
| 14 | 14 |
tb$tile=paste("h",sprintf("%02d",tb$ih),"v",sprintf("%02d",tb$iv),sep="")
|
| 15 | 15 |
save(tb,file="modlandTiles.Rdata") |
| 16 | 16 |
load("modlandTiles.Rdata")
|
| ... | ... | |
| 27 | 27 |
|
| 28 | 28 |
### list of tiles to process |
| 29 | 29 |
tiles=c("h11v08","h21v09","h08v04","h09v04","h08v05","h09v05","h20v11","h31v11")
|
| 30 |
tiles=tiles[c(5,7,8)]
|
|
| 30 |
tiles=tiles[1]
|
|
| 31 | 31 |
tile_bb=tb[tb$tile%in%tiles,] |
| 32 | 32 |
|
| 33 | 33 |
### get list of files to process |
| ... | ... | |
| 75 | 75 |
## identify which have been completed |
| 76 | 76 |
fdone=data.frame(path=list.files(outdir,pattern="nc$",recursive=T)) |
| 77 | 77 |
fdone$date=substr(basename(as.character(fdone$path)),14,21) |
| 78 |
fdone$tile=substr(basename(as.character(fdone$path)),14,21)
|
|
| 78 |
fdone$tile=substr(basename(as.character(fdone$path)),7,12)
|
|
| 79 | 79 |
|
| 80 | 80 |
## identify which date-tiles have already been run |
| 81 | 81 |
proclist$done=paste(proclist$tile,proclist$date,sep="_")%in%substr(basename(as.character(fdone$path)),7,21) |
| 82 | 82 |
|
| 83 | 83 |
### report on what has already been processed |
| 84 |
print(paste("Overview of completed tile-days (",round(sum(proclist$done)/nrow(proclist),2),"%)"))
|
|
| 84 |
print(paste(sum(!proclist$done)," out of ",nrow(proclist)," (",round(sum(!proclist$done)/nrow(proclist),2),"%) remain"))
|
|
| 85 | 85 |
table(tile=proclist$tile[proclist$done],year=proclist$year[proclist$done]) |
| 86 | 86 |
|
| 87 |
|
|
| 88 |
#updatedone=F #update the "done" list using the |
|
| 89 |
#if(updatedone&exists("fdly")){ #update using table from below
|
|
| 90 |
# done[alldates%in%fdly$dateid[fdly$drop]]=F |
|
| 91 |
#} |
|
| 92 |
|
|
| 93 |
## Identify which dates still need to be processed |
|
| 94 |
## This vector will be used to tell mpiexec which days to include |
|
| 95 |
#notdone=alldates[!done] |
|
| 96 |
|
|
| 97 | 87 |
script="/u/awilso10/environmental-layers/climate/procedures/MOD06_L2_process.r" |
| 98 |
climatescript="/u/awilso10/environmental-layers/climate/procedures/MOD06_Climatology.r" |
|
| 99 | 88 |
|
| 100 | 89 |
## write the table processed by mpiexec |
| 101 | 90 |
write.table(paste("--verbose ",script," --date ",proclist$date[!proclist$done]," --verbose T --tile ",proclist$tile[!proclist$done],sep=""),
|
| ... | ... | |
| 116 | 105 |
|
| 117 | 106 |
CORES=400 |
| 118 | 107 |
HDIR=/u/armichae/pr/ |
| 119 |
source $HDIR/etc/environ.sh |
|
| 108 |
# source $HDIR/etc/environ.sh |
|
| 109 |
source /u/awilso10/environ.sh |
|
| 120 | 110 |
source /u/awilso10/.bashrc |
| 121 | 111 |
IDIR=/nobackupp1/awilso10/mod06/ |
| 122 | 112 |
##WORKLIST=$HDIR/var/run/pxrRgrs/work.txt |
| ... | ... | |
| 141 | 131 |
### Now submit the script to generate the climatologies |
| 142 | 132 |
|
| 143 | 133 |
tiles |
| 144 |
ctiles=tiles[c(1,3)] #subset to only some tiles (for example if some aren't finished yet)? |
|
| 134 |
ctiles=tiles#[c(1,3)] #subset to only some tiles (for example if some aren't finished yet)?
|
|
| 145 | 135 |
climatescript="/u/awilso10/environmental-layers/climate/procedures/MOD06_Climatology.r" |
| 146 | 136 |
|
| 147 | 137 |
## write the table processed by mpiexec |
| 148 | 138 |
write.table(paste("--verbose ",climatescript," --verbose T --tile ",ctiles,sep=""),
|
| 149 | 139 |
file=paste("notdone_climate.txt",sep=""),row.names=F,col.names=F,quote=F)
|
| 150 | 140 |
|
| 141 |
## delay start until previous jobs have finished? |
|
| 142 |
delay=F |
|
| 143 |
## check running jobs to get JobID of job you want to wait for |
|
| 144 |
system("qstat -u awilso10")
|
|
| 145 |
## enter JobID here: |
|
| 146 |
job="881394.pbspl1.nas.nasa.gov" |
|
| 147 |
|
|
| 151 | 148 |
### qsub script |
| 152 | 149 |
cat(paste("
|
| 153 | 150 |
#PBS -S /bin/bash |
| 154 |
#PBS -l select=1:ncpus=8:mpiprocs=8
|
|
| 151 |
#PBS -l select=50:ncpus=8:mpiprocs=8
|
|
| 155 | 152 |
##PBS -l select=2:ncpus=4:mpiprocs=4 |
| 156 | 153 |
#PBS -l walltime=5:00:00 |
| 157 | 154 |
#PBS -j n |
| ... | ... | |
| 159 | 156 |
#PBS -N mod06_climate |
| 160 | 157 |
#PBS -q normal |
| 161 | 158 |
#PBS -V |
| 159 |
",if(delay) paste("#PBS -W depend=afterany:",job,sep=""),"
|
|
| 162 | 160 |
|
| 163 |
CORES=8
|
|
| 161 |
CORES=400
|
|
| 164 | 162 |
HDIR=/u/armichae/pr/ |
| 165 |
source $HDIR/etc/environ.sh |
|
| 163 |
# source $HDIR/etc/environ.sh |
|
| 164 |
source /u/awilso10/environ.sh |
|
| 166 | 165 |
source /u/awilso10/.bashrc |
| 167 | 166 |
IDIR=/nobackupp1/awilso10/mod06/ |
| 168 | 167 |
##WORKLIST=$HDIR/var/run/pxrRgrs/work.txt |
Also available in: Unified diff
Updates to MOD06 Climatology Script and MDO06 processing to isolate the effects of the MOD35 -landcover bias