Project

General

Profile

« Previous | Next » 

Revision d91f0663

Added by Adam Wilson about 11 years ago

nearly finished first global run of MOD35 cloud frequency

View differences:

climate/procedures/MOD35_Climatology.r
34 34
 fdly=data.frame(path=list.files(outdir,pattern="nc$",full=T),stringsAsFactors=F)
35 35
  fdly$file=basename(fdly$path)
36 36
  fdly$dateid=substr(fdly$file,14,21)
37
  fdly$date=as.Date(substr(fdly$file,14,21),"%Y%m%d")
37
  fdly$date=as.Date(substr(fdly$file,14,21),"%Y%m%d") #23,30 for alltiles
38 38
  fdly$month=format(fdly$date,"%m")
39 39
  fdly$year=format(fdly$date,"%Y")
40 40
nrow(fdly)
......
55 55

  
56 56
## merge all daily files to create a single file with all dates
57 57
system(paste(ncopath,"ncrcat -O ",outdir,"/*nc ",outdir2,"/MOD35_",tile,"_daily.nc",sep=""))
58
#system(paste(ncopath,"ncrcat -O ",outdir,"/*alltests*nc ",outdir2,"/MOD35_",tile,"_daily_alltiles.nc",sep=""))
58 59
system(paste("ncdump -h ",outdir2,"/MOD35_",tile,"_daily.nc",sep=""))
59 60
 
60 61
## Update attributes
climate/procedures/Pleiades_MOD35.R
134 134
  nodes=100
135 135
  threads=nodes*8
136 136
  writeLines(paste(" ################### \n Hours per date-tile:",round(time1[3]/60/60,2),"\n Date-tiles to process:",sum(tp)," \n Estimated CPU time: ",hours,"hours \n  With ",threads,"threads:",round(hours/threads,2),"hours \n ###################"))
137
  summaryRprof("/nobackupp1/awilso10/mod35/log/profile.out")
137 138
}
138 139

  
139 140
### Set up submission script
140 141
queue="devel"
141 142
queue="normal" #"devel"
142
nodes=50
143
walltime=2
143
queue="long" #"devel"
144
nodes=120
145
walltime=24
144 146

  
145 147
### write qsub script to disk
146 148
cat(paste("
......
198 200

  
199 201
## check which tiles have been processed and are on lou with a filename "MOD35_[tile].nc"
200 202
cdone=data.frame(path="",tile="")  #use this if you want to re-run everything
201
cdone=data.frame(path=sapply(strsplit(basename(
202
                   system("ssh lou 'find MOD35/summary -name \"MOD35_h[0-9][0-9]v[0-9][0-9].nc\"' ",intern=T)),split="_"),function(x) x[2]))
203
#cdone=data.frame(path=sapply(strsplit(basename(
204
#                   system("ssh lou 'find MOD35/summary -name \"MOD35_h[0-9][0-9]v[0-9][0-9].nc\"' ",intern=T)),split="_"),function(x) x[2]))
203 205
cdone=data.frame(path=sapply(strsplit(basename(
204 206
                   system("find summary -name \"MOD35_h[0-9][0-9]v[0-9][0-9].nc\"",intern=T)),split="_"),function(x) x[2]))
205 207
cdone$tile=substr(basename(as.character(cdone$path)),1,6)
......
216 218
## enter JobID here:
217 219
job="2031668.pbspl1.nas.nasa.gov"
218 220

  
221

  
222
queue="devel"
223
nodes=50
224
walltime=2
225

  
219 226
### qsub script
220 227
cat(paste("
221 228
#PBS -S /bin/bash
222
#PBS -l select=4:ncpus=8:mem=94
223
#PBS -l walltime=2:00:00
229
#PBS -l select=",nodes,":ncpus=8:mem=94
230
#PBS -l walltime=",walltime,":00:00
224 231
#PBS -j n
225 232
#PBS -m be
226 233
#PBS -N mod35_climate
227
#PBS -q devel
228
##PBS -q normal
229
##PBS -q ldan
234
#PBS -q ",queue,"
230 235
#PBS -V
231 236
",if(delay) paste("#PBS -W depend=afterany:",job,sep="")," 
232 237

  
233
CORES=32
238
CORES=",nodes*8,"
234 239
HDIR=/u/armichae/pr/
235 240
  source $HDIR/etc/environ.sh
236 241
  source /pleiades/u/awilso10/environ.sh

Also available in: Unified diff