Project

General

Profile

« Previous | Next » 

Revision be64daa8

Added by Adam Wilson over 11 years ago

Updated swath-grid section of MOD35 processing to add buffer and (hopefully) reduce edge artifacts in tiles

View differences:

climate/procedures/Pleiades_MOD35.R
75 75 
#### Generate submission file
76 76 
startdate="2000-03-01"
77 77 
stopdate="2011-12-31"
78 
## just 2009
79 
startdate="2009-01-01"
80 
stopdate="2009-12-31"
78 
## just 2005
79 
startdate="2005-01-01"
80 
stopdate="2005-12-31"
81 81 

  		




  82
  82
  
    
alldates=format(seq(as.Date(startdate),as.Date(stopdate),1),"%Y%m%d")


  		




  83
  83
  
    

  		




  ......




  102
  102
  
    
script="/u/awilso10/environmental-layers/climate/procedures/MOD35_L2_process.r"


  		




  103
  103
  
    

  		




  104
  104
  
    
## write the table processed by mpiexec


  		




  105
  
  
    
tp=(!proclist$done)&proclist$avail  #date-tiles to process


  		




  
  105
  
    
tp=((!proclist$done)&proclist$avail)  #date-tiles to process


  		




  106
  106
  
    
table(Available=proclist$avail,Completed=proclist$done)


  		




  107
  107
  
    

  		




  108
  108
  
    
write.table(paste("--verbose ",script," --date ",proclist$date[tp]," --verbose T --tile ",proclist$tile[tp],sep=""),


  		




  ......




  111
  111
  
    
### qsub script


  		




  112
  112
  
    
cat(paste("


  		




  113
  113
  
    
#PBS -S /bin/bash


  		




  114
  
  
    
#PBS -l select=50:ncpus=8:mpiprocs=8


  		




  115
  
  
    
##PBS -l select=2:ncpus=8:mpiprocs=8


  		




  116
  
  
    
##PBS -l select=2:ncpus=4:mpiprocs=4


  		




  
  114
  
    
#PBS -l select=100:ncpus=8:mpiprocs=8


  		




  
  115
  
    
##PBS -l select=20:ncpus=8:mpiprocs=8


  		




  117
  116
  
    
#PBS -l walltime=5:00:00


  		




  
  117
  
    
##PBS -l walltime=2:00:00


  		




  118
  118
  
    
#PBS -j n


  		




  119
  119
  
    
#PBS -m be


  		




  120
  120
  
    
#PBS -N mod35


  		




  121
  121
  
    
#PBS -q normal


  		




  
  122
  
    
##PBS -q devel


  		




  122
  123
  
    
#PBS -V


  		




  123
  124
  
    

  		




  124
  
  
    
CORES=400


  		




  
  125
  
    
CORES=800


  		




  
  126
  
    
#CORES=160


  		




  
  127
  
    

  		




  125
  128
  
    
HDIR=/u/armichae/pr/


  		




  126
  129
  
    
#  source $HDIR/etc/environ.sh


  		




  127
  130
  
    
  source /u/awilso10/environ.sh


  		




  ......




  150
  153
  
    
### Now submit the script to generate the climatologies


  		




  151
  154
  
    

  		




  152
  155
  
    
tiles


  		




  153
  
  
    
ctiles=tiles[c(1:3)]  #subset to only some tiles (for example if some aren't finished yet)?


  		




  
  156
  
    
ctiles=tiles#[c(1:3)]  #subset to only some tiles (for example if some aren't finished yet)?


  		




  154
  157
  
    
climatescript="/pleiades/u/awilso10/environmental-layers/climate/procedures/MOD35_Climatology.r"


  		




  155
  158
  
    

  		




  
  159
  
    
## check which tiles have been processed and are on lou with a filename "MOD35_[tile].nc"


  		




  
  160
  
    
cdone=data.frame(path=sapply(strsplit(basename(


  		




  
  161
  
    
                   system("ssh lou 'find MOD35/summary -name \"MOD35_h[0-9][0-9]v[0-9][0-9].nc\"' ",intern=T)),split="_"),function(x) x[2]))


  		




  
  162
  
    
cdone$tile=substr(basename(as.character(cdone$path)),1,6)


  		




  
  163
  
    
print(paste(length(ctiles[!ctiles%in%cdone$tile]),"Tiles still need to be processed: /n ",ctiles[!ctiles%in%cdone$tile]))


  		




  
  164
  
    

  		




  156
  165
  
    
## write the table processed by mpiexec


  		




  157
  
  
    
write.table(paste("--verbose ",climatescript," --verbose T --tile ",ctiles,sep=""),


  		




  
  166
  
    
write.table(paste("--verbose ",climatescript," --verbose T --tile ",ctiles[!ctiles%in%cdone$tile],sep=""),


  		




  158
  167
  
    
file=paste("notdone_climate.txt",sep=""),row.names=F,col.names=F,quote=F)


  		




  159
  168
  
    

  		




  160
  169
  
    
## delay start until previous jobs have finished?


  		




  ......




  177
  186
  
    
",if(delay) paste("#PBS -W depend=afterany:",job,sep="")," 


  		




  178
  187
  
    

  		




  179
  188
  
    
CORES=16


  		




  180
  
  
    
HDIR=/pleiades/u/armichae/pr/


  		




  
  189
  
    
HDIR=/u/armichae/pr/


  		




  181
  190
  
    
  source $HDIR/etc/environ.sh


  		




  182
  191
  
    
  source /pleiades/u/awilso10/environ.sh


  		




  183
  192
  
    
  source /pleiades/u/awilso10/.bashrc


  		




  184
  
  
    
  source /pleiades/u/awilso10/moduleload


  		




  185
  193
  
    
IDIR=/nobackupp1/awilso10/mod35/


  		




  186
  194
  
    
##WORKLIST=$HDIR/var/run/pxrRgrs/work.txt


  		




  187
  195
  
    
WORKLIST=$IDIR/notdone_climate.txt


  		




  ......




  209
  217
  
    

  		




  210
  218
  
    
#################################################################


  		




  211
  219
  
    
### copy the files back to Yale


  		




  212
  
  
    
summarydir="summary"


  		




  213
  
  
    

  		




  214
  
  
    
sumfiles=list.files("summary",pattern="^MOD06_.*[0-9][.]nc",full=T)


  		




  215
  
  
    

  		




  216
  
  
    
system(paste("scp ",paste(sumfiles,collapse=" ")," adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod06/summary",sep=""))


  		




  217
  
  
    

  		




  218
  
  
    
#system(paste("scp ",tsdir,"/MOD06_",tile,"*.nc adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod06/summary",sep=""))


  		




  219
  
  
    
#system(paste("scp ",paste(fs$path[40421:40422],collapse=" ")," adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod06/swaths",sep=""))


  		




  220
  
  
    

  		




  221
  
  
    

  		




  222
  220
  
    

  		




  
  221
  
    
system("ssh lou")


  		




  
  222
  
    
#scp `find MOD35/summary -name "MOD35_h[0-9][0-9]v[0-9][0-9].nc"` adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod35/summary/


  		




  
  223
  
    
rsync -vv `find MOD35/summary -name "MOD35_h[0-9][0-9]v[0-9][0-9].nc"` adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod35/summary/


  		




  
  224
  
    
exit


  		




  223
  225
  
    

  		




  224
  226
  
    

  		












Also available in:  Unified diff