Project

General

Profile

« Previous | Next » 

Revision aba23d60

Added by Adam Wilson over 11 years ago

updated MOD35 processing script to work in separate temporary directories and transfer daily cloud data to lou for archiving in preparation for global processing (which cannot be done in personal directory). Also set up MOD35_Climatology script to be submitted to LDAN queue to run on lou rather than Pleiades. However environment still not set up correctly and script fails when reading netcdf files. Andrew is working on it.

View differences:

climate/procedures/Pleiades_MOD35.R
12 12 
## get MODLAND tile information
13 13 
tb=read.table("http://landweb.nascom.nasa.gov/developers/sn_tiles/sn_bound_10deg.txt",skip=6,nrows=648,header=T)
14 14 
tb$tile=paste("h",sprintf("%02d",tb$ih),"v",sprintf("%02d",tb$iv),sep="")
15 
tb=tb[tb$lon_min!=-999,]
15 16 
save(tb,file="modlandTiles.Rdata")
16 17 
load("modlandTiles.Rdata")
17 18 

  		




  ......




  24
  25
  
    
#tile="h09v04"  # Oregon


  		




  25
  26
  
    
tile="h21v09"  #Kenya


  		




  26
  27
  
    

  		




  27
  
  
    

  		




  28
  28
  
    
### list of tiles to process


  		




  29
  29
  
    
tiles=c("h11v08","h21v09","h08v04","h09v04","h08v05","h09v05","h20v11","h31v11")


  		




  30
  
  
    
tiles=tiles[c(1,4)]


  		




  
  30
  
    
tiles=c("h10v08","h11v08","h12v08","h10v07","h11v07","h12v07")  # South America


  		




  
  31
  
    

  		




  
  32
  
    
## subset to MODLAND tiles


  		




  
  33
  
    
  modlandtiles=system("ls -r /nobackupp4/datapool/modis/MOD11A1.005/2010* | grep hdf$ | cut -c18-23 | sort | uniq - ",intern=T)


  		




  
  34
  
    
 tb$land=tb$tile%in%modlandtiles


  		




  
  35
  
    
tiles=tb$tile[tb$land]


  		




  
  36
  
    

  		




  
  37
  
    
## subset tile corner matrix to tiles selected above


  		




  31
  38
  
    
tile_bb=tb[tb$tile%in%tiles,]


  		




  32
  39
  
    

  		




  33
  40
  
    
### get list of files to process


  		




  34
  41
  
    
datadir="/nobackupp4/datapool/modis/MOD35_L2.006/"


  		




  35
  
  
    
#datadir="/nobackupp1/awilso10/mod06/data"   #for data downloaded from 


  		




  36
  42
  
    

  		




  37
  43
  
    
outdir="daily/" #paste("daily/",tile,sep="")


  		




  38
  44
  
    

  		




  ......




  62
  68
  
    
  


  		




  63
  69
  
    
if(verbose) print(paste("###############",nrow(fs)," swath IDs recieved from database"))


  		




  64
  70
  
    

  		




  65
  
  
    

  		




  66
  71
  
    
## get all unique dates


  		




  67
  72
  
    
fs$dateid=format(as.Date(paste(fs$year,fs$day,sep=""),"%Y%j"),"%Y%m%d")


  		




  68
  73
  
    
alldates=unique(fs$dateid[fs$exists])


  		




  69
  74
  
    

  		




  70
  75
  
    
#### Generate submission file


  		




  71
  
  
    
alldates=format(seq(as.Date("2000-03-01"),as.Date("2011-12-31"),1),"%Y%m%d")


  		




  
  76
  
    
startdate="2000-03-01"


  		




  
  77
  
    
stopdate="2011-12-31"


  		




  
  78
  
    
## just 2009


  		




  
  79
  
    
startdate="2009-01-01"


  		




  
  80
  
    
stopdate="2009-12-31"


  		




  
  81
  
    

  		




  
  82
  
    
alldates=format(seq(as.Date(startdate),as.Date(stopdate),1),"%Y%m%d")


  		




  
  83
  
    

  		




  72
  84
  
    
proclist=expand.grid(date=alldates,tile=tiles)


  		




  73
  85
  
    
proclist$year=substr(proclist$date,1,4)


  		




  74
  
  
    
  


  		




  
  86
  
    

  		




  
  87
  
    
## identify tile-dates with no available swaths


  		




  
  88
  
    
avail=unique(cbind.data.frame(tile=fs$tile,date=fs$dateid)[fs$exists, ])


  		




  
  89
  
    
proclist$avail=paste(proclist$tile,proclist$date,sep="_")%in%paste(avail$tile,avail$date,sep="_")


  		




  
  90
  
    

  		




  75
  91
  
    
## identify which have been completed


  		




  76
  
  
    
fdone=data.frame(path=list.files(outdir,pattern="nc$",recursive=T))


  		




  
  92
  
    
fdone=data.frame(path=system("ssh lou 'find MOD35/daily -name \"*.nc\"' ",intern=T))


  		




  
  93
  
    
#fdone=data.frame(path=list.files(outdir,pattern="nc$",recursive=T))


  		




  77
  94
  
    
fdone$date=substr(basename(as.character(fdone$path)),14,21)


  		




  78
  95
  
    
fdone$tile=substr(basename(as.character(fdone$path)),7,12)


  		




  79
  
  
    

  		




  80
  
  
    
## identify which date-tiles have already been run


  		




  81
  96
  
    
proclist$done=paste(proclist$tile,proclist$date,sep="_")%in%substr(basename(as.character(fdone$path)),7,21)


  		




  82
  97
  
    

  		




  83
  98
  
    
### report on what has already been processed


  		




  ......




  87
  102
  
    
script="/u/awilso10/environmental-layers/climate/procedures/MOD35_L2_process.r"


  		




  88
  103
  
    

  		




  89
  104
  
    
## write the table processed by mpiexec


  		




  90
  
  
    
write.table(paste("--verbose ",script," --date ",proclist$date[!proclist$done]," --verbose T --tile ",proclist$tile[!proclist$done],sep=""),


  		




  
  105
  
    
tp=(!proclist$done)&proclist$avail  #date-tiles to process


  		




  
  106
  
    
table(Available=proclist$avail,Completed=proclist$done)


  		




  
  107
  
    

  		




  
  108
  
    
write.table(paste("--verbose ",script," --date ",proclist$date[tp]," --verbose T --tile ",proclist$tile[tp],sep=""),


  		




  91
  109
  
    
file=paste("notdone.txt",sep=""),row.names=F,col.names=F,quote=F)


  		




  92
  110
  
    

  		




  93
  111
  
    
### qsub script


  		




  ......




  132
  150
  
    
### Now submit the script to generate the climatologies


  		




  133
  151
  
    

  		




  134
  152
  
    
tiles


  		




  135
  
  
    
ctiles=tiles#[c(2)]  #subset to only some tiles (for example if some aren't finished yet)?


  		




  136
  
  
    
climatescript="/u/awilso10/environmental-layers/climate/procedures/MOD35_Climatology.r"


  		




  
  153
  
    
ctiles=tiles[c(1:3)]  #subset to only some tiles (for example if some aren't finished yet)?


  		




  
  154
  
    
climatescript="/pleiades/u/awilso10/environmental-layers/climate/procedures/MOD35_Climatology.r"


  		




  137
  155
  
    

  		




  138
  156
  
    
## write the table processed by mpiexec


  		




  139
  157
  
    
write.table(paste("--verbose ",climatescript," --verbose T --tile ",ctiles,sep=""),


  		




  ......




  149
  167
  
    
### qsub script


  		




  150
  168
  
    
cat(paste("


  		




  151
  169
  
    
#PBS -S /bin/bash


  		




  152
  
  
    
##PBS -l select=50:ncpus=8:mpiprocs=8


  		




  153
  
  
    
#PBS -l select=4:ncpus=4:mpiprocs=4


  		




  154
  
  
    
#PBS -l walltime=2:00:00


  		




  
  170
  
    
#PBS -l select=1:ncpus=16:mem=94


  		




  
  171
  
    
#PBS -l walltime=24:00:00


  		




  155
  172
  
    
#PBS -j n


  		




  156
  173
  
    
#PBS -m be


  		




  157
  174
  
    
#PBS -N mod35_climate


  		




  158
  
  
    
#PBS -q devel


  		




  
  175
  
    
#PBS -q ldan


  		




  159
  176
  
    
#PBS -V


  		




  160
  177
  
    
",if(delay) paste("#PBS -W depend=afterany:",job,sep="")," 


  		




  161
  178
  
    

  		




  162
  179
  
    
CORES=16


  		




  163
  
  
    
HDIR=/u/armichae/pr/


  		




  164
  
  
    
#  source $HDIR/etc/environ.sh


  		




  165
  
  
    
  source /u/awilso10/environ.sh


  		




  166
  
  
    
  source /u/awilso10/.bashrc


  		




  
  180
  
    
HDIR=/pleiades/u/armichae/pr/


  		




  
  181
  
    
  source $HDIR/etc/environ.sh


  		




  
  182
  
    
  source /pleiades/u/awilso10/environ.sh


  		




  
  183
  
    
  source /pleiades/u/awilso10/.bashrc


  		




  
  184
  
    
  source /pleiades/u/awilso10/moduleload


  		




  167
  185
  
    
IDIR=/nobackupp1/awilso10/mod35/


  		




  168
  186
  
    
##WORKLIST=$HDIR/var/run/pxrRgrs/work.txt


  		




  169
  187
  
    
WORKLIST=$IDIR/notdone_climate.txt


  		




  170
  188
  
    
EXE=Rscript


  		




  171
  189
  
    
LOGSTDOUT=$IDIR/log/climatology_stdout


  		




  172
  190
  
    
LOGSTDERR=$IDIR/log/climatology_stderr


  		




  173
  
  
    
### use mpiexec to parallelize across days


  		




  
  191
  
    
### use mpiexec to parallelize across tiles


  		




  174
  192
  
    
mpiexec -np $CORES pxargs -a $WORKLIST -p $EXE -v -v -v --work-analyze 1> $LOGSTDOUT 2> $LOGSTDERR


  		




  175
  193
  
    
",sep=""),file=paste("mod35_climatology_qsub",sep=""))


  		




  176
  194
  
    

  		












Also available in:  Unified diff