Project

General

Profile

« Previous | Next » 

Revision 95354b03

Added by Adam Wilson over 11 years ago

Updated to use new swtif and remove additional QC that identified strips introduced by old swtif. Some tiling artifacts still present.

View differences:

climate/procedures/Pleiades_MOD35.R
1 1 
#### Script to facilitate processing of MOD06 data
2
2 
### This script is meant to be run iteratively, rather than unsupervised. There are several steps that require manual checking (such as choosing the number of cores, etc.)
3 

  		




  
  4
  
    
## working directory


  		




  3
  5
  
    
setwd("/nobackupp1/awilso10/mod35")


  		




  4
  6
  
    

  		




  
  7
  
    
## load libraries


  		




  5
  8
  
    
library(rgdal)


  		




  6
  9
  
    
library(raster)


  		




  7
  10
  
    
library(RSQLite)


  		




  8
  11
  
    

  		




  9
  
  
    

  		




  
  12
  
    
## flag to increase verbosity of output


  		




  10
  13
  
    
verbose=T


  		




  11
  14
  
    

  		




  12
  15
  
    
## get MODLAND tile information


  		




  ......




  19
  22
  
    
## Choose some tiles to process


  		




  20
  23
  
    
### list of tiles to process


  		




  21
  24
  
    
tiles=c("h10v08","h11v08","h12v08","h10v07","h11v07","h12v07")  # South America


  		




  22
  
  
    
## a northern block of tiles


  		




  
  25
  
    
## or a northern block of tiles


  		




  23
  26
  
    
tiles=apply(expand.grid(paste("h",11:17,sep=""),v=c("v00","v01","v02","v03","v04")),1,function(x) paste(x,collapse="",sep=""))


  		




  24
  27
  
    
## subset to MODLAND tiles


  		




  25
  28
  
    
alltiles=system("ls -r MODTILES/ | grep tif$ | cut -c1-6 | sort | uniq - ",intern=T)


  		




  ......




  99
  102
  
    

  		




  100
  103
  
    
### report on what has already been processed


  		




  101
  104
  
    
print(paste(sum(!proclist$done)," out of ",nrow(proclist)," (",round(100*sum(!proclist$done)/nrow(proclist),2),"%) remain"))


  		




  
  105
  
    
stem(table(tile=proclist$tile[proclist$done],year=proclist$year[proclist$done]))


  		




  102
  106
  
    
table(tile=proclist$tile[proclist$done],year=proclist$year[proclist$done])


  		




  103
  107
  
    
table(table(tile=proclist$tile[!proclist$done],year=proclist$year[!proclist$done]))


  		




  104
  108
  
    

  		




  ......




  112
  116
  
    
tp=T  # rerun everything


  		




  113
  117
  
    
tp=((!proclist$done)&proclist$avail)  #date-tiles to process


  		




  114
  118
  
    
table(Available=proclist$avail,Completed=proclist$done)


  		




  
  119
  
    
table(tp)


  		




  115
  120
  
    

  		




  116
  121
  
    
write.table(paste("--verbose ",script," --date ",proclist$date[tp]," --verbose T --tile ",proclist$tile[tp],sep=""),


  		




  117
  122
  
    
file=paste("notdone.txt",sep=""),row.names=F,col.names=F,quote=F)


  		




  118
  123
  
    

  		




  
  124
  
    
## try running it once for a single tile-date to get estimate of time/tile-day


  		




  
  125
  
    
test=F


  		




  
  126
  
    
if(test){


  		




  
  127
  
    
  i=2


  		




  
  128
  
    
  time1=system.time(system(paste("Rscript --verbose ",script," --date ",proclist$date[i]," --verbose T --tile ",proclist$tile[i],sep="")))


  		




  
  129
  
    
  hours=round(length(proclist$date[tp])*142/60/60)


  		




  
  130
  
    
  hours=round(length(proclist$date[tp])*time1[3]/60/60,1)


  		




  
  131
  
    
  hours/240


  		




  
  132
  
    
  print(paste("Based on runtime of previous command, it will take",hours," hours to process the full set"))


  		




  
  133
  
    
}


  		




  
  134
  
    

  		




  
  135
  
    

  		




  119
  136
  
    
### qsub script


  		




  120
  137
  
    
cat(paste("


  		




  121
  138
  
    
#PBS -S /bin/bash


  		




  122
  
  
    
#PBS -l select=20:ncpus=8:mpiprocs=8


  		




  
  139
  
    
#PBS -l select=28:ncpus=8:mpiprocs=8


  		




  123
  140
  
    
##PBS -l select=100:ncpus=8:mpiprocs=8


  		




  124
  141
  
    
##PBS -l walltime=8:00:00


  		




  125
  
  
    
#PBS -l walltime=4:00:00


  		




  
  142
  
    
#PBS -l walltime=2:00:00


  		




  126
  143
  
    
#PBS -j n


  		




  127
  144
  
    
#PBS -m be


  		




  128
  145
  
    
#PBS -N mod35


  		




  129
  
  
    
#PBS -q normal


  		




  130
  
  
    
##PBS -q devel


  		




  
  146
  
    
##PBS -q normal


  		




  
  147
  
    
#PBS -q devel


  		




  131
  148
  
    
#PBS -V


  		




  132
  149
  
    

  		




  133
  150
  
    
#CORES=800


  		




  134
  
  
    
CORES=160


  		




  
  151
  
    
CORES=224


  		




  135
  152
  
    

  		




  136
  153
  
    
HDIR=/u/armichae/pr/


  		




  137
  154
  
    
  source $HDIR/etc/environ.sh


  		




  ......




  152
  169
  
    
system(paste("cat notdone.txt | head",sep=""))


  		




  153
  170
  
    
system(paste("cat notdone.txt | wc -l ",sep=""))


  		




  154
  171
  
    

  		




  
  172
  
    

  		




  155
  173
  
    
## Submit it


  		




  156
  174
  
    
system(paste("qsub mod35_qsub",sep=""))


  		




  157
  175
  
    

  		












Also available in:  Unified diff