Project

General

Profile

« Previous | Next » 

Revision 0bddce86

Added by Adam Wilson about 12 years ago

Script now runs except ncap2 is missing from the nco module

View differences:

climate/procedures/MOD06_L2_process.r
1 
#!/bin/r
2 

  		




  3
  1
  
    
###################################################################################


  		




  4
  2
  
    
###  R code to aquire and process MOD06_L2 cloud data from the MODIS platform


  		




  5
  3
  
    

  		




  ......




  8
  6
  
    
## Then cycle through each element of the list and evaluate the expressions.


  		




  9
  7
  
    
eval(parse(text=args))


  		




  10
  8
  
    

  		




  11
  
  
    
system("source ~/moduleload")


  		




  
  9
  
    
#system("module list")


  		




  
  10
  
    
#system("source ~/moduleload")


  		




  
  11
  
    
#system("module list")


  		




  12
  12
  
    

  		




  13
  13
  
    
print(args)


  		




  14
  14
  
    

  		




  ......




  18
  18
  
    

  		




  19
  19
  
    
print(paste("Processing tile",tile," for date",date))


  		




  20
  20
  
    

  		




  
  21
  
    
#system("module list")


  		




  
  22
  
    
#system("ldd /u/awilso10/R/x86_64-unknown-linux-gnu-library/2.15/rgdal/libs/rgdal.so")


  		




  
  23
  
    

  		




  21
  24
  
    
## load libraries


  		




  22
  25
  
    
require(reshape)


  		




  23
  
  
    
#require(ncdf4)


  		




  24
  26
  
    
require(geosphere)


  		




  25
  27
  
    
require(raster)


  		




  26
  
  
    
#require(rgdal)


  		




  
  28
  
    
library(rgdal)


  		




  27
  29
  
    
require(spgrass6)


  		




  28
  30
  
    

  		




  29
  31
  
    

  		




  ......




  126
  128
  
    
### Function to extract various SDSs from a single gridded HDF file and use QA data to throw out 'bad' observations


  		




  127
  129
  
    
loadcloud<-function(date,fs){


  		




  128
  130
  
    
  tf=paste(tempdir(),"/grass", Sys.getpid(),"/", sep="")


  		




  
  131
  
    
  dir.create(tf)


  		




  
  132
  
    

  		




  129
  133
  
    
  print(paste("Set up temporary grass session in",tf))


  		




  130
  134
  
    

  		




  131
  135
  
    
  ## set up temporary grass instance for this PID


  		




  ......




  140
  144
  
    

  		




  141
  145
  
    
  ## Identify which files to process


  		




  142
  146
  
    
  tfs=fs$file[fs$dateid==date]


  		




  
  147
  
    
  ## drop swaths that did not produce an output file (typically due to not overlapping the ROI)


  		




  143
  148
  
    
  tfs=tfs[tfs%in%list.files(tempdir())]


  		




  144
  149
  
    
  nfs=length(tfs)


  		




  145
  150
  
    

  		




  ......




  227
  232
  
    
   


  		




  228
  233
  
    
  ncfile=paste(outdir,"/MOD06_",date,".nc",sep="")


  		




  229
  234
  
    
  execGRASS("r.out.gdal",input="mod06",output=ncfile,type="Int16",nodata=-32768,flags=c("quiet"),createopt=c("WRITE_GDAL_TAGS=YES","WRITE_LONLAT=NO"),format="netCDF")


  		




  230
  
  
    
  system(paste("/nasa/nco/3.9.8/bin/ncecat -O -u time ",ncfile," ",ncfile,sep=""))


  		




  231
  
  
    
  system(paste("/nasa/nco/3.9.8/bin/ncap -O -s 'time[time]=",as.integer(fs$date[fs$dateid==date]-as.Date("2000-01-01")),"'",ncfile," ",ncfile))


  		




  232
  
  
    
  system(paste("/nasa/nco/3.9.8/bin/ncatted -a calendar,time,c,c,\"standard\" -a long_name,time,c,c,\"time\" -a units,time,c,c,\"days since 2000-01-01 12:00:00\"",ncfile))


  		




  233
  
  
    
  system(paste("/nasa/nco/3.9.8/bin/ncrename -v Band1,CER -v Band2,COT -v Band3,CLD",ncfile))


  		




  234
  
  
    
  system(paste("/nasa/nco/3.9.8/bin/ncatted -a scale_factor,CER,o,d,0.01 -a units,CER,o,c,\"micron\" -a missing_value,CER,o,d,-32768 -a long_name,CER,o,c,\"Cloud Particle Effective Radius\"",ncfile))


  		




  235
  
  
    
  system(paste("/nasa/nco/3.9.8/bin/ncatted -a scale_factor,COT,o,d,0.01 -a units,COT,o,c,\"none\" -a missing_value,COT,o,d,-32768 -a long_name,COT,o,c,\"Cloud Optical Thickness\"",ncfile))


  		




  236
  
  
    
  system(paste("/nasa/nco/3.9.8/bin/ncatted -a scale_factor,CLD,o,d,0.01 -a units,CLD,o,c,\"none\" -a missing_value,CLD,o,d,-32768 -a long_name,CLD,o,c,\"Cloud Mask\"",ncfile))


  		




  
  235
  
    
  ncopath="/nasa/sles11/nco/4.0.8/gcc/mpt/bin/"


  		




  
  236
  
    
  system(paste(ncopath,"ncecat -O -u time ",ncfile," ",ncfile,sep=""))


  		




  
  237
  
    
  system(paste(ncopath,"ncap2 -O -s 'time[time]=",as.integer(fs$date[fs$dateid==date]-as.Date("2000-01-01")),"'",ncfile," ",ncfile,sep=""))


  		




  
  238
  
    
  system(paste(ncopath,"ncatted -a calendar,time,c,c,\"standard\" -a long_name,time,c,c,\"time\" -a units,time,c,c,\"days since 2000-01-01 12:00:00\" ",ncfile,sep=""))


  		




  
  239
  
    
  system(paste(ncopath,"ncrename -v Band1,CER -v Band2,COT -v Band3,CLD ",ncfile,sep=""))


  		




  
  240
  
    
  system(paste(ncopath,"ncatted -a scale_factor,CER,o,d,0.01 -a units,CER,o,c,\"micron\" -a missing_value,CER,o,d,-32768 -a long_name,CER,o,c,\"Cloud Particle Effective Radius\" ",ncfile,sep=""))


  		




  
  241
  
    
  system(paste(ncopath,"ncatted -a scale_factor,COT,o,d,0.01 -a units,COT,o,c,\"none\" -a missing_value,COT,o,d,-32768 -a long_name,COT,o,c,\"Cloud Optical Thickness\" ",ncfile,sep=""))


  		




  
  242
  
    
  system(paste(ncopath,"ncatted -a scale_factor,CLD,o,d,0.01 -a units,CLD,o,c,\"none\" -a missing_value,CLD,o,d,-32768 -a long_name,CLD,o,c,\"Cloud Mask\" ",ncfile,sep=""))


  		




  237
  243
  
    
   


  		




  238
  244
  
    
  


  		




  239
  245
  
    
### delete the temporary files 


  		












  

    
      climate/procedures/Pleiades.R
    
  





  1
  1
  
    
#### Script to facilitate processing of MOD06 data


  		




  2
  2
  
    

  		




  3
  3
  
    
setwd("/nobackupp1/awilso10/mod06")


  		




  
  4
  
    
library(rgdal)


  		




  
  5
  
    
library(raster)


  		




  4
  6
  
    

  		




  5
  7
  
    
## get MODLAND tile information


  		




  6
  8
  
    
tb=read.table("http://landweb.nascom.nasa.gov/developers/sn_tiles/sn_bound_10deg.txt",skip=6,nrows=648,header=T)


  		




  ......




  43
  45
  
    
tile="h11v08"   #can move this to submit script if needed


  		




  44
  46
  
    
script="/u/awilso10/environmental-layers/climate/procedures/MOD06_L2_process.r"


  		




  45
  47
  
    
#write.table(paste("--verbose ",script," date=",notdone," tile=\"",tile,"\"",sep=""),file="notdone.txt",row.names=F,col.names=F,quote=F)


  		




  46
  
  
    
write.table(paste("--verbose ",script," date=",notdone[1:30],sep=""),file="notdone.txt",row.names=F,col.names=F,quote=F)


  		




  
  48
  
    
#write.table(paste("--verbose ",script," date=",notdone[1:30],sep=""),file="notdone.txt",row.names=F,col.names=F,quote=F)


  		




  
  49
  
    
write.table(notdone[1:30],file="notdone.txt",row.names=F,col.names=F,quote=F)


  		




  47
  50
  
    

  		




  48
  51
  
    
save(fs,alldates,gridfile,td,file="allfiles.Rdata")


  		




  49
  52
  
    

  		




  
  53
  
    
## run script


  		




  
  54
  
    
cat(paste("


  		




  
  55
  
    
#! /bin/bash


  		




  
  56
  
    
source ~/moduleload


  		




  
  57
  
    
source ~/.bashrc


  		




  
  58
  
    
Rscript --verbose --vanilla /u/awilso10/environmental-layers/climate/procedures/MOD06_L2_process.r date=$1


  		




  
  59
  
    
#Rscript --verbose --vanilla rtest


  		




  
  60
  
    
",sep=""),file="MOD06_process2")


  		




  
  61
  
    
system("chmod +x MOD06_process2")


  		




  
  62
  
    

  		




  
  63
  
    
cat(paste("


  		




  
  64
  
    
library(rgdal)


  		




  
  65
  
    
GDALinfo


  		




  
  66
  
    
",sep=""),file="rtest")


  		




  
  67
  
    

  		




  
  68
  
    

  		




  50
  69
  
    
## Submission script


  		




  51
  70
  
    

  		




  52
  71
  
    
cat(paste("


  		




  ......




  76
  95
  
    
## export a few important variables


  		




  77
  96
  
    
  export NNODES=32


  		




  78
  97
  
    
  export R_LIBS=\"/u/awilso10/R/x86_64-unknown-linux-gnu-library/2.15/\"


  		




  79
  
  
    
## load modules


  		




  80
  
  
    
#  module load gcc comp-intel/2012.0.032 netcdf mpi-sgi/mpt.2.06r6 hdf4 udunits R nco


  		




  81
  98
  
    
## Run the script!


  		




  82
  99
  
    
## current version not parallelizing across nodes!


  		




  83
  
  
    
  TMPDIR=$TMPDIR Rscript --verbose --vanilla /u/awilso10/environmental-layers/climate/procedures/MOD06_L2_process.r date=20000403


  		




  
  100
  
    
#  TMPDIR=$TMPDIR Rscript --verbose --vanilla /u/awilso10/environmental-layers/climate/procedures/MOD06_L2_process.r date=20000403


  		




  84
  101
  
    

  		




  85
  102
  
    
WORKLIST=notdone.txt


  		




  86
  
  
    
EXE="Rscript"


  		




  
  103
  
    
#EXE=\"Rscript\"


  		




  
  104
  
    
EXE="./MOD06_process2"


  		




  87
  105
  
    
LOG=log/log_DataCompile.log


  		




  
  106
  
    
MQUEUE=/nobackupp4/pvotava/software/share/mqueue-eg/mqueue/mqueue


  		




  88
  107
  
    

  		




  89
  
  
    
TMPDIR=$TMPDIR mpiexec -np $NNODES  /nobackupp4/pvotava/software/share/mqueue-eg/mqueue/mqueue -l $WORKLIST -p $EXE -v -v -v --random-starts 2-4 --work-analyze #> $LOG


  		




  90
  
  
    
#mpiexec -np 2  /nobackupp4/pvotava/software/share/mqueue-eg/mqueue/mqueue -l testrun.txt -p $EXE -v -v -v  #> $LOG


  		




  91
  
  
    
#TMPDIR=$TMPDIR mpiexec -np $NNODES  /nobackupp4/pvotava/software/share/mqueue-eg/mqueue/mqueue -l $WORKLIST -p $EXE -v -v -v #> $LOG


  		




  
  108
  
    
TMPDIR=$TMPDIR mpiexec -np $NNODES $MQUEUE -l $WORKLIST -p $EXE -v -v -v --random-starts 2-4 --work-analyze #> $LOG


  		




  92
  109
  
    
exit 0


  		




  93
  110
  
    
",sep=""),file="MOD06_process")


  		




  94
  111
  
    

  		




  ......




  98
  115
  
    

  		




  99
  116
  
    
## check queue status


  		




  100
  117
  
    
system("/u/scicon/tools/bin/node_stats.sh")


  		




  101
  
  
    
system("/u/scicon/tools/bin/qtop.pl 479343")


  		




  
  118
  
    
system("/u/scicon/tools/bin/qtop.pl 492352")


  		




  102
  119
  
    

  		




  103
  120
  
    
## Submit it (and keep the pid)!


  		




  104
  121
  
    
system("qsub MOD06_process")


  		




  
  122
  
    
system("/u/scicon/tools/bin/pdsh_gdb -j 493281 -d tmp -s -u awilso10")


  		




  105
  123
  
    

  		




  106
  124
  
    
## work in interactive mode


  		




  107
  125
  
    
# system("qsub -I -l walltime=2:00:00 -lselect=2:ncpus=16:model=san -q devel")


  		












Also available in:  Unified diff