Revision 807fa48c
Added by Adam Wilson about 12 years ago
| climate/procedures/Pleiades.R | ||
|---|---|---|
| 1 | 1 |
#### Script to facilitate processing of MOD06 data |
| 2 | 2 |
|
| 3 |
|
|
| 4 | 3 |
setwd("/nobackupp1/awilso10/mod06")
|
| 5 | 4 |
|
| 6 |
### get list of files to process |
|
| 7 |
datadir="/nobackupp4/datapool/modis/MOD06_L2.005/" |
|
| 8 |
|
|
| 9 |
fs=data.frame( |
|
| 10 |
path=list.files(datadir,full=T,recursive=T,pattern="hdf"), |
|
| 11 |
file=basename(list.files(datadir,full=F,recursive=T,pattern="hdf"))) |
|
| 12 |
fs$date=as.Date(substr(fs$file,11,17),"%Y%j") |
|
| 13 |
fs$month=format(fs$date,"%m") |
|
| 14 |
fs$year=format(fs$date,"%Y") |
|
| 15 |
fs$time=substr(fs$file,19,22) |
|
| 16 |
fs$datetime=as.POSIXct(strptime(paste(substr(fs$file,11,17),substr(fs$file,19,22)), '%Y%j %H%M')) |
|
| 17 |
fs$dateid=format(fs$date,"%Y%m%d") |
|
| 18 |
fs$path=as.character(fs$path) |
|
| 19 |
fs$file=as.character(fs$file) |
|
| 20 |
|
|
| 21 |
## get all unique dates |
|
| 22 |
alldates=unique(fs$dateid) |
|
| 23 |
|
|
| 24 |
|
|
| 25 | 5 |
## get MODLAND tile information |
| 26 | 6 |
tb=read.table("http://landweb.nascom.nasa.gov/developers/sn_tiles/sn_bound_10deg.txt",skip=6,nrows=648,header=T)
|
| 27 | 7 |
tb$tile=paste("h",sprintf("%02d",tb$ih),"v",sprintf("%02d",tb$iv),sep="")
|
| 28 |
### use MODIS tile as ROI |
|
| 29 |
#modt=readOGR("modgrid","modis_sinusoidal_grid_world",)
|
|
| 30 |
#modt@data[,colnames(tb)[3:6]]=tb[match(paste(modt$h,modt$v),paste(tb$ih,tb$iv)),3:6] |
|
| 31 |
#write.csv(modt@data,file="modistile.csv") |
|
| 32 |
|
|
| 33 |
|
|
| 34 |
## write it out |
|
| 35 |
save(fs,tb,file="allfiles.Rdata") |
|
| 36 |
#save(alldates,file="alldates.Rdata") |
|
| 37 |
|
|
| 38 |
## identify which have been completed |
|
| 39 |
outdir="2_daily" |
|
| 40 |
done=alldates%in%substr(list.files(outdir),5,12) |
|
| 41 |
table(done) |
|
| 42 |
notdone=alldates[!done] |
|
| 43 |
|
|
| 44 |
#notdone=alldates[1:4] |
|
| 45 |
|
|
| 46 |
save(notdone,file="notdone.Rdata") |
|
| 47 |
|
|
| 48 |
|
|
| 49 |
## vars |
|
| 50 |
vars=as.data.frame(matrix(c( |
|
| 51 |
"Cloud_Effective_Radius", "CER", |
|
| 52 |
"Cloud_Effective_Radius_Uncertainty", "CERU", |
|
| 53 |
"Cloud_Optical_Thickness", "COT", |
|
| 54 |
"Cloud_Optical_Thickness_Uncertainty", "COTU", |
|
| 55 |
"Cloud_Water_Path", "CWP", |
|
| 56 |
"Cloud_Water_Path_Uncertainty", "CWPU", |
|
| 57 |
"Cloud_Phase_Optical_Properties", "CPOP", |
|
| 58 |
"Cloud_Multi_Layer_Flag", "CMLF", |
|
| 59 |
"Cloud_Mask_1km", "CM1", |
|
| 60 |
"Quality_Assurance_1km", "QA"), |
|
| 61 |
byrow=T,ncol=2,dimnames=list(1:10,c("variable","varid"))),stringsAsFactors=F)
|
|
| 62 |
save(vars,file="vars.Rdata") |
|
| 63 |
|
|
| 8 |
save(tb,file="modlandTiles.Rdata") |
|
| 64 | 9 |
|
| 65 | 10 |
### Submission script |
| 66 | 11 |
|
| 67 | 12 |
cat(paste("
|
| 68 |
#PBS -S /bin/sh |
|
| 69 |
#PBS -J 700-899 |
|
| 70 |
###PBS -J 1-",length(notdone)," |
|
| 13 |
#PBS -S /bin/bash |
|
| 14 |
#PBS -l select=1:ncpus=4:mpiprocs=4:model=wes |
|
| 15 |
####old PBS -l select=64:ncpus=4:mpiprocs=4:model=wes |
|
| 16 |
####### old: select=48:ncpus=8:mpiprocs=8:model=neh |
|
| 71 | 17 |
#PBS -l walltime=0:10:00 |
| 72 |
#PBS -l ncpus=100 |
|
| 73 | 18 |
#PBS -j oe |
| 74 |
#PBS -o log/log_^array_index^ |
|
| 75 | 19 |
#PBS -m e |
| 20 |
#PBS -V |
|
| 21 |
####PBS -W group_list=s1007 |
|
| 22 |
#PBS -q devel |
|
| 23 |
###PBS -o log/log_^array_index^ |
|
| 24 |
#PBS -o log/log_DataCompile |
|
| 76 | 25 |
#PBS -M adam.wilson@yale.edu |
| 77 | 26 |
#PBS -N MOD06 |
| 78 | 27 |
|
| 28 |
source /usr/share/modules/init/bash |
|
| 29 |
|
|
| 79 | 30 |
## cd to working directory |
| 80 | 31 |
cd /nobackupp1/awilso10/mod06 |
| 81 | 32 |
|
| 82 | 33 |
## set some memory limits |
| 83 | 34 |
# ulimit -d 1500000 -m 1500000 -v 1500000 #limit memory usage |
| 84 | 35 |
source /usr/local/lib/global.profile |
| 36 |
source /u/awilso10/.bashrc |
|
| 85 | 37 |
## export a few important variables |
| 86 | 38 |
export PATH=$PATH:/nobackupp1/awilso10/bin:/nobackupp1/awilso10/software/bin |
| 87 | 39 |
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/nobackupp1/awilso10/software/lib |
| 88 | 40 |
export MRTDATADIR=/nobackupp1/awilso10/software/heg/data |
| 89 | 41 |
export PGSHOME=/nobackupp1/awilso10/software/heg |
| 90 |
export MRTBINDIR=/nobackup1/awilso10/software/TOOLKIT_MTD |
|
| 42 |
export MRTBINDIR=/nobackupp1/awilso10/software/TOOLKIT_MTD
|
|
| 91 | 43 |
export R_LIBS=\"/u/awilso10/R/x86_64-unknown-linux-gnu-library/2.15/\" |
| 44 |
export TMPDIR=/nobackupp1/awilso10/mod06/tmp |
|
| 92 | 45 |
## load modules |
| 93 | 46 |
module load gcc mpi-sgi/mpt.2.06r6 hdf4 udunits R |
| 94 | 47 |
## Run the script! |
| 95 |
Rscript --verbose --vanilla /u/awilso10/environmental-layers/climate/procedures/MOD06_L2_data_compile_Pleiades.r i=${PBS_ARRAY_INDEX}
|
|
| 96 |
rm -r $TMPDIR |
|
| 48 |
TMPDIR=$TMPDIR Rscript --verbose --vanilla /u/awilso10/environmental-layers/climate/procedures/MOD06_L2_process.r |
|
| 97 | 49 |
exit 0 |
| 98 | 50 |
",sep=""),file="MOD06_process") |
| 99 | 51 |
|
| 100 | 52 |
### Check the file |
| 101 | 53 |
system("cat MOD06_process")
|
| 102 |
#system("chmod +x MOD06_process")
|
|
| 54 |
#system("cat ~/environmental-layers/climate/procedures/MOD06_L2_process.r")
|
|
| 55 |
|
|
| 56 |
## Submit it (and keep the pid)! |
|
| 57 |
pid=system("qsub -q devel MOD06_process",intern=T); pid; pid=strsplit(pid,split="[.]")[[1]][1]
|
|
| 58 |
|
|
| 59 |
#system("qsub MOD06_process")
|
|
| 103 | 60 |
|
| 104 |
## Submit it! |
|
| 105 |
#system("qsub -q devel MOD06_process")
|
|
| 106 |
system("qsub MOD06_process")
|
|
| 61 |
## work in interactive mode |
|
| 62 |
#system("qsub -I -lselect=1:ncpus=2:model=wes -q devel")
|
|
| 107 | 63 |
|
| 108 | 64 |
## check progress |
| 109 | 65 |
system("qstat -u awilso10")
|
| 110 |
system("qstat -t 391843[]")
|
|
| 111 |
system("qstat -f 391843[2]")
|
|
| 66 |
system(paste("qstat -t -x",pid))
|
|
| 112 | 67 |
|
| 113 |
#system("qstat devel ")
|
|
| 68 |
system("qstat devel ")
|
|
| 114 | 69 |
#system("qstat | grep awilso10")
|
| 115 | 70 |
|
| 116 |
#print(paste(max(0,length(system("qstat",intern=T))-2)," processes running"))
|
|
| 117 |
# system("ssh c0-8.farm.caes.ucdavis.edu")
|
|
| 118 |
# system("qalter -p +1024 25964") #decrease priority of job to run extraction below.
|
|
| 119 |
system("cat log/InterpScript.o55934.2")
|
|
| 120 | 71 |
|
| 121 |
## check log |
|
| 122 |
system(paste("cat",list.files("log",pattern="InterpScript",full=T)[100]))
|
|
| 123 |
#system(paste("cat",list.files("log",pattern="InterpScript",full=T)[13]," | grep \"Temporary Directory\""))
|
|
| 72 |
### copy the files back to Yale |
|
| 73 |
system("scp 2_daily/* adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/")
|
|
Also available in: Unified diff
Script is successfully running and producing the summary files, though the output looks strange. Maybe a problem with sinusoidal output of HEG? Use Pleiades.R to drive the submission and MOD06_L2_process as the processing script