| 1 |
3682f238
|
Adam M. Wilson @ pfe
|
#### Script to facilitate processing of MOD06 data
|
| 2 |
|
|
|
| 3 |
|
|
setwd("/nobackupp1/awilso10/mod06")
|
| 4 |
|
|
|
| 5 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
## get MODLAND tile information
|
| 6 |
|
|
tb=read.table("http://landweb.nascom.nasa.gov/developers/sn_tiles/sn_bound_10deg.txt",skip=6,nrows=648,header=T)
|
| 7 |
|
|
tb$tile=paste("h",sprintf("%02d",tb$ih),"v",sprintf("%02d",tb$iv),sep="")
|
| 8 |
807fa48c
|
Adam M. Wilson @ pfe
|
save(tb,file="modlandTiles.Rdata")
|
| 9 |
3682f238
|
Adam M. Wilson @ pfe
|
|
| 10 |
|
|
### Submission script
|
| 11 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
|
| 12 |
|
|
cat(paste("
|
| 13 |
807fa48c
|
Adam M. Wilson @ pfe
|
#PBS -S /bin/bash
|
| 14 |
92fd8a10
|
Adam M. Wilson @ pfe
|
##PBS -l select=1:ncpus=16:model=san
|
| 15 |
|
|
###PBS -l select=4:ncpus=8:model=neh
|
| 16 |
|
|
#PBS -l select=1:ncpus=12:model=wes
|
| 17 |
807fa48c
|
Adam M. Wilson @ pfe
|
####### old: select=48:ncpus=8:mpiprocs=8:model=neh
|
| 18 |
92fd8a10
|
Adam M. Wilson @ pfe
|
#PBS -l walltime=10:00:00
|
| 19 |
3682f238
|
Adam M. Wilson @ pfe
|
#PBS -j oe
|
| 20 |
|
|
#PBS -m e
|
| 21 |
807fa48c
|
Adam M. Wilson @ pfe
|
#PBS -V
|
| 22 |
|
|
####PBS -W group_list=s1007
|
| 23 |
92fd8a10
|
Adam M. Wilson @ pfe
|
###PBS -q devel
|
| 24 |
3959e686
|
Adam M. Wilson @ pfe
|
#PBS -o log/log_^array_index^
|
| 25 |
807fa48c
|
Adam M. Wilson @ pfe
|
#PBS -o log/log_DataCompile
|
| 26 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
#PBS -M adam.wilson@yale.edu
|
| 27 |
|
|
#PBS -N MOD06
|
| 28 |
|
|
|
| 29 |
807fa48c
|
Adam M. Wilson @ pfe
|
source /usr/share/modules/init/bash
|
| 30 |
|
|
|
| 31 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
## cd to working directory
|
| 32 |
|
|
cd /nobackupp1/awilso10/mod06
|
| 33 |
|
|
|
| 34 |
|
|
## set some memory limits
|
| 35 |
|
|
# ulimit -d 1500000 -m 1500000 -v 1500000 #limit memory usage
|
| 36 |
|
|
source /usr/local/lib/global.profile
|
| 37 |
807fa48c
|
Adam M. Wilson @ pfe
|
source /u/awilso10/.bashrc
|
| 38 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
## export a few important variables
|
| 39 |
92fd8a10
|
Adam M. Wilson @ pfe
|
export NCORES=24 # use to limit mclapply() to set nubmer of cores, should be select*ncpus above
|
| 40 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
export PATH=$PATH:/nobackupp1/awilso10/bin:/nobackupp1/awilso10/software/bin
|
| 41 |
|
|
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/nobackupp1/awilso10/software/lib
|
| 42 |
|
|
export R_LIBS=\"/u/awilso10/R/x86_64-unknown-linux-gnu-library/2.15/\"
|
| 43 |
807fa48c
|
Adam M. Wilson @ pfe
|
export TMPDIR=/nobackupp1/awilso10/mod06/tmp
|
| 44 |
92fd8a10
|
Adam M. Wilson @ pfe
|
## HEG related variables
|
| 45 |
|
|
export MRTDATADIR=/nobackupp1/awilso10/software/heg/data
|
| 46 |
|
|
export PGSHOME=/nobackupp1/awilso10/software/heg/TOOLKIT_MTD
|
| 47 |
|
|
export HEGUSER=ME
|
| 48 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
## load modules
|
| 49 |
3959e686
|
Adam M. Wilson @ pfe
|
module load gcc mpi-sgi/mpt.2.06r6 hdf4 udunits R nco
|
| 50 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
## Run the script!
|
| 51 |
92fd8a10
|
Adam M. Wilson @ pfe
|
## current version not parallelizing across nodes!
|
| 52 |
807fa48c
|
Adam M. Wilson @ pfe
|
TMPDIR=$TMPDIR Rscript --verbose --vanilla /u/awilso10/environmental-layers/climate/procedures/MOD06_L2_process.r
|
| 53 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
exit 0
|
| 54 |
92fd8a10
|
Adam M. Wilson @ pfe
|
exit 0
|
| 55 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
",sep=""),file="MOD06_process")
|
| 56 |
|
|
|
| 57 |
|
|
### Check the file
|
| 58 |
|
|
system("cat MOD06_process")
|
| 59 |
807fa48c
|
Adam M. Wilson @ pfe
|
#system("cat ~/environmental-layers/climate/procedures/MOD06_L2_process.r")
|
| 60 |
|
|
|
| 61 |
92fd8a10
|
Adam M. Wilson @ pfe
|
## check queue status
|
| 62 |
|
|
system("/u/scicon/tools/bin/node_stats.sh")
|
| 63 |
|
|
|
| 64 |
807fa48c
|
Adam M. Wilson @ pfe
|
## Submit it (and keep the pid)!
|
| 65 |
cf724805
|
Adam M. Wilson @ pfe
|
pid=system("qsub MOD06_process",intern=T); pid; pid=strsplit(pid,split="[.]")[[1]][1]
|
| 66 |
807fa48c
|
Adam M. Wilson @ pfe
|
|
| 67 |
|
|
#system("qsub MOD06_process")
|
| 68 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
|
| 69 |
807fa48c
|
Adam M. Wilson @ pfe
|
## work in interactive mode
|
| 70 |
92fd8a10
|
Adam M. Wilson @ pfe
|
# system("qsub -I -l walltime=1:00:00 -lselect=2:ncpus=16:model=san -q devel")
|
| 71 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
|
| 72 |
|
|
## check progress
|
| 73 |
|
|
system("qstat -u awilso10")
|
| 74 |
92fd8a10
|
Adam M. Wilson @ pfe
|
system(paste("/u/scicon/tools/bin/qps ",pid))
|
| 75 |
807fa48c
|
Adam M. Wilson @ pfe
|
system(paste("qstat -t -x",pid))
|
| 76 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
|
| 77 |
807fa48c
|
Adam M. Wilson @ pfe
|
system("qstat devel ")
|
| 78 |
9d52d7e0
|
Adam M. Wilson @ pfe
|
#system("qstat | grep awilso10")
|
| 79 |
|
|
|
| 80 |
|
|
|
| 81 |
807fa48c
|
Adam M. Wilson @ pfe
|
### copy the files back to Yale
|
| 82 |
3959e686
|
Adam M. Wilson @ pfe
|
list.files("2_daily")
|
| 83 |
|
|
system("scp 2_daily/* adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/Venezuela")
|
| 84 |
|
|
|
| 85 |
|
|
system("scp /tmp/Rtmp6I6tFn/MOD06_L2.A2000061.1615.051.2010273184629.hdf adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/Venezuela")
|
| 86 |
|
|
system("scp 2_daily/MOD06_20000410.nc adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/Venezuela")
|
| 87 |
cf724805
|
Adam M. Wilson @ pfe
|
|
| 88 |
|
|
|
| 89 |
|
|
list.files(" /tmp/Rtmp6I6tFn")
|
