outdir="2_daily"  #directory for separate daily files

outdir2="3_summary" #directory for combined daily files and summarized files

  ## load a MOD11A1 file to define grid

gridfile=list.files("/nobackupp4/datapool/modis/MOD11A1.005/2006.01.27/",pattern=paste(tile,".*hdf$",sep=""),full=T)[1]

  td=readGDAL(paste("HDF4_EOS:EOS_GRID:\"",gridfile,"\":MODIS_Grid_Daily_1km_LST:Night_view_angl",sep=""))

  projection(td)="+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs +datum=WGS84 +ellps=WGS84 "

### get list of files to process

datadir="/nobackupp4/datapool/modis/MOD06_L2.005/"

fs=data.frame(path=list.files(datadir,full=T,recursive=T,pattern="hdf"),stringsAsFactors=F)

fs$file=basename(fs$path)

fs$date=as.Date(substr(fs$file,11,17),"%Y%j")

fs$month=format(fs$date,"%m")

fs$year=format(fs$date,"%Y")

fs$time=substr(fs$file,19,22)

fs$datetime=as.POSIXct(strptime(paste(substr(fs$file,11,17),substr(fs$file,19,22)), '%Y%j %H%M'))

fs$dateid=format(fs$date,"%Y%m%d")

fs$path=as.character(fs$path)

fs$file=as.character(fs$file)

## get all unique dates

alldates=unique(fs$dateid)

#### Generate submission file

## identify which have been completed

done=alldates%in%substr(list.files(outdir),7,14)

table(done)

notdone=alldates[!done]  #these are the dates that still need to be processed

tile="h11v08"   #can move this to submit script if needed

script="/u/awilso10/environmental-layers/climate/procedures/MOD06_L2_process.r"

#write.table(paste("--verbose ",script," date=",notdone," tile=\"",tile,"\"",sep=""),file="notdone.txt",row.names=F,col.names=F,quote=F)

write.table(paste("--verbose ",script," date=",notdone[1:30],sep=""),file="notdone.txt",row.names=F,col.names=F,quote=F)

save(fs,alldates,gridfile,td,file="allfiles.Rdata")

## Submission script

#PBS -l select=2:ncpus=16:model=san

#PBS -o log/log_DataCompile.log

  source /usr/local/lib/global.profile

  export NNODES=32

#  module load gcc comp-intel/2012.0.032 netcdf mpi-sgi/mpt.2.06r6 hdf4 udunits R nco

  TMPDIR=$TMPDIR Rscript --verbose --vanilla /u/awilso10/environmental-layers/climate/procedures/MOD06_L2_process.r date=20000403

WORKLIST=notdone.txt

EXE="Rscript"

LOG=log/log_DataCompile.log

TMPDIR=$TMPDIR mpiexec -np $NNODES  /nobackupp4/pvotava/software/share/mqueue-eg/mqueue/mqueue -l $WORKLIST -p $EXE -v -v -v --random-starts 2-4 --work-analyze #> $LOG

#mpiexec -np 2  /nobackupp4/pvotava/software/share/mqueue-eg/mqueue/mqueue -l testrun.txt -p $EXE -v -v -v  #> $LOG

#TMPDIR=$TMPDIR mpiexec -np $NNODES  /nobackupp4/pvotava/software/share/mqueue-eg/mqueue/mqueue -l $WORKLIST -p $EXE -v -v -v #> $LOG

exit 0

system("/u/scicon/tools/bin/qtop.pl 479343")

####################################

################################################################################

## now generate the climatologies

fdly=data.frame(

  path=list.files(outdir,pattern="nc$",full=T),

  file=list.files(outdir,pattern="nc$"))

fdly$date=as.Date(substr(fdly$file,7,14),"%Y%m%d")

fdly$month=format(fdly$date,"%m")

fdly$year=format(fdly$date,"%Y")

## check validity (via npar and ntime) of nc files

for(i in 1:nrow(fdly)){

  fdly$ntime[i]=as.numeric(system(paste("cdo  sinfo ",fdly$path[i]),intern=T))

  fdly$npar[i]=as.numeric(system(paste("cdo -s npar ",fdly$path[i]),intern=T))

  print(i)

}

## Combine all days within years into a single file (can't mergetime all days at once because this opens too many files)

tsdir=paste(tempdir(),"/summary",sep="")

dir.create(tsdir)

lapply(unique(fdly$year),function(y){

  system(paste("cdo -O mergetime ",paste(fdly$path[fdly$year==y],collapse=" ")," ",tsdir,"/MOD09_",tile,"_",y,"_daily.nc",sep=""))

  print(paste("Finished merging daily files for year",y))

})

## Combine the year-by-year files into a single daily file

system(paste("cdo -O mergetime ",paste(list.files(tsdir,full=T,pattern="daily[.]nc$"),collapse=" ")," ",outdir2,"/MOD09_",tile,"_daily.nc",sep=""))

system(paste("cdo -O monmean ",outdir2,"/MOD09_",tile,"_daily.nc ",outdir2,"/",tile,"_monmean.nc",sep=""))

system(paste("cdo -O ymonmean ",outdir2,"/MOD09_",tile,"_daily.nc ",outdir2,"/",tile,"_ymonmean.nc",sep=""))

system(paste("cdo -O ymonstd ",outdir2,"/MOD09_",tile,"_daily.nc ",outdir2,"/",tile,"_ymonstd.nc",sep=""))

print("Finished!   %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")

## quit R

q("no")

#################################################################