###  Script to compile the monthly cloud data from earth engine into a netcdf file for further processing

library(raster)

library(doMC)

library(foreach)

#library(doMPI)

setwd(wd)

addstats=F

if(addstats){

    df[,c("max","min","mean","sd")]=do.call(rbind.data.frame,mclapply(1:nrow(df),function(i) as.numeric(sub("^.*[=]","",grep("STATISTICS",system(paste("gdalinfo -stats",df$path[i]),inter=T),value=T)))))

    table(df$sd==0)

}

rerun=F  # set to true to recalculate all dates even if file already exists

## Loop over existing months to build composite netcdf files

foreach(date=unique(df$date)) %dopar% {

  ## Define output and check if it already exists

  if(!rerun&file.exists(ncfile)) return(NA)

  system(paste("gdalbuildvrt -overwrite -srcnodata -32768 ",vrtfile," ",paste(df$path[df$date==date],collapse=" ")))

  ## Warp to WGS84 grid and convert to netcdf

  ops="-t_srs 'EPSG:4326' -multi -r cubic -te -90 -90 0 90 -tr 0.008333333333333 -0.008333333333333"

  ops="-t_srs 'EPSG:4326' -multi -r cubic -te -180 -90 180 90 -tr 0.008333333333333 -0.008333333333333"

  system(paste("gdalwarp -overwrite ",ops," -srcnodata -32768 -dstnodata -32768 -of netCDF ",vrtfile," ",ncfile," -ot Int16"))

#  system(paste("gdalwarp -overwrite ",ops," -srcnodata -32768 -dstnodata -32768 -of netCDF ",vrtfile," ",tffile," -ot Int16"))

  setwd(wd)

## create temporary nc file with time information to append to MOD06 data

  cat(paste("

    netcdf time {

      dimensions:

        time = 1 ;

      variables:

        int time(time) ;

      time:units = \"days since 2000-01-01 00:00:00\" ;

      time:calendar = \"gregorian\";

      time:long_name = \"time of observation\"; 

    data:

      time=",as.integer(date-as.Date("2000-01-01")),";

    }"),file=paste(tempdir,"/",date,"_time.cdl",sep=""))

system(paste("ncgen -o ",tempdir,"/",date,"_time.nc ",tempdir,"/",date,"_time.cdl",sep=""))

system(paste("ncks -A ",tempdir,"/",date,"_time.nc ",ncfile,sep=""))

## add other attributes

  system(paste("ncrename -v Band1,CF ",ncfile,sep=""))

  system(paste("ncatted ",

               " -a scale_factor,CF,o,f,\"0.1\" ",

               " -a long_name,CF,o,c,\"Cloud Frequency(%)\" ",

               " -a NETCDF_VARNAME,CF,o,c,\"Cloud Frequency(%)\" ",

               " -a title,global,o,c,\"Cloud Climatology from MOD09 Cloud Mask\" ",

               " -a institution,global,o,c,\"Jetz Lab, EEB, Yale University, New Haven, CT\" ",

               " -a source,global,o,c,\"Derived from MOD09GA Daily Data\" ",

               " -a comment,global,o,c,\"Developed by Adam M. Wilson (adam.wilson@yale.edu / http://adamwilson.us)\" ",

               ## add the fillvalue attribute back (without changing the actual values)

#system(paste("ncatted -a _FillValue,CF,o,b,-32768 ",ncfile,sep=""))

if(as.numeric(system(paste("cdo -s ntime ",ncfile),intern=T))<1) {

  print(paste(ncfile," has no time, deleting"))

  print(paste(basename(ncfile)," Finished"))

#system(paste("ncdump -h ",list.files(tempdir,pattern="mod09.*.nc$",full=T)[10]))

system(paste("cdo -O  -chname,CF,CF_annual -timmean data/cloud_daily.nc  data/cloud_mean.nc"))

### generate the monthly mean and sd

system(paste("cdo  -O -chname,CF,CF_sd -ymonstd data/cloud_daily.nc data/cloud_ymonsd.nc"))

#if(!file.exists("data/mod09_metrics.nc")) {

#    system("cdo -chname,CF,CFmin -timmin data/mod09_clim_mean.nc data/mod09_min.nc")

#    system("cdo -chname,CF,CFmax -timmax data/mod09_clim_mean.nc data/mod09_max.nc")

#    system("cdo -chname,CF,CFsd -timstd data/mod09_clim_mean.nc data/mod09_std.nc")

#    system("cdo -f nc2 merge data/mod09_std.nc data/mod09_min.nc data/mod09_max.nc data/mod09_metrics.nc")

    system("cdo merge -chname,CF,CFmin -timmin data/cloud_clim_mean.nc -chname,CF,CFmax -timmax data/cloud_clim_mean.nc  -chname,CF,CFsd -timstd data/cloud_clim_mean.nc  data/cloud_metrics.nc")

#}

### Long term summaries

seasconc <- function(x,return.Pc=T,return.thetat=F) {

          #################################################################################################

          ## Precipitation Concentration function

          ## This function calculates Precipitation Concentration based on Markham's (1970) technique as described in Schulze (1997)

          ## South Africa Atlas of Agrohydology and Climatology - R E Schulze, M Maharaj, S D Lynch, B J Howe, and B Melvile-Thomson

          ## Pages 37-38

          #################################################################################################

          ## x is a vector of precipitation quantities - the mean for each factor in "months" will be taken,

          ## so it does not matter if the data are daily or monthly, as long as the "months" factor correctly

          ## identifies them into 12 monthly bins, collapse indicates whether the data are already summarized as monthly means.

          #################################################################################################

          theta=seq(30,360,30)*(pi/180)                                       # set up angles for each month & convert to radians

                  if(sum(is.na(x))==12) { return(cbind(Pc=NA,thetat=NA)) ; stop}

                  if(return.Pc) {

                              rt=sqrt(sum(x * cos(theta))^2 + sum(x * sin(theta))^2)    # the magnitude of the summation

                                        Pc=as.integer(round((rt/sum(x))*100))}

                  if(return.thetat){

                              s1=sum(x*sin(theta),na.rm=T); s2=sum(x*cos(theta),na.rm=T)

                                        if(s1>=0 & s2>=0)  {thetat=abs((180/pi)*(atan(sum(x*sin(theta),na.rm=T)/sum(x*cos(theta),na.rm=T))))}

                                        if(s1>0 & s2<0)  {thetat=180-abs((180/pi)*(atan(sum(x*sin(theta),na.rm=T)/sum(x*cos(theta),na.rm=T))))}

                                        if(s1<0 & s2<0)  {thetat=180+abs((180/pi)*(atan(sum(x*sin(theta),na.rm=T)/sum(x*cos(theta),na.rm=T))))}

                                        if(s1<0 & s2>0)  {thetat=360-abs((180/pi)*(atan(sum(x*sin(theta),na.rm=T)/sum(x*cos(theta),na.rm=T))))}

                             thetat=as.integer(round(thetat))

                            }

                  if(return.thetat&return.Pc) return(c(conc=Pc,theta=thetat))

                  if(return.Pc)          return(Pc)

                  if(return.thetat)  return(thetat)

        }