#MODIFIED ON: 09/16/2014            

#in_dir1 <- "/nobackupp4/aguzman4/climateLayers/output10Deg/reg1/" #On NEX

in_dir1 <- "/nobackupp4/aguzman4/climateLayers/output20Deg2/"

in_dir_reg <- list.dirs(path=in_dir1,recursive=FALSE) #get the list regions processed for this run

in_dir_list <- list.dirs(path=in_dir_reg,recursive=FALSE) #get the list of tiles/directories with outputs 

in_dir_subset <- in_dir_list[grep("subset",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

#in_dir_shp <- in_dir_list[grep("shapefiles",basename(in_dir_subset),invert=FALSE)] #select directory with shapefiles...

#[27] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/outLogs"    

#[28] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/sept01"     

#[29] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/serial"     

#[30] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/subset"     

#[31] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/testFit" 

#in_dir_subset <- in_dir_list[grep("subset",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

#in_dir_subset <- in_dir_list[grep("serial",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

#in_dir_subset <- in_dir_list[grep("sept01",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

#in_dir_subset <- in_dir_list[grep("testFit",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

#Only 6 folders/regions contain information  

#[1] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/subset"

#[2] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg3/subset"

#[3] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg4/subset"

#[4] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg5/subset"

#[5] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg6/subset"

#[6] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg7/subset"

in_dir_reg <- dirname(in_dir_subset)

in_dir_list <- list.dirs(path=in_dir_reg,recursive=FALSE) #get the list of directories with resutls by 10x10 degree tiles

#select only directories used for predictions

in_dir_list <- in_dir_list[grep(".*._.*.",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

#use subset for now:

#in_dir_list <- c(

#"/nobackupp4/aguzman4/climateLayers/output10Deg/reg1/40.0_-120.0/",

#"/nobackupp4/aguzman4/climateLayers/output10Deg/reg1/35.0_-115.0/")

out_prefix<-"run6_global_analyses_09162014"

#diagnostics_obj_gam_fitting_dailyTmax7__08062014.RData

#lf_diagnostic_obj <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="diagnostics_.*.RData",full.names=T)})

#lf_diagnostic_obj <- lf_diagnostic_obj[grep("lk_min",lf_diagnostic_obj,invert=T)] #remove object that have lk_min...

#lf_raster_obj <- c("/nobackupp4/aguzman4/climateLayers/output20Deg/reg2//-10.0_-70.0//raster_prediction_obj_gam_CAI_dailyTmax-10.0_-70.0_run5_global_analyses_08252014.RData"

#  ,"/nobackupp4/aguzman4/climateLayers/output20Deg/reg4//40.0_0.0//raster_prediction_obj_gam_CAI_dailyTmax40.0_0.0_run5_global_analyses_08252014.RData"   

#  ,"/nobackupp4/aguzman4/climateLayers/output20Deg/reg4//50.0_0.0//raster_prediction_obj_gam_CAI_dailyTmax50.0_0.0_run5_global_analyses_08252014.RData"

#  ,"/nobackupp4/aguzman4/climateLayers/output20Deg/reg6//60.0_40.0//raster_prediction_obj_gam_CAI_dailyTmax60.0_40.0_run5_global_analyses_08252014.RData"

#  ,"/nobackupp4/aguzman4/climateLayers/output20Deg/reg6//30.0_40.0//raster_prediction_obj_gam_CAI_dailyTmax30.0_40.0_run5_global_analyses_08252014.RData"

#  ,"/nobackupp4/aguzman4/climateLayers/output20Deg/reg8//40.0_130.0//raster_prediction_obj_gam_CAI_dailyTmax40.0_130.0_run5_global_analyses_08252014.RData")#

robj1 <- load_obj(list_raster_obj_files[[4]]) #This is an example tile

names(robj1$method_mod_obj[[1]]) #for January 1, 2010

names(robj1$method_mod_obj[[1]]$dailyTmax) #for January

## daily fit info:

tb_diagnostic_s_list <- mclapply(list_raster_obj_files,FUN=function(x){try(x<-load_obj(x));try(x[["tb_diagnostic_s"]])},mc.preschedule=FALSE,mc.cores = 6)                           

names(tb_diagnostic_s_list) <- list_names_tile_id

tb_diagnostic_s_tmp <- remove_from_list_fun(tb_diagnostic_s_list)$list

#df_tile_processed$tb_diag <- remove_from_list_fun(tb_diagnostic_v_list)$valid

tb_diagnostic_s_NA <- do.call(rbind.fill,tb_diagnostic_s_tmp) #create a df for NA tiles with all accuracy metrics

tile_id_tmp <- lapply(1:length(tb_diagnostic_s_tmp),

                     FUN=function(i,x,y){rep(y[i],nrow(x[[i]]))},x=tb_diagnostic_s_tmp,y=names(tb_diagnostic_s_tmp))

tb_diagnostic_s_NA$tile_id <- unlist(tile_id_tmp) #adding identifier for tile

tb_diagnostic_s_NA <- merge(tb_diagnostic_s_NA,df_tile_processed[,1:2],by="tile_id")

write.table((tb_diagnostic_s_NA),

            file=file.path(out_dir,paste("tb_diagnostic_s_NA","_",out_prefix,".txt",sep="")),sep=",")

####### process gam fitting diagnostic info

#/nobackupp4/aguzman4/climateLayers/output20Deg/reg5/20.0_30.0//diagnostics_obj_gam_fitting_TMax_9_mod2_08062014.RData

#lf_diagnostic_obj <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="diagnostics_.*.RData",full.names=T)})

#lf_diagnostic_obj <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="diagnostics_obj_gam_fitting_TMax_*_mod*_08062014.RData",full.names=T)})

#lf_diagnostic_obj <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="diagnostics_obj_gam_fitting_TMax_.*._mod.*._08062014.RData",full.names=T)})

#lf_diagnostic_obj <- lf_diagnostic_obj[grep("lk_min",lf_diagnostic_obj,invert=T)] #remove object that have lk_min...

#names(lf_diagnostic_obj) <- list_names_tile_id

#lf_diagnostic_obj_tmp <- remove_from_list_fun(lf_diagnostic_obj)$list

#df_tile_processed$tb_diag <- remove_from_list_fun(tb_diagnostic_v_list)$valid

#gam_diagnostic_tb_list <- vector("list",length=length(lf_diagnostic_obj_tmp))

#for(i in 1:length(lf_diagnostic_obj_tmp)){

#  l_diagnostic_obj_tmp <- lf_diagnostic_obj_tmp[[i]]

#  tile_id_name <-  names(lf_diagnostic_obj_tmp)[i]

#  #l_diagnostic_obj_tmp <- l_diagnostic_obj_tmp[grep("lk_min",l_diagnostic_obj_tmp,invert=T)] #remove object that have lk_min...

#  l_diagnostic_obj_tmp_list <- lapply(l_diagnostic_obj_tmp,FUN=function(x){try(x<-load_obj(x));try(x[["df_diagnostics"]])})#,mc.preschedule=FALSE,mc.cores = 6)                            

#  gam_diagnostic_tb <- do.call(rbind.fill,l_diagnostic_obj_tmp_list)#create a df for NA tiles with all accuracy metrics

#  gam_diagnostic_tb$tile_id <- tile_id_name

#  gam_diagnostic_tb_list[[i]] <- gam_diagnostic_tb    

#}

#gam_diagnostic_df <- do.call(rbind.fill,gam_diagnostic_tb_list) #create a df for NA tiles with all accuracy metrics

#write.table(gam_diagnostic_df,

#            file=file.path(out_dir,paste("gam_diagnostic_df_",out_prefix,".txt",sep="")),sep=",")

#Now look at the 100 tiles of 10x10

#lf_test<-list.files("/nobackupp4/aguzman4/climateLayers/output10Deg/*/*/","diagnostics_obj_gam_fitting*")  

#lf_test <-list.files("/nobackupp4/aguzman4/climateLayers/output10Deg/","diagnostics_obj_gam_fitting.*.RData",recursive=T,full.names=T)

#gam_diagnostic_10x10tb_list <- vector("list",length=length(lf_test))

#lf_diagnostic_obj_tmp <- lf_test  

#for(i in 1:length( lf_diagnostic_obj_tmp)){

#  l_diagnostic_obj_tmp <-  lf_diagnostic_obj_tmp[[i]]

#  tile_coord <-  basename(dirname(lf_diagnostic_obj_tmp[i]))

#  #l_diagnostic_obj_tmp <- l_diagnostic_obj_tmp[grep("lk_min",l_diagnostic_obj_tmp,invert=T)] #remove object that have lk_min...

#  l_diagnostic_obj_tmp_list <- lapply(l_diagnostic_obj_tmp,FUN=function(x){try(x<-load_obj(x));try(x[["df_diagnostics"]])})#,mc.preschedule=FALSE,mc.cores = 6)                            

#  gam_diagnostic_tb <- do.call(rbind.fill,l_diagnostic_obj_tmp_list)#create a df for NA tiles with all accuracy metrics

#  gam_diagnostic_tb$tile_coord <- tile_coord

#  gam_diagnostic_10x10tb_list[[i]] <- gam_diagnostic_tb    

#}

#gam_diagnostic_10x10_df <- do.call(rbind.fill,gam_diagnostic_10x10tb_list) #create a df for NA tiles with all accuracy metrics

#list_tile_coord <- unique(gam_diagnostic_10x10_df$tile_coord)

#list_tile_id <- paste("tile_",1:length(list_tile_coord),sep="")

#tile_id_df <- data.frame(tile_coord=list_tile_coord,tile_id=list_tile_id)

#gam_diagnostic_10x10_df <- merge(gam_diagnostic_10x10_df,tile_id_df,by="tile_coord")

# write.table(gam_diagnostic_10x10_df,

#             file=file.path(out_dir,paste("gam_diagnostic_10x10_df_",out_prefix,".txt",sep="")),sep=",")

#################

###Table 3: monthly station information with predictions for all tiles

l_shp <- unlist(lapply(1:length(list_shp_world),FUN=function(i){paste(strsplit(list_shp_world[i],"_")[[1]][2:3],collapse="_")}))

                           FUN=function(x){paste(x,dates_l,".*.tif",sep="")})

date_l# <- paste("clim_month_",1:12,sep="")

pred_data_info <- lapply(1:length(list_raster_obj_files[1]),FUN=extract_daily_training_testing_info,list_param=list_param_training_testing_info)

lf_cp_shp_pattern <- gsub(".shp","*",base_name(lf_cp_shp))