#MODIFIED ON: 05/12/2014            

#pattern_raster_obj <- gam

#list.files(path=in_dir_list[1],pattern="gam_CAI_mod.*.RData")

#list_raster_obj_files <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="gam_CAI_mod.*.RData",full.names=T)})

#names(list_raster_obj_files)<- 

names(list_raster_obj_files)<- list_names_tile_coord

###################### PART I: Generate tables to collect information over all tiles in North America ##########

#Table 1: Average accuracy metrics

#Table 2: daily accuracy metrics for all tiles

#data_month_list <- lapply(list_raster_obj_files,FUN=function(x){x<-load_obj(x);x[["summary_metrics_v"]]$avg})                           

#robj1$tb_diagnostic_v[1:10,] #first 10 rows of accuarcy metrics per day and model (for specific tile)

#robj1$summary_metrics_v #first 10 rows of accuarcy metrics per day and model (for specific tile)

#summary_metrics_v_list <- lapply(list_raster_obj_files,FUN=function(x){x<-load_obj(x);x[["summary_metrics_v"]]$avg$rmse})                           

#summary_metrics_v_list <- lapply(list_raster_obj_files,FUN=function(x){try(x<- load_obj(x);x[["summary_metrics_v"]]$avg)})     

summary_metrics_v_list <- mclapply(list_raster_obj_files,FUN=function(x){try( x<- load_obj(x)); try(x[["summary_metrics_v"]]$avg)},mc.preschedule=FALSE,mc.cores = 5)                           

list_names_tile_id <- paste("tile",1:length(list_raster_obj_files),sep="_")

#names(summary_metrics_v_list) <- list_names_tile_coord

remove_from_list_fun <- function(l_x,condition_class ="try-error"){

  index <- vector("list",length(l_x))

  for (i in 1:length(l_x)){

    if (inherits(l_x[[i]],condition_class)){

      index[[i]] <- FALSE #remove from list

    }else{

      index[[i]] <- TRUE

    }

  }

  l_x<-l_x[unlist(index)] #remove from list all elements using subset

  return(l_x)

}

summary_metrics_v_list <- remove_from_list_fun(summary_metrics_v_list)

#summary_metrics_v_NA <- do.call(rbind,summary_metrics_v_list) #create a df for NA tiles with all accuracy metrics

summary_metrics_v_NA <- do.call(rbind.fill,summary_metrics_v_list) #create a df for NA tiles with all accuracy metrics

tile_id <- lapply(1:length(summary_metrics_v_list),

                     FUN=function(i,x,y){rep(y[i],nrow(x[[i]]))},x=summary_metrics_v_list,y=list_names_tile_id)

summary_metrics_v_NA$tile_id <-unlist(tile_id)

#summary_metrics_v_NA$tile_coord <-unlist(tile_coord)

#Function to collect all the tables from tiles into a table

names(tb_diagnostic_v_list)

tb_diagnostic_v_NA <- do.call(rbind.fill,tb_diagnostic_v_list) #create a df for NA tiles with all accuracy metrics

tile_coord <- lapply(1:length(tb_diagnostic_v_list),

                     FUN=function(i,x){rep(names(x)[i],nrow(x[[i]]))},x=tb_diagnostic_v_list)

tile_id <- lapply(1:length(tb_diagnostic_v_list),

                     FUN=function(i,x,y){rep(y[i],nrow(x[[i]]))},x=tb_diagnostic_v_list,y=list_names_tile_id)

tb_diagnostic_v_NA$tile_id <- unlist(tile_id) #adding identifier for tile

tb_diagnostic_v_NA$tile_coord <- unlist(tile_coord) #adding identifier for tile

            file=file.path(out_dir,paste("tb_diagnostic_v2_NA","_",out_prefix,".txt",sep="")),sep=",")

#load data_month for specific tiles

data_month <- extract_from_list_obj(robj1$clim_method_mod_obj,"data_month")

names(data_month) #this contains LST means (mm_1, mm_2 etc.) as well as TMax and other info

#problem with tile 12...the raster ojbect has missing sub object

#data_month_list <- lapply(1:length(list_raster_obj_files),x=list_raster_obj_files,

#                          FUN=function(i,x){x<-load_obj(x[[i]]);

#                                            extract_from_list_obj(x$validation_mod_month_obj,"data_s")})                           

data_month_list <- lapply(1:length(list_raster_obj_files),x=list_raster_obj_files,

                          FUN=function(i,x){x<-load_obj(x[[i]]);

                                            extract_from_list_obj(x$clim_method_mod_obj,"data_month")})                           

names(data_month_list) <- paste("tile","_",1:length(data_month_list),sep="")

#names(data_month_list) <- basename(in_dir_list) #use folder id instead

tile_id <- lapply(1:length(data_month_list),

                  FUN=function(i,x){rep(names(x)[i],nrow(x[[i]]))},x=data_month_list)

data_month_NAM <- do.call(rbind.fill,data_month_list) #combined data_month for "NAM" North America

data_month_NAM$tile_id <- unlist(tile_id)

#plot(mm_01 ~ elev_s,data=data_month_NAM) #Jan across all tiles

#plot(mm_06 ~ elev_s,data=data_month_NAM) #June across all tiles

#plot(TMax ~ mm_01,data=data_month_NAM) #monthly tmax averages and LST across all tiles

#copy back to atlas

system("scp -p ./*.txt ./*.tif parmentier@atlas.nceas.ucsb.edu:/data/project/layers/commons/NEX_data/output_run2_global_analyses_05122014")

#list.files(path=in_dir_list[[1]],pattern=".*predicted.*20100101.*.tif")

#some files are

#"/nobackupp4/aguzman4/climateLayers/output/30.0_-115.0/gam_fusion_dailyTmax_predicted_mod_kr_0_1_20100901_30_130.0_-115.0.tif"

#"/nobackupp4/aguzman4/climateLayers/output/30.0_-115.0/gam_fusion_dailyTmax_predicted_mod_kr_20100901_30_130.0_-115.0.tif"

#list_tif_files_dates <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern=".*month.*.tif",full.names=T)})

#list_tif_files_dates <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern=".*predicted.*20100101.*.tif",full.names=T)})

#".*predicted_mod1_0_1.*20100101.*.tif"

#tb <- tb_diagnostic_v_NA

#get specific dates from tb

#dates_l <- unique(tb$date)

##Function to list predicted tif

list_tif_fun <- function(i,in_dir_list,pattern_str){

  #list.files(path=x,pattern=".*predicted_mod1_0_1.*20100101.*.tif",full.names=T)})

  pat_str<- pattern_str[i]

  list_tif_files_dates <-lapply(in_dir_list,

         FUN=function(x,pat_str){list.files(path=x,pattern=pat_str,full.names=T)},pat_str=pat_str)

  return(list_tif_files_dates)

} 

######################################################

####### PART 3: EXAMINE STATIONS AND MODEL FITTING ###

### Stations and model fitting ###

#summarize location and number of training and testing used by tiles

names(robj1$clim_method_mod_obj[[1]]$data_month) # monthly data for January

#names(robj1$validation_mod_month_obj[[1]]$data_s) # daily for January with predictions

#note that there is no holdout in the current run at the monthly time scale:

robj1$clim_method_mod_obj[[1]]$data_month_v #zero rows for testing stations at monthly timescale

#load data_month for specific tiles

data_month <- extract_from_list_obj(robj1$clim_method_mod_obj,"data_month")

names(data_month) #this contains LST means (mm_1, mm_2 etc.) as well as TMax and other info

#problem with tile 12...the raster ojbect has missing sub object

#data_month_list <- lapply(1:length(list_raster_obj_files),x=list_raster_obj_files,

#                          FUN=function(i,x){x<-load_obj(x[[i]]);

#                                            extract_from_list_obj(x$validation_mod_month_obj,"data_s")})                           

### make this part a function:

#create a table for every month, day and tiles...

data_month_list <- lapply(1:length(list_raster_obj_files),x=list_raster_obj_files,

                          FUN=function(i,x){x<-load_obj(x[[i]]);

                                            extract_from_list_obj(x$clim_method_mod_obj,"data_month")})                           

names(data_month_list) <- paste("tile","_",1:length(data_month_list),sep="")

#names(data_month_list) <- basename(in_dir_list) #use folder id instead

list_names_tile_id <- paste("tile",1:length(list_raster_obj_files),sep="_")

#tile_id <- lapply(1:length(data_month_list),

#                  FUN=function(i,x){rep(names(x)[i],nrow(x[[i]]))},x=data_month_list)

data_month_NAM <- do.call(rbind.fill,data_month_list) #combined data_month for "NAM" North America

data_month_NAM$tile_id <- unlist(tile_id)

names(robj1$validation_mod_day_obj[[1]]$data_s) # daily for January with predictions

dim(robj1$validation_mod_month_obj[[1]]$data_s) # daily for January with predictions

use_day=TRUE

use_month=TRUE

list_param_training_testing_info <- list(list_raster_obj_files,use_month,use_day,list_names_tile_id)

names(list_param_training_testing_info) <- c("list_raster_obj_files","use_month","use_day","list_names_tile_id")

list_param <- list_param_training_testing_info

pred_data_info <- lapply(1:length(list_raster_obj_files),FUN=extract_daily_training_testing_info,list_param=list_param_training_testing_info)

pred_data_month_info <- do.call(rbind,lapply(pred_data_info,function(x){x$pred_data_month_info}))

pred_data_day_info <- do.call(rbind,lapply(pred_data_info,function(x){x$pred_data_day_info}))