#MODIFIED ON: 09/16/2014            

in_dir1 <- "/nobackupp4/aguzman4/climateLayers/output20Deg2/"

in_dir_reg <- list.dirs(path=in_dir1,recursive=FALSE) #get the list regions processed for this run

in_dir_list <- list.dirs(path=in_dir_reg,recursive=FALSE) #get the list of tiles/directories with outputs 

#in_dir_list <- in_dir_list[grep("bak",basename(basename(in_dir_list)),invert=TRUE)] #the first one is the in_dir1

in_dir_subset <- in_dir_list[grep("subset",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

in_dir_shp <- file.path(in_dir_subset,"shapefiles")

#in_dir_shp <- in_dir_list[grep("shapefiles",basename(in_dir_subset),invert=FALSE)] #select directory with shapefiles...

#[27] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/outLogs"    

#[28] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/sept01"     

#[29] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/serial"     

#[30] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/subset"     

#[31] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/testFit" 

#in_dir_subset <- in_dir_list[grep("subset",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

#in_dir_subset <- in_dir_list[grep("serial",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

#in_dir_subset <- in_dir_list[grep("sept01",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

#in_dir_subset <- in_dir_list[grep("testFit",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

#Only 6 folders/regions contain information  

#[1] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg2/subset"

#[2] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg3/subset"

#[3] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg4/subset"

#[4] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg5/subset"

#[5] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg6/subset"

#[6] "/nobackupp4/aguzman4/climateLayers/output20Deg2//reg7/subset"

in_dir_reg <- dirname(in_dir_subset)

in_dir_list <- list.dirs(path=in_dir_reg,recursive=FALSE) #get the list of directories with resutls by 10x10 degree tiles

#select only directories used for predictions

in_dir_list <- in_dir_list[grep(".*._.*.",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

out_prefix<-"run6_global_analyses_09162014"

#lf_diagnostic_obj <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="diagnostics_.*.RData",full.names=T)})

#lf_diagnostic_obj <- lf_diagnostic_obj[grep("lk_min",lf_diagnostic_obj,invert=T)] #remove object that have lk_min...

#out_prefix_str <- paste(basename(in_dir_list),out_prefix,sep="_") 

#lf_raster_obj <- create_raster_prediction_obj(in_dir_list,interpolation_method, y_var_name,out_prefix_str,out_path_list=NULL)

#lf_raster_obj <- c("/nobackupp4/aguzman4/climateLayers/output20Deg/reg2//-10.0_-70.0//raster_prediction_obj_gam_CAI_dailyTmax-10.0_-70.0_run5_global_analyses_08252014.RData"

#  ,"/nobackupp4/aguzman4/climateLayers/output20Deg/reg4//40.0_0.0//raster_prediction_obj_gam_CAI_dailyTmax40.0_0.0_run5_global_analyses_08252014.RData"   

#  ,"/nobackupp4/aguzman4/climateLayers/output20Deg/reg4//50.0_0.0//raster_prediction_obj_gam_CAI_dailyTmax50.0_0.0_run5_global_analyses_08252014.RData"

#  ,"/nobackupp4/aguzman4/climateLayers/output20Deg/reg6//60.0_40.0//raster_prediction_obj_gam_CAI_dailyTmax60.0_40.0_run5_global_analyses_08252014.RData"

#  ,"/nobackupp4/aguzman4/climateLayers/output20Deg/reg6//30.0_40.0//raster_prediction_obj_gam_CAI_dailyTmax30.0_40.0_run5_global_analyses_08252014.RData"

#  ,"/nobackupp4/aguzman4/climateLayers/output20Deg/reg8//40.0_130.0//raster_prediction_obj_gam_CAI_dailyTmax40.0_130.0_run5_global_analyses_08252014.RData")#

robj1 <- load_obj(list_raster_obj_files[[4]]) #This is an example tile

#robj1 <- load_obj(lf_raster_obj[4]) #This is tile tile

#list_tb_diagnostic_v <- mclapply(lf_validation_obj,FUN=function(x){try( x<- load_obj(x)); try(extract_from_list_obj(x,"metrics_v"))},mc.preschedule=FALSE,mc.cores = 6)                           

#names(list_tb_diagnostic_v) <- list_names_tile_id

#For 177 tiles but only xx RData boject it takes xxx min

#summary_metrics_v_list <- lapply(summary_metrics_v_list,FUN=function(x){try(x$avg)})

#lf_diagnostic_obj <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="diagnostics_obj_gam_fitting_TMax_.*._mod.*._08062014.RData",full.names=T)})

#names(lf_diagnostic_obj) <- list_names_tile_id

#lf_diagnostic_obj_tmp <- remove_from_list_fun(lf_diagnostic_obj)$list

#gam_diagnostic_tb_list <- vector("list",length=length(lf_diagnostic_obj_tmp))

#for(i in 1:length(lf_diagnostic_obj_tmp)){

#  l_diagnostic_obj_tmp <- lf_diagnostic_obj_tmp[[i]]

#  tile_id_name <-  names(lf_diagnostic_obj_tmp)[i]

#  #l_diagnostic_obj_tmp <- l_diagnostic_obj_tmp[grep("lk_min",l_diagnostic_obj_tmp,invert=T)] #remove object that have lk_min...

#  l_diagnostic_obj_tmp_list <- lapply(l_diagnostic_obj_tmp,FUN=function(x){try(x<-load_obj(x));try(x[["df_diagnostics"]])})#,mc.preschedule=FALSE,mc.cores = 6)                            

#  gam_diagnostic_tb <- do.call(rbind.fill,l_diagnostic_obj_tmp_list)#create a df for NA tiles with all accuracy metrics

#  gam_diagnostic_tb$tile_id <- tile_id_name

#  gam_diagnostic_tb_list[[i]] <- gam_diagnostic_tb    

#}

#gam_diagnostic_df <- do.call(rbind.fill,gam_diagnostic_tb_list) #create a df for NA tiles with all accuracy metrics

#write.table(gam_diagnostic_df,

#            file=file.path(out_dir,paste("gam_diagnostic_df_",out_prefix,".txt",sep="")),sep=",")

#lf_test <-list.files("/nobackupp4/aguzman4/climateLayers/output10Deg/","diagnostics_obj_gam_fitting.*.RData",recursive=T,full.names=T)

#gam_diagnostic_10x10tb_list <- vector("list",length=length(lf_test))

#lf_diagnostic_obj_tmp <- lf_test  

#for(i in 1:length( lf_diagnostic_obj_tmp)){

#  l_diagnostic_obj_tmp <-  lf_diagnostic_obj_tmp[[i]]

#  tile_coord <-  basename(dirname(lf_diagnostic_obj_tmp[i]))

#  #l_diagnostic_obj_tmp <- l_diagnostic_obj_tmp[grep("lk_min",l_diagnostic_obj_tmp,invert=T)] #remove object that have lk_min...

#  l_diagnostic_obj_tmp_list <- lapply(l_diagnostic_obj_tmp,FUN=function(x){try(x<-load_obj(x));try(x[["df_diagnostics"]])})#,mc.preschedule=FALSE,mc.cores = 6)                            

#  gam_diagnostic_tb <- do.call(rbind.fill,l_diagnostic_obj_tmp_list)#create a df for NA tiles with all accuracy metrics

#  gam_diagnostic_tb$tile_coord <- tile_coord

#  gam_diagnostic_10x10tb_list[[i]] <- gam_diagnostic_tb    

#}

#gam_diagnostic_10x10_df <- do.call(rbind.fill,gam_diagnostic_10x10tb_list) #create a df for NA tiles with all accuracy metrics

#list_tile_coord <- unique(gam_diagnostic_10x10_df$tile_coord)

#list_tile_id <- paste("tile_",1:length(list_tile_coord),sep="")

#tile_id_df <- data.frame(tile_coord=list_tile_coord,tile_id=list_tile_id)

#gam_diagnostic_10x10_df <- merge(gam_diagnostic_10x10_df,tile_id_df,by="tile_coord")

#dates_l <- unique(robj1$tb_diagnostic_s$date) #list of dates to query tif

#create date!!!

idx <- seq(as.Date('2010-01-01'), as.Date('2010-12-31'), 'day')

#idx <- seq(as.Date('20100101'), as.Date('20101231'), 'day')

#date_l <- strptime(idx[1], "%Y%m%d") # interpolation date being processed

dates_l <- format(idx, "%Y%m%d") # interpolation date being processed

#Need to check how many dates were predicted (have tif) !!! make a table with that!! 

#l_pattern_models <- lapply(c("_mod1_0_1.*","_mod2_0_1.*","_mod3_0_1.*","_mod_kr_0_1.*"),

#                           FUN=function(x){paste("*.",month_l,x,".*.tif",sep="")})

#generate this automatically!!!

l_pattern_models <- lapply(c("_mod1_0_1.*","_mod2_0_1.*","_mod_kr_0_1.*"),

#mod_id <- c(1:(nb_mod-1),"_kr")

#pred_pattern_str <- paste(".*predicted_mod",mod_id,"_0_1.*",sep="")

#,".*predicted_mod2_0_1.*",".*predicted_mod3_0_1.*",".*predicted_mod_kr_0_1.*")

#l_pattern_models <- lapply(c(".*predicted_mod1_0_1.*",".*predicted_mod2_0_1.*",".*predicted_mod3_0_1.*",".*predicted_mod_kr_0_1.*"),

#                           FUN=function(x){paste(x,dates_l,".*.tif",sep="")})

#l_pattern_models <- lapply(pred_pattern_str,

                           FUN=function(x){paste(x,dates_l,".*.tif",sep="")})

#l_pattern_models <- lapply(c(".*delta_dailyTmax_mod1_del_0_1.*",".*delta_dailyTmax_mod2_del_0_1.*",".*delta_dailyTmax_mod3_del_0_1.*",".*delta_dailyTmax_mod_kr_del_0_1.*"),

#                           FUN=function(x){paste(x,dates_l,".*.tif",sep="")})

l_pattern_models <- lapply(c(".*delta_dailyTmax_mod1_del_0_1.*",".*delta_dailyTmax_mod2_del_0_1.*",".*delta_dailyTmax_mod_kr_del_0_1.*"),

#out_prefix_s <- paste(name_method,c("predicted_mod1_0_01","predicted_mod2_0_01","predicted_mod3_0_01","predicted_mod_kr_0_1"),sep="")

out_prefix_s <- paste(name_method,c("predicted_mod1_0_01","predicted_mod2_0_01","predicted_mod_kr_0_1"),sep="")

pred_data_info <- lapply(1:length(list_raster_obj_files[1]),FUN=extract_daily_training_testing_info,list_param=list_param_training_testing_info)

#system("scp -p ./*.txt parmentier@atlas.nceas.ucsb.edu:/data/project/layers/commons/NEX_data/output_run6_global_analyses_09162014")

#Copy all shapefiles in one unique directory

Atlas_dir <- file.path("/data/project/layers/commons/NEX_data/",basename(out_dir),"shapefiles")

lf_cp_shp_pattern <- gsub(".shp","*",base_name(lf_cp_shp))

lf_cp_shp_pattern <- file.path(dirname(lf_cp_shp),lf_cp_shp_pattern)

filenames_NEX <- paste(lf_cp_shp_pattern,collapse=" ")  #copy raster prediction object

cmd_str <- paste("scp -p",filenames_NEX,paste(Atlas_hostname,Atlas_dir,sep=":"), sep=" ")

system(cmd_str)

###Copy shapefiles in the separate directories?