#Part 1 create summary tables and inputs files for figure in part 2 and part 3.

#AUTHOR: Benoit Parmentier 

#CREATED ON: 03/23/2014  

#Version: 4

#PROJECT: Environmental Layers project  

#TO DO:

# - Separate call in a master script for assessment

# - add second stage in the master script for assessment

# - add mosaicing in the master script for assessment

#First source these files:

#Resolved call issues from R.

  year_predicted <- list_param_run_assessment_prediction$list_year_predicted[i] 

  #region_name is not null then restrict the assessment to a specific region

  in_dir_list <- list.dirs(path=in_dir1,recursive=FALSE) #get the list regions processed for this run

  #basename(in_dir_list)

  #                       y=in_dir_list) 

  #in_dir_list <- in_dir_list[grep("bak",basename(basename(in_dir_list)),invert=TRUE)] #the first one is the in_dir1

  #this was changed on 10052015 because the shapefiles were not matching!!!

  #select only directories used for predictions

  in_dir_reg <- in_dir_list[grep(".*._.*.",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...

  #in_dir_reg <- in_dir_list[grep("july_tiffs",basename(in_dir_reg),invert=TRUE)] #select directory with shapefiles...

  in_dir_list <- in_dir_reg

  ##raster_prediction object : contains testing and training stations with RMSE and model object

  in_dir_list_tmp <- file.path(in_dir_list,year_predicted)

  list_raster_obj_files <- lapply(in_dir_list_tmp,FUN=function(x){list.files(path=x,pattern="^raster_prediction_obj.*.RData",full.names=T)})

  list_names_tile_coord <- lapply(list_raster_obj_files,FUN=function(x){basename(dirname(x))})

  list_names_tile_id <- paste("tile",1:length(list_raster_obj_files),sep="_")

  names(list_raster_obj_files)<- list_names_tile_id

  lf_covar_obj <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="covar_obj.*.RData",full.names=T)})

  lf_covar_tif <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="covar.*.tif",full.names=T)})

  lf_sub_sampling_obj_files <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern=paste("^sub_sampling_obj_",interpolation_method,".*.RData",sep=""),full.names=T)})

  lf_sub_sampling_obj_daily_files <- lapply(in_dir_list_tmp,FUN=function(x){list.files(path=x,pattern="^sub_sampling_obj_daily.*.RData",full.names=T)})

  ################################################################

  ######## PART 1: Generate tables to collect information:

  ######## over all tiles in North America 

  df_tile_processed$tile_id <- unlist(list_names_tile_id) #Arbitrary tiling number!!

  df_tile_processed$path_NEX <- in_dir_list

  df_tile_processed$year_predicted <- year_predicted

  #lf_sub_sampling_obj_files

  ################

  #### Table 1: Average accuracy metrics per tile and predictions

  summary_metrics_v_NA$lat <- lat

  summary_metrics_v_NA$lon <- long

  write.table(as.data.frame(summary_metrics_v_NA),

              file=file.path(out_dir,paste("summary_metrics_v2_NA_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  #################

  ###Table 2: daily validation/testing accuracy metrics for all tiles

  #this takes about 15min for 28 tiles (reg4)

  write.table((tb_diagnostic_v_NA),

              file=file.path(out_dir,paste("tb_diagnostic_v_NA_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  #################

  ###Table 3: monthly fit/training accuracy information for all tiles

  write.table((tb_month_diagnostic_s_NA),

              file=file.path(out_dir,paste("tb_month_diagnostic_s_NA_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  #################

  ###Table 4: daily fit/training accuracy information with predictions for all tiles

  write.table((tb_diagnostic_s_NA),

              file=file.path(out_dir,paste("tb_diagnostic_s_NA_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  ##### Table 5: Add later on: daily info

  ### with also data_s and data_v saved!!!

  write.table((data_month_NAM),

              file=file.path(out_dir,paste("data_month_s_NAM_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  data_day_s_list <- mclapply(list_raster_obj_files,FUN=function(x){try(x<-load_obj(x));try(extract_from_list_obj(x$validation_mod_obj,"data_s"))},mc.preschedule=FALSE,mc.cores = num_cores)    

  data_day_v_list <- mclapply(list_raster_obj_files,FUN=function(x){try(x<-load_obj(x));try(extract_from_list_obj(x$validation_mod_obj,"data_v"))},mc.preschedule=FALSE,mc.cores = num_cores)    

              file=file.path(out_dir,paste("data_day_s_NAM_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  write.table((data_day_v_NAM),

              file=file.path(out_dir,paste("data_day_v_NAM_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  #### Recover subsampling data

  #For tiles with many stations, there is a subsampling done in terms of distance (spatial pruning) and 

  data_month_v_subsampling_list <- mclapply(lf_sub_sampling_obj_files,FUN=function(x){try(x<-load_obj(x));try(extract_from_list_obj(x$validation_mod_month_obj,"data_removed"))},mc.preschedule=FALSE,mc.cores = 6)                           

  #test <- mclapply(list_raster_obj_files[1:6],FUN=function(x){try(x<-load_obj(x));try(extract_from_list_obj(x$validation_mod_month_obj,"data_s"))},mc.preschedule=FALSE,mc.cores = 6)                           

  names(data_month_v_subsampling_list) <- list_names_tile_id

  data_month_v_subsampling_tmp <- remove_from_list_fun(data_month_v_subsampling_list)$list

  #df_tile_processed$metrics_v <- remove_from_list_fun(data_month_s_list)$valid

  #if no stations have been removed then there are no validation stations !!!

  if(length(data_month_v_subsampling_tmp)!=0){

    tile_id <- lapply(1:length(data_month_v_subsampling_tmp),

                      FUN=function(i,x){try(rep(names(x)[i],nrow(x[[i]])))},x=data_month_v_subsampling_tmp)

    data_month_v_subsmapling_NAM <- do.call(rbind.fill,ddata_month_v_subsampling_tmp) #combined data_month for "NAM" North America

    data_month_v_subsampling_NAM$tile_id <- unlist(tile_id)

    write.table((data_month_v_subsampling_NAM),

                file=file.path(out_dir,paste("data_month_v_subsampling_NAM_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  }

  ######################################################

  ####### PART 3: EXAMINE STATIONS AND MODEL FITTING ###

  ### Stations and model fitting ###

  #summarize location and number of training and testing used by tiles

  #names(data_month) #this contains LST means (mm_1, mm_2 etc.) as well as TMax and other info

  use_day=TRUE

  use_month=TRUE

  list_names_tile_id <- df_tile_processed$tile_id

  list_raster_obj_files[list_names_tile_id]

  #list_names_tile_id <- c("tile_1","tile_2")

  #debug(extract_daily_training_testing_info)

  #pred_data_info <- extract_daily_training_testing_info(1,list_param=list_param_training_testing_info)

  pred_data_info <- mclapply(1:length(list_raster_obj_files[list_names_tile_id]),FUN=extract_daily_training_testing_info,list_param=list_param_training_testing_info,mc.preschedule=FALSE,mc.cores = num_cores)

  pred_data_info_tmp <- remove_from_list_fun(pred_data_info)$list #remove data not predicted

  ##Add tile nanmes?? it is alreaready there

  #names(pred_data_info)<-list_names_tile_id

              file=file.path(out_dir,paste("pred_data_month_info_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  write.table(pred_data_day_info,

              file=file.path(out_dir,paste("pred_data_day_info_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  ######################################################

  #get shape files for the region being assessed:

  list_shp_world <- list.files(path=in_dir_shp,pattern=".*.shp",full.names=T)

  l_shp <- gsub(".shp","",basename(list_shp_world))

  l_shp <- sub("shp_","",l_shp)

  l_shp <- unlist(lapply(1:length(l_shp),

                         FUN=function(i){paste(strsplit(l_shp[i],"_")[[1]][1:2],collapse="_")}))

  matching_index <- match(basename(in_dir_list),l_shp)

  list_shp_reg_files <- list_shp_world[matching_index]

  df_tile_processed$shp_files <-list_shp_reg_files

  tx<-strsplit(as.character(df_tile_processed$tile_coord),"_")

  lat<- as.numeric(lapply(1:length(tx),function(i,x){x[[i]][1]},x=tx))

  long<- as.numeric(lapply(1:length(tx),function(i,x){x[[i]][2]},x=tx))

  write.table(df_tiles_all,

              file=file.path(out_dir,paste("df_tiles_all_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  #Copy to local home directory on NAS-NEX

  #

  #save a list of all files...

  write.table(df_tiles_all,

              file=file.path(out_dir,"shapefiles",paste("df_tiles_all_",year_predicted,"_",out_prefix,".txt",sep="")),sep=",")

  ###Prepare files for copying back?

  ## Prepare list of files to return...

}

##################### END OF SCRIPT ######################