#MODIFIED ON: 12/11/2015            

#sub_sampling_obj_daily_gam_CAI_10.0_-75.0.RData

#sub_sampling_obj_gam_CAI_10.0_-75.0.RData

lf_sub_sampling_obj_files <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern=paste("^sub_sampling_obj_",interpolation_method,".*.RData",sep=""),full.names=T)})

lf_sub_sampling_obj_daily_files <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="^sub_sampling_obj_daily.*.RData",full.names=T)})

data_day_v_tmp <- remove_from_list_fun(data_day_v_list)$list

data_day_s_NAM <- do.call(rbind.fill,data_day_s_tmp) #combined data_month for "NAM" North America

tile_id <- lapply(1:length(data_day_v_tmp),

                  FUN=function(i,x){rep(names(x)[i],nrow(x[[i]]))},x=data_day_v_tmp)

data_day_v_NAM <- do.call(rbind.fill,data_day_v_tmp) #combined data_month for "NAM" North America

data_day_v_NAM$tile_id <- unlist(tile_id)

write.table((data_day_v_NAM),

            file=file.path(out_dir,paste("data_day_v_NAM","_",out_prefix,".txt",sep="")),sep=",")

#### Recover subsampling data

#For tiles with many stations, there is a subsampling done in terms of distance (spatial pruning) and 

#in terms of station numbers if there are still too many stations to consider. This is done at the 

#daily and monthly stages.

#lf_sub_sampling_obj_files <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern=paste("^sub_sampling_obj_",interpolation_method,".*.RData",sep=""),full.names=T)})

#lf_sub_sampling_obj_daily_files <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="^sub_sampling_obj_daily.*.RData",full.names=T)})

#sub_sampling_obj <- try(load_obj(lf_sub_sampling_obj_files[[3]])) #This is an example tile

#data_removed contains the validation data...

#this data can be used for validation of the product. Note that it may be missing for some tiles

#as no stations are removed if there are too enough stations in the tile

#this will need to be checked later on...

data_month_v_subsampling_list <- mclapply(lf_sub_sampling_obj_files,FUN=function(x){try(x<-load_obj(x));try(extract_from_list_obj(x$validation_mod_month_obj,"data_removed"))},mc.preschedule=FALSE,mc.cores = 6)                           

#test <- mclapply(list_raster_obj_files[1:6],FUN=function(x){try(x<-load_obj(x));try(extract_from_list_obj(x$validation_mod_month_obj,"data_s"))},mc.preschedule=FALSE,mc.cores = 6)                           

names(data_month_v_subsampling_list) <- list_names_tile_id

data_month_v_subsampling_tmp <- remove_from_list_fun(data_month_v_subsampling_list)$list

#df_tile_processed$metrics_v <- remove_from_list_fun(data_month_s_list)$valid

#if no stations have been removed then there are no validation stations !!!

if(length(data_month_v_subsampling_tmp)!=0){

  tile_id <- lapply(1:length(data_month_v_subsampling_tmp),

                  FUN=function(i,x){try(rep(names(x)[i],nrow(x[[i]])))},x=data_month_v_subsampling_tmp)

  data_month_v_subsmapling_NAM <- do.call(rbind.fill,ddata_month_v_subsampling_tmp) #combined data_month for "NAM" North America

  data_month_v_subsampling_NAM$tile_id <- unlist(tile_id)

  write.table((data_month_v_subsampling_NAM),

            file=file.path(out_dir,paste("data_month_v_subsampling_NAM","_",out_prefix,".txt",sep="")),sep=",")

}

## Do the same for daily...

## End of potential function started in line 317...this section will be cut down for simplicity

#This will be a separate script?? or function?

#output_run10_1500x4500_global_analyses_pred_1992_10052015

Atlas_hostname <- "parmentier@atlas.nceas.ucsb.edu"

#create remotely on Atlas

cmd_str <- paste("ssh ",Atlas_hostname," mkdir ",Atlas_dir,"/{/,mosaic,shapefiles,tiles}", sep="")