Revision dfc5ca84
Added by Benoit Parmentier about 10 years ago
| climate/research/oregon/interpolation/global_run_scalingup_assessment_part1.R | ||
|---|---|---|
| 5 | 5 |
#Part 1 create summary tables and inputs for figure in part 2 and part 3. |
| 6 | 6 |
#AUTHOR: Benoit Parmentier |
| 7 | 7 |
#CREATED ON: 03/23/2014 |
| 8 |
#MODIFIED ON: 10/21/2014
|
|
| 8 |
#MODIFIED ON: 10/29/2014
|
|
| 9 | 9 |
#Version: 3 |
| 10 | 10 |
#PROJECT: Environmental Layers project |
| 11 | 11 |
#TO DO: |
| ... | ... | |
| 171 | 171 |
} |
| 172 | 172 |
|
| 173 | 173 |
### Function: |
| 174 |
pred_data_info_fun <- function(k,list_data,pred_mod,sampling_dat_info){
|
|
| 175 |
#Summarizing input info from sampling and df used in training/testing |
|
| 176 |
|
|
| 177 |
data <- list_data[[k]] |
|
| 178 |
sampling_dat <- sampling_dat_info[[k]] |
|
| 179 |
if(data!="try-error"){
|
|
| 180 |
n <- nrow(data) |
|
| 181 |
n_mod <- vector("numeric",length(pred_mod))
|
|
| 182 |
for(j in 1:length(pred_mod)){
|
|
| 183 |
n_mod[j] <- sum(!is.na(data[[pred_mod[j]]])) |
|
| 174 |
pred_data_info_fun <- function(k,list_data,pred_mod,sampling_dat_info){
|
|
| 175 |
#Summarizing input info from sampling and df used in training/testing |
|
| 176 |
|
|
| 177 |
data <- list_data[[k]] |
|
| 178 |
sampling_dat <- sampling_dat_info[[k]] |
|
| 179 |
if(data!="try-error"){
|
|
| 180 |
n <- nrow(data) |
|
| 181 |
n_mod <- vector("numeric",length(pred_mod))
|
|
| 182 |
for(j in 1:length(pred_mod)){
|
|
| 183 |
n_mod[j] <- sum(!is.na(data[[pred_mod[j]]])) |
|
| 184 |
} |
|
| 185 |
n <- rep(n,length(pred_mod)) |
|
| 186 |
sampling_dat <- sampling_dat[rep(seq_len(nrow(sampling_dat)), each=length(pred_mod)),] |
|
| 187 |
row.names(sampling_dat) <- NULL |
|
| 188 |
df_n <- data.frame(n,n_mod,pred_mod) |
|
| 189 |
df_n <- cbind(df_n,sampling_dat) |
|
| 190 |
}else{
|
|
| 191 |
n <- rep(NA,length(pred_mod)) |
|
| 192 |
n_mod <- vector("numeric",length(pred_mod))
|
|
| 193 |
n_mod <- rep(NA,length(pred_mod)) |
|
| 194 |
df_n <- data.frame(n,n_mod,pred_mod) |
|
| 195 |
sampling_dat <- sampling_dat[rep(seq_len(nrow(sampling_dat)), each=length(pred_mod)),] |
|
| 196 |
row.names(sampling_dat) <- NULL |
|
| 197 |
df_n <- data.frame(n,n_mod,pred_mod) |
|
| 198 |
df_n <- cbind(df_n,sampling_dat) |
|
| 199 |
|
|
| 184 | 200 |
} |
| 185 |
n <- rep(n,length(pred_mod)) |
|
| 186 |
sampling_dat <- sampling_dat[rep(seq_len(nrow(sampling_dat)), each=length(pred_mod)),] |
|
| 187 |
row.names(sampling_dat) <- NULL |
|
| 188 |
df_n <- data.frame(n,n_mod,pred_mod) |
|
| 189 |
df_n <- cbind(df_n,sampling_dat) |
|
| 190 |
}else{
|
|
| 191 |
n <- rep(NA,length(pred_mod)) |
|
| 192 |
n_mod <- vector("numeric",length(pred_mod))
|
|
| 193 |
n_mod <- rep(NA,length(pred_mod)) |
|
| 194 |
df_n <- data.frame(n,n_mod,pred_mod) |
|
| 195 |
sampling_dat <- sampling_dat[rep(seq_len(nrow(sampling_dat)), each=length(pred_mod)),] |
|
| 196 |
row.names(sampling_dat) <- NULL |
|
| 197 |
df_n <- data.frame(n,n_mod,pred_mod) |
|
| 198 |
df_n <- cbind(df_n,sampling_dat) |
|
| 199 |
|
|
| 201 |
|
|
| 202 |
return(df_n) |
|
| 200 | 203 |
} |
| 201 |
|
|
| 202 |
return(df_n) |
|
| 203 |
} |
|
| 204 | 204 |
|
| 205 | 205 |
extract_daily_training_testing_info <- function(i,list_param){
|
| 206 | 206 |
#This function extracts training and testing information from the raster object produced for each tile |
| ... | ... | |
| 467 | 467 |
in_dir_list_all <- lapply(in_dir_list,function(x){list.dirs(path=x,recursive=F)})
|
| 468 | 468 |
#in_dir_list_all <- in_dir_list |
| 469 | 469 |
#in_dir_list <- list.dirs(path=in_dir_reg,recursive=FALSE) #get the list of tiles/directories with outputs |
| 470 |
in_dir_list <- unlist(in_dir_list_all[c(2)]) #only region 3 has informatation at this stage |
|
| 470 |
#in_dir_list <- unlist(in_dir_list_all[c(2)]) #only region 3 has informatation at this stage |
|
| 471 |
in_dir_list <- unlist(in_dir_list_all) #[c(2)]) #only region 3 has informatation at this stage |
|
| 471 | 472 |
|
| 472 | 473 |
#in_dir_list <- in_dir_list[grep("bak",basename(basename(in_dir_list)),invert=TRUE)] #the first one is the in_dir1
|
| 473 | 474 |
in_dir_subset <- in_dir_list[grep("subset",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...
|
| ... | ... | |
| 497 | 498 |
# the last directory contains shapefiles |
| 498 | 499 |
y_var_name <- "dailyTmax" |
| 499 | 500 |
interpolation_method <- c("gam_CAI")
|
| 500 |
out_prefix<-"run8_global_analyses_10212014"
|
|
| 501 |
out_prefix<-"run8_global_analyses_10292014"
|
|
| 501 | 502 |
|
| 502 | 503 |
#out_dir<-"/data/project/layers/commons/NEX_data/" #On NCEAS Atlas |
| 503 | 504 |
out_dir <- "/nobackup/bparmen1/" #on NEX |
| ... | ... | |
| 1009 | 1010 |
#for i in 1:length(df_tiled_processed$tile_coord) |
| 1010 | 1011 |
#output_atlas_dir <- "/data/project/layers/commons/NEX_data/output_run3_global_analyses_06192014/output10Deg/reg1" |
| 1011 | 1012 |
#output_atlas_dir <- "/data/project/layers/commons/NEX_data/output_run5_global_analyses_08252014/output20Deg" |
| 1012 |
output_atlas_dir <- "/data/project/layers/commons/NEX_data/output_run8_global_analyses_10212014"
|
|
| 1013 |
output_atlas_dir <- "/data/project/layers/commons/NEX_data/output_run8_global_analyses_10292014"
|
|
| 1013 | 1014 |
#Make directories on ATLAS |
| 1014 | 1015 |
#for (i in 1:length(df_tile_processed$tile_coord)){
|
| 1015 | 1016 |
# create_dir_fun(file.path(output_atlas_dir,as.character(df_tile_processed$tile_coord[i])),out_suffix=NULL) |
| ... | ... | |
| 1043 | 1044 |
|
| 1044 | 1045 |
lf_cp_shp_pattern <- gsub(".shp","*",basename(lf_cp_shp))
|
| 1045 | 1046 |
lf_cp_shp_pattern <- file.path(dirname(lf_cp_shp),lf_cp_shp_pattern) |
| 1047 |
|
|
| 1046 | 1048 |
filenames_NEX <- paste(lf_cp_shp_pattern,collapse=" ") #copy raster prediction object |
| 1047 | 1049 |
|
| 1048 | 1050 |
cmd_str <- paste("scp -p",filenames_NEX,paste(Atlas_hostname,Atlas_dir,sep=":"), sep=" ")
|
| ... | ... | |
| 1050 | 1052 |
|
| 1051 | 1053 |
###Copy shapefiles in the separate directories? |
| 1052 | 1054 |
#lf_cp_shp <- list.files(in_dir_shp, ".shp",full.names=T) |
| 1053 |
list_tile_scp <- 1:6 |
|
| 1055 |
#list_tile_scp <- 1:6
|
|
| 1054 | 1056 |
|
| 1055 |
for (j in 1:length(list_tile_scp)){
|
|
| 1056 |
tile_nb <- list_tile_scp[j] |
|
| 1057 |
|
|
| 1058 |
in_dir_tile <-dirname(df_tile_processed$shp_files[tile_nb]) |
|
| 1059 |
#/data/project/layers/commons/NEX_data/output_run2_05122014/output |
|
| 1060 |
#output_atlas_dir |
|
| 1061 |
#Atlas_dir <- file.path(file.path("/data/project/layers/commons/NEX_data/",basename(out_dir),"output"),in_dir_tile)
|
|
| 1062 |
Atlas_dir <- file.path(output_atlas_dir,as.character(df_tile_processed$tile_coord[j]),"/shapefiles") |
|
| 1057 |
#for (j in 1:length(list_tile_scp)){
|
|
| 1058 |
# tile_nb <- list_tile_scp[j]
|
|
| 1059 |
# |
|
| 1060 |
# in_dir_tile <-dirname(df_tile_processed$shp_files[tile_nb])
|
|
| 1061 |
# #/data/project/layers/commons/NEX_data/output_run2_05122014/output
|
|
| 1062 |
# #output_atlas_dir
|
|
| 1063 |
# #Atlas_dir <- file.path(file.path("/data/project/layers/commons/NEX_data/",basename(out_dir),"output"),in_dir_tile)
|
|
| 1064 |
# Atlas_dir <- file.path(output_atlas_dir,as.character(df_tile_processed$tile_coord[j]),"/shapefiles")
|
|
| 1063 | 1065 |
|
| 1064 |
Atlas_hostname <- "parmentier@atlas.nceas.ucsb.edu" |
|
| 1065 |
|
|
| 1066 |
lf_cp_shp_pattern <- gsub(".shp","*",lf_cp_shp)
|
|
| 1066 |
# Atlas_hostname <- "parmentier@atlas.nceas.ucsb.edu"
|
|
| 1067 |
# |
|
| 1068 |
# lf_cp_shp_pattern <- gsub(".shp","*",lf_cp_shp)
|
|
| 1067 | 1069 |
|
| 1068 | 1070 |
#filenames_NEX <- paste(lf_cp_shp,collapse=" ") #copy raster prediction object |
| 1069 |
filenames_NEX <- paste(lf_cp_shp_pattern,collapse=" ") #copy raster prediction object |
|
| 1071 |
# filenames_NEX <- paste(lf_cp_shp_pattern,collapse=" ") #copy raster prediction object
|
|
| 1070 | 1072 |
|
| 1071 |
cmd_str <- paste("scp -p",filenames_NEX,paste(Atlas_hostname,Atlas_dir,sep=":"), sep=" ")
|
|
| 1072 |
system(cmd_str) |
|
| 1073 |
} |
|
| 1073 |
# cmd_str <- paste("scp -p",filenames_NEX,paste(Atlas_hostname,Atlas_dir,sep=":"), sep=" ")
|
|
| 1074 |
# system(cmd_str)
|
|
| 1075 |
#}
|
|
| 1074 | 1076 |
|
| 1075 | 1077 |
#### FIRST COPY DATA FOR SPECIFIC TILES ##### |
| 1076 | 1078 |
#Copy specific tiles info back...This assumes that the tree structre |
Also available in: Unified diff
run 8 NEX assessment part 1 for scaling 10x30, Asia, Africa, South America