Revision 54460160
Added by Benoit Parmentier over 9 years ago
| climate/research/oregon/interpolation/global_run_scalingup_assessment_part1.R | ||
|---|---|---|
| 5 | 5 |
#Part 1 create summary tables and inputs files for figure in part 2 and part 3. |
| 6 | 6 |
#AUTHOR: Benoit Parmentier |
| 7 | 7 |
#CREATED ON: 03/23/2014 |
| 8 |
#MODIFIED ON: 03/23/2015
|
|
| 8 |
#MODIFIED ON: 03/25/2015
|
|
| 9 | 9 |
#Version: 4 |
| 10 | 10 |
#PROJECT: Environmental Layers project |
| 11 | 11 |
#TO DO: |
| ... | ... | |
| 15 | 15 |
# |
| 16 | 16 |
#First source these files: |
| 17 | 17 |
#Resolved call issues from R. |
| 18 |
source /nobackupp6/aguzman4/climateLayers/sharedModules/etc/environ.sh |
|
| 19 |
MODULEPATH=$MODULEPATH:/nex/modules/files |
|
| 20 |
module load pythonkits/gdal_1.10.0_python_2.7.3_nex |
|
| 18 |
#source /nobackupp6/aguzman4/climateLayers/sharedModules/etc/environ.sh
|
|
| 19 |
#MODULEPATH=$MODULEPATH:/nex/modules/files
|
|
| 20 |
#module load pythonkits/gdal_1.10.0_python_2.7.3_nex
|
|
| 21 | 21 |
|
| 22 | 22 |
# These are the names and number for the current subset regions used for global runs: |
| 23 | 23 |
#reg1 - North America (NAM) |
| ... | ... | |
| 71 | 71 |
#master directory containing the definition of tile size and tiles predicted |
| 72 | 72 |
#in_dir1 <- "/nobackupp6/aguzman4/climateLayers/output1000x3000_km/" |
| 73 | 73 |
in_dir1 <- "/nobackupp6/aguzman4/climateLayers/output1500x4500_km" #PARAM1 |
| 74 |
in_dir1b <- "/nobackupp6/aguzman4/climateLayers/output1500x4500_km/singles" #PARAM1, add for now in_dir1 can be a list... |
|
| 75 |
|
|
| 74 | 76 |
|
| 75 | 77 |
region_names <- c("reg1","reg2","reg3","reg4","reg5","reg6") #selected region names, #PARAM2
|
| 78 |
region_namesb <- c("reg_1b","reg_2b","reg_6b") #selected region names, #PARAM2
|
|
| 79 |
|
|
| 76 | 80 |
y_var_name <- "dailyTmax" #PARAM3 |
| 77 | 81 |
interpolation_method <- c("gam_CAI") #PARAM4
|
| 78 |
out_prefix<-"run10_1500x4500_global_analyses_03232015" #PARAM5
|
|
| 82 |
out_prefix<-"run10_1500x4500_global_analyses_03252015" #PARAM5
|
|
| 79 | 83 |
|
| 80 | 84 |
#out_dir<-"/data/project/layers/commons/NEX_data/" #On NCEAS Atlas |
| 81 | 85 |
#out_dir <- "/nobackup/bparmen1/" #on NEX |
| ... | ... | |
| 134 | 138 |
#list of shapefiles used to define tiles |
| 135 | 139 |
in_dir_shp_list <- list.files(in_dir_shp,".shp",full.names=T) |
| 136 | 140 |
|
| 141 |
## load problematic tiles |
|
| 142 |
|
|
| 143 |
in_dir_listb <- list.dirs(path=in_dir1b,recursive=FALSE) #get the list regions processed for this run |
|
| 144 |
#basename(in_dir_list) |
|
| 145 |
in_dir_listb<- lapply(region_namesb,FUN=function(x,y){y[grep(x,basename(y),invert=FALSE)]},y=in_dir_listb)
|
|
| 146 |
|
|
| 147 |
in_dir_list_allb <- lapply(in_dir_listb,function(x){list.dirs(path=x,recursive=F)})
|
|
| 148 |
in_dir_listb <- unlist(in_dir_list_allb) |
|
| 149 |
#in_dir_list <- in_dir_list[grep("bak",basename(basename(in_dir_list)),invert=TRUE)] #the first one is the in_dir1
|
|
| 150 |
in_dir_subsetb <- in_dir_listb[grep("subset",basename(in_dir_listb),invert=FALSE)] #select directory with shapefiles...
|
|
| 151 |
in_dir_shpb <- file.path(in_dir_subsetb,"shapefiles") |
|
| 152 |
|
|
| 153 |
#select only directories used for predictions |
|
| 154 |
in_dir_regb <- in_dir_listb[grep(".*._.*.",basename(in_dir_listb),invert=FALSE)] #select directory with shapefiles...
|
|
| 155 |
#in_dir_reg <- in_dir_list[grep("july_tiffs",basename(in_dir_reg),invert=TRUE)] #select directory with shapefiles...
|
|
| 156 |
in_dir_listb <- in_dir_regb |
|
| 157 |
|
|
| 158 |
in_dir_listb <- in_dir_listb[grep("bak",basename(basename(in_dir_listb)),invert=TRUE)] #the first one is the in_dir1
|
|
| 159 |
#list of shapefiles used to define tiles |
|
| 160 |
in_dir_shp_listb <- list.files(in_dir_shpb,".shp",full.names=T) |
|
| 161 |
|
|
| 162 |
|
|
| 163 |
#### Combine now... |
|
| 164 |
|
|
| 165 |
in_dir_list <- c(in_dir_list,in_dir_listb) |
|
| 166 |
in_dir_reg <- c(in_dir_reg,in_dir_regb) |
|
| 167 |
in_dir_shp <- c(in_dir_shp,in_dir_shpb) |
|
| 168 |
in_dir_shp_list <- c(in_dir_shp_list,in_dir_shp_listb) |
|
| 169 |
#in_dir_list <- c(in_dir_list,in_dir_listb) |
|
| 170 |
|
|
| 137 | 171 |
#system("ls /nobackup/bparmen1")
|
| 138 | 172 |
|
| 139 | 173 |
if(create_out_dir_param==TRUE){
|
| ... | ... | |
| 231 | 265 |
|
| 232 | 266 |
################# |
| 233 | 267 |
###Table 2: daily validation/testing accuracy metrics for all tiles |
| 234 |
#this takes about 25min
|
|
| 268 |
#this takes about 55min
|
|
| 235 | 269 |
#tb_diagnostic_v_list <- lapply(list_raster_obj_files,FUN=function(x){x<-load_obj(x);x[["tb_diagnostic_v"]]})
|
| 236 | 270 |
tb_diagnostic_v_list <- mclapply(list_raster_obj_files,FUN=function(x){try(x<-load_obj(x));try(x[["tb_diagnostic_v"]])},mc.preschedule=FALSE,mc.cores = num_cores)
|
| 237 | 271 |
|
| ... | ... | |
| 250 | 284 |
write.table((tb_diagnostic_v_NA), |
| 251 | 285 |
file=file.path(out_dir,paste("tb_diagnostic_v_NA","_",out_prefix,".txt",sep="")),sep=",")
|
| 252 | 286 |
|
| 287 |
##Take where shutdown took place after pathcing |
|
| 288 |
summary_metrics_v_NA <- read.table(file=file.path(out_dir,paste("summary_metrics_v2_NA_",out_prefix,".txt",sep="")),sep=",")
|
|
| 289 |
#fname <- file.path(out_dir,paste("summary_metrics_v2_NA_",out_suffix,".txt",sep=""))
|
|
| 290 |
tb_diagnostic_v_NA <- read.table(file=file.path(out_dir,paste("tb_diagnostic_v_NA","_",out_prefix,".txt",sep="")),sep=",")
|
|
| 291 |
#tb_diagnostic_s_NA_run10_global_analyses_11302014.txt |
|
| 292 |
#tb_s <- read.table(file=file.path(out_dir,paste("tb_diagnostic_s_NA","_",out_suffix,".txt",sep="")),sep=",")
|
|
| 293 |
|
|
| 294 |
#tb_month_s <- read.table(file=file.path(out_dir,paste("tb_month_diagnostic_s_NA","_",out_suffix,".txt",sep="")),sep=",")
|
|
| 295 |
#pred_data_month_info <- read.table(file=file.path(out_dir,paste("pred_data_month_info_",out_suffix,".txt",sep="")),sep=",")
|
|
| 296 |
#pred_data_day_info <- read.table(file=file.path(out_dir,paste("pred_data_day_info_",out_suffix,".txt",sep="")),sep=",")
|
|
| 297 |
#df_tile_processed <- read.table(file=file.path(out_dir,paste("df_tile_processed_",out_suffix,".txt",sep="")),sep=",")
|
|
| 298 |
|
|
| 253 | 299 |
################# |
| 254 | 300 |
###Table 3: monthly fit/training accuracy information for all tiles |
| 255 | 301 |
|
| ... | ... | |
| 378 | 424 |
#get shape files for the region being assessed: |
| 379 | 425 |
|
| 380 | 426 |
list_shp_world <- list.files(path=in_dir_shp,pattern=".*.shp",full.names=T) |
| 381 |
l_shp <- unlist(lapply(1:length(list_shp_world), |
|
| 382 |
FUN=function(i){paste(strsplit(list_shp_world[i],"_")[[1]][3:4],collapse="_")}))
|
|
| 383 |
l_shp <- gsub(".shp","",l_shp)
|
|
| 427 |
l_shp <- gsub(".shp","",basename(list_shp_world))
|
|
| 428 |
l_shp <- sub("shp_","",l_shp)
|
|
| 429 |
|
|
| 430 |
#l_shp <- unlist(lapply(1:length(list_shp_world), |
|
| 431 |
# FUN=function(i){paste(strsplit(list_shp_world[i],"_")[[1]][3:4],collapse="_")}))
|
|
| 432 |
l_shp <- unlist(lapply(1:length(l_shp), |
|
| 433 |
FUN=function(i){paste(strsplit(l_shp[i],"_")[[1]][1:2],collapse="_")}))
|
|
| 434 |
|
|
| 384 | 435 |
matching_index <- match(basename(in_dir_list),l_shp) |
| 385 | 436 |
list_shp_reg_files <- list_shp_world[matching_index] |
| 386 | 437 |
df_tile_processed$shp_files <-list_shp_world[matching_index] |
Also available in: Unified diff
global asssessment part 1, 1500x4500km including missing tiles in region 1