Project

General

Profile

« Previous | Next » 

Revision 8bb64f7a

Added by Benoit Parmentier over 9 years ago

global assessment part2, clean up of code, 1500x4500km til size production of figures

View differences:

climate/research/oregon/interpolation/global_run_scalingup_assessment_part2.R
5 5
#Analyses, figures, tables and data are also produced in the script.
6 6
#AUTHOR: Benoit Parmentier 
7 7
#CREATED ON: 03/23/2014  
8
#MODIFIED ON: 02/16/2015            
8
#MODIFIED ON: 03/07/2015            
9 9
#Version: 4
10 10
#PROJECT: Environmental Layers project     
11
#COMMENTS: analyses for run 10 global analyses, Europe, Australia, 1000x300km
11
#COMMENTS: analyses for run 10 global analyses,all regions 1500x4500km and other tiles
12
#TODO:
13
#1) Split functions and master script
14
#2) Make this is a script/function callable from the shell/bash
15
#3) Check image format for tif
16

  
12 17
#################################################################################################
13 18

  
14 19
### Loading R library and packages        
......
160 165
  col_mfrow <- 1
161 166
  row_mfrow <- 1
162 167

  
163
  png(filename=paste("Figure9_",names(r),"_map_processed_region_",region_name,"_",out_prefix,".png",sep=""),
168
  png(filename=paste("Figure9_",names(r),"_map_processed_region_",region_name,"_",out_suffix,".png",sep=""),
164 169
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
165 170

  
166 171
  #plot(reg_layer)
......
319 324
#out_dir <- "/nobackup/bparmen1/" #on NEX
320 325
#in_dir_shp <- "/nobackupp4/aguzman4/climateLayers/output4/subset/shapefiles/"
321 326

  
322
y_var_name <- "dailyTmax"
323
interpolation_method <- c("gam_CAI")
324
#out_prefix<-"run10_global_analyses_01282015"
325
#out_prefix <- "output_run10_1000x3000_global_analyses_02102015"
326
out_prefix <- "run10_1000x3000_global_analyses_02162015"
327
y_var_name <- "dailyTmax" #PARAM1
328
interpolation_method <- c("gam_CAI") #PARAM2
329
#out_suffix<-"run10_global_analyses_01282015"
330
#out_suffix <- "output_run10_1000x3000_global_analyses_02102015"
331
out_suffix <- "run10_1500x4500_global_analyses_03052015" #PARAM3
332
out_dir <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_03052015" #PARAM4
333
create_out_dir_param <- FALSE #PARAM 5
327 334

  
328
mosaic_plot <- FALSE
335
mosaic_plot <- FALSE #PARAM6
329 336

  
337
#if daily mosaics NULL then mosaicas all days of the year
330 338
day_to_mosaic <- c("20100101","20100102","20100103","20100104","20100105",
331 339
                   "20100301","20100302","20100303","20100304","20100305",
332 340
                   "20100501","20100502","20100503","20100504","20100505",
333 341
                   "20100701","20100702","20100703","20100704","20100705",
334 342
                   "20100901","20100902","20100903","20100904","20100905",
335
                   "20101101","20101102","20101103","20101104","20101105")
336

  
343
                   "20101101","20101102","20101103","20101104","20101105") #PARAM7
344
  
337 345
#CRS_locs_WGS84 <- CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84
338
CRS_WGS84 <- CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84
346
CRS_WGS84 <- CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84 #CONSTANT1
347
CRS_locs_WGS84<-CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84
339 348

  
340
proj_str<- CRS_WGS84
341
file_format <- ".rst"
342
NA_value <- -9999
349
proj_str<- CRS_WGS84 #PARAM 8 #check this parameter
350
file_format <- ".rst" #PARAM 9
351
NA_value <- -9999 #PARAM10
343 352
NA_flag_val <- NA_value
344
out_suffix <-out_prefix  
353
                                   
354
tile_size <- "1500x4500" #PARAM 11
355
mulitple_region <- TRUE #PARAM 12
356

  
357
region_name <- "world" #PARAM 13
358

  
359
########################## START SCRIPT ##############################
360

  
361

  
362
####### PART 1: Read in data ########
345 363

  
346
#out_dir <-paste(out_dir,"_",out_prefix,sep="")
347
create_out_dir_param <- FALSE
348
#out_dir <-"/data/project/layers/commons/NEX_data/output_run10_global_analyses_01282015/"
349
out_dir <- "/data/project/layers/commons/NEX_data/output_run10_1000x3000_global_analyses_02162015"
350 364
if(create_out_dir_param==TRUE){
351
  out_dir <- create_dir_fun(out_dir,out_prefix)
365
  out_dir <- create_dir_fun(out_dir,out_suffix)
352 366
  setwd(out_dir)
353 367
}else{
354 368
  setwd(out_dir) #use previoulsy defined directory
355 369
}
356 370

  
357 371
setwd(out_dir)
358
                                   
359
CRS_locs_WGS84<-CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84
360
CRS_WGS84<-c("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84
361

  
362
region_name <- "world"
363
tile_size <- "1000x3000"
364 372

  
365 373
###Table 1: Average accuracy metrics
366 374
###Table 2: daily accuracy metrics for all tiles
367 375

  
368
summary_metrics_v <- read.table(file=file.path(out_dir,paste("summary_metrics_v2_NA_",out_prefix,".txt",sep="")),sep=",")
369
#fname <- file.path(out_dir,paste("summary_metrics_v2_NA_",out_prefix,".txt",sep=""))
370
tb <- read.table(file=file.path(out_dir,paste("tb_diagnostic_v_NA","_",out_prefix,".txt",sep="")),sep=",")
376
summary_metrics_v <- read.table(file=file.path(out_dir,paste("summary_metrics_v2_NA_",out_suffix,".txt",sep="")),sep=",")
377
#fname <- file.path(out_dir,paste("summary_metrics_v2_NA_",out_suffix,".txt",sep=""))
378
tb <- read.table(file=file.path(out_dir,paste("tb_diagnostic_v_NA","_",out_suffix,".txt",sep="")),sep=",")
371 379
#tb_diagnostic_s_NA_run10_global_analyses_11302014.txt
372
tb_s <- read.table(file=file.path(out_dir,paste("tb_diagnostic_s_NA","_",out_prefix,".txt",sep="")),sep=",")
380
tb_s <- read.table(file=file.path(out_dir,paste("tb_diagnostic_s_NA","_",out_suffix,".txt",sep="")),sep=",")
373 381

  
374
tb_month_s <- read.table(file=file.path(out_dir,paste("tb_month_diagnostic_s_NA","_",out_prefix,".txt",sep="")),sep=",")
375
#pred_data_month_info <- read.table(file=file.path(out_dir,paste("pred_data_month_info_",out_prefix,".txt",sep="")),sep=",")
376
#pred_data_day_info <- read.table(file=file.path(out_dir,paste("pred_data_day_info_",out_prefix,".txt",sep="")),sep=",")
377
df_tile_processed <- read.table(file=file.path(out_dir,paste("df_tile_processed_",out_prefix,".txt",sep="")),sep=",")
378

  
379
########################## START SCRIPT ##############################
382
tb_month_s <- read.table(file=file.path(out_dir,paste("tb_month_diagnostic_s_NA","_",out_suffix,".txt",sep="")),sep=",")
383
#pred_data_month_info <- read.table(file=file.path(out_dir,paste("pred_data_month_info_",out_suffix,".txt",sep="")),sep=",")
384
#pred_data_day_info <- read.table(file=file.path(out_dir,paste("pred_data_day_info_",out_suffix,".txt",sep="")),sep=",")
385
df_tile_processed <- read.table(file=file.path(out_dir,paste("df_tile_processed_",out_suffix,".txt",sep="")),sep=",")
380 386

  
381 387
#add column for tile size later on!!!
382 388

  
......
390 396
tb_s$pred_mod <- as.character(tb_s$pred_mod)
391 397
tb_s$tile_id <- as.character(tb_s$tile_id)
392 398

  
393
mulitple_region <- TRUE
394 399

  
395 400
#multiple regions?
396 401
if(mulitple_region==TRUE){
......
403 408

  
404 409
}
405 410

  
406
###
407
"/data/project/layers/commons/NEX_data/output_run8_global_analyses_10212014/tb_diagnostic_v_NA_run8_global_analyses_10212014.txt"
408

  
409
#drop 3b
410 411
tb_all <- tb
411
#tb <- subset(tb,reg!="reg3b")
412 412

  
413 413
summary_metrics_v_all <- summary_metrics_v 
414
#summary_metrics_v <- subset(summary_metrics_v,reg!="reg3b")
415

  
416
#tb_s_all <- tb_s
417
#tb_s_all <- subset(tb_s,reg!="reg3b")
418

  
419
#tb_month_s_all <- tb_month_s
420
#tb_month_s <- subset(tb_month_s,reg!="reg3b")
421

  
422
#df_tile_processed_all <- df_tile_processed
423
#df_tile_processed <- subset(df_tile_processed,reg!="reg3b")
424

  
425
#pred_data_month_info_all <- pred_data_month_info
426
#pred_data_month_info <- subset(pred_data_month_info,reg!="reg3b")
427

  
428
#pred_data_month_info_all <- pred_data_month_info
429
#pred_data_month_info <- subset(pred_data_month_info,reg!="reg3b")
430 414

  
431
#path_reg3 <- "/data/project/layers/commons/NEX_data/output_run8_global_analyses_10212014"
432
#summary_metrics_v_reg3 <- read.table(file.path(path_reg3,"summary_metrics_v2_NA_run8_global_analyses_10212014.txt"),sep=",")
433
#tb_diagnostic_v_NA_run8_global_analyses_10212014.txt
434
#tb_month_diagnostic_s_NA_run8_global_analyses_10212014.txt
435
#tb_diagnostic_s_NA_run8_global_analyses_10212014.txt
436
#pred_data_month_info_run8_global_analyses_10212014.txt
437
#pred_data_day_info_run8_global_analyses_10212014.txt
438
#df_tile_processed_run8_global_analyses_10212014.txt
415
############ PART 2: PRODUCE FIGURES ################
439 416

  
440
###############
417
###########################
441 418
### Figure 1: plot location of the study area with tiles processed
442 419

  
443 420
#df_tiled_processed <- na.omit(df_tile_processed) #remove other list of folders irrelevant
......
448 425
list_shp_reg_files <- file.path("/data/project/layers/commons/NEX_data/",out_dir,
449 426
          "shapefiles",basename(list_shp_reg_files))
450 427

  
428

  
429
### Potential function starts here:
430
#function(in_dir,out_dir,list_shp_reg_files,title_str,region_name,num_cores,out_suffix,out_suffix)
431

  
451 432
### First get background map to display where study area is located
452
#can make this more general later on..       
433
#can make this more general later on..should have this already in a local directory on Atlas or NEX!!!!
434

  
453 435
if (region_name=="USA"){
454 436
  usa_map <- getData('GADM', country='USA', level=1) #Get US map
455 437
  #usa_map <- getData('GADM', country=region_name,level=1) #Get US map, this is not working right now
......
472 454
#collect info: read in all shapfiles
473 455
#This is slow...make a function and use mclapply??
474 456
#/data/project/layers/commons/NEX_data/output_run6_global_analyses_09162014/shapefiles
475

  
457
  
476 458
for(i in 1:length(list_shp_reg_files)){
477 459
  #path_to_shp <- dirname(list_shp_reg_files[[i]])
478 460
  path_to_shp <- file.path(out_dir,"/shapefiles")
......
495 477

  
496 478
#fun <- function(i,list_shp_files)
497 479
#coord_names <- c("lon","lat")
498
#l_ras#t <- rasterize_df_fun(test,coord_names,proj_str,out_suffix=out_prefix,out_dir=".",file_format,NA_flag_val,tolerance_val=0.000120005)
480
#l_ras#t <- rasterize_df_fun(test,coord_names,proj_str,out_suffix=out_suffix,out_dir=".",file_format,NA_flag_val,tolerance_val=0.000120005)
499 481

  
500 482
#remove try-error polygons...we loose three tiles because they extend beyond -180 deg
501 483
tmp <- shps_tiles
......
512 494
col_mfrow <- 1 
513 495
row_mfrow <- 1
514 496

  
515
png(filename=paste("Figure1_tile_processed_region_",region_name,"_",out_prefix,".png",sep=""),
497
png(filename=paste("Figure1_tile_processed_region_",region_name,"_",out_suffix,".png",sep=""),
516 498
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
517 499

  
518 500
plot(reg_layer)
......
552 534
  col_mfrow <- 1
553 535
  row_mfrow <- 1
554 536

  
555
  png(filename=paste("Figure2a_boxplot_with_oultiers_by_tiles_",model_name[i],"_",out_prefix,".png",sep=""),
537
  png(filename=paste("Figure2a_boxplot_with_oultiers_by_tiles_",model_name[i],"_",out_suffix,".png",sep=""),
556 538
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
557 539
  
558 540
  boxplot(rmse~tile_id,data=subset(tb,tb$pred_mod==model_name[i]))
......
568 550
  res_pix <- 480
569 551
  col_mfrow <- 1
570 552
  row_mfrow <- 1
571
  png(filename=paste("Figure2b_boxplot_scaling_by_tiles","_",model_name[i],"_",out_prefix,".png",sep=""),
553
  png(filename=paste("Figure2b_boxplot_scaling_by_tiles","_",model_name[i],"_",out_suffix,".png",sep=""),
572 554
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
573 555

  
574 556
  model_name <- unique(tb$pred_mod)
......
587 569
col_mfrow <- 1
588 570
row_mfrow <- 1
589 571

  
590
png(filename=paste("Figure3a_boxplot_overall_region_with_oultiers_",model_name[i],"_",out_prefix,".png",sep=""),
572
png(filename=paste("Figure3a_boxplot_overall_region_with_oultiers_",model_name[i],"_",out_suffix,".png",sep=""),
591 573
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
592 574

  
593 575
boxplot(rmse~pred_mod,data=tb)#,names=tb$pred_mod)
......
596 578
dev.off()
597 579

  
598 580
## Figure 3b
599
png(filename=paste("Figure3b_boxplot_overall_region_scaling_",model_name[i],"_",out_prefix,".png",sep=""),
581
png(filename=paste("Figure3b_boxplot_overall_region_scaling_",model_name[i],"_",out_suffix,".png",sep=""),
600 582
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
601 583

  
602 584
boxplot(rmse~pred_mod,data=tb,ylim=c(0,5),outline=FALSE)#,names=tb$pred_mod)
......
628 610
#     col_mfrow <- 1
629 611
#     row_mfrow <- 1
630 612
#   
631
#     png(filename=paste("Figure4_models_predicted_surfaces_",model_name[i],"_",name_method_var,"_",data_selected,"_",out_prefix,".png",sep=""),
613
#     png(filename=paste("Figure4_models_predicted_surfaces_",model_name[i],"_",name_method_var,"_",data_selected,"_",out_suffix,".png",sep=""),
632 614
#        width=col_mfrow*res_pix,height=row_mfrow*res_pix)
633 615
#   
634 616
#     plot(r_pred)
......
659 641
  col_mfrow <- 1
660 642
  row_mfrow <- 1
661 643

  
662
  png(filename=paste("Figure5_ac_metrics_ranked_",model_name[i],"_",out_prefix,".png",sep=""),
644
  png(filename=paste("Figure5_ac_metrics_ranked_",model_name[i],"_",out_suffix,".png",sep=""),
663 645
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
664 646
  x<- as.character(df_ac_mod$tile_id)
665 647
  barplot(df_ac_mod$rmse, names.arg=x)
......
689 671
#   col_mfrow <- 1
690 672
#   row_mfrow <- 1
691 673
# 
692
#   png(filename=paste("Figure6_ac_metrics_map_centroids_tile_",model_name[i],"_",out_prefix,".png",sep=""),
674
#   png(filename=paste("Figure6_ac_metrics_map_centroids_tile_",model_name[i],"_",out_suffix,".png",sep=""),
693 675
#     width=col_mfrow*res_pix,height=row_mfrow*res_pix)
694 676
# 
695 677
#   plot(r_pred)  
......
726 708
  col_mfrow <- 1
727 709
  row_mfrow <- 1
728 710

  
729
  png(filename=paste("Figure6_ac_metrics_map_centroids_tile_",model_name[i],"_",out_prefix,".png",sep=""),
711
  png(filename=paste("Figure6_ac_metrics_map_centroids_tile_",model_name[i],"_",out_suffix,".png",sep=""),
730 712
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
731 713

  
732 714
  #plot(r_pred)  
......
785 767
  row_mfrow <- 1
786 768

  
787 769
  png(filename=paste("Figure7a_ac_metrics_map_centroids_tile_",model_name[j],"_","missing_day_",threshold_missing_day[i],
788
                     "_",out_prefix,".png",sep=""),
770
                     "_",out_suffix,".png",sep=""),
789 771
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
790 772
  
791 773
  model_name[j]
......
815 797

  
816 798
# 
817 799
## Figure 7a
818
png(filename=paste("Figure7a_number_daily_predictions_per_models","_",out_prefix,".png",sep=""),
800
png(filename=paste("Figure7a_number_daily_predictions_per_models","_",out_suffix,".png",sep=""),
819 801
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
820 802

  
821 803
xyplot(n~pred_mod | tile_id,data=subset(as.data.frame(summary_metrics_v),
......
844 826
histogram(test$predicted~test$tile_id)
845 827
#table(tb)
846 828
## Figure 7b
847
#png(filename=paste("Figure7b_number_daily_predictions_per_models","_",out_prefix,".png",sep=""),
829
#png(filename=paste("Figure7b_number_daily_predictions_per_models","_",out_suffix,".png",sep=""),
848 830
#    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
849 831

  
850 832
#xyplot(n~month | tile_id + pred_mod,data=subset(as.data.frame(tb_month_s),
......
868 850
col_mfrow <- 1
869 851
row_mfrow <- 1
870 852

  
871
png(filename=paste("Figure8a_boxplot_overall_separated_by_region_with_oultiers_",model_name[i],"_",out_prefix,".png",sep=""),
853
png(filename=paste("Figure8a_boxplot_overall_separated_by_region_with_oultiers_",model_name[i],"_",out_suffix,".png",sep=""),
872 854
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
873 855

  
874 856
p<- bwplot(rmse~pred_mod | reg, data=tb,
......
877 859
dev.off()
878 860

  
879 861
## Figure 8b
880
png(filename=paste("Figure8b_boxplot_overall_separated_by_region_scaling_",model_name[i],"_",out_prefix,".png",sep=""),
862
png(filename=paste("Figure8b_boxplot_overall_separated_by_region_scaling_",model_name[i],"_",out_suffix,".png",sep=""),
881 863
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
882 864

  
883 865
boxplot(rmse~pred_mod,data=tb,ylim=c(0,5),outline=FALSE)#,names=tb$pred_mod)
......
911 893
#out_dir_str <- out_dir
912 894
#reg_name <- "reg6_1000x3000"
913 895
#lapply()
914
#list_param_plot_daily_mosaics <- list(lf_m=lf_m,reg_name=reg_name,out_dir_str=out_dir_str,out_suffix=out_prefix)
915
#list_param_plot_daily_mosaics <- list(lf_m=lf_m,reg_name=reg_name,out_dir_str=out_dir_str,out_suffix=out_prefix,l_dates=day_to_mosaic)
896
#list_param_plot_daily_mosaics <- list(lf_m=lf_m,reg_name=reg_name,out_dir_str=out_dir_str,out_suffix=out_suffix)
897
#list_param_plot_daily_mosaics <- list(lf_m=lf_m,reg_name=reg_name,out_dir_str=out_dir_str,out_suffix=out_suffix,l_dates=day_to_mosaic)
916 898

  
917 899
#lf_m_mask_reg4_1500x4500 <- mclapply(1:2,FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 6)
918 900
#debug(plot_daily_mosaics)
......
927 909
  out_dir_str <- out_dir
928 910
  reg_name <- paste(l_reg_name[i],"_",tile_size,sep="") #make this automatic
929 911
  #lapply()
930
  list_param_plot_daily_mosaics <- list(lf_m=lf_m,reg_name=reg_name,out_dir_str=out_dir_str,out_suffix=out_prefix,l_dates=day_to_mosaic)
912
  list_param_plot_daily_mosaics <- list(lf_m=lf_m,reg_name=reg_name,out_dir_str=out_dir_str,out_suffix=out_suffix,l_dates=day_to_mosaic)
931 913
  #lf_m_mask_reg4_1500x4500 <- mclapply(1:2,FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 6)
932 914

  
933 915
  lf_mosaics_mask_reg[[i]] <- mclapply(1:length(lf_m),FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 10)
......
1058 1040
#  out_dir_str <- out_dir
1059 1041
  #reg_name <- paste(l_reg_name[i],"_",tile_size,sep="") #make this automatic
1060 1042
  #lapply()
1061
#  list_param_plot_daily_mosaics <- list(lf_m=lf_m,reg_name=tile_size,out_dir_str=out_dir_str,out_suffix=out_prefix,l_dates=day_to_mosaic)
1043
#  list_param_plot_daily_mosaics <- list(lf_m=lf_m,reg_name=tile_size,out_dir_str=out_dir_str,out_suffix=out_suffix,l_dates=day_to_mosaic)
1062 1044
  #lf_m_mask_reg4_1500x4500 <- mclapply(1:2,FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 6)
1063 1045

  
1064 1046
  #lf_world_mask_reg[[i]] <- mclapply(1:length(lf_m),FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 10)

Also available in: Unified diff