Project

General

Profile

« Previous | Next » 

Revision b7bbbe32

Added by Benoit Parmentier almost 10 years ago

NEX part2 assessment major clean up, creating global plot and mosaics for 1000x3000km

View differences:

climate/research/oregon/interpolation/global_run_scalingup_assessment_part2.R
360 360
CRS_WGS84<-c("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84
361 361

  
362 362
region_name <- "world"
363
tile_size <- "1000x3000"
363 364

  
364 365
###Table 1: Average accuracy metrics
365 366
###Table 2: daily accuracy metrics for all tiles
......
371 372
tb_s <- read.table(file=file.path(out_dir,paste("tb_diagnostic_s_NA","_",out_prefix,".txt",sep="")),sep=",")
372 373

  
373 374
tb_month_s <- read.table(file=file.path(out_dir,paste("tb_month_diagnostic_s_NA","_",out_prefix,".txt",sep="")),sep=",")
374
pred_data_month_info <- read.table(file=file.path(out_dir,paste("pred_data_month_info_",out_prefix,".txt",sep="")),sep=",")
375
pred_data_day_info <- read.table(file=file.path(out_dir,paste("pred_data_day_info_",out_prefix,".txt",sep="")),sep=",")
375
#pred_data_month_info <- read.table(file=file.path(out_dir,paste("pred_data_month_info_",out_prefix,".txt",sep="")),sep=",")
376
#pred_data_day_info <- read.table(file=file.path(out_dir,paste("pred_data_day_info_",out_prefix,".txt",sep="")),sep=",")
376 377
df_tile_processed <- read.table(file=file.path(out_dir,paste("df_tile_processed_",out_prefix,".txt",sep="")),sep=",")
377 378

  
378 379
########################## START SCRIPT ##############################
379 380

  
381
#add column for tile size later on!!!
382

  
380 383
tb$pred_mod <- as.character(tb$pred_mod)
381 384
summary_metrics_v$pred_mod <- as.character(summary_metrics_v$pred_mod)
382 385
summary_metrics_v$tile_id <- as.character(summary_metrics_v$tile_id)
383 386
df_tile_processed$tile_id <- as.character(df_tile_processed$tile_id)
384 387

  
388
tb_month_s$pred_mod <- as.character(tb_month_s$pred_mod)
389
tb_month_s$tile_id<- as.character(tb_month_s$tile_id)
390
tb_s$pred_mod <- as.character(tb_s$pred_mod)
391
tb_s$tile_id <- as.character(tb_s$tile_id)
392

  
385 393
mulitple_region <- TRUE
386 394

  
387 395
#multiple regions?
388 396
if(mulitple_region==TRUE){
389 397
  df_tile_processed$reg <- basename(dirname(as.character(df_tile_processed$path_NEX)))
390
  tb <- merge(tb,df_tile_processed,by="tile_id")
391 398
  
399
  tb <- merge(tb,df_tile_processed,by="tile_id")
400
  tb_s <- merge(tb_s,df_tile_processed,by="tile_id")
401
  tb_month_s<- merge(tb_month_s,df_tile_processed,by="tile_id")
402
  summary_metrics_v <- merge(summary_metrics_v,df_tile_processed,by="tile_id")
403

  
392 404
}
393 405

  
394 406
###
......
399 411
tb <- subset(tb,reg!="reg3b")
400 412

  
401 413
summary_metrics_v_all <- summary_metrics_v 
414
summary_metrics_v <- subset(summary_metrics_v,reg!="reg3b")
402 415

  
403
tb <- read.table(file=file.path(out_dir,paste("tb_diagnostic_v_NA","_",out_prefix,".txt",sep="")),sep=",")
404
#tb_diagnostic_s_NA_run10_global_analyses_11302014.txt
405
tb_s <- read.table(file=file.path(out_dir,paste("tb_diagnostic_s_NA","_",out_prefix,".txt",sep="")),sep=",")
416
tb_s_all <- tb_s
417
tb_s_all <- subset(tb_s,reg!="reg3b")
406 418

  
407
tb_month_s <- read.table(file=file.path(out_dir,paste("tb_month_diagnostic_s_NA","_",out_prefix,".txt",sep="")),sep=",")
408
pred_data_month_info <- read.table(file=file.path(out_dir,paste("pred_data_month_info_",out_prefix,".txt",sep="")),sep=",")
409
pred_data_day_info <- read.table(file=file.path(out_dir,paste("pred_data_day_info_",out_prefix,".txt",sep="")),sep=",")
410
df_tile_processed <- read.table(file=file.path(out_dir,paste("df_tile_processed_",out_prefix,".txt",sep="")),sep=",")
419
tb_month_s_all <- tb_month_s
420
tb_month_s <- subset(tb_month_s,reg!="reg3b")
421

  
422
df_tile_processed_all <- df_tile_processed
423
df_tile_processed <- subset(df_tile_processed,reg!="reg3b")
424

  
425
#pred_data_month_info_all <- pred_data_month_info
426
#pred_data_month_info <- subset(pred_data_month_info,reg!="reg3b")
427

  
428
#pred_data_month_info_all <- pred_data_month_info
429
#pred_data_month_info <- subset(pred_data_month_info,reg!="reg3b")
430

  
431
#path_reg3 <- "/data/project/layers/commons/NEX_data/output_run8_global_analyses_10212014"
432
#summary_metrics_v_reg3 <- read.table(file.path(path_reg3,"summary_metrics_v2_NA_run8_global_analyses_10212014.txt"),sep=",")
433
#tb_diagnostic_v_NA_run8_global_analyses_10212014.txt
434
#tb_month_diagnostic_s_NA_run8_global_analyses_10212014.txt
435
#tb_diagnostic_s_NA_run8_global_analyses_10212014.txt
436
#pred_data_month_info_run8_global_analyses_10212014.txt
437
#pred_data_day_info_run8_global_analyses_10212014.txt
438
#df_tile_processed_run8_global_analyses_10212014.txt
411 439

  
412 440
###############
413 441
### Figure 1: plot location of the study area with tiles processed
......
706 734
  #plot(ac_mod1,cex=sqrt(ac_mod1$rmse),pch=1,add=T)
707 735
  #plot(ac_mod,cex=(ac_mod$rmse^2)/10,pch=1,col="red",add=T)
708 736

  
709
  coordinates(ac_mod) <- ac_mod[,c("lon","lat")] 
710
  #coordinates(ac_mod) <- ac_mod[,c("lon.x","lat.x")] #solve this later
737
  #coordinates(ac_mod) <- ac_mod[,c("lon","lat")] 
738
  coordinates(ac_mod) <- ac_mod[,c("lon.x","lat.x")] #solve this later
711 739
  p_shp <- layer(sp.polygons(reg_layer, lwd=1, col='black'))
712 740
  #title("(a) Mean for 1 January")
713 741
  p <- bubble(ac_mod,"rmse",main=paste("Averrage RMSE per tile and by ",model_name[i]))
......
729 757
sum(df_tile_processed$metrics_v)/length(df_tile_processed$metrics_v) #62.5% of tiles with info
730 758

  
731 759
#coordinates
732
coordinates(summary_metrics_v) <- c("lon","lat")
733
#coordinates(summary_metrics_v) <- c("lon.y","lat.y")
760
#coordinates(summary_metrics_v) <- c("lon","lat")
761
coordinates(summary_metrics_v) <- c("lon.y","lat.y")
734 762

  
735 763
threshold_missing_day <- c(367,365,300,200)
736 764

  
......
832 860
##########################################################
833 861
##### Figure 8: Breaking down accuracy by regions!! #####
834 862

  
835
summary_metrics_v <- merge(summary_metrics_v,df_tile_processed,by="tile_id")
863
#summary_metrics_v <- merge(summary_metrics_v,df_tile_processed,by="tile_id")
836 864
table(summary_metrics_v$reg)
837 865

  
838 866
## Figure 8a
......
875 903
    full.names=T))
876 904
  )
877 905
}
906
names(lf_mosaics_reg) <- l_reg_name
878 907

  
879 908
#This part should be automated...
880 909
#plot Australia
......
891 920

  
892 921
#lf_m_mask_reg6_1000x3000 <- mclapply(1:length(lf_m),FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 10)
893 922

  
894
#### North America
895
lf_m <- lf_mosaics_reg[[1]]
896
out_dir_str <- out_dir
897
reg_name <- "reg1_1500x4500"
898
#lapply()
899
list_param_plot_daily_mosaics <- list(lf_m=lf_m,reg_name=reg_name,out_dir_str=out_dir_str,out_suffix=out_prefix,l_dates=day_to_mosaic)
900
#lf_m_mask_reg4_1500x4500 <- mclapply(1:2,FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 6)
923
lf_mosaics_mask_reg <- vector("list",length=length(l_reg_name))
924
for(i in 1:length(l_reg_name)){
925
  #
926
  lf_m <- lf_mosaics_reg[[i]]
927
  out_dir_str <- out_dir
928
  reg_name <- paste(l_reg_name[i],"_",tile_size,sep="") #make this automatic
929
  #lapply()
930
  list_param_plot_daily_mosaics <- list(lf_m=lf_m,reg_name=reg_name,out_dir_str=out_dir_str,out_suffix=out_prefix,l_dates=day_to_mosaic)
931
  #lf_m_mask_reg4_1500x4500 <- mclapply(1:2,FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 6)
932

  
933
  lf_mosaics_mask_reg[[i]] <- mclapply(1:length(lf_m),FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 10)
934
}
935
################## WORLD MOSAICS NEEDS MAJOR CLEAN UP OF CODE HERE
936
##make functions!!
937
###Combine mosaics with modified code from Alberto
938

  
939
#use list from above!!
940

  
941
test_list <-list.files(path=out_dir,    
942
           pattern=paste("reg.*._CAI_TMAX_clim_month_20100101_mod1_all_mean.tif",sep=""), 
943
)
944

  
945
test_list<-lapply(1:30,FUN=function(i){lapply(1:x[[i]]},x=lf_mosaics_mask_reg)
946
test_list<-unlist(test_list)
947
mosaic_list_mean <- vector("list",length=1)
948
mosaic_list_mean[[1]] <- test_list 
949
out_rastnames <- "world_test_mosaic_20100101"
950
out_path <- out_dir
951

  
952
list_param_mosaic<-list(mosaic_list_mean,out_path,out_rastnames,file_format,NA_flag_val,out_suffix)
953
names(list_param_mosaic)<-c("mosaic_list","out_path","out_rastnames","file_format","NA_flag_val","out_suffix")
954
#mean_m_list <-mclapply(1:12, list_param=list_param_mosaic, mosaic_m_raster_list,mc.preschedule=FALSE,mc.cores = 6) #This is the end bracket from mclapply(...) statement
955

  
956
lf <- mosaic_m_raster_list(1,list_param_mosaic)
957

  
958
debug(mosaic_m_raster_list)
959
mosaic_list<-list_param$mosaic_list
960
out_path<-list_param$out_path
961
out_names<-list_param$out_rastnames
962
file_format <- list_param$file_format
963
NA_flag_val <- list_param$NA_flag_val
964
out_suffix <- list_param$out_suffix
965

  
966
##Now mosaic for mean: should reorder files!!
967
#out_rastnames_mean<-paste("_",lst_pat,"_","mean",out_suffix,sep="")
968
#list_date_names<-c("jan","feb","mar","apr","may","jun","jul","aug","sep","oct","nov","dec")
969
#mosaic_list<-split(tmp_str1,list_date_names)
970
#new_list<-vector("list",length(mosaic_list))
971
# for (i in 1:length(list_date_names)){
972
#    j<-grep(list_date_names[i],mosaic_list,value=FALSE)
973
#    names(mosaic_list)[j]<-list_date_names[i]
974
#    new_list[i]<-mosaic_list[j]
975
#  }
976
#  mosaic_list_mean <-new_list #list ready for mosaicing
977
#  out_rastnames_mean<-paste(list_date_names,out_rastnames_mean,sep="")
978
  
979
  ### Now mosaic tiles...Note that function could be improved to use less memory
980

  
901 981

  
902
lf_m_mask_reg1_1500x4500 <- mclapply(1:length(lf_m),FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 10)
982
################## PLOTTING WORLD MOSAICS ################
983

  
984
lf_world_pred <- list.files(pattern="world.*2010090.*.tif$")
985

  
986
r_test <- raster(lf_world_pred[6])
987
m <- c(-Inf, -100, NA,  
988
         -100, 100, 1, 
989
         100, Inf, NA) #can change the thresholds later
990
rclmat <- matrix(m, ncol=3, byrow=TRUE)
991
rc <- reclassify(r_test, rclmat,filename=paste("rc_tmp_",i,".tif",sep=""),dataType="FLT4S",overwrite=T)
992
#file_name <- unlist(strsplit(basename(lf_m[i]),"_"))
993
  
994
#date_proc <- file_name[7] #specific tot he current naming of files
995
#date_proc <- l_dates[i]
996
date_proc<- "2010010905"
997

  
998
#paste(raster_name[1:7],collapse="_")
999
#add filename option later
1000
extension_str <- extension(filename(r_test))
1001
raster_name_tmp <- gsub(extension_str,"",basename(filename(r_test)))
1002
raster_name <- file.path(out_dir_str,paste(raster_name_tmp,"_masked.tif",sep=""))
1003
r_pred <- mask(r_test,rc,filename=raster_name,overwrite=TRUE)
1004

  
1005
res_pix <- 1200
1006
#res_pix <- 480
1007

  
1008
col_mfrow <- 1
1009
row_mfrow <- 1
1010
  
1011
png(filename=paste("Figure10_clim_world_mosaics_day_","_",date_proc,"_",tile_size,"_",out_suffix,".png",sep=""),
1012
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
1013

  
1014
  plot(r_pred,main=paste("Predicted on ",date_proc , " ", tile_size,sep=""),cex.main=1.5)
1015
dev.off()
1016

  
1017

  
1018
#lf_world_mask_reg <- vector("list",length=length(lf_world_pred))
1019
#for(i in 1:length(lf_world_pred)){
1020
  #
1021
#  lf_m <- lf_mosaics_reg[i]
1022
#  out_dir_str <- out_dir
1023
  #reg_name <- paste(l_reg_name[i],"_",tile_size,sep="") #make this automatic
1024
  #lapply()
1025
#  list_param_plot_daily_mosaics <- list(lf_m=lf_m,reg_name=tile_size,out_dir_str=out_dir_str,out_suffix=out_prefix,l_dates=day_to_mosaic)
1026
  #lf_m_mask_reg4_1500x4500 <- mclapply(1:2,FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 6)
1027

  
1028
  #lf_world_mask_reg[[i]] <- mclapply(1:length(lf_m),FUN=plot_daily_mosaics,list_param=list_param_plot_daily_mosaics,mc.preschedule=FALSE,mc.cores = 10)
1029
}
903 1030

  
904 1031
##################### END OF SCRIPT ######################

Also available in: Unified diff