Revision 52dfbd74
Added by Benoit Parmentier about 8 years ago
climate/research/oregon/interpolation/global_product_assessment_part0.R | ||
---|---|---|
4 | 4 |
#This part 2 of the assessment focuses on graphics to explore the spatial patterns of raster times series as figures and movie |
5 | 5 |
#AUTHOR: Benoit Parmentier |
6 | 6 |
#CREATED ON: 10/03/2016 |
7 |
#MODIFIED ON: 10/24/2016
|
|
7 |
#MODIFIED ON: 10/27/2016
|
|
8 | 8 |
#Version: 1 |
9 | 9 |
#PROJECT: Environmental Layers project |
10 | 10 |
#COMMENTS: Initial commit, script based on part NASA biodiversity conferenc |
... | ... | |
260 | 260 |
table(df_time_series$missing) |
261 | 261 |
table(df_time_series$year) |
262 | 262 |
|
263 |
############################# |
|
264 |
##### Creating animation based on prediction |
|
265 |
|
|
266 |
##### |
|
267 |
NAvalue(r_stack) |
|
268 |
plot(r_stack,y=6,zlim=c(-10000,10000)) #this is not rescaled |
|
269 |
#plot(r_stack,zlim=c(-50,50),col=matlab.like(255)) |
|
270 |
var_name <- "dailyTmax" |
|
271 |
|
|
272 |
#debug(plot_and_animate_raster_time_series) |
|
273 |
|
|
274 |
#metric_name <- "var_pred" #use RMSE if accuracy |
|
275 |
#df_raster <- read.table("df_raster_global_assessment_reg6_10102016.txt",sep=",",header=T) |
|
276 |
#plot_figure <- |
|
277 |
#function_product_assessment_part2_functions <- "global_product_assessment_part2_functions_10222016.R" |
|
278 |
#source(file.path(script_path,function_product_assessment_part2_functions)) #source all functions used in this script |
|
279 |
|
|
280 |
#undebug(plot_and_animate_raster_time_series) |
|
281 |
range_year <- c(1984,1985) |
|
282 |
subset_df_time_series <- subset(df_time_series,year%in% range_year) |
|
283 |
subset_df_time_series <- subset_df_time_series[!is.na(subset_df_time_series$lf),] |
|
284 |
|
|
285 |
lf_subset <- file.path(subset_df_time_series$dir,subset_df_time_series$lf) |
|
286 |
range_year_str <- paste(range_year, sep = "_", collapse = "_") |
|
287 |
|
|
288 |
out_suffix_str <- paste(range_year_str,out_suffix,sep="_") |
|
289 |
|
|
290 |
#started on 10/22/2016 at 9.57 |
|
291 |
animation_obj <- plot_and_animate_raster_time_series(lf_subset, |
|
292 |
item_no, |
|
293 |
region_name, |
|
294 |
var_name, |
|
295 |
metric_name, |
|
296 |
NA_flag_val, |
|
297 |
filenames_figures=NULL, |
|
298 |
frame_speed=60, |
|
299 |
animation_format=".gif", |
|
300 |
zlim_val=NULL, |
|
301 |
plot_figure=T, |
|
302 |
generate_animation=T, |
|
303 |
num_cores=num_cores, |
|
304 |
out_suffix=out_suffix_str, |
|
305 |
out_dir=out_dir) |
|
306 |
|
|
307 |
zlim_val <- c(-2000,5000) |
|
308 |
animation_obj <- plot_and_animate_raster_time_series(lf_subset, |
|
309 |
item_no, |
|
310 |
region_name, |
|
311 |
var_name, |
|
312 |
metric_name, |
|
313 |
NA_flag_val, |
|
314 |
filenames_figures=NULL, |
|
315 |
frame_speed=60, |
|
316 |
animation_format=".gif", |
|
317 |
zlim_val=zlim_val, |
|
318 |
plot_figure=T, |
|
319 |
generate_animation=T, |
|
320 |
num_cores=num_cores, |
|
321 |
out_suffix=out_suffix_str, |
|
322 |
out_dir=out_dir) |
|
323 |
|
|
324 |
#ffmpeg -i yeay.gif outyeay.mp4 |
|
325 |
|
|
326 |
#/Applications/ffmpeg -r 25 -i input%3d.png -vcodec libx264 -x264opts keyint=25 -pix_fmt yuv420p -r 25 ../output.mp4 |
|
327 |
|
|
328 |
#ffmpeg -f gif -i file.gif -c:v libx264 outfile.mp4 |
|
329 |
|
|
330 |
#ffmpeg -i animation_frame_60_-2500_6000_.gif animation_frame_60_-2500_6000_.mp4 |
|
331 |
|
|
332 |
#ffmpeg -i animation_frame_60_-2500_6000_.gif animation_frame_60_-2500_6000_.mp4 |
|
333 |
|
|
334 |
#ffmpeg -f gif -i animation_frame_60_-2500_6000_.gif -vcodec libx264 -x264opts keyint=25 -pix_fmt yuv420p -r 25 outfile.mp4 |
|
335 |
#ffmpeg -f gif -i animation_frame_60_-2500_6000_.gif -vcodec libx264 -x264opts keyint=11 -pix_fmt yuv420p -r 11 outfile.mp4 |
|
336 |
|
|
337 |
#ffmpeg -r 10 -i animation_frame_60_-2500_6000_.gif animation.avi |
|
338 |
|
|
339 |
#ffmpeg -f gif -i animation_frame_60_-2500_6000_.gif -vcodec libx264 -x264opts -pix_fmt yuv420p outfile.mp4 |
|
340 | 263 |
|
341 | 264 |
#combine polygon |
342 | 265 |
#http://gis.stackexchange.com/questions/155328/merging-multiple-spatialpolygondataframes-into-1-spdf-in-r |
... | ... | |
365 | 288 |
|
366 | 289 |
#On NEX |
367 | 290 |
#contains all data from the run by Alberto |
368 |
#in_dir1 <- " /nobackupp6/aguzman4/climateLayers/out_15x45/" #On NEX
|
|
291 |
in_dir1 <- "/nobackupp6/aguzman4/climateLayers/out" #On NEX
|
|
369 | 292 |
#parent output dir for the current script analyes |
370 | 293 |
#out_dir <- "/nobackup/bparmen1/" #on NEX |
371 | 294 |
#in_dir_shp <- "/nobackupp4/aguzman4/climateLayers/output4/subset/shapefiles/" |
... | ... | |
376 | 299 |
# "output_run_global_analyses_pred_2011_reg4","output_run_global_analyses_pred_2012_reg4", |
377 | 300 |
# "output_run_global_analyses_pred_2013_reg4","output_run_global_analyses_pred_2014_reg4") |
378 | 301 |
#in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt" |
379 |
#in_dir <- "" #PARAM 0
|
|
380 |
#y_var_name <- "dailyTmax" #PARAM1
|
|
381 |
#interpolation_method <- c("gam_CAI") #PARAM2
|
|
382 |
#out_suffix <- "global_analyses_overall_assessment_reg4_02072016"
|
|
302 |
in_dir <- "" #PARAM 0 |
|
303 |
y_var_name <- "dailyTmax" #PARAM1 |
|
304 |
interpolation_method <- c("gam_CAI") #PARAM2 |
|
305 |
out_suffix <- "predictions_assessment_reg6_10302016"
|
|
383 | 306 |
#out_suffix <- "output_run10_1000x3000_global_analyses_02102015" |
384 | 307 |
#out_suffix <- "run10_1500x4500_global_analyses_pred_1992_10052015" #PARAM3 |
385 | 308 |
#out_dir <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_10052015" #PARAM4 |
... | ... | |
391 | 314 |
#CRS_locs_WGS84<-CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84 |
392 | 315 |
#proj_str<- CRS_WGS84 #PARAM 8 #check this parameter |
393 | 316 |
#file_format <- ".rst" #PARAM 9 |
394 |
#NA_value <- -9999 #PARAM10
|
|
395 |
#NA_flag_val <- NA_value
|
|
317 |
NA_value <- -9999 #PARAM10 |
|
318 |
NA_flag_val <- NA_value |
|
396 | 319 |
#multiple_region <- TRUE #PARAM 12 |
397 |
#region_name <- "world" #PARAM 13
|
|
398 |
#countries_shp <-"/data/project/layers/commons/NEX_data/countries.shp" #PARAM 13, copy this on NEX too
|
|
320 |
region_name <- "reg6" #PARAM 13
|
|
321 |
countries_shp <-"/data/project/layers/commons/NEX_data/countries.shp" #PARAM 13, copy this on NEX too |
|
399 | 322 |
#plot_region <- TRUE |
400 |
#num_cores <- 6 #PARAM 14 |
|
401 |
#region_name <- c("reg4") #reference region to merge if necessary, if world all the regions are together #PARAM 16 |
|
402 |
#use previous files produced in step 1a and stored in a data.frame |
|
403 |
#df_assessment_files <- "df_assessment_files_reg4_1984_run_global_analyses_pred_12282015.txt" #PARAM 17 |
|
404 |
#threshold_missing_day <- c(367,365,300,200) #PARM18 |
|
405 |
|
|
406 |
#list_param_run_assessment_plottingin_dir <- list(in_dir,y_var_name, interpolation_method, out_suffix, |
|
407 |
# out_dir, create_out_dir_param, mosaic_plot, proj_str, file_format, NA_value, |
|
408 |
# multiple_region, countries_shp, plot_region, num_cores, |
|
409 |
# region_name, df_assessment_files, threshold_missing_day) |
|
410 |
|
|
411 |
#names(list_param_run_assessment_plottingin_dir) <- c("in_dir","y_var_name","interpolation_method","out_suffix", |
|
412 |
# "out_dir","create_out_dir_param","mosaic_plot","proj_str","file_format","NA_value", |
|
413 |
# "multiple_region","countries_shp","plot_region","num_cores", |
|
414 |
# "region_name","df_assessment_files","threshold_missing_day") |
|
415 |
|
|
416 |
#run_assessment_plotting_prediction_fun(list_param_run_assessment_plottingin_dir) |
|
417 |
|
|
418 |
####### PARSE INPUT ARGUMENTS/PARAMETERS ##### |
|
419 |
in_dir_list_filename <- list_param_run_assessment_plotting$in_dir_list_filename #PARAM 0 |
|
420 |
in_dir <- list_param_run_assessment_plotting$in_dir #PARAM 1 |
|
421 |
y_var_name <- list_param_run_assessment_plotting$y_var_name #PARAM2 |
|
422 |
interpolation_method <- list_param_run_assessment_plotting$interpolation_method #c("gam_CAI") #PARAM3 |
|
423 |
out_suffix <- list_param_run_assessment_plotting$out_suffix #PARAM4 |
|
424 |
out_dir <- list_param_run_assessment_plotting$out_dir # PARAM5 |
|
425 |
create_out_dir_param <- list_param_run_assessment_plotting$create_out_dir_param # FALSE #PARAM 6 |
|
426 |
mosaic_plot <- list_param_run_assessment_plotting$mosaic_plot #FALSE #PARAM7 |
|
427 |
proj_str<- list_param_run_assessment_plotting$proj_str #CRS_WGS84 #PARAM 8 #check this parameter |
|
428 |
file_format <- list_param_run_assessment_plotting$file_format #".rst" #PARAM 9 |
|
429 |
NA_flag_val <- list_param_run_assessment_plotting$NA_flag_val #-9999 #PARAM10 |
|
430 |
multiple_region <- list_param_run_assessment_plotting$multiple_region # <- TRUE #PARAM 11 |
|
431 |
countries_shp <- list_param_run_assessment_plotting$countries_shp #<- "world" #PARAM 12 |
|
432 |
plot_region <- list_param_run_assessment_plotting$plot_region # PARAM13 |
|
433 |
num_cores <- list_param_run_assessment_plotting$num_cores # 6 #PARAM 14 |
|
434 |
region_name <- list_param_run_assessment_plotting$region_name #<- "world" #PARAM 15 |
|
435 |
#df_assessment_files_name <- list_param_run_assessment_plotting$df_assessment_files_name #PARAM 16 |
|
436 |
threshold_missing_day <- list_param_run_assessment_plotting$threshold_missing_day #PARM17 |
|
437 |
year_predicted <- list_param_run_assessment_plotting$year_predicted |
|
438 |
|
|
439 |
NA_value <- NA_flag_val |
|
440 |
metric_name <- "rmse" #to be added to the code later... |
|
323 |
num_cores <- 6 #PARAM 14 |
|
324 |
#/nobackupp6/aguzman4/climateLayers/out/reg6/subset/shapefiles |
|
325 |
list_year_predicted <- c(2000,2012,2013) #year still on disk for reg6 |
|
441 | 326 |
|
442 |
##################### START SCRIPT ################# |
|
327 |
#from script: |
|
328 |
#interpolation/global_run_scalingup_assessment_part1a.R |
|
329 |
|
|
330 |
############################## |
|
331 |
#### Parameters and constants |
|
443 | 332 |
|
444 |
####### PART 1: Read in data ######## |
|
445 |
out_dir <- in_dir |
|
446 |
if(create_out_dir_param==TRUE){ |
|
447 |
out_dir <- create_dir_fun(out_dir,out_suffix) |
|
448 |
setwd(out_dir) |
|
449 |
}else{ |
|
450 |
setwd(out_dir) #use previoulsy defined directory |
|
451 |
} |
|
452 | 333 |
|
453 |
setwd(out_dir) |
|
334 |
in_dir1 <- list_param_run_assessment_prediction$in_dir1 |
|
335 |
region_name <- list_param_run_assessment_prediction$region_name #e.g. c("reg23","reg4") #run only for one region |
|
336 |
y_var_name <- list_param_run_assessment_prediction$y_var_name # e.g. dailyTmax" #PARAM3 |
|
337 |
interpolation_method <- list_param_run_assessment_prediction$interpolation_method #c("gam_CAI") #PARAM4 |
|
338 |
out_prefix <- list_param_run_assessment_prediction$out_prefix #output suffix e.g."run_global_analyses_pred_12282015" #PARAM5 |
|
339 |
out_dir <- list_param_run_assessment_prediction$out_dir #<- "/nobackupp8/bparmen1/" #PARAM6 |
|
340 |
create_out_dir_param <-list_param_run_assessment_prediction$create_out_dir_param #if TRUE output dir created #PARAM7 |
|
341 |
proj_str <- list_param_run_assessment_prediction$proj_str # CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84, #PARAM8 |
|
342 |
list_year_predicted <- list_param_run_assessment_prediction$list_year_predicted # 1984:2004 |
|
343 |
file_format <- list_param_run_assessment_prediction$file_format #<- ".tif" #format for mosaiced files #PARAM10 |
|
344 |
NA_flag_val <- list_param_run_assessment_prediction$NA_flag_val #<- -9999 #No data value, #PARAM11 |
|
345 |
num_cores <- list_param_run_assessment_prediction$num_cores #<- 6 #number of cores used #PARAM13 |
|
346 |
plotting_figures <- list_param_run_assessment_prediction$plotting_figures #if true run part2 of assessment |
|
347 |
|
|
348 |
##for plotting assessment function |
|
349 |
|
|
350 |
mosaic_plot <- list_param_run_assessment_prediction$mosaic_plot #PARAM14 |
|
351 |
day_to_mosaic <- list_param_run_assessment_prediction$day_to_mosaic #PARAM15 |
|
352 |
multiple_region <- list_param_run_assessment_prediction$multiple_region #PARAM16 |
|
353 |
countries_shp <- list_param_run_assessment_prediction$countries_shp #PARAM17 |
|
354 |
plot_region <- list_param_run_assessment_prediction$plot_region #PARAM18 |
|
355 |
threshold_missing_day <- list_param_run_assessment_prediction$threshold_missing_day #PARM20 |
|
356 |
|
|
357 |
########################## START SCRIPT ######################################### |
|
358 |
|
|
359 |
#Need to make this a function to run as a job... |
|
360 |
|
|
361 |
######################## PART0: Read content of predictions first.... ##### |
|
362 |
#function looped over i, correspoding to year predicted |
|
454 | 363 |
|
455 | 364 |
list_outfiles <- vector("list", length=35) #collect names of output files, this should be dynamic? |
456 | 365 |
list_outfiles_names <- vector("list", length=35) #collect names of output files |
457 |
counter_fig <- 0 #index of figure to collect outputs |
|
366 |
|
|
367 |
|
|
368 |
#### STart here |
|
369 |
###### This is from assessment 1 |
|
458 | 370 |
|
459 |
#i <- year_predicted |
|
460 |
###Table 1: Average accuracy metrics |
|
461 |
###Table 2: daily accuracy metrics for all tiles |
|
371 |
#for each polygon find you overlap!! |
|
372 |
#plot number of overlap |
|
373 |
#for specific each find prediction... |
|
374 |
year_predicted <- list_param_run_assessment_prediction$list_year_predicted[i] |
|
462 | 375 |
|
463 |
if(!is.null(in_dir_list_filename)){ |
|
464 |
in_dir_list <- as.list(read.table(in_dir_list_filename,stringsAsFactors=F)[,1]) |
|
376 |
in_dir1 <- file.path(in_dir1,region_name) |
|
377 |
|
|
378 |
list_outfiles <- vector("list", length=14) #collect names of output files |
|
379 |
|
|
380 |
in_dir_list <- list.dirs(path=in_dir1,recursive=FALSE) #get the list regions processed for this run |
|
381 |
#basename(in_dir_list) |
|
382 |
# y=in_dir_list) |
|
383 |
|
|
384 |
#in_dir_list_all <- unlist(lapply(in_dir_list,function(x){list.dirs(path=x,recursive=F)})) |
|
385 |
in_dir_list_all <- in_dir_list |
|
386 |
in_dir_subset <- in_dir_list_all[grep("subset",basename(in_dir_list_all),invert=FALSE)] #select directory with shapefiles... |
|
387 |
in_dir_shp <- file.path(in_dir_subset,"shapefiles") |
|
388 |
|
|
389 |
|
|
390 |
#select only directories used for predictions |
|
391 |
#nested structure, we need to go to higher level to obtain the tiles... |
|
392 |
in_dir_reg <- in_dir_list[grep(".*._.*.",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles... |
|
393 |
#in_dir_reg <- in_dir_list[grep("july_tiffs",basename(in_dir_reg),invert=TRUE)] #select directory with shapefiles... |
|
394 |
in_dir_list <- in_dir_reg |
|
395 |
|
|
396 |
|
|
397 |
in_dir_list <- in_dir_list[grep("bak",basename(basename(in_dir_list)),invert=TRUE)] #the first one is the in_dir1 |
|
398 |
#list of shapefiles used to define tiles |
|
399 |
in_dir_shp_list <- list.files(in_dir_shp,".shp",full.names=T) |
|
400 |
|
|
401 |
## load problematic tiles or additional runs |
|
402 |
#modify later... |
|
403 |
|
|
404 |
#system("ls /nobackup/bparmen1") |
|
405 |
out_dir <- in_dir |
|
406 |
if(create_out_dir_param==TRUE){ |
|
407 |
out_dir <- create_dir_fun(out_dir,out_prefix) |
|
408 |
setwd(out_dir) |
|
465 | 409 |
}else{ |
466 |
pattern_str <- paste0("^output_",region_name,".*.") |
|
467 |
in_dir_list_all <- list.dirs(path=in_dir,recursive = T) |
|
468 |
in_dir_list <- in_dir_list_all[grep(pattern_str,basename(in_dir_list_all),invert=FALSE)] #select directory with shapefiles... |
|
469 |
#in_dir_shp <- file.path(in_dir_list_all,"shapefiles") |
|
410 |
setwd(out_dir) #use previoulsy defined directory |
|
470 | 411 |
} |
471 |
#pattern_str <- file.path(in_dir,paste0("output_",region_name,".*.")) |
|
472 |
#test <- Sys.glob(pattern_str,FALSE) |
|
473 |
# searchStr = paste(in_dir_tiles_tmp,"/*/",year_processed,"/gam_CAI_dailyTmax_predicted_",pred_mod_name,"*",day_to_mosaic[i],"*.tif",sep="") |
|
474 |
# #print(searchStr) |
|
475 |
# Sys.glob(searchStr)}) |
|
476 |
|
|
477 |
#lf_mosaic <- lapply(1:length(day_to_mosaic),FUN=function(i){ |
|
478 |
# searchStr = paste(in_dir_tiles_tmp,"/*/",year_processed,"/gam_CAI_dailyTmax_predicted_",pred_mod_name,"*",day_to_mosaic[i],"*.tif",sep="") |
|
479 |
# #print(searchStr) |
|
480 |
# Sys.glob(searchStr)}) |
|
481 |
|
|
482 |
##Read in data list from in_dir_list |
|
483 |
#list_tb_fname <- list.files(path=file.path(in_dir,in_dir_list),"tb_diagnostic_v_NA_.*.txt",full.names=T) |
|
484 |
#list_df_fname <- list.files(path=file.path(in_dir,in_dir_list),"df_tile_processed_.*..txt",full.names=T) |
|
485 |
#list_summary_metrics_v_fname <- list.files(path=file.path(in_dir,in_dir_list),"summary_metrics_v2_NA_.*.txt",full.names=T) |
|
486 |
#list_tb_s_fname <- list.files(path=file.path(in_dir,in_dir_list),"tb_diagnostic_s_NA.*.txt",full.names=T) |
|
487 |
#list_tb_month_s_fname <- list.files(path=file.path(in_dir,in_dir_list),"tb_month_diagnostic_s.*.txt",full.names=T) |
|
488 |
#list_data_month_s_fname <- list.files(path=file.path(in_dir,in_dir_list),"data_month_s.*.txt",full.names=T) |
|
489 |
#list_data_s_fname <- list.files(path=file.path(in_dir,in_dir_list),"data_day_s.*.txt",full.names=T) |
|
490 |
#list_data_v_fname <- list.files(path=file.path(in_dir,in_dir_list),"data_day_v.*.txt",full.names=T) |
|
491 |
#list_pred_data_month_info_fname <- list.files(path=file.path(in_dir,in_dir_list),"pred_data_month_info.*.txt",full.names=T) |
|
492 |
#list_pred_data_day_info_fname <- list.files(path=file.path(in_dir,in_dir_list),"pred_data_day_info.*.txt",full.names=T) |
|
493 | 412 |
|
494 |
list_tb_fname <- list.files(path=in_dir_list,"tb_diagnostic_v_NA_.*.txt",full.names=T) |
|
495 |
list_df_fname <- list.files(path=in_dir_list,"df_tile_processed_.*..txt",full.names=T) |
|
496 |
list_summary_metrics_v_fname <- list.files(path=in_dir_list,"summary_metrics_v2_NA_.*.txt",full.names=T) |
|
497 |
list_tb_s_fname <- list.files(path=in_dir_list,"tb_diagnostic_s_NA.*.txt",full.names=T) |
|
498 |
list_tb_month_s_fname <- list.files(path=in_dir_list,"tb_month_diagnostic_s.*.txt",full.names=T) |
|
499 |
list_data_month_s_fname <- list.files(path=in_dir_list,"data_month_s.*.txt",full.names=T) |
|
500 |
list_data_s_fname <- list.files(path=in_dir_list,"data_day_s.*.txt",full.names=T) |
|
501 |
list_data_v_fname <- list.files(path=in_dir_list,"data_day_v.*.txt",full.names=T) |
|
502 |
list_pred_data_month_info_fname <- list.files(path=in_dir_list,"pred_data_month_info.*.txt",full.names=T) |
|
503 |
list_pred_data_day_info_fname <- list.files(path=in_dir_list,"pred_data_day_info.*.txt",full.names=T) |
|
504 |
|
|
505 |
#need to fix this !! has all of the files in one list (for a region) |
|
506 |
#list_shp <- list.files(path=file.path(in_dir,file.path(in_dir_list,"shapefiles")),"*.shp",full.names=T) |
|
413 |
setwd(out_dir) |
|
507 | 414 |
|
508 | 415 |
############################ END OF SCRIPT ################################## |
Also available in: Unified diff
clean up of lising of tiles and shapefiles by region