Project

General

Profile

« Previous | Next » 

Revision 52dfbd74

Added by Benoit Parmentier about 8 years ago

clean up of lising of tiles and shapefiles by region

View differences:

climate/research/oregon/interpolation/global_product_assessment_part0.R
4 4
#This part 2 of the assessment focuses on graphics to explore the spatial patterns of raster times series as figures and movie
5 5
#AUTHOR: Benoit Parmentier 
6 6
#CREATED ON: 10/03/2016  
7
#MODIFIED ON: 10/24/2016            
7
#MODIFIED ON: 10/27/2016            
8 8
#Version: 1
9 9
#PROJECT: Environmental Layers project     
10 10
#COMMENTS: Initial commit, script based on part NASA biodiversity conferenc 
......
260 260
table(df_time_series$missing)
261 261
table(df_time_series$year)
262 262

  
263
#############################
264
##### Creating animation based on prediction
265

  
266
#####
267
NAvalue(r_stack)
268
plot(r_stack,y=6,zlim=c(-10000,10000)) #this is not rescaled
269
#plot(r_stack,zlim=c(-50,50),col=matlab.like(255))
270
var_name <- "dailyTmax"
271

  
272
#debug(plot_and_animate_raster_time_series)
273

  
274
#metric_name <- "var_pred" #use RMSE if accuracy
275
#df_raster <- read.table("df_raster_global_assessment_reg6_10102016.txt",sep=",",header=T)
276
#plot_figure <- 
277
#function_product_assessment_part2_functions <- "global_product_assessment_part2_functions_10222016.R"
278
#source(file.path(script_path,function_product_assessment_part2_functions)) #source all functions used in this script 
279

  
280
#undebug(plot_and_animate_raster_time_series)
281
range_year <- c(1984,1985)
282
subset_df_time_series <- subset(df_time_series,year%in% range_year)
283
subset_df_time_series <- subset_df_time_series[!is.na(subset_df_time_series$lf),]
284

  
285
lf_subset <- file.path(subset_df_time_series$dir,subset_df_time_series$lf)
286
range_year_str <- paste(range_year, sep = "_", collapse = "_")
287

  
288
out_suffix_str <- paste(range_year_str,out_suffix,sep="_")
289

  
290
#started on 10/22/2016 at 9.57
291
animation_obj <- plot_and_animate_raster_time_series(lf_subset, 
292
                                                     item_no,
293
                                                     region_name,
294
                                                     var_name,
295
                                                     metric_name,
296
                                                     NA_flag_val,
297
                                                     filenames_figures=NULL,
298
                                                     frame_speed=60,
299
                                                     animation_format=".gif",
300
                                                     zlim_val=NULL,
301
                                                     plot_figure=T,
302
                                                     generate_animation=T,
303
                                                     num_cores=num_cores,
304
                                                     out_suffix=out_suffix_str,
305
                                                     out_dir=out_dir)
306
  
307
zlim_val <- c(-2000,5000)
308
animation_obj <- plot_and_animate_raster_time_series(lf_subset, 
309
                                                     item_no,
310
                                                     region_name,
311
                                                     var_name,
312
                                                     metric_name,
313
                                                     NA_flag_val,
314
                                                     filenames_figures=NULL,
315
                                                     frame_speed=60,
316
                                                     animation_format=".gif",
317
                                                     zlim_val=zlim_val,
318
                                                     plot_figure=T,
319
                                                     generate_animation=T,
320
                                                     num_cores=num_cores,
321
                                                     out_suffix=out_suffix_str,
322
                                                     out_dir=out_dir)
323

  
324
#ffmpeg -i yeay.gif outyeay.mp4
325

  
326
#/Applications/ffmpeg -r 25 -i input%3d.png -vcodec libx264 -x264opts keyint=25 -pix_fmt yuv420p -r 25 ../output.mp4
327

  
328
#ffmpeg -f gif -i file.gif -c:v libx264 outfile.mp4
329

  
330
#ffmpeg -i animation_frame_60_-2500_6000_.gif animation_frame_60_-2500_6000_.mp4
331

  
332
#ffmpeg -i animation_frame_60_-2500_6000_.gif animation_frame_60_-2500_6000_.mp4
333

  
334
#ffmpeg -f gif -i animation_frame_60_-2500_6000_.gif -vcodec libx264 -x264opts keyint=25 -pix_fmt yuv420p -r 25 outfile.mp4
335
#ffmpeg -f gif -i animation_frame_60_-2500_6000_.gif -vcodec libx264 -x264opts keyint=11 -pix_fmt yuv420p -r 11 outfile.mp4
336

  
337
#ffmpeg -r 10 -i animation_frame_60_-2500_6000_.gif animation.avi
338

  
339
#ffmpeg -f gif -i animation_frame_60_-2500_6000_.gif -vcodec libx264 -x264opts -pix_fmt yuv420p outfile.mp4
340 263

  
341 264
#combine polygon
342 265
#http://gis.stackexchange.com/questions/155328/merging-multiple-spatialpolygondataframes-into-1-spdf-in-r
......
365 288

  
366 289
#On NEX
367 290
#contains all data from the run by Alberto
368
#in_dir1 <- " /nobackupp6/aguzman4/climateLayers/out_15x45/" #On NEX
291
in_dir1 <- "/nobackupp6/aguzman4/climateLayers/out" #On NEX
369 292
#parent output dir for the current script analyes
370 293
#out_dir <- "/nobackup/bparmen1/" #on NEX
371 294
#in_dir_shp <- "/nobackupp4/aguzman4/climateLayers/output4/subset/shapefiles/"
......
376 299
#                  "output_run_global_analyses_pred_2011_reg4","output_run_global_analyses_pred_2012_reg4",
377 300
#                  "output_run_global_analyses_pred_2013_reg4","output_run_global_analyses_pred_2014_reg4")
378 301
#in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt"
379
#in_dir <- "" #PARAM 0
380
#y_var_name <- "dailyTmax" #PARAM1
381
#interpolation_method <- c("gam_CAI") #PARAM2
382
#out_suffix <- "global_analyses_overall_assessment_reg4_02072016"
302
in_dir <- "" #PARAM 0
303
y_var_name <- "dailyTmax" #PARAM1
304
interpolation_method <- c("gam_CAI") #PARAM2
305
out_suffix <- "predictions_assessment_reg6_10302016"
383 306
#out_suffix <- "output_run10_1000x3000_global_analyses_02102015"
384 307
#out_suffix <- "run10_1500x4500_global_analyses_pred_1992_10052015" #PARAM3
385 308
#out_dir <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_10052015" #PARAM4
......
391 314
#CRS_locs_WGS84<-CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84
392 315
#proj_str<- CRS_WGS84 #PARAM 8 #check this parameter
393 316
#file_format <- ".rst" #PARAM 9
394
#NA_value <- -9999 #PARAM10
395
#NA_flag_val <- NA_value
317
NA_value <- -9999 #PARAM10
318
NA_flag_val <- NA_value
396 319
#multiple_region <- TRUE #PARAM 12
397
#region_name <- "world" #PARAM 13
398
#countries_shp <-"/data/project/layers/commons/NEX_data/countries.shp" #PARAM 13, copy this on NEX too
320
region_name <- "reg6" #PARAM 13
321
countries_shp <-"/data/project/layers/commons/NEX_data/countries.shp" #PARAM 13, copy this on NEX too
399 322
#plot_region <- TRUE
400
#num_cores <- 6 #PARAM 14
401
#region_name <- c("reg4") #reference region to merge if necessary, if world all the regions are together #PARAM 16
402
#use previous files produced in step 1a and stored in a data.frame
403
#df_assessment_files <- "df_assessment_files_reg4_1984_run_global_analyses_pred_12282015.txt" #PARAM 17
404
#threshold_missing_day <- c(367,365,300,200) #PARM18
405

  
406
#list_param_run_assessment_plottingin_dir <- list(in_dir,y_var_name, interpolation_method, out_suffix, 
407
#                      out_dir, create_out_dir_param, mosaic_plot, proj_str, file_format, NA_value,
408
#                      multiple_region, countries_shp, plot_region, num_cores, 
409
#                      region_name, df_assessment_files, threshold_missing_day) 
410

  
411
#names(list_param_run_assessment_plottingin_dir) <- c("in_dir","y_var_name","interpolation_method","out_suffix", 
412
#                      "out_dir","create_out_dir_param","mosaic_plot","proj_str","file_format","NA_value",
413
#                      "multiple_region","countries_shp","plot_region","num_cores", 
414
#                      "region_name","df_assessment_files","threshold_missing_day") 
415

  
416
#run_assessment_plotting_prediction_fun(list_param_run_assessment_plottingin_dir) 
417

  
418
  ####### PARSE INPUT ARGUMENTS/PARAMETERS #####
419
  in_dir_list_filename <- list_param_run_assessment_plotting$in_dir_list_filename #PARAM 0
420
  in_dir <- list_param_run_assessment_plotting$in_dir #PARAM 1
421
  y_var_name <- list_param_run_assessment_plotting$y_var_name #PARAM2
422
  interpolation_method <- list_param_run_assessment_plotting$interpolation_method #c("gam_CAI") #PARAM3
423
  out_suffix <- list_param_run_assessment_plotting$out_suffix #PARAM4
424
  out_dir <- list_param_run_assessment_plotting$out_dir # PARAM5
425
  create_out_dir_param <- list_param_run_assessment_plotting$create_out_dir_param # FALSE #PARAM 6
426
  mosaic_plot <- list_param_run_assessment_plotting$mosaic_plot #FALSE #PARAM7
427
  proj_str<- list_param_run_assessment_plotting$proj_str #CRS_WGS84 #PARAM 8 #check this parameter
428
  file_format <- list_param_run_assessment_plotting$file_format #".rst" #PARAM 9
429
  NA_flag_val <- list_param_run_assessment_plotting$NA_flag_val #-9999 #PARAM10
430
  multiple_region <- list_param_run_assessment_plotting$multiple_region # <- TRUE #PARAM 11
431
  countries_shp <- list_param_run_assessment_plotting$countries_shp #<- "world" #PARAM 12
432
  plot_region <- list_param_run_assessment_plotting$plot_region # PARAM13 
433
  num_cores <- list_param_run_assessment_plotting$num_cores # 6 #PARAM 14
434
  region_name <- list_param_run_assessment_plotting$region_name #<- "world" #PARAM 15
435
  #df_assessment_files_name <- list_param_run_assessment_plotting$df_assessment_files_name #PARAM 16
436
  threshold_missing_day <- list_param_run_assessment_plotting$threshold_missing_day #PARM17
437
  year_predicted <- list_param_run_assessment_plotting$year_predicted
438
 
439
  NA_value <- NA_flag_val 
440
  metric_name <- "rmse" #to be added to the code later...
323
num_cores <- 6 #PARAM 14
324
#/nobackupp6/aguzman4/climateLayers/out/reg6/subset/shapefiles
325
list_year_predicted <- c(2000,2012,2013) #year still on disk for reg6
441 326
  
442
  ##################### START SCRIPT #################
327
  #from script:
328
  #interpolation/global_run_scalingup_assessment_part1a.R
329

  
330
  ##############################
331
  #### Parameters and constants  
443 332
  
444
  ####### PART 1: Read in data ########
445
  out_dir <- in_dir
446
  if(create_out_dir_param==TRUE){
447
    out_dir <- create_dir_fun(out_dir,out_suffix)
448
    setwd(out_dir)
449
  }else{
450
    setwd(out_dir) #use previoulsy defined directory
451
  }
452 333

  
453
  setwd(out_dir)
334
  in_dir1 <- list_param_run_assessment_prediction$in_dir1 
335
  region_name <- list_param_run_assessment_prediction$region_name #e.g. c("reg23","reg4") #run only for one region
336
  y_var_name <- list_param_run_assessment_prediction$y_var_name # e.g. dailyTmax" #PARAM3
337
  interpolation_method <- list_param_run_assessment_prediction$interpolation_method #c("gam_CAI") #PARAM4
338
  out_prefix <- list_param_run_assessment_prediction$out_prefix #output suffix e.g."run_global_analyses_pred_12282015" #PARAM5
339
  out_dir <- list_param_run_assessment_prediction$out_dir #<- "/nobackupp8/bparmen1/" #PARAM6
340
  create_out_dir_param <-list_param_run_assessment_prediction$create_out_dir_param #if TRUE output dir created #PARAM7
341
  proj_str <- list_param_run_assessment_prediction$proj_str # CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84, #PARAM8
342
  list_year_predicted <- list_param_run_assessment_prediction$list_year_predicted # 1984:2004
343
  file_format <- list_param_run_assessment_prediction$file_format #<- ".tif" #format for mosaiced files #PARAM10
344
  NA_flag_val <- list_param_run_assessment_prediction$NA_flag_val #<- -9999  #No data value, #PARAM11
345
  num_cores <- list_param_run_assessment_prediction$num_cores #<- 6 #number of cores used #PARAM13
346
  plotting_figures <- list_param_run_assessment_prediction$plotting_figures #if true run part2 of assessment
347
  
348
  ##for plotting assessment function
349
  
350
  mosaic_plot <- list_param_run_assessment_prediction$mosaic_plot  #PARAM14
351
  day_to_mosaic <- list_param_run_assessment_prediction$day_to_mosaic #PARAM15
352
  multiple_region <- list_param_run_assessment_prediction$multiple_region #PARAM16
353
  countries_shp <- list_param_run_assessment_prediction$countries_shp #PARAM17
354
  plot_region <- list_param_run_assessment_prediction$plot_region #PARAM18
355
  threshold_missing_day <- list_param_run_assessment_prediction$threshold_missing_day #PARM20
356

  
357
  ########################## START SCRIPT #########################################
358
  
359
  #Need to make this a function to run as a job...
360
  
361
  ######################## PART0: Read content of predictions first.... #####
362
  #function looped over i, correspoding to year predicted
454 363
  
455 364
  list_outfiles <- vector("list", length=35) #collect names of output files, this should be dynamic?
456 365
  list_outfiles_names <- vector("list", length=35) #collect names of output files
457
  counter_fig <- 0 #index of figure to collect outputs
366

  
367

  
368
  #### STart here
369
  ###### This is from assessment 1
458 370
  
459
  #i <- year_predicted
460
  ###Table 1: Average accuracy metrics
461
  ###Table 2: daily accuracy metrics for all tiles
371
  #for each polygon find you overlap!!
372
  #plot number of overlap
373
  #for specific each find prediction...
374
  year_predicted <- list_param_run_assessment_prediction$list_year_predicted[i] 
462 375

  
463
  if(!is.null(in_dir_list_filename)){
464
    in_dir_list <- as.list(read.table(in_dir_list_filename,stringsAsFactors=F)[,1])
376
  in_dir1 <- file.path(in_dir1,region_name)
377
  
378
  list_outfiles <- vector("list", length=14) #collect names of output files
379
  
380
  in_dir_list <- list.dirs(path=in_dir1,recursive=FALSE) #get the list regions processed for this run
381
  #basename(in_dir_list)
382
  #                       y=in_dir_list) 
383
  
384
  #in_dir_list_all  <- unlist(lapply(in_dir_list,function(x){list.dirs(path=x,recursive=F)}))
385
  in_dir_list_all <- in_dir_list
386
  in_dir_subset <- in_dir_list_all[grep("subset",basename(in_dir_list_all),invert=FALSE)] #select directory with shapefiles...
387
  in_dir_shp <- file.path(in_dir_subset,"shapefiles")
388
  
389
  
390
  #select only directories used for predictions
391
  #nested structure, we need to go to higher level to obtain the tiles...
392
  in_dir_reg <- in_dir_list[grep(".*._.*.",basename(in_dir_list),invert=FALSE)] #select directory with shapefiles...
393
  #in_dir_reg <- in_dir_list[grep("july_tiffs",basename(in_dir_reg),invert=TRUE)] #select directory with shapefiles...
394
  in_dir_list <- in_dir_reg
395
  
396
  
397
  in_dir_list <- in_dir_list[grep("bak",basename(basename(in_dir_list)),invert=TRUE)] #the first one is the in_dir1
398
  #list of shapefiles used to define tiles
399
  in_dir_shp_list <- list.files(in_dir_shp,".shp",full.names=T)
400
  
401
  ## load problematic tiles or additional runs
402
  #modify later...
403
  
404
  #system("ls /nobackup/bparmen1")
405
  out_dir <- in_dir
406
  if(create_out_dir_param==TRUE){
407
    out_dir <- create_dir_fun(out_dir,out_prefix)
408
    setwd(out_dir)
465 409
  }else{
466
    pattern_str <- paste0("^output_",region_name,".*.")
467
    in_dir_list_all <- list.dirs(path=in_dir,recursive = T)
468
    in_dir_list <- in_dir_list_all[grep(pattern_str,basename(in_dir_list_all),invert=FALSE)] #select directory with shapefiles...
469
    #in_dir_shp <- file.path(in_dir_list_all,"shapefiles")
410
    setwd(out_dir) #use previoulsy defined directory
470 411
  }
471
  #pattern_str <- file.path(in_dir,paste0("output_",region_name,".*."))
472
  #test <- Sys.glob(pattern_str,FALSE)
473
  #  searchStr = paste(in_dir_tiles_tmp,"/*/",year_processed,"/gam_CAI_dailyTmax_predicted_",pred_mod_name,"*",day_to_mosaic[i],"*.tif",sep="")
474
  #  #print(searchStr)
475
  #  Sys.glob(searchStr)})
476

  
477
  #lf_mosaic <- lapply(1:length(day_to_mosaic),FUN=function(i){
478
  #  searchStr = paste(in_dir_tiles_tmp,"/*/",year_processed,"/gam_CAI_dailyTmax_predicted_",pred_mod_name,"*",day_to_mosaic[i],"*.tif",sep="")
479
  #  #print(searchStr)
480
  #  Sys.glob(searchStr)})
481

  
482
  ##Read in data list from in_dir_list
483
  #list_tb_fname <- list.files(path=file.path(in_dir,in_dir_list),"tb_diagnostic_v_NA_.*.txt",full.names=T)
484
  #list_df_fname <- list.files(path=file.path(in_dir,in_dir_list),"df_tile_processed_.*..txt",full.names=T)
485
  #list_summary_metrics_v_fname <- list.files(path=file.path(in_dir,in_dir_list),"summary_metrics_v2_NA_.*.txt",full.names=T)
486
  #list_tb_s_fname <- list.files(path=file.path(in_dir,in_dir_list),"tb_diagnostic_s_NA.*.txt",full.names=T)
487
  #list_tb_month_s_fname <- list.files(path=file.path(in_dir,in_dir_list),"tb_month_diagnostic_s.*.txt",full.names=T)
488
  #list_data_month_s_fname <- list.files(path=file.path(in_dir,in_dir_list),"data_month_s.*.txt",full.names=T)
489
  #list_data_s_fname <- list.files(path=file.path(in_dir,in_dir_list),"data_day_s.*.txt",full.names=T)
490
  #list_data_v_fname <- list.files(path=file.path(in_dir,in_dir_list),"data_day_v.*.txt",full.names=T)
491
  #list_pred_data_month_info_fname <- list.files(path=file.path(in_dir,in_dir_list),"pred_data_month_info.*.txt",full.names=T)
492
  #list_pred_data_day_info_fname <- list.files(path=file.path(in_dir,in_dir_list),"pred_data_day_info.*.txt",full.names=T)
493 412
  
494
  list_tb_fname <- list.files(path=in_dir_list,"tb_diagnostic_v_NA_.*.txt",full.names=T)
495
  list_df_fname <- list.files(path=in_dir_list,"df_tile_processed_.*..txt",full.names=T)
496
  list_summary_metrics_v_fname <- list.files(path=in_dir_list,"summary_metrics_v2_NA_.*.txt",full.names=T)
497
  list_tb_s_fname <- list.files(path=in_dir_list,"tb_diagnostic_s_NA.*.txt",full.names=T)
498
  list_tb_month_s_fname <- list.files(path=in_dir_list,"tb_month_diagnostic_s.*.txt",full.names=T)
499
  list_data_month_s_fname <- list.files(path=in_dir_list,"data_month_s.*.txt",full.names=T)
500
  list_data_s_fname <- list.files(path=in_dir_list,"data_day_s.*.txt",full.names=T)
501
  list_data_v_fname <- list.files(path=in_dir_list,"data_day_v.*.txt",full.names=T)
502
  list_pred_data_month_info_fname <- list.files(path=in_dir_list,"pred_data_month_info.*.txt",full.names=T)
503
  list_pred_data_day_info_fname <- list.files(path=in_dir_list,"pred_data_day_info.*.txt",full.names=T)
504
  
505
  #need to fix this !! has all of the files in one list (for a region)
506
  #list_shp <- list.files(path=file.path(in_dir,file.path(in_dir_list,"shapefiles")),"*.shp",full.names=T)
413
  setwd(out_dir)
507 414

  
508 415
############################ END OF SCRIPT ##################################

Also available in: Unified diff