Project

General

Profile

« Previous | Next » 

Revision f9023de7

Added by Benoit Parmentier over 8 years ago

mosaicing script, clean up of code and debugging for options

View differences:

climate/research/oregon/interpolation/global_run_scalingup_mosaicing.R
5 5
#Analyses, figures, tables and data are also produced in the script.
6 6
#AUTHOR: Benoit Parmentier 
7 7
#CREATED ON: 04/14/2015  
8
#MODIFIED ON: 04/07/2016            
8
#MODIFIED ON: 04/08/2016            
9 9
#Version: 6
10 10
#PROJECT: Environmental Layers project     
11 11
#COMMENTS: analyses run for reg4 1991 for test of mosaicing using 1500x4500km and other tiles
......
79 79
  #27) algorithm: python or R, if R use mosaic function for R, if python use modified gdal merge, PARAM 27
80 80
  #28) match_extent : if "FALSE" try without matching geographic extent #PARAM 28 
81 81
  #29) list_models : if NULL use y~1 formula #PARAM 29
82

  
82
  #30) layers_option: mosaic to create as a layer from var_pred (e.g. TMax), res_training, res_testing, ac_testing
83
  
83 84
  ###OUTPUT
84 85
  # 
85 86
  #
......
167 168
  #list_models <- paste(var_pred,"~","1",sep=" ") #if null then this is the default...
168 169
  layers_option <- list_param_run_mosaicing_prediction$layers_option
169 170
  
171
  
172
  #################################################################
170 173
  ####### PART 1: Read in data and process data ########
174
  ########################################################
171 175
  
172 176
  #out_dir <- in_dir #PARAM 11
173 177
  #in_dir_tiles <- file.path(in_dir,"tiles") #this is valid both for Atlas and NEX
......
230 234
  in_dir_tiles_tmp <- file.path(in_dir, region_name)
231 235
  #fix this later and add the year..
232 236
  #gam_CAI_dailyTmax_predicted_mod1
233

  
237
  #this is very slow!!! it takes 8 minutes?!
234 238
  lf_mosaic <- lapply(1:length(day_to_mosaic),FUN=function(i){list.files(path=file.path(in_dir_tiles_tmp),    
235 239
                                                                                    pattern=paste("gam_CAI_dailyTmax_predicted_",pred_mod_name,".*.",day_to_mosaic[i],".*.tif$",sep=""),full.names=T,recursive=T)})
236 240

  
237
  #reg_lf_mosaic[[k]] <- list.files(path=file.path(in_dir_tiles_tmp),pattern=paste(".*.",day_to_mosaic[i],".*.tif$",sep=""),full.names=T,recursive=T)
241
  #########################################################################
238 242
  ##################### PART 2: produce the mosaic ##################
243
  ######################################################################
239 244
  
240 245
  #This is is assuming a list of file for a region!! 
241 246
  #this is where the main function for mosaicing region starts!!
......
292 297
  ### Now create accuracy surfaces from residuals...
293 298
    
294 299
  if(layers_option=="res_testing"){
295

  
300
    #This part took 19 minutes and 45 seconds
301
    
296 302
    ## Create accuracy surface by kriging
297 303
    num_cores_tmp <-num_cores
298 304
    lf_day_tiles  <- lf_mosaic #list of raster files by dates
......
357 363
  ##Run for data_day_s
358 364
  ##
359 365
  
360
  if(layers_option=="res_testing"){
366
  if(layers_option=="res_training"){
367
    #This part took 19 minutes and 40 seconds
361 368
    
362 369
    data_df <- data_day_s # data.frame table/spdf containing stations with residuals and variable
363 370
    
......
413 420
    lf_accuracy_residuals_data_s_raster <- lapply(1:length(list_create_accuracy_residuals_raster_obj),FUN=function(i,x){as.character(unlist(extract_from_list_obj(x[[i]]$list_pred_res_obj,"raster_name")))},x=list_create_accuracy_residuals_raster_obj)
414 421
    
415 422
  }
416
  ##took 31 minutes to generate the residuals maps for each tiles (28) for region 4
417
  ##Revised on 04/07 for three dates, it took 40 minutes
418 423
  
419 424
  ######################################################
420
  #### PART 2: GENERATE MOSAIC FOR LIST OF FILES #####
425
  #### PART 3: GENERATE MOSAIC FOR LIST OF FILES #####
421 426
  #################################
422 427
  #### Mosaic tiles for the variable predicted and accuracy metric
423 428
    
......
427 432
  list_mosaic_obj <- vector("list",length=length(day_to_mosaic))
428 433
  for(i in 1:length(day_to_mosaic)){
429 434
    #
430
    mosaic_method <- "use_edge_weights" #this is distance from edge
431
    out_suffix_tmp <- paste(interpolation_method,y_var_name,day_to_mosaic[i],out_suffix,sep="_")
432
    #debug(mosaicFiles)
433
    #can also loop through methods!!!
434
    #python_bin <- "/usr/bin/" #python gdal bin, on Atlas NCEAS
435
    #python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules/bin" #on NEX
436
    #gdal_merge_sum_noDataTest.py
435
    
436
    if(layers_option=="var_pred"){
437
      
438
      mosaic_method <- "use_edge_weights" #this is distance from edge
439
      out_suffix_tmp <- paste(interpolation_method,y_var_name,day_to_mosaic[i],out_suffix,sep="_")
440
      #debug(mosaicFiles)
441
      #can also loop through methods!!!
442
      #python_bin <- "/usr/bin/" #python gdal bin, on Atlas NCEAS
443
      #python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules/bin" #on NEX
444
      #gdal_merge_sum_noDataTest.py
437 445
      
438
    mosaic_edge_obj_prediction <- mosaicFiles(lf_mosaic[[i]],
446
      mosaic_obj <- mosaicFiles(lf_mosaic[[i]],
439 447
                                                mosaic_method="use_edge_weights",
440 448
                                                num_cores=num_cores,
441 449
                                                r_mask_raster_name=infile_mask,
......
448 456
                                                file_format=file_format,
449 457
                                                out_suffix=out_suffix_tmp,
450 458
                                                out_dir=out_dir)
451
    #runs in 15-16 minutes for 3 dates and mosaicing of 28 tiles...  
459
      #runs in 15-16 minutes for 3 dates and mosaicing of 28 tiles...
460
      list_mosaic_obj[[i]] <- mosaic_obj
461
    }
452 462
    
453
    ## Now accuracy based on center of centroids
454
    mosaic_method <- "use_edge_weights" #this is distance from edge
455
    #Adding metric name in the name...
456
    out_suffix_tmp <- paste(interpolation_method,metric_name,day_to_mosaic[i],out_suffix,sep="_")
457
      
458
    #debug(mosaicFiles)
459
    #can also loop through methods!!!
460
    mosaic_edge_obj_accuracy <- mosaicFiles(lf_accuracy_raster[[i]],
463
    if(layers_option=="ac_testing"){
464
      ## Now accuracy based on center of centroids
465
      mosaic_method <- "use_edge_weights" #this is distance from edge
466
      #Adding metric name in the name...
467
      out_suffix_tmp <- paste(interpolation_method,metric_name,day_to_mosaic[i],out_suffix,sep="_")
468
    
469
      #debug(mosaicFiles)
470
      #can also loop through methods!!!
471
      mosaic_obj <- mosaicFiles(lf_accuracy_raster[[i]],
461 472
                                              mosaic_method="use_edge_weights",
462 473
                                              num_cores=num_cores,
463 474
                                              r_mask_raster_name=infile_mask,
......
469 480
                                              file_format=file_format,
470 481
                                              out_suffix=out_suffix_tmp,
471 482
                                              out_dir=out_dir)
472
    ##Took 39 minutes for 28 tiles and one date...!!!  
473
    list_mosaic_obj[[i]] <- list(prediction=mosaic_edge_obj_prediction,accuracy=mosaic_edge_obj_accuracy)
474
      
483
      ##Took 29 minutes for 28 tiles and one date...!!! 
484
      list_mosaic_obj[[i]] <- mosaic_obj
485
    }
486
    
487
    #list_mosaic_obj[[i]] <- list(prediction=mosaic_edge_obj_prediction,accuracy=mosaic_edge_obj_accuracy)
488

  
475 489
    ### produce residuals mosaics
476
    #for now add data_day_s in the name!!
477
    mosaic_method <- "use_edge_weights" #this is distance from edge
478
    out_suffix_tmp <- paste(interpolation_method,"kriged_residuals","data_day_s",day_to_mosaic[i],out_suffix,sep="_")
479
    #lf_tmp<-list.files(pattern="*kriged_residuals.*.tif",full.names=T)
480
    lf_tmp <- lf_accuracy_residuals_raster[[i]]
481
    #lf_accuracy_residuals_raster[[i]]
482
    #debug(mosaicFiles)
483
    mosaic_edge_obj_residuals <- mosaicFiles(lf_tmp,
490
    if(layers_option=="res_testing"){
491
      #for now add data_day_s in the name!!
492
      mosaic_method <- "use_edge_weights" #this is distance from edge
493
      out_suffix_tmp <- paste(interpolation_method,"kriged_residuals","data_day_v",day_to_mosaic[i],out_suffix,sep="_")
494
      #lf_tmp<-list.files(pattern="*kriged_residuals.*.tif",full.names=T)
495
      lf_tmp <- lf_accuracy_residuals_raster[[i]]
496
      #lf_accuracy_residuals_raster[[i]]
497
      #debug(mosaicFiles)
498
      mosaic_obj <- mosaicFiles(lf_tmp,
484 499
                                               mosaic_method="use_edge_weights",
485 500
                                               num_cores=num_cores,
486 501
                                               r_mask_raster_name=infile_mask,
......
493 508
                                               file_format=file_format,
494 509
                                               out_suffix=out_suffix_tmp,
495 510
                                               out_dir=out_dir)
496
    #Took 11 to 12 minues for one day and 28 tiles in region 4
497
    list_mosaic_obj[[i]] <- list(prediction=mosaic_edge_obj_prediction,accuracy=mosaic_edge_obj_accuracy,mosaic_edge_obj_residuals)
498
    #}
499
    
500
    ##End of mosaicing function for region predictions
501
  }
502
  
503
  
504
  #####################
505
  ###### PART 2: Analysis and figures for the outputs of mosaic function #####
506
  
507
  #### compute and aspect and slope with figures
508
  #list_lf_mosaic_obj <- vector("list",length(day_to_mosaic))
509
  #lf_mean_mosaic <- vector("list",length(mosaicing_method))#2methods only
510
  #l_method_mosaic <- vector("list",length(mosaicing_method))
511
  #list_out_suffix <- vector("list",length(mosaicing_method))
512
  
513
  #for(i in 1:length(day_to_mosaic)){
514
  #  list_lf_mosaic_obj[[i]] <- list.files(path=out_dir,pattern=paste("*",day_to_mosaic[i],
515
  #                                                                   "_.*.RData",sep=""))
516
  #  lf_mean_mosaic[[i]] <- unlist(lapply(list_lf_mosaic_obj[[i]],function(x){load_obj(x)[["mean_mosaic"]]}))
517
  #  l_method_mosaic[[i]] <- paste(unlist(lapply(list_lf_mosaic_obj[[i]],function(x){load_obj(x)[["method"]]})),day_to_mosaic[i],sep="_")
518
  #  list_out_suffix[[i]] <- unlist(paste(l_method_mosaic[[i]],day_to_mosaic[[i]],out_suffix,sep="_"))
519
  #}
520
  
521
  #if(plot_figures==TRUE){
522
    
523
    #list_param_plot_mosaic <- list(lf_mosaic=unlist(lf_mean_mosaic),
524
    #                               method=unlist(l_method_mosaic),
525
    #                               out_suffix=unlist(list_out_suffix))
511
      #Took 11 to 12 minues for one day and 28 tiles in region 4
512
      list_mosaic_obj[[i]] <- mosaic_obj
513
    }      
526 514
    
527
    #plot and produce png movie...
528
    #plot_mosaic(1,list_param=list_param_plot_mosaic)
529
    #num_cores <- 1
530
    #l_png_files <- mclapply(1:length(unlist(lf_mean_mosaic)),FUN=plot_mosaic,
531
    #                        list_param= list_param_plot_mosaic,
532
    #                        mc.preschedule=FALSE,mc.cores = num_cores)
533
    
534
    #lf_plot<- list.files(pattern="r_m_use.*.mask.*.tif$")
535
    #lf_mean_mosaic <- lf_plot
536
    
537
    
538
    #list_param_plot_mosaic <- list(lf_raster_fname=unlist(lf_mean_mosaic[1:2]),
539
    #                               screenRast=TRUE,
540
    #                               l_dates=day_to_mosaic,
541
    #                               out_dir_str=out_dir,
542
    #                               out_prefix_str <- "dailyTmax_",
543
    #                               out_suffix_str=out_suffix)
544
    #debug(plot_screen_raster_val)
545
    #plot_screen_raster_val(1,list_param_plot_mosaic)
546
    
547
    
548
    #num_cores <- 2
549
    #l_png_files <- mclapply(1:length(unlist(lf_mean_mosaic)[1:2]),FUN=plot_screen_raster_val,
550
    #                        list_param= list_param_plot_mosaic,
551
    #                        mc.preschedule=FALSE,mc.cores = num_cores)
552
    
553
    
554
    #list_param_plot_mosaic <- list(lf_raster_fname=unlist(lf_mean_mosaic[4:6]),
555
    #                               screenRast=FALSE,
556
    #                               l_dates=day_to_mosaic,
557
    #                               out_dir_str=out_dir,
558
    #                               out_prefix_str <- paste("rmse_",out_suffix,sep=""),
559
    #                               out_suffix_str=paste("rmse_",out_suffix,sep=""))
560
    #num_cores <- 3
561
    #l_png_files_rmse <- mclapply(1:length(unlist(lf_mean_mosaic)[4:6]),FUN=plot_screen_raster_val,
562
    #                             list_param= list_param_plot_mosaic,
563
    #                             mc.preschedule=FALSE,mc.cores = num_cores)
515
    ### produce residuals mosaics
516
    if(layers_option=="res_training"){
517
      #for now add data_day_s in the name!!
518
      mosaic_method <- "use_edge_weights" #this is distance from edge
519
      out_suffix_tmp <- paste(interpolation_method,"kriged_residuals","data_day_s",day_to_mosaic[i],out_suffix,sep="_")
520
      #lf_tmp<-list.files(pattern="*kriged_residuals.*.tif",full.names=T)
521
      lf_tmp <- lf_accuracy_residuals_raster[[i]]
522
      #lf_accuracy_residuals_raster[[i]]
523
      #debug(mosaicFiles)
524
      mosaic_obj <- mosaicFiles(lf_tmp,
525
                                               mosaic_method="use_edge_weights",
526
                                               num_cores=num_cores,
527
                                               r_mask_raster_name=infile_mask,
528
                                               python_bin=python_bin,
529
                                               mosaic_python=mosaic_python,
530
                                               algorithm=algorithm,
531
                                               match_extent=match_extent,
532
                                               df_points=NULL,
533
                                               NA_flag=NA_flag_val,
534
                                               file_format=file_format,
535
                                               out_suffix=out_suffix_tmp,
536
                                               out_dir=out_dir)
537
      list_mosaic_obj[[i]] <- mosaic_obj
538
      #Took 11 to 12 minues for one day and 28 tiles in region 4
539
    }
564 540
    
565
  #}
541
    ##End of mosaicing function for region predictions
542
  }## end of day_to_mosaic loop
566 543
  
567 544
  ##Create return object
568
  #list of mosaiced files
569
  
545
  #list of mosaiced files: get the list of files now to include in the output object!!
546
  mosaicing_prediction_obj <- list(list_mosaic_obj,layer_option)
547
  names(mosaicing_prediction_obj) <- c("list_mosaic_obj","layer_option")
570 548
  return(run_mosaicing_prediction_obj)
549
  
571 550
}
572 551

  
573 552
###############

Also available in: Unified diff