Project

General

Profile

« Previous | Next » 

Revision 500fdcdc

Added by Benoit Parmentier about 9 years ago

adding python script option from gdal modified merged Alberto

View differences:

climate/research/oregon/interpolation/global_run_scalingup_mosaicing_function.R
4 4
#Different options to explore mosaicing are tested. This script only contains functions.
5 5
#AUTHOR: Benoit Parmentier 
6 6
#CREATED ON: 04/14/2015  
7
#MODIFIED ON: 10/26/2015            
7
#MODIFIED ON: 10/27/2015            
8 8
#Version: 1
9 9
#PROJECT: Environmental Layers project     
10 10
#COMMENTS: first commit of function script to test mosaicing using 1500x4500km and other tiles
......
24 24
library(sp)                                  # Spatial pacakge with class definition by Bivand et al.
25 25
library(spdep)                               # Spatial pacakge with methods and spatial stat. by Bivand et al.
26 26
library(rgdal)                               # GDAL wrapper for R, spatial utilities
27
library(gstat)                               # Kriging and co-kriging by Pebesma et al.
27
#library(gstat)                               # Kriging and co-kriging by Pebesma et al., not on NEX
28 28
library(fields)                              # NCAR Spatial Interpolation methods such as kriging, splines
29 29
library(raster)                              # Hijmans et al. package for raster processing
30 30
library(gdata)                               # various tools with xls reading, cbindX
......
35 35
library(reshape)                             # Change shape of object, summarize results 
36 36
library(plotrix)                             # Additional plotting functions
37 37
library(plyr)                                # Various tools including rbind.fill
38
library(spgwr)                               # GWR method
39
library(automap)                             # Kriging automatic fitting of variogram using gstat
40
library(rgeos)                               # Geometric, topologic library of functions
38
#library(spgwr)                               # GWR method, not on NEX
39
#library(automap)                             # Kriging automatic fitting of variogram using gstat, not on NEX
40
#library(rgeos)                               # Geometric, topologic library of functions
41 41
#RPostgreSQL                                 # Interface R and Postgres, not used in this script
42 42
library(gridExtra)
43 43
#Additional libraries not used in workflow
......
414 414
  return(raster_name)
415 415
}
416 416

  
417
mosaicFiles <- function(lf_mosaic,mosaic_method="unweighted",num_cores=1,r_mask_raster_name=NULL,python_bin=NULL,df_points=NULL,NA_flag_val=-9999,file_format=".tif",out_suffix=NULL,out_dir=NULL){
417
mosaicFiles <- function(lf_mosaic,mosaic_method="unweighted",num_cores=1,r_mask_raster_name=NULL,python_bin=NULL,mosaic_python="/nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py",algorithm="R",df_points=NULL,NA_flag_val=-9999,file_format=".tif",out_suffix=NULL,out_dir=NULL){
418 418
  #This functions mosaics tiles/files give a list of files
419 419
  #There are four options to mosaic:   use_sine_weights,use_edge,use_linear_weights, unweighted
420 420
  #Sine weights fits sine fuctions across rows and column producing elliptical/spherical patterns from center
......
434 434
  #9)out_suffix: output suffix, default is NULL, it is recommended to add the variable name
435 435
  #             here e.g. dailyTmax and date!!
436 436
  #10)out_dir: output directory, default is NULL
437
  #11)algorithm: use R or python function
437 438
  #
438 439
  #OUTPUT:
439 440
  # Object is produced with 3 components:
......
565 566
    
566 567
    lf_files <- unlist(list_weights)
567 568

  
568
    
569 569
    ##Maching resolution is probably only necessary for the r mosaic function
570 570
    #MOdify later to take into account option R or python...
571 571
    list_param_raster_match <- list(lf_files,rast_ref,file_format,python_bin,out_suffix,out_dir)
......
586 586
    #####################
587 587
    ###### PART 4: compute the weighted mean with the mosaic function #####
588 588

  
589
    ##Make this a function later
590
    #list_weights_m <- list(list_linear_weights_m,list_edge_weights_m,list_sine_weights_m)
591
    #list_weights_prod_m <- list(list_linear_weights_prod_m,list_edge_weights_prod_m,list_sine_weights_prod_m)
592
    #list_methods <- c("linear","edge","sine")
593
    list_mosaiced_files <- vector("list",length=1)
594

  
595
    list_args_weights <- list_weights_m
596
    list_args_weights_prod <- list_weights_prod_m
597
    method_str <- method
598
  
599
    #making a list of raster object before mosaicing
600
    list_args_weights <- lapply(1:length(list_args_weights), FUN=function(i,x){raster(x[[i]])},x=list_args_weights)
601

  
602
    #get the list of weights product into raster objects
589
    if(algorithm=="python"){
590
      
591
      #The file to do the merge is /nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py. Sample call below.
592
      #python /nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py --config GDAL_CACHEMAX=1500 --overwrite=TRUE -o  outputname.tif --optfile input.txt
593
      #lf_day_to_mosaic <- list_weights_m
594
      
595
      #pattern_str <- paste("*.","predicted_mod1",".*.",day_to_mosaic[i],".*.tif",sep="")
596
      #lf_day_to_mosaic <- lapply(1:length(unlist(in_dir_mosaics)),FUN=function(k){list.files(path=unlist(in_dir_mosaics)[k],pattern=pattern_str,full.names=T,recursive=T)}) 
597
      #lf_day_to_mosaic <- unlist(lf_day_to_mosaic)
598
      #write.table(lf_day_to_mosaic,file=file.path(out_dir,paste("list_to_mosaics_",day_to_mosaic[i],".txt",sep="")))
599
      #filename_list_mosaics <- file.path(out_dir,paste("list_to_mosaics_",day_to_mosaic[i],".txt",sep=""))
600
      filename_list_mosaics_weights_m <- file.path(out_dir,paste("list_to_mosaics_","weights_m_",mosaic_method,"_",out_suffix,".txt",sep=""))
601
      filename_list_mosaics_prod_weights_m <- file.path(out_dir,paste("list_to_mosaics_","prod_weights_m_",mosaic_method,"_",out_suffix,".txt",sep=""))
602
      
603
      writeLines(unlist(list_weights_m),con=filename_list_mosaics_weights_m) #weights files to mosaic 
604
      writeLines(unlist(list_weights_prod_m),con=filename_list_mosaics_prod_weights_m) #prod weights files to mosaic
605

  
606
      #out_mosaic_name_weights_m <- r_weights_sum_raster_name <- file.path(out_dir,paste("r_weights_sum_m_",mosaic_method,"_weighted_mean_",out_suffix,".tif",sep=""))
607
      #out_mosaic_name_prod_weights_m <- r_weights_sum_raster_name <- file.path(out_dir,paste("r_prod_weights_sum_m_",mosaic_method,"_weighted_mean_",out_suffix,".tif",sep=""))
608
      out_mosaic_name_weights_m  <- file.path(out_dir,paste("r_weights_sum_m_",mosaic_method,"_weighted_mean_",out_suffix,".tif",sep=""))
609
      out_mosaic_name_prod_weights_m <- file.path(out_dir,paste("r_prod_weights_sum_m_",mosaic_method,"_weighted_mean_",out_suffix,".tif",sep=""))
610

  
611
      #in_file_to_mosaics <- filename_list_mosaics        
612
      #in_dir_mosaics <- file.path(in_dir1,region_names[i])
613
      #out_dir_mosaics <- "/nobackupp6/aguzman4/climateLayers/output1000x3000_km/reg5/mosaicsMean"
614
      #Can be changed to have mosaics in different dir..
615
      #out_dir_mosaics <- out_dir
616
      #prefix_str <- "reg4_1500x4500"
617
      #tile_size <- basename(dirname(in_dir[[i]]))
618
      #tile_size <- basename(in_dir1)
619

  
620
      #prefix_str <- paste(region_names[i],"_",tile_size,sep="")
621
      #mod_str <- "mod1" #use mod2 which corresponds to model with LST and elev
622
      #out_mosaic_name <- paste(region,"_mosaics_",mod_str,"_",tile_size,"_",day_to_mosaic[i],"_",out_prefix,".tif",sep="")
623
      
624
      ## Mosaic sum weights...
625
      input_file <- filename_list_mosaics_weights_m
626
      module_path <- mosaic_python #this should be a parameter for the function...
627

  
628
      mosaic_python_merge <- function(module_path,module_name,input_file,out_mosaic_name){
629
        #d
630
        #out_mosaic_name <- r_weights_sum_raster_name <- file.path(out_dir,paste("r_weights_sum_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep=""))
631
        cmd_str <- paste("python", file.path(module_path,module_name),
632
                         "--config GDAL_CACHEMAX=1500",
633
                         "--overwrite=TRUE",
634
                         paste("-o",out_mosaic_name,sep=" "),
635
                         paste("--optfile", input_file,sep=" "))
636
        system(cmd_str)
637
        #list(out_mosaic_name,cmd_str)
638
        return(out_mosaic_name)
639
      }
640
      #python /nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py 
641
      #--config GDAL_CACHEMAX=1500 --overwrite=TRUE -o  outputname.tif --optfile input.txt
642
      r_weights_sum_raster_name <- mosaic_python_merge(module_path=mosaic_python,
643
                                                       module_name="gdal_merge_sum.py",
644
                                                       input_file=filename_list_mosaics_weights_m,
645
                                                       out_mosaic_name=out_mosaic_name_weights_m)
646

  
647
      r_prod_sum_raster_name <- mosaic_python_merge(module_path=mosaic_python,
648
                                                    module_name="gdal_merge_sum.py",
649
                                                    input_file=filename_list_mosaics_prod_weights_m,
650
                                                    out_mosaic_name=out_mosaic_name_prod_weights_m)
651
      
652
    }
653
    
654
    if(algorithm=="R"){
655
      
656
      #The file to do the merge is /nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py. Sample call below.
657
      #python /nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py --config GDAL_CACHEMAX=1500 --overwrite=TRUE -o  outputname.tif --optfile input.txt
658
          ##Make this a function later
659
      #list_weights_m <- list(list_linear_weights_m,list_edge_weights_m,list_sine_weights_m)
660
      #list_weights_prod_m <- list(list_linear_weights_prod_m,list_edge_weights_prod_m,list_sine_weights_prod_m)
661
      #list_methods <- c("linear","edge","sine")
662
      list_mosaiced_files <- vector("list",length=1)
663

  
664
      list_args_weights <- list_weights_m
665
      list_args_weights_prod <- list_weights_prod_m
666
      method_str <- method
667
  
668
      #making a list of raster object before mosaicing
669
      list_args_weights <- lapply(1:length(list_args_weights), FUN=function(i,x){raster(x[[i]])},x=list_args_weights)
670

  
671
      #get the list of weights product into raster objects
672

  
673
      list_args_weights_prod <- lapply(1:length(list_args_weights_prod), FUN=function(i,x){raster(x[[i]])},x=list_args_weights_prod)
674
      list_args_weights_prod$fun <- "sum" #use sum while mosaicing
675
      list_args_weights_prod$na.rm <- TRUE #deal with NA by removal
676
      r_weights_sum_raster_name <- file.path(out_dir,paste("r_weights_sum_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep=""))
677
      list_args_weights$filename <- r_weights_sum_raster_name
678
      list_args_weights$overwrite<- TRUE
679
      list_args_weights_prod$overwrite<- TRUE  #add to overwrite existing image  
680
    
681
      list_args_weights$fun <- "sum" #we want the sum to compute the weighted mean
682
      list_args_weights$na.rm <- TRUE
683
      r_prod_sum_raster_name <- file.path(out_dir,paste("r_prod_sum_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep=""))
684
      list_args_weights_prod$filename <- r_prod_sum_raster_name
685

  
686
      #Mosaic files: this is where we can use Alberto Python function but modified with option for
687
      #sum in addition ot the current option for mean.
688
      #This took 23 minutes!
689
      r_weights_sum <- do.call(mosaic,list_args_weights) #weights sum image mosaiced
690
      #This took 23 minutes with the R function
691
      r_prod_sum <- do.call(mosaic,list_args_weights_prod) #weights sum product image mosacied
603 692

  
604
    list_args_weights_prod <- lapply(1:length(list_args_weights_prod), FUN=function(i,x){raster(x[[i]])},x=list_args_weights_prod)
605
    list_args_weights_prod$fun <- "sum" #use sum while mosaicing
606
    list_args_weights_prod$na.rm <- TRUE #deal with NA by removal
607
    r_weights_sum_raster_name <- file.path(out_dir,paste("r_weights_sum_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep=""))
608
    list_args_weights$filename <- r_weights_sum_raster_name
609
    list_args_weights$overwrite<- TRUE
610
    list_args_weights_prod$overwrite<- TRUE  #add to overwrite existing image  
693
    }
611 694
    
612
    list_args_weights$fun <- "sum" #we want the sum to compute the weighted mean
613
    list_args_weights$na.rm <- TRUE
614
    r_prod_sum_raster_name <- file.path(out_dir,paste("r_prod_sum_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep=""))
615
    list_args_weights_prod$filename <- r_prod_sum_raster_name
616

  
617
    #Mosaic files: this is where we can use Alberto Python function but modified with option for
618
    #sum in addition ot the current option for mean.
619
    #This took 23 minutes!
620
    r_weights_sum <- do.call(mosaic,list_args_weights) #weights sum image mosaiced
621
    #This took 23 minutes with the R function
622
    r_prod_sum <- do.call(mosaic,list_args_weights_prod) #weights sum product image mosacied
623 695

  
624 696
    #r_m_weighted_mean <- r_prod_sum/r_weights_sum #this is the mosaic using weighted mean...
625 697

  
626
    r_m_weighted_mean_raster_name <- file.path(out_dir,paste("r_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep=""))
698
    r_m_weighted_mean_raster_name <- file.path(out_dir,paste("r_m_",mosaic_method,"_weighted_mean_",out_suffix,".tif",sep=""))
627 699

  
628 700
    if(is.null(python_bin)){
629 701
      python_bin=""
......
640 712
    #writeRaster(r_m_weighted_mean, NAflag=NA_flag_val,filename=raster_name,overwrite=TRUE)  
641 713
    #now use the mask
642 714
    if(!is.null(r_mask_raster_name)){
643
      r_m_weighted_mean_mask_raster_name <- file.path(out_dir,paste("r_m_",method_str,"_weighted_mean_mask_",out_suffix,".tif",sep=""))
715
      r_m_weighted_mean_mask_raster_name <- file.path(out_dir,paste("r_m_",method_mosaic_method,"_weighted_mean_mask_",out_suffix,".tif",sep=""))
644 716
      mask(raster(r_m_weighted_mean_raster_name),mask=raster(r_mask_raster_name),
645 717
           filename=r_m_weighted_mean_mask_raster_name,overwrite=TRUE)
646 718
      raster_name <- r_m_weighted_mean_mask_raster_name
......
854 926
}
855 927

  
856 928

  
929
plot_screen_raster_val<-function(i,list_param){
930
  ##USAGE###
931
  #Screen raster list and produced plot as png.
932
  fname <-list_param$lf_raster_fname[i]
933
  screenRast <- list_param$screenRast
934
  l_dates<- list_param$l_dates
935
  out_dir_str <- list_param$out_dir_str
936
  prefix_str <-list_param$prefix_str
937
  out_suffix_str <- list_param$out_suffix_str
938
  
939
  ### START SCRIPT ####
940
  date_proc <- l_dates[i]
941
  
942
  if(screenRast==TRUE){
943
    r_test <- raster(fname)
944

  
945
    m <- c(-Inf, -100, NA,  
946
         -100, 100, 1, 
947
         100, Inf, NA) #can change the thresholds later
948
    rclmat <- matrix(m, ncol=3, byrow=TRUE)
949
    rc <- reclassify(r_test, rclmat,filename=paste("rc_tmp_",i,".tif",sep=""),dataType="FLT4S",overwrite=T)
950
    #file_name <- unlist(strsplit(basename(lf_m[i]),"_"))
951
    extension_str <- extension(filename(r_test))
952
    raster_name_tmp <- gsub(extension_str,"",basename(filename(r_test)))
953
    raster_name <- file.path(out_dir_str,paste(raster_name_tmp,"_masked.tif",sep=""))
954
  
955
    r_pred <- mask(r_test,rc,filename=raster_name,overwrite=TRUE)
956
  }else{
957
    r_pred <- raster(fname)
958
  }
959
  
960
  #date_proc <- file_name[7] #specific tot he current naming of files
961
  #date_proc<- "2010010101"
962

  
963
  #paste(raster_name[1:7],collapse="_")
964
  #add filename option later
965

  
966
  res_pix <- 960
967
  #res_pix <- 480
968

  
969
  col_mfrow <- 1
970
  row_mfrow <- 1
971
  
972
#  png(filename=paste("Figure10_clim_world_mosaics_day_","_",date_proc,"_",tile_size,"_",out_suffix,".png",sep=""),
973
#    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
974
  png(filename=paste(prefix_str,"_",date_proc,"_","_",out_suffix_str,".png",sep=""),
975
    width=col_mfrow*res_pix,height=row_mfrow*res_pix)
976

  
977
  plot(r_pred,main=paste("Predicted on ",date_proc , " ", sep=""),cex.main=1.5)
978
  dev.off()
979

  
980
}
981

  
982

  
983

  
857 984
##################### END OF SCRIPT ######################
858 985

  

Also available in: Unified diff