Revision 500fdcdc
Added by Benoit Parmentier about 9 years ago
climate/research/oregon/interpolation/global_run_scalingup_mosaicing_function.R | ||
---|---|---|
4 | 4 |
#Different options to explore mosaicing are tested. This script only contains functions. |
5 | 5 |
#AUTHOR: Benoit Parmentier |
6 | 6 |
#CREATED ON: 04/14/2015 |
7 |
#MODIFIED ON: 10/26/2015
|
|
7 |
#MODIFIED ON: 10/27/2015
|
|
8 | 8 |
#Version: 1 |
9 | 9 |
#PROJECT: Environmental Layers project |
10 | 10 |
#COMMENTS: first commit of function script to test mosaicing using 1500x4500km and other tiles |
... | ... | |
24 | 24 |
library(sp) # Spatial pacakge with class definition by Bivand et al. |
25 | 25 |
library(spdep) # Spatial pacakge with methods and spatial stat. by Bivand et al. |
26 | 26 |
library(rgdal) # GDAL wrapper for R, spatial utilities |
27 |
library(gstat) # Kriging and co-kriging by Pebesma et al.
|
|
27 |
#library(gstat) # Kriging and co-kriging by Pebesma et al., not on NEX
|
|
28 | 28 |
library(fields) # NCAR Spatial Interpolation methods such as kriging, splines |
29 | 29 |
library(raster) # Hijmans et al. package for raster processing |
30 | 30 |
library(gdata) # various tools with xls reading, cbindX |
... | ... | |
35 | 35 |
library(reshape) # Change shape of object, summarize results |
36 | 36 |
library(plotrix) # Additional plotting functions |
37 | 37 |
library(plyr) # Various tools including rbind.fill |
38 |
library(spgwr) # GWR method
|
|
39 |
library(automap) # Kriging automatic fitting of variogram using gstat
|
|
40 |
library(rgeos) # Geometric, topologic library of functions |
|
38 |
#library(spgwr) # GWR method, not on NEX
|
|
39 |
#library(automap) # Kriging automatic fitting of variogram using gstat, not on NEX
|
|
40 |
#library(rgeos) # Geometric, topologic library of functions
|
|
41 | 41 |
#RPostgreSQL # Interface R and Postgres, not used in this script |
42 | 42 |
library(gridExtra) |
43 | 43 |
#Additional libraries not used in workflow |
... | ... | |
414 | 414 |
return(raster_name) |
415 | 415 |
} |
416 | 416 |
|
417 |
mosaicFiles <- function(lf_mosaic,mosaic_method="unweighted",num_cores=1,r_mask_raster_name=NULL,python_bin=NULL,df_points=NULL,NA_flag_val=-9999,file_format=".tif",out_suffix=NULL,out_dir=NULL){ |
|
417 |
mosaicFiles <- function(lf_mosaic,mosaic_method="unweighted",num_cores=1,r_mask_raster_name=NULL,python_bin=NULL,mosaic_python="/nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py",algorithm="R",df_points=NULL,NA_flag_val=-9999,file_format=".tif",out_suffix=NULL,out_dir=NULL){
|
|
418 | 418 |
#This functions mosaics tiles/files give a list of files |
419 | 419 |
#There are four options to mosaic: use_sine_weights,use_edge,use_linear_weights, unweighted |
420 | 420 |
#Sine weights fits sine fuctions across rows and column producing elliptical/spherical patterns from center |
... | ... | |
434 | 434 |
#9)out_suffix: output suffix, default is NULL, it is recommended to add the variable name |
435 | 435 |
# here e.g. dailyTmax and date!! |
436 | 436 |
#10)out_dir: output directory, default is NULL |
437 |
#11)algorithm: use R or python function |
|
437 | 438 |
# |
438 | 439 |
#OUTPUT: |
439 | 440 |
# Object is produced with 3 components: |
... | ... | |
565 | 566 |
|
566 | 567 |
lf_files <- unlist(list_weights) |
567 | 568 |
|
568 |
|
|
569 | 569 |
##Maching resolution is probably only necessary for the r mosaic function |
570 | 570 |
#MOdify later to take into account option R or python... |
571 | 571 |
list_param_raster_match <- list(lf_files,rast_ref,file_format,python_bin,out_suffix,out_dir) |
... | ... | |
586 | 586 |
##################### |
587 | 587 |
###### PART 4: compute the weighted mean with the mosaic function ##### |
588 | 588 |
|
589 |
##Make this a function later |
|
590 |
#list_weights_m <- list(list_linear_weights_m,list_edge_weights_m,list_sine_weights_m) |
|
591 |
#list_weights_prod_m <- list(list_linear_weights_prod_m,list_edge_weights_prod_m,list_sine_weights_prod_m) |
|
592 |
#list_methods <- c("linear","edge","sine") |
|
593 |
list_mosaiced_files <- vector("list",length=1) |
|
594 |
|
|
595 |
list_args_weights <- list_weights_m |
|
596 |
list_args_weights_prod <- list_weights_prod_m |
|
597 |
method_str <- method |
|
598 |
|
|
599 |
#making a list of raster object before mosaicing |
|
600 |
list_args_weights <- lapply(1:length(list_args_weights), FUN=function(i,x){raster(x[[i]])},x=list_args_weights) |
|
601 |
|
|
602 |
#get the list of weights product into raster objects |
|
589 |
if(algorithm=="python"){ |
|
590 |
|
|
591 |
#The file to do the merge is /nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py. Sample call below. |
|
592 |
#python /nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py --config GDAL_CACHEMAX=1500 --overwrite=TRUE -o outputname.tif --optfile input.txt |
|
593 |
#lf_day_to_mosaic <- list_weights_m |
|
594 |
|
|
595 |
#pattern_str <- paste("*.","predicted_mod1",".*.",day_to_mosaic[i],".*.tif",sep="") |
|
596 |
#lf_day_to_mosaic <- lapply(1:length(unlist(in_dir_mosaics)),FUN=function(k){list.files(path=unlist(in_dir_mosaics)[k],pattern=pattern_str,full.names=T,recursive=T)}) |
|
597 |
#lf_day_to_mosaic <- unlist(lf_day_to_mosaic) |
|
598 |
#write.table(lf_day_to_mosaic,file=file.path(out_dir,paste("list_to_mosaics_",day_to_mosaic[i],".txt",sep=""))) |
|
599 |
#filename_list_mosaics <- file.path(out_dir,paste("list_to_mosaics_",day_to_mosaic[i],".txt",sep="")) |
|
600 |
filename_list_mosaics_weights_m <- file.path(out_dir,paste("list_to_mosaics_","weights_m_",mosaic_method,"_",out_suffix,".txt",sep="")) |
|
601 |
filename_list_mosaics_prod_weights_m <- file.path(out_dir,paste("list_to_mosaics_","prod_weights_m_",mosaic_method,"_",out_suffix,".txt",sep="")) |
|
602 |
|
|
603 |
writeLines(unlist(list_weights_m),con=filename_list_mosaics_weights_m) #weights files to mosaic |
|
604 |
writeLines(unlist(list_weights_prod_m),con=filename_list_mosaics_prod_weights_m) #prod weights files to mosaic |
|
605 |
|
|
606 |
#out_mosaic_name_weights_m <- r_weights_sum_raster_name <- file.path(out_dir,paste("r_weights_sum_m_",mosaic_method,"_weighted_mean_",out_suffix,".tif",sep="")) |
|
607 |
#out_mosaic_name_prod_weights_m <- r_weights_sum_raster_name <- file.path(out_dir,paste("r_prod_weights_sum_m_",mosaic_method,"_weighted_mean_",out_suffix,".tif",sep="")) |
|
608 |
out_mosaic_name_weights_m <- file.path(out_dir,paste("r_weights_sum_m_",mosaic_method,"_weighted_mean_",out_suffix,".tif",sep="")) |
|
609 |
out_mosaic_name_prod_weights_m <- file.path(out_dir,paste("r_prod_weights_sum_m_",mosaic_method,"_weighted_mean_",out_suffix,".tif",sep="")) |
|
610 |
|
|
611 |
#in_file_to_mosaics <- filename_list_mosaics |
|
612 |
#in_dir_mosaics <- file.path(in_dir1,region_names[i]) |
|
613 |
#out_dir_mosaics <- "/nobackupp6/aguzman4/climateLayers/output1000x3000_km/reg5/mosaicsMean" |
|
614 |
#Can be changed to have mosaics in different dir.. |
|
615 |
#out_dir_mosaics <- out_dir |
|
616 |
#prefix_str <- "reg4_1500x4500" |
|
617 |
#tile_size <- basename(dirname(in_dir[[i]])) |
|
618 |
#tile_size <- basename(in_dir1) |
|
619 |
|
|
620 |
#prefix_str <- paste(region_names[i],"_",tile_size,sep="") |
|
621 |
#mod_str <- "mod1" #use mod2 which corresponds to model with LST and elev |
|
622 |
#out_mosaic_name <- paste(region,"_mosaics_",mod_str,"_",tile_size,"_",day_to_mosaic[i],"_",out_prefix,".tif",sep="") |
|
623 |
|
|
624 |
## Mosaic sum weights... |
|
625 |
input_file <- filename_list_mosaics_weights_m |
|
626 |
module_path <- mosaic_python #this should be a parameter for the function... |
|
627 |
|
|
628 |
mosaic_python_merge <- function(module_path,module_name,input_file,out_mosaic_name){ |
|
629 |
#d |
|
630 |
#out_mosaic_name <- r_weights_sum_raster_name <- file.path(out_dir,paste("r_weights_sum_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep="")) |
|
631 |
cmd_str <- paste("python", file.path(module_path,module_name), |
|
632 |
"--config GDAL_CACHEMAX=1500", |
|
633 |
"--overwrite=TRUE", |
|
634 |
paste("-o",out_mosaic_name,sep=" "), |
|
635 |
paste("--optfile", input_file,sep=" ")) |
|
636 |
system(cmd_str) |
|
637 |
#list(out_mosaic_name,cmd_str) |
|
638 |
return(out_mosaic_name) |
|
639 |
} |
|
640 |
#python /nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py |
|
641 |
#--config GDAL_CACHEMAX=1500 --overwrite=TRUE -o outputname.tif --optfile input.txt |
|
642 |
r_weights_sum_raster_name <- mosaic_python_merge(module_path=mosaic_python, |
|
643 |
module_name="gdal_merge_sum.py", |
|
644 |
input_file=filename_list_mosaics_weights_m, |
|
645 |
out_mosaic_name=out_mosaic_name_weights_m) |
|
646 |
|
|
647 |
r_prod_sum_raster_name <- mosaic_python_merge(module_path=mosaic_python, |
|
648 |
module_name="gdal_merge_sum.py", |
|
649 |
input_file=filename_list_mosaics_prod_weights_m, |
|
650 |
out_mosaic_name=out_mosaic_name_prod_weights_m) |
|
651 |
|
|
652 |
} |
|
653 |
|
|
654 |
if(algorithm=="R"){ |
|
655 |
|
|
656 |
#The file to do the merge is /nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py. Sample call below. |
|
657 |
#python /nobackupp6/aguzman4/climateLayers/sharedCode/gdal_merge_sum.py --config GDAL_CACHEMAX=1500 --overwrite=TRUE -o outputname.tif --optfile input.txt |
|
658 |
##Make this a function later |
|
659 |
#list_weights_m <- list(list_linear_weights_m,list_edge_weights_m,list_sine_weights_m) |
|
660 |
#list_weights_prod_m <- list(list_linear_weights_prod_m,list_edge_weights_prod_m,list_sine_weights_prod_m) |
|
661 |
#list_methods <- c("linear","edge","sine") |
|
662 |
list_mosaiced_files <- vector("list",length=1) |
|
663 |
|
|
664 |
list_args_weights <- list_weights_m |
|
665 |
list_args_weights_prod <- list_weights_prod_m |
|
666 |
method_str <- method |
|
667 |
|
|
668 |
#making a list of raster object before mosaicing |
|
669 |
list_args_weights <- lapply(1:length(list_args_weights), FUN=function(i,x){raster(x[[i]])},x=list_args_weights) |
|
670 |
|
|
671 |
#get the list of weights product into raster objects |
|
672 |
|
|
673 |
list_args_weights_prod <- lapply(1:length(list_args_weights_prod), FUN=function(i,x){raster(x[[i]])},x=list_args_weights_prod) |
|
674 |
list_args_weights_prod$fun <- "sum" #use sum while mosaicing |
|
675 |
list_args_weights_prod$na.rm <- TRUE #deal with NA by removal |
|
676 |
r_weights_sum_raster_name <- file.path(out_dir,paste("r_weights_sum_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep="")) |
|
677 |
list_args_weights$filename <- r_weights_sum_raster_name |
|
678 |
list_args_weights$overwrite<- TRUE |
|
679 |
list_args_weights_prod$overwrite<- TRUE #add to overwrite existing image |
|
680 |
|
|
681 |
list_args_weights$fun <- "sum" #we want the sum to compute the weighted mean |
|
682 |
list_args_weights$na.rm <- TRUE |
|
683 |
r_prod_sum_raster_name <- file.path(out_dir,paste("r_prod_sum_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep="")) |
|
684 |
list_args_weights_prod$filename <- r_prod_sum_raster_name |
|
685 |
|
|
686 |
#Mosaic files: this is where we can use Alberto Python function but modified with option for |
|
687 |
#sum in addition ot the current option for mean. |
|
688 |
#This took 23 minutes! |
|
689 |
r_weights_sum <- do.call(mosaic,list_args_weights) #weights sum image mosaiced |
|
690 |
#This took 23 minutes with the R function |
|
691 |
r_prod_sum <- do.call(mosaic,list_args_weights_prod) #weights sum product image mosacied |
|
603 | 692 |
|
604 |
list_args_weights_prod <- lapply(1:length(list_args_weights_prod), FUN=function(i,x){raster(x[[i]])},x=list_args_weights_prod) |
|
605 |
list_args_weights_prod$fun <- "sum" #use sum while mosaicing |
|
606 |
list_args_weights_prod$na.rm <- TRUE #deal with NA by removal |
|
607 |
r_weights_sum_raster_name <- file.path(out_dir,paste("r_weights_sum_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep="")) |
|
608 |
list_args_weights$filename <- r_weights_sum_raster_name |
|
609 |
list_args_weights$overwrite<- TRUE |
|
610 |
list_args_weights_prod$overwrite<- TRUE #add to overwrite existing image |
|
693 |
} |
|
611 | 694 |
|
612 |
list_args_weights$fun <- "sum" #we want the sum to compute the weighted mean |
|
613 |
list_args_weights$na.rm <- TRUE |
|
614 |
r_prod_sum_raster_name <- file.path(out_dir,paste("r_prod_sum_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep="")) |
|
615 |
list_args_weights_prod$filename <- r_prod_sum_raster_name |
|
616 |
|
|
617 |
#Mosaic files: this is where we can use Alberto Python function but modified with option for |
|
618 |
#sum in addition ot the current option for mean. |
|
619 |
#This took 23 minutes! |
|
620 |
r_weights_sum <- do.call(mosaic,list_args_weights) #weights sum image mosaiced |
|
621 |
#This took 23 minutes with the R function |
|
622 |
r_prod_sum <- do.call(mosaic,list_args_weights_prod) #weights sum product image mosacied |
|
623 | 695 |
|
624 | 696 |
#r_m_weighted_mean <- r_prod_sum/r_weights_sum #this is the mosaic using weighted mean... |
625 | 697 |
|
626 |
r_m_weighted_mean_raster_name <- file.path(out_dir,paste("r_m_",method_str,"_weighted_mean_",out_suffix,".tif",sep=""))
|
|
698 |
r_m_weighted_mean_raster_name <- file.path(out_dir,paste("r_m_",mosaic_method,"_weighted_mean_",out_suffix,".tif",sep=""))
|
|
627 | 699 |
|
628 | 700 |
if(is.null(python_bin)){ |
629 | 701 |
python_bin="" |
... | ... | |
640 | 712 |
#writeRaster(r_m_weighted_mean, NAflag=NA_flag_val,filename=raster_name,overwrite=TRUE) |
641 | 713 |
#now use the mask |
642 | 714 |
if(!is.null(r_mask_raster_name)){ |
643 |
r_m_weighted_mean_mask_raster_name <- file.path(out_dir,paste("r_m_",method_str,"_weighted_mean_mask_",out_suffix,".tif",sep=""))
|
|
715 |
r_m_weighted_mean_mask_raster_name <- file.path(out_dir,paste("r_m_",method_mosaic_method,"_weighted_mean_mask_",out_suffix,".tif",sep=""))
|
|
644 | 716 |
mask(raster(r_m_weighted_mean_raster_name),mask=raster(r_mask_raster_name), |
645 | 717 |
filename=r_m_weighted_mean_mask_raster_name,overwrite=TRUE) |
646 | 718 |
raster_name <- r_m_weighted_mean_mask_raster_name |
... | ... | |
854 | 926 |
} |
855 | 927 |
|
856 | 928 |
|
929 |
plot_screen_raster_val<-function(i,list_param){ |
|
930 |
##USAGE### |
|
931 |
#Screen raster list and produced plot as png. |
|
932 |
fname <-list_param$lf_raster_fname[i] |
|
933 |
screenRast <- list_param$screenRast |
|
934 |
l_dates<- list_param$l_dates |
|
935 |
out_dir_str <- list_param$out_dir_str |
|
936 |
prefix_str <-list_param$prefix_str |
|
937 |
out_suffix_str <- list_param$out_suffix_str |
|
938 |
|
|
939 |
### START SCRIPT #### |
|
940 |
date_proc <- l_dates[i] |
|
941 |
|
|
942 |
if(screenRast==TRUE){ |
|
943 |
r_test <- raster(fname) |
|
944 |
|
|
945 |
m <- c(-Inf, -100, NA, |
|
946 |
-100, 100, 1, |
|
947 |
100, Inf, NA) #can change the thresholds later |
|
948 |
rclmat <- matrix(m, ncol=3, byrow=TRUE) |
|
949 |
rc <- reclassify(r_test, rclmat,filename=paste("rc_tmp_",i,".tif",sep=""),dataType="FLT4S",overwrite=T) |
|
950 |
#file_name <- unlist(strsplit(basename(lf_m[i]),"_")) |
|
951 |
extension_str <- extension(filename(r_test)) |
|
952 |
raster_name_tmp <- gsub(extension_str,"",basename(filename(r_test))) |
|
953 |
raster_name <- file.path(out_dir_str,paste(raster_name_tmp,"_masked.tif",sep="")) |
|
954 |
|
|
955 |
r_pred <- mask(r_test,rc,filename=raster_name,overwrite=TRUE) |
|
956 |
}else{ |
|
957 |
r_pred <- raster(fname) |
|
958 |
} |
|
959 |
|
|
960 |
#date_proc <- file_name[7] #specific tot he current naming of files |
|
961 |
#date_proc<- "2010010101" |
|
962 |
|
|
963 |
#paste(raster_name[1:7],collapse="_") |
|
964 |
#add filename option later |
|
965 |
|
|
966 |
res_pix <- 960 |
|
967 |
#res_pix <- 480 |
|
968 |
|
|
969 |
col_mfrow <- 1 |
|
970 |
row_mfrow <- 1 |
|
971 |
|
|
972 |
# png(filename=paste("Figure10_clim_world_mosaics_day_","_",date_proc,"_",tile_size,"_",out_suffix,".png",sep=""), |
|
973 |
# width=col_mfrow*res_pix,height=row_mfrow*res_pix) |
|
974 |
png(filename=paste(prefix_str,"_",date_proc,"_","_",out_suffix_str,".png",sep=""), |
|
975 |
width=col_mfrow*res_pix,height=row_mfrow*res_pix) |
|
976 |
|
|
977 |
plot(r_pred,main=paste("Predicted on ",date_proc , " ", sep=""),cex.main=1.5) |
|
978 |
dev.off() |
|
979 |
|
|
980 |
} |
|
981 |
|
|
982 |
|
|
983 |
|
|
857 | 984 |
##################### END OF SCRIPT ###################### |
858 | 985 |
|
Also available in: Unified diff
adding python script option from gdal modified merged Alberto