Revision e4762c0b
Added by Benoit Parmentier almost 9 years ago
climate/research/oregon/interpolation/global_run_scalingup_assessment_part1.R | ||
---|---|---|
5 | 5 |
#Part 1 create summary tables and inputs files for figure in part 2 and part 3. |
6 | 6 |
#AUTHOR: Benoit Parmentier |
7 | 7 |
#CREATED ON: 03/23/2014 |
8 |
#MODIFIED ON: 12/11/2015
|
|
8 |
#MODIFIED ON: 12/28/2015
|
|
9 | 9 |
#Version: 4 |
10 | 10 |
#PROJECT: Environmental Layers project |
11 | 11 |
#TO DO: |
12 |
# - generate delta and clim mosaic
|
|
13 |
# - Wrap in function to be able to extract information via a job?
|
|
14 |
# - Parallelize the region mosaics generation (by dates)?
|
|
12 |
# - |
|
13 |
# - Make this a function...
|
|
14 |
# - Drop mosaicing from the script
|
|
15 | 15 |
# |
16 | 16 |
#First source these files: |
17 | 17 |
#Resolved call issues from R. |
... | ... | |
62 | 62 |
script_path <- "/nobackupp8/bparmen1/env_layers_scripts" #path to script |
63 | 63 |
source(file.path(script_path,function_analyses_paper1)) #source all functions used in this script |
64 | 64 |
|
65 |
|
|
66 | 65 |
############################## |
67 | 66 |
#### Parameters and constants |
68 | 67 |
|
69 | 68 |
#Make this a function |
70 | 69 |
#reg1 (North Am), reg2(Europe),reg3(Asia), reg4 (South Am), reg5 (Africa), reg6 (Australia-Asia) |
71 | 70 |
#master directory containing the definition of tile size and tiles predicted |
72 |
in_dir1 <- "/nobackupp6/aguzman4/climateLayers/out_15x45/"
|
|
71 |
in_dir1 <- "/nobackupp6/aguzman4/climateLayers/out/" |
|
73 | 72 |
#/nobackupp6/aguzman4/climateLayers/out_15x45/1982 |
74 | 73 |
|
75 |
region_names <- c("reg23","reg4") #selected region names, #PARAM2 |
|
74 |
#region_names <- c("reg23","reg4") #selected region names, #PARAM2 |
|
75 |
region_names <- c("reg4") |
|
76 | 76 |
#region_names <- c("1992") #no specific region here so use date |
77 | 77 |
#region_names <- c("reg1","reg2","reg3","reg4","reg5","reg6") #selected region names, #PARAM2 |
78 | 78 |
#region_namesb <- c("reg_1b","reg_1c","reg_2b","reg_3b","reg_6b") #selected region names, #PARAM2 |
79 | 79 |
|
80 | 80 |
y_var_name <- "dailyTmax" #PARAM3 |
81 | 81 |
interpolation_method <- c("gam_CAI") #PARAM4 |
82 |
out_prefix<-"run10_1500x4500_global_analyses_pred_1992_12072015" #PARAM5
|
|
82 |
out_prefix<-"run_global_analyses_pred_12282015" #PARAM5
|
|
83 | 83 |
|
84 | 84 |
#output_run10_1500x4500_global_analyses_pred_2003_04102015/ |
85 | 85 |
|
... | ... | |
91 | 91 |
|
92 | 92 |
CRS_locs_WGS84 <- CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84, #PARAM8 |
93 | 93 |
|
94 |
day_to_mosaic <- c("19920101","19920102","19920103,19920104,19920105") |
|
95 |
|
|
96 |
|
|
94 |
#day_to_mosaic <- c("19920101","19920102","19920103,19920104,19920105") |
|
97 | 95 |
#day_to_mosaic <- NULL #if day to mosaic is null then mosaic all dates? |
96 |
year_predicted <- 1984:2004 |
|
98 | 97 |
|
99 | 98 |
file_format <- ".tif" #format for mosaiced files #PARAM10 |
100 | 99 |
NA_flag_val <- -9999 #No data value, #PARAM11 |
... | ... | |
161 | 160 |
|
162 | 161 |
##raster_prediction object : contains testing and training stations with RMSE and model object |
163 | 162 |
|
164 |
|
|
165 | 163 |
list_raster_obj_files <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="^raster_prediction_obj.*.RData",full.names=T)}) |
166 | 164 |
basename(dirname(list_raster_obj_files[[1]])) |
167 | 165 |
list_names_tile_coord <- lapply(list_raster_obj_files,FUN=function(x){basename(dirname(x))}) |
... | ... | |
518 | 516 |
write.table(df_tiles_all, |
519 | 517 |
file=file.path(out_dir,"shapefiles",paste("df_tiles_all_",out_prefix,".txt",sep="")),sep=",") |
520 | 518 |
|
521 |
###################################################### |
|
522 |
####### PART 2 CREATE MOSAIC OF PREDICTIONS PER DAY, Delta surfaces and clim ### |
|
523 |
|
|
524 |
#if mosaicing_tiles==TRUE then do it? |
|
525 |
#dates_l <- unique(robj1$tb_diagnostic_s$date) #list of dates to query tif |
|
526 |
#create date!!! |
|
527 |
if(is.null(day_to_mosaic)){ |
|
528 |
|
|
529 |
#idx <- seq(as.Date('2010-01-01'), as.Date('2010-12-31'), 'day') |
|
530 |
#idx <- seq(as.Date('20100101'), as.Date('20101231'), 'day') |
|
531 |
#date_l <- strptime(idx[1], "%Y%m%d") # interpolation date being processed |
|
532 |
#dates_l <- format(idx, "%Y%m%d") # interpolation date being processed |
|
533 |
day_to_mosaic <- dates_predicted #should be 365 days... |
|
534 |
#l_dates <- day_to_mosaic |
|
535 |
} |
|
536 |
#else{ |
|
537 |
# l_dates <- paste(day_to_mosaic,collapse=",") |
|
538 |
#} |
|
539 |
|
|
540 |
## make this a function? report on number of tiles used for mosaic... |
|
541 |
|
|
542 |
#inputs: build a pattern to find files |
|
543 |
#y_var_name <- "dailyTmax" #set up in parameters of this script |
|
544 |
#interpolation_method <- c("gam_CAI") #set up in parameters of the script |
|
545 |
name_method <- paste(interpolation_method,"_",y_var_name,"_",sep="") |
|
546 |
##Use python code written by Alberto Guzman |
|
547 |
|
|
548 |
#system("MODULEPATH=$MODULEPATH:/nex/modules/files") |
|
549 |
#system("module load /nex/modules/files/pythonkits/gdal_1.10.0_python_2.7.3_nex") |
|
550 |
|
|
551 |
#module_path <- "" |
|
552 |
module_path <- "/nobackupp6/aguzman4/climateLayers/sharedCode/" |
|
553 |
#/nobackupp6/aguzman4/climateLayers/sharedCode/mosaicUsingGdalMerge.py |
|
554 |
l_dates <- paste(day_to_mosaic,collapse=",",sep=" ") |
|
555 |
## use region 2 first |
|
556 |
|
|
557 |
### FIRST mosaics by processing region |
|
558 |
#make this a function later...with following param |
|
559 |
#input: |
|
560 |
#region_names |
|
561 |
#in_dir1 |
|
562 |
##out_dir , not ehta out_dir moasic s can be created in rhe future function |
|
563 |
#mod_str <- mod1 |
|
564 |
#For the time being use mean,median from python function by Alberto... |
|
565 |
#Solved issue about calls from R |
|
566 |
#First run 3 lines in the bash shell |
|
567 |
#source /nobackupp6/aguzman4/climateLayers/sharedModules/etc/environ.sh |
|
568 |
#MODULEPATH=$MODULEPATH:/nex/modules/files |
|
569 |
#module load pythonkits/gdal_1.10.0_python_2.7.3_nex |
|
570 |
|
|
571 |
#recombine region first: |
|
572 |
region_names_mosaic <- list(region_names) |
|
573 |
names(region_names_mosaic) <- "reg5" |
|
574 |
in_dir_mosaics <- lapply(region_names_mosaic,FUN=function(x){file.path(in_dir1,x)}) |
|
575 |
|
|
576 |
for (j in 1:length(region_names_mosaic)){ |
|
577 |
#for(i in 1:length(region_names)) |
|
578 |
in_dir_mosaics <- lapply(region_names_mosaic,FUN=function(x){file.path(in_dir1,x)}) |
|
579 |
#in_dir_mosaics <- paste(region_names_mosaic,collapse=" ") |
|
580 |
#out_dir_mosaics <- "/nobackupp6/aguzman4/climateLayers/output1000x3000_km/reg5/mosaicsMean" |
|
581 |
#Can be changed to have mosaics in different dir.. |
|
582 |
out_dir_mosaics <- out_dir |
|
583 |
#prefix_str <- "reg4_1500x4500" |
|
584 |
#tile_size <- basename(dirname(in_dir[[i]])) |
|
585 |
tile_size <- basename(in_dir1) |
|
586 |
|
|
587 |
prefix_str <- paste(names(region_names_mosaic)[j],"_",tile_size,sep="") |
|
588 |
|
|
589 |
mod_str <- "mod1" #use mod2 which corresponds to model with LST and elev |
|
590 |
|
|
591 |
cmd_str <- paste("python", file.path(module_path,"mosaicUsingGdalMerge.py"), |
|
592 |
in_dir_mosaics, |
|
593 |
out_dir_mosaics, |
|
594 |
prefix_str, |
|
595 |
"--mods", mod_str, |
|
596 |
"--date", l_dates,sep=" ") |
|
597 |
system(cmd_str) |
|
598 |
|
|
599 |
} |
|
600 |
|
|
601 |
### SECOND mosaics globally from regional mosaics... |
|
602 |
### Now find out how many files were predicted |
|
603 |
# will be useful later on |
|
604 |
# Transform this into a function that takes in a list of files!!! We can skip the region stage to reduce the number of files.. |
|
605 |
|
|
606 |
#sh /nobackupp6/aguzman4/climateLayers/sharedCode/shMergeFromFile.sh list_mosaics_20100901.txt world_mosaics_1000x3000_20100901.tif |
|
607 |
in_dir_str <- in_dir_mosaics |
|
608 |
region <- "reg5" #can be the world... |
|
609 |
for (i in 1:length(day_to_mosaic)){ |
|
610 |
pattern_str <- paste("*.","predicted_mod1",".*.",day_to_mosaic[i],".*.tif",sep="") |
|
611 |
lf_day_to_mosaic <- lapply(1:length(unlist(in_dir_mosaics)),FUN=function(k){list.files(path=unlist(in_dir_mosaics)[k],pattern=pattern_str,full.names=T,recursive=T)}) |
|
612 |
lf_day_to_mosaic <- unlist(lf_day_to_mosaic) |
|
613 |
#write.table(lf_day_to_mosaic,file=file.path(out_dir,paste("list_to_mosaics_",day_to_mosaic[i],".txt",sep=""))) |
|
614 |
writeLines(lf_day_to_mosaic,con=file.path(out_dir,paste("list_to_mosaics_",day_to_mosaic[i],".txt",sep=""))) |
|
615 |
in_file_to_mosaics <- file.path(out_dir,paste("list_to_mosaics_",day_to_mosaic[i],".txt",sep="")) |
|
616 |
#in_dir_mosaics <- file.path(in_dir1,region_names[i]) |
|
617 |
#out_dir_mosaics <- "/nobackupp6/aguzman4/climateLayers/output1000x3000_km/reg5/mosaicsMean" |
|
618 |
#Can be changed to have mosaics in different dir.. |
|
619 |
#out_dir_mosaics <- out_dir |
|
620 |
#prefix_str <- "reg4_1500x4500" |
|
621 |
#tile_size <- basename(dirname(in_dir[[i]])) |
|
622 |
tile_size <- basename(in_dir1) |
|
623 |
|
|
624 |
#prefix_str <- paste(region_names[i],"_",tile_size,sep="") |
|
625 |
mod_str <- "mod1" #use mod2 which corresponds to model with LST and elev |
|
626 |
out_mosaic_name <- paste(region,"_mosaics_",mod_str,"_",tile_size,"_",day_to_mosaic[i],"_",out_prefix,".tif",sep="") |
|
627 |
module_path <- "/nobackupp6/aguzman4/climateLayers/sharedCode" #this should be a parameter for the function... |
|
628 |
cmd_str <- paste("sh", file.path(module_path,"shMergeFromFile.sh"), |
|
629 |
in_file_to_mosaics, |
|
630 |
out_mosaic_name, |
|
631 |
sep=" ") |
|
632 |
system(cmd_str) |
|
633 |
|
|
634 |
} |
|
635 |
|
|
636 |
|
|
637 | 519 |
########### LAST PART: COPY SOME DATA BACK TO ATLAS ##### |
638 | 520 |
|
639 | 521 |
#This will be a separate script?? or function? |
Also available in: Unified diff
assessment part1, clean up before splitting of function in part1a for automation process for stage 6