Revision f9d99622
Added by Benoit Parmentier almost 9 years ago
climate/research/oregon/interpolation/master_script_stage_8.R | ||
---|---|---|
13 | 13 |
#STAGE 8: Comparison of predictions across regions and years with figure generation. |
14 | 14 |
#AUTHOR: Benoit Parmentier |
15 | 15 |
#CREATED ON: 12/29/2015 |
16 |
#MODIFIED ON: 02/04/2016
|
|
16 |
#MODIFIED ON: 02/08/2016
|
|
17 | 17 |
#PROJECT: NCEAS-IPLANT-NASA: Environment Layers |
18 | 18 |
|
19 | 19 |
#First source these files: |
... | ... | |
35 | 35 |
#There are 21 parameters, 1 constant and 8 arguments (drawn from the parameters) for the Rscript call. |
36 | 36 |
#The arguments are passed directly from Rscript: |
37 | 37 |
#var <- args[1] # variable being interpolated #param 1, arg 1 |
38 |
#in_dir1 <- args[2] # This is the output directory containing global prediction e.g./nobackupp6/aguzman4/climateLayers/out/ param 5, arg 2
|
|
38 |
#in_dir <- args[2] # This is the output directory containing global predictions for multiple years e.g./nobackupp6/aguzman4/climateLayers/out/ param 5, arg 2
|
|
39 | 39 |
#region_name <- args[3] # region e.g. "reg4" param 6, arg 3 |
40 | 40 |
#out_prefix <- args[4] # this is used in creating an output directory,include region name? param 7, arg 4 |
41 | 41 |
#out_dir <- args[5] # output parent dir, can be home dir or other, param 8, arg 5) |
42 | 42 |
#create_out_dir_param <- args[6] # if TRUE create a output from "output"+out_prefix param 9, arg 6 |
43 |
#list_year_predicted <- args[7] # enter as list but currently runs on the first element of the list, param 10, arg 7
|
|
43 |
#list_year_predicted <- args[7] # enter a range, min and max year #arg 7
|
|
44 | 44 |
#num_cores <- args[8] #number of cores used # param 13, arg 8 |
45 | 45 |
#max_mem <- args[9] # maximum memory, param 21 |
46 |
in_dir_list_filename <- args[10] |
|
47 |
#in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt" |
|
48 |
run_figure_by_year <- args[11] # param 10, arg 7, if true will create figures for individual years... |
|
46 | 49 |
|
47 | 50 |
#var = "TMAX" # variable being interpolated #param 1, arg 1 |
48 | 51 |
#in_dir1 = "/nobackupp6/aguzman4/climateLayers/out/" #param 5, arg 2 |
... | ... | |
53 | 56 |
#list_year_predicted = c(2010) # param 10, arg 7 |
54 | 57 |
#num_cores = 6 #number of cores used # param 13, arg 8 |
55 | 58 |
#max_mem = 1e+07 #param 21, arg 9 |
59 |
#in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt" |
|
60 |
#in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt" |
|
61 |
#run_figure_by_year <- args[11] # param 10, arg 7, if true will create figures for individual years... |
|
62 |
#run_figure_by_year <- TRUE # param 10, arg 7, if true will create figures for individual years... |
|
56 | 63 |
|
57 | 64 |
### Testing several years on the bridge before running jobs on nodes with qsub |
58 | 65 |
#This can be made in a data.frame to run through... |
59 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_12282015 /nobackupp8/bparmen1/ TRUE 2010 6 1e+07 |
|
60 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2011_reg4 /nobackupp8/bparmen1/ TRUE 2011 6 1e+07 |
|
61 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2012_reg4 /nobackupp8/bparmen1/ TRUE 2012 6 1e+07 |
|
62 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2013_reg4 /nobackupp8/bparmen1/ TRUE 2013 6 1e+07 |
|
63 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2014_reg4 /nobackupp8/bparmen1/ TRUE 2014 6 1e+07 |
|
64 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2009_reg4 /nobackupp8/bparmen1/ TRUE 2009 6 1e+07 |
|
65 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2010_reg4 /nobackupp8/bparmen1/ TRUE 2010 6 1e+07 |
|
66 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_8_02072016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_12282015 /nobackupp8/bparmen1/ TRUE 2010 6 1e+07 |
|
67 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_8_02072016.R TMAX /data/project/layers/commons/NEX_data/reg4_assessment reg4 global_analyses_overall_assessment_reg4_02072016 /data/project/layers/commons/NEX_data/reg4_assessment TRUE 2010,2013 6 1e+07 stage6_reg4_in_dir_list_02072016.txt TRUE |
|
66 | 68 |
|
67 | 69 |
###Loading R library and packages ou |
68 | 70 |
library(RPostgreSQL) |
... | ... | |
104 | 106 |
#CALLED FROM MASTER SCRIPT: |
105 | 107 |
|
106 | 108 |
script_path <- "/nobackupp8/bparmen1/env_layers_scripts" #path to script |
109 |
script_path <- "/home/parmentier/Data/IPLANT_project/env_layers_scripts" |
|
107 | 110 |
function_assessment_part1_functions <- "global_run_scalingup_assessment_part1_functions_02112015.R" #PARAM12 |
108 | 111 |
function_assessment_part1a <-"global_run_scalingup_assessment_part1a_01042016.R" |
109 |
function_assessment_part2 <- "global_run_scalingup_assessment_part2_02032016.R"
|
|
112 |
function_assessment_part2 <- "global_run_scalingup_assessment_part2_02072016.R"
|
|
110 | 113 |
function_assessment_part2_functions <- "global_run_scalingup_assessment_part2_functions_01032016.R" |
114 |
function_assessment_part3 <- "global_run_scalingup_assessment_part3_02082016.R" |
|
111 | 115 |
source(file.path(script_path,function_assessment_part1_functions)) #source all functions used in this script |
112 | 116 |
source(file.path(script_path,function_assessment_part1a)) #source all functions used in this script |
113 | 117 |
source(file.path(script_path,function_assessment_part2)) #source all functions used in this script |
114 | 118 |
source(file.path(script_path,function_assessment_part2_functions)) #source all functions used in this script |
119 |
source(file.path(script_path,function_assessment_part3)) #source all functions used in this script |
|
115 | 120 |
|
116 | 121 |
### Parameters, constants and arguments ### |
117 |
stages_to_run<-c(0,0,0,0,0,6) #this stage 2 to 5 currently stored in another file.
|
|
122 |
stages_to_run<-c(0,0,0,0,0,0,0,8) #this stage 2 to 5 currently stored in another file.
|
|
118 | 123 |
|
119 | 124 |
CRS_locs_WGS84<-CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #constant 1 |
120 | 125 |
|
121 |
#var<-"TMAX" # variable being interpolated #param 1, arg 1 |
|
122 | 126 |
var <- args[1] # variable being interpolated #param 1, arg 1 |
127 |
var<-"TMAX" # variable being interpolated #param 1, arg 1 |
|
123 | 128 |
|
124 | 129 |
##Add for precip later... |
125 | 130 |
if (var == "TMAX") { |
... | ... | |
142 | 147 |
#reg1 (North Am), reg2(Europe),reg3(Asia), reg4 (South Am), reg5 (Africa), reg6 (Australia-Asia) |
143 | 148 |
#master directory containing the definition of tile size and tiles predicted |
144 | 149 |
#in_dir1 <- "/nobackupp6/aguzman4/climateLayers/out/" #param 5, arg 2 |
145 |
in_dir1 <- args[2] #param 5, arg 2 |
|
150 |
in_dir <- args[2] #param 5, arg 2 |
|
151 |
in_dir <- "/data/project/layers/commons/NEX_data/reg4_assessment" |
|
152 |
|
|
153 |
#in_dir <- "" #PARAM 0 |
|
154 |
#y_var_name <- "dailyTmax" #PARAM1 |
|
155 |
#interpolation_method <- c("gam_CAI") #PARAM2 |
|
146 | 156 |
|
147 | 157 |
#region_names <- c("reg1","reg2","reg3","reg4","reg5","reg6") #selected region names, #PARAM2 |
148 | 158 |
#region_names <- c("reg23","reg4") #selected region names, |
149 | 159 |
#run assessment by region, this is a unique region only |
150 |
#region_name <- c("reg4") #param 6, arg 3 |
|
151 | 160 |
region_name <- args[3] #param 6, arg 3 |
161 |
region_name <- c("reg4") #param 6, arg 3 |
|
152 | 162 |
|
153 | 163 |
#out_prefix <- "run_global_analyses_pred_12282015" #param 7, arg 4 |
154 | 164 |
#out_dir <- "/nobackupp8/bparmen1/" #param 8, arg 5 |
155 | 165 |
#out_dir <-paste(out_dir,"_",out_prefix,sep="") |
156 | 166 |
create_out_dir_param <- TRUE #param 9, arg 6 |
157 | 167 |
out_prefix <- args[4] #param 7, arg 4 |
168 |
out_suffix <- args[4] #param 7, arg 4s |
|
169 |
out_suffix <- "global_analyses_overall_assessment_reg4_02072016" |
|
170 |
|
|
158 | 171 |
out_dir <- args[5] #param 8, arg 5 |
172 |
out_dir <- "/data/project/layers/commons/NEX_data/reg4_assessment" |
|
173 |
|
|
159 | 174 |
#out_dir <-paste(out_dir,"_",out_prefix,sep="") |
160 | 175 |
create_out_dir_param <- args[6] #param 9, arg 6 |
176 |
create_out_dir_param <- TRUE #param 9, arg |
|
161 | 177 |
|
162 | 178 |
#list_year_predicted <- 1984:2004 |
163 | 179 |
#list_year_predicted <- c("2014") # param 10, arg 7 |
164 | 180 |
#year_predicted <- list_year_predicted[1] |
165 |
list_year_predicted <- args[7] # param 10, arg 7 |
|
181 |
list_year_predicted <- args[7] # param 10, arg 7, min and max year |
|
182 |
|
|
183 |
#run_figure_by_year <- TRUE # param 10, arg 7 |
|
184 |
list_year_predicted <- "2010,2013" |
|
166 | 185 |
|
167 | 186 |
file_format <- ".tif" #format for mosaiced files # param 11 |
168 | 187 |
NA_flag_val <- -9999 #No data value, # param 12 |
169 | 188 |
#num_cores <- 6 #number of cores used # param 13, arg 8 |
170 | 189 |
plotting_figures <- TRUE #running part2 of assessment to generate figures... # param 14 |
171 | 190 |
num_cores <- args[8] #number of cores used # param 13, arg 8 |
191 |
num_cores <-6 #number of cores used # param 13, arg 8 |
|
172 | 192 |
|
173 | 193 |
##Additional parameters used in part 2, some these may be removed as code is simplified |
174 | 194 |
mosaic_plot <- FALSE #param 15 |
175 | 195 |
day_to_mosaic <- c("19920102","19920103","19920103") #param 16, not in use... |
176 | 196 |
multiple_region <- TRUE #param 17 |
177 |
countries_shp <- "/nobackupp8/bparmen1/NEX_data/countries.shp" #param 18 |
|
178 |
#countries_shp <-"/data/project/layers/commons/NEX_data/countries.shp" #Atlas
|
|
197 |
#countries_shp <- "/nobackupp8/bparmen1/NEX_data/countries.shp" #param 18
|
|
198 |
countries_shp <-"/data/project/layers/commons/NEX_data/countries.shp" #Atlas |
|
179 | 199 |
plot_region <- TRUE # param 19 |
180 | 200 |
threshold_missing_day <- c(367,365,300,200) # param 20 |
181 | 201 |
|
182 | 202 |
#max number of cells to read in memory |
183 | 203 |
max_mem <- args[9] #param 21 |
184 |
in_dir_list_filename <- args[10] #param 22 |
|
185 |
#rasterOptions(maxmemory=1e+07,timer=TRUE) |
|
186 |
|
|
204 |
max_mem <- 1e+07 |
|
205 |
#in_dir_list_filename <- args[10] #param 22 |
|
206 |
in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt" |
|
207 |
#in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt" |
|
208 |
run_figure_by_year <- args[11] # param 10, arg 7, if true will create figures for individual years... |
|
209 |
run_figure_by_year <- TRUE # param 10, arg 7, if true will create figures for individual years... |
|
187 | 210 |
|
188 |
list_param_run_assessment_part2_plotting <-list( |
|
189 |
in_dir,y_var_name, interpolation_method, out_suffix, |
|
190 |
out_dir, create_out_dir_param, mosaic_plot, proj_str, file_format, NA_flag_val, |
|
191 |
multiple_region, countries_shp, plot_region, num_cores, |
|
192 |
region_name, df_assessment_files_name, threshold_missing_day,year_predicted |
|
193 |
) |
|
194 |
|
|
195 |
names(list_param_run_assessment_part2_plotting) <- c( |
|
196 |
"in_dir","y_var_name","interpolation_method","out_suffix", |
|
197 |
"out_dir","create_out_dir_param","mosaic_plot","proj_str","file_format","NA_flag_val", |
|
198 |
"multiple_region","countries_shp","plot_region","num_cores", |
|
199 |
"region_name","df_assessment_files_name","threshold_missing_day","year_predicted" |
|
200 |
) |
|
211 |
#### Prepare parameters for the production of figures... |
|
212 |
proj_str <- CRS_interp |
|
213 |
list_in_dir <- as.list(read.table(in_dir_list_filename,stringsAsFactors=F)[,1]) |
|
201 | 214 |
|
202 | 215 |
list_param_run_assessment_combined_region_plotting_prediction <-list( |
203 | 216 |
in_dir_list_filename, |
204 | 217 |
in_dir,y_var_name, interpolation_method, out_suffix, |
205 | 218 |
out_dir, create_out_dir_param, mosaic_plot, proj_str, file_format, NA_flag_val, |
206 | 219 |
multiple_region, countries_shp, plot_region, num_cores, |
207 |
region_name, df_assessment_files_name, threshold_missing_day,year_predicted) |
|
220 |
region_name, df_assessment_files_name, threshold_missing_day,list_year_predicted)
|
|
208 | 221 |
|
209 | 222 |
names(list_param_run_assessment_combined_region_plotting_prediction) <- c( |
210 | 223 |
"in_dir_list_filename", |
... | ... | |
224 | 237 |
#Step 1: run individual figure production if needed: |
225 | 238 |
if(run_figure_by_year==TRUE){ |
226 | 239 |
#debug(run_assessment_plotting_prediction_fun) |
227 |
df_assessment_figures_files <- |
|
228 |
run_assessment_plotting_prediction_fun(list_param_run_assessment_plotting) |
|
240 |
l_list_param_run_assessment_part2_plotting <- vector("list",length=length(list_in_dir)) |
|
241 |
list_df_assessment_files <- lapply(list_in_dir,FUN=function(x,y){list.files(path=file.path(y,x),pattern="df_assessment_files_.*.txt",full.names=T)},y=in_dir) |
|
242 |
for(j in 1:length(list_in_dir)){ |
|
243 |
|
|
244 |
df_assessment_files_name <- list_df_assessment_files[[j]] |
|
245 |
df_assessment <- read.table(df_assessment_files_name,stringsAsFactors=F,sep=",") |
|
246 |
year_tmp <- unique(df_assessment$year) |
|
247 |
reg_tmp <- unique(df_assessment$reg) |
|
248 |
in_dir_tmp <- file.path(in_dir,list_in_dir[[j]]) |
|
249 |
out_suffix_tmp <- paste("s8_",year_tmp,"_",out_suffix,sep="") |
|
250 |
create_out_dir_param_tmp <- FALSE |
|
251 |
out_dir_tmp <- in_dir_tmp #write output in the existing dir from NEX |
|
252 |
|
|
253 |
list_param_run_assessment_part2_plotting <-list( |
|
254 |
in_dir_tmp,y_var_name, interpolation_method, out_suffix_tmp, |
|
255 |
out_dir_tmp, create_out_dir_param_tmp, mosaic_plot, proj_str, file_format, NA_flag_val, |
|
256 |
multiple_region, countries_shp, plot_region, num_cores, |
|
257 |
region_name, df_assessment_files_name, threshold_missing_day,year_tmp |
|
258 |
) |
|
259 |
|
|
260 |
names(list_param_run_assessment_part2_plotting) <- c( |
|
261 |
"in_dir","y_var_name","interpolation_method","out_suffix", |
|
262 |
"out_dir","create_out_dir_param","mosaic_plot","proj_str","file_format","NA_flag_val", |
|
263 |
"multiple_region","countries_shp","plot_region","num_cores", |
|
264 |
"region_name","df_assessment_files_name","threshold_missing_day","year_predicted" |
|
265 |
) |
|
266 |
#undebug(run_assessment_plotting_prediction_fun) |
|
267 |
df_assessment_figures_files <- |
|
268 |
run_assessment_plotting_prediction_fun(list_param_run_assessment_part2_plotting) |
|
269 |
l_list_param_run_assessment_part2_plotting[[j]] <- list_param_run_assessment_part2_plotting |
|
270 |
list_df_assessment_files[[j]] <- df_assessment_figures_files |
|
271 |
} |
|
229 | 272 |
} |
230 | 273 |
|
231 | 274 |
#Step 2: run combination of all files... |
... | ... | |
233 | 276 |
#function_assessment_part2 <- "global_run_scalingup_assessment_part2_01032016.R" |
234 | 277 |
#source(file.path(script_path,function_assessment_part2)) #source all functions used in this script |
235 | 278 |
|
236 |
#debug(run_assessment_combined_region_plotting_prediction_fun)
|
|
279 |
debug(run_assessment_combined_region_plotting_prediction_fun) |
|
237 | 280 |
df_assessment_combined_figures_files <- |
238 | 281 |
run_assessment_combined_region_plotting_prediction_fun(list_param_run_assessment_combined_region_plotting_prediction) |
239 | 282 |
} |
Also available in: Unified diff
making stage 8 callable from the shell