Project

General

Profile

« Previous | Next » 

Revision f9d99622

Added by Benoit Parmentier almost 9 years ago

making stage 8 callable from the shell

View differences:

climate/research/oregon/interpolation/master_script_stage_8.R
13 13
#STAGE 8: Comparison of predictions across regions and years with figure generation.
14 14
#AUTHOR: Benoit Parmentier                                                                        
15 15
#CREATED ON: 12/29/2015  
16
#MODIFIED ON: 02/04/2016  
16
#MODIFIED ON: 02/08/2016  
17 17
#PROJECT: NCEAS-IPLANT-NASA: Environment Layers                                                                           
18 18

  
19 19
#First source these files:
......
35 35
#There are 21 parameters, 1 constant and 8 arguments (drawn from the parameters) for the Rscript call.
36 36
#The arguments are passed directly from Rscript:
37 37
#var <- args[1] # variable being interpolated #param 1, arg 1
38
#in_dir1 <- args[2] # This is the output directory containing global prediction e.g./nobackupp6/aguzman4/climateLayers/out/ param 5, arg 2
38
#in_dir <- args[2] # This is the output directory containing global predictions for multiple years e.g./nobackupp6/aguzman4/climateLayers/out/ param 5, arg 2
39 39
#region_name <- args[3] # region e.g. "reg4" param 6, arg 3
40 40
#out_prefix <- args[4] # this is used in creating an output directory,include region name? param 7, arg 4
41 41
#out_dir <- args[5] # output parent dir, can be home dir or other, param 8, arg 5)
42 42
#create_out_dir_param <- args[6] # if TRUE create a output from "output"+out_prefix param 9, arg 6
43
#list_year_predicted <- args[7] # enter as list but currently runs on the first element of the list, param 10, arg 7
43
#list_year_predicted <- args[7] # enter a range, min and max year #arg 7
44 44
#num_cores <- args[8] #number of cores used # param 13, arg 8
45 45
#max_mem <- args[9] # maximum memory, param 21
46
in_dir_list_filename <- args[10]
47
#in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt"
48
run_figure_by_year <- args[11] # param 10, arg 7, if true will create figures for individual years...
46 49

  
47 50
#var = "TMAX" # variable being interpolated #param 1, arg 1
48 51
#in_dir1 = "/nobackupp6/aguzman4/climateLayers/out/" #param 5, arg 2
......
53 56
#list_year_predicted = c(2010) # param 10, arg 7
54 57
#num_cores = 6 #number of cores used # param 13, arg 8
55 58
#max_mem = 1e+07 #param 21, arg 9
59
#in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt"
60
#in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt"
61
#run_figure_by_year <- args[11] # param 10, arg 7, if true will create figures for individual years...
62
#run_figure_by_year <- TRUE # param 10, arg 7, if true will create figures for individual years...
56 63

  
57 64
### Testing several years on the bridge before running jobs on nodes with qsub
58 65
#This can be made in a data.frame to run through...  
59
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_12282015 /nobackupp8/bparmen1/ TRUE 2010 6 1e+07
60
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2011_reg4 /nobackupp8/bparmen1/ TRUE 2011 6 1e+07
61
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2012_reg4 /nobackupp8/bparmen1/ TRUE 2012 6 1e+07
62
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2013_reg4 /nobackupp8/bparmen1/ TRUE 2013 6 1e+07
63
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2014_reg4 /nobackupp8/bparmen1/ TRUE 2014 6 1e+07
64
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2009_reg4 /nobackupp8/bparmen1/ TRUE 2009 6 1e+07
65
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01222016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_2010_reg4 /nobackupp8/bparmen1/ TRUE 2010 6 1e+07
66
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_8_02072016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 run_global_analyses_pred_12282015 /nobackupp8/bparmen1/ TRUE 2010 6 1e+07
67
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_8_02072016.R TMAX /data/project/layers/commons/NEX_data/reg4_assessment reg4 global_analyses_overall_assessment_reg4_02072016 /data/project/layers/commons/NEX_data/reg4_assessment TRUE 2010,2013 6 1e+07 stage6_reg4_in_dir_list_02072016.txt TRUE
66 68

  
67 69
###Loading R library and packages  ou 
68 70
library(RPostgreSQL)
......
104 106
#CALLED FROM MASTER SCRIPT:
105 107

  
106 108
script_path <- "/nobackupp8/bparmen1/env_layers_scripts" #path to script
109
script_path <- "/home/parmentier/Data/IPLANT_project/env_layers_scripts"
107 110
function_assessment_part1_functions <- "global_run_scalingup_assessment_part1_functions_02112015.R" #PARAM12
108 111
function_assessment_part1a <-"global_run_scalingup_assessment_part1a_01042016.R"
109
function_assessment_part2 <- "global_run_scalingup_assessment_part2_02032016.R"
112
function_assessment_part2 <- "global_run_scalingup_assessment_part2_02072016.R"
110 113
function_assessment_part2_functions <- "global_run_scalingup_assessment_part2_functions_01032016.R"
114
function_assessment_part3 <- "global_run_scalingup_assessment_part3_02082016.R"
111 115
source(file.path(script_path,function_assessment_part1_functions)) #source all functions used in this script 
112 116
source(file.path(script_path,function_assessment_part1a)) #source all functions used in this script 
113 117
source(file.path(script_path,function_assessment_part2)) #source all functions used in this script 
114 118
source(file.path(script_path,function_assessment_part2_functions)) #source all functions used in this script 
119
source(file.path(script_path,function_assessment_part3)) #source all functions used in this script 
115 120

  
116 121
### Parameters, constants and arguments ###
117
stages_to_run<-c(0,0,0,0,0,6) #this stage 2 to 5 currently stored in another file.
122
stages_to_run<-c(0,0,0,0,0,0,0,8) #this stage 2 to 5 currently stored in another file.
118 123

  
119 124
CRS_locs_WGS84<-CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #constant 1
120 125

  
121
#var<-"TMAX" # variable being interpolated #param 1, arg 1
122 126
var <- args[1] # variable being interpolated #param 1, arg 1
127
var<-"TMAX" # variable being interpolated #param 1, arg 1
123 128

  
124 129
##Add for precip later...
125 130
if (var == "TMAX") {
......
142 147
#reg1 (North Am), reg2(Europe),reg3(Asia), reg4 (South Am), reg5 (Africa), reg6 (Australia-Asia)
143 148
#master directory containing the definition of tile size and tiles predicted
144 149
#in_dir1 <- "/nobackupp6/aguzman4/climateLayers/out/" #param 5, arg 2
145
in_dir1 <- args[2] #param 5, arg 2
150
in_dir <- args[2] #param 5, arg 2
151
in_dir <- "/data/project/layers/commons/NEX_data/reg4_assessment"
152

  
153
#in_dir <- "" #PARAM 0
154
#y_var_name <- "dailyTmax" #PARAM1
155
#interpolation_method <- c("gam_CAI") #PARAM2
146 156

  
147 157
#region_names <- c("reg1","reg2","reg3","reg4","reg5","reg6") #selected region names, #PARAM2
148 158
#region_names <- c("reg23","reg4") #selected region names,
149 159
#run assessment by region, this is a unique region only 
150
#region_name <- c("reg4") #param 6, arg 3
151 160
region_name <- args[3] #param 6, arg 3
161
region_name <- c("reg4") #param 6, arg 3
152 162

  
153 163
#out_prefix <- "run_global_analyses_pred_12282015" #param 7, arg 4
154 164
#out_dir <- "/nobackupp8/bparmen1/" #param 8, arg 5
155 165
#out_dir <-paste(out_dir,"_",out_prefix,sep="")
156 166
create_out_dir_param <- TRUE #param 9, arg 6
157 167
out_prefix <- args[4] #param 7, arg 4
168
out_suffix <- args[4] #param 7, arg 4s
169
out_suffix <- "global_analyses_overall_assessment_reg4_02072016"
170

  
158 171
out_dir <- args[5] #param 8, arg 5
172
out_dir <- "/data/project/layers/commons/NEX_data/reg4_assessment"
173

  
159 174
#out_dir <-paste(out_dir,"_",out_prefix,sep="")
160 175
create_out_dir_param <- args[6] #param 9, arg 6
176
create_out_dir_param <- TRUE #param 9, arg 
161 177

  
162 178
#list_year_predicted <- 1984:2004
163 179
#list_year_predicted <- c("2014") # param 10, arg 7
164 180
#year_predicted <- list_year_predicted[1]
165
list_year_predicted <- args[7] # param 10, arg 7
181
list_year_predicted <- args[7] # param 10, arg 7, min and max year
182

  
183
#run_figure_by_year <- TRUE # param 10, arg 7
184
list_year_predicted <- "2010,2013"
166 185

  
167 186
file_format <- ".tif" #format for mosaiced files # param 11
168 187
NA_flag_val <- -9999  #No data value, # param 12
169 188
#num_cores <- 6 #number of cores used # param 13, arg 8
170 189
plotting_figures <- TRUE #running part2 of assessment to generate figures... # param 14
171 190
num_cores <- args[8] #number of cores used # param 13, arg 8
191
num_cores <-6 #number of cores used # param 13, arg 8
172 192
  
173 193
##Additional parameters used in part 2, some these may be removed as code is simplified
174 194
mosaic_plot <- FALSE #param 15
175 195
day_to_mosaic <- c("19920102","19920103","19920103") #param 16, not in use...
176 196
multiple_region <- TRUE #param 17
177
countries_shp <- "/nobackupp8/bparmen1/NEX_data/countries.shp" #param 18
178
#countries_shp <-"/data/project/layers/commons/NEX_data/countries.shp" #Atlas
197
#countries_shp <- "/nobackupp8/bparmen1/NEX_data/countries.shp" #param 18
198
countries_shp <-"/data/project/layers/commons/NEX_data/countries.shp" #Atlas
179 199
plot_region <- TRUE  # param 19
180 200
threshold_missing_day <- c(367,365,300,200) # param 20
181 201

  
182 202
#max number of cells to read in memory
183 203
max_mem <- args[9] #param 21
184
in_dir_list_filename <- args[10] #param 22
185
#rasterOptions(maxmemory=1e+07,timer=TRUE)
186

  
204
max_mem <- 1e+07
205
#in_dir_list_filename <- args[10] #param 22
206
in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt"
207
#in_dir_list_filename <- "/data/project/layers/commons/NEX_data/reg4_assessment/stage6_reg4_in_dir_list_02072016.txt"
208
run_figure_by_year <- args[11] # param 10, arg 7, if true will create figures for individual years...
209
run_figure_by_year <- TRUE # param 10, arg 7, if true will create figures for individual years...
187 210

  
188
list_param_run_assessment_part2_plotting <-list(  
189
    in_dir,y_var_name, interpolation_method, out_suffix,
190
    out_dir, create_out_dir_param, mosaic_plot, proj_str, file_format, NA_flag_val,
191
    multiple_region, countries_shp, plot_region, num_cores,
192
    region_name, df_assessment_files_name, threshold_missing_day,year_predicted
193
  )
194

  
195
names(list_param_run_assessment_part2_plotting) <- c(
196
  "in_dir","y_var_name","interpolation_method","out_suffix",
197
    "out_dir","create_out_dir_param","mosaic_plot","proj_str","file_format","NA_flag_val",
198
    "multiple_region","countries_shp","plot_region","num_cores",
199
    "region_name","df_assessment_files_name","threshold_missing_day","year_predicted"
200
  )
211
#### Prepare parameters for the production of figures...
212
proj_str <- CRS_interp
213
list_in_dir <- as.list(read.table(in_dir_list_filename,stringsAsFactors=F)[,1])
201 214

  
202 215
list_param_run_assessment_combined_region_plotting_prediction <-list(  
203 216
    in_dir_list_filename,
204 217
    in_dir,y_var_name, interpolation_method, out_suffix,
205 218
    out_dir, create_out_dir_param, mosaic_plot, proj_str, file_format, NA_flag_val,
206 219
    multiple_region, countries_shp, plot_region, num_cores,
207
    region_name, df_assessment_files_name, threshold_missing_day,year_predicted)
220
    region_name, df_assessment_files_name, threshold_missing_day,list_year_predicted)
208 221

  
209 222
names(list_param_run_assessment_combined_region_plotting_prediction) <- c(
210 223
    "in_dir_list_filename",
......
224 237
  #Step 1: run individual figure production if needed:
225 238
  if(run_figure_by_year==TRUE){
226 239
    #debug(run_assessment_plotting_prediction_fun)
227
    df_assessment_figures_files <-
228
    run_assessment_plotting_prediction_fun(list_param_run_assessment_plotting)
240
    l_list_param_run_assessment_part2_plotting <- vector("list",length=length(list_in_dir))
241
    list_df_assessment_files <- lapply(list_in_dir,FUN=function(x,y){list.files(path=file.path(y,x),pattern="df_assessment_files_.*.txt",full.names=T)},y=in_dir)
242
    for(j in 1:length(list_in_dir)){
243
      
244
      df_assessment_files_name <- list_df_assessment_files[[j]]
245
      df_assessment <- read.table(df_assessment_files_name,stringsAsFactors=F,sep=",")
246
      year_tmp <- unique(df_assessment$year)
247
      reg_tmp <- unique(df_assessment$reg)
248
      in_dir_tmp <- file.path(in_dir,list_in_dir[[j]])
249
      out_suffix_tmp <- paste("s8_",year_tmp,"_",out_suffix,sep="")
250
      create_out_dir_param_tmp <- FALSE
251
      out_dir_tmp <- in_dir_tmp #write output in the existing dir from NEX
252
      
253
      list_param_run_assessment_part2_plotting <-list(  
254
             in_dir_tmp,y_var_name, interpolation_method, out_suffix_tmp,
255
             out_dir_tmp, create_out_dir_param_tmp, mosaic_plot, proj_str, file_format, NA_flag_val,
256
             multiple_region, countries_shp, plot_region, num_cores,
257
            region_name, df_assessment_files_name, threshold_missing_day,year_tmp
258
      )
259
      
260
      names(list_param_run_assessment_part2_plotting) <- c(
261
            "in_dir","y_var_name","interpolation_method","out_suffix",
262
            "out_dir","create_out_dir_param","mosaic_plot","proj_str","file_format","NA_flag_val",
263
            "multiple_region","countries_shp","plot_region","num_cores",
264
            "region_name","df_assessment_files_name","threshold_missing_day","year_predicted"
265
      )
266
      #undebug(run_assessment_plotting_prediction_fun)
267
      df_assessment_figures_files <-
268
      run_assessment_plotting_prediction_fun(list_param_run_assessment_part2_plotting)
269
      l_list_param_run_assessment_part2_plotting[[j]] <- list_param_run_assessment_part2_plotting
270
      list_df_assessment_files[[j]] <- df_assessment_figures_files
271
    }
229 272
  }
230 273

  
231 274
  #Step 2: run combination of all files...
......
233 276
  #function_assessment_part2 <- "global_run_scalingup_assessment_part2_01032016.R"
234 277
  #source(file.path(script_path,function_assessment_part2)) #source all functions used in this script
235 278

  
236
  #debug(run_assessment_combined_region_plotting_prediction_fun)
279
  debug(run_assessment_combined_region_plotting_prediction_fun)
237 280
  df_assessment_combined_figures_files <-
238 281
  run_assessment_combined_region_plotting_prediction_fun(list_param_run_assessment_combined_region_plotting_prediction)
239 282
}

Also available in: Unified diff