Project

General

Profile

« Previous | Next » 

Revision b4057554

Added by Benoit Parmentier over 8 years ago

stage 7 mosaicing, major clean up and modification to allow call from shell for jobs on NEX

View differences:

climate/research/oregon/interpolation/master_script_stage_7.R
30 30
##################################################################################################
31 31

  
32 32
### PARAMETERS DEFINED IN THE SCRIPT
33
#There are 21 parameters, 1 constant and 8 arguments (drawn from the parameters) for the Rscript call.
34
#The arguments are passed directly from Rscript:
35
#var <- args[1] # variable being interpolated #param 1, arg 1
36
#in_dir1 <- args[2] # This is the output directory containing global prediction e.g./nobackupp6/aguzman4/climateLayers/out/ param 5, arg 2
37
#region_name <- args[3] # region e.g. "reg4" param 6, arg 3
38
#out_prefix <- args[4] # this is used in creating an output directory,include region name? param 7, arg 4
39
#out_dir <- args[5] # output parent dir, can be home dir or other, param 8, arg 5)
40
#create_out_dir_param <- args[6] # if TRUE create a output from "output"+out_prefix param 9, arg 6
41
#list_year_predicted <- args[7] # enter as list but currently runs on the first element of the list, param 10, arg 7
42
#num_cores <- args[8] #number of cores used # param 13, arg 8
43
#max_mem <- args[9] # maximum memory, param 21
33

  
44 34

  
45 35
###Loading R library and packages  ou 
46 36
library(RPostgreSQL)
......
73 63

  
74 64
#script_path <- "/home/parmentier/Data/IPLANT_project/env_layers_scripts"
75 65
script_path <- "/nobackupp8/bparmen1/env_layers_scripts" #path to script
76
function_mosaicing_functions <- "global_run_scalingup_mosaicing_function_04112016.R" #PARAM12
77
function_mosaicing <-"global_run_scalingup_mosaicing_04102016.R"
66
function_mosaicing_functions <- "global_run_scalingup_mosaicing_function_04102016.R" #PARAM12
67
function_mosaicing <-"global_run_scalingup_mosaicing_04112016.R"
78 68
source(file.path(script_path,function_mosaicing)) #source all functions used in this script 
79 69
source(file.path(script_path,function_mosaicing_functions)) #source all functions used in this script 
80 70

  
......
99 89
#Data is on ATLAS or NASA NEX
100 90

  
101 91
### PARAMETERS DEFINED IN THE SCRIPT
102
#There are 21 parameters, 1 constant and 8 arguments (drawn from the parameters) for the Rscript call.
92
#There are 31 parameters, 1 constant and 17 arguments (drawn from the parameters) for the Rscript call.
103 93
#The arguments are passed directly from Rscript:
104 94
#var <- args[1] # variable being interpolated #param 1, arg 1
105 95
#in_dir <- args[2] # This is the output directory containing global prediction e.g./nobackupp6/aguzman4/climateLayers/out/ param 5, arg 2
106
#region_name <- args[3] # region e.g. "reg4" param 6, arg 3
96
#region_name <- args[3] # region e.g. "reg4" param 3, arg 3
107 97
#out_suffix <- args[4] # formely out_prefix, this is used in creating an output directory, it is suggested to use "reg4" or same as region_name
108
#out_dir <- args[5] # output parent dir, can be home dir or other, param 8, arg 5
109
#create_out_dir_param <- args[6] # if TRUE create a output from "output"+out_prefix param 9, arg 6
110
#year_predicted <- args[7] # enter as list but currently runs on the first element of the list, param 10, arg 7
111
#num_cores <- args[8] #number of cores used # param 13, arg 8
112
#max_mem <- args[9] # maximum memory, param 21
113
#mosaicing_method <- arg[10] #PARAM5
114
#metric_name <- arg[11] #"rmse" #RMSE, MAE etc. #PARAM 8
115
#day_to_mosaic_range <- arg[12] #c("19910101","19910103") #if null run all year
116
#infile_mask <- arg[12] # "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_reg4.tif"
117
#df_assessment_files_name <- arg[13] #"/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt" # data.frame with all files used in assessmnet, PARAM 21
118
#algorithm <- arg[14] #"python" #PARAM 28 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
119
#layers_option <- arg[15] #c("var_pred") #options are:
120
#res_training, res_testing,ac_training, ac_testing, var_pred
121
#tmp_files <- arg[16] #FALSE
122

  
123
#mosaicing_method <- c("unweighted","use_edge_weights") #PARAM5
124
#metric_name <- "rmse" #RMSE, MAE etc. #PARAM 8
125
#day_to_mosaic_range <- c("19910101","19910103") #if null run all year
126
#infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_reg4.tif"
127
#df_assessment_files_name <- "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt" # data.frame with all files used in assessmnet, PARAM 21
128
#algorithm <- "python" #PARAM 28 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
129
#layers_option <- c("var_pred") #options are:
130
#res_training, res_testing,ac_training, ac_testing, var_pred
131
#tmp_files <- FALSE
98
#out_dir <- args[5] # output parent dir, can be home dir or other, param 5, arg 5
99
#create_out_dir_param <- args[6] # if TRUE create a output from "output"+out_prefix param 6, arg 6
100
#year_predicted <- args[7] # enter as list but currently runs on the first element of the list, param 7, arg 7
101
#num_cores <- args[8] #number of cores used # param 8, arg 8
102
#max_mem <- args[9] # maximum memory, param 9
103
#mosaicing_method <- arg[10] #PARAM10
104
#metric_name <- arg[11] #"rmse" #RMSE, MAE etc. #PARAM 11
105
#day_to_mosaic_range <- arg[12] #c("19910101","19910103") #if null run all year, param 12
106
#infile_mask <- arg[13] # "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_reg4.tif"
107
#df_assessment_files_name <- arg[14] #"/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt" # data.frame with all files used in assessmnet, PARAM 21
108
#algorithm <- arg[15] #"python" #PARAM 28 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
109
#layers_option <- arg[16] #c("var_pred") #options are:res_training, res_testing,ac_training, ac_testing, var_pred
110
#tmp_files <- arg[17] #FALSE
111

  
112
### Use the following values to run code from the shell:
113
#var<-"TMAX" # variable being interpolated #param 1, arg 1
114
#in_dir <- "/nobackupp6/aguzman4/climateLayers/out/" #PARAM2,arg 2
115
#region_name <- "reg4" #PARAM 3, arg 3 #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3
116
#out_suffix <- "reg4" #PARAM 4, arg 4
117
#out_dir <- "/nobackupp8/bparmen1/climateLayers/out/reg4" #PARAM 5,arg 5 use this location for now
118
#create_out_dir_param <- TRUE #PARAM 6, arg 6
119
#year_predicted <- 1991 #PARAM 7, arg 7
120
#num_cores <- 6 #PARAM 8, arg 8         
121
#max_mem = 1e+07 #param 9, arg 9
122
#mosaicing_method <- use_edge_weights" #PARAM10, arg 10
123
#metric_name <- "rmse" #RMSE, MAE etc. #PARAM 11, arg 11
124
#day_start <- "19910101" #PARAM 12
125
#day_end <- "19910101" #PARAM 13
126
#infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_reg4.tif" #PARAM 14, arg 14
127
#df_assessment_files_name <- "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt"  # data.frame with all files used in assessmnet, PARAM 15
128
#algorithm <- "python" #PARAM 16 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
129
#layers_option <- c("var_pred") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
130
#tmp_files <- FALSE #arg 18, param 18
131

  
132
#path_assessment <- NOT USED "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991" #PARAM 14a, arg 14
133

  
134
### Testing several years on the bridge before running jobs on nodes with qsub
135
#Use the following command to run as script via the shell on the bridge 
136
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_7_04112016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 reg4 /nobackupp8/bparmen1/climateLayers/out/reg4 TRUE 1991 6 1e+07 use_edge_weights rmse 19910101 19910103 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_reg4.tif /nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt python var_pred FALSE 
137

  
138
############################
132 139

  
133 140
var <- args[1] # variable being interpolated #param 1, arg 1
134
var<-"TMAX" # variable being interpolated #param 1, arg 1
135

  
136
var<-"TMAX" # variable being interpolated
137
if (var == "TMAX") {
138
  y_var_name <- "dailyTmax"
139
  y_var_month <- "TMax"
140
}
141
if (var == "TMIN") {
142
  y_var_name <- "dailyTmin"
143
  y_var_month <- "TMin"
144
}
141
#var<-"TMAX" # variable being interpolated #param 1, arg 1
145 142

  
146 143
#PARAM 2
147 144
#in_dir <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015" #NCEAS
148 145
#in_dir <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_12072015" #NEX
146
#in_dir <- "/nobackupp6/aguzman4/climateLayers/out/" #PARAM2
147
in_dir <- args[2] #PARAM2
149 148

  
150
in_dir <- "/nobackupp6/aguzman4/climateLayers/out/"
151
in_dir <- args[2]
152
interpolation_method <- c("gam_CAI") #PARAM3
153

  
154

  
155
#var <- args[1] # variable being interpolated #param 1, arg 1
156
#in_dir <- args[2] # This is the output directory containing global prediction e.g./nobackupp6/aguzman4/climateLayers/out/ param 5, arg 2
157
#region_name <- args[3] # region e.g. "reg4" param 6, arg 3
158
#out_suffix <- args[4] # formely out_prefix, this is used in creating an output directory, it is suggested to use "reg4" or same as region_name
159
#out_dir <- args[5] # output parent dir, can be home dir or other, param 8, arg 5
160
#create_out_dir_param <- args[6] # if TRUE create a output from "output"+out_prefix param 9, arg 6
161
#year_predicted <- args[7] # enter as list but currently runs on the first element of the list, param 10, arg 7
162
#num_cores <- args[8] #number of cores used # param 13, arg 8
163
#max_mem <- args[9] # maximum memory, param 21
164
#mosaicing_method <- args[10] #PARAM5
165
#metric_name <- args[11] #"rmse" #RMSE, MAE etc. #PARAM 8
166
#day_to_mosaic_range <- arg[12] #c("19910101","19910103") #if null run all year
167
#infile_mask <- args[13] # "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_reg4.tif"
168
#df_assessment_files_name <- args[14] #"/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt" # data.frame with all files used in assessmnet, PARAM 21
169
#algorithm <- args[15] #"python" #PARAM 28 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
170
#layers_option <- args[16] #c("var_pred") #options are:
171
#res_training, res_testing,ac_training, ac_testing, var_pred
172
#tmp_files <- args[17] #FALSE
173

  
174
region_name <- args[3]
175
region_name <- "reg4" #PARAM 4 #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3
149
region_name <- args[3] #PARAM3
150
#region_name <- "reg4" #PARAM 3 #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3
176 151

  
177 152
#out_suffix <- paste(region_name,"_","run10_1500x4500_global_analyses_pred_1991_04052016",sep="") #PARAM 6
178 153
#out_suffix_str <- "run10_1500x4500_global_analyses_pred_1991_04052016" #PARAM 7
179 154

  
180
out_suffix <- args[4]
181
out_suffix <- region_name #PARAM 6
182
out_suffix_str <- region_name #PARAM 7
183
#out_dir <- in_dir #PARAM 11
184
out_dir <- args[5]
185
out_dir <- "/nobackupp8/bparmen1/climateLayers/out/reg4" #PARAM 11, use this location for now
186
create_out_dir_param <- args[6]
187
create_out_dir_param <- TRUE #PARAM 12
155
out_suffix <- args[4] #PARAM 4
156
#out_suffix <- region_name #PARAM 4
157
out_suffix_str <- region_name #PARAM 4
158
out_dir <- args[5] #PARAM 5
159
#out_dir <- "/nobackupp8/bparmen1/climateLayers/out/reg4" #PARAM 5, use this location for now
160
create_out_dir_param <- args[6] #PARAM 6
161
#create_out_dir_param <- TRUE #PARAM 6
188 162

  
189
year_predicted <- args[7]
190
year_predicted <- 1991 #PARAM 31
163
year_predicted <- args[7] #PARAM 7
164
#year_predicted <- 1991 #PARAM 7
191 165

  
192
num_cores <- args[8]
193
num_cores <- 6 #PARAM 17         
166
num_cores <- args[8] #PARAM 8
167
#num_cores <- 6 #PARAM 8         
194 168

  
195 169
#max number of cells to read in memory
196
max_mem<-args[9]
170
max_mem<-args[9] #PARAM 9
197 171

  
198
mosaicing_method <- c("unweighted","use_edge_weights") #PARAM5
199
mosaicing_method <- args[10]
172
#mosaicing_method <- c("unweighted","use_edge_weights") #PARAM10
173
mosaicing_method <- args[10] #PARAM10
200 174

  
201 175
metric_name <- args[11]
202
metric_name <- "rmse" #RMSE, MAE etc. #PARAM 8
203
#if daily mosaics NULL then mosaicas all days of the year #PARAM 13
204
#day_to_mosaic <- c("19910101","19910102","19910103") #,"19920104","19920105") #PARAM9, two dates note in /tiles for now on NEX
205
day_to_mosaic_range <- c("19910101","19910103") #if null run all year
206
day_to_mosaic_range <- c("19910101","19910101") #if null run all year
207
day_to_mosaic_range <- args[12]
208

  
209
###Separate folder for masks by regions, should be listed as just the dir!!... #PARAM 20
210
infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_reg4.tif"
211
inflie_mask <- args[13]
176
#metric_name <- "rmse" #RMSE, MAE etc. #PARAM 11
177
#if daily mosaics NULL then mosaicas all days of the year #PARAM 12
178
#day_to_mosaic_range <- c("19910101","19910103") #if null run all year #PARAM 12
179
#day_to_mosaic_range <- c("19910101","19910101") #if null run all year #PARAM 12
180
#day_to_mosaic_range <- args[12] #PARAM 12
181
day_start <- args[12] #PARAM 12
182
day_end <- args[13] #PARAM 13
183

  
184
###Separate folder for masks by regions, should be listed as just the dir!!... #PARAM 14
185
#infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_reg4.tif" #PARAM 14
186
infile_mask <- args[14]
212 187
#infile_mask <- "/data/project/layers/commons/NEX_data/regions_input_files/r_mask_reg4.tif"
213 188
## All of this is interesting so use df_assessment!!
214 189

  
215 190
#path_assessment <- "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991"
216 191
#path_assessment <- file.path(in_dir,region_name,"assessment",paste("output_",region_name,year_processed,sep=""))
217
df_assessment_files_name <- args[14]
218
df_assessment_files_name <- "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt" # data.frame with all files used in assessmnet, PARAM 21
219
algorithm <- args[15]
220
algorithm <- "python" #PARAM 28 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
192

  
193
df_assessment_files_name <- args[15] #PARAM 15
194
#df_assessment_files_name <- "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt" # data.frame with all files used in assessmnet, PARAM 14
195
algorithm <- args[16] #PARAM 16
196
#algorithm <- "python" #PARAM 15 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
221 197
#algorithm <- "R" #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
222 198

  
223
layers_option <- c("var_pred") #options are:
199
layers_option <- args[17] # PARAM 17 options are:
200
#layers_option <- c("var_pred") #options are:
224 201
#res_training, res_testing,ac_training, ac_testing, var_pred
225
tmp_files <- FALSE
226

  
227
pred_mod_name <- "mod1" #PARAM 9
228
var_pred <- "res_mod1" #used in residuals mapping #PARAM 10
229

  
230

  
231
#CRS_WGS84 <- CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84 #CONSTANT1
232
#CRS_locs_WGS84<-CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84
233
#proj_str<- CRS_WGS84 #PARAM 8 #check this parameter
234
 
235
file_format <- ".tif" #PARAM 14
236
NA_value <- -9999 #PARAM 15
237
NA_flag_val <- NA_value #PARAM 16
238
     
239
#region_names <- c("reg23","reg4") #selected region names, ##PARAM 18 
240
use_autokrige <- F #PARAM 19
241
proj_str <- CRS_locs_WGS84
242

  
243
#in_dir can be on NEX or Atlas
244

  
202
tmp_files <- args[18] #PARAM 18
203
#tmp_files <- FALSE
204
interpolation_method <- c("gam_CAI") #PARAM19
205
pred_mod_name <- "mod1" #PARAM 20
206
var_pred <- "res_mod1" #used in residuals mapping #PARAM 21
207
proj_str<- CRS_WGS84 #PARAM 22 #check this parameter
208
file_format <- ".tif" #PARAM 23
209
NA_value <- -9999 #PARAM 24
210
NA_flag_val <- NA_value #PARAM 24
211
use_autokrige <- F #PARAM 25
212
proj_str <- CRS_locs_WGS84 #PARAM 26
245 213
#python script and gdal on NEX NASA:
246
mosaic_python <- "/nobackupp6/aguzman4/climateLayers/sharedCode/"
247
python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules2/bin"
214
mosaic_python <- "/nobackupp6/aguzman4/climateLayers/sharedCode/" #PARAM 27
215
python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules2/bin" #PARAM 28
248 216
#python script and gdal on Atlas NCEAS
249
#mosaic_python <- "/data/project/layers/commons/NEX_data/sharedCode" #PARAM 26
250
#python_bin <- "/usr/bin" #PARAM 27
251

  
217
#mosaic_python <- "/data/project/layers/commons/NEX_data/sharedCode" #PARAM 29
218
#python_bin <- "/usr/bin" #PARAM 30
252 219
match_extent <- "FALSE" #PARAM 29 #try without matching!!!
253

  
254 220
#for residuals...
255 221
list_models <- NULL #PARAM 30
256 222
#list_models <- paste(var_pred,"~","1",sep=" ") #if null then this is the default...
257 223
  
224
if (var == "TMAX") {
225
  y_var_name <- "dailyTmax"
226
  y_var_month <- "TMax"
227
}
228
if (var == "TMIN") {
229
  y_var_name <- "dailyTmin"
230
  y_var_month <- "TMin"
231
}
232

  
233
if(!(is.null(day_start)) & !(is.null(day_end))){
234
  day_to_mosaic_range <- c(day_start,day_end) #if null run all year #PARAM 12
235
}else{
236
  day_to_mosaic_range <- NULL
237
}
238

  
239
#browser()
258 240

  
259 241
#rasterOptions(maxmemory=1e+07,timer=TRUE)
260 242
list_param_run_mosaicing_prediction <- list(in_dir,y_var_name,interpolation_method,region_name,
261
                 mosaicing_method,out_suffix,out_suffix_str,metric_name,pred_mod_name,var_pred,
243
                 mosaicing_method,out_suffix,out_suffix_str,metric_name,pred_mod_name,var_pred, out_dir,
262 244
                 create_out_dir_param,day_to_mosaic_range,year_predicted,proj_str,file_format,NA_value,num_cores,
263 245
                 use_autokrige,infile_mask,df_assessment_files_name,mosaic_python,
264 246
                 python_bin,algorithm,match_extent,list_models,layers_option,tmp_files)
265 247
param_names <- c("in_dir","y_var_name","interpolation_method","region_name",
266
                 "mosaicing_method","out_suffix","out_suffix_str","metric_name","pred_mod_name","var_pred",
248
                 "mosaicing_method","out_suffix","out_suffix_str","metric_name","pred_mod_name","var_pred","out_dir",
267 249
                 "create_out_dir_param","day_to_mosaic_range","year_predicted","proj_str","file_format","NA_value","num_cores",
268 250
                 "use_autokrige","infile_mask","df_assessment_files_name","mosaic_python",
269 251
                 "python_bin","algorithm","match_extent","list_models","layers_option","tmp_files")

Also available in: Unified diff