Revision 395c00f1
Added by Benoit Parmentier over 8 years ago
climate/research/oregon/interpolation/master_script_stage_7.R | ||
---|---|---|
14 | 14 |
|
15 | 15 |
#AUTHOR: Benoit Parmentier |
16 | 16 |
#CREATED ON: 01/01/2016 |
17 |
#MODIFIED ON: 06/12/2016
|
|
17 |
#MODIFIED ON: 06/19/2016
|
|
18 | 18 |
#PROJECT: NCEAS INPLANT: Environment and Organisms |
19 | 19 |
|
20 | 20 |
#First source these files: |
... | ... | |
27 | 27 |
# |
28 | 28 |
## TODO: |
29 | 29 |
# |
30 |
## Comments: dealing with flags and datatype in output mosaics
|
|
30 |
## Comments: Fixed accuracy bugs and tested command line script for jobs
|
|
31 | 31 |
# |
32 |
|
|
33 |
### Testing several years on the bridge before running jobs on nodes with qsub |
|
34 |
#Use the following command to run as script via the shell on the bridge |
|
35 |
#run mosaic of predictions |
|
36 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_7_06192016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 reg4_1999 /nobackupp8/bparmen1/climateLayers/out/reg4 TRUE 1999 6 1e+07 use_edge_weights rmse 19990107 19990108 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif /nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt python var_pred FALSE Int16 100 -100,100 |
|
37 |
#run mosaic for kriged residuals testing |
|
38 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_7_06192016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 reg4_1999 /nobackupp8/bparmen1/climateLayers/out/reg4 TRUE 1999 6 1e+07 use_edge_weights rmse 19990107 19990108 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif /nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt python res_testing FALSE Int16 100 -100,100 |
|
39 |
#run mosaic for ac testing, rmse |
|
40 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_7_06192016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 reg4_1999 /nobackupp8/bparmen1/climateLayers/out/reg4 TRUE 1999 6 1e+07 use_edge_weights rmse 19990107 19990108 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif /nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt python ac_testing FALSE Int16 100 -100,100 |
|
41 |
#run mosaic for ac testing, n |
|
42 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_7_06192016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 reg4_1999 /nobackupp8/bparmen1/climateLayers/out/reg4 TRUE 1999 6 1e+07 use_edge_weights n 19990107 19990108 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif /nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt python ac_testing FALSE Int16 100 -100,100 |
|
43 |
|
|
32 | 44 |
################################################################################################## |
33 | 45 |
|
34 | 46 |
### PARAMETERS DEFINED IN THE SCRIPT |
... | ... | |
65 | 77 |
|
66 | 78 |
#script_path <- "/home/parmentier/Data/IPLANT_project/env_layers_scripts" |
67 | 79 |
script_path <- "/nobackupp8/bparmen1/env_layers_scripts" #path to script |
68 |
function_mosaicing_functions <- "global_run_scalingup_mosaicing_function_06122016.R" #PARAM12
|
|
69 |
function_mosaicing <-"global_run_scalingup_mosaicing_06122016.R"
|
|
80 |
function_mosaicing_functions <- "global_run_scalingup_mosaicing_function_06192016.R" #PARAM12
|
|
81 |
function_mosaicing <-"global_run_scalingup_mosaicing_06192016.R"
|
|
70 | 82 |
source(file.path(script_path,function_mosaicing)) #source all functions used in this script |
71 | 83 |
source(file.path(script_path,function_mosaicing_functions)) #source all functions used in this script |
72 | 84 |
|
... | ... | |
96 | 108 |
#The 20 arguments are passed directly from Rscript: |
97 | 109 |
|
98 | 110 |
### Use the following values to run code from the shell: |
99 |
var <- "TMAX" # variable being interpolated #param 1, arg 1 |
|
100 |
in_dir <- "/nobackupp6/aguzman4/climateLayers/out/" #PARAM2,arg 2 |
|
101 |
region_name <- "reg4" #PARAM 3, arg 3 #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3 |
|
102 |
out_suffix <- "reg4_1999" #PARAM 4, arg 4 |
|
103 |
out_suffix_str <- region_name #PARAM 4, CONST 3 |
|
104 |
out_dir <- "/nobackupp8/bparmen1/climateLayers/out/reg4" #PARAM 5,arg 5 use this location for now |
|
105 |
create_out_dir_param <- TRUE #PARAM 6, arg 6 |
|
106 |
year_predicted <- 1999 #PARAM 7, arg 7 |
|
107 |
num_cores <- 6 #PARAM 8, arg 8 |
|
108 |
max_mem = 1e+07 #param 9, arg 9 |
|
109 |
mosaicing_method <- "use_edge_weights" #PARAM10, arg 10 |
|
110 |
#metric_name <- "rmse" # "mae", "r" for MAE, R etc.; can also be ns or nv? #PARAM 11, arg 11 |
|
111 |
metric_name <- "n" |
|
112 |
#metric_name <- "mae" |
|
113 |
|
|
114 |
#day_start <- "19990701" #PARAM 12 arg 12 |
|
115 |
#day_end <- "19990703" #PARAM 13 arg 13 |
|
116 |
day_start <- "19990107" #PARAM 12 arg 12 |
|
117 |
day_end <- "19990108" #PARAM 13 arg 13 |
|
118 |
infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif" #PARAM 14, arg 14 |
|
119 |
df_assessment_files_name <- "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt" # data.frame with all files used in assessmnet, PARAM 15 |
|
120 |
algorithm <- "python" #PARAM 16 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann |
|
121 |
#layers_option <- c("var_pred") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred |
|
122 |
layers_option <- c("ac_training") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred |
|
123 |
#layers_option <- c("res_training") # #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred |
|
124 |
#layers_option <- c("res_testing") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred |
|
125 |
#layers_option <- c("ac_testing") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred |
|
126 |
|
|
127 |
tmp_files <- TRUE #arg 18, param 18, keep temp files if TRUE |
|
128 |
data_type <- "Int16" #, param 19, use int32 for output layers mosaiced |
|
129 |
scaling <- 100 #, param 20, if null use 1 |
|
130 |
values_range <- "-100,100" |
|
131 |
NA_flag_val <- -32768 #should be here |
|
132 |
|
|
133 | 111 |
# var <- "TMAX" # variable being interpolated #param 1, arg 1 |
134 | 112 |
# in_dir <- "/nobackupp6/aguzman4/climateLayers/out/" #PARAM2,arg 2 |
135 | 113 |
# region_name <- "reg4" #PARAM 3, arg 3 #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3 |
136 |
# out_suffix <- "reg4_1991" #PARAM 4, arg 4
|
|
114 |
# out_suffix <- "reg4_1999" #PARAM 4, arg 4
|
|
137 | 115 |
# out_suffix_str <- region_name #PARAM 4, CONST 3 |
138 | 116 |
# out_dir <- "/nobackupp8/bparmen1/climateLayers/out/reg4" #PARAM 5,arg 5 use this location for now |
139 | 117 |
# create_out_dir_param <- TRUE #PARAM 6, arg 6 |
140 |
# year_predicted <- 1991 #PARAM 7, arg 7
|
|
118 |
# year_predicted <- 1999 #PARAM 7, arg 7
|
|
141 | 119 |
# num_cores <- 6 #PARAM 8, arg 8 |
142 | 120 |
# max_mem = 1e+07 #param 9, arg 9 |
143 | 121 |
# mosaicing_method <- "use_edge_weights" #PARAM10, arg 10 |
144 |
# metric_name <- "rmse" #RMSE, MAE etc. #PARAM 11, arg 11 |
|
145 |
# day_start <- "19910101" #PARAM 12 arg 12 |
|
146 |
# day_end <- "19910103" #PARAM 13 arg 13 |
|
147 |
# #infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_reg4.tif" #PARAM 14, arg 14 |
|
148 |
# infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST.tif" |
|
149 |
# df_assessment_files_name <- "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt" # data.frame with all files used in assessmnet, PARAM 15 |
|
150 |
# algorithm <- "python" #PARAM 16 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann |
|
151 |
# layers_option <- c("var_pred") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred |
|
152 |
# tmp_files <- FALSE #arg 18, param 18 |
|
153 |
# data_type <- "Int16" #, param 19, use int32 for output layers mosaiced |
|
154 |
# scaling <- 100 #, param 20, if NULL don't rescale values |
|
155 |
# values_range <- "-100,100" #args 21, if NULL don't screen values |
|
122 |
# metric_name <- "rmse" # "mae", "r" for MAE, R etc.; can also be ns or nv? #PARAM 11, arg 11 |
|
123 |
#metric_name <- "n" |
|
124 |
#metric_name <- "mae" |
|
125 |
|
|
126 |
#day_start <- "19990701" #PARAM 12 arg 12 |
|
127 |
#day_end <- "19990703" #PARAM 13 arg 13 |
|
128 |
#day_start <- "19990107" #PARAM 12 arg 12 |
|
129 |
#day_end <- "19990108" #PARAM 13 arg 13 |
|
130 |
#infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif" #PARAM 14, arg 14 |
|
131 |
#df_assessment_files_name <- "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt" # data.frame with all files used in assessmnet, PARAM 15 |
|
132 |
#algorithm <- "python" #PARAM 16 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann |
|
133 |
#layers_option <- c("var_pred") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred |
|
134 |
#layers_option <- c("ac_training") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred |
|
135 |
#layers_option <- c("res_training") # #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred |
|
136 |
#layers_option <- c("res_testing") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred |
|
137 |
#layers_option <- c("ac_testing") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred |
|
138 |
|
|
139 |
#tmp_files <- TRUE #arg 18, param 18, keep temp files if TRUE |
|
140 |
#data_type <- "Int16" #, param 19, use int32 for output layers mosaiced |
|
141 |
#scaling <- 100 #, param 20, if null use 1 |
|
142 |
#values_range <- "-100,100" |
|
156 | 143 |
#NA_flag_val <- -32768 #should be here |
157 | 144 |
|
158 | 145 |
|
159 | 146 |
#path_assessment <- NOT USED "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991" #PARAM 14a, arg 14 |
160 | 147 |
|
161 |
### Testing several years on the bridge before running jobs on nodes with qsub |
|
162 |
#Use the following command to run as script via the shell on the bridge |
|
163 |
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_7_04232016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 reg4_1991 /nobackupp8/bparmen1/climateLayers/out/reg4 TRUE 1991 6 1e+07 use_edge_weights rmse 19910101 19910103 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST.tif /nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt python var_pred FALSE Int16 100 -100,100 |
|
164 | 148 |
|
165 | 149 |
############################ |
166 | 150 |
|
Also available in: Unified diff
testing master script stage 7 mosaicing from command line for future jobs