Project

General

Profile

« Previous | Next » 

Revision 395c00f1

Added by Benoit Parmentier over 8 years ago

testing master script stage 7 mosaicing from command line for future jobs

View differences:

climate/research/oregon/interpolation/master_script_stage_7.R
14 14

  
15 15
#AUTHOR: Benoit Parmentier                                                                        
16 16
#CREATED ON: 01/01/2016  
17
#MODIFIED ON: 06/12/2016  
17
#MODIFIED ON: 06/19/2016  
18 18
#PROJECT: NCEAS INPLANT: Environment and Organisms                                                                           
19 19

  
20 20
#First source these files:
......
27 27
#
28 28
## TODO:
29 29
# 
30
## Comments: dealing with flags and datatype in output mosaics
30
## Comments: Fixed accuracy bugs and tested command line script for jobs
31 31
# 
32

  
33
### Testing several years on the bridge before running jobs on nodes with qsub
34
#Use the following command to run as script via the shell on the bridge 
35
#run mosaic of predictions
36
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_7_06192016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 reg4_1999 /nobackupp8/bparmen1/climateLayers/out/reg4 TRUE 1999 6 1e+07 use_edge_weights rmse 19990107 19990108 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif /nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt python var_pred FALSE Int16 100 -100,100
37
#run mosaic for kriged residuals testing
38
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_7_06192016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 reg4_1999 /nobackupp8/bparmen1/climateLayers/out/reg4 TRUE 1999 6 1e+07 use_edge_weights rmse 19990107 19990108 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif /nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt python res_testing FALSE Int16 100 -100,100
39
#run mosaic for ac testing, rmse
40
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_7_06192016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 reg4_1999 /nobackupp8/bparmen1/climateLayers/out/reg4 TRUE 1999 6 1e+07 use_edge_weights rmse 19990107 19990108 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif /nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt python ac_testing FALSE Int16 100 -100,100
41
#run mosaic for ac testing, n
42
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_7_06192016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 reg4_1999 /nobackupp8/bparmen1/climateLayers/out/reg4 TRUE 1999 6 1e+07 use_edge_weights n 19990107 19990108 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif /nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt python ac_testing FALSE Int16 100 -100,100
43

  
32 44
##################################################################################################
33 45

  
34 46
### PARAMETERS DEFINED IN THE SCRIPT
......
65 77

  
66 78
#script_path <- "/home/parmentier/Data/IPLANT_project/env_layers_scripts"
67 79
script_path <- "/nobackupp8/bparmen1/env_layers_scripts" #path to script
68
function_mosaicing_functions <- "global_run_scalingup_mosaicing_function_06122016.R" #PARAM12
69
function_mosaicing <-"global_run_scalingup_mosaicing_06122016.R"
80
function_mosaicing_functions <- "global_run_scalingup_mosaicing_function_06192016.R" #PARAM12
81
function_mosaicing <-"global_run_scalingup_mosaicing_06192016.R"
70 82
source(file.path(script_path,function_mosaicing)) #source all functions used in this script 
71 83
source(file.path(script_path,function_mosaicing_functions)) #source all functions used in this script 
72 84

  
......
96 108
#The 20 arguments are passed directly from Rscript:
97 109

  
98 110
### Use the following values to run code from the shell:
99
var <- "TMAX" # variable being interpolated #param 1, arg 1
100
in_dir <- "/nobackupp6/aguzman4/climateLayers/out/" #PARAM2,arg 2
101
region_name <- "reg4" #PARAM 3, arg 3 #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3
102
out_suffix <- "reg4_1999" #PARAM 4, arg 4
103
out_suffix_str <- region_name #PARAM 4, CONST 3
104
out_dir <- "/nobackupp8/bparmen1/climateLayers/out/reg4" #PARAM 5,arg 5 use this location for now
105
create_out_dir_param <- TRUE #PARAM 6, arg 6
106
year_predicted <- 1999 #PARAM 7, arg 7
107
num_cores <- 6 #PARAM 8, arg 8         
108
max_mem = 1e+07 #param 9, arg 9
109
mosaicing_method <- "use_edge_weights" #PARAM10, arg 10
110
#metric_name <- "rmse" # "mae", "r" for MAE, R etc.; can also be ns or nv? #PARAM 11, arg 11
111
metric_name <- "n"
112
#metric_name <- "mae"
113

  
114
#day_start <- "19990701" #PARAM 12 arg 12
115
#day_end <- "19990703" #PARAM 13 arg 13
116
day_start <- "19990107" #PARAM 12 arg 12
117
day_end <- "19990108" #PARAM 13 arg 13
118
infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif" #PARAM 14, arg 14
119
df_assessment_files_name <- "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt"  # data.frame with all files used in assessmnet, PARAM 15
120
algorithm <- "python" #PARAM 16 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
121
#layers_option <- c("var_pred") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
122
layers_option <- c("ac_training") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
123
#layers_option <- c("res_training") # #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
124
#layers_option <- c("res_testing") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
125
#layers_option <- c("ac_testing") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
126

  
127
tmp_files <- TRUE #arg 18, param 18, keep temp files if TRUE
128
data_type <- "Int16" #, param 19, use int32 for output layers mosaiced
129
scaling <- 100 #, param 20, if null use 1
130
values_range <- "-100,100"
131
NA_flag_val <- -32768 #should be here
132

  
133 111
# var <- "TMAX" # variable being interpolated #param 1, arg 1
134 112
# in_dir <- "/nobackupp6/aguzman4/climateLayers/out/" #PARAM2,arg 2
135 113
# region_name <- "reg4" #PARAM 3, arg 3 #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3
136
# out_suffix <- "reg4_1991" #PARAM 4, arg 4
114
# out_suffix <- "reg4_1999" #PARAM 4, arg 4
137 115
# out_suffix_str <- region_name #PARAM 4, CONST 3
138 116
# out_dir <- "/nobackupp8/bparmen1/climateLayers/out/reg4" #PARAM 5,arg 5 use this location for now
139 117
# create_out_dir_param <- TRUE #PARAM 6, arg 6
140
# year_predicted <- 1991 #PARAM 7, arg 7
118
# year_predicted <- 1999 #PARAM 7, arg 7
141 119
# num_cores <- 6 #PARAM 8, arg 8         
142 120
# max_mem = 1e+07 #param 9, arg 9
143 121
# mosaicing_method <- "use_edge_weights" #PARAM10, arg 10
144
# metric_name <- "rmse" #RMSE, MAE etc. #PARAM 11, arg 11
145
# day_start <- "19910101" #PARAM 12 arg 12
146
# day_end <- "19910103" #PARAM 13 arg 13
147
# #infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_reg4.tif" #PARAM 14, arg 14
148
# infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST.tif"
149
# df_assessment_files_name <- "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt"  # data.frame with all files used in assessmnet, PARAM 15
150
# algorithm <- "python" #PARAM 16 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
151
# layers_option <- c("var_pred") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
152
# tmp_files <- FALSE #arg 18, param 18
153
# data_type <- "Int16" #, param 19, use int32 for output layers mosaiced
154
# scaling <- 100 #, param 20, if NULL don't rescale values
155
# values_range <- "-100,100" #args 21, if NULL don't screen values
122
# metric_name <- "rmse" # "mae", "r" for MAE, R etc.; can also be ns or nv? #PARAM 11, arg 11
123
#metric_name <- "n"
124
#metric_name <- "mae"
125

  
126
#day_start <- "19990701" #PARAM 12 arg 12
127
#day_end <- "19990703" #PARAM 13 arg 13
128
#day_start <- "19990107" #PARAM 12 arg 12
129
#day_end <- "19990108" #PARAM 13 arg 13
130
#infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif" #PARAM 14, arg 14
131
#df_assessment_files_name <- "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1999/df_assessment_files_reg4_1999_reg4_1999.txt"  # data.frame with all files used in assessmnet, PARAM 15
132
#algorithm <- "python" #PARAM 16 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
133
#layers_option <- c("var_pred") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
134
#layers_option <- c("ac_training") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
135
#layers_option <- c("res_training") # #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
136
#layers_option <- c("res_testing") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
137
#layers_option <- c("ac_testing") #arg 17 ,param 17, options are:#res_training, res_testing,ac_training, ac_testing, var_pred
138

  
139
#tmp_files <- TRUE #arg 18, param 18, keep temp files if TRUE
140
#data_type <- "Int16" #, param 19, use int32 for output layers mosaiced
141
#scaling <- 100 #, param 20, if null use 1
142
#values_range <- "-100,100"
156 143
#NA_flag_val <- -32768 #should be here
157 144

  
158 145

  
159 146
#path_assessment <- NOT USED "/nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991" #PARAM 14a, arg 14
160 147

  
161
### Testing several years on the bridge before running jobs on nodes with qsub
162
#Use the following command to run as script via the shell on the bridge 
163
#Rscript /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_7_04232016.R TMAX /nobackupp6/aguzman4/climateLayers/out/ reg4 reg4_1991 /nobackupp8/bparmen1/climateLayers/out/reg4 TRUE 1991 6 1e+07 use_edge_weights rmse 19910101 19910103 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST.tif /nobackupp6/aguzman4/climateLayers/out/reg4/assessment/output_reg4_1991/df_assessment_files_reg4_1991_reg4_1991.txt python var_pred FALSE Int16 100 -100,100
164 148

  
165 149
############################
166 150

  

Also available in: Unified diff