Project

General

Profile

« Previous | Next » 

Revision 4b605f4a

Added by Benoit Parmentier about 8 years ago

  • ID 4b605f4ae8bd048332b9c35bc9dfbc15f836981d
  • Parent 84022daa

full testing from shell with removal of tmp files

View differences:

climate/research/oregon/interpolation/global_product_assessment_part0.R
9 9 
#
10 10 
#AUTHOR: Benoit Parmentier
11 11 
#CREATED ON: 10/27/2016
12 
#MODIFIED ON: 11/28/2016
12 
#MODIFIED ON: 12/01/2016
13 13 
#Version: 1
14 14 
#PROJECT: Environmental Layers project
15 15 
#COMMENTS:
......
20 20 
#source /nobackupp6/aguzman4/climateLayers/sharedModules2/etc/environ.sh
21 21 
#
22 22 
#setfacl -Rm u:aguzman4:rwx /nobackupp6/aguzman4/climateLayers/LST_tempSpline/
23 
#COMMIT: making callable from shel for function of number of predictions for day with missing tiles
23 
#COMMIT: testing option to remove tmp files
24 24 

  		




  25
  25
  
    
### Testing several years on the bridge before running jobs on nodes with qsub


  		




  26
  26
  
    
#Use the following command to run as script via the shell on the bridge 


  		




  ......




  93
  93
  
    
source(file.path(script_path,function_assessment_part3)) #source all functions used in this script 


  		




  94
  94
  
    

  		




  95
  95
  
    
#Product assessment


  		




  96
  
  
    
function_product_assessment_part0_functions <- "global_product_assessment_part0_functions_11302016.R"


  		




  
  96
  
    
function_product_assessment_part0_functions <- "global_product_assessment_part0_functions_12012016.R"


  		




  97
  97
  
    
source(file.path(script_path,function_product_assessment_part0_functions)) #source all functions used in this script 


  		




  98
  98
  
    
##Don't load part 1 and part2, mosaic package does not work on NEX


  		




  99
  99
  
    
#function_product_assessment_part1_functions <- "global_product_assessment_part1_functions_09192016b.R"


  		




  ......




  104
  104
  
    
###############################


  		




  105
  105
  
    
####### Parameters, constants and arguments ###


  		




  106
  106
  
    

  		




  107
  
  
    
#Rscript /nobackupp8/bparmen1/env_layers_scripts/global_product_assessment_part0_11272016.R TMAX /nobackupp6/aguzman4/climateLayers/out/reg6/assessment reg6 predictions_assessment_reg6_10302016 /nobackupp8/bparmen1/climateLayers/out/reg6/assessment TRUE 2000 6 1e+07 9 rmse 20000101 20001231 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg6.tif /nobackupp6/aguzman4/climateLayers/out var_pred


  		




  
  107
  
    
#Rscript /nobackupp8/bparmen1/env_layers_scripts/global_product_assessment_part0_12012016.R TMAX /nobackupp6/aguzman4/climateLayers/out/reg6/assessment reg6 predictions_assessment_reg6_10302016 /nobackupp8/bparmen1/climateLayers/out/reg6/assessment TRUE 2000 6 1e+07 9 rmse 20000101 20001231 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg6.tif /nobackupp6/aguzman4/climateLayers/out var_pred FALSE FALSE


  		




  
  108
  
    
#Rscript /nobackupp8/bparmen1/env_layers_scripts/global_product_assessment_part0_12012016.R TMAX /nobackupp6/aguzman4/climateLayers/out/reg6/assessment reg6 predictions_tiles_assessment_reg6_2000 /nobackupp8/bparmen1/climateLayers/out/reg6/assessment TRUE 2000 6 1e+07 9 rmse 20000101 20001231 /nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg6.tif /nobackupp6/aguzman4/climateLayers/out var_pred FALSE FALSE


  		




  108
  109
  
    

  		




  109
  110
  
    
### ARGUMENTS: inputs parameters set from the command line


  		




  110
  111
  
    

  		




  111
  112
  
    
var <- args[1] # variable being interpolated #param 1, arg 1


  		




  112
  
  
    
in_dir <- args[2] #PARAM2,#region_name <- "reg4" #PARAM 3 #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3


  		




  113
  
  
    
region_name <- args[3] #PARAM3


  		




  114
  
  
    
out_suffix <- args[4] #PARAM 4


  		




  115
  
  
    
out_suffix_str <- region_name #PARAM 4, CONST 3


  		




  116
  
  
    
out_dir <- args[5] #PARAM 5


  		




  117
  
  
    
create_out_dir_param <- args[6] #PARAM 6


  		




  118
  
  
    
year_predicted <- args[7] #PARAM 7


  		




  119
  
  
    
num_cores <- args[8] #PARAM 8


  		




  120
  
  
    
max_mem<-args[9] #PARAM 9


  		




  121
  
  
    
##mosaicing_method <- c("unweighted","use_edge_weights") #PARAM10


  		




  122
  
  
    
item_no <- args[10] #PARAM10


  		




  123
  
  
    
metric_name <- args[11]


  		




  124
  
  
    
day_start <- args[12] #PARAM 12


  		




  125
  
  
    
day_end <- args[13] #PARAM 13


  		




  126
  
  
    
infile_mask <- args[14]


  		




  
  113
  
    
in_dir <- args[2] #input dir containing tiles predictions from stage 4 workflow


  		




  
  114
  
    
region_name <- args[3] #PARAM3  #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3


  		




  
  115
  
    
out_suffix <- args[4] #PARAM 4 # output suffix, add region and year of assessment


  		




  
  116
  
    
out_dir <- args[5] #PARAM 5, parent output dir, a new dir is generated using the "output_"+out_suffix 


  		




  
  117
  
    
create_out_dir_param <- args[6] #PARAM 6, if true create out_dir otherwise use given out_dir


  		




  
  118
  
    
year_predicted <- args[7] #PARAM 7, year being assessed


  		




  
  119
  
    
num_cores <- args[8] #PARAM 8, number of cores used in the parraleliation


  		




  
  120
  
    
max_mem<-args[9] #PARAM 9, maximum memory used in raster package


  		




  
  121
  
    
item_no <- args[10] #PARAM10, string position of date in tile tif prediciton, use 9 as default


  		




  
  122
  
    
metric_name <- args[11] #PARAM 11, prediction or accuracy: rmse, mae


  		




  
  123
  
    
day_start <- args[12] #PARAM 12, start of day to process


  		




  
  124
  
    
day_end <- args[13] #PARAM 13, end of day to process


  		




  
  125
  
    
infile_mask <- args[14]#PARAM 14, input mask file for the region


  		




  127
  126
  
    
in_dir1 <- args[15] #PARAM 15, files containing assessment information


  		




  128
  
  
    
layers_option <- args[16] # PARAM 17 options are:


  		




  129
  
  
    
tmp_files <- args[17]


  		




  
  127
  
    
layers_option <- args[16] # PARAM 16 options are: prediction or accuracy


  		




  
  128
  
    
tmp_files <- args[17] # PARAM 17, if FALSE, temporary files are removed


  		




  
  129
  
    
plotting_figures <- args[18]# PARAM 18, if TRUE, png files are produced for missing tiles and day predicted


  		




  130
  130
  
    

  		




  131
  131
  
    
#### values used for testing


  		




  132
  
  
    
var <- "TMAX" # variable being interpolated #PARAM 1, arg 1


  		




  133
  
  
    
in_dir <- "/nobackupp6/aguzman4/climateLayers/out/reg6/assessment" #PARAM2


  		




  134
  
  
    
region_name <- c("reg6") #PARAM 3, arg 3


  		




  135
  
  
    
out_suffix <- "predictions_assessment_reg6_11302016" #PARAM 4


  		




  136
  
  
    
#out_suffix_str <- region_name #PARAM 4, CONST 3


  		




  137
  
  
    
out_dir <- "/nobackupp6/aguzman4/climateLayers/out/reg6/assessment" #PARAM 5


  		




  138
  
  
    
out_dir <- "/nobackupp8/bparmen1/climateLayers/out/reg6/assessment"


  		




  139
  
  
    
create_out_dir_param <- TRUE #PARAM 12, arg 6


  		




  140
  
  
    
year_predicted <- c(2000) #PARAM 7, arg7


  		




  141
  
  
    
num_cores <- 6 #number of cores used # PARAM 8, arg 8


  		




  142
  
  
    
max_mem <- 1e+07 #PARAM 9


  		




  143
  
  
    
#mosaicing_method <- args[10] #PARAM10


  		




  144
  
  
    
item_no <- 9 #PARAM 10, arg 10


  		




  145
  
  
    
metric_name <- "rmse" # "mae", "r" for MAE, R etc.; can also be ns or nv? #PARAM 11, arg 11


  		




  146
  
  
    
day_start <- "20000101" #PARAM 12, arg 12


  		




  147
  
  
    
day_end <- "20001231" #PARAM 13, arg 13


  		




  148
  
  
    
infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg6.tif" #PARAM 14, arg 14


  		




  149
  
  
    
in_dir1 <- "/nobackupp6/aguzman4/climateLayers/out" # PARAM 15 On NEX


  		




  150
  
  
    
layers_option <- c("var_pred") #PARAM 16, arg 16


  		




  151
  
  
    
tmp_files <- FALSE #PARAM 17, arg 17


  		




  
  132
  
    
# var <- "TMAX" # variable being interpolated #PARAM 1, arg 1


  		




  
  133
  
    
# in_dir <- "/nobackupp6/aguzman4/climateLayers/out/reg6/assessment" #PARAM2


  		




  
  134
  
    
# region_name <- c("reg6") #PARAM 3, arg 3


  		




  
  135
  
    
# out_suffix <- "predictions_tiles_assessment_reg6_2000" #PARAM 4


  		




  
  136
  
    
# #out_suffix_str <- region_name #PARAM 4, CONST 3


  		




  
  137
  
    
# #out_dir <- "/nobackupp6/aguzman4/climateLayers/out/reg6/assessment" #PARAM 5


  		




  
  138
  
    
# out_dir <- "/nobackupp8/bparmen1/climateLayers/out/reg6/assessment"


  		




  
  139
  
    
# create_out_dir_param <- TRUE #PARAM 12, arg 6


  		




  
  140
  
    
# year_predicted <- c(2000) #PARAM 7, arg7


  		




  
  141
  
    
# num_cores <- 6 #number of cores used # PARAM 8, arg 8


  		




  
  142
  
    
# max_mem <- 1e+07 #PARAM 9


  		




  
  143
  
    
# #mosaicing_method <- args[10] #PARAM10


  		




  
  144
  
    
# item_no <- 9 #PARAM 10, arg 10


  		




  
  145
  
    
# metric_name <- "rmse" # "mae", "r" for MAE, R etc.; can also be ns or nv? #PARAM 11, arg 11


  		




  
  146
  
    
# day_start <- "20000101" #PARAM 12, arg 12


  		




  
  147
  
    
# day_end <- "20001231" #PARAM 13, arg 13


  		




  
  148
  
    
# infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg6.tif" #PARAM 14, arg 14


  		




  
  149
  
    
# in_dir1 <- "/nobackupp6/aguzman4/climateLayers/out" # PARAM 15 On NEX


  		




  
  150
  
    
# layers_option <- c("var_pred") #PARAM 16, arg 16


  		




  
  151
  
    
# tmp_files <- FALSE #PARAM 17, arg 17


  		




  
  152
  
    
# plotting_figures <- FALSE #PARAm 18, arg 18


  		




  152
  153
  
    

  		




  153
  154
  
    
###################


  		




  154
  155
  
    
### CONSTANT: not set from command line


  		




  ......




  162
  163
  
    
interpolation_method<-c("gam_CAI") #PARAM 2


  		




  163
  164
  
    
day_to_mosaic_range <- NULL #PARAM 7


  		




  164
  165
  
    
file_format <- ".tif" #format for mosaiced files # PARAM 14


  		




  165
  
  
    
plotting_figures <- TRUE #running part2 of assessment to generate figures... # PARAM 13


  		




  166
  166
  
    
#python_bin <- "/usr/bin" #PARAM 15, NCEAS


  		




  167
  167
  
    
python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules2/bin" #PARAM 15"


  		




  168
  168
  
    
in_dir_list_filename <- NULL # PARAM 16, if NULL, use the in_dir directory to search for info


  		




  ......




  171
  171
  
    
lf_raster <- NULL #list of raster to consider #PARAM 18


  		




  172
  172
  
    
scaling <- 1


  		




  173
  173
  
    
data_type <- "Int16"


  		




  174
  
  
    
#CRS_interp <- "+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0" #param 3


  		




  175
  
  
    
#list_models<-c("y_var ~ s(lat,lon,k=5) + s(elev_s,k=3) + s(LST,k=3)") #param 4


  		




  
  174
  
    
out_suffix_str <- region_name #PARAM 4, CONST 3


  		




  176
  175
  
    

  		




  177
  176
  
    
##Add for precip later...


  		




  178
  177
  
    
if (var == "TMAX") {


  		




  ......




  220
  219
  
    
                                             "pred_mod_name","metric_name")


  		




  221
  220
  
    

  		




  222
  221
  
    
#Product assessment


  		




  223
  
  
    
function_product_assessment_part0_functions <- "global_product_assessment_part0_functions_11302016.R"


  		




  224
  
  
    
source(file.path(script_path,function_product_assessment_part0_functions)) #source all functions used in this script 


  		




  
  222
  
    
#function_product_assessment_part0_functions <- "global_product_assessment_part0_functions_12012016.R"


  		




  
  223
  
    
#source(file.path(script_path,function_product_assessment_part0_functions)) #source all functions used in this script 


  		




  225
  224
  
    

  		




  226
  225
  
    
#debug(predictions_tiles_missing_fun)


  		




  227
  
  
    
#Started at 11.06pm


  		




  
  226
  
    
#Started at 9.35am


  		




  228
  227
  
    
obj_predictions_tiles_missing_fun <- predictions_tiles_missing_fun(list_param=list_param_predictions_tiles_missing)


  		




  229
  
  
    

  		




  
  228
  
    
 


  		




  
  229
  
    
###Generate summary from object here to simplify output?


  		




  230
  230
  
    

  		




  231
  231
  
    
############################ END OF SCRIPT ##################################


  		




  232
  232
  
    

  		












Also available in:  Unified diff