Revision 526d623d
Added by Benoit Parmentier almost 9 years ago
| climate/research/oregon/interpolation/global_run_scalingup_mosaicing.R | ||
|---|---|---|
| 5 | 5 |
#Analyses, figures, tables and data are also produced in the script. |
| 6 | 6 |
#AUTHOR: Benoit Parmentier |
| 7 | 7 |
#CREATED ON: 04/14/2015 |
| 8 |
#MODIFIED ON: 12/30/2015
|
|
| 8 |
#MODIFIED ON: 01/01/2016
|
|
| 9 | 9 |
#Version: 5 |
| 10 | 10 |
#PROJECT: Environmental Layers project |
| 11 | 11 |
#COMMENTS: analyses run for reg4 1992 for test of mosaicing using 1500x4500km and other tiles |
| 12 | 12 |
#TODO: |
| 13 |
#1) Make this is a script/function callable from the shell/bas |
|
| 13 |
#1) Make this is a script/function callable from the shell/bash
|
|
| 14 | 14 |
#2) clean up temporary files, it builds currently on the disk |
| 15 | 15 |
#3) fix output folder for some of output files: create a mosaic output folder if doesn't exist? |
| 16 |
#4) create a helper function for inputs/arguments to automate...?? Could also be in the assessment stage |
|
| 16 |
#4) create a helper function for inputs/arguments to automate (optparse pacakge)...?? |
|
| 17 |
#Could also be in the assessment stage |
|
| 17 | 18 |
|
| 18 | 19 |
### Before running, the gdal modules and other environment parameters need to be set if on NEX-NASA. |
| 19 | 20 |
### This can be done by running the following commands: |
| ... | ... | |
| 70 | 71 |
#in_dir <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_12072015" #NEX |
| 71 | 72 |
|
| 72 | 73 |
in_dir_tiles <- file.path(in_dir,"tiles") #this is valid both for Atlas and NEX |
| 73 |
|
|
| 74 | 74 |
y_var_name <- "dailyTmax" #PARAM2 |
| 75 | 75 |
interpolation_method <- c("gam_CAI") #PARAM3
|
| 76 | 76 |
region_name <- "reg4" #PARAM 4 #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3 |
| ... | ... | |
| 100 | 100 |
use_autokrige <- F #PARAM 19 |
| 101 | 101 |
|
| 102 | 102 |
###Separate folder for masks by regions, should be listed as just the dir!!... #PARAM 20 |
| 103 |
#infile_mask <- "/nobackupp8/bparmen1/regions_input_files/r_mask_reg4.tif" |
|
| 103 |
#infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_reg4.tif"
|
|
| 104 | 104 |
infile_mask <- "/data/project/layers/commons/NEX_data/regions_input_files/r_mask_reg4.tif" |
| 105 |
|
|
| 106 |
#tb_accuracy_name <- file.path(in_dir,paste("tb_diagnostic_v_NA","_",out_suffix_str,".txt",sep=""))
|
|
| 107 |
#tb_accuracy_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_12072015.txt" #PARAM 21 |
|
| 108 |
#data_month_s_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/data_month_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt" #PARAM 22 |
|
| 109 |
#data_day_v_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/data_day_v_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt" #PARAM 23 |
|
| 110 |
#data_day_s_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/data_day_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt" ##PARAM 24 |
|
| 111 |
#df_tile_processed_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/df_tile_processed_run10_1500x4500_global_analyses_pred_1992_12072015.txt" ##PARAM 25 |
|
| 105 |
## All of this is interesting so use df_assessment!! |
|
| 106 |
df_assessment_files_name <- "df_assessment_files_reg4_1984_run_global_analyses_pred_12282015.txt" # data.frame with all files used in assessmnet, PARAM 21 |
|
| 112 | 107 |
|
| 113 | 108 |
#in_dir can be on NEX or Atlas |
| 114 |
## All of this is interesting so use df_assessment!! |
|
| 115 |
tb_v_accuracy_name <- file.path(in_dir, basename(df_assessment_files$files[2])) #PARAM 21 |
|
| 116 |
tb_s_accuracy_name <- file.path(in_dir, basename(df_assessment_files$files[4])) #PARAM 21 |
|
| 117 |
data_month_s_name <- file.path(in_dir,basename(df_assessment_files$files[5])) #PARAM 22 |
|
| 118 |
data_day_v_name <- file.path(in_dir, basename(df_assessment_files$files[6])) #PARAM 23 |
|
| 119 |
data_day_s_name <- file.path(in_dir, basename(df_assessment_files$files[7])) ##PARAM 24 |
|
| 120 |
df_tile_processed_name <- file.path(in_dir, basename(df_assessment_files$files[11])) |
|
| 121 |
pred_data_month_info <- file.path(in_dir, basename(df_assessment_files$files[9])) |
|
| 122 |
pred_data_day_info <- file.path(in_dir, basename(df_assessment_files$files[10])) |
|
| 123 | 109 |
|
| 124 | 110 |
#python script and gdal on NEX NASA: |
| 125 | 111 |
#mosaic_python <- "/nobackupp6/aguzman4/climateLayers/sharedCode/" |
| ... | ... | |
| 130 | 116 |
|
| 131 | 117 |
algorithm <- "python" #PARAM 28 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann |
| 132 | 118 |
#algorithm <- "R" #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann |
| 133 |
|
|
| 134 | 119 |
match_extent <- "FALSE" #PARAM 29 #try without matching!!! |
| 135 | 120 |
|
| 136 | 121 |
#for residuals... |
| ... | ... | |
| 234 | 219 |
|
| 235 | 220 |
###Separate folder for masks by regions, should be listed as just the dir!!... #PARAM 20 |
| 236 | 221 |
#infile_mask <- "/nobackupp8/bparmen1/regions_input_files/r_mask_reg4.tif" |
| 237 |
infile_mask <- list_param_run_mosaicing_prediction$infile_mask #"/data/project/layers/commons/NEX_data/regions_input_files/r_mask_reg4.tif"
|
|
| 222 |
infile_mask <- list_param_run_mosaicing_prediction$infile_mask # input mask used in defining the region
|
|
| 238 | 223 |
|
| 239 | 224 |
#in_dir can be on NEX or Atlas |
| 240 |
tb_v_accuracy_name <- list_param_run_mosaicing_prediction$tb_accuracy_name #<- file.path(in_dir,"tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_12072015.txt") #PARAM 21 |
|
| 241 |
tb_s_accuracy_name <- list_param_run_mosaicing_prediction$tb_accuracy_name #<- file.path(in_dir,"tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_12072015.txt") #PARAM 21 |
|
| 242 |
data_month_s_name <- list_param_run_mosaicing_prediction$data_month_s_name # file.path(in_dir,"data_month_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt") #PARAM 22 |
|
| 243 |
data_day_v_name <- list_param_run_mosaicing_prediction$data_day_v_name # file.path(in_dir,"data_day_v_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt") #PARAM 23 |
|
| 244 |
data_day_s_name <- list_param_run_mosaicing_prediction$data_day_s_name # file.path(in_dir,"data_day_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt") ##PARAM 24 |
|
| 245 |
df_tile_processed_name <- list_param_run_mosaicing_prediction$df_tile_processed_name # file.path(in_dir,"df_tile_processed_run10_1500x4500_global_analyses_pred_1992_12072015.txt") ##PARAM 25 |
|
| 246 |
|
|
| 225 |
df_assessment_files_name <- list_param_run_mosaicing_prediction$df_assessment_files_name # data.frame with all files used in assessmnet, PARAM 21 |
|
| 226 |
|
|
| 247 | 227 |
#python script and gdal on NEX NASA: |
| 248 | 228 |
#mosaic_python <- "/nobackupp6/aguzman4/climateLayers/sharedCode/" |
| 249 | 229 |
#python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules2/bin" |
| 250 | 230 |
#python script and gdal on Atlas NCEAS |
| 251 |
mosaic_python <- list_param_run_mosaicing_prediction$mosaic_python # "/data/project/layers/commons/NEX_data/sharedCode" #PARAM 26
|
|
| 252 |
python_bin <- list_param_run_mosaicing_prediction$python_bin # "/usr/bin" #PARAM 27
|
|
| 231 |
mosaic_python <- list_param_run_mosaicing_prediction$mosaic_python # "/data/project/layers/commons/NEX_data/sharedCode" #PARAM 22
|
|
| 232 |
python_bin <- list_param_run_mosaicing_prediction$python_bin # "/usr/bin" #PARAM 23
|
|
| 253 | 233 |
|
| 254 |
algorithm <- list_param_run_mosaicing_prediction$algorithm #"python" #PARAM 28 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
|
|
| 234 |
algorithm <- list_param_run_mosaicing_prediction$algorithm #"python" #PARAM 24 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
|
|
| 255 | 235 |
#algorithm <- "R" #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann |
| 256 | 236 |
|
| 257 |
match_extent <- list_param_run_mosaicing_prediction$match_extent #"FALSE" #PARAM 29 #try without matching!!!
|
|
| 237 |
match_extent <- list_param_run_mosaicing_prediction$match_extent #"FALSE" #PARAM 25 #try without matching!!!
|
|
| 258 | 238 |
|
| 259 | 239 |
#for residuals... |
| 260 |
list_models <- list_param_run_mosaicing_prediction$list_models # NULL #PARAM 30
|
|
| 240 |
list_models <- list_param_run_mosaicing_prediction$list_models # NULL #PARAM 26
|
|
| 261 | 241 |
#list_models <- paste(var_pred,"~","1",sep=" ") #if null then this is the default... |
| 262 | 242 |
|
| 263 | 243 |
####### PART 1: Read in data and process data ######## |
| ... | ... | |
| 278 | 258 |
|
| 279 | 259 |
setwd(out_dir) |
| 280 | 260 |
|
| 261 |
### Read in assessment and accuracy files |
|
| 262 |
df_assessment_files <- read.table(df_assessment_files_name,stringsAsFactors=F,sep=",") |
|
| 263 |
|
|
| 264 |
tb_v_accuracy_name <- file.path(in_dir, basename(df_assessment_files$files[2])) |
|
| 265 |
tb_s_accuracy_name <- file.path(in_dir, basename(df_assessment_files$files[4])) |
|
| 266 |
tb_s_month_accuracy_name <- file.path(in_dir, basename(df_assessment_files$files[3])) |
|
| 267 |
data_month_s_name <- file.path(in_dir,basename(df_assessment_files$files[5])) |
|
| 268 |
data_day_v_name <- file.path(in_dir, basename(df_assessment_files$files[6])) |
|
| 269 |
data_day_s_name <- file.path(in_dir, basename(df_assessment_files$files[7])) |
|
| 270 |
#data_month_v_name <- file.path(in_dir,basename(df_assessment_files$files[8])) |
|
| 271 |
pred_data_month_info_name <- file.path(in_dir, basename(df_assessment_files$files[9])) |
|
| 272 |
pred_data_day_info_name <- file.path(in_dir, basename(df_assessment_files$files[10])) |
|
| 273 |
df_tile_processed_name <- file.path(in_dir, basename(df_assessment_files$files[11])) |
|
| 274 |
|
|
| 281 | 275 |
# accuracy table by tiles |
| 282 |
tb <- read.table(tb_accuracy_name,sep=",") |
|
| 283 |
# textfiles of stations by month |
|
| 284 |
data_month_s <- read.table(file.path(data_month_s_name),sep=",") |
|
| 285 |
data_day_s <- read.table(file.path(data_day_s_name),sep=",") #daily testing/validation stations by dates and tiles |
|
| 286 |
data_day_v <- read.table(file.path(data_day_v_name),sep=",") #daily training stations by dates and tiles |
|
| 287 |
|
|
| 288 |
df_tile_processed <- read.table( df_tile_processed_name,sep=",") |
|
| 276 |
tb <- read.table(tb_v_accuracy_name,sep=",") |
|
| 277 |
tb_s <- read.table(tb_s_accuracy_name,sep=",") |
|
| 278 |
data_month_s <- read.table(data_month_s_name,sep=",") # textfiles of stations by month |
|
| 279 |
data_day_s <- read.table(data_day_s_name,sep=",") #daily testing/validation stations by dates and tiles |
|
| 280 |
data_day_v <- read.table(data_day_v_name,sep=",") #daily training stations by dates and tiles |
|
| 281 |
df_tile_processed <- read.table(df_tile_processed_name,sep=",") |
|
| 289 | 282 |
|
| 290 | 283 |
#list all files to mosaic for a list of day |
| 291 | 284 |
#Take into account multiple region in some cases!!! |
Also available in: Unified diff
modifying main mosaicing script for call from shell and stage 7 on NEX