Project

General

Profile

« Previous | Next » 

Revision 1b4f99e6

Added by Benoit Parmentier almost 9 years ago

adding option to create residuals surface from stations using kriging

View differences:

climate/research/oregon/interpolation/global_run_scalingup_mosaicing.R
5 5
#Analyses, figures, tables and data are also produced in the script.
6 6
#AUTHOR: Benoit Parmentier 
7 7
#CREATED ON: 04/14/2015  
8
#MODIFIED ON: 12/04/2015            
8
#MODIFIED ON: 12/12/2015            
9 9
#Version: 5
10 10
#PROJECT: Environmental Layers project     
11 11
#COMMENTS: analyses run for reg4 1992 for test of mosaicing using 1500x4500km and other tiles
......
58 58

  
59 59
#### FUNCTION USED IN SCRIPT
60 60

  
61
function_mosaicing <-"global_run_scalingup_mosaicing_function_12042015.R"
61
function_mosaicing <-"global_run_scalingup_mosaicing_function_12122015.R"
62 62

  
63 63
#in_dir_script <-"/home/parmentier/Data/IPLANT_project/env_layers_scripts" #NCEAS UCSB
64 64
in_dir_script <- "/nobackupp8/bparmen1/env_layers_scripts" #NASA NEX
......
71 71

  
72 72
#in_dir <- "/data/project/layers/commons/NEX_data/mosaicing_data_test" #PARAM1
73 73
#in_dir <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_10052015" #PARAM4
74
in_dir <- "/nobackupp8/bparmen1/test_output_run10_1500x4500_global_analyses_pred_1992_10052015" #NEX
74
in_dir <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015" #NEX
75
#in_dir <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_12072015" #NEX
75 76

  
76 77
in_dir_tiles <- file.path(in_dir,"tiles")
77 78
#in_dir_tiles <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015/tiles" #North America
......
83 84
interpolation_method <- c("gam_CAI") #PARAM3
84 85
region_name <- "reg4" #PARAM 4 #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3
85 86
mosaicing_method <- c("unweighted","use_edge_weights") #PARAM5
86
out_suffix <- paste(region_name,"_","run10_1500x4500_global_analyses_pred_1992_10052015",sep="") #PARAM 6
87
out_suffix_str <- "run10_1500x4500_global_analyses_pred_1992_10052015"
87
out_suffix <- paste(region_name,"_","run10_1500x4500_global_analyses_pred_1992_12072015",sep="") #PARAM 6
88
out_suffix_str <- "run10_1500x4500_global_analyses_pred_1992_12072015"
88 89
metric_name <- "rmse" #RMSE, MAE etc.
89 90
pred_mod_name <- "mod1"
90 91

  
......
104 105
NA_flag_val <- NA_value
105 106
     
106 107
num_cores <- 6 #PARAM 12                  
108
region_names <- c("reg23","reg4") #selected region names, #PARAM2 
107 109

  
110
###Make a separate folder for masks by regions...
108 111
#infile_mask <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_10052015/r_mask_reg4.tif"
109 112
infile_mask <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015/r_mask_reg4.tif"
110 113

  
111 114
#tb_accuracy_name <- file.path(in_dir,paste("tb_diagnostic_v_NA","_",out_suffix_str,".txt",sep=""))
112 115
#tb_accuracy_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_10052015/tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_10052015.txt"
113
tb_accuracy_name <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015/tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_10052015.txt"
114

  
116
#tb_accuracy_name <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015/tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_10052015.txt"
117
tb_accuracy_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015"
118
data_month_s_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/data_month_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt"
119
df_tile_processed_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/df_tile_processed_run10_1500x4500_global_analyses_pred_1992_12072015.txt"
115 120
#python script and gdal on NEX NASA:
116
mosaic_python <- "/nobackupp6/aguzman4/climateLayers/sharedCode/"
117
python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules2/bin"
121
#mosaic_python <- "/nobackupp6/aguzman4/climateLayers/sharedCode/"
122
#python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules2/bin"
118 123
#python script and gdal on Atlas NCEAS
119
#mosaic_python <- "/data/project/layers/commons/NEX_data/sharedCode"
120
#python_bin <- "/usr/bin"
124
mosaic_python <- "/data/project/layers/commons/NEX_data/sharedCode"
125
python_bin <- "/usr/bin"
121 126

  
122 127
algorithm <- "python" #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
123 128
#algorithm <- "R" #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann
......
140 145

  
141 146
setwd(out_dir)
142 147

  
143

  
148
# accuracy table by tiles
144 149
tb <- read.table(file=tb_accuracy_name,sep=",")
150
# textfiles of stations by month
151
data_month_s <- read.table(file.path(data_month_s_name),sep=",")
152
df_tile_processed <- read.table( df_tile_processed_name,sep=",")
145 153

  
146 154
#list all files to mosaic for a list of day
155
#Take into account multiple region in some cases!!!      
147 156
lf_mosaic <- lapply(1:length(day_to_mosaic),FUN=function(i){list.files(path=file.path(in_dir_tiles),    
148
           pattern=paste(".*.",day_to_mosaic[i],".*.tif$",sep=""),full.names=T)})
149
       
157
           pattern=paste(".*.",day_to_mosaic[i],".*.tif$",sep=""),full.names=T,recursive=T)})
158

  
150 159
##################### PART 2: produce the mosaic ##################
151 160

  
152 161
##########################
......
174 183
                       "days_to_process","num_cores","NA_flag_val","file_format","out_dir_str","out_suffix_str") 
175 184
list_raster_created_obj <- lapply(1:length(day_to_mosaic),FUN=create_accuracy_metric_raster,
176 185
                                  list_param=list_param_accuracy_metric_raster)
186

  
187
#interpolation_method_day_function_multisampling.R
188
## Create accuracy surface by kriging
189
data_tmp <- subset(data_month_s,month==1)
190
data_tmp$tile_id <- as.character(data_tmp$tile_id)
191
var_pred <- "res_mod1"
192
list_models<- c("res_mod1 ~ 1")
193
df_tile_processed
194
#list_models<-c("y_var ~ lat*lon + elev_s + N_w*E_w",
195
#               "y_var ~ lat*lon + elev_s + DISTOC",
196
#               "y_var ~ lat*lon + elev_s + LST",
197
#               "y_var ~ lat*lon + elev_s + LST + I(LST*LC1)")
198

  
199
##By regions
200
create_accuracy_metric_raster_kriging <- function(data_df,df_tile_processed,var_pred,lf_ref_rast,out_dir,out_suffix){
201
  #This function generates kriged values for a giving set of formula and data input.
202
  #Inputs:
203
  #lf: list of raster files
204
  #tb: data.frame table #fitting or validation table with all days
205
  #metric_name: accuracy metric selected to be mapped, RMSE, MAE etc.
206
  #pred_mod_name: model selected, such as mod1, mod_kr etc.
207
  #y_var_name: variable being modeled e.g."dailyTmax", dailyTmin, precip  
208
  #interpolation_method: names of the interpolation/modeling method
209
  #date_processed: day being processed , e.g. 19920101
210
  #num_cores : number of cores used in the parallelization
211
  #NA_flag_val: value used as flag in the raster 
212
  #file_format: e.g. tif., .rst
213
  #out_dir_str: output directory
214
  #out_suffix_str: output suffix
215
  #Outputs:
216
  #raster list of weights and product of wegihts and inuts
217
  #TODO: 
218

  
219
  # - improve efficiency
220
  #
221
  #promote to spdf
222
  list_formulas <- lapply(list_models,as.formula,env=.GlobalEnv) #mulitple arguments passed to lapply!!
223

  
224
  #data_training <- data_df 
225
  coordinates(data_df)<-cbind(data_df$x,data_df$y)
226
  #proj4string(data_s)<-proj_str
227
  #coordinates(data_v)<-cbind(data_v$x,data_v$y)
228
  #proj4string(data_v)<-proj_str
229
  lf_day_tiles <- lf_mosaic[[i]] #i index for time...
230
  
231
  #Now match the correct tiles with data used in kriging...
232
  #match the correct tile!!! df_tile_processed
233
  #pattern_str <- as.character(unique(df_tile_processed$tile_coord))
234
  list_tile_coord <- as.character(df_tile_processed$tile_coord)
235
  pattern_str <- glob2rx(paste("*",list_tile_coord,"*","*.tif",sep=""))
236
  keywords_str <- pattern_str
237
  tmp_str2 <-unlist(lapply(keywords_str,grep,lf_day_tiles,value=TRUE))
238
  df_raster_pred_tiles_tmp <- data.frame(files =tmp_str2, tile_coord=list_tile_coord)
239
  df_raster_pred_tiles <- merge(df_raster_pred_tiles_tmp,df_tile_processed,by="tile_coord")
240
  df_raster_pred_tiles$path_NEX <- as.character(df_raster_pred_tiles$path_NEX)
241
  df_raster_pred_tiles$reg <- basename(dirname(df_raster_pred_tiles$path_NEX))
242
  df_raster_pred_tiles$files <- as.character(df_raster_pred_tiles$files)
243
  
244
  j <- 1 #loops across tiles from set of files
245
  
246
  r_stack <- df_raster_pred_tiles$files[j]
247
  tile_selected <- as.character(df_raster_pred_tiles$tile_id[j])
248
  data_training <- subset(data_df,tile_id==tile_selected) #df_raster_pred_tiles$files
249
  #r_stack <- stack(lf_day[j])
250
  out_filename <- "test.tif"
251
  #debug(predict_auto_krige_raster_model)
252
  test <- predict_auto_krige_raster_model(list_formulas,r_stack,data_training,out_filename)
253
  #identify residuals
254
  
255
  #call kriging function
256
  
257
  #write output
258
}
259

  
177 260
#debug(create_accuracy_metric_raster)
178 261
#list_raster_created_obj <- lapply(1:1,FUN=create_accuracy_metric_raster,
179 262
#                                  list_param=list_param_accuracy_metric_raster)

Also available in: Unified diff