Revision c81c68f9
Added by Benoit Parmentier over 8 years ago
climate/research/oregon/interpolation/global_run_scalingup_mosaicing.R | ||
---|---|---|
279 | 279 |
|
280 | 280 |
#There a 28 files for reg4, South America |
281 | 281 |
|
282 |
####################################### |
|
283 |
################### PART I: Accuracy layers by tiles ############# |
|
284 |
###Depending on value of layers_option, create accuracy surfaces based on day testing metric (e.g. rmse)... |
|
285 |
#Generate accuracy layers using tiles definitions and output from the accuracy assessment |
|
286 |
|
|
287 |
|
|
288 |
#################################### |
|
289 |
###Depending on value of layers_option, create accuracy surfaces based on testing residuals... |
|
290 |
#browser() |
|
291 |
if(layers_option=="res_testing"){ |
|
292 |
#This part took 19 minutes and 45 seconds |
|
293 |
|
|
294 |
## Create accuracy surface by kriging |
|
295 |
num_cores_tmp <-num_cores |
|
296 |
lf_day_tiles <- lf_mosaic #list of raster files by dates |
|
297 |
data_df <- data_day_v # data.frame table/spdf containing stations with residuals and variable |
|
298 |
#df_tile_processed #tiles processed during assessment usually by region |
|
299 |
#var_pred #variable being modeled |
|
300 |
#if not list of models is provided generate one |
|
301 |
if(is.null(list_models)){ |
|
302 |
list_models <- paste(var_pred,"~","1",sep=" ") |
|
303 |
} |
|
304 |
|
|
305 |
#use_autokrige #if TRUE use automap/gstat package |
|
306 |
#y_var_name #"dailyTmax" #PARAM2 |
|
307 |
#interpolation_method #c("gam_CAI") #PARAM3, need to select reg!! |
|
308 |
#date_processed #can be a monthly layer |
|
309 |
#num_cores #number of cores used |
|
310 |
#NA_flag_val |
|
311 |
#file_format |
|
312 |
out_dir_str <- out_dir #change to specific separate dir?? |
|
313 |
#out_suffix_str <- out_suffix |
|
314 |
days_to_process <- day_to_mosaic |
|
315 |
out_suffix_str <- paste("data_day_v_",out_suffix,sep="") |
|
316 |
#browser() |
|
317 |
df_tile_processed$path_NEX <- as.character(df_tile_processed$path_NEX) |
|
318 |
df_tile_processed$reg <- basename(dirname(df_tile_processed$path_NEX)) |
|
319 |
|
|
320 |
##By regions, selected earlier |
|
321 |
#for(k in 1:length(region_names)){ |
|
322 |
df_tile_processed_reg <- subset(df_tile_processed,reg==region_selected)#use reg4 |
|
323 |
#i<-1 #loop by days/date to process!! |
|
324 |
#test on the first day |
|
325 |
list_param_create_accuracy_residuals_raster <- list(lf_day_tiles,data_df,df_tile_processed_reg, |
|
326 |
var_pred,list_models,use_autokrige,y_var_name,interpolation_method, |
|
327 |
days_to_process,num_cores_tmp,NA_flag_val,file_format,out_dir_str, |
|
328 |
out_suffix_str) |
|
329 |
names(list_param_create_accuracy_residuals_raster) <- c("lf_day_tiles","data_df","df_tile_processed_reg", |
|
330 |
"var_pred","list_models","use_autokrige","y_var_name","interpolation_method", |
|
331 |
"days_to_process","num_cores_tmp","NA_flag_val","file_format","out_dir_str", |
|
332 |
"out_suffix_str") |
|
333 |
#browser() |
|
334 |
list_create_accuracy_residuals_raster_obj <- lapply(1:length(day_to_mosaic),FUN=create_accuracy_residuals_raster, |
|
335 |
list_param=list_param_create_accuracy_residuals_raster) |
|
336 |
|
|
337 |
#undebug(create_accuracy_residuals_raster) |
|
338 |
#list_create_accuracy_residuals_raster_obj <- lapply(1:1,FUN=create_accuracy_residuals_raster, |
|
339 |
# list_param=list_param_create_accuracy_residuals_raster) |
|
340 |
|
|
341 |
#create_accuracy_residuals_raster_obj <- create_accuracy_residuals_raster(1, list_param_create_accuracy_residuals_raster_obj) |
|
342 |
|
|
343 |
#note that three tiles did not produce a residuals surface!!! find out more later, join the output |
|
344 |
#to df_raste_tile to keep track of which one did not work... |
|
345 |
#lf_accuracy_residuals_raster <- as.character(unlist(lapply(1:length(list_create_accuracy_residuals_raster_obj),FUN=function(i,x){unlist(extract_from_list_obj(x[[i]]$list_pred_res_obj,"raster_name"))},x=list_create_accuracy_residuals_raster_obj))) |
|
346 |
lf_accuracy_residuals_testing_raster <- lapply(1:length(list_create_accuracy_residuals_raster_obj),FUN=function(i,x){as.character(unlist(extract_from_list_obj(x[[i]]$list_pred_res_obj,"raster_name")))},x=list_create_accuracy_residuals_raster_obj) |
|
347 |
|
|
348 |
#Plot as quick check |
|
349 |
#r1 <- raster(lf_mosaic[[1]][1]) |
|
350 |
#list_create_accuracy_residuals_raster_obj |
|
351 |
#browser() |
|
352 |
} |
|
353 |
|
|
354 |
######################################### |
|
355 |
###Depending on value of layers_option, create accuracy surfaces based on training residuals... |
|
356 |
## |
|
357 |
|
|
358 |
if(layers_option=="res_training"){ |
|
359 |
#This part took 19 minutes and 40 seconds |
|
360 |
|
|
361 |
data_df <- data_day_s # data.frame table/spdf containing stations with residuals and variable |
|
362 |
|
|
363 |
num_cores_tmp <-num_cores |
|
364 |
lf_day_tiles <- lf_mosaic #list of raster files by dates |
|
365 |
#data_df <- data_day_v # data.frame table/spdf containing stations with residuals and variable |
|
366 |
#df_tile_processed #tiles processed during assessment usually by region |
|
367 |
#var_pred #variable being modeled |
|
368 |
#if not list of models is provided generate one |
|
369 |
if(is.null(list_models)){ |
|
370 |
list_models <- paste(var_pred,"~","1",sep=" ") |
|
371 |
} |
|
372 |
|
|
373 |
#use_autokrige #if TRUE use automap/gstat package |
|
374 |
#y_var_name #"dailyTmax" #PARAM2 |
|
375 |
#interpolation_method #c("gam_CAI") #PARAM3, need to select reg!! |
|
376 |
#date_processed #can be a monthly layer |
|
377 |
#num_cores #number of cores used |
|
378 |
#NA_flag_val |
|
379 |
#file_format |
|
380 |
out_dir_str <- out_dir |
|
381 |
out_suffix_str <- paste("data_day_s_",out_suffix,sep="") |
|
382 |
days_to_process <- day_to_mosaic |
|
383 |
df_tile_processed$path_NEX <- as.character(df_tile_processed$path_NEX) |
|
384 |
df_tile_processed$reg <- basename(dirname(df_tile_processed$path_NEX)) |
|
385 |
|
|
386 |
##By regions, selected earlier |
|
387 |
#for(k in 1:length(region_names)){ |
|
388 |
df_tile_processed_reg <- subset(df_tile_processed,reg==region_selected)#use reg4 |
|
389 |
#i<-1 #loop by days/date to process!! |
|
390 |
#test on the first day |
|
391 |
list_param_create_accuracy_residuals_raster <- list(lf_day_tiles,data_df,df_tile_processed_reg, |
|
392 |
var_pred,list_models,use_autokrige,y_var_name,interpolation_method, |
|
393 |
days_to_process,num_cores_tmp,NA_flag_val,file_format,out_dir_str, |
|
394 |
out_suffix_str) |
|
395 |
names(list_param_create_accuracy_residuals_raster) <- c("lf_day_tiles","data_df","df_tile_processed_reg", |
|
396 |
"var_pred","list_models","use_autokrige","y_var_name","interpolation_method", |
|
397 |
"days_to_process","num_cores_tmp","NA_flag_val","file_format","out_dir_str", |
|
398 |
"out_suffix_str") |
|
399 |
#browser() #21 minutes and 40 second to get here |
|
400 |
list_create_accuracy_residuals_raster_obj <- lapply(1:length(day_to_mosaic),FUN=create_accuracy_residuals_raster, |
|
401 |
list_param=list_param_create_accuracy_residuals_raster) |
|
402 |
|
|
403 |
#undebug(create_accuracy_residuals_raster) |
|
404 |
#list_create_accuracy_residuals_raster_obj <- lapply(1:1,FUN=create_accuracy_residuals_raster, |
|
405 |
# list_param=list_param_create_accuracy_residuals_raster) |
|
406 |
|
|
407 |
#create_accuracy_residuals_raster_obj <- create_accuracy_metric_raster(1, list_param_create_accuracy_residuals_raster_obj) |
|
408 |
|
|
409 |
#note that three tiles did not produce a residuals surface!!! find out more later, join the output |
|
410 |
#to df_raste_tile to keep track of which one did not work... |
|
411 |
#lf_accuracy_residuals_raster <- as.character(unlist(lapply(1:length(list_create_accuracy_residuals_raster_obj),FUN=function(i,x){unlist(extract_from_list_obj(x[[i]]$list_pred_res_obj,"raster_name"))},x=list_create_accuracy_residuals_raster_obj))) |
|
412 |
lf_accuracy_residuals_training_raster <- lapply(1:length(list_create_accuracy_residuals_raster_obj),FUN=function(i,x){as.character(unlist(extract_from_list_obj(x[[i]]$list_pred_res_obj,"raster_name")))},x=list_create_accuracy_residuals_raster_obj) |
|
413 |
|
|
414 |
} |
|
415 |
|
|
416 |
#browser() |
|
417 |
|
|
418 | 282 |
###################################################### |
419 | 283 |
#### PART 3: GENERATE MOSAIC FOR LIST OF FILES ##### |
420 | 284 |
################################# |
... | ... | |
470 | 334 |
df <- tb #for ac_testing |
471 | 335 |
days_to_process <- day_to_mosaic[i] |
472 | 336 |
|
473 |
browser() |
|
337 |
#browser() |
|
338 |
|
|
474 | 339 |
debug(generate_ac_assessment_layers_by_tile) |
475 |
lf_accuracy_testing_raster<- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name, |
|
340 |
lf_accuracy_testing_raster <- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name,
|
|
476 | 341 |
var_pred,list_models,use_autokrige,pred_mod_name, |
477 | 342 |
y_var_name,interpolation_method, |
478 | 343 |
days_to_process,num_cores,NA_flag_val,file_format, |
... | ... | |
509 | 374 |
scaling=scaling, |
510 | 375 |
values_range=values_range) |
511 | 376 |
##Took 12-13 minutes for 28 tiles and one date...!!! |
377 |
lf_accuracy_testing_raster |
|
378 |
|
|
379 |
if(tmp_files==F){ #if false...delete all files with "_tmp" |
|
380 |
lf_tmp <- lf_accuracy_testing_raster |
|
381 |
##now delete temporary files... |
|
382 |
file.remove(lf_accuracy_testing_raster) |
|
383 |
} |
|
512 | 384 |
list_mosaic_obj[[i]] <- mosaic_obj |
513 | 385 |
} |
514 | 386 |
|
... | ... | |
520 | 392 |
df <- tb_s #for ac_testing |
521 | 393 |
days_to_process <- day_to_mosaic[i] |
522 | 394 |
|
523 |
browser() |
|
395 |
#browser()
|
|
524 | 396 |
debug(generate_ac_assessment_layers_by_tile) |
525 | 397 |
lf_accuracy_training_raster<- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name, |
526 | 398 |
var_pred,list_models,use_autokrige,pred_mod_name, |
... | ... | |
567 | 439 |
if(layers_option=="res_testing"){ |
568 | 440 |
|
569 | 441 |
lf <- lf_mosaic[i] |
570 |
df <- tb_s #for ac_testing
|
|
442 |
df <- data_day_v #for ac_testing
|
|
571 | 443 |
days_to_process <- day_to_mosaic[i] |
572 | 444 |
|
573 |
browser() |
|
445 |
#browser()
|
|
574 | 446 |
debug(generate_ac_assessment_layers_by_tile) |
575 |
lf_accuracy_training_raster<- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name,
|
|
447 |
lf_accuracy_residuals_testing_raster <- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name,
|
|
576 | 448 |
var_pred,list_models,use_autokrige,pred_mod_name, |
577 | 449 |
y_var_name,interpolation_method, |
578 | 450 |
days_to_process,num_cores,NA_flag_val,file_format, |
... | ... | |
608 | 480 |
|
609 | 481 |
### produce residuals mosaics |
610 | 482 |
if(layers_option=="res_training"){ |
483 |
|
|
484 |
lf <- lf_mosaic[i] |
|
485 |
df <- data_day_s #for ac_training |
|
486 |
days_to_process <- day_to_mosaic[i] |
|
487 |
|
|
488 |
#browser() |
|
489 |
debug(generate_ac_assessment_layers_by_tile) |
|
490 |
lf_accuracy_residuals_training_raster <- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name, |
|
491 |
var_pred,list_models,use_autokrige,pred_mod_name, |
|
492 |
y_var_name,interpolation_method, |
|
493 |
days_to_process,num_cores,NA_flag_val,file_format, |
|
494 |
out_dir,out_suffix) #### create a function to generate accuracy layers by tiles |
|
495 |
|
|
611 | 496 |
#for now add data_day_s in the name!! |
612 | 497 |
mosaic_method <- "use_edge_weights" #this is distance from edge |
613 | 498 |
out_suffix_tmp <- paste(interpolation_method,"kriged_residuals","data_day_s",day_to_mosaic[i],out_suffix,sep="_") |
Also available in: Unified diff
cleaning up code and removing options to mosaicing part