Project

General

Profile

« Previous | Next » 

Revision c81c68f9

Added by Benoit Parmentier over 8 years ago

cleaning up code and removing options to mosaicing part

View differences:

climate/research/oregon/interpolation/global_run_scalingup_mosaicing.R
279 279

  
280 280
  #There a 28 files for reg4, South America
281 281
    
282
  #######################################
283
  ################### PART I: Accuracy layers by tiles #############
284
  ###Depending on value of layers_option, create accuracy surfaces based on day testing metric (e.g. rmse)...
285
  #Generate accuracy layers using tiles definitions and output from the accuracy assessment
286
  
287

  
288
  ####################################
289
  ###Depending on value of layers_option, create accuracy surfaces based on testing residuals...
290
  #browser()
291
  if(layers_option=="res_testing"){
292
    #This part took 19 minutes and 45 seconds
293
    
294
    ## Create accuracy surface by kriging
295
    num_cores_tmp <-num_cores
296
    lf_day_tiles  <- lf_mosaic #list of raster files by dates
297
    data_df <- data_day_v # data.frame table/spdf containing stations with residuals and variable
298
    #df_tile_processed  #tiles processed during assessment usually by region
299
    #var_pred  #variable being modeled
300
    #if not list of models is provided generate one
301
    if(is.null(list_models)){
302
      list_models <- paste(var_pred,"~","1",sep=" ")
303
    }
304
    
305
    #use_autokrige #if TRUE use automap/gstat package
306
    #y_var_name  #"dailyTmax" #PARAM2
307
    #interpolation_method #c("gam_CAI") #PARAM3, need to select reg!!
308
    #date_processed #can be a monthly layer
309
    #num_cores #number of cores used
310
    #NA_flag_val 
311
    #file_format 
312
    out_dir_str <- out_dir #change to specific separate dir??
313
    #out_suffix_str <- out_suffix 
314
    days_to_process <- day_to_mosaic
315
    out_suffix_str <- paste("data_day_v_",out_suffix,sep="") 
316
    #browser()
317
    df_tile_processed$path_NEX <- as.character(df_tile_processed$path_NEX) 
318
    df_tile_processed$reg <- basename(dirname(df_tile_processed$path_NEX))
319
    
320
    ##By regions, selected earlier
321
    #for(k in 1:length(region_names)){
322
    df_tile_processed_reg <- subset(df_tile_processed,reg==region_selected)#use reg4
323
    #i<-1 #loop by days/date to process!!
324
    #test on the first day 
325
    list_param_create_accuracy_residuals_raster <- list(lf_day_tiles,data_df,df_tile_processed_reg,
326
                                                        var_pred,list_models,use_autokrige,y_var_name,interpolation_method,
327
                                                        days_to_process,num_cores_tmp,NA_flag_val,file_format,out_dir_str,
328
                                                        out_suffix_str) 
329
    names(list_param_create_accuracy_residuals_raster) <- c("lf_day_tiles","data_df","df_tile_processed_reg",
330
                                                            "var_pred","list_models","use_autokrige","y_var_name","interpolation_method",
331
                                                            "days_to_process","num_cores_tmp","NA_flag_val","file_format","out_dir_str",
332
                                                            "out_suffix_str") 
333
    #browser()  
334
    list_create_accuracy_residuals_raster_obj <- lapply(1:length(day_to_mosaic),FUN=create_accuracy_residuals_raster,
335
                                                        list_param=list_param_create_accuracy_residuals_raster)
336
    
337
    #undebug(create_accuracy_residuals_raster)
338
    #list_create_accuracy_residuals_raster_obj <- lapply(1:1,FUN=create_accuracy_residuals_raster,
339
    #                                list_param=list_param_create_accuracy_residuals_raster)
340
    
341
    #create_accuracy_residuals_raster_obj <- create_accuracy_residuals_raster(1, list_param_create_accuracy_residuals_raster_obj)
342
    
343
    #note that three tiles did not produce a residuals surface!!! find out more later, join the output
344
    #to df_raste_tile to keep track of which one did not work...
345
    #lf_accuracy_residuals_raster <- as.character(unlist(lapply(1:length(list_create_accuracy_residuals_raster_obj),FUN=function(i,x){unlist(extract_from_list_obj(x[[i]]$list_pred_res_obj,"raster_name"))},x=list_create_accuracy_residuals_raster_obj))) 
346
    lf_accuracy_residuals_testing_raster <- lapply(1:length(list_create_accuracy_residuals_raster_obj),FUN=function(i,x){as.character(unlist(extract_from_list_obj(x[[i]]$list_pred_res_obj,"raster_name")))},x=list_create_accuracy_residuals_raster_obj)
347
    
348
    #Plot as quick check
349
    #r1 <- raster(lf_mosaic[[1]][1]) 
350
    #list_create_accuracy_residuals_raster_obj
351
    #browser()  
352
  }
353
  
354
  #########################################
355
  ###Depending on value of layers_option, create accuracy surfaces based on training residuals...
356
  ##
357
  
358
  if(layers_option=="res_training"){
359
    #This part took 19 minutes and 40 seconds
360
    
361
    data_df <- data_day_s # data.frame table/spdf containing stations with residuals and variable
362
    
363
    num_cores_tmp <-num_cores
364
    lf_day_tiles  <- lf_mosaic #list of raster files by dates
365
    #data_df <- data_day_v # data.frame table/spdf containing stations with residuals and variable
366
    #df_tile_processed  #tiles processed during assessment usually by region
367
    #var_pred  #variable being modeled
368
    #if not list of models is provided generate one
369
    if(is.null(list_models)){
370
      list_models <- paste(var_pred,"~","1",sep=" ")
371
    }
372
    
373
    #use_autokrige #if TRUE use automap/gstat package
374
    #y_var_name  #"dailyTmax" #PARAM2
375
    #interpolation_method #c("gam_CAI") #PARAM3, need to select reg!!
376
    #date_processed #can be a monthly layer
377
    #num_cores #number of cores used
378
    #NA_flag_val 
379
    #file_format 
380
    out_dir_str <- out_dir
381
    out_suffix_str <- paste("data_day_s_",out_suffix,sep="") 
382
    days_to_process <- day_to_mosaic
383
    df_tile_processed$path_NEX <- as.character(df_tile_processed$path_NEX) 
384
    df_tile_processed$reg <- basename(dirname(df_tile_processed$path_NEX))
385
    
386
    ##By regions, selected earlier
387
    #for(k in 1:length(region_names)){
388
    df_tile_processed_reg <- subset(df_tile_processed,reg==region_selected)#use reg4
389
    #i<-1 #loop by days/date to process!!
390
    #test on the first day 
391
    list_param_create_accuracy_residuals_raster <- list(lf_day_tiles,data_df,df_tile_processed_reg,
392
                                                        var_pred,list_models,use_autokrige,y_var_name,interpolation_method,
393
                                                        days_to_process,num_cores_tmp,NA_flag_val,file_format,out_dir_str,
394
                                                        out_suffix_str) 
395
    names(list_param_create_accuracy_residuals_raster) <- c("lf_day_tiles","data_df","df_tile_processed_reg",
396
                                                            "var_pred","list_models","use_autokrige","y_var_name","interpolation_method",
397
                                                            "days_to_process","num_cores_tmp","NA_flag_val","file_format","out_dir_str",
398
                                                            "out_suffix_str") 
399
    #browser()  #21 minutes and 40 second to get here
400
    list_create_accuracy_residuals_raster_obj <- lapply(1:length(day_to_mosaic),FUN=create_accuracy_residuals_raster,
401
                                                        list_param=list_param_create_accuracy_residuals_raster)
402
    
403
    #undebug(create_accuracy_residuals_raster)
404
    #list_create_accuracy_residuals_raster_obj <- lapply(1:1,FUN=create_accuracy_residuals_raster,
405
    #                                list_param=list_param_create_accuracy_residuals_raster)
406
    
407
    #create_accuracy_residuals_raster_obj <- create_accuracy_metric_raster(1, list_param_create_accuracy_residuals_raster_obj)
408
    
409
    #note that three tiles did not produce a residuals surface!!! find out more later, join the output
410
    #to df_raste_tile to keep track of which one did not work...
411
    #lf_accuracy_residuals_raster <- as.character(unlist(lapply(1:length(list_create_accuracy_residuals_raster_obj),FUN=function(i,x){unlist(extract_from_list_obj(x[[i]]$list_pred_res_obj,"raster_name"))},x=list_create_accuracy_residuals_raster_obj))) 
412
    lf_accuracy_residuals_training_raster <- lapply(1:length(list_create_accuracy_residuals_raster_obj),FUN=function(i,x){as.character(unlist(extract_from_list_obj(x[[i]]$list_pred_res_obj,"raster_name")))},x=list_create_accuracy_residuals_raster_obj)
413
    
414
  }
415
  
416
  #browser()
417
  
418 282
  ######################################################
419 283
  #### PART 3: GENERATE MOSAIC FOR LIST OF FILES #####
420 284
  #################################
......
470 334
      df <- tb #for ac_testing
471 335
      days_to_process <- day_to_mosaic[i]
472 336
  
473
      browser()
337
      #browser()
338
      
474 339
      debug(generate_ac_assessment_layers_by_tile)
475
      lf_accuracy_testing_raster<- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name,
340
      lf_accuracy_testing_raster <- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name,
476 341
                                                    var_pred,list_models,use_autokrige,pred_mod_name,
477 342
                                                    y_var_name,interpolation_method,
478 343
                                                    days_to_process,num_cores,NA_flag_val,file_format,
......
509 374
                                scaling=scaling,
510 375
                                values_range=values_range)
511 376
      ##Took 12-13 minutes for 28 tiles and one date...!!! 
377
      lf_accuracy_testing_raster
378
      
379
      if(tmp_files==F){ #if false...delete all files with "_tmp"
380
        lf_tmp <- lf_accuracy_testing_raster
381
        ##now delete temporary files...
382
        file.remove(lf_accuracy_testing_raster)
383
      }
512 384
      list_mosaic_obj[[i]] <- mosaic_obj
513 385
    }
514 386
    
......
520 392
      df <- tb_s #for ac_testing
521 393
      days_to_process <- day_to_mosaic[i]
522 394
  
523
      browser()
395
      #browser()
524 396
      debug(generate_ac_assessment_layers_by_tile)
525 397
      lf_accuracy_training_raster<- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name,
526 398
                                                    var_pred,list_models,use_autokrige,pred_mod_name,
......
567 439
    if(layers_option=="res_testing"){
568 440
      
569 441
      lf <- lf_mosaic[i]
570
      df <- tb_s #for ac_testing
442
      df <- data_day_v #for ac_testing
571 443
      days_to_process <- day_to_mosaic[i]
572 444
  
573
      browser()
445
      #browser()
574 446
      debug(generate_ac_assessment_layers_by_tile)
575
      lf_accuracy_training_raster<- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name,
447
      lf_accuracy_residuals_testing_raster <- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name,
576 448
                                                    var_pred,list_models,use_autokrige,pred_mod_name,
577 449
                                                    y_var_name,interpolation_method,
578 450
                                                    days_to_process,num_cores,NA_flag_val,file_format,
......
608 480
    
609 481
    ### produce residuals mosaics
610 482
    if(layers_option=="res_training"){
483
      
484
      lf <- lf_mosaic[i]
485
      df <- data_day_s #for ac_training
486
      days_to_process <- day_to_mosaic[i]
487
  
488
      #browser()
489
      debug(generate_ac_assessment_layers_by_tile)
490
      lf_accuracy_residuals_training_raster <- generate_ac_assessment_layers_by_tile(lf,layers_option,df,df_tile_processed,metric_name,
491
                                                    var_pred,list_models,use_autokrige,pred_mod_name,
492
                                                    y_var_name,interpolation_method,
493
                                                    days_to_process,num_cores,NA_flag_val,file_format,
494
                                                    out_dir,out_suffix)   #### create a function to generate accuracy layers by tiles
495

  
611 496
      #for now add data_day_s in the name!!
612 497
      mosaic_method <- "use_edge_weights" #this is distance from edge
613 498
      out_suffix_tmp <- paste(interpolation_method,"kriged_residuals","data_day_s",day_to_mosaic[i],out_suffix,sep="_")

Also available in: Unified diff