Project

General

Profile

« Previous | Next » 

Revision e4b8a604

Added by Benoit Parmentier about 9 years ago

testing code on NEX for accuracy metrics kriging of resiudals and gdalmerge modified script

View differences:

climate/research/oregon/interpolation/global_run_scalingup_mosaicing.R
14 14 
#2) generalize to run dates and region fast (use python mosaic Alberto code)
15 15 
#3) clean up temporary files, it builds currently on the disk
16 16 
#4) fix output folder for some of output files
17 
#
17 
#5) create a helper function for inputs/arguments to automate...?? Could also be in the assessment stage
18 18 

  		




  19
  19
  
    
### Before running, the gdal modules and other environment parameters need to be set if on NEX-NASA.


  		




  20
  20
  
    
### This can be done by running the following commands:


  		




  ......




  60
  60
  
    

  		




  61
  61
  
    
function_mosaicing <-"global_run_scalingup_mosaicing_function_12172015.R"


  		




  62
  62
  
    

  		




  63
  
  
    
in_dir_script <-"/home/parmentier/Data/IPLANT_project/env_layers_scripts" #NCEAS UCSB


  		




  64
  
  
    
#in_dir_script <- "/nobackupp8/bparmen1/env_layers_scripts" #NASA NEX


  		




  
  63
  
    
#in_dir_script <-"/home/parmentier/Data/IPLANT_project/env_layers_scripts" #NCEAS UCSB


  		




  
  64
  
    
in_dir_script <- "/nobackupp8/bparmen1/env_layers_scripts" #NASA NEX


  		




  65
  65
  
    
source(file.path(in_dir_script,function_mosaicing))


  		




  66
  66
  
    

  		




  67
  67
  
    
load_obj <- function(f)


  		




  ......




  90
  90
  
    

  		




  91
  91
  
    
#in_dir <- "/data/project/layers/commons/NEX_data/mosaicing_data_test" #PARAM1


  		




  92
  92
  
    
#in_dir <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_10052015" #PARAM4


  		




  93
  
  
    
in_dir <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015" #NEX


  		




  94
  
  
    
#in_dir <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_12072015" #NEX


  		




  
  93
  
    
#in_dir <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015" #NEX


  		




  
  94
  
    
in_dir <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_12072015" #NEX


  		




  95
  95
  
    

  		




  96
  96
  
    
in_dir_tiles <- file.path(in_dir,"tiles")


  		




  97
  97
  
    
#in_dir_tiles <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015/tiles" #North America


  		




  ......




  127
  127
  
    
region_names <- c("reg23","reg4") #selected region names, ##PARAM 18 


  		




  128
  128
  
    
use_autokrige <- F #PARAM 19


  		




  129
  129
  
    

  		




  130
  
  
    
###Make a separate folder for masks by regions... #PARAM 20


  		




  131
  
  
    
#infile_mask <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_10052015/r_mask_reg4.tif"


  		




  132
  
  
    
infile_mask <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015/r_mask_reg4.tif"


  		




  
  130
  
    
###Separate folder for masks by regions, should be listed as just the dir!!... #PARAM 20


  		




  
  131
  
    
infile_mask <- "/nobackupp8/bparmen1/regions_input_files/r_mask_reg4.tif"


  		




  
  132
  
    
#infile_mask <- "/data/project/layers/commons/NEX_data/regions_input_files/r_mask_reg4.tif"


  		




  133
  133
  
    

  		




  134
  134
  
    
#tb_accuracy_name <- file.path(in_dir,paste("tb_diagnostic_v_NA","_",out_suffix_str,".txt",sep=""))


  		




  135
  
  
    
#tb_accuracy_name <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015/tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_10052015.txt"


  		




  136
  
  
    
tb_accuracy_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_12072015.txt" #PARAM 21


  		




  137
  
  
    
data_month_s_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/data_month_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt" #PARAM 22


  		




  138
  
  
    
data_day_v_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/data_day_v_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt" #PARAM 23


  		




  139
  
  
    
data_day_s_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/data_day_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt" ##PARAM 24


  		




  140
  
  
    
df_tile_processed_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/df_tile_processed_run10_1500x4500_global_analyses_pred_1992_12072015.txt" ##PARAM 25


  		




  
  135
  
    
#tb_accuracy_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_12072015.txt" #PARAM 21


  		




  
  136
  
    
#data_month_s_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/data_month_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt" #PARAM 22


  		




  
  137
  
    
#data_day_v_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/data_day_v_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt" #PARAM 23


  		




  
  138
  
    
#data_day_s_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/data_day_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt" ##PARAM 24


  		




  
  139
  
    
#df_tile_processed_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/df_tile_processed_run10_1500x4500_global_analyses_pred_1992_12072015.txt" ##PARAM 25


  		




  
  140
  
    

  		




  
  141
  
    
tb_accuracy_name <- file.path(in_dir,"tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_12072015.txt") #PARAM 21


  		




  
  142
  
    
data_month_s_name <- file.path(in_dir,"data_month_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt") #PARAM 22


  		




  
  143
  
    
data_day_v_name <- file.path(in_dir,"data_day_v_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt") #PARAM 23


  		




  
  144
  
    
data_day_s_name <- file.path(in_dir,"data_day_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt") ##PARAM 24


  		




  
  145
  
    
df_tile_processed_name <- file.path(in_dir,"df_tile_processed_run10_1500x4500_global_analyses_pred_1992_12072015.txt") ##PARAM 25


  		




  141
  146
  
    

  		




  142
  147
  
    
#python script and gdal on NEX NASA:


  		




  143
  
  
    
#mosaic_python <- "/nobackupp6/aguzman4/climateLayers/sharedCode/"


  		




  144
  
  
    
#python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules2/bin"


  		




  
  148
  
    
mosaic_python <- "/nobackupp6/aguzman4/climateLayers/sharedCode/"


  		




  
  149
  
    
python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules2/bin"


  		




  145
  150
  
    
#python script and gdal on Atlas NCEAS


  		




  146
  
  
    
mosaic_python <- "/data/project/layers/commons/NEX_data/sharedCode" #PARAM 26


  		




  147
  
  
    
python_bin <- "/usr/bin" #PARAM 27


  		




  
  151
  
    
#mosaic_python <- "/data/project/layers/commons/NEX_data/sharedCode" #PARAM 26


  		




  
  152
  
    
#python_bin <- "/usr/bin" #PARAM 27


  		




  148
  153
  
    

  		




  149
  154
  
    
algorithm <- "python" #PARAM 28 #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann


  		




  150
  155
  
    
#algorithm <- "R" #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann


  		




  ......




  340
  345
  
    
  


  		




  341
  346
  
    
  list_mosaic_obj[[i]] <- list(prediction=mosaic_edge_obj_prediction,accuracy=mosaic_edge_obj_accuracy)


  		




  342
  347
  
    

  		




  
  348
  
    
  ### produce residuals mosaics


  		




  343
  349
  
    
  mosaic_method <- "use_edge_weights" #this is distance from edge


  		




  344
  
  
    
  out_suffix_tmp <- paste(interpolation_method,metric_name,day_to_mosaic[i],out_suffix,sep="_")


  		




  345
  
  
    

  		




  346
  
  
    
  mosaic_edge_obj_residuals <- mosaicFiles(lf_accuracy_residuals_raster[[i]],


  		




  
  350
  
    
  out_suffix_tmp <- paste(interpolation_method,"kriged_residuals",day_to_mosaic[i],out_suffix,sep="_")


  		




  
  351
  
    
  lf_tmp<-list.files(pattern="*kriged_residuals.*.tif",full.names=T)


  		




  
  352
  
    
  #lf_accuracy_residuals_raster[[i]]


  		




  
  353
  
    
  mosaic_edge_obj_residuals <- mosaicFiles(lf_tmp,


  		




  347
  354
  
    
                                        mosaic_method="use_edge_weights",


  		




  348
  355
  
    
                                        num_cores=num_cores,


  		




  349
  356
  
    
                                        r_mask_raster_name=infile_mask,


  		




  ......




  356
  363
  
    
                                        out_suffix=out_suffix_tmp,


  		




  357
  364
  
    
                                        out_dir=out_dir)


  		




  358
  365
  
    
  


  		




  359
  
  
    
  list_mosaic_obj[[i]] <- list(prediction=mosaic_edge_obj_prediction,accuracy=mosaic_edge_obj_accuracy)


  		




  
  366
  
    
  list_mosaic_obj[[i]] <- list(prediction=mosaic_edge_obj_prediction,accuracy=mosaic_edge_obj_accuracy,mosaic_edge_obj_residuals)


  		




  360
  367
  
    
  


  		




  361
  368
  
    
}


  		




  362
  369
  
    

  		












Also available in:  Unified diff