Revision 1b4f99e6
Added by Benoit Parmentier almost 9 years ago
climate/research/oregon/interpolation/global_run_scalingup_mosaicing.R | ||
---|---|---|
5 | 5 |
#Analyses, figures, tables and data are also produced in the script. |
6 | 6 |
#AUTHOR: Benoit Parmentier |
7 | 7 |
#CREATED ON: 04/14/2015 |
8 |
#MODIFIED ON: 12/04/2015
|
|
8 |
#MODIFIED ON: 12/12/2015
|
|
9 | 9 |
#Version: 5 |
10 | 10 |
#PROJECT: Environmental Layers project |
11 | 11 |
#COMMENTS: analyses run for reg4 1992 for test of mosaicing using 1500x4500km and other tiles |
... | ... | |
58 | 58 |
|
59 | 59 |
#### FUNCTION USED IN SCRIPT |
60 | 60 |
|
61 |
function_mosaicing <-"global_run_scalingup_mosaicing_function_12042015.R"
|
|
61 |
function_mosaicing <-"global_run_scalingup_mosaicing_function_12122015.R"
|
|
62 | 62 |
|
63 | 63 |
#in_dir_script <-"/home/parmentier/Data/IPLANT_project/env_layers_scripts" #NCEAS UCSB |
64 | 64 |
in_dir_script <- "/nobackupp8/bparmen1/env_layers_scripts" #NASA NEX |
... | ... | |
71 | 71 |
|
72 | 72 |
#in_dir <- "/data/project/layers/commons/NEX_data/mosaicing_data_test" #PARAM1 |
73 | 73 |
#in_dir <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_10052015" #PARAM4 |
74 |
in_dir <- "/nobackupp8/bparmen1/test_output_run10_1500x4500_global_analyses_pred_1992_10052015" #NEX |
|
74 |
in_dir <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015" #NEX |
|
75 |
#in_dir <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_12072015" #NEX |
|
75 | 76 |
|
76 | 77 |
in_dir_tiles <- file.path(in_dir,"tiles") |
77 | 78 |
#in_dir_tiles <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015/tiles" #North America |
... | ... | |
83 | 84 |
interpolation_method <- c("gam_CAI") #PARAM3 |
84 | 85 |
region_name <- "reg4" #PARAM 4 #reg4 South America, Africa reg5,Europe reg2, North America reg1, Asia reg3 |
85 | 86 |
mosaicing_method <- c("unweighted","use_edge_weights") #PARAM5 |
86 |
out_suffix <- paste(region_name,"_","run10_1500x4500_global_analyses_pred_1992_10052015",sep="") #PARAM 6
|
|
87 |
out_suffix_str <- "run10_1500x4500_global_analyses_pred_1992_10052015"
|
|
87 |
out_suffix <- paste(region_name,"_","run10_1500x4500_global_analyses_pred_1992_12072015",sep="") #PARAM 6
|
|
88 |
out_suffix_str <- "run10_1500x4500_global_analyses_pred_1992_12072015"
|
|
88 | 89 |
metric_name <- "rmse" #RMSE, MAE etc. |
89 | 90 |
pred_mod_name <- "mod1" |
90 | 91 |
|
... | ... | |
104 | 105 |
NA_flag_val <- NA_value |
105 | 106 |
|
106 | 107 |
num_cores <- 6 #PARAM 12 |
108 |
region_names <- c("reg23","reg4") #selected region names, #PARAM2 |
|
107 | 109 |
|
110 |
###Make a separate folder for masks by regions... |
|
108 | 111 |
#infile_mask <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_10052015/r_mask_reg4.tif" |
109 | 112 |
infile_mask <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015/r_mask_reg4.tif" |
110 | 113 |
|
111 | 114 |
#tb_accuracy_name <- file.path(in_dir,paste("tb_diagnostic_v_NA","_",out_suffix_str,".txt",sep="")) |
112 | 115 |
#tb_accuracy_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_10052015/tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_10052015.txt" |
113 |
tb_accuracy_name <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015/tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_10052015.txt" |
|
114 |
|
|
116 |
#tb_accuracy_name <- "/nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015/tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_10052015.txt" |
|
117 |
tb_accuracy_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015" |
|
118 |
data_month_s_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/data_month_s_NAM_run10_1500x4500_global_analyses_pred_1992_12072015.txt" |
|
119 |
df_tile_processed_name <- "/data/project/layers/commons/NEX_data/output_run10_1500x4500_global_analyses_pred_1992_12072015/df_tile_processed_run10_1500x4500_global_analyses_pred_1992_12072015.txt" |
|
115 | 120 |
#python script and gdal on NEX NASA: |
116 |
mosaic_python <- "/nobackupp6/aguzman4/climateLayers/sharedCode/" |
|
117 |
python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules2/bin" |
|
121 |
#mosaic_python <- "/nobackupp6/aguzman4/climateLayers/sharedCode/"
|
|
122 |
#python_bin <- "/nobackupp6/aguzman4/climateLayers/sharedModules2/bin"
|
|
118 | 123 |
#python script and gdal on Atlas NCEAS |
119 |
#mosaic_python <- "/data/project/layers/commons/NEX_data/sharedCode"
|
|
120 |
#python_bin <- "/usr/bin"
|
|
124 |
mosaic_python <- "/data/project/layers/commons/NEX_data/sharedCode" |
|
125 |
python_bin <- "/usr/bin" |
|
121 | 126 |
|
122 | 127 |
algorithm <- "python" #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann |
123 | 128 |
#algorithm <- "R" #if R use mosaic function for R, if python use modified gdalmerge script from Alberto Guzmann |
... | ... | |
140 | 145 |
|
141 | 146 |
setwd(out_dir) |
142 | 147 |
|
143 |
|
|
148 |
# accuracy table by tiles |
|
144 | 149 |
tb <- read.table(file=tb_accuracy_name,sep=",") |
150 |
# textfiles of stations by month |
|
151 |
data_month_s <- read.table(file.path(data_month_s_name),sep=",") |
|
152 |
df_tile_processed <- read.table( df_tile_processed_name,sep=",") |
|
145 | 153 |
|
146 | 154 |
#list all files to mosaic for a list of day |
155 |
#Take into account multiple region in some cases!!! |
|
147 | 156 |
lf_mosaic <- lapply(1:length(day_to_mosaic),FUN=function(i){list.files(path=file.path(in_dir_tiles), |
148 |
pattern=paste(".*.",day_to_mosaic[i],".*.tif$",sep=""),full.names=T)}) |
|
149 |
|
|
157 |
pattern=paste(".*.",day_to_mosaic[i],".*.tif$",sep=""),full.names=T,recursive=T)})
|
|
158 |
|
|
150 | 159 |
##################### PART 2: produce the mosaic ################## |
151 | 160 |
|
152 | 161 |
########################## |
... | ... | |
174 | 183 |
"days_to_process","num_cores","NA_flag_val","file_format","out_dir_str","out_suffix_str") |
175 | 184 |
list_raster_created_obj <- lapply(1:length(day_to_mosaic),FUN=create_accuracy_metric_raster, |
176 | 185 |
list_param=list_param_accuracy_metric_raster) |
186 |
|
|
187 |
#interpolation_method_day_function_multisampling.R |
|
188 |
## Create accuracy surface by kriging |
|
189 |
data_tmp <- subset(data_month_s,month==1) |
|
190 |
data_tmp$tile_id <- as.character(data_tmp$tile_id) |
|
191 |
var_pred <- "res_mod1" |
|
192 |
list_models<- c("res_mod1 ~ 1") |
|
193 |
df_tile_processed |
|
194 |
#list_models<-c("y_var ~ lat*lon + elev_s + N_w*E_w", |
|
195 |
# "y_var ~ lat*lon + elev_s + DISTOC", |
|
196 |
# "y_var ~ lat*lon + elev_s + LST", |
|
197 |
# "y_var ~ lat*lon + elev_s + LST + I(LST*LC1)") |
|
198 |
|
|
199 |
##By regions |
|
200 |
create_accuracy_metric_raster_kriging <- function(data_df,df_tile_processed,var_pred,lf_ref_rast,out_dir,out_suffix){ |
|
201 |
#This function generates kriged values for a giving set of formula and data input. |
|
202 |
#Inputs: |
|
203 |
#lf: list of raster files |
|
204 |
#tb: data.frame table #fitting or validation table with all days |
|
205 |
#metric_name: accuracy metric selected to be mapped, RMSE, MAE etc. |
|
206 |
#pred_mod_name: model selected, such as mod1, mod_kr etc. |
|
207 |
#y_var_name: variable being modeled e.g."dailyTmax", dailyTmin, precip |
|
208 |
#interpolation_method: names of the interpolation/modeling method |
|
209 |
#date_processed: day being processed , e.g. 19920101 |
|
210 |
#num_cores : number of cores used in the parallelization |
|
211 |
#NA_flag_val: value used as flag in the raster |
|
212 |
#file_format: e.g. tif., .rst |
|
213 |
#out_dir_str: output directory |
|
214 |
#out_suffix_str: output suffix |
|
215 |
#Outputs: |
|
216 |
#raster list of weights and product of wegihts and inuts |
|
217 |
#TODO: |
|
218 |
|
|
219 |
# - improve efficiency |
|
220 |
# |
|
221 |
#promote to spdf |
|
222 |
list_formulas <- lapply(list_models,as.formula,env=.GlobalEnv) #mulitple arguments passed to lapply!! |
|
223 |
|
|
224 |
#data_training <- data_df |
|
225 |
coordinates(data_df)<-cbind(data_df$x,data_df$y) |
|
226 |
#proj4string(data_s)<-proj_str |
|
227 |
#coordinates(data_v)<-cbind(data_v$x,data_v$y) |
|
228 |
#proj4string(data_v)<-proj_str |
|
229 |
lf_day_tiles <- lf_mosaic[[i]] #i index for time... |
|
230 |
|
|
231 |
#Now match the correct tiles with data used in kriging... |
|
232 |
#match the correct tile!!! df_tile_processed |
|
233 |
#pattern_str <- as.character(unique(df_tile_processed$tile_coord)) |
|
234 |
list_tile_coord <- as.character(df_tile_processed$tile_coord) |
|
235 |
pattern_str <- glob2rx(paste("*",list_tile_coord,"*","*.tif",sep="")) |
|
236 |
keywords_str <- pattern_str |
|
237 |
tmp_str2 <-unlist(lapply(keywords_str,grep,lf_day_tiles,value=TRUE)) |
|
238 |
df_raster_pred_tiles_tmp <- data.frame(files =tmp_str2, tile_coord=list_tile_coord) |
|
239 |
df_raster_pred_tiles <- merge(df_raster_pred_tiles_tmp,df_tile_processed,by="tile_coord") |
|
240 |
df_raster_pred_tiles$path_NEX <- as.character(df_raster_pred_tiles$path_NEX) |
|
241 |
df_raster_pred_tiles$reg <- basename(dirname(df_raster_pred_tiles$path_NEX)) |
|
242 |
df_raster_pred_tiles$files <- as.character(df_raster_pred_tiles$files) |
|
243 |
|
|
244 |
j <- 1 #loops across tiles from set of files |
|
245 |
|
|
246 |
r_stack <- df_raster_pred_tiles$files[j] |
|
247 |
tile_selected <- as.character(df_raster_pred_tiles$tile_id[j]) |
|
248 |
data_training <- subset(data_df,tile_id==tile_selected) #df_raster_pred_tiles$files |
|
249 |
#r_stack <- stack(lf_day[j]) |
|
250 |
out_filename <- "test.tif" |
|
251 |
#debug(predict_auto_krige_raster_model) |
|
252 |
test <- predict_auto_krige_raster_model(list_formulas,r_stack,data_training,out_filename) |
|
253 |
#identify residuals |
|
254 |
|
|
255 |
#call kriging function |
|
256 |
|
|
257 |
#write output |
|
258 |
} |
|
259 |
|
|
177 | 260 |
#debug(create_accuracy_metric_raster) |
178 | 261 |
#list_raster_created_obj <- lapply(1:1,FUN=create_accuracy_metric_raster, |
179 | 262 |
# list_param=list_param_accuracy_metric_raster) |
Also available in: Unified diff
adding option to create residuals surface from stations using kriging