Revision 55bbc59b
Added by Benoit Parmentier about 9 years ago
climate/research/oregon/interpolation/global_run_scalingup_mosaicing.R | ||
---|---|---|
5 | 5 |
#Analyses, figures, tables and data are also produced in the script. |
6 | 6 |
#AUTHOR: Benoit Parmentier |
7 | 7 |
#CREATED ON: 04/14/2015 |
8 |
#MODIFIED ON: 10/09/2015
|
|
8 |
#MODIFIED ON: 10/10/2015
|
|
9 | 9 |
#Version: 5 |
10 | 10 |
#PROJECT: Environmental Layers project |
11 | 11 |
#COMMENTS: analyses run for reg4 1992 for test of mosaicing using 1500x4500km and other tiles |
... | ... | |
46 | 46 |
|
47 | 47 |
#### FUNCTION USED IN SCRIPT |
48 | 48 |
|
49 |
function_mosaicing <-"global_run_scalingup_mosaicing_function_10052015.R"
|
|
49 |
function_mosaicing <-"global_run_scalingup_mosaicing_function_10102015.R"
|
|
50 | 50 |
|
51 | 51 |
in_dir_script <-"/home/parmentier/Data/IPLANT_project/env_layers_scripts" |
52 | 52 |
source(file.path(in_dir_script,function_mosaicing)) |
... | ... | |
70 | 70 |
mosaicing_method <- c("unweighted","use_edge_weights") #PARAM5 |
71 | 71 |
out_suffix <- paste(region_name,"_","run10_1500x4500_global_analyses_pred_1992_10052015",sep="") #PARAM 6 |
72 | 72 |
out_suffix_str <- "run10_1500x4500_global_analyses_pred_1992_10052015" |
73 |
metric_name <- "rmse" #RMSE, MAE etc. |
|
74 |
pred_mod_name <- "mod1" |
|
73 | 75 |
|
74 | 76 |
#PARAM3 |
75 | 77 |
out_dir <- in_dir #PARAM 7 |
... | ... | |
105 | 107 |
setwd(out_dir) |
106 | 108 |
|
107 | 109 |
|
108 |
|
|
109 |
#must be cleaned up |
|
110 | 110 |
tb <- read.table(file=file.path(in_dir,paste("tb_diagnostic_v_NA","_",out_suffix_str,".txt",sep="")),sep=",") |
111 |
#tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_10052015.txt |
|
112 |
|
|
113 |
### Start new function here |
|
114 |
|
|
115 |
## Add numcores |
|
116 |
## use mclapply |
|
117 |
create_accuracy_metric_raster <- function(i, list_param){ |
|
118 |
#This function generates weights from a point location on a raster layer. |
|
119 |
#Note that the weights are normatlized on 0-1 scale using max and min values. |
|
120 |
#Inputs: |
|
121 |
#lf: list of raster files |
|
122 |
#tb: data.frame table #fitting or validation table with all days |
|
123 |
#metric_name <- list_param$metric_name #RMSE, MAE etc. |
|
124 |
#pred_mod_name <- list_param$pred_mod_name #mod1, mod_kr etc. |
|
125 |
#y_var_name <- list_param$y_var_name #"dailyTmax" #PARAM2 |
|
126 |
#interpolation_method <- list_param$interpolation_method #c("gam_CAI") #PARAM3 |
|
127 |
#date_processed <- list_param$days_to_process[i] |
|
128 |
#NA_flag_val <- list_param$NA_flag_val |
|
129 |
#NAflag,file_format,out_suffix etc... |
|
130 |
#file_format <- list_param$file_format |
|
131 |
#out_dir_str <- list_param$out_dir |
|
132 |
#out_suffix_str <- list_param$out_suffix |
|
133 |
#Outputs: |
|
134 |
#raster list of weights and product of wegihts and inuts |
|
135 |
#TODO: |
|
136 |
|
|
137 |
# - improve efficiency |
|
138 |
# |
|
139 |
############ |
|
140 |
|
|
141 |
lf <- list_param$lf #list of files to mosaic |
|
142 |
tb <- list_param$tb #fitting or validation table with all days |
|
143 |
metric_name <- list_param$metric_name #RMSE, MAE etc. |
|
144 |
pred_mod_name <- list_param$pred_mod_name #mod1, mod_kr etc. |
|
145 |
y_var_name <- list_param$y_var_name #"dailyTmax" #PARAM2 |
|
146 |
interpolation_method <- list_param$interpolation_method #c("gam_CAI") #PARAM3 |
|
147 |
date_processed <- list_param$days_to_process[i] |
|
148 |
NA_flag_val <- list_param$NA_flag_val |
|
149 |
#NAflag,file_format,out_suffix etc... |
|
150 |
file_format <- list_param$file_format |
|
151 |
out_dir_str <- list_param$out_dir |
|
152 |
out_suffix_str <- list_param$out_suffix |
|
153 |
|
|
154 |
####### START SCRIPT ##### |
|
155 |
|
|
156 |
#r_in <- raster(lf[i]) #input image |
|
157 |
|
|
158 |
#date_processed <- day_to_mosaic[i] |
|
159 |
#lf_to_mosaic <-list.files(path=file.path(in_dir_tiles), |
|
160 |
# pattern=paste(".*.",date_processed,".*.tif$",sep=""),full.names=T) #choosing date 2...20100901 |
|
161 |
|
|
162 |
lf_tmp <- gsub(file_format,"",lf) |
|
163 |
tx<-strsplit(as.character(lf_tmp),"_") |
|
164 |
#deal with the fact that we have number "1" attached to the out_suffix (centroids of tiles) |
|
165 |
pos_lat <- lapply(1:length(tx),function(i,x){length(x[[i]])-1},x=tx) |
|
166 |
pos_lon <- lapply(1:length(tx),function(i,x){length(x[[i]])},x=tx) |
|
167 |
lat_val <- unlist(lapply(1:length(tx),function(i,x,y){x[[i]][pos_lat[[i]]]},x=tx,y=pos_lat)) |
|
168 |
lat <- as.character(lapply(1:length(lat_val),function(i,x){substr(x[[i]],2,nchar(x[i]))},x=lat_val)) #first number not in the coordinates |
|
169 |
long <- as.character(lapply(1:length(tx),function(i,x,y){x[[i]][pos_lon[[i]]]},x=tx,y=lon_lat)) |
|
170 |
|
|
171 |
df_centroids <- data.frame(long=as.numeric(long),lat=as.numeric(lat)) |
|
172 |
df_centroids$ID <- as.numeric(1:nrow(df_centroids)) |
|
173 |
df_centroids$tile_coord <- paste(lat,long,sep="_") |
|
174 |
df_centroids$files <- lf |
|
175 |
df_centroids$date <- date_processed |
|
176 |
write.table(df_centroids,paste("df_centroids_",date_processed,"_",out_suffix,".txt",sep=""),sep=',') |
|
177 |
|
|
178 |
#sprintf(" %3.1f", df_centroids$lat) |
|
179 |
|
|
180 |
tb_date <- subset(tb,date==date_processed & pred_mod==pred_mod_name) |
|
181 |
tb_date$tile_coord <- as.character(tb_date$tile_coord) |
|
182 |
df_centroids <- merge(df_centroids,tb_date,by="tile_coord") |
|
183 |
|
|
184 |
#r1 <- raster(lf[i]) |
|
185 |
|
|
186 |
#function(j,df_centroids,metric_name,date_processed,interpolation,) |
|
187 |
#loop over files, make this a function later, works for now |
|
188 |
#use mclapply |
|
189 |
list_raster_name <- vector("list",length=length(lf)) |
|
190 |
|
|
191 |
for(j in 1:length(lf)){ |
|
192 |
|
|
193 |
inFilename <- df_centroids$files[j] |
|
194 |
r1 <- raster(inFilename) |
|
195 |
r1[] <- df_centroids[[metric_name]][j] #improve this |
|
196 |
#set1f <- function(x){rep(NA, x)} |
|
197 |
#r_init <- init(r_in, fun=set1f) |
|
198 |
lf_tmp <- gsub(file_format,"",lf) |
|
199 |
|
|
200 |
extension_str <- extension(inFilename) |
|
201 |
raster_name_tmp <- gsub(extension_str,"",basename(inFilename)) |
|
202 |
outFilename <- file.path(out_dir,paste(raster_name_tmp,"_",metric_name,"_",out_suffix,file_format,sep="")) #for use in function later... |
|
203 |
|
|
204 |
#raster_name <- file.path(out_dir_str,paste(raster_name_tmp,"_",out_suffix,file_format,sep=""))#output file |
|
205 |
|
|
206 |
#raster_name_tmp <- paste("r_",metric_name,"_",interpolation_method,"_",date_processed,"_",out_suffix_str,sep="") |
|
207 |
#raster_name <- file.path(out_dir_str,raster_name_tmp) |
|
208 |
writeRaster(r1, NAflag=NA_flag_val,filename=outFilename,overwrite=TRUE) |
|
209 |
list_raster_name[[j]] <- outFilename |
|
210 |
} |
|
211 |
|
|
212 |
|
|
213 |
raster_created_obj <- list(list_raster_name,df_centroids) |
|
214 |
names(raster_created_obj) <- c("list_raster_name","df_centroids") |
|
215 |
return(raster_created_obj) |
|
216 |
|
|
217 |
} |
|
218 | 111 |
|
219 |
#### end of function |
|
220 | 112 |
|
221 | 113 |
lf_mosaic1 <-list.files(path=file.path(in_dir_tiles), |
222 | 114 |
pattern=paste(".*.",day_to_mosaic[1],".*.tif$",sep=""),full.names=T) #choosing date 2...20100901 |
... | ... | |
224 | 116 |
lf_mosaic2 <-list.files(path=file.path(in_dir_tiles), |
225 | 117 |
pattern=paste(".*.",day_to_mosaic[2],".*.tif$",sep=""),full.names=T) #choosing date 2...20100901 |
226 | 118 |
|
227 |
lf <- lf_mosaic1 #list of files to mosaic |
|
119 |
|
|
120 |
##################### PART 2: produce the mosaic ################## |
|
121 |
|
|
122 |
########################## |
|
123 |
#### First generate rmse images for each date and tile for the region |
|
124 |
|
|
125 |
lf <- lf_mosaic1 #list of files to mosaic for date 1, there a 28 files for reg4, South America |
|
126 |
|
|
228 | 127 |
#tb <- list_param$tb #fitting or validation table with all days |
229 |
metric_name <- "rmse" #RMSE, MAE etc. |
|
230 |
pred_mod_name <- "mod1" |
|
128 |
#metric_name <- "rmse" #RMSE, MAE etc.
|
|
129 |
#pred_mod_name <- "mod1"
|
|
231 | 130 |
#y_var_name |
232 | 131 |
#interpolation_method #c("gam_CAI") #PARAM3 |
233 | 132 |
days_to_process <- day_to_mosaic |
... | ... | |
241 | 140 |
|
242 | 141 |
names(list_param_accuracy_metric_raster) <- c("lf","tb","metric_name","pred_mod_name","y_var_name","interpolation_method", |
243 | 142 |
"days_to_process","NA_flag_val","file_format","out_dir_str","out_suffix_str") |
244 |
debug(create_accuracy_metric_raster) |
|
143 |
#debug(create_accuracy_metric_raster)
|
|
245 | 144 |
raster_created_obj <- create_accuracy_metric_raster(1, list_param_accuracy_metric_raster) |
246 | 145 |
|
247 |
lf_accuracy_raster <- unlist(raster_created_obj$list_raster_name) |
|
248 |
|
|
249 |
lf_r_rmse <- mclapply(1:length(days_to_process),FUN=create_accuracy_metric_raster,list_param=list_param_accuracy_metric_raster,mc.preschedule=FALSE,mc.cores = 3) |
|
146 |
#Extract list of files for rmse and date 1 (19920101), there should be 28 raster images |
|
147 |
lf_accuracy_raster <- unlist(raster_created_obj$list_raster_name) |
|
250 | 148 |
|
251 |
#lf_mosaic <- lf_mosaic[1:20]
|
|
149 |
#Plot as quick check
|
|
252 | 150 |
r1 <- raster(lf_mosaic1[1]) |
253 |
r2 <- raster(lf_mosaic2[2]) |
|
151 |
#r2 <- raster(lf_mosaic2[2])
|
|
254 | 152 |
|
255 | 153 |
plot(r1) |
256 |
plot(r2) |
|
154 |
#plot(r2) |
|
155 |
|
|
156 |
################################# |
|
157 |
#### Second mosaic tiles for the variable and accuracy metrid |
|
257 | 158 |
|
258 | 159 |
|
259 | 160 |
#methods availbable:use_sine_weights,use_edge,use_linear_weights |
... | ... | |
263 | 164 |
for(i in 1:length(day_to_mosaic)){ |
264 | 165 |
|
265 | 166 |
mosaic_method <- "use_edge_weights" #this is distance from edge |
266 |
out_suffix_str <- paste(day_to_mosaic[i],out_suffix,sep="_") |
|
267 |
#undebug(mosaicFiles)
|
|
167 |
out_suffix_str <- paste(interpolation_method,y_var_name,day_to_mosaic[i],out_suffix,sep="_")
|
|
168 |
#debug(mosaicFiles) |
|
268 | 169 |
#can also loop through methods!!! |
170 |
python_bin <- "/usr/bin/" #python gdal bin |
|
269 | 171 |
mosaic_edge_obj_prediction <- mosaicFiles(lf_mosaic1,mosaic_method="use_edge_weights", |
270 | 172 |
num_cores=num_cores, |
271 |
python_bin=NULL,
|
|
173 |
python_bin=python_bin,
|
|
272 | 174 |
df_points=NULL,NA_flag=NA_flag_val, |
273 | 175 |
file_format=file_format,out_suffix=out_suffix_str, |
274 | 176 |
out_dir=out_dir) |
275 | 177 |
|
276 | 178 |
mosaic_method <- "use_edge_weights" #this is distance from edge |
277 |
out_suffix_str <- paste(day_to_mosaic[i],out_suffix,sep="_") |
|
278 |
#undebug(mosaicFiles) |
|
179 |
out_suffix_str <- paste(interpolation_method,metric_name,day_to_mosaic[i],out_suffix,sep="_") |
|
180 |
|
|
181 |
#debug(mosaicFiles) |
|
279 | 182 |
#can also loop through methods!!! |
280 | 183 |
mosaic_edge_obj_accuracy <- mosaicFiles(lf_accuracy_raster,mosaic_method="use_edge_weights", |
281 | 184 |
num_cores=num_cores, |
282 |
python_bin=NULL,
|
|
185 |
python_bin=python_bin,
|
|
283 | 186 |
df_points=NULL,NA_flag=NA_flag_val, |
284 | 187 |
file_format=file_format,out_suffix=out_suffix_str, |
285 | 188 |
out_dir=out_dir) |
286 | 189 |
|
287 |
#mosaic_unweighted_obj <- mosaicFiles(lf_mosaic1,mosaic_method="unweighted", |
|
288 |
# num_cores=num_cores, |
|
289 |
# python_bin=NULL, |
|
290 |
# df_points=NULL,NA_flag=NA_flag_val, |
|
291 |
# file_format=file_format,out_suffix=out_suffix_str, |
|
292 |
# out_dir=out_dir) |
|
293 |
|
|
294 |
#list_mosaic_obj[[i]] <- list(unweighted=mosaic_unweighted_obj,edge=mosaic_edge_obj) |
|
295 |
#list_mosaic_obj[[i]] <- list(prediction=mosaic_edge_obj_accuracy,accuracy=mosaic_edge_obj_accuracy) |
|
296 |
|
|
297 | 190 |
list_mosaic_obj[[i]] <- list(prediction=mosaic_edge_obj_prediction,accuracy=mosaic_edge_obj_accuracy) |
298 | 191 |
} |
299 | 192 |
|
... | ... | |
318 | 211 |
list_param_plot_mosaic <- list(lf_mosaic=unlist(lf_mean_mosaic), |
319 | 212 |
method=unlist(l_method_mosaic), |
320 | 213 |
out_suffix=unlist(list_out_suffix)) |
321 |
#undebug(plot_mosaic) |
|
214 |
|
|
215 |
#plot and produce png movie... |
|
322 | 216 |
#plot_mosaic(1,list_param=list_param_plot_mosaic) |
323 | 217 |
num_cores <- 4 |
324 | 218 |
l_png_files <- mclapply(1:length(unlist(lf_mean_mosaic)),FUN=plot_mosaic, |
325 | 219 |
list_param= list_param_plot_mosaic, |
326 | 220 |
mc.preschedule=FALSE,mc.cores = num_cores) |
327 | 221 |
|
328 |
num_cores <- 2 |
|
329 |
l_diff_png_files <- mclapply(1:length(lf1),FUN=plot_diff_raster,list_param= list_param_plot_diff, |
|
330 |
mc.preschedule=FALSE,mc.cores = num_cores) |
|
331 | 222 |
|
332 | 223 |
############### |
333 | 224 |
|
Also available in: Unified diff
running mosaicing script for accuracy metric mosaic, clean up