Project

General

Profile

« Previous | Next » 

Revision 55bbc59b

Added by Benoit Parmentier about 9 years ago

running mosaicing script for accuracy metric mosaic, clean up

View differences:

climate/research/oregon/interpolation/global_run_scalingup_mosaicing.R
5 5
#Analyses, figures, tables and data are also produced in the script.
6 6
#AUTHOR: Benoit Parmentier 
7 7
#CREATED ON: 04/14/2015  
8
#MODIFIED ON: 10/09/2015            
8
#MODIFIED ON: 10/10/2015            
9 9
#Version: 5
10 10
#PROJECT: Environmental Layers project     
11 11
#COMMENTS: analyses run for reg4 1992 for test of mosaicing using 1500x4500km and other tiles
......
46 46

  
47 47
#### FUNCTION USED IN SCRIPT
48 48

  
49
function_mosaicing <-"global_run_scalingup_mosaicing_function_10052015.R"
49
function_mosaicing <-"global_run_scalingup_mosaicing_function_10102015.R"
50 50

  
51 51
in_dir_script <-"/home/parmentier/Data/IPLANT_project/env_layers_scripts"
52 52
source(file.path(in_dir_script,function_mosaicing))
......
70 70
mosaicing_method <- c("unweighted","use_edge_weights") #PARAM5
71 71
out_suffix <- paste(region_name,"_","run10_1500x4500_global_analyses_pred_1992_10052015",sep="") #PARAM 6
72 72
out_suffix_str <- "run10_1500x4500_global_analyses_pred_1992_10052015"
73
metric_name <- "rmse" #RMSE, MAE etc.
74
pred_mod_name <- "mod1"
73 75

  
74 76
#PARAM3
75 77
out_dir <- in_dir #PARAM 7
......
105 107
setwd(out_dir)
106 108

  
107 109

  
108

  
109
#must be cleaned up
110 110
tb <- read.table(file=file.path(in_dir,paste("tb_diagnostic_v_NA","_",out_suffix_str,".txt",sep="")),sep=",")
111
#tb_diagnostic_v_NA_run10_1500x4500_global_analyses_pred_1992_10052015.txt
112

  
113
### Start new function here
114

  
115
## Add numcores
116
## use mclapply
117
create_accuracy_metric_raster <- function(i, list_param){
118
  #This function generates weights from a point location on a raster layer.
119
  #Note that the weights are normatlized on 0-1 scale using max and min values.
120
  #Inputs:
121
  #lf: list of raster files
122
  #tb: data.frame table #fitting or validation table with all days
123
  #metric_name <- list_param$metric_name #RMSE, MAE etc.
124
  #pred_mod_name <- list_param$pred_mod_name #mod1, mod_kr etc.
125
  #y_var_name <- list_param$y_var_name #"dailyTmax" #PARAM2
126
  #interpolation_method <- list_param$interpolation_method #c("gam_CAI") #PARAM3
127
  #date_processed <- list_param$days_to_process[i]
128
  #NA_flag_val <- list_param$NA_flag_val
129
  #NAflag,file_format,out_suffix etc...
130
  #file_format <- list_param$file_format
131
  #out_dir_str <- list_param$out_dir
132
  #out_suffix_str <- list_param$out_suffix
133
  #Outputs:
134
  #raster list of weights and product of wegihts and inuts
135
  #TODO: 
136

  
137
  # - improve efficiency
138
  #
139
  ############
140
  
141
  lf <- list_param$lf #list of files to mosaic
142
  tb <- list_param$tb #fitting or validation table with all days
143
  metric_name <- list_param$metric_name #RMSE, MAE etc.
144
  pred_mod_name <- list_param$pred_mod_name #mod1, mod_kr etc.
145
  y_var_name <- list_param$y_var_name #"dailyTmax" #PARAM2
146
  interpolation_method <- list_param$interpolation_method #c("gam_CAI") #PARAM3
147
  date_processed <- list_param$days_to_process[i]
148
  NA_flag_val <- list_param$NA_flag_val
149
  #NAflag,file_format,out_suffix etc...
150
  file_format <- list_param$file_format
151
  out_dir_str <- list_param$out_dir
152
  out_suffix_str <- list_param$out_suffix
153
   
154
  ####### START SCRIPT #####
155
  
156
  #r_in <- raster(lf[i]) #input image
157

  
158
  #date_processed <- day_to_mosaic[i]
159
  #lf_to_mosaic <-list.files(path=file.path(in_dir_tiles),    
160
  #         pattern=paste(".*.",date_processed,".*.tif$",sep=""),full.names=T) #choosing date 2...20100901
161

  
162
  lf_tmp <- gsub(file_format,"",lf)
163
  tx<-strsplit(as.character(lf_tmp),"_")
164
  #deal with the fact that we have number "1" attached to the out_suffix (centroids of tiles)
165
  pos_lat <- lapply(1:length(tx),function(i,x){length(x[[i]])-1},x=tx)
166
  pos_lon <- lapply(1:length(tx),function(i,x){length(x[[i]])},x=tx)
167
  lat_val <- unlist(lapply(1:length(tx),function(i,x,y){x[[i]][pos_lat[[i]]]},x=tx,y=pos_lat))
168
  lat <- as.character(lapply(1:length(lat_val),function(i,x){substr(x[[i]],2,nchar(x[i]))},x=lat_val)) #first number not in the coordinates
169
  long <- as.character(lapply(1:length(tx),function(i,x,y){x[[i]][pos_lon[[i]]]},x=tx,y=lon_lat))
170

  
171
  df_centroids <- data.frame(long=as.numeric(long),lat=as.numeric(lat))
172
  df_centroids$ID <- as.numeric(1:nrow(df_centroids))
173
  df_centroids$tile_coord <- paste(lat,long,sep="_")
174
  df_centroids$files <- lf
175
  df_centroids$date <- date_processed
176
  write.table(df_centroids,paste("df_centroids_",date_processed,"_",out_suffix,".txt",sep=""),sep=',')
177

  
178
  #sprintf(" %3.1f", df_centroids$lat)
179

  
180
  tb_date <- subset(tb,date==date_processed & pred_mod==pred_mod_name)
181
  tb_date$tile_coord <- as.character(tb_date$tile_coord)
182
  df_centroids <- merge(df_centroids,tb_date,by="tile_coord")
183

  
184
  #r1 <- raster(lf[i])
185
  
186
  #function(j,df_centroids,metric_name,date_processed,interpolation,)
187
  #loop over files, make this a function later, works for now
188
  #use mclapply  
189
  list_raster_name <- vector("list",length=length(lf))
190
  
191
  for(j in 1:length(lf)){ 
192
    
193
    inFilename <- df_centroids$files[j]
194
    r1 <- raster(inFilename)
195
    r1[] <- df_centroids[[metric_name]][j] #improve this
196
    #set1f <- function(x){rep(NA, x)}
197
    #r_init <- init(r_in, fun=set1f)
198
    lf_tmp <- gsub(file_format,"",lf)
199
  
200
    extension_str <- extension(inFilename)
201
    raster_name_tmp <- gsub(extension_str,"",basename(inFilename))
202
    outFilename <- file.path(out_dir,paste(raster_name_tmp,"_",metric_name,"_",out_suffix,file_format,sep="")) #for use in function later...
203
  
204
    #raster_name <- file.path(out_dir_str,paste(raster_name_tmp,"_",out_suffix,file_format,sep=""))#output file
205

  
206
    #raster_name_tmp <- paste("r_",metric_name,"_",interpolation_method,"_",date_processed,"_",out_suffix_str,sep="")
207
    #raster_name <- file.path(out_dir_str,raster_name_tmp)
208
    writeRaster(r1, NAflag=NA_flag_val,filename=outFilename,overwrite=TRUE)  
209
    list_raster_name[[j]] <- outFilename
210
  }
211
  
212
    
213
  raster_created_obj <- list(list_raster_name,df_centroids)
214
  names(raster_created_obj) <- c("list_raster_name","df_centroids")
215
  return(raster_created_obj)
216

  
217
}
218 111

  
219
#### end of function
220 112

  
221 113
lf_mosaic1 <-list.files(path=file.path(in_dir_tiles),    
222 114
           pattern=paste(".*.",day_to_mosaic[1],".*.tif$",sep=""),full.names=T) #choosing date 2...20100901
......
224 116
lf_mosaic2 <-list.files(path=file.path(in_dir_tiles),    
225 117
           pattern=paste(".*.",day_to_mosaic[2],".*.tif$",sep=""),full.names=T) #choosing date 2...20100901
226 118

  
227
lf <- lf_mosaic1 #list of files to mosaic
119

  
120
##################### PART 2: produce the mosaic ##################
121

  
122
##########################
123
#### First generate rmse images for each date and tile for the region
124

  
125
lf <- lf_mosaic1 #list of files to mosaic for date 1, there a 28 files for reg4, South America
126

  
228 127
#tb <- list_param$tb #fitting or validation table with all days
229
metric_name <- "rmse" #RMSE, MAE etc.
230
pred_mod_name <- "mod1"
128
#metric_name <- "rmse" #RMSE, MAE etc.
129
#pred_mod_name <- "mod1"
231 130
#y_var_name 
232 131
#interpolation_method #c("gam_CAI") #PARAM3
233 132
days_to_process <- day_to_mosaic
......
241 140

  
242 141
names(list_param_accuracy_metric_raster) <- c("lf","tb","metric_name","pred_mod_name","y_var_name","interpolation_method",
243 142
                       "days_to_process","NA_flag_val","file_format","out_dir_str","out_suffix_str") 
244
debug(create_accuracy_metric_raster)
143
#debug(create_accuracy_metric_raster)
245 144
raster_created_obj <- create_accuracy_metric_raster(1, list_param_accuracy_metric_raster)
246 145

  
247
lf_accuracy_raster <- unlist(raster_created_obj$list_raster_name)
248

  
249
lf_r_rmse <- mclapply(1:length(days_to_process),FUN=create_accuracy_metric_raster,list_param=list_param_accuracy_metric_raster,mc.preschedule=FALSE,mc.cores = 3)                           
146
#Extract list of files for rmse and date 1 (19920101), there should be 28 raster images
147
lf_accuracy_raster <- unlist(raster_created_obj$list_raster_name) 
250 148

  
251
#lf_mosaic <- lf_mosaic[1:20]
149
#Plot as quick check
252 150
r1 <- raster(lf_mosaic1[1]) 
253
r2 <- raster(lf_mosaic2[2]) 
151
#r2 <- raster(lf_mosaic2[2]) 
254 152

  
255 153
plot(r1)
256
plot(r2)
154
#plot(r2)
155

  
156
#################################
157
#### Second mosaic tiles for the variable and accuracy metrid
257 158

  
258 159

  
259 160
#methods availbable:use_sine_weights,use_edge,use_linear_weights
......
263 164
for(i in 1:length(day_to_mosaic)){
264 165
  
265 166
  mosaic_method <- "use_edge_weights" #this is distance from edge
266
  out_suffix_str <- paste(day_to_mosaic[i],out_suffix,sep="_")
267
  #undebug(mosaicFiles)
167
  out_suffix_str <- paste(interpolation_method,y_var_name,day_to_mosaic[i],out_suffix,sep="_")
168
  #debug(mosaicFiles)
268 169
  #can also loop through methods!!!
170
  python_bin <- "/usr/bin/" #python gdal bin
269 171
  mosaic_edge_obj_prediction <- mosaicFiles(lf_mosaic1,mosaic_method="use_edge_weights",
270 172
                                        num_cores=num_cores,
271
                                        python_bin=NULL,
173
                                        python_bin=python_bin,
272 174
                                        df_points=NULL,NA_flag=NA_flag_val,
273 175
                                        file_format=file_format,out_suffix=out_suffix_str,
274 176
                                        out_dir=out_dir)
275 177
  
276 178
  mosaic_method <- "use_edge_weights" #this is distance from edge
277
  out_suffix_str <- paste(day_to_mosaic[i],out_suffix,sep="_")
278
  #undebug(mosaicFiles)
179
  out_suffix_str <- paste(interpolation_method,metric_name,day_to_mosaic[i],out_suffix,sep="_")
180

  
181
  #debug(mosaicFiles)
279 182
  #can also loop through methods!!!
280 183
  mosaic_edge_obj_accuracy <- mosaicFiles(lf_accuracy_raster,mosaic_method="use_edge_weights",
281 184
                                        num_cores=num_cores,
282
                                        python_bin=NULL,
185
                                        python_bin=python_bin,
283 186
                                        df_points=NULL,NA_flag=NA_flag_val,
284 187
                                        file_format=file_format,out_suffix=out_suffix_str,
285 188
                                        out_dir=out_dir)
286 189
  
287
  #mosaic_unweighted_obj <- mosaicFiles(lf_mosaic1,mosaic_method="unweighted",
288
  #                                      num_cores=num_cores,
289
  #                                      python_bin=NULL,
290
  #                                      df_points=NULL,NA_flag=NA_flag_val,
291
  #                                      file_format=file_format,out_suffix=out_suffix_str,
292
  #                                      out_dir=out_dir)
293

  
294
  #list_mosaic_obj[[i]] <- list(unweighted=mosaic_unweighted_obj,edge=mosaic_edge_obj)
295
  #list_mosaic_obj[[i]] <- list(prediction=mosaic_edge_obj_accuracy,accuracy=mosaic_edge_obj_accuracy)
296

  
297 190
  list_mosaic_obj[[i]] <- list(prediction=mosaic_edge_obj_prediction,accuracy=mosaic_edge_obj_accuracy)
298 191
}
299 192

  
......
318 211
list_param_plot_mosaic <- list(lf_mosaic=unlist(lf_mean_mosaic),
319 212
                               method=unlist(l_method_mosaic),
320 213
                               out_suffix=unlist(list_out_suffix))
321
#undebug(plot_mosaic)
214

  
215
#plot and produce png movie...
322 216
#plot_mosaic(1,list_param=list_param_plot_mosaic)
323 217
num_cores <- 4
324 218
l_png_files <- mclapply(1:length(unlist(lf_mean_mosaic)),FUN=plot_mosaic,
325 219
                        list_param= list_param_plot_mosaic,
326 220
                        mc.preschedule=FALSE,mc.cores = num_cores)
327 221

  
328
num_cores <- 2
329
l_diff_png_files <- mclapply(1:length(lf1),FUN=plot_diff_raster,list_param= list_param_plot_diff,
330
                        mc.preschedule=FALSE,mc.cores = num_cores)
331 222

  
332 223
###############
333 224

  

Also available in: Unified diff