Revision bfc56b81
Added by Benoit Parmentier about 8 years ago
climate/research/oregon/interpolation/global_product_assessment_part2.R | ||
---|---|---|
19 | 19 |
# |
20 | 20 |
#setfacl -Rmd user:aguzman4:rwx /nobackupp8/bparmen1/output_run10_1500x4500_global_analyses_pred_1992_10052015 |
21 | 21 |
|
22 |
#COMMIT: testing function to generate animation and moving to function script
|
|
22 |
#COMMIT: generating animation for reg6 (Australia and South East Asia)
|
|
23 | 23 |
|
24 | 24 |
################################################################################################# |
25 | 25 |
|
... | ... | |
87 | 87 |
#Product assessment |
88 | 88 |
function_product_assessment_part1_functions <- "global_product_assessment_part1_functions_09192016b.R" |
89 | 89 |
source(file.path(script_path,function_product_assessment_part1_functions)) #source all functions used in this script |
90 |
function_product_assessment_part2_functions <- "global_product_assessment_part2_functions_10102016.R" |
|
90 |
function_product_assessment_part2_functions <- "global_product_assessment_part2_functions_10102016b.R"
|
|
91 | 91 |
source(file.path(script_path,function_product_assessment_part2_functions)) #source all functions used in this script |
92 | 92 |
|
93 | 93 |
############################### |
... | ... | |
127 | 127 |
#master directory containing the definition of tile size and tiles predicted |
128 | 128 |
#in_dir <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg5/assessment" |
129 | 129 |
#in_dir_mosaic <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg5/mosaic/mosaic" |
130 |
in_dir <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg1/assessment"
|
|
131 |
in_dir_mosaic <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg1/mosaics/mosaic"
|
|
130 |
in_dir <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg6/assessment"
|
|
131 |
in_dir_mosaic <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg6/mosaics/mosaic" #predicted mosaic
|
|
132 | 132 |
|
133 |
region_name <- c("reg1") #param 6, arg 3
|
|
134 |
out_suffix <- "_global_assessment_reg1_10032016"
|
|
133 |
region_name <- c("reg6") #param 6, arg 3
|
|
134 |
out_suffix <- "global_assessment_reg6_10102016"
|
|
135 | 135 |
|
136 | 136 |
create_out_dir_param <- TRUE #param 9, arg 6 |
137 | 137 |
|
138 | 138 |
|
139 |
out_dir <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg1/assessment"
|
|
139 |
out_dir <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg6/assessment"
|
|
140 | 140 |
|
141 | 141 |
#run_figure_by_year <- TRUE # param 10, arg 7 |
142 | 142 |
|
... | ... | |
156 | 156 |
|
157 | 157 |
#infile_mask <- "/nobackupp8/bparmen1/NEX_data/regions_input_files/r_mask_LST_reg4.tif" |
158 | 158 |
#infile_mask <- "/data/project/layers/commons/NEX_data/regions_input_files/r_mask_LST_reg5.tif" |
159 |
infile_mask <- "/data/project/layers/commons/NEX_data/regions_input_files/r_mask_LST_reg1.tif"
|
|
159 |
infile_mask <- "/data/project/layers/commons/NEX_data/regions_input_files/r_mask_LST_reg6.tif"
|
|
160 | 160 |
|
161 | 161 |
#run_figure_by_year <- TRUE # param 10, arg 7 |
162 | 162 |
list_year_predicted <- "1984,2014" |
... | ... | |
164 | 164 |
#if scaling is null then perform no scaling!! |
165 | 165 |
|
166 | 166 |
#df_centroids_fname <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg4/mosaic/output_reg5_1999/df_centroids_19990701_reg5_1999.txt" |
167 |
df_centroids_fname <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg1/mosaic/output_reg1_1984/df_centroids_19840101_reg1_1984.txt"
|
|
167 |
df_centroids_fname <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg6/mosaic/output_reg6_1984/df_centroids_19840101_reg6_1984.txt"
|
|
168 | 168 |
#/nobackupp6/aguzman4/climateLayers/out/reg1/assessment//output_reg1_1984/df_assessment_files_reg1_1984_reg1_1984.txt |
169 | 169 |
|
170 | 170 |
#dates to plot and analyze |
... | ... | |
180 | 180 |
NA_flag_val_mosaic <- -32768 |
181 | 181 |
in_dir_list_filename <- NULL #if NULL, use the in_dir directory to search for info |
182 | 182 |
countries_shp <-"/data/project/layers/commons/NEX_data/countries.shp" #Atlas |
183 |
|
|
183 |
lf_raster <- NULL #list of raster to consider |
|
184 | 184 |
|
185 | 185 |
##################### START SCRIPT ################# |
186 | 186 |
|
... | ... | |
215 | 215 |
|
216 | 216 |
NAvalue(r_stack) |
217 | 217 |
plot(r_stack,y=6,zlim=c(-10000,10000)) #this is not rescaled |
218 |
plot(r_stack,zlim=c(-50,50),col=matlab.like(255)) |
|
218 |
#plot(r_stack,zlim=c(-50,50),col=matlab.like(255))
|
|
219 | 219 |
|
220 | 220 |
#plot(r_mosaic_scaled,y=6,zlim=c(-50,50)) |
221 | 221 |
#plot(r_mosaic_scaled,zlim=c(-50,50),col=matlab.like(255)) |
... | ... | |
281 | 281 |
##################################### PART 5 ###### |
282 | 282 |
##### Plotting specific days for the mosaics |
283 | 283 |
|
284 |
function_product_assessment_part2_functions <- "global_product_assessment_part2_functions_10102016.R" |
|
285 |
source(file.path(script_path,function_product_assessment_part2_functions)) #source all functions used in this script |
|
284 |
#function_product_assessment_part2_functions <- "global_product_assessment_part2_functions_10102016.R"
|
|
285 |
#source(file.path(script_path,function_product_assessment_part2_functions)) #source all functions used in this script
|
|
286 | 286 |
|
287 | 287 |
#NA_flag_val_mosaic <- -3399999901438340239948148078125514752.000 |
288 | 288 |
r_stack_subset <- subset(r_stack,1:11) |
... | ... | |
298 | 298 |
lf_mosaic_plot_fig <- lapply(1:2, |
299 | 299 |
FUN=plot_raster_mosaic, |
300 | 300 |
list_param=list_param_plot_raster_mosaic) |
301 |
|
|
302 |
### Now run for the full time series |
|
303 |
#13.26 Western time: start |
|
301 | 304 |
l_dates <- list_dates_produced_date_val |
302 | 305 |
r_stack_subset <- r_stack |
306 |
zlim_val <- NULL |
|
307 |
list_param_plot_raster_mosaic <- list(l_dates,r_stack_subset,NA_flag_val,out_dir,out_suffix, |
|
308 |
region_name,variable_name, zlim_val) |
|
309 |
names(list_param_plot_raster_mosaic) <- c("l_dates","r_mosaiced_scaled","NA_flag_val_mosaic","out_dir","out_suffix", |
|
310 |
"region_name","variable_name","zlim_val") |
|
303 | 311 |
|
312 |
lf_mosaic_plot_fig <- mclapply(1:length(l_dates[1:11]), |
|
313 |
FUN=plot_raster_mosaic, |
|
314 |
list_param=list_param_plot_raster_mosaic, |
|
315 |
mc.preschedule=FALSE, |
|
316 |
mc.cores = num_cores) |
|
317 |
##start at 14.12 |
|
318 |
##finished at 16.47 |
|
304 | 319 |
lf_mosaic_plot_fig <- mclapply(1:length(l_dates), |
305 | 320 |
FUN=plot_raster_mosaic, |
306 | 321 |
list_param=list_param_plot_raster_mosaic, |
... | ... | |
313 | 328 |
zlim_val_str <- paste(zlim_val,sep="_",collapse="_") |
314 | 329 |
out_suffix_movie <- paste(zlim_val_str,"_",out_suffix,sep="") |
315 | 330 |
} |
316 |
r_stack_subset <- subset(r_stack,1:11) |
|
317 |
l_dates <- list_dates_produced_date_val[1:11] |
|
331 |
#r_stack_subset <- subset(r_stack,1:11) |
|
332 |
#l_dates <- list_dates_produced_date_val[1:11] |
|
333 |
|
|
334 |
filenames_figures_mosaic_test <- "list_figures_animation_test_reg6.txt" |
|
335 |
|
|
336 |
write.table(unlist(lf_mosaic_plot_fig[1:11]),filenames_figures_mosaic_test,row.names = F,col.names = F,quote = F) |
|
318 | 337 |
|
319 |
filenames_figures_mosaic <- "test_animation_reg1.txt" |
|
338 |
filenames_figures_mosaic <- paste0("list_figures_animation_",out_suffix_movie,".txt") |
|
339 |
|
|
340 |
write.table(unlist(lf_mosaic_plot_fig),filenames_figures_mosaic,row.names = F,col.names = F,quote = F) |
|
320 | 341 |
|
321 |
write.table(unlist(lf_mosaic_plot_fig[1:11]),filenames_figures_mosaic,row.names = F,col.names = F,quote = F) |
|
322 | 342 |
#now generate movie with imageMagick |
323 | 343 |
frame_speed <- 60 |
324 | 344 |
animation_format <- ".gif" |
325 | 345 |
out_suffix_str <- out_suffix |
326 |
|
|
346 |
#started |
|
327 | 347 |
#debug(generate_animation_from_figures_fun) |
328 |
generate_animation_from_figures_fun(filenames_figures= filenames_figures_mosaic, |
|
348 |
generate_animation_from_figures_fun(filenames_figures= filenames_figures_mosaic_test,
|
|
329 | 349 |
frame_speed=frame_speed, |
330 | 350 |
format_file=animation_format, |
331 | 351 |
out_suffix=out_suffix_str, |
332 | 352 |
out_dir=out_dir, |
333 |
out_filename_figure_animation=NULL)
|
|
334 |
|
|
335 |
generate_animation_from_figures_fun(filenames_figures= filenames_figures_mosaic,
|
|
353 |
out_filename_figure_animation="test_reg6_animation.gif")
|
|
354 |
|
|
355 |
generate_animation_from_figures_fun(filenames_figures= unlist(lf_mosaic_plot_fig[1:11]),
|
|
336 | 356 |
frame_speed=frame_speed, |
337 | 357 |
format_file=animation_format, |
338 | 358 |
out_suffix=out_suffix_str, |
339 | 359 |
out_dir=out_dir, |
340 |
out_filename_figure_animation="test.gif") |
|
341 |
|
|
342 |
|
|
343 |
generate_animation_from_figures_fun(filenames_figures= unlist(lf_mosaic_plot_fig[1:11]), |
|
360 |
out_filename_figure_animation="test2_reg6_animation.gif") |
|
361 |
#started 17.36 Western time on Oct 10 and 18.18 |
|
362 |
generate_animation_from_figures_fun(filenames_figures= filenames_figures_mosaic, |
|
344 | 363 |
frame_speed=frame_speed, |
345 | 364 |
format_file=animation_format, |
346 |
out_suffix=out_suffix_str,
|
|
365 |
out_suffix=out_suffix_movie,
|
|
347 | 366 |
out_dir=out_dir, |
348 |
out_filename_figure_animation="test2.gif") |
|
367 |
out_filename_figure_animation=NULL) |
|
368 |
|
|
349 | 369 |
|
370 |
############ Now accuracy |
|
350 | 371 |
#### PLOT ACCURACY METRICS: First test #### |
351 | 372 |
##this will be cleaned up later: |
352 | 373 |
|
353 |
#dir_ac_mosaics <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg4/mosaic/output_reg4_1999" |
|
354 |
lf_tmp <-list.files(path=dir_ac_mosaics,pattern="r_m_use_edge_weights_weighted_mean_mask_gam_CAI_.*.ac.*._reg4_1999.tif") |
|
374 |
pattern_str <-"*.tif" |
|
375 |
in_dir_mosaic <- in_dir_mosaic_RMSE |
|
376 |
lf_raster_rmse <- list.files(path=in_dir_mosaic,pattern=pattern_str,recursive=F,full.names=T) |
|
377 |
r_stack <- stack(lf_raster,quick=T) #this is very fast now with the quick option! |
|
378 |
#save(r_mosaic,file="r_mosaic.RData") |
|
355 | 379 |
|
356 |
r_mosaiced_ac <- stack(lf_tmp) |
|
357 |
l_dates <- unlist(lapply(1:length(lf_tmp),FUN=extract_date,x=basename(lf_tmp),item_no=14)) |
|
358 |
variable_name |
|
359 |
zlim_val <- NULL |
|
360 |
list_param_plot_raster_mosaic <- list(l_dates,r_mosaiced_ac,NA_flag_val_mosaic,out_dir,out_suffix, |
|
361 |
region_name,variable_name, zlim_val) |
|
362 |
names(list_param_plot_raster_mosaic) <- c("l_dates","r_mosaiced_scaled","NA_flag_val_mosaic","out_dir","out_suffix", |
|
363 |
"region_name","variable_name","zlim_val") |
|
364 |
#debug(plot_raster_mosaic) |
|
365 |
plot_raster_mosaic(1,list_param_plot_raster_mosaic) |
|
366 |
lf_mosaic_plot_fig <- mclapply(1:length(lf_tmp), |
|
367 |
FUN=plot_raster_mosaic, |
|
368 |
list_param=list_param_plot_raster_mosaic, |
|
369 |
mc.preschedule=FALSE, |
|
370 |
mc.cores = num_cores) |
|
380 |
NAvalue(r_stack) |
|
381 |
plot(r_stack,y=6,zlim=c(-10000,10000)) #this is not rescaled |
|
382 |
#plot(r_stack,zlim=c(-50,50),col=matlab.like(255)) |
|
371 | 383 |
|
372 |
#### Now plot kriged residuals from mosaiced surfaces |
|
384 |
#plot(r_mosaic_scaled,y=6,zlim=c(-50,50)) |
|
385 |
#plot(r_mosaic_scaled,zlim=c(-50,50),col=matlab.like(255)) |
|
373 | 386 |
|
374 |
lf_tmp_res <-list.files(path=dir_ac_mosaics,pattern="r_m_use_edge_weights_weighted_mean_mask_gam_CAI_.*.residuals.*._reg4_1999.tif",full.names=T) |
|
387 |
#debug(extract_date) |
|
388 |
#test <- extract_date(6431,lf_mosaic_list,12) #extract item number 12 from the name of files to get the data |
|
389 |
#list_dates_produced <- unlist(mclapply(1:length(lf_raster),FUN=extract_date,x=lf_raster,item_no=13,mc.preschedule=FALSE,mc.cores = num_cores)) |
|
390 |
lf_mosaic_list <- lf_raster |
|
391 |
list_dates_produced <- mclapply(1:2, |
|
392 |
FUN=extract_date, |
|
393 |
x=lf_mosaic_list, |
|
394 |
item_no=13, |
|
395 |
mc.preschedule=FALSE, |
|
396 |
mc.cores = 2) |
|
397 |
item_no <-13 |
|
398 |
list_dates_produced <- unlist(mclapply(1:length(lf_raster), |
|
399 |
FUN=extract_date, |
|
400 |
x=lf_raster, |
|
401 |
item_no=item_no, |
|
402 |
mc.preschedule=FALSE, |
|
403 |
mc.cores = num_cores)) |
|
375 | 404 |
|
376 |
l_dates <- unlist(lapply(1:length(lf_tmp_res),FUN=extract_date,x=basename(lf_tmp),item_no=14)) |
|
377 |
variable_name |
|
378 |
zlim_val <- NULL |
|
379 |
r_mosaiced_res <- stack(lf_tmp_res) |
|
380 |
list_param_plot_raster_mosaic <- list(l_dates,r_mosaiced_res,NA_flag_val_mosaic,out_dir,out_suffix, |
|
381 |
region_name,variable_name, zlim_val) |
|
382 |
names(list_param_plot_raster_mosaic) <- c("l_dates","r_mosaiced_scaled","NA_flag_val_mosaic","out_dir","out_suffix", |
|
383 |
"region_name","variable_name","zlim_val") |
|
384 |
#debug(plot_raster_mosaic) |
|
385 |
plot_raster_mosaic(1,list_param_plot_raster_mosaic) |
|
386 |
lf_mosaic_plot_fig_res <- mclapply(1:length(lf_tmp_res), |
|
387 |
FUN=plot_raster_mosaic, |
|
388 |
list_param=list_param_plot_raster_mosaic, |
|
389 |
mc.preschedule=FALSE, |
|
390 |
mc.cores = num_cores) |
|
405 |
list_dates_produced_date_val <- as.Date(strptime(list_dates_produced,"%Y%m%d")) |
|
406 |
month_str <- format(list_dates_produced_date_val, "%b") ## Month, char, abbreviated |
|
407 |
year_str <- format(list_dates_produced_date_val, "%Y") ## Year with century |
|
408 |
day_str <- as.numeric(format(list_dates_produced_date_val, "%d")) ## numeric month |
|
391 | 409 |
|
392 |
### New plot of residuals surface with zlim |
|
393 |
zlim_val <- c(-60,60) |
|
394 |
#r_mosaiced_res <- stack(lf_tmp_res) |
|
395 |
list_param_plot_raster_mosaic <- list(l_dates,r_mosaiced_res,NA_flag_val_mosaic,out_dir,out_suffix, |
|
396 |
region_name,variable_name, zlim_val) |
|
397 |
names(list_param_plot_raster_mosaic) <- c("l_dates","r_mosaiced_scaled","NA_flag_val_mosaic","out_dir","out_suffix", |
|
398 |
"region_name","variable_name","zlim_val") |
|
399 |
#debug(plot_raster_mosaic) |
|
400 |
#plot_raster_mosaic(1,list_param_plot_raster_mosaic) |
|
401 |
lf_mosaic_plot_fig_res <- mclapply(1:length(lf_tmp_res), |
|
402 |
FUN=plot_raster_mosaic, |
|
403 |
list_param=list_param_plot_raster_mosaic, |
|
404 |
mc.preschedule=FALSE, |
|
405 |
mc.cores = num_cores) |
|
410 |
df_raster <- data.frame(lf=basename(lf_raster), |
|
411 |
date=list_dates_produced_date_val, |
|
412 |
month_str=month_str, |
|
413 |
year=year_str, |
|
414 |
day=day_str, |
|
415 |
dir=dirname(lf_mosaic_list)) |
|
406 | 416 |
|
417 |
df_raster_fname <- file.path(out_dir,paste0("df_raster_",out_suffix,".txt")) |
|
418 |
write.table(df_raster,file= df_raster_fname,sep=",",row.names = F) |
|
407 | 419 |
|
408 | 420 |
############################ END OF SCRIPT ################################## |
Also available in: Unified diff
more changes and generating animation for reg6 (Australia and South East Asia)