Revision 7d545a71
Added by Benoit Parmentier over 8 years ago
climate/research/oregon/interpolation/global_product_assessment_part1.R | ||
---|---|---|
4 | 4 |
#Combining tables and figures for individual runs for years and tiles. |
5 | 5 |
#AUTHOR: Benoit Parmentier |
6 | 6 |
#CREATED ON: 05/15/2016 |
7 |
#MODIFIED ON: 05/16/2016
|
|
7 |
#MODIFIED ON: 05/22/2016
|
|
8 | 8 |
#Version: 1 |
9 | 9 |
#PROJECT: Environmental Layers project |
10 | 10 |
#COMMENTS: Initial commit, script based on part NASA biodiversity conferenc |
... | ... | |
158 | 158 |
l_dates <- c("19920101","19920102","19920103","19920701","19920702","19920703") #dates to plot and analze |
159 | 159 |
|
160 | 160 |
df_points_extracted_fname <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg4/mosaic/int_mosaics/data_points_extracted.txt" |
161 |
NA_flag_val_mosaic <- -3399999901438340239948148078125514752.000 |
|
161 |
df_points_extracted_fname <- NULL #if null compute on the fly |
|
162 |
#r_mosaic_fname <- "r_mosaic.RData" |
|
163 |
r_mosaic_fname <- NULL #if null create a stack from input dir |
|
164 |
|
|
165 |
#NA_flag_val_mosaic <- -3399999901438340239948148078125514752.000 |
|
166 |
NA_flag_val_mosaic <- -32768 |
|
162 | 167 |
in_dir_list_filename <- NULL #if NULL, use the in_dir directory to search for info |
163 | 168 |
|
164 | 169 |
##################### START SCRIPT ################# |
... | ... | |
176 | 181 |
|
177 | 182 |
########### #################### |
178 | 183 |
|
184 |
##Get the assessment information for every year for the matching region |
|
179 | 185 |
if(!is.null(in_dir_list_filename)){ |
180 | 186 |
in_dir_list <- as.list(read.table(in_dir_list_filename,stringsAsFactors=F)[,1]) |
181 | 187 |
}else{ |
... | ... | |
185 | 191 |
#in_dir_shp <- file.path(in_dir_list_all,"shapefiles") |
186 | 192 |
} |
187 | 193 |
|
194 |
## Now get the list of files for assessment of global product |
|
188 | 195 |
list_tb_fname <- list.files(path=in_dir_list,"tb_diagnostic_v_NA_.*.txt",full.names=T) |
189 | 196 |
list_df_fname <- list.files(path=in_dir_list,"df_tile_processed_.*..txt",full.names=T) |
190 | 197 |
list_summary_metrics_v_fname <- list.files(path=in_dir_list,"summary_metrics_v2_NA_.*.txt",full.names=T) |
... | ... | |
196 | 203 |
list_pred_data_month_info_fname <- list.files(path=in_dir_list,"pred_data_month_info.*.txt",full.names=T) |
197 | 204 |
list_pred_data_day_info_fname <- list.files(path=in_dir_list,"pred_data_day_info.*.txt",full.names=T) |
198 | 205 |
|
199 |
### Get data information
|
|
206 |
### Get station data information: create a data.frame with stations info
|
|
200 | 207 |
data_date <- unlist(lapply(list_data_s_fname, function(x){unlist(strsplit(basename(x),"_"))[5]})) |
201 | 208 |
df_data_s_fname <- data.frame(data_date,region_name,dataset="data_s",file=list_data_s_fname) |
202 | 209 |
|
... | ... | |
207 | 214 |
df_data_v_fname$year <- as.character(df_data_v_fname$year_str) |
208 | 215 |
|
209 | 216 |
|
217 |
############### PART1: Select stations for specific list of dates ############# |
|
218 |
#### Extract corresponding stationsfor given dates and plot stations used |
|
210 | 219 |
## Use station from specific year and date? |
211 | 220 |
|
212 | 221 |
list_dates_val <- as.Date(strptime(l_dates,"%Y%m%d")) |
213 |
month_str <- format(list_dates_val, "%b") ## Month, char, abbreviated |
|
214 |
year_str <- format(list_dates_val, "%Y") ## Year with century |
|
215 |
day_str <- as.numeric(format(list_dates_val, "%d")) ## numeric month |
|
222 |
l_dates_month_str <- format(list_dates_val, "%b") ## Month, char, abbreviated
|
|
223 |
l_dates_year_str <- format(list_dates_val, "%Y") ## Year with century
|
|
224 |
l_dates_day_str <- as.numeric(format(list_dates_val, "%d")) ## numeric month
|
|
216 | 225 |
|
217 |
for(i in 1:length(unique(year_str))){ |
|
218 |
df_data_v_fname$data_date==year_str[i] |
|
226 |
list_year_str <- unique(l_dates_year_str) |
|
227 |
list_df_stations <- vector("list",length=length(list_year_str)) |
|
228 |
|
|
229 |
for(i in 1:length(list_year_str)){ |
|
230 |
subset(df_data_v_fname,year=list_year_str) |
|
231 |
df_data_v_fname$data_date==l_dates_year_str[i] |
|
219 | 232 |
filename_tmp <- as.character(df_data_v_fname$file[9]) |
220 | 233 |
df_stations_tmp <- read.table(filename_tmp,stringsAsFactors=F,sep=",") |
221 |
|
|
222 | 234 |
} |
223 | 235 |
|
224 | 236 |
df_points <- subset(df_stations,df_stations_tmp$date==l_dates[1]) |
... | ... | |
226 | 238 |
plot(df_points,add=T) |
227 | 239 |
coordinates(df_points) <- cbind(df_points$x,df_points$y) |
228 | 240 |
proj4string(df_points) <- CRS_locs_WGS84 |
241 |
## No spatial duplicates |
|
242 |
df_points_day <- remove.duplicates(df_points) |
|
243 |
|
|
244 |
############### PART2: Select stations by ID to build a time series ############# |
|
245 |
#### Extract corresponding stationsfor given dates and plot stations used |
|
246 |
## Use station from specific year and date? |
|
229 | 247 |
|
230 | 248 |
##################### |
231 | 249 |
#select one station based on id or coordinates and find that in the list of data.frame?? |
... | ... | |
268 | 286 |
#rbind.fill(mtcars[c("mpg", "wt")], mtcars[c("wt", "cyl")]) |
269 | 287 |
data_stations <- rbind.fill(data_v_subset, data_s_subset) |
270 | 288 |
|
289 |
#aggregate(breaks ~ wool + tension, data = warpbreaks, mean) |
|
290 |
#aggregate(df_points ~ wool + tension, data = warpbreaks, mean) |
|
291 |
|
|
271 | 292 |
coordinates(data_stations) <- cbind(data_stations$x,data_stations$y) |
272 | 293 |
proj4string(data_stations) <- CRS_locs_WGS84 |
273 | 294 |
|
274 | 295 |
dim(data_stations) |
275 | 296 |
|
276 | 297 |
|
298 |
############### PART1: Make raster stack and display maps ############# |
|
299 |
#### Extract corresponding raster for given dates and plot stations used |
|
277 | 300 |
|
278 | 301 |
##Now grab year year 1992 or matching year...maybe put this in a data.frame?? |
279 | 302 |
|
303 |
if(is.null(r_mosaic=) |
|
280 | 304 |
pattern_str <-"*.tif" |
281 | 305 |
lf_mosaic_list <- list.files(path=in_dir_mosaic,pattern=pattern_str,recursive=F,full.names=T) |
282 | 306 |
r_mosaic <- stack(lf_mosaic_list) |
... | ... | |
326 | 350 |
lf_mosaic_plot_fig <- mclapply(1:length(lf_mosaic_scaled),FUN=plot_raster_mosaic,list_param=list_param_plot_raster_mosaic,mc.preschedule=FALSE,mc.cores = num_cores) |
327 | 351 |
|
328 | 352 |
############### PART2: temporal profile ############# |
329 |
#### Extract time series |
|
353 |
#### Extract time series from raster stack
|
|
330 | 354 |
### |
331 | 355 |
#-65,-22 |
332 | 356 |
|
333 | 357 |
#Use the global output?? |
334 | 358 |
|
335 |
dim(data_stations) |
|
359 |
dim(data_stations) # |
|
360 |
#> dim(data_stations) |
|
361 |
#[1] 100458 90 |
|
362 |
#This is a lot of replication!!! okay cut it down |
|
363 |
|
|
364 |
##23.09 (on 05/22) |
|
365 |
#df_points_day_extracted <- extract(r_mosaic,data_stations,df=T) |
|
366 |
#df_points_day_extracted_fname <- paste0("df_points_day_extracted_fname",".txt") |
|
367 |
#write.table(df_points_day_extracted,file=df_points_day_extracted_fname) #5.1 Giga |
|
368 |
#4.51 (on 05/23) |
|
369 |
|
|
370 |
if(is.null(df_points_extracted_fname)){ |
|
371 |
|
|
372 |
##10.41 (on 05/22) |
|
373 |
df_points_day_extracted <- extract(r_mosaic,df_points_day,df=T) |
|
374 |
#17.19 (on 05/23) |
|
375 |
df_points_day_extracted_fname <- paste0("df_points_day_extracted_fname2",".txt") |
|
376 |
#17.27 (on 05/23) |
|
377 |
write.table(df_points_day_extracted,file=df_points_day_extracted_fname,sep=",",row.names = F) |
|
378 |
#17.19 (on 05/23) |
|
379 |
|
|
380 |
}else{ |
|
381 |
df_points_extracted <- read.table(df_points_extracted_fname,sep=",") |
|
382 |
} |
|
336 | 383 |
|
337 | 384 |
|
338 |
df_points_extracted <- extract(r_mosaic,data_stations,df=T) |
|
339 |
df_points_extracted_fname <- paste0("df_points_extracted_fname",".txt") |
|
340 |
write.table(df_points_extracted,file=df_points_extracted_fname) |
|
385 |
#df_points_extracted <- extract(r_mosaic,data_stations,df=T)
|
|
386 |
#df_points_extracted_fname <- paste0("df_points_extracted_fname",".txt")
|
|
387 |
#write.table(df_points_extracted,file=df_points_extracted_fname)
|
|
341 | 388 |
|
342 |
df_points <- read.table(df_points_extracted_fname,sep=",") |
|
343 |
df_points_tmp <- df_points |
|
344 |
df_points <- as.data.frame(t(df_points)) |
|
345 |
names(df_points) <- paste0("ID_",1:ncol(df_points)) |
|
389 |
#df_points <- read.table(df_points_extracted_fname,sep=",")
|
|
390 |
#df_points_tmp <- df_points
|
|
391 |
#df_points <- as.data.frame(t(df_points))
|
|
392 |
#names(df_points) <- paste0("ID_",1:ncol(df_points))
|
|
346 | 393 |
|
347 | 394 |
#df_centroids <- read.table(df_centroids_fname,sep=",") |
348 | 395 |
|
349 |
coordinates(df_centroids)<- c("long","lat") |
|
350 |
proj4string(df_centroids) <- CRS_locs_WGS84 |
|
396 |
#coordinates(df_centroids)<- c("long","lat")
|
|
397 |
#proj4string(df_centroids) <- CRS_locs_WGS84
|
|
351 | 398 |
## Checking new files: |
352 | 399 |
in_dir_mosaic <- "/nobackupp6/aguzman4/climateLayers/out/reg4/mosaics2/mosaic" |
353 | 400 |
#/nobackupp6/aguzman4/climateLayers/out/reg4/mosaics2/mosaic/output_reg4_*/r_m_use_edge_weights_weighted_mean_mask_gam_CAI_dailyTmax_*_reg4_*.tif |
Also available in: Unified diff
more changes assembling station with training and testing for specific day