Project

General

Profile

« Previous | Next » 

Revision 7d545a71

Added by Benoit Parmentier over 8 years ago

more changes assembling station with training and testing for specific day

View differences:

climate/research/oregon/interpolation/global_product_assessment_part1.R
4 4
#Combining tables and figures for individual runs for years and tiles.
5 5
#AUTHOR: Benoit Parmentier 
6 6
#CREATED ON: 05/15/2016  
7
#MODIFIED ON: 05/16/2016            
7
#MODIFIED ON: 05/22/2016            
8 8
#Version: 1
9 9
#PROJECT: Environmental Layers project     
10 10
#COMMENTS: Initial commit, script based on part NASA biodiversity conferenc 
......
158 158
l_dates <- c("19920101","19920102","19920103","19920701","19920702","19920703") #dates to plot and analze
159 159

  
160 160
df_points_extracted_fname <- "/data/project/layers/commons/NEX_data/climateLayers/out/reg4/mosaic/int_mosaics/data_points_extracted.txt"
161
NA_flag_val_mosaic <- -3399999901438340239948148078125514752.000
161
df_points_extracted_fname <- NULL #if null compute on the fly
162
#r_mosaic_fname <- "r_mosaic.RData"
163
r_mosaic_fname <- NULL #if null create a stack from input dir
164

  
165
#NA_flag_val_mosaic <- -3399999901438340239948148078125514752.000
166
NA_flag_val_mosaic <- -32768
162 167
in_dir_list_filename <- NULL #if NULL, use the in_dir directory to search for info
163 168

  
164 169
##################### START SCRIPT #################
......
176 181

  
177 182
###########  ####################
178 183

  
184
##Get the assessment information for every year for the matching region
179 185
if(!is.null(in_dir_list_filename)){
180 186
  in_dir_list <- as.list(read.table(in_dir_list_filename,stringsAsFactors=F)[,1])
181 187
}else{
......
185 191
  #in_dir_shp <- file.path(in_dir_list_all,"shapefiles")
186 192
}
187 193

  
194
## Now get the list of files for assessment of global product
188 195
list_tb_fname <- list.files(path=in_dir_list,"tb_diagnostic_v_NA_.*.txt",full.names=T)
189 196
list_df_fname <- list.files(path=in_dir_list,"df_tile_processed_.*..txt",full.names=T)
190 197
list_summary_metrics_v_fname <- list.files(path=in_dir_list,"summary_metrics_v2_NA_.*.txt",full.names=T)
......
196 203
list_pred_data_month_info_fname <- list.files(path=in_dir_list,"pred_data_month_info.*.txt",full.names=T)
197 204
list_pred_data_day_info_fname <- list.files(path=in_dir_list,"pred_data_day_info.*.txt",full.names=T)
198 205

  
199
### Get data information
206
### Get station data information: create a data.frame with stations info
200 207
data_date <- unlist(lapply(list_data_s_fname, function(x){unlist(strsplit(basename(x),"_"))[5]}))
201 208
df_data_s_fname <- data.frame(data_date,region_name,dataset="data_s",file=list_data_s_fname)
202 209

  
......
207 214
df_data_v_fname$year <- as.character(df_data_v_fname$year_str)
208 215

  
209 216

  
217
############### PART1: Select stations for specific list of dates #############
218
#### Extract corresponding stationsfor given dates and plot stations used
210 219
## Use station from specific year and date?
211 220

  
212 221
list_dates_val <- as.Date(strptime(l_dates,"%Y%m%d"))
213
month_str <- format(list_dates_val, "%b") ## Month, char, abbreviated
214
year_str <- format(list_dates_val, "%Y") ## Year with century
215
day_str <- as.numeric(format(list_dates_val, "%d")) ## numeric month
222
l_dates_month_str <- format(list_dates_val, "%b") ## Month, char, abbreviated
223
l_dates_year_str <- format(list_dates_val, "%Y") ## Year with century
224
l_dates_day_str <- as.numeric(format(list_dates_val, "%d")) ## numeric month
216 225

  
217
for(i in 1:length(unique(year_str))){
218
  df_data_v_fname$data_date==year_str[i]
226
list_year_str <- unique(l_dates_year_str)
227
list_df_stations <- vector("list",length=length(list_year_str))
228

  
229
for(i in 1:length(list_year_str)){
230
  subset(df_data_v_fname,year=list_year_str)
231
  df_data_v_fname$data_date==l_dates_year_str[i]
219 232
  filename_tmp <- as.character(df_data_v_fname$file[9])
220 233
  df_stations_tmp <- read.table(filename_tmp,stringsAsFactors=F,sep=",")
221
  
222 234
}
223 235

  
224 236
df_points <- subset(df_stations,df_stations_tmp$date==l_dates[1])
......
226 238
plot(df_points,add=T)
227 239
coordinates(df_points) <- cbind(df_points$x,df_points$y)
228 240
proj4string(df_points) <- CRS_locs_WGS84
241
## No spatial duplicates
242
df_points_day <- remove.duplicates(df_points)
243

  
244
############### PART2: Select stations by ID to build a time series #############
245
#### Extract corresponding stationsfor given dates and plot stations used
246
## Use station from specific year and date?
229 247

  
230 248
#####################
231 249
#select one station based on id or coordinates and find that in the list of data.frame??
......
268 286
#rbind.fill(mtcars[c("mpg", "wt")], mtcars[c("wt", "cyl")])
269 287
data_stations <- rbind.fill(data_v_subset, data_s_subset)
270 288

  
289
#aggregate(breaks ~ wool + tension, data = warpbreaks, mean)
290
#aggregate(df_points ~ wool + tension, data = warpbreaks, mean)
291

  
271 292
coordinates(data_stations) <- cbind(data_stations$x,data_stations$y)
272 293
proj4string(data_stations) <- CRS_locs_WGS84
273 294

  
274 295
dim(data_stations)
275 296

  
276 297

  
298
############### PART1: Make raster stack and display maps #############
299
#### Extract corresponding raster for given dates and plot stations used
277 300

  
278 301
##Now grab year year 1992 or matching year...maybe put this in a data.frame??
279 302

  
303
if(is.null(r_mosaic=)
280 304
pattern_str <-"*.tif"
281 305
lf_mosaic_list <- list.files(path=in_dir_mosaic,pattern=pattern_str,recursive=F,full.names=T)
282 306
r_mosaic <- stack(lf_mosaic_list)
......
326 350
lf_mosaic_plot_fig <- mclapply(1:length(lf_mosaic_scaled),FUN=plot_raster_mosaic,list_param=list_param_plot_raster_mosaic,mc.preschedule=FALSE,mc.cores = num_cores)                         
327 351

  
328 352
############### PART2: temporal profile #############
329
#### Extract time series
353
#### Extract time series from raster stack
330 354
###
331 355
#-65,-22
332 356

  
333 357
#Use the global output??
334 358

  
335
dim(data_stations)
359
dim(data_stations) #
360
#> dim(data_stations)
361
#[1] 100458     90
362
#This is a lot of replication!!! okay cut it down
363

  
364
##23.09 (on 05/22)
365
#df_points_day_extracted <- extract(r_mosaic,data_stations,df=T)
366
#df_points_day_extracted_fname <- paste0("df_points_day_extracted_fname",".txt") 
367
#write.table(df_points_day_extracted,file=df_points_day_extracted_fname) #5.1 Giga
368
#4.51 (on 05/23)
369

  
370
if(is.null(df_points_extracted_fname)){
371
  
372
  ##10.41 (on 05/22)
373
  df_points_day_extracted <- extract(r_mosaic,df_points_day,df=T)
374
  #17.19 (on 05/23)
375
  df_points_day_extracted_fname <- paste0("df_points_day_extracted_fname2",".txt")
376
  #17.27 (on 05/23)
377
  write.table(df_points_day_extracted,file=df_points_day_extracted_fname,sep=",",row.names = F) 
378
  #17.19 (on 05/23)
379

  
380
}else{
381
  df_points_extracted <- read.table(df_points_extracted_fname,sep=",")
382
}
336 383

  
337 384

  
338
df_points_extracted <- extract(r_mosaic,data_stations,df=T)
339
df_points_extracted_fname <- paste0("df_points_extracted_fname",".txt")
340
write.table(df_points_extracted,file=df_points_extracted_fname) 
385
#df_points_extracted <- extract(r_mosaic,data_stations,df=T)
386
#df_points_extracted_fname <- paste0("df_points_extracted_fname",".txt")
387
#write.table(df_points_extracted,file=df_points_extracted_fname) 
341 388

  
342
df_points <- read.table(df_points_extracted_fname,sep=",") 
343
df_points_tmp <- df_points
344
df_points <- as.data.frame(t(df_points))
345
names(df_points) <- paste0("ID_",1:ncol(df_points))
389
#df_points <- read.table(df_points_extracted_fname,sep=",") 
390
#df_points_tmp <- df_points
391
#df_points <- as.data.frame(t(df_points))
392
#names(df_points) <- paste0("ID_",1:ncol(df_points))
346 393

  
347 394
#df_centroids <- read.table(df_centroids_fname,sep=",")
348 395

  
349
coordinates(df_centroids)<- c("long","lat")
350
proj4string(df_centroids) <- CRS_locs_WGS84
396
#coordinates(df_centroids)<- c("long","lat")
397
#proj4string(df_centroids) <- CRS_locs_WGS84
351 398
## Checking new files:
352 399
in_dir_mosaic <- "/nobackupp6/aguzman4/climateLayers/out/reg4/mosaics2/mosaic"
353 400
#/nobackupp6/aguzman4/climateLayers/out/reg4/mosaics2/mosaic/output_reg4_*/r_m_use_edge_weights_weighted_mean_mask_gam_CAI_dailyTmax_*_reg4_*.tif

Also available in: Unified diff