1
|
################## Kriging Multisampling method assessment #######################################
|
2
|
########################### Kriging and Cokriging ###############################################
|
3
|
#This script interpolates station values for the Oregon case study using Univeral Kriging. #
|
4
|
#The script uses LST monthly averages as input variables and loads the station data #
|
5
|
#from a shape file with projection information. #
|
6
|
#Note that this program: #
|
7
|
#1)assumes that the shape file is in the current working. #
|
8
|
#2)relevant variables were extracted from raster images before performing the regressions #
|
9
|
# and stored shapefile #
|
10
|
#This scripts predicts tmax using autokrige, gstat and LST derived from MOD11A1. #
|
11
|
#also included and assessed using the RMSE,MAE,ME and R2 from validation dataset. #
|
12
|
#The dates must be provided as a textfile. Method is assesed using multisampling with variation #
|
13
|
#of validation sample with different hold out proportions. #
|
14
|
#AUTHOR: Benoit Parmentier #
|
15
|
#DATE: 08/31/2012 #
|
16
|
#PROJECT: NCEAS INPLANT: Environment and Organisms --TASK#364-- #
|
17
|
##################################################################################################
|
18
|
|
19
|
###Loading R library and packages
|
20
|
#library(gtools) # loading some useful tools
|
21
|
library(mgcv) # GAM package by Wood 2006 (version 2012)
|
22
|
library(sp) # Spatial pacakge with class definition by Bivand et al. 2008
|
23
|
library(spdep) # Spatial pacakge with methods and spatial stat. by Bivand et al. 2012
|
24
|
library(rgdal) # GDAL wrapper for R, spatial utilities (Keitt et al. 2012)
|
25
|
library(gstat) # Kriging and co-kriging by Pebesma et al. 2004
|
26
|
library(automap) # Automated Kriging based on gstat module by Hiemstra et al. 2008
|
27
|
library(spgwr)
|
28
|
library(maptools)
|
29
|
library(graphics)
|
30
|
library(parallel) # Urbanek S. and Ripley B., package for multi cores & parallel processing
|
31
|
library(raster)
|
32
|
library(rasterVis)
|
33
|
library(fields) # May be used later...
|
34
|
library(reshape)
|
35
|
### Parameters and argument
|
36
|
|
37
|
infile1<- "ghcn_or_tmax_covariates_06262012_OR83M.shp" #GHCN shapefile containing variables for modeling 2010
|
38
|
#infile2<-"list_10_dates_04212012.txt" #List of 10 dates for the regression
|
39
|
infile2<-"list_2_dates_04212012.txt"
|
40
|
#infile2<-"list_365_dates_04212012.txt"
|
41
|
infile3<-"LST_dates_var_names.txt" #LST dates name
|
42
|
infile4<-"models_interpolation_05142012.txt" #Interpolation model names
|
43
|
infile5<-"mean_day244_rescaled.rst" #Raster or grid for the locations of predictions
|
44
|
#infile6<-"lst_climatology.txt"
|
45
|
infile6<-"LST_files_monthly_climatology.txt"
|
46
|
inlistf<-"list_files_05032012.txt" #Stack of images containing the Covariates
|
47
|
|
48
|
#path<-"/home/parmentier/Data/IPLANT_project/data_Oregon_stations_07192012_GAM"
|
49
|
path<-"/home/parmentier/Data/IPLANT_project/data_Oregon_stations_07152012" #Jupiter LOCATION on Atlas for kriging
|
50
|
|
51
|
#Station location of the study area
|
52
|
#stat_loc<-read.table(paste(path,"/","location_study_area_OR_0602012.txt",sep=""),sep=",", header=TRUE)
|
53
|
#GHCN Database for 1980-2010 for study area (OR)
|
54
|
#data3<-read.table(paste(path,"/","ghcn_data_TMAXy1980_2010_OR_0602012.txt",sep=""),sep=",", header=TRUE) #Not needing at this stage...
|
55
|
|
56
|
nmodels<-9 #number of models running
|
57
|
y_var_name<-"dailyTmax" #variable value being modeled...("value" in the GHCND database)
|
58
|
predval<-1 # if set to 1, full interpolation raster produced for the study area
|
59
|
prederr<-0 # if set to 0, no uncertain error (e.g. standard error or kriging std dev) is produced
|
60
|
prop<-0.3 #Proportion of testing retained for validation
|
61
|
#prop<-0.25
|
62
|
seed_number<- 100 #Seed number for random sampling
|
63
|
out_prefix<-"_08312012_365d_Kriging_multi_samp3" #User defined output prefix
|
64
|
setwd(path)
|
65
|
|
66
|
nb_sample<-15
|
67
|
prop_min<-0.1
|
68
|
prop_max<-0.7
|
69
|
step<-0.1
|
70
|
|
71
|
#source("fusion_function_07192012.R")
|
72
|
source("KrigingUK_function_multisampling_08312012.R")
|
73
|
############ START OF THE SCRIPT ##################
|
74
|
|
75
|
###Reading the station data and setting up for models' comparison
|
76
|
filename<-sub(".shp","",infile1) #Removing the extension from file.
|
77
|
ghcn<-readOGR(".", filename) #reading shapefile
|
78
|
|
79
|
CRS<-proj4string(ghcn) #Storing projection information (ellipsoid, datum,etc.)
|
80
|
|
81
|
mean_LST<- readGDAL(infile5) #Reading the whole raster in memory. This provides a grid for kriging
|
82
|
proj4string(mean_LST)<-CRS #Assigning coordinate information to prediction grid.
|
83
|
|
84
|
ghcn <- transform(ghcn,Northness = cos(ASPECT*pi/180)) #Adding a variable to the dataframe
|
85
|
ghcn <- transform(ghcn,Eastness = sin(ASPECT*pi/180)) #adding variable to the dataframe.
|
86
|
ghcn <- transform(ghcn,Northness_w = sin(slope*pi/180)*cos(ASPECT*pi/180)) #Adding a variable to the dataframe
|
87
|
ghcn <- transform(ghcn,Eastness_w = sin(slope*pi/180)*sin(ASPECT*pi/180)) #adding variable to the dataframe.
|
88
|
|
89
|
#Remove NA for LC and CANHEIGHT
|
90
|
ghcn$LC1[is.na(ghcn$LC1)]<-0
|
91
|
ghcn$LC3[is.na(ghcn$LC3)]<-0
|
92
|
ghcn$CANHEIGHT[is.na(ghcn$CANHEIGHT)]<-0
|
93
|
|
94
|
dates <-readLines(paste(path,"/",infile2, sep=""))
|
95
|
LST_dates <-readLines(paste(path,"/",infile3, sep=""))
|
96
|
models <-readLines(paste(path,"/",infile4, sep=""))
|
97
|
|
98
|
##Extracting the variables values from the raster files
|
99
|
|
100
|
lines<-read.table(paste(path,"/",inlistf,sep=""), sep=" ") #Column 1 contains the names of raster files
|
101
|
inlistvar<-lines[,1]
|
102
|
inlistvar<-paste(path,"/",as.character(inlistvar),sep="")
|
103
|
covar_names<-as.character(lines[,2]) #Column two contains short names for covaraites
|
104
|
|
105
|
s_raster<- stack(inlistvar) #Creating a stack of raster images from the list of variables.
|
106
|
layerNames(s_raster)<-covar_names #Assigning names to the raster layers
|
107
|
projection(s_raster)<-CRS
|
108
|
|
109
|
#stat_val<- extract(s_raster, ghcn3) #Extracting values from the raster stack for every point location in coords data frame.
|
110
|
pos<-match("ASPECT",layerNames(s_raster)) #Find column with name "value"
|
111
|
r1<-raster(s_raster,layer=pos) #Select layer from stack
|
112
|
pos<-match("slope",layerNames(s_raster)) #Find column with name "value"
|
113
|
r2<-raster(s_raster,layer=pos) #Select layer from stack
|
114
|
N<-cos(r1*pi/180)
|
115
|
E<-sin(r1*pi/180)
|
116
|
Nw<-sin(r2*pi/180)*cos(r1*pi/180) #Adding a variable to the dataframe
|
117
|
Ew<-sin(r2*pi/180)*sin(r1*pi/180) #Adding variable to the dataframe.
|
118
|
|
119
|
pos<-match("LC1",layerNames(s_raster)) #Find column with name "value"
|
120
|
LC1<-raster(s_raster,layer=pos) #Select layer from stack
|
121
|
s_raster<-dropLayer(s_raster,pos)
|
122
|
LC1[is.na(LC1)]<-0
|
123
|
pos<-match("LC3",layerNames(s_raster)) #Find column with name "value"
|
124
|
LC3<-raster(s_raster,layer=pos) #Select layer from stack
|
125
|
s_raster<-dropLayer(s_raster,pos)
|
126
|
LC3[is.na(LC3)]<-0
|
127
|
pos<-match("CANHEIGHT",layerNames(s_raster)) #Find column with name "value"
|
128
|
CANHEIGHT<-raster(s_raster,layer=pos) #Select layer from stack
|
129
|
s_raster<-dropLayer(s_raster,pos)
|
130
|
CANHEIGHT[is.na(CANHEIGHT)]<-0
|
131
|
|
132
|
xy<-coordinates(r1) #get x and y projected coordinates...
|
133
|
xy_latlon<-project(xy, CRS, inv=TRUE) # find lat long for projected coordinats (or pixels...)
|
134
|
lon<-raster(xy_latlon) #Transform a matrix into a raster object ncol=ncol(r1), nrow=nrow(r1))
|
135
|
ncol(lon)<-ncol(r1)
|
136
|
nrow(lon)<-nrow(r1)
|
137
|
extent(lon)<-extent(r1)
|
138
|
projection(lon)<-CRS #At this stage this is still an empty raster with 536 nrow and 745 ncell
|
139
|
lat<-lon
|
140
|
values(lon)<-xy_latlon[,1]
|
141
|
values(lat)<-xy_latlon[,2]
|
142
|
|
143
|
r<-stack(N,E,Nw,Ew,lon,lat,LC1,LC3,CANHEIGHT)
|
144
|
rnames<-c("Northness","Eastness","Northness_w","Eastness_w", "lon","lat","LC1","LC3","CANHEIGHT")
|
145
|
layerNames(r)<-rnames
|
146
|
s_raster<-addLayer(s_raster, r)
|
147
|
|
148
|
#s_sgdf<-as(s_raster,"SpatialGridDataFrame") #Conversion to spatial grid data frame
|
149
|
|
150
|
####### Preparing LST stack of climatology...
|
151
|
|
152
|
#l=list.files(pattern="mean_month.*rescaled.rst")
|
153
|
l <-readLines(paste(path,"/",infile6, sep=""))
|
154
|
molst<-stack(l) #Creating a raster stack...
|
155
|
#setwd(old)
|
156
|
molst<-molst-273.16 #K->C #LST stack of monthly average...
|
157
|
idx <- seq(as.Date('2010-01-15'), as.Date('2010-12-15'), 'month')
|
158
|
molst <- setZ(molst, idx)
|
159
|
layerNames(molst) <- month.abb
|
160
|
|
161
|
###### Preparing tables for model assessment: specific diagnostic/metrics
|
162
|
|
163
|
#Model assessment: specific diagnostics/metrics
|
164
|
results_m1<- matrix(1,1,nmodels+3) #Diagnostic metrics specific to the modeleling framework
|
165
|
results_m2<- matrix(1,1,nmodels+3)
|
166
|
results_m3<- matrix(1,1,nmodels+3)
|
167
|
#results_RMSE_f<- matrix(1,length(models)+3)
|
168
|
|
169
|
#Model assessment: general diagnostic/metrics
|
170
|
results_RMSE <- matrix(1,1,nmodels+3)
|
171
|
results_MAE <- matrix(1,1,nmodels+3)
|
172
|
results_ME <- matrix(1,1,nmodels+3) #There are 8 models for kriging!!!
|
173
|
results_R2 <- matrix(1,1,nmodels+3) #Coef. of determination for the validation dataset
|
174
|
|
175
|
results_RMSE_f<- matrix(1,1,nmodels+3) #RMSE fit, RMSE for the training dataset
|
176
|
results_MAE_f <- matrix(1,1,nmodels+3)
|
177
|
results_R2_f <- matrix(1,1,nmodels+3)
|
178
|
|
179
|
######### Preparing daily values for training and testing
|
180
|
|
181
|
#Screening for bad values: value is tmax in this case
|
182
|
#ghcn$value<-as.numeric(ghcn$value)
|
183
|
ghcn_all<-ghcn
|
184
|
ghcn_test<-subset(ghcn,ghcn$value>-150 & ghcn$value<400)
|
185
|
ghcn_test2<-subset(ghcn_test,ghcn_test$ELEV_SRTM>0)
|
186
|
ghcn<-ghcn_test2
|
187
|
#coords<- ghcn[,c('x_OR83M','y_OR83M')]
|
188
|
|
189
|
##Sampling: training and testing sites...
|
190
|
|
191
|
#set.seed(seed_number) #Using a seed number allow results based on random number to be compared...
|
192
|
|
193
|
nel<-length(dates)
|
194
|
dates_list<-vector("list",nel) #list of one row data.frame
|
195
|
|
196
|
prop_range<-(seq(from=prop_min,to=prop_max,by=step))*100
|
197
|
sn<-length(dates)*nb_sample*length(prop_range)
|
198
|
|
199
|
for(i in 1:length(dates)){
|
200
|
d_tmp<-rep(dates[i],nb_sample*length(prop_range)) #repeating same date
|
201
|
s_nb<-rep(1:nb_sample,length(prop_range)) #number of random sample per proportion
|
202
|
prop_tmp<-sort(rep(prop_range, nb_sample))
|
203
|
tab_run_tmp<-cbind(d_tmp,s_nb,prop_tmp)
|
204
|
dates_list[[i]]<-tab_run_tmp
|
205
|
}
|
206
|
|
207
|
sampling_dat<-as.data.frame(do.call(rbind,dates_list))
|
208
|
names(sampling_dat)<-c("date","run_samp","prop")
|
209
|
|
210
|
for(i in 2:3){ # start of the for loop #1
|
211
|
sampling_dat[,i]<-as.numeric(as.character(sampling_dat[,i]))
|
212
|
}
|
213
|
|
214
|
sampling_dat$date<- as.character(sampling_dat[,1])
|
215
|
#ghcn.subsets <-lapply(dates, function(d) subset(ghcn, date==d)) #this creates a list of 10 or 365 subsets dataset based on dates
|
216
|
ghcn.subsets <-lapply(as.character(sampling_dat$date), function(d) subset(ghcn, date==d)) #this creates a list of 10 or 365 subsets dataset based on dates
|
217
|
|
218
|
sampling<-vector("list",length(ghcn.subsets))
|
219
|
|
220
|
for(i in 1:length(ghcn.subsets)){
|
221
|
n<-nrow(ghcn.subsets[[i]])
|
222
|
prop<-(sampling_dat$prop[i])/100
|
223
|
ns<-n-round(n*prop) #Create a sample from the data frame with 70% of the rows
|
224
|
nv<-n-ns #create a sample for validation with prop of the rows
|
225
|
ind.training <- sample(nrow(ghcn.subsets[[i]]), size=ns, replace=FALSE) #This selects the index position for 70% of the rows taken randomly
|
226
|
ind.testing <- setdiff(1:nrow(ghcn.subsets[[i]]), ind.training)
|
227
|
sampling[[i]]<-ind.training
|
228
|
}
|
229
|
|
230
|
######## Prediction for the range of dates
|
231
|
|
232
|
#krig_mod<-mclapply(1:length(dates), runKriging,mc.preschedule=FALSE,mc.cores = 8) #This is the end bracket from mclapply(...) statement
|
233
|
krig_mod<-mclapply(1:length(ghcn.subsets), runKriging,mc.preschedule=FALSE,mc.cores = 8) #This is the end bracket from mclapply(...) statement
|
234
|
#krig_mod<-mclapply(1:1, runKriging,mc.preschedule=FALSE,mc.cores = 1) #This is the end bracket from mclapply(...) statement
|
235
|
|
236
|
save(krig_mod,file= paste(path,"/","results2_krig_mod_",out_prefix,".RData",sep=""))
|
237
|
load("results2_krig_mod__08312012_365d_Kriging_multi_samp3.RData")
|
238
|
tb<-krig_mod[[1]][[3]][0,] #empty data frame with metric table structure that can be used in rbinding...
|
239
|
tb_tmp<-krig_mod #copy
|
240
|
|
241
|
for (i in 1:length(tb_tmp)){
|
242
|
tmp<-tb_tmp[[i]][[3]]
|
243
|
tb<-rbind(tb,tmp)
|
244
|
}
|
245
|
rm(tb_tmp)
|
246
|
|
247
|
for(i in 4:nmodels+3){ # start of the for loop #1
|
248
|
tb[,i]<-as.numeric(as.character(tb[,i]))
|
249
|
}
|
250
|
|
251
|
metrics<-as.character(unique(tb$metric)) #Name of accuracy metrics (RMSE,MAE etc.)
|
252
|
tb_metric_list<-vector("list",length(metrics))
|
253
|
|
254
|
for(i in 1:length(metrics)){ # Reorganizing information in terms of metrics
|
255
|
metric_name<-paste("tb_",metrics[i],sep="")
|
256
|
tb_metric<-subset(tb, metric==metrics[i])
|
257
|
tb_metric<-cbind(tb_metric,sampling_dat[,2:3])
|
258
|
assign(metric_name,tb_metric)
|
259
|
tb_metric_list[[i]]<-tb_metric
|
260
|
}
|
261
|
|
262
|
tb_diagnostic<-do.call(rbind,tb_metric_list)
|
263
|
tb_diagnostic[["prop"]]<-as.factor(tb_diagnostic[["prop"]])
|
264
|
|
265
|
t<-melt(tb_diagnostic,
|
266
|
measure=c("mod1","mod2","mod3","mod4", "mod5", "mod6", "mod7", "mod8","mod9"),
|
267
|
id=c("dates","metric","prop"),
|
268
|
na.rm=F)
|
269
|
avg_tb<-cast(t,metric+prop~variable,mean)
|
270
|
median_tb<-cast(t,metric+prop~variable,mean)
|
271
|
avg_tb[["prop"]]<-as.numeric(as.character(avg_tb[["prop"]]))
|
272
|
avg_RMSE<-subset(avg_tb,metric=="RMSE")
|
273
|
|
274
|
# Save before plotting
|
275
|
|
276
|
write.table(avg_tb, file= paste(path,"/","results2_fusion_Assessment_measure_avg_",out_prefix,".txt",sep=""), sep=",")
|
277
|
write.table(median_tb, file= paste(path,"/","results2_fusion_Assessment_measure_median_",out_prefix,".txt",sep=""), sep=",")
|
278
|
write.table(tb_diagnostic, file= paste(path,"/","results2_fusion_Assessment_measure",out_prefix,".txt",sep=""), sep=",")
|
279
|
write.table(tb, file= paste(path,"/","results2_fusion_Assessment_measure_all",out_prefix,".txt",sep=""), sep=",")
|
280
|
|
281
|
#save(krig_mod,file= paste(path,"/","results2_krig_mod_",out_prefix,".RData",sep=""))
|
282
|
|
283
|
# get the range for the x and y axis
|
284
|
X11()
|
285
|
xrange <- range(avg_tb$prop)
|
286
|
yrange <- c(0,3.6)
|
287
|
|
288
|
# set up the plot
|
289
|
plot(xrange, yrange, type="n", xlab="Proportion of hold out in %",
|
290
|
ylab="RMSE" )
|
291
|
colors <- rainbow(nmodels)
|
292
|
linetype <- c(1:nmodels)
|
293
|
plotchar <- seq(1,1+nmodels,1)
|
294
|
|
295
|
# add lines
|
296
|
for (i in 1:(nmodels)) {
|
297
|
avg_tb_RMSE <- subset(avg_tb, metric=="RMSE")
|
298
|
x<-avg_tb_RMSE[["prop"]]
|
299
|
mod_name<-paste("mod",i,sep="")
|
300
|
y<-avg_tb_RMSE[[mod_name]]
|
301
|
lines(x, y, type="b", lwd=1.5,
|
302
|
lty=1, col=colors[i], pch=plotchar[i])
|
303
|
}
|
304
|
|
305
|
# add a title and subtitle
|
306
|
title("RMSE for fusion and GAM models")
|
307
|
|
308
|
# add a legend
|
309
|
legend("bottomright",legend=1:(nmodels), cex=1.2, col=colors,
|
310
|
pch=plotchar, lty=linetype, title="mod")
|
311
|
|
312
|
|
313
|
#### END OF SCRIPT
|