Project

General

Profile

Download (23.9 KB) Statistics
| Branch: | Revision:
1
runFusion <- function(i) {            # loop over dates
2
  
3
  date<-strptime(dates[i], "%Y%m%d")   # interpolation date being processed
4
  month<-strftime(date, "%m")          # current month of the date being processed
5
  LST_month<-paste("mm_",month,sep="") # name of LST month to be matched
6
  
7
  ###Regression part 1: Creating a validation dataset by creating training and testing datasets
8
  
9
  mod_LST <-ghcn.subsets[[i]][,match(LST_month, names(ghcn.subsets[[i]]))]  #Match interpolation date and monthly LST average
10
  ghcn.subsets[[i]] = transform(ghcn.subsets[[i]],LST = mod_LST)            #Add the variable LST to the subset dataset
11
  #n<-nrow(ghcn.subsets[[i]])
12
  #ns<-n-round(n*prop)   #Create a sample from the data frame with 70% of the rows
13
  #nv<-n-ns              #create a sample for validation with prop of the rows
14
  #ind.training <- sample(nrow(ghcn.subsets[[i]]), size=ns, replace=FALSE) #This selects the index position for 70% of the rows taken randomly
15
  ind.training<-sampling[[i]]
16
  ind.testing <- setdiff(1:nrow(ghcn.subsets[[i]]), ind.training)
17
  data_s <- ghcn.subsets[[i]][ind.training, ]   #Training dataset currently used in the modeling
18
  data_v <- ghcn.subsets[[i]][ind.testing, ]    #Testing/validation dataset
19
  
20
  ns<-nrow(data_s)
21
  nv<-nrow(data_v)
22
  #i=1
23
  date_proc<-dates[i]
24
  date_proc<-strptime(dates[i], "%Y%m%d")   # interpolation date being processed
25
  mo<-as.integer(strftime(date_proc, "%m"))          # current month of the date being processed
26
  day<-as.integer(strftime(date_proc, "%d"))
27
  year<-as.integer(strftime(date_proc, "%Y"))
28

    
29
  datelabel=format(ISOdate(year,mo,day),"%b %d, %Y")
30
  
31
  ###########
32
  #  STEP 1 - 10 year monthly averages
33
  ###########
34
  
35
  #l=list.files(pattern="mean_month.*rescaled.rst")
36
  l <-readLines(paste(path,"/",infile6, sep=""))
37
  molst<-stack(l)  #Creating a raster stack...
38
  #setwd(old)
39
  molst=molst-273.16  #K->C
40
  idx <- seq(as.Date('2010-01-15'), as.Date('2010-12-15'), 'month')
41
  molst <- setZ(molst, idx)
42
  layerNames(molst) <- month.abb
43
  themolst<-raster(molst,mo) #current month being processed saved in a raster image
44
  plot(themolst)
45
  
46
  ###########
47
  # STEP 2 - Weather station means across same days: Monthly mean calculation
48
  ###########
49
  
50
  ##Added by Benoit ######
51
  date1<-ISOdate(data3$year,data3$month,data3$day) #Creating a date object from 3 separate column
52
  date2<-as.POSIXlt(as.Date(date1))
53
  data3$date<-date2
54
  d<-subset(data3,year>=2000 & mflag=="0" ) #Selecting dataset 2000-2010 with good quality: 193 stations
55
  #May need some screeing??? i.e. range of temp and elevation...
56
  d1<-aggregate(value~station+month, data=d, mean)  #Calculate monthly mean for every station in OR
57
  id<-as.data.frame(unique(d1$station))     #Unique station in OR for year 2000-2010: 193 but 7 loss of monthly avg    
58
  
59
  dst<-merge(d1, stat_loc, by.x="station", by.y="STAT_ID")   #Inner join all columns are retained
60
  
61
  #This allows to change only one name of the data.frame
62
  pos<-match("value",names(dst)) #Find column with name "value"
63
  names(dst)[pos]<-c("TMax")
64
  dst$TMax<-dst$TMax/10                #TMax is the average max temp for months
65
  #dstjan=dst[dst$month==9,]  #dst contains the monthly averages for tmax for every station over 2000-2010
66
  ##############
67
  
68
  modst=dst[dst$month==mo,] #Subsetting dataset for the relevant month of the date being processed
69
  
70
  ##########
71
  # STEP 3 - get LST at stations
72
  ##########
73
  
74
  sta_lola=modst[,c("lon","lat")] #Extracting locations of stations for the current month..
75
  
76
  proj_str="+proj=lcc +lat_1=43 +lat_2=45.5 +lat_0=41.75 +lon_0=-120.5 +x_0=400000 +y_0=0 +ellps=GRS80 +units=m +no_defs";
77
  lookup<-function(r,lat,lon) {
78
    xy<-project(cbind(lon,lat),proj_str);
79
    cidx<-cellFromXY(r,xy);
80
    return(r[cidx])
81
  }
82
  sta_tmax_from_lst=lookup(themolst,sta_lola$lat,sta_lola$lon) #Extracted values of LST for the stations
83
  
84
  #########
85
  # STEP 4 - bias at stations     
86
  #########
87
  
88
  sta_bias=sta_tmax_from_lst-modst$TMax; #That is the difference between the monthly LST mean and monthly station mean
89
  #Added by Benoit
90
  modst$LSTD_bias<-sta_bias  #Adding bias to data frame modst containning the monthly average for 10 years
91
  
92
  bias_xy=project(as.matrix(sta_lola),proj_str)
93
  # windows()
94
  png(paste("LST_TMax_scatterplot_",dates[i],out_prefix,".png", sep=""))
95
  plot(modst$TMax,sta_tmax_from_lst,xlab="Station mo Tmax",ylab="LST mo Tmax",main=paste("LST vs TMax for",datelabel,sep=" "))
96
  abline(0,1)
97
  dev.off()
98
  
99
  #added by Benoit 
100
  #x<-ghcn.subsets[[i]]  #Holds both training and testing for instance 161 rows for Jan 1
101
  x<-data_v
102
  d<-data_s
103
  
104
  pos<-match("value",names(d)) #Find column with name "value"
105
  names(d)[pos]<-c("dailyTmax")
106
  names(x)[pos]<-c("dailyTmax")
107
  d$dailyTmax=(as.numeric(d$dailyTmax))/10 #stored as 1/10 degree C to allow integer storage
108
  x$dailyTmax=(as.numeric(x$dailyTmax))/10 #stored as 1/10 degree C to allow integer storage
109
  pos<-match("station",names(d)) #Find column with name "value"
110
  names(d)[pos]<-c("id")
111
  names(x)[pos]<-c("id")
112
  names(modst)[1]<-c("id")       #modst contains the average tmax per month for every stations...
113
  dmoday=merge(modst,d,by="id")  #LOOSING DATA HERE!!! from 113 t0 103
114
  xmoday=merge(modst,x,by="id")  #LOOSING DATA HERE!!! from 48 t0 43
115
  names(dmoday)[4]<-c("lat")
116
  names(dmoday)[5]<-c("lon")     #dmoday contains all the the information: BIAS, monn
117
  names(xmoday)[4]<-c("lat")
118
  names(xmoday)[5]<-c("lon")     #dmoday contains all the the information: BIAS, monn
119
  
120
  data_v<-xmoday
121
  ###
122
  
123
  #dmoday contains the daily tmax values for training with TMax being the monthly station tmax mean
124
  #xmoday contains the daily tmax values for validation with TMax being the monthly station tmax mean
125
  
126
  # windows()
127
  #png(paste("LST_TMax_scatterplot_",dates[i],out_prefix,".png", sep=""))
128
  png(paste("Daily_tmax_monthly_TMax_scatterplot_",dates[i],out_prefix,".png", sep=""))
129
  plot(dailyTmax~TMax,data=dmoday,xlab="Mo Tmax",ylab=paste("Daily for",datelabel),main="across stations in OR")
130
  #savePlot(paste("Daily_tmax_monthly_TMax_scatterplot_",dates[i],out_prefix,".png", sep=""), type="png")
131
  #png(paste("LST_TMax_scatterplot_",dates[i],out_prefix,".png", sep=""))
132
  dev.off()
133
  
134
  ########
135
  # STEP 5 - interpolate bias
136
  ########
137
  
138
  # ?? include covariates like elev, distance to coast, cloud frequency, tree height
139
  #library(fields)
140
  #windows()
141
  quilt.plot(sta_lola,sta_bias,main="Bias at stations",asp=1)
142
  US(add=T,col="magenta",lwd=2)
143
  #fitbias<-Tps(bias_xy,sta_bias) #use TPS or krige
144
  
145
  #Adding options to use only training stations: 07/11/2012
146
  bias_xy=project(as.matrix(sta_lola),proj_str)
147
  #bias_xy2=project(as.matrix(c(dmoday$lon,dmoday$lat),proj_str)
148
  if(bias_val==1){
149
    sta_bias<-dmoday$LSTD_bias
150
    bias_xy<-cbind(dmoday$x_OR83M,dmoday$y_OR83M)
151
  }
152
  
153
  fitbias<-Krig(bias_xy,sta_bias,theta=1e5) #use TPS or krige 
154
  #The output is a krig object using fields
155
  mod9a<-fitbias
156
  # Creating plot of bias surface and saving it
157
  #X11()
158
  png(paste("Bias_surface_LST_TMax_",dates[i],out_prefix,".png", sep="")) #Create file to write a plot
159
  datelabel2=format(ISOdate(year,mo,day),"%B ") #added by Benoit, label
160
  surface(fitbias,col=rev(terrain.colors(100)),asp=1,main=paste("Interpolated bias for",datelabel2,sep=" ")) #Plot to file
161
  #savePlot(paste("Bias_surface_LST_TMax_",dates[i],out_prefix,".png", sep=""), type="png")
162
  dev.off()  #Release the hold to the file
163
  
164
  #US(add=T,col="magenta",lwd=2)
165
  
166
  ##########
167
  # STEP 7 - interpolate delta across space
168
  ##########
169
  
170
  daily_sta_lola=dmoday[,c("lon","lat")] #could be same as before but why assume merge does this - assume not
171
  daily_sta_xy=project(as.matrix(daily_sta_lola),proj_str)
172
  daily_delta=dmoday$dailyTmax-dmoday$TMax
173
  #windows()
174
  quilt.plot(daily_sta_lola,daily_delta,asp=1,main="Station delta for Jan 15")
175
  US(add=T,col="magenta",lwd=2)
176
  #fitdelta<-Tps(daily_sta_xy,daily_delta) #use TPS or krige
177
  fitdelta<-Krig(daily_sta_xy,daily_delta,theta=1e5) #use TPS or krige
178
  #Kriging using fields package
179
  mod9b<-fitdelta
180
  # Creating plot of bias surface and saving it
181
  #X11()
182
  png(paste("Delta_surface_LST_TMax_",dates[i],out_prefix,".png", sep=""))
183
  surface(fitdelta,col=rev(terrain.colors(100)),asp=1,main=paste("Interpolated delta for",datelabel,sep=" "))
184
  #savePlot(paste("Delta_surface_LST_TMax_",dates[i],out_prefix,".png", sep=""), type="png")
185
  dev.off()
186
  #US(add=T,col="magenta",lwd=2)
187
  #
188
  
189
  #### Added by Benoit on 06/19
190
  data_s<-dmoday #put the 
191
  data_s$daily_delta<-daily_delta
192
  
193
  
194
  #data_s$y_var<-daily_delta  #y_var is the variable currently being modeled, may be better with BIAS!!
195
  data_s$y_var<-data_s$LSTD_bias
196
  #data_s$y_var<-(data_s$dailyTmax)*10
197
  #Model and response variable can be changed without affecting the script
198
  
199
  mod1<- try(gam(y_var~ s(lat) + s (lon) + s (ELEV_SRTM), data=data_s))
200
  mod2<- try(gam(y_var~ s(lat,lon)+ s(ELEV_SRTM), data=data_s)) #modified nesting....from 3 to 2
201
  mod3<- try(gam(y_var~ s(lat) + s (lon) + s (ELEV_SRTM) +  s (Northness)+ s (Eastness) + s(DISTOC), data=data_s))
202
  mod4<- try(gam(y_var~ s(lat) + s (lon) + s(ELEV_SRTM) + s(Northness) + s (Eastness) + s(DISTOC) + s(LST), data=data_s))
203
  mod5<- try(gam(y_var~ s(lat,lon) +s(ELEV_SRTM) + s(Northness,Eastness) + s(DISTOC) + s(LST), data=data_s))
204
  mod6<- try(gam(y_var~ s(lat,lon) +s(ELEV_SRTM) + s(Northness,Eastness) + s(DISTOC) + s(LST)+s(LC1), data=data_s))
205
  mod7<- try(gam(y_var~ s(lat,lon) +s(ELEV_SRTM) + s(Northness,Eastness) + s(DISTOC) + s(LST)+s(LC3), data=data_s))
206
  mod8<- try(gam(y_var~ s(lat,lon) +s(ELEV_SRTM) + s(Northness,Eastness) + s(DISTOC) + s(LST) + s(LC1), data=data_s))
207
  
208
  #Added
209
  #tmax_predicted=themolst+daily_delta_rast-bias_rast #Final surface?? but daily_rst
210
  
211
  #### Added by Benoit ends
212
  
213
  #########
214
  # STEP 8 - assemble final answer - T=LST+Bias(interpolated)+delta(interpolated)
215
  #########
216

    
217
  
218
  bias_rast=interpolate(themolst,fitbias) #interpolation using function from raster package
219
  #themolst is raster layer, fitbias is "Krig" object from bias surface
220
  #plot(bias_rast,main="Raster bias") #This not displaying...
221
  
222
  #Saving kriged surface in raster images
223
  data_name<-paste("bias_LST_",dates[[i]],sep="")
224
  raster_name<-paste("fusion_",data_name,out_prefix,".rst", sep="")
225
  writeRaster(bias_rast, filename=raster_name,overwrite=TRUE)  #Writing the data in a raster file format...(IDRISI)
226
  
227
  daily_delta_rast=interpolate(themolst,fitdelta) #Interpolation of the bias surface...
228
  
229
  #plot(daily_delta_rast,main="Raster Daily Delta")
230
  
231
  #Saving kriged surface in raster images
232
  data_name<-paste("daily_delta_",dates[[i]],sep="")
233
  raster_name<-paste("fusion_",data_name,out_prefix,".rst", sep="")
234
  writeRaster(daily_delta_rast, filename=raster_name,overwrite=TRUE)  #Writing the data in a raster file format...(IDRISI)
235
  
236
  tmax_predicted=themolst+daily_delta_rast-bias_rast #Final surface  as a raster layer...
237
  #tmax_predicted=themolst+daily_delta_rast+bias_rast #Added by Benoit, why is it -bias_rast
238
  #plot(tmax_predicted,main="Predicted daily")
239
  
240
  #Saving kriged surface in raster images
241
  data_name<-paste("tmax_predicted_",dates[[i]],sep="")
242
  raster_name<-paste("fusion_",data_name,out_prefix,".rst", sep="")
243
  writeRaster(tmax_predicted, filename=raster_name,overwrite=TRUE)  #Writing the data in a raster file format...(IDRISI)
244
  
245
  ########
246
  # check: assessment of results: validation
247
  ########
248
  RMSE<-function(x,y) {return(mean((x-y)^2)^0.5)}
249
  MAE_fun<-function(x,y) {return(mean(abs(x-y)))}
250
  #ME_fun<-function(x,y){return(mean(abs(y)))}
251
  #FIT ASSESSMENT
252
  sta_pred_data_s=lookup(tmax_predicted,data_s$lat,data_s$lon)
253
  rmse_fit=RMSE(sta_pred_data_s,data_s$dailyTmax)
254
  mae_fit=MAE_fun(sta_pred_data_s,data_s$dailyTmax)
255
    
256
  sta_pred=lookup(tmax_predicted,data_v$lat,data_v$lon)
257
  #sta_pred=lookup(tmax_predicted,daily_sta_lola$lat,daily_sta_lola$lon)
258
  #rmse=RMSE(sta_pred,dmoday$dailyTmax)
259
  #pos<-match("value",names(data_v)) #Find column with name "value"
260
  #names(data_v)[pos]<-c("dailyTmax")
261
  tmax<-data_v$dailyTmax
262
  #data_v$dailyTmax<-tmax
263
  rmse=RMSE(sta_pred,tmax)
264
  mae<-MAE_fun(sta_pred,tmax)
265
  r2<-cor(sta_pred,tmax)^2              #R2, coef. of var
266
  me<-mean(sta_pred-tmax)
267
 
268
  #plot(sta_pred~dmoday$dailyTmax,xlab=paste("Actual daily for",datelabel),ylab="Pred daily",main=paste("RMSE=",rmse))
269
  
270
  png(paste("Predicted_tmax_versus_observed_scatterplot_",dates[i],out_prefix,".png", sep=""))
271
  plot(sta_pred~tmax,xlab=paste("Actual daily for",datelabel),ylab="Pred daily",main=paste("RMSE=",rmse))
272
  abline(0,1)
273
  #savePlot(paste("Predicted_tmax_versus_observed_scatterplot_",dates[i],out_prefix,".png", sep=""), type="png")
274
  dev.off()
275
  #resid=sta_pred-dmoday$dailyTmax
276
  resid=sta_pred-tmax
277
  quilt.plot(daily_sta_lola,resid)
278
  
279
  ### END OF BRIAN's code
280
  
281
  ### Added by benoit
282
  #Store results using TPS
283
#   j=9
284
#   results_RMSE[i,1]<- dates[i]    #storing the interpolation dates in the first column
285
#   results_RMSE[i,2]<- ns          #number of stations used in the training stage
286
#   results_RMSE[i,3]<- "RMSE"
287
#   results_RMSE[i,j+3]<- rmse  #Storing RMSE for the model j
288
#   
289
#   results_RMSE_f[i,1]<- dates[i]    #storing the interpolation dates in the first column
290
#   results_RMSE_f[i,2]<- ns          #number of stations used in the training stage
291
#   results_RMSE_f[i,3]<- "RMSE"
292
#   results_RMSE_f[i,j+3]<- rmse_fit  #Storing RMSE for the model j
293
#   
294
  ns<-nrow(data_s) #This is added to because some loss of data might have happened because of the averaging...
295
  nv<-nrow(data_v)       
296
  
297
  ### Added by benoit
298
  #Store results using TPS
299
  j=9
300
  results_RMSE[1]<- dates[i]    #storing the interpolation dates in the first column
301
  results_RMSE[2]<- ns          #number of stations used in the training stage
302
  results_RMSE[3]<- "RMSE"
303
  results_RMSE[j+3]<- rmse  #Storing RMSE for the model j
304
  
305
  results_RMSE_f[1]<- dates[i]    #storing the interpolation dates in the first column
306
  results_RMSE_f[2]<- ns          #number of stations used in the training stage
307
  results_RMSE_f[3]<- "RMSE_f"
308
  results_RMSE_f[j+3]<- rmse_fit  #Storing RMSE for the model j
309
  
310
  results_MAE_f[1]<- dates[i]    #storing the interpolation dates in the first column
311
  results_MAE_f[2]<- ns          #number of stations used in the training stage
312
  results_MAE_f[3]<- "RMSE_f"
313
  results_MAE_f[j+3]<- mae_fit  #Storing RMSE for the model j
314

    
315
  results_MAE[1]<- dates[i]    #storing the interpolation dates in the first column
316
  results_MAE[2]<- ns          #number of stations used in the training stage
317
  results_MAE[3]<- "MAE"
318
  results_MAE[j+3]<- mae  #Storing RMSE for the model j
319

    
320
  results_ME[1]<- dates[i]    #storing the interpolation dates in the first column
321
  results_ME[2]<- ns          #number of stations used in the training stage
322
  results_ME[3]<- "ME"
323
  results_ME[j+3]<- me  #Storing RMSE for the model j
324
  
325
  results_R2[1]<- dates[i]    #storing the interpolation dates in the first column
326
  results_R2[2]<- ns          #number of stations used in the training stage
327
  results_R2[3]<- "R2"
328
  results_R2[j+3]<- r2  #Storing RMSE for the model j
329
  
330
  #ns<-nrow(data_s) #This is added to because some loss of data might have happened because of the averaging...
331
  #nv<-nrow(data_v)
332
  
333
  for (j in 1:length(models)){
334
    
335
    ##Model assessment: specific diagnostic/metrics for GAM
336
    
337
    name<-paste("mod",j,sep="")  #modj is the name of The "j" model (mod1 if j=1) 
338
    mod<-get(name)               #accessing GAM model ojbect "j"
339
    
340
    #If mod "j" is not a model object
341
    if (inherits(mod,"try-error")) {
342
      results_AIC[1]<- dates[i]  #storing the interpolation dates in the first column
343
      results_AIC[2]<- ns        #number of stations used in the training stage
344
      results_AIC[3]<- "AIC"
345
      results_AIC[j+3]<- NA
346
      
347
      results_GCV[1]<- dates[i]  #storing the interpolation dates in the first column
348
      results_GCV[2]<- ns        #number of stations used in the training 
349
      results_GCV[3]<- "GCV"
350
      results_GCV[j+3]<- NA
351
      
352
      results_DEV[1]<- dates[i]  #storing the interpolation dates in the first column
353
      results_DEV[2]<- ns        #number of stations used in the training stage
354
      results_DEV[3]<- "DEV"
355
      results_DEV[j+3]<- NA
356
      
357
      results_RMSE_f[1]<- dates[i]  #storing the interpolation dates in the first column
358
      results_RMSE_f[2]<- ns        #number of stations used in the training stage
359
      results_RMSE_f[3]<- "RSME_f"
360
      results_RMSE_f[j+3]<- NA
361
      
362
      results_MAE_f[1]<- dates[i]  #storing the interpolation dates in the first column
363
      results_MAE_f[2]<- ns        #number of stations used in the training stage
364
      results_MAE_f[3]<- "MAE_f"
365
      results_MAE_f[j+3]<-NA
366
      
367
      results_RMSE[1]<- dates[i]    #storing the interpolation dates in the first column
368
      results_RMSE[2]<- ns          #number of stations used in the training stage
369
      results_RMSE[3]<- "RMSE"
370
      results_RMSE[j+3]<- NA  #Storing RMSE for the model j
371
      results_MAE[1]<- dates[i]     #storing the interpolation dates in the first column
372
      results_MAE[2]<- ns           #number of stations used in the training stage
373
      results_MAE[3]<- "MAE"
374
      results_MAE[j+3]<- NA    #Storing MAE for the model j
375
      results_ME[1]<- dates[i]      #storing the interpolation dates in the first column
376
      results_ME[2]<- ns            #number of stations used in the training stage
377
      results_ME[3]<- "ME"
378
      results_ME[j+3]<- NA      #Storing ME for the model j
379
      results_R2[1]<- dates[i]      #storing the interpolation dates in the first column
380
      results_R2[2]<- ns            #number of stations used in the training stage
381
      results_R2[3]<- "R2"
382
      results_R2[j+3]<- NA      #Storing R2 for the model j
383
      
384
    }
385
    
386
    #If mod is a modelobject
387
    
388
    #If mod "j" is not a model object
389
    if (inherits(mod,"gam")) {
390
      
391
      results_AIC[1]<- dates[i]  #storing the interpolation dates in the first column
392
      results_AIC[2]<- ns        #number of stations used in the training stage
393
      results_AIC[3]<- "AIC"
394
      results_AIC[j+3]<- AIC (mod)
395
      
396
      results_GCV[1]<- dates[i]  #storing the interpolation dates in the first column
397
      results_GCV[2]<- ns        #number of stations used in the training 
398
      results_GCV[3]<- "GCV"
399
      results_GCV[j+3]<- mod$gcv.ubre
400
      
401
      results_DEV[1]<- dates[i]  #storing the interpolation dates in the first column
402
      results_DEV[2]<- ns        #number of stations used in the training stage
403
      results_DEV[3]<- "DEV"
404
      results_DEV[j+3]<- mod$deviance
405
      
406
      sta_LST_s=lookup(themolst,data_s$lat,data_s$lon)
407
      sta_delta_s=lookup(daily_delta_rast,data_s$lat,data_s$lon) #delta surface has been calculated before!!
408
      sta_bias_s= mod$fit
409
      #Need to extract values from the kriged delta surface...
410
      #sta_delta= lookup(delta_surface,data_v$lat,data_v$lon)
411
      #tmax_predicted=sta_LST+sta_bias-y_mod$fit
412
      tmax_predicted_s= sta_LST_s-sta_bias_s+sta_delta_s
413
      
414
      results_RMSE_f[1]<- dates[i]  #storing the interpolation dates in the first column
415
      results_RMSE_f[2]<- ns        #number of stations used in the training stage
416
      results_RMSE_f[3]<- "RSME_f"
417
      results_RMSE_f[j+3]<- sqrt(sum((tmax_predicted_s-data_s$dailyTmax)^2)/ns)
418
      
419
      results_MAE_f[1]<- dates[i]  #storing the interpolation dates in the first column
420
      results_MAE_f[2]<- ns        #number of stations used in the training stage
421
      results_MAE_f[3]<- "MAE_f"
422
      results_MAE_f[j+3]<-sum(abs(tmax_predicted_s-data_s$dailyTmax))/ns
423
      
424
      ##Model assessment: general diagnostic/metrics
425
      ##validation: using the testing data
426
      
427
      #data_v$y_var<-data_v$LSTD_bias
428
      #data_v$y_var<-tmax
429
      y_mod<- predict(mod, newdata=data_v, se.fit = TRUE) #Using the coeff to predict new values.
430
      
431
      ####ADDED ON JULY 5th
432
      sta_LST_v=lookup(themolst,data_v$lat,data_v$lon)
433
      sta_delta_v=lookup(daily_delta_rast,data_v$lat,data_v$lon) #delta surface has been calculated before!!
434
      sta_bias_v= y_mod$fit
435
      #Need to extract values from the kriged delta surface...
436
      #sta_delta= lookup(delta_surface,data_v$lat,data_v$lon)
437
      #tmax_predicted=sta_LST+sta_bias-y_mod$fit
438
      tmax_predicted_v= sta_LST_v-sta_bias_v+sta_delta_v
439
      
440
      #data_v$tmax<-(data_v$tmax)/10
441
      res_mod<- data_v$dailyTmax - tmax_predicted_v              #Residuals for the model for fusion
442
      #res_mod<- data_v$y_var - y_mod$fit                  #Residuals for the model
443
      
444
      RMSE_mod <- sqrt(sum(res_mod^2)/nv)                 #RMSE FOR REGRESSION STEP 1: GAM     
445
      MAE_mod<- sum(abs(res_mod))/nv                     #MAE, Mean abs. Error FOR REGRESSION STEP 1: GAM   
446
      ME_mod<- sum(res_mod)/nv                            #ME, Mean Error or bias FOR REGRESSION STEP 1: GAM
447
      R2_mod<- cor(data_v$dailyTmax,tmax_predicted_v)^2              #R2, coef. of var FOR REGRESSION STEP 1: GAM
448
      
449
      results_RMSE[1]<- dates[i]    #storing the interpolation dates in the first column
450
      results_RMSE[2]<- ns          #number of stations used in the training stage
451
      results_RMSE[3]<- "RMSE"
452
      results_RMSE[j+3]<- RMSE_mod  #Storing RMSE for the model j
453
      results_MAE[1]<- dates[i]     #storing the interpolation dates in the first column
454
      results_MAE[2]<- ns           #number of stations used in the training stage
455
      results_MAE[3]<- "MAE"
456
      results_MAE[j+3]<- MAE_mod    #Storing MAE for the model j
457
      results_ME[1]<- dates[i]      #storing the interpolation dates in the first column
458
      results_ME[2]<- ns            #number of stations used in the training stage
459
      results_ME[3]<- "ME"
460
      results_ME[j+3]<- ME_mod      #Storing ME for the model j
461
      results_R2[1]<- dates[i]      #storing the interpolation dates in the first column
462
      results_R2[2]<- ns            #number of stations used in the training stage
463
      results_R2[3]<- "R2"
464
      results_R2[j+3]<- R2_mod      #Storing R2 for the model j
465
      
466
      #Saving residuals and prediction in the dataframes: tmax predicted from GAM
467
      pred<-paste("pred_mod",j,sep="")
468
      #data_v[[pred]]<-as.numeric(y_mod$fit)
469
      data_v[[pred]]<-as.numeric(tmax_predicted_v)
470
      data_s[[pred]]<-as.numeric(tmax_predicted_s) #Storing model fit values (predicted on training sample)
471
      #data_s[[pred]]<-as.numeric(mod$fit) #Storing model fit values (predicted on training sample)
472
      
473
      name2<-paste("res_mod",j,sep="")
474
      data_v[[name2]]<-as.numeric(res_mod)
475
      temp<-tmax_predicted_s-data_s$dailyTmax
476
      data_s[[name2]]<-as.numeric(temp)
477
      #end of loop calculating RMSE
478
    }
479
  }
480
  
481
  #if (i==length(dates)){
482
  
483
  #Specific diagnostic measures related to the testing datasets
484

    
485
  results_table_RMSE<-as.data.frame(results_RMSE)
486
  results_table_MAE<-as.data.frame(results_MAE)
487
  results_table_ME<-as.data.frame(results_ME)
488
  results_table_R2<-as.data.frame(results_R2)
489
  results_table_RMSE_f<-as.data.frame(results_RMSE_f)
490
  results_table_MAE_f<-as.data.frame(results_MAE_f)
491
  
492
  results_table_AIC<-as.data.frame(results_AIC)
493
  results_table_GCV<-as.data.frame(results_GCV)
494
  results_table_DEV<-as.data.frame(results_DEV)
495
  
496
  tb_metrics1<-rbind(results_table_RMSE,results_table_MAE, results_table_ME, results_table_R2,results_table_RMSE_f,results_table_MAE_f)   #
497
  tb_metrics2<-rbind(results_table_AIC,results_table_GCV, results_table_DEV)
498
  cname<-c("dates","ns","metric","mod1", "mod2","mod3", "mod4", "mod5", "mod6", "mod7","mod8","mod9")
499
  colnames(tb_metrics1)<-cname
500
  cname<-c("dates","ns","metric","mod1", "mod2","mod3", "mod4", "mod5", "mod6", "mod7","mod8")
501
  colnames(tb_metrics2)<-cname
502
  #colnames(results_table_RMSE)<-cname
503
  #colnames(results_table_RMSE_f)<-cname
504
  #tb_diagnostic1<-results_table_RMSE      #measures of validation
505
  #tb_diagnostic2<-results_table_RMSE_f    #measures of fit
506
  
507
  #write.table(tb_diagnostic1, file= paste(path,"/","results_fusion_Assessment_measure1",out_prefix,".txt",sep=""), sep=",")
508
  
509
  #}  
510
  print(paste(dates[i],"processed"))
511
  mod_obj<-list(mod1,mod2,mod3,mod4,mod5,mod6,mod7,mod8,mod9a,mod9b)
512
  # end of the for loop1
513
  #results_list<-list(data_s,data_v,tb_metrics1,tb_metrics2)
514
  results_list<-list(data_s,data_v,tb_metrics1,tb_metrics2,mod_obj)
515
  return(results_list)
516
  #return(tb_diagnostic1)
517
}
(8-8/12)