LST Temporal Aggregation Evaluation

====

```{r,echo=FALSE,message=FALSE}

## get repo info

githash=system("git --git-dir /Users/adamw/work/environmental-layers/.git log --pretty=format:'%h' -n 1",intern=T)

ghead=paste("Compiled on",date()," using code hash:",githash)

#(`r ghead`)

```

### Adam M. Wilson and Giuseppe Amatulli

A short script to explore the implications of a 3-month moving window Land Surface Temperature Climatology.

```{r,echo=FALSE,results='hide',message=F}

## some setup

 uploadimages=T  # upload all images to imgur.com for easy viewing on github

opts_knit$set(progress = TRUE, verbose = TRUE,root.dir="~/Downloads",base.url = NULL)

if(uploadimages) opts_knit$set(upload.fun =imgur_upload)

opts_chunk$set(fig.width=12, fig.height=8, cache=!uploadimages)

### libraries

library(sp)

library(reshape)

library(plyr)

library(latticeExtra)

library(knitr)

## Import the data

setwd("~")

dlst=read.csv("/Users/adamw/Downloads/lst_tmax.csv")

## add month names

dlst$month=factor(month.name[dlst$month],ordered=T,levels=month.name)

## add 3 month moving window

dlst2=unique(dlst[,c("station","month","TMax","lst","lstp1","lstm1")])

dlst2$lst3m=apply(dlst2[,c("lst","lstp1","lstm1")],1,mean)

dlst2$lst2m=apply(dlst2[,c("lstp1","lstm1")],1,mean)

```

# Data

The following data were extracted from the shapefiles used to fit the tiled interpolations that Alberto has already run. They are limited to the CONUS region and include mean monthly maximum temperature observed at the station locations, as well as the LST from the month before, the month of, and the month following.  Here's what the data look like:

```{r,results="asis", echo=F}

kable(head(dlst[,-1]), format="markdown")

```

# TMax~LST relationship seasonal variability 

First let's explore the variability of the TMax~LST relationship by month, grouped by tile.  In this plot grey lines indicate a Tmax~LST regression within each tile (stations may be present in multiple tiles). Variability among grey lines represents spatial variability in the fitted lapse rate and the heavy black line is overall mean relationship (by month).

```{r,echo=FALSE}

xyplot (TMax~lst | month, groups=tile,data=dlst , 

        main="Tmax~LST at station locations by month",

        sub="Colors indicate  initial tiled-CONUS TMAX interpolation",

        ylab="Observations Tmax", xlab="LST",cex=.2,pch=16,auto.key=list(space="right"))+

  layer(panel.abline(0,1,lwd=3,col="black"))+

  glayer(panel.abline(lm (y~x),col="grey"),groups=dlst$tile,under=T)

```

# Comparison of monthly means with 3-month moving window

Here we compare the monthly means with a three month moving window (e.g. January mean LST with December-February mean LST).  Note that the relationship is very good (R^2 >0.95) but slightly weaker in winter months, likely due primarily to seasonal minimums.  Heavy black line is 1:1, and thin line is the fitted regression.

```{r,echo=FALSE}

xyplot (lst~lst3m | month,data=dlst2 , 

        main="Tmax~LST at station locations by month",

        ylab="LST (monthly)", xlab="LST (3-month Mean)",cex=.5,pch=16,auto.key=list(space="right"))+

  layer(panel.abline(0,1,lwd=3,col="black"))+

  layer(panel.ablineq(lm(y ~ x), r.sq = TRUE, at = 0.1, digits=2,adj=0:1,pos=4,offset=3,col="black"), style = 2)

```

# Comparison of monthly means with 2-month moving window that does not include the month

Here we compare the monthly means with a two month moving window that does not include the month of interest (e.g. January mean LST with December and February mean LST, but not including the January LST).  Heavy black line is 1:1, and thin line is the fitted regression.

```{r,echo=FALSE}

xyplot (lst~lst2m | month,data=dlst2 , 

        main="LST at station locations by month",

        ylab="LST (monthly)", xlab="LST (2-month Mean)",cex=.5,pch=16,auto.key=list(space="right"))+

  layer(panel.abline(0,1,lwd=3,col="black"))+

  layer(panel.ablineq(lm(y ~ x), r.sq = TRUE, at = 0.2, digits=2,adj=0:1,pos=4,offset=2,col="black"), style = 2)

```

Now let's look at the differences between the 1-month and 2-month LST values.  These represent a measure of how wrong we would be if we only had data from the two surrounding months and not the month in question.  

```{r,fig.height=3}

histogram(dlst2$lst-dlst2$lst2m,col="grey",xlab="Anomolies (1 month - 2 month means)")

```

The 95th quantile of the absolute value is only `r round(quantile(abs(dlst2$lst-dlst2$lst2m),na.rm=T,0.95),1)`, so the differences are quite small. From this analysis, it appears that broadening the temporal window will maintain a relatively consistent estimate of LST (R^2 ranged from 0.88-0.9) even when using only data from the surrounding months.

Let's see how the seasonal cycle is represented by these proxies for a few randomly selected stations.  Here the red line is the observed TMax data, the heavy black line represents the mean LST in that month, and the green line is a three-month moving window, while the purple line is the 2-month window (not including the month of interest).

```{rm,echo=FALSE}

## subset to 10 stations

dlst2l=melt(dlst2,id.vars=c("station","month"),measure.vars=c("TMax","lst","lst3m","lst2m"))

dlst2ls=dlst2l[dlst2l$station%in%sample(unique(dlst2l$station),10),]

```

```{r,echo=FALSE}

trellis.par.set(superpose.line = list(col = c("red","black","green","blue"),lwd=c(2,2,1,1)),

                superpose.symbol = list(col = c("red","black","green","blue"),lwd=c(2,2,1,1)))

xyplot (value~month|station, data=dlst2ls, groups=variable,type="l",

        main="LST at station locations by month",

        ylab="LST (monthly)", xlab="LST (2-month Mean)",cex=.5,pch=16,auto.key=list(space="right"),scales=list(x=list(rot=45)),

        layout=c(2,5))

```

# Processing Options

## Solution 1 (relatively easy)

1. Spatial interpolation: fill in LST when another observation is available within  some small distance (3? 5? 7? kilometers)

2. Temporal interpolation: Estimate the monthly value at the 15th day by linear interpolation (in the temporal domain) considering the temporally closest observations in the previous and following month.

This is more robust than the simple mean of 3 months.  For example, imagine using a simple mean in the following case:  

* month -1: no observation 

* month  0: no observation                                     

* month +1: observation in the end of the month

Would lead to a prediction = month +1.

Or, alternatively, one in which:

* month -1: has data for the full month

* month 0: has no data

* month +1: has only data near end of month

In this case a mean would be 'pulled' towards the mean value of month-1.  Using a fixed number of observations in t-1 and t+1 and a linear interpolation that accounts for the time of those observations would prevent 

## New Solution 2 (also relatively easy)

1. Create 12 monthly LST climatologies if at least 5 or 10 observations are available for each pixel-month.

2. Fill in missing months using temporal interpolation:

   * Fit function through all available months (rather than just +/- 1 month) to capture seasonal variabilty.

   * Use some function that can capture the cyclical nature (sin, spline, polynomial, etc.)

Remember from above that the LST curves are fairly smooth (caveat: we've only looked at locations in the CONUS dataset).  So let's try fitting a simple sin function to estimate missing months:

```{r,echo=T}

### Curve fitting function

### Set parameter starting values and limits

parms=list(phi=5,A=.05)  #starting parameters

low=list(phi=-999,A=0)  #lower bounds - constrain s3 to be positive 

high=list(phi=999,A=999)  #lower bounds - constrain s3 to be positive

## define a simple model estimating a sin function with variable shift and amplitude

fit.model<-function(parms,x) {

      fit=sin(parms[["phi"]]+(x*2*pi))*abs(parms[["A"]])

      fit[fit==Inf]=.Machine$double.xmax  #if Inf, replace with maximum value

      fit[fit==-Inf]=.Machine$double.xmin  #if Inf, replace with maximum value

    list(fit=fit)

  } 

## function to minimize the RMSE of predictions

 minimize.model<-function(parms,x,y) {

    ss=sum((y-fit.model(parms,x)$fit)^2)

    ss=min(ss,.Machine$double.xmax)  #if Inf, replace with maximum value

    ss=max(ss,.Machine$double.xmin)  #if -Inf, replace with maximum value

    list(ss=ss)

  }

## Process station data and make predictions for each month

makepreds=function(dat,drop=NULL){

  dat=na.omit(dat)

  dat=dat[dat$variable=="lst",]

  ## drop

  if(nrow(dat)==0){ return(NULL)}

  ## drop some proportion of data if drop is specified

  if(!is.null(drop)) dat=dat[sample(1:nrow(dat),round(nrow(dat)*(1-drop))),]

  dat=dat[order(dat$month),]

  dat$time=(1:nrow(dat))/nrow(dat)

  xbar=mean(dat$value,na.rm=T)

  fit=optim(unlist(parms),minimize.model,

            x=dat$time,y=dat$value-xbar,

            control=list(trace=0,maxit=10000),method="L-BFGS-B",lower=low,upper=high)

  dat$pred=xbar+sin(fit$par[["phi"]]+(dat$time*2*pi))*fit$par[["A"]]

  dat$A=fit$par[["A"]]

  dat$phi=fit$par[["phi"]]

  dat

}

## apply the function

d4=ddply(dlst2ls,.(station),.fun=makepreds)

d4l=melt(d4,id.vars=c("station","month"),measure.vars=c("value","pred"))

```

Let's see what that looks  for the 10 example stations above. The red line is the observed LST value and the black line is the predicted LST from the sinusoidal function.

```{r,echo=FALSE}

xyplot (value~month|station, data=d4l, groups=variable,type="l",

        main="LST at station locations by month",

        ylab="LST (monthly)", xlab="LST (2-month Mean)",cex=.5,pch=16,auto.key=list(space="right"),scales=list(x=list(rot=45)))

```

And a histogram of the residuals for these stations:

```{r,echo=FALSE,fig.height=3}

d4$anomoly=d4$value-d4$pred

hist(d4$anomoly,col="grey",main="Histogram of residuals for predictions from sinusoidal model",xlab="Degrees C")

```

Not bad... Now let's drop 25% of the observations from each station and do it again:

```{r,echo=F}

## apply the function

d4a=ddply(dlst2ls,.(station),.fun=makepreds,drop=.25)

d4a$anomoly=d4a$value-d4a$pred

d4al=melt(d4a,id.vars=c("station","month"),measure.vars=c("value","pred"))

xyplot (value~month|station, data=d4al, groups=variable,type="l",

        main="LST at station locations by month",

        ylab="LST (monthly)", xlab="LST (2-month Mean)",cex=.5,pch=16,auto.key=list(space="right"),scales=list(x=list(rot=45)))

```

And the  of residuals for these 10 stations with 25% of the observations removed:

```{r,echo=FALSE,fig.height=3}

hist(d4a$anomoly,col="grey",main="Histogram of residuals for predictions from sinusoidal model",xlab="Degrees C")

```

### Apply this function to the full CONUS dataset described above.

Now look at the distributions of the residuals for the full conus dataset.

```{r,echo=FALSE,fig.height=3}

## apply the function

d5=ddply(dlst2l,.(station),.fun=makepreds,drop=.25)

d5$anomoly=d5$value-d5$pred

hist(d5$anomoly,col="grey",main="Histogram of residuals for predictions from sinusoidal model",xlab="Degrees C")

```

And summarize the residuals into RMSE's by station:

```{r, echo=FALSE,fig.height=3}

d5_rmse=ddply(d5,.(station),.fun=function(dat){

  sqrt(mean( (dat$pred - dat$value)^2, na.rm = TRUE))

})

hist(d5_rmse$V1,col="grey",main="Distribution of station RMSE's for predictions from sinusoidal model",xlab="RMSE")

```

So the vast majority of stations will have a RMSE of <5 using this simple method even if 25% of the data are missing.

# Next steps  

We should pick 3-4 problematic tiles and 1 tile with little missing data and explore how these various options differ in infilling LST.  It would also be ideal if we could complete the interpolation using these different datasets and compare the resulting estimates of TMax over these tiles. 

#  Remaining Questions  

1. How does this variability compare to spatial variability (e.g. which is worse, spatial or temporal smoothing)?

2. How do the anomolies from different temporal windows vary spatially?