| 1 |
35d59dc1
|
Adam M. Wilson @ pfe
|
### Process a folder of daily MOD35 HDF files to produce a climatology
|
| 2 |
|
|
|
| 3 |
aba23d60
|
Adam M. Wilson @ pfe
|
.libPaths("/pleiades/u/awilso10/R/x86_64-unknown-linux-gnu-library/2.15/")
|
| 4 |
|
|
|
| 5 |
35d59dc1
|
Adam M. Wilson @ pfe
|
## import commandline arguments
|
| 6 |
aba23d60
|
Adam M. Wilson @ pfe
|
library(getopt,lib="/pleiades/u/awilso10/R/x86_64-unknown-linux-gnu-library/2.15/")
|
| 7 |
35d59dc1
|
Adam M. Wilson @ pfe
|
## get options
|
| 8 |
|
|
opta <- getopt(matrix(c(
|
| 9 |
|
|
'tile', 't', 1, 'character',
|
| 10 |
|
|
'verbose','v',1,'logical'
|
| 11 |
|
|
), ncol=4, byrow=TRUE))
|
| 12 |
|
|
|
| 13 |
b3344197
|
Adam M. Wilson @ pfe
|
tile=opta$tile #tile="h00v08"
|
| 14 |
35d59dc1
|
Adam M. Wilson @ pfe
|
verbose=opta$verbose #print out extensive information for debugging?
|
| 15 |
|
|
|
| 16 |
aba23d60
|
Adam M. Wilson @ pfe
|
## set working directory
|
| 17 |
b3344197
|
Adam M. Wilson @ pfe
|
setwd("/nobackupp1/awilso10/mod35")
|
| 18 |
|
|
#setwd("/u/awilso10/MOD35")
|
| 19 |
aba23d60
|
Adam M. Wilson @ pfe
|
|
| 20 |
35d59dc1
|
Adam M. Wilson @ pfe
|
### directory containing daily files
|
| 21 |
|
|
outdir=paste("daily/",tile,"/",sep="") #directory for separate daily files
|
| 22 |
|
|
|
| 23 |
|
|
### directory to hold climatology
|
| 24 |
|
|
outdir2="summary" #directory for combined daily files and summarized files
|
| 25 |
|
|
if(!file.exists(outdir2)) dir.create(outdir2)
|
| 26 |
|
|
|
| 27 |
|
|
### path to NCO
|
| 28 |
|
|
ncopath="/nasa/sles11/nco/4.0.8/gcc/mpt/bin/"
|
| 29 |
|
|
|
| 30 |
|
|
################################################################################
|
| 31 |
|
|
## Get list of all daily files
|
| 32 |
be64daa8
|
Adam M. Wilson @ pfe
|
if(verbose) print(paste("Checking daily output in preparation for generating climatology:",tile))
|
| 33 |
35d59dc1
|
Adam M. Wilson @ pfe
|
|
| 34 |
|
|
fdly=data.frame(path=list.files(outdir,pattern="nc$",full=T),stringsAsFactors=F)
|
| 35 |
|
|
fdly$file=basename(fdly$path)
|
| 36 |
|
|
fdly$dateid=substr(fdly$file,14,21)
|
| 37 |
d91f0663
|
Adam M. Wilson @ pfe
|
fdly$date=as.Date(substr(fdly$file,14,21),"%Y%m%d") #23,30 for alltiles
|
| 38 |
35d59dc1
|
Adam M. Wilson @ pfe
|
fdly$month=format(fdly$date,"%m")
|
| 39 |
|
|
fdly$year=format(fdly$date,"%Y")
|
| 40 |
|
|
nrow(fdly)
|
| 41 |
|
|
|
| 42 |
|
|
## print some summaries
|
| 43 |
be64daa8
|
Adam M. Wilson @ pfe
|
if(verbose) print(paste("Summary of available daily files:",tile))
|
| 44 |
35d59dc1
|
Adam M. Wilson @ pfe
|
print(table(fdly$year))
|
| 45 |
|
|
print(table(fdly$month))
|
| 46 |
|
|
#print(table(fdly$fvar))
|
| 47 |
|
|
|
| 48 |
|
|
#################################################################################
|
| 49 |
|
|
## Combine the year-by-year files into a single daily file in the summary directory (for archiving)
|
| 50 |
be64daa8
|
Adam M. Wilson @ pfe
|
if(verbose) print(paste("Merging daily files into single file output:",tile))
|
| 51 |
35d59dc1
|
Adam M. Wilson @ pfe
|
|
| 52 |
|
|
## create temporary directory to put intermediate files (will be deleted when R quits)
|
| 53 |
|
|
tsdir=paste(tempdir(),"/summary",sep="")
|
| 54 |
|
|
if(!file.exists(tsdir)) dir.create(tsdir,recursive=T)
|
| 55 |
|
|
|
| 56 |
|
|
## merge all daily files to create a single file with all dates
|
| 57 |
|
|
system(paste(ncopath,"ncrcat -O ",outdir,"/*nc ",outdir2,"/MOD35_",tile,"_daily.nc",sep=""))
|
| 58 |
d91f0663
|
Adam M. Wilson @ pfe
|
#system(paste(ncopath,"ncrcat -O ",outdir,"/*alltests*nc ",outdir2,"/MOD35_",tile,"_daily_alltiles.nc",sep=""))
|
| 59 |
e4e30b86
|
Adam M. Wilson @ pfe
|
system(paste("ncdump -h ",outdir2,"/MOD35_",tile,"_daily.nc",sep=""))
|
| 60 |
1523a1e7
|
Adam M. Wilson
|
|
| 61 |
35d59dc1
|
Adam M. Wilson @ pfe
|
## Update attributes
|
| 62 |
|
|
system(paste(ncopath,"ncatted ",
|
| 63 |
b3344197
|
Adam M. Wilson @ pfe
|
" -a title,global,o,c,\"MODIS Cloud Product (MOD35) Summaries\" ",
|
| 64 |
35d59dc1
|
Adam M. Wilson @ pfe
|
" -a institution,global,o,c,\"Yale University\" ",
|
| 65 |
b3344197
|
Adam M. Wilson @ pfe
|
" -a source,global,o,c,\"MODIS Collection 6 Cloud Mask (MOD35)\" ",
|
| 66 |
35d59dc1
|
Adam M. Wilson @ pfe
|
" -a comment,global,o,c,\"Compiled by Adam M. Wilson (adam.wilson@yale.edu)\" ",
|
| 67 |
|
|
outdir2,"/MOD35_",tile,"_daily.nc",sep=""))
|
| 68 |
|
|
|
| 69 |
78f94710
|
Adam M. Wilson
|
## report on daily file:
|
| 70 |
|
|
#ncfile=paste(outdir2,"/MOD35_",tile,"_daily.nc",sep="")
|
| 71 |
|
|
system(paste("ncdump -h ",outdir2,"/MOD35_",tile,"_daily.nc | head -20 ",sep=""))
|
| 72 |
|
|
system(paste("cdo showyear ",outdir2,"/MOD35_",tile,"_daily.nc",sep=""))
|
| 73 |
|
|
system(paste("cdo showmon ",outdir2,"/MOD35_",tile,"_daily.nc",sep=""))
|
| 74 |
|
|
#system(paste("cdo showdate ",outdir2,"/MOD35_",tile,"_daily.nc",sep=""))
|
| 75 |
|
|
|
| 76 |
|
|
|
| 77 |
35d59dc1
|
Adam M. Wilson @ pfe
|
### produce a monthly timeseries?
|
| 78 |
|
|
#system(paste("cdo -O monmean ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_monmean.nc",sep=""))
|
| 79 |
|
|
|
| 80 |
|
|
#############################
|
| 81 |
|
|
## Generate the Climatologies
|
| 82 |
be64daa8
|
Adam M. Wilson @ pfe
|
if(verbose) print(paste("Generate monthly climatologies: ",tile))
|
| 83 |
35d59dc1
|
Adam M. Wilson @ pfe
|
|
| 84 |
|
|
myear=as.integer(max(fdly$year)) #this year will be used in all dates of monthly climatologies (and day will = 15)
|
| 85 |
|
|
|
| 86 |
b3344197
|
Adam M. Wilson @ pfe
|
## Overall Means
|
| 87 |
|
|
if(verbose) print(paste("Calculating the overall mean:",tile))
|
| 88 |
e4e30b86
|
Adam M. Wilson @ pfe
|
system(paste("cdo -O -b I8 -v sorttimestamp -setyear,",myear," -setmon,1 -setday,1 -mulc,100 -timmean -lec,1 ",outdir2,"/MOD35_",tile,"_daily.nc ",outdir2,"/MOD35_",tile,"_mean.nc",sep=""),wait=T)
|
| 89 |
|
|
system(paste(ncopath,"ncrename -v CMday,CFday -v CMnight,CFnight ",outdir2,"/MOD35_",tile,"_mean.nc",sep=""))
|
| 90 |
b3344197
|
Adam M. Wilson @ pfe
|
system(paste(ncopath,"ncatted ",
|
| 91 |
e4e30b86
|
Adam M. Wilson @ pfe
|
" -a long_name,CFday,o,c,\"Daytime Cloud Frequency\" ",
|
| 92 |
|
|
" -a missing_value,CFday,o,b,255 ",
|
| 93 |
|
|
" -a _FillValue,CFday,d,b,255 ",
|
| 94 |
|
|
" -a long_name,CFnight,o,c,\"Nighttime Cloud Frequency\" ",
|
| 95 |
|
|
" -a missing_value,CFnight,o,b,255 ",
|
| 96 |
|
|
" -a _FillValue,CFnight,d,b,255 ",
|
| 97 |
|
|
outdir2,"/MOD35_",tile,"_mean.nc",sep=""))
|
| 98 |
b3344197
|
Adam M. Wilson @ pfe
|
|
| 99 |
35d59dc1
|
Adam M. Wilson @ pfe
|
## Monthly means
|
| 100 |
be64daa8
|
Adam M. Wilson @ pfe
|
if(verbose) print(paste("Calculating the monthly means:",tile))
|
| 101 |
e4e30b86
|
Adam M. Wilson @ pfe
|
system(paste("cdo -O -b I8 sorttimestamp -setyear,",myear," -setday,15 -mulc,100 -ymonmean -lec,1 ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""),wait=T)
|
| 102 |
|
|
system(paste(ncopath,"ncrename -v CMday,CFday -v CMnight,CFnight ",tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""))
|
| 103 |
c24e32a8
|
Adam M. Wilson @ pfe
|
system(paste(ncopath,"ncatted ",
|
| 104 |
e4e30b86
|
Adam M. Wilson @ pfe
|
" -a long_name,CFday,o,c,\"Daytime Cloud Frequency\" ",
|
| 105 |
|
|
" -a units,CFday,o,c,\"Proportion (%)\" ",
|
| 106 |
|
|
" -a missing_value,CFday,o,b,255 ",
|
| 107 |
|
|
" -a _FillValue,CFday,d,b,255 ",
|
| 108 |
|
|
" -a long_name,CFnight,o,c,\"Nighttime Cloud Frequency\" ",
|
| 109 |
|
|
" -a units,CFnight,o,c,\"Proportion (%)\" ",
|
| 110 |
|
|
" -a missing_value,CFnight,o,b,255 ",
|
| 111 |
|
|
" -a _FillValue,CFnight,d,b,255 ",
|
| 112 |
c24e32a8
|
Adam M. Wilson @ pfe
|
tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""))
|
| 113 |
b83b17ae
|
Adam M. Wilson
|
|
| 114 |
|
|
|
| 115 |
c24e32a8
|
Adam M. Wilson @ pfe
|
## Monthly Mean
|
| 116 |
|
|
#months=c("01","02","03","04","05","06","07","08","09","10","11","12")
|
| 117 |
|
|
# month="02"
|
| 118 |
|
|
|
| 119 |
aba23d60
|
Adam M. Wilson @ pfe
|
#system(paste("cdo -O sorttimestamp -setyear,",myear," -mulc,-1 -subc,100 -ydrunmean,30 ",outdir2,"/MOD35_",tile,"_daily.nc ",outdir2,"/MOD35_",tile,"_ydrunmean30.nc &",sep=""),wait=T)
|
| 120 |
c24e32a8
|
Adam M. Wilson @ pfe
|
#system(paste("scp summary/MOD35_",tile,".nc adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod35/",sep=""))
|
| 121 |
|
|
#system(paste("ncdump -h ",tsdir,"/MOD35_",tile,"_ymonmean.nc ",sep=""))
|
| 122 |
b83b17ae
|
Adam M. Wilson
|
|
| 123 |
35d59dc1
|
Adam M. Wilson @ pfe
|
## Monthly standard deviation
|
| 124 |
be64daa8
|
Adam M. Wilson @ pfe
|
if(verbose) print(paste("Calculating the monthly SD:",tile))
|
| 125 |
e4e30b86
|
Adam M. Wilson @ pfe
|
system(paste("cdo -O -b I8 sorttimestamp -setyear,",myear," -setday,15 -ymonstd -mulc,100 -monmean -lec,1 ",
|
| 126 |
aba23d60
|
Adam M. Wilson @ pfe
|
outdir2,"/MOD35_",tile,"_daily.nc ",
|
| 127 |
|
|
tsdir,"/MOD35_",tile,"_ymonstd.nc",sep=""))
|
| 128 |
e4e30b86
|
Adam M. Wilson @ pfe
|
system(paste(ncopath,"ncrename -v CMday,CFday_sd -v CMnight,CFnight_sd ",tsdir,"/MOD35_",tile,"_ymonstd.nc",sep=""))
|
| 129 |
35d59dc1
|
Adam M. Wilson @ pfe
|
system(paste(ncopath,"ncatted ",
|
| 130 |
e4e30b86
|
Adam M. Wilson @ pfe
|
" -a long_name,CFnight_sd,o,c,\"Standard Deviation of monthly nighttime cloud frequency\" ",
|
| 131 |
|
|
" -a long_name,CFday_sd,o,c,\"Standard Deviation of monthly daytime cloud frequency\" ",
|
| 132 |
35d59dc1
|
Adam M. Wilson @ pfe
|
tsdir,"/MOD35_",tile,"_ymonstd.nc",sep=""))
|
| 133 |
|
|
|
| 134 |
1523a1e7
|
Adam M. Wilson
|
## frequency of cloud days p(clear<90%)
|
| 135 |
e4e30b86
|
Adam M. Wilson @ pfe
|
#if(verbose) print(paste("Calculating the proportion of cloudy and probably cloudy days:",tile))
|
| 136 |
|
|
#system(paste("cdo -O -b I8 sorttimestamp -setyear,",myear," -setday,15 -ymonmean -mulc,100 -lec,90 -selvar,PClear ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymoncld01.nc",sep=""))
|
| 137 |
|
|
#system(paste(ncopath,"ncrename -v PClear,CF ",tsdir,"/MOD35_",tile,"_ymoncld01.nc",sep=""))
|
| 138 |
|
|
#system(paste(ncopath,"ncatted ",
|
| 139 |
|
|
#" -a long_name,CF,o,c,\"Cloud Frequency: Proportion of Days with probability of clear < 90%\" ",
|
| 140 |
|
|
#" -a units,CF,o,c,\"Proportion (%)\" ",
|
| 141 |
|
|
#tsdir,"/MOD35_",tile,"_ymoncld01.nc",sep=""))
|
| 142 |
35d59dc1
|
Adam M. Wilson @ pfe
|
|
| 143 |
|
|
## number of observations
|
| 144 |
be64daa8
|
Adam M. Wilson @ pfe
|
if(verbose) print(paste("Calculating the number of missing variables:",tile))
|
| 145 |
e4e30b86
|
Adam M. Wilson @ pfe
|
system(paste("cdo -O -b I8 sorttimestamp -setyear,",myear," -setday,15 -ymonmean -mulc,100 -eqc,9999 -setmisstoc,9999 ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymonmiss.nc",sep=""))
|
| 146 |
|
|
system(paste(ncopath,"ncrename -v CMday,CFday_pmiss -v CMnight,CFnight_pmiss ",tsdir,"/MOD35_",tile,"_ymonmiss.nc",sep=""))
|
| 147 |
35d59dc1
|
Adam M. Wilson @ pfe
|
system(paste(ncopath,"ncatted ",
|
| 148 |
e4e30b86
|
Adam M. Wilson @ pfe
|
" -a long_name,CFday_pmiss,o,c,\"Proportion of Days with missing data\" ",
|
| 149 |
|
|
" -a units,CFday_pmiss,o,c,\"Proportion (%)\" ",
|
| 150 |
|
|
" -a long_name,CFnight_pmiss,o,c,\"Proportion of Days with missing data\" ",
|
| 151 |
|
|
" -a units,CFnight_pmiss,o,c,\"Proportion (%)\" ",
|
| 152 |
78f94710
|
Adam M. Wilson
|
tsdir,"/MOD35_",tile,"_ymonmiss.nc",sep=""))
|
| 153 |
35d59dc1
|
Adam M. Wilson @ pfe
|
|
| 154 |
|
|
## TODO: fix projection information so GDAL can read it correctly.
|
| 155 |
|
|
## clean up variables?
|
| 156 |
|
|
|
| 157 |
|
|
## append variables to a single file
|
| 158 |
be64daa8
|
Adam M. Wilson @ pfe
|
if(verbose) print(paste("Append all monthly climatologies into a single file:",tile))
|
| 159 |
35d59dc1
|
Adam M. Wilson @ pfe
|
system(paste(ncopath,"ncks -O ",tsdir,"/MOD35_",tile,"_ymonmean.nc ",tsdir,"/MOD35_",tile,"_ymon.nc",sep=""))
|
| 160 |
|
|
system(paste(ncopath,"ncks -A ",tsdir,"/MOD35_",tile,"_ymonstd.nc ",tsdir,"/MOD35_",tile,"_ymon.nc",sep=""))
|
| 161 |
e4e30b86
|
Adam M. Wilson @ pfe
|
#system(paste(ncopath,"ncks -A ",tsdir,"/MOD35_",tile,"_ymoncld01.nc ",tsdir,"/MOD35_",tile,"_ymon.nc",sep=""))
|
| 162 |
35d59dc1
|
Adam M. Wilson @ pfe
|
system(paste(ncopath,"ncks -A ",tsdir,"/MOD35_",tile,"_ymonmiss.nc ",tsdir,"/MOD35_",tile,"_ymon.nc",sep=""))
|
| 163 |
|
|
|
| 164 |
|
|
## append sinusoidal grid from one of input files as CDO doesn't transfer all attributes
|
| 165 |
be64daa8
|
Adam M. Wilson @ pfe
|
if(verbose) print(paste("Clean up file (update attributes, flip latitudes, add grid description:",tile))
|
| 166 |
35d59dc1
|
Adam M. Wilson @ pfe
|
|
| 167 |
|
|
#system(paste(ncopath,"ncea -d time,0,1 -v sinusoidal ",list.files(outdir,full=T,pattern="[.]nc$")[1]," ",tsdir,"/sinusoidal.nc",sep=""))
|
| 168 |
|
|
#system(paste(ncopath,"ncks -A -d time,0,1 -v sinusoidal ",list.files(outdir,full=T,pattern="[.]nc$")[1]," ",tsdir,"/MOD35_",tile,"_ymon.nc",sep=""))
|
| 169 |
|
|
|
| 170 |
|
|
## invert latitude so it plays nicely with gdal
|
| 171 |
|
|
system(paste(ncopath,"ncpdq -O -a -y ",tsdir,"/MOD35_",tile,"_ymon.nc ",outdir2,"/MOD35_",tile,".nc",sep=""))
|
| 172 |
|
|
|
| 173 |
b716a3f0
|
Adam M. Wilson @ pfe
|
## proj string taken from GDAL-written MODIS tile
|
| 174 |
|
|
projstring="PROJCS[\"Sinusoidal (Sanson-Flamsteed)\",GEOGCS[\"wgs84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS_1984\",6378137,298.257223563],TOWGS84[0,0,0,0,0,0,0]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433]],PROJECTION[\"Sinusoidal\"],PARAMETER[\"longitude_of_center\",0],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"Meter\",1]]"
|
| 175 |
|
|
|
| 176 |
35d59dc1
|
Adam M. Wilson @ pfe
|
## update attributes
|
| 177 |
|
|
system(paste(ncopath,"ncatted ",
|
| 178 |
b716a3f0
|
Adam M. Wilson @ pfe
|
" -a false_easting,sinusoidal,o,d,0. ",
|
| 179 |
|
|
" -a false_northing,sinusoidal,o,d,0. ",
|
| 180 |
|
|
" -a longitude_of_central_meridian,sinusoidal,o,d,0. ",
|
| 181 |
|
|
" -a longitude_of_prime_meridian,sinusoidal,o,d,0. ",
|
| 182 |
|
|
" -a semi_major_axis,sinusoidal,o,d,6378137. ",
|
| 183 |
|
|
" -a inverse_flattening,sinusoidal,o,d,298.257223563 ",
|
| 184 |
|
|
" -a spatial_ref,sinusoidal,o,c,",projstring,
|
| 185 |
|
|
" -a GeoTransform,sinusoidal,o,c,\"-7783653.638366 926.6254331391661 0 1111950.519767 0 -926.6254331391667\" ",
|
| 186 |
35d59dc1
|
Adam M. Wilson @ pfe
|
" -a units,time,o,c,\"days since 2000-1-1 0:0:0\" ",
|
| 187 |
|
|
" -a title,global,o,c,\"MODIS Cloud Product (MOD35) Climatology\" ",
|
| 188 |
b716a3f0
|
Adam M. Wilson @ pfe
|
" -a tile,global,o,c,\"",tile,"\" ",
|
| 189 |
35d59dc1
|
Adam M. Wilson @ pfe
|
" -a institution,global,o,c,\"Yale University\" ",
|
| 190 |
1523a1e7
|
Adam M. Wilson
|
" -a source,global,o,c,\"MODIS Cloud Product (MOD35) Collection 6\" ",
|
| 191 |
35d59dc1
|
Adam M. Wilson @ pfe
|
" -a comment,global,o,c,\"Compiled by Adam M. Wilson (adam.wilson@yale.edu)\" ",
|
| 192 |
|
|
outdir2,"/MOD35_",tile,".nc",sep=""))
|
| 193 |
|
|
|
| 194 |
|
|
|
| 195 |
|
|
print(paste("############################### Processed ",nrow(fdly),"days for tile:",tile," ###################################################"))
|
| 196 |
|
|
print("Years:")
|
| 197 |
|
|
print(table(fdly$fyear))
|
| 198 |
|
|
print("Months:")
|
| 199 |
|
|
print(table(fdly$fmonth))
|
| 200 |
1523a1e7
|
Adam M. Wilson
|
|
| 201 |
35d59dc1
|
Adam M. Wilson @ pfe
|
## quit R
|
| 202 |
|
|
q("no")
|
| 203 |
|
|
|
