1
|
#!/usr/bin/python
|
2
|
#
|
3
|
# Early version of assorted Python code to download MODIS 11A1 (daily)
|
4
|
# data associated with specified dates and tiles, then interface with
|
5
|
# GRASS to calculate temporally averaged LST in a way that accounts for
|
6
|
# QC flags.
|
7
|
#
|
8
|
# TODO:
|
9
|
# - functionalize to encapsulate high level procedural steps
|
10
|
# - create a 'main' function to allow script to be run as a command
|
11
|
# that can accept arguments?
|
12
|
# - put downloaded data somewhere other than working directory?
|
13
|
# - write code to set up and take down a temporary GRASS location/mapset
|
14
|
# - write code to export climatology rasters to file (geotiff? compressed?)
|
15
|
# - calculate other aggregate stats? (stddev, ...)
|
16
|
# - deal with nightly LST too?
|
17
|
# - deal with more than just first two bits of QC flags?
|
18
|
# e.g.: r.mapcalc 'qc_level2 = (qc>>2) & 0x03'
|
19
|
#
|
20
|
# Jim Regetz
|
21
|
# NCEAS
|
22
|
# Created on 16-May-2012
|
23
|
|
24
|
import os
|
25
|
import datetime
|
26
|
import ftplib
|
27
|
import grass.script as gs
|
28
|
|
29
|
#------------------
|
30
|
# helper functions
|
31
|
#------------------
|
32
|
|
33
|
# quick function to reformat e.g. 200065 to 2000.03.05
|
34
|
def yj_to_ymd(year, doy):
|
35
|
date = datetime.datetime.strptime('%d%d' % (year, doy), '%Y%j')
|
36
|
return date.strftime('%Y.%m.%d')
|
37
|
|
38
|
# quick function to return list of dirs in wd
|
39
|
def list_contents():
|
40
|
listing = []
|
41
|
ftp.dir(listing.append)
|
42
|
contents = [item.split()[-1] for item in listing[1:]]
|
43
|
return contents
|
44
|
|
45
|
# create raster of pixelwise N for a list of input rasters
|
46
|
def calc_nobs(maplist, name, overwrite=False):
|
47
|
denominator = '(%s)' % '+'.join(['if(!isnull(%s))' % m
|
48
|
for m in maplist])
|
49
|
gs.mapcalc('nobs_%s = %s' % (name, denominator), overwrite=overwrite)
|
50
|
|
51
|
# create raster of pixelwise means for a list of input rasters
|
52
|
def calc_mean(maplist, name, overwrite=False):
|
53
|
numerator = '(%s)' % '+'.join(['if(isnull(%s), 0, %s)' % (m, m)
|
54
|
for m in maplist])
|
55
|
denominator = '(%s)' % '+'.join(['if(!isnull(%s))' % m
|
56
|
for m in maplist])
|
57
|
gs.mapcalc('mean_%s = %s/%s' % (name, numerator, denominator),
|
58
|
overwrite=overwrite)
|
59
|
|
60
|
# ...combined version of the above two functions...
|
61
|
def calc_clim(maplist, name, overwrite=False):
|
62
|
denominator = '(%s)' % '+'.join(['if(!isnull(%s))' % m
|
63
|
for m in maplist])
|
64
|
gs.mapcalc('nobs_%s = %s' % (name, denominator), overwrite=overwrite)
|
65
|
numerator = '(%s)' % '+'.join(['if(isnull(%s), 0, %s)' % (m, m)
|
66
|
for m in maplist])
|
67
|
#gs.mapcalc('mean_%s = round(float(%s)/nobs_%s)' % (name, numerator, name),
|
68
|
gs.mapcalc('mean_%s = %s/nobs_%s' % (name, numerator, name),
|
69
|
overwrite=overwrite)
|
70
|
|
71
|
def load_qc_adjusted_lst(hdf):
|
72
|
"""Load LST_Day_1km into GRASS from the specified hdf file, and
|
73
|
nullify any cells for which QA flags indicate anything other than
|
74
|
high quality.
|
75
|
|
76
|
Argument:
|
77
|
hdf -- local path to the 11A1 HDF file
|
78
|
|
79
|
Returns the name of the QC-adjusted LST map created in GRASS.
|
80
|
"""
|
81
|
lstname = 'LST_' + '_'.join(os.path.basename(hdf).split('.')[1:3])
|
82
|
# read in lst grid
|
83
|
gs.run_command('r.in.gdal',
|
84
|
input='HDF4_EOS:EOS_GRID:%s:MODIS_Grid_Daily_1km_LST:LST_Day_1km' % hdf,
|
85
|
output=lstname)
|
86
|
# read in qc grid
|
87
|
gs.run_command('r.in.gdal',
|
88
|
input = 'HDF4_EOS:EOS_GRID:%s:MODIS_Grid_Daily_1km_LST:QC_Day' % hdf,
|
89
|
output = 'qc_this_day')
|
90
|
gs.run_command('g.region', rast=lstname)
|
91
|
# null out any LST cells for which QC is not high [00]
|
92
|
gs.mapcalc('${lst} = if((qc_this_day & 0x03)==0, ${lst}, null())',
|
93
|
lst=lstname, overwrite=True)
|
94
|
# clean up
|
95
|
gs.run_command('g.remove', rast='qc_this_day')
|
96
|
# return name of qc-adjusted LST raster in GRASS
|
97
|
return lstname
|
98
|
|
99
|
#----------------------------------------
|
100
|
# set up a (temporary?) GRASS data store
|
101
|
#----------------------------------------
|
102
|
|
103
|
## TODO ##
|
104
|
|
105
|
# hack to fix datum???
|
106
|
gs.run_command('g.proj', flags='c',
|
107
|
proj4='+proj=sinu +a=6371007.181 +b=6371007.181 +ellps=sphere')
|
108
|
# proj4='+proj=sinu +R=6371007.181 +nadgrids=@null +wktext')
|
109
|
|
110
|
#-------------------------
|
111
|
# test download & process
|
112
|
#-------------------------
|
113
|
|
114
|
tile = 'h09v04'
|
115
|
year = 2000
|
116
|
start_doy = 122
|
117
|
end_doy = 152
|
118
|
|
119
|
# connect to data pool and change to the right directory
|
120
|
ftp = ftplib.FTP('e4ftl01.cr.usgs.gov')
|
121
|
ftp.login('anonymous', '')
|
122
|
ftp.cwd('MOLT/MOD11A1.005')
|
123
|
|
124
|
# make list of daily directories to traverse
|
125
|
day_dirs = listdirs()
|
126
|
days_to_get = [yj_to_ymd(year, x) for x in range(start_doy, end_doy+1)]
|
127
|
if not all [day in day_dirs for day in days_to_get]:
|
128
|
error
|
129
|
|
130
|
# using v004 data, this was taking 60 seconds for the 8 hdf/xml pairs on
|
131
|
# atlas [16-May-2012] ... each individual hdf is ~23MB
|
132
|
hdfs = list()
|
133
|
for day in days_to_get:
|
134
|
ftp.cwd(day)
|
135
|
files_to_get = [file for file in list_contents()
|
136
|
if tile in file and file[-3:]=='hdf']
|
137
|
if len(files_to_get)==0:
|
138
|
# no file found -- print message and move on
|
139
|
print 'no hdf found on server for tile', tile, 'on', day
|
140
|
continue
|
141
|
elif 1<len(files_to_get):
|
142
|
# multiple files found! -- just use the first...
|
143
|
print 'multiple hdfs found on server for tile', tile, 'on', day
|
144
|
file = files_to_get[0]
|
145
|
ftp.retrbinary('RETR %s' % file, open(file, 'wb').write)
|
146
|
hdfs.append(file)
|
147
|
ftp.cwd('..')
|
148
|
|
149
|
# politely disconnect
|
150
|
ftp.quit()
|
151
|
|
152
|
|
153
|
#-----------------------------------------------------------------------
|
154
|
gs.os.environ['GRASS_OVERWRITE'] = '1'
|
155
|
# load and qc-adjust all daily LSTs, return list of the GRASS map names
|
156
|
LST = [load_qc_adjusted_lst(hdf) for hdf in hdfs]
|
157
|
# calculate mean and nobs rasters
|
158
|
calc_clim(LST, 'LST_%s_%d_%03d_%03d' % (tile, year, start_doy, end_doy))
|
159
|
# clean up
|
160
|
gs.run_command('g.remove', rast=','.join(LST))
|
161
|
gs.os.environ['GRASS_OVERWRITE'] = '0'
|
162
|
#-----------------------------------------------------------------------
|
163
|
|
164
|
|
165
|
#
|
166
|
# test 31-day mean with local files
|
167
|
#
|
168
|
|
169
|
# Note that some statements below are redundant with (and possibly
|
170
|
# slightly different) from code above -- still need to consolidate
|
171
|
|
172
|
start_doy = 122
|
173
|
end_doy = 152
|
174
|
|
175
|
gs.os.environ['GRASS_OVERWRITE'] = '1'
|
176
|
ordir = '/home/layers/data/climate/MOD11A1.004-OR-orig-2000'
|
177
|
LST = list()
|
178
|
for doy in range(start_doy, end_doy+1):
|
179
|
fileglob = 'MOD11A1.A%d%03d.%s*hdf' % (year, doy, tile)
|
180
|
try:
|
181
|
hdfpath = glob.glob(os.path.join(ordir, fileglob))[0]
|
182
|
except:
|
183
|
print '*** problem finding', fileglob, '***'
|
184
|
continue
|
185
|
hdfname = os.path.basename(hdfpath)
|
186
|
LST.append('LST_' + '_'.join(hdfname.split('.')[1:3]))
|
187
|
# TODO: handle failures here...
|
188
|
gs.run_command('r.in.gdal',
|
189
|
input='HDF4_EOS:EOS_GRID:%s:MODIS_Grid_Daily_1km_LST:LST_Day_1km' % hdfpath,
|
190
|
output=LST[-1])
|
191
|
gs.os.environ['GRASS_OVERWRITE'] = '0'
|
192
|
gs.run_command('g.region', rast=LST[0])
|
193
|
#numerator = '(%s)' % '+'.join(['if(isnull(%s), 0, %s)' % (lst, lst) for lst in LST])
|
194
|
#denominator = '(%s)' % '+'.join(['if(!isnull(%s))' % lst for lst in LST])
|
195
|
#gs.mapcalc('LST_mean_%s = %s / %s' % (tile, numerator, denominator))
|
196
|
calc_mean(LST, 'LST_mean_%s_%d_%03d_%03d' % (tile, year, start_doy, end_doy))
|
197
|
# clean up
|
198
|
gs.run_command('g.remove', rast=','.join(LST))
|