Revision 2a8747de
Added by Benoit Parmentier over 11 years ago
climate/research/oregon/modis-lst/LST_downloading_and_climatology.py | ||
---|---|---|
23 | 23 |
# - deal with nightly LST too? |
24 | 24 |
# - deal with more than just first two bits of QC flags? |
25 | 25 |
# e.g.: r.mapcalc 'qc_level2 = (qc>>2) & 0x03' |
26 |
# 0x03?? that means 0000_0011 and >>2 means shit twice on the right for |
|
26 |
# 0x03?? that means 0000_0011 and >>2 means shift twice on the right for
|
|
27 | 27 |
# - record all downloads to a log file? |
28 | 28 |
# |
29 | 29 |
# Jim Regetz |
... | ... | |
37 | 37 |
import datetime, calendar |
38 | 38 |
import ftplib |
39 | 39 |
import grass.script as gs |
40 |
import argparse |
|
40 | 41 |
|
41 | 42 |
#from datetime import date |
42 | 43 |
#from datetime import datetime |
... | ... | |
90 | 91 |
|
91 | 92 |
return(date_seq) |
92 | 93 |
|
93 |
#Added on January 16, 2012 by Benoit NCEAS |
|
94 |
def list_raster_per_month(listmaps): |
|
95 |
"""Determine starting and ending numeric day-of-year (doy) |
|
96 |
asscociated with the specified month and year. |
|
97 |
|
|
98 |
Arguments: |
|
99 |
maplist -- list of raster grass-digit integer year |
|
100 |
month -- integer month (1-12) |
|
101 |
year_range -- list of years |
|
102 |
|
|
103 |
Returns tuple of start and end doy for that month/year. |
|
104 |
""" |
|
105 |
list_maps_month=list() |
|
106 |
nb_month=12 |
|
107 |
for i in range(1,nb_month+1): |
|
108 |
#i=i+1 |
|
109 |
#filename = listfiles_wd2[i] #list the files of interest |
|
110 |
#monthlist[:]=[] #empty the list |
|
111 |
monthlist=list() |
|
112 |
#monthlist2[:]=[] #empty the list |
|
113 |
j=0 #counter |
|
114 |
for mapname in listmaps: |
|
115 |
#tmp_str=LST[0] |
|
116 |
date_map=mapname.split('_')[1][1:8] |
|
117 |
#date = filename[filename.rfind('_MOD')-8:filename.rfind('_MOD')] |
|
118 |
d=datetime.datetime.strptime(date_map,'%Y%j') #This produces a date object from a string extracted from the file name. |
|
119 |
month=d.strftime('%m') #Extract the month of the current map |
|
120 |
if int(month)==i: |
|
121 |
#monthlist.append(listmaps[j]) |
|
122 |
monthlist.append(mapname) |
|
123 |
#monthlist2.append(listmaps[j]) |
|
124 |
j=j+1 |
|
125 |
list_maps_month.append(monthlist) #This is a list of a list containing a list for each month |
|
126 |
#list_files2.append(monthlist2) |
|
127 |
return(list_maps_month) |
|
128 |
|
|
94 |
|
|
129 | 95 |
# quick function to return list of dirs in wd |
130 | 96 |
def list_contents(ftp): |
131 | 97 |
"""Parse ftp directory listing into list of names of the files |
... | ... | |
218 | 184 |
hdfs.append(os.path.abspath(files[0])) |
219 | 185 |
return hdfs |
220 | 186 |
|
187 |
#Added on January 16, 2012 by Benoit NCEAS |
|
188 |
def list_raster_per_month(listmaps): |
|
189 |
"""Determine starting and ending numeric day-of-year (doy) |
|
190 |
asscociated with the specified month and year. |
|
191 |
|
|
192 |
Arguments: |
|
193 |
maplist -- list of raster grass-digit integer year |
|
194 |
month -- integer month (1-12) |
|
195 |
year_range -- list of years |
|
196 |
|
|
197 |
Returns tuple of start and end doy for that month/year. |
|
198 |
""" |
|
199 |
list_maps_month=list() |
|
200 |
nb_month=12 |
|
201 |
for i in range(1,nb_month+1): |
|
202 |
#i=i+1 |
|
203 |
#filename = listfiles_wd2[i] #list the files of interest |
|
204 |
#monthlist[:]=[] #empty the list |
|
205 |
monthlist=list() |
|
206 |
#monthlist2[:]=[] #empty the list |
|
207 |
j=0 #counter |
|
208 |
for mapname in listmaps: |
|
209 |
#tmp_str=LST[0] |
|
210 |
date_map=mapname.split('_')[1][1:8] |
|
211 |
#date = filename[filename.rfind('_MOD')-8:filename.rfind('_MOD')] |
|
212 |
d=datetime.datetime.strptime(date_map,'%Y%j') #This produces a date object from a string extracted from the file name. |
|
213 |
month=d.strftime('%m') #Extract the month of the current map |
|
214 |
if int(month)==i: |
|
215 |
#monthlist.append(listmaps[j]) |
|
216 |
monthlist.append(mapname) |
|
217 |
#monthlist2.append(listmaps[j]) |
|
218 |
j=j+1 |
|
219 |
list_maps_month.append(monthlist) #This is a list of a list containing a list for each month |
|
220 |
#list_files2.append(monthlist2) |
|
221 |
return(list_maps_month) |
|
222 |
|
|
223 |
|
|
221 | 224 |
def calc_clim(maplist, name, overwrite=False): |
222 | 225 |
"""Generate some climatalogies in GRASS based on the input list of |
223 | 226 |
maps. As usual, current GRASS region settings apply. Produces the |
... | ... | |
285 | 288 |
# proj4='+proj=sinu +a=6371007.181 +b=6371007.181 +ellps=sphere') |
286 | 289 |
## proj4='+proj=sinu +R=6371007.181 +nadgrids=@null +wktext') |
287 | 290 |
|
291 |
########## START OF SCRIPT ############## |
|
292 |
#### Modified by Benoit on May 13, 2013 |
|
293 |
|
|
294 |
### INPUT Parameters |
|
295 |
#Inputs from R?? there are 9 parameters |
|
296 |
#tiles = ['h11v08','h11v07','h12v07','h12v08','h10v07','h10v08'] #These tiles correspond to Venezuela. |
|
297 |
tiles= ['h12v08'] |
|
298 |
#start_year = 2001 |
|
299 |
#end_year = 2010 |
|
300 |
#start_month=1 |
|
301 |
#end_month=12 |
|
302 |
#hdfdir = '/home/layers/commons/modis/MOD11A1_tiles' #destination file where hdf files are stored locally after download. |
|
303 |
#night=1 # if 1 then produce night climatology |
|
304 |
#out_suffix="_03192013" |
|
305 |
#download=1 # if 1 then download files |
|
288 | 306 |
|
289 |
#### MOdified by Benoit in March 2013 |
|
307 |
parser = argparse.ArgumentParser() |
|
308 |
parser.add_argument("tiles", type=str, help="list of Modis tiles") |
|
309 |
parser.add_argument("start_year", type=int, help="start year") |
|
310 |
parser.add_argument("end_year", type=int, help="end year") |
|
311 |
parser.add_argument("start_month", type=int, help="start month") |
|
312 |
parser.add_argument("end_month", type=int, help="end month") |
|
313 |
parser.add_argument("hdfdir", type=str, help="destination/source directory for hdf file") |
|
314 |
parser.add_argument("night", type=int, help="night") |
|
315 |
parser.add_argument("download", type=int, help="out_suffix") |
|
316 |
parser.add_argument("out_suffix", type=str, help="out_suffix") |
|
317 |
|
|
318 |
myargs = parser.parse_args() |
|
319 |
|
|
320 |
tiles = myargs.tiles #These tiles correspond to Venezuela. |
|
321 |
start_year = myargs.start_year |
|
322 |
end_year = myargs.start_year |
|
323 |
end_month = myargs.end_month |
|
324 |
start_month= myargs.start_month |
|
325 |
hdfdir = myargs.hdfdir |
|
326 |
night= myargs.night # if 1 then produce night climatology |
|
327 |
out_suffix= myargs.out_suffix #"_03192013" |
|
328 |
download=myargs.download# if 1 then download files |
|
290 | 329 |
|
291 |
#Parameters |
|
292 |
#Inputs from R?? |
|
293 |
tiles = ['h11v08','h11v07','h12v07','h12v08','h10v07','h10v08'] #These tiles correspond to Venezuela. |
|
294 |
start_year = 2001 |
|
295 |
end_year = 2010 |
|
296 |
end_month=12 |
|
297 |
start_month=1 |
|
298 |
hdfdir = '/home/parmentier/Data/benoit_test' #destination file where hdf files are stored locally after download. |
|
299 |
night=1 # if 1 then produce night climatology |
|
300 |
out_suffix="_03192013" |
|
301 |
download=1 # if 1 then download files |
|
330 |
tiles =tiles.split(",") #Create a list from string |
|
331 |
#need to add on the fly creation of folder for each tile!! |
|
302 | 332 |
|
303 | 333 |
################## First step: download data ################### |
304 | 334 |
# Added tile loop |
305 | 335 |
year_list=range(start_year,end_year+1) #list of year to loop through |
306 |
#for testing... |
|
307 |
#year_list=[2001,2002] |
|
308 |
#hdfdir = '/home/parmentier/Data/benoit_test2' |
|
309 |
#tiles = ['h10v07','h10v08','h12v07'] |
|
336 |
|
|
310 | 337 |
if download==1: |
338 |
|
|
311 | 339 |
for tile in tiles: |
312 | 340 |
for year in year_list: |
313 | 341 |
start_doy = 1 |
... | ... | |
320 | 348 |
# modify loop to take into account "hdfs", list of hdf files needed in the next steps… |
321 | 349 |
################# Second step: compute climatology ################# |
322 | 350 |
## Process tile by tile... |
323 |
##start loop for tile |
|
324 |
#tile= 'h12v08' |
|
325 |
#tiles = ['h12v07','h12v08'] |
|
326 | 351 |
var_name = ['LST_Day_1km','LST_Night_1km'] |
327 | 352 |
if night==1: |
328 | 353 |
lst_var = var_name[1] |
... | ... | |
388 | 413 |
gs.mapcalc(' clim_rescaled = ('+ clims[j-1 ]+ ' * 0.02) -273.15') |
389 | 414 |
gs.run_command('r.out.gdal', input= 'clim_rescaled', output=clims[j-1]+ out_suffix+'.tif', type='Float32') |
390 | 415 |
#clims = calc_clim(LST, 'LST_%s_%d_%02d' % (tile, year, month)) |
391 |
# clean up if necessary |
|
392 | 416 |
|
393 |
#gs.run_command('g.remove', rast=','.join(LST)) |
|
394 |
#gs.os.environ['GRASS_OVERWRITE'] = '0' |
|
417 |
# clean up GRASS DATABASE |
|
418 |
|
|
419 |
gs.run_command('g.remove', rast=','.join(LST)) |
|
420 |
gs.os.environ['GRASS_OVERWRITE'] = '0' |
|
395 | 421 |
|
396 |
# clean up
|
|
397 |
#gs.run_command('g.gisenv', set='LOCATION_NAME=%s' % orig_location)
|
|
398 |
#gs.run_command('g.gisenv', set='MAPSET=%s' % orig_mapset)
|
|
399 |
#shutil.rmtree(os.path.join(gs.gisenv()['GISDBASE'], tmp_location))
|
|
422 |
#clean up |
|
423 |
gs.run_command('g.gisenv', set='LOCATION_NAME=%s' % orig_location) |
|
424 |
gs.run_command('g.gisenv', set='MAPSET=%s' % orig_mapset) |
|
425 |
shutil.rmtree(os.path.join(gs.gisenv()['GISDBASE'], tmp_location)) |
|
400 | 426 |
|
401 | 427 |
return None |
428 |
|
|
429 |
|
Also available in: Unified diff
LST downloading and climatology python script passing arguments from shell