Revision 15936a3e
Added by Adam Wilson about 11 years ago
climate/procedures/ee.MOD09.py | ||
---|---|---|
9 | 9 |
import datetime |
10 | 10 |
import wget |
11 | 11 |
import os |
12 |
from subprocess import call |
|
12 | 13 |
|
13 | 14 |
#import logging |
14 | 15 |
#logging.basicConfig() |
... | ... | |
16 | 17 |
## set working directory (where files will be downloaded) |
17 | 18 |
os.chdir('/home/adamw/acrobates/adamw/projects/cloud/data/mod09') |
18 | 19 |
|
19 |
#MY_SERVICE_ACCOUNT = '511722844190@developer.gserviceaccount.com' # replace with your service account
|
|
20 |
#MY_PRIVATE_KEY_FILE = '/home/adamw/EarthEngine-privatekey.p12' # replace with you private key file path
|
|
20 |
MY_SERVICE_ACCOUNT = '511722844190@developer.gserviceaccount.com' # replace with your service account |
|
21 |
MY_PRIVATE_KEY_FILE = '/home/adamw/EarthEngine-privatekey.p12' # replace with you private key file path |
|
21 | 22 |
|
22 |
MY_SERVICE_ACCOUNT = '205878743334-4mrtqgu0n5rnsv1vanrvv6atqk6vu8am@developer.gserviceaccount.com' |
|
23 |
MY_PRIVATE_KEY_FILE = '/home/adamw/EarthEngine_Jeremy-privatekey.p12' |
|
23 |
#MY_SERVICE_ACCOUNT = '205878743334-4mrtqgu0n5rnsv1vanrvv6atqk6vu8am@developer.gserviceaccount.com'
|
|
24 |
#MY_PRIVATE_KEY_FILE = '/home/adamw/EarthEngine_Jeremy-privatekey.p12'
|
|
24 | 25 |
|
25 | 26 |
ee.Initialize(ee.ServiceAccountCredentials(MY_SERVICE_ACCOUNT, MY_PRIVATE_KEY_FILE)) |
26 | 27 |
|
27 | 28 |
## set map center to speed up viewing |
28 |
ee.mapclient.centerMap(-121.767, 46.852, 11) |
|
29 |
#ee.mapclient.centerMap(-121.767, 46.852, 11)
|
|
29 | 30 |
|
30 | 31 |
#/////////////////////////////////// |
31 | 32 |
#// Function to extract cloud flags |
32 |
def getmod09(img): return(img.select(['state_1km']).expression("((b(0)/1024)%2)")); |
|
33 |
|
|
33 |
def getmod09(img): return(img.select(['state_1km']).expression("((b(0)/1024)%2)"));
|
|
34 |
|
|
34 | 35 |
#// Date ranges |
35 |
yearstart=2000
|
|
36 |
yearstop=2010
|
|
36 |
yearstart=2001
|
|
37 |
yearstop=2001
|
|
37 | 38 |
monthstart=1 |
38 |
monthstop=12
|
|
39 |
monthstop=2 |
|
39 | 40 |
|
40 | 41 |
#//////////////////////////////////////////////////// |
41 | 42 |
# Loop through months and get monthly % missing data |
42 | 43 |
|
43 | 44 |
## set a year-month if you don't want to run the loop (for testing) |
44 | 45 |
year=2001 |
45 |
month=1 |
|
46 |
|
|
47 |
for year in range(yearstart,yearstop+1): { |
|
48 |
for month in range(monthstart,monthstop); { |
|
49 |
|
|
50 |
print('Processing '+str(year)+'_'+str(month)) |
|
51 |
|
|
52 |
## MOD09 internal cloud flag for this year-month |
|
53 |
## to filter by a date range: filterDate(datetime.datetime(yearstart,monthstart,1),datetime.datetime(yearstop,monthstop,31)) |
|
54 |
mod09 = ee.ImageCollection("MOD09GA").filter(ee.Filter.calendarRange(year,year,"year")).filter(ee.Filter.calendarRange(month,month,"month")).map(getmod09); |
|
55 |
|
|
56 |
## calculate mean cloudiness (%), rename band, multiply by 100, and convert to integer |
|
57 |
mod09a=mod09.mean().select([0], ['MOD09_'+str(year)+'_'+str(month)]).multiply(ee.Image(100)).byte(); |
|
58 |
|
|
59 |
## print info to confirm there is data |
|
60 |
#mod09a.getInfo() |
|
61 |
|
|
62 |
## add to plot to confirm it's working |
|
63 |
#ee.mapclient.addToMap(mod09a, {'range': '0,100'}, 'MOD09') |
|
64 |
|
|
65 |
## define region for download |
|
66 |
region='[[-72, -1], [-72, 11], [-59, 11], [-59, -1]]' #h11v08 |
|
67 |
# '[[-90, -1], [-90, 20], [-49, 20], [-49, -1]]' // Northern S. America |
|
68 |
# '[[-180, -90], [-180, 90], [180, 90], [180, -90]]' //global |
|
69 |
# '[[-180, -60], [-180, 90], [180, 90], [180, -60]]' // Western Hemisphere |
|
70 |
|
|
71 |
## Define tiles |
|
72 |
|
|
73 |
|
|
74 |
## build the URL and name the object (so that when it's unzipped we know what it is!) |
|
75 |
path =mod09a.getDownloadUrl({ |
|
76 |
'name': 'mod09_'+str(year)+"_"+str(month), # name the file (otherwise it will be a uninterpretable hash) |
|
77 |
'scale': 926, # resolution in meters |
|
78 |
'crs': 'EPSG:4326', # MODIS sinusoidal |
|
79 |
'region': region # region defined above |
|
80 |
}); |
|
81 |
|
|
82 |
## download with wget |
|
83 |
wget.download(path) |
|
84 |
|
|
85 |
}} |
|
46 |
month=2 |
|
47 |
|
|
48 |
|
|
49 |
## define the regions to be processed |
|
50 |
regions=['[[-180, -60], [-180, 0], [0, 0], [0, -60]]', # SW |
|
51 |
'[[-180, 0], [-180, 90], [0, 90], [0, 0]]', # NW |
|
52 |
'[[0, 0], [0, 90], [180, 90], [180, 0]]', # NE |
|
53 |
'[[0, 0], [0, -60], [180, -60], [180, 0]]'] # SE |
|
54 |
## name the regions (these names will be used in the file names |
|
55 |
## must be same length as regions list above |
|
56 |
rnames=['SW','NW','NE','SE'] |
|
57 |
|
|
58 |
## Loop over regions, years, months to generate monthly timeseries |
|
59 |
for r in range(0,len(regions)): # loop over regions |
|
60 |
for year in range(yearstart,yearstop+1): # loop over years |
|
61 |
for month in range(monthstart,monthstop+1): # loop over months |
|
62 |
print('Processing '+rnames[r]+"_"+str(year)+'_'+str(month)) |
|
63 |
|
|
64 |
# output filename |
|
65 |
filename='mod09_'+rnames[r]+"_"+str(year)+"_"+str(month) |
|
66 |
|
|
67 |
# Check if file already exists and continue if so... |
|
68 |
if(os.path.exists(filename+".zip")): |
|
69 |
print("File exists:"+filename) |
|
70 |
continue |
|
71 |
|
|
72 |
# MOD09 internal cloud flag for this year-month |
|
73 |
# to filter by a date range: filterDate(datetime.datetime(yearstart,monthstart,1),datetime.datetime(yearstop,monthstop,31)) |
|
74 |
mod09 = ee.ImageCollection("MOD09GA").filter(ee.Filter.calendarRange(year,year,"year")).filter(ee.Filter.calendarRange(month,month,"month")).map(getmod09); |
|
75 |
# calculate mean cloudiness (%), rename band, multiply by 100, and convert to integer |
|
76 |
mod09a=mod09.mean().select([0], ['MOD09_'+str(year)+'_'+str(month)]).multiply(ee.Image(100)).byte(); |
|
77 |
|
|
78 |
# Next few lines for testing only |
|
79 |
# print info to confirm there is data |
|
80 |
# mod09a.getInfo() |
|
81 |
# add to plot to confirm it's working |
|
82 |
# ee.mapclient.addToMap(mod09a, {'range': '0,100'}, 'MOD09') |
|
83 |
|
|
84 |
# build the URL and name the object (so that when it's unzipped we know what it is!) |
|
85 |
path =mod09a.getDownloadUrl({ |
|
86 |
'name': filename, # name the file (otherwise it will be a uninterpretable hash) |
|
87 |
'scale': 1000, # resolution in meters |
|
88 |
'crs': 'EPSG:4326', # MODIS sinusoidal |
|
89 |
'region': regions[r] # region defined above |
|
90 |
}); |
|
91 |
|
|
92 |
# Sometimes EE will serve a corrupt zipped file with no error |
|
93 |
# to check this, use a while loop that keeps going till there is an unzippable file. |
|
94 |
# This has the potential for an infinite loop... |
|
95 |
|
|
96 |
while(not(os.path.exists(filename+".zip"))): |
|
97 |
# download with wget |
|
98 |
print("Downloading "+filename) |
|
99 |
wget.download(path) |
|
100 |
# try to unzip it |
|
101 |
print("Unzipping "+filename) |
|
102 |
zipstatus=call("unzip "+filename+".zip",shell=True) |
|
103 |
# if file doesn't exists or it didn't unzip, remove it and try again |
|
104 |
if(zipstatus==9): |
|
105 |
print("ERROR: "+filename+" unzip-able") |
|
106 |
os.remove(filename) |
|
107 |
|
|
108 |
print 'Finished' |
|
86 | 109 |
|
87 | 110 |
|
Also available in: Unified diff
updated EE script to loop over regions (currently global quarters) and check if uncorrupted files exist before downloading