Project

General

Profile

« Previous | Next » 

Revision 15936a3e

Added by Adam Wilson about 11 years ago

updated EE script to loop over regions (currently global quarters) and check if uncorrupted files exist before downloading

View differences:

climate/procedures/ee.MOD09.py
9 9
import datetime
10 10
import wget
11 11
import os
12
from subprocess import call
12 13

  
13 14
#import logging
14 15
#logging.basicConfig()
......
16 17
## set working directory (where files will be downloaded)
17 18
os.chdir('/home/adamw/acrobates/adamw/projects/cloud/data/mod09')
18 19

  
19
#MY_SERVICE_ACCOUNT = '511722844190@developer.gserviceaccount.com'  # replace with your service account
20
#MY_PRIVATE_KEY_FILE = '/home/adamw/EarthEngine-privatekey.p12'       # replace with you private key file path
20
MY_SERVICE_ACCOUNT = '511722844190@developer.gserviceaccount.com'  # replace with your service account
21
MY_PRIVATE_KEY_FILE = '/home/adamw/EarthEngine-privatekey.p12'       # replace with you private key file path
21 22

  
22
MY_SERVICE_ACCOUNT = '205878743334-4mrtqgu0n5rnsv1vanrvv6atqk6vu8am@developer.gserviceaccount.com'
23
MY_PRIVATE_KEY_FILE = '/home/adamw/EarthEngine_Jeremy-privatekey.p12'
23
#MY_SERVICE_ACCOUNT = '205878743334-4mrtqgu0n5rnsv1vanrvv6atqk6vu8am@developer.gserviceaccount.com'
24
#MY_PRIVATE_KEY_FILE = '/home/adamw/EarthEngine_Jeremy-privatekey.p12'
24 25

  
25 26
ee.Initialize(ee.ServiceAccountCredentials(MY_SERVICE_ACCOUNT, MY_PRIVATE_KEY_FILE))
26 27

  
27 28
## set map center to speed up viewing
28
ee.mapclient.centerMap(-121.767, 46.852, 11)
29
#ee.mapclient.centerMap(-121.767, 46.852, 11)
29 30

  
30 31
#///////////////////////////////////
31 32
#// Function to extract cloud flags
32
def getmod09(img): return(img.select(['state_1km']).expression("((b(0)/1024)%2)"));
33

  
33
def getmod09(img): return(img.select(['state_1km']).expression("((b(0)/1024)%2)")); 
34
 
34 35
#// Date ranges
35
yearstart=2000
36
yearstop=2010
36
yearstart=2001
37
yearstop=2001
37 38
monthstart=1
38
monthstop=12
39
monthstop=2
39 40

  
40 41
#////////////////////////////////////////////////////
41 42
# Loop through months and get monthly % missing data
42 43

  
43 44
## set a year-month if you don't want to run the loop (for testing)
44 45
year=2001
45
month=1
46

  
47
for year in range(yearstart,yearstop+1): {
48
for month in range(monthstart,monthstop); {
49
    
50
print('Processing '+str(year)+'_'+str(month))
51

  
52
## MOD09 internal cloud flag for this year-month
53
## to filter by a date range:  filterDate(datetime.datetime(yearstart,monthstart,1),datetime.datetime(yearstop,monthstop,31))
54
mod09 = ee.ImageCollection("MOD09GA").filter(ee.Filter.calendarRange(year,year,"year")).filter(ee.Filter.calendarRange(month,month,"month")).map(getmod09);
55

  
56
## calculate mean cloudiness (%), rename band, multiply by 100, and convert to integer
57
mod09a=mod09.mean().select([0], ['MOD09_'+str(year)+'_'+str(month)]).multiply(ee.Image(100)).byte();
58

  
59
## print info to confirm there is data
60
#mod09a.getInfo()
61

  
62
## add to plot to confirm it's working
63
#ee.mapclient.addToMap(mod09a, {'range': '0,100'}, 'MOD09')
64

  
65
## define region for download
66
region='[[-72, -1], [-72, 11], [-59, 11], [-59, -1]]'  #h11v08
67
#  '[[-90, -1], [-90, 20], [-49, 20], [-49, -1]]'  // Northern S. America
68
#  '[[-180, -90], [-180, 90], [180, 90], [180, -90]]'  //global
69
#  '[[-180, -60], [-180, 90], [180, 90], [180, -60]]'  // Western Hemisphere
70

  
71
## Define tiles
72

  
73

  
74
## build the URL and name the object (so that when it's unzipped we know what it is!)
75
path =mod09a.getDownloadUrl({
76
  'name': 'mod09_'+str(year)+"_"+str(month),  # name the file (otherwise it will be a uninterpretable hash)
77
  'scale': 926,                               # resolution in meters
78
  'crs': 'EPSG:4326',                         # MODIS sinusoidal
79
  'region': region                            # region defined above
80
});
81

  
82
## download with wget
83
wget.download(path)
84

  
85
}}
46
month=2
47

  
48

  
49
## define the regions to be processed
50
regions=['[[-180, -60], [-180, 0], [0, 0], [0, -60]]',  # SW
51
         '[[-180, 0], [-180, 90], [0, 90], [0, 0]]',    # NW
52
         '[[0, 0], [0, 90], [180, 90], [180, 0]]',      # NE
53
         '[[0, 0], [0, -60], [180, -60], [180, 0]]']    # SE
54
## name the regions (these names will be used in the file names
55
## must be same length as regions list above
56
rnames=['SW','NW','NE','SE']
57

  
58
## Loop over regions, years, months to generate monthly timeseries
59
for r in range(0,len(regions)):                                    # loop over regions
60
  for year in range(yearstart,yearstop+1):              # loop over years
61
    for month in range(monthstart,monthstop+1):         # loop over months
62
      print('Processing '+rnames[r]+"_"+str(year)+'_'+str(month))
63

  
64
      # output filename
65
      filename='mod09_'+rnames[r]+"_"+str(year)+"_"+str(month)
66

  
67
      # Check if file already exists and continue if so...
68
      if(os.path.exists(filename+".zip")):
69
        print("File exists:"+filename)
70
        continue
71

  
72
      # MOD09 internal cloud flag for this year-month
73
      # to filter by a date range:  filterDate(datetime.datetime(yearstart,monthstart,1),datetime.datetime(yearstop,monthstop,31))
74
      mod09 = ee.ImageCollection("MOD09GA").filter(ee.Filter.calendarRange(year,year,"year")).filter(ee.Filter.calendarRange(month,month,"month")).map(getmod09);
75
      # calculate mean cloudiness (%), rename band, multiply by 100, and convert to integer
76
      mod09a=mod09.mean().select([0], ['MOD09_'+str(year)+'_'+str(month)]).multiply(ee.Image(100)).byte();
77
      
78
      # Next few lines for testing only
79
      # print info to confirm there is data
80
      # mod09a.getInfo()
81
      # add to plot to confirm it's working
82
      # ee.mapclient.addToMap(mod09a, {'range': '0,100'}, 'MOD09')
83

  
84
      # build the URL and name the object (so that when it's unzipped we know what it is!)
85
      path =mod09a.getDownloadUrl({
86
          'name': filename,  # name the file (otherwise it will be a uninterpretable hash)
87
          'scale': 1000,                              # resolution in meters
88
          'crs': 'EPSG:4326',                         # MODIS sinusoidal
89
          'region': regions[r]                        # region defined above
90
          });
91

  
92
      # Sometimes EE will serve a corrupt zipped file with no error
93
      # to check this, use a while loop that keeps going till there is an unzippable file.  
94
      # This has the potential for an infinite loop...
95

  
96
      while(not(os.path.exists(filename+".zip"))):
97
        # download with wget
98
        print("Downloading "+filename) 
99
        wget.download(path)
100
        # try to unzip it
101
        print("Unzipping "+filename)
102
        zipstatus=call("unzip "+filename+".zip",shell=True)
103
        # if file doesn't exists or it didn't unzip, remove it and try again      
104
        if(zipstatus==9):
105
          print("ERROR: "+filename+" unzip-able")
106
          os.remove(filename)
107

  
108
print 'Finished'
86 109

  
87 110

  

Also available in: Unified diff