Project

General

Profile

« Previous | Next » 

Revision c24e32a8

Added by Adam Wilson over 11 years ago

Fixed the 'missing stripes' problem in the climatology script. Source of problem was related to the _FillValue attribute being incorrectly set to a short integer when the data was saved as byte. Changing both the attribute and the data to type 'byte' appears to have fixed the problem.

View differences:

climate/procedures/MOD35_Climatology.r
79 79
80 80 
## Update attributes
81 81 
system(paste(ncopath,"ncatted ",
82 
" -a valid_min,PClear,o,b,0 ",
83 
" -a valid_max,PClear,o,b,100 ",
84 
#" -a valid_range,PClear,o,b,\"0,255\" ",
85 
#" -a missing_value,PClear,o,b,255 ",
86 
#" -a _FillValue,PClear,d,b,255 ",
82 87 
" -a units,time,o,c,\"days since 2000-1-1 0:0:0\" ",
83 88 
" -a title,global,o,c,\"MODIS Cloud Product (MOD35) Daily Timeseries\" ",
84 89 
" -a institution,global,o,c,\"Yale University\" ",
......
105 110 

  		




  106
  111
  
    
## subset dates


  		




  107
  112
  
    
## due to bug (?) in CDO tools, only 10 years of data can be processed at a time or strange areas of NAs appear.


  		




  108
  
  
    
datesubset="-seldate,2002-01-01,2012-01-01"


  		




  
  113
  
    
datesubset="-seldate,2000-01-01,2011-12-31"


  		




  109
  114
  
    

  		




  110
  115
  
    
## Monthly means


  		




  111
  116
  
    
if(verbose) print("Calculating the monthly means")


  		




  112
  
  
    
system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -mulc,-1 -subc,100 -ymonmean ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""),wait=T)


  		




  113
  
  
    

  		




  114
  
  
    
#system(paste("cdo -v -O selindexbox,1,100,1100,1200 ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_dailysmall.nc",sep=""),wait=T)


  		




  115
  
  
    
#system(paste("cdo -v -O sorttimestamp -setyear,",myear," -setday,15 -mulc,-1 -subc,100 -ymonmean  ",outdir2,"/MOD35_",tile,"_dailysmall.nc ",tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""),wait=T)


  		




  
  117
  
    
system(paste("cdo -O -b I8 -v sorttimestamp -setyear,",myear," -setday,15 -mulc,-1 -subc,100 -ymonmean ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""),wait=T)


  		




  
  118
  
    
system(paste(ncopath,"ncrename -v PClear,PCloud ",tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""))


  		




  
  119
  
    
system(paste(ncopath,"ncatted ",


  		




  
  120
  
    
" -a long_name,PCloud,o,c,\"Mean Probability of Cloud\" ",


  		




  
  121
  
    
" -a missing_value,PCloud,o,b,255 ",


  		




  
  122
  
    
" -a _FillValue,PCloud,d,b,255 ",


  		




  
  123
  
    
tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""))


  		




  116
  124
  
    

  		




  117
  
  
    
system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -mulc,-1 -subc,100 -ymonmean  ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""),wait=T)


  		




  118
  125
  
    

  		




  119
  
  
    
system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -mulc,-1 -subc,100 -monmean   ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_monmean.nc",sep=""),wait=T)


  		




  
  126
  
    
## Monthly Mean


  		




  
  127
  
    
#months=c("01","02","03","04","05","06","07","08","09","10","11","12")


  		




  
  128
  
    
#  month="02"


  		




  
  129
  
    

  		




  
  130
  
    
#ymonmean=function(month){


  		




  
  131
  
    
#  tfile=paste(tempdir(),"/",tile,"_",month,"_",Sys.getpid(),".txt",sep="")


  		




  
  132
  
    
#  write.table(paste(fdly$path[fdly$month==month],collapse=" "),tfile,col.names=F,row.names=F,quote=F)


  		




  
  133
  
    
#  system(paste("cat ",tfile," | ",ncopath,"ncra -O -o ",tsdir,"/MOD35_",tile,"_",month,".nc",sep=""))


  		




  
  134
  
    
#  system(paste("cdo -O -setyear,",myear," -setmon,",month," -setday,15 -mulc,-1 -subc,100  ",tsdir,"/MOD35_",tile,"_",month,".nc ",tsdir,"/MOD35_",tile,"_",month,"b.nc",sep=""))


  		




  
  135
  
    
#  system(paste(ncopath,"ncrename -v PClear,PCloud ",tsdir,"/MOD35_",tile,"_",month,"b.nc",sep=""))


  		




  
  136
  
    
#  system(paste(ncopath,"ncatted ",


  		




  
  137
  
    
#" -a long_name,PCloud,o,c,\"Mean Probability of Cloud\" ",


  		




  
  138
  
    
#" -a missing_value,PCloud,o,b,255 ",


  		




  
  139
  
    
#" -a _FillValue,PCloud,d,b,255 ",


  		




  
  140
  
    
#tsdir,"/MOD35_",tile,"_",month,"b.nc",sep=""))


  		




  
  141
  
    
#}


  		




  
  142
  
    
#mclapply(months,ymonmean)


  		




  120
  143
  
    

  		




  
  144
  
    
## merge to a single file


  		




  
  145
  
    
#  system(paste("cdo -O -b I8 -v -mergetime ",paste(tsdir,"/MOD35_",tile,"_",months,"b.nc ",sep="",collapse=" ")," ",tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""))


  		




  121
  146
  
    

  		




  122
  
  
    
system(paste("cdo -O sorttimestamp -setyear,",myear," -mulc,-1 -subc,100 -ydrunmean,30 ",datesubset," ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ydrunmean30.nc",sep=""),wait=T)


  		




  
  147
  
    
#system(paste("cdo -v -O selindexbox,1,100,1100,1200 ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_dailysmall.nc",sep=""),wait=T)


  		




  
  148
  
    
#system(paste("cdo -v -O sorttimestamp -setyear,",myear," -setday,15 -mulc,-1 -subc,100 -ymonmean  ",outdir2,"/MOD35_",tile,"_dailysmall.nc ",tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""),wait=T)


  		




  
  149
  
    
#system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -mulc,-1 -subc,100 -timmean -selmon,2 ",datesubset," ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""),wait=T)


  		




  
  150
  
    
#system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -mulc,-1 -subc,100 -monmean   ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_monmean.nc",sep=""),wait=T)


  		




  
  151
  
    
#system(paste("cdo -O sorttimestamp -setyear,",myear," -mulc,-1 -subc,100 -ydrunmean,30 ",datesubset," ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ydrunmean30.nc",sep=""),wait=T)


  		




  
  152
  
    
#system(paste("scp ",tsdir,"/MOD35_",tile,"_ymonmean.nc adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod35/",sep=""))


  		




  
  153
  
    
#system(paste("scp summary/MOD35_",tile,".nc adamw@acrobates.eeb.yale.edu:/data/personal/adamw/projects/interp/data/modis/mod35/",sep=""))


  		




  
  154
  
    
#system(paste("ncdump -h ",tsdir,"/MOD35_",tile,"_ymonmean.nc ",sep=""))


  		




  123
  155
  
    

  		




  124
  
  
    
system(paste(ncopath,"ncrename -v PClear,PCloud ",tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""))


  		




  125
  
  
    
system(paste(ncopath,"ncatted ",


  		




  126
  
  
    
" -a long_name,PCloud,o,c,\"Mean Probability of Cloud\" ",


  		




  127
  
  
    
tsdir,"/MOD35_",tile,"_ymonmean.nc",sep=""))


  		




  128
  156
  
    

  		




  129
  157
  
    
## Monthly standard deviation


  		




  130
  158
  
    
if(verbose) print("Calculating the monthly SD")


  		




  131
  
  
    
system(paste("cdo -O sorttimestamp -setyear,",myear," -setday,15 -ymonstd  ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymonstd.nc",sep=""))


  		




  
  159
  
    
system(paste("cdo -O -b I8 sorttimestamp -setyear,",myear," -setday,15 -ymonstd  ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymonstd.nc",sep=""))


  		




  132
  160
  
    
system(paste(ncopath,"ncrename -v PClear,PCloud_sd ",tsdir,"/MOD35_",tile,"_ymonstd.nc",sep=""))


  		




  133
  161
  
    
system(paste(ncopath,"ncatted ",


  		




  134
  162
  
    
" -a long_name,PCloud_sd,o,c,\"Standard Deviation of p(cloud)\" ",


  		




  ......




  136
  164
  
    

  		




  137
  165
  
    
## frequency of cloud days p(clear<90%)  


  		




  138
  166
  
    
if(verbose) print("Calculating the proportion of cloudy and probably cloudy days")


  		




  139
  
  
    
system(paste("cdo -O  sorttimestamp -setyear,",myear," -setday,15 -ymonmean  -mulc,100 -lec,90 -selvar,PClear ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymoncld01.nc",sep=""))


  		




  
  167
  
    
system(paste("cdo -O -b I8 sorttimestamp -setyear,",myear," -setday,15 -ymonmean  -mulc,100 -lec,90 -selvar,PClear ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymoncld01.nc",sep=""))


  		




  140
  168
  
    
system(paste(ncopath,"ncrename -v PClear,CF ",tsdir,"/MOD35_",tile,"_ymoncld01.nc",sep=""))


  		




  141
  169
  
    
system(paste(ncopath,"ncatted ",


  		




  142
  170
  
    
" -a long_name,CF,o,c,\"Cloud Frequency: Proportion of Days with probability of clear < 90%\" ",


  		




  ......




  145
  173
  
    

  		




  146
  174
  
    
## number of observations


  		




  147
  175
  
    
if(verbose) print("Calculating the number of missing variables")


  		




  148
  
  
    
system(paste("cdo -O sorttimestamp  -setyear,",myear," -setday,15 -ymonmean -mulc,100  -eqc,9999 -setmisstoc,9999   -selvar,PClear ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymonmiss.nc",sep=""))


  		




  
  176
  
    
system(paste("cdo -O -b I8 sorttimestamp  -setyear,",myear," -setday,15 -ymonmean -mulc,100  -eqc,9999 -setmisstoc,9999   -selvar,PClear ",outdir2,"/MOD35_",tile,"_daily.nc ",tsdir,"/MOD35_",tile,"_ymonmiss.nc",sep=""))


  		




  149
  177
  
    
system(paste(ncopath,"ncrename -v PClear,Pmiss ",tsdir,"/MOD35_",tile,"_ymonmiss.nc",sep=""))


  		




  150
  178
  
    
system(paste(ncopath,"ncatted ",


  		




  151
  179
  
    
             " -a long_name,Pmiss,o,c,\"Proportion of Days with missing data\" ",


  		












  

    
      climate/procedures/MOD35_L2_process.r
    
  





  290
  290
  
    
system(paste(ncopath,"ncks -A ",tempdir(),"/time.nc ",ncfile,sep=""))


  		




  291
  291
  
    
## add other attributes


  		




  292
  292
  
    
  system(paste(ncopath,"ncrename -v Band1,PClear ",ncfile,sep=""))


  		




  293
  
  
    
  system(paste(ncopath,"ncatted -a scale_factor,PClear,o,s,1 -a units,PClear,o,c,\"Probability (%)\" -a missing_value,PClear,o,s,255 -a _FillValue,PClear,o,s,255 -a long_name,PClear,o,c,\"Probability of Clear Sky\" ",ncfile,sep=""))


  		




  294
  
  
    

  		




  295
  
  
    
                                        #  system(paste(ncopath,"ncatted -a sourcecode,global,o,c,",script," ",ncfile,sep=""))


  		




  
  293
  
    
  system(paste(ncopath,"ncatted ",


  		




  
  294
  
    
" -a units,PClear,o,c,\"Probability (%)\" ",


  		




  
  295
  
    
" -a missing_value,PClear,o,b,255 ",


  		




  
  296
  
    
" -a _FillValue,PClear,o,b,255 ",


  		




  
  297
  
    
" -a valid_range,PClear,o,b,\"0,100\" ",


  		




  
  298
  
    
" -a long_name,PClear,o,c,\"Probability of Clear Sky\" ",


  		




  
  299
  
    
ncfile,sep=""))


  		




  
  300
  
    
#system(paste(ncopath,"ncatted -a sourcecode,global,o,c,",script," ",ncfile,sep=""))


  		




  296
  301
  
    
   


  		




  297
  302
  
    
  


  		




  298
  303
  
    
### delete the temporary files 


  		












  

    
      climate/procedures/Pleiades_MOD35.R
    
  





  126
  126
  
    

  		




  127
  127
  
    
## Submit it


  		




  128
  128
  
    
system(paste("qsub mod35_qsub",sep=""))


  		




  
  129
  
    
system("qstat -u awilso10")


  		




  129
  130
  
    

  		




  130
  131
  
    
#######################################################


  		




  131
  132
  
    
### Now submit the script to generate the climatologies


  		












Also available in:  Unified diff