Project

General

Profile

« Previous | Next » 

Revision 3193de08

Added by Benoit Parmentier over 11 years ago

master script, debugging covariates production for Oregon study area

View differences:

climate/research/oregon/interpolation/master_script_temp.R
10 10
#STAGE 5: Output analyses: assessment of results for specific dates...
11 11
#
12 12
#AUTHOR: Benoit Parmentier                                                                       
13
#DATE: 05/27/2013                                                                                 
13
#DATE: 05/30/2013                                                                                 
14 14

  
15 15
#PROJECT: NCEAS INPLANT: Environment and Organisms --TASK#363, TASK$568--   
16 16

  
17
##Comments and TODO:
17
## TODO:
18 18
# Modify code for stage 1 and call python script from R
19 19
# Modify code for stage 2, make it a function and fully automated (distoc var)
20 20
# Add options to run only specific stage + additional out_suffix?
21 21
# Make master script a function?
22 22
# Add log file for master script,add function to collect inputs and outputs
23
# 
23
# Comments for run:
24
#Testing full code for 5 stages (no downloading) for Oregon region.
24 25
##################################################################################################
25 26

  
26 27
###Loading R library and packages   
......
54 55
modis_download_script <- file.path(script_path,"modis_download_05142013.py") # LST modis download python script
55 56
clim_script <- file.path(script_path,"climatology_05142013.py") # LST climatology python script
56 57
grass_setting_script <- file.path(script_path,"grass-setup.R") #Set up system shell environment for python+GRASS
57
#source(file.path(script_path,"download_and_produce_MODIS_LST_climatology_05162013.R"))
58
source(file.path(script_path,"covariates_production_temperatures_05272013.R"))
58
source(file.path(script_path,"download_and_produce_MODIS_LST_climatology_05302013.R"))
59
source(file.path(script_path,"covariates_production_temperatures_05302013.R"))
59 60
source(file.path(script_path,"Database_stations_covariates_processing_function_05212013.R"))
60 61
source(file.path(script_path,"GAM_fusion_analysis_raster_prediction_multisampling_05212013.R"))
61 62
source(file.path(script_path,"results_interpolation_date_output_analyses_05062013.R"))
......
68 69
source(file.path(script_path,"GAM_fusion_function_multisampling_validation_metrics_05062013.R"))
69 70

  
70 71
#stages_to_run<-c(1,2,3,4,5) #May decide on antoher strategy later on...
71
stages_to_run<-c(0,2,3,4,5) #May decide on antoher strategy later on...
72
stages_to_run<-c(1,2,3,4,5) #May decide on antoher strategy later on...
72 73

  
73 74
var<-"TMAX" # variable being interpolated
74
out_prefix<-"_365d_GAM_fus_all_lst_05242013"                #User defined output prefix
75
out_suffix<-"_VE_05242013"
76
out_suffix_modis <-"_05242013"
75
out_prefix<-"_365d_GAM_fus_all_lst_05302013"                #User defined output prefix
76
out_suffix<-"_OR_05302013"
77
out_suffix_modis <-"_05302013"
77 78

  
78 79
#interpolation_method<-c("gam_fusion","gam_CAI") #other otpions to be added later
79 80
#interpolation_method<-c("gam_CAI") #other otpions to be added later
80 81
interpolation_method<-c("gam_fusion") #other otpions to be added later
81 82

  
82
#Change input path?? for LST stage ??
83
in_path  <- "/home/parmentier/Data/IPLANT_project/Venezuela_interpolation/Venezuela_01142013/input_data/"
84
#in_path <- out_path
85
out_path <- paste("/home/parmentier/Data/IPLANT_project/Venezuela_interpolation/Venezuela_01142013/output_data",
86
                  out_prefix,"/",sep="")
83
#out_path <- paste("/home/parmentier/Data/IPLANT_project/Venezuela_interpolation/Venezuela_01142013/output_data",
84
#                  out_prefix,"/",sep="")
85
out_path<-"/home/parmentier/Data/IPLANT_project/Oregon_interpolation/Oregon_03142013/output_data"
86
out_path <-paste(out_path,out_prefix,sep="")
87

  
87 88
if (!file.exists(out_path)){
88 89
  dir.create(out_path)
89 90
  #} else{
......
101 102
#infile_covariates<- "/home/parmentier/Data/IPLANT_project/Venezuela_interpolation/Venezuela_01142013/covariates_Oregon_region_TMAX__OR_04052013.tif" #Oregon covar TMAX from earlier codes...for continuity
102 103
#infile_reg_outline=""  #input region outline defined by polygon: none for Venezuela
103 104
#This is the shape file of outline of the study area                                                      #It is an input/output of the covariate script
104
#infile_reg_outline <- "OR83M_state_outline.shp"  #input region outline defined by polygon: Oregon
105
infile_reg_outline <- "/home/parmentier/Data/IPLANT_project/Oregon_interpolation/Oregon_03142013/OR83M_state_outline.shp"  #input region outline defined by polygon: Oregon
105 106
#infile_reg_outline <-"OR83M_state_outline.shp" #remove this parameter!!!
106
#covar_names see stage 2
107
#ref_rast_name<-""  #local raster name defining resolution, exent, local projection--. set on the fly?? 
108
#this may be redundant with infile_reg_outline
109
ref_rast_name<-"/home/parmentier/Data/IPLANT_project/Oregon_interpolation/Oregon_03142013/mean_day244_rescaled.rst"  #local raster name defining resolution, exent: oregon
107 110

  
108
#lc_path<-"/home/layers/data/land-cover/lc-consensus-global"
109
#infile_modis_grid<-"/home/layers/commons/modis/modis_sinusoidal/modis_sinusoidal_grid_world.shp" #Give path!!! NEED TO CHANGE THIS...
110
#infile_elev<-"/home/layers/data/terrain/dem-cgiar-srtm-1km-tif/srtm_1km.tif"  #this is the global file: replace later with the input produced by the DEM team
111
#infile_canheight<-"/home/layers/data/land-cover/treeheight-simard2011/Simard_Pinto_3DGlobalVeg_JGR.tif"              #Canopy height
111
#covar_names see stage 2
112 112

  
113
list_tiles_modis <- c("h11v08,h11v07,h12v07,h12v08,h10v07,h10v08") #tile for Venezuela and surrounding area
114
#list_tiles_modis <- c("h08v04","h09v04") #tiles for Oregon
113
#list_tiles_modis <- c("h11v08,h11v07,h12v07,h12v08,h10v07,h10v08") #tile for Venezuela and surrounding area
114
list_tiles_modis <- c("h08v04,h09v04") #tiles for Oregon
115 115
  
116
infile_reg_outline=""  #input region outline defined by polygon: none for Venezuela
117
#infile_reg_outline <- "OR83M_state_outline.shp"  #input region outline defined by polygon: Oregon
118
  
119
CRS_interp<-"+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs";
120
#CRS_interp <-"+proj=lcc +lat_1=43 +lat_2=45.5 +lat_0=41.75 +lon_0=-120.5 +x_0=400000 +y_0=0 +ellps=GRS80 +units=m +no_defs";
116
#CRS_interp<-"+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs";
117
CRS_interp <-"+proj=lcc +lat_1=43 +lat_2=45.5 +lat_0=41.75 +lon_0=-120.5 +x_0=400000 +y_0=0 +ellps=GRS80 +units=m +no_defs";
121 118
CRS_locs_WGS84<-CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84
122
out_region_name<-"_venezuela_region" #generated on the fly
123
#out_suffix<-"_VE_05252013"
124
ref_rast_name<-""  #local raster name defining resolution, exent, local projection--. set on the fly?? 
125
                  #this may be redundant with infile_reg_outline
126
#ref_rast_name<-"mean_day244_rescaled.rst"  #local raster name defining resolution, exent: oregon
119
#out_region_name<-"_venezuela_region" #generated on the fly
120
out_region_name<-"_oregon_region" #generated on the fly
127 121
  
128 122
#The names of covariates can be changed...these names should be output/input from covar script!!!
129 123
rnames<-c("x","y","lon","lat","N","E","N_w","E_w","elev_s","slope","aspect","CANHEIGHT","DISTOC")
......
140 134
#list_tiles_modis <- c("h08v04,h09v04") #tiles for Oregon #defined above...
141 135
start_year = "2001"
142 136
end_year = "2010"
143
#end_year = "2002" #for testing (year included?)
144 137
#hdfdir =  '/home/layers/commons/modis/MOD11A1_tiles' #destination file where hdf files are stored locally after download.
145
#hdfdir =  '/home/parmentier/Data/IPLANT_project/MOD11A1_tiles'
146 138
hdfdir <- "/data/project/layers/commons/modis/MOD11A1_tiles"
147
download=1
148
clim_calc=0
149
#out_suffix_modis="_05252013"
150
#end_month= "12"
151
#start_month= "1"
139
download=0
140
clim_calc=1
152 141

  
153 142
list_param_download_clim_LST_script <- list(list_tiles_modis,start_year,end_year,hdfdir,
154 143
                                            var,grass_setting_script,modis_download_script, clim_script,
......
156 145
names(list_param_download_clim_LST_script)<-c("list_tiles_modis","start_year","end_year","hdfdir",
157 146
                                              "var","grass_setting_script","modis_download_script","clim_script",
158 147
                                              "download","clim_calc","out_suffix_modis")
148
no_tiles <- length(unlist(strsplit(list_tiles_modis,",")))  # transform string into separate element in char vector
159 149

  
160 150
if (stages_to_run[1]==1){
151
  #clim_production_obj <-mclapply(1:2, list_param=list_param_download_clim_LST_script, download_calculate_MODIS_LST_climatology,mc.preschedule=FALSE,mc.cores = 2) #This is the end bracket from mclapply(...) statement
152
  clim_production_obj <-lapply(1:no_tiles, list_param=list_param_download_clim_LST_script, download_calculate_MODIS_LST_climatology) #,mc.preschedule=FALSE,mc.cores = 2) #This is the end bracket from mclapply(...) statement
161 153
  
162
  download_calculate_MODIS_LST_climatology(1,list_param_download_clim_LST_script)
163

  
164 154
}
165 155
#Collect LST climatology list as output???
166 156

  
167 157
############ STAGE 2: Covariate production ################
168 158

  
169 159
#list of 17 parameters
170
list_param_covar_production<-list(var,in_path,out_path,lc_path,infile_modis_grid,infile_elev,infile_canheight,
160
list_param_covar_production<-list(var,out_path,lc_path,infile_modis_grid,infile_elev,infile_canheight,
171 161
                                  infile_distoc,list_tiles_modis,infile_reg_outline,CRS_interp,CRS_locs_WGS84,out_region_name,
172 162
                                  out_suffix,out_suffix_modis,ref_rast_name,hdfdir,covar_names) 
173 163

  
174
names(list_param_covar_production)<-c("var","in_path","out_path","lc_path","infile_modis_grid","infile_elev","infile_canheight",
164
names(list_param_covar_production)<-c("var","out_path","lc_path","infile_modis_grid","infile_elev","infile_canheight",
175 165
                                      "infile_distoc","list_tiles_modis","infile_reg_outline","CRS_interp","CRS_locs_WGS84","out_region_name",
176 166
                                      "out_suffix","out_suffix_modis","ref_rast_name","hdfdir","covar_names") 
177 167

  
......
192 182
range_years<-c("2010","2011") #right bound not included in the range!!
193 183
range_years_clim<-c("2000","2011") #right bound not included in the range!!
194 184
infile_ghncd_data <-"/home/layers/data/climate/ghcn/v2.92-upd-2012052822/ghcnd-stations.txt"                              #This is the textfile of station locations from GHCND
195
qc_flags_stations<-c("0","S")    #flags allowed for screening after the query from the GHCND??
185
#qc_flags_stations<-c("0","S")    #flags allowed for screening after the query from the GHCND??
186
qc_flags_stations<-c("0")    #flags allowed for screening after the query from the GHCND??
187

  
196 188
#infile_covariates and infile_reg_outline defined in stage 2 or at the start of script...
197 189

  
198 190
#list of 12 parameters for input in the function...
199 191

  
200
list_param_prep<-list(db.name,var,range_years,range_years_clim,infile_reg_outline,infile_ghncd_data,infile_covariates,CRS_locs_WGS84,in_path,out_path,covar_names,qc_flags_stations,out_prefix)
201
cnames<-c("db.name","var","range_years","range_years_clim","infile_reg_outline","infile_ghncd_data","infile_covariates","CRS_locs_WGS84","in_path","out_path","covar_names","qc_flags_stations","out_prefix")
192
list_param_prep<-list(db.name,var,range_years,range_years_clim,infile_reg_outline,infile_ghncd_data,infile_covariates,CRS_locs_WGS84,out_path,covar_names,qc_flags_stations,out_prefix)
193
cnames<-c("db.name","var","range_years","range_years_clim","infile_reg_outline","infile_ghncd_data","infile_covariates","CRS_locs_WGS84","out_path","covar_names","qc_flags_stations","out_prefix")
202 194
names(list_param_prep)<-cnames
203 195

  
204 196
##### RUN SCRIPT TO GET STATION DATA WITH COVARIATES #####
......
218 210

  
219 211
#names(outfiles_obj)<- c("loc_stations","loc_stations_ghcn","daily_covar_ghcn_data","monthly_covar_ghcn_data")
220 212

  
221
list_param_data_prep<-list(infile_monthly,infile_daily,infile_locs,infile_covariates,covar_names,var,out_prefix,CRS_locs_WGS84)
222
names(list_param_data_prep)<-c("infile_monthly","infile_daily","infile_locs","infile_covariates","covar_names","var","out_prefix","CRS_locs_WGS84")
213
list_param_data_prep <- list(infile_monthly,infile_daily,infile_locs,infile_covariates,covar_names,var,out_prefix,CRS_locs_WGS84)
214
names(list_param_data_prep) <- c("infile_monthly","infile_daily","infile_locs","infile_covariates","covar_names","var","out_prefix","CRS_locs_WGS84")
223 215

  
224 216
#Set additional parameters
225 217
#Input for sampling function...
......
232 224
dates_selected<-"" # if empty string then predict for the full year specified earlier
233 225

  
234 226
#Models to run...this can be change for each run
235
list_models<-c("y_var ~ s(x,y)",
227

  
228
list_models<-c("y_var ~ s(elev_s)",
236 229
               "y_var ~ s(LST)",
237
               "y_var ~ s(x,y,LST)",
238
               "y_var ~ s(LST,elev_s)",
230
               "y_var ~ s(elev_s,LST)",
231
               "y_var ~ s(lat) + s(lon)+ s(elev_s)",
232
               "y_var ~ s(lat,lon,elev_s)",
239 233
               "y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST)", 
240
               "y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(LC2)",
241
               "y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(LC6)")
242
#               "y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(DISTOC)")
243

  
234
               "y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(LC2)",	
235
               "y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(LC6)", 
236
               "y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(DISTOC)")
244 237
#Default name of LST avg to be matched               
245 238
lst_avg<-c("mm_01","mm_02","mm_03","mm_04","mm_05","mm_06","mm_07","mm_08","mm_09","mm_10","mm_11","mm_12")  
246 239

  
247 240
#Collect all parameters in a list
248 241
list_param_raster_prediction<-list(list_param_data_prep,
249 242
                                seed_number,nb_sample,step,constant,prop_minmax,dates_selected,
250
                                list_models,lst_avg,in_path,out_path,script_path,
243
                                list_models,lst_avg,out_path,script_path,
251 244
                                interpolation_method)
252 245
names(list_param_raster_prediction)<-c("list_param_data_prep",
253 246
                                "seed_number","nb_sample","step","constant","prop_minmax","dates_selected",
254
                                "list_models","lst_avg","in_path","out_path","script_path",
247
                                "list_models","lst_avg","out_path","script_path",
255 248
                                "interpolation_method")
256 249

  
257 250
raster_prediction_obj <-raster_prediction_fun(list_param_raster_prediction)
258 251

  
259 252
############## STAGE 5: OUTPUT ANALYSES ##################
260 253

  
261
date_selected_results<-c("20100101") ##This is for year 2000!!!
262
#raster_prediciton_obj<-load_obj(paste("raster_prediction_obj","_","interpolation_method,
263
#                                y_var_name,out_prefix,sep="")
264
list_param_results_analyses<-list(in_path,out_path,script_path,raster_prediction_obj,interpolation_method,
254
date_selected_results<-c("20100101") 
255

  
256
list_param_results_analyses<-list(out_path,script_path,raster_prediction_obj,interpolation_method,
265 257
                                  infile_covariates,covar_names,date_selected_results,var,out_prefix)
266
names(list_param_results_analyses)<-c("in_path","out_path","script_path","raster_prediction_obj","interpolation_method",
258
names(list_param_results_analyses)<-c("out_path","script_path","raster_prediction_obj","interpolation_method",
267 259
                     "infile_covariates","covar_names","date_selected_results","var","out_prefix")
268 260
#plots_assessment_by_date<-function(j,list_param){
269 261
if (stages_to_run[5]==5){
270 262
  #source(file.path(script_path,"results_interpolation_date_output_analyses_05062013.R"))
271
  plots_assessment_by_date(1,list_param_results_analyses)
263
  #Use lapply or mclapply
264
  summary_v_day <-plots_assessment_by_date(1,list_param_results_analyses)
272 265
  #Call as function...
273 266
}
274 267
  

Also available in: Unified diff