Revision 3193de08
Added by Benoit Parmentier over 11 years ago
climate/research/oregon/interpolation/master_script_temp.R | ||
---|---|---|
10 | 10 |
#STAGE 5: Output analyses: assessment of results for specific dates... |
11 | 11 |
# |
12 | 12 |
#AUTHOR: Benoit Parmentier |
13 |
#DATE: 05/27/2013
|
|
13 |
#DATE: 05/30/2013
|
|
14 | 14 |
|
15 | 15 |
#PROJECT: NCEAS INPLANT: Environment and Organisms --TASK#363, TASK$568-- |
16 | 16 |
|
17 |
##Comments and TODO:
|
|
17 |
## TODO: |
|
18 | 18 |
# Modify code for stage 1 and call python script from R |
19 | 19 |
# Modify code for stage 2, make it a function and fully automated (distoc var) |
20 | 20 |
# Add options to run only specific stage + additional out_suffix? |
21 | 21 |
# Make master script a function? |
22 | 22 |
# Add log file for master script,add function to collect inputs and outputs |
23 |
# |
|
23 |
# Comments for run: |
|
24 |
#Testing full code for 5 stages (no downloading) for Oregon region. |
|
24 | 25 |
################################################################################################## |
25 | 26 |
|
26 | 27 |
###Loading R library and packages |
... | ... | |
54 | 55 |
modis_download_script <- file.path(script_path,"modis_download_05142013.py") # LST modis download python script |
55 | 56 |
clim_script <- file.path(script_path,"climatology_05142013.py") # LST climatology python script |
56 | 57 |
grass_setting_script <- file.path(script_path,"grass-setup.R") #Set up system shell environment for python+GRASS |
57 |
#source(file.path(script_path,"download_and_produce_MODIS_LST_climatology_05162013.R"))
|
|
58 |
source(file.path(script_path,"covariates_production_temperatures_05272013.R"))
|
|
58 |
source(file.path(script_path,"download_and_produce_MODIS_LST_climatology_05302013.R"))
|
|
59 |
source(file.path(script_path,"covariates_production_temperatures_05302013.R"))
|
|
59 | 60 |
source(file.path(script_path,"Database_stations_covariates_processing_function_05212013.R")) |
60 | 61 |
source(file.path(script_path,"GAM_fusion_analysis_raster_prediction_multisampling_05212013.R")) |
61 | 62 |
source(file.path(script_path,"results_interpolation_date_output_analyses_05062013.R")) |
... | ... | |
68 | 69 |
source(file.path(script_path,"GAM_fusion_function_multisampling_validation_metrics_05062013.R")) |
69 | 70 |
|
70 | 71 |
#stages_to_run<-c(1,2,3,4,5) #May decide on antoher strategy later on... |
71 |
stages_to_run<-c(0,2,3,4,5) #May decide on antoher strategy later on...
|
|
72 |
stages_to_run<-c(1,2,3,4,5) #May decide on antoher strategy later on...
|
|
72 | 73 |
|
73 | 74 |
var<-"TMAX" # variable being interpolated |
74 |
out_prefix<-"_365d_GAM_fus_all_lst_05242013" #User defined output prefix
|
|
75 |
out_suffix<-"_VE_05242013"
|
|
76 |
out_suffix_modis <-"_05242013"
|
|
75 |
out_prefix<-"_365d_GAM_fus_all_lst_05302013" #User defined output prefix
|
|
76 |
out_suffix<-"_OR_05302013"
|
|
77 |
out_suffix_modis <-"_05302013"
|
|
77 | 78 |
|
78 | 79 |
#interpolation_method<-c("gam_fusion","gam_CAI") #other otpions to be added later |
79 | 80 |
#interpolation_method<-c("gam_CAI") #other otpions to be added later |
80 | 81 |
interpolation_method<-c("gam_fusion") #other otpions to be added later |
81 | 82 |
|
82 |
#Change input path?? for LST stage ??
|
|
83 |
in_path <- "/home/parmentier/Data/IPLANT_project/Venezuela_interpolation/Venezuela_01142013/input_data/"
|
|
84 |
#in_path <- out_path
|
|
85 |
out_path <- paste("/home/parmentier/Data/IPLANT_project/Venezuela_interpolation/Venezuela_01142013/output_data",
|
|
86 |
out_prefix,"/",sep="") |
|
83 |
#out_path <- paste("/home/parmentier/Data/IPLANT_project/Venezuela_interpolation/Venezuela_01142013/output_data",
|
|
84 |
# out_prefix,"/",sep="")
|
|
85 |
out_path<-"/home/parmentier/Data/IPLANT_project/Oregon_interpolation/Oregon_03142013/output_data"
|
|
86 |
out_path <-paste(out_path,out_prefix,sep="")
|
|
87 |
|
|
87 | 88 |
if (!file.exists(out_path)){ |
88 | 89 |
dir.create(out_path) |
89 | 90 |
#} else{ |
... | ... | |
101 | 102 |
#infile_covariates<- "/home/parmentier/Data/IPLANT_project/Venezuela_interpolation/Venezuela_01142013/covariates_Oregon_region_TMAX__OR_04052013.tif" #Oregon covar TMAX from earlier codes...for continuity |
102 | 103 |
#infile_reg_outline="" #input region outline defined by polygon: none for Venezuela |
103 | 104 |
#This is the shape file of outline of the study area #It is an input/output of the covariate script |
104 |
#infile_reg_outline <- "OR83M_state_outline.shp" #input region outline defined by polygon: Oregon
|
|
105 |
infile_reg_outline <- "/home/parmentier/Data/IPLANT_project/Oregon_interpolation/Oregon_03142013/OR83M_state_outline.shp" #input region outline defined by polygon: Oregon
|
|
105 | 106 |
#infile_reg_outline <-"OR83M_state_outline.shp" #remove this parameter!!! |
106 |
#covar_names see stage 2 |
|
107 |
#ref_rast_name<-"" #local raster name defining resolution, exent, local projection--. set on the fly?? |
|
108 |
#this may be redundant with infile_reg_outline |
|
109 |
ref_rast_name<-"/home/parmentier/Data/IPLANT_project/Oregon_interpolation/Oregon_03142013/mean_day244_rescaled.rst" #local raster name defining resolution, exent: oregon |
|
107 | 110 |
|
108 |
#lc_path<-"/home/layers/data/land-cover/lc-consensus-global" |
|
109 |
#infile_modis_grid<-"/home/layers/commons/modis/modis_sinusoidal/modis_sinusoidal_grid_world.shp" #Give path!!! NEED TO CHANGE THIS... |
|
110 |
#infile_elev<-"/home/layers/data/terrain/dem-cgiar-srtm-1km-tif/srtm_1km.tif" #this is the global file: replace later with the input produced by the DEM team |
|
111 |
#infile_canheight<-"/home/layers/data/land-cover/treeheight-simard2011/Simard_Pinto_3DGlobalVeg_JGR.tif" #Canopy height |
|
111 |
#covar_names see stage 2 |
|
112 | 112 |
|
113 |
list_tiles_modis <- c("h11v08,h11v07,h12v07,h12v08,h10v07,h10v08") #tile for Venezuela and surrounding area |
|
114 |
#list_tiles_modis <- c("h08v04","h09v04") #tiles for Oregon
|
|
113 |
#list_tiles_modis <- c("h11v08,h11v07,h12v07,h12v08,h10v07,h10v08") #tile for Venezuela and surrounding area
|
|
114 |
list_tiles_modis <- c("h08v04,h09v04") #tiles for Oregon
|
|
115 | 115 |
|
116 |
infile_reg_outline="" #input region outline defined by polygon: none for Venezuela |
|
117 |
#infile_reg_outline <- "OR83M_state_outline.shp" #input region outline defined by polygon: Oregon |
|
118 |
|
|
119 |
CRS_interp<-"+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs"; |
|
120 |
#CRS_interp <-"+proj=lcc +lat_1=43 +lat_2=45.5 +lat_0=41.75 +lon_0=-120.5 +x_0=400000 +y_0=0 +ellps=GRS80 +units=m +no_defs"; |
|
116 |
#CRS_interp<-"+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs"; |
|
117 |
CRS_interp <-"+proj=lcc +lat_1=43 +lat_2=45.5 +lat_0=41.75 +lon_0=-120.5 +x_0=400000 +y_0=0 +ellps=GRS80 +units=m +no_defs"; |
|
121 | 118 |
CRS_locs_WGS84<-CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84 |
122 |
out_region_name<-"_venezuela_region" #generated on the fly |
|
123 |
#out_suffix<-"_VE_05252013" |
|
124 |
ref_rast_name<-"" #local raster name defining resolution, exent, local projection--. set on the fly?? |
|
125 |
#this may be redundant with infile_reg_outline |
|
126 |
#ref_rast_name<-"mean_day244_rescaled.rst" #local raster name defining resolution, exent: oregon |
|
119 |
#out_region_name<-"_venezuela_region" #generated on the fly |
|
120 |
out_region_name<-"_oregon_region" #generated on the fly |
|
127 | 121 |
|
128 | 122 |
#The names of covariates can be changed...these names should be output/input from covar script!!! |
129 | 123 |
rnames<-c("x","y","lon","lat","N","E","N_w","E_w","elev_s","slope","aspect","CANHEIGHT","DISTOC") |
... | ... | |
140 | 134 |
#list_tiles_modis <- c("h08v04,h09v04") #tiles for Oregon #defined above... |
141 | 135 |
start_year = "2001" |
142 | 136 |
end_year = "2010" |
143 |
#end_year = "2002" #for testing (year included?) |
|
144 | 137 |
#hdfdir = '/home/layers/commons/modis/MOD11A1_tiles' #destination file where hdf files are stored locally after download. |
145 |
#hdfdir = '/home/parmentier/Data/IPLANT_project/MOD11A1_tiles' |
|
146 | 138 |
hdfdir <- "/data/project/layers/commons/modis/MOD11A1_tiles" |
147 |
download=1 |
|
148 |
clim_calc=0 |
|
149 |
#out_suffix_modis="_05252013" |
|
150 |
#end_month= "12" |
|
151 |
#start_month= "1" |
|
139 |
download=0 |
|
140 |
clim_calc=1 |
|
152 | 141 |
|
153 | 142 |
list_param_download_clim_LST_script <- list(list_tiles_modis,start_year,end_year,hdfdir, |
154 | 143 |
var,grass_setting_script,modis_download_script, clim_script, |
... | ... | |
156 | 145 |
names(list_param_download_clim_LST_script)<-c("list_tiles_modis","start_year","end_year","hdfdir", |
157 | 146 |
"var","grass_setting_script","modis_download_script","clim_script", |
158 | 147 |
"download","clim_calc","out_suffix_modis") |
148 |
no_tiles <- length(unlist(strsplit(list_tiles_modis,","))) # transform string into separate element in char vector |
|
159 | 149 |
|
160 | 150 |
if (stages_to_run[1]==1){ |
151 |
#clim_production_obj <-mclapply(1:2, list_param=list_param_download_clim_LST_script, download_calculate_MODIS_LST_climatology,mc.preschedule=FALSE,mc.cores = 2) #This is the end bracket from mclapply(...) statement |
|
152 |
clim_production_obj <-lapply(1:no_tiles, list_param=list_param_download_clim_LST_script, download_calculate_MODIS_LST_climatology) #,mc.preschedule=FALSE,mc.cores = 2) #This is the end bracket from mclapply(...) statement |
|
161 | 153 |
|
162 |
download_calculate_MODIS_LST_climatology(1,list_param_download_clim_LST_script) |
|
163 |
|
|
164 | 154 |
} |
165 | 155 |
#Collect LST climatology list as output??? |
166 | 156 |
|
167 | 157 |
############ STAGE 2: Covariate production ################ |
168 | 158 |
|
169 | 159 |
#list of 17 parameters |
170 |
list_param_covar_production<-list(var,in_path,out_path,lc_path,infile_modis_grid,infile_elev,infile_canheight,
|
|
160 |
list_param_covar_production<-list(var,out_path,lc_path,infile_modis_grid,infile_elev,infile_canheight, |
|
171 | 161 |
infile_distoc,list_tiles_modis,infile_reg_outline,CRS_interp,CRS_locs_WGS84,out_region_name, |
172 | 162 |
out_suffix,out_suffix_modis,ref_rast_name,hdfdir,covar_names) |
173 | 163 |
|
174 |
names(list_param_covar_production)<-c("var","in_path","out_path","lc_path","infile_modis_grid","infile_elev","infile_canheight",
|
|
164 |
names(list_param_covar_production)<-c("var","out_path","lc_path","infile_modis_grid","infile_elev","infile_canheight", |
|
175 | 165 |
"infile_distoc","list_tiles_modis","infile_reg_outline","CRS_interp","CRS_locs_WGS84","out_region_name", |
176 | 166 |
"out_suffix","out_suffix_modis","ref_rast_name","hdfdir","covar_names") |
177 | 167 |
|
... | ... | |
192 | 182 |
range_years<-c("2010","2011") #right bound not included in the range!! |
193 | 183 |
range_years_clim<-c("2000","2011") #right bound not included in the range!! |
194 | 184 |
infile_ghncd_data <-"/home/layers/data/climate/ghcn/v2.92-upd-2012052822/ghcnd-stations.txt" #This is the textfile of station locations from GHCND |
195 |
qc_flags_stations<-c("0","S") #flags allowed for screening after the query from the GHCND?? |
|
185 |
#qc_flags_stations<-c("0","S") #flags allowed for screening after the query from the GHCND?? |
|
186 |
qc_flags_stations<-c("0") #flags allowed for screening after the query from the GHCND?? |
|
187 |
|
|
196 | 188 |
#infile_covariates and infile_reg_outline defined in stage 2 or at the start of script... |
197 | 189 |
|
198 | 190 |
#list of 12 parameters for input in the function... |
199 | 191 |
|
200 |
list_param_prep<-list(db.name,var,range_years,range_years_clim,infile_reg_outline,infile_ghncd_data,infile_covariates,CRS_locs_WGS84,in_path,out_path,covar_names,qc_flags_stations,out_prefix)
|
|
201 |
cnames<-c("db.name","var","range_years","range_years_clim","infile_reg_outline","infile_ghncd_data","infile_covariates","CRS_locs_WGS84","in_path","out_path","covar_names","qc_flags_stations","out_prefix")
|
|
192 |
list_param_prep<-list(db.name,var,range_years,range_years_clim,infile_reg_outline,infile_ghncd_data,infile_covariates,CRS_locs_WGS84,out_path,covar_names,qc_flags_stations,out_prefix) |
|
193 |
cnames<-c("db.name","var","range_years","range_years_clim","infile_reg_outline","infile_ghncd_data","infile_covariates","CRS_locs_WGS84","out_path","covar_names","qc_flags_stations","out_prefix") |
|
202 | 194 |
names(list_param_prep)<-cnames |
203 | 195 |
|
204 | 196 |
##### RUN SCRIPT TO GET STATION DATA WITH COVARIATES ##### |
... | ... | |
218 | 210 |
|
219 | 211 |
#names(outfiles_obj)<- c("loc_stations","loc_stations_ghcn","daily_covar_ghcn_data","monthly_covar_ghcn_data") |
220 | 212 |
|
221 |
list_param_data_prep<-list(infile_monthly,infile_daily,infile_locs,infile_covariates,covar_names,var,out_prefix,CRS_locs_WGS84)
|
|
222 |
names(list_param_data_prep)<-c("infile_monthly","infile_daily","infile_locs","infile_covariates","covar_names","var","out_prefix","CRS_locs_WGS84")
|
|
213 |
list_param_data_prep <- list(infile_monthly,infile_daily,infile_locs,infile_covariates,covar_names,var,out_prefix,CRS_locs_WGS84)
|
|
214 |
names(list_param_data_prep) <- c("infile_monthly","infile_daily","infile_locs","infile_covariates","covar_names","var","out_prefix","CRS_locs_WGS84")
|
|
223 | 215 |
|
224 | 216 |
#Set additional parameters |
225 | 217 |
#Input for sampling function... |
... | ... | |
232 | 224 |
dates_selected<-"" # if empty string then predict for the full year specified earlier |
233 | 225 |
|
234 | 226 |
#Models to run...this can be change for each run |
235 |
list_models<-c("y_var ~ s(x,y)", |
|
227 |
|
|
228 |
list_models<-c("y_var ~ s(elev_s)", |
|
236 | 229 |
"y_var ~ s(LST)", |
237 |
"y_var ~ s(x,y,LST)", |
|
238 |
"y_var ~ s(LST,elev_s)", |
|
230 |
"y_var ~ s(elev_s,LST)", |
|
231 |
"y_var ~ s(lat) + s(lon)+ s(elev_s)", |
|
232 |
"y_var ~ s(lat,lon,elev_s)", |
|
239 | 233 |
"y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST)", |
240 |
"y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(LC2)", |
|
241 |
"y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(LC6)") |
|
242 |
# "y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(DISTOC)") |
|
243 |
|
|
234 |
"y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(LC2)", |
|
235 |
"y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(LC6)", |
|
236 |
"y_var ~ s(lat,lon) + s(elev_s) + s(N_w,E_w) + s(LST) + s(DISTOC)") |
|
244 | 237 |
#Default name of LST avg to be matched |
245 | 238 |
lst_avg<-c("mm_01","mm_02","mm_03","mm_04","mm_05","mm_06","mm_07","mm_08","mm_09","mm_10","mm_11","mm_12") |
246 | 239 |
|
247 | 240 |
#Collect all parameters in a list |
248 | 241 |
list_param_raster_prediction<-list(list_param_data_prep, |
249 | 242 |
seed_number,nb_sample,step,constant,prop_minmax,dates_selected, |
250 |
list_models,lst_avg,in_path,out_path,script_path,
|
|
243 |
list_models,lst_avg,out_path,script_path, |
|
251 | 244 |
interpolation_method) |
252 | 245 |
names(list_param_raster_prediction)<-c("list_param_data_prep", |
253 | 246 |
"seed_number","nb_sample","step","constant","prop_minmax","dates_selected", |
254 |
"list_models","lst_avg","in_path","out_path","script_path",
|
|
247 |
"list_models","lst_avg","out_path","script_path", |
|
255 | 248 |
"interpolation_method") |
256 | 249 |
|
257 | 250 |
raster_prediction_obj <-raster_prediction_fun(list_param_raster_prediction) |
258 | 251 |
|
259 | 252 |
############## STAGE 5: OUTPUT ANALYSES ################## |
260 | 253 |
|
261 |
date_selected_results<-c("20100101") ##This is for year 2000!!! |
|
262 |
#raster_prediciton_obj<-load_obj(paste("raster_prediction_obj","_","interpolation_method, |
|
263 |
# y_var_name,out_prefix,sep="") |
|
264 |
list_param_results_analyses<-list(in_path,out_path,script_path,raster_prediction_obj,interpolation_method, |
|
254 |
date_selected_results<-c("20100101") |
|
255 |
|
|
256 |
list_param_results_analyses<-list(out_path,script_path,raster_prediction_obj,interpolation_method, |
|
265 | 257 |
infile_covariates,covar_names,date_selected_results,var,out_prefix) |
266 |
names(list_param_results_analyses)<-c("in_path","out_path","script_path","raster_prediction_obj","interpolation_method",
|
|
258 |
names(list_param_results_analyses)<-c("out_path","script_path","raster_prediction_obj","interpolation_method", |
|
267 | 259 |
"infile_covariates","covar_names","date_selected_results","var","out_prefix") |
268 | 260 |
#plots_assessment_by_date<-function(j,list_param){ |
269 | 261 |
if (stages_to_run[5]==5){ |
270 | 262 |
#source(file.path(script_path,"results_interpolation_date_output_analyses_05062013.R")) |
271 |
plots_assessment_by_date(1,list_param_results_analyses) |
|
263 |
#Use lapply or mclapply |
|
264 |
summary_v_day <-plots_assessment_by_date(1,list_param_results_analyses) |
|
272 | 265 |
#Call as function... |
273 | 266 |
} |
274 | 267 |
|
Also available in: Unified diff
master script, debugging covariates production for Oregon study area