Project

General

Profile

« Previous | Next » 

Revision e4762c0b

Added by Benoit Parmentier almost 9 years ago

assessment part1, clean up before splitting of function in part1a for automation process for stage 6

View differences:

climate/research/oregon/interpolation/global_run_scalingup_assessment_part1.R
5 5
#Part 1 create summary tables and inputs files for figure in part 2 and part 3.
6 6
#AUTHOR: Benoit Parmentier 
7 7
#CREATED ON: 03/23/2014  
8
#MODIFIED ON: 12/11/2015            
8
#MODIFIED ON: 12/28/2015            
9 9
#Version: 4
10 10
#PROJECT: Environmental Layers project  
11 11
#TO DO:
12
# - generate delta and clim mosaic
13
# - Wrap in function to be able to extract information via a job?
14
# - Parallelize the region mosaics generation (by dates)?
12
# - 
13
# - Make this a function...
14
# - Drop mosaicing from the script
15 15
#
16 16
#First source these files:
17 17
#Resolved call issues from R.
......
62 62
script_path <- "/nobackupp8/bparmen1/env_layers_scripts" #path to script
63 63
source(file.path(script_path,function_analyses_paper1)) #source all functions used in this script 
64 64

  
65

  
66 65
##############################
67 66
#### Parameters and constants  
68 67

  
69 68
#Make this a function
70 69
#reg1 (North Am), reg2(Europe),reg3(Asia), reg4 (South Am), reg5 (Africa), reg6 (Australia-Asia)
71 70
#master directory containing the definition of tile size and tiles predicted
72
in_dir1 <- "/nobackupp6/aguzman4/climateLayers/out_15x45/"
71
in_dir1 <- "/nobackupp6/aguzman4/climateLayers/out/"
73 72
#/nobackupp6/aguzman4/climateLayers/out_15x45/1982
74 73

  
75
region_names <- c("reg23","reg4") #selected region names, #PARAM2 
74
#region_names <- c("reg23","reg4") #selected region names, #PARAM2 
75
region_names <- c("reg4")
76 76
#region_names <- c("1992") #no specific region here so use date
77 77
#region_names <- c("reg1","reg2","reg3","reg4","reg5","reg6") #selected region names, #PARAM2
78 78
#region_namesb <- c("reg_1b","reg_1c","reg_2b","reg_3b","reg_6b") #selected region names, #PARAM2
79 79

  
80 80
y_var_name <- "dailyTmax" #PARAM3
81 81
interpolation_method <- c("gam_CAI") #PARAM4
82
out_prefix<-"run10_1500x4500_global_analyses_pred_1992_12072015" #PARAM5
82
out_prefix<-"run_global_analyses_pred_12282015" #PARAM5
83 83

  
84 84
#output_run10_1500x4500_global_analyses_pred_2003_04102015/
85 85

  
......
91 91

  
92 92
CRS_locs_WGS84 <- CRS("+proj=longlat +ellps=WGS84 +datum=WGS84 +towgs84=0,0,0") #Station coords WGS84, #PARAM8
93 93

  
94
day_to_mosaic <- c("19920101","19920102","19920103,19920104,19920105")
95

  
96

  
94
#day_to_mosaic <- c("19920101","19920102","19920103,19920104,19920105")
97 95
#day_to_mosaic <- NULL #if day to mosaic is null then mosaic all dates?
96
year_predicted <- 1984:2004
98 97

  
99 98
file_format <- ".tif" #format for mosaiced files #PARAM10
100 99
NA_flag_val <- -9999  #No data value, #PARAM11
......
161 160

  
162 161
##raster_prediction object : contains testing and training stations with RMSE and model object
163 162

  
164

  
165 163
list_raster_obj_files <- lapply(in_dir_list,FUN=function(x){list.files(path=x,pattern="^raster_prediction_obj.*.RData",full.names=T)})
166 164
basename(dirname(list_raster_obj_files[[1]]))
167 165
list_names_tile_coord <- lapply(list_raster_obj_files,FUN=function(x){basename(dirname(x))})
......
518 516
write.table(df_tiles_all,
519 517
            file=file.path(out_dir,"shapefiles",paste("df_tiles_all_",out_prefix,".txt",sep="")),sep=",")
520 518

  
521
######################################################
522
####### PART 2 CREATE MOSAIC OF PREDICTIONS PER DAY, Delta surfaces and clim ###
523

  
524
#if mosaicing_tiles==TRUE then do it?
525
#dates_l <- unique(robj1$tb_diagnostic_s$date) #list of dates to query tif
526
#create date!!!
527
if(is.null(day_to_mosaic)){
528
  
529
  #idx <- seq(as.Date('2010-01-01'), as.Date('2010-12-31'), 'day')
530
  #idx <- seq(as.Date('20100101'), as.Date('20101231'), 'day')
531
  #date_l <- strptime(idx[1], "%Y%m%d") # interpolation date being processed
532
  #dates_l <- format(idx, "%Y%m%d") # interpolation date being processed
533
  day_to_mosaic <- dates_predicted #should be 365 days...
534
  #l_dates <- day_to_mosaic
535
}
536
#else{
537
#  l_dates <- paste(day_to_mosaic,collapse=",")
538
#}
539

  
540
## make this a function? report on number of tiles used for mosaic...
541

  
542
#inputs: build a pattern to find files
543
#y_var_name <- "dailyTmax" #set up in parameters of this script
544
#interpolation_method <- c("gam_CAI") #set up in parameters of the script
545
name_method <- paste(interpolation_method,"_",y_var_name,"_",sep="")
546
##Use python code written by Alberto Guzman
547

  
548
#system("MODULEPATH=$MODULEPATH:/nex/modules/files")
549
#system("module load /nex/modules/files/pythonkits/gdal_1.10.0_python_2.7.3_nex")
550

  
551
#module_path <- ""
552
module_path <- "/nobackupp6/aguzman4/climateLayers/sharedCode/"
553
#/nobackupp6/aguzman4/climateLayers/sharedCode/mosaicUsingGdalMerge.py
554
l_dates <- paste(day_to_mosaic,collapse=",",sep=" ")
555
## use region 2 first
556

  
557
### FIRST mosaics by processing region
558
#make this a function later...with following param
559
#input:
560
#region_names
561
#in_dir1
562
##out_dir , not ehta out_dir moasic s can be created in rhe future function
563
#mod_str <- mod1
564
#For the time being use mean,median from python function by Alberto...
565
#Solved issue about calls from R
566
#First run 3 lines in the bash shell
567
#source /nobackupp6/aguzman4/climateLayers/sharedModules/etc/environ.sh 
568
#MODULEPATH=$MODULEPATH:/nex/modules/files
569
#module load pythonkits/gdal_1.10.0_python_2.7.3_nex
570

  
571
#recombine region first:
572
region_names_mosaic <- list(region_names)
573
names(region_names_mosaic) <- "reg5"
574
in_dir_mosaics <- lapply(region_names_mosaic,FUN=function(x){file.path(in_dir1,x)})
575

  
576
for (j in 1:length(region_names_mosaic)){
577
  #for(i in 1:length(region_names))
578
  in_dir_mosaics <- lapply(region_names_mosaic,FUN=function(x){file.path(in_dir1,x)})
579
  #in_dir_mosaics <- paste(region_names_mosaic,collapse=" ")
580
  #out_dir_mosaics <- "/nobackupp6/aguzman4/climateLayers/output1000x3000_km/reg5/mosaicsMean"
581
  #Can be changed to have mosaics in different dir..
582
  out_dir_mosaics <- out_dir
583
  #prefix_str <- "reg4_1500x4500"
584
  #tile_size <- basename(dirname(in_dir[[i]]))
585
  tile_size <- basename(in_dir1)
586

  
587
  prefix_str <- paste(names(region_names_mosaic)[j],"_",tile_size,sep="")
588

  
589
  mod_str <- "mod1" #use mod2 which corresponds to model with LST and elev
590

  
591
  cmd_str <- paste("python", file.path(module_path,"mosaicUsingGdalMerge.py"),
592
                 in_dir_mosaics,
593
                 out_dir_mosaics,
594
                 prefix_str,
595
                 "--mods", mod_str,
596
                 "--date", l_dates,sep=" ")
597
  system(cmd_str)
598

  
599
}
600

  
601
### SECOND mosaics globally from regional mosaics...
602
### Now find out how many files were predicted
603
# will be useful later on
604
# Transform this into a function that takes in a list of files!!! We can skip the region stage to reduce the number of files..
605

  
606
#sh /nobackupp6/aguzman4/climateLayers/sharedCode/shMergeFromFile.sh list_mosaics_20100901.txt world_mosaics_1000x3000_20100901.tif
607
in_dir_str <- in_dir_mosaics
608
region <- "reg5"  #can be the world...
609
for (i in 1:length(day_to_mosaic)){
610
  pattern_str <- paste("*.","predicted_mod1",".*.",day_to_mosaic[i],".*.tif",sep="")
611
  lf_day_to_mosaic <- lapply(1:length(unlist(in_dir_mosaics)),FUN=function(k){list.files(path=unlist(in_dir_mosaics)[k],pattern=pattern_str,full.names=T,recursive=T)}) 
612
  lf_day_to_mosaic <- unlist(lf_day_to_mosaic)
613
  #write.table(lf_day_to_mosaic,file=file.path(out_dir,paste("list_to_mosaics_",day_to_mosaic[i],".txt",sep="")))
614
  writeLines(lf_day_to_mosaic,con=file.path(out_dir,paste("list_to_mosaics_",day_to_mosaic[i],".txt",sep="")))
615
  in_file_to_mosaics <- file.path(out_dir,paste("list_to_mosaics_",day_to_mosaic[i],".txt",sep=""))        
616
  #in_dir_mosaics <- file.path(in_dir1,region_names[i])
617
  #out_dir_mosaics <- "/nobackupp6/aguzman4/climateLayers/output1000x3000_km/reg5/mosaicsMean"
618
  #Can be changed to have mosaics in different dir..
619
  #out_dir_mosaics <- out_dir
620
  #prefix_str <- "reg4_1500x4500"
621
  #tile_size <- basename(dirname(in_dir[[i]]))
622
  tile_size <- basename(in_dir1)
623

  
624
  #prefix_str <- paste(region_names[i],"_",tile_size,sep="")
625
  mod_str <- "mod1" #use mod2 which corresponds to model with LST and elev
626
  out_mosaic_name <- paste(region,"_mosaics_",mod_str,"_",tile_size,"_",day_to_mosaic[i],"_",out_prefix,".tif",sep="")
627
  module_path <- "/nobackupp6/aguzman4/climateLayers/sharedCode" #this should be a parameter for the function...
628
  cmd_str <- paste("sh", file.path(module_path,"shMergeFromFile.sh"),
629
                 in_file_to_mosaics,
630
                 out_mosaic_name,
631
                 sep=" ")
632
  system(cmd_str)
633

  
634
}
635

  
636

  
637 519
########### LAST PART: COPY SOME DATA BACK TO ATLAS #####
638 520

  
639 521
#This will be a separate script?? or function?

Also available in: Unified diff