Revision 1580701e
Added by Benoit Parmentier almost 9 years ago
| climate/research/oregon/interpolation/inputCreate_stage_6.py | ||
|---|---|---|
| 22 | 22 |
# |
| 23 | 23 |
#Output: |
| 24 | 24 |
# Places number of files into directory. |
| 25 |
#/nobackupp6/aguzman4/climateLayers/out/ |
|
| 26 |
# |
|
| 27 |
# |
|
| 28 |
# |
|
| 29 |
# |
|
| 25 | 30 |
|
| 26 | 31 |
def main(): |
| 27 |
usage = "usage: %prog [options] <inLatLonList> <outputFolder> <outDirForShFiles> <filesPerDir> <lowFeatureCount> <highFeatureCount> <scriptFn> <prefix> <year>\n"+\ |
|
| 32 |
#usage = "usage: %prog [options] <inLatLonList> <outputFolder> <outDirForShFiles> <filesPerDir> <lowFeatureCount> <highFeatureCount> <scriptFn> <prefix> <year>\n"+\ |
|
| 33 |
# "Prepare input for stage 4 " |
|
| 34 |
#var <- args[1] # variable being interpolated #param 1, arg 1 |
|
| 35 |
#in_dir1 <- args[2] #param 5, arg 2 |
|
| 36 |
#region_name <- args[3] #param 6, arg 3 |
|
| 37 |
#out_prefix <- args[4] #param 7, arg 4 |
|
| 38 |
#out_dir <- args[5] #param 8, arg 5 |
|
| 39 |
#create_out_dir_param <- args[6] #param 9, arg 6 |
|
| 40 |
#list_year_predicted <- args[7] # param 10, arg 7, replaced by yearInt here |
|
| 41 |
#num_cores <- args[8] #number of cores used # param 13, arg 8 |
|
| 42 |
#max_mem <- args[9] #param 21 |
|
| 43 |
|
|
| 44 |
#python /nobackupp6/aguzman4/climateLayers/finalCode/environmental-layers/climate/research/world/interpolation/runSetup/inputCreate_stage_4.py |
|
| 45 |
#/nobackupp6/aguzman4/climateLayers/inputLayers/regionCSV/list_reg4.csv |
|
| 46 |
#/nobackupp6/aguzman4/climateLayers/out/reg4/ |
|
| 47 |
#/nobackupp6/aguzman4/climateLayers/out/reg4/serialStage6/2010/ |
|
| 48 |
#10 |
|
| 49 |
#0 |
|
| 50 |
#250000 |
|
| 51 |
#/nobackupp6/aguzman4/climateLayers/finalCode/environmental-layers/climate/research/world/interpolation/master_script_stage_4.R |
|
| 52 |
#r4 |
|
| 53 |
#2010 |
|
| 54 |
|
|
| 55 |
inFn = "/nobackupp6/aguzman4/climateLayers/inputLayers/regionCSV/list_reg4.csv" |
|
| 56 |
outputFolder = "/nobackupp6/aguzman4/climateLayers/out/reg4/" |
|
| 57 |
outDir = "/nobackupp6/aguzman4/climateLayers/out/reg4/serialStage6/2010/" |
|
| 58 |
fPer = 10 #subdivide the job for NEX |
|
| 59 |
lowTresh = 0 |
|
| 60 |
hiTresh = int(250000) |
|
| 61 |
scriptFn = "/nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01182016.R" |
|
| 62 |
#scriptFn = "/nobackupp6/aguzman4/climateLayers/finalCode/environmental-layers/climate/research/world/interpolation/master_script_stage_4.R" |
|
| 63 |
prefix = r4 |
|
| 64 |
year = 2010 #give a range of date here |
|
| 65 |
var = "TMAX" # variable being interpolated #param 1, arg 1 |
|
| 66 |
in_dir1 = "/nobackupp6/aguzman4/climateLayers/out/" #param 5, arg 2 |
|
| 67 |
region_name = "reg4" #param 6, arg 3 |
|
| 68 |
out_prefix = "run_global_analyses_pred_12282015" #param 7, arg 4 |
|
| 69 |
out_dir = "/nobackupp8/bparmen1/" #param 8, arg 5 |
|
| 70 |
create_out_dir_param = "TRUE" #param 9, arg 6 |
|
| 71 |
list_year_predicted = 2010 # param 10, arg 7 |
|
| 72 |
num_cores = 6 #number of cores used # param 13, arg 8 |
|
| 73 |
max_mem = 1e+07 #param 21, arg 9 |
|
| 74 |
|
|
| 75 |
usage = "usage: %prog [options] <inLatLonList> <outputFolder> <outDirForShFiles> <filesPerDir> <lowFeatureCount> <highFeatureCount> <scriptFn> <prefix> "+\ |
|
| 76 |
" <var> <in_dir1> <region_name> <out_prefix> <out_dir> <create_out_dir_param> <yearInt> <num_cores> <max_mem> "+\ |
|
| 28 | 77 |
"Prepare input for stage 4 " |
| 29 | 78 |
parser = OptionParser(usage=usage) |
| 30 | 79 |
opts,args=parser.parse_args() |
| ... | ... | |
| 44 | 93 |
inPtr = open(inFn,"r") |
| 45 | 94 |
if inPtr is None: |
| 46 | 95 |
print "Error: %s doesnt' exist" % inFn |
| 47 |
lines = inPtr.readlines() |
|
| 96 |
lines = inPtr.readlines() #tile directory one per line, 28 for region 4
|
|
| 48 | 97 |
inPtr.close() |
| 49 | 98 |
|
| 50 | 99 |
found = 0 |
| ... | ... | |
| 59 | 108 |
print "Warning: %s doesn't exists, creating" % wrap |
| 60 | 109 |
wrapFn = os.path.abspath(os.path.dirname(sys.argv[0]))+"/wrapper.sh" |
| 61 | 110 |
shutil.copyfile(wrapFn,wrap) |
| 62 |
os.chmod(wrap,0750) |
|
| 111 |
os.chmod(wrap,0750) #set permission...
|
|
| 63 | 112 |
|
| 64 | 113 |
qs = "%s/serial_TEMPLATE.qsub" % outDir |
| 65 | 114 |
if os.path.exists(qs) is False: |
| ... | ... | |
| 105 | 154 |
#notFound += 1 |
| 106 | 155 |
print "Error finding %s" % searchStr |
| 107 | 156 |
|
| 108 |
sortedShpFiles = sorted(shpFiles,key=lambda x: x[1]) |
|
| 157 |
sortedShpFiles = sorted(shpFiles,key=lambda x: x[1]) #sort by numer of stations...
|
|
| 109 | 158 |
#print sortedShpFiles |
| 110 | 159 |
|
| 111 | 160 |
stCFn = "%s/stationCounts.txt" % outDir |
| ... | ... | |
| 119 | 168 |
fPtrSt.write(outSt) |
| 120 | 169 |
fPtrSt.close() |
| 121 | 170 |
|
| 171 |
|
|
| 122 | 172 |
stDim = len(sortedShpFiles)-1 |
| 123 | 173 |
|
| 124 | 174 |
stMin = int(sortedShpFiles[0][1]) |
| ... | ... | |
| 186 | 236 |
##Input files that contain Rscript command |
| 187 | 237 |
#binSingle: this contains a list of tiles to run ranked by time |
| 188 | 238 |
#run by year |
| 189 |
for b in binSingle: |
|
| 190 |
ll = b[2]#"%s_%s" % (baseSpl[1],baseSpl[2]) |
|
| 239 |
#for b in binSingle: |
|
| 240 |
for year in yearInt: |
|
| 241 |
#ll = b[2]#"%s_%s" % (baseSpl[1],baseSpl[2]) |
|
| 191 | 242 |
#check for existence of files from stage2 |
| 192 |
metTest = "%s/%s/%s/met_stations_outfiles_obj_gam_CAI_%s.RData" % (outputFolder,ll,year,ll) |
|
| 243 |
#metTest = "%s/%s/%s/met_stations_outfiles_obj_gam_CAI_%s.RData" % (outputFolder,ll,year,ll)
|
|
| 193 | 244 |
|
| 194 | 245 |
#if os.path.exists(metTest) is False: |
| 195 | 246 |
# print "No met object %s" % metTest |
| ... | ... | |
| 200 | 251 |
# print "Method object exists" |
| 201 | 252 |
# continue |
| 202 | 253 |
|
| 203 |
yearInt = int(year)+1 |
|
| 204 |
outStr = "Rscript %s %s wgs84Grid %s %s %s %s/subset/mean_LST_%s_jan_6_wgs84.tif FALSE %s/%s/covar_obj_%s.RData %s/%s/%s/met_stations_outfiles_obj_gam_CAI_%s.RData 10 4800 %s %s > %s/outLogs/%s_stage4_%s.log 2> %s/outLogs/%s_stage4_err_%s.log" % (scriptFn,ll,ll,outputFolder,b[0],outputFolder,ll,outputFolder,ll,ll,outputFolder,ll,year,ll,year,yearInt,outputFolder,ll,year,outputFolder,ll,year) |
|
| 254 |
#yearInt = int(year)+1 #interval of year e.g. 201-2015 |
|
| 255 |
#var <- args[1] # variable being interpolated #param 1, arg 1 |
|
| 256 |
#in_dir1 <- args[2] #param 5, arg 2 |
|
| 257 |
#region_name <- args[3] #param 6, arg 3 |
|
| 258 |
#out_prefix <- args[4] #param 7, arg 4 |
|
| 259 |
#out_dir <- args[5] #param 8, arg 5 |
|
| 260 |
#create_out_dir_param <- args[6] #param 9, arg 6 |
|
| 261 |
#list_year_predicted <- args[7] # param 10, arg 7 |
|
| 262 |
#num_cores <- args[8] #number of cores used # param 13, arg 8 |
|
| 263 |
#max_mem <- args[9] #param 21 |
|
| 264 |
list_year_predicted = list(year) #list with one element |
|
| 265 |
#"Rscript %s %s %s %s %s %s %s %s" % (var,in_dir1,region_name,out_prefix,out_dir,create_out_dir_param,list_year_predicted,num_cores,max_mem) |
|
| 266 |
#var = "TMAX" # variable being interpolated #param 1, arg 1 |
|
| 267 |
#in_dir1 = "/nobackupp6/aguzman4/climateLayers/out/" #param 5, arg 2 |
|
| 268 |
#region_name = "reg4" #param 6, arg 3 |
|
| 269 |
#out_prefix = "run_global_analyses_pred_12282015" #param 7, arg 4 |
|
| 270 |
#out_dir = "/nobackupp8/bparmen1/" #param 8, arg 5 |
|
| 271 |
#create_out_dir_param = "TRUE" #param 9, arg 6 |
|
| 272 |
#list_year_predicted = c(2010) # param 10, arg 7 |
|
| 273 |
#num_cores = 6 #number of cores used # param 13, arg 8 |
|
| 274 |
#max_mem = 1e+07 #param 21, arg 9 |
|
| 275 |
|
|
| 276 |
#scriptFn= /nobackupp8/bparmen1/env_layers_scripts/master_script_stage_6_01182016.R |
|
| 277 |
outStr = "Rscript %s %s %s %s %s %s %s %s" % (scriptFn,var,in_dir1,region_name,out_prefix,out_dir,create_out_dir_param,list_year_predicted,num_cores,max_mem) |
|
| 278 |
|
|
| 279 |
#outStr = "Rscript %s %s wgs84Grid %s %s %s %s/subset/mean_LST_%s_jan_6_wgs84.tif FALSE %s/%s/covar_obj_%s.RData %s/%s/%s/met_stations_outfiles_obj_gam_CAI_%s.RData 10 4800 %s %s > %s/outLogs/%s_stage4_%s.log 2> %s/outLogs/%s_stage4_err_%s.log" % (scriptFn,ll,ll,outputFolder,b[0],outputFolder,ll,outputFolder,ll,ll,outputFolder,ll,year,ll,year,yearInt,outputFolder,ll,year,outputFolder,ll,year) |
|
| 205 | 280 |
#print outStr |
| 206 | 281 |
|
| 207 | 282 |
outFnSt = "%s/dirSub_%d/input_%d.in" % (outDir,dirCounter,rankCount) |
| ... | ... | |
| 370 | 445 |
#python /nobackupp6/aguzman4/climateLayers/finalCode/environmental-layers/climate/research/world/interpolation/runSetup/inputCreate_stage_4.py /nobackupp6/aguzman4/climateLayers/inputLayers/regionCSV/list_reg4.csv /nobackupp6/aguzman4/climateLayers/out/reg4/ /nobackupp6/aguzman4/climateLayers/out/reg4/serialStage6/2010/ 10 0 250000 /nobackupp6/aguzman4/climateLayers/finalCode/environmental-layers/climate/research/world/interpolation/master_script_stage_4.R r4 2010 |
| 371 | 446 |
#python /nobackupp6/aguzman4/climateLayers/finalCode/environmental-layers/climate/research/world/interpolation/runSetup/master_script_stage_6_01182016.R /nobackupp6/aguzman4/climateLayers/inputLayers/regionCSV/list_reg4.csv /nobackupp6/aguzman4/climateLayers/out/reg4/ /nobackupp6/aguzman4/climateLayers/out/reg4/serialStage6/2010/ 10 0 250000 /nobackupp6/aguzman4/climateLayers/finalCode/environmental-layers/climate/research/world/interpolation/master_script_stage_6_01182016.R r4 2010 |
| 372 | 447 |
|
| 448 |
#Calculate times it takes by dates and region? |
|
| 449 |
#Add all bin count together for each year |
|
| 450 |
#loop over year? |
|
| 451 |
|
|
| 452 |
#inFn = "/nobackupp6/aguzman4/climateLayers/inputLayers/regionCSV/list_reg4.csv" |
|
| 453 |
#outputFolder = "/nobackupp6/aguzman4/climateLayers/out/reg4/" |
|
| 454 |
#outDir = "/nobackupp6/aguzman4/climateLayers/out/reg4/serialStage6/2010/" |
|
| 455 |
#fPer = 10 #subdivide the job for NEX |
|
| 456 |
lowTresh = 0 |
|
| 457 |
hiTresh = int(250000) |
|
| 458 |
scriptFn = "/nobackupp6/aguzman4/climateLayers/finalCode/environmental-layers/climate/research/world/interpolation/master_script_stage_4.R" |
|
| 459 |
prefix = r4 |
|
| 460 |
year = 2010 #give a range of date here |
|
| 461 |
var = "TMAX" # variable being interpolated #param 1, arg 1 |
|
| 462 |
in_dir1 = "/nobackupp6/aguzman4/climateLayers/out/" #param 5, arg 2 |
|
| 463 |
region_name = "reg4" #param 6, arg 3 |
|
| 464 |
out_prefix = "run_global_analyses_pred_12282015" #param 7, arg 4 |
|
| 465 |
out_dir = "/nobackupp8/bparmen1/" #param 8, arg 5 |
|
| 466 |
create_out_dir_param = "TRUE" #param 9, arg 6 |
|
| 467 |
list_year_predicted = 2010 # param 10, arg 7 |
|
| 468 |
num_cores = 6 #number of cores used # param 13, arg 8 |
|
| 469 |
max_mem = 1e+07 #param 21, arg 9 |
|
| 470 |
|
|
| 471 |
|
|
| 472 |
inPtr = open(inFn,"r") |
|
| 473 |
if inPtr is None: |
|
| 474 |
print "Error: %s doesnt' exist" % inFn |
|
| 475 |
lines = inPtr.readlines() |
|
| 476 |
inPtr.close() |
|
| 477 |
|
|
| 478 |
found = 0 |
|
| 479 |
notFound = 0 |
|
| 480 |
|
|
| 481 |
if os.path.exists(outDir) is False: |
|
| 482 |
print "Warning: %s doesn't exists, creating" % outDir |
|
| 483 |
os.mkdir(outDir) |
|
| 484 |
|
|
| 485 |
wrap = "%s/wrapper.sh" % outDir #this is the wrapper to launch all the codes together |
|
| 486 |
if os.path.exists(wrap) is False: |
|
| 487 |
print "Warning: %s doesn't exists, creating" % wrap |
|
| 488 |
wrapFn = os.path.abspath(os.path.dirname(sys.argv[0]))+"/wrapper.sh" |
|
| 489 |
shutil.copyfile(wrapFn,wrap) |
|
| 490 |
os.chmod(wrap,0750) |
|
| 491 |
|
|
| 492 |
qs = "%s/serial_TEMPLATE.qsub" % outDir |
|
| 493 |
if os.path.exists(qs) is False: |
|
| 494 |
print "Warning: %s doesn't exists, creating" % qs |
|
| 495 |
qsFn = os.path.abspath(os.path.dirname(sys.argv[0]))+"/serial_TEMPLATE.qsub" |
|
| 496 |
shutil.copyfile(qsFn,qs) |
|
| 497 |
os.chmod(wrap,0750) |
|
| 498 |
|
|
Also available in: Unified diff
more changes to python stage 4 script to make it work for stage 6