Revision c2c63fac
Added by selv in ga254@bulldogj over 10 years ago
- ID c2c63facd25c412a03c181085ab015200c35ef1c
- Parent 0fe072a4
| terrain/procedures/dem_variables/AEROSOL/sc1_convert_hdf.sh | ||
|---|---|---|
| 1 |
# data downleded from |
|
| 2 |
# wget -r ftp://ladsweb.nascom.nasa.gov/allData/51/MYD08_M3/ ; wget -r ftp://ladsweb.nascom.nasa.gov/allData/51/MOD08_M3/ |
|
| 3 |
|
|
| 4 |
# for file in /lustre0/scratch/ga254/dem_bj/AEROSOL/ladsweb.nascom.nasa.gov/allData/51/M?D08_M3/*/*/*.hdf ; do qsub -v file=$file /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/AEROSOL/sc1_convert_hdf.sh ; done |
|
| 5 |
|
|
| 6 |
# this are the data usefull. Now condisidering only Corrected_Optical_Depth_Land_Mean_Mean |
|
| 7 |
|
|
| 8 |
# SUBDATASET_44_NAME=HDF4_EOS:EOS_GRID:"MOD08_M3.A2002244.051.2010309214319.hdf":mod08:Corrected_Optical_Depth_Land_Mean_Mean |
|
| 9 |
# SUBDATASET_44_DESC=[3x180x360] Corrected_Optical_Depth_Land_Mean_Mean mod08 (16-bit integer) |
|
| 10 |
# SUBDATASET_45_NAME=HDF4_EOS:EOS_GRID:"MOD08_M3.A2002244.051.2010309214319.hdf":mod08:Corrected_Optical_Depth_Land_Mean_Std |
|
| 11 |
# SUBDATASET_45_DESC=[3x180x360] Corrected_Optical_Depth_Land_Mean_Std mod08 (16-bit integer) |
|
| 12 |
# SUBDATASET_46_NAME=HDF4_EOS:EOS_GRID:"MOD08_M3.A2002244.051.2010309214319.hdf":mod08:Corrected_Optical_Depth_Land_Mean_Min |
|
| 13 |
# SUBDATASET_46_DESC=[3x180x360] Corrected_Optical_Depth_Land_Mean_Min mod08 (16-bit integer) |
|
| 14 |
# SUBDATASET_47_NAME=HDF4_EOS:EOS_GRID:"MOD08_M3.A2002244.051.2010309214319.hdf":mod08:Corrected_Optical_Depth_Land_Mean_Max |
|
| 15 |
# SUBDATASET_47_DESC=[3x180x360] Corrected_Optical_Depth_Land_Mean_Max mod08 (16-bit integer) |
|
| 16 |
# SUBDATASET_48_NAME=HDF4_EOS:EOS_GRID:"MOD08_M3.A2002244.051.2010309214319.hdf":mod08:Corrected_Optical_Depth_Land_Std_Deviation_Mean |
|
| 17 |
|
|
| 18 |
# Corrected aerosol optical depth (Land) at 0.47, 0.55, and 0.66 microns: Mean of Daily Mean |
|
| 19 |
# (0.466, 0.553, 0.644 and 2.119 μm, representing MODIS channels 3, 4, 1 and 7, respectively) |
|
| 20 |
|
|
| 21 |
|
|
| 22 |
#PBS -S /bin/bash |
|
| 23 |
#PBS -q fas_normal |
|
| 24 |
#PBS -l mem=1gb |
|
| 25 |
#PBS -l walltime=0:02:00 |
|
| 26 |
#PBS -l nodes=1:ppn=1 |
|
| 27 |
#PBS -V |
|
| 28 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 29 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 30 |
|
|
| 31 |
module load Tools/Python/2.7.3 |
|
| 32 |
module load Libraries/GDAL/1.10.0 |
|
| 33 |
module load Libraries/OSGEO/1.10.0 |
|
| 34 |
|
|
| 35 |
# file=$1 |
|
| 36 |
|
|
| 37 |
INDIR=/lustre0/scratch/ga254/dem_bj/AEROSOL/tif |
|
| 38 |
|
|
| 39 |
filename=`basename $file .hdf` |
|
| 40 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 'HDF4_EOS:EOS_GRID:''"'$file'"'':mod08:Corrected_Optical_Depth_Land_Mean_Mean' $INDIR/$filename.tif |
|
| 41 |
rm -f $INDIR/$filename.tif.aux.xml |
|
| 42 |
|
|
| 43 |
|
|
| 44 |
|
|
| terrain/procedures/dem_variables/AEROSOL/sc2_mean_optical_depth.sh | ||
|---|---|---|
| 1 |
# for day in 00 03 06 09 12 15 18 21 24 27 30 33 ; do qsub -v day=$day /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/AEROSOL/sc2_mean_optical_depth.sh ; done |
|
| 2 |
|
|
| 3 |
# bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/AEROSOL/sc2_mean_optical_depth.sh 00 |
|
| 4 |
|
|
| 5 |
#PBS -S /bin/bash |
|
| 6 |
#PBS -q fas_normal |
|
| 7 |
#PBS -l mem=1gb |
|
| 8 |
#PBS -l walltime=0:10:00 |
|
| 9 |
#PBS -l nodes=1:ppn=1 |
|
| 10 |
#PBS -V |
|
| 11 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 12 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 13 |
|
|
| 14 |
module load Tools/Python/2.7.3 |
|
| 15 |
module load Libraries/GDAL/1.10.0 |
|
| 16 |
module load Libraries/OSGEO/1.10.0 |
|
| 17 |
|
|
| 18 |
export day=$day |
|
| 19 |
|
|
| 20 |
export INDIR=/lustre0/scratch/ga254/dem_bj/AEROSOL/tif |
|
| 21 |
export OUTDIR=/lustre0/scratch/ga254/dem_bj/AEROSOL |
|
| 22 |
|
|
| 23 |
|
|
| 24 |
ls $INDIR/M?D08_M3.A20??${day}?.051.*.tif | xargs -n 1 -P 10 bash -c $'
|
|
| 25 |
file=$1 |
|
| 26 |
export filename=`basename $file .tif` |
|
| 27 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -b 1 $file $OUTDIR/tif_stack/${filename}_b1.tif
|
|
| 28 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -b 2 $file $OUTDIR/tif_stack/${filename}_b2.tif
|
|
| 29 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -b 3 $file $OUTDIR/tif_stack/${filename}_b3.tif
|
|
| 30 |
' _ |
|
| 31 |
|
|
| 32 |
echo start calculate the mean and the median for $OUTDIR/tif_mean/day${dayr}_mean.tif $OUTDIR/tif_mean/day${dayr}_median.tif
|
|
| 33 |
|
|
| 34 |
if [ $day -eq 00 ] ; then export dayr=1 ; fi |
|
| 35 |
if [ $day -eq 03 ] ; then export dayr=32 ; fi |
|
| 36 |
if [ $day -eq 06 ] ; then export dayr=61 ; fi |
|
| 37 |
if [ $day -eq 09 ] ; then export dayr=91 ; fi |
|
| 38 |
if [ $day -eq 12 ] ; then export dayr=122 ; fi |
|
| 39 |
if [ $day -eq 15 ] ; then export dayr=153 ; fi |
|
| 40 |
if [ $day -eq 18 ] ; then export dayr=183 ; fi |
|
| 41 |
if [ $day -eq 21 ] ; then export dayr=214 ; fi |
|
| 42 |
if [ $day -eq 24 ] ; then export dayr=245 ; fi |
|
| 43 |
if [ $day -eq 27 ] ; then export dayr=275 ; fi |
|
| 44 |
if [ $day -eq 30 ] ; then export dayr=306 ; fi |
|
| 45 |
if [ $day -eq 33 ] ; then export dayr=336 ; fi |
|
| 46 |
|
|
| 47 |
|
|
| 48 |
echo mean median | xargs -n 1 -P 2 bash -c $' |
|
| 49 |
par=$1 |
|
| 50 |
pkmosaic -min -1 -cr $par $(for tif in $OUTDIR/tif_stack/M?D08_M3.A20??${day}?.051.*.tif ; do echo -i $tif ; done ) -o $OUTDIR/tif_$par/day${dayr}_$par.tif
|
|
| 51 |
|
|
| 52 |
' _ |
|
| 53 |
rm -f $OUTDIR/tif_stack/M?D08_M3.A20??${day}?.051.*_b[1-3].tif
|
|
| 54 |
|
|
| 55 |
|
|
| 56 |
# cp tif_mean/day001_mean.tif tif_mean/day365_mean.tif # copiato a mano per il day_estimation |
|
| 57 |
|
|
| terrain/procedures/dem_variables/AEROSOL/sc2_mean_optical_depth_055.sh | ||
|---|---|---|
| 1 |
# for day in 00 03 06 09 12 15 18 21 24 27 30 33 ; do qsub -v day=$day /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/AEROSOL/sc2_mean_optical_depth_055.sh ; done |
|
| 2 |
|
|
| 3 |
# bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/AEROSOL/sc2_mean_optical_depth_055.sh 00 |
|
| 4 |
|
|
| 5 |
#PBS -S /bin/bash |
|
| 6 |
#PBS -q fas_normal |
|
| 7 |
#PBS -l mem=1gb |
|
| 8 |
#PBS -l walltime=1:00:00 |
|
| 9 |
#PBS -l nodes=1:ppn=1 |
|
| 10 |
#PBS -V |
|
| 11 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 12 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 13 |
|
|
| 14 |
module load Tools/Python/2.7.3 |
|
| 15 |
module load Libraries/GDAL/1.10.0 |
|
| 16 |
module load Libraries/OSGEO/1.10.0 |
|
| 17 |
|
|
| 18 |
export day=${day}
|
|
| 19 |
|
|
| 20 |
export INDIR=/lustre0/scratch/ga254/dem_bj/AEROSOL/tif |
|
| 21 |
export OUTDIR=/lustre0/scratch/ga254/dem_bj/AEROSOL |
|
| 22 |
|
|
| 23 |
# take out only band 055 |
|
| 24 |
|
|
| 25 |
ls $INDIR/M?D08_M3.A20??${day}?.051.*.tif | xargs -n 1 -P 10 bash -c $'
|
|
| 26 |
file=$1 |
|
| 27 |
export filename=`basename $file .tif` |
|
| 28 |
gdal_translate -ot Float32 -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -b 2 $file $OUTDIR/tif_stack/${filename}_b2.tif
|
|
| 29 |
' _ |
|
| 30 |
|
|
| 31 |
echo start calculate the mean and the median for $OUTDIR/tif_mean/day${dayr}_mean.tif $OUTDIR/tif_mean/day${dayr}_median.tif
|
|
| 32 |
|
|
| 33 |
if [ $day -eq 00 ] ; then export dayr=1 ; fi |
|
| 34 |
if [ $day -eq 03 ] ; then export dayr=32 ; fi |
|
| 35 |
if [ $day -eq 06 ] ; then export dayr=61 ; fi |
|
| 36 |
if [ $day -eq 09 ] ; then export dayr=91 ; fi |
|
| 37 |
if [ $day -eq 12 ] ; then export dayr=122 ; fi |
|
| 38 |
if [ $day -eq 15 ] ; then export dayr=153 ; fi |
|
| 39 |
if [ $day -eq 18 ] ; then export dayr=183 ; fi |
|
| 40 |
if [ $day -eq 21 ] ; then export dayr=214 ; fi |
|
| 41 |
if [ $day -eq 24 ] ; then export dayr=245 ; fi |
|
| 42 |
if [ $day -eq 27 ] ; then export dayr=275 ; fi |
|
| 43 |
if [ $day -eq 30 ] ; then export dayr=306 ; fi |
|
| 44 |
if [ $day -eq 33 ] ; then export dayr=336 ; fi |
|
| 45 |
|
|
| 46 |
|
|
| 47 |
echo mean median | xargs -n 1 -P 2 bash -c $' |
|
| 48 |
par=$1 |
|
| 49 |
|
|
| 50 |
pkmosaic -srcnodata -9999 --dstnodata -9999 -ot Float32 -min -100 -cr $par $(for tif in $OUTDIR/tif_stack/M?D08_M3.A20??${day}?.051.*.tif ; do echo -i $tif ; done ) -o $OUTDIR/tif_$par/day${dayr}_$par.tif
|
|
| 51 |
|
|
| 52 |
pkgetmask -max -100 -data 0 -nodata 1 -i $OUTDIR/tif_$par/day${dayr}_$par.tif -o $OUTDIR/tif_$par/day${dayr}_mask$par.tif
|
|
| 53 |
|
|
| 54 |
pkfillnodata -co COMPRESS=LZW -co ZLEVEL=9 -d 1 -m $OUTDIR/tif_$par/day${dayr}_mask$par.tif -i $OUTDIR/tif_$par/day${dayr}_$par.tif -o $OUTDIR/tif_$par/day${dayr}_fill$par.tif
|
|
| 55 |
|
|
| 56 |
gdalwarp -co COMPRESS=LZW -co ZLEVEL=9 -overwrite -ot Int16 -wt Int16 -srcnodata -9999 -dstnodata -9999 -r bilinear -tr 0.008333333300000 0.008333333300000 $OUTDIR/tif_$par/day${dayr}_fill$par.tif $OUTDIR/tif_$par/day${dayr}_res_$par.tif
|
|
| 57 |
|
|
| 58 |
' _ |
|
| 59 |
|
|
| 60 |
# rm -f $OUTDIR/tif_stack/M?D08_M3.A20??${day}?.051.*_b[1-3].tif
|
|
| 61 |
|
|
| 62 |
|
|
| 63 |
# cp tif_mean/day001_mean.tif tif_mean/day365_mean.tif # copiato a mano per il day_estimation |
|
| 64 |
|
|
| terrain/procedures/dem_variables/AEROSOL/sc3_dayly_optical.sh | ||
|---|---|---|
| 1 |
# calculate the linke days based on linear trend between 2 estimation |
|
| 2 |
# eventualmente integrare nel lista degli if la parte $mont -lt 7 ... vedere sc2_real-sky-horiz-solar_Monthradiation.sh |
|
| 3 |
# change the data type in the input file because gdal_calc in case of byte do not support the operation a-b be negative... |
|
| 4 |
# for file in months_orig/*.tif ; do filename=`basename $file ` ; gdal_translate -ot Int16 -co COMPRESS=LZW -co ZLEVEL=9 $file months/$filename ; done |
|
| 5 |
# seq 0 13 | xargs -n 1 -P 10 bash /mnt/data2/scratch/GMTED2010/scripts/sc1b_dayly_cloud.sh |
|
| 6 |
# for file in `seq 1 365` ;do ls linke$file.tif ; done # for control check |
|
| 7 |
|
|
| 8 |
|
|
| 9 |
# cp |
|
| 10 |
# for day in 1 32 61 91 122 153 183 214 245 275 306 336 ; do qsub -v day=$day /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/AEROSOL/sc3_dayly_optical.sh ; done |
|
| 11 |
|
|
| 12 |
# bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/AEROSOL/sc3_dayly_optical.sh 32 |
|
| 13 |
|
|
| 14 |
#PBS -S /bin/bash |
|
| 15 |
#PBS -q fas_normal |
|
| 16 |
#PBS -l mem=1gb |
|
| 17 |
#PBS -l walltime=0:10:00 |
|
| 18 |
#PBS -l nodes=1:ppn=1 |
|
| 19 |
#PBS -V |
|
| 20 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 21 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 22 |
|
|
| 23 |
module load Tools/Python/2.7.3 |
|
| 24 |
module load Libraries/GDAL/1.10.0 |
|
| 25 |
module load Libraries/OSGEO/1.10.0 |
|
| 26 |
|
|
| 27 |
# export day=$day |
|
| 28 |
export day=$1 |
|
| 29 |
|
|
| 30 |
export INDIR=/lustre0/scratch/ga254/dem_bj/AEROSOL/tif_mean |
|
| 31 |
export OUTDIR=/lustre0/scratch/ga254/dem_bj/AEROSOL/day_estimation |
|
| 32 |
|
|
| 33 |
|
|
| 34 |
if [ $day -eq 1 ] ; then export dayend=32 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 35 |
if [ $day -eq 32 ] ; then export dayend=61 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 36 |
if [ $day -eq 61 ] ; then export dayend=91 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 37 |
if [ $day -eq 91 ] ; then export dayend=122 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 38 |
if [ $day -eq 122 ] ; then export dayend=153 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 39 |
if [ $day -eq 153 ] ; then export dayend=183 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 40 |
if [ $day -eq 183 ] ; then export dayend=214 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 41 |
if [ $day -eq 214 ] ; then export dayend=245 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 42 |
if [ $day -eq 245 ] ; then export dayend=275 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 43 |
if [ $day -eq 275 ] ; then export dayend=306 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 44 |
if [ $day -eq 306 ] ; then export dayend=336 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 45 |
if [ $day -eq 336 ] ; then export dayend=365 ; gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $INDIR/day${day}_mean.tif $OUTDIR/day${day}_mean.tif ; fi
|
|
| 46 |
|
|
| 47 |
|
|
| 48 |
export nseq=$(expr $dayend - $day - 1 ) |
|
| 49 |
export fact=$(awk -v nseq=$nseq 'BEGIN { print 1/(nseq + 1) }' )
|
|
| 50 |
|
|
| 51 |
echo start to process $1 |
|
| 52 |
|
|
| 53 |
for n in `seq 1 $nseq` ; do |
|
| 54 |
|
|
| 55 |
echo processing day $(expr $day + $n) |
|
| 56 |
rm -f $OUTDIR/tmpday$(expr $day + $n)_mean.tif |
|
| 57 |
gdal_calc.py --type=Float32 --NoDataValue=-9999 -A $OUTDIR/day${day}_mean.tif -B $OUTDIR/day${dayend}_mean.tif --calc="( A + ((B-A) * $fact * $n ) )" --outfile=$OUTDIR/tmpday$(expr $day + $n)_mean.tif --co=COMPRESS=LZW --co=ZLEVEL=9 --type Float32 --overwrite
|
|
| 58 |
gdal_translate -a_nodata -9999 -co COMPRESS=LZW -co ZLEVEL=9 -ot Float32 $OUTDIR/tmpday$(expr $day + $n)_mean.tif $OUTDIR/day$(expr $day + $n)_mean.tif |
|
| 59 |
rm -f $OUTDIR/tmpday$(expr $day + $n)_mean.tif |
|
| 60 |
done |
|
| 61 |
|
|
| 62 |
exit |
|
| 63 |
|
|
| 64 |
# quality controll # funziona tutto |
|
| 65 |
|
|
| 66 |
for day in `seq 1 365` ; do echo $day `gdallocationinfo -valonly day${day}_mean.tif 200 100 ` ; done
|
|
| 67 |
|
|
| 68 |
|
|
| terrain/procedures/dem_variables/AEROSOL/sc3_dayly_optical_055.sh | ||
|---|---|---|
| 1 |
# dayli estimation trend |
|
| 2 |
|
|
| 3 |
# copy the input tif from /lustre0/scratch/ga254/dem_bj/AEROSOL/tif_mean |
|
| 4 |
# cd /lustre0/scratch/ga254/dem_bj/AEROSOL/tif_mean |
|
| 5 |
|
|
| 6 |
# ls day*_res_mean.tif | xargs -n 1 -P 12 bash -c $' |
|
| 7 |
# gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata -9999 -ot Float32 $1 /lustre0/scratch/ga254/dem_bj/AEROSOL/day_estimation/$1 |
|
| 8 |
# ' |
|
| 9 |
# cp /lustre0/scratch/ga254/dem_bj/AEROSOL/day_estimation/day1_res_mean.tif /lustre0/scratch/ga254/dem_bj/AEROSOL/day_estimation/day365_res_mean.tif |
|
| 10 |
|
|
| 11 |
# for day in 1 32 61 91 122 153 183 214 245 275 306 336 ; do qsub -v day=$day /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/AEROSOL/sc3_dayly_optical_055.sh ; done |
|
| 12 |
|
|
| 13 |
# bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/AEROSOL/sc3_dayly_optical.sh 32 |
|
| 14 |
|
|
| 15 |
#PBS -S /bin/bash |
|
| 16 |
#PBS -q fas_normal |
|
| 17 |
#PBS -l mem=1gb |
|
| 18 |
#PBS -l walltime=3:00:00 |
|
| 19 |
#PBS -l nodes=1:ppn=1 |
|
| 20 |
#PBS -V |
|
| 21 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 22 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 23 |
|
|
| 24 |
module load Tools/Python/2.7.3 |
|
| 25 |
module load Libraries/GDAL/1.10.0 |
|
| 26 |
module load Libraries/OSGEO/1.10.0 |
|
| 27 |
|
|
| 28 |
export day=$day |
|
| 29 |
# export day=$1 |
|
| 30 |
|
|
| 31 |
export INDIR=/lustre0/scratch/ga254/dem_bj/AEROSOL/tif_mean |
|
| 32 |
export OUTDIR=/lustre0/scratch/ga254/dem_bj/AEROSOL/day_estimation |
|
| 33 |
|
|
| 34 |
|
|
| 35 |
|
|
| 36 |
if [ $day -eq 1 ] ; then export dayend=32 ; fi |
|
| 37 |
if [ $day -eq 32 ] ; then export dayend=61 ; fi |
|
| 38 |
if [ $day -eq 61 ] ; then export dayend=91 ; fi |
|
| 39 |
if [ $day -eq 91 ] ; then export dayend=122 ; fi |
|
| 40 |
if [ $day -eq 122 ] ; then export dayend=153 ;fi |
|
| 41 |
if [ $day -eq 153 ] ; then export dayend=183 ;fi |
|
| 42 |
if [ $day -eq 183 ] ; then export dayend=214 ;fi |
|
| 43 |
if [ $day -eq 214 ] ; then export dayend=245 ;fi |
|
| 44 |
if [ $day -eq 245 ] ; then export dayend=275 ;fi |
|
| 45 |
if [ $day -eq 275 ] ; then export dayend=306 ;fi |
|
| 46 |
if [ $day -eq 306 ] ; then export dayend=336 ;fi |
|
| 47 |
if [ $day -eq 336 ] ; then export dayend=365 ;fi |
|
| 48 |
|
|
| 49 |
|
|
| 50 |
export nseq=$(expr $dayend - $day - 1 ) |
|
| 51 |
export fact=$(awk -v nseq=$nseq 'BEGIN { print 1/(nseq + 1) }' )
|
|
| 52 |
|
|
| 53 |
echo start to process $1 |
|
| 54 |
|
|
| 55 |
for n in `seq 1 $nseq` ; do |
|
| 56 |
|
|
| 57 |
echo processing day $(expr $day + $n) |
|
| 58 |
rm -f $OUTDIR/tmpday$(expr $day + $n)_res_mean.tif |
|
| 59 |
gdal_calc.py --NoDataValue -9999 --type=Float32 -A $OUTDIR/day${day}_res_mean.tif -B $OUTDIR/day${dayend}_res_mean.tif --calc="( A + ((B-A) * $fact * $n ) )" --outfile=$OUTDIR/tmpday$(expr $day + $n)_res_mean.tif --co=COMPRESS=LZW --co=ZLEVEL=9 --type Float32 --overwrite
|
|
| 60 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -ot Float32 $OUTDIR/tmpday$(expr $day + $n)_res_mean.tif $OUTDIR/day$(expr $day + $n)_res_mean.tif |
|
| 61 |
rm -f $OUTDIR/tmpday$(expr $day + $n)_res_mean.tif |
|
| 62 |
done |
|
| 63 |
|
|
| 64 |
exit |
|
| 65 |
|
|
| 66 |
# quality controll # funziona tutto |
|
| 67 |
|
|
| 68 |
for day in `seq 1 365` ; do echo $day `gdallocationinfo -valonly day${day}_res_mean.tif 200 100 ` ; done
|
|
| 69 |
|
|
| 70 |
|
|
| terrain/procedures/dem_variables/GSHHG/#sc1_rasterize_10m.sh# | ||
|---|---|---|
| 1 |
case |
|
| 2 |
# for tile in $( awk '{ print $1 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt ) ; do bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/GSHHG/sc1_rasterize.sh $tile ; done
|
|
| 3 |
|
|
| 4 |
# for tile in $( awk '{ if (NR>1 ) print $1 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt ) ; do qsub -v tile=$tile /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/GSHHG/sc1_rasterize_10m.sh ; sleep 2000 ; done
|
|
| 5 |
|
|
| 6 |
# The geography data come in five resolutions: |
|
| 7 |
|
|
| 8 |
# F full resolution: Original (full) data resolution. |
|
| 9 |
# H high resolution: About 80 % reduction in size and quality. |
|
| 10 |
# I intermediate resolution: Another ~80 % reduction. |
|
| 11 |
# L low resolution: Another ~80 % reduction. |
|
| 12 |
# C crude resolution: Another ~80 % reduction. |
|
| 13 |
|
|
| 14 |
# Unlike the shoreline polygons at all resolutions, the lower resolution rivers are not guaranteed not to cross. |
|
| 15 |
# Shorelines are furthermore organized into 4 hierarchical levels: |
|
| 16 |
|
|
| 17 |
# L1: boundary between land and ocean. |
|
| 18 |
# L2: boundary between lake and land. |
|
| 19 |
# L3: boundary between island-in-lake and lake. |
|
| 20 |
# L4: boundary between pond-in-island and island. |
|
| 21 |
|
|
| 22 |
#PBS -S /bin/bash |
|
| 23 |
#PBS -q fas_normal |
|
| 24 |
#PBS -l mem=1gb |
|
| 25 |
#PBS -l walltime=4:00:00 |
|
| 26 |
#PBS -l nodes=1:ppn=1 |
|
| 27 |
#PBS -V |
|
| 28 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 29 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 30 |
|
|
| 31 |
|
|
| 32 |
module load Tools/Python/2.7.3 |
|
| 33 |
module load Libraries/GDAL/1.10.0 |
|
| 34 |
module load Libraries/OSGEO/1.10.0 |
|
| 35 |
|
|
| 36 |
|
|
| 37 |
# tile=$1 |
|
| 38 |
|
|
| 39 |
INPUT=/lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_shp/f/GSHHS_f_L1.shp |
|
| 40 |
SHPOUT=/lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_shp_clip |
|
| 41 |
TIFOUT=/lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_tif |
|
| 42 |
|
|
| 43 |
xmin=$( awk -v tile=$tile '{ if($1==tile) print $4 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 44 |
ymin=$( awk -v tile=$tile '{ if($1==tile) print $9 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 45 |
xmax=$( awk -v tile=$tile '{ if($1==tile) print $10 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 46 |
ymax=$( awk -v tile=$tile '{ if($1==tile) print $5 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 47 |
|
|
| 48 |
rm -f $SHPOUT/$tile.* |
|
| 49 |
echo clip the shp by $tile |
|
| 50 |
ogr2ogr -spat $xmin $ymin $xmax $ymax -skipfailures $SHPOUT/$tile.shp $INPUT |
|
| 51 |
|
|
| 52 |
echo rasterize |
|
| 53 |
rm -f $TIFOUT/$tile.tif |
|
| 54 |
# rasterize 10 meter |
|
| 55 |
gdal_rasterize -tr 0.000083333333333 0.000083333333333 -burn 1 -te $xmin $ymin $xmax $ymax -co COMPRESS=LZW -co ZLEVEL=9 -ot Byte -l $tile $SHPOUT/$tile.shp $TIFOUT/$tile.tif |
|
| 56 |
pkfilter -co COMPRESS=LZW -ot Float32 -class 1 -dx 100 -dy 100 -f density -d 100 -i $TIFOUT/$tile.tif -o ${TIFOUT}_1km/${tile}_1km.tif
|
|
| 57 |
|
|
| 58 |
oft-calc -ot UInt16 ${TIFOUT}_1km/${tile}_1km.tif ${TIFOUT}_1km/${tile}_1kmPerc.tif <<EOF
|
|
| 59 |
1 |
|
| 60 |
#1 100 * |
|
| 61 |
EOF |
|
| 62 |
|
|
| 63 |
|
|
| 64 |
# rm -f ${TIFOUT}_1km/${tile}_1kmPerc.tif ${TIFOUT}_1km/${tile}_1km.tif $TIFOUT/$tile.tif $SHPOUT/$tile.*
|
|
| 65 |
|
|
| 66 |
|
|
| 67 |
|
|
| 68 |
exit |
|
| terrain/procedures/dem_variables/GSHHG/sc1_rasterize.sh | ||
|---|---|---|
| 1 |
# for tile in $( awk '{ print $1 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt ) ; do bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/GSHHG/sc1_rasterize.sh $tile ; done
|
|
| 2 |
|
|
| 3 |
# for tile in $( awk '{ print $1 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt ) ; do qsub -v tile=$tile /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/GSHHG/sc1_rasterize.sh ; done
|
|
| 4 |
|
|
| 5 |
# The geography data come in five resolutions: |
|
| 6 |
|
|
| 7 |
# F full resolution: Original (full) data resolution. |
|
| 8 |
# H high resolution: About 80 % reduction in size and quality. |
|
| 9 |
# I intermediate resolution: Another ~80 % reduction. |
|
| 10 |
# L low resolution: Another ~80 % reduction. |
|
| 11 |
# C crude resolution: Another ~80 % reduction. |
|
| 12 |
|
|
| 13 |
# Unlike the shoreline polygons at all resolutions, the lower resolution rivers are not guaranteed not to cross. |
|
| 14 |
# Shorelines are furthermore organized into 4 hierarchical levels: |
|
| 15 |
|
|
| 16 |
# L1: boundary between land and ocean. |
|
| 17 |
# L2: boundary between lake and land. |
|
| 18 |
# L3: boundary between island-in-lake and lake. |
|
| 19 |
# L4: boundary between pond-in-island and island. |
|
| 20 |
|
|
| 21 |
#PBS -S /bin/bash |
|
| 22 |
#PBS -q fas_normal |
|
| 23 |
#PBS -l mem=1gb |
|
| 24 |
#PBS -l walltime=1:00:00 |
|
| 25 |
#PBS -l nodes=1:ppn=1 |
|
| 26 |
#PBS -V |
|
| 27 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 28 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 29 |
|
|
| 30 |
|
|
| 31 |
module load Tools/Python/2.7.3 |
|
| 32 |
module load Libraries/GDAL/1.10.0 |
|
| 33 |
module load Libraries/OSGEO/1.10.0 |
|
| 34 |
|
|
| 35 |
|
|
| 36 |
# tile=$1 |
|
| 37 |
|
|
| 38 |
INPUT=/lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_shp/f/GSHHS_f_L1.shp |
|
| 39 |
SHPOUT=/lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_shp_clip |
|
| 40 |
TIFOUT=/lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_tif |
|
| 41 |
|
|
| 42 |
|
|
| 43 |
xmin=$( awk -v tile=$tile '{ if($1==tile) print $4 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 44 |
ymin=$( awk -v tile=$tile '{ if($1==tile) print $9 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 45 |
xmax=$( awk -v tile=$tile '{ if($1==tile) print $10 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 46 |
ymax=$( awk -v tile=$tile '{ if($1==tile) print $5 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 47 |
|
|
| 48 |
rm -f $SHPOUT/$tile.* |
|
| 49 |
echo clip the shp by $tile |
|
| 50 |
ogr2ogr -spat $xmin $ymin $xmax $ymax -skipfailures $SHPOUT/$tile.shp $INPUT |
|
| 51 |
|
|
| 52 |
echo rasterize |
|
| 53 |
rm -f $TIFOUT/$tile.tif |
|
| 54 |
gdal_rasterize -tr 0.00083333333333 0.00083333333333 -burn 1 -te $xmin $ymin $xmax $ymax -co COMPRESS=LZW -co ZLEVEL=9 -ot Byte -l $tile $SHPOUT/$tile.shp $TIFOUT/$tile.tif |
|
| 55 |
|
|
| 56 |
pkfilter -co COMPRESS=LZW -ot Byte -class 1 -dx 10 -dy 10 -f density -d 10 -i $TIFOUT/$tile.tif -o ${TIFOUT}_1km/${tile}_1km.tif
|
|
| 57 |
|
|
| 58 |
exit |
|
| 59 |
|
|
| 60 |
# in case of better resolution |
|
| 61 |
|
|
| 62 |
# gdal_rasterize -tr 0.000083333333333 0.000083333333333 -burn 1 -te $xmin $ymin $xmax $ymax -co COMPRESS=LZW -co ZLEVEL=9 -ot Byte -l $tile $SHPOUT/$tile.shp $TIFOUT/$tile.tif |
|
| 63 |
# pkfilter -co COMPRESS=LZW -ot Float32 -class 1 -dx 100 -dy 100 -f density -d 10 -i $TIFOUT/$tile.tif -o ${TIFOUT}_1km/${tile}_1km.tif
|
|
| 64 |
|
|
| 65 |
# oft-calc -ot Float32 ${TIFOUT}_1km/${tile}_1km.tif ${TIFOUT}_1km/${tile}_1kmPerc.tif <<EOF
|
|
| 66 |
# 1 |
|
| 67 |
# #1 100 * |
|
| 68 |
# EOF |
|
| 69 |
|
|
| 70 |
# gdal_calc.py -A ${TIFOUT}_1km/${tile}_1km.tif --calc="(A * 100 )" --co=COMPRESS=LZW --co=ZLEVEL=9 --overwrite --outfile ${TIFOUT}_1km/${tile}_1kmPerc.tif
|
|
| 71 |
# gdal_translate -ot Byte -co COMPRESS=LZW -co ZLEVEL=9 ${TIFOUT}_1km/${tile}_1kmPerc.tif ${TIFOUT}_1km/${tile}_1kmPercC.tif
|
|
| 72 |
|
|
| 73 |
# rm -f ${TIFOUT}_1km/${tile}_1kmPerc.tif ${TIFOUT}_1km/${tile}_1km.tif $TIFOUT/$tile.tif $SHPOUT/$tile.*
|
|
| 74 |
|
|
| 75 |
|
|
| terrain/procedures/dem_variables/GSHHG/sc1_rasterize_10m.sh | ||
|---|---|---|
| 1 |
# for tile in $( awk '{ print $1 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt ) ; do bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/GSHHG/sc1_rasterize.sh $tile ; done
|
|
| 2 |
|
|
| 3 |
# for tile in $( awk '{ if (NR>1 ) print $1 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt ) ; do qsub -v tile=$tile /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/GSHHG/sc1_rasterize_10m.sh ; sleep 2000 ; done
|
|
| 4 |
|
|
| 5 |
# The geography data come in five resolutions: |
|
| 6 |
|
|
| 7 |
# F full resolution: Original (full) data resolution. |
|
| 8 |
# H high resolution: About 80 % reduction in size and quality. |
|
| 9 |
# I intermediate resolution: Another ~80 % reduction. |
|
| 10 |
# L low resolution: Another ~80 % reduction. |
|
| 11 |
# C crude resolution: Another ~80 % reduction. |
|
| 12 |
|
|
| 13 |
# Unlike the shoreline polygons at all resolutions, the lower resolution rivers are not guaranteed not to cross. |
|
| 14 |
# Shorelines are furthermore organized into 4 hierarchical levels: |
|
| 15 |
|
|
| 16 |
# L1: boundary between land and ocean. |
|
| 17 |
# L2: boundary between lake and land. |
|
| 18 |
# L3: boundary between island-in-lake and lake. |
|
| 19 |
# L4: boundary between pond-in-island and island. |
|
| 20 |
|
|
| 21 |
#PBS -S /bin/bash |
|
| 22 |
#PBS -q fas_normal |
|
| 23 |
#PBS -l mem=1gb |
|
| 24 |
#PBS -l walltime=4:00:00 |
|
| 25 |
#PBS -l nodes=1:ppn=1 |
|
| 26 |
#PBS -V |
|
| 27 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 28 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 29 |
|
|
| 30 |
|
|
| 31 |
module load Tools/Python/2.7.3 |
|
| 32 |
module load Libraries/GDAL/1.10.0 |
|
| 33 |
module load Libraries/OSGEO/1.10.0 |
|
| 34 |
|
|
| 35 |
|
|
| 36 |
# tile=$1 |
|
| 37 |
|
|
| 38 |
INPUT=/lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_shp/f/GSHHS_f_L1.shp |
|
| 39 |
SHPOUT=/lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_shp_clip |
|
| 40 |
TIFOUT=/lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_tif |
|
| 41 |
|
|
| 42 |
xmin=$( awk -v tile=$tile '{ if($1==tile) print $4 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 43 |
ymin=$( awk -v tile=$tile '{ if($1==tile) print $9 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 44 |
xmax=$( awk -v tile=$tile '{ if($1==tile) print $10 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 45 |
ymax=$( awk -v tile=$tile '{ if($1==tile) print $5 }' /lustre0/scratch/ga254/dem_bj/GSHHG/geo_file/tile_lat_long_10d.txt)
|
|
| 46 |
|
|
| 47 |
rm -f $SHPOUT/$tile.* |
|
| 48 |
echo clip the shp by $tile |
|
| 49 |
ogr2ogr -spat $xmin $ymin $xmax $ymax -skipfailures $SHPOUT/$tile.shp $INPUT |
|
| 50 |
|
|
| 51 |
echo rasterize |
|
| 52 |
rm -f $TIFOUT/$tile.tif |
|
| 53 |
# rasterize 10 meter |
|
| 54 |
gdal_rasterize -tr 0.000083333333333 0.000083333333333 -burn 1 -te $xmin $ymin $xmax $ymax -co COMPRESS=LZW -co ZLEVEL=9 -ot Byte -l $tile $SHPOUT/$tile.shp $TIFOUT/$tile.tif |
|
| 55 |
pkfilter -co COMPRESS=LZW -ot Float32 -class 1 -dx 100 -dy 100 -f density -d 100 -i $TIFOUT/$tile.tif -o ${TIFOUT}_1km/${tile}_1km.tif
|
|
| 56 |
|
|
| 57 |
oft-calc -ot UInt16 ${TIFOUT}_1km/${tile}_1km.tif ${TIFOUT}_1km/${tile}_1kmPerc.tif <<EOF
|
|
| 58 |
1 |
|
| 59 |
#1 100 * |
|
| 60 |
EOF |
|
| 61 |
|
|
| 62 |
|
|
| 63 |
# rm -f ${TIFOUT}_1km/${tile}_1kmPerc.tif ${TIFOUT}_1km/${tile}_1km.tif $TIFOUT/$tile.tif $SHPOUT/$tile.*
|
|
| 64 |
|
|
| 65 |
|
|
| 66 |
|
|
| 67 |
exit |
|
| terrain/procedures/dem_variables/GSHHG/sc2_merge.sh | ||
|---|---|---|
| 1 |
#PBS -S /bin/bash |
|
| 2 |
#PBS -q fas_normal |
|
| 3 |
#PBS -l mem=1gb |
|
| 4 |
#PBS -l walltime=1:00:00 |
|
| 5 |
#PBS -l nodes=1:ppn=1 |
|
| 6 |
#PBS -V |
|
| 7 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 8 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 9 |
|
|
| 10 |
|
|
| 11 |
module load Tools/Python/2.7.3 |
|
| 12 |
module load Libraries/GDAL/1.10.0 |
|
| 13 |
module load Libraries/OSGEO/1.10.0 |
|
| 14 |
|
|
| 15 |
|
|
| 16 |
TIFIN=/lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_tif_1km |
|
| 17 |
|
|
| 18 |
|
|
| 19 |
gdalbuildvrt -overwrite -tr 0.0083333333333 0.0083333333333 $TIFIN/land_perc.vrt $TIFIN/h??v??_1km.tif |
|
| 20 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -ot Byte $TIFIN/land_perc.vrt $TIFIN/land_frequency_GSHHS_f_L1.tif |
|
| 21 |
|
|
| 22 |
rm /lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_shp_clip/* $TIFIN/land_perc.vrt $TIFIN/h??v??_1km.tif /lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_tif/*.tif |
|
| 23 |
|
|
| terrain/procedures/dem_variables/GSHHG/sc2_merge_10m.sh | ||
|---|---|---|
| 1 |
# qsub /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/GSHHG/sc2_merge_10m.sh |
|
| 2 |
|
|
| 3 |
#PBS -S /bin/bash |
|
| 4 |
#PBS -q fas_normal |
|
| 5 |
#PBS -l mem=1gb |
|
| 6 |
#PBS -l walltime=10:00:00 |
|
| 7 |
#PBS -l nodes=1:ppn=2 |
|
| 8 |
#PBS -V |
|
| 9 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 10 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 11 |
|
|
| 12 |
|
|
| 13 |
module load Tools/Python/2.7.3 |
|
| 14 |
module load Libraries/GDAL/1.10.0 |
|
| 15 |
module load Libraries/OSGEO/1.10.0 |
|
| 16 |
|
|
| 17 |
|
|
| 18 |
TIFIN=/lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_tif_1km |
|
| 19 |
|
|
| 20 |
|
|
| 21 |
gdalbuildvrt -overwrite -tr 0.0083333333333 0.0083333333333 $TIFIN/land_perc.vrt $TIFIN/h??v??_1km.tif |
|
| 22 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -ot Float32 $TIFIN/land_perc.vrt $TIFIN/land_frequency_m10fltGSHHS_f_L1.tif |
|
| 23 |
|
|
| 24 |
gdalbuildvrt -overwrite -tr 0.0083333333333 0.0083333333333 $TIFIN/land_perc.vrt $TIFIN/h??v??_1kmPerc.tif |
|
| 25 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -ot Float32 $TIFIN/land_perc.vrt $TIFIN/land_frequency_m10intGSHHS_f_L1.tif |
|
| 26 |
|
|
| 27 |
|
|
| 28 |
# rm /lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_shp_clip/* $TIFIN/land_perc.vrt $TIFIN/h??v??_1km.tif /lustre0/scratch/ga254/dem_bj/GSHHG/GSHHS_tif/*.tif |
|
| 29 |
|
|
| terrain/procedures/dem_variables/WDPA/old/#sc3_extrac_area_all.sh# | ||
|---|---|---|
| 1 |
|
|
| 2 |
INDIR=/lustre0/scratch/ga254/dem_bj/WDPA/rasterize_all |
|
| 3 |
# pkmosaic -i $INDIR/WDPA_point_Jan2014EPSG4326Buf.tif -i $INDIR/WDPA_point_Jan2014.tif -i $INDIR/WDPA_point_Jan2014.tif -cr max -o $INDIR/WDPA_all.tif |
|
| 4 |
|
|
| 5 |
|
|
| 6 |
# extract the area for each cell |
|
| 7 |
|
|
| 8 |
rm -f /lustre0/scratch/ga254/dem_bj/WDPA/shp_out/360x114global_area_all.* |
|
| 9 |
|
|
| 10 |
pkextract -m $INDIR/WDPA_all.tif -msknodata 0 -i /lustre0/scratch/ga254/dem_bj/GEO_AREA/area_tif/30arc-sec-Area_prj6974.tif \ |
|
| 11 |
-l -r sum -bn AREA_all -lt String -s /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114global/360x114global.shp \ |
|
| 12 |
-o /lustre0/scratch/ga254/dem_bj/WDPA/shp_out/360x114global_area_all.shp |
|
| 13 |
|
|
| 14 |
|
|
| 15 |
|
|
| 16 |
|
|
| 17 |
|
|
| 18 |
|
|
| terrain/procedures/dem_variables/WDPA/old/sc1_rasterize_point.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
|
|
| 3 |
# echo 1 2 3 4 5 6 7 8 9 | xargs -n 1 -P 10 bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/WDPA/sc1_rasterize_point.sh |
|
| 4 |
|
|
| 5 |
|
|
| 6 |
#PBS -S /bin/bash |
|
| 7 |
#PBS -q fas_normal |
|
| 8 |
#PBS -l mem=10gb |
|
| 9 |
#PBS -l walltime=4:00:00 |
|
| 10 |
#PBS -l nodes=1:ppn=4 |
|
| 11 |
#PBS -V |
|
| 12 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 13 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 14 |
|
|
| 15 |
|
|
| 16 |
# load moduels |
|
| 17 |
|
|
| 18 |
module load Tools/Python/2.7.3 |
|
| 19 |
module load Libraries/GDAL/1.10.0 |
|
| 20 |
module load Tools/PKTOOLS/2.4.2 |
|
| 21 |
module load Libraries/OSGEO/1.10.0 |
|
| 22 |
|
|
| 23 |
|
|
| 24 |
|
|
| 25 |
export RASTERIZE=/lustre0/scratch/ga254/dem_bj/WDPA/rasterize_buf |
|
| 26 |
export SHP=/lustre0/scratch/ga254/dem_bj/WDPA/shp_buf |
|
| 27 |
n=$1 |
|
| 28 |
|
|
| 29 |
# attribute of the shapefile land NO-MARINE |
|
| 30 |
|
|
| 31 |
# IUCN_CAT (String) = Ia 11 |
|
| 32 |
# IUCN_CAT (String) = Ib 12 |
|
| 33 |
# IUCN_CAT (String) = II 2 |
|
| 34 |
# IUCN_CAT (String) = III 3 |
|
| 35 |
# IUCN_CAT (String) = IV 4 |
|
| 36 |
# IUCN_CAT (String) = Not Applicable 20 |
|
| 37 |
# IUCN_CAT (String) = Not Reported 21 |
|
| 38 |
# IUCN_CAT (String) = V 5 |
|
| 39 |
# IUCN_CAT (String) = VI 6 |
|
| 40 |
|
|
| 41 |
|
|
| 42 |
# attribute of the shapefile MARINE |
|
| 43 |
|
|
| 44 |
# IUCN_CAT (String) = Ia 111 |
|
| 45 |
# IUCN_CAT (String) = Ib 112 |
|
| 46 |
# IUCN_CAT (String) = II 102 |
|
| 47 |
# IUCN_CAT (String) = III 103 |
|
| 48 |
# IUCN_CAT (String) = IV 104 |
|
| 49 |
# IUCN_CAT (String) = Not Applicable 120 |
|
| 50 |
# IUCN_CAT (String) = Not Reported 121 |
|
| 51 |
# IUCN_CAT (String) = V 105 |
|
| 52 |
# IUCN_CAT (String) = VI 106 |
|
| 53 |
|
|
| 54 |
|
|
| 55 |
|
|
| 56 |
echo rasterize the land |
|
| 57 |
|
|
| 58 |
if [ $n -eq 1 ] ; then IUCN_CAT='"Ia"' ; BURN=11 ; fi |
|
| 59 |
if [ $n -eq 2 ] ; then IUCN_CAT='"Ib"' ; BURN=12 ; fi |
|
| 60 |
if [ $n -eq 3 ] ; then IUCN_CAT='"II"' ; BURN=2 ; fi |
|
| 61 |
if [ $n -eq 4 ] ; then IUCN_CAT='"III"'; BURN=3 ; fi |
|
| 62 |
if [ $n -eq 5 ] ; then IUCN_CAT='"IV"' ; BURN=4 ; fi |
|
| 63 |
if [ $n -eq 6 ] ; then IUCN_CAT='"Not Applicable"' ; BURN=20 ; fi |
|
| 64 |
if [ $n -eq 7 ] ; then IUCN_CAT='"Not Reported"' ; BURN=21 ; fi |
|
| 65 |
if [ $n -eq 8 ] ; then IUCN_CAT='"V"' ; BURN=5 ; fi |
|
| 66 |
if [ $n -eq 9 ] ; then IUCN_CAT='"VI"' ; BURN=6 ; fi |
|
| 67 |
|
|
| 68 |
|
|
| 69 |
rm -f $SHP/WDPA_point_Jan2014EPSG4326BufIUCN$BURN.shp |
|
| 70 |
|
|
| 71 |
ogr2ogr -sql "SELECT * FROM WDPA_point_Jan2014EPSG4326Buf WHERE ( IUCN_CAT = ${IUCN_CAT} ) AND ( MARINE = '0') " $SHP/WDPA_point_Jan2014EPSG4326BufIUCN$BURN.shp $SHP/WDPA_point_Jan2014EPSG4326Buf.shp
|
|
| 72 |
|
|
| 73 |
|
|
| 74 |
|
|
| 75 |
rm -f $RASTERIZE/WDPA_point_Jan2014EPSG4326BufIUCN$BURN.tif |
|
| 76 |
|
|
| 77 |
gdal_rasterize -ot Byte -a_srs EPSG:4326 -l WDPA_point_Jan2014EPSG4326BufIUCN$BURN -a IUCN_CAT \ |
|
| 78 |
-burn $BURN -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te $(getCorners4Gwarp /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/6/IUCN6.tif ) -co COMPRESS=LZW \ |
|
| 79 |
-co ZLEVEL=9 $SHP/WDPA_point_Jan2014EPSG4326BufIUCN$BURN.shp $RASTERIZE/WDPA_point_Jan2014EPSG4326BufIUCN$BURN.tif |
|
| 80 |
|
|
| 81 |
|
|
| 82 |
echo rasterize the marine |
|
| 83 |
|
|
| 84 |
if [ $n -eq 1 ] ; then IUCN_CAT='"Ia"' ; BURN=111 ; fi |
|
| 85 |
if [ $n -eq 2 ] ; then IUCN_CAT='"Ib"' ; BURN=112 ; fi |
|
| 86 |
if [ $n -eq 3 ] ; then IUCN_CAT='"II"' ; BURN=102 ; fi |
|
| 87 |
if [ $n -eq 4 ] ; then IUCN_CAT='"III"'; BURN=103 ; fi |
|
| 88 |
if [ $n -eq 5 ] ; then IUCN_CAT='"IV"' ; BURN=104 ; fi |
|
| 89 |
if [ $n -eq 6 ] ; then IUCN_CAT='"Not Applicable"' ; BURN=120 ; fi |
|
| 90 |
if [ $n -eq 7 ] ; then IUCN_CAT='"Not Reported"' ; BURN=121 ; fi |
|
| 91 |
if [ $n -eq 8 ] ; then IUCN_CAT='"V"' ; BURN=105 ; fi |
|
| 92 |
if [ $n -eq 9 ] ; then IUCN_CAT='"VI"' ; BURN=106 ; fi |
|
| 93 |
|
|
| 94 |
|
|
| 95 |
rm -f $SHP/WDPA_point_Jan2014EPSG4326BufIUCN$BURN.shp |
|
| 96 |
|
|
| 97 |
ogr2ogr -sql "SELECT * FROM WDPA_point_Jan2014EPSG4326Buf WHERE ( IUCN_CAT = ${IUCN_CAT} ) AND ( MARINE = '1') " $SHP/WDPA_point_Jan2014EPSG4326BufIUCN$BURN.shp $SHP/WDPA_point_Jan2014EPSG4326Buf.shp
|
|
| 98 |
|
|
| 99 |
rm -f $RASTERIZE/WDPA_point_Jan2014EPSG4326BufIUCN$BURN.tif |
|
| 100 |
gdal_rasterize -ot Byte -a_srs EPSG:4326 -l WDPA_point_Jan2014EPSG4326BufIUCN$BURN -a IUCN_CAT \ |
|
| 101 |
-burn $BURN -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te $(getCorners4Gwarp /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/6/IUCN6.tif ) -co COMPRESS=LZW \ |
|
| 102 |
-co ZLEVEL=9 $SHP/WDPA_point_Jan2014EPSG4326BufIUCN$BURN.shp $RASTERIZE/WDPA_point_Jan2014EPSG4326BufIUCN$BURN.tif |
|
| 103 |
|
|
| 104 |
|
|
| 105 |
|
|
| 106 |
|
|
| 107 |
|
|
| 108 |
|
|
| 109 |
|
|
| 110 |
|
|
| terrain/procedures/dem_variables/WDPA/old/sc1_rasterize_point_buf_area.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
module load Tools/Python/2.7.3 |
|
| 3 |
module load Libraries/GDAL/1.10.0 |
|
| 4 |
module load Tools/PKTOOLS/2.4.2 |
|
| 5 |
module load Libraries/OSGEO/1.10.0 |
|
| 6 |
|
|
| 7 |
|
|
| 8 |
# rasterize buffer |
|
| 9 |
|
|
| 10 |
export RASTERIZE=/lustre0/scratch/ga254/dem_bj/WDPA/rasterize_all |
|
| 11 |
export SHP=/lustre0/scratch/ga254/dem_bj/WDPA |
|
| 12 |
|
|
| 13 |
gdal_rasterize -ot Byte -a_srs EPSG:4326 -l WDPA_point_Jan2014EPSG4326Buf \ |
|
| 14 |
-burn 1 -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te $(getCorners4Gwarp /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/6/IUCN6.tif ) -co COMPRESS=LZW \ |
|
| 15 |
-co ZLEVEL=9 $SHP/shp_buf/WDPA_point_Jan2014EPSG4326Buf.shp $RASTERIZE/WDPA_point_Jan2014EPSG4326Buf.tif |
|
| 16 |
|
|
| 17 |
# rasterize point |
|
| 18 |
rm -f $RASTERIZE/WDPA_point_Jan2014.tif |
|
| 19 |
|
|
| 20 |
gdal_rasterize -ot Byte -a_srs EPSG:4326 -l WDPA_point_Jan2014 -burn 1 -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te $(getCorners4Gwarp /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/6/IUCN6.tif ) -co COMPRESS=LZW \ |
|
| 21 |
-co ZLEVEL=9 $SHP/shp_input/WDPA_Jan2014.shp/WDPA_point_Jan2014.shp $RASTERIZE/WDPA_point_Jan2014.tif |
|
| 22 |
|
|
| 23 |
|
|
| 24 |
# rasterize areas |
|
| 25 |
|
|
| 26 |
gdal_rasterize -ot Byte -a_srs EPSG:4326 -l -burn 1 -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te $(getCorners4Gwarp /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/6/IUCN6.tif ) -co COMPRESS=LZW \ |
|
| 27 |
-co ZLEVEL=9 $SHP/shp_input/WDPA_Jan2014.shp/WDPA_poly_Jan2014.shp $RASTERIZE/WDPA_poly_Jan2014.tif |
|
| 28 |
|
|
| 29 |
|
|
| 30 |
# merge all of them using gdal_mosaic |
|
| 31 |
|
|
| 32 |
|
|
| terrain/procedures/dem_variables/WDPA/old/sc1_rasterize_tile.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
# awk '{ if ( NR > 1 ) print $1 }' /lustre0/scratch/ga254/dem_bj/GMTED2010/geo_file/tiles-te_noOverlap.txt | xargs -n 1 -P 10 bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/WDPA/area_intersection.sh
|
|
| 3 |
|
|
| 4 |
# for tile in $(awk '{ if ( NR > 1 ) print $1 }' /lustre0/scratch/ga254/dem_bj/GMTED2010/geo_file/tiles-te_noOverlap.txt) ; do qsub -v tile=$tile /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/WDPA/sc1_rasterize_tile.sh ; done
|
|
| 5 |
|
|
| 6 |
|
|
| 7 |
# after compile gdal with the following set |
|
| 8 |
# ./configure --with-fgdb=/usr/local/FileGDB_API --with-geos=yes |
|
| 9 |
|
|
| 10 |
# DIR=/mnt/data2/scratch/WDPA |
|
| 11 |
# apps/ogr2ogr /mnt/data2/scratch/WDPA/shp_input/WDPA_Jan2014.shp /mnt/data2/scratch/WDPA/shp_input/WDPA_Jan2014_Public/WDPA_Jan2014.gdb |
|
| 12 |
|
|
| 13 |
|
|
| 14 |
|
|
| 15 |
#PBS -S /bin/bash |
|
| 16 |
#PBS -q fas_normal |
|
| 17 |
#PBS -l mem=10gb |
|
| 18 |
#PBS -l walltime=4:00:00 |
|
| 19 |
#PBS -l nodes=1:ppn=4 |
|
| 20 |
#PBS -V |
|
| 21 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 22 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 23 |
|
|
| 24 |
|
|
| 25 |
# load moduels |
|
| 26 |
|
|
| 27 |
module load Tools/Python/2.7.3 |
|
| 28 |
module load Libraries/GDAL/1.10.0 |
|
| 29 |
module load Tools/PKTOOLS/2.4.2 |
|
| 30 |
module load Libraries/OSGEO/1.10.0 |
|
| 31 |
|
|
| 32 |
|
|
| 33 |
|
|
| 34 |
# export tile=$1 |
|
| 35 |
export tile=$tile |
|
| 36 |
|
|
| 37 |
export RASTERIZE=/lustre0/scratch/ga254/dem_bj/WDPA/rasterize |
|
| 38 |
export SHP=/lustre0/scratch/ga254/dem_bj/WDPA/shp_input/WDPA_Jan2014.shp |
|
| 39 |
|
|
| 40 |
# attribute of the shapefile land NO-MARINE |
|
| 41 |
|
|
| 42 |
# IUCN_CAT (String) = Ia 11 |
|
| 43 |
# IUCN_CAT (String) = Ib 12 |
|
| 44 |
# IUCN_CAT (String) = II 2 |
|
| 45 |
# IUCN_CAT (String) = III 3 |
|
| 46 |
# IUCN_CAT (String) = IV 4 |
|
| 47 |
# IUCN_CAT (String) = Not Applicable 20 |
|
| 48 |
# IUCN_CAT (String) = Not Reported 21 |
|
| 49 |
# IUCN_CAT (String) = V 5 |
|
| 50 |
# IUCN_CAT (String) = VI 6 |
|
| 51 |
|
|
| 52 |
|
|
| 53 |
# attribute of the shapefile MARINE |
|
| 54 |
|
|
| 55 |
# IUCN_CAT (String) = Ia 111 |
|
| 56 |
# IUCN_CAT (String) = Ib 112 |
|
| 57 |
# IUCN_CAT (String) = II 102 |
|
| 58 |
# IUCN_CAT (String) = III 103 |
|
| 59 |
# IUCN_CAT (String) = IV 104 |
|
| 60 |
# IUCN_CAT (String) = Not Applicable 120 |
|
| 61 |
# IUCN_CAT (String) = Not Reported 121 |
|
| 62 |
# IUCN_CAT (String) = V 105 |
|
| 63 |
# IUCN_CAT (String) = VI 106 |
|
| 64 |
|
|
| 65 |
|
|
| 66 |
export geo_string=$( grep $tile /lustre0/scratch/ga254/dem_bj/GMTED2010/geo_file/tiles-te_noOverlap.txt | awk '{ print $2,$3,$4,$5 }' )
|
|
| 67 |
|
|
| 68 |
echo clip the large shp |
|
| 69 |
|
|
| 70 |
rm -f $SHP/shp_clip/WDPA_poly_Jan2014_$tile.* |
|
| 71 |
|
|
| 72 |
ogr2ogr -spat $geo_string $SHP/shp_clip/WDPA_poly_Jan2014_$tile.shp $SHP/WDPA_poly_Jan2014.shp |
|
| 73 |
|
|
| 74 |
|
|
| 75 |
|
|
| 76 |
echo start the resterize |
|
| 77 |
|
|
| 78 |
for n in 1 2 3 4 5 6 7 8 9 ; do |
|
| 79 |
# for n in 3 ; do |
|
| 80 |
|
|
| 81 |
echo rasterize the land |
|
| 82 |
|
|
| 83 |
if [ $n -eq 1 ] ; then IUCN_CAT='"Ia"' ; BURN=11 ; fi |
|
| 84 |
if [ $n -eq 2 ] ; then IUCN_CAT='"Ib"' ; BURN=12 ; fi |
|
| 85 |
if [ $n -eq 3 ] ; then IUCN_CAT='"II"' ; BURN=2 ; fi |
|
| 86 |
if [ $n -eq 4 ] ; then IUCN_CAT='"III"'; BURN=3 ; fi |
|
| 87 |
if [ $n -eq 5 ] ; then IUCN_CAT='"IV"' ; BURN=4 ; fi |
|
| 88 |
if [ $n -eq 6 ] ; then IUCN_CAT='"Not Applicable"' ; BURN=20 ; fi |
|
| 89 |
if [ $n -eq 7 ] ; then IUCN_CAT='"Not Reported"' ; BURN=21 ; fi |
|
| 90 |
if [ $n -eq 8 ] ; then IUCN_CAT='"V"' ; BURN=5 ; fi |
|
| 91 |
if [ $n -eq 9 ] ; then IUCN_CAT='"VI"' ; BURN=6 ; fi |
|
| 92 |
|
|
| 93 |
|
|
| 94 |
rm -f $SHP/shp_clip/WDPA_poly_Jan2014_${tile}L${BURN}.*
|
|
| 95 |
|
|
| 96 |
ogr2ogr -sql "SELECT * FROM WDPA_poly_Jan2014_${tile} WHERE ( IUCN_CAT = ${IUCN_CAT} ) AND ( MARINE = '0') " $SHP/shp_clip/WDPA_poly_Jan2014_${tile}L${BURN}.shp $SHP/shp_clip/WDPA_poly_Jan2014_${tile}.shp
|
|
| 97 |
|
|
| 98 |
|
|
| 99 |
rm -f $RASTERIZE/${tile}_IUCN${BURN}_L.tif
|
|
| 100 |
gdal_rasterize -ot Byte -a_srs EPSG:4326 -l WDPA_poly_Jan2014_${tile}L${BURN} -a IUCN_CAT \
|
|
| 101 |
-burn $BURN -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te $geo_string -co COMPRESS=LZW \ |
|
| 102 |
-co ZLEVEL=9 $SHP/shp_clip/WDPA_poly_Jan2014_${tile}L${BURN}.shp $RASTERIZE/${BURN}/${tile}_IUCN${BURN}_L.tif
|
|
| 103 |
|
|
| 104 |
# |
|
| 105 |
rm -f $SHP/shp_clip/WDPA_poly_Jan2014_${tile}L.*
|
|
| 106 |
|
|
| 107 |
|
|
| 108 |
echo rasterize the marine |
|
| 109 |
|
|
| 110 |
if [ $n -eq 1 ] ; then IUCN_CAT='"Ia"' ; BURN=111 ; fi |
|
| 111 |
if [ $n -eq 2 ] ; then IUCN_CAT='"Ib"' ; BURN=112 ; fi |
|
| 112 |
if [ $n -eq 3 ] ; then IUCN_CAT='"II"' ; BURN=102 ; fi |
|
| 113 |
if [ $n -eq 4 ] ; then IUCN_CAT='"III"'; BURN=103 ; fi |
|
| 114 |
if [ $n -eq 5 ] ; then IUCN_CAT='"IV"' ; BURN=104 ; fi |
|
| 115 |
if [ $n -eq 6 ] ; then IUCN_CAT='"Not Applicable"' ; BURN=120 ; fi |
|
| 116 |
if [ $n -eq 7 ] ; then IUCN_CAT='"Not Reported"' ; BURN=121 ; fi |
|
| 117 |
if [ $n -eq 8 ] ; then IUCN_CAT='"V"' ; BURN=105 ; fi |
|
| 118 |
if [ $n -eq 9 ] ; then IUCN_CAT='"VI"' ; BURN=106 ; fi |
|
| 119 |
|
|
| 120 |
rm -f $SHP/shp_clip/WDPA_poly_Jan2014_${tile}M${BURN}.*
|
|
| 121 |
ogr2ogr -sql "SELECT * FROM WDPA_poly_Jan2014_${tile} WHERE ( IUCN_CAT = ${IUCN_CAT} ) AND ( MARINE = '1') " $SHP/shp_clip/WDPA_poly_Jan2014_${tile}M${BURN}.shp $SHP/shp_clip/WDPA_poly_Jan2014_${tile}.shp
|
|
| 122 |
|
|
| 123 |
rm -f $RASTERIZE/${tile}_IUCN${BURN}_M.tif
|
|
| 124 |
|
|
| 125 |
gdal_rasterize -ot Byte -a_srs EPSG:4326 -l WDPA_poly_Jan2014_${tile}M${BURN} -a IUCN_CAT \
|
|
| 126 |
-burn $BURN -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te $geo_string -co COMPRESS=LZW \ |
|
| 127 |
-co ZLEVEL=9 $SHP/shp_clip/WDPA_poly_Jan2014_${tile}M${BURN}.shp $RASTERIZE/${BURN}/${tile}_IUCN${BURN}_M.tif
|
|
| 128 |
|
|
| 129 |
# rm -f $SHP/shp_clip/WDPA_poly_Jan2014_${tile}M.*
|
|
| 130 |
|
|
| 131 |
done |
|
| 132 |
|
|
| 133 |
exit |
|
| 134 |
|
|
| 135 |
|
|
| 136 |
|
|
| 137 |
|
|
| 138 |
|
|
| terrain/procedures/dem_variables/WDPA/old/sc1_rasterize_tile_LAND.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
# awk '{ if ( NR > 1 ) print $1 }' /lustre0/scratch/ga254/dem_bj/GMTED2010/geo_file/tiles-te_noOverlap.txt | xargs -n 1 -P 10 bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/WDPA/sc1_rasterize_tile_all.sh
|
|
| 3 |
|
|
| 4 |
# for tile in $(awk '{ if ( NR > 1 ) print $1 }' /lustre0/scratch/ga254/dem_bj/GMTED2010/geo_file/tiles-te_noOverlap.txt) ; do qsub -v tile=$tile /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/WDPA/sc1_rasterize_tile.sh ; done
|
|
| 5 |
|
|
| 6 |
|
|
| 7 |
# after compile gdal with the following set |
|
| 8 |
# ./configure --with-fgdb=/usr/local/FileGDB_API --with-geos=yes |
|
| 9 |
|
|
| 10 |
|
|
| 11 |
# DIR=/mnt/data2/scratch/WDPA |
|
| 12 |
# apps/ogr2ogr /mnt/data2/scratch/WDPA/shp_input/WDPA_Jan2014.shp /mnt/data2/scratch/WDPA/shp_input/WDPA_Jan2014_Public/WDPA_Jan2014.gdb |
|
| 13 |
|
|
| 14 |
|
|
| 15 |
|
|
| 16 |
#PBS -S /bin/bash |
|
| 17 |
#PBS -q fas_normal |
|
| 18 |
#PBS -l mem=10gb |
|
| 19 |
#PBS -l walltime=4:00:00 |
|
| 20 |
#PBS -l nodes=1:ppn=4 |
|
| 21 |
#PBS -V |
|
| 22 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 23 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 24 |
|
|
| 25 |
|
|
| 26 |
# load moduels |
|
| 27 |
|
|
| 28 |
module load Tools/Python/2.7.3 |
|
| 29 |
module load Libraries/GDAL/1.10.0 |
|
| 30 |
# module load Tools/PKTOOLS/2.4.2 |
|
| 31 |
module load Libraries/OSGEO/1.10.0 |
|
| 32 |
|
|
| 33 |
|
|
| 34 |
|
|
| 35 |
# export tile=$1 |
|
| 36 |
export tile=$1 |
|
| 37 |
|
|
| 38 |
export RASTERIZE=/lustre0/scratch/ga254/dem_bj/WDPA/rasterize_all/tiles |
|
| 39 |
export SHP=/lustre0/scratch/ga254/dem_bj/WDPA |
|
| 40 |
|
|
| 41 |
|
|
| 42 |
export geo_string=$( grep $tile /lustre0/scratch/ga254/dem_bj/GMTED2010/geo_file/tiles-te_noOverlap.txt | awk '{ print $2,$3,$4,$5 }' )
|
|
| 43 |
|
|
| 44 |
echo clip the large shp by $geo_string |
|
| 45 |
|
|
| 46 |
rm -f $SHP/shp_input/WDPA_Jan2014.shp/shp_clip/WDPA_poly_Jan2014_$tile.shp |
|
| 47 |
|
|
| 48 |
ogr2ogr -skipfailures -spat $geo_string $SHP/shp_input/WDPA_Jan2014.shp/shp_clip/WDPA_poly_Jan2014_$tile.shp $SHP/shp_input/WDPA_Jan2014.shp/WDPA_poly_Jan2014.shp |
|
| 49 |
|
|
| 50 |
|
|
| 51 |
|
|
| 52 |
rm -f $RASTERIZE/${tile}.tif
|
|
| 53 |
gdal_rasterize -ot Byte -a_srs EPSG:4326 -l WDPA_poly_Jan2014_$tile -burn 1 -a_nodata 0 -tr 0.008333333333333 0.008333333333333 \ |
|
| 54 |
-te $geo_string -co COMPRESS=LZW -co ZLEVEL=9 $SHP/shp_input/WDPA_Jan2014.shp/shp_clip/WDPA_poly_Jan2014_$tile.shp $RASTERIZE/${tile}.tif
|
|
| 55 |
|
|
| 56 |
exit |
|
| 57 |
|
|
| 58 |
|
|
| 59 |
|
|
| 60 |
|
|
| 61 |
|
|
| terrain/procedures/dem_variables/WDPA/old/sc1_rasterize_tile_all.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
# awk '{ if ( NR > 1 ) print $1 }' /lustre0/scratch/ga254/dem_bj/GMTED2010/geo_file/tiles-te_noOverlap.txt | xargs -n 1 -P 10 bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/WDPA/sc1_rasterize_tile_all.sh
|
|
| 3 |
|
|
| 4 |
# for tile in $(awk '{ if ( NR > 1 ) print $1 }' /lustre0/scratch/ga254/dem_bj/GMTED2010/geo_file/tiles-te_noOverlap.txt) ; do qsub -v tile=$tile /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/WDPA/sc1_rasterize_tile.sh ; done
|
|
| 5 |
|
|
| 6 |
|
|
| 7 |
# after compile gdal with the following set |
|
| 8 |
# ./configure --with-fgdb=/usr/local/FileGDB_API --with-geos=yes |
|
| 9 |
|
|
| 10 |
|
|
| 11 |
# DIR=/mnt/data2/scratch/WDPA |
|
| 12 |
# apps/ogr2ogr /mnt/data2/scratch/WDPA/shp_input/WDPA_Jan2014.shp /mnt/data2/scratch/WDPA/shp_input/WDPA_Jan2014_Public/WDPA_Jan2014.gdb |
|
| 13 |
|
|
| 14 |
|
|
| 15 |
|
|
| 16 |
#PBS -S /bin/bash |
|
| 17 |
#PBS -q fas_normal |
|
| 18 |
#PBS -l mem=10gb |
|
| 19 |
#PBS -l walltime=4:00:00 |
|
| 20 |
#PBS -l nodes=1:ppn=4 |
|
| 21 |
#PBS -V |
|
| 22 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 23 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 24 |
|
|
| 25 |
|
|
| 26 |
# load moduels |
|
| 27 |
|
|
| 28 |
module load Tools/Python/2.7.3 |
|
| 29 |
module load Libraries/GDAL/1.10.0 |
|
| 30 |
module load Tools/PKTOOLS/2.4.2 |
|
| 31 |
module load Libraries/OSGEO/1.10.0 |
|
| 32 |
|
|
| 33 |
|
|
| 34 |
|
|
| 35 |
# export tile=$1 |
|
| 36 |
export tile=$1 |
|
| 37 |
|
|
| 38 |
export RASTERIZE=/lustre0/scratch/ga254/dem_bj/WDPA/rasterize_all/tiles |
|
| 39 |
export SHP=/lustre0/scratch/ga254/dem_bj/WDPA |
|
| 40 |
|
|
| 41 |
|
|
| 42 |
export geo_string=$( grep $tile /lustre0/scratch/ga254/dem_bj/GMTED2010/geo_file/tiles-te_noOverlap.txt | awk '{ print $2,$3,$4,$5 }' )
|
|
| 43 |
|
|
| 44 |
echo clip the large shp by $geo_string |
|
| 45 |
|
|
| 46 |
rm -f $SHP/shp_input/WDPA_Jan2014.shp/shp_clip/WDPA_poly_Jan2014_$tile.shp |
|
| 47 |
|
|
| 48 |
ogr2ogr -skipfailures -spat $geo_string $SHP/shp_input/WDPA_Jan2014.shp/shp_clip/WDPA_poly_Jan2014_$tile.shp $SHP/shp_input/WDPA_Jan2014.shp/WDPA_poly_Jan2014.shp |
|
| 49 |
|
|
| 50 |
|
|
| 51 |
|
|
| 52 |
rm -f $RASTERIZE/${tile}.tif
|
|
| 53 |
gdal_rasterize -ot Byte -a_srs EPSG:4326 -l WDPA_poly_Jan2014_$tile -burn 1 -a_nodata 0 -tr 0.008333333333333 0.008333333333333 \ |
|
| 54 |
-te $geo_string -co COMPRESS=LZW -co ZLEVEL=9 $SHP/shp_input/WDPA_Jan2014.shp/shp_clip/WDPA_poly_Jan2014_$tile.shp $RASTERIZE/${tile}.tif
|
|
| 55 |
|
|
| 56 |
exit |
|
| 57 |
|
|
| 58 |
|
|
| 59 |
|
|
| 60 |
|
|
| 61 |
|
|
| terrain/procedures/dem_variables/WDPA/old/sc1a_rasterize_shp.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
gdal_rasterize -ot UInt16 -a_srs EPSG:4326 -l 360x114global -a ID -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te -180 -90 +180 +90 -co COMPRESS=LZW \ |
|
| 3 |
-co ZLEVEL=9 /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114global/360x114global.shp /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114globalID.tif |
|
| 4 |
|
|
| 5 |
gdal_rasterize -ot UInt16 -a_srs EPSG:4326 -l 360x114global -a ID_1 -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te -180 -90 +180 +90 -co COMPRESS=LZW \ |
|
| 6 |
-co ZLEVEL=9 /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114global/360x114global.shp /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114globalID_1.tif |
|
| 7 |
|
|
| 8 |
gdal_rasterize -ot UInt16 -a_srs EPSG:4326 -l 360x114global -a HBWID -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te -180 -90 +180 +90 -co COMPRESS=LZW \ |
|
| 9 |
-co ZLEVEL=9 /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114global/360x114global.shp /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114globalHBWID.tif |
|
| terrain/procedures/dem_variables/WDPA/old/sc2_merge_tile.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
# for dir in 102 103 104 105 106 11 111 112 12 120 121 2 20 21 3 4 5 6; do qsub -v DIR=$dir /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/WDPA/sc2_merge_tile.sh ; done |
|
| 3 |
|
|
| 4 |
# echo 102 103 104 105 106 11 111 112 12 120 121 2 20 21 3 4 5 6 | xargs -n 1 -P 20 bash /lustre0/scratch/ga254/scripts_bj/environmental-layers/terrain/procedures/dem_variables/WDPA/sc2_merge_tile.sh |
|
| 5 |
|
|
| 6 |
|
|
| 7 |
#PBS -S /bin/bash |
|
| 8 |
#PBS -q fas_normal |
|
| 9 |
#PBS -l mem=10gb |
|
| 10 |
#PBS -l walltime=4:00:00 |
|
| 11 |
#PBS -l nodes=1:ppn=4 |
|
| 12 |
#PBS -V |
|
| 13 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 14 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 15 |
|
|
| 16 |
|
|
| 17 |
# load moduels |
|
| 18 |
|
|
| 19 |
module load Tools/Python/2.7.3 |
|
| 20 |
module load Libraries/GDAL/1.10.0 |
|
| 21 |
module load Tools/PKTOOLS/2.4.2 |
|
| 22 |
module load Libraries/OSGEO/1.10.0 |
|
| 23 |
|
|
| 24 |
# export DIR=$DIR |
|
| 25 |
export DIR=$1 |
|
| 26 |
|
|
| 27 |
|
|
| 28 |
export RASTERIZE=/lustre0/scratch/ga254/dem_bj/WDPA/rasterize |
|
| 29 |
|
|
| 30 |
rm -f $RASTERIZE/$DIR/a.tif $RASTERIZE/$DIR/b.tif $RASTERIZE/$DIR/c.tif $RASTERIZE/$DIR/d.tif $RASTERIZE/$DIR/e.tif |
|
| 31 |
|
|
| 32 |
/home2/ga254/bin/gdal_merge_bylines.py -o $RASTERIZE/$DIR/a.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/$DIR/[0-1]_?_IUCN*.tif |
|
| 33 |
/home2/ga254/bin/gdal_merge_bylines.py -o $RASTERIZE/$DIR/b.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/$DIR/[2-3]_?_IUCN*.tif |
|
| 34 |
/home2/ga254/bin/gdal_merge_bylines.py -o $RASTERIZE/$DIR/c.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/$DIR/[4-5]_?_IUCN*.tif |
|
| 35 |
/home2/ga254/bin/gdal_merge_bylines.py -o $RASTERIZE/$DIR/d.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/$DIR/[6-7]_?_IUCN*.tif |
|
| 36 |
/home2/ga254/bin/gdal_merge_bylines.py -o $RASTERIZE/$DIR/e.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/$DIR/[8-9]_?_IUCN*.tif |
|
| 37 |
|
|
| 38 |
rm -f $RASTERIZE/$DIR/tmpIUCN$DIR.tif |
|
| 39 |
|
|
| 40 |
/home2/ga254/bin/gdal_merge_bylines.py -ul_lr -180 +90 +180 -90 -o $RASTERIZE/$DIR/tmpIUCN$DIR.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/$DIR/[a-e].tif |
|
| 41 |
|
|
| 42 |
rm -f $RASTERIZE/$DIR/a.tif $RASTERIZE/$DIR/b.tif $RASTERIZE/$DIR/c.tif $RASTERIZE/$DIR/d.tif $RASTERIZE/$DIR/e.tif |
|
| 43 |
|
|
| 44 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -outsize -a_nodata 0 $RASTERIZE/$DIR/IUCN$DIR.tif $RASTERIZE/$DIR/tmpIUCN$DIR.tif |
|
| 45 |
rm -f $RASTERIZE/$DIR/tmpIUCN$DIR.tif |
|
| terrain/procedures/dem_variables/WDPA/old/sc2_merge_tile_all.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
|
|
| 3 |
|
|
| 4 |
#PBS -S /bin/bash |
|
| 5 |
#PBS -q fas_normal |
|
| 6 |
#PBS -l mem=10gb |
|
| 7 |
#PBS -l walltime=4:00:00 |
|
| 8 |
#PBS -l nodes=1:ppn=4 |
|
| 9 |
#PBS -V |
|
| 10 |
#PBS -o /lustre0/scratch/ga254/stdout |
|
| 11 |
#PBS -e /lustre0/scratch/ga254/stderr |
|
| 12 |
|
|
| 13 |
|
|
| 14 |
# load moduels |
|
| 15 |
|
|
| 16 |
module load Tools/Python/2.7.3 |
|
| 17 |
module load Libraries/GDAL/1.10.0 |
|
| 18 |
# module load Tools/PKTOOLS/2.4.2 |
|
| 19 |
module load Libraries/OSGEO/1.10.0 |
|
| 20 |
|
|
| 21 |
|
|
| 22 |
|
|
| 23 |
export RASTERIZE=/lustre0/scratch/ga254/dem_bj/WDPA/rasterize_all |
|
| 24 |
|
|
| 25 |
rm -f $RASTERIZE/a.tif $RASTERIZE/b.tif $RASTERIZE/c.tif $RASTERIZE/d.tif $RASTERIZE/e.tif |
|
| 26 |
|
|
| 27 |
echo a b c d e | xargs -n 1 -P 5 bash -c $' |
|
| 28 |
|
|
| 29 |
l=$1 |
|
| 30 |
|
|
| 31 |
if [ $l = "a" ];then |
|
| 32 |
/home2/ga254/bin/gdal_merge_bylines.py -o $RASTERIZE/$l.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/tiles/[0-1]_?.tif |
|
| 33 |
fi |
|
| 34 |
|
|
| 35 |
if [ $l = "b" ];then |
|
| 36 |
/home2/ga254/bin/gdal_merge_bylines.py -o $RASTERIZE/$l.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/tiles/[2-3]_?.tif |
|
| 37 |
fi |
|
| 38 |
if [ $l = "c" ];then |
|
| 39 |
/home2/ga254/bin/gdal_merge_bylines.py -o $RASTERIZE/$l.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/tiles/[4-5]_?.tif |
|
| 40 |
fi |
|
| 41 |
if [ $l = "d" ];then |
|
| 42 |
/home2/ga254/bin/gdal_merge_bylines.py -o $RASTERIZE/$l.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/tiles/[6-7]_?.tif |
|
| 43 |
fi |
|
| 44 |
if [ $l = "e" ];then |
|
| 45 |
/home2/ga254/bin/gdal_merge_bylines.py -o $RASTERIZE/$l.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/tiles/[8-9]_?.tif |
|
| 46 |
fi |
|
| 47 |
' _ |
|
| 48 |
|
|
| 49 |
|
|
| 50 |
|
|
| 51 |
rm -f $RASTERIZE/WDPA_poly_Jan2014.tif |
|
| 52 |
|
|
| 53 |
/home2/ga254/bin/gdal_merge_bylines.py -ul_lr -180 +90 +180 -90 -o $RASTERIZE/WDPA_poly_Jan2014.tif -ot Byte -co COMPRESS=LZW $RASTERIZE/[a-e].tif |
|
| 54 |
|
|
| 55 |
# rm -f $RASTERIZE/a.tif $RASTERIZE/b.tif $RASTERIZE/c.tif $RASTERIZE/d.tif $RASTERIZE/e.tif |
|
| 56 |
|
|
| 57 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -a_nodata 0 $RASTERIZE/WDPA_poly_Jan2014.tif $RASTERIZE/tmpIUCN$DIR.tif |
|
| 58 |
rm -f $RASTERIZE/tmpIUCN$DIR.tif |
|
| terrain/procedures/dem_variables/WDPA/old/sc3_extrac_area-oft-stat.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
# calculate area for each cell |
|
| 3 |
|
|
| 4 |
TIF_GRID=/lustre0/scratch/ga254/dem_bj/WDPA/tif_grid |
|
| 5 |
OUTDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat |
|
| 6 |
|
|
| 7 |
|
|
| 8 |
|
|
| 9 |
# calcuate area for ecah cell in a specific IUCN class |
|
| 10 |
|
|
| 11 |
echo 102 103 104 105 106 11 111 112 12 120 121 2 20 21 3 4 5 6 | xargs -n 1 -P 18 bash -c $' |
|
| 12 |
|
|
| 13 |
TIF_GRID=/lustre0/scratch/ga254/dem_bj/WDPA/tif_grid |
|
| 14 |
OUTDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat |
|
| 15 |
IUCN=$1 |
|
| 16 |
ulx=-180 |
|
| 17 |
uly=$(getCorners4Gtranslate /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/$IUCN/IUCN$IUCN.tif | awk \'{print $2}\')
|
|
| 18 |
lrx=+180 |
|
| 19 |
lry=$(getCorners4Gtranslate /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/$IUCN/IUCN$IUCN.tif | awk \'{print $4}\')
|
|
| 20 |
|
|
| 21 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -projwin $ulx $uly $lrx $lry $TIF_GRID/360x114globalID.tif $TIF_GRID/360x114globalIDclip$IUCN.tif |
|
| 22 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -projwin $ulx $uly $lrx $lry /lustre0/scratch/ga254/dem_bj/GEO_AREA/area_tif/30arc-sec-Area_prj6974.tif $TIF_GRID/30arc-sec-Area_prj6974clip$IUCN.tif |
|
| 23 |
|
|
| 24 |
|
|
| 25 |
pksetmask -co COMPRESS=LZW -co ZLEVEL=9 -i $TIF_GRID/360x114globalIDclip$IUCN.tif -m /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/$IUCN/IUCN$IUCN.tif -msknodata 0 -nodata 0 -o $TIF_GRID/360x114globalID$IUCN.tif |
|
| 26 |
|
|
| 27 |
oft-stat -nostd -i $TIF_GRID/30arc-sec-Area_prj6974clip$IUCN.tif -o $OUTDIR/${IUCN}_stat.txt -um $TIF_GRID/360x114globalID$IUCN.tif
|
|
| 28 |
|
|
| 29 |
' _ |
|
| 30 |
|
|
| 31 |
|
|
| 32 |
# calculate the area for each class |
|
| 33 |
|
|
| 34 |
echo 102 103 104 105 106 11 111 112 12 120 121 2 20 21 3 4 5 6 | xargs -n 1 -P 18 bash -c $' |
|
| 35 |
|
|
| 36 |
INDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat |
|
| 37 |
IUCN=$1 |
|
| 38 |
awk \'{ print $1 , int ($2*$3 ) }\' $INDIR/${IUCN}_stat.txt > $INDIR/${IUCN}_stat_sum.txt
|
|
| 39 |
|
|
| 40 |
' _ |
|
| 41 |
|
|
| 42 |
|
|
| 43 |
# calculate the area for each cell |
|
| 44 |
INDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat |
|
| 45 |
oft-stat -nostd -i /lustre0/scratch/ga254/dem_bj/WDPA/tif_grid/30arc-sec-Area_prj6974clip20.tif -o $OUTDIR/ID_area_stat.txt -um $TIF_GRID/360x114globalID.tif |
|
| 46 |
awk '{ print $1 , int ($2*$3 ) }' $INDIR/ID_area_stat.txt > $INDIR/ID_area_sum.txt
|
|
| 47 |
|
|
| 48 |
|
|
| 49 |
|
|
| 50 |
# join and calculate the percentage |
|
| 51 |
|
|
| 52 |
echo 102 103 104 105 106 11 111 112 12 120 121 2 20 21 3 4 5 6 | xargs -n 1 -P 18 bash -c $' |
|
| 53 |
|
|
| 54 |
INDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat |
|
| 55 |
IUCN=$1 |
|
| 56 |
|
|
| 57 |
echo join |
|
| 58 |
join -1 1 -2 1 -a 1 <(sort -k 1,1 $INDIR/ID_area_sum.txt) <(sort -k 1,1 $INDIR/${IUCN}_stat_sum.txt) > $INDIR/${IUCN}_stat_sum_j.txt
|
|
| 59 |
echo perc |
|
| 60 |
awk \'{ if (NF==3) {print $1 , (100/$2)*$3 } else { print $1 , "0" } }\' $INDIR/${IUCN}_stat_sum_j.txt > $INDIR/${IUCN}_stat_perc.txt
|
|
| 61 |
|
|
| 62 |
' _ |
|
| 63 |
|
|
| 64 |
|
|
| 65 |
INDIR=/lustre0/scratch/ga254/dem_bj/WDPA |
|
| 66 |
rm -f /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114global/360x114global_join.* |
|
| 67 |
ogr2ogr $INDIR/shp_grid/360x114global/360x114global_join.shp $INDIR/shp_grid/360x114global/360x114global.shp |
|
| 68 |
|
|
| 69 |
INDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat |
|
| 70 |
for IUCN in 102 103 104 105 106 11 111 112 12 120 121 2 20 21 3 4 5 6 ; do |
|
| 71 |
oft-addattr-new.py /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114global/360x114global_join.shp ID IUCN${IUCN} Float $INDIR/${IUCN}_stat_perc.txt 0
|
|
| 72 |
done |
|
| 73 |
|
|
| 74 |
|
|
| 75 |
# create tif file |
|
| 76 |
|
|
| 77 |
|
|
| 78 |
|
|
| 79 |
echo 102 103 104 105 106 11 111 112 12 120 121 2 20 21 3 4 5 6 | xargs -n 1 -P 18 bash -c $' |
|
| 80 |
IUCN=$1 |
|
| 81 |
pkreclass -co COMPRESS=LZW -co ZLEVEL=9 -ot Byte -msknodata 0 -nodata 0 -code /lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat/${IUCN}_stat_perc.txt -i /lustre0/scratch/ga254/dem_bj/WDPA/tif_grid/360x114globalID${IUCN}.tif -o /lustre0/scratch/ga254/dem_bj/WDPA/tif_grid/perc$IUCN.tif
|
|
| 82 |
|
|
| 83 |
' _ |
|
| 84 |
|
|
| 85 |
|
|
| 86 |
|
|
| 87 |
|
|
| 88 |
|
|
| 89 |
|
|
| 90 |
|
|
| 91 |
|
|
| terrain/procedures/dem_variables/WDPA/old/sc3_extrac_area.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
# extract the area for each cell |
|
| 3 |
|
|
| 4 |
rm -f /lustre0/scratch/ga254/dem_bj/WDPA/shp_out/360x114global_areaPrj6974.* |
|
| 5 |
pkextract -i /lustre0/scratch/ga254/dem_bj/GEO_AREA/area_tif/30arc-sec-Area_prj6974.tif \ |
|
| 6 |
-l -r sum -bn AREAp6974 -lt String -s /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114global/360x114global.shp \ |
|
| 7 |
-o /lustre0/scratch/ga254/dem_bj/WDPA/shp_out/360x114global_areaPrj6974.shp |
|
| 8 |
|
|
| 9 |
|
|
| 10 |
rm -f /lustre0/scratch/ga254/dem_bj/WDPA/shp_out/360x114globalIUCN*.* |
|
| 11 |
|
|
| 12 |
|
|
| 13 |
# extract the area for each cell useing as a mask the different IUCN classes |
|
| 14 |
|
|
| 15 |
echo 103 104 105 106 11 111 112 12 120 121 2 20 21 3 4 5 6 | xargs -n 1 -P 8 bash -c $' |
|
| 16 |
|
|
| 17 |
dir=$1 |
|
| 18 |
rm -f /lustre0/scratch/ga254/dem_bj/WDPA/shp_out/360x114globalIUCN$dir.* |
|
| 19 |
|
|
| 20 |
pkextract -m /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/$dir/IUCN*.tif -msknodata 0 -i /lustre0/scratch/ga254/dem_bj/GEO_AREA/area_tif/30arc-sec-Area_prj6974.tif -l -r sum -bn IUCN$dir -lt String -s /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114global/360x114global.shp -o /lustre0/scratch/ga254/dem_bj/WDPA/shp_out/360x114globalIUCN$dir.shp |
|
| 21 |
|
|
| 22 |
' _ |
|
| 23 |
|
|
| 24 |
|
|
| 25 |
# dir=103 |
|
| 26 |
# pkextract -m /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/$dir/IUCN*.tif -msknodata 0 -i /lustre0/scratch/ga254/dem_bj/GEO_AREA/area_tif/30arc-sec-Area_prj6974.tif -l -r sum -bn IUCN$dir -lt String -s /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114globalID.tif -o /lustre0/scratch/ga254/dem_bj/WDPA/shp_out/test.shp |
|
| 27 |
|
|
| 28 |
|
|
| 29 |
|
|
| 30 |
|
|
| 31 |
|
|
| terrain/procedures/dem_variables/WDPA/old/sc3_extrac_area_LAND.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
OUTDIR=/lustre0/scratch/ga254/dem_bj/WDPA/rasterize_land |
|
| 3 |
DIR=/lustre0/scratch/ga254/dem_bj/WDPA/ |
|
| 4 |
|
|
| 5 |
|
|
| 6 |
pkmosaic $( for class in 11 12 2 20 21 3 4 5 6; do echo -i rasterize/$class/IUCN$class.tif ; done ) -cr max -o $OUTDIR/WDPA_landpoly.tif |
|
| 7 |
|
|
| 8 |
gdal_translate -a_ullr $(getCorners4Gtranslate rasterize/6/IUCN6.tif) -co COMPRESS=LZW -co ZLEVEL=9 -a_srs EPSG:4326 $OUTDIR/WDPA_landpoly.tif $OUTDIR/WDPA_landpoly2.tif |
|
| 9 |
|
|
| 10 |
# rasterize point |
|
| 11 |
gdal_rasterize -ot Byte -a_srs EPSG:4326 -l WDPA_point_Jan2014 -sql "SELECT * FROM WDPA_point_Jan2014 WHERE MARINE = '0'" \ |
|
| 12 |
-burn 1 -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te $(getCorners4Gwarp /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/6/IUCN6.tif ) -co COMPRESS=LZW \ |
|
| 13 |
-co ZLEVEL=9 /lustre0/scratch/ga254/dem_bj/WDPA/shp_input/WDPA_Jan2014.shp/WDPA_point_Jan2014.shp $OUTDIR/WDPA_point_Jan2014EPSG4326.tif |
|
| 14 |
|
|
| 15 |
# rasterize buffer |
|
| 16 |
gdal_rasterize -ot Byte -a_srs EPSG:4326 -l WDPA_point_Jan2014EPSG4326Buf -sql "SELECT * FROM WDPA_point_Jan2014EPSG4326Buf WHERE MARINE = '0'" \ |
|
| 17 |
-burn 1 -a_nodata 0 -tr 0.008333333333333 0.008333333333333 -te $(getCorners4Gwarp /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/6/IUCN6.tif ) -co COMPRESS=LZW \ |
|
| 18 |
-co ZLEVEL=9 /lustre0/scratch/ga254/dem_bj/WDPA/shp_buf/WDPA_point_Jan2014EPSG4326Buf.shp $OUTDIR/WDPA_point_Jan2014EPSG4326BufLAND.tif |
|
| 19 |
|
|
| 20 |
# merge point buffer poly |
|
| 21 |
|
|
| 22 |
pkmosaic -i $OUTDIR/WDPA_point_Jan2014EPSG4326BufLAND.tif -i $OUTDIR/WDPA_point_Jan2014EPSG4326.tif -i $OUTDIR/WDPA_landpoly2.tif -cr max -o $OUTDIR/WDPA_land_all.tif |
|
| 23 |
gdal_translate -a_ullr $(getCorners4Gtranslate rasterize/6/IUCN6.tif) -co COMPRESS=LZW -co ZLEVEL=9 -a_srs EPSG:4326 $OUTDIR/WDPA_land_all.tif $OUTDIR/WDPA_land_all2.tif |
|
| 24 |
|
|
| 25 |
|
|
| 26 |
# extract the area for each cell |
|
| 27 |
|
|
| 28 |
|
|
| 29 |
TIF_GRID=/lustre0/scratch/ga254/dem_bj/WDPA/tif_grid |
|
| 30 |
IUCN=$1 |
|
| 31 |
ulx=-180 |
|
| 32 |
uly=$(getCorners4Gtranslate /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/6/IUCN6.tif | awk '{print $2}')
|
|
| 33 |
lrx=+180 |
|
| 34 |
lry=$(getCorners4Gtranslate /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/6/IUCN6.tif | awk '{print $4}')
|
|
| 35 |
|
|
| 36 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -projwin $ulx $uly $lrx $lry $TIF_GRID/360x114globalID.tif $TIF_GRID/360x114globalIDclip.tif |
|
| 37 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -projwin $ulx $uly $lrx $lry /lustre0/scratch/ga254/dem_bj/GEO_AREA/area_tif/30arc-sec-Area_prj6974.tif $TIF_GRID/30arc-sec-Area_prj6974clip.tif |
|
| 38 |
|
|
| 39 |
|
|
| 40 |
pksetmask -co COMPRESS=LZW -co ZLEVEL=9 -i $TIF_GRID/360x114globalIDclip.tif -m $OUTDIR/WDPA_land_all2.tif -msknodata 0 -nodata 0 -o $TIF_GRID/360x114globalIDland.tif |
|
| 41 |
|
|
| 42 |
|
|
| 43 |
INDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat |
|
| 44 |
|
|
| 45 |
oft-stat-sum -nostd -i /lustre0/scratch/ga254/dem_bj/WDPA/tif_grid/30arc-sec-Area_prj6974clip.tif -o $INDIR/ID_area_stat.txt -um $TIF_GRID/360x114globalIDclip.tif |
|
| 46 |
oft-stat-sum -nostd -i /lustre0/scratch/ga254/dem_bj/WDPA/tif_grid/30arc-sec-Area_prj6974clip.tif -o $INDIR/land_stat.txt -um $TIF_GRID/360x114globalIDland.tif |
|
| 47 |
|
|
| 48 |
# calculate the percentage |
|
| 49 |
|
|
| 50 |
|
|
| 51 |
|
|
| 52 |
|
|
| 53 |
echo join |
|
| 54 |
join -1 1 -2 1 -a 1 <(sort -k 1,1 $INDIR/ID_area_stat.txt) <(sort -k 1,1 $INDIR/land_stat.txt) > $INDIR/stat_sum_j.txt |
|
| 55 |
echo perc |
|
| 56 |
awk '{ if (NF==5) {print $1 , (100/$3)*$5 } else { print $1 , "0" } }' $INDIR/stat_sum_j.txt > $INDIR/stat_perc.txt
|
|
| 57 |
|
|
| 58 |
# start to merge with the existing shapefile |
|
| 59 |
|
|
| 60 |
# copy the spafile to a new one |
|
| 61 |
rm -f shp_grid/360x114global/360x114global_land.* |
|
| 62 |
ogr2ogr shp_grid/360x114global/360x114global_land.shp shp_grid/360x114global/360x114global.shp |
|
| 63 |
oft-addattr-new.py /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114global/360x114global_land.shp ID LAND Float $INDIR/stat_perc.txt 0 |
|
| 64 |
|
|
| 65 |
|
|
| 66 |
|
|
| 67 |
openev /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114global/360x114global_land.shp $TIF_GRID/360x114globalIDland.tif |
|
| 68 |
|
|
| 69 |
|
|
| terrain/procedures/dem_variables/WDPA/old/sc3_extrac_area_all-py.sh | ||
|---|---|---|
| 1 |
|
|
| 2 |
# calculate area for each cell |
|
| 3 |
|
|
| 4 |
TIF_GRID=/lustre0/scratch/ga254/dem_bj/WDPA/tif_grid |
|
| 5 |
OUTDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_py |
|
| 6 |
|
|
| 7 |
|
|
| 8 |
|
|
| 9 |
# calcuate area for ecah cell in a specific IUCN class |
|
| 10 |
|
|
| 11 |
|
|
| 12 |
|
|
| 13 |
TIF_GRID=/lustre0/scratch/ga254/dem_bj/WDPA/tif_grid |
|
| 14 |
OUTDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat |
|
| 15 |
IUCN=$1 |
|
| 16 |
ulx=-180 |
|
| 17 |
uly=$(getCorners4Gtranslate /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/102/IUCN102.tif | awk '{print $2}')
|
|
| 18 |
lrx=+180 |
|
| 19 |
lry=$(getCorners4Gtranslate /lustre0/scratch/ga254/dem_bj/WDPA/rasterize/102/IUCN102.tif | awk '{print $4}')
|
|
| 20 |
|
|
| 21 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -projwin $ulx $uly $lrx $lry $TIF_GRID/360x114globalID.tif $TIF_GRID/360x114globalIDclip.tif |
|
| 22 |
gdal_translate -co COMPRESS=LZW -co ZLEVEL=9 -projwin $ulx $uly $lrx $lry /lustre0/scratch/ga254/dem_bj/GEO_AREA/area_tif/30arc-sec-Area_prj6974.tif $TIF_GRID/30arc-sec-Area_prj6974clip.tif |
|
| 23 |
|
|
| 24 |
|
|
| 25 |
pksetmask -co COMPRESS=LZW -co ZLEVEL=9 -i $TIF_GRID/360x114globalIDclip.tif -m $RASTERIZE/WDPA_poly_Jan2014.tif -msknodata 0 -nodata 0 -o $TIF_GRID/360x114globalIDall.tif |
|
| 26 |
|
|
| 27 |
oft-stat -nostd -i $TIF_GRID/30arc-sec-Area_prj6974clip.tif -o $OUTDIR/${IUCN}_stat.txt -um $TIF_GRID/360x114globalIDall.tif
|
|
| 28 |
|
|
| 29 |
|
|
| 30 |
|
|
| 31 |
|
|
| 32 |
|
|
| 33 |
|
|
| 34 |
# calculate the area for each class |
|
| 35 |
|
|
| 36 |
echo 102 103 104 105 106 11 111 112 12 120 121 2 20 21 3 4 5 6 | xargs -n 1 -P 18 bash -c $' |
|
| 37 |
|
|
| 38 |
INDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat |
|
| 39 |
IUCN=$1 |
|
| 40 |
awk \'{ print $1 , int ($2*$3 ) }\' $INDIR/${IUCN}_stat.txt > $INDIR/${IUCN}_stat_sum.txt
|
|
| 41 |
|
|
| 42 |
' _ |
|
| 43 |
|
|
| 44 |
|
|
| 45 |
# calculate the area for each cell |
|
| 46 |
INDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat |
|
| 47 |
oft-stat -nostd -i /lustre0/scratch/ga254/dem_bj/WDPA/tif_grid/30arc-sec-Area_prj6974clip20.tif -o $OUTDIR/ID_area_stat.txt -um $TIF_GRID/360x114globalID.tif |
|
| 48 |
awk '{ print $1 , int ($2*$3 ) }' $INDIR/ID_area_stat.txt > $INDIR/ID_area_sum.txt
|
|
| 49 |
|
|
| 50 |
|
|
| 51 |
|
|
| 52 |
# join and calculate the percentage |
|
| 53 |
|
|
| 54 |
echo 102 103 104 105 106 11 111 112 12 120 121 2 20 21 3 4 5 6 | xargs -n 1 -P 18 bash -c $' |
|
| 55 |
|
|
| 56 |
INDIR=/lustre0/scratch/ga254/dem_bj/WDPA/txt_oft-stat |
|
| 57 |
IUCN=$1 |
|
| 58 |
|
|
| 59 |
echo join |
|
| 60 |
join -1 1 -2 1 -a 1 <(sort -k 1,1 $INDIR/ID_area_sum.txt) <(sort -k 1,1 $INDIR/${IUCN}_stat_sum.txt) > $INDIR/${IUCN}_stat_sum_j.txt
|
|
| 61 |
echo perc |
|
| 62 |
awk \'{ if (NF==3) {print $1 , (100/$2)*$3 } else { print $1 , "0" } }\' $INDIR/${IUCN}_stat_sum_j.txt > $INDIR/${IUCN}_stat_perc.txt
|
|
| 63 |
|
|
| 64 |
' _ |
|
| 65 |
|
|
| 66 |
|
|
| 67 |
INDIR=/lustre0/scratch/ga254/dem_bj/WDPA |
|
| 68 |
rm -f /lustre0/scratch/ga254/dem_bj/WDPA/shp_grid/360x114global/360x114global_join.* |
|
| 69 |
ogr2ogr $INDIR/shp_grid/360x114global/360x114global_join.shp $INDIR/shp_grid/360x114global/360x114global.shp |
|
Also available in: Unified diff
push before to move ti github