| 1 |
6cc19306
|
Jim Regetz
|
# Full sequence of (bash) commands used to transform the original
|
| 2 |
|
|
# organisms@jupiter home directory structure to the new layers@atlas
|
| 3 |
|
|
# home directory structure. This was done after first rsyncing the
|
| 4 |
|
|
# entire ~organisms home directory from jupiter onto atlas at
|
| 5 |
|
|
# /home/organisms, and creating a new 'layers' user with home directory
|
| 6 |
|
|
# /home/layers.
|
| 7 |
|
|
#
|
| 8 |
|
|
# Jim Regetz
|
| 9 |
|
|
# NCEAS
|
| 10 |
|
|
# Created on 23-Feb-2012
|
| 11 |
|
|
|
| 12 |
|
|
export ORGANISMS="/home/organisms"
|
| 13 |
|
|
export LAYERS="/home/layers"
|
| 14 |
|
|
|
| 15 |
|
|
#=======================================================================
|
| 16 |
|
|
# set up some directories
|
| 17 |
|
|
#=======================================================================
|
| 18 |
|
|
|
| 19 |
|
|
# create master home for all datasets
|
| 20 |
|
|
mkdir $LAYERS/data
|
| 21 |
|
|
mkdir $LAYERS/data/climate
|
| 22 |
|
|
mkdir $LAYERS/data/terrain
|
| 23 |
|
|
mkdir $LAYERS/data/land-cover
|
| 24 |
|
|
# create (temporary) home for associated readmes, documentation, etc
|
| 25 |
|
|
mkdir $LAYERS/documentation
|
| 26 |
|
|
mkdir $LAYERS/documentation/climate
|
| 27 |
|
|
mkdir $LAYERS/documentation/terrain
|
| 28 |
|
|
mkdir $LAYERS/documentation/land-cover
|
| 29 |
|
|
# create (temporary) home for processing code
|
| 30 |
|
|
mkdir $LAYERS/code
|
| 31 |
|
|
mkdir $LAYERS/code/climate
|
| 32 |
|
|
mkdir $LAYERS/code/terrain
|
| 33 |
|
|
mkdir $LAYERS/code/land-cover
|
| 34 |
|
|
# create (temporary) home for other cruft that i'm not quite ready to delete
|
| 35 |
|
|
mkdir $LAYERS/cruft
|
| 36 |
|
|
# create (temporary?) home for everything else
|
| 37 |
|
|
mkdir $LAYERS/experimental
|
| 38 |
|
|
mkdir $LAYERS/experimental/terrain
|
| 39 |
|
|
mkdir $LAYERS/experimental/land-cover
|
| 40 |
|
|
|
| 41 |
|
|
#=======================================================================
|
| 42 |
|
|
# carry out file migration, reorganization, and cleanup
|
| 43 |
|
|
#=======================================================================
|
| 44 |
|
|
|
| 45 |
|
|
#========#
|
| 46 |
|
|
# Marine #
|
| 47 |
|
|
#========#
|
| 48 |
|
|
|
| 49 |
|
|
# easy one ... just delete, as we're retaining this work on the original
|
| 50 |
|
|
# server
|
| 51 |
|
|
rm -rf Marine/
|
| 52 |
|
|
|
| 53 |
|
|
|
| 54 |
|
|
#=====#
|
| 55 |
|
|
# DEM #
|
| 56 |
|
|
#=====#
|
| 57 |
|
|
|
| 58 |
|
|
#
|
| 59 |
|
|
# SRTM 1km
|
| 60 |
|
|
#
|
| 61 |
|
|
|
| 62 |
|
|
# migrate original rar file
|
| 63 |
|
|
mkdir $LAYERS/data/terrain/dem-cgiar-srtm-1km-ascii
|
| 64 |
|
|
mv $ORGANISMS/DEM/SRTM_1km_ASCII.rar $LAYERS/data/terrain/dem-cgiar-srtm-1km-ascii/srtm_1km_ascii.rar
|
| 65 |
|
|
# migrate clipped/reprojected bil
|
| 66 |
|
|
mv $ORGANISMS/Oregon_SRTM $LAYERS/data/terrain/dem-cgiar-srtm-1km-OR/
|
| 67 |
|
|
|
| 68 |
|
|
#
|
| 69 |
|
|
# CGIAR SRTM 90m
|
| 70 |
|
|
#
|
| 71 |
|
|
|
| 72 |
|
|
# SRTM 90m zips
|
| 73 |
|
|
mv $ORGANISMS/DEM/cgiarSrtm/archive $LAYERS/data/terrain/dem-cgiar-srtm-90m-zips
|
| 74 |
|
|
|
| 75 |
|
|
# SRTM 90m tiles and processed descendents
|
| 76 |
|
|
# first flatten out SRTM 90m directories
|
| 77 |
|
|
mv $ORGANISMS/DEM/cgiarSrtm/SRTM_90m_ASCII_4_1/Tiles_Resampled/Mosaiced \
|
| 78 |
|
|
$ORGANISMS/DEM/cgiarSrtm/SRTM_90m_ASCII_4_1_Resampled_Mosaiced
|
| 79 |
|
|
mv $ORGANISMS/DEM/cgiarSrtm/SRTM_90m_ASCII_4_1/Tiles_Resampled \
|
| 80 |
|
|
$ORGANISMS/DEM/cgiarSrtm/SRTM_90m_ASCII_4_1_Resampled
|
| 81 |
|
|
# now migrate to ~layers
|
| 82 |
|
|
mv $ORGANISMS/DEM/cgiarSrtm/SRTM_90m_ASCII_4_1 \
|
| 83 |
|
|
$LAYERS/data/terrain/dem-cgiar-srtm-90m-asc
|
| 84 |
|
|
mv $ORGANISMS/DEM/cgiarSrtm/SRTM_90m_ASCII_4_1_Resampled \
|
| 85 |
|
|
$LAYERS/data/terrain/dem-cgiar-srtm-90m-asc-resamp
|
| 86 |
|
|
mv $ORGANISMS/DEM/cgiarSrtm/SRTM_90m_ASCII_4_1_Resampled_Mosaiced \
|
| 87 |
|
|
$LAYERS/data/terrain/dem-cgiar-srtm-90m-asc-resamp-mosaic
|
| 88 |
|
|
|
| 89 |
|
|
# SRTM 90m tile index (zipped)
|
| 90 |
|
|
mkdir $LAYERS/data/terrain/dem-cgiar-srtm-90m-index-zip
|
| 91 |
|
|
mv $ORGANISMS/DEM/cgiarSrtm/srtm-tile-index_v4.1_shp.zip \
|
| 92 |
|
|
$LAYERS/data/terrain/dem-cgiar-srtm-90m-index-zip/
|
| 93 |
|
|
|
| 94 |
|
|
# SRTM 90m tile index (unzipped)
|
| 95 |
|
|
mv $ORGANISMS/DEM/cgiarSrtm/index \
|
| 96 |
|
|
$LAYERS/data/terrain/dem-cgiar-srtm-90m-index
|
| 97 |
|
|
|
| 98 |
|
|
# move readme to deal with later
|
| 99 |
|
|
mv $ORGANISMS/DEM/cgiarSrtm/README.txt $LAYERS/documentation/terrain/cgiar-srtm-90m-readme.txt
|
| 100 |
|
|
|
| 101 |
|
|
# remove now-empty directory
|
| 102 |
|
|
rmdir $ORGANISMS/DEM/cgiarSrtm
|
| 103 |
|
|
|
| 104 |
ff302b98
|
Jim Regetz
|
#
|
| 105 |
|
|
# ASTER GDEM1
|
| 106 |
|
|
#
|
| 107 |
|
|
|
| 108 |
|
|
# --- first do cleanup ---
|
| 109 |
|
|
# remove duplicate files (confirmed by diffing)
|
| 110 |
|
|
rm $ORGANISMS/DEM/asterGdem/ListOfEurasiaDropouts.txt
|
| 111 |
|
|
rm $ORGANISMS/DEM/asterGdem/BadFileCheckLogVulcanAfterNewFiles.txt
|
| 112 |
|
|
rm $ORGANISMS/DEM/asterGdem/BadFileCheckLogBeforeCorrection.txt
|
| 113 |
|
|
# remove misc trivial/obsolete scripts
|
| 114 |
|
|
rm $ORGANISMS/DEM/asterGdem/translateToImg.sh
|
| 115 |
|
|
rm $ORGANISMS/DEM/asterGdem/incorrectTilesJuly2011/replacedJuly4/copyDownTifs.sh
|
| 116 |
|
|
rm $ORGANISMS/DEM/asterGdem/incorrectTilesJuly2011/checkBadAsterGDEMFiles.sh
|
| 117 |
|
|
# remove file listings and/or gdalinfo output with insufficient context
|
| 118 |
|
|
rm $ORGANISMS/DEM/asterGdem/incorrectTilesJuly2011/BadFileCheckLogVulcanAfterNewFiles.txt
|
| 119 |
|
|
rm $ORGANISMS/DEM/asterGdem/incorrectTilesJuly2011/ListOfEurasiaDropouts.txt
|
| 120 |
|
|
rm $ORGANISMS/DEM/asterGdem/incorrectTilesJuly2011/BadFileCheckLogBeforeCorrection.txt
|
| 121 |
|
|
rm $ORGANISMS/DEM/asterGdem/incorrectTilesJuly2011/replacedJuly4/Juy4NumFiles.txt
|
| 122 |
|
|
rm $ORGANISMS/DEM/asterGdem/incorrectTilesJuly2011/replacedJuly4/July4NumFiles.txt
|
| 123 |
|
|
rm $ORGANISMS/DEM/asterGdem/incorrectTilesJuly2011/replacedJuly4/July4Files.txt
|
| 124 |
|
|
# remove now-empty directory
|
| 125 |
|
|
rmdir $ORGANISMS/DEM/asterGdem/incorrectTilesJuly2011/replacedJuly4
|
| 126 |
|
|
# remove reeves-generated ArcGIS aux/rrd files
|
| 127 |
|
|
find $ORGANISMS/DEM/asterGdem/ -name "*.aux" -exec rm {} \;
|
| 128 |
|
|
find $ORGANISMS/DEM/asterGdem/ -name "*.rrd" -exec rm {} \;
|
| 129 |
|
|
# remove gdal-generated metadata files
|
| 130 |
|
|
find $ORGANISMS/DEM/asterGdem/ -name "*.tif.aux.xml" -exec rm {} \;
|
| 131 |
|
|
find $ORGANISMS/DEM/asterGdem/ -name "*.dem.aux.xml" -exec rm {} \;
|
| 132 |
|
|
# remove img tiles (for which we have the original tifs in all cases)
|
| 133 |
|
|
rm $ORGANISMS/DEM/asterGdem/TilesBelowN59/*.{img,img.aux.xml}
|
| 134 |
|
|
|
| 135 |
|
|
# migrate Robinson R scripts
|
| 136 |
|
|
mv -i $ORGANISMS/DEM/asterGdem/R_files \
|
| 137 |
|
|
$LAYERS/code/terrain/asterGdem
|
| 138 |
|
|
# migrate Robinson GDEM1 assessment docs (incl QGIS project files)
|
| 139 |
|
|
mkdir $LAYERS/experimental/terrain/gdem-v1
|
| 140 |
|
|
mv -i $ORGANISMS/DEM/asterGdem/*.ods \
|
| 141 |
|
|
$LAYERS/experimental/terrain/gdem-v1/
|
| 142 |
|
|
find $ORGANISMS/DEM/asterGdem -name "*.qgs" -exec mv {} \
|
| 143 |
|
|
$LAYERS/experimental/terrain/gdem-v1/ \;
|
| 144 |
|
|
# migrate remaining tree as the GDEM v1 dataset itself
|
| 145 |
|
|
mv $ORGANISMS/DEM/asterGdem \
|
| 146 |
|
|
$LAYERS/data/terrain/dem-aster-gdem1-30m-orig
|
| 147 |
|
|
|
| 148 |
6cc19306
|
Jim Regetz
|
#
|
| 149 |
|
|
# ASTER GDEM2
|
| 150 |
|
|
#
|
| 151 |
|
|
|
| 152 |
|
|
# remove gdal-generated metadata files
|
| 153 |
|
|
rm $ORGANISMS/DEM/asterGdem2/*.aux.xml
|
| 154 |
|
|
rm $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/*.aux.xml
|
| 155 |
|
|
rm $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/Mosaiced/N59to60/*.aux.xml
|
| 156 |
|
|
|
| 157 |
|
|
# first flatten out directories
|
| 158 |
|
|
mv $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/Mosaiced/N59to60 \
|
| 159 |
|
|
$ORGANISMS/DEM/asterGdem2-90m_NoPixelOffset-Mosaiced-N59to60
|
| 160 |
|
|
mv $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset/Mosaiced \
|
| 161 |
|
|
$ORGANISMS/DEM/asterGdem2-90m_NoPixelOffset-Mosaiced
|
| 162 |
|
|
mv $ORGANISMS/DEM/asterGdem2/90m_NoPixelOffset \
|
| 163 |
|
|
$ORGANISMS/DEM/asterGdem2-90m_NoPixelOffset
|
| 164 |
|
|
# now migrate to ~layers
|
| 165 |
|
|
mv $ORGANISMS/DEM/asterGdem2 \
|
| 166 |
|
|
$LAYERS/data/terrain/dem-aster-gdem2-30m-orig
|
| 167 |
|
|
mv $ORGANISMS/DEM/asterGdem2-90m_NoPixelOffset \
|
| 168 |
|
|
$LAYERS/data/terrain/dem-aster-gdem2-90m-resamp
|
| 169 |
|
|
mv $ORGANISMS/DEM/asterGdem2-90m_NoPixelOffset-Mosaiced \
|
| 170 |
|
|
$LAYERS/data/terrain/dem-aster-gdem2-90m-resamp-mosaic-N60to83
|
| 171 |
|
|
mv $ORGANISMS/DEM/asterGdem2-90m_NoPixelOffset-Mosaiced-N59to60 \
|
| 172 |
|
|
$LAYERS/data/terrain/dem-aster-gdem2-90m-resamp-mosaic-N59to60
|
| 173 |
|
|
|
| 174 |
|
|
#
|
| 175 |
|
|
# CDED
|
| 176 |
|
|
#
|
| 177 |
|
|
|
| 178 |
|
|
# remove duplicate files (confirmed by diffing)
|
| 179 |
|
|
rm $ORGANISMS/DEM/CanadaDED/084b\ \(1\).zip
|
| 180 |
|
|
rm $ORGANISMS/DEM/CanadaDED/Monday/114p\ \(1\).zip
|
| 181 |
|
|
rm $ORGANISMS/DEM/CanadaDED/mosaicMostCanadaDem.sh
|
| 182 |
|
|
rm $ORGANISMS/DEM/CanadaDED/Monday/mergeCanada104105.{tif,aux,rrd}
|
| 183 |
|
|
# migrate zips
|
| 184 |
|
|
mkdir $LAYERS/data/terrain/dem-cded-zip
|
| 185 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/0*.zip \
|
| 186 |
|
|
$LAYERS/data/terrain/dem-cded-zip/
|
| 187 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/Monday/1*.zip \
|
| 188 |
|
|
$LAYERS/data/terrain/dem-cded-zip/
|
| 189 |
|
|
# migrate documentation
|
| 190 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/CanadaDEDTechRefManual.pdf \
|
| 191 |
|
|
$LAYERS/documentation/terrain/cded-TechRefManual.pdf
|
| 192 |
|
|
# migrate some stuff to experimental, for now...
|
| 193 |
|
|
mkdir $LAYERS/experimental/terrain/dem-cded
|
| 194 |
|
|
# ... vrt files, what the heck
|
| 195 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/*demFile.vrt \
|
| 196 |
|
|
$LAYERS/experimental/terrain/dem-cded/
|
| 197 |
|
|
# ... misc scripts and log files
|
| 198 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/in*DemFile.txt \
|
| 199 |
|
|
$LAYERS/experimental/terrain/dem-cded/
|
| 200 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/unzipFiles.sh \
|
| 201 |
|
|
$LAYERS/experimental/terrain/dem-cded/unzipFiles_0.sh
|
| 202 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/Monday/unzipFiles.sh \
|
| 203 |
|
|
$LAYERS/experimental/terrain/dem-cded/unzipFiles_1.sh
|
| 204 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/{.,/Monday}/*.sh \
|
| 205 |
|
|
$LAYERS/experimental/terrain/dem-cded/
|
| 206 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/*.log \
|
| 207 |
|
|
$LAYERS/experimental/terrain/dem-cded/
|
| 208 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/moscomps \
|
| 209 |
|
|
$LAYERS/experimental/terrain/dem-cded/
|
| 210 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/Monday/filesToMosaic \
|
| 211 |
|
|
$LAYERS/experimental/terrain/dem-cded/
|
| 212 |
|
|
mv -i $ORGANISMS/DEM/CanadaDED/Monday/*Log \
|
| 213 |
|
|
$LAYERS/experimental/terrain/dem-cded/
|
| 214 |
|
|
# remove extracted data (no plans to use it), after determining that these are
|
| 215 |
|
|
# the set of files matching globs "*.dem" or "*_fgdc_*.{xml,html}"
|
| 216 |
|
|
# # ---- bash code ---#
|
| 217 |
|
|
# # list out all files contained in the zips
|
| 218 |
|
|
# cd $LAYERS/data/terrain/dem-cded-zip && ls \
|
| 219 |
|
|
# | xargs -I+ sh -c 'unzip -l + | grep -e " 200[7-9]-"' \
|
| 220 |
|
|
# | awk '{print $4}' > /home/layers/cded-extracts.txt
|
| 221 |
|
|
# # ----- R code -----#
|
| 222 |
|
|
# # read in the extracted file names
|
| 223 |
|
|
# extracts <- scan("/home/layers/cded-extracts.txt", what="")
|
| 224 |
|
|
# # read in all CDED files stored in the organisms home dir
|
| 225 |
|
|
# files <- list.files(path=c("/home/organisms/DEM/CanadaDED",
|
| 226 |
|
|
# "/home/organisms/DEM/CanadaDED/Monday"))
|
| 227 |
|
|
# # note that all zip contents had been extracted (...based on filename)
|
| 228 |
|
|
# all(extracts %in% files)
|
| 229 |
|
|
# ## [1] TRUE
|
| 230 |
|
|
# # isolate files *not* extracted from the zips
|
| 231 |
|
|
# others <- setdiff(files, extracts)
|
| 232 |
|
|
# # show that all extracted filenames match one of two patterns
|
| 233 |
|
|
# all(grepl("dem$", extracts) | grepl("_fgdc_", extracts))
|
| 234 |
|
|
# ## [1] TRUE
|
| 235 |
|
|
# # and show that no other files match these
|
| 236 |
|
|
# any(grepl("dem$", others) | grepl("_fgdc_", others))
|
| 237 |
|
|
# ## [1] FALSE
|
| 238 |
|
|
# # ------------------#
|
| 239 |
|
|
rm $ORGANISMS/DEM/CanadaDED/*.dem
|
| 240 |
|
|
rm $ORGANISMS/DEM/CanadaDED/*_fgdc_*.{xml,html}
|
| 241 |
|
|
rm $ORGANISMS/DEM/CanadaDED/Monday/*.dem
|
| 242 |
|
|
rm $ORGANISMS/DEM/CanadaDED/Monday/*_fgdc_*.{xml,html}
|
| 243 |
|
|
# now remove items of uncertain origin
|
| 244 |
|
|
rm $ORGANISMS/DEM/CanadaDED/*.dem.aux.xml
|
| 245 |
|
|
rm $ORGANISMS/DEM/CanadaDED/Monday/*.dem.aux.xml
|
| 246 |
|
|
rm $ORGANISMS/DEM/CanadaDED/*dem?.{aux,rrd}
|
| 247 |
|
|
rm $ORGANISMS/DEM/CanadaDED/Monday/*dem?.{aux,rrd}
|
| 248 |
|
|
rm $ORGANISMS/DEM/CanadaDED/*demFile.{aux,rrd,vrt.aux,tif.aux.xml,tif,vrt.rrd}
|
| 249 |
|
|
rm $ORGANISMS/DEM/CanadaDED/CDEMPlySRTMDf.aux
|
| 250 |
|
|
rm -r $ORGANISMS/DEM/CanadaDED/cdemplysrtmdf
|
| 251 |
|
|
rm -r $ORGANISMS/DEM/CanadaDED/info
|
| 252 |
|
|
# what the hell, remove other generated tifs because I really can't account for
|
| 253 |
|
|
# the details of their creation...
|
| 254 |
|
|
rm $ORGANISMS/DEM/CanadaDED/CanadaDemMos*.{tif,rrd,aux,aux.xml,tif.xml,img,vat.dbf}
|
| 255 |
|
|
rm $ORGANISMS/DEM/CanadaDED/mergeCanada104105.{tif,aux,rrd}
|
| 256 |
|
|
|
| 257 |
|
|
# remove now-empty directories
|
| 258 |
|
|
rmdir $ORGANISMS/DEM/CanadaDED/Monday
|
| 259 |
|
|
rmdir $ORGANISMS/DEM/CanadaDED
|
| 260 |
|
|
|
| 261 |
|
|
|
| 262 |
|
|
#
|
| 263 |
|
|
# FUSED DEM
|
| 264 |
|
|
#
|
| 265 |
|
|
|
| 266 |
|
|
# remove gdal-generated metadata files
|
| 267 |
|
|
rm $ORGANISMS/DEM/GlobalProduct/*.aux.xml
|
| 268 |
|
|
# migrate to ~layers
|
| 269 |
|
|
mv $ORGANISMS/DEM/GlobalProduct $LAYERS/data/terrain/dem-fused
|
| 270 |
|
|
|
| 271 |
|
|
#
|
| 272 |
|
|
# GTOPO30
|
| 273 |
|
|
#
|
| 274 |
|
|
|
| 275 |
ff302b98
|
Jim Regetz
|
# migrate Nunokawa-clipped data -- first version
|
| 276 |
|
|
mkdir $LAYERS/data/terrain/dem-usgs-gtopo30-clipped-nunokawa-v1
|
| 277 |
|
|
find $ORGANISMS/DEM/usgsGTOPO30 -name "*_Clipped.dem" \
|
| 278 |
|
|
-exec mv {} $LAYERS/data/terrain/dem-usgs-gtopo30-clipped-nunokawa-v1 \;
|
| 279 |
|
|
# migrate Nunokawa-clipped data -- second version
|
| 280 |
|
|
mv -i $ORGANISMS/DEM/Yuni/Data/GTOPO30 \
|
| 281 |
|
|
$LAYERS/data/terrain/dem-usgs-gtopo30-clipped-nunokawa-v2
|
| 282 |
|
|
|
| 283 |
|
|
# migrate Robinson-clipped data
|
| 284 |
|
|
mkdir $LAYERS/data/terrain/dem-usgs-gtopo30-clipped-robinson
|
| 285 |
|
|
mv $ORGANISMS/DEM/asterGdem/N59to81_E20to59/e020n90/E020N90_clipped.dem \
|
| 286 |
|
|
$LAYERS/data/terrain/dem-usgs-gtopo30-clipped-robinson/
|
| 287 |
|
|
mv $ORGANISMS/DEM/asterGdem/N59to81_E140to179/e140n90/E140N90_clipped.dem \
|
| 288 |
|
|
$LAYERS/data/terrain/dem-usgs-gtopo30-clipped-robinson/
|
| 289 |
|
|
mv $ORGANISMS/DEM/asterGdem/N59to81_W140to101/w140n90/W140N90_clipped.dem \
|
| 290 |
|
|
$LAYERS/data/terrain/dem-usgs-gtopo30-clipped-robinson/
|
| 291 |
|
|
mv $ORGANISMS/DEM/asterGdem/N59to81_E60to99/USGS_ErosDEM_N59to81E60to99/E060N90_clipped.dem \
|
| 292 |
|
|
$LAYERS/data/terrain/dem-usgs-gtopo30-clipped-robinson/
|
| 293 |
|
|
mv $ORGANISMS/DEM/asterGdem/N59to81_W60to21/w060n90/W060N90_clipped.dem \
|
| 294 |
|
|
$LAYERS/data/terrain/dem-usgs-gtopo30-clipped-robinson/
|
| 295 |
|
|
mv $ORGANISMS/DEM/asterGdem/N59to81_W100to61/w100n90/W100N90_clipped.dem \
|
| 296 |
|
|
$LAYERS/data/terrain/dem-usgs-gtopo30-clipped-robinson/
|
| 297 |
|
|
mv $ORGANISMS/DEM/asterGdem/N59to81_E100to139/e100n90/E100N90_clipped.dem \
|
| 298 |
|
|
$LAYERS/data/terrain/dem-usgs-gtopo30-clipped-robinson/
|
| 299 |
|
|
mv $ORGANISMS/DEM/asterGdem/N59to81_W180to141/w180n90/W180N90_clipped.dem \
|
| 300 |
|
|
$LAYERS/data/terrain/dem-usgs-gtopo30-clipped-robinson/
|
| 301 |
|
|
mv $ORGANISMS/DEM/asterGdem/N59to81_W20toE19/w020n90/W020N90_clipped.dem \
|
| 302 |
|
|
$LAYERS/data/terrain/dem-usgs-gtopo30-clipped-robinson/
|
| 303 |
|
|
# remove mysterious extra GeoTIFF (sans extension) with timestamp similar to
|
| 304 |
|
|
# the above, but that was mysteriously the same as the like-named
|
| 305 |
|
|
# clipped-nunokawa-v1 file (created months later)
|
| 306 |
|
|
rm $ORGANISMS/DEM/asterGdem/N59to81_E60to99/USGS_ErosDEM_N59to81E60to99/E060N90_Clipped
|
| 307 |
|
|
|
| 308 |
6cc19306
|
Jim Regetz
|
# remove gdal-generated metadata file
|
| 309 |
|
|
rm $ORGANISMS/DEM/usgsGTOPO30/e020n90/*.aux.xml
|
| 310 |
ff302b98
|
Jim Regetz
|
# remove Mac OS X file metadata cruft
|
| 311 |
|
|
find $ORGANISMS/DEM/usgsGTOPO30 -name ".DS_Store" -exec rm {} \;
|
| 312 |
|
|
find $ORGANISMS/DEM/usgsGTOPO30 -name "._*" -exec rm {} \;
|
| 313 |
|
|
# now migrate original (?) GTOPO30 data
|
| 314 |
6cc19306
|
Jim Regetz
|
mv $ORGANISMS/DEM/usgsGTOPO30 $LAYERS/data/terrain/dem-usgs-gtopo30
|
| 315 |
ff302b98
|
Jim Regetz
|
# confirmed that we had identical copies of the above GTOPO DEM data
|
| 316 |
|
|
# # ---- bash code ---#
|
| 317 |
|
|
# for i in `find $ORGANISMS/DEM/asterGdem -name "*.DEM" -exec basename {} \;`
|
| 318 |
|
|
# do
|
| 319 |
|
|
# diff -q `find $ORGANISMS/DEM/asterGdem -name $i` \
|
| 320 |
|
|
# `find $LAYERS/data/terrain/dem-usgs-gtopo30 -name $i`
|
| 321 |
|
|
# done
|
| 322 |
|
|
# # ------------------#
|
| 323 |
|
|
# (and in fact manually confirmed that the containing directories themselves
|
| 324 |
|
|
# were duplicated, albeit with different names)
|
| 325 |
|
|
# ...so remove the duplicates...
|
| 326 |
|
|
find $ORGANISMS/DEM/asterGdem -name "*.DEM" | sed 's/\/[EW].*//' | xargs -I+ rm -r +
|
| 327 |
6cc19306
|
Jim Regetz
|
|
| 328 |
|
|
#
|
| 329 |
|
|
# GMTED2010
|
| 330 |
|
|
#
|
| 331 |
|
|
|
| 332 |
|
|
# remove gdal-generated metadata files
|
| 333 |
|
|
rm $ORGANISMS/GMTED2010/*.aux.xml
|
| 334 |
|
|
# remove ArcMap(?) schema lock file
|
| 335 |
|
|
rm $ORGANISMS/GMTED2010/GMTED2010_Spatial_Metadata/GMTED2010_Spatial_Metadata.shp.IGSKMNCNWK00726.6284.8844.sr.lock
|
| 336 |
|
|
# migrate original (I think) downloaded data (OR only)
|
| 337 |
|
|
mkdir $LAYERS/data/terrain/dem-gmted2010-mea300-OR-tiles
|
| 338 |
|
|
mv $ORGANISMS/GMTED2010/30N150W_20101117_gmted_mea300.tif \
|
| 339 |
|
|
$LAYERS/data/terrain/dem-gmted2010-mea300-OR-tiles/
|
| 340 |
|
|
mv $ORGANISMS/GMTED2010/30N120W_20101117_gmted_mea300.tif \
|
| 341 |
|
|
$LAYERS/data/terrain/dem-gmted2010-mea300-OR-tiles/
|
| 342 |
|
|
# migrate merged OR tiles (i think)
|
| 343 |
|
|
mkdir $LAYERS/data/terrain/dem-gmted2010-mea300-OR-mosaic
|
| 344 |
|
|
mv $ORGANISMS/GMTED2010/OR_Coverage.tif \
|
| 345 |
|
|
$LAYERS/data/terrain/dem-gmted2010-mea300-OR-mosaic/
|
| 346 |
|
|
# migrate OR clipped/reprojected data
|
| 347 |
|
|
mkdir $LAYERS/data/terrain/dem-gmted2010-mea300-OR-clip-proj-sinu-tif
|
| 348 |
|
|
mv $ORGANISMS/GMTED2010/OR_ClippedToMODIS_InSinu.tif \
|
| 349 |
|
|
$LAYERS/data/terrain/dem-gmted2010-mea300-OR-clip-proj-sinu-tif/
|
| 350 |
|
|
mkdir $LAYERS/data/terrain/dem-gmted2010-mea300-OR-clip-proj-sinu-bil
|
| 351 |
|
|
mv $ORGANISMS/GMTED2010/OR_ClippedToMODIS_InSinu.{prj,hdr,bil} \
|
| 352 |
|
|
$LAYERS/data/terrain/dem-gmted2010-mea300-OR-clip-proj-sinu-bil/
|
| 353 |
|
|
# migrate shapefile
|
| 354 |
|
|
mv $ORGANISMS/GMTED2010/GMTED2010_Spatial_Metadata \
|
| 355 |
|
|
$LAYERS/data/terrain/dem-gmted2010-spatial-metadata
|
| 356 |
|
|
# migrate documentation
|
| 357 |
|
|
mv $ORGANISMS/GMTED2010/of2011-1073.pdf $LAYERS/documentation/terrain/gmted2010-report.pdf
|
| 358 |
|
|
mv $ORGANISMS/GMTED2010/ReadMe $LAYERS/documentation/terrain/gmted2010-readme.txt
|
| 359 |
|
|
# remove now-empty directory
|
| 360 |
|
|
rmdir $ORGANISMS/GMTED2010
|
| 361 |
|
|
|
| 362 |
|
|
#
|
| 363 |
|
|
# Nunokawa content
|
| 364 |
|
|
#
|
| 365 |
|
|
|
| 366 |
|
|
# TODO: deal with data
|
| 367 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_82N.tif # Int16 59N-82N GDEM2 elevation
|
| 368 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_below.tif # Int16 59N-60N GDEM2 elevation
|
| 369 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_above.tif # Int16 60N-61N GDEM2 elevation
|
| 370 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_straddle.tif # Int16 59N-61N GDEM2 elevation (GDEM2 above, GDEM2 below)
|
| 371 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_straddle_a.tif # Flt32 59N-61N GDEM2-based aspect
|
| 372 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_straddle_s.tif # Flt32 59N-61N GDEM2-based slope
|
| 373 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/aster2_*_below_blendgau.tif # Flt32 59N-60N GDEM2/SRTM blended elevation
|
| 374 |
|
|
|
| 375 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_straddle.tif # Int16 59N-61N GDEM2/SRTM unblended elevation (GDEM2 above, SRTM below)
|
| 376 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_straddle_a.tif # Flt32 59N-61N unblended-based aspect
|
| 377 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_straddle_s.tif # Flt32 59N-61N unblended-based aspect
|
| 378 |
|
|
|
| 379 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_blendgau.tif # Int16 59N-61N GDEM2/SRTM blended elevation (GDEM2 above, blended below)
|
| 380 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_blendgau_a.tif # Flt32 59N-61N blend-based aspect
|
| 381 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/fused_*_blendgau_s.tif # Flt32 59N-61N blend-based aspect
|
| 382 |
|
|
|
| 383 |
|
|
$ORGANISMS/DEM/Yuni/Data/srtm/srtm_*_below.tif # Int16 59N-60N SRTM elevation
|
| 384 |
|
|
$ORGANISMS/DEM/Yuni/Data/srtm/srtm_*_below_a.tif # Flt32 59N-60N SRTM-based aspect
|
| 385 |
|
|
$ORGANISMS/DEM/Yuni/Data/srtm/srtm_*_below_below_s.tif # Flt32 59N-60N SRTM-based slope
|
| 386 |
|
|
|
| 387 |
|
|
$ORGANISMS/DEM/Yuni/Data/aster2/aster.vrt # VRT - GDEM2 from ~59N-82N (with 1/2-pixel offset)
|
| 388 |
|
|
$ORGANISMS/DEM/Yuni/Data/srtm/srtm.vrt # VRT - SRTM from ~55N-60N (with 1/2-pixel offset)
|
| 389 |
|
|
|
| 390 |
|
|
# remove duplicate file
|
| 391 |
|
|
rm $ORGANISMS/DEM/Yuni/Data/aster2/fused_w180w141
|
| 392 |
|
|
# remove redundant backup files (but leaving some others)
|
| 393 |
|
|
rm $ORGANISMS/DEM/Yuni/scripts/toAnalyzeData/aspect.r~
|
| 394 |
|
|
rm $ORGANISMS/DEM/Yuni/scripts/toAnalyzeData/check.r~
|
| 395 |
|
|
rm $ORGANISMS/DEM/Yuni/scripts/toAnalyzeData/meanElv.r~
|
| 396 |
|
|
rm $ORGANISMS/DEM/Yuni/scripts/toAnalyzeData/meanElv_onlyN59.r~
|
| 397 |
|
|
rm $ORGANISMS/DEM/Yuni/scripts/toAnalyzeData/
|
| 398 |
|
|
rm $ORGANISMS/DEM/Yuni/scripts/toProduceData/assembling.r~
|
| 399 |
|
|
rm $ORGANISMS/DEM/Yuni/scripts/toProduceData/gaussian.r~
|
| 400 |
|
|
rm $ORGANISMS/DEM/Yuni/scripts/toProduceData/mkVrt_Tiff.r~
|
| 401 |
|
|
rm $ORGANISMS/DEM/Yuni/scripts/toProduceData/slope_aspect.r~
|
| 402 |
|
|
# migrate scripts files
|
| 403 |
|
|
mv $ORGANISMS/DEM/Yuni/scripts \
|
| 404 |
|
|
$LAYERS/code/terrain/nunokawa-scripts
|
| 405 |
|
|
# migrate documents
|
| 406 |
|
|
mv -i $ORGANISMS/DEM/Yuni/documents \
|
| 407 |
|
|
$LAYERS/documentation/terrain/nunokawa-documents
|
| 408 |
|
|
mv -i $ORGANISMS/DEM/Yuni/missing_gdem2_tiles.txt \
|
| 409 |
|
|
$LAYERS/documentation/terrain/nunokawa-documents/
|
| 410 |
|
|
mv -i $ORGANISMS/DEM/Yuni/metadata.txt \
|
| 411 |
|
|
$LAYERS/documentation/terrain/nunokawa-documents/
|
| 412 |
|
|
|
| 413 |
|
|
#
|
| 414 |
|
|
# Remaining DEM content
|
| 415 |
|
|
#
|
| 416 |
|
|
|
| 417 |
|
|
mv $ORGANISMS/DEM/DEM_ProcessingScripts $LAYERS/code/terrain/
|
| 418 |
|
|
mv $ORGANISMS/DEM/ReadMeInProgress.txt $LAYERS/documentation/terrain/misc-partial-readme.txt
|
| 419 |
|
|
|
| 420 |
|
|
|
| 421 |
|
|
rm $ORGANISMS/DEM/checklog
|
| 422 |
|
|
mv $ORGANISMS/DEM/checkBadAsterGDEMFiles.sh $LAYERS/cruft/
|
| 423 |
|
|
mv $ORGANISMS/DEM/CheckGDEMLog.txt $LAYERS/cruft/
|
| 424 |
|
|
|
| 425 |
|
|
|
| 426 |
|
|
#=========#
|
| 427 |
|
|
# CLIMATE #
|
| 428 |
|
|
#=========#
|
| 429 |
|
|
|
| 430 |
|
|
#
|
| 431 |
|
|
# CRU
|
| 432 |
|
|
#
|
| 433 |
|
|
|
| 434 |
|
|
# migrate original gz data from CRU (or so it appears...)
|
| 435 |
|
|
mkdir $LAYERS/data/climate/cru-ts-3.0-1901-2006-gzip
|
| 436 |
|
|
mv $ORGANISMS/CRU_TS_3.0/*.gz $LAYERS/data/climate/cru-ts-3.0-1901-2006-gzip/
|
| 437 |
|
|
# I confirmed that the gunzipped files are identical to the gz contents (by
|
| 438 |
|
|
# gunzipping to /tmp and then diffing):w
|
| 439 |
|
|
# ...so let's delete the uncompressed versions
|
| 440 |
|
|
rm $ORGANISMS/CRU_TS_3.0/cru_ts_3_00.1901.2006.pre.nc
|
| 441 |
|
|
rm $ORGANISMS/CRU_TS_3.0/cru_ts_3_00.1901.2006.tmn.nc
|
| 442 |
|
|
rm $ORGANISMS/CRU_TS_3.0/cru_ts_3_00.1901.2006.tmp.nc
|
| 443 |
|
|
rm $ORGANISMS/CRU_TS_3.0/cru_ts_3_00.1901.2006.tmx.nc
|
| 444 |
|
|
# migrate extracted ASCII grid files
|
| 445 |
|
|
mv $ORGANISMS/CRU_TS_3.0/PRE $LAYERS/data/climate/cru-ts-3.0-1967-2006-asc-pre
|
| 446 |
|
|
mv $ORGANISMS/CRU_TS_3.0/TMN $LAYERS/data/climate/cru-ts-3.0-1967-2006-asc-tmn
|
| 447 |
|
|
mv $ORGANISMS/CRU_TS_3.0/TMP $LAYERS/data/climate/cru-ts-3.0-1967-2006-asc-tmp
|
| 448 |
|
|
mv $ORGANISMS/CRU_TS_3.0/TMX $LAYERS/data/climate/cru-ts-3.0-1967-2006-asc-tmx
|
| 449 |
|
|
# migrate R script for asc extraction
|
| 450 |
|
|
mv -i $ORGANISMS/CRU_TS_3.0/R/cru_3.0_data_extract.r $LAYERS/code/climate/
|
| 451 |
|
|
# rename then migrate readmes
|
| 452 |
|
|
rename 's/(Read_Me)/clim-cru-ts-3.0-\1/' $ORGANISMS/CRU_TS_3.0/Read_Me_*.txt
|
| 453 |
|
|
mv $ORGANISMS/CRU_TS_3.0/*Read_Me_*.txt $LAYERS/documentation/climate/
|
| 454 |
|
|
# now remove empty directory structure
|
| 455 |
|
|
find $ORGANISMS/CRU_TS_3.0 -depth -type d -exec rmdir {} \;
|
| 456 |
|
|
|
| 457 |
|
|
#
|
| 458 |
|
|
# GSOD
|
| 459 |
|
|
#
|
| 460 |
|
|
|
| 461 |
|
|
# for now, just move the whole thing as-is
|
| 462 |
|
|
mv $ORGANISMS/gsod $LAYERS/data/climate/gsod
|
| 463 |
|
|
|
| 464 |
|
|
#
|
| 465 |
|
|
# GHCN
|
| 466 |
|
|
#
|
| 467 |
|
|
|
| 468 |
|
|
# for now, just move the whole thing as-is
|
| 469 |
|
|
mv $ORGANISMS/ghcn $LAYERS/data/climate/ghcn
|
| 470 |
|
|
|
| 471 |
|
|
#
|
| 472 |
|
|
# MODIS LST
|
| 473 |
|
|
#
|
| 474 |
|
|
|
| 475 |
|
|
# migrate MOD11A1 V4 daily 1km L3 LST (March 1-7, 2000)
|
| 476 |
|
|
mv $ORGANISMS/MODIS\ LST/MOD11A1.004 $LAYERS/data/climate/MOD11A1.004-daily-1km-L3-LST/
|
| 477 |
|
|
# migrate MOD11A2 V4 8-day 1km L3 LST (March 5, 2000)
|
| 478 |
|
|
mkdir $LAYERS/data/climate/MOD11A2.004-8day-1km-L3-LST
|
| 479 |
|
|
mv $ORGANISMS/MODIS\ LST/2000.03.05 $LAYERS/data/climate/MOD11A2.004-8day-1km-L3-LST/
|
| 480 |
|
|
# also remove identical copies of the MOD11A2 data (confirmed by diffing)
|
| 481 |
|
|
rm $ORGANISMS/Modis_2000.03.05/MOD11A2.A2000065.*.hdf
|
| 482 |
|
|
rm $ORGANISMS/Modis_2000.03.05/MOD11A2.A2000065.*.hdf.xml
|
| 483 |
|
|
rmdir $ORGANISMS/Modis_2000.03.05
|
| 484 |
|
|
# remove now-empty directory
|
| 485 |
|
|
rmdir $ORGANISMS/MODIS\ LST
|
| 486 |
|
|
|
| 487 |
|
|
#
|
| 488 |
|
|
# MODIS LST -- Oregon
|
| 489 |
|
|
#
|
| 490 |
|
|
|
| 491 |
|
|
# migrate original MODIS OR hdfs (2001-2010?)
|
| 492 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-orig
|
| 493 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/MOD11A1.*.hdf \
|
| 494 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-orig/
|
| 495 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/MOD11A1.*.hdf.xml \
|
| 496 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-orig/
|
| 497 |
|
|
# migrate original MODIS OR hdfs (2000, not used)
|
| 498 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-orig-2000
|
| 499 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/2000\ _WithheldFromAVG/MOD11A1.A2000*.hdf \
|
| 500 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-orig-2000/
|
| 501 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/2000\ _WithheldFromAVG/MOD11A1.A2000*.hdf.xml \
|
| 502 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-orig-2000/
|
| 503 |
|
|
# migrate ClearDayCov OR img extracts (2001-2010?)
|
| 504 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-img
|
| 505 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/MOD11A1.*.img \
|
| 506 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-img/
|
| 507 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/MOD11A1.*.img.aux.xml \
|
| 508 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-img/
|
| 509 |
|
|
# migrate ClearDayCov OR img extracts (2000, not used)
|
| 510 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-img-2000
|
| 511 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/2000_WithheldFromAVG/MOD11A1.A2000*.005_ClearDay.img \
|
| 512 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-img-2000/
|
| 513 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/2000_WithheldFromAVG/MOD11A1.A2000*.005_ClearDay.img.aux.xml \
|
| 514 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-img-2000/
|
| 515 |
|
|
|
| 516 |
|
|
# migrate renamed ClearDayCov OR img extracts (2001-2010?)
|
| 517 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-img-renamed
|
| 518 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/ByDate/*.img \
|
| 519 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-img-renamed/
|
| 520 |
|
|
# migrate renamed ClearDayCov OR img extracts (2000, not used)
|
| 521 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-img-renamed-2000
|
| 522 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/ByDate/2000_WithheldFromAVG/MOD11A1.A.2000.*.img \
|
| 523 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-img-renamed-2000/
|
| 524 |
|
|
|
| 525 |
|
|
# migrate renamed ClearDayCov OR img extracts with corrected pixels
|
| 526 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-img-renamed-fixed
|
| 527 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/ByDate/BadPixelsCorrected_Tiles/*.img \
|
| 528 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-img-renamed-fixed/
|
| 529 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/ByDate/BadPixelsCorrected_Tiles/*.img.aux.xml \
|
| 530 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-img-renamed-fixed/
|
| 531 |
|
|
|
| 532 |
|
|
# migrate ClearDayCov OR daily avg tiles
|
| 533 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-tiles-img
|
| 534 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Tiles/day*.img \
|
| 535 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-tiles-img/
|
| 536 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Tiles/day*.img.aux.xml \
|
| 537 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-tiles-img/
|
| 538 |
|
|
# migrate ClearDayCov OR daily avg mosaic imgs
|
| 539 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-mosaics-img
|
| 540 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Mosaics/Day*_Mosaic_DailyAvg_Scaled.img \
|
| 541 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-mosaics-img
|
| 542 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Mosaics/Day*_Mosaic_DailyAvg_Scaled.img.aux.xml \
|
| 543 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-mosaics-img
|
| 544 |
|
|
# migrate ClearDayCov OR daily avg mosaic bils
|
| 545 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-mosaics-bil
|
| 546 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Mosaics/Day*_Mosaic_DailyAvg_Scaled.prj \
|
| 547 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-mosaics-bil/
|
| 548 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Mosaics/Day*_Mosaic_DailyAvg_Scaled.bil \
|
| 549 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-mosaics-bil/
|
| 550 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Mosaics/Day*_Mosaic_DailyAvg_Scaled.bil.aux.xml \
|
| 551 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-mosaics-bil/
|
| 552 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Mosaics/Day*_Mosaic_DailyAvg_Scaled.hdr \
|
| 553 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-mosaics-bil/
|
| 554 |
|
|
|
| 555 |
|
|
# migrate ClearDayCov OR monthly avg tiles
|
| 556 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-monthlyavg-tiles-img
|
| 557 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Tiles/*_AVG_Scaled.img \
|
| 558 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-monthlyavg-tiles-img/
|
| 559 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Tiles/*_AVG_Scaled.img.aux.xml \
|
| 560 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-monthlyavg-tiles-img/
|
| 561 |
|
|
# migrate ClearDayCov OR monthly avg mosaic imgs
|
| 562 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-monthlyavg-mosaics-img
|
| 563 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Mosaics/*Mosaic_MonthlyAvg_Scaled.img \
|
| 564 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-monthlyavg-mosaics-img/
|
| 565 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Mosaics/*Mosaic_MonthlyAvg_Scaled.img.aux.xml \
|
| 566 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-monthlyavg-mosaics-img/
|
| 567 |
|
|
# migrate ClearDayCov OR monthly avg mosaic bils
|
| 568 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-monthlyavg-mosaics-bil
|
| 569 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Mosaics/*Mosaic_MonthlyAvg_Scaled.prj \
|
| 570 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-monthlyavg-mosaics-bil/
|
| 571 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Mosaics/*Mosaic_MonthlyAvg_Scaled.bil \
|
| 572 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-monthlyavg-mosaics-bil/
|
| 573 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Mosaics/*Mosaic_MonthlyAvg_Scaled.bil.aux.xml \
|
| 574 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-monthlyavg-mosaics-bil/
|
| 575 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Mosaics/*Mosaic_MonthlyAvg_Scaled.hdr \
|
| 576 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-monthlyavg-mosaics-bil/
|
| 577 |
|
|
|
| 578 |
|
|
# migrate *old* (wrong?) ClearDayCov OR daily avg tiles
|
| 579 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-tiles-img-OLD
|
| 580 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_DailyAverage_Tiles_Old/Day_*_Average_Scaled_*.img \
|
| 581 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-tiles-img-OLD/
|
| 582 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_DailyAverage_Tiles_Old/Day_*_Average_Scaled_*.img.aux.xml \
|
| 583 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-tiles-img-OLD/
|
| 584 |
|
|
# migrate *old* (wrong?) ClearDayCov OR daily avg mosaics
|
| 585 |
|
|
mkdir $LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-mosaics-img-OLD
|
| 586 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_DailyAverage_Mosaic_Old/Day_*_ClearDay_Average.img \
|
| 587 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-mosaics-img-OLD/
|
| 588 |
|
|
mv $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_DailyAverage_Mosaic_Old/Day_*_ClearDay_Average.img.aux.xml \
|
| 589 |
|
|
$LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-mosaics-img-OLD/
|
| 590 |
|
|
|
| 591 |
|
|
# remove identical copies of (old) ClearDayCov daily avgs for tile h08v04
|
| 592 |
|
|
# *Note*: I first confirmed that these are duplicate copies:
|
| 593 |
|
|
# for i in {1..366};
|
| 594 |
|
|
# do
|
| 595 |
|
|
# diff -q $ORGANISMS/MODIS_LST_Oregon/DailyAvg1_old/Day_"$i"_Average_Scaled.img \
|
| 596 |
|
|
# $LAYERS/data/climate/MOD11A1.004-OR-clearday-dailyavg-tiles-img-OLD/Day_"$i"_Average_Scaled_h08v04.img
|
| 597 |
|
|
# done
|
| 598 |
|
|
rm $ORGANISMS/MODIS_LST_Oregon/DailyAvg1_old/Day_{1..366}_Average_Scaled.img
|
| 599 |
|
|
rm $ORGANISMS/MODIS_LST_Oregon/DailyAvg1_old/Day_{1..366}_Average_Scaled.img.aux.xml
|
| 600 |
|
|
# remove now-empty directory
|
| 601 |
|
|
rmdir $ORGANISMS/MODIS_LST_Oregon/DailyAvg1_old
|
| 602 |
|
|
|
| 603 |
|
|
# remove misc bil statistics (stx) files)
|
| 604 |
|
|
rm $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Mosaics/Day11_Mosaic_DailyAvg_Scaled.stx
|
| 605 |
|
|
rm $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Mosaics/Day1_Mosaic_DailyAvg_Scaled.stx
|
| 606 |
|
|
rm $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Mosaics/AprMosaic_MonthlyAvg_Scaled.stx
|
| 607 |
|
|
|
| 608 |
|
|
# remove pyc file
|
| 609 |
|
|
rm $ORGANISMS/MODIS_LST_Oregon/GDAL_Extract_ClearDay_from_HDF_Files.pyc
|
| 610 |
|
|
|
| 611 |
|
|
# move and (if needed) rename documentation files
|
| 612 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/1\ ReadMe\ First.txt \
|
| 613 |
|
|
$LAYERS/documentation/climate/clim-MOD11A1.004-OR-orig-readme.txt
|
| 614 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/MODIS_Oregon_ProcessingInfo.doc \
|
| 615 |
|
|
$LAYERS/documentation/climate/
|
| 616 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/2000\ _WithheldFromAVG/ReadMe \
|
| 617 |
|
|
$LAYERS/documentation/climate/clim-MOD11A1.004-OR-orig-2000-readme.txt
|
| 618 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/1\ ReadMe\ First.txt \
|
| 619 |
|
|
$LAYERS/documentation/climate/clim-MOD11A1.004-OR-clearday-readme.txt
|
| 620 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/ByDate/BadPixelsCorrected_Tiles/1\ ReadMe\ First \
|
| 621 |
|
|
$LAYERS/documentation/climate/clim-MOD11A1.004-OR-clearday-img-renamed-fixed-readme.txt
|
| 622 |
|
|
|
| 623 |
|
|
# move scripts
|
| 624 |
|
|
mkdir $LAYERS/code/climate/modis-lst-oregon
|
| 625 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/GDAL_Extract_ClearDay_from_HDF_Files.py \
|
| 626 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 627 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/QCTagCheck \
|
| 628 |
|
|
$LAYERS/code/climate/modis-lst-oregon/QCTagCheck.py
|
| 629 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ConvertMODIStobil.py \
|
| 630 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 631 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Tiles/DayAVGCalcs_h08v04TIles.r \
|
| 632 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 633 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Tiles/DayAVGCalcs_h09v04TIles.r \
|
| 634 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 635 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_DailyAvgs_Mosaics/LST_Mosaicing_DailyAvgs.r \
|
| 636 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 637 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Mosaics/LST_Mosaicing_MonthlyAvgs.R \
|
| 638 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 639 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Mosaics/Conversion_hdf_to_img.py \
|
| 640 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 641 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/ByDate/BadPixelsCorrected_Tiles/h09v04Tiles_BadPixelCorrection.r \
|
| 642 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 643 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/ByDate/BadPixelsCorrected_Tiles/h08v04Tiles_BadPixelCorrection.r \
|
| 644 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 645 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/ClearDay_Original_IMG_Extracts/Convert\ MODIS\ Day\ of\ Year\ to\ Date\ File\ Name.py \
|
| 646 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 647 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Tiles/MonthlyAvgCalcs_h08v04Tiles.r \
|
| 648 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 649 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/ClearDayGDAL/NewClearDay_MonthlyAvgs_Tiles/MonthlyAvgCalcs_h09v04Tiles.r \
|
| 650 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 651 |
|
|
|
| 652 |
|
|
# move last few things...
|
| 653 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/hdf.txt \
|
| 654 |
|
|
$LAYERS/cruft/clim-MOD11A1.004-OR-orig-hdf.txt
|
| 655 |
|
|
mv -i $ORGANISMS/MODIS_LST_Oregon/SDS_PctFills.txt \
|
| 656 |
|
|
$LAYERS/cruft/clim-MOD11A1.004-OR-orig-SDS_PctFills.txt
|
| 657 |
|
|
|
| 658 |
|
|
# remove all the now-empty directories...
|
| 659 |
|
|
# tree $ORGANISMS/MODIS_LST_Oregon
|
| 660 |
|
|
# /home/organisms/MODIS_LST_Oregon
|
| 661 |
|
|
# .
|
| 662 |
|
|
# ├── 2000 _WithheldFromAVG
|
| 663 |
|
|
# └── ClearDayGDAL
|
| 664 |
|
|
# ├── ClearDay_DailyAverage_Mosaic_Old
|
| 665 |
|
|
# ├── ClearDay_DailyAverage_Tiles_Old
|
| 666 |
|
|
# ├── ClearDay_Original_IMG_Extracts
|
| 667 |
|
|
# │  ├── 2000_WithheldFromAVG
|
| 668 |
|
|
# │  └── ByDate
|
| 669 |
|
|
# │  ├── 2000_WithheldFromAVG
|
| 670 |
|
|
# │  └── BadPixelsCorrected_Tiles
|
| 671 |
|
|
# ├── NewClearDay_DailyAvgs_Mosaics
|
| 672 |
|
|
# ├── NewClearDay_DailyAvgs_Tiles
|
| 673 |
|
|
# ├── NewClearDay_MonthlyAvgs_Mosaics
|
| 674 |
|
|
# └── NewClearDay_MonthlyAvgs_Tiles
|
| 675 |
|
|
find $ORGANISMS/MODIS_LST_Oregon -depth -type d -exec rmdir {} \;
|
| 676 |
|
|
|
| 677 |
|
|
# last but not least, deal with top level R directory
|
| 678 |
|
|
# ...move some (old? superceded?) scripts
|
| 679 |
|
|
mv -i $ORGANISMS/R/ReClass Clear Day Rasters h08v04.r \
|
| 680 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 681 |
|
|
mv -i $ORGANISMS/R/Calc Clear Day Daily Avg.r \
|
| 682 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 683 |
|
|
mv -i $ORGANISMS/R/Mosaic Clear Day Average Grids.r \
|
| 684 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 685 |
|
|
mv -i $ORGANISMS/R/Calc Clear Day Monthly Avg.r \
|
| 686 |
|
|
$LAYERS/code/climate/modis-lst-oregon/
|
| 687 |
|
|
# don't know how this was created, but it's clearly derived data...
|
| 688 |
|
|
mv -i $ORGANISMS/R/Modis_LSTdaily_climatology $LAYERS/data/clim-modis-lstdaily
|
| 689 |
|
|
# remove now-empty directory
|
| 690 |
|
|
rmdir $ORGANISMS/R
|
| 691 |
|
|
|
| 692 |
|
|
#
|
| 693 |
|
|
# TRMM
|
| 694 |
|
|
#
|
| 695 |
|
|
|
| 696 |
|
|
# this was an *empty* directory! removing...
|
| 697 |
|
|
rmdir $ORGANISMS/TRMM
|
| 698 |
|
|
|
| 699 |
|
|
#
|
| 700 |
|
|
# WorldClim
|
| 701 |
|
|
#
|
| 702 |
|
|
|
| 703 |
|
|
# migrate each one into its own directory
|
| 704 |
|
|
mkdir $LAYERS/data/climate/worldclim-bio-30s-esri-zip
|
| 705 |
|
|
mv $ORGANISMS/WorldClim/bio_30s_esri.zip $LAYERS/data/climate/worldclim-bio-30s-esri-zip/
|
| 706 |
|
|
mkdir $LAYERS/data/climate/worldclim-prec-30s-esri-zip
|
| 707 |
|
|
mv $ORGANISMS/WorldClim/prec_30s_esri.zip $LAYERS/data/climate/worldclim-prec-30s-esri-zip/
|
| 708 |
|
|
mkdir $LAYERS/data/climate/worldclim-tmax-30s-esri-zip
|
| 709 |
|
|
mv $ORGANISMS/WorldClim/tmax_30s_esri.zip $LAYERS/data/climate/worldclim-tmax-30s-esri-zip/
|
| 710 |
|
|
mkdir $LAYERS/data/climate/worldclim-tmean-30s-esri-zip
|
| 711 |
|
|
mv $ORGANISMS/WorldClim/tmean_30s_esri.zip $LAYERS/data/climate/worldclim-tmean-30s-esri-zip/
|
| 712 |
|
|
mkdir $LAYERS/data/climate/worldclim-tmin-30s-esri-zip
|
| 713 |
|
|
mv $ORGANISMS/WorldClim/tmin_30s_esri.zip $LAYERS/data/climate/worldclim-tmin-30s-esri-zip/
|
| 714 |
|
|
# remove now-empty directory
|
| 715 |
|
|
rmdir $ORGANISMS/WorldClim
|
| 716 |
|
|
|
| 717 |
|
|
|
| 718 |
|
|
#============#
|
| 719 |
|
|
# LAND COVER #
|
| 720 |
|
|
#============#
|
| 721 |
|
|
|
| 722 |
|
|
#
|
| 723 |
|
|
# GLCNMO
|
| 724 |
|
|
#
|
| 725 |
|
|
|
| 726 |
|
|
# for now, just move the whole thing as-is
|
| 727 |
|
|
mv $ORGANISMS/GLCNMO $LAYERS/experimental/land-cover/
|
| 728 |
|
|
|
| 729 |
|
|
#
|
| 730 |
|
|
# GlobCover
|
| 731 |
|
|
#
|
| 732 |
|
|
# migrate what seems like the original data (?)
|
| 733 |
|
|
mkdir $LAYERS/data/land-cover/GlobCover-v2.2
|
| 734 |
|
|
mv -i $ORGANISMS/GlobCoverv2.2/GLOBCOVER_200412_200606_V2.2_Global_CLA* \
|
| 735 |
|
|
$LAYERS/data/land-cover/GlobCover-v2.2/
|
| 736 |
|
|
# migrate mysterious derivatives of the above
|
| 737 |
|
|
mkdir $LAYERS/data/land-cover/GlobCover-v2.2-arcgis
|
| 738 |
|
|
mv -i $ORGANISMS/GlobCoverv2.2/globcov22_beh $LAYERS/data/land-cover/GlobCover-v2.2-arcgis/
|
| 739 |
|
|
mv -i $ORGANISMS/GlobCoverv2.2/globcoverv22 $LAYERS/data/land-cover/GlobCover-v2.2-arcgis/
|
| 740 |
|
|
# migrate reference info to documentation area
|
| 741 |
|
|
mv -i $ORGANISMS/GlobCoverv2.2/Globcover_* $LAYERS/documentation/land-cover/
|
| 742 |
|
|
# remove now-empty directory
|
| 743 |
|
|
rmdir $ORGANISMS/GlobCoverv2.2
|
| 744 |
|
|
|
| 745 |
|
|
#
|
| 746 |
|
|
# LandCover (whatever that is)
|
| 747 |
|
|
#
|
| 748 |
|
|
|
| 749 |
|
|
# remove gdal-generated metadata file
|
| 750 |
|
|
rm $ORGANISMS/LandCover/Layer10_Clip1.tif.aux.xml
|
| 751 |
|
|
# migrate original OR landcover tiles
|
| 752 |
|
|
mkdir $LAYERS/data/land-cover/lc-OR-tiles
|
| 753 |
|
|
mv $ORGANISMS/LandCover/Layer*_Clip[1-3].tif \
|
| 754 |
|
|
$LAYERS/data/land-cover/lc-OR-tiles/
|
| 755 |
|
|
# migrate OR landcover mosaics
|
| 756 |
|
|
mkdir $LAYERS/data/land-cover/lc-OR-mosaic
|
| 757 |
|
|
mv $ORGANISMS/LandCover/Layer*_Mosaic.tif \
|
| 758 |
|
|
$LAYERS/data/land-cover/lc-OR-mosaic/
|
| 759 |
|
|
# migrate OR clipped/reprojected landcover mosaics
|
| 760 |
|
|
mkdir $LAYERS/data/land-cover/lc-OR-mosaic-clip-proj-sinu
|
| 761 |
|
|
mv $ORGANISMS/LandCover/Layer*_ClippedToMODIS_Sinu.* \
|
| 762 |
|
|
$LAYERS/data/land-cover/lc-OR-mosaic-clip-proj-sinu/
|
| 763 |
|
|
# migrate OR multiband clipped/reprojected landcover mosaic
|
| 764 |
|
|
mkdir $LAYERS/data/land-cover/lc-OR-mosaic-clip-proj-sinu-multiband
|
| 765 |
|
|
mv $ORGANISMS/LandCover/Final_Multiband_SinuGrid.* \
|
| 766 |
|
|
$LAYERS/data/land-cover/lc-OR-mosaic-clip-proj-sinu-multiband/
|
| 767 |
|
|
# migrate documentation
|
| 768 |
|
|
mv $ORGANISMS/LandCover/ReadMe $LAYERS/documentation/land-cover/landcover-ReadMe.txt
|
| 769 |
|
|
mv $ORGANISMS/LandCover/LCT\ calculations.xlsx $LAYERS/documentation/land-cover/
|
| 770 |
|
|
mv $ORGANISMS/LandCover/landcover_method_v3.docx $LAYERS/documentation/land-cover/
|
| 771 |
|
|
# migrate scripts
|
| 772 |
|
|
mv $ORGANISMS/LandCover/CheckForNoDataValues_LandCoverFiles.txt $LAYERS/code/land-cover/
|
| 773 |
|
|
# remove now-empty directory
|
| 774 |
|
|
rmdir $ORGANISMS/LandCover
|
| 775 |
|
|
|
| 776 |
|
|
#
|
| 777 |
|
|
# Tree height
|
| 778 |
|
|
#
|
| 779 |
|
|
|
| 780 |
|
|
# migrate $ORGANISMS/Tree_height/Lefsky2010
|
| 781 |
|
|
mkdir $LAYERS/data/land-cover/treeheight-lefsky2010
|
| 782 |
|
|
mv $ORGANISMS/Tree_height/Lefsky2010/x2_080809_global.img \
|
| 783 |
|
|
$LAYERS/data/land-cover/treeheight-lefsky2010/
|
| 784 |
|
|
mv $ORGANISMS/Tree_height/Lefsky2010/p_080809_global.img* \
|
| 785 |
|
|
$LAYERS/data/land-cover/treeheight-lefsky2010/
|
| 786 |
|
|
# migrate associated publication do documentation area
|
| 787 |
|
|
mv $ORGANISMS/Tree_height/Lefsky2010/Lefsky\ et\ al.\ 2010\ GeoRLett.pdf \
|
| 788 |
|
|
$LAYERS/documentation/land-cover/treeheight-Lefsky2010-GeoRLett.pdf
|
| 789 |
|
|
# remove now-empty directory
|
| 790 |
|
|
rmdir $ORGANISMS/Tree_height/Lefsky2010
|
| 791 |
|
|
|
| 792 |
|
|
# migrate $ORGANISMS/Tree_height/Simard2011
|
| 793 |
|
|
mkdir $LAYERS/data/land-cover/treeheight-simard2011
|
| 794 |
|
|
mv $ORGANISMS/Tree_height/Simard2011/Simard_Pinto_3DGlobalVeg_JGR.tif \
|
| 795 |
|
|
$LAYERS/data/land-cover/treeheight-simard2011/
|
| 796 |
|
|
# remove gdal-generated metadata file
|
| 797 |
|
|
rm $ORGANISMS/Tree_height/Simard2011/Simard_Pinto_3DGlobalVeg_JGR.tif.aux.xml
|
| 798 |
|
|
# migrate clipped/reprojected file (bil)
|
| 799 |
|
|
mkdir $LAYERS/data/land-cover/treeheight-simard2011-OR-clip-proj-sinu-bil
|
| 800 |
|
|
mv $ORGANISMS/Tree_height/Simard2011/GlobalCanopy_ClippedToMODIS_InSinu.* \
|
| 801 |
|
|
$LAYERS/data/land-cover/treeheight-simard2011-OR-clip-proj-sinu-bil/
|
| 802 |
|
|
# migrate documentation
|
| 803 |
|
|
mv $ORGANISMS/Tree_height/Simard2011/SimardEtAl2011_GlobalTreeHeight_WithLIDAR.pdf \
|
| 804 |
|
|
$LAYERS/documentation/land-cover/treeheight-SimardEtAl_GlobalTreeHeight_WithLIDAR.pdf
|
| 805 |
|
|
mv $ORGANISMS/Tree_height/Simard2011/ReadMe \
|
| 806 |
|
|
$LAYERS/documentation/land-cover/treeheight-ReadMe.txt
|
| 807 |
|
|
# remove remaining files as per instructions from Natalie Robinson
|
| 808 |
|
|
rm $ORGANISMS/Tree_height/Simard2011/Test_Clip.tif
|
| 809 |
|
|
rm $ORGANISMS/Tree_height/Simard2011/GDALTranslate_test.*
|
| 810 |
|
|
rm $ORGANISMS/Tree_height/Simard2011/GDALTranslate_DT_int16.*
|
| 811 |
|
|
# remove now-empty directory
|
| 812 |
|
|
rmdir $ORGANISMS/Tree_height/Simard2011
|
| 813 |
|
|
|
| 814 |
|
|
# lastely, remove now-empty Tree_height directory
|
| 815 |
|
|
rmdir $ORGANISMS/Tree_height
|
| 816 |
|
|
|
| 817 |
|
|
|
| 818 |
|
|
#=================#
|
| 819 |
|
|
# Remaining stuff #
|
| 820 |
|
|
#=================#
|
| 821 |
|
|
|
| 822 |
|
|
#
|
| 823 |
|
|
# top level directory
|
| 824 |
|
|
#
|
| 825 |
|
|
|
| 826 |
|
|
# move top level notes
|
| 827 |
|
|
mv $ORGANISMS/Oregon_CaseStudy_FileProcessingInfo $LAYERS/documentation/
|
| 828 |
|
|
mv $ORGANISMS/ReProjection_To_MODIS $LAYERS/documentation/
|
| 829 |
|
|
mv $ORGANISMS/README.txt $LAYERS/documentation/organisms-homedir-readme.txt
|
| 830 |
|
|
mv $ORGANISMS/CHANGES.txt $LAYERS/documentation/organisms-homedir-changes.txt
|
| 831 |
|
|
# move old file extension summary
|
| 832 |
|
|
mv -i $ORGANISMS/file-extension-summary.txt $LAYERS/cruft/
|
| 833 |
|
|
|
| 834 |
|
|
#
|
| 835 |
|
|
# Desktop
|
| 836 |
|
|
#
|
| 837 |
|
|
|
| 838 |
|
|
# move MRT installer to cruft
|
| 839 |
|
|
mv $ORGANISMS/Desktop/MRT_download_Linux $LAYERS/cruft/
|
| 840 |
|
|
# remove orphaned bil metadata file
|
| 841 |
|
|
rm $ORGANISMS/Desktop/ASTER_Test.bil.aux.xml
|
| 842 |
|
|
# remove empty Desktop dir
|
| 843 |
|
|
rmdir $ORGANISMS/Desktop
|
| 844 |
|
|
|
| 845 |
|
|
#
|
| 846 |
|
|
# pyhdf
|
| 847 |
|
|
#
|
| 848 |
|
|
|
| 849 |
|
|
# move pyhdf package to cruft
|
| 850 |
|
|
mv $ORGANISMS/pyhdf $LAYERS/cruft/
|
| 851 |
|
|
|
| 852 |
|
|
#
|
| 853 |
|
|
# GIS/GDD
|
| 854 |
|
|
#
|
| 855 |
|
|
|
| 856 |
|
|
# confirmed that /tmean contains unzipped tmean, except there are additional
|
| 857 |
|
|
# aux files presumably autogenerated by Arc -- I'm find with deleting them
|
| 858 |
|
|
# # ---- bash code ---#
|
| 859 |
|
|
# cd /tmp && unzip $ORGANISMS/GIS/GDD/tmean_30s_esri.zip
|
| 860 |
|
|
# diff -qr /tmp/tmean /home/organisms/GIS/GDD/tmean_30s_esri/tmean/
|
| 861 |
|
|
# done
|
| 862 |
|
|
# # ------------------#
|
| 863 |
|
|
# delete the uncompressed tmean_30s_esri directory
|
| 864 |
|
|
rm -r $ORGANISMS/GIS/GDD/tmean_30s_esri
|
| 865 |
|
|
|
| 866 |
|
|
# confirmed that WorldClim zips are identical to what we already have elsewhere
|
| 867 |
|
|
# # ---- bash code ---#
|
| 868 |
|
|
# for var in {'prec','tmin','tmax','tmean','bio'}
|
| 869 |
|
|
# do
|
| 870 |
|
|
# diff -s $ORGANISMS/GIS/GDD/"$var"_30s_esri.zip \
|
| 871 |
|
|
# $LAYERS/data/climate/worldclim-"$var"-30s-esri-zip/*.zip
|
| 872 |
|
|
# done
|
| 873 |
|
|
# done
|
| 874 |
|
|
# # ------------------#
|
| 875 |
|
|
# delete the duplicate worldclim zips
|
| 876 |
|
|
rm $ORGANISMS/GIS/GDD/prec_30s_esri.zip
|
| 877 |
|
|
rm $ORGANISMS/GIS/GDD/tmin_30s_esri.zip
|
| 878 |
|
|
rm $ORGANISMS/GIS/GDD/tmax_30s_esri.zip
|
| 879 |
|
|
rm $ORGANISMS/GIS/GDD/tmean_30s_esri.zip
|
| 880 |
|
|
rm $ORGANISMS/GIS/GDD/bio_30s_esri.zip
|
| 881 |
|
|
|
| 882 |
|
|
# confirmed that all MODIS files here are identical to (though in fact they are
|
| 883 |
|
|
# only a subset of) what we already have elsewhere
|
| 884 |
|
|
# # ---- bash code ---#
|
| 885 |
|
|
# for dir in `ls $ORGANISMS/GIS/GDD/MOD11A1.004`
|
| 886 |
|
|
# do
|
| 887 |
|
|
# echo checking "$dir"...
|
| 888 |
|
|
# ls $ORGANISMS/GIS/GDD/MOD11A1.004/"$dir" \
|
| 889 |
|
|
# | xargs -I+ \
|
| 890 |
|
|
# diff -q $ORGANISMS/GIS/GDD/MOD11A1.004/"$dir"/+ \
|
| 891 |
|
|
# $LAYERS/data/climate/MOD11A1.004-daily-1km-L3-LST/"$dir"/+
|
| 892 |
|
|
# done
|
| 893 |
|
|
# # ------------------#
|
| 894 |
|
|
# delete the duplicate MODIS data
|
| 895 |
|
|
# note that the last 3 dirs were already empty...
|
| 896 |
|
|
rm $ORGANISMS/GIS/GDD/MOD11A1.004/2000.03.01/MOD11A1.A2000*.{hdf,hdf.xml}
|
| 897 |
|
|
rmdir $ORGANISMS/GIS/GDD/MOD11A1.004/2000.03.01
|
| 898 |
|
|
rm $ORGANISMS/GIS/GDD/MOD11A1.004/2000.03.02/MOD11A1.A2000*.{hdf,hdf.xml}
|
| 899 |
|
|
rmdir $ORGANISMS/GIS/GDD/MOD11A1.004/2000.03.02
|
| 900 |
|
|
rm $ORGANISMS/GIS/GDD/MOD11A1.004/2000.03.03/MOD11A1.A2000*.{hdf,hdf.xml}
|
| 901 |
|
|
rmdir $ORGANISMS/GIS/GDD/MOD11A1.004/2000.03.03
|
| 902 |
|
|
rm $ORGANISMS/GIS/GDD/MOD11A1.004/2000.03.04/MOD11A1.A2000*.{hdf,hdf.xml}
|
| 903 |
|
|
rmdir $ORGANISMS/GIS/GDD/MOD11A1.004/2000.03.04
|
| 904 |
|
|
rmdir $ORGANISMS/GIS/GDD/MOD11A1.004/2000.03.05
|
| 905 |
|
|
rmdir $ORGANISMS/GIS/GDD/MOD11A1.004/2000.03.06
|
| 906 |
|
|
rmdir $ORGANISMS/GIS/GDD/MOD11A1.004/2000.03.07
|
| 907 |
|
|
rmdir $ORGANISMS/GIS/GDD/MOD11A1.004
|
| 908 |
|
|
|
| 909 |
|
|
# migrate remaining text (script?) file
|
| 910 |
|
|
mv $ORGANISMS/GIS/GDD/map\ algebra.txt $LAYERS/code/climate/gdd-worldclim-tmean-map-algebra.txt
|
| 911 |
|
|
# remove empty directory
|
| 912 |
|
|
rmdir $ORGANISMS/GIS/GDD
|
| 913 |
|
|
|
| 914 |
|
|
#
|
| 915 |
|
|
# GIS/terrain
|
| 916 |
|
|
#
|
| 917 |
|
|
|
| 918 |
|
|
# remove empty test directory
|
| 919 |
|
|
rmdir $ORGANISMS/GIS/terrain/test
|
| 920 |
|
|
# for now just migrate everything else to experimental area
|
| 921 |
|
|
mv $ORGANISMS/GIS/terrain $LAYERS/experimental/terrain/arcgis
|
| 922 |
|
|
|
| 923 |
|
|
# remove empty directory
|
| 924 |
|
|
rmdir $ORGANISMS/GIS
|
| 925 |
|
|
|
| 926 |
|
|
#
|
| 927 |
|
|
# Oregon
|
| 928 |
|
|
#
|
| 929 |
|
|
|
| 930 |
|
|
# TODO
|
| 931 |
|
|
|
| 932 |
|
|
#
|
| 933 |
|
|
# steph
|
| 934 |
|
|
#
|
| 935 |
|
|
|
| 936 |
|
|
# delete as per verbal instructions from Stephanie Pau
|
| 937 |
|
|
rm -rf $ORGANISMS/steph
|
| 938 |
|
|
|
| 939 |
|
|
#
|
| 940 |
|
|
# topo
|
| 941 |
|
|
#
|
| 942 |
|
|
|
| 943 |
|
|
# for now just migrate to experimental area
|
| 944 |
|
|
mv $ORGANISMS/topo $LAYERS/experimental/
|
| 945 |
|
|
|
| 946 |
|
|
#
|
| 947 |
|
|
# temp_benoit
|
| 948 |
|
|
#
|
| 949 |
|
|
|
| 950 |
|
|
# for now just migrate to experimental area
|
| 951 |
|
|
mv $ORGANISMS/temp_benoit $LAYERS/experimental/
|
| 952 |
|
|
|
| 953 |
|
|
|
| 954 |
|
|
#=======================================================================
|
| 955 |
|
|
# fix up permissions and ownership
|
| 956 |
|
|
#=======================================================================
|
| 957 |
|
|
|
| 958 |
|
|
chgrp -R layers $LAYERS/data
|
| 959 |
|
|
find $LAYERS/data -type f -exec chmod 640 {} \;
|
| 960 |
|
|
find $LAYERS/data -type d -exec chmod g-s {} \;
|
| 961 |
|
|
find $LAYERS/data -type d -exec chmod 750 {} \;
|
| 962 |
7526fb1c
|
Jim Regetz
|
|
| 963 |
|
|
|
| 964 |
|
|
#=======================================================================
|
| 965 |
|
|
# now migrate code into git repository clone
|
| 966 |
|
|
#=======================================================================
|
| 967 |
|
|
|
| 968 |
|
|
# nunokawa terrain scripts
|
| 969 |
|
|
mkdir terrain/research/gtopo30
|
| 970 |
|
|
mv -i code/terrain/nunokawa-scripts/toProduceData/clipUSGS.r terrain/research/gtopo30/
|
| 971 |
|
|
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/check.r terrain/tests/
|
| 972 |
|
|
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/meanElv_OnlyN59.r terrain/research/north-60/
|
| 973 |
|
|
rm code/terrain/nunokawa-scripts/toAnalyzeData/meanElv.r
|
| 974 |
|
|
rm code/terrain/nunokawa-scripts/toAnalyzeData/Deltas.r~
|
| 975 |
|
|
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/Deltas.r terrain/research/north-60/
|
| 976 |
|
|
rm code/terrain/nunokawa-scripts/toAnalyzeData/rmse_cor.r~
|
| 977 |
|
|
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/rmse_cor.r terrain/research/north-60/
|
| 978 |
|
|
rm code/terrain/nunokawa-scripts/toAnalyzeData/slope.r~
|
| 979 |
|
|
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/{aspect,slope}.r terrain/research/north-60/
|
| 980 |
|
|
mv -i code/terrain/nunokawa-scripts/toAnalyzeData/negativeTable.r terrain/research/north-60/
|
| 981 |
|
|
mv -i code/terrain/nunokawa-scripts/toProduceData/*.{r,r~} terrain/research/north-60/
|
| 982 |
|
|
|
| 983 |
|
|
# robinson terrain scripts
|
| 984 |
|
|
mv -i code/terrain/DEM_ProcessingScripts/Aster_CheckMosaicedTilesExtents.py terrain/tests/
|
| 985 |
|
|
mv -i code/terrain/DEM_ProcessingScripts/CheckPixelValuesAtOverlapZones.txt terrain/tests/
|
| 986 |
|
|
mv -i code/terrain/DEM_ProcessingScripts/Aster\&SRTM_* terrain/tests/
|
| 987 |
|
|
mv -i code/terrain/DEM_ProcessingScripts/Gaussian_Blend.r terrain/procedures/
|
| 988 |
|
|
mv -i terrain/tests/Aster\&SRTM_* terrain/procedures/
|
| 989 |
|
|
mv -i code/terrain/DEM_ProcessingScripts/SRTM_ClipToN59to60.txt terrain/procedures/
|
| 990 |
|
|
mv -i code/terrain/DEM_ProcessingScripts/Mosaicing_AllTiles_East\&WestHemispheres.txt terrain/procedures/
|
| 991 |
|
|
mv -i code/terrain/DEM_ProcessingScripts/AsterMosaicingScripts terrain/procedures/
|
| 992 |
|
|
|
| 993 |
|
|
# robinson land-cover check
|
| 994 |
|
|
mv -i code/land-cover/CheckForNoDataValues_LandCoverFiles.txt land-cover/tests/
|
| 995 |
|
|
|
| 996 |
|
|
# robinson/donoghue Oregon MODIS LST processing code
|
| 997 |
|
|
mkdir climate/research/oregon
|
| 998 |
|
|
mkdir climate/research/oregon/modis-lst
|
| 999 |
|
|
mv -i code/climate/modis-lst-oregon/*.{r,R,py} climate/research/oregon/modis-lst/
|
| 1000 |
|
|
|
| 1001 |
|
|
# misc old (authorless) climate code
|
| 1002 |
|
|
mv -i code/climate/cru_3.0_data_extract.r climate/procedures/
|
| 1003 |
|
|
mv -i code/climate/gdd-worldclim-tmean-map-algebra.txt climate/extra/
|
| 1004 |
|
|
|
| 1005 |
|
|
# TODO: set all file permissions to 644 before committing
|
