1
|
#!/usr/bin/bash
|
2
|
#
|
3
|
# Bash script used to download GHCN data and generate a simple log file
|
4
|
# recording basic info about what was obtained. The downloaded data and
|
5
|
# log file are stored in the directory specified by $DESTDIR below. Note
|
6
|
# that the subdirectory name is hardcoded to reflect the GHCN version in
|
7
|
# effect at the time this script was written and executed; it would
|
8
|
# need be modified if this script is used again in the future, either
|
9
|
# manually in the code or perhaps by parsing the ghcnd-version.txt file
|
10
|
# (if it can be assumed that the structure of this file is consistent
|
11
|
# over time).
|
12
|
#
|
13
|
# Written and executed by Jim Regetz on 29-May-2012, with post-hoc
|
14
|
# tweaking of filesystem metadata to preserve everything as owned by
|
15
|
# 'layers' and read-only to all project collaborators.
|
16
|
|
17
|
FTPDIR="ftp://ftp.ncdc.noaa.gov/pub/data/ghcn/daily"
|
18
|
BASEDIR="/home/layers/data/climate/ghcn"
|
19
|
DESTDIR="$BASEDIR/v2.92-upd-2012052822"
|
20
|
|
21
|
# create destination for data
|
22
|
if [ -d $DESTDIR ]
|
23
|
then
|
24
|
echo "target directory $DESTDIR already exists"
|
25
|
exit 1
|
26
|
fi
|
27
|
mkdir $DESTDIR
|
28
|
|
29
|
# set up log file
|
30
|
LOGFILE="$DESTDIR/download-ghcn-daily.log"
|
31
|
if [ -f $LOGFILE ]
|
32
|
then
|
33
|
echo "first remove existing log file '$LOGFILE'"
|
34
|
exit 1
|
35
|
fi
|
36
|
echo "==========================================
|
37
|
Download log for GHCN daily data
|
38
|
Timestamp: `date`
|
39
|
User: $USER
|
40
|
==========================================" >>$LOGFILE
|
41
|
echo -e "\n" >>$LOGFILE
|
42
|
|
43
|
# log snapshot of current FTP directory listing
|
44
|
echo "Remote directory listing:" >>$LOGFILE
|
45
|
curl "$FTPDIR/" >>$LOGFILE 2>/dev/null
|
46
|
echo -e "\n" >>$LOGFILE
|
47
|
|
48
|
# files we want to get...
|
49
|
FILES=(
|
50
|
"readme.txt"
|
51
|
"status.txt"
|
52
|
"ghcnd-countries.txt"
|
53
|
"ghcnd-inventory.txt"
|
54
|
"ghcnd-states.txt"
|
55
|
"ghcnd-stations.txt"
|
56
|
"ghcnd-version.txt"
|
57
|
"ghcnd_all.tar.gz"
|
58
|
)
|
59
|
|
60
|
# download each file and record some info to log
|
61
|
for file in ${FILES[@]}
|
62
|
do
|
63
|
TARGET=$DESTDIR/$file
|
64
|
if [ -f $TARGET ]
|
65
|
then
|
66
|
echo "$TARGET already exists -- skipping"
|
67
|
else
|
68
|
printf '%72s\n' | tr ' ' '-' >>$LOGFILE
|
69
|
curl -o $TARGET "$FTPDIR/$file" >>$LOGFILE \
|
70
|
-w "URL: %{url_effective}\nSize: %{size_download}\nTime: %{time_total}\n"
|
71
|
echo "Location: $TARGET" >>$LOGFILE
|
72
|
echo "MD5: `md5sum $TARGET | sed 's/ .*//'`" >>$LOGFILE
|
73
|
echo -e "\n" >>$LOGFILE
|
74
|
fi
|
75
|
done
|