1
|
#!/bin/bash -e
|
2
|
. "$(dirname "${BASH_SOURCE[0]}")"/../table.run
|
3
|
|
4
|
if self_not_included; then
|
5
|
|
6
|
full_table="${table%_plants}"
|
7
|
|
8
|
herbaria_filter.ih.csv_/make()
|
9
|
{
|
10
|
echo_func; set_make_vars; check_target_exists
|
11
|
|
12
|
use_local
|
13
|
limit= query='SELECT DISTINCT acronym FROM herbaria.ih ORDER BY acronym' \
|
14
|
to_target pg_export
|
15
|
}
|
16
|
|
17
|
ih_herbarium/make() # stores IH herbaria
|
18
|
{
|
19
|
echo_func; set_make_vars
|
20
|
|
21
|
use_local_remote
|
22
|
local table="$target_stem"
|
23
|
if remaking; then mk_drop|mysql; fi
|
24
|
|
25
|
herbaria_filter.ih.csv_/make
|
26
|
(benign_error=1 mysql <<'EOF'
|
27
|
CREATE TABLE `ih_herbarium` (
|
28
|
`institution_code` varchar(255) NOT NULL PRIMARY KEY
|
29
|
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='from IH.vegpath.org?&Index+Herbariorum'
|
30
|
EOF
|
31
|
) || skip_table || return 0
|
32
|
append=1 mysql_import <"$top_dir"/herbaria_filter.ih.csv_
|
33
|
}
|
34
|
|
35
|
plant_fraction/make() # runtime: 1 hr ("10603 rows affected (53 min 44.68 sec)")
|
36
|
{
|
37
|
echo_func; set_make_vars; mk_table_esc
|
38
|
|
39
|
use_local_remote
|
40
|
if remaking; then table=plant_fraction table_esc= mk_drop|mysql; fi
|
41
|
(benign_error=1 mysql <<'EOF'
|
42
|
CREATE TABLE `plant_fraction` (
|
43
|
`institution_code` varchar(255) NOT NULL PRIMARY KEY,
|
44
|
`plant_fraction` float NOT NULL,
|
45
|
KEY `plant_fraction` (`plant_fraction`)
|
46
|
) ENGINE=MyISAM DEFAULT CHARSET=utf8
|
47
|
EOF
|
48
|
) || table=plant_fraction skip_table || return 0
|
49
|
mysql <<EOF
|
50
|
INSERT INTO "plant_fraction"
|
51
|
SELECT
|
52
|
institution_code
|
53
|
, COUNT(NULLIF(
|
54
|
family LIKE '%aceae%' OR family IN ('Compositae','Gramineae','Palmae','Guttiferae','Cruciferae','Labiatae','Umbelliferae','Leguminosae')
|
55
|
, false))/COUNT(*)
|
56
|
AS plant_fraction
|
57
|
FROM ${limit:+(SELECT * FROM }$table_esc ${limit:+$(prefix= limit)) s}
|
58
|
GROUP BY institution_code
|
59
|
ORDER BY institution_code
|
60
|
EOF
|
61
|
}
|
62
|
|
63
|
plant_fraction/seal()
|
64
|
{
|
65
|
echo_func; set_make_vars
|
66
|
|
67
|
use_local_remote
|
68
|
table=plant_fraction table_esc= mysql_seal_table
|
69
|
}
|
70
|
|
71
|
plant_fraction_for_herbaria_filter/make()
|
72
|
{
|
73
|
echo_func; set_make_vars; mk_table_esc
|
74
|
|
75
|
plant_fraction/make
|
76
|
use_local_remote
|
77
|
if remaking; then
|
78
|
table=plant_fraction_for_herbaria_filter table_esc= mk_drop|mysql
|
79
|
fi
|
80
|
(benign_error=1 mysql <<'EOF'
|
81
|
CREATE VIEW `plant_fraction_for_herbaria_filter` AS
|
82
|
SELECT * FROM `plant_fraction`
|
83
|
WHERE plant_fraction >= 0.8
|
84
|
EOF
|
85
|
) || table=plant_fraction_for_herbaria_filter skip_table || return 0
|
86
|
}
|
87
|
|
88
|
herbaria_filter/make() # runtime: ~0 ("0m2.235s"): CSV import, index scans only
|
89
|
{
|
90
|
echo_func; set_make_vars
|
91
|
|
92
|
use_local_remote
|
93
|
local table="$target_stem"
|
94
|
if remaking; then mk_drop|mysql; fi
|
95
|
|
96
|
ih_herbarium/make
|
97
|
plant_fraction_for_herbaria_filter/make
|
98
|
|
99
|
(benign_error=1 mysql <<'EOF'
|
100
|
CREATE TABLE `herbaria_filter` (
|
101
|
`institution_code` varchar(255) NOT NULL PRIMARY KEY
|
102
|
) ENGINE=MyISAM DEFAULT CHARSET=utf8
|
103
|
EOF
|
104
|
) || skip_table || return 0
|
105
|
mysql <<EOF
|
106
|
INSERT IGNORE INTO $table
|
107
|
SELECT institution_code FROM ih_herbarium
|
108
|
;
|
109
|
INSERT IGNORE INTO $table
|
110
|
SELECT institution_code FROM plant_fraction_for_herbaria_filter
|
111
|
;
|
112
|
EOF
|
113
|
}
|
114
|
|
115
|
herbaria_filter/seal()
|
116
|
{
|
117
|
echo_func; set_make_vars
|
118
|
|
119
|
use_local_remote
|
120
|
table=herbaria_filter table_esc= mysql_seal_table
|
121
|
}
|
122
|
|
123
|
table/make() # makes the filter view
|
124
|
{
|
125
|
echo_func; set_make_vars; mk_table_esc
|
126
|
|
127
|
herbaria_filter/make
|
128
|
use_local_remote
|
129
|
if remaking; then mk_drop|mysql; fi
|
130
|
(benign_error=1 mysql <<EOF
|
131
|
CREATE VIEW $table AS
|
132
|
SELECT * FROM $full_table
|
133
|
WHERE deleted IS NULL
|
134
|
AND institution_code IN (SELECT * FROM herbaria_filter)
|
135
|
AND (
|
136
|
family IS NULL
|
137
|
OR (family LIKE '%aceae%' OR family IN ('Compositae','Gramineae','Palmae','Guttiferae','Cruciferae','Labiatae','Umbelliferae','Leguminosae'))
|
138
|
)
|
139
|
EOF
|
140
|
) || skip_table || return 0
|
141
|
}
|
142
|
|
143
|
# table.tsv.gz/make() runtime: 35 min ("34m55.802s")
|
144
|
|
145
|
func_override table.tsv/make__src_table_run
|
146
|
table.tsv/make() # usage: [remake=1] [limit=...] .../run table.tsv/make
|
147
|
# note that incremental output is provided right away with --quick (unbuffered),
|
148
|
# but takes awhile to become visible in Macfusion sshfs
|
149
|
#
|
150
|
# to view runtime when using `screen`:
|
151
|
# press Ctrl-A [ , use up-arrow, and then press Esc to leave copy mode
|
152
|
{
|
153
|
echo_func; set_make_vars; check_target_exists
|
154
|
|
155
|
table/make
|
156
|
self_make table.tsv/make__src_table_run
|
157
|
}
|
158
|
|
159
|
fi
|