Revision 9568
Added by Aaron Marcuse-Kubitza over 11 years ago
inputs/GBIF/_MySQL/MySQL.data.sql.run | ||
---|---|---|
1 | 1 |
#!/bin/bash -e |
2 | 2 |
. "$(dirname "${BASH_SOURCE[0]}")"/../../../lib/runscripts/table_dir.run |
3 |
. "$(dirname "${BASH_SOURCE[0]}")"/../../../lib/sh/binsearch.sh
|
|
3 |
. "$(dirname "${BASH_SOURCE[0]}")"/../../../lib/sh/resume_import.sh
|
|
4 | 4 |
|
5 | 5 |
if self_not_included; then |
6 | 6 |
|
7 | 7 |
|
8 | 8 |
#### import resuming |
9 | 9 |
|
10 |
|
|
11 |
### is_pkey_imported() |
|
12 |
|
|
13 | 10 |
table=raw_occurrence_record |
14 | 11 |
pkey_name=id |
15 | 12 |
|
16 |
is_pkey_imported__int() # usage: pkey_name=... pkey=# is_pkey_imported__int |
|
17 |
{ |
|
18 |
echo_func; kw_params pkey_name pkey; : "${pkey_name?}" "${pkey?}" |
|
19 |
test "$pkey" || { log++ echo_run echo 0; return; } |
|
20 |
|
|
21 |
mk_table_esc |
|
22 |
use_local_remote |
|
23 |
data_only=1 mysql_ANSI \ |
|
24 |
<<<"SELECT COUNT(*) FROM $table_esc WHERE $pkey_name = $pkey" |
|
25 |
} |
|
26 |
|
|
27 |
func_override is_pkey_imported__int__no_cache |
|
28 |
is_pkey_imported__int() # caches the last result for efficiency |
|
29 |
{ |
|
30 |
local cache_key="$(declare -p pkey_name pkey) $*"; load_cache |
|
31 |
if ! cached; then save_cache "$(${FUNCNAME}__no_cache "$@")" || return; fi |
|
32 |
echo_cached_value |
|
33 |
} |
|
34 |
|
|
35 |
is_pkey_imported() |
|
36 |
{ echo_func; local int; int="$(is_pkey_imported__int)"; int2bool "$int"; } |
|
37 |
|
|
38 |
|
|
39 |
get_pkey_at_pos() # usage: i=# get_pkey_at_pos |
|
40 |
{ |
|
41 |
echo_func; log++; kw_params i; : "${i?}"; mk_table_esc |
|
42 |
piped_cmd echo_run tail -c +"$i" "$top_file"\ |
|
43 |
|echo_run sed -n '/^INSERT INTO '"$table_esc"' VALUES \(([0-9]+),.*$/{ |
|
44 |
s//\1/p |
|
45 |
q # stop after first match |
|
46 |
}' |
|
47 |
} |
|
48 |
|
|
49 |
is_pkey_at_pos_imported() |
|
50 |
{ echo_func; local pkey; pkey="$(get_pkey_at_pos)"; is_pkey_imported; } |
|
51 |
|
|
52 |
import_resume_pos() # usage: [min=#] [max=#] import_resume_pos |
|
53 |
{ |
|
54 |
echo_func; kw_params min max; local min="${min-0}" |
|
55 |
if ! isset max; then local max; max="$(file_size "$top_file")"; fi |
|
56 |
binsearch is_pkey_at_pos_imported |
|
57 |
} |
|
58 | 13 |
# import_resume_pos() takes 6 min to run, with 37 iterations |
59 | 14 |
|
60 |
resume_import() # usage: [pos=#] resume_import |
|
61 |
{ |
|
62 |
echo_func; kw_params pos |
|
63 |
if ! isset pos; then local pos; pos="$(import_resume_pos)"; fi |
|
64 |
|
|
65 |
use_local_remote |
|
66 |
echo_run tail -c +"$pos" "$top_file"|mysql |
|
67 |
} |
|
68 |
|
|
69 | 15 |
fi |
lib/sh/resume_import.sh | ||
---|---|---|
1 |
#!/bin/bash -e |
|
2 |
. "$(dirname "${BASH_SOURCE[0]}")"/util.sh |
|
3 |
. "$(dirname "${BASH_SOURCE[0]}")"/binsearch.sh |
|
4 |
|
|
5 |
if self_not_included; then |
|
6 |
|
|
7 |
|
|
8 |
### is_pkey_imported() |
|
9 |
|
|
10 |
is_pkey_imported__int() # usage: pkey_name=... pkey=# is_pkey_imported__int |
|
11 |
{ |
|
12 |
echo_func; kw_params pkey_name pkey; : "${pkey_name?}" "${pkey?}" |
|
13 |
test "$pkey" || { log++ echo_run echo 0; return; } |
|
14 |
|
|
15 |
mk_table_esc |
|
16 |
use_local_remote |
|
17 |
data_only=1 mysql_ANSI \ |
|
18 |
<<<"SELECT COUNT(*) FROM $table_esc WHERE $pkey_name = $pkey" |
|
19 |
} |
|
20 |
|
|
21 |
func_override is_pkey_imported__int__no_cache |
|
22 |
is_pkey_imported__int() # caches the last result for efficiency |
|
23 |
{ |
|
24 |
local cache_key="$(declare -p pkey_name pkey) $*"; load_cache |
|
25 |
if ! cached; then save_cache "$(${FUNCNAME}__no_cache "$@")" || return; fi |
|
26 |
echo_cached_value |
|
27 |
} |
|
28 |
|
|
29 |
is_pkey_imported() |
|
30 |
{ echo_func; local int; int="$(is_pkey_imported__int)"; int2bool "$int"; } |
|
31 |
|
|
32 |
|
|
33 |
get_pkey_at_pos() # usage: i=# get_pkey_at_pos |
|
34 |
{ |
|
35 |
echo_func; log++; kw_params i; : "${i?}"; mk_table_esc |
|
36 |
piped_cmd echo_run tail -c +"$i" "$top_file"\ |
|
37 |
|echo_run sed -n '/^INSERT INTO '"$table_esc"' VALUES \(([0-9]+),.*$/{ |
|
38 |
s//\1/p |
|
39 |
q # stop after first match |
|
40 |
}' |
|
41 |
} |
|
42 |
|
|
43 |
is_pkey_at_pos_imported() |
|
44 |
{ echo_func; local pkey; pkey="$(get_pkey_at_pos)"; is_pkey_imported; } |
|
45 |
|
|
46 |
import_resume_pos() # usage: [min=#] [max=#] import_resume_pos |
|
47 |
{ |
|
48 |
echo_func; kw_params min max; local min="${min-0}" |
|
49 |
if ! isset max; then local max; max="$(file_size "$top_file")"; fi |
|
50 |
binsearch is_pkey_at_pos_imported |
|
51 |
} |
|
52 |
|
|
53 |
resume_import() # usage: [pos=#] resume_import |
|
54 |
{ |
|
55 |
echo_func; kw_params pos |
|
56 |
if ! isset pos; then local pos; pos="$(import_resume_pos)"; fi |
|
57 |
|
|
58 |
use_local_remote |
|
59 |
echo_run tail -c +"$pos" "$top_file"|mysql |
|
60 |
} |
|
61 |
|
|
62 |
fi |
|
0 | 63 |
Also available in: Unified diff
added lib/sh/resume_import.sh and use it in inputs/GBIF/_MySQL/MySQL.data.sql.run