Project

General

Profile

« Previous | Next » 

Revision 9568

added lib/sh/resume_import.sh and use it in inputs/GBIF/_MySQL/MySQL.data.sql.run

View differences:

inputs/GBIF/_MySQL/MySQL.data.sql.run
1 1
#!/bin/bash -e
2 2
. "$(dirname "${BASH_SOURCE[0]}")"/../../../lib/runscripts/table_dir.run
3
. "$(dirname "${BASH_SOURCE[0]}")"/../../../lib/sh/binsearch.sh
3
. "$(dirname "${BASH_SOURCE[0]}")"/../../../lib/sh/resume_import.sh
4 4

  
5 5
if self_not_included; then
6 6

  
7 7

  
8 8
#### import resuming
9 9

  
10

  
11
### is_pkey_imported()
12

  
13 10
table=raw_occurrence_record
14 11
pkey_name=id
15 12

  
16
is_pkey_imported__int() # usage: pkey_name=... pkey=# is_pkey_imported__int
17
{
18
	echo_func; kw_params pkey_name pkey; : "${pkey_name?}" "${pkey?}"
19
	test "$pkey" || { log++ echo_run echo 0; return; }
20
	
21
	mk_table_esc
22
	use_local_remote
23
	data_only=1 mysql_ANSI \
24
<<<"SELECT COUNT(*) FROM $table_esc WHERE $pkey_name = $pkey"
25
}
26

  
27
func_override is_pkey_imported__int__no_cache
28
is_pkey_imported__int() # caches the last result for efficiency
29
{
30
	local cache_key="$(declare -p pkey_name pkey) $*"; load_cache
31
	if ! cached; then save_cache "$(${FUNCNAME}__no_cache "$@")" || return; fi
32
	echo_cached_value
33
}
34

  
35
is_pkey_imported()
36
{ echo_func; local int; int="$(is_pkey_imported__int)"; int2bool "$int"; }
37

  
38

  
39
get_pkey_at_pos() # usage: i=# get_pkey_at_pos
40
{
41
	echo_func; log++; kw_params i; : "${i?}"; mk_table_esc
42
	piped_cmd echo_run tail -c +"$i" "$top_file"\
43
	|echo_run sed -n '/^INSERT INTO '"$table_esc"' VALUES \(([0-9]+),.*$/{
44
s//\1/p
45
q # stop after first match
46
}'
47
}
48

  
49
is_pkey_at_pos_imported()
50
{ echo_func; local pkey; pkey="$(get_pkey_at_pos)"; is_pkey_imported; }
51

  
52
import_resume_pos() # usage: [min=#] [max=#] import_resume_pos
53
{
54
	echo_func; kw_params min max; local min="${min-0}"
55
	if ! isset max; then local max; max="$(file_size "$top_file")"; fi
56
	binsearch is_pkey_at_pos_imported
57
}
58 13
# import_resume_pos() takes 6 min to run, with 37 iterations
59 14

  
60
resume_import() # usage: [pos=#] resume_import
61
{
62
	echo_func; kw_params pos
63
	if ! isset pos; then local pos; pos="$(import_resume_pos)"; fi
64
	
65
	use_local_remote
66
	echo_run tail -c +"$pos" "$top_file"|mysql
67
}
68

  
69 15
fi
lib/sh/resume_import.sh
1
#!/bin/bash -e
2
. "$(dirname "${BASH_SOURCE[0]}")"/util.sh
3
. "$(dirname "${BASH_SOURCE[0]}")"/binsearch.sh
4

  
5
if self_not_included; then
6

  
7

  
8
### is_pkey_imported()
9

  
10
is_pkey_imported__int() # usage: pkey_name=... pkey=# is_pkey_imported__int
11
{
12
	echo_func; kw_params pkey_name pkey; : "${pkey_name?}" "${pkey?}"
13
	test "$pkey" || { log++ echo_run echo 0; return; }
14
	
15
	mk_table_esc
16
	use_local_remote
17
	data_only=1 mysql_ANSI \
18
<<<"SELECT COUNT(*) FROM $table_esc WHERE $pkey_name = $pkey"
19
}
20

  
21
func_override is_pkey_imported__int__no_cache
22
is_pkey_imported__int() # caches the last result for efficiency
23
{
24
	local cache_key="$(declare -p pkey_name pkey) $*"; load_cache
25
	if ! cached; then save_cache "$(${FUNCNAME}__no_cache "$@")" || return; fi
26
	echo_cached_value
27
}
28

  
29
is_pkey_imported()
30
{ echo_func; local int; int="$(is_pkey_imported__int)"; int2bool "$int"; }
31

  
32

  
33
get_pkey_at_pos() # usage: i=# get_pkey_at_pos
34
{
35
	echo_func; log++; kw_params i; : "${i?}"; mk_table_esc
36
	piped_cmd echo_run tail -c +"$i" "$top_file"\
37
	|echo_run sed -n '/^INSERT INTO '"$table_esc"' VALUES \(([0-9]+),.*$/{
38
s//\1/p
39
q # stop after first match
40
}'
41
}
42

  
43
is_pkey_at_pos_imported()
44
{ echo_func; local pkey; pkey="$(get_pkey_at_pos)"; is_pkey_imported; }
45

  
46
import_resume_pos() # usage: [min=#] [max=#] import_resume_pos
47
{
48
	echo_func; kw_params min max; local min="${min-0}"
49
	if ! isset max; then local max; max="$(file_size "$top_file")"; fi
50
	binsearch is_pkey_at_pos_imported
51
}
52

  
53
resume_import() # usage: [pos=#] resume_import
54
{
55
	echo_func; kw_params pos
56
	if ! isset pos; then local pos; pos="$(import_resume_pos)"; fi
57
	
58
	use_local_remote
59
	echo_run tail -c +"$pos" "$top_file"|mysql
60
}
61

  
62
fi
0 63

  

Also available in: Unified diff