to test the import:

# -> "to set up the VM"

# restore the "booted" snapshot under "+pg_restore"

# start the VM

ssh -t 128.111.220.240

# update system time so wait times can be calculated correctly:

sudo service ntp stop; sudo ntpdate -s time.nist.gov; sudo service ntp start

* to test a different revision:

# {

# pick [rev] from r12025 (2014-2-2, last success)..r13002 (2014-3-30, 1st failure)

svn revert --depth infinity .

svn up --force -r [rev] # runtime: 1 min ("0m50.945s")

# fix concurrent regressions (which occurred during the revision range):

for patch in r12689 r12760 r12865 r12872 r12994; do

	svn di -c "$patch"|patch --force --fuzz=3 -p 0

done

	# use `patch` instead of `svn merge` because `svn merge` sometimes inexplicably

	# doens't apply patches

	# --fuzz: allow patching even when surrounding lines have changed

# if any of the commands below are broken, try a different revision

yes|make mk_db rm_initial_public schemas/reinstall # mk_db: install new extensions

yes|make inputs/.TNRS/reinstall

yes|make schemas/public/reinstall # sanity check before running the import

# }

$0 # nested shell

export version=test_import

declare -ax inputs; inputs=(inputs/ARIZ/)

. bin/import_all # runtime: 50 min ("0:45:07.105389"+"4:05.03")

# at "This will delete the test_import schema", type y and press ENTER

# determine if the revision works:

# {

# wait 8-10 min

# wait until pg_stat_activity shows an in# query

# see if pg_stat_activity shows the buggy query

#  (https://projects.nceas.ucsb.edu/nceas/issues/902) -> the revision is broken,

#  or an in# query after in#6 -> the revision is OK

# }

# stop the import:

# {

kill % # cancel after_import()

. bin/stop_imports

sudo pkill -f 'postgres: bien vegbien'

# }

to set up the VM:

ln -s /Volumes/vegbiendev/vegbiendev.vdi ~/VirtualBox\ VMs/vegbiendev/vegbiendev.vdi

screen

# press ENTER

l=1 overwrite=1 inplace=1 src=/Volumes/vegbiendev/vegbiendev.vdi dest=~/VirtualBox\ VMs/vegbiendev/vegbiendev.file.vdi put # runtime: 5 h ("4:43:16")

	# move vegbiendev.vdi to the internal hard drive

# press Ctrl-D

(cd ~/VirtualBox\ VMs/vegbiendev/; mv vegbiendev.file.vdi vegbiendev.vdi)

# open VirtualBox

# select the vegbiendev VM

# restore the "+bootloader+VirtualBox Guest Additions+eth1" snapshot

# open Settings > System tab

# in the Motherboard subtab, set Base Memory to 2048 MB

# in the Processor subtab, set Processor(s) to 1

# boot the VM

# when asked whether to skip mounting, press S

ssh -t 128.111.220.240

rm inputs/GBIF/_MySQL/GBIFPortalDB-2013-02-20.data.sql

sed --in-place -r 's/^(shared_buffers = )\S+/\11000MB/' schemas/postgresql.conf

	# = <1 GB; decrease to match VM's allocated memory (2 GB)

sudo mkdir -p /var/lib/postgresql/9.3/main/

sudo chown postgres:postgres /var/lib/postgresql/9.3/main/

sudo -E -u postgres /usr/lib/postgresql/9.3/bin/initdb -D /var/lib/postgresql/9.3/main/

sudo ln -sf /etc/postgresql/9.3/main/*.conf /var/lib/postgresql/9.3/main/

make postgres_reload

# at "Modifying postgresql.conf and pg_hba.conf", type y and press ENTER

# at "kernel.shmmax ... Press ENTER to continue", press ENTER

# at "restart PostgreSQL manually ... Press ENTER to continue":

# open a new window

ssh -t 128.111.220.240

# run what it says

# press Ctrl-D

# return to the previous window

# press ENTER

* to use a database backup:

# {

make mk_db rm_initial_public # do things that are not part of the backup

screen

# press ENTER

l=1 src='frenzy.nceas.ucsb.edu:~/bien/' dest=. put backups/vegbien.r12025.backup # runtime: 25 min ("0:25:24")

# press Ctrl-D

sudo chgrp postgres backups/vegbien.r12025.backup

sudo halt

# at "System halted", close the VM and *uncheck* Restore current snapshot

# create a VM snapshot named "+initdb+database backup file"

# boot the VM

# when asked whether to skip mounting, press S

ssh -t 128.111.220.240

screen

# press ENTER

time for schema in temp util py_util geoscrub herbaria IRMNG IUCN NCBI newWorld TNRS ARIZ; do

	bin/psql_verbose_vegbien <<<"CREATE SCHEMA \"$schema\";"

	"time" sudo -E -u postgres pg_restore --exit-on-error --verbose --dbname=vegbien --schema="$schema" backups/vegbien.r12025.backup

		# need to run as DB superuser to restore py_util

done # runtime: 15 min ("11:44:05" - ~11:30)

	# must use for loop because pg_restore doesn't support multiple --schema

	# options (it just uses the last one)

	# (http://postgresql.1045698.n5.nabble.com/pg-restore-should-accept-multiple-t-switches-td2399138.html)

# press Ctrl-D

sudo halt

# at "System halted", close the VM and *uncheck* Restore current snapshot

# create a VM snapshot named "+pg_restore"

# boot the VM

# when asked whether to skip mounting, press S

# create a VM snapshot named "booted"

# close the VM and *check* Restore current snapshot

# }

* to use a clean database:

# {

(cd inputs/; for dir in IRMNG IUCN newWorld; do mv "$dir" ."$dir"; done)

(cd inputs/; for dir in */; do dir="${dir%/}"; case "$dir" in analytical_db|ARIZ) : ;; *) mv "$dir" .archive/"$dir";; esac; done)

make misc-Linux db inputs/install schemas/public/reinstall # runtime: 15 min ("15:11:16".."15:26:28") @starscream.vegbiendev

# at "This will delete the current public schema of your VegBIEN DB", type y and press ENTER

# }

to install system updates:

# boot the VM

# when asked whether to skip mounting, press S

sudo apt-get update

sudo apt-get dist-upgrade # runtime: 12 min ("14:20:15".."14:31:50") @starscream.vegbiendev

sudo reboot # some system updates require system restart

# when asked whether to skip mounting, press S

sudo halt

# at "System halted", close the VM and *uncheck* Restore current snapshot

# create a VM snapshot named "+system updates"

# boot the VM

# when asked whether to skip mounting, press S

# create a VM snapshot named "booted"

to enable VM to access VirtualBox shared folders:

(currently doesn't work, likely because of a bug in VirtualBox Guest Additions when installed on something other than a clean Linux install)

sudo sed --in-place -r 's/^vboxsf:.*:$/&aaronmk/' /etc/group

# press Ctrl-D

ssh -t 128.111.220.240

to retrieve the original contents of the backup from the VM:

# open VirtualBox

# select the vegbiendev VM

# restore the "+LiveCD+ssh+root pw+mount" snapshot

# **IMPORTANT**: can't use inplace=1 optimization because this messes up the

#  shared hardlinks used by ~/Documents/BIEN/vegbiendev/

# --link-dest: relative to dest, not currdir, so need abs path

l=1 overwrite=1 src=root@128.111.220.198:/mnt/ dest=~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT/ sudo -E put --link-dest="$HOME"/Documents/BIEN/vegbiendev/ --exclude=/home/bien/svn/ --exclude=/home/aaronmk/Dropbox/

	# --exclude=/home/aaronmk/Dropbox/: this is not used by the BIEN scripts

sudo mkdir ~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT/home/bien/svn/

# --exclude of specific files: these have compressed versions

l=1 overwrite=1 src=root@128.111.220.198:/mnt/ dest=~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT/ sudo -E put --link-dest="$HOME"/Documents/BIEN/vegbiendev/home/aaronmk/bien/ --no-owner --no-group --exclude=/backups/.vegbien.r10848.backup.aYCFxI --exclude='/backups/vegbien.*.backup' --exclude=/inputs/GBIF/_archive/Specimen/GBIF.txt --exclude=/inputs/GBIF/_MySQL/GBIFPortalDB-2013-02-20.data.sql --exclude=/inputs/GBIF/raw_occurrence_record_plants/_archive/table.2013-5-13.tsv --exclude=/inputs/GBIF/raw_occurrence_record_plants/table.tsv --exclude=/inputs/VegBank/_archive/2012-8-30/vegbank.sql --exclude='/inputs/FIA/_archive/2011-10-17/Organism/*_Extract.csv' --exclude='*.CSV' --exclude='*.gz' --exclude='*.log.sql' --exclude='*.zip'                                          home/bien/svn/

l=1 overwrite=1 src=root@128.111.220.198:/mnt/ dest=~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT/ sudo -E put --link-dest="$HOME"/Documents/BIEN/vegbiendev/home/aaronmk/bien/ --no-owner --no-group --exclude=/backups/.vegbien.r10848.backup.aYCFxI --exclude='/backups/vegbien.*.backup' --exclude=/inputs/GBIF/_archive/Specimen/GBIF.txt --exclude=/inputs/GBIF/_MySQL/GBIFPortalDB-2013-02-20.data.sql --exclude=/inputs/GBIF/raw_occurrence_record_plants/_archive/table.2013-5-13.tsv --exclude=/inputs/GBIF/raw_occurrence_record_plants/table.tsv --exclude=/inputs/VegBank/_archive/2012-8-30/vegbank.sql --include='/inputs/FIA/_archive/2011-10-17/Organism/*_Extract.csv' --include='*.CSV' --include='*.gz' --include='*.log.sql' --include='*.zip' --include='*/' --exclude='**' --no-perms home/bien/svn/

	# --no-perms: enable --link-dest to work in spite of perms changes (the

	#  perms on these files are not important to the BIEN scripts)

# generate list of all the files whose permissions were changed since the

#  backup, but which are extracted with their changed permissions instead of

#  their original ones in the backup:

l=  overwrite=1 src=root@128.111.220.198:/mnt/ dest=~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT/ sudo -E put --link-dest="$HOME"/Documents/BIEN/vegbiendev/home/aaronmk/bien/ --no-owner --no-group --exclude=/backups/.vegbien.r10848.backup.aYCFxI --exclude='/backups/vegbien.*.backup' --exclude=/inputs/GBIF/_archive/Specimen/GBIF.txt --exclude=/inputs/GBIF/_MySQL/GBIFPortalDB-2013-02-20.data.sql --exclude=/inputs/GBIF/raw_occurrence_record_plants/_archive/table.2013-5-13.tsv --exclude=/inputs/GBIF/raw_occurrence_record_plants/table.tsv --exclude=/inputs/VegBank/_archive/2012-8-30/vegbank.sql home/bien/svn/ >~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT.perms_changed.txt

to retrieve the contents of the VM, with the VirtualBox changes added:

# open VirtualBox

# select the vegbiendev VM

# restore the "(+VirtualBox changes)+root pw" snapshot

# boot the VM

# **IMPORTANT**: can't use inplace=1 optimization because this messes up the

#  shared hardlinks used by ~/Documents/BIEN/vegbiendev/

# --link-dest: relative to dest, not currdir, so need abs path

l=1 overwrite=1 src=root@128.111.220.240:/ dest=~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT.VirtualBox/ sudo -E put --link-dest="$HOME"/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT/ --exclude=/home/bien/svn/ --exclude=/home/aaronmk/Dropbox/

	# --exclude=/home/aaronmk/Dropbox/: this is not used by the BIEN scripts

sudo mkdir ~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT.VirtualBox/home/bien/svn/

# --exclude of specific files: these have compressed versions

l=1 overwrite=1 src=root@128.111.220.240:/ dest=~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT.VirtualBox/ sudo -E put --link-dest="$HOME"/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT/home/bien/svn/ --no-owner --no-group --exclude=/backups/.vegbien.r10848.backup.aYCFxI --exclude='/backups/vegbien.*.backup' --exclude=/inputs/GBIF/_archive/Specimen/GBIF.txt --exclude=/inputs/GBIF/_MySQL/GBIFPortalDB-2013-02-20.data.sql --exclude=/inputs/GBIF/raw_occurrence_record_plants/_archive/table.2013-5-13.tsv --exclude=/inputs/GBIF/raw_occurrence_record_plants/table.tsv --exclude=/inputs/VegBank/_archive/2012-8-30/vegbank.sql --exclude='/inputs/FIA/_archive/2011-10-17/Organism/*_Extract.csv' --exclude='*.CSV' --exclude='*.gz' --exclude='*.log.sql' --exclude='*.zip'                                          home/bien/svn/

l=1 overwrite=1 src=root@128.111.220.240:/ dest=~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT.VirtualBox/ sudo -E put --link-dest="$HOME"/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT/home/bien/svn/ --no-owner --no-group --exclude=/backups/.vegbien.r10848.backup.aYCFxI --exclude='/backups/vegbien.*.backup' --exclude=/inputs/GBIF/_archive/Specimen/GBIF.txt --exclude=/inputs/GBIF/_MySQL/GBIFPortalDB-2013-02-20.data.sql --exclude=/inputs/GBIF/raw_occurrence_record_plants/_archive/table.2013-5-13.tsv --exclude=/inputs/GBIF/raw_occurrence_record_plants/table.tsv --exclude=/inputs/VegBank/_archive/2012-8-30/vegbank.sql --include='/inputs/FIA/_archive/2011-10-17/Organism/*_Extract.csv' --include='*.CSV' --include='*.gz' --include='*.log.sql' --include='*.zip' --include='*/' --exclude='**' --no-perms home/bien/svn/

	# --no-perms: enable --link-dest to work in spite of perms changes (the

	#  perms on these files are not important to the BIEN scripts)

l=  overwrite=1 src=root@128.111.220.240:/ dest=~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT.VirtualBox/ sudo -E put --link-dest="$HOME"/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT/home/bien/svn/ --no-owner --no-group --exclude=/backups/.vegbien.r10848.backup.aYCFxI --exclude='/backups/vegbien.*.backup' --exclude=/inputs/GBIF/_archive/Specimen/GBIF.txt --exclude=/inputs/GBIF/_MySQL/GBIFPortalDB-2013-02-20.data.sql --exclude=/inputs/GBIF/raw_occurrence_record_plants/_archive/table.2013-5-13.tsv --exclude=/inputs/GBIF/raw_occurrence_record_plants/table.tsv --exclude=/inputs/VegBank/_archive/2012-8-30/vegbank.sql home/bien/svn/|diff /dev/stdin ~/Documents/BIEN/vegbiendev.2014-2-2_1-07-32PT.perms_changed.txt 

# ensure there are no file changes

to configure the VM to support VirtualBox:

# open VirtualBox

# select the vegbiendev VM

# restore the "restore of 2014-02-02 01:07:32 backup (DO NOT DELETE)" snapshot

# boot the VM from ~/software/ubuntu-12.04.4-desktop-amd64.iso

# install a bootloader

# shut down the VM

# eject the CD image

# boot the VM

# when asked whether to skip mounting, press S

# install VirtualBox Guest Additions

# set up the network connection:

# {

ssh -t aaronmk@128.111.220.240

sudo nano /etc/network/interfaces

# remove the entry for eth0 (which doesn't work under VirtualBox)

# add the following entry:

---

auto eth1

iface eth1 inet dhcp

---

# }