swh:1:snp:5789d5563ae0ff1fac9d4620526a6e02863df326
Tip revision: be2f5bad92e89df92965388e6030a4b14a11c864 authored by Thomas B. Mooney on 27 August 2019, 19:37:10 UTC
Merge pull request #1941 from apaul7/add_is_10x
Merge pull request #1941 from apaul7/add_is_10x
Tip revision: be2f5ba
genome-env
#!/bin/bash
ORIG_SET="$(set +o | tr '\n' ';')"
set -o errexit
set -o pipefail
function usage {
sed 's/^\ \{4\}//' <<EOS
usage: $THIS [-hsuw] [command args]
See \`$THIS -h' for detailed help.
EOS
}
function help {
sed 's/^\ \{4\}//' <<EOS
NAME - $THIS
DESCRIPTION
===========
\`$THIS' is used as a wrapper command to setup an environment for \`genome'. For most use cases no options need to be specified. For example,
$THIS genome --help
If you know you will need to run several commands in succession you can start a new shell by doing,
$THIS \$SHELL
OPTIONS
=======
-h Display this detailed help.
-M Disable migration of database.
-m sqitch_repo
The path to a genome-sqitch repo to use instead of the default.
-D Disable configuration of database.
-d db_snapshot_name
The name of the database snapshot to use instead of the default.
-i db_id:db_host:db_port
The connection information of an existing database to use. Format is name:host:port
-p Persist the test-db outside of this shell.
-R Disable rebuilding of the Meta DB.
-U Disable configuration of UR.
-u ur_repo
The path to a UR repo to use instead of the default.
-t Enable testing mode.
EOS
}
function main {
local TOP
if $GIT rev-parse --is-inside-work-tree 1> /dev/null 2> /dev/null
then
TOP="$($GIT rev-parse --show-toplevel)"
else
TOP="$PWD"
fi
local DISABLE_DB DISABLE_MIGRATIONS DISABLE_UR DISABLE_METADB_REBUILD \
DB_SNAPSHOT_NAME DB_ID DB_HOST DB_PORT SQITCH_REPO UR_REPO \
TESTING_MODE=""
parse_opts "$@"
shift $((OPTIND-1))
prepare_submodules
validate_bash
if test -z "$DISABLE_METADB_REBUILD"
then
rebuild_meta_db
fi
local TESTDBSERVER_DB_NAME TESTDBSERVER_DB_HOST TESTDBSERVER_DB_PORT
if test -z "$DISABLE_DB"
then
if test -n "$DB_ID"
then
export TESTDBSERVER_DB_NAME=$DB_ID
export TESTDBSERVER_DB_HOST=$DB_HOST
export TESTDBSERVER_DB_PORT=$DB_PORT
else
setup_test_db "$TOP"
fi
fi
setup_env "$TOP"
log.info ''
log.info "Genome Perl: $(genome-perl -e 'printf(qq(%vd\n), $^V)')"
log.info "Genome Prove: $(genome-prove --version)"
for MODULE in UR Genome
do
log.info "${MODULE}: $(genome-perl -M$MODULE -e "print \$INC{q(${MODULE}.pm)}, qq(\n)")"
done
if test -z "$DISABLE_DB"
then
if test -n "$DB_ID"
then
log.info "Using existing test DB"
else
log.info "Test DB Snapshot: $DB_SNAPSHOT_NAME"
fi
log.info "Test DB Name: $TESTDBSERVER_DB_NAME"
log.info "Test DB Host: ${TESTDBSERVER_DB_HOST}"
log.info "Test DB Port: ${TESTDBSERVER_DB_PORT}"
fi
log.info ''
if test -n "$TESTING_MODE"
then
# Have to use `genome config get` after setup_env.
eval "$(genome config set-env log_usage 0)"
eval "$(genome config set-env sys_services_files_url 'file:///')"
eval "$(genome config set-env test_url "$(genome config get sys_services_files_url)$(genome config get test_inputs)")"
eval "$(genome config set-env testing 1)"
unset WF_USE_FLOW
export UR_DBI_NO_COMMIT=1
export GE_ENV_NAME="genome-test-env"
fi
if test "$#" -gt 0
then
log.info "$@"...
eval "$ORIG_SET"
"$@"
EXIT=$?
else
echo 'Press <Ctrl>-D or type `exit` to return to normal environment.'
echo ''
unset PROMPT_COMMAND
export PS1="(${GE_ENV_NAME}) \w $ "
eval "$ORIG_SET"
"$BASH" --norc --noprofile
EXIT=$?
fi
if test -z "$DISABLE_DB" -a -z "$PERSIST_DB"
then
cleanup_test_db
fi
exit $EXIT
}
function validate_bash {
if test "$#" -eq 0 -a -z "$BASH"
then
fatal $DEP_ERR 'only `bash` shells are supported at this time'
fi
}
function parse_opts {
DISABLE_DB="" DISABLE_MIGRATIONS="" DISABLE_UR="" DB_SNAPSHOT_NAME="" DB_ID="" DB_HOST="" DB_PORT="" SQITCH_REPO="" UR_REPO="" PERSIST_DB="" DISABLE_METADB_REBUILD=""
local opts
while getopts :hMm:Dd:i:pRUu:Ww:t opts "$@"
do
case $opts in
h)
help
exit 0
;;
M)
DISABLE_MIGRATIONS="TRUE"
;;
m)
SQITCH_REPO="$OPTARG"
;;
D)
DISABLE_DB="TRUE"
DISABLE_MIGRATIONS="TRUE"
;;
d)
DB_SNAPSHOT_NAME="$OPTARG"
;;
i)
args=(${OPTARG//:/ })
DB_ID=${args[0]}
DB_HOST=${args[1]}
DB_PORT=${args[2]}
PERSIST_DB="TRUE"
;;
p) PERSIST_DB="TRUE"
;;
R) DISABLE_METADB_REBUILD="TRUE"
;;
U)
DISABLE_UR="TRUE"
;;
u)
UR_REPO="$OPTARG"
;;
t)
TESTING_MODE=1
;;
:)
fatal $OPT_ERR "option -$OPTARG requires an argument"
;;
\?)
fatal $OPT_ERR "illegal option -- $OPTARG"
;;
esac
done
validate_opts
set_defaults
}
function validate_opts {
if test -n "$DISABLE_DB" -a -n "$DB_SNAPSHOT_NAME"
then
fatal $OPT_ERR "cannot use -D and -d options together"
fi
if test -n "$DISABLE_DB" -a -n "$DB_ID"
then
fatal $OPT_ERR "cannot use -D and -i options together"
fi
if test -n "$DB_SNAPSHOT_NAME" -a -n "$DB_ID"
then
fatal $OPT_ERR "cannot use -d and -i options together"
fi
if test -n "$DISABLE_UR" -a -n "$UR_REPO"
then
fatal $OPT_ERR "cannot use -U and -u options together"
fi
if test -n "$DISABLE_MIGRATIONS" -a -n "$SQITCH_REPO"
then
fatal $OPT_ERR "cannot use -M and -m options together"
fi
if test -n "$DB_SNAPSHOT_NAME"
then
if ! $TESTDB template list | cut -f 1 | $GREP -q "^${DB_SNAPSHOT_NAME}$"
then
fatal $OPT_ERR "invalid snapshot name -- $DB_SNAPSHOT_NAME"
fi
fi
if test -n "$DB_ID"
then
if ! $TESTDB database list | cut -f 1 | $GREP -q "^${DB_ID}$"
then
fatal $OPT_ERR "invalid database name -- $DB_ID"
fi
fi
if test -n "$SQITCH_REPO"
then
assert_dir "-m" "$SQITCH_REPO"
fi
if test -n "$UR_REPO"
then
assert_dir "-u" "$UR_REPO"
fi
}
function assert_dir {
if ! test -d "$2"
then
fatal $OPT_ERR "$1 value must be a directory -- $2"
fi
}
declare -a SUBMODULES=()
function append_submodule {
if test "${#SUBMODULES[@]}" -gt 0
then
SUBMODULES=("${SUBMODULES[@]}" "$@")
else
SUBMODULES=("$@")
fi
}
function set_defaults {
if test -z "$DISABLE_MIGRATIONS" -a -z "$SQITCH_REPO"
then
local SQITCH_GENOME='sqitch/genome'
append_submodule $SQITCH_GENOME
SQITCH_REPO="$TOP/$SQITCH_GENOME"
fi
if test -z "$DB_SNAPSHOT_NAME" -a -z "$DB_ID"
then
DB_SNAPSHOT_NAME="b06f70c"
fi
if test -z "$UR_REPO" -a -z "$DISABLE_UR"
then
if test -z "$TOP" -o ! -d "$TOP/ur"
then
fatal $OPT_ERR "-u must be specified if not in a Genome repo"
fi
append_submodule ur
UR_REPO="$TOP/ur"
fi
}
DEP_ERR=1 # 1 = dependency error
OPT_ERR=2 # 2 = option error
BUILD_ERR=3 # 3 = build error
function fatal {
local EXIT="$1"
shift
log.error "$@"
exit $EXIT
}
function log.error {
echo "$THIS: $@" >&2
}
function log.info {
echo "$@" 1>&2
}
# __quiet_git only show Git output if there Git exits non-zero so that we can
# provide "pretty" output around the Git commands.
function __quiet_git {
(
set +o errexit
local OUT="$($GIT "$@" 2>&1)"
local GIT_EXIT=$?
if test $GIT_EXIT -ne 0
then
fatal $BUILD_ERR -e "$OUT"
fi
)
}
# prepare_submodules tries to (essentially) do a fast-forward update on
# submodules when the submodules are used as the default repo.
function prepare_submodules {
if test "${#SUBMODULES}" -eq 0
then
return
fi
(
log.info -e "\n=> Preparing Submodules..."
cd "$($GIT rev-parse --show-cdup)"
__quiet_git submodule sync "${SUBMODULES[@]}"
for submodule in "${SUBMODULES[@]}"
do
log.info -n " $submodule: "
if $GIT submodule status $submodule | $GREP -q ^-
then
__quiet_git submodule update --init $submodule
log.info "OK"
else
(
local EXPECTED_HEAD=$($GIT ls-tree HEAD $submodule | awk '{print $3}')
cd $submodule
local HEAD=$($GIT rev-parse HEAD)
if ! $GIT cat-file -e $EXPECTED_HEAD
then
__quiet_git fetch --all
fi
# Essentialy trying to do a fast-forward only update but
# since submodules are not branch-based we can't just do
# a `git pull --ff-only`.
if ! $GIT status --porcelain | $GREP -q . \
&& $GIT merge-base --is-ancestor $HEAD $EXPECTED_HEAD
then
cd - 1> /dev/null
__quiet_git submodule update --init $submodule
log.info "OK"
else
fatal $BUILD_ERR "submodule has local changes"
fi
)
fi
done
)
}
# regen_db creates/updates SQLite (binary) database files from ASCII dump files
# that are in source control.
function regen_db {
local DB_TXT="$1"
log.info -n '.'
local DB_BIN=${DB_TXT/-dump/}
if test "$DB_BIN" -ot "$DB_TXT"
then
rm -f "$DB_BIN"
$SQLITE3 "$DB_BIN" < "$DB_TXT"
fi
}
# rebuild_meta_db scans for SQLite ASCII dump files that are (usually) in
# source control and calls regen_db on them.
function rebuild_meta_db {
log.info -en "\n=> Rebuilding Meta DB..."
if $GIT rev-parse --is-inside-work-tree 1> /dev/null 2> /dev/null
then
# ls-files is fast but then we have to also query submodules
# foreach defines $name which serves to prefix the submodule path
( $GIT ls-files \
&& $GIT submodule foreach "$GIT ls-files | sed \"s|^|\$name/|\"" ) \
| $GREP -P '\.sqlite3n?-dump$' \
| while read DB_TXT
do
regen_db "$DB_TXT"
done
else
find "$PWD" -iname '*sqlite3-dump' -or -iname '*sqlite3n-dump' | while read DB_TXT; do
regen_db "$DB_TXT"
done
fi
log.info ' done'
}
# setup_test_db creates an ephemeral test database, runs the latest migrations
# on it, and provides shell variables used to configure GMSchema later.
function setup_test_db {
local BASE_DIR="$1"
echo "=> Creating test database..."
eval "$($TESTDB database create --timeout 30 --bash --owner $TESTDBSERVER_DB_USER --based-on $DB_SNAPSHOT_NAME)"
if test -z "$TESTDBSERVER_DB_NAME"
then
fatal $BUILD_ERR "failed to create test database"
fi
if test -z "$DISABLE_MIGRATIONS"
then
echo -n "=> Migrating test database..."
(
local SQITCH_TMP="$(mktemp -d -t genome-env-sqitch-XXXXX)"
trap "rm -rf \"$SQITCH_TMP\"" EXIT
cp -aT "$SQITCH_REPO" "$SQITCH_TMP"
cd "$SQITCH_TMP"
$SQITCH config core.pg.host $TESTDBSERVER_DB_HOST
$SQITCH config core.pg.username $TESTDBSERVER_DB_USER
$SQITCH config core.pg.password $TESTDBSERVER_DB_PASS
$SQITCH config core.pg.db_name $TESTDBSERVER_DB_NAME
if $SQITCH status | grep -q 'Nothing to deploy'
then
echo " nothing to deploy"
else
echo ""
$SQITCH deploy
fi
)
fi
}
# cleanup_test_db reverses setup_test_db by deleting any new disk allocations
# from the test database, then deletes the test database
function cleanup_test_db {
echo "=> Removing disk allocations created during $THIS..."
UR_DBI_NO_COMMIT=0 genome model admin remove-disk-allocations-from-testdb
test-db delete database $TESTDBSERVER_DB_NAME > /dev/null
}
# assert_module_missing only works with single word module names, e.g. UR and Genome.
function assert_module_missing {
local MODULE=$1
local MODULE_PATH="$(genome-perl -M${MODULE} -e "print \$INC{q($MODULE.pm)}, qq(\\n)" 2> /dev/null )"
if test -n "$MODULE_PATH"
then
fatal $BUILD_ERR "found shadow module: $MODULE ($MODULE_PATH)"
fi
}
# assert_module_found only works with single word module names, e.g. UR and Genome.
function assert_module_found {
local MODULE=$1
local MODULE_PATH="$(genome-perl -M${MODULE} -e "print \$INC{q($MODULE.pm)}, qq(\\n)")"
if test -z "$MODULE_PATH"
then
fatal $BUILD_ERR "module not found: $MODULE"
fi
}
# setup_env tries to clean the current environment and then updates the
# environment to run tests.
function setup_env {
log.info -e "\n=> Setting Up Test Environment..."
local BIN MODULE \
BASE_DIR="$1" \
SNAPSHOT_LIB="/gsc/scripts/opt/genome/current/user/lib/perl" \
SNAPSHOT_BIN="/gsc/scripts/opt/genome/current/user/bin"
# remove deployed bin directory from PATH
PATH="$(echo $PATH | tr : "\n" | $GREP -v "$SNAPSHOT_BIN" | tr '\n' : | sed 's/:$//')"
export PATH=$BASE_DIR/bin:$PATH
# remove deployed lib/perl directory from PATH
export PERL5LIB="$(echo $PERL5LIB | tr : "\n" | $GREP -v "$SNAPSHOT_LIB" | tr '\n' : | sed 's/:$//')"
assert_module_missing Genome
if test -z "$DISABLE_UR"
then
export PATH=$UR_REPO/bin:$PATH
assert_module_missing UR
fi
if which genome | $GREP -q "$SNAPSHOT_BIN"
then
fatal $BUILD_ERR "genome found in $SNAPSHOT_BIN"
fi
if test -z "$DISABLE_UR"
then
export PERL5LIB=$UR_REPO/lib:$PERL5LIB
assert_module_found UR
if which ur | $GREP -q "$SNAPSHOT_BIN"
then
fatal $BUILD_ERR "ur found in $SNAPSHOT_BIN"
fi
fi
export PERL5LIB=$BASE_DIR/lib/perl:$PERL5LIB
assert_module_found Genome
if test -z "$DISABLE_DB"
then
eval "$(genome config set-env ds_gmschema_server "dbname=${TESTDBSERVER_DB_NAME};host=${TESTDBSERVER_DB_HOST};port=${TESTDBSERVER_DB_PORT}")"
eval "$(genome config set-env ds_gmschema_login "$TESTDBSERVER_DB_USER")"
eval "$(genome config set-env ds_gmschema_auth "$TESTDBSERVER_DB_PASS")"
fi
hash -r
}
function default_value {
if test -z $(eval "echo \$$1")
then
eval "$1=\"$2\""
fi
}
THIS="$(basename "$0")"
GIT="$(type -p git)" || fatal $DEP_ERR "git is required"
SQITCH="$(type -p sqitch)" || fatal $DEP_ERR "sqitch is required"
SQLITE3="$(type -p sqlite3)" || fatal $DEP_ERR "sqlite3 is required"
GREP="$(type -p grep)" || fatal $DEP_ERR "grep is required"
TESTDB="$(type -p test-db)" || fatal $DEP_ERR "test-db is required"
default_value TESTDBSERVER_DB_USER "genome"
default_value TESTDBSERVER_DB_PASS "mypassword"
default_value TESTDBSERVER_URL "https://apipe-test-db.gsc.wustl.edu"
default_value GE_ENV_NAME "genome-env"
export TESTDBSERVER_URL
if test -z "$GE_NO_REDIRECT"
then
# delegate to local version if it is found
if $GIT rev-parse --is-inside-work-tree 1> /dev/null 2> /dev/null
then
WORK_TREE="$($GIT rev-parse --show-toplevel)"
NEW_THIS="${WORK_TREE}/bin/${THIS}"
if test -x "$NEW_THIS"
then
export GE_NO_REDIRECT=1
exec "$NEW_THIS" "$@"
fi
fi
fi
set -o nounset
main "$@"