https://github.com/genome/genome
Raw File
Tip revision: 8a7edd8a9bb376b4fec1621bd820ba8f651381f6 authored by Thomas B. Mooney on 21 October 2015, 13:14:20 UTC
Merge pull request #1060 from tmooney/stop_testing_58
Tip revision: 8a7edd8
genome-env
#!/bin/bash

ORIG_SET="$(set +o | tr '\n' ';')"

set -o errexit
set -o pipefail

function usage {
    sed 's/^\ \{4\}//' <<EOS
    usage: $THIS [-hsuw] [command args]
        See \`$THIS -h' for detailed help.
EOS
}

function help {
    sed 's/^\ \{4\}//' <<EOS
    NAME - $THIS

    DESCRIPTION
    ===========

    \`$THIS' is used as a wrapper command to setup an environment for \`genome'.  For most use cases no options need to be specified.  For example,

        $THIS genome --help

    If you know you will need to run several commands in succession you can start a new shell by doing,

        $THIS \$SHELL

    OPTIONS
    =======

    -h    Display this detailed help.

    -M    Disable migration of database.

    -m sqitch_repo
          The path to a genome-sqitch repo to use instead of the default.

    -D    Disable configuration of database.

    -d db_snapshot_name
          The name of the database snapshot to use instead of the default.

    -i db_id:db_host:db_port
          The connection information of an existing database to use.  Format is name:host:port

    -p    Persist the test-db outside of this shell.

    -U    Disable configuration of UR.

    -u ur_repo
          The path to a UR repo to use instead of the default.

    -W    Disable configuration of Workflow.

    -w workflow_repo
          The path to a Workflow repo to use instead of the default.
    -t    Enable testing mode.
EOS
}

function main {
    local TOP
    if $GIT rev-parse --is-inside-work-tree 1> /dev/null 2> /dev/null
    then
        TOP="$($GIT rev-parse --show-toplevel)"
    else
        TOP="$PWD"
    fi

    local DISABLE_DB DISABLE_MIGRATIONS DISABLE_UR DISABLE_WF \
        DB_SNAPSHOT_NAME DB_ID DB_HOST DB_PORT SQITCH_REPO UR_REPO WF_REPO \
        TESTING_MODE=""
    parse_opts "$@"
    shift $((OPTIND-1))

    prepare_submodules

    validate_bash

    rebuild_meta_db
    local TESTDBSERVER_DB_NAME TESTDBSERVER_DB_HOST TESTDBSERVER_DB_PORT
    if test -z "$DISABLE_DB"
    then
        if test -n "$DB_ID"
        then
            export TESTDBSERVER_DB_NAME=$DB_ID
            export TESTDBSERVER_DB_HOST=$DB_HOST
            export TESTDBSERVER_DB_PORT=$DB_PORT
        else
            setup_test_db "$TOP"
        fi
    fi


    setup_env "$TOP"

    log.info ''
    log.info "Genome Perl: $(genome-perl -e 'printf(qq(%vd\n), $^V)')"
    log.info "Genome Prove: $(genome-prove --version)"
    for MODULE in UR Workflow Genome
    do
        log.info "${MODULE}: $(genome-perl -M$MODULE -e "print \$INC{q(${MODULE}.pm)}, qq(\n)")"
    done

    if test -z "$DISABLE_DB"
    then
        if test -n "$DB_ID"
        then
            log.info "Using existing test DB"
        else
            log.info "Test DB Snapshot: $DB_SNAPSHOT_NAME"
        fi
        log.info "Test DB Name: $TESTDBSERVER_DB_NAME"
        log.info "Test DB Host: ${TESTDBSERVER_DB_HOST}"
        log.info "Test DB Port: ${TESTDBSERVER_DB_PORT}"
    fi
    log.info ''

    if test -n "$TESTING_MODE"
    then
        # Have to use `genome config get` after setup_env.
        eval "$(genome config set-env log_usage 0)"
        eval "$(genome config set-env sys_services_files_url 'file:///')"
        eval "$(genome config set-env test_url "$(genome config get sys_services_files_url)$(genome config get test_inputs)")"
        eval "$(genome config set-env testing 1)"
        unset WF_USE_FLOW
        export UR_DBI_NO_COMMIT=1
        export GE_ENV_NAME="genome-test-env"
    fi

    if test "$#" -gt 0
    then
        log.info "$@"...
        eval "$ORIG_SET"
        "$@"
        EXIT=$?
    else
        echo 'Press <Ctrl>-D or type `exit` to return to normal environment.'
        echo ''
        unset PROMPT_COMMAND
        export PS1="(${GE_ENV_NAME}) \w $ "
        eval "$ORIG_SET"
        "$BASH" --norc --noprofile
        EXIT=$?
    fi

    if test -z "$DISABLE_DB" -a -z "$PERSIST_DB"
    then
        cleanup_test_db
    fi
    exit $EXIT
}

function validate_bash {
    if test "$#" -eq 0 -a -z "$BASH"
    then
        fatal $DEP_ERR 'only `bash` shells are supported at this time'
    fi
}

function parse_opts {
    DISABLE_DB="" DISABLE_MIGRATIONS="" DISABLE_UR="" DISABLE_WF="" DB_SNAPSHOT_NAME="" DB_ID="" DB_HOST="" DB_PORT="" SQITCH_REPO="" UR_REPO="" WF_REPO="" PERSIST_DB=""

    local opts
    while getopts :hMm:Dd:i:pUu:Ww:t opts "$@"
    do
        case $opts in
            h)
                help
                exit 0
                ;;
            M)
                DISABLE_MIGRATIONS="TRUE"
                ;;
            m)
                SQITCH_REPO="$OPTARG"
                ;;
            D)
                DISABLE_DB="TRUE"
                DISABLE_MIGRATIONS="TRUE"
                ;;
            d)
                DB_SNAPSHOT_NAME="$OPTARG"
                ;;
            i)
                args=(${OPTARG//:/ })
                DB_ID=${args[0]}
                DB_HOST=${args[1]}
                DB_PORT=${args[2]}
                PERSIST_DB="TRUE"
                ;;
            p)  PERSIST_DB="TRUE"
                ;;
            U)
                DISABLE_UR="TRUE"
                ;;
            u)
                UR_REPO="$OPTARG"
                ;;
            W)
                DISABLE_WF="TRUE"
                ;;
            w)
                WF_REPO="$OPTARG"
                ;;
            t)
                TESTING_MODE=1
                ;;
            :)
                fatal $OPT_ERR "option -$OPTARG requires an argument"
                ;;
            \?)
                fatal $OPT_ERR "illegal option -- $OPTARG"
                ;;
        esac
    done

    validate_opts
    set_defaults
}

function validate_opts {
    if test -n "$DISABLE_DB" -a -n "$DB_SNAPSHOT_NAME"
    then
        fatal $OPT_ERR "cannot use -D and -d options together"
    fi

    if test -n "$DISABLE_DB" -a -n "$DB_ID"
    then
        fatal $OPT_ERR "cannot use -D and -i options together"
    fi

    if test -n "$DB_SNAPSHOT_NAME" -a -n "$DB_ID"
    then
        fatal $OPT_ERR "cannot use -d and -i options together"
    fi

    if test -n "$DISABLE_UR" -a -n "$UR_REPO"
    then
        fatal $OPT_ERR "cannot use -U and -u options together"
    fi

    if test -n "$DISABLE_MIGRATIONS" -a -n "$SQITCH_REPO"
    then
        fatal $OPT_ERR "cannot use -M and -m options together"
    fi

    if test -n "$DISABLE_WF" -a -n "$WF_REPO"
    then
        fatal $OPT_ERR "cannot use -W and -w options together"
    fi

    if test -n "$DB_SNAPSHOT_NAME"
    then
        if ! $TESTDB template list | cut -f 1 | $GREP -q "^${DB_SNAPSHOT_NAME}$"
        then
            fatal $OPT_ERR "invalid snapshot name -- $DB_SNAPSHOT_NAME"
        fi
    fi

    if test -n "$DB_ID"
    then
        if ! $TESTDB database list | cut -f 1 | $GREP -q "^${DB_ID}$"
        then
            fatal $OPT_ERR "invalid database name -- $DB_ID"
        fi
    fi

    if test -n "$SQITCH_REPO"
    then
        assert_dir "-m" "$SQITCH_REPO"
    fi

    if test -n "$UR_REPO"
    then
        assert_dir "-u" "$UR_REPO"
    fi

    if test -n "$WF_REPO"
    then
        assert_dir "-w" "$WF_REPO"
    fi
}

function assert_dir {
    if ! test -d "$2"
    then
        fatal $OPT_ERR "$1 value must be a directory -- $2"
    fi
}

declare -a SUBMODULES=()
function append_submodule {
    if test "${#SUBMODULES[@]}" -gt 0
    then
        SUBMODULES=("${SUBMODULES[@]}" "$@")
    else
        SUBMODULES=("$@")
    fi
}

function set_defaults {
    if test -z "$DISABLE_MIGRATIONS" -a -z "$SQITCH_REPO"
    then
        local SQITCH_GENOME='sqitch/genome'
        append_submodule $SQITCH_GENOME
        SQITCH_REPO="$TOP/$SQITCH_GENOME"
    fi

    if test -z "$DB_SNAPSHOT_NAME" -a -z "$DB_ID"
    then
        DB_SNAPSHOT_NAME="715f122"
    fi

    if test -z "$UR_REPO" -a -z "$DISABLE_UR"
    then
        if test -z "$TOP" -o ! -d "$TOP/ur"
        then
            fatal $OPT_ERR "-u must be specified if not in a Genome repo"
        fi
        append_submodule ur
        UR_REPO="$TOP/ur"
    fi

    if test -z "$WF_REPO" -a -z "$DISABLE_WF"
    then
        if test -z "$TOP" -o ! -d "$TOP/workflow"
        then
            fatal $OPT_ERR "-w must be specified if not in a Genome repo"
        fi
        append_submodule workflow
        WF_REPO="$TOP/workflow"
    fi
}

DEP_ERR=1 # 1 = dependency error
OPT_ERR=2 # 2 = option error
BUILD_ERR=3 # 3 = build error
function fatal {
    local EXIT="$1"
    shift
    log.error "$@"
    exit $EXIT
}

function log.error {
    echo "$THIS: $@" >&2
}

function log.info {
    echo "$@" 1>&2
}

# __quiet_git only show Git output if there Git exits non-zero so that we can
# provide "pretty" output around the Git commands.
function __quiet_git {
    (
        set +o errexit
        local OUT="$($GIT "$@" 2>&1)"
        local GIT_EXIT=$?
        if test $GIT_EXIT -ne 0
        then
            fatal $BUILD_ERR -e "$OUT"
        fi
    )
}

# prepare_submodules tries to (essentially) do a fast-forward update on
# submodules when the submodules are used as the default repo.
function prepare_submodules {
    if test "${#SUBMODULES}" -eq 0
    then
        return
    fi

    (
        log.info -e "\n=> Preparing Submodules..."
        cd "$($GIT rev-parse --show-cdup)"

        __quiet_git submodule sync "${SUBMODULES[@]}"

        for submodule in "${SUBMODULES[@]}"
        do
            log.info -n "   $submodule: "
            if $GIT submodule status $submodule | $GREP -q ^-
            then
                __quiet_git submodule update --init $submodule
                log.info "OK"
            else
                (
                    local EXPECTED_HEAD=$($GIT ls-tree HEAD $submodule | awk '{print $3}')
                    cd $submodule
                    local HEAD=$($GIT rev-parse HEAD)

                    if ! $GIT cat-file -e $EXPECTED_HEAD
                    then
                        __quiet_git fetch --all
                    fi

                    # Essentialy trying to do a fast-forward only update but
                    # since submodules are not branch-based we can't just do
                    # a `git pull --ff-only`.
                    if ! $GIT status --porcelain | $GREP -q . \
                        && $GIT merge-base --is-ancestor $HEAD $EXPECTED_HEAD
                    then
                        cd - 1> /dev/null
                        __quiet_git submodule update --init $submodule
                        log.info "OK"
                    else
                        fatal $BUILD_ERR "submodule has local changes"
                    fi
                )
            fi
        done
    )
}

# regen_db creates/updates SQLite (binary) database files from ASCII dump files
# that are in source control.
function regen_db {
    local DB_TXT="$1"
    log.info -n '.'
    local DB_BIN=${DB_TXT/-dump/}
    if test "$DB_BIN" -ot "$DB_TXT"
    then
        rm -f "$DB_BIN"
        $SQLITE3 "$DB_BIN" < "$DB_TXT"
    fi
}

# rebuild_meta_db scans for SQLite ASCII dump files that are (usually) in
# source control and calls regen_db on them.
function rebuild_meta_db {
    log.info -en "\n=> Rebuilding Meta DB..."

    if $GIT rev-parse --is-inside-work-tree 1> /dev/null 2> /dev/null
    then
        # ls-files is fast but then we have to also query submodules
        # foreach defines $name which serves to prefix the submodule path
        ( $GIT ls-files \
            && $GIT submodule foreach "$GIT ls-files | sed \"s|^|\$name/|\"" ) \
        | $GREP -P '\.sqlite3n?-dump$' \
        | while read DB_TXT
        do
            regen_db "$DB_TXT"
        done
    else
        find "$PWD" -iname '*sqlite3-dump' -or -iname '*sqlite3n-dump' | while read DB_TXT; do
            regen_db "$DB_TXT"
        done
    fi

    log.info ' done'
}

# setup_test_db creates an ephemeral test database, runs the latest migrations
# on it, and provides shell variables used to configure GMSchema later.
function setup_test_db {
    local BASE_DIR="$1"
    echo "=> Creating test database..."
    eval "$($TESTDB database create --timeout 30 --bash --owner $TESTDBSERVER_DB_USER --based-on $DB_SNAPSHOT_NAME)"
    if test -z "$TESTDBSERVER_DB_NAME"
    then
        fatal $BUILD_ERR "failed to create test database"
    fi

    if test -z "$DISABLE_MIGRATIONS"
    then
        echo -n "=> Migrating test database..."
        (
            local SQITCH_TMP="$(mktemp -d -t genome-env-sqitch-XXXXX)"
            trap "rm -rf \"$SQITCH_TMP\"" EXIT

            cp -aT "$SQITCH_REPO" "$SQITCH_TMP"
            cd "$SQITCH_TMP"
            $SQITCH config core.pg.host     $TESTDBSERVER_DB_HOST
            $SQITCH config core.pg.username $TESTDBSERVER_DB_USER
            $SQITCH config core.pg.password $TESTDBSERVER_DB_PASS
            $SQITCH config core.pg.db_name  $TESTDBSERVER_DB_NAME
            if $SQITCH status | grep -q 'Nothing to deploy'
            then
                echo " nothing to deploy"
            else
                echo ""
                $SQITCH deploy
            fi
        )
    fi
}

# cleanup_test_db reverses setup_test_db by deleting any new disk allocations
# from the test database, then deletes the test database
function cleanup_test_db {
    echo "=> Removing disk allocations created during $THIS..."
    genome model admin remove-disk-allocations-from-testdb
    test-db delete database $TESTDBSERVER_DB_NAME > /dev/null
}

# assert_module_missing only works with single word module names, e.g. UR, Workflow, and Genome.
function assert_module_missing {
    local MODULE=$1
    local MODULE_PATH="$(genome-perl -M${MODULE} -e "print \$INC{q($MODULE.pm)}, qq(\\n)" 2> /dev/null )"
    if test -n "$MODULE_PATH"
    then
        fatal $BUILD_ERR "found shadow module: $MODULE ($MODULE_PATH)"
    fi
}

# assert_module_found only works with single word module names, e.g. UR, Workflow, and Genome.
function assert_module_found {
    local MODULE=$1
    local MODULE_PATH="$(genome-perl -M${MODULE} -e "print \$INC{q($MODULE.pm)}, qq(\\n)")"
    if test -z "$MODULE_PATH"
    then
        fatal $BUILD_ERR "module not found: $MODULE"
    fi
}

# setup_env tries to clean the current environment and then updates the
# environment to run tests.
function setup_env {
    log.info -e "\n=> Setting Up Test Environment..."

    local BIN MODULE \
        BASE_DIR="$1" \
        SNAPSHOT_LIB="/gsc/scripts/opt/genome/current/user/lib/perl" \
        SNAPSHOT_BIN="/gsc/scripts/opt/genome/current/user/bin"

    # remove deployed bin directory from PATH
    PATH="$(echo $PATH | tr : "\n" | $GREP -v "$SNAPSHOT_BIN" | tr '\n' : | sed 's/:$//')"
    export PATH=$BASE_DIR/bin:$PATH

    # remove deployed lib/perl directory from PATH
    export PERL5LIB="$(echo $PERL5LIB | tr : "\n" | $GREP -v "$SNAPSHOT_LIB" | tr '\n' : | sed 's/:$//')"

    assert_module_missing Genome

    if test -z "$DISABLE_UR"
    then
        export PATH=$UR_REPO/bin:$PATH

        assert_module_missing UR
    fi

    if test -z "$DISABLE_WF"
    then
        export PATH=$WF_REPO/bin:$PATH

        assert_module_missing Workflow
    fi

    if which genome | $GREP -q "$SNAPSHOT_BIN"
    then
        fatal $BUILD_ERR "genome found in $SNAPSHOT_BIN"
    fi

    if test -z "$DISABLE_UR"
    then
        export PERL5LIB=$UR_REPO/lib:$PERL5LIB
        assert_module_found UR

        if which ur | $GREP -q "$SNAPSHOT_BIN"
        then
            fatal $BUILD_ERR "ur found in $SNAPSHOT_BIN"
        fi
    fi

    if test -z "$DISABLE_WF"
    then
        export PERL5LIB=$WF_REPO/lib:$PERL5LIB
        assert_module_found Workflow

        if which workflow | $GREP -q "$SNAPSHOT_BIN"
        then
            fatal $BUILD_ERR "workflow found in $SNAPSHOT_BIN"
        fi
    fi

    export PERL5LIB=$BASE_DIR/lib/perl:$PERL5LIB
    assert_module_found Genome

    if test -z "$DISABLE_DB"
    then
        eval "$(genome config set-env ds_gmschema_server "dbname=${TESTDBSERVER_DB_NAME};host=${TESTDBSERVER_DB_HOST};port=${TESTDBSERVER_DB_PORT}")"
        eval "$(genome config set-env ds_gmschema_login "$TESTDBSERVER_DB_USER")"
        eval "$(genome config set-env ds_gmschema_auth "$TESTDBSERVER_DB_PASS")"
    fi

    hash -r
}

function default_value {
    if test -z $(eval "echo \$$1")
    then
        eval "$1=\"$2\""
    fi
}

THIS="$(basename "$0")"

GIT="$(type -p git)" || fatal $DEP_ERR "git is required"
SQITCH="$(type -p sqitch)" || fatal $DEP_ERR "sqitch is required"
SQLITE3="$(type -p sqlite3)" || fatal $DEP_ERR "sqlite3 is required"
GREP="$(type -p grep)" || fatal $DEP_ERR "grep is required"
TESTDB="$(type -p test-db)" || fatal $DEP_ERR "test-db is required"

default_value TESTDBSERVER_DB_USER "genome"
default_value TESTDBSERVER_DB_PASS "mypassword"
default_value TESTDBSERVER_URL "https://apipe-test-db.gsc.wustl.edu"
default_value GE_ENV_NAME "genome-env"

export TESTDBSERVER_URL

if test -z "$GE_NO_REDIRECT"
then
    # delegate to local version if it is found
    if $GIT rev-parse --is-inside-work-tree 1> /dev/null 2> /dev/null
    then
        WORK_TREE="$($GIT rev-parse --show-toplevel)"
        NEW_THIS="${WORK_TREE}/bin/${THIS}"
        if test -x "$NEW_THIS"
        then
            export GE_NO_REDIRECT=1
            exec "$NEW_THIS" "$@"
        fi
    fi
fi

set -o nounset

main "$@"
back to top