https://github.com/Microsoft/CNTK
Raw File
Tip revision: 7ae9d0e2658329c24ed060129b27af91b1cc0f69 authored by Mark Hillebrand on 18 January 2016, 08:33:42 UTC
License change
Tip revision: 7ae9d0e
build-and-test
#!/bin/bash

# Setting some default values
BUILD=1
RUN=1
CLEAN_AFTER=0
CLEAN_BEFORE=0
RANDOM_OUTPUT=0
FLAVORS="debug:release"
TARGETS="CPU:GPU"

# parsing command line arguments:
while [[ $# > 0 ]]
do
key="$1"

case $key in
    -h|--help)
    echo "Usage: build-and-test [options]"
    echo "Options:"
    echo "  -q|--quiet-build - redirect build output to file"
    echo "  -r|--run-only - elides build step, runs the binaries that have already been built"
    echo "  -b|--build-only - just build, do not run"
    echo "  -f|--flavors <flavor1:flavor2...> - which flavor to build/run (by default $FLAVORS)"
    echo "  -t|--targets <target1:target2...> - which target to run (by default $TARGETS)"
    echo "  -cb|--clean-build - clean up the enlistment binaries before build"
    echo "  -rnd|--random-output-suffix - add random suffix to output directory"
    echo "  -o|--output-directory <output_dir> - specify output directory to use (by default those will be in <cntk_root>.run-<operating_system>)"
    echo "The root directory used to build and run CNTK is hosts the Scripts directory that contains this script"
    exit 1
    ;;
    -q|--quiet)
    QUIET_BUILD=1
    ;;
    -r|--run-only)
    BUILD=0
    RUN=1
    ;;
    -rnd|--random-output-suffix)
    RANDOM_OUTPUT=1
    ;;
    -b|--build-only)
    BUILD=1
    RUN=0
    ;;
    -cb|--clean-build)
    CLEAN_BEFORE=1
    BUILD=1
    ;;
    -f|--flavors)
    FLAVORS="$2"
    shift # past argument
    ;;
    -t|--targets)
    TARGETS="$2"
    shift # past argument
    ;;
    -o|--output-directory)
    OUTPUT_DIR="$2"
    shift # past argument
    ;;
    *)
    echo Unkown option $key
    exit 1
    ;;
esac
shift # past argument or value
done

# Step 0 -- Validate all necessary prerequisites and check for incompatible options
# It is possible to use this script on Windows to build CNTK
# from Cygwin window with Visual C++ environment loaded.
# In that case OS environment variable will be set and we 
# can use it to differentiate from Linux.
if [[ $CLEAN_BEFORE == 1 && $RUN == 1 && $BUILD == 0 ]]; then
    echo "============ ERROR: Incompatible options RUN and CLEAN_BEFORE set without BUILD ============"
    exit 1
fi

if [[ $OS == "Windows_NT" && $OSTYPE == "cygwin" ]]; then
    DEBUG_DIR=Debug
    RELEASE_DIR=Release
    PREFIX_DIR=x64
    BIN_NAME=CNTK.exe
    BUILD_OS="windows"
  
    if [[ $VS120COMNTOOLS == "" ]]; then
        echo "============ Visual Studio 12.0 environment not properly setup or VS not installed ============"
        echo "============ Please find and run the appropriate vcvarsall.bat script ============"
        exit 1
    fi

    if [[ $ACML_PATH == "" ]]; then
        echo "============ ACML path not set  ============"
        echo "============ ACML libraries are needed to successfully build CNTK ============"
        exit 1
    fi
elif [[ $OSTYPE == "linux-gnu" ]]; then
    DEBUG_DIR=build/debug
    RELEASE_DIR=build/release
    PREFIX_DIR=
    # Make sure no dependencies on current directory
    BIN_NAME=bin/cntk
    MAKEFILE=Makefile
    BUILD_OS="linux"
else
    echo "============ ERROR: Unsupported OS ============"
    echo "============ Scripts supports only building from Linux and Windows through Cygwin ============"
    exit 1
fi

# Step 1 -- Prepare temporary folders and files, tweak settings if necessary

# Get to the root path from which we know how to build and run
SCRIPT=`readlink -f $0`
SCRIPT_DIR=`dirname $SCRIPT`
CNTK_ROOT=`dirname $SCRIPT_DIR`

# Setup the output directory
if [[ $OUTPUT_DIR == "" ]]; then
    OUTPUT_DIR="$CNTK_ROOT/.run-$BUILD_OS"
fi

# Add random number at the end of the output directory to prevent overwriting previous results
if [[ $RANDOM_OUTPUT == 1 ]]; then
    OUTPUT_DIR="$OUTPUT_DIR-$RANDOM"
fi

echo "============ Creating CNTK temp directory in $TMP_ROOT ============"
mkdir -p $OUTPUT_DIR || exit $?

CONF_FILE="$OUTPUT_DIR/Simple.conf"
BUILD_FILE="$OUTPUT_DIR/Build"
RUN_FILE="$OUTPUT_DIR/Result"

if ! [[ -d "$CNTK_ROOT/MachineLearning" ]]; then
    echo "============ ERROR: Build script located in the wrong directory ($SCRIPT_DIR) ============"
    exit 1
fi

cd $CNTK_ROOT

if ! [[ -f $CONF_FILE ]]; then
    cp Demos/Simple/Simple.config $CONF_FILE || exit $?

    # This chmod is necessary due to restrictive Cygwin interpretation of Windows permissions.
    # Cygwin interprets Windows permissions as ----rwx---, which lacks read permissions for user.
    chmod a+r $CONF_FILE || exit $?
fi

if [[ $QUIET_BUILD == 1 ]]; then
    echo "============ WARNING: You have selected quiet build. All build output will be placed in ($OUTPUT_DIR) ============"
fi

# Initialize lists of flavors and targets
flavorArray=(${FLAVORS//:/ })
targetArray=(${TARGETS//:/ })

# Step 2 -- Build the project debug and release, if requested
if [[ $BUILD == 1 ]]; then
    # Step 2 -- Perform necessary builds
    for FLAVOR in "${flavorArray[@]}"
    do
        # Our make is too noisy right now and it is difficult to spot
        # issues from stdout and stderr. In the quiet mode these are
        # redirected to a file where they could be examined after the fact
        if [[ $QUIET_BUILD == 1 ]]; then
            exec 6>$BUILD_FILE.$FLAVOR.out || exit $?
            exec 7>$BUILD_FILE.$FLAVOR.err || exit $?
        else
            exec 6>&1 || exit $?
            exec 7>&2 || exit $?
        fi

        echo "============ Building CNTK $FLAVOR (clean=$CLEAN_BEFORE)  ============"

        if [[ $OS == "Windows_NT" ]]; then
            if [[ $CLEAN_BEFORE == 1 ]]; then
                msbuild.exe /property:Configuration=$FLAVOR /t:Clean 1>&6 2>&7 || exit $?
            fi
            msbuild.exe /property:Configuration=$FLAVOR /m 1>&6 2>&7 || exit $?
        else
            BUILD_DIR=build/$FLAVOR
            ./configure --with-build-top=$BUILD_DIR --with-acml=$ACML_PATH --with-buildtype=$FLAVOR
            if [[ $CLEAN_BEFORE == 1 ]]; then
                make -C $BUILD_DIR -f $MAKEFILE clean 1>&6 2>&7 || exit $?
            fi
            make -C $BUILD_DIR -j -f $MAKEFILE 1>&6 2>&7 || exit $?
        fi
        if [[ $QUIET_BUILD == 1 ]]; then
          chmod a+r $BUILD_FILE.*
        fi
    done
fi

# Step 3 -- Run the project tests, both debug and release, if requested
if [[ $RUN == 1 ]]; then

    cd $PREFIX_DIR

    for TARGET in "${targetArray[@]}"
    do
        # These sed scripts are simply toggling DeviceNumber argument in the config file
        # If it is set to Auto, it will pick GPU over CPU. At -1 CPU is selected.
        if [[ $TARGET == CPU ]]; then
            sed -i -e 's/^DeviceNumber.*/DeviceNumber=-1/g' $CONF_FILE || exit $?
        else
            sed -i -e 's/^DeviceNumber.*/DeviceNumber=Auto/g' $CONF_FILE || exit $?
        fi

        for FLAVOR in "${flavorArray[@]}"
        do
            if [[ $FLAVOR == "debug" ]]; then
                FLAVOR_DIR="$DEBUG_DIR"
            else
                FLAVOR_DIR="$RELEASE_DIR"
            fi
            OUT_FILE="$RUN_FILE.$FLAVOR.$TARGET.out"

            BIN_PATH=$CNTK_ROOT/$FLAVOR_DIR/$BIN_NAME 
            if ! [[ -f $BIN_PATH ]]; then
                echo "============ ERROR: CNTK did not build properly for flavor ($FLAVOR)  ============"
                echo "Missing file: $BIN_PATH"
                exit 1
            fi

            echo "============ Running CNTK for ($FLAVOR) ($TARGET), output in ($RUN_FILE.*) ============"
            cd $CNTK_ROOT/Demos
            rm -rf models
            if [[ $OS == "Windows_NT" ]]; then
                # We have to use cygpath on Windows to modify the file paths into the format readable by cntk.
                time $BIN_PATH configFile="`cygpath -w $CONF_FILE`" &>$OUT_FILE || exit $?
            else
                time $BIN_PATH configFile=$CONF_FILE &>$OUT_FILE || exit $?
            fi
            chmod a+r $RUN_FILE.*

            # Check if execution was successful
            grep -q "Using $TARGET" "$OUT_FILE" || {
                echo "============ ERROR: Run output (in $OUT_FILE) did not contain information about target device ($TARGET) ============"
                exit 1
            }

            grep -q "EXCEPTION" "$OUT_FILE" && {
                echo "============ ERROR: Run output in ($OUT_FILE) contains exceptions ============"
                grep "EXCEPTION" "$OUT_FILE"
                exit 1
            }
        done
    done
fi

# Step 5 -- Optionally clean after builds and tests
if [[ $CLEAN_AFTER == 1 ]]; then
    rm -rf models
    cd $CNTK_ROOT
    for FLAVOR in debug release
    do
        echo "============ Cleaning up CNTK $FLAVOR  ============"
        if [[ $OS == "Windows_NT" ]]; then
            msbuild.exe /property:Configuration=$FLAVOR /t:clean 1>&6 2>&7 || exit $?
        else
            make BUILDTYPE=$FLAVOR -f $MAKEFILE clean 1>&6 2>&7 || exit $?
        fi
    done
    rm -rf $OUTPUT_DIR
fi

echo "============ Build and test of CNTK was successful!  ============"
back to top