https://github.com/Microsoft/CNTK
Tip revision: e9162e18e1c51b88f5c92ac6eb6623943eae353e authored by Amit Agarwal on 08 May 2016, 01:37:50 UTC
Synchronize batch norm nodes' running means and variances across parallel workers at end of each epoch to ensure accurate restartability across epochs and add and end-to-end test for this scenario
Synchronize batch norm nodes' running means and variances across parallel workers at end of each epoch to ensure accurate restartability across epochs and add and end-to-end test for this scenario
Tip revision: e9162e1
build-and-test
#!/bin/bash
# Setting some default values
BUILD=1
RUN=1
CLEAN_AFTER=0
CLEAN_BEFORE=0
RANDOM_OUTPUT=0
CODE_COVERAGE=no
FLAVORS="debug:release"
TARGETS="cpu:gpu"
TESTTARGETS="cpu:gpu"
# parsing command line arguments:
while [[ $# > 0 ]]
do
key="$1"
case $key in
-h|--help)
echo "Usage: build-and-test [options]"
echo "Options:"
echo " -q|--quiet-build - redirect build output to file"
echo " -r|--run-only - elides build step, runs the binaries that have already been built"
echo " -b|--build-only - just build, do not run"
echo " -f|--flavors <flavor1:flavor2...> - which flavor to build (by default $FLAVORS)"
echo " -t|--targets <target1:target2...> - which target to build (by default $TARGETS)"
echo " -tt|--test-targets <testtarget1:testtarget2...> - which target to test (by default $TESTTARGETS)"
echo " -cc|--code-coverage - build with support for code coverage (gcov)"
echo " -cb|--clean-build - clean up the enlistment binaries before build"
echo " -cba|--clean-build-after - clean up the enlistment binaries after build"
echo " -rnd|--random-output-suffix - add random suffix to output directory"
echo " -o|--output-directory <output_dir> - specify output directory to use (by default those will be in <cntk_root>.run-<operating_system>)"
echo "The root directory used to build and run CNTK is hosts the Scripts directory that contains this script"
exit 1
;;
-q|--quiet)
QUIET_BUILD=1
;;
-r|--run-only)
BUILD=0
RUN=1
;;
-rnd|--random-output-suffix)
RANDOM_OUTPUT=1
;;
-b|--build-only)
BUILD=1
RUN=0
;;
-cb|--clean-build)
CLEAN_BEFORE=1
BUILD=1
;;
-cba|--clean-build-after)
CLEAN_AFTER=1
BUILD=1
;;
-f|--flavors)
FLAVORS="${2,,}"
shift # past argument
;;
-t|--targets)
TARGETS="${2,,}"
shift # past argument
;;
-tt|--test-targets)
TESTTARGETS="${2,,}"
shift # past argument
;;
-cc|--code-coverage)
CODE_COVERAGE=yes
;;
-o|--output-directory)
OUTPUT_DIR="$2"
shift # past argument
;;
*)
echo Unkown option $key
exit 1
;;
esac
shift # past argument or value
done
# Step 0 -- Validate all necessary prerequisites and check for incompatible options
# It is possible to use this script on Windows to build CNTK
# from Cygwin window with Visual C++ environment loaded.
# In that case OS environment variable will be set and we
# can use it to differentiate from Linux.
if [[ $CLEAN_BEFORE == 1 && $RUN == 1 && $BUILD == 0 ]]; then
echo "============ ERROR: Incompatible options RUN and CLEAN_BEFORE set without BUILD ============"
exit 1
fi
if [[ $OS == "Windows_NT" && $OSTYPE == "cygwin" ]]; then
DEBUG_DIR=Debug
RELEASE_DIR=Release
PREFIX_DIR=x64
BIN_NAME=CNTK.exe
BUILD_OS="windows"
if [[ $VS120COMNTOOLS == "" ]]; then
echo "============ Visual Studio 12.0 environment not properly setup or VS not installed ============"
echo "============ Please find and run the appropriate vcvarsall.bat script ============"
exit 1
fi
if [[ $ACML_PATH == "" ]]; then
echo "============ ACML path not set ============"
echo "============ ACML libraries are needed to successfully build CNTK ============"
exit 1
fi
if [[ "${TARGETS,,}" =~ "1bitsgd" && "${TARGETS,,}" =~ "gpu" ]]; then
echo "============ Cannot specify both GPU and 1bit-SGD as targets, please choose one ============"
exit 1
fi
elif [[ $OSTYPE == "linux-gnu" ]]; then
DEBUG_DIR=build/$TARGET/debug
RELEASE_DIR=build/$TARGET/release
PREFIX_DIR=
# Make sure no dependencies on current directory
BIN_NAME=bin/cntk
MAKEFILE=Makefile
BUILD_OS="linux"
else
echo "============ ERROR: Unsupported OS ============"
echo "============ Script supports only building from Linux and Windows through Cygwin ============"
exit 1
fi
# Step 1 -- Prepare temporary folders and files, tweak settings if necessary
# Get to the root path from which we know how to build and run
SCRIPT=`readlink -f $0`
SCRIPT_DIR=`dirname $SCRIPT`
CNTK_ROOT=`dirname $SCRIPT_DIR`
# Setup the output directory
if [[ $OUTPUT_DIR == "" ]]; then
OUTPUT_DIR="$CNTK_ROOT/.run-$BUILD_OS"
fi
# Add random number at the end of the output directory to prevent overwriting previous results
if [[ $RANDOM_OUTPUT == 1 ]]; then
OUTPUT_DIR="$OUTPUT_DIR-$RANDOM"
fi
echo "============ Creating CNTK temp directory in $OUTPUT_DIR ============"
mkdir -p $OUTPUT_DIR || exit $?
CONF_FILE="$OUTPUT_DIR/Simple.cntk"
BUILD_FILE="$OUTPUT_DIR/Build"
RUN_FILE="$OUTPUT_DIR/Result"
if ! [[ -d "$CNTK_ROOT/Source" ]]; then
echo "============ ERROR: Build script located in the wrong directory ($SCRIPT_DIR) ============"
exit 1
fi
cd $CNTK_ROOT
if ! [[ -f $CONF_FILE ]]; then
cp Examples/Other/Simple2d/Config/Simple.cntk $CONF_FILE || exit $?
# This chmod is necessary due to restrictive Cygwin interpretation of Windows permissions.
# Cygwin interprets Windows permissions as ----rwx---, which lacks read permissions for user.
chmod a+r $CONF_FILE || exit $?
fi
if [[ $QUIET_BUILD == 1 ]]; then
echo "============ WARNING: You have selected quiet build. All build output will be placed in ($OUTPUT_DIR) ============"
fi
# Initialize lists of flavors and targets
flavorArray=(${FLAVORS//:/ })
targetArray=(${TARGETS//:/ })
testTargetArray=(${TESTTARGETS//:/ })
# Step 2 -- Build the project for all requested flavors and targets
if [[ $BUILD == 1 ]]; then
for FLAVOR in "${flavorArray[@]}"
do
for TARGET in "${targetArray[@]}"
do
echo "============ Building CNTK $TARGET/$FLAVOR (clean=$CLEAN_BEFORE) ============"
# Our make is too noisy right now and it is difficult to spot
# issues from stdout and stderr. In the quiet mode these are
# redirected to a file where they could be examined after the fact
if [[ $QUIET_BUILD == 1 ]]; then
exec 6>$BUILD_FILE.$FLAVOR.out || exit $?
exec 7>$BUILD_FILE.$FLAVOR.err || exit $?
else
exec 6>&1 || exit $?
exec 7>&2 || exit $?
fi
if [[ $OS == "Windows_NT" ]]; then
OneBitSGDOPT=/property:CNTK_ENABLE_1BitSGD=
if [[ $TARGET == "cpu" ]]; then
CONFIGURATION=${FLAVOR}_CpuOnly
else
CONFIGURATION=${FLAVOR}
if [[ $TARGET == "1bitsgd" ]]; then
OneBitSGDOPT=/property:CNTK_ENABLE_1BitSGD=true
fi
fi
if [[ $CLEAN_BEFORE == 1 ]]; then
msbuild.exe /nologo /verbosity:m /property:Configuration=$CONFIGURATION /t:Clean 1>&6 2>&7 || exit $?
fi
msbuild.exe /nologo /verbosity:m /m /property:Configuration=$CONFIGURATION $OneBitSGDOPT 1>&6 2>&7 || exit $?
else
OneBitSGDOPT=no
BUILD_DIR=build/$TARGET/$FLAVOR
if [[ $TARGET == "cpu" ]]; then
CUDAOPT=no
else
CUDAOPT=yes
if [[ $TARGET == "1bitsgd" ]]; then
OneBitSGDOPT=yes
fi
fi
./configure --with-build-top=$BUILD_DIR --with-acml=$ACML_PATH --with-buildtype=$FLAVOR --cuda=$CUDAOPT --with-code-coverage=$CODE_COVERAGE --1bitsgd=$OneBitSGDOPT
if [[ $CLEAN_BEFORE == 1 ]]; then
make -C $BUILD_DIR -f $MAKEFILE clean 1>&6 2>&7 || exit $?
fi
make -C $BUILD_DIR -j `nproc` -f $MAKEFILE 1>&6 2>&7 || exit $?
fi
if [[ $QUIET_BUILD == 1 ]]; then
chmod a+r $BUILD_FILE.*
fi
done
done
fi
# Step 3 -- Run the project tests, both debug and release, if requested
if [[ $RUN == 1 ]]; then
cd $PREFIX_DIR
echo "============ cp Examples/Other/Simple2d/Config/Simple.cntk $CONF_FILE ============"
echo "============ cd $CNTK_ROOT/Examples/Other/Simple2d/Data ============"
for FLAVOR in "${flavorArray[@]}"
do
for TARGET in "${targetArray[@]}"
do
for TESTTARGET in "${testTargetArray[@]}"
do
# Determine how to set the deviceId parameter in the configuration.
# 0 will pick the first GPU (or fail. -1 will pick the CPU.
if [[ $TESTTARGET == gpu ]]; then
DEVICE_ID=0
else
DEVICE_ID=-1
fi
if [[ $TESTTARGET == gpu && $TARGET == cpu ]]; then
# CPU-only builds cannot run GPU test targets
continue;
fi
if [[ $OSTYPE == "linux-gnu" ]]; then
FLAVOR_DIR=build/$TARGET/$FLAVOR
else
if [[ $FLAVOR == "debug" ]]; then
FLAVOR_DIR="$DEBUG_DIR"
else
FLAVOR_DIR="$RELEASE_DIR"
fi
if [[ $TARGET == cpu ]]; then
FLAVOR_DIR="${FLAVOR}_CpuOnly"
fi
fi
OUT_FILE="$RUN_FILE.$FLAVOR.$TARGET.$TESTTARGET.out"
BIN_PATH=$CNTK_ROOT/$PREFIX_DIR/$FLAVOR_DIR/$BIN_NAME
if ! [[ -f $BIN_PATH ]]; then
echo "============ ERROR: CNTK did not build properly for $TARGET/$FLAVOR ============"
echo "Missing file: $BIN_PATH"
exit 1
fi
if [[ $OS == "Windows_NT" ]]; then
echo "============ Running $BIN_PATH configFile=`cygpath -w $CONF_FILE` for $TARGET/$FLAVOR (on $TESTTARGET) ============"
else
echo "============ Running $BIN_PATH configFile=$CONF_FILE for $TARGET/$FLAVOR (on $TESTTARGET) ============"
fi
echo "============ output in ($OUT_FILE) ============"
cd $CNTK_ROOT/Examples/Other/Simple2d/Data
rm -rf "$OUTPUT_DIR/Models"
if [[ $OS == "Windows_NT" ]]; then
# We have to use cygpath on Windows to modify the file paths into the format readable by cntk.
time $BIN_PATH configFile="`cygpath -w $CONF_FILE`" deviceId=$DEVICE_ID OutputDir="`cygpath -w $OUTPUT_DIR`" &>$OUT_FILE || exit $?
else
time $BIN_PATH configFile=$CONF_FILE deviceId=$DEVICE_ID OutputDir="$OUTPUT_DIR" &>$OUT_FILE || exit $?
fi
chmod a+r $RUN_FILE.*
# Check if execution was successful
grep -q "Using ${TESTTARGET^^}" "$OUT_FILE" || {
echo "============ ERROR: Run output (in $OUT_FILE) did not contain information about target device ($TESTTARGET) ============"
exit 1
}
grep -q "EXCEPTION" "$OUT_FILE" && {
echo "============ ERROR: Run output in ($OUT_FILE) contains exceptions ============"
grep "EXCEPTION" "$OUT_FILE"
exit 1
}
done
done
done
fi
# Step 5 -- Optionally clean after builds and tests
if [[ $CLEAN_AFTER == 1 ]]; then
cd $CNTK_ROOT
for FLAVOR in "${flavorArray[@]}"
do
for TARGET in "${targetArray[@]}"
do
echo "============ Cleaning up CNTK $TARGET/$FLAVOR ============"
if [[ $OS == "Windows_NT" ]]; then
if [[ $TARGET == "cpu" ]]; then
CONFIGURATION=${FLAVOR}_CpuOnly
else
CONFIGURATION=${FLAVOR}
fi
msbuild.exe /nologo /verbosity:m /property:Configuration=$CONFIGURATION /t:clean 1>&6 2>&7 || exit $?
else
make -C build/$TARGET/$FLAVOR -f $MAKEFILE clean 1>&6 2>&7 || exit $?
fi
done
done
rm -rf "$OUTPUT_DIR"
fi
echo "============ Build and test of CNTK was successful! ============"