Revision 4cc4c077d5e516e831796bfbd8ab2094a5be9171 authored by Carlos Arguelles on 28 December 2018, 16:46:08 UTC, committed by Carlos Arguelles on 28 December 2018, 16:46:08 UTC
1 parent 3748135
run_tests
#!/bin/sh
# Copyright (c) 2014, Christopher Weaver
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# main settings
# the name of the file in which to store results
REPORT_FILE="Report.txt"
# a file listing tests which are expected to fail; not required to exist
EXPECTED_FAILURES_FILE="Expected_Failures.txt"
# the suffix on the names of tests
TEST_SUFFIX=".test.cpp"
# the suffix on the names of optional extra compilation flags for tests
TEST_FLAGS_SUFFIX=".flags"
# the suffix on the names of expected test ouput files
REF_OUTPUT_SUFFIX=".output.txt"
# the flag included in a test name to indicate that it is required to fail compilation
COMPILE_FAIL_FLAG=".compile_fail"
# the subsitute suffix used for the version of a compile-fail test with the failure inducing code removed
CLEANED_TEST_SUFFIX=".clean.cpp"
# the subdirectory in which to stash produced files
PRODUCT_DIR="products"
# fetch any compilation flags set during library configuration
if [ -f env_vars.sh ]; then
. ./env_vars.sh
fi
COMPILE_FLAGS="-I../include $CFLAGS $CXXFLAGS $LDFLAGS -I./ -L../lib"
CLEAN_COMMAND_FILE="clean_fail.sed"
NAME_FILTER_REGEX='\('$COMPILE_FAIL_FLAG'\)*'$TEST_SUFFIX'$'
# output format settings
PASS_LABEL="PASS"
FAIL_LABEL="FAIL"
EXPECTED_FAIL_LABEL="FAIL (Expected)"
MISSING_REFERENCE_LABEL="NOT RUN (Reference output missing)"
if [ $# -eq 0 ] ; then
# locate all tests
TESTS=`ls -1 *$TEST_SUFFIX`
else
# collect the specified tests
TESTS=`for test in ${1+"$@"}; do ls ${test}*${TEST_SUFFIX} ; done`
SHOW_RESULTS=1
fi
# clear the report file
cat /dev/null > "$REPORT_FILE"
# set up for pretty printing
LONGEST_NAME_LEN=`echo "$TESTS" | sed "s|${NAME_FILTER_REGEX}||" | awk '{ if(length($1) > longest){ longest=length($1); } } END{ print longest; }'`
# make the product directory if necessary
if [ ! -d $PRODUCT_DIR ] ; then
mkdir $PRODUCT_DIR
fi
echo "$TESTS" | awk '{ tests++ } END{ if(tests==1) print "Running 1 test"; else print "Running "tests" tests"}'
# run all of the tests
for TEST in $TESTS ; do
NAME=`echo $TEST | sed "s|${NAME_FILTER_REGEX}||"`
PRETTY_NAME=`awk 'BEGIN{ printf("%-'${LONGEST_NAME_LEN}'s\n","'${NAME}'"); }'`
REF_OUTPUT="${NAME}${REF_OUTPUT_SUFFIX}"
# only run a test if its reference output exists
if [ -f "$REF_OUTPUT" ];then
# fetch any extra flags needed specifically by this test
if [ -f "${NAME}${TEST_FLAGS_SUFFIX}" ]; then
EXTRA_FLAGS=`cat ${NAME}${TEST_FLAGS_SUFFIX}`
else
EXTRA_FLAGS=""
fi
SHOULD_FAIL_COMPILE=`echo "$TEST" | grep "$COMPILE_FAIL_FLAG"`
if [ "$SHOULD_FAIL_COMPILE" ] ; then
# try to compile
RESULT=`${CXX} $TEST ${COMPILE_FLAGS} ${EXTRA_FLAGS} -o ${PRODUCT_DIR}/${NAME} 2>&1 | grep -c -f $REF_OUTPUT`
# check that the compiler printed the expected error message
if [ "$RESULT" -eq 0 ] ; then
REPORT_ENTRY="$PRETTY_NAME : $FAIL_LABEL (compilation did not produce required error)"
if [ -n "$SHOW_RESULTS" ];then
echo "$RESULT"
fi
else
# compilation failed with the expected error,
#but does it succeed if we remove the deliberately offending code?
CLEANED_TEST=${PRODUCT_DIR}/${NAME}${CLEANED_TEST_SUFFIX}
cat $TEST | sed -f ${CLEAN_COMMAND_FILE} > $CLEANED_TEST
RESULT=`${CXX} $CLEANED_TEST ${COMPILE_FLAGS} ${EXTRA_FLAGS} -o ${PRODUCT_DIR}/${NAME} 2>&1 && ${PRODUCT_DIR}/${NAME}`
SUCCESS=$?
if [ "$SUCCESS" -ne 0 ] ; then
REPORT_ENTRY="$PRETTY_NAME : $FAIL_LABEL (failed to compile or run after cleaning)"
if [ -n "$SHOW_RESULTS" ];then
echo "$RESULT"
fi
else
REPORT_ENTRY="$PRETTY_NAME : $PASS_LABEL"
fi
fi
else
# try to compile
RESULT=`$CXX $TEST ${COMPILE_FLAGS} ${EXTRA_FLAGS} -o ${PRODUCT_DIR}/${NAME} 2>&1`
COMPILE_SUCCESS=$?
# check that compilation succeeded
if [ "$COMPILE_SUCCESS" -ne 0 ] ; then
REPORT_ENTRY="$PRETTY_NAME : $FAIL_LABEL (failed to compile)"
if [ -n "$SHOW_RESULTS" ];then
echo "$RESULT"
fi
else
# run the resulting program and see if it worked
TMPFILE=`mktemp ${PRODUCT_DIR}/test_XXXXXX`
${PRODUCT_DIR}/${NAME} > $TMPFILE
RUN_SUCCESS=$?
RESULT=`diff -u "$REF_OUTPUT" "$TMPFILE"`
rm $TMPFILE
#check that it ran to completion and exited cleanly
if [ "$RUN_SUCCESS" -ne 0 ] ; then
REPORT_ENTRY="$PRETTY_NAME : $FAIL_LABEL (failed to run)"
if [ -n "$SHOW_RESULTS" ];then
echo "$RESULT"
fi
else
#check that the output matches the reference
if [ -n "$RESULT" ];then
REPORT_ENTRY="$PRETTY_NAME : $FAIL_LABEL"
if [ -n "$SHOW_RESULTS" ];then
echo "$RESULT"
fi
else
REPORT_ENTRY="$PRETTY_NAME : $PASS_LABEL"
fi
fi
fi
fi
else
REPORT_ENTRY="$PRETTY_NAME : $MISSING_REFERENCE_LABEL"
fi
# if the expected failures file exists, search it to see if this test is listed
if [ -f "$EXPECTED_FAILURES_FILE" ];then
EXPECTED=`grep "$NAME" $EXPECTED_FAILURES_FILE`
fi
FAILED=`echo $REPORT_ENTRY | grep $FAIL_LABEL`
if [ -n "$EXPECTED" -a -n "$FAILED" ] ; then
REPORT_ENTRY="$PRETTY_NAME : $EXPECTED_FAIL_LABEL"
fi
# record the test result
echo "$REPORT_ENTRY" | tee -a "$REPORT_FILE"
if [ -n "$RESULT" -a -z "$EXPECTED" ]; then
echo "$RESULT" >> "$REPORT_FILE"
fi
done
# collect some statistics
NUM_TESTS=`echo "$TESTS" | wc -l`
NUM_PASS=`grep -c ": $PASS_LABEL" "$REPORT_FILE"`
NUM_FAIL=`grep -c ": $FAIL_LABEL" "$REPORT_FILE"`
NUM_EXP_FAIL=`grep -c ": $EXPECTED_FAIL_LABEL" "$REPORT_FILE"`
NUM_NOT_RUN=`grep -c ": $MISSING_REFERENCE_LABEL" "$REPORT_FILE"`
# figure out whether to use plural endings or not
if [ "$NUM_TESTS" -ne "1" ] ;then
TEST_PLURAL="s"
fi
if [ "$NUM_PASS" -ne "1" ] ;then
PASS_PLURAL="es"
fi
if [ "$NUM_FAIL" -ne "1" ];then
FAILURE_PLURAL="s"
fi
if [ "$NUM_EXP_FAIL" -ge "1" ] ;then
EXPECTED_NOTE=" ($NUM_EXP_FAIL expected)"
fi
if [ "$NUM_NOT_RUN" -ge "1" ] ;then
if [ "$NUM_NOT_RUN" -ne "1" ] ;then
NOT_RUN_PLURAL="s"
fi
NOT_RUN_NOTE=", $NUM_NOT_RUN test$NOT_RUN_PLURAL could not be run"
fi
# record the statistics
REPORT_ENTRY="$NUM_TESTS Test$TEST_PLURAL: $NUM_PASS pass$PASS_PLURAL, $NUM_FAIL failure$FAILURE_PLURAL$EXPECTED_NOTE$NOT_RUN_NOTE"
echo "$REPORT_ENTRY" | tee -a "$REPORT_FILE"
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...