Revision 41697f9d282e991562c166ecc362922a8cb8fb84 authored by Jeff Bezanson on 02 November 2017, 21:59:43 UTC, committed by Jeff Bezanson on 29 December 2017, 20:45:37 UTC
1 parent 2a56a37
runtests.jl
# This file is a part of Julia. License is MIT: https://julialang.org/license
using Test
using Distributed
using Base.Printf: @sprintf
include("choosetests.jl")
include("testenv.jl")
tests, net_on, exit_on_error, seed = choosetests(ARGS)
tests = unique(tests)
const max_worker_rss = if haskey(ENV, "JULIA_TEST_MAXRSS_MB")
parse(Int, ENV["JULIA_TEST_MAXRSS_MB"]) * 2^20
else
typemax(Csize_t)
end
function test_path(test)
if test in STDLIBS
test_file = joinpath(STDLIB_DIR, test, "test", "runtests")
if !isfile(test_file * ".jl")
error("Standard library $test did not provide a `test/runtests.jl` file")
end
return test_file
else
return test
end
end
const node1_tests = String[]
function move_to_node1(t)
if t in tests
splice!(tests, findfirst(equalto(t), tests))
push!(node1_tests, t)
end
end
# Base.compile only works from node 1, so compile test is handled specially
move_to_node1("compile")
move_to_node1("SharedArrays")
# In a constrained memory environment, run the "distributed" test after all other tests
# since it starts a lot of workers and can easily exceed the maximum memory
max_worker_rss != typemax(Csize_t) && move_to_node1("Distributed")
cd(dirname(@__FILE__)) do
n = 1
if net_on
n = min(Sys.CPU_CORES, length(tests))
n > 1 && addprocs_with_testenv(n)
BLAS.set_num_threads(1)
end
skipped = 0
@everywhere include("testdefs.jl")
#pretty print the information about gc and mem usage
name_align = maximum([length("Test (Worker)"); map(x -> length(x) + 3 + ndigits(nworkers()), tests)])
elapsed_align = length("Time (s)")
gc_align = length("GC (s)")
percent_align = length("GC %")
alloc_align = length("Alloc (MB)")
rss_align = length("RSS (MB)")
print_with_color(:white, rpad("Test (Worker)",name_align," "), " | ")
print_with_color(:white, "Time (s) | GC (s) | GC % | Alloc (MB) | RSS (MB)\n")
results=[]
print_lock = ReentrantLock()
function print_testworker_stats(test, wrkr, resp)
lock(print_lock)
try
print_with_color(:white, rpad(test*" ($wrkr)", name_align, " "), " | ")
time_str = @sprintf("%7.2f",resp[2])
print_with_color(:white, rpad(time_str,elapsed_align," "), " | ")
gc_str = @sprintf("%5.2f",resp[5].total_time/10^9)
print_with_color(:white, rpad(gc_str,gc_align," "), " | ")
# since there may be quite a few digits in the percentage,
# the left-padding here is less to make sure everything fits
percent_str = @sprintf("%4.1f",100*resp[5].total_time/(10^9*resp[2]))
print_with_color(:white, rpad(percent_str,percent_align," "), " | ")
alloc_str = @sprintf("%5.2f",resp[3]/2^20)
print_with_color(:white, rpad(alloc_str,alloc_align," "), " | ")
rss_str = @sprintf("%5.2f",resp[6]/2^20)
print_with_color(:white, rpad(rss_str,rss_align," "), "\n")
finally
unlock(print_lock)
end
end
@sync begin
for p in workers()
@async begin
while length(tests) > 0
test = popfirst!(tests)
local resp
wrkr = p
try
resp = remotecall_fetch(runtests, wrkr, test, test_path(test); seed=seed)
catch e
resp = [e]
end
push!(results, (test, resp))
if resp[1] isa Exception
if exit_on_error
skipped = length(tests)
empty!(tests)
end
elseif resp[end] > max_worker_rss
if n > 1
rmprocs(wrkr, waitfor=30)
p = addprocs_with_testenv(1)[1]
remotecall_fetch(include, p, "testdefs.jl")
else # single process testing
error("Halting tests. Memory limit reached : $resp > $max_worker_rss")
end
end
!isa(resp[1], Exception) && print_testworker_stats(test, wrkr, resp)
end
if p != 1
# Free up memory =)
rmprocs(p, waitfor=30)
end
end
end
end
n > 1 && length(node1_tests) > 1 && print("\nExecuting tests that run on node 1 only:\n")
for t in node1_tests
# As above, try to run each test
# which must run on node 1. If
# the test fails, catch the error,
# and either way, append the results
# to the overall aggregator
isolate = true
t == "SharedArrays" && (isolate = false)
local resp
try
resp = eval(Expr(:call, () -> runtests(t, test_path(t), isolate, seed=seed))) # runtests is defined by the include above
print_testworker_stats(t, 1, resp)
catch e
resp = [e]
end
push!(results, (t, resp))
end
#=
` Construct a testset on the master node which will hold results from all the
test files run on workers and on node1. The loop goes through the results,
inserting them as children of the overall testset if they are testsets,
handling errors otherwise.
Since the workers don't return information about passing/broken tests, only
errors or failures, those Result types get passed `nothing` for their test
expressions (and expected/received result in the case of Broken).
If a test failed, returning a `RemoteException`, the error is displayed and
the overall testset has a child testset inserted, with the (empty) Passes
and Brokens from the worker and the full information about all errors and
failures encountered running the tests. This information will be displayed
as a summary at the end of the test run.
If a test failed, returning an `Exception` that is not a `RemoteException`,
it is likely the julia process running the test has encountered some kind
of internal error, such as a segfault. The entire testset is marked as
Errored, and execution continues until the summary at the end of the test
run, where the test file is printed out as the "failed expression".
=#
o_ts = Test.DefaultTestSet("Overall")
Test.push_testset(o_ts)
for res in results
if isa(res[2][1], Test.DefaultTestSet)
Test.push_testset(res[2][1])
Test.record(o_ts, res[2][1])
Test.pop_testset()
elseif isa(res[2][1], Tuple{Int,Int})
fake = Test.DefaultTestSet(res[1])
for i in 1:res[2][1][1]
Test.record(fake, Test.Pass(:test, nothing, nothing, nothing))
end
for i in 1:res[2][1][2]
Test.record(fake, Test.Broken(:test, nothing))
end
Test.push_testset(fake)
Test.record(o_ts, fake)
Test.pop_testset()
elseif isa(res[2][1], RemoteException) && isa(res[2][1].captured.ex, Test.TestSetException)
println("Worker $(res[2][1].pid) failed running test $(res[1]):")
Base.showerror(STDOUT,res[2][1].captured)
fake = Test.DefaultTestSet(res[1])
for i in 1:res[2][1].captured.ex.pass
Test.record(fake, Test.Pass(:test, nothing, nothing, nothing))
end
for i in 1:res[2][1].captured.ex.broken
Test.record(fake, Test.Broken(:test, nothing))
end
for t in res[2][1].captured.ex.errors_and_fails
Test.record(fake, t)
end
Test.push_testset(fake)
Test.record(o_ts, fake)
Test.pop_testset()
elseif isa(res[2][1], Exception)
# If this test raised an exception that is not a remote testset exception,
# i.e. not a RemoteException capturing a TestSetException that means
# the test runner itself had some problem, so we may have hit a segfault,
# deserialization errors or something similar. Record this testset as Errored.
fake = Test.DefaultTestSet(res[1])
Test.record(fake, Test.Error(:test_error, res[1], res[2][1], [], LineNumberNode(1)))
Test.push_testset(fake)
Test.record(o_ts, fake)
Test.pop_testset()
else
error(string("Unknown result type : ", typeof(res)))
end
end
println()
Test.print_test_results(o_ts,1)
if !o_ts.anynonpass
println(" \033[32;1mSUCCESS\033[0m")
else
println(" \033[31;1mFAILURE\033[0m\n")
skipped > 0 &&
println("$skipped test", skipped > 1 ? "s were" : " was", " skipped due to failure.")
println("The global RNG seed was 0x$(hex(seed)).\n")
Test.print_test_errors(o_ts)
throw(Test.FallbackTestSetException("Test run finished with errors"))
end
end
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...