From 7a316682dffc3e43eccac1b21624277bd2dd857c Mon Sep 17 00:00:00 2001 From: Lyndon White Date: Tue, 3 Jul 2018 13:34:19 +0800 Subject: [PATCH 1/3] Add function to isolate tests --- test/runtests.jl | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 86467c8..1f16249 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -15,6 +15,26 @@ copy!(view(cameras,:,:,2), camera) square = Gray{N0f8}[0.1 0.2 0.3; 0.4 0.5 0.6; 0.7 0.6 0.9] rgb_rect = rand(RGB{N0f8}, 2, 3) + +""" +Run the encludes code in a sperate process, +and make sure that process fails. +""" +macro ensure_fails_tests(expr) + quote + code = quote + using Base.Test, + using ReferenceTests + $expr + end + command = (Base.cmd_gen(((Base.julia_cmd(),), ("-e",), (code,)))) + ps = spawn(command) + wait(ps) # don't actually run it Async + kill(ps) + @test ps.exitcode != 0 + end +end + @testset "io2str" begin @test_throws ArgumentError eval(@macroexpand @io2str(::IO)) @test_throws ArgumentError @io2str(2) @@ -50,11 +70,11 @@ end multiline string that does indeed end with a new line. """ - @test_throws ErrorException @test_reference "references/string1.txt" "intentionally wrong to check that this message prints" - @test_throws ErrorException @test_reference "references/wrong.txt" "intentional error to check that this message prints" - @test_throws ErrorException @test_reference "references/string5.txt" """ + @ensure_fails_tests(@test_reference "references/string1.txt" "intentionally wrong to check that this message prints") + @ensure_fails_tests(@test_reference "references/wrong.txt" "intentional error to check that this message prints") + @ensure_fails_tests(@test_reference "references/string5.txt" """ This is an incorrect - multiline string that does not end with a new line.""" + multiline string that does not end with a new line.""") end @testset "images as txt using ImageInTerminal" begin @@ -78,13 +98,13 @@ end @testset "images as PNG" begin @test_reference "references/camera.png" imresize(camera, (64,64)) - @test_throws ErrorException @test_reference "references/wrongfilename.png" imresize(camera, (64,64)) - @test_throws ErrorException @test_reference "references/camera.png" imresize(lena, (64,64)) + @ensure_fails_tests(@test_reference "references/wrongfilename.png" imresize(camera, (64,64))) + @ensure_fails_tests(@test_reference "references/camera.png" imresize(lena, (64,64))) end using DataFrames, CSVFiles @testset "DataFrame as CSV" begin @test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["a","b","c"]) - @test_throws ErrorException @test_reference "references/wrongfilename.csv" DataFrame(v1=[1,2,3], v2=["a","b","c"]) - @test_throws ErrorException @test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["c","b","c"]) + @ensure_fails_tests(@test_reference "references/wrongfilename.csv" DataFrame(v1=[1,2,3], v2=["a","b","c"])) + @ensure_fails_tests(@test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["c","b","c"])) end From 9449f20400be1e239d7c79b3e88b5ead080e96c6 Mon Sep 17 00:00:00 2001 From: Lyndon White Date: Tue, 3 Jul 2018 13:35:49 +0800 Subject: [PATCH 2/3] fail tests no throw error --- src/core.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core.jl b/src/core.jl index 97977b7..3200d8d 100644 --- a/src/core.jl +++ b/src/core.jl @@ -94,7 +94,8 @@ function _test_reference(equiv, rendermode, file::File, actual::T) where T @test false end else - error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images") + warn("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images") + @test false end end catch ex @@ -113,7 +114,8 @@ function _test_reference(equiv, rendermode, file::File, actual::T) where T @test false end else - error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images") + warn("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images") + @test false end else rethrow(ex) From 56457b609d0043c0501f89d74a73fb74f6629353 Mon Sep 17 00:00:00 2001 From: Lyndon White Date: Tue, 3 Jul 2018 14:04:37 +0800 Subject: [PATCH 3/3] Use a nonpropergating testset --- test/runtests.jl | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 1f16249..6f1b521 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -15,24 +15,27 @@ copy!(view(cameras,:,:,2), camera) square = Gray{N0f8}[0.1 0.2 0.3; 0.4 0.5 0.6; 0.7 0.6 0.9] rgb_rect = rand(RGB{N0f8}, 2, 3) +struct NonPropagatingTestSet <: AbstractTestset + description::String + results::Vector +end +NonPropagatingTestSet(desc) = NonPropagatingTestSet(desc, []) +record(ts::NonPropagatingTestSet, t) = (push!(ts.results, t); t) +function finish(ts::ReportingTestSet) + # Teststs are supposed to throw errors if they have failures and no parent testset + # or if they do have a parent, then push themselves into the parents results + # but we are going to do none of that. +end + -""" -Run the encludes code in a sperate process, -and make sure that process fails. -""" macro ensure_fails_tests(expr) - quote - code = quote - using Base.Test, - using ReferenceTests - $expr - end - command = (Base.cmd_gen(((Base.julia_cmd(),), ("-e",), (code,)))) - ps = spawn(command) - wait(ps) # don't actually run it Async - kill(ps) - @test ps.exitcode != 0 + quote + ts = @testset NonPropagatingTestSet "Ensure Fails" begin + $expr end + @assert length(ts.results)==1 + @test ts.results[1] isa Test.Fail + end end @testset "io2str" begin