From 6b3e9032bedbfe393f985ccbbef9e04300ca4cbc Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Sun, 6 Oct 2019 14:09:41 +0800 Subject: [PATCH] dispatch test_reference pipeline according to TestMode --- src/ReferenceTests.jl | 1 + src/test_reference.jl | 77 +++++++++++++++++++++++++++++++------------ src/testmode.jl | 47 ++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 21 deletions(-) create mode 100644 src/testmode.jl diff --git a/src/ReferenceTests.jl b/src/ReferenceTests.jl index ef7c750..949ea2d 100644 --- a/src/ReferenceTests.jl +++ b/src/ReferenceTests.jl @@ -15,6 +15,7 @@ export @test_reference, psnr_equality +include("testmode.jl") include("utils.jl") include("test_reference.jl") include("fileio.jl") diff --git a/src/test_reference.jl b/src/test_reference.jl index d9cdb31..42789de 100644 --- a/src/test_reference.jl +++ b/src/test_reference.jl @@ -99,6 +99,7 @@ function test_reference( path = file.filename dir, filename = splitdir(path) + testmode = TESTMODE() # infer the default rendermode here # since `nothing` is always passed to this method from @@ -112,19 +113,11 @@ function test_reference( println("Reference file for \"$filename\" does not exist.") render(rendermode, raw_actual) - if !isinteractive() - error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images") - end - - if !input_bool("Create reference file with above content (path: $path)?") + continue_test = _create_new_reference(testmode, file, raw_actual) + if !continue_test @test false - else - mkpath(dir) - savefile(file, raw_actual) - @info("Please run the tests again for any changes to take effect") + return nothing end - - return nothing # skip current test case end # file exists @@ -137,21 +130,63 @@ function test_reference( end if equiv(reference, actual) - @test true # to increase test counter if reached + @test true else # post-processing when test fails println("Test for \"$filename\" failed.") render(rendermode, reference, actual) - if !isinteractive() - error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images") - end + increase_fail_count = _update_reference(testmode, file, reference, actual) + increase_fail_count && @test false + end +end - if !input_bool("Replace reference with actual result (path: $path)?") - @test false - else - savefile(file, actual) - @info("Please run the tests again for any changes to take effect") - end +function _create_new_reference(::NonInteractiveMode, file::File, actual)::Bool + error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images") + + # # automatically create new reference file and continue test + # path = file.filename + # dir, filename = splitdir(path) + + # println("Create new reference file \"$filename\".") + # mkpath(dir) + # savefile(file, actual) + + # continue_test = true + # return continue_test +end + +function _create_new_reference(::InteractiveMode, file::File, actual)::Bool + path = file.filename + dir = splitdir(path)[1] + + if !input_bool("Create reference file with above content (path: $path)?") + continue_test = false + else + mkpath(dir) + savefile(file, actual) + continue_test = true + end + return continue_test +end + +function _update_reference(::NonInteractiveMode, file::File, reference, actual)::Bool + error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images") + + # # Do not update reference + # increase_fail_count = true + # return increase_fail_count +end + +function _update_reference(::InteractiveMode, file::File, reference, actual)::Bool + path = file.filename + + if !input_bool("Replace reference with actual result (path: $path)?") + increase_fail_count = true + else + savefile(file, actual) + @info("Please run the tests again for any changes to take effect") + increase_fail_count = false end + return increase_fail_count end diff --git a/src/testmode.jl b/src/testmode.jl new file mode 100644 index 0000000..6e621c4 --- /dev/null +++ b/src/testmode.jl @@ -0,0 +1,47 @@ +################################################# +# Test mode +# This controls how test cases are handled +abstract type TestMode end +struct InteractiveMode <: TestMode end +struct NonInteractiveMode <: TestMode end + +""" +Predefined CI environment variables + +# References + +* Travis: https://docs.travis-ci.com/user/environment-variables/#default-environment-variables +* Appveyor: https://www.appveyor.com/docs/environment-variables/ +""" +const CI_ENVIRONMENTS = Dict( + "CI" => "true", + "APPVEYOR" => "true", + "TRAVIS" => "true", + "CONTINUOUS_INTEGRATION" => "true", + "DEBIAN_FRONTEND" => "noninteractive" +) + +function TESTMODE() + global GLOBAL_TESTMODE + if !isdefined(ReferenceTests, :GLOBAL_TESTMODE) + # it's only called once in runtime + GLOBAL_TESTMODE = _get_testmode() + end + return GLOBAL_TESTMODE +end + +function _get_testmode() + # test if this package is used in a CI environment + common_keys = collect(intersect(keys(ENV), keys(CI_ENVIRONMENTS))) + matched_envs = map(common_keys) do k + # some variables might have different cases in different CI platforms + # e.g., in Appveyor, ENV["CI] is "True" in Windows and "true" in Ubuntu. + lowercase(ENV[k])==lowercase(CI_ENVIRONMENTS[k]) + end + has_testenv = any(matched_envs) + has_testenv && return NonInteractiveMode() + + # fallback + @info "You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update references." + return isinteractive() ? InteractiveMode() : NonInteractiveMode() +end