dispatch test_reference pipeline according to TestMode

src/ReferenceTests.jl

@@ -15,6 +15,7 @@ export
     @test_reference,
     psnr_equality
 
+include("testmode.jl")
 include("utils.jl")
 include("test_reference.jl")
 include("fileio.jl")

src/test_reference.jl

@@ -99,6 +99,7 @@ function test_reference(
 
     path = file.filename
     dir, filename = splitdir(path)
+    testmode = TESTMODE()
 
     # infer the default rendermode here
     # since `nothing` is always passed to this method from
@@ -112,19 +113,11 @@ function test_reference(
         println("Reference file for \"$filename\" does not exist.")
         render(rendermode, raw_actual)
 
-        if !isinteractive()
-            error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images")
-        end
-
-        if !input_bool("Create reference file with above content (path: $path)?")
+        continue_test = _create_new_reference(testmode, file, raw_actual)
+        if !continue_test
             @test false
-        else
-            mkpath(dir)
-            savefile(file, raw_actual)
-            @info("Please run the tests again for any changes to take effect")
+            return nothing
         end
-
-        return nothing # skip current test case
     end
 
     # file exists
@@ -137,21 +130,63 @@ function test_reference(
     end
 
     if equiv(reference, actual)
-        @test true # to increase test counter if reached
+        @test true
     else
         # post-processing when test fails
         println("Test for \"$filename\" failed.")
         render(rendermode, reference, actual)
 
-        if !isinteractive()
-            error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images")
-        end
+        increase_fail_count = _update_reference(testmode, file, reference, actual)
+        increase_fail_count && @test false
+    end
+end
 
-        if !input_bool("Replace reference with actual result (path: $path)?")
-            @test false
-        else
-            savefile(file, actual)
-            @info("Please run the tests again for any changes to take effect")
-        end
+function _create_new_reference(::NonInteractiveMode, file::File, actual)::Bool
+    error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images")
+
+    # # automatically create new reference file and continue test
+    # path = file.filename
+    # dir, filename = splitdir(path)
+
+    # println("Create new reference file \"$filename\".")
+    # mkpath(dir)
+    # savefile(file, actual)
+
+    # continue_test = true
+    # return continue_test
+end
+
+function _create_new_reference(::InteractiveMode, file::File, actual)::Bool
+    path = file.filename
+    dir = splitdir(path)[1]
+
+    if !input_bool("Create reference file with above content (path: $path)?")
+        continue_test = false
+    else
+        mkpath(dir)
+        savefile(file, actual)
+        continue_test = true
+    end
+    return continue_test
+end
+
+function _update_reference(::NonInteractiveMode, file::File, reference, actual)::Bool
+    error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images")
+
+    # # Do not update reference
+    # increase_fail_count = true
+    # return increase_fail_count
+end
+
+function _update_reference(::InteractiveMode, file::File, reference, actual)::Bool
+    path = file.filename
+
+    if !input_bool("Replace reference with actual result (path: $path)?")
+        increase_fail_count = true
+    else
+        savefile(file, actual)
+        @info("Please run the tests again for any changes to take effect")
+        increase_fail_count = false
     end
+    return increase_fail_count
 end

src/testmode.jl

@@ -0,0 +1,47 @@
+#################################################
+# Test mode
+# This controls how test cases are handled
+abstract type TestMode end
+struct InteractiveMode <: TestMode end
+struct NonInteractiveMode <: TestMode end
+
+"""
+Predefined CI environment variables
+
+# References
+
+* Travis: https://docs.travis-ci.com/user/environment-variables/#default-environment-variables
+* Appveyor: https://www.appveyor.com/docs/environment-variables/
+"""
+const CI_ENVIRONMENTS = Dict(
+    "CI" => "true",
+    "APPVEYOR" => "true",
+    "TRAVIS" => "true",
+    "CONTINUOUS_INTEGRATION" => "true",
+    "DEBIAN_FRONTEND" => "noninteractive"
+)
+
+function TESTMODE()
+    global GLOBAL_TESTMODE
+    if !isdefined(ReferenceTests, :GLOBAL_TESTMODE)
+        # it's only called once in runtime
+        GLOBAL_TESTMODE = _get_testmode()
+    end
+    return GLOBAL_TESTMODE
+end
+
+function _get_testmode()
+    # test if this package is used in a CI environment
+    common_keys = collect(intersect(keys(ENV), keys(CI_ENVIRONMENTS)))
+    matched_envs = map(common_keys) do k
+        # some variables might have different cases in different CI platforms
+        # e.g., in Appveyor, ENV["CI] is "True" in Windows and "true" in Ubuntu.
+        lowercase(ENV[k])==lowercase(CI_ENVIRONMENTS[k])
+    end
+    has_testenv = any(matched_envs)
+    has_testenv && return NonInteractiveMode()
+
+    # fallback
+    @info "You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update references."
+    return isinteractive() ? InteractiveMode() : NonInteractiveMode()
+end