diff --git a/README.md b/README.md index 30d79cc..00586f9 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,9 @@ using ReferenceTests If you put the above code into your `test/runtests.jl` and execute the file in an interactive julia session (i.e. with -`include`), then it will trigger an interactive dialog if the -results don't match. This dialog allows the user to update the -reference files. If you do not want to be prompted, just -delete the reference data before running the tests. +`include`), then it will trigger an interactive dialog if the refrence is missing or the results don't match. +This dialog allows the user to create/update the reference files. +If you do not want to be prompted, set `ENV["JULIA_REFERENCETESTS_UPDATE"]=true` before running. ![readme1](https://user-images.githubusercontent.com/10854026/30002940-3ba480b0-90b6-11e7-93f6-148ac38bd695.png) @@ -91,13 +90,11 @@ test suite. These tests are easy to run via `pkg> test` but the child process used within `pkg> test` is non-interactive, so the update prompt will not show if there are mismatches. -To update references within a package test suite, there are three options: +To update/create references within a package test suite, there are two options: - Set the environment variable `JULIA_REFERENCETESTS_UPDATE` to `"true"` - and run `pkg> test`, which will force update any non-matches. You can then - check changes to any git-tracked reference images before commit. -- Delete any reference images you wish to update and run `pkg> test`, given - that missing references are created automatically. + and run `pkg> test`, which will force update any non-matches and create any missing files. + You can then check changes to any git-tracked reference images before commit. - Run the `test/runtests.jl` interactively. This may be easier using the [`TestEnv.jl`](https://github.com/JuliaTesting/TestEnv.jl) package, given that the test environment used by `pkg> test` is a merge of the diff --git a/src/test_reference.jl b/src/test_reference.jl index 98af86f..9e340a2 100644 --- a/src/test_reference.jl +++ b/src/test_reference.jl @@ -117,9 +117,7 @@ function test_reference( rendermode=nothing; kw...) where {F <: DataFormat, T} - reference_path = reference_file.filename - reference_dir, reference_filename = splitdir(reference_path) - + reference_filename = basename(reference_file.filename) actual = _convert(F, raw_actual; kw...) # infer the default rendermode here @@ -129,14 +127,14 @@ function test_reference( rendermode = default_rendermode(F, actual) end - if !isfile(reference_path) # when reference file doesn't exists - mkpath(reference_dir) - savefile(reference_file, actual) - @info """Reference file for \"$reference_filename\" did not exist. It has been created: + if !isfile(reference_file.filename) # when reference file doesn't exists + actual_path = save_mismatch_file(reference_file, actual) + + @info """Reference file for \"$reference_filename\" did not exist: $(render(rendermode, actual)) - """ new_reference = reference_path + """ actual=actual_path - @info "Please run the tests again for any changes to take effect" + handle_mismatch(reference_file, actual_path) return nothing # skip current test case end @@ -152,33 +150,44 @@ function test_reference( @test true # to increase test counter if reached else # When test fails # Saving actual file so user can look at it - actual_path = joinpath(mismatch_staging_dir(), reference_filename) - actual_file = typeof(reference_file)(actual_path) - savefile(actual_file, actual) + actual_path = save_mismatch_file(reference_file, actual) # Report to user. @info """Reference Test for \"$reference_filename\" failed: $(render(rendermode, reference, actual)) - """ reference = reference_path actual = actual_path - - if !isinteractive() && !force_update() - error(""" - To update the reference images either run the tests interactively with 'include(\"test/runtests.jl\")', - or to force-update all failing reference images set the environment variable `JULIA_REFERENCETESTS_UPDATE` - to "true" and re-run the tests via Pkg. - """) - end - - if force_update() || input_bool("Replace reference with actual result?") - mv(actual_path, reference_path; force=true) # overwrite old file it - @info "Please run the tests again for any changes to take effect" - else - @test false - end + """ reference=reference_file.filename actual=actual_path + handle_mismatch(reference_file, actual_path) + end +end + +function save_mismatch_file(reference_file::File{F}, actual) where F + # Saving actual file so user can look at it + actual_path = joinpath(mismatch_staging_dir(), basename(reference_file.filename)) + actual_file = File{F}(actual_path) + + savefile(actual_file, actual) + return actual_path +end + +function handle_mismatch(reference_file, actual_path) + if !isinteractive() && !force_update() + error(""" + To update the reference images either run the tests interactively with 'include(\"test/runtests.jl\")', + or to force-update/create all failing reference images set the environment variable `JULIA_REFERENCETESTS_UPDATE` + to "true" and re-run all tests. + """) + end + + if force_update() || input_bool("Replace reference with actual result?") + mkpath(dirname(reference_file.filename)) + mv(actual_path, reference_file.filename; force=true) # overwrite old file + @info "Please run the tests again for any changes to take effect" + else + @test false end end -force_update() = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE", "false")) === true +force_update() = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE", "false")) """ mismatch_staging_dir() diff --git a/test/runtests.jl b/test/runtests.jl index e30914a..8691307 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -162,13 +162,15 @@ end mktempdir() do path newfilename = joinpath(path, "newfilename.$ext") @assert !isfile(newfilename) - with_logger(test_logger) do - @test_reference newfilename val # this should create it + withenv("JULIA_REFERENCETESTS_UPDATE"=>true) do + with_logger(test_logger) do + @test_reference newfilename val # this should create it + end end @test isfile(newfilename) # Was created @test_reference newfilename val # Matches expected content end -end + end @testset "Create new image as txt" begin # This is a separate testset as need to use the `size` argument to ``@test_reference` @@ -176,7 +178,9 @@ end newfilename = joinpath(path, "new_camera.txt") @assert !isfile(newfilename) with_logger(test_logger) do - @test_reference newfilename camera size=(5,10) # this should create it + withenv("JULIA_REFERENCETESTS_UPDATE"=>true) do + @test_reference newfilename camera size=(5,10) # this should create it + end end @test isfile(newfilename) # Was created @test_reference newfilename camera size=(5,10) # Matches expected content