Add test_warn macro to Base.Test for checking for warnings (#19903)

* add test_warn macro to Base.Test for verifying warning messages

* use test_warn in compilation tests

* add test for #18725

* doc links for test_warn

* compile test no longer needs to save STDERR

* rewrite pkg tests in terms of test_nowarn

Author: stevengj

NEWS.md

@@ -146,6 +146,9 @@ Library improvements
     implemented; the semantics are as if the `Nullable` were a container with
     zero or one elements ([#16961]).
 
+  * New `@test_warn` and `@test_nowarn` macros in the `Base.Test` module to
+    test for the presence or absence of warning messages ([#19903]).
+
   * `logging` can be used to redirect `info`, `warn`, and `error` messages
     either universally or on a per-module/function basis ([#16213]).
 

base/test.jl

@@ -13,7 +13,7 @@ and summarize them at the end of the test set with `@testset`.
 """
 module Test
 
-export @test, @test_throws, @test_broken, @test_skip
+export @test, @test_throws, @test_broken, @test_skip, @test_warn, @test_nowarn
 export @testset
 # Legacy approximate testing functions, yet to be included
 export @test_approx_eq_eps, @inferred
@@ -356,6 +356,57 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype)
     record(get_testset(), testres)
 end
 
+#-----------------------------------------------------------------------
+# Test for warning messages
+
+ismatch_warn(s::AbstractString, output) = contains(output, s)
+ismatch_warn(s::Regex, output) = ismatch(s, output)
+ismatch_warn(s::Function, output) = s(output)
+ismatch_warn(S::Union{AbstractArray,Tuple}, output) = all(s -> ismatch_warn(s, output), S)
+
+"""
+    @test_warn msg expr
+
+Test whether evaluating `expr` results in [`STDERR`](@ref) output that contains
+the `msg` string or matches the `msg` regular expression.  If `msg` is
+a boolean function, tests whether `msg(output)` returns `true`.  If `msg` is a
+tuple or array, checks that the error output contains/matches each item in `msg`.
+Returns the result of evaluating `expr`.
+
+See also [`@test_nowarn`](@ref) to check for the absence of error output.
+"""
+macro test_warn(msg, expr)
+    quote
+        let fname = tempname(), have_color = Base.have_color
+            try
+                eval(Base, :(have_color = false))
+                ret = open(fname, "w") do f
+                    redirect_stderr(f) do
+                        $(esc(expr))
+                    end
+                end
+                @test ismatch_warn($(esc(msg)), readstring(fname))
+                ret
+            finally
+                eval(Base, Expr(:(=), :have_color, have_color))
+                rm(fname, force=true)
+            end
+        end
+    end
+end
+
+"""
+    @test_nowarn expr
+
+Test whether evaluating `expr` results in empty [`STDERR`](@ref) output
+(no warnings or other messages).  Returns the result of evaluating `expr`.
+"""
+macro test_nowarn(expr)
+    quote
+        @test_warn r"^(?!.)"s $expr
+    end
+end
+
 #-----------------------------------------------------------------------
 
 # The AbstractTestSet interface is defined by two methods:

doc/src/stdlib/test.md

@@ -171,6 +171,8 @@ checks using either `@test a ≈ b` (where `≈`, typed via tab completion of `\
 
 ```@docs
 Base.Test.@inferred
+Base.Test.@test_warn
+Base.Test.@test_nowarn
 ```
 
 ## Broken Tests

test/compile.jl

@@ -2,20 +2,6 @@
 
 using Base.Test
 
-function redirected_stderr(expected)
-    rd, wr = redirect_stderr()
-    t = @async begin
-        read = readstring(rd) # also makes sure the kernel isn't being forced to buffer the output
-        if !contains(read, expected)
-            @show expected
-            @show read
-            @test false
-        end
-        nothing
-    end
-    return t
-end
-
 Foo_module = :Foo4b3a94a1a081a8cb
 FooBase_module = :FooBase4b3a94a1a081a8cb
 @eval module ConflictingBindings
@@ -29,7 +15,6 @@ using .ConflictingBindings
 # so we disable it for the tests below
 withenv( "JULIA_DEBUG_LOADING" => nothing ) do
 
-olderr = STDERR
 dir = mktempdir()
 dir2 = mktempdir()
 insert!(LOAD_PATH, 1, dir)
@@ -136,14 +121,9 @@ try
 
     # use _require_from_serialized to ensure that the test fails if
     # the module doesn't reload from the image:
-    t = redirected_stderr("WARNING: replacing module Foo4b3a94a1a081a8cb.\nWARNING: Method definition ")
-    try
+    @test_warn "WARNING: replacing module Foo4b3a94a1a081a8cb.\nWARNING: Method definition " begin
         @test isa(Base._require_from_serialized(myid(), Foo_module, cachefile, #=broadcast-load=#false), Array{Any,1})
-    finally
-        close(STDERR)
-        redirect_stderr(olderr)
     end
-    wait(t)
 
     let Foo = getfield(Main, Foo_module)
         @test_throws MethodError Foo.foo(17) # world shouldn't be visible yet
@@ -217,17 +197,13 @@ try
           end
           """)
 
-    t = redirected_stderr("ERROR: LoadError: Declaring __precompile__(false) is not allowed in files that are being precompiled.\nStacktrace:\n [1] __precompile__")
-    try
+    @test_warn "ERROR: LoadError: Declaring __precompile__(false) is not allowed in files that are being precompiled.\nStacktrace:\n [1] __precompile__" try
         Base.compilecache("Baz") # from __precompile__(false)
         error("__precompile__ disabled test failed")
     catch exc
-        close(STDERR)
-        redirect_stderr(olderr)
         isa(exc, ErrorException) || rethrow(exc)
         !isempty(search(exc.msg, "__precompile__(false)")) && rethrow(exc)
     end
-    wait(t)
 
     # Issue #12720
     FooBar1_file = joinpath(dir, "FooBar1.jl")
@@ -273,14 +249,7 @@ try
     fb_uuid1 = Base.module_uuid(Main.FooBar1)
     @test fb_uuid != fb_uuid1
 
-    t = redirected_stderr("WARNING: replacing module FooBar.")
-    try
-        reload("FooBar")
-    finally
-        close(STDERR)
-        redirect_stderr(olderr)
-    end
-    wait(t)
+    @test_warn "WARNING: replacing module FooBar." reload("FooBar")
     @test fb_uuid != Base.module_uuid(Main.FooBar)
     @test fb_uuid1 == Base.module_uuid(Main.FooBar1)
     fb_uuid = Base.module_uuid(Main.FooBar)
@@ -289,14 +258,7 @@ try
     @test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji"))
     @test !Base.stale_cachefile(FooBar_file, joinpath(dir2, "FooBar.ji"))
 
-    t = redirected_stderr("WARNING: replacing module FooBar1.")
-    try
-        reload("FooBar1")
-    finally
-        close(STDERR)
-        redirect_stderr(olderr)
-    end
-    wait(t)
+    @test_warn "WARNING: replacing module FooBar1." reload("FooBar1")
     @test fb_uuid == Base.module_uuid(Main.FooBar)
     @test fb_uuid1 != Base.module_uuid(Main.FooBar1)
 
@@ -314,22 +276,14 @@ try
           error("break me")
           end
           """)
-    t = redirected_stderr("ERROR: LoadError: break me\nStacktrace:\n [1] error")
-    try
+    @test_warn "ERROR: LoadError: break me\nStacktrace:\n [1] error" try
         Base.require(:FooBar)
         error("\"LoadError: break me\" test failed")
     catch exc
-        close(STDERR)
-        redirect_stderr(olderr)
         isa(exc, ErrorException) || rethrow(exc)
         !isempty(search(exc.msg, "ERROR: LoadError: break me")) && rethrow(exc)
     end
-    wait(t)
 finally
-    if STDERR != olderr
-        close(STDERR)
-        redirect_stderr(olderr)
-    end
     splice!(Base.LOAD_CACHE_PATH, 1:2)
     splice!(LOAD_PATH, 1)
     rm(dir, recursive=true)

test/core.jl

@@ -4822,6 +4822,17 @@ end
 @test f14893() == 14893
 @test M14893.f14893() == 14893
 
+# issue #18725
+@test_nowarn eval(Main, :(begin
+    f18725(x) = 1
+    f18725(x) = 2
+end))
+@test Main.f18725(0) == 2
+@test_warn "WARNING: Method definition f18725(Any) in module Module18725" eval(Main, :(module Module18725
+    f18725(x) = 1
+    f18725(x) = 2
+end))
+
 # issue #19599
 f19599{T}(x::((S)->Vector{S})(T)...) = 1
 @test f19599([1],[1]) == 1