Distinguish between test errors and test failures

A test error occurs when an error is thrown or an assert fails outside
of the test. While test failures only occur when an assert fails inside
a test.

Call setfenv on 'assert' to replace 'error' with 'fail', such that
'fail' throws a special failure table instead of a string. Hence,
assertion failures will throw a failure table and are distinct from
errors. Additionally, the failure table contains the same error string
that would have been thrown with 'error'.

Author: o-lim

Diff for: busted/compatibility.lua

@@ -1,4 +1,17 @@
 return {
+  getfenv = getfenv or function(f)
+    f = (type(f) == 'function' and f or debug.getinfo(f + 1, 'f').func)
+    local name, value
+    local up = 0
+
+    repeat
+      up = up + 1
+      name, value = debug.getupvalue(f, up)
+    until name == '_ENV' or name == nil
+
+    return (name and value or _G)
+  end,
+
   setfenv = setfenv or function(f, t)
     f = (type(f) == 'function' and f or debug.getinfo(f + 1, 'f').func)
     local name

Diff for: busted/core.lua

@@ -1,3 +1,24 @@
+local metatype = function(obj)
+  local otype = type(obj)
+  if otype == 'table' then
+    local mt = getmetatable(obj)
+    if mt and mt.__type then
+      return mt.__type
+    end
+  end
+  return otype
+end
+
+local failureMt = {
+  __index = {},
+  __tostring = function(e) return e.message end,
+  __type = 'failure'
+}
+
+local getfenv = require 'busted.compatibility'.getfenv
+local setfenv = require 'busted.compatibility'.setfenv
+local throw = error
+
 return function()
   local mediator = require 'mediator'()
 
@@ -16,6 +37,12 @@ return function()
     level = level or  3
 
     local info = debug.getinfo(level, 'Sl')
+    while info.what == 'C' or info.short_src:match('luassert[/\\].*%.lua$') or
+          info.short_src:match('busted[/\\].*%.lua$') do
+      level = level + 1
+      info = debug.getinfo(level, 'Sl')
+    end
+
     info.traceback = debug.traceback('', level)
     info.message = msg
 
@@ -64,6 +91,21 @@ return function()
     return parent
   end
 
+  function busted.fail(msg, level)
+    local _, emsg = pcall(error, msg, level+2)
+    local e = { message = emsg }
+    setmetatable(e, failureMt)
+    throw(e, level+1)
+  end
+
+  function busted.replaceErrorWithFail(callable)
+    local env = {}
+    local f = getmetatable(callable).__call or callable
+    setmetatable(env, { __index = getfenv(f) })
+    env.error = busted.fail
+    setfenv(f, env)
+  end
+
   function busted.safe(descriptor, run, element, setenv)
     if setenv and (type(run) == 'function' or getmetatable(run).__call) then
       -- prioritize __call if it exists, like in files
@@ -72,15 +114,19 @@ return function()
 
     busted.context.push(element)
     local trace, message
+    local status = 'success'
 
     local ret = { xpcall(run, function(msg)
-      message = busted.rewriteMessage(element, msg)
+      local errType = metatype(msg)
+      status = (errType == 'string' and 'error' or errType)
+      message = busted.rewriteMessage(element, tostring(msg))
       trace = busted.getTrace(element, 3, msg)
     end) }
 
     if not ret[1] then
-      busted.publish({ 'error', descriptor }, element, busted.context.parent(element), message, trace)
+      busted.publish({ status, descriptor }, element, busted.context.parent(element), message, trace)
     end
+    ret[1] = status
 
     busted.context.pop()
     return unpack(ret)

Diff for: busted/init.lua

@@ -41,7 +41,7 @@ return function(busted)
   local file = function(file)
     busted.publish({ 'file', 'start' }, file.name)
 
-    if busted.safe('file', file.run, file, true) then
+    if busted.safe('file', file.run, file, true) == 'success' then
       busted.execute(file)
     end
 
@@ -60,7 +60,7 @@ return function(busted)
       randomize = true
     end
 
-    if busted.safe('describe', describe.run, describe) then
+    if busted.safe('describe', describe.run, describe) == 'success' then
       if randomize then
         shuffle(busted.context.children(describe))
       end
@@ -87,9 +87,9 @@ return function(busted)
 
     busted.publish({ 'test', 'start' }, element, parent)
 
-    local res = busted.safe('element', element.run, element)
+    local status = busted.safe('element', element.run, element)
     if not element.env.done then
-      busted.publish({ 'test', 'end' }, element, parent, res and 'success' or 'failure')
+      busted.publish({ 'test', 'end' }, element, parent, status)
       if finally then busted.safe('finally', finally, element) end
       dexecAll('after_each', parent, true)
     end
@@ -119,5 +119,8 @@ return function(busted)
   mock   = require 'luassert.mock'
   stub   = require 'luassert.stub'
 
+  busted.replaceErrorWithFail(assert)
+  busted.replaceErrorWithFail(assert.True)
+
   return busted
 end

Diff for: busted/outputHandlers/base.lua

@@ -27,6 +27,7 @@ return function(busted)
     busted.subscribe({ 'suite', 'end' }, handler.baseSuiteEnd)
     busted.subscribe({ 'test', 'start' }, handler.baseTestStart, { predicate = handler.cancelOnPending })
     busted.subscribe({ 'test', 'end' }, handler.baseTestEnd, { predicate = handler.cancelOnPending })
+    busted.subscribe({ 'failure' }, handler.baseError)
     busted.subscribe({ 'error' }, handler.baseError)
   end
 
@@ -87,6 +88,7 @@ return function(busted)
 
   handler.baseTestEnd = function(element, parent, status, debug)
 
+    local isError
     local insertTable
     local id = tostring(element)
 
@@ -99,9 +101,13 @@ return function(busted)
     elseif status == 'failure' then
       insertTable = handler.failures
       handler.failuresCount = handler.failuresCount + 1
+    elseif status == 'error' then
+      insertTable = handler.errors
+      handler.errorsCount = handler.errorsCount + 1
+      isError = true
     end
 
-    insertTable[id] = handler.format(element, parent, nil, debug)
+    insertTable[id] = handler.format(element, parent, nil, debug, isError)
 
     if handler.inProgress[id] then
       for k, v in pairs(handler.inProgress[id]) do

Diff for: busted/outputHandlers/plainTerminal.lua

@@ -29,6 +29,8 @@ return function(options, busted)
       if failure.message then
         string = string .. ' → ' ..  failure.message .. '\n'
       end
+      string = string ..
+        handler.getFullName(failure)
     else
       string = string ..
         failure.trace.short_src .. ' @ ' ..
@@ -44,7 +46,7 @@ return function(options, busted)
       end
     end
 
-    if options.verbose and failure.trace.traceback then
+    if options.verbose and failure.trace and failure.trace.traceback then
       string = string .. '\n' .. failure.trace.traceback
     end
 
@@ -104,6 +106,8 @@ return function(options, busted)
         string = pendingDot
       elseif status == 'failure' then
         string = failureDot
+      elseif status == 'error' then
+        string = errorDot
       end
 
       io.write(string)

Diff for: busted/outputHandlers/utfTerminal.lua

@@ -30,6 +30,8 @@ return function(options, busted)
       if failure.message then
         string = string .. ' → ' ..  ansicolors('%{cyan}' .. failure.message) .. '\n'
       end
+      string = string ..
+        ansicolors('%{bright}' .. handler.getFullName(failure))
     else
       string = string ..
         ansicolors('%{cyan}' .. failure.trace.short_src) .. ' @ ' ..
@@ -45,7 +47,7 @@ return function(options, busted)
       end
     end
 
-    if options.verbose and failure.trace.traceback then
+    if options.verbose and failure.trace and failure.trace.traceback then
       string = string .. '\n' .. failure.trace.traceback
     end
 
@@ -105,6 +107,8 @@ return function(options, busted)
         string = pendingDot
       elseif status == 'failure' then
         string = failureDot
+      elseif status == 'error' then
+        string = errorDot
       end
 
       io.write(string)
@@ -143,7 +147,7 @@ return function(options, busted)
     return nil, true
   end
 
-  local s = busted.subscribe({ 'test', 'end' }, handler.testEnd, { predicate = handler.cancelOnPending })
+  busted.subscribe({ 'test', 'end' }, handler.testEnd, { predicate = handler.cancelOnPending })
   busted.subscribe({ 'suite', 'end' }, handler.suiteEnd)
   busted.subscribe({ 'error', 'file' }, handler.error)
   busted.subscribe({ 'error', 'describe' }, handler.error)

Diff for: spec/cl_errors.lua

@@ -0,0 +1,14 @@
+-- supporting testfile; belongs to 'cl_spec.lua'
+
+describe('Tests the busted error detection through the commandline', function()
+
+  it('is a test that throws an error #testerr', function()
+    error('force an error')
+  end)
+
+  it('is a test with a Lua error #luaerr', function()
+    local foo
+    foo.bar = nil
+  end)
+end)
+

Diff for: spec/cl_spec.lua

@@ -36,6 +36,13 @@ local execute = function(cmd)
   return not not success, modexit(exitcode)
 end
 
+local run = function(cmd)
+  local p = io.popen(cmd, 'r')
+  local out = p:read('*a')
+  p:close()
+  return out
+end
+
 
 it('Tests the busted command-line options', function()
 
@@ -178,5 +185,121 @@ describe('Tests failing tests through the commandline', function()
     assert.is_equal(8, exitcode)
     error_end()
   end)
+
+  it('tests failing support functions as errors', function()
+    error_start()
+    local result = run('bin/busted --output=plainTerminal --pattern=cl_failing_support.lua$')
+    local _, numErrors = result:gsub('Error → .-\n','')
+    assert.is_equal(8, numErrors)
+    error_end()
+  end)
+end)
+
+describe('Tests distinguish between errors and failures', function()
+  it('by detecting errors as test errors', function()
+    error_start()
+    local result = run('bin/busted --output=plainTerminal --pattern=cl_errors.lua$ --tags=testerr')
+    local errmsg = result:match('(Error → .-)\n')
+    assert.is_truthy(errmsg)
+    error_end()
+  end)
+
+  it('by detecting assert failures as test failures', function()
+    error_start()
+    local result = run('bin/busted --output=plainTerminal --pattern=cl_two_failures.lua$')
+    local failmsg = result:match('(Failure → .-)\n')
+    assert.is_truthy(failmsg)
+    error_end()
+  end)
+
+  it('by detecting Lua runtime errors as test errors', function()
+    error_start()
+    local result = run('bin/busted --output=plainTerminal --pattern=cl_errors.lua$ --tags=luaerr')
+    local failmsg = result:match('(Error → .-)\n')
+    assert.is_truthy(failmsg)
+    error_end()
+  end)
+end)
+
+describe('Tests stack trackback', function()
+  it('when throwing an error', function()
+    error_start()
+    local result = run('bin/busted --verbose --pattern=cl_errors.lua$ --tags=testerr')
+    local errmsg = result:match('(stack traceback:.*)\n')
+    local expected = [[stack traceback:
+	./spec/cl_errors.lua:6: in function <./spec/cl_errors.lua:5>
+]]
+    assert.is_equal(expected, errmsg)
+    error_end()
+  end)
+
+  it('when assertion fails', function()
+    error_start()
+    local result = run('bin/busted --verbose --pattern=cl_two_failures.lua$ --tags=err1')
+    local errmsg = result:match('(stack traceback:.*)\n')
+    local expected = [[stack traceback:
+	./spec/cl_two_failures.lua:6: in function <./spec/cl_two_failures.lua:5>
+]]
+    assert.is_equal(expected, errmsg)
+    error_end()
+  end)
+
+  it('when Lua runtime error', function()
+    error_start()
+    local result = run('bin/busted --verbose --pattern=cl_errors.lua$ --tags=luaerr')
+    local errmsg = result:match('(stack traceback:.*)\n')
+    local expected = [[stack traceback:
+	./spec/cl_errors.lua:11: in function <./spec/cl_errors.lua:9>
+]]
+    assert.is_equal(expected, errmsg)
+    error_end()
+  end)
+end)
+
+describe('Tests error messages through the command line', function()
+  it('when throwing errors in a test', function()
+    error_start()
+    local result = run('bin/busted --output=plainTerminal --pattern=cl_errors.lua$ --tags=testerr')
+    local errmsg = result:match('(Error → .-)\n')
+    local expected = "Error → ./spec/cl_errors.lua:6: force an error"
+    assert.is_equal(expected, errmsg)
+    error_end()
+  end)
+
+  it('when running a non-compiling testfile', function()
+    error_start()
+    local result = run('bin/busted --output=plainTerminal --pattern=cl_compile_fail.lua$')
+    local errmsg = result:match('(Error → .-)\n')
+    local expected = "Error → ./spec/cl_compile_fail.lua:3: '=' expected near 'here'"
+    assert.is_equal(expected, errmsg)
+    error_end()
+  end)
+
+  it('when a testfile throws errors', function()
+    error_start()
+    local result = run('bin/busted --output=plainTerminal --pattern=cl_execute_fail.lua$')
+    local errmsg = result:match('(Error → .-)\n')
+    local expected = 'Error → ./spec/cl_execute_fail.lua:4: This compiles fine, but throws an error when being run'
+    assert.is_equal(expected, errmsg)
+    error_end()
+  end)
+
+  it('when output library not found', function()
+    error_start()
+    local result = run('bin/busted --pattern=cl_two_failures.lua$ --output=not_found_here')
+    local errmsg = result:match('(.-)\n')
+    local expected = 'Cannot load output library: not_found_here'
+    assert.is_equal(expected, errmsg)
+    error_end()
+  end)
+
+  it('when no test files matching Lua pattern', function()
+    error_start()
+    local result = run('bin/busted --output=plainTerminal --pattern=this_filename_does_simply_not_exist$')
+    local errmsg = result:match('(.-)\n')
+    local expected = 'No test files found matching Lua pattern: this_filename_does_simply_not_exist$'
+    assert.is_equal(expected, errmsg)
+    error_end()
+  end)
 end)
 --]]