diff --git a/CHANGELOG.md b/CHANGELOG.md index cedd155d..c512a1d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Check docs generation with LDoc. - Add `--repeat-group` (`-R`) option to run tests in a circle within the group. +- Add `flaky` test mark. ## 0.5.7 diff --git a/README.rst b/README.rst index f16e526a..850a7065 100644 --- a/README.rst +++ b/README.rst @@ -242,6 +242,8 @@ List of luatest functions +--------------------------------------------------------------------+-----------------------------------------------+ | ``xfail_if (condition, message)`` | Mark test as xfail if condition is met. | +--------------------------------------------------------------------+-----------------------------------------------+ +| ``flaky (message)`` | Mark test as flaky. | ++--------------------------------------------------------------------+-----------------------------------------------+ | ``skip (message)`` | Skip a running test. | +--------------------------------------------------------------------+-----------------------------------------------+ | ``skip_if (condition, message)`` | Skip a running test if condition is met. | @@ -283,6 +285,22 @@ keep xfail tests in sync with an issue tracker. XFail only applies to the errors raised by the luatest assertions. Regular Lua errors still cause the test failure. +.. _xfail: + +--------------------------------- +Flaky +--------------------------------- + +The ``flaky`` mark can be used to temporarily remove flaky error from your tests. + +.. code-block:: Lua + + local g = t.group() + g.test_fail = function() + t.flaky("Can be failed and I don't know why ") + t.assert_equals(1, math.random(0,1)) + end + .. _capturing-output: --------------------------------- diff --git a/luatest/assertions.lua b/luatest/assertions.lua index 1e183936..9ffe8207 100644 --- a/luatest/assertions.lua +++ b/luatest/assertions.lua @@ -16,6 +16,7 @@ local prettystr_pairs = pp.tostring_pair local M = {} local xfail = false +local flaky = false -- private exported functions (for testing) M.private = {} @@ -26,6 +27,13 @@ function M.private.is_xfail() return xfail_status end + +function M.private.is_flaky() + local flaky_status = flaky + flaky = false + return flaky_status +end + --[[-- EPS is meant to help with Lua's floating point math in simple corner @@ -169,6 +177,14 @@ function M.xfail_if(condition, message) end end +--- Mark a test as xfail if condition is met +-- +-- @param condition +-- @string message +function M.flaky(message) + flaky = message or true +end + --- Check that two values are equal. -- Tables are compared by value. -- diff --git a/luatest/output/generic.lua b/luatest/output/generic.lua index 4e6f45eb..b39a6497 100644 --- a/luatest/output/generic.lua +++ b/luatest/output/generic.lua @@ -50,7 +50,7 @@ end -- luacheck: pop function Output.mt:status_line(colors) - colors = colors or {success = '', failure = '', reset = '', xfail = ''} + colors = colors or {success = '', failure = '', reset = '', xfail = '', flaky = ''} -- return status line string according to results local tests = self.result.tests local s = { @@ -69,7 +69,10 @@ function Output.mt:status_line(colors) if #tests.error > 0 then table.insert(s, string.format("%s%d %s%s", colors.failure, #tests.error, 'errored', colors.reset)) end - if #tests.fail == 0 and #tests.error == 0 and #tests.xsuccess == 0 then + if #tests.flaky > 0 then + table.insert(s, string.format("%s%d %s%s", colors.flaky, #tests.flaky, 'flaky', colors.reset)) + end + if #tests.fail == 0 and #tests.error == 0 and #tests.xsuccess == 0 and #tests.flaky == 0 then table.insert(s, '0 failed') end if #tests.skip > 0 then diff --git a/luatest/output/text.lua b/luatest/output/text.lua index bf06dc2a..5551ca23 100644 --- a/luatest/output/text.lua +++ b/luatest/output/text.lua @@ -113,6 +113,7 @@ function Output.mt:end_suite() failure = self.class.ERROR_COLOR_CODE, reset = self.class.RESET_TERM, xfail = self.class.WARN_COLOR_CODE, + flaky = self.class.WARN_COLOR_CODE, })) if self.result.notSuccessCount == 0 then print('OK') diff --git a/luatest/runner.lua b/luatest/runner.lua index 98655649..00f89885 100644 --- a/luatest/runner.lua +++ b/luatest/runner.lua @@ -299,6 +299,7 @@ function Runner.mt:start_suite(selected_count, not_selected_count) skip = {}, xfail = {}, xsuccess = {}, + flaky = {}, }, } self.output.result = self.result @@ -324,7 +325,7 @@ function Runner.mt:update_status(node, err) elseif not node:is('success') then return elseif err.status == 'fail' or err.status == 'error' or err.status == 'skip' - or err.status == 'xfail' or err.status == 'xsuccess' then + or err.status == 'xfail' or err.status == 'xsuccess' or err.status == 'flaky' then node:update_status(err.status, err.message, err.trace) else error('No such status: ' .. pp.tostring(err.status)) @@ -340,7 +341,7 @@ function Runner.mt:end_test(node) if node:is('error') or node:is('fail') or node:is('xsuccess') then self.result.aborted = self.fail_fast elseif not node:is('success') and not node:is('skip') - and not node:is('xfail') then + and not node:is('xfail') and not node:is('flaky') then error('No such node status: ' .. pp.tostring(node.status)) end end @@ -377,6 +378,7 @@ function Runner.mt:protected_call(instance, method, pretty_name) -- check if test was marked as xfail and reset xfail flag local xfail = assertions.private.is_xfail() + local flaky = assertions.private.is_flaky() if type(err.message) ~= 'string' then err.message = pp.tostring(err.message) @@ -399,6 +401,11 @@ function Runner.mt:protected_call(instance, method, pretty_name) return err end + if flaky and err.status ~= 'error' and err.status ~= 'skip' then + err.status = 'flaky' + return err + end + -- reformat / improve the stack trace if pretty_name then -- we do have the real method name err.trace = err.trace:gsub("in (%a+) 'method'", "in %1 '" .. pretty_name .. "'") diff --git a/test/flaky_test.lua b/test/flaky_test.lua new file mode 100644 index 00000000..6f769b73 --- /dev/null +++ b/test/flaky_test.lua @@ -0,0 +1,40 @@ +local t = require('luatest') +local g = t.group() + +local helper = require('test.helper') + +g.test_failed = function() + local result = helper.run_suite(function(lu2) + local pg = lu2.group('flaky') + pg.test_fail = function() + lu2.flaky() + lu2.assert_equals(2 + 2, 5) + end + end) + + t.assert_equals(result, 0) +end + +g.test_succeeded = function() + local result = helper.run_suite(function(lu2) + local pg = lu2.group('flaky') + pg.test_success = function() + lu2.flaky() + lu2.assert_equals(2 + 3, 5) + end + end) + + t.assert_equals(result, 0) +end + +g.test_error = function() + local result = helper.run_suite(function(lu2) + local pg = lu2.group('flaky') + pg.test_error = function() + lu2.flaky() + error('Boom!') + end + end) + + t.assert_equals(result, 1) +end