Match xfail behavior for pytest and unitttest with TAP spec (#57)

* Add a test for unittest.expectedFailure behavior.

* Handle xfail behavior changes.

* Add release docs about the behavior change.

* Make the unittest test assertion formatting consistent with other tests.

Author: mblayman

docs/releases.rst

@@ -6,6 +6,9 @@ Version 3.2, To Be Released
 
 * Add support for Python 3.8.
 * Add support for Python 3.9.
+* Handle ``unittest.expectedFailure`` and ``pytest.xfail``
+  in a way that is more consistent
+  with the TAP specification.
 
 Version 3.1, Released March 25, 2020
 ------------------------------------

src/pytest_tap/plugin.py

@@ -98,36 +98,50 @@ def pytest_runtest_logreport(report):
     # Handle xfails first because they report in unusual ways.
     # Non-strict xfails will include `wasxfail` while strict xfails won't.
     if hasattr(report, "wasxfail"):
-        directive = ""
+        reason = ""
+        # pytest adds an ugly "reason: " for expectedFailure
+        # even though the standard library doesn't accept a reason for that decorator.
+        # Ignore the "reason: " from pytest.
+        if report.wasxfail and report.wasxfail != "reason: ":
+            reason = ": {}".format(report.wasxfail)
+
         if report.skipped:
-            directive = "TODO expected failure: {}".format(report.wasxfail)
+            directive = "TODO expected failure{}".format(reason)
+            tracker.add_not_ok(testcase, description, directive=directive)
         elif report.passed:
-            directive = "TODO unexpected success: {}".format(report.wasxfail)
-
-        tracker.add_ok(testcase, description, directive=directive)
+            directive = "TODO unexpected success{}".format(reason)
+            tracker.add_ok(testcase, description, directive=directive)
     elif report.passed:
         tracker.add_ok(testcase, description)
     elif report.failed:
         diagnostics = _make_as_diagnostics(report)
 
-        # strict xfail mode should include the todo directive.
-        # The only indicator that strict xfail occurred for this report
-        # is to check longrepr.
-        directive = ""
-        if isinstance(report.longrepr, str) and "[XPASS(strict)]" in report.longrepr:
-            directive = "TODO"
+        # pytest treats an unexpected success from unitest.expectedFailure as a failure.
+        # To match up with TAPTestResult and the TAP spec, treat the pass
+        # as an ok with a todo directive instead.
+        if isinstance(report.longrepr, str) and "Unexpected success" in report.longrepr:
+            tracker.add_ok(testcase, description, directive="TODO unexpected success")
+            return
 
-        tracker.add_not_ok(
-            testcase, description, directive=directive, diagnostics=diagnostics
-        )
+        # A strict xfail that passes (i.e., XPASS) should be marked as a failure.
+        # The only indicator that strict xfail occurred for XPASS is to check longrepr.
+        if isinstance(report.longrepr, str) and "[XPASS(strict)]" in report.longrepr:
+            tracker.add_not_ok(
+                testcase,
+                description,
+                directive="unexpected success: {}".format(report.longrepr),
+            )
+            return
+
+        tracker.add_not_ok(testcase, description, diagnostics=diagnostics)
     elif report.skipped:
         reason = report.longrepr[2].split(":", 1)[1].strip()
         tracker.add_skip(testcase, description, reason)
 
 
 def _make_as_diagnostics(report):
     """Format a report as TAP diagnostic output."""
-    lines = report.longreprtext.splitlines(True)
+    lines = report.longreprtext.splitlines(keepends=True)
     return format_as_diagnostics(lines)
 
 

tests/test_plugin.py

@@ -57,7 +57,7 @@ def test_stream(testdir, sample_test_file):
             "ok 3 test_stream.py::test_params[foo]",
             "ok 4 test_stream.py::test_params[bar]",
             "ok 5 test_stream.py::test_skipped # SKIP some reason",
-            "ok 6 test_stream.py::test_broken # TODO expected failure: a reason",
+            "not ok 6 test_stream.py::test_broken # TODO expected failure: a reason",
         ]
     )
 
@@ -78,7 +78,7 @@ def test_combined(testdir, sample_test_file):
         "ok 3 test_combined.py::test_params[foo]",
         "ok 4 test_combined.py::test_params[bar]",
         "ok 5 test_combined.py::test_skipped # SKIP some reason",
-        "ok 6 test_combined.py::test_broken # TODO expected failure: a reason",
+        "not ok 6 test_combined.py::test_broken # TODO expected failure: a reason",
     ]
     # If the dependencies for version 13 happen to be installed, tweak the output.
     if ENABLE_VERSION_13:
@@ -112,34 +112,114 @@ def test_outdir(testdir, sample_test_file):
     assert testresults.check()
 
 
-def test_xfail_strict_function(testdir):
-    """An xfail with strict on will fail when it unexpectedly passes.
+def test_xfail_no_reason(testdir):
+    """xfails output gracefully when no reason is provided."""
+    testdir.makepyfile(
+        """
+        import pytest
 
-    The xfail should look like an xfail by including the TODO directive.
+        @pytest.mark.xfail(strict=False)
+        def test_unexpected_success():
+            assert True
+
+        @pytest.mark.xfail(strict=False)
+        def test_expected_failure():
+            assert False
     """
+    )
+    result = testdir.runpytest_subprocess("--tap-stream")
+
+    result.stdout.fnmatch_lines(
+        [
+            "ok 1 test_xfail_no_reason.py::test_unexpected_success "
+            "# TODO unexpected success",
+            "not ok 2 test_xfail_no_reason.py::test_expected_failure "
+            "# TODO expected failure",
+        ]
+    )
+
+
+def test_xfail_nonstrict(testdir):
+    """Non-strict xfails are treated as TODO directives."""
     testdir.makepyfile(
         """
         import pytest
 
-        @pytest.mark.xfail(reason='a reason')
-        def test_unexpected_pass():
+        @pytest.mark.xfail(strict=False, reason='a reason')
+        def test_unexpected_success():
             assert True
 
-        @pytest.mark.xfail(reason='a reason', strict=True)
-        def test_broken():
+        @pytest.mark.xfail(strict=False, reason='a reason')
+        def test_expected_failure():
+            assert False
+    """
+    )
+    result = testdir.runpytest_subprocess("--tap-stream")
+
+    result.stdout.fnmatch_lines(
+        [
+            "ok 1 test_xfail_nonstrict.py::test_unexpected_success "
+            "# TODO unexpected success: a reason",
+            "not ok 2 test_xfail_nonstrict.py::test_expected_failure "
+            "# TODO expected failure: a reason",
+        ]
+    )
+
+
+def test_xfail_strict(testdir):
+    """xfail strict mode handles expected behavior."""
+    testdir.makepyfile(
+        """
+        import pytest
+
+        @pytest.mark.xfail(strict=True, reason='a reason')
+        def test_unexpected_success():
             assert True
+
+        @pytest.mark.xfail(strict=True, reason='a reason')
+        def test_expected_failure():
+            assert False
+    """
+    )
+    result = testdir.runpytest_subprocess("--tap-stream")
+
+    result.stdout.fnmatch_lines(
+        [
+            "not ok 1 test_xfail_strict.py::test_unexpected_success "
+            "# unexpected success: [XPASS(strict)] a reason",
+            "not ok 2 test_xfail_strict.py::test_expected_failure "
+            "# TODO expected failure: a reason",
+        ]
+    )
+
+
+def test_unittest_expected_failure(testdir):
+    """The plugin handles unittest's expectedFailure decorator behavior."""
+    testdir.makepyfile(
+        """
+        import pytest
+        import unittest
+
+        class TestExpectedFailure(unittest.TestCase):
+            @unittest.expectedFailure
+            def test_when_failing(self):
+                assert False
+
+            @unittest.expectedFailure
+            def test_when_passing(self):
+                assert True
     """
     )
     result = testdir.runpytest_subprocess("--tap-stream")
 
     result.stdout.fnmatch_lines(
         [
-            (
-                "ok 1 test_xfail_strict_function.py::test_unexpected_pass "
-                "# TODO unexpected success: a reason"
-            ),
-            "not ok 2 test_xfail_strict_function.py::test_broken # TODO",
-            "# [XPASS(strict)] a reason",
+            "not ok 1 test_unittest_expected_failure.py::"
+            "TestExpectedFailure.test_when_failing "
+            "# TODO expected failure",
+            "ok 2 test_unittest_expected_failure.py::"
+            "TestExpectedFailure.test_when_passing "
+            "# TODO unexpected success",
         ]
     )