fix(ci-visibility): proper statuscode with pytest-xdist + ATR

ddtrace/contrib/internal/pytest/_atr_utils.py

@@ -4,7 +4,6 @@
 import pytest
 
 from ddtrace.contrib.internal.pytest._retry_utils import RetryOutcomes
-from ddtrace.contrib.internal.pytest._retry_utils import RetryTestReport
 from ddtrace.contrib.internal.pytest._retry_utils import _get_outcome_from_retry
 from ddtrace.contrib.internal.pytest._retry_utils import _get_retry_attempt_string
 from ddtrace.contrib.internal.pytest._retry_utils import set_retry_num
@@ -22,25 +21,23 @@
 log = get_logger(__name__)
 
 
-class _ATR_RETRY_OUTCOMES:
+class _ATR_RETRY_OUTCOMES(PYTEST_STATUS):
     ATR_ATTEMPT_PASSED = "dd_atr_attempt_passed"
     ATR_ATTEMPT_FAILED = "dd_atr_attempt_failed"
     ATR_ATTEMPT_SKIPPED = "dd_atr_attempt_skipped"
-    ATR_FINAL_PASSED = "dd_atr_final_passed"
-    ATR_FINAL_FAILED = "dd_atr_final_failed"
 
 
 class _QUARANTINE_ATR_RETRY_OUTCOMES(_ATR_RETRY_OUTCOMES):
     ATR_ATTEMPT_PASSED = "dd_quarantine_atr_attempt_passed"
     ATR_ATTEMPT_FAILED = "dd_quarantine_atr_attempt_failed"
     ATR_ATTEMPT_SKIPPED = "dd_quarantine_atr_attempt_skipped"
-    ATR_FINAL_PASSED = "dd_quarantine_atr_final_passed"
     ATR_FINAL_FAILED = "dd_quarantine_atr_final_failed"
+    ATR_FINAL_PASSED = "dd_quarantine_atr_final_passed"
 
 
 _FINAL_OUTCOMES: t.Dict[TestStatus, str] = {
-    TestStatus.PASS: _ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED,
-    TestStatus.FAIL: _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED,
+    TestStatus.PASS: _ATR_RETRY_OUTCOMES.PASSED,
+    TestStatus.FAIL: _ATR_RETRY_OUTCOMES.FAILED,
 }
 
 
@@ -82,13 +79,14 @@ def atr_handle_retries(
     atr_outcome = _atr_do_retries(item, outcomes)
     longrepr = InternalTest.stash_get(test_id, "failure_longrepr")
 
-    final_report = RetryTestReport(
+    final_report = pytest_TestReport(
         nodeid=item.nodeid,
         location=item.location,
-        keywords=item.keywords,
+        keywords={k: 1 for k in item.keywords},
         when="call",
         longrepr=longrepr,
         outcome=final_outcomes[atr_outcome],
+        user_properties=item.user_properties + [("dd_retry_reason", "auto_test_retry")],
     )
     item.ihook.pytest_runtest_logreport(report=final_report)
 
@@ -113,6 +111,13 @@ def _atr_do_retries(item: pytest.Item, outcomes: RetryOutcomes) -> TestStatus:
     return InternalTest.atr_get_final_status(test_id)
 
 
+def get_user_property(report, key, default=None):
+    for k, v in report.user_properties:
+        if k == key:
+            return v
+    return default
+
+
 def _atr_write_report_for_status(
     terminalreporter: _pytest.terminal.TerminalReporter,
     status_key: str,
@@ -122,8 +127,14 @@ def _atr_write_report_for_status(
     markedup_strings: t.List[str],
     color: str,
     delete_reports: bool = True,
+    retry_reason="auto_test_retry",
 ):
-    reports = terminalreporter.getreports(status_key)
+    reports = [
+        report
+        for report in terminalreporter.getreports(report_outcome)
+        if get_user_property(report, "dd_retry_reason") == retry_reason
+    ]
+
     markup_kwargs = {color: True}
     if reports:
         text = f"{len(reports)} {status_text}"
@@ -133,15 +144,6 @@ def _atr_write_report_for_status(
         for report in reports:
             line = f"{terminalreporter._tw.markup(status_text.upper(), **markup_kwargs)} {report.nodeid}"
             terminalreporter.write_line(line)
-            report.outcome = report_outcome
-            # Do not re-append a report if a report already exists for the item in the reports
-            for existing_reports in terminalreporter.stats.get(report_outcome, []):
-                if existing_reports.nodeid == report.nodeid:
-                    break
-            else:
-                terminalreporter.stats.setdefault(report_outcome, []).append(report)
-        if delete_reports:
-            del terminalreporter.stats[status_key]
 
 
 def _atr_prepare_attempts_strings(
@@ -177,7 +179,7 @@ def atr_pytest_terminal_summary_post_yield(terminalreporter: _pytest.terminal.Te
 
     _atr_write_report_for_status(
         terminalreporter,
-        status_key=_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED,
+        status_key=_ATR_RETRY_OUTCOMES.FAILED,
         status_text="failed",
         report_outcome=PYTEST_STATUS.FAILED,
         raw_strings=raw_summary_strings,
@@ -187,7 +189,7 @@ def atr_pytest_terminal_summary_post_yield(terminalreporter: _pytest.terminal.Te
 
     _atr_write_report_for_status(
         terminalreporter,
-        status_key=_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED,
+        status_key=_ATR_RETRY_OUTCOMES.PASSED,
         status_text="passed",
         report_outcome=PYTEST_STATUS.PASSED,
         raw_strings=raw_summary_strings,
@@ -280,10 +282,10 @@ def atr_get_teststatus(report: pytest_TestReport) -> _pytest_report_teststatus_r
             "s",
             (f"ATR RETRY {_get_retry_attempt_string(report.nodeid)}SKIPPED", {"yellow": True}),
         )
-    if report.outcome == _ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED:
-        return (_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, ".", ("ATR FINAL STATUS: PASSED", {"green": True}))
-    if report.outcome == _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED:
-        return (_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, "F", ("ATR FINAL STATUS: FAILED", {"red": True}))
+    # if report.outcome == _ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED:
+    #     return (_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, ".", ("ATR FINAL STATUS: PASSED", {"green": True}))
+    # if report.outcome == _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED:
+    #     return (_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, "F", ("ATR FINAL STATUS: FAILED", {"red": True}))
     return None
 
 

ddtrace/contrib/internal/pytest/_attempt_to_fix.py

@@ -4,14 +4,14 @@
 import pytest
 
 from ddtrace.contrib.internal.pytest._retry_utils import RetryOutcomes
-from ddtrace.contrib.internal.pytest._retry_utils import RetryTestReport
 from ddtrace.contrib.internal.pytest._retry_utils import _get_outcome_from_retry
 from ddtrace.contrib.internal.pytest._retry_utils import _get_retry_attempt_string
 from ddtrace.contrib.internal.pytest._retry_utils import set_retry_num
 from ddtrace.contrib.internal.pytest._types import _pytest_report_teststatus_return_type
 from ddtrace.contrib.internal.pytest._types import pytest_TestReport
 from ddtrace.contrib.internal.pytest._utils import _get_test_id_from_item
 from ddtrace.contrib.internal.pytest._utils import _TestOutcome
+from ddtrace.contrib.internal.pytest._utils import PYTEST_STATUS
 from ddtrace.contrib.internal.pytest.constants import USER_PROPERTY_QUARANTINED
 from ddtrace.ext.test_visibility.api import TestStatus
 from ddtrace.internal.logger import get_logger
@@ -22,19 +22,17 @@
 log = get_logger(__name__)
 
 
-class _RETRY_OUTCOMES:
+class _RETRY_OUTCOMES(PYTEST_STATUS):
     ATTEMPT_PASSED = "dd_fix_attempt_passed"
     ATTEMPT_FAILED = "dd_fix_attempt_failed"
     ATTEMPT_SKIPPED = "dd_fix_attempt_skipped"
     FINAL_PASSED = "dd_fix_final_passed"
-    FINAL_FAILED = "dd_fix_final_failed"
-    FINAL_SKIPPED = "dd_fix_final_skipped"
 
 
 _FINAL_OUTCOMES: t.Dict[TestStatus, str] = {
     TestStatus.PASS: _RETRY_OUTCOMES.FINAL_PASSED,
-    TestStatus.FAIL: _RETRY_OUTCOMES.FINAL_FAILED,
-    TestStatus.SKIP: _RETRY_OUTCOMES.FINAL_SKIPPED,
+    TestStatus.FAIL: _RETRY_OUTCOMES.FAILED,
+    TestStatus.SKIP: _RETRY_OUTCOMES.SKIPPED,
 }
 
 
@@ -60,14 +58,16 @@ def attempt_to_fix_handle_retries(
     if when == "call":
         if test_outcome.status == TestStatus.FAIL:
             original_result.outcome = outcomes.FAILED
+        elif test_outcome.status == TestStatus.PASS:
+            original_result.outcome = outcomes.PASSED
         elif test_outcome.status == TestStatus.SKIP:
             original_result.outcome = outcomes.SKIPPED
         return
 
     retries_outcome = _do_retries(item, outcomes)
     longrepr = InternalTest.stash_get(test_id, "failure_longrepr")
 
-    final_report = RetryTestReport(
+    final_report = pytest_TestReport(
         nodeid=item.nodeid,
         location=item.location,
         keywords=item.keywords,
@@ -114,36 +114,20 @@ def attempt_to_fix_get_teststatus(report: pytest_TestReport) -> _pytest_report_t
             "s",
             (f"ATTEMPT TO FIX RETRY {_get_retry_attempt_string(report.nodeid)}SKIPPED", {"yellow": True}),
         )
-    if report.outcome == _RETRY_OUTCOMES.FINAL_PASSED:
-        return (_RETRY_OUTCOMES.FINAL_PASSED, ".", ("ATTEMPT TO FIX FINAL STATUS: PASSED", {"green": True}))
-    if report.outcome == _RETRY_OUTCOMES.FINAL_FAILED:
-        return (_RETRY_OUTCOMES.FINAL_FAILED, "F", ("ATTEMPT TO FIX FINAL STATUS: FAILED", {"red": True}))
-    if report.outcome == _RETRY_OUTCOMES.FINAL_SKIPPED:
-        return (_RETRY_OUTCOMES.FINAL_SKIPPED, "s", ("ATTEMPT TO FIX FINAL STATUS: SKIPPED", {"yellow": True}))
     return None
 
 
 def attempt_to_fix_pytest_terminal_summary_post_yield(terminalreporter: _pytest.terminal.TerminalReporter):
     # Flaky tests could have passed their initial attempt, so they need to be removed from the passed stats to avoid
     # overcounting:
-    flaky_node_ids = {report.nodeid for report in terminalreporter.stats.get(_RETRY_OUTCOMES.FINAL_FAILED, [])}
+    flaky_node_ids = {report.nodeid for report in terminalreporter.stats.get(_RETRY_OUTCOMES.FAILED, [])}
     passed_reports = terminalreporter.stats.get("passed")
     if passed_reports:
         terminalreporter.stats["passed"] = [report for report in passed_reports if report.nodeid not in flaky_node_ids]
 
+    terminalreporter.stats.pop(_RETRY_OUTCOMES.FINAL_PASSED, None)
     terminalreporter.stats.pop(_RETRY_OUTCOMES.ATTEMPT_PASSED, None)
     terminalreporter.stats.pop(_RETRY_OUTCOMES.ATTEMPT_FAILED, None)
     terminalreporter.stats.pop(_RETRY_OUTCOMES.ATTEMPT_SKIPPED, None)
-    terminalreporter.stats.pop(_RETRY_OUTCOMES.FINAL_PASSED, None)
-
-    failed_tests = terminalreporter.stats.pop(_RETRY_OUTCOMES.FINAL_FAILED, [])
-    for report in failed_tests:
-        if USER_PROPERTY_QUARANTINED not in report.user_properties:
-            terminalreporter.stats.setdefault("failed", []).append(report)
-
-    skipped_tests = terminalreporter.stats.pop(_RETRY_OUTCOMES.FINAL_SKIPPED, [])
-    for report in skipped_tests:
-        if USER_PROPERTY_QUARANTINED not in report.user_properties:
-            terminalreporter.stats.setdefault("skipped", []).append(report)
 
     # TODO: report list of attempt-to-fix results.