DEVINFRA-630: record "rerun" test results

pyproject.toml

@@ -412,6 +412,7 @@ module = [
     "sentry.tempest.endpoints.*",
     "sentry.tempest.migrations.*",
     "sentry.testutils.helpers.task_runner",
+    "sentry.testutils.pytest.json_report_reruns",
     "sentry.testutils.skips",
     "sentry.toolbar.utils.*",
     "sentry.types.*",

src/sentry/testutils/pytest/__init__.py

@@ -8,4 +8,5 @@
     "sentry.testutils.pytest.relay",
     "sentry.testutils.pytest.metrics",
     "sentry.testutils.pytest.stale_database_reads",
+    "sentry.testutils.pytest.json_report_reruns",
 ]

src/sentry/testutils/pytest/json_report_reruns.py

@@ -0,0 +1,33 @@
+import pytest_jsonreport.plugin  # type:ignore[import-untyped]
+
+TestItem = dict[str, list[object]]
+
+
+class PytestRerunJSONReporter:
+    json_tests: None | dict[str, TestItem] = None
+
+    def pytest_plugin_registered(self, plugin, manager):
+        del manager
+        if isinstance(plugin, pytest_jsonreport.plugin.JSONReport):
+            self.json_tests = plugin._json_tests
+
+    def pytest_json_runtest_stage(self, report):
+        assert self.json_tests is not None
+
+        nodeid = report.nodeid
+        json_testitem = self.json_tests[nodeid]
+        if report.when in json_testitem:
+            # this is a new result of some kind -- record all prior data as a
+            # "rerun" and start over fresh
+            reruns = json_testitem.setdefault("reruns", [])
+            reruns.append(
+                {
+                    key: json_testitem.pop(key)
+                    for key in ("setup", "call", "teardown")
+                    if key in json_testitem
+                }
+            )
+
+
+def pytest_configure(config):
+    config.pluginmanager.register(PytestRerunJSONReporter())