fixup! feat: store more benchmark metadata in results

Author: adriencaccia

src/pytest_codspeed/benchmark.py

@@ -17,7 +17,7 @@ def has_args(item: pytest.Item) -> bool:
 
 
 @dataclass
-class Benchmark:
+class BenchmarkMetadata:
     file: str
     module: str
     groups: list[str]
@@ -26,7 +26,7 @@ class Benchmark:
     args_names: list[str]
 
     @classmethod
-    def from_item(cls, item: pytest.Item) -> Benchmark:
+    def from_item(cls, item: pytest.Item) -> BenchmarkMetadata:
         file = str(get_git_relative_path(item.path))
         module = "::".join(
             [node.name for node in item.listchain() if isinstance(node, pytest.Class)]

src/pytest_codspeed/instruments/__init__.py

@@ -9,7 +9,7 @@
 
     import pytest
 
-    from pytest_codspeed.benchmark import Benchmark
+    from pytest_codspeed.benchmark import BenchmarkMetadata
     from pytest_codspeed.plugin import CodSpeedConfig
 
     T = TypeVar("T")
@@ -28,7 +28,7 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]: ...
     @abstractmethod
     def measure(
         self,
-        benchmark: Benchmark,
+        benchmark_metadata: BenchmarkMetadata,
         fn: Callable[P, T],
         *args: P.args,
         **kwargs: P.kwargs,

src/pytest_codspeed/instruments/valgrind/__init__.py

@@ -14,7 +14,7 @@
 
     from pytest import Session
 
-    from pytest_codspeed.benchmark import Benchmark
+    from pytest_codspeed.benchmark import BenchmarkMetadata
     from pytest_codspeed.instruments import P, T
     from pytest_codspeed.instruments.valgrind._wrapper import LibType
     from pytest_codspeed.plugin import CodSpeedConfig
@@ -28,7 +28,7 @@ class ValgrindInstrument(Instrument):
 
     def __init__(self, config: CodSpeedConfig) -> None:
         self.benchmark_count = 0
-        self.benchmarks: list[Benchmark] = []
+        self.benchmarks_metadata: list[BenchmarkMetadata] = []
         self.should_measure = os.environ.get("CODSPEED_ENV") is not None
         if self.should_measure:
             self.lib = get_lib()
@@ -57,13 +57,13 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
 
     def measure(
         self,
-        benchmark: Benchmark,
+        benchmark_metadata: BenchmarkMetadata,
         fn: Callable[P, T],
         *args: P.args,
         **kwargs: P.kwargs,
     ) -> T:
         self.benchmark_count += 1
-        self.benchmarks.append(benchmark)
+        self.benchmarks_metadata.append(benchmark_metadata)
         if self.lib is None:  # Thus should_measure is False
             return fn(*args, **kwargs)
 
@@ -81,7 +81,7 @@ def __codspeed_root_frame__() -> T:
         finally:
             # Ensure instrumentation is stopped even if the test failed
             self.lib.stop_instrumentation()
-            self.lib.dump_stats_at(benchmark.to_json_string().encode("ascii"))
+            self.lib.dump_stats_at(benchmark_metadata.to_json_string().encode("ascii"))
 
     def report(self, session: Session) -> None:
         reporter = session.config.pluginmanager.get_plugin("terminalreporter")
@@ -94,5 +94,5 @@ def report(self, session: Session) -> None:
     def get_result_dict(self) -> dict[str, Any]:
         return {
             "instrument": {"type": self.instrument},
-            "benchmarks": [asdict(bench) for bench in self.benchmarks],
+            "benchmarks": [asdict(bench) for bench in self.benchmarks_metadata],
         }

src/pytest_codspeed/instruments/walltime.py

@@ -11,7 +11,7 @@
 from rich.table import Table
 from rich.text import Text
 
-from pytest_codspeed.benchmark import Benchmark
+from pytest_codspeed.benchmark import BenchmarkMetadata
 from pytest_codspeed.instruments import Instrument
 
 if TYPE_CHECKING:
@@ -115,13 +115,13 @@ def from_list(
 
 
 @dataclass
-class WalltimeBenchmark(Benchmark):
+class WalltimeBenchmark(BenchmarkMetadata):
     config: WalltimeBenchmarkConfig
     stats: WalltimeBenchmarkStats
 
 
 def run_benchmark(
-    benchmark: Benchmark,
+    benchmark_metadata: BenchmarkMetadata,
     fn: Callable[P, T],
     args,
     kwargs,
@@ -182,7 +182,7 @@ def run_benchmark(
     )
 
     return WalltimeBenchmark(
-        **asdict(benchmark),
+        **asdict(benchmark_metadata),
         config=config,
         stats=stats,
     ), out
@@ -200,13 +200,13 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
 
     def measure(
         self,
-        benchmark: Benchmark,
+        benchmark_metadata: BenchmarkMetadata,
         fn: Callable[P, T],
         *args: P.args,
         **kwargs: P.kwargs,
     ) -> T:
         bench, out = run_benchmark(
-            benchmark=benchmark,
+            benchmark_metadata=benchmark_metadata,
             fn=fn,
             args=args,
             kwargs=kwargs,

src/pytest_codspeed/plugin.py

@@ -13,7 +13,7 @@
 import pytest
 from _pytest.fixtures import FixtureManager
 
-from pytest_codspeed.benchmark import Benchmark
+from pytest_codspeed.benchmark import BenchmarkMetadata
 from pytest_codspeed.instruments import (
     MeasurementMode,
     get_instrument_from_mode,
@@ -263,8 +263,8 @@ def _measure(
         gc.collect()
         gc.disable()
     try:
-        benchmark = Benchmark.from_item(item)
-        return plugin.instrument.measure(benchmark, fn, *args, **kwargs)
+        benchmark_metadata = BenchmarkMetadata.from_item(item)
+        return plugin.instrument.measure(benchmark_metadata, fn, *args, **kwargs)
     finally:
         # Ensure GC is re-enabled even if the test failed
         if is_gc_enabled:

tests/conftest.py

@@ -84,6 +84,7 @@ def run_pytest_codspeed_with_mode(
     # create empty `.git` folder in the rootdir to simulate a git repository
     if not pytester.path.joinpath(".git").exists():
         pytester.mkdir(".git")
+    # subprocess is important for inline-snapshot to work properly
     return pytester.runpytest_subprocess(
         *csargs,
         *args,

tests/test_benchmark.py renamed to tests/test_benchmark_metadata.py

@@ -1,6 +1,6 @@
 import pytest
 
-from pytest_codspeed.plugin import Benchmark
+from pytest_codspeed.plugin import BenchmarkMetadata
 
 
 class ClassWithStr:
@@ -9,31 +9,31 @@ def __str__(self):
 
 
 benchmarks = [
-    Benchmark(
+    BenchmarkMetadata(
         file="test_benchmark_results.py",
         module="TestClass::TestNested",
         groups=["bench-group"],
         name="test_fully_parametrized",
         args=[58.3, "baz"],
         args_names=["a", "b"],
     ),
-    Benchmark(
+    BenchmarkMetadata(
         file="another_test_file.py",
         module="AnotherClass::AnotherTest",
         groups=["another-group"],
         name="test_another_case",
         args=[42, "foo"],
         args_names=["x", "y"],
     ),
-    Benchmark(
+    BenchmarkMetadata(
         file="yet_another_test_file.py",
         module="",
         groups=[],
         name="test_yet_another_case",
         args=[],
         args_names=[],
     ),
-    Benchmark(
+    BenchmarkMetadata(
         file="test_complex_args.py",
         module="",
         groups=[],
@@ -65,7 +65,7 @@ def __str__(self):
         ),
     ],
 )
-def test_benchmark_to_json(bench, expected_json):
+def test_benchmark_metadata_to_json(bench, expected_json):
     assert bench.to_json_string() == expected_json
 
 
@@ -78,5 +78,5 @@ def test_benchmark_to_json(bench, expected_json):
         (benchmarks[3], "test_complex_args[x0-foo-z2-w3-class_str]"),
     ],
 )
-def test_benchmark_display_name(bench, expected_display_name):
+def test_benchmark_metadata_display_name(bench, expected_display_name):
     assert bench.display_name == expected_display_name