reworked with pytest hook generate_tests

Author: felix314159

src/pytest_plugins/execute/eth_config/eth_config.py

@@ -78,9 +78,6 @@ def pytest_configure(config: pytest.Config) -> None:
     network_configs_path = config.getoption("network_config_file")
     clients = config.getoption("clients")
     rpc_endpoint = config.getoption("rpc_endpoint")
-
-    # set flags for defining whether to run majority eth_config test or not, and how
-    config.option.majority_eth_config_test_enabled = False
     config.option.majority_clients = []  # List[str]
 
     # load provided networks file
@@ -106,12 +103,11 @@ def pytest_configure(config: pytest.Config) -> None:
     for c in clients:
         if c not in EXECUTION_CLIENTS:
             pytest.exit(f"Unsupported client was passed: {c}")
-    print(f"Activating majority mode\nProvided client list: {clients}")
+    print(f"Provided client list: {clients}")
 
     # store majority mode configuration
     if ".ethpandaops.io" in rpc_endpoint:
         print("Ethpandaops RPC detected, toggling majority test on")
-        config.option.majority_eth_config_test_enabled = True
         config.option.majority_clients = clients  # List[str]
     else:
         print("No ethpandaops RPC detected, majority test will not be toggled on")
@@ -130,6 +126,7 @@ def pytest_configure(config: pytest.Config) -> None:
     try:
         print("Will now briefly check whether eth_config is supported by target rpc..")
         eth_rpc.config()
+        print("Connection check ok (successfully got eth_config response)")
     except Exception as e:
         pytest.exit(f"RPC endpoint {rpc_endpoint} does not support `eth_config`: {e}")
 
@@ -140,19 +137,19 @@ def rpc_endpoint(request) -> str:
     return request.config.getoption("rpc_endpoint")
 
 
-@pytest.fixture(autouse=True, scope="session")
-def eth_rpc(rpc_endpoint: str, request) -> Dict[str, List[EthRPC]]:
-    """Generate list of RPC URLs which define which clients are tested."""
-    # generate all cl+el client combinations
-    config = request.config
-    el_clients: List[str] = config.getoption("majority_clients")  # besu, erigon, ..
+# @pytest.fixture(autouse=True, scope="session")
+# def eth_rpc(rpc_endpoint: str) -> EthRPC:
+#     """Initialize ethereum RPC client for the execution client under test."""
+#     return EthRPC(rpc_endpoint)
+
 
-    assert len(el_clients) > 0
-    if "geth" in el_clients and "fusaka-devnet-3" in rpc_endpoint:
-        pytest.exit("fusaka-devnet-3 geth does not support eth_config!")
+def all_rpc_endpoints(config) -> Dict[str, List[EthRPC]]:
+    """Derive a mapping of exec clients to the RPC URLs they are reachable at."""
+    rpc_endpoint = config.getoption("rpc_endpoint")
+    el_clients: List[str] = config.getoption("majority_clients")  # besu, erigon, ..
+    if len(el_clients) == 0:
+        return {}
 
-    # generate client-specific URLs from provided rpc_endpoint (it does not matter which client the original rpc_endpoint specifies)  # noqa: E501
-    # we want all combinations of consensus and execution clients (sometimes an exec client is only reachable via a subset of consensus client combinations)  # noqa: E501
     pattern = r"(.*?@rpc\.)([^-]+)-([^-]+)(-.*)"
     url_dict: Dict[str, List[EthRPC]] = {
         exec_client: [
@@ -174,3 +171,37 @@ def eth_rpc(rpc_endpoint: str, request) -> Dict[str, List[EthRPC]]:
     #     ...
     # }
     return url_dict
+
+
+def pytest_generate_tests(metafunc: pytest.Metafunc):
+    """Generate tests for all clients under test."""
+    # all_rpc_endpoints is a dictionary with the name of the exec client as key
+    # and the possible URLs to contact it (different cl combinations) as value list
+    all_rpc_endpoints_dict = all_rpc_endpoints(metafunc.config)
+
+    if metafunc.definition.name == "test_eth_config_majority":
+        if len(all_rpc_endpoints_dict) < 2:
+            # The test function is not run because we only have a single client, so no majority comparison  # noqa: E501
+            print("Skipping eth_config majority because less than 2 exec clients were passed")
+            metafunc.parametrize(
+                ["all_rpc_endpoints"],
+                [],
+            )
+        else:
+            metafunc.parametrize(
+                ["all_rpc_endpoints"],
+                [[all_rpc_endpoints_dict]],  # interpret it as a single argument dict
+                scope="function",
+            )
+    else:
+        metafunc.parametrize(
+            ["eth_rpc"],
+            [
+                pytest.param(
+                    rpc_endpoint,
+                    id=endpoint_name,
+                )
+                for endpoint_name, rpc_endpoint in all_rpc_endpoints_dict.items()
+            ],
+            scope="function",
+        )

src/pytest_plugins/execute/eth_config/execute_eth_config.py

@@ -12,28 +12,26 @@
 from .types import NetworkConfig
 
 
-@pytest.fixture(scope="session")
-def eth_config_response(request) -> EthConfigResponse | None:
-    """Get the `eth_config` response from the user-provided RPC endpoint."""
-    config = request.config
-    eth_rpc_url = config.getoption("rpc_endpoint")
-    eth_rpc_target = EthRPC(eth_rpc_url)
-    return eth_rpc_target.config()
+@pytest.fixture(scope="function")
+def eth_config_response(eth_rpc: List[EthRPC]) -> EthConfigResponse | None:
+    """Get the `eth_config` response from the client to be verified by all tests."""
+    assert len(eth_rpc) > 0
+    return eth_rpc[0].config()  # just pick the first of possible URLs for this exec client
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def network(request) -> NetworkConfig:
     """Get the network that will be used to verify all tests."""
     return request.config.network
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def current_time() -> int:
     """Get the `eth_config` response from the client to be verified by all tests."""
     return int(time.time())
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def expected_eth_config(network: NetworkConfig, current_time: int) -> EthConfigResponse:
     """Calculate the current fork value to verify against the client's response."""
     print(f"Network provided: {network}, Type: {type(network)}")
@@ -43,13 +41,8 @@ def expected_eth_config(network: NetworkConfig, current_time: int) -> EthConfigR
 def test_eth_config_current(
     eth_config_response: EthConfigResponse | None,
     expected_eth_config: EthConfigResponse,
-    request,
 ) -> None:
     """Validate `current` field of the `eth_config` RPC endpoint."""
-    config = request.config
-    if config.getoption("network_config_file") is None:
-        pytest.skip("Skipping test because no 'network_config_file' was specified")
-
     assert eth_config_response is not None, "Client did not return a valid `eth_config` response."
     assert eth_config_response.current is not None, (
         "Client did not return a valid `current` fork config."
@@ -65,13 +58,8 @@ def test_eth_config_current(
 def test_eth_config_current_fork_id(
     eth_config_response: EthConfigResponse | None,
     expected_eth_config: EthConfigResponse,
-    request,
 ) -> None:
     """Validate `forkId` field within the `current` configuration object."""
-    config = request.config
-    if config.getoption("network_config_file") is None:
-        pytest.skip("Skipping test because no 'network_config_file' was specified")
-
     assert eth_config_response is not None, "Client did not return a valid `eth_config` response."
     assert eth_config_response.current is not None, (
         "Client did not return a valid `current` fork config."
@@ -89,13 +77,8 @@ def test_eth_config_current_fork_id(
 def test_eth_config_next(
     eth_config_response: EthConfigResponse | None,
     expected_eth_config: EthConfigResponse,
-    request,
 ) -> None:
     """Validate `next` field of the `eth_config` RPC endpoint."""
-    config = request.config
-    if config.getoption("network_config_file") is None:
-        pytest.skip("Skipping test because no 'network_config_file' was specified")
-
     assert eth_config_response is not None, "Client did not return a valid `eth_config` response."
     expected_next = expected_eth_config.next
     if expected_next is None:
@@ -116,13 +99,8 @@ def test_eth_config_next(
 def test_eth_config_next_fork_id(
     eth_config_response: EthConfigResponse | None,
     expected_eth_config: EthConfigResponse,
-    request,
 ) -> None:
     """Validate `forkId` field within the `next` configuration object."""
-    config = request.config
-    if config.getoption("network_config_file") is None:
-        pytest.skip("Skipping test because no 'network_config_file' was specified")
-
     assert eth_config_response is not None, "Client did not return a valid `eth_config` response."
     expected_next = expected_eth_config.next
     if expected_next is None:
@@ -151,13 +129,8 @@ def test_eth_config_next_fork_id(
 def test_eth_config_last(
     eth_config_response: EthConfigResponse | None,
     expected_eth_config: EthConfigResponse,
-    request,
 ) -> None:
     """Validate `last` field of the `eth_config` RPC endpoint."""
-    config = request.config
-    if config.getoption("network_config_file") is None:
-        pytest.skip("Skipping test because no 'network_config_file' was specified")
-
     expected_last = expected_eth_config.last
     assert eth_config_response is not None, "Client did not return a valid `eth_config` response."
     if expected_last is None:
@@ -178,13 +151,8 @@ def test_eth_config_last(
 def test_eth_config_last_fork_id(
     eth_config_response: EthConfigResponse | None,
     expected_eth_config: EthConfigResponse,
-    request,
 ) -> None:
     """Validate `forkId` field within the `last` configuration object."""
-    config = request.config
-    if config.getoption("network_config_file") is None:
-        pytest.skip("Skipping test because no 'network_config_file' was specified")
-
     assert eth_config_response is not None, "Client did not return a valid `eth_config` response."
     expected_last = expected_eth_config.last
     if expected_last is None:
@@ -211,24 +179,15 @@ def test_eth_config_last_fork_id(
 
 
 def test_eth_config_majority(
-    eth_rpc: Dict[str, List[EthRPC]],
-    request,
+    all_rpc_endpoints: Dict[str, List[EthRPC]],
 ) -> None:
     """Queries devnet exec clients for their eth_config and fails if not all have the same response."""  # noqa: E501
-    # decide whether to run this test
-    config = request.config
-    run_this_test_bool = config.getoption(name="majority_eth_config_test_enabled")
-    if not run_this_test_bool:
-        pytest.skip("Skipping eth_config majority test")
-
-    assert eth_rpc is not None
-
     responses = dict()  # Dict[exec_client_name : response] # noqa: C408
     client_to_url_used_dict = dict()  # noqa: C408
-    for exec_client in eth_rpc.keys():
+    for exec_client in all_rpc_endpoints.keys():
         # try only as many consensus+exec client combinations until you receive a response
         # if all combinations for a given exec client fail we panic
-        for eth_rpc_target in eth_rpc[exec_client]:
+        for eth_rpc_target in all_rpc_endpoints[exec_client]:
             response = eth_rpc_target.config(timeout=10)
             if response is None:
                 # safely split url to not leak rpc_endpoint in logs
@@ -246,11 +205,14 @@ def test_eth_config_majority(
 
             break  # no need to gather more responses for this client
 
-    assert len(responses.keys()) == len(eth_rpc.keys()), "Failed to get an eth_config response "
-    f" from each specified execution client. Full list of execution clients is {eth_rpc.keys()} "
-    f"but we were only able to gather eth_config responses from: {responses.keys()}\nWill try "
-    "again with a different consensus-execution client combination for this execution client"
-
+    assert len(responses.keys()) == len(all_rpc_endpoints.keys()), (
+        "Failed to get an eth_config response "
+        f" from each specified execution client. Full list of execution clients is "
+        f"{all_rpc_endpoints.keys()} but we were only able to gather eth_config responses "
+        f"from: {responses.keys()}\n"
+        "Will try again with a different consensus-execution client combination for "
+        "this execution client"
+    )
     # determine hashes of client responses
     client_to_hash_dict = dict()  # Dict[exec_client : response hash] # noqa: C408
     for client in responses.keys():