gas and reorg tests, docs

book/src/SUMMARY.md

@@ -12,7 +12,6 @@
   - [NodeSet Compat Environment](./framework/nodeset_compatibility.md)
   - [Creating your own components](./developing/developing_components.md)
   - [Asserting Logs](./developing/asserting_logs.md)
-  - [Fork Testing](./framework/fork.md)
   - [Quick Contracts Deployment](./framework/quick_deployment.md)
   - [Verifying Contracts](./framework/verify.md)
   - [NodeSet with External Blockchain]()
@@ -55,7 +54,8 @@
 - [Testing Maturity Model](framework/testing.md)
   - [Smoke]()
   - [Performance]()
-  - [Chaos]()
+  - [Chaos](./framework/chaos/chaos.md)
+  - [Fork Testing](./framework/fork.md)
 - [Interactive](framework/interactive.md)
 - [Libraries](./libraries.md)
   - [Seth](./libs/seth.md)

book/src/framework/chaos/chaos.md

@@ -0,0 +1,59 @@
+# Chaos Testing
+
+We offer Docker and Kubernetes boilerplates designed to test the resilience of `NodeSet` and `Blockchain`, which you can customize and integrate into your pipeline.
+
+
+## Goals
+
+We recommend structuring your tests as a linear suite that applies various chaos experiments and verifies the outcomes using a [load](../../libs/wasp.md) testing suite. Focus on critical user metrics, such as:
+
+- The ratio of successful responses to failed responses
+- The nth percentile of response latency
+
+Next, evaluate observability:
+
+- Ensure proper alerts are triggered during failures (manual or automated)
+- Verify the service recovers within the expected timeframe (manual or automated)
+
+In summary, the **primary** focus is on meeting user expectations and maintaining SLAs, while the **secondary** focus is on observability and making operational part smoother.
+
+
+## Docker
+
+For Docker, we utilize [Pumba](https://github.com/alexei-led/pumba) to conduct chaos experiments, including:
+
+- Container reboots
+- Network simulations (such as delays, packet loss, corruption, etc., using the tc tool)
+- Stress testing for CPU and memory usage
+
+Additionally, we offer a [resources](../../framework/components/resources.md) API that allows you to test whether your software can operate effectively in low-resource environments.
+
+You can also use [fake](../../framework/components/mocking.md) package to create HTTP chaos experiments.
+
+Given the complexity of `Kubernetes`, we recommend starting with `Docker` first. Identifying faulty behavior in your services early—such as cascading latency—can prevent more severe issues when scaling up. Addressing these problems at a smaller scale can save significant time and effort later.
+
+Check `NodeSet` + `Blockchain` template [here]().
+
+## Kubernetes
+
+We utilize a subset of [ChaosMesh](https://chaos-mesh.org/) experiments that can be safely executed on an isolated node group. These include:
+
+- [Pod faults](https://chaos-mesh.org/docs/simulate-pod-chaos-on-kubernetes/)
+
+- [Network faults](https://chaos-mesh.org/docs/simulate-network-chaos-on-kubernetes/) – We focus on delay and partition experiments, as others may impact pods outside the dedicated node group.
+
+- [HTTP faults](https://chaos-mesh.org/docs/simulate-http-chaos-on-kubernetes/)
+
+Check `NodeSet` + `Blockchain` template [here]().
+
+## Blockchain
+
+We also offer a set of blockchain-specific experiments, which typically involve API calls to blockchain simulators to execute certain actions. These include:
+
+- Adjusting gas prices
+
+- Introducing chain reorganizations (setting a new head)
+
+- Utilizing developer APIs (e.g., Anvil)
+
+Check [gas]() and [reorg]() examples.

framework/examples/myproject/chaos.toml → framework/examples/myproject/chaos/chaos.toml

@@ -1,7 +1,7 @@
 
 [blockchain_a]
-  type = "anvil"
   docker_cmd_params = ["-b", "1"]
+  type = "anvil"
 
 [data_provider]
   port = 9111

framework/examples/myproject/chaos/chaos_blockchain_evm_gas_test.go

@@ -0,0 +1,123 @@
+package chaos
+
+import (
+	"context"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/clclient"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake"
+	ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/rpc"
+	"github.com/stretchr/testify/require"
+	"math/big"
+	"testing"
+	"time"
+)
+
+func printBlockBaseFee(t *testing.T, url string) {
+	ec, err := ethclient.Dial(url)
+	require.NoError(t, err)
+	bn, err := ec.BlockNumber(context.Background())
+	require.NoError(t, err)
+	b, err := ec.BlockByNumber(context.Background(), big.NewInt(int64(bn)))
+	require.NoError(t, err)
+	t.Logf("Current block base fee: %d", b.BaseFee())
+}
+
+type CfgGas struct {
+	BlockchainA        *blockchain.Input `toml:"blockchain_a" validate:"required"`
+	MockerDataProvider *fake.Input       `toml:"data_provider" validate:"required"`
+	NodeSet            *ns.Input         `toml:"nodeset" validate:"required"`
+}
+
+func TestBlockchainGasChaos(t *testing.T) {
+	in, err := framework.Load[CfgGas](t)
+	require.NoError(t, err)
+
+	// Can replace deployments with CRIB here
+
+	bc, err := blockchain.NewBlockchainNetwork(in.BlockchainA)
+	require.NoError(t, err)
+	_, err = fake.NewFakeDataProvider(in.MockerDataProvider)
+	require.NoError(t, err)
+	out, err := ns.NewSharedDBNodeSet(in.NodeSet, bc)
+	require.NoError(t, err)
+
+	c, err := clclient.New(out.CLNodes)
+	require.NoError(t, err)
+
+	// !! This value must match anvil block speed to set gas values for every block !!
+	blockEvery := 1 * time.Second
+	waitBetweenTests := 1 * time.Minute
+
+	gasControlFunc := func(t *testing.T, r *rpc.RPCClient, url string) {
+		startGasPrice := big.NewInt(2e9)
+		// ramp
+		for i := 0; i < 10; i++ {
+			printBlockBaseFee(t, url)
+			t.Logf("Setting block base fee: %d", startGasPrice)
+			err := r.AnvilSetNextBlockBaseFeePerGas(startGasPrice)
+			require.NoError(t, err)
+			startGasPrice = startGasPrice.Add(startGasPrice, big.NewInt(1e9))
+			time.Sleep(blockEvery)
+		}
+		// hold
+		for i := 0; i < 10; i++ {
+			printBlockBaseFee(t, url)
+			t.Logf("Setting block base fee: %d", startGasPrice)
+			err := r.AnvilSetNextBlockBaseFeePerGas(startGasPrice)
+			require.NoError(t, err)
+		}
+		// release
+		for i := 0; i < 10; i++ {
+			printBlockBaseFee(t, url)
+			time.Sleep(blockEvery)
+		}
+	}
+
+	testCases := []struct {
+		name             string
+		chainURL         string
+		increase         *big.Int
+		waitBetweenTests time.Duration
+		gasFunc          func(t *testing.T, r *rpc.RPCClient, url string)
+		validate         func(t *testing.T, c []*clclient.ChainlinkClient)
+	}{
+		{
+			name:             "Slow and low",
+			chainURL:         bc.Nodes[0].HostHTTPUrl,
+			waitBetweenTests: 30 * time.Second,
+			increase:         big.NewInt(1e9),
+			gasFunc:          gasControlFunc,
+			validate: func(t *testing.T, c []*clclient.ChainlinkClient) {
+				// add more clients and validate
+			},
+		},
+		{
+			name:             "Fast and degen",
+			chainURL:         bc.Nodes[0].HostHTTPUrl,
+			waitBetweenTests: 30 * time.Second,
+			increase:         big.NewInt(5e9),
+			gasFunc:          gasControlFunc,
+			validate: func(t *testing.T, c []*clclient.ChainlinkClient) {
+				// add more clients and validate
+			},
+		},
+	}
+
+	// Start WASP load test here, apply average load profile that you expect in production!
+	// Configure timeouts and validate all the test cases until the test ends
+
+	// Run test cases
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Log(tc.name)
+			printBlockBaseFee(t, tc.chainURL)
+			r := rpc.New(tc.chainURL, nil)
+			tc.gasFunc(t, r, tc.chainURL)
+			tc.validate(t, c)
+			time.Sleep(waitBetweenTests)
+		})
+	}
+}

framework/examples/myproject/chaos/chaos_blockchain_evm_reorg_test.go

@@ -0,0 +1,126 @@
+package chaos
+
+import (
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/clclient"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/clnode"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake"
+	ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/rpc"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+type CfgReorgTwoChains struct {
+	BlockchainA        *blockchain.Input `toml:"blockchain_a" validate:"required"`
+	BlockchainB        *blockchain.Input `toml:"blockchain_b" validate:"required"`
+	MockerDataProvider *fake.Input       `toml:"data_provider" validate:"required"`
+	NodeSet            *ns.Input         `toml:"nodeset" validate:"required"`
+}
+
+func TestBlockchainReorgChaos(t *testing.T) {
+	in, err := framework.Load[CfgReorgTwoChains](t)
+	require.NoError(t, err)
+
+	// Can replace deployments with CRIB here
+
+	bcA, err := blockchain.NewBlockchainNetwork(in.BlockchainA)
+	require.NoError(t, err)
+	bcB, err := blockchain.NewBlockchainNetwork(in.BlockchainB)
+	require.NoError(t, err)
+	// create network configs for 2 EVM networks
+	srcNetworkCfg, err := clnode.NewNetworkCfg(&clnode.EVMNetworkConfig{
+		MinIncomingConfirmations: 1,
+		MinContractPayment:       "0.00001 link",
+		ChainID:                  bcA.ChainID,
+		EVMNodes: []*clnode.EVMNode{
+			{
+				SendOnly: false,
+				Order:    100,
+			},
+		},
+	}, bcA)
+	dstNetworkConfig, err := clnode.NewNetworkCfg(&clnode.EVMNetworkConfig{
+		MinIncomingConfirmations: 1,
+		MinContractPayment:       "0.00001 link",
+		ChainID:                  bcA.ChainID,
+		EVMNodes: []*clnode.EVMNode{
+			{
+				SendOnly: false,
+				Order:    100,
+			},
+		},
+	}, bcB)
+	// override the configuration to connect with 2 networks
+	in.NodeSet.NodeSpecs[0].Node.TestConfigOverrides = srcNetworkCfg + dstNetworkConfig
+	// create a node set
+	nodesOut, err := ns.NewSharedDBNodeSet(in.NodeSet, bcA)
+	require.NoError(t, err)
+
+	c, err := clclient.New(nodesOut.CLNodes)
+	require.NoError(t, err)
+
+	testCases := []struct {
+		name       string
+		wait       time.Duration
+		chainURL   string
+		reorgDepth int
+		validate   func(c []*clclient.ChainlinkClient) error
+	}{
+		{
+			name:       "Reorg src with depth: 1",
+			wait:       30 * time.Second,
+			chainURL:   bcA.Nodes[0].HostHTTPUrl,
+			reorgDepth: 1,
+			validate: func(c []*clclient.ChainlinkClient) error {
+				// add clients and validate
+				return nil
+			},
+		},
+		{
+			name:       "Reorg dst with depth: 1",
+			wait:       30 * time.Second,
+			chainURL:   bcB.Nodes[0].HostHTTPUrl,
+			reorgDepth: 1,
+			validate: func(c []*clclient.ChainlinkClient) error {
+				return nil
+			},
+		},
+		{
+			name:       "Reorg src with depth: 5",
+			wait:       30 * time.Second,
+			chainURL:   bcA.Nodes[0].HostHTTPUrl,
+			reorgDepth: 5,
+			validate: func(c []*clclient.ChainlinkClient) error {
+				return nil
+			},
+		},
+		{
+			name:       "Reorg dst with depth: 5",
+			wait:       30 * time.Second,
+			chainURL:   bcB.Nodes[0].HostHTTPUrl,
+			reorgDepth: 5,
+			validate: func(c []*clclient.ChainlinkClient) error {
+				return nil
+			},
+		},
+	}
+
+	// Start WASP load test here, apply average load profile that you expect in production!
+	// Configure timeouts and validate all the test cases until the test ends
+
+	// Run test cases
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Log(tc.name)
+			r := rpc.New(tc.chainURL, nil)
+			err := r.GethSetHead(tc.reorgDepth)
+			require.NoError(t, err)
+			time.Sleep(tc.wait)
+			err = tc.validate(c)
+			require.NoError(t, err)
+		})
+	}
+}