Merge branch 'main' of github.com:smartcontractkit/chainlink-testing-…

…framework into flakeguardAggregations

book/src/SUMMARY.md

@@ -12,7 +12,6 @@
   - [NodeSet Compat Environment](./framework/nodeset_compatibility.md)
   - [Creating your own components](./developing/developing_components.md)
   - [Asserting Logs](./developing/asserting_logs.md)
-  - [Fork Testing](./framework/fork.md)
   - [Quick Contracts Deployment](./framework/quick_deployment.md)
   - [Verifying Contracts](./framework/verify.md)
   - [NodeSet with External Blockchain]()
@@ -55,7 +54,8 @@
 - [Testing Maturity Model](framework/testing.md)
   - [Smoke]()
   - [Performance]()
-  - [Chaos]()
+  - [Chaos](./framework/chaos/chaos.md)
+  - [Fork Testing](./framework/fork.md)
 - [Interactive](framework/interactive.md)
 - [Libraries](./libraries.md)
   - [Seth](./libs/seth.md)

book/src/framework/chaos/chaos.md

@@ -0,0 +1,59 @@
+# Chaos Testing
+
+We offer Docker and Kubernetes boilerplates designed to test the resilience of `NodeSet` and `Blockchain`, which you can customize and integrate into your pipeline.
+
+
+## Goals
+
+We recommend structuring your tests as a linear suite that applies various chaos experiments and verifies the outcomes using a [load](../../libs/wasp.md) testing suite. Focus on critical user metrics, such as:
+
+- The ratio of successful responses to failed responses
+- The nth percentile of response latency
+
+Next, evaluate observability:
+
+- Ensure proper alerts are triggered during failures (manual or automated)
+- Verify the service recovers within the expected timeframe (manual or automated)
+
+In summary, the **primary** focus is on meeting user expectations and maintaining SLAs, while the **secondary** focus is on observability and making operational part smoother.
+
+
+## Docker
+
+For Docker, we utilize [Pumba](https://github.com/alexei-led/pumba) to conduct chaos experiments, including:
+
+- Container reboots
+- Network simulations (such as delays, packet loss, corruption, etc., using the tc tool)
+- Stress testing for CPU and memory usage
+
+Additionally, we offer a [resources](../../framework/components/resources.md) API that allows you to test whether your software can operate effectively in low-resource environments.
+
+You can also use [fake](../../framework/components/mocking.md) package to create HTTP chaos experiments.
+
+Given the complexity of `Kubernetes`, we recommend starting with `Docker` first. Identifying faulty behavior in your services early—such as cascading latency—can prevent more severe issues when scaling up. Addressing these problems at a smaller scale can save significant time and effort later.
+
+Check `NodeSet` + `Blockchain` template [here]().
+
+## Kubernetes
+
+We utilize a subset of [ChaosMesh](https://chaos-mesh.org/) experiments that can be safely executed on an isolated node group. These include:
+
+- [Pod faults](https://chaos-mesh.org/docs/simulate-pod-chaos-on-kubernetes/)
+
+- [Network faults](https://chaos-mesh.org/docs/simulate-network-chaos-on-kubernetes/) – We focus on delay and partition experiments, as others may impact pods outside the dedicated node group.
+
+- [HTTP faults](https://chaos-mesh.org/docs/simulate-http-chaos-on-kubernetes/)
+
+Check `NodeSet` + `Blockchain` template [here]().
+
+## Blockchain
+
+We also offer a set of blockchain-specific experiments, which typically involve API calls to blockchain simulators to execute certain actions. These include:
+
+- Adjusting gas prices
+
+- Introducing chain reorganizations (setting a new head)
+
+- Utilizing developer APIs (e.g., Anvil)
+
+Check [gas]() and [reorg]() examples.

book/src/framework/components/chainlink/nodeset.md

@@ -42,6 +42,8 @@ This component requires some Blockchain to be deployed, add this to config
 Then configure NodeSet
 ```toml
 [nodeset]
+  # unique NodeSet name
+  name = "don"
   # amount of Chainlink nodes to spin up
   nodes = 5
   # Override mode: can be "all" or "each"

book/src/framework/components/state.md

@@ -44,6 +44,7 @@ One node set is enough for any kind of testing, if you need more nodes consider
 You can also define a custom set of ports for any node.
 ```toml
 [nodeset]
+  name = "don"
   nodes = 5
   override_mode = "each"
 

book/src/framework/nodeset_capabilities.md

@@ -21,6 +21,7 @@ Create a configuration file `smoke.toml`
   docker_cmd_params = ["-b", "1"]
 
 [nodeset]
+  name = "don"
   nodes = 5
   override_mode = "all"
 

book/src/framework/nodeset_compatibility.md

@@ -9,6 +9,7 @@ Create a configuration file `smoke.toml`
   docker_cmd_params = ["-b", "1"]
 
 [nodeset]
+  name = "don"
   nodes = 5
   override_mode = "each"
 

book/src/framework/nodeset_docker_rebuild.md

@@ -9,6 +9,7 @@ Create a configuration file `smoke.toml`
   docker_cmd_params = ["-b", "1"]
 
 [nodeset]
+  name = "don"
   nodes = 5
   override_mode = "all"
 

book/src/framework/nodeset_environment.md

@@ -9,6 +9,7 @@ Create a configuration file `smoke.toml`
   docker_cmd_params = ["-b", "1"]
 
 [nodeset]
+  name = "don"
   nodes = 5
   override_mode = "all"
 

framework/.changeset/v0.5.0.md

@@ -0,0 +1,2 @@
+- Allow multiple node sets simultaneously
+- Add chaos examples for gas, reorg, k8s and docker

framework/cmd/docker.go

@@ -14,7 +14,7 @@ func removeTestContainers() error {
 	cmd := exec.Command("bash", "-c", `
 		docker ps -aq --filter "label=framework=ctf" | xargs -r docker rm -f && \
 		docker network ls --filter "label=framework=ctf" -q | xargs -r docker network rm && \
-		docker volume rm postgresql_data || true
+		docker volume ls -q | xargs -r docker volume rm || true
 	`)
 	framework.L.Debug().Msg("Running command")
 	if framework.L.GetLevel() == zerolog.DebugLevel {

framework/components/clnode/clnode.go

@@ -244,7 +244,10 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 		},
 		ExposedPorts: exposedPorts,
 		Entrypoint:   generateEntryPoint(),
-		WaitingFor:   wait.ForHTTP("/").WithPort(DefaultHTTPPort).WithStartupTimeout(1 * time.Minute).WithPollInterval(200 * time.Millisecond),
+		WaitingFor: wait.ForHTTP("/").
+			WithPort(DefaultHTTPPort).
+			WithStartupTimeout(1 * time.Minute).
+			WithPollInterval(200 * time.Millisecond),
 	}
 	if in.Node.HTTPPort != 0 && in.Node.P2PPort != 0 {
 		req.HostConfigModifier = func(h *container.HostConfig) {

framework/components/clnode/clnode_test.go

@@ -24,7 +24,7 @@ func checkBasicOutputs(t *testing.T, output *clnode.Output) {
 	require.Contains(t, output.Node.DockerP2PUrl, "cl-node")
 	require.NotNil(t, output.PostgreSQL)
 	require.Contains(t, output.PostgreSQL.Url, "postgresql://chainlink:[email protected]")
-	require.Contains(t, output.PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@ns-postgresql")
+	require.Contains(t, output.PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@pg")
 }
 
 func TestComponentDockerNodeWithSharedDB(t *testing.T) {
@@ -35,6 +35,7 @@ func TestComponentDockerNodeWithSharedDB(t *testing.T) {
 				DbInput: &postgres.Input{
 					Image:      "postgres:12.0",
 					Port:       16000,
+					Name:       "pg-1",
 					VolumeName: "a",
 				},
 				Node: &clnode.NodeInput{
@@ -70,6 +71,7 @@ func TestComponentDockerNodeWithDB(t *testing.T) {
 				DbInput: &postgres.Input{
 					Image:      "postgres:12.0",
 					Port:       15000,
+					Name:       "pg-2",
 					VolumeName: "b",
 				},
 				Node: &clnode.NodeInput{

framework/components/postgres/postgres.go

@@ -48,10 +48,10 @@ func NewPostgreSQL(in *Input) (*Output, error) {
 
 	bindPort := fmt.Sprintf("%s/tcp", Port)
 	var containerName string
-	if in.Name != "" {
-		containerName = framework.DefaultTCName(in.Name)
+	if in.Name == "" {
+		containerName = "ns-postgresql"
 	} else {
-		containerName = framework.DefaultTCName("ns-postgresql")
+		containerName = in.Name
 	}
 
 	var sqlCommands []string
@@ -116,7 +116,7 @@ func NewPostgreSQL(in *Input) (*Output, error) {
 		},
 		WaitingFor: tcwait.ForExec([]string{"psql", "-h", "127.0.0.1",
 			"-U", User, "-p", Port, "-c", "select", "1", "-d", Database}).
-			WithStartupTimeout(15 * time.Second).
+			WithStartupTimeout(40 * time.Second).
 			WithPollInterval(200 * time.Millisecond),
 	}
 	var portToExpose int

framework/components/simple_node_set/node_set.go

@@ -20,6 +20,7 @@ const (
 
 // Input is a node set configuration input
 type Input struct {
+	Name               string          `toml:"name" validate:"required"`
 	Nodes              int             `toml:"nodes" validate:"required"`
 	HTTPPortRangeStart int             `toml:"http_port_range_start"`
 	P2PPortRangeStart  int             `toml:"p2p_port_range_start"`
@@ -75,6 +76,10 @@ func printURLs(out *Output) {
 }
 
 func sharedDBSetup(in *Input, bcOut *blockchain.Output) (*Output, error) {
+	in.DbInput.Name = fmt.Sprintf("%s-%s", in.Name, "ns-postgresql")
+	in.DbInput.VolumeName = in.Name
+
+	// create database for each node
 	in.DbInput.Databases = in.Nodes
 	dbOut, err := postgres.NewPostgreSQL(in.DbInput)
 	if err != nil {
@@ -116,9 +121,7 @@ func sharedDBSetup(in *Input, bcOut *blockchain.Output) (*Output, error) {
 				return nil, fmt.Errorf("custom_ports can be used only with override_mode = 'each'")
 			}
 		}
-		if in.NodeSpecs[overrideIdx].Node.Name == "" {
-			nodeName = fmt.Sprintf("node%d", i)
-		}
+
 		eg.Go(func() error {
 			var net string
 			net, err = clnode.NewNetworkCfgOneNetworkAllNodes(bcOut)
@@ -128,6 +131,8 @@ func sharedDBSetup(in *Input, bcOut *blockchain.Output) (*Output, error) {
 			if in.NodeSpecs[overrideIdx].Node.TestConfigOverrides != "" {
 				net = in.NodeSpecs[overrideIdx].Node.TestConfigOverrides
 			}
+			nodeName = fmt.Sprintf("node%d", i)
+			nodeWithNodeSetPrefixName := fmt.Sprintf("%s-%s", in.Name, nodeName)
 
 			nodeSpec := &clnode.Input{
 				DbInput: in.DbInput,
@@ -137,7 +142,7 @@ func sharedDBSetup(in *Input, bcOut *blockchain.Output) (*Output, error) {
 					DebuggerPort:            dlvPortStart + i,
 					CustomPorts:             in.NodeSpecs[overrideIdx].Node.CustomPorts,
 					Image:                   in.NodeSpecs[overrideIdx].Node.Image,
-					Name:                    nodeName,
+					Name:                    nodeWithNodeSetPrefixName,
 					PullImage:               in.NodeSpecs[overrideIdx].Node.PullImage,
 					DockerFilePath:          in.NodeSpecs[overrideIdx].Node.DockerFilePath,
 					DockerContext:           in.NodeSpecs[overrideIdx].Node.DockerContext,

framework/components/simple_node_set/nodeset_test.go

@@ -24,13 +24,13 @@ func checkBasicOutputs(t *testing.T, output *ns.Output) {
 	require.NotNil(t, output.CLNodes)
 	require.Len(t, output.CLNodes, 2)
 	require.Contains(t, output.CLNodes[0].PostgreSQL.Url, "postgresql://chainlink:[email protected]")
-	require.Contains(t, output.CLNodes[0].PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@ns-postgresql-")
+	require.Contains(t, output.CLNodes[0].PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@don")
 	require.Contains(t, output.CLNodes[0].Node.HostURL, "127.0.0.1")
 	require.Contains(t, output.CLNodes[0].Node.DockerURL, "node")
 	require.Contains(t, output.CLNodes[0].Node.DockerP2PUrl, "node")
 
 	require.Contains(t, output.CLNodes[1].PostgreSQL.Url, "postgresql://chainlink:[email protected]")
-	require.Contains(t, output.CLNodes[1].PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@ns-postgresql-")
+	require.Contains(t, output.CLNodes[1].PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@don")
 	require.Contains(t, output.CLNodes[1].Node.HostURL, "127.0.0.1")
 	require.Contains(t, output.CLNodes[1].Node.DockerURL, "node")
 	require.Contains(t, output.CLNodes[1].Node.DockerP2PUrl, "node")
@@ -47,6 +47,7 @@ func TestComponentDockerNodeSetSharedDB(t *testing.T) {
 				ChainID: "31337",
 			},
 			nodeSetInput: &ns.Input{
+				Name:         "don-1",
 				Nodes:        2,
 				OverrideMode: "all",
 				DbInput: &postgres.Input{
@@ -74,6 +75,7 @@ func TestComponentDockerNodeSetSharedDB(t *testing.T) {
 				ChainID: "31337",
 			},
 			nodeSetInput: &ns.Input{
+				Name:               "don-2",
 				Nodes:              2,
 				OverrideMode:       "each",
 				HTTPPortRangeStart: 20000,

framework/components/simple_node_set/reload.go

@@ -17,7 +17,7 @@ func UpgradeNodeSet(t *testing.T, in *Input, bc *blockchain.Output, wait time.Du
 	if _, err := framework.SaveContainerLogs(uniq); err != nil {
 		return nil, err
 	}
-	_, err := chaos.ExecPumba("rm --volumes=false re2:^node.*|ns-postgresql.*", wait)
+	_, err := chaos.ExecPumba(fmt.Sprintf("rm --volumes=false re2:^%s-node.*|%s-ns-postgresql.*", in.Name, in.Name), wait)
 	if err != nil {
 		return nil, err
 	}

framework/examples/myproject/chaos.toml → framework/examples/myproject/chaos/chaos.toml

@@ -1,12 +1,13 @@
 
 [blockchain_a]
-  type = "anvil"
   docker_cmd_params = ["-b", "1"]
+  type = "anvil"
 
 [data_provider]
   port = 9111
 
 [nodeset]
+  name = "don"
   nodes = 5
   override_mode = "all"