test(integration): improve integration test suite coverage

.github/workflows/rust.yml

@@ -289,11 +289,14 @@ jobs:
       - name: Install cargo-nextest
         uses: taiki-e/install-action@v2
         with:
-          tool: cargo-nextest
+          tool: cargo-nextest@0.9.130
 
       - name: Run contra integration tests with coverage
         run: make -C integration integration-coverage-contra
 
+      - name: Run Redis-backend coverage tests
+        run: make -C integration integration-coverage-contra-redis
+
       - name: Build escrow with test-tree for indexer tests
         run: make -C contra-escrow-program build-test
 

Makefile

@@ -64,22 +64,10 @@ integration-test:
 	@echo "Running integration tests for all projects..."
 	@$(MAKE) -C contra-escrow-program integration-test
 	@$(MAKE) -C contra-withdraw-program integration-test
-	@echo "Running contra integration test (with production build)..."
-	@cd integration && cargo nextest run --test contra_integration
-	@echo "Running reconciliation integration tests..."
-	@cd integration && cargo test --test reconciliation_integration -- --nocapture
-	@echo "Running mint idempotency integration tests..."
-	@cd integration && cargo test --test mint_idempotency_integration -- --nocapture
-	@echo "Running gap detection integration tests..."
-	@cd integration && cargo test --test gap_detection_integration -- --nocapture
-	@echo "Running truncate integration tests..."
-	@cd integration && cargo test --test truncate_integration -- --nocapture
-	@echo "Building escrow with test-tree for indexer and operator lifecycle tests..."
-	@$(MAKE) -C contra-escrow-program build-test
-	@echo "Running indexer integration test (with test-tree build)..."
-	@cd integration && cargo test --features test-tree --test indexer_integration -- --nocapture
-	@echo "Running operator lifecycle integration tests (with test-tree build)..."
-	@cd integration && cargo test --features test-tree --test operator_lifecycle_integration -- --nocapture
+	@# Delegate to integration/Makefile so escrow-feature grouping + the full
+	@# `[[test]]` set stay in one place. See integration/Makefile:integration-test
+	@# for the contra/test-tree/prod group ordering.
+	@$(MAKE) -C integration integration-test
 
 ci-unit-test:
 	@echo "Running CI unit tests for core + indexer..."
@@ -94,44 +82,107 @@ ci-integration-test:
 	@$(MAKE) ci-integration-test-build-test-tree
 
 ci-integration-test-build-test-tree:
+	@# Order matters: prod-feature first (escrow already prod from caller),
+	@# then rebuild test-tree, then run only test-tree-feature tests. Mixing
+	@# escrow features in a single run poisons the validator_helper Once.
 	@$(MAKE) ci-integration-test-prebuilt
 	@echo "Building escrow with test-tree for indexer and operator lifecycle tests..."
 	@$(MAKE) -C contra-escrow-program build-test
-	@echo "Running indexer integration test (with test-tree build)..."
+	@echo "=== test-tree feature group ==="
 	@cd integration && cargo test --features test-tree --test indexer_integration -- --nocapture
-	@echo "Running operator lifecycle integration tests (with test-tree build)..."
 	@cd integration && cargo test --features test-tree --test operator_lifecycle_integration -- --nocapture
+	@cd integration && cargo test --features test-tree --test withdrawal_null_nonce -- --nocapture
 
 ci-integration-test-prebuilt:
-	@echo "Running contra integration test (with production build)..."
+	@# Prod-feature suites that assume a prod escrow `.so` is already in
+	@# target/deploy/. Caller is responsible for the build. Keep in sync with
+	@# integration/Makefile's `integration-coverage-contra` (contra group) and
+	@# the prod-feature subset of `integration-coverage-indexer`.
+	@echo "=== contra group (prod escrow) ==="
 	@cd integration && cargo nextest run --test contra_integration
-	@echo "Running reconciliation integration tests..."
+	@cd integration && cargo test --test test_rpc_http_malformed -- --nocapture
+	@cd integration && cargo test --test test_simulate_transaction_guards -- --nocapture
+	@cd integration && cargo test --test test_health_endpoint_standalone -- --nocapture
+	@cd integration && cargo test --test test_sequencer_zero_deadline -- --nocapture
+	@cd integration && cargo test --test test_signatures_corruption_guard -- --nocapture
+	@cd integration && cargo test --test test_node_config_validation -- --nocapture
+	@cd integration && cargo test --test test_redis_cache_path -- --nocapture --test-threads=1
+	@echo "=== prod-feature indexer group ==="
 	@cd integration && cargo test --test reconciliation_integration -- --nocapture
-	@echo "Running mint idempotency integration tests..."
 	@cd integration && cargo test --test mint_idempotency_integration -- --nocapture
-	@echo "Running gap detection integration tests..."
 	@cd integration && cargo test --test gap_detection_integration -- --nocapture
-	@echo "Running truncate integration tests..."
 	@cd integration && cargo test --test truncate_integration -- --nocapture
+	@cd integration && cargo test --test resync_integration -- --nocapture
+	@cd integration && cargo test --test reconciliation_e2e_test -- --nocapture
+	@cd integration && cargo test --test mock_rpc_retry -- --nocapture
+	@cd integration && cargo test --test checkpoint_partial_flush -- --nocapture
+	@cd integration && cargo test --test remint_recovery -- --nocapture
+	@cd integration && cargo test --test bootstrap_validation -- --nocapture
+	@cd integration && cargo test --test yellowstone_wiring -- --nocapture
+	@cd integration && cargo test --test malformed_yellowstone_update -- --nocapture
+	@cd integration && cargo test --test yellowstone_reconnect_gap -- --nocapture
+	@cd integration && cargo test --test yellowstone_inner_and_unknown -- --nocapture
+	@cd integration && cargo test --test harness_sanity -- --nocapture
+	@cd integration && cargo test --test sender_mint_idempotency -- --nocapture
+	@cd integration && cargo test --test sender_mint_validator_encodings -- --nocapture
+	@cd integration && cargo test --test sender_mint_signature_failures -- --nocapture
+	@cd integration && cargo test --test sender_poll_rpc_error -- --nocapture
+	@cd integration && cargo test --test sender_sign_and_send_error -- --nocapture
+	@cd integration && cargo test --test sender_max_retries -- --nocapture
+	@cd integration && cargo test --test sender_onchain_error_arms -- --nocapture
+	@cd integration && cargo test --test jit_mint_helper -- --nocapture
+	@cd integration && cargo test --test storage_update_failure -- --nocapture
+	@cd integration && cargo test --test processor_quarantine -- --nocapture
+	@cd integration && cargo test --test state_recovery_malformed -- --nocapture
+	@cd integration && cargo test --test remint_flow -- --nocapture
+	@cd integration && cargo test --test mint_builder_validation -- --nocapture
+	@cd integration && cargo test --test sender_channel_close -- --nocapture
+	@cd integration && cargo test --test sender_cancellation_drain -- --nocapture
 
 # CI-focused integration target that runs indexer integration tests only.
+# Same group-ordering invariant as integration/Makefile:integration-coverage-indexer:
+#   1. Build test-tree → run test-tree-feature tests.
+#   2. Rebuild escrow as prod → run prod-feature indexer tests.
 ci-integration-test-indexer:
-	@echo "Building escrow with test-tree for indexer and operator lifecycle tests..."
+	@echo "Building escrow with test-tree for test-tree-feature group..."
 	@$(MAKE) -C contra-escrow-program build-test
-	@echo "Running indexer integration test (with test-tree build)..."
+	@echo "=== test-tree feature group ==="
 	@cd integration && cargo test --features test-tree --test indexer_integration -- --nocapture
-	@echo "Running reconciliation integration tests..."
+	@cd integration && cargo test --features test-tree --test operator_lifecycle_integration -- --nocapture
+	@cd integration && cargo test --features test-tree --test withdrawal_null_nonce -- --nocapture
+	@echo "=== Rebuilding escrow in prod for prod-feature indexer group ==="
+	@$(MAKE) -C contra-escrow-program build-no-clients
+	@echo "=== prod-feature indexer group ==="
 	@cd integration && cargo test --test reconciliation_integration -- --nocapture
-	@echo "Running mint idempotency integration tests..."
 	@cd integration && cargo test --test mint_idempotency_integration -- --nocapture
-	@echo "Running gap detection integration tests..."
 	@cd integration && cargo test --test gap_detection_integration -- --nocapture
-	@echo "Running truncate integration tests..."
 	@cd integration && cargo test --test truncate_integration -- --nocapture
-	@echo "Running operator lifecycle integration tests (with test-tree build)..."
-	@cd integration && cargo test --features test-tree --test operator_lifecycle_integration -- --nocapture
-	@echo "Running reconciliation e2e tests..."
-	@cd indexer && cargo test --test reconciliation_e2e_test -- --nocapture
+	@cd integration && cargo test --test resync_integration -- --nocapture
+	@cd integration && cargo test --test reconciliation_e2e_test -- --nocapture
+	@cd integration && cargo test --test mock_rpc_retry -- --nocapture
+	@cd integration && cargo test --test checkpoint_partial_flush -- --nocapture
+	@cd integration && cargo test --test remint_recovery -- --nocapture
+	@cd integration && cargo test --test bootstrap_validation -- --nocapture
+	@cd integration && cargo test --test yellowstone_wiring -- --nocapture
+	@cd integration && cargo test --test malformed_yellowstone_update -- --nocapture
+	@cd integration && cargo test --test yellowstone_reconnect_gap -- --nocapture
+	@cd integration && cargo test --test yellowstone_inner_and_unknown -- --nocapture
+	@cd integration && cargo test --test harness_sanity -- --nocapture
+	@cd integration && cargo test --test sender_mint_idempotency -- --nocapture
+	@cd integration && cargo test --test sender_mint_validator_encodings -- --nocapture
+	@cd integration && cargo test --test sender_mint_signature_failures -- --nocapture
+	@cd integration && cargo test --test sender_poll_rpc_error -- --nocapture
+	@cd integration && cargo test --test sender_sign_and_send_error -- --nocapture
+	@cd integration && cargo test --test sender_max_retries -- --nocapture
+	@cd integration && cargo test --test sender_onchain_error_arms -- --nocapture
+	@cd integration && cargo test --test jit_mint_helper -- --nocapture
+	@cd integration && cargo test --test storage_update_failure -- --nocapture
+	@cd integration && cargo test --test processor_quarantine -- --nocapture
+	@cd integration && cargo test --test state_recovery_malformed -- --nocapture
+	@cd integration && cargo test --test remint_flow -- --nocapture
+	@cd integration && cargo test --test mint_builder_validation -- --nocapture
+	@cd integration && cargo test --test sender_channel_close -- --nocapture
+	@cd integration && cargo test --test sender_cancellation_drain -- --nocapture
 
 # Backward-compatible aliases.
 unit-test-ci: ci-unit-test

README.md

@@ -274,6 +274,38 @@ docker compose --profile auth up
 
 Once running, the auth service is available at `http://localhost:${AUTH_PORT}` (default `8903`). See [auth/README.md](auth/README.md) for the full API reference, role definitions, and wallet verification flow.
 
+## Pinned tooling versions
+
+CI pins the following tool versions. Local development environments should
+match these to keep coverage reports and test behavior reproducible between
+CI and local runs.
+
+| Tool             | Version    | Install command                                        |
+|------------------|------------|--------------------------------------------------------|
+| Rust toolchain   | `1.91.0`   | Pinned in `rust-toolchain.toml` — `rustup` picks it up automatically |
+| cargo-llvm-cov   | `0.8.4`    | `cargo install cargo-llvm-cov@0.8.4`                   |
+| cargo-nextest    | `0.9.130`  | `cargo install cargo-nextest@0.9.130 --locked`         |
+| Solana CLI       | `2.2.19`   | `sh -c "$(curl -sSfL https://release.anza.xyz/v2.2.19/install)"` |
+| Node.js          | `22.x`     | See your distro's package manager                      |
+| pnpm             | `10.15.1`  | `npm install -g pnpm@10.15.1` (also pinned via `packageManager` in each `package.json`) |
+
+Container images used by integration tests (pulled automatically by
+`testcontainers` at test time):
+
+| Image                | Notes                                          |
+|----------------------|------------------------------------------------|
+| `postgres:11-alpine` | Default of `testcontainers-modules 0.13`.      |
+| `redis:7`            | Warmed in CI before each integration run.      |
+
+Source of truth for tool versions:
+- Rust + `cargo-llvm-cov` + pnpm: [`.github/actions/setup-environment/action.yml`](.github/actions/setup-environment/action.yml)
+- `cargo-nextest`: [`.github/workflows/rust.yml`](.github/workflows/rust.yml) (`Install cargo-nextest` step)
+- Solana CLI: [`.github/actions/setup-solana/action.yml`](.github/actions/setup-solana/action.yml)
+- pnpm (also): `packageManager` field in `contra-escrow-program/package.json`
+
+When bumping any version, update the CI config **and** this section in the
+same PR so the two stay in sync.
+
 ## License
 
 This project is licensed under the MIT License. See [LICENSE](LICENSE) for details.

contra-escrow-program/Makefile

@@ -6,7 +6,7 @@ PROGRAM_SO := ../target/deploy/contra_escrow_program.so
 PROGRAM_KEYPAIR := ../target/deploy/contra_escrow_program-keypair.json
 PROGRAM_ENV_KEY := ESCROW_PROGRAM_ID
 
-.PHONY: install build build-test fmt generate-idl generate-clients
+.PHONY: install build build-test build-no-clients fmt generate-idl generate-clients
 .PHONY: unit-test integration-test integration-test-no-build all-test
 .PHONY: unit-coverage coverage-html all-coverage
 .PHONY: build-localnet build-devnet deploy-devnet
@@ -18,12 +18,20 @@ install:
 # Build the program
 build:
 	$(MAKE) generate-clients
-	cargo-build-sbf
+	cd program && cargo-build-sbf
 
 # Build the program with test-tree feature for integration tests
 build-test:
 	cd program && cargo-build-sbf --features test-tree
 
+# Build the prod program WITHOUT regenerating IDL / clients. Use after
+# `make build` (or after restoring a CI artifact that already contains
+# `clients/` and `idl/`) when you only need to rebuild the BPF binary —
+# e.g. mid-test-suite when toggling between test-tree and prod features.
+# Avoids the pnpm dependency that `generate-clients` requires.
+build-no-clients:
+	cd program && cargo-build-sbf
+
 # Format / lint code
 fmt:
 	cargo fmt --all

indexer/Cargo.toml

@@ -17,6 +17,12 @@ datasource-yellowstone = [
 # Test features
 test-tree = [] # Use smaller tree size for integration tests
 
+# Expose `Storage::Mock` and `mock::MockStorage` outside of `#[cfg(test)]`
+# so integration-test binaries in other crates can inject deterministic
+# storage faults without touching production code paths. Never enable
+# for a release build.
+test-mock-storage = []
+
 [lib]
 path = "src/lib.rs"
 

indexer/src/indexer/datasource/rpc_polling/rpc.rs

@@ -42,7 +42,8 @@ impl RpcPoller {
                         "encoding": self.encoding.to_string(),
                         "transactionDetails": "full",
                         "maxSupportedTransactionVersion": 0,
-                        "rewards": false
+                        "rewards": false,
+                        "commitment": self.commitment.to_string(),
                     }
                 ]
             }))
@@ -226,6 +227,39 @@ mod tests {
         assert!(result.unwrap_err().to_string().contains("RPC error"));
     }
 
+    /// Regression: `get_block` must forward the configured commitment in the
+    /// request body. A previous version omitted the field, so the server
+    /// defaulted to `finalized` regardless of `RpcPoller::new`'s argument —
+    /// causing `solana-test-validator` to reply `-32009` for slots that were
+    /// only confirmed.
+    #[tokio::test]
+    async fn test_get_block_forwards_configured_commitment() {
+        let mut server = Server::new_async().await;
+        let m = server
+            .mock("POST", "/")
+            .match_body(mockito::Matcher::PartialJson(json!({
+                "method": "getBlock",
+                "params": [
+                    101,
+                    { "commitment": "confirmed" }
+                ]
+            })))
+            .with_status(200)
+            .with_body(r#"{"jsonrpc":"2.0","result":null,"id":1}"#)
+            .expect(1)
+            .create_async()
+            .await;
+
+        let poller = RpcPoller::new(
+            server.url(),
+            UiTransactionEncoding::Json,
+            CommitmentLevel::Confirmed,
+        );
+        let _ = poller.get_block(101).await;
+
+        m.assert_async().await;
+    }
+
     // ============================================================================
     // get_latest_slot Tests
     // ============================================================================