Deploy workflow testing

.github/actions/export-repository-variables/action.yml

@@ -26,8 +26,8 @@ runs:
             "BB_RPC_URL_WS_",
             "BB_RPC_BIND_HOST_",
             "BB_RPC_ALLOW_IP_",
-            "BB_API_URL_HTTP_",
-            "BB_API_URL_WS_",
+            "BB_TEST_API_URL_HTTP_",
+            "BB_TEST_API_URL_WS_",
         )
 
         def write_env_var(env_file, key, value):

.github/scripts/prepare_deploy_plan.py

@@ -20,6 +20,12 @@ def matchable_name(coin: str) -> str:
     return coin + "=main"
 
 
+def test_coin_name(coin: str) -> str:
+    if coin.endswith("_archive"):
+        return coin[: -len("_archive")]
+    return coin
+
+
 def load_runner_map(vars_map: dict) -> dict:
     prefix = "BB_RUNNER_"
     mapping = {}
@@ -82,6 +88,7 @@ def main() -> None:
 
     deploy_matrix = []
     e2e_names = []
+    test_coins = []
 
     for coin in requested:
         if coin not in runner_map:
@@ -91,16 +98,22 @@ def main() -> None:
         if not coin_cfg_path.exists():
             fail(f"unknown coin '{coin}' (missing {coin_cfg_path})")
 
-        test_cfg = tests_cfg.get(coin)
+        lookup_coin = test_coin_name(coin)
+        test_cfg = tests_cfg.get(lookup_coin)
         if not isinstance(test_cfg, dict) or "connectivity" not in test_cfg:
-            fail(f"coin '{coin}' has no connectivity tests in tests/tests.json")
+            fail(
+                f"coin '{coin}' maps to test coin '{lookup_coin}' "
+                "which has no connectivity tests in tests/tests.json"
+            )
 
         deploy_matrix.append({"coin": coin, "runner": runner_map[coin]})
-        e2e_names.append(matchable_name(coin))
+        e2e_names.append(matchable_name(lookup_coin))
+        test_coins.append(lookup_coin)
 
     unique_names = sorted(set(e2e_names))
     if not unique_names:
         fail("no coins selected after validation")
+    unique_test_coins = sorted(set(test_coins))
 
     escaped = [re.escape(name) for name in unique_names]
     e2e_regex = "TestIntegration/(" + "|".join(escaped) + ")/api"
@@ -113,6 +126,7 @@ def main() -> None:
         out.write(f"deploy_matrix={json.dumps(deploy_matrix, separators=(',', ':'))}\n")
         out.write(f"e2e_regex={e2e_regex}\n")
         out.write(f"coins_csv={','.join(requested)}\n")
+        out.write(f"test_coins_csv={','.join(unique_test_coins)}\n")
 
     print("Selected coins:", ", ".join(requested))
     print("E2E regex:", e2e_regex)

.github/scripts/wait_for_blockbook_sync.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import ssl
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+
+def fail(message: str) -> None:
+    print(f"error: {message}", file=sys.stderr)
+    raise SystemExit(1)
+
+
+def parse_requested_coins(raw: str) -> list[str]:
+    text = raw.strip()
+    if not text:
+        fail("COINS_INPUT is empty")
+
+    seen = set()
+    result = []
+    for part in text.split(","):
+        coin = part.strip().lower()
+        if not coin or coin in seen:
+            continue
+        seen.add(coin)
+        result.append(coin)
+    if not result:
+        fail("COINS_INPUT resolved to an empty list")
+    return result
+
+
+def normalize_http_base(raw: str) -> str:
+    parsed = urllib.parse.urlparse(raw.strip())
+    if parsed.scheme not in ("http", "https"):
+        fail(f"unsupported HTTP scheme {parsed.scheme!r} in {raw!r}")
+    if not parsed.netloc:
+        fail(f"missing host in {raw!r}")
+    return urllib.parse.urlunparse(
+        (parsed.scheme, parsed.netloc, parsed.path or "/", "", "", "")
+    ).rstrip("/")
+
+
+def should_upgrade_to_https(status: int, body: bytes, base_url: str) -> bool:
+    if status != 400:
+        return False
+    if "http request to an https server" not in body.decode("utf-8", "replace").lower():
+        return False
+    parsed = urllib.parse.urlparse(base_url)
+    return parsed.scheme == "http"
+
+
+def upgrade_http_base_to_https(raw: str) -> str:
+    parsed = urllib.parse.urlparse(raw)
+    if parsed.scheme != "http":
+        return raw
+    return urllib.parse.urlunparse(
+        ("https", parsed.netloc, parsed.path, "", "", "")
+    ).rstrip("/")
+
+
+def resolve_http_base(coin: str) -> str:
+    value = os.environ.get("BB_TEST_API_URL_HTTP_" + coin, "").strip()
+    if not value:
+        fail(f"missing BB_TEST_API_URL_HTTP_{coin} for selected test coin {coin!r}")
+    return normalize_http_base(value)
+
+
+def preview_body(body: bytes, limit: int = 200) -> str:
+    text = body.decode("utf-8", "replace").strip()
+    if len(text) <= limit:
+        return text
+    return text[: limit - 3] + "..."
+
+
+def fetch_status(base_url: str, request_timeout: int) -> tuple[int, bytes]:
+    request = urllib.request.Request(base_url + "/api/status")
+    context = ssl._create_unverified_context()
+    with urllib.request.urlopen(request, timeout=request_timeout, context=context) as resp:
+        return resp.getcode(), resp.read()
+
+
+def parse_sync_state(body: bytes) -> tuple[bool, str]:
+    try:
+        payload = json.loads(body)
+    except json.JSONDecodeError as exc:
+        return False, f"invalid JSON: {exc}"
+
+    blockbook = payload.get("blockbook")
+    if not isinstance(blockbook, dict):
+        return False, "response missing blockbook object"
+
+    in_sync = blockbook.get("inSync")
+    best_height = blockbook.get("bestHeight")
+    summary = f"inSync={in_sync!r}, bestHeight={best_height!r}"
+    return in_sync is True, summary
+
+
+def main() -> None:
+    coins = parse_requested_coins(os.environ.get("COINS_INPUT", ""))
+    timeout_seconds = int(os.environ.get("SYNC_TIMEOUT_SECONDS", "1800"))
+    poll_seconds = int(os.environ.get("SYNC_POLL_SECONDS", "10"))
+    request_timeout = int(os.environ.get("SYNC_REQUEST_TIMEOUT_SECONDS", "20"))
+
+    pending = {}
+    last_seen = {}
+    for coin in coins:
+        if coin in pending:
+            continue
+        pending[coin] = resolve_http_base(coin)
+        last_seen[coin] = "not checked yet"
+
+    deadline = time.monotonic() + timeout_seconds
+    print(
+        "Waiting for Blockbook sync:",
+        ", ".join(f"{coin} -> {base}" for coin, base in sorted(pending.items())),
+        flush=True,
+    )
+
+    while pending:
+        for coin in sorted(list(pending)):
+            base_url = pending[coin]
+            try:
+                status, body = fetch_status(base_url, request_timeout)
+            except urllib.error.HTTPError as exc:
+                status = exc.code
+                body = exc.read()
+            except Exception as exc:
+                last_seen[coin] = f"{base_url}/api/status request failed: {exc}"
+                continue
+
+            if should_upgrade_to_https(status, body, base_url):
+                base_url = upgrade_http_base_to_https(base_url)
+                pending[coin] = base_url
+                try:
+                    status, body = fetch_status(base_url, request_timeout)
+                except urllib.error.HTTPError as exc:
+                    status = exc.code
+                    body = exc.read()
+                except Exception as exc:
+                    last_seen[coin] = f"{base_url}/api/status request failed: {exc}"
+                    continue
+
+            if status != 200:
+                last_seen[coin] = (
+                    f"{base_url}/api/status returned HTTP {status}: {preview_body(body)}"
+                )
+                continue
+
+            in_sync, summary = parse_sync_state(body)
+            last_seen[coin] = f"{base_url}/api/status returned HTTP 200: {summary}"
+            if in_sync:
+                print(f"{coin}: synced ({summary})", flush=True)
+                del pending[coin]
+
+        if not pending:
+            break
+
+        remaining_seconds = int(max(0, deadline - time.monotonic()))
+        if remaining_seconds == 0:
+            break
+
+        details = "; ".join(
+            f"{coin}: {last_seen[coin]}" for coin in sorted(pending)
+        )
+        print(
+            f"Still waiting for Blockbook sync ({remaining_seconds}s left): {details}",
+            flush=True,
+        )
+        time.sleep(min(poll_seconds, remaining_seconds))
+
+    if pending:
+        details = "; ".join(
+            f"{coin}: {last_seen[coin]}" for coin in sorted(pending)
+        )
+        fail(
+            f"timed out after {timeout_seconds}s waiting for Blockbook sync. {details}"
+        )
+
+    print("All selected Blockbook instances are synced.", flush=True)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/deploy.yml

@@ -22,6 +22,7 @@ jobs:
       deploy_matrix: ${{ steps.plan.outputs.deploy_matrix }}
       e2e_regex: ${{ steps.plan.outputs.e2e_regex }}
       coins_csv: ${{ steps.plan.outputs.coins_csv }}
+      test_coins_csv: ${{ steps.plan.outputs.test_coins_csv }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -57,11 +58,34 @@ jobs:
       - name: Deploy blockbook package
         run: ./contrib/scripts/deploy-blockbook-local.sh "${{ matrix.coin }}"
 
-  e2e-tests:
-    name: E2E Tests (post-deploy)
+  wait-for-sync:
+    name: Wait For Sync
     needs: [prepare, deploy]
     if: ${{ needs.deploy.result == 'success' }}
     runs-on: [self-hosted, bb-dev-selfhosted]
+    timeout-minutes: 31
+    env:
+      COINS_INPUT: ${{ needs.prepare.outputs.test_coins_csv }}
+      SYNC_TIMEOUT_SECONDS: "1800"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref != '' && inputs.ref || github.ref }}
+
+      - name: Export repository variables
+        uses: ./.github/actions/export-repository-variables
+        with:
+          vars_json: ${{ toJSON(vars) }}
+
+      - name: Wait for Blockbook sync
+        run: python3 ./.github/scripts/wait_for_blockbook_sync.py
+
+  e2e-tests:
+    name: E2E Tests (post-deploy)
+    needs: [prepare, deploy, wait-for-sync]
+    if: ${{ needs.deploy.result == 'success' && needs.wait-for-sync.result == 'success' }}
+    runs-on: [self-hosted, bb-dev-selfhosted]
     env:
       E2E_REGEX: ${{ needs.prepare.outputs.e2e_regex }}
     steps:
@@ -76,4 +100,4 @@ jobs:
           vars_json: ${{ toJSON(vars) }}
 
       - name: Run e2e tests
-        run: make test-e2e ARGS="-v -run ${E2E_REGEX}"
+        run: make test-e2e ARGS="-v"

Makefile

@@ -7,8 +7,8 @@ NO_CACHE = false
 TCMALLOC =
 PORTABLE = 0
 ARGS ?=
-# Forward BB_RPC_* and BB_API_* overrides into Docker for build/test tooling.
-BB_RPC_ENV := $(shell env | awk -F= '/^BB_RPC_(URL_HTTP|URL_WS|BIND_HOST|ALLOW_IP)_|^BB_API_URL_(HTTP|WS)_/ {print "-e " $$1}')
+# Forward BB_RPC_* and BB_TEST_API_* overrides into Docker for build/test tooling.
+BB_RPC_ENV := $(shell env | awk -F= '/^BB_RPC_(URL_HTTP|URL_WS|BIND_HOST|ALLOW_IP)_|^BB_TEST_API_URL_(HTTP|WS)_/ {print "-e " $$1}')
 
 TARGETS=$(subst .json,, $(shell ls configs/coins))
 
@@ -27,7 +27,7 @@ test-integration: .bin-image
 	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-integration ARGS="$(ARGS)"
 
 test-e2e: .bin-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-e2e ARGS="$(ARGS)"
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -e E2E_REGEX $(BB_RPC_ENV) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-e2e ARGS="$(ARGS)"
 
 test-connectivity: .bin-image
 	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-connectivity ARGS="$(ARGS)"

build/docker/bin/Makefile

@@ -30,7 +30,7 @@ test-integration: prepare-sources
 	cd $(BLOCKBOOK_SRC) && go test -tags 'integration' `go list github.com/trezor/blockbook/tests/...` -run 'TestIntegration/.*/(rpc|sync)' -timeout 30m $(ARGS)
 
 test-e2e: prepare-sources
-	cd $(BLOCKBOOK_SRC) && go test -tags 'integration' `go list github.com/trezor/blockbook/tests/...` -run 'TestIntegration/.*/api' -timeout 30m $(ARGS)
+	cd $(BLOCKBOOK_SRC) && go test -tags 'integration' `go list github.com/trezor/blockbook/tests/...` -run "$${E2E_REGEX:-TestIntegration/.*/api}" -timeout 30m $(ARGS)
 
 test-connectivity: prepare-sources
 	cd $(BLOCKBOOK_SRC) && go test -tags 'integration' github.com/trezor/blockbook/tests -run 'TestIntegration/.*/connectivity' -timeout 30m $(ARGS)

contrib/scripts/blockbook_status.sh

@@ -10,7 +10,7 @@ else
   host="localhost"
 fi
 
-var="BB_API_URL_HTTP_${coin}"
+var="BB_TEST_API_URL_HTTP_${coin}"
 base_url="${!var-}"
 [[ -n "$base_url" ]] || die "environment variable ${var} is not set"
 command -v curl >/dev/null 2>&1 || die "curl is not installed"

docs/testing.md

@@ -89,12 +89,12 @@ Example:
 HTTP connectivity verifies both back-end and Blockbook accessibility:
 
 * back-end: UTXO chains call `getblockchaininfo`, EVM chains call `web3_clientVersion`
-* Blockbook: calls `GET /api/status` (resolved from `BB_API_URL_HTTP_<coin alias>` or local `ports.blockbook_public`)
+* Blockbook: calls `GET /api/status` (resolved from `BB_TEST_API_URL_HTTP_<coin alias>` or local `ports.blockbook_public`)
 
 WebSocket connectivity also verifies both surfaces:
 
 * back-end: validates `web3_clientVersion` and opens a `newHeads` subscription
-* Blockbook: connects to `/websocket` (or `BB_API_URL_WS_<coin alias>`) and calls `getInfo`
+* Blockbook: connects to `/websocket` (or `BB_TEST_API_URL_WS_<coin alias>`) and calls `getInfo`
 
 ### Blockbook API end-to-end tests
 
@@ -109,7 +109,7 @@ Phase 1 covers smoke checks for:
 
 Endpoint resolution uses coin alias and this precedence:
 
-1. `BB_API_URL_HTTP_<coin alias>` and `BB_API_URL_WS_<coin alias>`
+1. `BB_TEST_API_URL_HTTP_<coin alias>` and `BB_TEST_API_URL_WS_<coin alias>`
 2. localhost fallback from coin config port `ports.blockbook_public`
 3. when WS env var is missing, WS URL is derived from HTTP URL with `/websocket` path
 

tests/api/endpoint_resolution.go

@@ -12,12 +12,10 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-
-	"github.com/trezor/blockbook/common"
 )
 
 // ResolveEndpoints resolves Blockbook API endpoints for a coin alias using
-// BB_API_URL_* overrides first and coin config fallbacks.
+// exact BB_TEST_API_URL_* overrides first and coin config fallbacks.
 func ResolveEndpoints(coin string) (string, string, error) {
 	ep, err := resolveAPIEndpoints(coin)
 	if err != nil {
@@ -38,7 +36,7 @@ func resolveAPIEndpoints(coin string) (*apiEndpoints, error) {
 	}
 
 	httpURL := ""
-	if v, ok := common.LookupEnvWithArchiveFallback("BB_API_URL_HTTP_", alias); ok {
+	if v, ok := os.LookupEnv("BB_TEST_API_URL_HTTP_" + alias); ok {
 		httpURL = strings.TrimSpace(v)
 	}
 	if httpURL == "" {
@@ -53,7 +51,7 @@ func resolveAPIEndpoints(coin string) (*apiEndpoints, error) {
 	}
 
 	wsURL := ""
-	if v, ok := common.LookupEnvWithArchiveFallback("BB_API_URL_WS_", alias); ok {
+	if v, ok := os.LookupEnv("BB_TEST_API_URL_WS_" + alias); ok {
 		wsURL = strings.TrimSpace(v)
 	}
 	if wsURL == "" {