test(provider): guard standard benchmark duration arithmetic #4131

Workflow file for this run

	name: CI

	on:
	push:
	branches: [master, main]
	pull_request:

	jobs:
	release-integrity:
	name: Release Integrity
	runs-on: blacksmith-4vcpu-ubuntu-2404
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	- name: Check source, fallback, and installer parity
	run: \|
	./scripts/check-release-version.sh
	python3 scripts/test-provider-release-resolution.py
	python3 scripts/test-provider-release-toolchain.py
	python3 scripts/test-provider-release-cache.py
	python3 scripts/test-prepare-metal-toolchain.py
	python3 scripts/test-provider-release-pipeline.py
	python3 scripts/test-provider-release-publication.py
	python3 scripts/test-provider-signing-validation.py
	python3 scripts/test-provider-test-watchdog.py
	python3 scripts/test-app-attest-entitlements.py
	./scripts/sync-install-embed.sh check
	./scripts/test-prod-env-refresh.sh
	- name: Test startup observer against local stubs
	run: python3 -m unittest discover -s scripts/startup_measurement -t scripts -p 'test_*.py'
	- name: Test native GPU CI routing without a GPU
	run: python3 scripts/test-native-gpu-ci.py

	docs:
	name: Docs Lint
	runs-on: blacksmith-4vcpu-ubuntu-2404
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	with:
	fetch-depth: 0 # Frozen source links are checked at their stamped commits.
	- name: Test historical source-link validation
	run: python3 scripts/test-docs-check-historical-links.py
	- name: Check freshness stamps, links, cited paths, orphans
	run: ./scripts/docs-check.sh

	test-coordinator:
	name: Coordinator Tests
	runs-on: blacksmith-4vcpu-ubuntu-2404
	services:
	postgres:
	image: postgres:16
	env:
	POSTGRES_USER: testbed
	POSTGRES_PASSWORD: testbed
	POSTGRES_DB: testbed
	ports:
	- 5432:5432
	options: >-
	--health-cmd "pg_isready -U testbed -d testbed"
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	- uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
	with:
	go-version-file: go.mod
	- name: Run tests
	env:
	DATABASE_URL: postgres://testbed:testbed@127.0.0.1:5432/testbed?sslmode=disable
	run: \|
	# Word splitting is the point here: one argument per package path,
	# and Go package paths contain no spaces.
	# shellcheck disable=SC2046
	go test -race $(go list ./... \| grep -v /e2e)
	- name: Check formatting
	run: \|
	set -o pipefail
	# Dated report evidence preserves the exact captured source bytes.
	# Check tracked Go source outside that immutable archive.
	UNFORMATTED=$(git ls-files -z -- '.go' ':!:docs/reports/evidence/*' \| xargs -0 gofmt -l --)
	if [ -n "$UNFORMATTED" ]; then
	echo "Unformatted files:"
	echo "$UNFORMATTED"
	exit 1
	fi

	lint-coordinator:
	name: Coordinator Lint
	runs-on: blacksmith-4vcpu-ubuntu-2404
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	- uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
	with:
	go-version-file: go.mod
	- name: golangci-lint
	run: \|
	go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.1.6
	golangci-lint run

	test-prompt-sidecar:
	name: Prompt Sidecar Tests
	runs-on: blacksmith-4vcpu-ubuntu-2404
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	- uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
	with:
	go-version-file: go.mod
	- name: Install pinned Rust toolchain
	run: \|
	rustup toolchain install 1.88.0 --profile minimal --component rustfmt --component clippy
	rustup override set 1.88.0
	- name: Check formatting, lint, and tests
	working-directory: coordinator/promptsidecar
	run: \|
	cargo fmt --all -- --check
	cargo check --locked --all-targets
	cargo clippy --locked --all-targets -- -D warnings
	cargo test --locked --all-targets
	- name: Verify static Linux container build
	run: docker build --platform=linux/amd64 --target prompt-sidecar-builder -t darkbloom-prompt-sidecar-ci -f coordinator/Dockerfile .
	- name: Prove production Linux sidecar under real contract load
	timeout-minutes: 15
	run: \|
	container=$(docker create darkbloom-prompt-sidecar-ci)
	trap 'docker rm -f "$container" >/dev/null 2>&1 \|\| true' EXIT
	docker cp \
	"$container:/app/coordinator/promptsidecar/target/x86_64-unknown-linux-musl/release/promptsidecar" \
	"$RUNNER_TEMP/promptsidecar-linux"
	chmod +x "$RUNNER_TEMP/promptsidecar-linux"
	file "$RUNNER_TEMP/promptsidecar-linux" \| grep -Eq "statically linked\|static-pie linked"
	./scripts/verify-prompt-sidecar-linux.sh "$RUNNER_TEMP/promptsidecar-linux"

	test-provider:
	name: Provider Tests
	runs-on: blacksmith-12vcpu-macos-latest
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	with:
	submodules: recursive
	- name: Test Qwen resources in a relocated app
	run: python3 scripts/test-qwen4-packaged-resources.py
	- uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
	with:
	go-version-file: go.mod
	- name: Restore SwiftPM cache
	uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
	with:
	path: \|
	~/Library/Caches/org.swift.swiftpm
	~/Library/org.swift.swiftpm
	provider-swift/.build
	libs/mlx-swift-lm/.build
	key: spm-v3-${{ runner.os }}-${{ github.sha }}-${{ hashFiles('provider-swift/Package.resolved', 'libs/mlx-swift/Package.swift', 'libs/mlx-swift-lm/Package.swift') }}
	restore-keys: \|
	spm-v3-${{ runner.os }}-
	- name: Discard cached runtime resources before rebuilding
	run: \|
	for root in provider-swift/.build libs/mlx-swift-lm/.build; do
	[ -d "$root" ] \|\| continue
	find "$root" -type f -name mlx.metallib -delete
	# The inactive package may also contain a divergent debug bundle
	# from an older cache. Each subsequent build recreates its resources.
	find "$root" -mindepth 3 -maxdepth 3 -type d \
	-path "$root/-apple-macosx//*.bundle" -prune -exec rm -rf {} +
	done
	- name: Install pinned Rust toolchain for production prompt parity
	run: \|
	rustup toolchain install 1.88.0 --profile minimal
	rustup override set 1.88.0
	- name: Build provider-swift (incl. tests)
	working-directory: provider-swift
	# Build the TEST bundle too, so the .xctest bundle exists before we
	# place the metallib next to its runner binary (see next step).
	run: \|
	swift build --build-tests
	# The root helper is a separately packaged executable product and is
	# not guaranteed to be linked merely because the test bundle builds.
	swift build --product darkbloom-fan-helper
	- name: Build and stage source-matched mlx.metallib
	run: \|
	set -euo pipefail
	command -v cmake >/dev/null 2>&1 \|\| brew install cmake
	METALLIB_CACHE_DIR="$RUNNER_TEMP/metallib-cache" \
	./scripts/fetch-metallib.sh "$RUNNER_TEMP"
	metallib="$RUNNER_TEMP/mlx.metallib"
	test -s "$metallib" \|\| { echo "::error::source-matched mlx.metallib was not produced"; exit 1; }

	# MLX loads the metallib from beside the running executable. Stage the
	# helper-authorized artifact into every provider binary and XCTest
	# location that the preceding build created.
	provider_bin=$(cd provider-swift && swift build -c debug --show-bin-path)
	for destination in provider-swift/.build/debug "$provider_bin"; do
	mkdir -p "$destination"
	cp "$metallib" "$destination/mlx.metallib"
	done
	while IFS= read -r macos; do
	cp "$metallib" "$macos/mlx.metallib"
	echo "placed source-matched metallib in $macos"
	done < <(
	find provider-swift/.build -type d -path '*.xctest/Contents/MacOS' -print
	)
	- name: Run Swift tests
	working-directory: provider-swift
	# Reuse the built bundle and matched metallib. The exact allocator
	# suite runs in its own process even if the general suite fails.
	run: ../scripts/run-provider-tests.sh
	- name: Verify production prompt parity
	run: ./scripts/verify-prompt-parity.sh
	# FLAKE-ORDERING GUARD (audit fix, v0.8.0): every nested step below runs
	# under `if: ${{ !cancelled() && ... }}`. Before this, the nested paged
	# suites ran with the default `success()` condition, so ANY earlier step
	# failure — including a known-flaky unit test in `Run Swift tests` —
	# SKIPPED all six paged correctness gates. On the v0.8.0 PR head that is
	# exactly what happened: the unit step failed on unrelated flakes and
	# the paged gates reported `skipped`, i.e. the release's central
	# correctness signal was silenced by an unrelated flake. `!cancelled()`
	# keeps the gates running after an earlier failure; each nested RUN step
	# still requires its own build/metallib prerequisites to have succeeded
	# so a red build produces one attributable failure, not six cascading
	# ones. A nested-step failure fails the job as usual (no
	# continue-on-error anywhere in this chain).
	- name: Build nested mlx-swift-lm tests
	id: build-nested-tests
	if: ${{ !cancelled() }}
	working-directory: libs/mlx-swift-lm
	# Cold builds compile the full Cmlx C++/Metal tree (~20 min on the
	# 12-vcpu runner); warm runs restore libs/mlx-swift-lm/.build from
	# the SwiftPM cache and finish in ~1 min.
	timeout-minutes: 35
	run: \|
	# This standalone package otherwise resolves its remote mlx-swift
	# branch while loading the root helper's local-source metallib.
	swift package unedit --force mlx-swift >/dev/null 2>&1 \|\| true
	swift package edit --path ../mlx-swift mlx-swift
	swift build --build-tests
	- name: Place metallib for nested test bundles
	id: place-nested-metallib
	if: ${{ !cancelled() && steps.build-nested-tests.outcome == 'success' }}
	working-directory: libs/mlx-swift-lm
	run: \|
	set -euo pipefail
	cp "$RUNNER_TEMP/mlx.metallib" .build/debug/mlx.metallib
	for bundle in .build/debug/*PackageTests.xctest; do
	macos="$bundle/Contents/MacOS"
	if [ -d "$macos" ]; then
	cp "$RUNNER_TEMP/mlx.metallib" "$macos/mlx.metallib"
	fi
	done
	# NOTE (v0.8.0): the guards below exist because these gates once
	# reported GREEN while executing NOTHING. `Tests/BenchCBv2Tests`
	# depended on the BenchCBv2 EXECUTABLE target, so SwiftPM routed the
	# swift-testing pass at that binary, which rejects
	# `--test-bundle-path` and exits — every `@Test` in the package was
	# unreachable and only XCTestCase classes ran. `swift test` exits 0 in
	# that state, which is why it went unnoticed.
	#
	# FIXED: BenchCBv2 is now split into a `BenchCBv2Core` library plus a
	# thin executable shim, and the test target depends on the library.
	#
	# The guards STAY as a permanent tripwire. This failure mode is silent
	# by construction — a dark suite and a passing one are
	# indistinguishable by exit code — so any future change that re-routes
	# the swift-testing pass must be RED here, not green.
	# The suites below are already compiled by the `swift build
	# --build-tests` step above but were never executed, so the paged-KV
	# migration had no automated numerical gate. They add no compile time,
	# only run time; one step each so a slow or failing suite is
	# attributable straight from the job summary. They reuse the metallib
	# placed by the step above.
	#
	# The bodies are ONE script (scripts/run-nested-suite.sh), which is what
	# carries the executed-count and zero-skip tripwires. ALL six suites go
	# through it — a bare `swift test --filter` here would bypass the
	# tripwire and reintroduce the dark-gate hole the script closes (the
	# safety and prompt-hash suites below used to run bare for exactly that
	# reason). Inline copies of a check whose failure mode is silence are
	# that many chances to edit one and miss the rest.
	- name: Run nested paged safety tests
	if: ${{ !cancelled() && steps.place-nested-metallib.outcome == 'success' }}
	working-directory: libs/mlx-swift-lm
	# Runtime smoke over the JIT-compiled paged Metal kernels.
	timeout-minutes: 10
	run: ../../scripts/run-nested-suite.sh CBv2PagedSafetyTests
	- name: Run nested prompt-hash tests
	if: ${{ !cancelled() && steps.place-nested-metallib.outcome == 'success' }}
	working-directory: libs/mlx-swift-lm
	# Prefix-cache prompt-hash identity; pure Swift.
	timeout-minutes: 10
	run: ../../scripts/run-nested-suite.sh CBv2PrefixCacheHasherTests
	- name: Run nested paged eligibility tests
	if: ${{ !cancelled() && steps.place-nested-metallib.outcome == 'success' }}
	working-directory: libs/mlx-swift-lm
	# Pure Swift threadgroup-budget arithmetic; no GPU dispatch.
	timeout-minutes: 5
	run: ../../scripts/run-nested-suite.sh CBv2PagedEligibilityTests
	- name: Run nested paged backend tests
	if: ${{ !cancelled() && steps.place-nested-metallib.outcome == 'success' }}
	working-directory: libs/mlx-swift-lm
	# Pool bookkeeping, per-sequence page tables, windowed rings,
	# reservation admission, backend lifecycle. Small MLX arrays only.
	timeout-minutes: 10
	run: ../../scripts/run-nested-suite.sh CBv2PagedBackendTests
	- name: Run nested paged kernel parity tests
	if: ${{ !cancelled() && steps.place-nested-metallib.outcome == 'success' }}
	working-directory: libs/mlx-swift-lm
	# Heaviest of the set: fp32 decode-kernel parity, bitwise
	# batch-composition invariance, and a 200-step greedy token match.
	# Dispatches the JIT-compiled Metal kernels, exactly as
	# CBv2PagedSafetyTests' runtimeSmoke already does in this job.
	timeout-minutes: 20
	# The composition assertion opts in only in its own test process;
	# both invocations retain the nonempty/no-skips tripwires.
	run: ../../scripts/run-paged-kernel-tests.sh
	- name: Run nested KV-sharing contiguous/paged parity tests
	if: ${{ !cancelled() && steps.place-nested-metallib.outcome == 'success' }}
	working-directory: libs/mlx-swift-lm
	# Two-arm differential over TinyTestModel's KV-sharing variant. The
	# paged arm used to XCTSkip when the backend was unavailable; the
	# runner script now FAILS on any skip, because "paged unavailable on
	# the gate runner" is precisely the condition this lane must not be
	# green under.
	timeout-minutes: 10
	run: ../../scripts/run-nested-suite.sh CBv2KVSharingParityTests
	- name: Run Nemotron production SDK gates
	if: ${{ !cancelled() && steps.place-nested-metallib.outcome == 'success' }}
	working-directory: libs/mlx-swift-lm
	timeout-minutes: 20
	run: \|
	# Each filter has its own nonzero/no-skip tripwire. Continue after a
	# failure so one broken suite cannot hide another unexecuted gate.
	onboarding_failed=0
	for suite in SSMDecodeBoundsTests NemotronHTests \
	NemotronH35BackendParityTests NemotronH35StorageParityTests \
	NemotronH35MTPTests NemotronH35MTPPrimingTests \
	MutableInputKernelTests MutableInputExportTests \
	CBv2QwenMTPIntegrationTests CBv2MTPDepthControllerTests \
	PagedMTPBatchedColumnsTests CBv2CompleteCheckpointEngineTests \
	NemotronH35ProductionDefaultsTests \
	OpenAIServiceTests ToolCallParserIntegrationTests; do
	if ! ../../scripts/run-nested-suite.sh "$suite"; then
	onboarding_failed=1
	fi
	done
	exit "$onboarding_failed"
	- name: Run atomic installer artifact tests
	timeout-minutes: 2
	run: ./scripts/test-install-atomic.sh

	cache-swift:
	name: Swift Build + Cache
	runs-on: blacksmith-12vcpu-macos-latest
	if: github.event_name == 'push'
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	with:
	submodules: recursive
	- name: Restore SwiftPM cache
	uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
	with:
	path: \|
	~/Library/Caches/org.swift.swiftpm
	~/Library/org.swift.swiftpm
	provider-swift/.build
	libs/mlx-swift-lm/.build
	key: spm-v3-${{ runner.os }}-${{ github.sha }}-${{ hashFiles('provider-swift/Package.resolved', 'libs/mlx-swift/Package.swift', 'libs/mlx-swift-lm/Package.swift') }}
	restore-keys: \|
	spm-v3-${{ runner.os }}-
	- name: Discard metallibs restored by the generic SwiftPM cache
	run: \|
	for root in provider-swift/.build libs/mlx-swift-lm/.build; do
	[ ! -d "$root" ] \|\| find "$root" -type f -name mlx.metallib -delete
	done
	- name: Build provider-swift
	working-directory: provider-swift
	run: \|
	swift build -c release --product darkbloom
	swift build -c release --product darkbloom-fan-helper
	- name: Build nested mlx-swift-lm tests (warms nested cache)
	working-directory: libs/mlx-swift-lm
	# The test-provider job runs the CBv2 paged suites (safety, kernel
	# parity, backend, eligibility, KV-sharing parity) from this nested
	# .build; saving it here keeps that lane warm across PRs.
	timeout-minutes: 35
	run: \|
	# Keep the warmed nested workspace on the same local MLX source used
	# by provider builds and scripts/fetch-metallib.sh.
	swift package unedit --force mlx-swift >/dev/null 2>&1 \|\| true
	swift package edit --path ../mlx-swift mlx-swift
	swift build --build-tests
	- name: Save SwiftPM cache
	uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
	with:
	path: \|
	~/Library/Caches/org.swift.swiftpm
	~/Library/org.swift.swiftpm
	provider-swift/.build
	libs/mlx-swift-lm/.build
	key: spm-v3-${{ runner.os }}-${{ github.sha }}-${{ hashFiles('provider-swift/Package.resolved', 'libs/mlx-swift/Package.swift', 'libs/mlx-swift-lm/Package.swift') }}

	lint-console:
	name: Console UI Lint & Build
	runs-on: blacksmith-4vcpu-ubuntu-2404
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
	with:
	node-version: '22'
	cache: 'npm'
	cache-dependency-path: console-ui/package-lock.json
	- name: Install dependencies
	run: cd console-ui && npm ci
	- name: Lint
	run: cd console-ui && npx eslint src/
	- name: Build
	run: cd console-ui && npm run build

Navigation Menu

test(provider): guard standard benchmark duration arithmetic #4131

Workflow file

test(provider): guard standard benchmark duration arithmetic #4131

Uh oh!

Workflow file for this run