v0.2.108 #56
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Gateway update workflow. | |
| # | |
| # Auto-triggers on `release.published` — once a release is published on | |
| # GitHub (after release.yml runs cargo publish + creates the release with | |
| # binaries attached), this workflow signs HMAC-authed `POST /update` | |
| # requests to each gateway's `freenet-release-agent`. | |
| # | |
| # `workflow_dispatch` is kept for manual re-rollouts and for the smoke | |
| # test (`dry_run_workflow: true`). | |
| name: Gateway Update | |
| on: | |
| release: | |
| types: [published] | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: 'Version to roll out (e.g. 0.2.56, no leading v)' | |
| required: true | |
| type: string | |
| gateways: | |
| description: 'Comma-separated subset (nova,vega) or "all"' | |
| required: false | |
| type: string | |
| default: 'all' | |
| dry_run_workflow: | |
| description: 'Skip the actual POST, just compute and log the payload' | |
| required: false | |
| type: boolean | |
| default: false | |
| allow_binary_only_fallback: | |
| description: >- | |
| Accept a gateway whose release-agent predates the service_active | |
| field (binary-version-only verification). OFF by default: without | |
| service_active we cannot tell a running gateway from one whose binary | |
| was swapped but whose service is dead (the 2026-06-18 nova incident, | |
| #4492), so a missing field is treated as not-yet-converged and the | |
| rollout fails at the deadline. Turn this on ONLY for a deliberate | |
| rollout to a gateway you know runs an old agent. | |
| required: false | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: read | |
| concurrency: | |
| # On release-trigger the version is in github.event.release.tag_name; | |
| # on workflow_dispatch it's in inputs.version. The `||` coalesces. | |
| group: gateway-update-${{ github.event.release.tag_name || inputs.version }} | |
| cancel-in-progress: false | |
| jobs: | |
| plan: | |
| name: Plan rollout | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.plan.outputs.matrix }} | |
| version: ${{ steps.ver.outputs.version }} | |
| steps: | |
| - name: Resolve version | |
| id: ver | |
| # Pass values via env: rather than ${{ }} interpolation — a tag | |
| # name like `1.2.3"; curl evil; "` would otherwise template into | |
| # the script at expression-eval time, before the regex below | |
| # could reject it. Env binding makes it a regular shell string. | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| TAG_NAME: ${{ github.event.release.tag_name }} | |
| DISPATCH_VERSION: ${{ inputs.version }} | |
| run: | | |
| if [[ "$EVENT_NAME" == "release" ]]; then | |
| RAW="$TAG_NAME" | |
| else | |
| RAW="$DISPATCH_VERSION" | |
| fi | |
| VERSION="${RAW#v}" | |
| if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | |
| echo "::error::Version must be X.Y.Z. Got: $VERSION (raw: $RAW)" | |
| exit 1 | |
| fi | |
| echo "version=$VERSION" >> "$GITHUB_OUTPUT" | |
| echo "Resolved version: $VERSION" | |
| - name: Verify GitHub release exists and is latest | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| VERSION="${{ steps.ver.outputs.version }}" | |
| LATEST=$(gh api repos/${{ github.repository }}/releases/latest --jq .tag_name) | |
| LATEST="${LATEST#v}" | |
| if [[ "$LATEST" != "$VERSION" ]]; then | |
| echo "::error::Version $VERSION is not the GitHub 'latest' release ($LATEST). Refusing to roll out." | |
| echo "::error::The agent will reject this anyway (403 not_github_latest); failing fast here." | |
| echo "::error::Note: when triggered by release.published, the freshly-created release is normally already 'latest'." | |
| exit 1 | |
| fi | |
| echo "GitHub /releases/latest confirms $VERSION is current." | |
| - name: Build matrix | |
| id: plan | |
| run: | | |
| # Default to all gateways when triggered by release.published | |
| # (no inputs available); honor explicit subset on workflow_dispatch. | |
| REQUESTED="${{ inputs.gateways || 'all' }}" | |
| if [[ "$REQUESTED" == "all" ]]; then | |
| MATRIX='[{"name":"nova","url":"https://nova.locut.us/release-agent"},{"name":"vega","url":"https://vega.locut.us:8443/release-agent"}]' | |
| else | |
| # Build the array by filtering the full list. Cheap, transparent, | |
| # easy to read in workflow logs. | |
| ITEMS="[]" | |
| for tag in $(echo "$REQUESTED" | tr ',' ' '); do | |
| case "$tag" in | |
| nova) ITEMS=$(echo "$ITEMS" | jq -c '. + [{"name":"nova","url":"https://nova.locut.us/release-agent"}]') ;; | |
| vega) ITEMS=$(echo "$ITEMS" | jq -c '. + [{"name":"vega","url":"https://vega.locut.us:8443/release-agent"}]') ;; | |
| *) echo "::error::Unknown gateway tag: $tag"; exit 1 ;; | |
| esac | |
| done | |
| MATRIX="$ITEMS" | |
| fi | |
| echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" | |
| echo "Rollout plan: $MATRIX" | |
| rollout: | |
| name: Update ${{ matrix.gateway.name }} | |
| needs: plan | |
| runs-on: ubuntu-latest | |
| # Serialize so a failure on the first gateway aborts the rest. | |
| # Phase 3 may parallelize this — for now, blast-radius control wins. | |
| # On partial failure (e.g. nova upgraded, vega 5xx) the fleet is left | |
| # split-version; agent refuses downgrades, so recovery requires either | |
| # re-running with the failing gateway only or a manual emergency path. | |
| strategy: | |
| fail-fast: true | |
| max-parallel: 1 | |
| matrix: | |
| gateway: ${{ fromJson(needs.plan.outputs.matrix) }} | |
| steps: | |
| # Needed for the verify step, which sources | |
| # scripts/release-agent/verify-version-decision.sh so the workflow and | |
| # its regression test share one copy of the decision logic. | |
| - uses: actions/checkout@v7 | |
| - name: Resolve secret | |
| id: secret | |
| env: | |
| NOVA_SECRET: ${{ secrets.RELEASE_AGENT_HMAC_NOVA }} | |
| VEGA_SECRET: ${{ secrets.RELEASE_AGENT_HMAC_VEGA }} | |
| run: | | |
| case "${{ matrix.gateway.name }}" in | |
| nova) SECRET="$NOVA_SECRET" ;; | |
| vega) SECRET="$VEGA_SECRET" ;; | |
| *) echo "::error::No secret mapping for ${{ matrix.gateway.name }}"; exit 1 ;; | |
| esac | |
| if [[ -z "$SECRET" ]]; then | |
| echo "::error::Secret RELEASE_AGENT_HMAC_${{ matrix.gateway.name }} is not set" | |
| exit 1 | |
| fi | |
| # Mask immediately so any accidental subsequent echo redacts. | |
| echo "::add-mask::$SECRET" | |
| # Pass via env file, not stdout, so it never appears in logs. | |
| echo "secret=$SECRET" >> "$GITHUB_OUTPUT" | |
| - name: Sign and POST /update | |
| env: | |
| SECRET: ${{ steps.secret.outputs.secret }} | |
| GATEWAY_URL: ${{ matrix.gateway.url }} | |
| VERSION: ${{ needs.plan.outputs.version }} | |
| DRY_RUN_WORKFLOW: ${{ inputs.dry_run_workflow }} | |
| run: | | |
| set -euo pipefail | |
| ISSUED_AT=$(date +%s) | |
| BODY=$(jq -nc \ | |
| --arg v "$VERSION" \ | |
| --argjson t "$ISSUED_AT" \ | |
| '{version: $v, issued_at: $t}') | |
| SIG=$(printf '%s' "$BODY" \ | |
| | openssl dgst -sha256 -mac HMAC -macopt "hexkey:$SECRET" \ | |
| | awk '{print $NF}') | |
| echo "Target: $GATEWAY_URL" | |
| echo "Body: $BODY" | |
| echo "Sig: ${SIG:0:8}…(truncated)" | |
| if [[ "$DRY_RUN_WORKFLOW" == "true" ]]; then | |
| echo "::notice::dry_run_workflow=true; skipping actual POST" | |
| exit 0 | |
| fi | |
| # 30s connect timeout, 30s total. The agent returns within | |
| # ~1s on the happy path (it spawns sudo and returns 202 | |
| # before the script finishes). | |
| HTTP_STATUS=$(curl -sS -o /tmp/agent-response.json -w '%{http_code}' \ | |
| --connect-timeout 30 --max-time 30 \ | |
| -X POST "$GATEWAY_URL/update" \ | |
| -H 'Content-Type: application/json' \ | |
| -H "X-Signature: $SIG" \ | |
| --data-raw "$BODY") | |
| echo "HTTP $HTTP_STATUS" | |
| cat /tmp/agent-response.json | |
| echo | |
| case "$HTTP_STATUS" in | |
| 200|202) ;; | |
| 409) | |
| # 409 Conflict: an update is already in flight on this gateway | |
| # (an overlapping/legit re-run — e.g. a manual re-trigger landing | |
| # while the release.published rollout is still restarting the | |
| # service). The agent's in-flight guard rejected the duplicate to | |
| # prevent the #4271 double-stop. This is NOT a failure: the | |
| # in-progress update is the one that matters, so fall through to | |
| # the verify step below, which polls /version until the gateway | |
| # converges on the target version with an active service. | |
| echo "::notice::Gateway ${{ matrix.gateway.name }} reports an update already in progress (HTTP 409); skipping duplicate spawn and proceeding to verify." | |
| ;; | |
| 429) | |
| # 429 Too Many Requests: the rate-limit window hasn't elapsed | |
| # since the last accepted update. Like 409, this means an update | |
| # was recently kicked off, so the target is (or will be) applied. | |
| # Proceed to verify rather than failing the release. | |
| echo "::notice::Gateway ${{ matrix.gateway.name }} rate-limited this update (HTTP 429); a recent update is in effect. Proceeding to verify." | |
| ;; | |
| *) | |
| echo "::error::Gateway ${{ matrix.gateway.name }} rejected update: HTTP $HTTP_STATUS" | |
| exit 1 | |
| ;; | |
| esac | |
| - name: Verify version after update | |
| if: ${{ inputs.dry_run_workflow != true }} | |
| env: | |
| GATEWAY_URL: ${{ matrix.gateway.url }} | |
| VERSION: ${{ needs.plan.outputs.version }} | |
| # Empty on the release-trigger path (no inputs) → defaults to the | |
| # safe "false". See the allow_binary_only_fallback input docs. | |
| ALLOW_BINARY_ONLY_FALLBACK: ${{ inputs.allow_binary_only_fallback }} | |
| run: | | |
| set -euo pipefail | |
| # The agent restarts the freenet service after the script | |
| # finishes; /version will briefly 502/connection-refused while | |
| # the systemd unit cycles. Poll for up to 120s. | |
| # | |
| # Success requires BOTH the on-disk binary on the new version AND | |
| # the gateway SERVICE actually active on it. The binary-version check | |
| # alone is NOT sufficient: during the v0.2.71 release vega's binary | |
| # was swapped to 0.2.71 but the service failed to restart (old | |
| # process hung on shutdown → SIGKILL → unit left `failed`). | |
| # `/version` still reported 0.2.71 (on-disk), so this step reported | |
| # SUCCESS while vega's gateway was DOWN for ~5 minutes. The | |
| # `service_active` field closes that gap. | |
| # | |
| # The decision logic lives in verify-version-decision.sh so it is | |
| # unit-tested (verify-version-decision_test.sh, run in CI) and shared | |
| # with this workflow — see that script for the backward-compat and | |
| # swap-ordering rationale. | |
| source scripts/release-agent/verify-version-decision.sh | |
| DEADLINE=$(( $(date +%s) + 120 )) | |
| # Remember if we ever saw the old-agent (no service_active) response, | |
| # so the deadline error can point at the real cause. | |
| SAW_FALLBACK=false | |
| # #4567: require the gateway to report `success` on this many | |
| # CONSECUTIVE polls before declaring the update converged. Confirming | |
| # health at a single instant let a gateway that came up active then | |
| # died within the window (flap-after-up) still report green. The | |
| # release-agent's 2s service_active TTL cache means a crash-loop flaps | |
| # the field across the 5s polls, resetting the streak. Cost on the | |
| # happy path: (REQUIRED_SUCCESS_POLLS-1)*5s ≈ 10s. | |
| REQUIRED_SUCCESS_POLLS=3 | |
| SUCCESS_STREAK=0 | |
| while (( $(date +%s) < DEADLINE )); do | |
| RESPONSE=$(curl -sS --max-time 5 "$GATEWAY_URL/version" || true) | |
| INSTALLED=$(printf '%s' "$RESPONSE" | jq -r '.version // empty' 2>/dev/null || echo "") | |
| MANAGED_SERVICE=$(printf '%s' "$RESPONSE" | jq -r '.managed_service // empty' 2>/dev/null || echo "") | |
| DECISION=$(verify_version_decision "$RESPONSE" "$VERSION") | |
| case "$DECISION" in | |
| success) | |
| SUCCESS_STREAK=$(stability_streak "$DECISION" "$SUCCESS_STREAK") | |
| if [[ "$(stability_confirmed "$SUCCESS_STREAK" "$REQUIRED_SUCCESS_POLLS")" == "confirm" ]]; then | |
| echo "✓ Gateway ${{ matrix.gateway.name }} reports version $INSTALLED and service ${MANAGED_SERVICE:-?} is active (stable across $SUCCESS_STREAK consecutive polls)" | |
| exit 0 | |
| fi | |
| echo "Gateway ${{ matrix.gateway.name }} reports version $INSTALLED and service ${MANAGED_SERVICE:-?} active ($SUCCESS_STREAK/$REQUIRED_SUCCESS_POLLS consecutive healthy polls); confirming stability…" | |
| ;; | |
| success-fallback) | |
| # The agent predates the service_active field, so we cannot | |
| # confirm the SERVICE is up — only that the binary on disk is | |
| # the new version. On 2026-06-18 (#4492) nova hit exactly this: | |
| # binary swapped to the target but the gateway service was | |
| # dead, and this branch reported the rollout green. So by | |
| # default we DO NOT accept binary-only as success: treat it | |
| # like `wait` (keep polling, fail at the deadline) and tell the | |
| # operator to update the agent. Only the explicit | |
| # allow_binary_only_fallback opt-in restores the old behaviour. | |
| # The accept/hold policy lives in fallback_decision() (sourced | |
| # above) so it is unit-tested rather than inline-untested. | |
| if [[ "$(fallback_decision "$DECISION" "${ALLOW_BINARY_ONLY_FALLBACK:-false}")" == "accept" ]]; then | |
| echo "::warning::Gateway ${{ matrix.gateway.name }} agent predates the service-health check (no 'service_active' in /version); allow_binary_only_fallback=true, so accepting binary-only verification. Update the release-agent on this gateway to enable the stronger check." | |
| echo "✓ Gateway ${{ matrix.gateway.name }} now reports version $INSTALLED (binary-only check)" | |
| exit 0 | |
| fi | |
| # Not confirmed as active this poll → break any success streak. | |
| SUCCESS_STREAK=0 | |
| SAW_FALLBACK=true | |
| echo "::warning::Gateway ${{ matrix.gateway.name }} agent predates the service-health check (no 'service_active' in /version); cannot confirm the service is actually running. Update the release-agent (>= the #4378 build) to enable the service-health gate, or re-run with allow_binary_only_fallback=true to accept binary-only verification." | |
| echo "Binary is $INSTALLED but service health is UNVERIFIABLE on this agent; treating as not-yet-converged…" | |
| ;; | |
| wait) | |
| # A non-success poll (service not active, unreachable, or | |
| # malformed) breaks the consecutive-success streak (#4567): a | |
| # gateway that flaps active→down cannot accumulate the run. | |
| SUCCESS_STREAK=0 | |
| if [[ "$INSTALLED" == "$VERSION" ]]; then | |
| echo "Binary is $INSTALLED but service ${MANAGED_SERVICE:-?} is NOT active yet; waiting for restart…" | |
| else | |
| echo "Waiting for ${{ matrix.gateway.name }} to report $VERSION (current: ${INSTALLED:-unreachable})…" | |
| fi | |
| ;; | |
| *) | |
| echo "::error::Unexpected decision token: $DECISION" | |
| exit 1 | |
| ;; | |
| esac | |
| sleep 5 | |
| done | |
| if [[ "$SAW_FALLBACK" == "true" ]]; then | |
| echo "::error::Gateway ${{ matrix.gateway.name }} reported version $VERSION but its release-agent predates the service_active field, so service health could not be confirmed (binary-only). Failing closed (#4492). Update the release-agent on ${{ matrix.gateway.name }}, or re-run with allow_binary_only_fallback=true if you accept binary-only verification." | |
| exit 1 | |
| fi | |
| echo "::error::Gateway ${{ matrix.gateway.name }} did not converge on $VERSION with an active service within 120s" | |
| exit 1 |