ci.yml

name: CI

# Cost optimization strategy (Ubicloud runners ~$0.13/run):
# 1. Skip CI entirely for docs-only changes (paths-ignore)
# 2. Skip expensive jobs on main push — PR already validated
# 3. Skip NAT validation for dependabot PRs — Test is sufficient for version bumps
# 4. Run fast checks (Fmt, Clippy) in parallel for quick fail-fast
# 5. Run required checks in merge_group to satisfy branch protection
# 6. Split unit/integration and simulation tests for parallelism
# 7. Tiered merge_group depth (#3973):
#    - PR runs the full suite — catches regressions in the change being made.
#    - Non-release merge_group runs only Unit & Integration on the heavy runner;
#      Simulation and NAT Validation are present-but-skipped to keep branch
#      protection happy without contending for ubicloud-standard-16 slots.
#    - Release merge_group (commit message starts with "build: release") runs
#      the full suite as the pre-publish gate, so what ships is known-green.

on:
  push:
    branches: [main]
    # Skip CI for docs-only changes on main (saves ~$0.13/run)
    paths-ignore:
      - 'docs/**'
      - '*.md'
      - 'LICENSE'
      - '.github/ISSUE_TEMPLATE/**'
      - '.github/FUNDING.yml'
  pull_request:
    paths-ignore:
      - 'docs/**'
      - '*.md'
      - 'LICENSE'
      - '.github/ISSUE_TEMPLATE/**'
      - '.github/FUNDING.yml'
  merge_group:

# Cancel in-progress runs when a new commit is pushed to the same branch.
# On main, never cancel — each merge must complete its CI run. Otherwise a
# fast succession of merges silently cancels earlier builds (#3311).
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  # Fast checks first - fail fast on simple issues
  # Order: fastest to slowest for single-runner efficiency

  # Fast checks on GitHub-hosted runners (don't block nova)
  conventional_commits:
    name: Conventional Commits
    runs-on: ubuntu-latest
    timeout-minutes: 5
    if: github.event_name == 'pull_request'

    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - name: Check PR title follows Conventional Commits
        uses: amannn/action-semantic-pull-request@v6
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          types: |
            feat
            fix
            docs
            style
            refactor
            perf
            test
            build
            ci
            chore
            revert
          requireScope: false
          subjectPattern: ^(?![A-Z]).+$
          subjectPatternError: |
            The subject "{subject}" found in the pull request title "{title}"
            didn't match the configured pattern. Please ensure that the subject
            doesn't start with an uppercase character.

  fmt_check:
    name: Fmt
    runs-on: ubuntu-latest
    timeout-minutes: 5
    # Run in parallel with conventional_commits (both are fast GitHub-hosted checks)

    steps:
      - uses: actions/checkout@v7

      - uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: 1.93.0
          components: rustfmt

      - name: Check code formatting
        run: cargo fmt -- --check

      # Locks the bash predicate that the merge_group release-gate keys off
      # (#3973). The actual release path is only exercised on a real release
      # merge_group entry, so a regression in the matching logic would
      # silently skip the pre-publish gate. This self-test catches predicate
      # quoting, anchoring, and case-sensitivity bugs on every CI run.
      - name: Self-test merge_group release detector
        run: |
          set -eu
          check() {
            local msg="$1"
            local expected="$2"  # "release" or "non-release"
            local actual
            if [[ "$msg" == build:\ release* ]]; then
              actual="release"
            else
              actual="non-release"
            fi
            if [[ "$actual" != "$expected" ]]; then
              echo "FAIL: '$msg' -> expected '$expected', got '$actual'"
              return 1
            fi
            echo "OK:   '$msg' -> $actual"
          }
          # Positive cases (release-gate must fire)
          check "build: release 0.2.49 (#3947)"                "release"
          check "build: release 1.0.0"                         "release"
          check "build: release 0.2.49"                        "release"
          # Negative cases (release-gate must NOT fire)
          check "chore: bump versions to 0.2.49"               "non-release"
          check "refactor(ops): retire SubOperationTracker"    "non-release"
          check "build(deps): bump rand from 0.8.5 to 0.9.2"   "non-release"
          check "build: replace local release build"           "non-release"
          check ""                                             "non-release"

      # Regression gate for the gateway-update verify decision (the vega
      # v0.2.71 incident: binary swapped, service down, workflow falsely
      # reported success). The decision logic in
      # scripts/release-agent/verify-version-decision.sh is sourced both by
      # gateway-update.yml and by this test, so they cannot drift. The
      # load-bearing case is "service_active:false must NOT report success".
      - name: Self-test gateway-update verify decision
        run: bash scripts/release-agent/verify-version-decision_test.sh

      # Regression gate for the independent post-deploy service-health check in
      # gateway-auto-update.sh (the 2026-06-18 v0.2.78 nova incident, #4492: a
      # stale deploy-local-gateway.sh exited 0 on a DEAD service and the wrapper
      # reported the update successful). The load-bearing case: a deploy script
      # that lies (exits 0 while the service is down) must still fail the update
      # via the independent `systemctl is-active` gate.
      - name: Self-test gateway-auto-update service-health gate
        run: bash scripts/release-agent/gateway-auto-update_test.sh

      # Regression gate for deploy-local-gateway.sh::verify_service — the
      # function that actually regressed on nova (#4492). Pins that a dead
      # service, a missing unit, and an active-but-no-process unit each cause a
      # non-zero exit, and that a healthy deploy exits 0.
      - name: Self-test deploy-local-gateway verify_service
        run: bash scripts/release-agent/deploy-local-gateway_test.sh

      # Regression gate for the release-announce River post (the
      # v0.2.74/v0.2.75 incident: the announce raced the gateway self-update,
      # a WS-port probe passed, but riverctl's room GET hit a mid-teardown
      # node and failed "room not found" while the workflow showed green).
      # The readiness poll (wait_for_room), owner-signing, and logging
      # functions are extracted verbatim from announce-to-river.sh and
      # exercised here, so the script and its test cannot drift.
      - name: Self-test announce-to-river
        run: bash scripts/release-agent/announce-to-river_test.sh

      # install.sh now sets up a supervised service by default (issue #4073) so
      # new nodes auto-update. The system-vs-user + lingering decision is the
      # load-bearing new logic; these smoke tests pin it without needing real
      # root/sudo. shellcheck guards the installer/uninstaller scripts too.
      - name: Install shellcheck
        run: sudo apt-get update && sudo apt-get install -y shellcheck
      - name: Lint install/uninstall scripts (shellcheck)
        run: shellcheck scripts/install.sh scripts/uninstall.sh scripts/test-install-sh.sh scripts/test-uninstall-sh.sh
      - name: Self-test install.sh service-mode decision
        run: sh scripts/test-install-sh.sh
      - name: Self-test uninstall.sh
        run: sh scripts/test-uninstall-sh.sh

      - name: Check for old-style mod.rs files
        run: |
          # New modules must use foo.rs + foo/ style, not foo/mod.rs
          # Exceptions: tests/common/mod.rs (Rust convention), src/bin/*/mod.rs (Cargo binary),
          #             benches/*/mod.rs (Cargo bench binary)
          bad_files=$(find crates apps -name "mod.rs" -not -path "*/target/*" \
            -not -path "*/tests/common/mod.rs" \
            -not -path "*/src/bin/*/mod.rs" \
            -not -path "*/benches/*/mod.rs" \
            2>/dev/null || true)
          if [ -n "$bad_files" ]; then
            echo "::error::Found old-style mod.rs files. Use foo.rs + foo/ style instead:"
            echo "$bad_files"
            exit 1
          fi

      # Regression gate for #4240: cargo publish only bundles files
      # inside the crate, so an include_str!/include_bytes! path that
      # walks outside the crate root ships a broken tarball that fails
      # to build for crates.io users.
      #
      # When adding a new embedded resource (include_str!/include_bytes!),
      # add its package-list path to `required` below. A general
      # auto-discovery variant (resolve every embed site in
      # crates/core/src/ and assert each appears in the package list)
      # is a sensible follow-up but is out of scope for this fix.
      - name: Verify freenet crate package includes embedded resources
        run: |
          set -eu
          required=(
            scripts/macos-bundle-updater.sh
          )
          package_list=$(cargo package --list -p freenet)
          fail=0
          for f in "${required[@]}"; do
            if ! echo "$package_list" | grep -qFx "$f"; then
              echo "::error::Missing from published freenet crate: $f"
              fail=1
            fi
          done
          if [ "$fail" -ne 0 ]; then
            echo "Files referenced by include_str!/include_bytes! must live"
            echo "inside crates/core/ so cargo publish bundles them."
            echo
            echo "Package contents:"
            echo "$package_list"
            exit 1
          fi

  # Grep-based lint for banned patterns in crates/core/ (DST-breaking APIs).
  # Only checks added lines in the diff — pre-existing violations are grandfathered.
  # Fast (~2s on GitHub-hosted runner), no Rust toolchain needed.
  rule_lint:
    name: Rule Lint
    runs-on: ubuntu-latest
    timeout-minutes: 5
    if: github.event_name == 'pull_request'

    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0

      # Self-test the rule #6 blocking-send linter (good/bad fixtures) so a
      # regression in the matcher fails CI before it can let a real offender
      # through or wrongly flag clean code.
      - name: Rule #6 linter self-test
        run: python3 .github/scripts/check_blocking_sends.py --self-test

      # Self-test the rules #1/#2/#3 linter (cfg-test-aware banned-pattern
      # checker). Exercises prod-code violations (must flag) and test-scoped
      # uses (must NOT flag, including the #4616 moved-test-module case).
      - name: Rules #1/#2/#3 linter self-test
        run: python3 .github/scripts/check_banned_patterns.py --self-test

      - name: Check for banned patterns in new code
        env:
          PR_TITLE: ${{ github.event.pull_request.title }}
        run: |
          BASE="${{ github.event.pull_request.base.sha }}"
          HEAD="${{ github.event.pull_request.head.sha }}"

          ERRORS=""

          # 1/2/3. std::time::Instant::now(), rand::thread_rng()/rand::random(),
          # tokio::net::UdpSocket — must use TimeSource / GlobalRng / Socket trait.
          # Implemented as a Python linter (check_banned_patterns.py) so it can
          # resolve each flagged line to its position in the HEAD file and check
          # whether the line is inside a #[cfg(test)]-gated scope. Test-scoped uses
          # are EXEMPT (tests legitimately use real clocks/RNG/sockets). Production
          # enforcement is UNCHANGED — a non-test line that adds any of the banned
          # patterns is still a hard failure. The linter has its own --self-test
          # (run as the "Rules #1/#2/#3 linter self-test" step above).
          # See .github/scripts/check_banned_patterns.py and
          # .claude/rules/testing.md for details.
          if ! BANNED_OUT=$(python3 .github/scripts/check_banned_patterns.py "$BASE" "$HEAD"); then
            ERRORS="$ERRORS
          $BANNED_OUT"
          fi

          # 6. Blocking .send(...).await on an event-loop-reachable bounded
          # channel sender (the #4145/#4231/#4466 incident class). Implemented
          # as a Python linter so it can span multi-line statements, skip
          # string-literal / comment contents, catch the P2pBridge::send wrapper
          # form, and honour a `// channel-safety: ok` annotation on the call
          # line OR the line above. The linter has its own --self-test (run as
          # the "Rule #6 linter self-test" step above). Diff-scoped to ADDED
          # lines only. See .claude/rules/channel-safety.md and
          # .github/scripts/check_blocking_sends.py.
          if ! BLOCKING_OUT=$(python3 .github/scripts/check_blocking_sends.py "$BASE" "$HEAD"); then
            ERRORS="$ERRORS
          $BLOCKING_OUT"
          fi

          # 4. Deleted test functions (removed #[test], #[tokio::test],
          # #[test_log::test], or #[freenet_test(...)] lines)
          # Check ALL Rust files for test additions/removals (not just crates/core/src/)
          # TEST_ATTR_RE matches the set of attributes this repo uses to mark
          # a function as a test:
          #   #[test]                                          - plain libtest
          #   #[tokio::test]                                   - async tests
          #   #[tokio::test(flavor = ..., worker_threads = N)] - parameterized async tests
          #   #[test_log::test]                                - tracing-subscriber-enabled tests
          #   #[test_log::test(tokio::test)]                   - parameterized tracing-subscriber-enabled
          #   #[freenet_test(...)]                             - Freenet integration-test macro
          # The character class [(\]] after `test` matches either `]` (plain
          # form) or `(` (parameterized form). Without that, parameterized
          # tokio::test attributes weren't counted as new tests and fix: PRs
          # using only such tests were incorrectly rejected.
          TEST_ATTR_RE='#\[((tokio|test_log)::)?test(\]|\()|#\[freenet_test'
          ALL_ADDED=$(git diff "$BASE"..."$HEAD" -- '**/*.rs' \
            | grep '^+' | grep -v '^+++' || true)
          ALL_ADDED_STRIPPED=$(echo "$ALL_ADDED" | sed 's|//.*||' || true)

          REMOVED=$(git diff "$BASE"..."$HEAD" -- 'crates/core/src/**/*.rs' 'crates/core/tests/**/*.rs' \
            | grep '^-' | grep -v '^---' || true)
          REMOVED_STRIPPED=$(echo "$REMOVED" | sed 's|//.*||' || true)
          # The test-exempt label allows mechanical refactors (e.g., retiring a
          # data structure whose unit tests can't be #[ignore]d because the
          # type itself is gone) to bypass both rule #4 (deleted tests) and
          # rule #5 (missing fix-PR test). Justification must be in PR comments.
          HAS_EXEMPT_LABEL=$(echo "${{ join(github.event.pull_request.labels.*.name, ',') }}" | grep -c 'test-exempt' || true)
          if [ "$HAS_EXEMPT_LABEL" -eq 0 ] && echo "$REMOVED_STRIPPED" | grep -qE "$TEST_ATTR_RE"; then
            # Check it wasn't just moved (also present in added lines across all files)
            REMOVED_TESTS=$(echo "$REMOVED_STRIPPED" | grep -cE "$TEST_ATTR_RE" || true)
            ADDED_TESTS=$(echo "$ALL_ADDED_STRIPPED" | grep -cE "$TEST_ATTR_RE" || true)
            if [ "$REMOVED_TESTS" -gt "$ADDED_TESTS" ]; then
              ERRORS="$ERRORS
          ERROR: Test function(s) removed — tests must not be deleted (use #[ignore] with a tracking issue if broken)
          Removed $REMOVED_TESTS test(s), added $ADDED_TESTS test(s)
          If the deletion is intentional (e.g., retiring a data structure whose
          tests can't be #[ignore]d because the type is gone), add the
          'test-exempt' label with a justification comment."
            fi
          fi

          # 5. fix: PRs must add at least one new test (check all Rust files, not just src/)
          # PR_TITLE is set via env: above to avoid shell injection from backticks in titles
          IS_FIX=$(echo "$PR_TITLE" | grep -ciE '^fix(\(.*\))?:' || true)

          if [ "$IS_FIX" -gt 0 ] && [ "$HAS_EXEMPT_LABEL" -eq 0 ]; then
            NEW_TESTS=$(echo "$ALL_ADDED_STRIPPED" | grep -cE "$TEST_ATTR_RE" || true)
            # Some fixes are to shell tooling (e.g. scripts/release-agent/*.sh),
            # whose regression tests are shell *_test.sh self-tests run in CI,
            # not Rust #[test]s. Count added assertion lines (`check ...`) in
            # added *_test.sh files so such fixes aren't forced to claim
            # test-exempt. Without this, a genuine shell regression test scores
            # zero against TEST_ATTR_RE (Rust-only) and the fix is wrongly
            # rejected — the gap that hid the announce-to-river fix's own test.
            NEW_SHELL_TESTS=$(git diff "$BASE"..."$HEAD" -- '**/*_test.sh' \
              | grep -E '^\+' | grep -vF '+++' \
              | grep -cE '^\+[[:space:]]*check[[:space:]]' || true)
            if [ "$NEW_TESTS" -eq 0 ] && [ "$NEW_SHELL_TESTS" -eq 0 ]; then
              ERRORS="$ERRORS
          ERROR: fix: PR must include at least one new regression test
          Add a test that reproduces the bug (fails without fix, passes with fix).
          A Rust #[test]/#[tokio::test]/etc. or an added 'check' assertion in a
          *_test.sh self-test both satisfy this.
          If this fix genuinely cannot have a test, add the 'test-exempt' label with a justification comment."
            fi
          fi

          if [ -n "$ERRORS" ]; then
            echo "::error::Rule lint failed — banned patterns found in new code"
            echo "$ERRORS"
            echo ""
            echo "These patterns break deterministic simulation testing (DST)."
            echo "See .claude/rules/testing.md and .claude/rules/code-style.md for alternatives."
            exit 1
          fi

          echo "Rule lint passed — no banned patterns in new code"

  # Clippy runs in parallel with fmt_check for faster fail-fast
  # Using Ubicloud for faster builds with more cores
  clippy_check:
    name: Clippy
    runs-on: ubicloud-standard-8
    timeout-minutes: 15
    # No dependencies - runs immediately in parallel with fmt_check

    env:
      RUST_LOG: error
      RUST_MIN_STACK: 268435456

    steps:
      - uses: actions/checkout@v7

      - name: Install mold linker
        run: |
          sudo apt-get update
          sudo apt-get install -y mold

      - uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: 1.93.0
          components: clippy
          targets: wasm32-unknown-unknown

      - uses: Swatinem/rust-cache@v2

      - name: clippy
        env:
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: clang
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -C link-arg=-fuse-ld=mold
        run: cargo clippy --locked -- -D warnings

      # Guards the opt-in `trace-ot` OpenTelemetry feature against silent
      # rot: it is excluded from the default build, so a refactor can break
      # its #[cfg(feature = "trace-ot")] code paths without any other CI job
      # noticing. See #4225.
      - name: clippy (trace-ot feature)
        env:
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: clang
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -C link-arg=-fuse-ld=mold
        run: cargo clippy --locked -p freenet --features trace-ot -- -D warnings

  # Windows compile check - catches missing imports, feature flags, and
  # platform-specific errors that Linux CI misses. Added after #3685 where
  # PRs #3669/#3680 introduced Windows-only code that failed to compile,
  # but wasn't caught until the release cross-compile workflow ran.
  windows_check:
    name: Windows Check
    runs-on: windows-latest
    timeout-minutes: 20
    needs: fmt_check
    if: github.event_name == 'pull_request' || github.event_name == 'merge_group'

    steps:
      - uses: actions/checkout@v7

      - uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: 1.93.0

      - uses: Swatinem/rust-cache@v2

      - name: Check compilation
        run: cargo check --locked -p freenet -p fdev

  test_unit:
    name: Unit & Integration
    # Always runs on the heavy runner. Unlike test_simulation and nat_validation,
    # this job has no runner-conditional and no skip_check: Unit & Integration is
    # the one heavy gate kept on every merge_group entry (release or not), so the
    # tiered model (#3973) doesn't apply here.
    runs-on: ubicloud-standard-16
    timeout-minutes: 30
    needs: fmt_check
    # Skip on push to main (PR already validated) and on release branches
    # themselves (the release version-bump PR is gated by its merge_group entry,
    # which runs the full suite). The release merge_group entry IS the
    # pre-publish gate — it must run the full suite so we know what ships is
    # green.
    if: |
      (github.event_name == 'pull_request' || github.event_name == 'merge_group') &&
      !startsWith(github.head_ref, 'release/v')

    env:
      # RUST_LOG controls tracing output level (used by both test-log and production code)
      RUST_LOG: error
      CARGO_TARGET_DIR: ${{ github.workspace }}/target
      # Increase rustc stack size to prevent SIGSEGV during compilation
      # 256MB required for LLVM optimization passes on large workspaces (rustc suggests this value)
      RUST_MIN_STACK: 268435456

    steps:
      - uses: actions/checkout@v7

      - name: Install mold linker
        run: |
          sudo apt-get update
          sudo apt-get install -y mold

      - uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: 1.93.0
          targets: wasm32-unknown-unknown

      - uses: Swatinem/rust-cache@v2
        if: success() || steps.test.conclusion == 'failure'
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install nextest
        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin

      - name: Build
        env:
          # Use mold linker to avoid rust-lld crashes (see issue #2519)
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: clang
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -C link-arg=-fuse-ld=mold
        run: |
          cargo build --locked
          export PATH="$PWD/target/debug:$PATH"
          make -C apps/freenet-ping -f run-ping.mk build

      - name: Clean test directories
        run: |
          # Remove freenet test directories from /tmp to avoid permission issues
          # when tests create directories with different user ownership
          rm -rf /tmp/freenet /tmp/freenet-* 2>/dev/null || true

      - name: Test
        env:
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: clang
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -C link-arg=-fuse-ld=mold
          # Default congestion control is FixedRate for CI stability.
          # To test BBR: set FREENET_CONGESTION_CONTROL=bbr and optionally FREENET_BBR_STARTUP_RATE
          # Reduce gateway connection backoff for CI (default 30s is too aggressive
          # for localhost tests under CI resource pressure). See issue #3078.
          FREENET_BACKOFF_BASE_SECS: "5"
        # Run unit and integration tests (simulation tests run in separate job).
        # nextest runs each test in its own process, providing full isolation
        # for global state (DashMaps, callbacks, etc.). Determinism tests no
        # longer need serialization — process isolation handles it (#3051).
        # Exclude freenet-ping-types: --no-default-features disables its std
        # feature, breaking test code that uses Ping::insert().
        # Skip blocked_peers tests (run serially below — they spin up 3 real
        # nodes each and timeout under parallel resource contention).
        # nextest retries only failed tests (via .config/nextest.toml [profile.ci]),
        # avoiding the 10-15 min penalty of re-running the entire workspace.
        run: |
          cargo nextest run --workspace --exclude freenet-ping-types \
            --no-default-features \
            --features trace,websocket,redb,wasmtime-backend,testing \
            --profile ci -E 'not test(blocked_peers)'

      - name: Test blocked-peers (serial)
        if: ${{ !cancelled() }}
        env:
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: clang
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -C link-arg=-fuse-ld=mold
          FREENET_BACKOFF_BASE_SECS: "5"
        # Blocked-peers tests each start 3 real nodes with 180s connection
        # timeouts. Running them sequentially prevents resource contention.
        run: |
          cargo nextest run -p freenet-ping-app \
            --features testing --profile ci \
            -E 'test(blocked_peers)' -j 1

      - name: Test ping-types
        if: ${{ !cancelled() }}
        # freenet-ping-types tests need the std feature (default).
        # Run separately since the main step uses --no-default-features.
        run: cargo nextest run -p freenet-ping-types --profile ci

  macos_unit:
    name: macOS Service Unit
    runs-on: macos-latest
    timeout-minutes: 30
    needs: fmt_check
    # Same skip conditions as test_unit (see test_unit comment for details).
    if: |
      (github.event_name == 'pull_request' || github.event_name == 'merge_group') &&
      !startsWith(github.head_ref, 'release/v')

    env:
      # RUST_LOG controls tracing output level (used by both test-log and production code)
      RUST_LOG: error
      CARGO_TARGET_DIR: ${{ github.workspace }}/target
      # Increase rustc stack size to prevent SIGSEGV during compilation
      # 256MB required for LLVM optimization passes on large workspaces (rustc suggests this value)
      RUST_MIN_STACK: 268435456

    steps:
      - uses: actions/checkout@v7

      - uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: 1.93.0
          targets: wasm32-unknown-unknown

      - uses: Swatinem/rust-cache@v2
        if: success() || steps.test.conclusion == 'failure'
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install nextest
        run: curl -LsSf https://get.nexte.st/latest/mac | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin

      - name: Build freenet bin
        run: cargo build --locked -p freenet --bin freenet

      - name: Test service commands
        run: cargo nextest run -p freenet --bin freenet --profile ci -E 'test(commands::service)'

  test_simulation:
    name: Simulation
    # Run on the heavy runner for PRs (validates the change) and for release
    # merge_group entries (the full-suite release gate). Skip the real steps on
    # non-release merge_group entries to avoid contending for ubicloud-standard-16
    # slots on every queue entry — the runner stays cheap there since every step
    # is gated off (see skip_check below).
    runs-on: "${{ (github.event_name == 'merge_group' && !startsWith(github.event.merge_group.head_commit.message, 'build: release')) && 'ubuntu-latest' || 'ubicloud-standard-16' }}"
    timeout-minutes: 40
    needs: fmt_check
    # Same skip conditions as test_unit (see test_unit comment for details)
    if: |
      (github.event_name == 'pull_request' || github.event_name == 'merge_group') &&
      !startsWith(github.head_ref, 'release/v')

    env:
      RUST_LOG: error
      RUST_MIN_STACK: 268435456

    steps:
      # Skip every real step on non-release merge_group entries. PR-level CI
      # already exercised the same code; running heavy ubicloud-standard-16
      # simulations on every merge-queue entry contends for runner slots
      # (issue #3973) without catching merge-time-specific regressions. The
      # release merge_group entry (commit message starts with "build: release")
      # still runs the full suite as the final pre-publish gate. The job
      # remains present in the skipped case so branch-protection's required-
      # check gate stays satisfied.
      - name: Detect skip
        id: skip_check
        env:
          EVENT_NAME: ${{ github.event_name }}
          MERGE_COMMIT_MSG: ${{ github.event.merge_group.head_commit.message }}
        run: |
          if [[ "$EVENT_NAME" == "merge_group" && "$MERGE_COMMIT_MSG" != build:\ release* ]]; then
            echo "skip=true" >> "$GITHUB_OUTPUT"
            echo "Skipping Simulation on non-release merge_group — covered by PR-level CI; full suite runs on release merge_group (#3973)"
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi

      - uses: actions/checkout@v7
        if: steps.skip_check.outputs.skip != 'true'

      - name: Install mold linker
        if: steps.skip_check.outputs.skip != 'true'
        run: |
          sudo apt-get update
          sudo apt-get install -y mold

      - uses: dtolnay/rust-toolchain@stable
        if: steps.skip_check.outputs.skip != 'true'
        with:
          toolchain: 1.93.0

      - uses: Swatinem/rust-cache@v2
        if: steps.skip_check.outputs.skip != 'true'

      - name: Install nextest
        if: steps.skip_check.outputs.skip != 'true'
        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin

      # Build the simulation test binaries and fdev up-front in one cargo
      # invocation so they share a single compile of freenet/deps. Then the
      # next two steps can run their *tests* in parallel without contending
      # for the cargo target-dir lock.
      - name: Build simulation tests and fdev
        if: steps.skip_check.outputs.skip != 'true'
        env:
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: clang
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -C link-arg=-fuse-ld=mold
        run: |
          # `--lib` is included so the in-crate simulation tests gated behind
          # `#[cfg(all(test, feature = "simulation_tests"))]` (e.g. the
          # governance ban-chain e2e module in
          # crates/core/src/contract/governance.rs) are built and run. They use
          # `pub(crate)` accessors (Ring::governance, Ring::contract_ban_list,
          # the test-only hosting_manager_* counters) that an external
          # tests/ integration binary cannot reach, so they MUST live in the
          # lib. Without `--lib` here this whole class of tests was compiled
          # out of the workspace test step (no simulation_tests feature) AND
          # excluded from this simulation step (only --test binaries), so it
          # never ran in CI. See issue #4301.
          cargo nextest run -p freenet --locked --no-run \
            --lib \
            --test simulation_integration \
            --test simulation_smoke \
            --test streaming_e2e \
            --test state_verification \
            --features simulation_tests,testing --profile ci
          cargo build -p fdev --release --locked

      - name: Run simulation tests (nextest first, then fdev load sims)
        if: steps.skip_check.outputs.skip != 'true'
        env:
          RUST_LOG: info,turmoil=warn
        run: |
          # Each fdev sim runs under `timeout` so one hanging sim cannot eat the
          # entire 30m budget silently. Each sim writes to its own log file so
          # we can upload them as artifacts and tell which one wedged.
          mkdir -p sim-logs

          # Phase 1: run the precise nextest simulation tests FIRST and ALONE
          # (already built above with --no-run). Previously these ran
          # concurrently with the fdev load sims below; under a high-contention
          # runner the Turmoil `start_paused` clock auto-advances past the 120s
          # RealTime connection idle timeout during `spawn_blocking` (WASM
          # execution), spuriously tearing down healthy connections and flaking
          # the precise streaming/GET sim tests on EVERY nextest retry (the fdev
          # sims run ~8-15m, overlapping all 3 retries). Running nextest uncontended
          # first removes that interference; the fdev load sims run afterwards
          # (still parallel among themselves). Use a ( ) subshell so `exit $rc`
          # ends the subshell, not this whole step.
          NEXTEST_FAILED=0
          (
            echo "== $(date -u +%H:%M:%S) nextest starting"
            # `--lib` makes the in-crate simulation_tests-gated tests (see the
            # build step above and issue #4301) available to run alongside the
            # integration binaries. The filterset then restricts execution to
            # ALL integration tests (`kind(test)`) PLUS only the simulation
            # e2e lib tests (`kind(lib) & test(sim_e2e_tests)`) — so the ~3000
            # ordinary freenet lib unit tests (already covered by the
            # workspace test job) are NOT re-run here. They were all built with
            # --no-run above.
            cargo nextest run -p freenet --locked \
              --lib \
              --test simulation_integration \
              --test simulation_smoke \
              --test streaming_e2e \
              --test state_verification \
              -E 'kind(test) | (kind(lib) & test(sim_e2e_tests))' \
              --features simulation_tests,testing --profile ci 2>&1
            rc=$?
            echo "== $(date -u +%H:%M:%S) nextest done rc=${rc}"
            exit $rc
          ) > sim-logs/nextest.log 2>&1 || NEXTEST_FAILED=1
          echo "nextest phase done (failed=${NEXTEST_FAILED}) — starting fdev load sims"

          # 15 minute cap per sim, SIGKILL 30s later if it refuses to die.
          # (#4404 placement migration roughly doubles per-sim cost when active,
          # so 8m could be exceeded on a slow release-branch runner; the job's
          # 40m timeout-minutes still covers the four sims running in parallel.)
          # Background directly (no function) so $! refers to the actual child
          # of this shell, which `wait` can then reap.
          {
            echo "== $(date -u +%H:%M:%S) ci-medium-50 starting"
            timeout --kill-after=30s 15m target/release/fdev test \
              --name "ci-medium-50" \
              --seed 0xDEADBEEF \
              --gateways 4 --nodes 46 --events 2000 \
              --ring-max-htl 12 --max-connections 20 --min-connections 6 \
              --latency-min 10 --latency-max 50 \
              --min-success-rate 1.0 \
              --print-summary --print-network-stats \
              single-process 2>&1
            rc=$?
            echo "== $(date -u +%H:%M:%S) ci-medium-50 done rc=${rc}"
            exit $rc
          } > sim-logs/ci-medium-50.log 2>&1 &
          PID1=$!

          {
            echo "== $(date -u +%H:%M:%S) ci-fault-loss starting"
            timeout --kill-after=30s 15m target/release/fdev test \
              --name "ci-fault-loss" \
              --seed 0xFA017001 \
              --gateways 3 --nodes 27 --events 1000 \
              --message-loss 0.15 \
              --latency-min 10 --latency-max 50 \
              --min-success-rate 0.80 \
              --print-summary --print-network-stats \
              single-process 2>&1
            rc=$?
            echo "== $(date -u +%H:%M:%S) ci-fault-loss done rc=${rc}"
            exit $rc
          } > sim-logs/ci-fault-loss.log 2>&1 &
          PID2=$!

          {
            echo "== $(date -u +%H:%M:%S) ci-high-latency starting"
            timeout --kill-after=30s 15m target/release/fdev test \
              --name "ci-high-latency" \
              --seed 0x1A7E0C71 \
              --gateways 2 --nodes 12 --events 500 \
              --latency-min 50 --latency-max 200 \
              --min-success-rate 0.95 \
              --print-summary --print-network-stats \
              single-process 2>&1
            rc=$?
            echo "== $(date -u +%H:%M:%S) ci-high-latency done rc=${rc}"
            exit $rc
          } > sim-logs/ci-high-latency.log 2>&1 &
          PID3=$!

          {
            echo "== $(date -u +%H:%M:%S) ci-churn-20 starting"
            timeout --kill-after=30s 15m target/release/fdev test \
              --name "ci-churn-20" \
              --seed 0xC102FEED \
              --gateways 2 --nodes 18 --events 500 \
              --ring-max-htl 10 --max-connections 15 --min-connections 4 \
              --latency-min 10 --latency-max 50 \
              --churn-rate 0.1 --churn-recovery-delay-ms 3000 \
              --churn-permanent-rate 0.05 --churn-tick-ms 5000 \
              --min-success-rate 0.80 \
              --print-summary --print-network-stats \
              single-process 2>&1
            rc=$?
            echo "== $(date -u +%H:%M:%S) ci-churn-20 done rc=${rc}"
            exit $rc
          } > sim-logs/ci-churn-20.log 2>&1 &
          PID4=$!

          echo "Started fdev: medium=$PID1 fault=$PID2 latency=$PID3 churn=$PID4"

          # Wait for the fdev sims and fail if any (or nextest above) failed
          FAILED=$NEXTEST_FAILED
          [ "$NEXTEST_FAILED" -eq 1 ] && echo "nextest FAILED"
          wait $PID1 || { echo "ci-medium-50 FAILED"; FAILED=1; }
          wait $PID2 || { echo "ci-fault-loss FAILED"; FAILED=1; }
          wait $PID3 || { echo "ci-high-latency FAILED"; FAILED=1; }
          wait $PID4 || { echo "ci-churn-20 FAILED"; FAILED=1; }

          echo "===== sim log tails ====="
          for f in sim-logs/*.log; do
            echo "----- $f (last 40 lines) -----"
            tail -n 40 "$f" || true
          done

          exit $FAILED

      - name: Upload simulation logs
        if: always() && steps.skip_check.outputs.skip != 'true'
        uses: actions/upload-artifact@v7
        with:
          name: simulation-logs
          path: sim-logs/
          if-no-files-found: ignore
          retention-days: 7

  nat_validation:
    name: NAT Validation
    needs: fmt_check
    # Run on the heavy runner for PRs (validates the change) and for release
    # merge_group entries (the full-suite release gate). Skip the real steps on
    # non-release merge_group entries to avoid contending for ubicloud-standard-16
    # slots on every queue entry — the runner stays cheap there since every step
    # is gated off (see skip_check below).
    runs-on: "${{ (github.event_name == 'merge_group' && !startsWith(github.event.merge_group.head_commit.message, 'build: release')) && 'ubuntu-latest' || 'ubicloud-standard-16' }}"
    timeout-minutes: 30
    # Skip on:
    # - push to main: PR already validated code before merge
    # - release branches: covered by the release merge_group gate
    # - dependabot PRs: just version bumps, Test job is sufficient
    if: |
      (github.event_name == 'pull_request' || github.event_name == 'merge_group') &&
      !startsWith(github.head_ref, 'release/v') &&
      !startsWith(github.head_ref, 'dependabot/')

    env:
      RUST_LOG: error
      RUST_MIN_STACK: 268435456

    steps:
      # Skip every real step on non-release merge_group entries. PR-level CI
      # already exercised the same code; running heavy ubicloud-standard-16
      # Docker NAT validation on every merge-queue entry contends for runner
      # slots (issue #3973) without catching merge-time-specific regressions.
      # The release merge_group entry (commit message starts with
      # "build: release") still runs the full suite as the final pre-publish
      # gate. The job remains present in the skipped case so branch-
      # protection's required-check gate stays satisfied.
      - name: Detect skip
        id: skip_check
        env:
          EVENT_NAME: ${{ github.event_name }}
          MERGE_COMMIT_MSG: ${{ github.event.merge_group.head_commit.message }}
        run: |
          if [[ "$EVENT_NAME" == "merge_group" && "$MERGE_COMMIT_MSG" != build:\ release* ]]; then
            echo "skip=true" >> "$GITHUB_OUTPUT"
            echo "Skipping NAT Validation on non-release merge_group — covered by PR-level CI; full suite runs on release merge_group (#3973)"
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi

      - uses: actions/checkout@v7
        if: steps.skip_check.outputs.skip != 'true'

      - name: Install dependencies
        if: steps.skip_check.outputs.skip != 'true'
        run: |
          sudo apt-get update
          sudo apt-get install -y mold liblzma-dev

      - uses: dtolnay/rust-toolchain@stable
        if: steps.skip_check.outputs.skip != 'true'
        with:
          toolchain: 1.93.0
          targets: wasm32-unknown-unknown

      - uses: Swatinem/rust-cache@v2
        if: steps.skip_check.outputs.skip != 'true'

      - name: Pull Docker images for NAT simulation
        if: steps.skip_check.outputs.skip != 'true'
        run: docker pull alpine:latest

      - name: Install nextest
        if: steps.skip_check.outputs.skip != 'true'
        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin

      - name: Build
        if: steps.skip_check.outputs.skip != 'true'
        env:
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: clang
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -C link-arg=-fuse-ld=mold
        run: |
          cargo build --locked
          export PATH="$PWD/target/debug:$PATH"
          make -C apps/freenet-ping -f run-ping.mk build

      - name: Run Docker NAT test
        if: steps.skip_check.outputs.skip != 'true'
        env:
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: clang
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -C link-arg=-fuse-ld=mold
          FREENET_TEST_DOCKER_NAT: "1"
          FREENET_BINARY_PATH: ${{ github.workspace }}/target/debug/freenet
        # Validates NAT hole-punching using Docker NAT simulation with
        # freenet-ping contract operations (PUT, SUBSCRIBE, UPDATE, GET).
        # Replaces the six-peer River regression test with a self-contained
        # test that doesn't depend on the River repository.
        run: |
          if ! docker info >/dev/null 2>&1; then
            echo "ERROR: Docker is not available but is REQUIRED for NAT hole punching tests"
            echo "A test that doesn't test what it claims to test is worse than no test"
            exit 1
          fi
          cargo nextest run -p freenet-ping-app --features testing \
            --profile ci -E 'test(docker_nat)' -- --ignored

  claude-ci-analysis:
    name: Claude CI Analysis
    runs-on: ubicloud-standard-4
    timeout-minutes: 30
    needs: [clippy_check, fmt_check]  # Removed test_all to avoid dependency skip cascade
    # Only run on PRs with claude-debug label when there's a failure
    if: |
      github.event_name == 'pull_request' &&
      failure() &&
      contains(github.event.pull_request.labels.*.name, 'claude-debug')

    permissions:
      contents: write
      pull-requests: write
      issues: read
      id-token: write
      actions: read

    steps:
      - name: Checkout repository
        uses: actions/checkout@v7
        with:
          fetch-depth: 10
          ref: ${{ github.event.pull_request.head.ref }}

      - name: Check Claude fix attempt count
        id: check-attempts
        run: |
          # Count how many times Claude has already tried to fix this
          ATTEMPT_COUNT=$(git log -10 --pretty=%B | grep -c "🤖 Claude CI fix attempt" || echo "0")
          echo "Current attempt count: $ATTEMPT_COUNT"

          if [ "$ATTEMPT_COUNT" -ge 2 ]; then
            echo "skip=true" >> $GITHUB_OUTPUT
            echo "❌ Claude has already made 2 fix attempts. Stopping to prevent infinite loop."
          else
            NEXT_ATTEMPT=$((ATTEMPT_COUNT + 1))
            echo "skip=false" >> $GITHUB_OUTPUT
            echo "attempt_number=$NEXT_ATTEMPT" >> $GITHUB_OUTPUT
            echo "✅ Proceeding with fix attempt $NEXT_ATTEMPT/2"
          fi

      - name: Run Claude CI Fix
        if: steps.check-attempts.outputs.skip != 'true'
        uses: anthropics/claude-code-action@v1
        with:
          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          prompt: |
            REPO: ${{ github.repository }}
            PR NUMBER: ${{ github.event.pull_request.number }}
            PR BRANCH: ${{ github.event.pull_request.head.ref }}
            FIX ATTEMPT: ${{ steps.check-attempts.outputs.attempt_number }}/2

            The CI workflow has failed. Your task is to:
            1. Analyze the CI failure logs to identify the root cause
            2. Fix the code to resolve the issue
            3. Commit and push your fixes to the PR branch

            IMPORTANT COMMIT MESSAGE FORMAT:
            Your commit message MUST include:
            - Clear description of what you fixed
            - The marker: "🤖 Claude CI fix attempt ${{ steps.check-attempts.outputs.attempt_number }}/2"
            - This prevents infinite loops and tracks attempt count

            Example commit message:
            "Fix: [description of fix]

            🤖 Claude CI fix attempt ${{ steps.check-attempts.outputs.attempt_number }}/2

            Co-authored-by: Claude <noreply@anthropic.com>"

            GUIDELINES:
            - Make focused, minimal changes to fix the specific CI failure
            - Do not make unrelated improvements
            - Use git commands to commit and push your changes
            - After pushing, CI will run automatically

          claude_args: '--allowed-tools "Bash(gh issue view:*),Bash(gh search:*),Bash(gh issue list:*),Bash(gh pr comment:*),Bash(gh pr view:*),Bash(gh pr list:*),Bash(gh api:*),Bash(gh run view:*),Bash(git status:*),Bash(git diff:*),Bash(git add:*),Bash(git commit:*),Bash(git push:*),Bash(git log:*)"'

  # Notify the private freenet-dev River room when CI fails on the main branch.
  # Only fires on push to main (post-merge): the merge queue already gates PRs,
  # so a failure on the main run means main is red — a bad merge, or a transient
  # infra/toolchain flake (the linked run shows which). On push to main only
  # fmt_check + clippy_check run (the heavy jobs are PR/merge_group-gated), so
  # those are the dependencies that define "main CI".
  notify_dev_room_main_failure:
    name: Notify dev room on main CI failure
    runs-on: ubuntu-latest
    needs: [fmt_check, clippy_check]
    if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/main'
    # Best-effort: a failure in this notifier itself must not add a second red
    # job to an already-red run (parity with the nightly notify job).
    continue-on-error: true
    steps:
      - uses: actions/checkout@v7
      - uses: ./.github/actions/river-dev-notify
        with:
          message: "🚨 main CI failed — ${{ github.repository }}@${{ github.sha }} — ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          bot-config: ${{ secrets.RIVER_DEV_BOT_CONFIG }}
          room-id: ${{ secrets.RIVER_DEV_ROOM_ID }}
          gateway-url: ${{ secrets.RIVER_GATEWAY_URL }}