feat(telemetry): shadow event sub-budget so shadow streams can't star… #957
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Benchmarks | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'crates/core/src/transport/**' | |
| - 'crates/core/benches/**' | |
| - '.github/workflows/benchmarks.yml' | |
| pull_request: | |
| paths: | |
| - 'crates/core/src/transport/**' | |
| - 'crates/core/benches/**' | |
| - '.github/workflows/benchmarks.yml' | |
| # Allow manual trigger | |
| workflow_dispatch: | |
| # Cancel in-progress runs when a new commit is pushed. | |
| # On main, never cancel — each merge must complete its run (#3311). | |
| concurrency: | |
| group: benchmarks-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| benchmark: | |
| name: Performance Benchmarks | |
| # NOTE: Using ubuntu-latest instead of self-hosted because the self-hosted | |
| # runner is missing liblzma-dev and doesn't have passwordless sudo. | |
| # When the self-hosted runner is updated, change back for consistent hardware. | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| # Don't fail the whole workflow if benchmarks detect regressions | |
| continue-on-error: true | |
| env: | |
| CARGO_TARGET_DIR: ${{ github.workspace }}/target | |
| # Reduce noise from logging during benchmarks | |
| RUST_LOG: error | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| # Need enough history to find merge-base with target branch | |
| fetch-depth: 0 | |
| # Install system dependencies required for linking | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y liblzma-dev | |
| - uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: 1.93.0 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| # Cache benchmarks separately from test builds | |
| prefix-key: bench | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| # Determine baseline commit (merge-base for PRs, current SHA for main) | |
| # This eliminates false positives from "missing recent improvements" | |
| - name: Determine Baseline Commit | |
| id: baseline-commit | |
| run: | | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| # For PRs: use merge-base (where branch diverged from main) | |
| BASE_SHA=$(git merge-base origin/${{ github.base_ref }} HEAD) | |
| echo "📍 Using merge-base for PR: $BASE_SHA" | |
| else | |
| # For main branch: use current commit | |
| BASE_SHA=${{ github.sha }} | |
| echo "📍 Using current commit for main: $BASE_SHA" | |
| fi | |
| echo "sha=$BASE_SHA" >> $GITHUB_OUTPUT | |
| # Download baseline from merge-base commit (eliminates stale baseline problem) | |
| - name: Download Baseline from Merge-Base | |
| id: baseline-cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: target/criterion | |
| # Try exact match with merge-base SHA | |
| key: criterion-baseline-main-${{ runner.os }}-${{ steps.baseline-commit.outputs.sha }} | |
| # Fall back to recent main baseline if merge-base not cached | |
| restore-keys: | | |
| criterion-baseline-main-${{ runner.os }}- | |
| - name: Report baseline status | |
| run: | | |
| echo "## Baseline Information" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ steps.baseline-cache.outputs.cache-hit }}" == "true" ]; then | |
| echo "✅ **Exact baseline match**: Using baseline from this exact commit" >> $GITHUB_STEP_SUMMARY | |
| echo "- Cache key: \`criterion-baseline-main-${{ runner.os }}-${{ github.sha }}\`" >> $GITHUB_STEP_SUMMARY | |
| elif [ -d "target/criterion" ]; then | |
| echo "⚠️ **Restored from fallback**: Using most recent main branch baseline (via restore-keys)" >> $GITHUB_STEP_SUMMARY | |
| echo "- Cache key prefix: \`criterion-baseline-main-${{ runner.os }}-\`" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Warning**: Baseline may be from an older commit if main has advanced" >> $GITHUB_STEP_SUMMARY | |
| # Try to determine baseline age | |
| if [ -f "target/criterion/.baseline_info" ]; then | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Baseline details:**" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| cat target/criterion/.baseline_info >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # Show when criterion directory was last modified | |
| BASELINE_AGE=$(find target/criterion -name "base" -type d -printf '%T+\n' 2>/dev/null | head -1 || echo "unknown") | |
| if [ "$BASELINE_AGE" != "unknown" ]; then | |
| echo "- Last modified: \`$BASELINE_AGE\`" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| else | |
| echo "⚠️ **No baseline found** - This run will establish the baseline" >> $GITHUB_STEP_SUMMARY | |
| echo "- All benchmarks will show as \"new\" with no comparison" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Compile benchmarks first (fast fail on compilation errors) | |
| - name: Compile Benchmarks | |
| run: | | |
| echo "## Compiling Benchmarks" >> $GITHUB_STEP_SUMMARY | |
| cargo bench --bench transport_ci --features bench --no-run --color=never 2>&1 | tee compile_output.txt | |
| echo "✅ Benchmark compilation successful" >> $GITHUB_STEP_SUMMARY | |
| # Find the benchmark binary path for direct execution | |
| BENCH_BIN=$(find target/release/deps -name 'transport_ci-*' -type f -executable | head -1) | |
| echo "BENCH_BIN=$BENCH_BIN" >> $GITHUB_ENV | |
| echo "Benchmark binary: $BENCH_BIN" | |
| # Run Transport Layer Benchmarks - execute pre-compiled binary directly | |
| - name: Run Transport Layer Benchmarks | |
| id: bench_transport | |
| run: | | |
| echo "## Transport Layer Performance (Streamlined)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Focused on what matters - sustained throughput and critical paths:" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Warm connection throughput**: Sustained bulk transfer (PRIMARY METRIC)" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Connection establishment**: Cold-start handshake timing" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Streaming buffer**: Lock-free buffer operations (critical path)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Note**: Micro-benchmarks moved to \`transport_extended\` (runs nightly)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Run pre-compiled benchmark binary directly (avoids cargo recompilation check) | |
| "$BENCH_BIN" --bench --color never 2>&1 | tee bench_output.txt | |
| # Parse benchmark results with structured script | |
| - name: Parse Benchmark Results | |
| id: parse_results | |
| run: | | |
| # Install Python (should already be available on ubuntu-latest) | |
| python3 --version | |
| # Run parser script | |
| if python3 scripts/parse_bench_output.py bench_output.txt > parsed_output.txt 2>&1; then | |
| echo "regression_detected=false" >> $GITHUB_OUTPUT | |
| PARSE_EXIT=0 | |
| else | |
| # Script exits with 1 if regressions found | |
| echo "regression_detected=true" >> $GITHUB_OUTPUT | |
| PARSE_EXIT=$? | |
| fi | |
| # Append parsed results to summary | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| cat bench_summary.md >> $GITHUB_STEP_SUMMARY 2>/dev/null || { | |
| echo "⚠️ Failed to parse benchmark results" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| # Strip ANSI escape codes for clean markdown output | |
| tail -50 bench_output.txt | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| } | |
| exit 0 # Don't fail the step even if regressions found | |
| # Save baseline metadata for tracking | |
| - name: Save Baseline Metadata | |
| if: github.ref == 'refs/heads/main' | |
| run: | | |
| mkdir -p target/criterion | |
| cat > target/criterion/.baseline_info <<EOF | |
| Commit: ${{ github.sha }} | |
| Branch: ${{ github.ref }} | |
| Timestamp: $(date -u +"%Y-%m-%d %H:%M:%S UTC") | |
| Workflow Run: ${{ github.run_id }} | |
| EOF | |
| # Save baseline for future comparisons (only on main branch) | |
| # Each main branch commit creates a new baseline with SHA in the key | |
| # PRs use restore-keys to find the most recent one | |
| - name: Save Baseline to Main | |
| if: github.ref == 'refs/heads/main' | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: target/criterion | |
| # Include SHA so each main commit has its own baseline | |
| key: criterion-baseline-main-${{ runner.os }}-${{ github.sha }} | |
| # Post comment on PR with regression summary | |
| - name: Comment on PR | |
| if: github.event_name == 'pull_request' && steps.parse_results.outputs.regression_detected == 'true' | |
| uses: actions/github-script@v9 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| // Try to read the parsed PR comment from the Python script | |
| let body; | |
| try { | |
| body = fs.readFileSync('bench_pr_comment.md', 'utf8'); | |
| } catch (error) { | |
| // Fallback to simple message if parsing failed | |
| body = `## ⚠️ Performance Benchmark Regressions Detected | |
| Some benchmarks show performance regressions compared to the baseline. | |
| [View full benchmark results](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}) | |
| > **Note:** This is informational only and does not block the PR.`; | |
| } | |
| // Add link to workflow run | |
| body += `\n\n[View full benchmark results and summary](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})`; | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: body | |
| }); | |
| # Upload benchmark results as artifact | |
| - name: Upload Benchmark Results | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: benchmark-results | |
| path: | | |
| bench_output.txt | |
| bench_results.json | |
| bench_summary.md | |
| bench_pr_comment.md | |
| target/criterion/**/report/index.html | |
| retention-days: 30 | |
| # Summary job that always succeeds (so PR can merge) | |
| benchmark-summary: | |
| name: Benchmark Summary | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| needs: benchmark | |
| # Always run, even if benchmark job "fails" (detected regression) | |
| if: always() | |
| steps: | |
| - name: Check Benchmark Status | |
| run: | | |
| if [ "${{ needs.benchmark.result }}" == "failure" ]; then | |
| echo "⚠️ Benchmarks detected performance regressions, but this is non-blocking." | |
| echo "Please review the benchmark results in the workflow summary." | |
| else | |
| echo "✅ Benchmarks completed successfully." | |
| fi |