diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b000357f6..bd845706e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -492,9 +492,32 @@ jobs: - name: Eval runner guards hold (offline, no model) run: python3 scripts/run-evals.py --selftest + # Guards .gitleaks.toml: the fixture allowlist must not blind the scanner (full history needed for the history scan). + gitleaks: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Install gitleaks 8.30.1 (pinned sha256) + run: | + set -euo pipefail + cd "$RUNNER_TEMP" + curl -fsSL -O https://github.com/gitleaks/gitleaks/releases/download/v8.30.1/gitleaks_8.30.1_linux_x64.tar.gz + echo "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb gitleaks_8.30.1_linux_x64.tar.gz" | sha256sum -c - + mkdir -p "$RUNNER_TEMP/bin" + tar -xzf gitleaks_8.30.1_linux_x64.tar.gz -C "$RUNNER_TEMP/bin" gitleaks + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + - name: Allowlist guard + run: REQUIRE_GITLEAKS=1 bash scripts/test_gitleaks_allowlist.sh + # Branch protection requires a check named exactly `gates`: it passes only when every shard job succeeded. gates: - needs: [selftests, enforce, perpr, evals, cheap] + needs: [selftests, enforce, perpr, evals, cheap, gitleaks] if: always() runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/.gitleaks.toml b/.gitleaks.toml index 977a7432d..fe7115d9f 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -11,6 +11,19 @@ paths = [ '''^docs/bench/secrets-heldout[0-9]+-[0-9]+/(secret-leaks|secret-decoys)\.txt$''', '''^docs/bench/secrets-heldout[0-9]+-[0-9]+/ledger\.jsonl$''', '''^docs/bench/g1-matrix/heldout/[^/]+/result\.json$''', - '''^scripts/test_secret_extra_rules_w5\.py$''', - '''^scripts/test_secret_precision\.py$''', +] + +# The two secret-test scripts (scripts/test_secret_extra_rules_w5.py and +# scripts/test_secret_precision.py) are exempt only for their fictional fixture +# values, matched by value and not by path, so a real key added to either file is +# still caught. These value regexes apply repo-wide, not only to those two files: +# a line containing one of the values is exempt anywhere. (A path in an AND allowlist is applied on its own by gitleaks +# 8.30.1 and would blind the whole file again.) +[[allowlists]] +description = "Fictional fixture values in the secret test scripts" +regexTarget = "match" +regexes = [ + '''eyJhbGciOiJIUzI1NiJ9\.eyJyb2xlIjoiYW5vbiJ9\.''', + '''sk_test_@@A2aMrXbJqxi4ai1aZSzJWHU7''', + '''pk_live_n91DMA5Cj6vP3w01RyvJgcjy''', ] diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 000000000..718f986a5 --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,19 @@ +3d4cd69aeadc07ec2527512127bf8874a0691d93:docs/bench/growth-r4-1018/results/outputs/rival/gy052-e92a8bf6-6bfceff8.txt:generic-api-key:40 +508000d654136f41e8b02be8e012ec1a7c12244d:docs/bench/differentiator-1019/calibration/answers/PERUN/ca2.txt:curl-auth-header:20 +508000d654136f41e8b02be8e012ec1a7c12244d:docs/bench/differentiator-1019/calibration/answers/RIVAL/ca2.txt:curl-auth-header:13 +508000d654136f41e8b02be8e012ec1a7c12244d:docs/bench/differentiator-1019/golden/fabrication-rival-ca2.txt:curl-auth-header:13 +508000d654136f41e8b02be8e012ec1a7c12244d:docs/bench/differentiator-1019/superseded-first-run/answers/PLAIN/ha4.txt:curl-auth-header:17 +508000d654136f41e8b02be8e012ec1a7c12244d:docs/bench/differentiator-1019/superseded-first-run/answers/PLAIN/ha7.txt:stripe-access-token:26 +52ebef8644f67a30e39405e47886308d077e5e47:docs/bench/bizops-fab-1019/outputs/cal/EDITED-cd2e78bc/xa2.txt:curl-auth-header:13 +52ebef8644f67a30e39405e47886308d077e5e47:docs/bench/bizops-fab-1019/outputs/cal/EDITED-ef4491a1/xa2.txt:curl-auth-header:13 +52ebef8644f67a30e39405e47886308d077e5e47:docs/bench/bizops-fab-1019/outputs/cal/EDITED-fb936c4e/xa2.txt:curl-auth-header:14 +52ebef8644f67a30e39405e47886308d077e5e47:docs/bench/bizops-fab-1019/outputs/heldout/CURRENT-39af659d/ya2.txt:curl-auth-header:1 +52ebef8644f67a30e39405e47886308d077e5e47:docs/bench/bizops-fab-1019/outputs/heldout/EDITED-fb936c4e/ya2.txt:curl-auth-header:11 +b28a90f83b67332599d18f13e0a61611980c8b40:docs/bench/bizops-shipped-1018/outputs/shipped/c150-124.txt:generic-api-key:26 +c3f14095a3d18bf4c37ab603f1ff0e8d9dcc4d1d:docs/bench/security-gates-heldout-1024/secret-leaks.txt:generic-api-key:16 +c3f14095a3d18bf4c37ab603f1ff0e8d9dcc4d1d:docs/bench/security-gates-heldout-1024/secret-leaks.txt:generic-api-key:3 +d50de27a76d3fa0b88bd3f41b35fc0692488e226:docs/bench/growth-outcome-r2-1019/ledger.jsonl:generic-api-key:2 +d68fb67ca2aaa36cf2bcc9c4329ed63d8aa48dca:docs/bench/security-gates-1024/secret-leaks.txt:generic-api-key:4 +de66350063b258c7af5c2a56841774ef428abaf4:docs/bench/security-gates-1024/rivals/gitleaks-v8.30.1.toml:aws-amazon-bedrock-api-key-short-lived:232 +e42fed4a7e94713d3b51487a45bce4f390b93a90:docs/bench/security-gates-heldout2-1024/secret-decoys.txt:curl-auth-header:23 +e42fed4a7e94713d3b51487a45bce4f390b93a90:docs/bench/security-gates-heldout2-1024/secret-leaks.txt:curl-auth-header:29 diff --git a/changelog.d/gitleaks-ci-guard.md b/changelog.d/gitleaks-ci-guard.md new file mode 100644 index 000000000..25b6fa0a7 --- /dev/null +++ b/changelog.d/gitleaks-ci-guard.md @@ -0,0 +1,3 @@ +### Added +- CI now runs scripts/test_gitleaks_allowlist.sh in a `gitleaks` job (full-history checkout, gitleaks 8.30.1 installed from its release tarball verified against a hardcoded sha256, REQUIRE_GITLEAKS=1), and the `gates` aggregate requires it. Before this no workflow called the script. +- The guard is strict (missing gitleaks or shallow history FAILs) only when REQUIRE_GITLEAKS=1 is set, so the per-PR test legs, which do not install gitleaks, SKIP it. diff --git a/changelog.d/gitleaks-narrow-allowlist.md b/changelog.d/gitleaks-narrow-allowlist.md new file mode 100644 index 000000000..558d3abf5 --- /dev/null +++ b/changelog.d/gitleaks-narrow-allowlist.md @@ -0,0 +1,3 @@ +### Fixed +- .gitleaks.toml no longer allowlists the whole of scripts/test_secret_extra_rules_w5.py and scripts/test_secret_precision.py by path. A second allowlist exempts only their three fictional fixture values by regex on the match, so a real key added to either file is detected. A path inside an AND allowlist is applied on its own by gitleaks 8.30.1 (it blinds the whole file), so the exemption is by value only. scripts/test_gitleaks_allowlist.sh now appends a fresh probe key to a temp copy of each file at its real relative path and requires detection, while the unmodified copies stay clean. +- A .gitleaksignore lists the fingerprints of the 19 remaining history hits, all triaged as fictional (YOUR_* placeholders, sha256 hashes, LLM bench outputs naming made-up APIs), so a full history scan reports 0 findings. scripts/test_gitleaks_allowlist.sh now asserts that when run in a git checkout. diff --git a/scripts/test_gitleaks_allowlist.sh b/scripts/test_gitleaks_allowlist.sh index 3f7d631a8..02538010f 100755 --- a/scripts/test_gitleaks_allowlist.sh +++ b/scripts/test_gitleaks_allowlist.sh @@ -1,12 +1,20 @@ #!/usr/bin/env bash # Guard for .gitleaks.toml: the fixture allowlist must not blind gitleaks. -# Skips (exit 0) when gitleaks is absent. Otherwise requires that +# Gitleaks absent: exit 1 when REQUIRE_GITLEAKS=1 is set (the ci.yml gitleaks job sets it), else SKIP (exit 0). +# A shallow checkout or missing .git skips only the history step; with REQUIRE_GITLEAKS=1 it FAILs. Otherwise requires that # (1) a fake AWS key in a temp file outside the allowlisted paths is detected # as an aws-access-token finding, both with the repo config and (control) with -# the default config, and (2) the allowlisted fixture tree scans clean. +# the default config, (2) the allowlisted fixture tree scans clean, and (3) a +# fresh key appended to a copy of each narrowly allowlisted test script, at its +# real relative path, is still detected while the unmodified copy stays clean, +# and (4) a history scan, when in a git checkout, reports 0 findings. set -uo pipefail cd "$(dirname "$0")/.." if ! command -v gitleaks >/dev/null 2>&1; then + if [ "${REQUIRE_GITLEAKS:-}" = 1 ]; then + echo "FAIL: gitleaks not installed (REQUIRE_GITLEAKS=1)" + exit 1 + fi echo "gitleaks not installed: SKIP test_gitleaks_allowlist" exit 0 fi @@ -48,4 +56,49 @@ for p in "${fixtures[@]}"; do exit 1 fi done + +# Narrow allowlist: only the fictional values in the 2 scripts are exempt. +cfg=$PWD/.gitleaks.toml +for f in scripts/test_secret_extra_rules_w5.py scripts/test_secret_precision.py; do + mkdir -p "$tmp/rel/$(dirname "$f")" && cp "$f" "$tmp/rel/$f" || { echo "FAIL: copy $f"; exit 1; } +done +# scan_rel: exit 0 clean, 1 only when rc==1 AND the report holds aws-access-token +# (same check as detects()); any other rc fails the script. +scan_rel() { + rm -f "$tmp/s.json" + (cd "$tmp/rel" && gitleaks dir --no-banner --config "$cfg" --report-format json --report-path "$tmp/s.json" . >/dev/null 2>&1) + local rc=$? + [ "$rc" -eq 0 ] && return 0 + if [ "$rc" -eq 1 ] && grep -q '"RuleID": *"aws-access-token"' "$tmp/s.json" 2>/dev/null; then return 1; fi + echo "FAIL: scan_rel unexpected gitleaks exit $rc without an aws-access-token finding" + exit 1 +} +if ! scan_rel; then + echo "FAIL: unmodified script copies flagged at their real relative paths" + exit 1 +fi +for f in scripts/test_secret_extra_rules_w5.py scripts/test_secret_precision.py; do + printf 'probe_key = "%s"\n' "AKIA""ZXCVBNM3QW""ERTY27" >> "$tmp/rel/$f" + if scan_rel; then + echo "FAIL: real key appended to $f not detected" + exit 1 + fi + cp "$f" "$tmp/rel/$f" +done +# History scan: .gitleaksignore covers the triaged fictional hits, so 0 findings remain. +strict=; [ "${REQUIRE_GITLEAKS:-}" = 1 ] && strict=1 +if [ "$(git rev-parse --is-shallow-repository 2>/dev/null)" = true ]; then + [ -n "$strict" ] && { echo "FAIL: history scan needs full history (shallow checkout, REQUIRE_GITLEAKS=1)"; exit 1; } + echo "SKIP: history scan (shallow checkout)" +elif [ ! -e .git ]; then + [ -n "$strict" ] && { echo "FAIL: history scan needs .git (REQUIRE_GITLEAKS=1)"; exit 1; } + echo "SKIP: history scan (no .git)" +else + rm -f "$tmp/h.json" + gitleaks git --no-banner --log-opts=HEAD -c .gitleaks.toml --report-format json --report-path "$tmp/h.json" >/dev/null 2>&1 + if [ "$?" -ne 0 ] || [ "$(grep -c '"Fingerprint"' "$tmp/h.json" 2>/dev/null)" != 0 ]; then + echo "FAIL: history scan reports findings (extend .gitleaksignore only after triage)" + exit 1 + fi +fi echo "PASS: test_gitleaks_allowlist"