Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -492,9 +492,32 @@ jobs:
- name: Eval runner guards hold (offline, no model)
run: python3 scripts/run-evals.py --selftest

# Guards .gitleaks.toml: the fixture allowlist must not blind the scanner (full history needed for the history scan).
gitleaks:
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Install gitleaks 8.30.1 (pinned sha256)
run: |
set -euo pipefail
cd "$RUNNER_TEMP"
curl -fsSL -O https://github.com/gitleaks/gitleaks/releases/download/v8.30.1/gitleaks_8.30.1_linux_x64.tar.gz
echo "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb gitleaks_8.30.1_linux_x64.tar.gz" | sha256sum -c -
mkdir -p "$RUNNER_TEMP/bin"
tar -xzf gitleaks_8.30.1_linux_x64.tar.gz -C "$RUNNER_TEMP/bin" gitleaks
echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
- name: Allowlist guard
run: REQUIRE_GITLEAKS=1 bash scripts/test_gitleaks_allowlist.sh

# Branch protection requires a check named exactly `gates`: it passes only when every shard job succeeded.
gates:
needs: [selftests, enforce, perpr, evals, cheap]
needs: [selftests, enforce, perpr, evals, cheap, gitleaks]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 5
Expand Down
17 changes: 15 additions & 2 deletions .gitleaks.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,19 @@ paths = [
'''^docs/bench/secrets-heldout[0-9]+-[0-9]+/(secret-leaks|secret-decoys)\.txt$''',
'''^docs/bench/secrets-heldout[0-9]+-[0-9]+/ledger\.jsonl$''',
'''^docs/bench/g1-matrix/heldout/[^/]+/result\.json$''',
'''^scripts/test_secret_extra_rules_w5\.py$''',
'''^scripts/test_secret_precision\.py$''',
]

# The two secret-test scripts (scripts/test_secret_extra_rules_w5.py and
# scripts/test_secret_precision.py) are exempt only for their fictional fixture
# values, matched by value and not by path, so a real key added to either file is
# still caught. These value regexes apply repo-wide, not only to those two files:
# a line containing one of the values is exempt anywhere. (A path in an AND allowlist is applied on its own by gitleaks
# 8.30.1 and would blind the whole file again.)
[[allowlists]]
description = "Fictional fixture values in the secret test scripts"
regexTarget = "match"
regexes = [
'''eyJhbGciOiJIUzI1NiJ9\.eyJyb2xlIjoiYW5vbiJ9\.''',
'''sk_test_@@A2aMrXbJqxi4ai1aZSzJWHU7''',
'''pk_live_n91DMA5Cj6vP3w01RyvJgcjy''',
]
19 changes: 19 additions & 0 deletions .gitleaksignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
3d4cd69aeadc07ec2527512127bf8874a0691d93:docs/bench/growth-r4-1018/results/outputs/rival/gy052-e92a8bf6-6bfceff8.txt:generic-api-key:40
508000d654136f41e8b02be8e012ec1a7c12244d:docs/bench/differentiator-1019/calibration/answers/PERUN/ca2.txt:curl-auth-header:20
508000d654136f41e8b02be8e012ec1a7c12244d:docs/bench/differentiator-1019/calibration/answers/RIVAL/ca2.txt:curl-auth-header:13
508000d654136f41e8b02be8e012ec1a7c12244d:docs/bench/differentiator-1019/golden/fabrication-rival-ca2.txt:curl-auth-header:13
508000d654136f41e8b02be8e012ec1a7c12244d:docs/bench/differentiator-1019/superseded-first-run/answers/PLAIN/ha4.txt:curl-auth-header:17
508000d654136f41e8b02be8e012ec1a7c12244d:docs/bench/differentiator-1019/superseded-first-run/answers/PLAIN/ha7.txt:stripe-access-token:26
52ebef8644f67a30e39405e47886308d077e5e47:docs/bench/bizops-fab-1019/outputs/cal/EDITED-cd2e78bc/xa2.txt:curl-auth-header:13
52ebef8644f67a30e39405e47886308d077e5e47:docs/bench/bizops-fab-1019/outputs/cal/EDITED-ef4491a1/xa2.txt:curl-auth-header:13
52ebef8644f67a30e39405e47886308d077e5e47:docs/bench/bizops-fab-1019/outputs/cal/EDITED-fb936c4e/xa2.txt:curl-auth-header:14
52ebef8644f67a30e39405e47886308d077e5e47:docs/bench/bizops-fab-1019/outputs/heldout/CURRENT-39af659d/ya2.txt:curl-auth-header:1
52ebef8644f67a30e39405e47886308d077e5e47:docs/bench/bizops-fab-1019/outputs/heldout/EDITED-fb936c4e/ya2.txt:curl-auth-header:11
b28a90f83b67332599d18f13e0a61611980c8b40:docs/bench/bizops-shipped-1018/outputs/shipped/c150-124.txt:generic-api-key:26
c3f14095a3d18bf4c37ab603f1ff0e8d9dcc4d1d:docs/bench/security-gates-heldout-1024/secret-leaks.txt:generic-api-key:16
c3f14095a3d18bf4c37ab603f1ff0e8d9dcc4d1d:docs/bench/security-gates-heldout-1024/secret-leaks.txt:generic-api-key:3
d50de27a76d3fa0b88bd3f41b35fc0692488e226:docs/bench/growth-outcome-r2-1019/ledger.jsonl:generic-api-key:2
d68fb67ca2aaa36cf2bcc9c4329ed63d8aa48dca:docs/bench/security-gates-1024/secret-leaks.txt:generic-api-key:4
de66350063b258c7af5c2a56841774ef428abaf4:docs/bench/security-gates-1024/rivals/gitleaks-v8.30.1.toml:aws-amazon-bedrock-api-key-short-lived:232
e42fed4a7e94713d3b51487a45bce4f390b93a90:docs/bench/security-gates-heldout2-1024/secret-decoys.txt:curl-auth-header:23
e42fed4a7e94713d3b51487a45bce4f390b93a90:docs/bench/security-gates-heldout2-1024/secret-leaks.txt:curl-auth-header:29
3 changes: 3 additions & 0 deletions changelog.d/gitleaks-ci-guard.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Added
- CI now runs scripts/test_gitleaks_allowlist.sh in a `gitleaks` job (full-history checkout, gitleaks 8.30.1 installed from its release tarball verified against a hardcoded sha256, REQUIRE_GITLEAKS=1), and the `gates` aggregate requires it. Before this no workflow called the script.
- The guard is strict (missing gitleaks or shallow history FAILs) only when REQUIRE_GITLEAKS=1 is set, so the per-PR test legs, which do not install gitleaks, SKIP it.
3 changes: 3 additions & 0 deletions changelog.d/gitleaks-narrow-allowlist.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Fixed
- .gitleaks.toml no longer allowlists the whole of scripts/test_secret_extra_rules_w5.py and scripts/test_secret_precision.py by path. A second allowlist exempts only their three fictional fixture values by regex on the match, so a real key added to either file is detected. A path inside an AND allowlist is applied on its own by gitleaks 8.30.1 (it blinds the whole file), so the exemption is by value only. scripts/test_gitleaks_allowlist.sh now appends a fresh probe key to a temp copy of each file at its real relative path and requires detection, while the unmodified copies stay clean.
- A .gitleaksignore lists the fingerprints of the 19 remaining history hits, all triaged as fictional (YOUR_* placeholders, sha256 hashes, LLM bench outputs naming made-up APIs), so a full history scan reports 0 findings. scripts/test_gitleaks_allowlist.sh now asserts that when run in a git checkout.
57 changes: 55 additions & 2 deletions scripts/test_gitleaks_allowlist.sh
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
#!/usr/bin/env bash
# Guard for .gitleaks.toml: the fixture allowlist must not blind gitleaks.
# Skips (exit 0) when gitleaks is absent. Otherwise requires that
# Gitleaks absent: exit 1 when REQUIRE_GITLEAKS=1 is set (the ci.yml gitleaks job sets it), else SKIP (exit 0).
# A shallow checkout or missing .git skips only the history step; with REQUIRE_GITLEAKS=1 it FAILs. Otherwise requires that
# (1) a fake AWS key in a temp file outside the allowlisted paths is detected
# as an aws-access-token finding, both with the repo config and (control) with
# the default config, and (2) the allowlisted fixture tree scans clean.
# the default config, (2) the allowlisted fixture tree scans clean, and (3) a
# fresh key appended to a copy of each narrowly allowlisted test script, at its
# real relative path, is still detected while the unmodified copy stays clean,
# and (4) a history scan, when in a git checkout, reports 0 findings.
set -uo pipefail
cd "$(dirname "$0")/.."
if ! command -v gitleaks >/dev/null 2>&1; then
if [ "${REQUIRE_GITLEAKS:-}" = 1 ]; then
echo "FAIL: gitleaks not installed (REQUIRE_GITLEAKS=1)"
exit 1
fi
echo "gitleaks not installed: SKIP test_gitleaks_allowlist"
exit 0
fi
Expand Down Expand Up @@ -48,4 +56,49 @@ for p in "${fixtures[@]}"; do
exit 1
fi
done

# Narrow allowlist: only the fictional values in the 2 scripts are exempt.
cfg=$PWD/.gitleaks.toml
for f in scripts/test_secret_extra_rules_w5.py scripts/test_secret_precision.py; do
mkdir -p "$tmp/rel/$(dirname "$f")" && cp "$f" "$tmp/rel/$f" || { echo "FAIL: copy $f"; exit 1; }
done
# scan_rel: exit 0 clean, 1 only when rc==1 AND the report holds aws-access-token
# (same check as detects()); any other rc fails the script.
scan_rel() {
rm -f "$tmp/s.json"
(cd "$tmp/rel" && gitleaks dir --no-banner --config "$cfg" --report-format json --report-path "$tmp/s.json" . >/dev/null 2>&1)
local rc=$?
[ "$rc" -eq 0 ] && return 0
if [ "$rc" -eq 1 ] && grep -q '"RuleID": *"aws-access-token"' "$tmp/s.json" 2>/dev/null; then return 1; fi
echo "FAIL: scan_rel unexpected gitleaks exit $rc without an aws-access-token finding"
exit 1
}
if ! scan_rel; then
echo "FAIL: unmodified script copies flagged at their real relative paths"
exit 1
fi
for f in scripts/test_secret_extra_rules_w5.py scripts/test_secret_precision.py; do
printf 'probe_key = "%s"\n' "AKIA""ZXCVBNM3QW""ERTY27" >> "$tmp/rel/$f"
if scan_rel; then
echo "FAIL: real key appended to $f not detected"
exit 1
fi
cp "$f" "$tmp/rel/$f"
done
# History scan: .gitleaksignore covers the triaged fictional hits, so 0 findings remain.
strict=; [ "${REQUIRE_GITLEAKS:-}" = 1 ] && strict=1
if [ "$(git rev-parse --is-shallow-repository 2>/dev/null)" = true ]; then
[ -n "$strict" ] && { echo "FAIL: history scan needs full history (shallow checkout, REQUIRE_GITLEAKS=1)"; exit 1; }
echo "SKIP: history scan (shallow checkout)"
elif [ ! -e .git ]; then
[ -n "$strict" ] && { echo "FAIL: history scan needs .git (REQUIRE_GITLEAKS=1)"; exit 1; }
echo "SKIP: history scan (no .git)"
else
rm -f "$tmp/h.json"
gitleaks git --no-banner --log-opts=HEAD -c .gitleaks.toml --report-format json --report-path "$tmp/h.json" >/dev/null 2>&1
if [ "$?" -ne 0 ] || [ "$(grep -c '"Fingerprint"' "$tmp/h.json" 2>/dev/null)" != 0 ]; then
echo "FAIL: history scan reports findings (extend .gitleaksignore only after triage)"
exit 1
fi
fi
echo "PASS: test_gitleaks_allowlist"
Loading