Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -840,6 +840,8 @@ jobs:
tar -xzf nuclei-templates.tar.gz --strip-components=1 -C "$HOME/nuclei-templates"
- name: Real-tool canaries
run: REQUIRE_REAL_HUB=1 bash scripts/test-review-checks-real-hub.sh
- name: Tool-only bench (frozen recall floor and false-positive ceiling, no model spend)
run: REQUIRE_REAL_HUB=1 python3 scripts/bench_tool_only.py

# Branch protection requires a check named exactly `gates`: it passes only when every shard job succeeded.
gates:
Expand Down
1 change: 1 addition & 0 deletions .gitleaksignore
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,4 @@ d68fb67ca2aaa36cf2bcc9c4329ed63d8aa48dca:docs/bench/security-gates-1024/secret-l
de66350063b258c7af5c2a56841774ef428abaf4:docs/bench/security-gates-1024/rivals/gitleaks-v8.30.1.toml:aws-amazon-bedrock-api-key-short-lived:232
e42fed4a7e94713d3b51487a45bce4f390b93a90:docs/bench/security-gates-heldout2-1024/secret-decoys.txt:curl-auth-header:23
e42fed4a7e94713d3b51487a45bce4f390b93a90:docs/bench/security-gates-heldout2-1024/secret-leaks.txt:curl-auth-header:29
78bab5190543d0c2a51554b7e427b44083331cb1:bench/tool-only/vuln/secret.py:generic-api-key:2
14 changes: 14 additions & 0 deletions bench/tool-only/answer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"note": "Frozen answer key for scripts/bench_tool_only.py. Keys are file:line under vuln/; values are CWE ids. clean/ holds a fixed twin per file and must stay silent. vuln/secret.py is generated by the scorer (not committed).",
"vuln/sqli.py:5": "CWE-89",
"vuln/cmdi.py:5": "CWE-78",
"vuln/secret.py:2": "CWE-798",
"vuln/pathtrav.py:8": "CWE-22",
"vuln/crypto.py:5": "CWE-327",
"vuln/deser.py:5": "CWE-502",
"vuln/debug.py:6": "CWE-489",
"vuln/requirements.txt:1": "CWE-1104",
"vuln/Dockerfile:2": "CWE-250",
"vuln/.github/workflows/inject.yml:7": "CWE-94",
"vuln/main.tf:3": "CWE-732"
}
9 changes: 9 additions & 0 deletions bench/tool-only/clean/.github/workflows/inject.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
on: pull_request_target
permissions: {}
jobs:
greet:
runs-on: ubuntu-latest
steps:
- env:
TITLE: ${{ github.event.pull_request.title }}
run: echo "$TITLE"
5 changes: 5 additions & 0 deletions bench/tool-only/clean/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
FROM python:3.12-slim
COPY . /app
USER 10001
HEALTHCHECK CMD ["true"]
CMD ["python", "/app/main.py"]
5 changes: 5 additions & 0 deletions bench/tool-only/clean/cmdi.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import subprocess


def ping(host: str):
return subprocess.call(["ping", "-c", "1", host])
5 changes: 5 additions & 0 deletions bench/tool-only/clean/crypto.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import hashlib


def hash_password(pw: str, salt: bytes) -> str:
return hashlib.scrypt(pw.encode(), salt=salt, n=2**14, r=8, p=1).hex()
6 changes: 6 additions & 0 deletions bench/tool-only/clean/debug.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from flask import Flask

app = Flask(__name__)

if __name__ == "__main__":
app.run(host="127.0.0.1")
5 changes: 5 additions & 0 deletions bench/tool-only/clean/deser.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import json


def load(blob: bytes):
return json.loads(blob)
11 changes: 11 additions & 0 deletions bench/tool-only/clean/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
resource "aws_s3_bucket" "assets" {
bucket = "acme-bench-assets"
}

resource "aws_s3_bucket_public_access_block" "assets" {
bucket = aws_s3_bucket.assets.id
block_public_acls = true
block_public_policy = true
ignore_public_acls = true
restrict_public_buckets = true
}
8 changes: 8 additions & 0 deletions bench/tool-only/clean/pathtrav.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
from flask import Flask, request, send_from_directory

app = Flask(__name__)


@app.route("/download")
def download():
return send_from_directory("/srv/files", request.args["name"])
1 change: 1 addition & 0 deletions bench/tool-only/clean/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
tomli==2.0.1
3 changes: 3 additions & 0 deletions bench/tool-only/clean/secret.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
import os

STRIPE_KEY = os.environ["STRIPE_KEY"]
5 changes: 5 additions & 0 deletions bench/tool-only/clean/sqli.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import sqlite3


def find_user(conn: sqlite3.Connection, name: str):
return conn.execute("SELECT * FROM users WHERE name = ?", (name,)).fetchall()
7 changes: 7 additions & 0 deletions bench/tool-only/vuln/.github/workflows/inject.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
on: pull_request_target
permissions: {}
jobs:
greet:
runs-on: ubuntu-latest
steps:
- run: echo "${{ github.event.pull_request.title }}"
4 changes: 4 additions & 0 deletions bench/tool-only/vuln/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
FROM python:3.12-slim
USER root
COPY . /app
CMD ["python", "/app/main.py"]
5 changes: 5 additions & 0 deletions bench/tool-only/vuln/cmdi.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import subprocess


def ping(host: str):
return subprocess.call("ping -c 1 " + host, shell=True)
5 changes: 5 additions & 0 deletions bench/tool-only/vuln/crypto.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import hashlib


def hash_password(pw: str) -> str:
return hashlib.md5(pw.encode()).hexdigest()
6 changes: 6 additions & 0 deletions bench/tool-only/vuln/debug.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from flask import Flask

app = Flask(__name__)

if __name__ == "__main__":
app.run(host="0.0.0.0", debug=True)
5 changes: 5 additions & 0 deletions bench/tool-only/vuln/deser.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import pickle


def load(blob: bytes):
return pickle.loads(blob)
4 changes: 4 additions & 0 deletions bench/tool-only/vuln/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
resource "aws_s3_bucket" "assets" {
bucket = "acme-bench-assets"
acl = "public-read"
}
8 changes: 8 additions & 0 deletions bench/tool-only/vuln/pathtrav.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
from flask import Flask, request, send_file

app = Flask(__name__)


@app.route("/download")
def download():
return send_file("/srv/files/" + request.args["name"])
1 change: 1 addition & 0 deletions bench/tool-only/vuln/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
django==2.2.0
5 changes: 5 additions & 0 deletions bench/tool-only/vuln/sqli.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import sqlite3


def find_user(conn: sqlite3.Connection, name: str):
return conn.execute("SELECT * FROM users WHERE name = '" + name + "'").fetchall()
2 changes: 2 additions & 0 deletions changelog.d/bench-tool-only.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
### Added
- bench/tool-only: 11 seeded fictional CWE fixtures with clean twins and an answer key, scored by scripts/bench_tool_only.py (per-CWE and overall recall, false positives on the clean twins; fails below the frozen recall floor or above the frozen false-positive ceiling; SKIPs when semgrep, gitleaks or trivy is absent). Runs in the real-hub CI job after the tools install.
67 changes: 67 additions & 0 deletions scripts/bench_tool_only.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#!/usr/bin/env python3
"""Frozen, zero-model-spend bench for the review_checks.sh hub.

Copies bench/tool-only/{vuln,clean} into a throwaway git repo, runs `review_checks.sh --full`, and prints JSON:
per-CWE recall, overall recall, and false positives (rows on the clean twins). A row hits an answer.json entry
when it is in the same file and on the seeded line (+/-2) and its rule is not a noise rule (one that also fires on
the clean twins), or is a file-level row (line 0) on the dependency fixture only. Rows of tool `not-checked` are unchecked surface, not findings, and are ignored.

Exit 0 pass or SKIP (a core tool is absent: the floors are measured with every tool installed, as in the ci.yml
real-hub job), 1 on SKIP when REQUIRE_REAL_HUB=1 (CI), 1 when recall < FLOOR_RECALL or false positives > MAX_FALSE_POSITIVES.
Raise the floor only after a measured improvement in CI; never lower it to make a run pass. Side effects: temp dir only.
"""
import json, os, shutil, subprocess, sys, tempfile
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
HUB = ROOT / ".claude/skills/deep-code-review/scripts/review_checks.sh"
CORE = ("semgrep", "gitleaks", "trivy", "bandit", "ruff", "git", "jq")
FLOOR_RECALL = 0.909 # frozen: 10/11 measured locally with semgrep, gitleaks, trivy, bandit, ruff; CWE-22 is caught only by a rule that also fires on its clean twin, so it no longer counts
DEP_FILE = "vuln/requirements.txt" # only fixture where a file-level (line 0) row counts
MAX_FALSE_POSITIVES = 27 # frozen: 27 noise rows on the clean twins measured in the CI real-hub job (full hub: bandit, ruff, trivy, checkov, devskim, semgrep); a local run with fewer tools sees fewer


def run(truth_path=ROOT / "bench/tool-only/answer.json", fixtures=ROOT / "bench/tool-only"):
truth = {k: v for k, v in json.loads(Path(truth_path).read_text()).items() if k != "note"}
with tempfile.TemporaryDirectory() as t:
for d in ("vuln", "clean"):
shutil.copytree(fixtures / d, Path(t) / d)
# generated here so no committed text matches a secret pattern; keep on line 2 (answer.json: vuln/secret.py:2)
(Path(t) / "vuln/secret.py").write_text("# Fictional credential for a bench fixture; not a real account.\nAPI_KEY = \"" + "q7Zk2XvB9mWc4T" + "nRj8LpD3sHf6YgA1eU" + "\"\n")
g = ["git", "-c", "user.name=b", "-c", "user.email=b@example.com"]
for a in (["init", "-q"], ["add", "-A"], ["commit", "-qm", "bench"]):
subprocess.run(g + a, cwd=t, check=True, capture_output=True)
out = subprocess.run(["bash", str(HUB), "--full"], cwd=t, capture_output=True, text=True).stdout
rows = [r for r in json.loads(out or "[]") if r.get("tool") != "not-checked" and r.get("rule") != "not-checked"]
noise = {r["rule"] for r in rows if r["file"].startswith("clean/")} # rules that also fire on clean twins prove nothing
found, per = set(), {}
for key, cwe in truth.items():
f, _, line = key.rpartition(":")
hit = any(r["file"] == f and r["rule"] not in noise and
(abs(r["line"] - int(line)) <= 2 or (r["line"] == 0 and f == DEP_FILE)) for r in rows)
per.setdefault(cwe, [0, 0])[1] += 1
if hit:
found.add(key); per[cwe][0] += 1
fps = sorted({(r["file"], r["line"], r["rule"]) for r in rows if r["file"].startswith("clean/")})
return {"recall": round(len(found) / len(truth), 3), "hits": len(found), "total": len(truth),
"per_cwe_recall": {c: h / n for c, (h, n) in sorted(per.items())},
"missed": sorted(set(truth) - found), "false_positives": len(fps),
"false_positive_rows": [list(x) for x in fps]}


def main():
if not HUB.exists() or any(shutil.which(c) is None for c in CORE):
print("SKIP bench_tool_only: " + ", ".join(c for c in CORE if shutil.which(c) is None) + " not installed")
return int(os.environ.get("REQUIRE_REAL_HUB") == "1")
res = run()
print(json.dumps(res, indent=2))
bad = res["recall"] < FLOOR_RECALL or \
res["false_positives"] > MAX_FALSE_POSITIVES
if bad:
print("FAIL bench_tool_only: recall %s (floor %s), false positives %s (max %s)" % (
res["recall"], FLOOR_RECALL, res["false_positives"], MAX_FALSE_POSITIVES), file=sys.stderr)
return int(bad)


if __name__ == "__main__":
sys.exit(main())
2 changes: 2 additions & 0 deletions scripts/ci-tests.d/bench-tool-only.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Registers scripts/bench_tool_only.py with CI (see the "Per-PR tests" step in ci.yml); it SKIPs (exit 0) without the real scanners and gates recall in the real-hub job.
python3 scripts/bench_tool_only.py
Loading