Skip to content

Commit b248e82

Browse files
review_checks: report semgrep as not checked when it scans zero files (#2289)
* review_checks: semgrep scanning 0 files is NOT CHECKED, not a clean pass When semgrep reports an empty paths.scanned list and no findings, the hub now records a not-checked row and prints "NOT CHECKED: semgrep scanned 0 files" instead of treating the chunk as clean. Adds a regression test. Tests: 4/4 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B4dfwe8bGsWUqekBt1dkaf * review_checks: semgrep errors with zero scanned files stay errored A semgrep chunk with errors, empty paths.scanned and no results was forced to rc=5 (zero files), losing the errored not-checked row. Guard that branch with not err. Tests: 3/3 semgrep scripts pass Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
1 parent b0fdae4 commit b248e82

3 files changed

Lines changed: 58 additions & 2 deletions

File tree

‎.claude/skills/deep-code-review/scripts/review_checks.sh‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,7 @@ rc = 3 if err else 0 # 3 = the scan was partial (rows may be empty); 4 = errors
480480
if err and not rows: # errors and no findings: partial when some scanned file raised no error, else exit 4 (nothing usable was scanned)
481481
bad = {e.get("path") if isinstance(e, dict) else None for e in err}
482482
rc = 3 if None not in bad and len(set((d.get("paths") or {}).get("scanned") or [])) > len(bad) else 4
483+
if kind == "semgrep" and not rows and not err and (d.get("paths") or {}).get("scanned") == []: rc = 5 # semgrep scanned none of the files (all filtered or excluded): not a clean pass
483484
if kind == "semgrep" and rc == 3: # name what semgrep could not fully read, at any level (a warn-level parse error is never downgraded to clean): counts by type and level on stderr, one not-checked row per errored path
484485
by, files = collections.Counter(), {} # "<type> (<level>)" -> errors; path -> its distinct "<type>, <level>" pairs. An error with no path is counted, not rowed
485486
for e in err if isinstance(err, list) else [err]:
@@ -642,7 +643,7 @@ if [ ! -e "$sgc" ]; then
642643
if [ -z "${REVIEW_SEMGREP_CONFIG:-}" ] && [ "$sgc" = "$here/../templates/semgrep-taint" ]; then skip semgrep "bundled rules missing at $sgc (reinstall the skill)"
643644
else skip semgrep "REVIEW_SEMGREP_CONFIG must be an existing local path"; fi
644645
elif have semgrep; then
645-
sg=() sgwas=$ran sgok=0 sgerr=0 sgj=() sgb=0 sgrx=0 sgbun=(--config "$sgc"); [ "$lease" != degraded ] || { sgj=(--jobs 1); w1+=" semgrep"; }
646+
sg=() sgwas=$ran sgok=0 sgerr=0 sgzero=0 sgj=() sgb=0 sgrx=0 sgbun=(--config "$sgc"); [ "$lease" != degraded ] || { sgj=(--jobs 1); w1+=" semgrep"; }
646647
while IFS= read -r f; do [ -f "$f" ] && sg+=("$f"); done <<<"$files"
647648
# sgrun CONFIG...: scan one chunk. sgreg=1 (seen by run_one) while the p/default pack (registry or snapshot) is in CONFIG (4 args, not 2), so its timeout is soft: the loop reruns with the bundled rules.
648649
sgrun() { local sgreg=0; [ $# -le 2 ] || sgreg=1; run_one semgrep env HOME="$sgh" XDG_CACHE_HOME="$sgh" SEMGREP_USER_HOME="$sgh" semgrep "$@" ${sgj[@]+"${sgj[@]}"} --json --quiet --metrics=off --disable-version-check -- "${sg[@]:i:200}"; }
@@ -663,14 +664,16 @@ elif have semgrep; then
663664
scan_ok semgrep || continue
664665
[ "$sgb" = 0 ] || case " $ran " in *" semgrep(bundled-only) "*) ;; *) ran+=" semgrep(bundled-only)" ;; esac
665666
e=0; SG_CFG=$sgnorm sjson semgrep "$out" >"$tsv" || e=$?
666-
if [ "$e" = 4 ]; then sgerr=$((sgerr + 1)) # errors on every scanned file and no results: nothing usable from this chunk; judged after the loop
667+
if [ "$e" = 5 ]; then sgzero=$((sgzero + 1)) # a chunk that scanned 0 files: judged after the loop (NOT CHECKED only when no chunk scanned anything)
668+
elif [ "$e" = 4 ]; then sgerr=$((sgerr + 1)) # errors on every scanned file and no results: nothing usable from this chunk; judged after the loop
667669
elif [ "$e" = 0 ] || [ "$e" = 3 ]; then
668670
sgok=1
669671
while IFS=$'\t' read -r p l r m; do add semgrep "$p" "$l" "$r" "$m"; done <"$tsv"
670672
# a "(note)" token in ran is a partial scope: the closing line and the record list it apart from a full run. On exit 3 sjson has already printed the partial line (error counts by type and level) and added the not-checked rows read above
671673
[ "$e" = 0 ] || case " $ran " in *" semgrep(errors) "*) ;; *) ran+=" semgrep(errors)" ;; esac
672674
else skip semgrep "unparseable output"; fi
673675
done
676+
if [ "$sgzero" -gt 0 ] && [ "$sgok" = 0 ] && [ "$sgerr" = 0 ]; then ran=$sgwas; skip semgrep "NOT CHECKED: semgrep scanned 0 files"; fi
674677
if [ "$sgerr" -gt 0 ] && [ "$sgok" = 1 ]; then # other chunks scanned fine: partial coverage, not a voided tool
675678
echo "review_checks: partial: semgrep errored with no findings on $sgerr chunk(s) of up to 200 files (those files were not scanned)" >&2
676679
add semgrep "" 0 not-checked "semgrep errored on $sgerr chunk(s) of up to 200 files; those files were not scanned"
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Registers the review_checks.sh semgrep zero-files test (scanned 0 files is NOT CHECKED, never a clean pass) with CI.
2+
bash scripts/test-review-checks-semgrep-zero-files.sh
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env bash
2+
# review_checks.sh: semgrep NOT CHECKED when every chunk scanned 0 files (paths.scanned empty, no findings): one not-checked row,
3+
# "NOT CHECKED: semgrep scanned 0 files" on stderr, semgrep out of the "ran" list. When another chunk scanned files normally, semgrep
4+
# is not reported as NOT CHECKED. Stub semgrep (one JSON per call) on a hermetic PATH, fictional content only.
5+
# REVIEW_CHECKS=<path> runs another copy of the script (used to prove the test fails against the pre-fix script).
6+
set -euo pipefail
7+
ROOT=$(cd "$(dirname "$0")/.." && pwd)
8+
RC=${REVIEW_CHECKS:-$ROOT/.claude/skills/deep-code-review/scripts/review_checks.sh}
9+
T=$(mktemp -d "${TMPDIR:-/tmp}/sgzero.XXXXXX") && [ -n "$T" ] && [ -d "$T" ] || { echo "no temp dir" >&2; exit 1; }
10+
trap 'cd /; rm -r -- "$T"' EXIT
11+
fail=0
12+
export DCR_CACHE_ROOT="$T/nocache" DCR_USER_BIN= PERUN_HEAVY_DIR="$T/heavy" REVIEW_LEASE_WAIT=0 REVIEW_SEMGREP_REGISTRY=0
13+
unset REVIEW_OSV REVIEW_OSV_DB_DIR OSV_SCANNER_LOCAL_DB_CACHE_DIRECTORY REVIEW_SEMGREP_CONFIG REVIEW_FIXTURE_ROWS REVIEW_TEST_CMD
14+
G() { git -c user.email=t@example.com -c user.name=t "$@"; }
15+
mkdir -p "$T/bin" "$T/sys"
16+
for b in bash env git grep sed head tail cat rm mkdir mktemp dirname basename python3 perl timeout gtimeout jq true sort uniq paste tr cut wc awk date cp mv ls find touch tee readlink sleep cmp nice uname; do
17+
p=$(type -P "$b") || continue; ln -s "$p" "$T/sys/$b" # only the system binaries review_checks.sh needs, never the host PATH
18+
done
19+
# stub: call N (1-based) prints $SG_DIR/N.json
20+
cat >"$T/bin/semgrep" <<'S'
21+
#!/usr/bin/env bash
22+
n=$(cat "$SG_DIR/n" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" >"$SG_DIR/n"; cat "$SG_DIR/$n.json"
23+
S
24+
chmod +x "$T/bin/semgrep"
25+
mkdir -p "$T/repo" && cd "$T/repo" && git init -q . && G commit -q --allow-empty -m base && git branch base
26+
mkdir src; for i in $(seq 1 201); do printf 'x = %s\n' "$i" >"src/f$i.py"; done
27+
G add -A && G commit -q -m change
28+
ZERO='{"results":[],"errors":[],"paths":{"scanned":[]}}'
29+
OKJ='{"results":[],"errors":[],"paths":{"scanned":["src/f1.py"]}}'
30+
# go J1 J2: run the hub (201 changed files = two semgrep chunks of 200 + 1); call 1 prints J1, call 2 prints J2
31+
go() {
32+
mkdir -p "$T/sg"; rm -f "$T/sg/n"; printf '%s\n' "$1" >"$T/sg/1.json"; printf '%s\n' "$2" >"$T/sg/2.json"
33+
PATH="$T/bin:$T/sys" SG_DIR="$T/sg" bash "$RC" --base base >"$T/out" 2>"$T/err" || true
34+
}
35+
eq() { if [ "$1" = "$2" ]; then echo "ok - $3"; else echo "FAIL - $3 (got $1, want $2)"; fail=1; fi; }
36+
nc() { jq '[.[] | select(.tool=="semgrep" and .rule=="not-checked")] | length' "$T/out"; }
37+
msg() { grep -c '^review_checks: not run: semgrep (NOT CHECKED: semgrep scanned 0 files)' "$T/err" || true; }
38+
39+
go "$ZERO" "$ZERO" # every chunk scanned nothing
40+
eq "$(msg)" 1 "all chunks zero: one NOT CHECKED line (not one per chunk)"
41+
eq "$(nc)" 1 "all chunks zero: one semgrep not-checked row"
42+
43+
go "$ZERO" "$OKJ" # one chunk zero, one scanned normally
44+
eq "$(msg)" 0 "mixed chunks: semgrep not reported as scanning 0 files"
45+
eq "$(nc)" 0 "mixed chunks: no semgrep not-checked row"
46+
47+
ERRJ='{"results":[],"errors":[{"type":"Timeout","level":"error","path":"src/f2.py"}],"paths":{"scanned":[]}}'
48+
go "$ERRJ" "$ZERO" # chunk 1 errored with nothing scanned, chunk 2 scanned 0: errors must not read as zero-files
49+
eq "$(msg)" 0 "errors + empty scanned: not reported as scanning 0 files"
50+
eq "$(jq '[.[]|select(.tool=="semgrep" and .rule=="not-checked")]|length' "$T/out")" 1 "errors + empty scanned: errored not-checked row kept"
51+
exit $fail

0 commit comments

Comments
 (0)