From 1f401950730195e1701b71f8f31b5a6e71fec687 Mon Sep 17 00:00:00 2001 From: Remi Antczak <140408771+remigiusz-antczak@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:08:07 +0100 Subject: [PATCH 1/2] review_checks: semgrep scanning 0 files is NOT CHECKED, not a clean pass When semgrep reports an empty paths.scanned list and no findings, the hub now records a not-checked row and prints "NOT CHECKED: semgrep scanned 0 files" instead of treating the chunk as clean. Adds a regression test. Tests: 4/4 Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01B4dfwe8bGsWUqekBt1dkaf --- .../deep-code-review/scripts/review_checks.sh | 7 ++- .../review-checks-semgrep-zero-files.sh | 2 + .../test-review-checks-semgrep-zero-files.sh | 46 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 scripts/ci-tests.d/review-checks-semgrep-zero-files.sh create mode 100755 scripts/test-review-checks-semgrep-zero-files.sh diff --git a/.claude/skills/deep-code-review/scripts/review_checks.sh b/.claude/skills/deep-code-review/scripts/review_checks.sh index 563f0fdb4..088db01dc 100755 --- a/.claude/skills/deep-code-review/scripts/review_checks.sh +++ b/.claude/skills/deep-code-review/scripts/review_checks.sh @@ -480,6 +480,7 @@ rc = 3 if err else 0 # 3 = the scan was partial (rows may be empty); 4 = errors if err and not rows: # errors and no findings: partial when some scanned file raised no error, else exit 4 (nothing usable was scanned) bad = {e.get("path") if isinstance(e, dict) else None for e in err} rc = 3 if None not in bad and len(set((d.get("paths") or {}).get("scanned") or [])) > len(bad) else 4 +if kind == "semgrep" and not rows and (d.get("paths") or {}).get("scanned") == []: rc = 5 # semgrep scanned none of the files (all filtered or excluded): not a clean pass if kind == "semgrep" and rc == 3: # name what semgrep could not fully read, at any level (a warn-level parse error is never downgraded to clean): counts by type and level on stderr, one not-checked row per errored path by, files = collections.Counter(), {} # " ()" -> errors; path -> its distinct ", " pairs. An error with no path is counted, not rowed for e in err if isinstance(err, list) else [err]: @@ -642,7 +643,7 @@ if [ ! -e "$sgc" ]; then if [ -z "${REVIEW_SEMGREP_CONFIG:-}" ] && [ "$sgc" = "$here/../templates/semgrep-taint" ]; then skip semgrep "bundled rules missing at $sgc (reinstall the skill)" else skip semgrep "REVIEW_SEMGREP_CONFIG must be an existing local path"; fi elif have semgrep; then - sg=() sgwas=$ran sgok=0 sgerr=0 sgj=() sgb=0 sgrx=0 sgbun=(--config "$sgc"); [ "$lease" != degraded ] || { sgj=(--jobs 1); w1+=" semgrep"; } + sg=() sgwas=$ran sgok=0 sgerr=0 sgzero=0 sgj=() sgb=0 sgrx=0 sgbun=(--config "$sgc"); [ "$lease" != degraded ] || { sgj=(--jobs 1); w1+=" semgrep"; } while IFS= read -r f; do [ -f "$f" ] && sg+=("$f"); done <<<"$files" # sgrun CONFIG...: scan one chunk. sgreg=1 (seen by run_one) while the p/default pack (registry or snapshot) is in CONFIG (4 args, not 2), so its timeout is soft: the loop reruns with the bundled rules. sgrun() { local sgreg=0; [ $# -le 2 ] || sgreg=1; run_one semgrep env HOME="$sgh" XDG_CACHE_HOME="$sgh" SEMGREP_USER_HOME="$sgh" semgrep "$@" ${sgj[@]+"${sgj[@]}"} --json --quiet --metrics=off --disable-version-check -- "${sg[@]:i:200}"; } @@ -663,7 +664,8 @@ elif have semgrep; then scan_ok semgrep || continue [ "$sgb" = 0 ] || case " $ran " in *" semgrep(bundled-only) "*) ;; *) ran+=" semgrep(bundled-only)" ;; esac e=0; SG_CFG=$sgnorm sjson semgrep "$out" >"$tsv" || e=$? - if [ "$e" = 4 ]; then sgerr=$((sgerr + 1)) # errors on every scanned file and no results: nothing usable from this chunk; judged after the loop + if [ "$e" = 5 ]; then sgzero=$((sgzero + 1)) # a chunk that scanned 0 files: judged after the loop (NOT CHECKED only when no chunk scanned anything) + elif [ "$e" = 4 ]; then sgerr=$((sgerr + 1)) # errors on every scanned file and no results: nothing usable from this chunk; judged after the loop elif [ "$e" = 0 ] || [ "$e" = 3 ]; then sgok=1 while IFS=$'\t' read -r p l r m; do add semgrep "$p" "$l" "$r" "$m"; done <"$tsv" @@ -671,6 +673,7 @@ elif have semgrep; then [ "$e" = 0 ] || case " $ran " in *" semgrep(errors) "*) ;; *) ran+=" semgrep(errors)" ;; esac else skip semgrep "unparseable output"; fi done + if [ "$sgzero" -gt 0 ] && [ "$sgok" = 0 ] && [ "$sgerr" = 0 ]; then ran=$sgwas; skip semgrep "NOT CHECKED: semgrep scanned 0 files"; fi if [ "$sgerr" -gt 0 ] && [ "$sgok" = 1 ]; then # other chunks scanned fine: partial coverage, not a voided tool echo "review_checks: partial: semgrep errored with no findings on $sgerr chunk(s) of up to 200 files (those files were not scanned)" >&2 add semgrep "" 0 not-checked "semgrep errored on $sgerr chunk(s) of up to 200 files; those files were not scanned" diff --git a/scripts/ci-tests.d/review-checks-semgrep-zero-files.sh b/scripts/ci-tests.d/review-checks-semgrep-zero-files.sh new file mode 100644 index 000000000..1bd1a59d0 --- /dev/null +++ b/scripts/ci-tests.d/review-checks-semgrep-zero-files.sh @@ -0,0 +1,2 @@ +# Registers the review_checks.sh semgrep zero-files test (scanned 0 files is NOT CHECKED, never a clean pass) with CI. +bash scripts/test-review-checks-semgrep-zero-files.sh diff --git a/scripts/test-review-checks-semgrep-zero-files.sh b/scripts/test-review-checks-semgrep-zero-files.sh new file mode 100755 index 000000000..569d56ebf --- /dev/null +++ b/scripts/test-review-checks-semgrep-zero-files.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# review_checks.sh: semgrep NOT CHECKED when every chunk scanned 0 files (paths.scanned empty, no findings): one not-checked row, +# "NOT CHECKED: semgrep scanned 0 files" on stderr, semgrep out of the "ran" list. When another chunk scanned files normally, semgrep +# is not reported as NOT CHECKED. Stub semgrep (one JSON per call) on a hermetic PATH, fictional content only. +# REVIEW_CHECKS= runs another copy of the script (used to prove the test fails against the pre-fix script). +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +RC=${REVIEW_CHECKS:-$ROOT/.claude/skills/deep-code-review/scripts/review_checks.sh} +T=$(mktemp -d "${TMPDIR:-/tmp}/sgzero.XXXXXX") && [ -n "$T" ] && [ -d "$T" ] || { echo "no temp dir" >&2; exit 1; } +trap 'cd /; rm -r -- "$T"' EXIT +fail=0 +export DCR_CACHE_ROOT="$T/nocache" DCR_USER_BIN= PERUN_HEAVY_DIR="$T/heavy" REVIEW_LEASE_WAIT=0 REVIEW_SEMGREP_REGISTRY=0 +unset REVIEW_OSV REVIEW_OSV_DB_DIR OSV_SCANNER_LOCAL_DB_CACHE_DIRECTORY REVIEW_SEMGREP_CONFIG REVIEW_FIXTURE_ROWS REVIEW_TEST_CMD +G() { git -c user.email=t@example.com -c user.name=t "$@"; } +mkdir -p "$T/bin" "$T/sys" +for b in bash env git grep sed head tail cat rm mkdir mktemp dirname basename python3 perl timeout gtimeout jq true sort uniq paste tr cut wc awk date cp mv ls find touch tee readlink sleep cmp nice uname; do + p=$(type -P "$b") || continue; ln -s "$p" "$T/sys/$b" # only the system binaries review_checks.sh needs, never the host PATH +done +# stub: call N (1-based) prints $SG_DIR/N.json +cat >"$T/bin/semgrep" <<'S' +#!/usr/bin/env bash +n=$(cat "$SG_DIR/n" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" >"$SG_DIR/n"; cat "$SG_DIR/$n.json" +S +chmod +x "$T/bin/semgrep" +mkdir -p "$T/repo" && cd "$T/repo" && git init -q . && G commit -q --allow-empty -m base && git branch base +mkdir src; for i in $(seq 1 201); do printf 'x = %s\n' "$i" >"src/f$i.py"; done +G add -A && G commit -q -m change +ZERO='{"results":[],"errors":[],"paths":{"scanned":[]}}' +OKJ='{"results":[],"errors":[],"paths":{"scanned":["src/f1.py"]}}' +# go J1 J2: run the hub (201 changed files = two semgrep chunks of 200 + 1); call 1 prints J1, call 2 prints J2 +go() { + mkdir -p "$T/sg"; rm -f "$T/sg/n"; printf '%s\n' "$1" >"$T/sg/1.json"; printf '%s\n' "$2" >"$T/sg/2.json" + PATH="$T/bin:$T/sys" SG_DIR="$T/sg" bash "$RC" --base base >"$T/out" 2>"$T/err" || true +} +eq() { if [ "$1" = "$2" ]; then echo "ok - $3"; else echo "FAIL - $3 (got $1, want $2)"; fail=1; fi; } +nc() { jq '[.[] | select(.tool=="semgrep" and .rule=="not-checked")] | length' "$T/out"; } +msg() { grep -c '^review_checks: not run: semgrep (NOT CHECKED: semgrep scanned 0 files)' "$T/err" || true; } + +go "$ZERO" "$ZERO" # every chunk scanned nothing +eq "$(msg)" 1 "all chunks zero: one NOT CHECKED line (not one per chunk)" +eq "$(nc)" 1 "all chunks zero: one semgrep not-checked row" + +go "$ZERO" "$OKJ" # one chunk zero, one scanned normally +eq "$(msg)" 0 "mixed chunks: semgrep not reported as scanning 0 files" +eq "$(nc)" 0 "mixed chunks: no semgrep not-checked row" +exit $fail From 3f4c66b73936019d0a643621b7d8ccb68b9888aa Mon Sep 17 00:00:00 2001 From: Remi Antczak <140408771+remigiusz-antczak@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:14:50 +0100 Subject: [PATCH 2/2] review_checks: semgrep errors with zero scanned files stay errored A semgrep chunk with errors, empty paths.scanned and no results was forced to rc=5 (zero files), losing the errored not-checked row. Guard that branch with not err. Tests: 3/3 semgrep scripts pass Co-Authored-By: Claude Opus 5.5 --- .claude/skills/deep-code-review/scripts/review_checks.sh | 2 +- scripts/test-review-checks-semgrep-zero-files.sh | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.claude/skills/deep-code-review/scripts/review_checks.sh b/.claude/skills/deep-code-review/scripts/review_checks.sh index 088db01dc..be57f3c14 100755 --- a/.claude/skills/deep-code-review/scripts/review_checks.sh +++ b/.claude/skills/deep-code-review/scripts/review_checks.sh @@ -480,7 +480,7 @@ rc = 3 if err else 0 # 3 = the scan was partial (rows may be empty); 4 = errors if err and not rows: # errors and no findings: partial when some scanned file raised no error, else exit 4 (nothing usable was scanned) bad = {e.get("path") if isinstance(e, dict) else None for e in err} rc = 3 if None not in bad and len(set((d.get("paths") or {}).get("scanned") or [])) > len(bad) else 4 -if kind == "semgrep" and not rows and (d.get("paths") or {}).get("scanned") == []: rc = 5 # semgrep scanned none of the files (all filtered or excluded): not a clean pass +if kind == "semgrep" and not rows and not err and (d.get("paths") or {}).get("scanned") == []: rc = 5 # semgrep scanned none of the files (all filtered or excluded): not a clean pass if kind == "semgrep" and rc == 3: # name what semgrep could not fully read, at any level (a warn-level parse error is never downgraded to clean): counts by type and level on stderr, one not-checked row per errored path by, files = collections.Counter(), {} # " ()" -> errors; path -> its distinct ", " pairs. An error with no path is counted, not rowed for e in err if isinstance(err, list) else [err]: diff --git a/scripts/test-review-checks-semgrep-zero-files.sh b/scripts/test-review-checks-semgrep-zero-files.sh index 569d56ebf..86617a797 100755 --- a/scripts/test-review-checks-semgrep-zero-files.sh +++ b/scripts/test-review-checks-semgrep-zero-files.sh @@ -43,4 +43,9 @@ eq "$(nc)" 1 "all chunks zero: one semgrep not-checked row" go "$ZERO" "$OKJ" # one chunk zero, one scanned normally eq "$(msg)" 0 "mixed chunks: semgrep not reported as scanning 0 files" eq "$(nc)" 0 "mixed chunks: no semgrep not-checked row" + +ERRJ='{"results":[],"errors":[{"type":"Timeout","level":"error","path":"src/f2.py"}],"paths":{"scanned":[]}}' +go "$ERRJ" "$ZERO" # chunk 1 errored with nothing scanned, chunk 2 scanned 0: errors must not read as zero-files +eq "$(msg)" 0 "errors + empty scanned: not reported as scanning 0 files" +eq "$(jq '[.[]|select(.tool=="semgrep" and .rule=="not-checked")]|length' "$T/out")" 1 "errors + empty scanned: errored not-checked row kept" exit $fail