From d3d1dbc49ad6c5779aa2be4cb9f0ffa0a89d188f Mon Sep 17 00:00:00 2001 From: Remi Antczak <140408771+remigiusz-antczak@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:31:46 +0100 Subject: [PATCH] dependency_currency: collapse transitive npm outdated rows per lockfile On an 83-repo audit corpus, 2,358 of 2,664 dep-currency "outdated" rows were transitive packages in npm lockfiles. They are not directly actionable, because they move when the direct dependencies that pull them in are upgraded, and they buried the direct and vulnerable rows. TSV output now prints the outdated rows of transitive npm packages as one outdated row per lockfile when there are more than 3. The row gives the count, how many are a major version behind, and the first 3 names; --json still lists every one. The lockfiles covered are package-lock.json, npm-shrinkwrap.json, yarn.lock and pnpm-lock.yaml. The direct set comes from three sources: - the sibling package.json; - the root and workspace entries of a package-lock v2/v3; - the pnpm importers. Nothing collapses in these cases: - the direct set cannot be read; - a yarn.lock sits beside a package.json that declares workspaces, because workspace manifests are not read. Direct, vulnerable, kev and eol rows are unchanged. Collapsed messages have tabs and newlines replaced, like every other TSV row. A failure while reading the direct set falls back to no collapse rather than dropping the TSV output. On a real corpus repo, outdated rows go from 699 to 60, and the 48 vulnerable and 2 eol rows are unchanged. Tests: 346/346 Co-Authored-By: Claude Opus 5.5 --- .../scripts/dependency_currency.py | 50 ++++++ SHA256SUMS | 6 +- .../dep-currency-transitive-collapse.md | 2 + ...dependency-currency-transitive-collapse.sh | 3 + ...dependency_currency_transitive_collapse.py | 161 ++++++++++++++++++ 5 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 changelog.d/dep-currency-transitive-collapse.md create mode 100644 scripts/ci-tests.d/dependency-currency-transitive-collapse.sh create mode 100644 scripts/test_dependency_currency_transitive_collapse.py diff --git a/.claude/skills/deep-code-review/scripts/dependency_currency.py b/.claude/skills/deep-code-review/scripts/dependency_currency.py index b9709c5ee..2e2f23e5a 100644 --- a/.claude/skills/deep-code-review/scripts/dependency_currency.py +++ b/.claude/skills/deep-code-review/scripts/dependency_currency.py @@ -41,6 +41,7 @@ unpinned, vendored or not-checked. The message carries name, current, latest, behind, vulnerable ids and the upgrade target. Not-checked rows that share file, ecosystem and note collapse to one row when there are more than 3 (rule not-checked, smallest line, message `N packages not checked in this file (first 3: a, b, c; --json lists every one): `); 3 or fewer print unchanged. +Outdated rows of transitive npm packages (lockfile names no sibling package.json or lock root declares directly) collapse the same way per lockfile when more than 3, into one outdated row (smallest line, first 3 names, a major-behind count). `--json` prints every row (also current ones) with those fields, never collapsed. Exit 0 always; the caller reads the rows. KNOWN CEILINGS @@ -1433,6 +1434,37 @@ def message(r): return "; ".join(p).replace("\t", " ").replace("\n", " ") +NPM_LOCKS = ("package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "pnpm-lock.yaml") +NPM_DEP_KEYS = ("dependencies", "devDependencies", "optionalDependencies", "peerDependencies") + + +def npm_direct(root, lock): + """Names a npm lockfile declares directly: its sibling package.json plus, for package-lock v2/v3 the deps of every workspace/root `packages` + entry (key without node_modules/) and, for pnpm, the importers. None when neither source can be read, or when a yarn.lock sits beside a + package.json with "workspaces" (caller collapses nothing either way). + # ponytail: yarn workspace package.jsons are not read, so a yarn workspaces repo never collapses (folding them would hide direct deps).""" + out, have = set(), False + try: + txt = read(root, os.path.join(os.path.dirname(lock), "package.json")) + if txt is not None: + have = True + pj = json.loads(txt) + if os.path.basename(lock) == "yarn.lock" and "workspaces" in pj: + return None + out.update(n for k in NPM_DEP_KEYS for n in pj.get(k) or {}) + ltxt = read(root, lock) + if os.path.basename(lock) == "pnpm-lock.yaml": + for direct in p_pnpm_root(ltxt).values(): + out.update(direct) + elif os.path.basename(lock) in ("package-lock.json", "npm-shrinkwrap.json"): + for k, e in (json.loads(ltxt).get("packages") or {}).items(): + if "node_modules/" not in k: + out.update(n for kk in NPM_DEP_KEYS for n in e.get(kk) or {}) + except Exception: # noqa: BLE001 a crash here would drop every TSV row + return None # unreadable or malformed (bad JSON, >5 MB, odd shape): fail safe, no collapse + return out if have or out else None + + def run(root, files, today, budget=45.0): """Collect, look up and return the sorted row dicts (every row, current ones included).""" global DEADLINE @@ -1505,9 +1537,27 @@ def main(argv=None): for r in rows: if rule_of(r) == "not-checked": groups.setdefault((r["file"], r["kind"], r["note"]), []).append(r) + tgroups, direct_of = {}, {} # outdated transitive npm rows per lockfile; direct names per lockfile (None: collapse nothing) + for r in rows: + if rule_of(r) == "outdated" and r["kind"] == "npm" and os.path.basename(r["file"]) in NPM_LOCKS: + if r["file"] not in direct_of: + direct_of[r["file"]] = npm_direct(a.root, r["file"]) + if direct_of[r["file"]] is not None and r["name"] not in direct_of[r["file"]]: + tgroups.setdefault(r["file"], []).append(r) + tcoll = {id(x) for g in tgroups.values() if len(g) > 3 for x in g} + tdone = set() done = set() for r in rows: rule = rule_of(r) + if id(r) in tcoll: # transitive outdated rows of one lockfile: more than 3 print as one row at the smallest line + if r["file"] not in tdone: + tdone.add(r["file"]) + g = tgroups[r["file"]] + first = ", ".join(x["name"] for x in g[:3]) + msg = (f"{len(g)} transitive npm packages outdated in this lockfile ({sum('major' in x['behind'] for x in g)} a major behind; " + f"first 3: {first}; --json lists every one): upgrade the direct dependencies that pull them in") + print(f"{r['file'].replace(chr(9), ' ').replace(chr(10), ' ')}\t{min(x['line'] for x in g)}\t{rule}\t{msg.replace(chr(9), ' ').replace(chr(10), ' ')}") + continue key = (r["file"], r["kind"], r["note"]) g = groups.get(key, []) if rule == "not-checked" else [] if rule == "current" or (len(g) > 3 and key in done): diff --git a/SHA256SUMS b/SHA256SUMS index eacc8ef0f..29b2b1854 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -366,7 +366,7 @@ f8d10be1934b7e69ab1eb19f5aef2bbfd21e16676cbb6cdcd27aaff476f22e9a .claude/skills d682969dcd1adb66f2fcfb986dfe3b3c3090a0ea6adf35847c31dce84d6bac32 .claude/skills/deep-code-review/scripts/cost_limit_map.py c5b40f3f0d3fb174e6c9490150c7b85723de33f95d3fab87d60214d4939401a8 .claude/skills/deep-code-review/scripts/coverage_ledger.py c724de8ef9b9a028c7bd5e5c4198caf92bb1bad9a579deb3e6c68d7f413c2b8e .claude/skills/deep-code-review/scripts/csrf_cookie_map.py -4af7fdb3f0914357d21f87f965b931d4a9c6098a969e4538a6cac76a466c9f00 .claude/skills/deep-code-review/scripts/dependency_currency.py +4a06df507430762add14cd677a4b60fa4dd06a7549ad9da0baf4eecf5b555ecd .claude/skills/deep-code-review/scripts/dependency_currency.py 36c1447a65135496c519397c04c6a74ef066bd01399d593907e598b8d2038498 .claude/skills/deep-code-review/scripts/derived_numbers.py 662031286d4a29aff2e1aaa5cb4fa287ab99f9c7b5124085ea47da3dc66622cc .claude/skills/deep-code-review/scripts/design_drift.py 254f1016d76feb5fc2c26925b74ccc1158343d5075efcf875ef1a60305bab827 .claude/skills/deep-code-review/scripts/dynamic_probe.py @@ -407,9 +407,9 @@ f735943637229b05b0ee249d118146720a07cfd690571c1fb2c06d0347fd5b38 .claude/skills b01cae69d80ca560cc414c1c70ef0ddfdbb910da5b68df4e9d46cf0daf3f8b16 .claude/skills/deep-code-review/scripts/provision_tools.sh e1d1d2eab715ef280b8c211a9da15ebe6b87969de89d7028d7f1cc82d64fc1f0 .claude/skills/deep-code-review/scripts/reaper_lint.py 7849fac82d7d6bb3f8e178ebf3149504feec70ea09c6ef8e21c8ce294c290b5d .claude/skills/deep-code-review/scripts/report_contract_lint.py -f1d9c65a46d038e3387455910ac6c9652cda78ce9905dc2e62b3d87349d06578 .claude/skills/deep-code-review/scripts/review_checks.sh +d5a6d773613413bd0f1cf592bdea78688367dce9db6b02c6b227b247358ae69d .claude/skills/deep-code-review/scripts/review_checks.sh 8c87c02a050a975e453b52ecacbaf01481c7134466a9c67feaf409d22e36d9f7 .claude/skills/deep-code-review/scripts/review_feedback.py -326898c4420a91a53b5fb8c59b63637e46e07e6583797bb570e3600004b8e6b2 .claude/skills/deep-code-review/scripts/route_auth_map.py +fc5a897a11098726503bd003ec21a2c4fa873a11408c11490981ac1db5a0f382 .claude/skills/deep-code-review/scripts/route_auth_map.py 859c67d6a6a781c873f47d5662d403da2321da53e6994ca3d0ccdbf74afd39ac .claude/skills/deep-code-review/scripts/score_review.py e9dfa337ad16d0022a3b7a87e0ba49acb5c2c9c0a5ee24396ff80523f2f8fa1a .claude/skills/deep-code-review/scripts/security_verify.py 90bbc7f05448c6606824423595ab1c43ee5724699c1fbe7f8f5be523be3e1b2f .claude/skills/deep-code-review/scripts/sibling_sweep.py diff --git a/changelog.d/dep-currency-transitive-collapse.md b/changelog.d/dep-currency-transitive-collapse.md new file mode 100644 index 000000000..e3feb009e --- /dev/null +++ b/changelog.d/dep-currency-transitive-collapse.md @@ -0,0 +1,2 @@ +### Changed +- `dependency_currency.py` TSV output collapses outdated transitive npm packages (lockfile names that no sibling `package.json` or lock root declares directly) into one outdated row per lockfile when there are more than 3, naming the first 3 and counting the major-behind ones. Direct rows, vulnerable rows, non-npm files and `--json` (every row) print unchanged. diff --git a/scripts/ci-tests.d/dependency-currency-transitive-collapse.sh b/scripts/ci-tests.d/dependency-currency-transitive-collapse.sh new file mode 100644 index 000000000..42d6cc949 --- /dev/null +++ b/scripts/ci-tests.d/dependency-currency-transitive-collapse.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Registers the dependency_currency.py TSV transitive-npm collapse tests: more than 3 outdated rows from one npm lockfile whose names no sibling package.json declares directly print as one row; direct rows, 3 or fewer, no readable package.json, vulnerable rows and --json (every row) unchanged. Offline (dep_row and osv_pass stubbed). +python3 scripts/test_dependency_currency_transitive_collapse.py diff --git a/scripts/test_dependency_currency_transitive_collapse.py b/scripts/test_dependency_currency_transitive_collapse.py new file mode 100644 index 000000000..816b7aee7 --- /dev/null +++ b/scripts/test_dependency_currency_transitive_collapse.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Regression tests for the TSV collapse of outdated transitive npm rows: more than 3 outdated rows from one npm lockfile whose names no sibling +package.json declares directly print as one outdated row (smallest line, first 3 names, major-behind count); direct rows, 3 or fewer, lockfiles +without a readable package.json (or a yarn.lock beside a "workspaces" package.json), vulnerable rows and --json (every row) stay unchanged. Offline: dep_row and osv_pass are stubbed.""" +import contextlib, importlib.util, io, json, os, tempfile, unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +spec = importlib.util.spec_from_file_location("dc", os.path.join(HERE, "..", ".claude/skills/deep-code-review/scripts/dependency_currency.py")) +dc = importlib.util.module_from_spec(spec) +spec.loader.exec_module(dc) + +MINOR, MAJOR = "2 releases, minor", "3 releases, major" + + +def npm_lock(names): + """A package-lock v3 text listing node_modules/ at 1.0.0 for every name.""" + pk = {"": {}} + pk.update({f"node_modules/{n}": {"version": "1.0.0"} for n in names}) + return json.dumps({"lockfileVersion": 3, "packages": pk}) + + +def make_root(files): + """Temp repo with files (path -> text); returns its path.""" + d = tempfile.mkdtemp() + for f, t in files.items(): + os.makedirs(os.path.dirname(os.path.join(d, f)) or d, exist_ok=True) + with open(os.path.join(d, f), "w") as fh: + fh.write(t) + return d + + +def stub_dep_row(plan): + """Replacement for dc.dep_row: name -> (behind, line) from plan, always outdated to 2.0.0.""" + def fake(f, eco, name, cur, today): + bh, line = plan[name] + return dc.row("", name, eco, "outdated", cur, "2.0.0", bh, "2.0.0", line=line) + return fake + + +def stub_osv(vulnerable): + """Replacement for dc.osv_pass: marks the names in vulnerable as vulnerable, leaves the rest clean.""" + def fake(rows): + for r in rows: + if r["name"] in vulnerable: + r["vulnerable"] = ["GHSA-fictional-0001"] + return fake + + +def run_main(root, files, plan, extra=(), vulnerable=()): + """Run dc.main over the given files with stubs; return stdout lines (or the parsed JSON when --json).""" + out = io.StringIO() + with mock.patch.object(dc, "dep_row", stub_dep_row(plan)), mock.patch.object(dc, "osv_pass", stub_osv(set(vulnerable))), \ + mock.patch("sys.stdin", io.StringIO("\n".join(files) + "\n")), contextlib.redirect_stdout(out): + dc.main(["--root", root, "--stdin", "--today", "2026-11-08", *extra]) + text = out.getvalue() + return json.loads(text) if "--json" in extra else [ln for ln in text.splitlines() if ln] + + +class TransitiveCollapse(unittest.TestCase): + def setUp(self): + self.plan = {} + + def project(self, direct, transitive, pkg=True): + """Write package.json (direct deps, when pkg) and a lock listing direct + transitive names; plans each name outdated, lines count up.""" + names = list(direct) + list(transitive) + for i, n in enumerate(names): + self.plan[n] = (MAJOR if i % 2 else MINOR, 20 + i) + files = {"package-lock.json": npm_lock(names)} + if pkg: + files["package.json"] = json.dumps({"dependencies": {n: "^1.0.0" for n in direct}}) + return make_root(files) + + def test_more_than_three_transitive_collapse_to_one_row(self): + root = self.project(["direct-a"], ["pkg-01", "pkg-02", "pkg-03", "pkg-04", "pkg-05"]) + lines = run_main(root, ["package-lock.json"], self.plan) + self.assertEqual(len(lines), 2, lines) + f, line, rule, msg = next(ln for ln in lines if "transitive" in ln).split("\t") + self.assertEqual((f, rule), ("package-lock.json", "outdated")) + self.assertEqual(line, "21") # smallest transitive line (direct-a took line 20) + self.assertEqual(msg, "5 transitive npm packages outdated in this lockfile (3 a major behind; first 3: pkg-01, pkg-02, pkg-03; " + "--json lists every one): upgrade the direct dependencies that pull them in") + + def test_direct_row_stays_separate(self): + root = self.project(["direct-a"], ["pkg-01", "pkg-02", "pkg-03", "pkg-04"]) + lines = run_main(root, ["package-lock.json"], self.plan) + self.assertEqual(len(lines), 2, lines) + self.assertTrue(any(ln.split("\t")[3].startswith("npm direct-a") for ln in lines), lines) + self.assertTrue(any("4 transitive npm packages" in ln for ln in lines), lines) + + def test_three_or_fewer_transitive_stay_unchanged(self): + root = self.project(["direct-a"], ["pkg-01", "pkg-02", "pkg-03"]) + lines = run_main(root, ["package-lock.json"], self.plan) + self.assertEqual(len(lines), 4, lines) + self.assertTrue(all("transitive npm packages" not in ln for ln in lines), lines) + + def test_no_package_json_no_collapse(self): + root = self.project([], ["pkg-01", "pkg-02", "pkg-03", "pkg-04", "pkg-05"], pkg=False) + lines = run_main(root, ["package-lock.json"], self.plan) + self.assertEqual(len(lines), 5, lines) + self.assertTrue(all("transitive npm packages" not in ln for ln in lines), lines) + + def test_vulnerable_row_never_collapsed(self): + root = self.project(["direct-a"], ["pkg-01", "pkg-02", "pkg-03", "pkg-04", "pkg-05"]) + lines = run_main(root, ["package-lock.json"], self.plan, vulnerable=("pkg-02",)) + self.assertEqual(len(lines), 3, lines) # direct-a row, pkg-02 vulnerable row, 4 transitive collapsed + vuln = [ln for ln in lines if ln.split("\t")[2] == "vulnerable"] + self.assertEqual(len(vuln), 1, lines) + self.assertIn("pkg-02", vuln[0]) + self.assertIn("4 transitive npm packages", "\n".join(lines)) + + def test_json_keeps_every_row(self): + names = ["pkg-01", "pkg-02", "pkg-03", "pkg-04", "pkg-05"] + root = self.project(["direct-a"], names) + rows = run_main(root, ["package-lock.json"], self.plan, extra=("--json",)) + listed = sorted(r["name"] for r in rows if r["status"] == "outdated" and r["name"] in names) + self.assertEqual(listed, names) + self.assertTrue(all(r["rule"] == "outdated" for r in rows if r["name"] in names)) + + def yarn_project(self, direct, transitive, pj_text): + """Write a yarn.lock listing direct + transitive names and the given package.json text; plans each name outdated.""" + names = list(direct) + list(transitive) + for i, n in enumerate(names): + self.plan[n] = (MINOR, 20 + i) + lock = "".join(f'{n}@^1.0.0:\n version "1.0.0"\n\n' for n in names) + return make_root({"yarn.lock": lock, "package.json": pj_text}) + + def test_yarn_lock_sibling_package_json_collapses(self): + root = self.yarn_project(["direct-a"], ["pkg-01", "pkg-02", "pkg-03", "pkg-04", "pkg-05"], + json.dumps({"dependencies": {"direct-a": "^1.0.0"}})) + lines = run_main(root, ["yarn.lock"], self.plan) + self.assertEqual(len(lines), 2, lines) + self.assertTrue(any(ln.split("\t")[3].startswith("npm direct-a") for ln in lines), lines) + self.assertTrue(any(ln.startswith("yarn.lock\t21\toutdated\t5 transitive npm packages") for ln in lines), lines) + + def test_yarn_workspaces_never_collapse(self): + root = self.yarn_project(["direct-a"], ["pkg-01", "pkg-02", "pkg-03", "pkg-04", "pkg-05"], + json.dumps({"workspaces": ["packages/*"], "dependencies": {"direct-a": "^1.0.0"}})) + lines = run_main(root, ["yarn.lock"], self.plan) + self.assertEqual(len(lines), 6, lines) # 1 direct + 5 transitive rows, nothing folded + self.assertTrue(all("transitive npm packages" not in ln for ln in lines), lines) + + def test_transitive_name_with_tab_is_one_four_field_line(self): + root = self.project(["direct-a"], ["pkg\t01", "pkg-02", "pkg-03", "pkg-04"]) + lines = run_main(root, ["package-lock.json"], self.plan) + collapsed = [ln for ln in lines if "transitive" in ln] + self.assertEqual(len(collapsed), 1, lines) + self.assertEqual(len(collapsed[0].split("\t")), 4, collapsed) + self.assertIn("first 3: pkg 01, pkg-02, pkg-03", collapsed[0]) + + def test_malformed_package_json_with_yarn_lock_no_collapse(self): + root = self.yarn_project(["direct-a"], ["pkg-01", "pkg-02", "pkg-03", "pkg-04", "pkg-05"], "{not json") + lines = run_main(root, ["yarn.lock"], self.plan) + self.assertTrue(all("transitive npm packages" not in ln for ln in lines), lines) + self.assertEqual(sum("outdated" in ln.split("\t")[2] for ln in lines), 6, lines) + + +if __name__ == "__main__": + res = unittest.TextTestRunner().run(unittest.defaultTestLoader.loadTestsFromTestCase(TransitiveCollapse)) + print(f"Tests: {res.testsRun - len(res.failures) - len(res.errors)}/{res.testsRun}") + raise SystemExit(not res.wasSuccessful())