pulumi
diff --git a/‎.claude/commands/docs-review/scripts/entity_key.py‎
Lines changed: 74 additions & 1 deletion b/‎.claude/commands/docs-review/scripts/entity_key.py‎
Lines changed: 74 additions & 1 deletion
diff --git a/‎.github/workflows/claims-reverify.yml‎
Lines changed: 26 additions & 5 deletions b/‎.github/workflows/claims-reverify.yml‎
Lines changed: 26 additions & 5 deletions
diff --git a/‎scripts/content-review/reverify-claims.py‎
Lines changed: 116 additions & 10 deletions b/‎scripts/content-review/reverify-claims.py‎
Lines changed: 116 additions & 10 deletions
@@ -30,7 +30,8 @@
 `volatile` marks the claim types worth cheap nightly re-verification straight
 from the index — assertions whose truth drifts with the outside world even
 when the page doesn't change: version pins, prices/rates/limits, and
-pricing/limit-flavored entity specs.
+pricing/limit-flavored entity specs. A `numerical` claim about the page's own
+example or tutorial output is excluded: see SELF_DESCRIBING_RE.
 
 No LLM involvement: derivation is pure text normalization, so keys are
 reproducible from the claim record alone. Run the smoke checks with
@@ -61,6 +62,30 @@
     re.IGNORECASE,
 )
 
+# A `numerical` claim counting something the page itself produced — "the
+# example update creates 3 resources", "this tutorial takes 15 minutes", "the
+# preview output shows 2 changes". The figure is a property of the prose, not
+# of the world: it can only change when the page changes, so the nightly lane
+# can never find drift in it, and the verifier has nothing to check it against
+# except the page it came from (which is what makes those checks circular —
+# see reverify-claims.py's own-corpus demotion). Excluded from `volatile`;
+# still keyed, still indexed, still re-checked whenever the page is reviewed.
+#
+# PRICING_LIMIT_RE vetoes the exclusion, because the two overlap on the one
+# phrasing where this rule would otherwise do damage: "the example stack costs
+# $12/month" is self-describing in form but the figure is an outside-world
+# price that drifts on its own. Keeping a stray example count in the nightly
+# pool costs one verifier call and a demoted verdict; dropping a real price
+# means never checking it again.
+SELF_DESCRIBING_RE = re.compile(
+    r"\b(?:examples?|samples?|tutorials?|walkthroughs?|screenshots?|"
+    r"meta[- ]description|"
+    r"(?:preview|update|command|terminal|console)\s+output|"
+    r"output\s+(?:shows|displays|lists|reports|contains|includes)|"
+    r"shown\s+(?:above|below))\b",
+    re.IGNORECASE,
+)
+
 MAX_SUBJECT_TOKENS = 5
 
 _STOPWORDS = {
@@ -147,6 +172,9 @@ def derive(claim: dict) -> tuple[str | None, bool]:
     volatile = ctype in ALWAYS_VOLATILE_TYPES or (
         ctype == "entity-spec" and bool(PRICING_LIMIT_RE.search(text))
     )
+    if (ctype == "numerical" and SELF_DESCRIBING_RE.search(text)
+            and not PRICING_LIMIT_RE.search(text)):
+        volatile = False
 
     if ctype not in KEYED_TYPES:
         return None, volatile
@@ -170,6 +198,17 @@ def derive(claim: dict) -> tuple[str | None, bool]:
     return f"{ctype}/{'-'.join(tokens)}", volatile
 
 
+def is_volatile(claim: dict) -> bool:
+    """Volatility of one claim record, re-derived from `type`/`text`.
+
+    Both inputs are persisted in the claims index, so `reverify-claims.py`
+    calls this to apply the current policy to snapshots that were stamped
+    under an older one — a narrowed rule takes effect on the whole index that
+    night rather than page by page as each is re-reviewed.
+    """
+    return derive(claim)[1]
+
+
 def stamp(claim: dict) -> dict:
     """Mutate `claim` in place with `entity_key` and `volatile`; returns it."""
     key, volatile = derive(claim)
@@ -234,6 +273,40 @@ def check(name: str, cond: bool) -> None:
                    "text": "Pulumi-hosted deployment runners run in AWS us-west-2."})
     check("hosting entity-spec is not volatile", v is False)
 
+    # Self-describing numerical claims: still keyed, never volatile. Each of
+    # these was in the nightly lane's first two live sweeps.
+    for text in ("This tutorial takes about 15 minutes to complete.",
+                 "The example update creates 3 resources.",
+                 "The preview output shows 2 changes.",
+                 "The page meta description characterizes the list as complete.",
+                 "The command output lists 4 stacks."):
+        k, v = derive({"type": "numerical", "text": text})
+        check(f"self-describing not volatile: {text[:34]}...", v is False)
+        check(f"self-describing still keyed: {text[:34]}...", k is not None)
+
+    # ...but a count of something the outside world owns still is, including
+    # the policy-pack row counts that share the "lists/complete" vocabulary.
+    k, v = derive({"type": "numerical",
+                   "text": "This page lists all 139 policies in the NIST SP 800-53 pack."})
+    check("policy-pack count stays volatile", v is True)
+    k, v = derive({"type": "numerical",
+                   "text": "An operation is retried a maximum of 100 times."})
+    check("external limit stays volatile", v is True)
+
+    # A price or limit inside self-describing phrasing keeps the claim in the
+    # nightly pool: the two regexes disagree here, and the figure still drifts
+    # on its own even though the sentence is about the page's own example.
+    for text in ("The example stack costs about $12 per month.",
+                 "The tutorial's cluster stays within the 5-node free limit.",
+                 "The example uses 2 of the 10 seats on the Team plan."):
+        k, v = derive({"type": "numerical", "text": text})
+        check(f"pricing vetoes the exclusion: {text[:34]}...", v is True)
+
+    # is_volatile() re-derives from a persisted record, ignoring its stored flag.
+    check("is_volatile re-derives from type/text",
+          is_volatile({"type": "numerical", "volatile": True,
+                       "text": "The example update creates 3 resources."}) is False)
+
     # Non-keyed types: no key, not volatile.
     k, v = derive({"type": "behavior",
                    "text": "`pulumi up` deploys all resources in the stack."})
 
@@ -29,11 +29,20 @@ name: "Scheduled jobs: Re-verify volatile claims"
 # TURNED ON 2026-08-11 with `CLAIMS_REVERIFY_COUNT` set explicitly to '25' in
 # pulumi/docs, so the first live run is a recorded decision rather than an
 # implicit default. At the time there were 63 volatile entities across 86
-# claim snapshots — a full sweep every 3 nights. Because the lane had never
-# executed, watch the first few reports for false-positive churn: most entities
-# are typed `numerical`, and some keys read like prose fragments rather than
-# version pins or limits, which would point at the volatile classifier rather
-# than at genuinely rotted facts.
+# claim snapshots — a full sweep every 3 nights.
+#
+# The first two live sweeps (Aug 11-12, 50 entities, 0 stale) confirmed both
+# of the things that warning called for; pulumi/docs#20851 fixed them:
+#   * 48 of 50 entities were typed `numerical`, and many were counts of the
+#     page's own example output ("the example update creates 3 resources")
+#     rather than version pins or limits. entity_key.py now excludes
+#     self-describing numerical claims from `volatile`, and this lane
+#     re-derives the flag rather than trusting what the snapshot stored.
+#   * 22 of 35 `verified` verdicts cited only Pulumi's own published docs —
+#     16 of them the published copy of the very page making the claim. Those
+#     are demoted to `unverifiable` now: the site is this repo rendered, so
+#     such a check can never detect drift. Expect `fresh` to read far lower
+#     than it did on those two nights; the difference was never real.
 #
 # DEGRADATION HEALTH: this lane only Slacks when it finds STALE entities, so a
 # dead lane (claims index gone, API key missing, every check inconclusive)
@@ -179,6 +188,18 @@ jobs:
           echo "--- report ---"
           cat .claims-reverify-report.json
 
+      # The report is the lane's only durable evidence: the run log masks every
+      # occurrence of the org name (so source URLs read as `www.***.com`) and
+      # ages out with the run. Uploaded unconditionally — a run worth debugging
+      # is usually one where the verify step failed.
+      - name: Upload re-verify report
+        if: always() && hashFiles('.claims-reverify-report.json') != ''
+        uses: actions/upload-artifact@v7
+        with:
+          name: claims-reverify-report
+          path: .claims-reverify-report.json
+          retention-days: 90
+
       # Counts only + a run link — the evidence lives in the report artifact
       # and the ledger markers; the fix arrives as a normal content-review PR.
       - name: Post stale summary to Slack
 
@@ -34,6 +34,12 @@
 anything else (`unverifiable`, errors) → inconclusive, reported but never
 marked — a flaky check must not burn review-queue slots.
 
+Evidence independence: a verdict whose only cited source is Pulumi's own
+published docs — a www.pulumi.com URL, or a `content/` source file — is
+demoted to `unverifiable` before that mapping (see `source_is_own_corpus`).
+The site is this repo rendered, so such a check has confirmed the page
+against itself and cannot detect drift in either direction.
+
 Writes `.claims-reverify-report.json` (plus `n_checked`/`n_stale`/`has_stale`
 to $GITHUB_OUTPUT) and, when CONTENT_REVIEW_LEDGER_URI is set, uploads each
 marked ledger object. Degrades gracefully: no API key, no claims dir, or no
@@ -53,19 +59,24 @@
 import importlib.util
 import json
 import os
+import re
 import sys
 from concurrent.futures import ThreadPoolExecutor
 from datetime import date, datetime, timezone
 from pathlib import Path
 
 HERE = Path(__file__).resolve().parents[2]
 VERIFY_CLAIMS = HERE / ".claude/commands/docs-review/scripts/verify-claims.py"
+ENTITY_KEY = HERE / ".claude/commands/docs-review/scripts/entity_key.py"
 
 SCHEMA_VERSION = 1
 DEFAULT_COUNT = 25
 MAX_CONCURRENCY = 8
 STALE_VERDICTS = {"contradicted", "mismatch", "framing-drift"}
 FRESH_VERDICTS = {"verified", "matches"}
+# Verdicts that assert something either way, and so have to rest on evidence
+# from outside the docs corpus to mean anything.
+DECIDED_VERDICTS = STALE_VERDICTS | FRESH_VERDICTS
 
 
 def log(msg: str) -> None:
@@ -86,6 +97,55 @@ def _load_verify_claims():
     return mod
 
 
+def _load_is_volatile():
+    """entity_key.is_volatile, or None if the module can't be loaded.
+
+    Re-deriving beats trusting the snapshot's stored `volatile` flag: the flag
+    was stamped whenever the page was last reviewed, so a narrowing of the
+    policy would otherwise only reach the index page by page over months. None
+    falls the caller back to the stored flag — a stale policy is a much smaller
+    problem than a lane that stops selecting anything."""
+    try:
+        spec = importlib.util.spec_from_file_location("entity_key", ENTITY_KEY)
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+        return mod.is_volatile
+    except Exception as e:  # noqa: BLE001 - never break the lane over this
+        warn(f"entity_key module unavailable ({e}); using stored volatile flags")
+        return None
+
+
+# ---- evidence independence ---------------------------------------------------
+
+# www.pulumi.com is this repo rendered and `content/` is its source, so a
+# verdict citing only those has checked the page against itself: it can never
+# go stale, and a `contradicted` from the same place is equally meaningless.
+# Demoted to `unverifiable` — reported, never counted fresh, never marked.
+_URL_RE = re.compile(r"https?://[^\s,;)\]]+", re.IGNORECASE)
+_OWN_HOST_RE = re.compile(r"^https?://(?:[\w-]+\.)*pulumi\.com(?:[/:?#]|$)", re.IGNORECASE)
+_PATH_RE = re.compile(
+    r"(?<![\w/-])[\w][\w./-]*\.(?:json|ya?ml|md|mdx|go|ts|tsx|js|py|cs|java|tf|toml)\b"
+)
+
+
+def source_is_own_corpus(source: str) -> bool:
+    """True when every source the verifier cited is our own published docs.
+
+    Positive evidence only: a source naming nothing identifiable (no URL, no
+    file path) is left alone rather than demoted — unrecognized is not the
+    same as circular.
+    """
+    src = source or ""
+    urls = _URL_RE.findall(src)
+    if any(not _OWN_HOST_RE.match(u) for u in urls):
+        return False
+    paths = _PATH_RE.findall(_URL_RE.sub(" ", src))
+    own_paths = [p for p in paths if p.startswith("content/") or "/content/" in p]
+    if len(paths) > len(own_paths):
+        return False
+    return bool(urls) or bool(own_paths)
+
+
 # ---- index loading -----------------------------------------------------------
 
 
@@ -105,16 +165,20 @@ def load_snapshots(claims_dir: Path) -> list[dict]:
     return out
 
 
-def volatile_entities(snapshots: list[dict]) -> dict[str, list[dict]]:
+def volatile_entities(snapshots: list[dict], is_volatile=None) -> dict[str, list[dict]]:
     """{entity_key: [assertion, ...]} over every volatile keyed claim.
 
     Each assertion carries the claim record plus its page provenance
     (path/slug/reviewed_at), so a stale verdict can fan back out to every
-    page asserting the entity."""
+    page asserting the entity.
+
+    `is_volatile` re-derives the flag under today's policy (see
+    `_load_is_volatile`); omitting it reads the snapshot's stored flag."""
+    decide = is_volatile or (lambda c: bool(c.get("volatile")))
     entities: dict[str, list[dict]] = {}
     for snap in snapshots:
         for c in snap.get("claims") or []:
-            if not isinstance(c, dict) or not c.get("entity_key") or not c.get("volatile"):
+            if not isinstance(c, dict) or not c.get("entity_key") or not decide(c):
                 continue
             entities.setdefault(c["entity_key"], []).append({
                 "claim": c,
@@ -239,7 +303,8 @@ def finish(report: dict, out_path: Path) -> int:
     out_path.write_text(json.dumps(report, indent=2) + "\n")
     m = report["meta"]
     log(f"checked={m['n_checked']} stale={m['n_stale']} fresh={m['n_fresh']} "
-        f"inconclusive={m['n_inconclusive']} (volatile entities={m['n_entities']}) -> {out_path}")
+        f"inconclusive={m['n_inconclusive']} (of which {m['n_demoted']} demoted for "
+        f"own-corpus evidence) (volatile entities={m['n_entities']}) -> {out_path}")
     write_outputs(report)
     return 0
 
@@ -258,15 +323,17 @@ def run(args) -> int:
         "checked_at": today.isoformat(),
         "entities": [],
         "meta": {"n_snapshots": 0, "n_entities": 0, "n_due": 0, "n_checked": 0,
-                 "n_stale": 0, "n_fresh": 0, "n_inconclusive": 0},
+                 "n_stale": 0, "n_fresh": 0, "n_inconclusive": 0, "n_demoted": 0},
     }
     out_path = Path(args.out)
 
     # The meta block doubles as the health observation consumed by
     # signal-health.py's reverify signal: `skipped` distinguishes "couldn't
-    # run" (degraded) from the quiet-night n_due=0 (healthy), and an
-    # all-inconclusive n_checked is the broken-verifier tell. Keep those
-    # semantics intact when touching the early-exit paths below.
+    # run" (degraded) from the quiet-night n_due=0 (healthy), an
+    # all-inconclusive n_checked is the no-drift-detected tell, and n_demoted
+    # says how much of that was own-corpus evidence rather than broken
+    # plumbing — the two want opposite fixes. Keep those semantics intact when
+    # touching the early-exit paths below.
     snapshots = load_snapshots(Path(args.claims_dir))
     report["meta"]["n_snapshots"] = len(snapshots)
     if not snapshots:
@@ -275,7 +342,7 @@ def run(args) -> int:
         return finish(report, out_path)
 
     ledger = load_ledger(Path(args.ledger_dir))
-    entities = volatile_entities(snapshots)
+    entities = volatile_entities(snapshots, _load_is_volatile())
     report["meta"]["n_entities"] = len(entities)
 
     unmarked = sorted(k for k, v in entities.items() if not already_marked(k, v, ledger))
@@ -299,9 +366,14 @@ def check_entity(key: str) -> dict:
         claim["__id"] = key
         claim["__route"] = vc.route_claim(claim, {})
         rec, err = vc.process_claim(api_key, claim, {}, args.model, repo_root, args.dry_run)
+        verdict = rec.get("verdict")
+        demoted_from = None
+        if verdict in DECIDED_VERDICTS and source_is_own_corpus(rec.get("source") or ""):
+            demoted_from, verdict = verdict, "unverifiable"
         return {
             "entity_key": key,
-            "verdict": rec.get("verdict"),
+            "verdict": verdict,
+            "demoted_from": demoted_from,
             "confidence": rec.get("confidence"),
             "evidence": rec.get("evidence"),
             "source": rec.get("source"),
@@ -320,6 +392,7 @@ def check_entity(key: str) -> dict:
     report["meta"]["n_stale"] = len(stale)
     report["meta"]["n_fresh"] = len(fresh)
     report["meta"]["n_inconclusive"] = len(results) - len(stale) - len(fresh)
+    report["meta"]["n_demoted"] = sum(1 for r in results if r["demoted_from"])
 
     if stale:
         changed = apply_markers(ledger, stale, today)
@@ -371,10 +444,43 @@ def snap(slug, reviewed_at, *claims):
     ents = volatile_entities(snaps)
     check("volatile keyed entities only",
           set(ents) == {"version/pulumi-gcp", "numerical/team-plan-price"})
+
     check("entity fans out across pages", len(ents["version/pulumi-gcp"]) == 2)
     check("representative is the freshest assertion",
           representative(ents["version/pulumi-gcp"])["text"] == "pulumi-gcp v8.3.0")
 
+    # Re-derived volatility overrides the snapshot's stored flag, so a
+    # narrowed policy reaches the whole index the night it ships.
+    stale_flag = {"entity_key": "numerical/example-resources-created", "volatile": True,
+                  "type": "numerical", "line_range": "L3", "verdict": "verified",
+                  "text": "The example update creates 3 resources."}
+    rederived = volatile_entities(
+        snaps + [snap("c", "2026-07-06", stale_flag)], _load_is_volatile())
+    check("re-derive drops a self-describing claim stamped volatile",
+          "numerical/example-resources-created" not in rederived)
+    check("re-derive keeps genuinely volatile entities",
+          set(rederived) == {"version/pulumi-gcp", "numerical/team-plan-price"})
+    check("no re-derive falls back to the stored flag",
+          "numerical/example-resources-created"
+          in volatile_entities(snaps + [snap("c", "2026-07-06", stale_flag)]))
+
+    # Evidence independence: only own-corpus-sourced verdicts are demoted, and
+    # an unrecognizable source is never treated as circular.
+    own = ["https://www.pulumi.com/docs/iac/get-started/aws/modify-program/",
+           "https://www.pulumi.com/docs/a/ and https://www.pulumi.com/docs/b/",
+           "repo:content/docs/iac/concepts/providers/_index.md"]
+    independent = ["https://docs.aws.amazon.com/config/latest/developerguide/x.html",
+                   "gh api repos/pulumi/pulumi/contents/pkg/resource/deploy/retries.go",
+                   "repo:data/policy_pack_policies/cis-aws.json (line 2872)",
+                   "https://github.com/pulumi/pulumi/blob/master/.goreleaser.yml",
+                   "https://www.pulumi.com/docs/a/ and https://docs.aws.amazon.com/x.html",
+                   "N/A - author's own estimate of their tutorial content",
+                   ""]
+    for s in own:
+        check(f"own-corpus source: {s[:44]}", source_is_own_corpus(s) is True)
+    for s in independent:
+        check(f"independent source: {s[:44] or '(empty)'}", source_is_own_corpus(s) is False)
+
     # Chunk rotation: deterministic, complete coverage across consecutive days.
     keys = [f"k{i}" for i in range(5)]
     d0 = date(2026, 7, 6)