adapters/claude-code: use CC skill system for inject; split skill telemetry fields (#14)

src/adapters/bare-agent.ts

@@ -164,15 +164,17 @@ export class BareAgentAdapter implements AgentAdapter {
       activeProvider = new LoggingProvider(this.provider, convLog)
     }
 
-    let skillLoaded = false
+    let skillProvided = false
+    let skillObserved: boolean | undefined
 
     // Build system prompt
     let system = "You are a helpful assistant that completes tasks by using tools. Work in the provided directory."
 
     if (task.skillContent && skillMode === "inject") {
       // Inject mode: embed full skill content in system prompt
       system += `\n\n<skill>\n${task.skillContent}\n</skill>`
-      skillLoaded = true
+      // Structural: the skill content is now in the model's context.
+      skillProvided = true
     } else if (task.skillContent && skillMode === "discover" && task.skillMeta) {
       // Discover mode: copy skill dir to workDir, show only name+description in prompt
       await copySkillToDiscoverDir(
@@ -199,8 +201,10 @@ Available skills:
     const beforeLLMHooks = this.hooks.beforeLLM
     const allToolCalls: ToolCall[] = []
 
-    // Discover-mode state that needs to be maintained across iterations
-    let discoverSkillLoaded = skillLoaded
+    // Discover-mode state that needs to be maintained across iterations.
+    // Tracks whether the once-only <load-skill> protocol has fired; renamed
+    // from discoverSkillLoaded as part of the skillProvided/skillObserved split.
+    let discoverSkillProvided = skillProvided
 
     // Create a wrapper provider that handles beforeLLM hooks and discover mode
     const wrappedProvider: LLMProvider = {
@@ -273,13 +277,18 @@ Available skills:
         // tool_use blocks in one turn, we actually execute them concurrently.
         parallelToolExecution: true,
         onAfterLLM: async (response, iteration) => {
-          if (skillMode === "discover" && task.skillContent && task.skillMeta && !discoverSkillLoaded) {
+          if (skillMode === "discover" && task.skillContent && task.skillMeta && !discoverSkillProvided) {
             const skillMatch = response.text.match(LOAD_SKILL_RE)
             if (skillMatch) {
               const requestedName = (skillMatch[1] ?? "").trim()
               if (requestedName === task.skillMeta.name) {
-                discoverSkillLoaded = true
-                skillLoaded = true
+                // <load-skill> firing is simultaneously structural (the harness
+                // is about to splice the skill content into the model's context
+                // on the next turn) and behavioral (the model just engaged with
+                // the skill registry by name).
+                discoverSkillProvided = true
+                skillProvided = true
+                skillObserved = true
               }
             }
           }
@@ -322,7 +331,17 @@ Available skills:
       durationMs,
       llmDurationMs: loopResult.llmDurationMs,
       workDir: task.workDir,
-      skillLoaded: task.skillContent ? skillLoaded : undefined,
+      ...(task.skillContent
+        ? {
+            skillProvided,
+            ...(skillObserved !== undefined ? { skillObserved } : {}),
+            skillMode,
+            // Deprecated mirror of skillProvided; kept for one release so
+            // downstream consumers can migrate. Remove once jit-optimize and
+            // bench consumers all read skillProvided.
+            skillLoaded: skillProvided,
+          }
+        : {}),
       runStatus,
       ...(statusDetail ? { statusDetail } : {}),
       ...(loopResult.error ? { adapterError: { exitCode: 1, stderr: loopResult.error.message } } : {}),

src/adapters/claude-code.ts

@@ -406,43 +406,40 @@ export function resolveUserClaudeDir(): string {
 }
 
 // ---------------------------------------------------------------------------
-// Skill-mode helpers
+// Skill detection helpers (init-event structural + Skill-tool behavioral)
 // ---------------------------------------------------------------------------
 
 /**
- * Sentinel string written into the injected system prompt so we can detect
- * — by grepping the assistant's text — whether the model actually read the
- * skill content. Mirrors the CONTEXT.md trick in opencode.ts.
+ * Structural signal for discover mode: does CC's `system.init` event
+ * announce that the named skill was loaded into its registry? This is
+ * independent of any model behavior — `true` here means CC knows about
+ * the skill regardless of whether the model invoked it.
+ *
+ * Matches both bare names ("bench-skill") and plugin-namespaced names
+ * ("plugin-x:bench-skill") that CC emits when a plugin registers a skill.
  */
-const SKILL_INJECT_SENTINEL = "<skvm-skill-injected/>"
-
-function injectedSystemPrompt(skillContent: string): string {
-  return `${SKILL_INJECT_SENTINEL}\n\n${skillContent}`
-}
-
-export function detectSkillInject(events: ClaudeCodeEvent[], snippet: string): boolean {
+export function detectSkillProvided_Discover(events: ClaudeCodeEvent[], skillName: string): boolean {
   for (const ev of events) {
-    if (ev.type !== "assistant" || !ev.message) continue
-    const content = Array.isArray(ev.message.content) ? ev.message.content : []
-    for (const c of content) {
-      if (!c || (c as { type?: string }).type !== "text") continue
-      const text = (c as ClaudeCodeContentText).text
-      if (text.includes(SKILL_INJECT_SENTINEL)) return true
-      if (snippet.length > 20 && text.includes(snippet)) return true
+    if (ev.type !== "system" || ev.subtype !== "init") continue
+    const skills = Array.isArray(ev.skills) ? ev.skills : []
+    for (const s of skills) {
+      if (typeof s !== "string") continue
+      if (s === skillName || s.endsWith(`:${skillName}`)) return true
     }
   }
   return false
 }
 
-export function detectSkillDiscover(events: ClaudeCodeEvent[], skillName: string): boolean {
-  const matchesName = (s: unknown): boolean =>
-    typeof s === "string" && (s === skillName || s.endsWith(`:${skillName}`))
-
+/**
+ * Behavioral signal for discover mode: did the model invoke the `Skill`
+ * (or lowercase `skill`) tool with input matching this skill's name?
+ * A tool_use without an `input.name`/`input.skill` is NOT counted — the
+ * older permissive heuristic conflated "model invoked some Skill tool"
+ * with "model invoked OUR skill" and produced false positives in tests
+ * that share a sandbox across skills.
+ */
+export function detectSkillObserved_Discover(events: ClaudeCodeEvent[], skillName: string): boolean {
   for (const ev of events) {
-    if (ev.type === "system" && ev.subtype === "init") {
-      const skills = Array.isArray(ev.skills) ? ev.skills : []
-      if (skills.some(matchesName)) return true
-    }
     if (ev.type !== "assistant" || !ev.message) continue
     const content = Array.isArray(ev.message.content) ? ev.message.content : []
     for (const c of content) {
@@ -451,12 +448,67 @@ export function detectSkillDiscover(events: ClaudeCodeEvent[], skillName: string
       if (tu.name !== "Skill" && tu.name !== "skill") continue
       const inputName = (tu.input as { name?: string; skill?: string })?.name
         ?? (tu.input as { name?: string; skill?: string })?.skill
-      if (!inputName || matchesName(inputName)) return true
+      if (!inputName) continue
+      if (inputName === skillName || inputName.endsWith(`:${skillName}`)) return true
     }
   }
   return false
 }
 
+// ---------------------------------------------------------------------------
+// Inject / Discover artifact builders
+// ---------------------------------------------------------------------------
+
+/**
+ * Build the on-disk artifacts and CLI arguments needed to register a skill
+ * with CC's native skill system in inject (forced-load) mode.
+ *
+ * Inject = "skill is required for this task". Writes the SKILL.md file into
+ * the project-scope `.claude/skills/` directory (CC announces this in the
+ * `system.init` event's `skills[]` array — that's our `skillProvided`
+ * signal, independent of model behavior). The returned `appendSystemPrompt`
+ * carries a MUST directive that mandates calling the Skill tool — this is
+ * what makes inject behaviorally different from discover.
+ *
+ * Factored out of run() for direct testability without spawning CC.
+ */
+export async function buildClaudeCodeInjectArtifacts(opts: {
+  workDir: string
+  skillContent: string
+  skillMeta: { name: string; description: string }
+}): Promise<{ appendSystemPrompt: string }> {
+  const { workDir, skillContent, skillMeta } = opts
+  const skillDir = path.join(workDir, ".claude", "skills", skillMeta.name)
+  await mkdir(skillDir, { recursive: true })
+  const frontmatter = `---\nname: ${skillMeta.name}\ndescription: ${skillMeta.description}\n---\n\n`
+  await Bun.write(path.join(skillDir, "SKILL.md"), frontmatter + skillContent)
+  const appendSystemPrompt =
+    `You have access to a project skill named \`${skillMeta.name}\` ` +
+    `(${skillMeta.description}). ` +
+    `You MUST invoke the Skill tool with name="${skillMeta.name}" ` +
+    `before any other action, and you MUST follow its instructions.`
+  return { appendSystemPrompt }
+}
+
+/**
+ * Discover-mode (progressive-disclosure) artifacts: write the skill file
+ * to `.claude/skills/<name>/SKILL.md` and return no system-prompt
+ * directive. CC's normal description-matching + Skill tool exposure path
+ * takes over from here; the model decides whether to invoke.
+ */
+export async function buildClaudeCodeDiscoverArtifacts(opts: {
+  workDir: string
+  skillContent: string
+  skillMeta: { name: string; description: string }
+}): Promise<Record<string, never>> {
+  const { workDir, skillContent, skillMeta } = opts
+  const skillDir = path.join(workDir, ".claude", "skills", skillMeta.name)
+  await mkdir(skillDir, { recursive: true })
+  const frontmatter = `---\nname: ${skillMeta.name}\ndescription: ${skillMeta.description}\n---\n\n`
+  await Bun.write(path.join(skillDir, "SKILL.md"), frontmatter + skillContent)
+  return {}
+}
+
 // ---------------------------------------------------------------------------
 // Adapter
 // ---------------------------------------------------------------------------
@@ -568,25 +620,35 @@ export class ClaudeCodeAdapter implements AgentAdapter {
   }): Promise<RunResult> {
     const skillMode = task.skillMode ?? "inject"
     const skillName = task.skillMeta?.name ?? "bench-skill"
-    let skillLoaded: boolean | undefined
+    const skillDesc = task.skillMeta?.description ?? "Benchmark skill injected by SkVM"
     let appendSystemPrompt: string | undefined
+    let skillProvided: boolean | undefined
+    let skillObserved: boolean | undefined
 
     if (task.skillContent) {
-      skillLoaded = false
       if (skillMode === "inject") {
-        // --append-system-prompt is the documented headless way to inject
-        // extra context. The sentinel lets us verify the model actually
-        // saw the skill (matches opencode's CONTEXT.md trick).
-        appendSystemPrompt = injectedSystemPrompt(task.skillContent)
+        // Forced load: write the skill into CC's native registry AND add a
+        // hard directive instructing the model to invoke the Skill tool.
+        // The structural confirmation (system.init event listing the skill)
+        // is independent of model behavior, which makes it robust against
+        // thinking-mode models that don't echo system prompt content.
+        const artifacts = await buildClaudeCodeInjectArtifacts({
+          workDir: task.workDir,
+          skillContent: task.skillContent,
+          skillMeta: { name: skillName, description: skillDesc },
+        })
+        appendSystemPrompt = artifacts.appendSystemPrompt
       } else {
-        // Discover: <workDir>/.claude/skills/ is project-scope, so it works
-        // regardless of sandbox HOME — managed-mode runs ship the skill too.
-        const skillDesc = task.skillMeta?.description ?? "Benchmark skill injected by SkVM"
-        const skillDir = path.join(task.workDir, ".claude", "skills", skillName)
-        await mkdir(skillDir, { recursive: true })
-        const frontmatter = `---\nname: ${skillName}\ndescription: ${skillDesc}\n---\n\n`
-        await Bun.write(path.join(skillDir, "SKILL.md"), frontmatter + task.skillContent)
+        // Discoverable: write the skill file without a force directive.
+        // CC's progressive disclosure based on description match takes over.
+        await buildClaudeCodeDiscoverArtifacts({
+          workDir: task.workDir,
+          skillContent: task.skillContent,
+          skillMeta: { name: skillName, description: skillDesc },
+        })
       }
+      // Provisional; overwritten from the init event after the run.
+      skillProvided = false
     }
 
     const startMs = performance.now()
@@ -646,18 +708,28 @@ export class ClaudeCodeAdapter implements AgentAdapter {
 
     const events = parseClaudeCodeStreamJSON(stdout)
 
-    if (task.skillContent && skillLoaded === false) {
-      if (skillMode === "inject") {
-        const snippet = task.skillContent.replace(/^#.*\n/m, "").trim().slice(0, 60)
-        skillLoaded = detectSkillInject(events, snippet)
-      } else {
-        skillLoaded = detectSkillDiscover(events, skillName)
-      }
+    if (task.skillContent) {
+      // Structural signal — same source in both modes: CC's init event
+      // either listed our skill or it didn't. Independent of model behavior.
+      skillProvided = detectSkillProvided_Discover(events, skillName)
+      // Behavioral signal — also same source in both modes. In inject mode
+      // the MUST directive should drive this to true on compliant models;
+      // in discover mode it depends on progressive disclosure.
+      skillObserved = detectSkillObserved_Discover(events, skillName)
     }
 
     const result = eventsToRunResult(events, task.workDir, durationMs)
-    if (skillLoaded !== undefined) {
-      result.skillLoaded = skillLoaded
+    if (skillProvided !== undefined) {
+      result.skillProvided = skillProvided
+      // Mirror to deprecated alias so any reader still on `skillLoaded`
+      // sees the same value during the migration window.
+      result.skillLoaded = skillProvided
+    }
+    if (skillObserved !== undefined) {
+      result.skillObserved = skillObserved
+    }
+    if (task.skillContent) {
+      result.skillMode = skillMode
     }
     if (timedOut) {
       result.runStatus = "timeout"

src/adapters/hermes.ts

@@ -345,15 +345,18 @@ export class HermesAdapter implements AgentAdapter {
     timeoutMs?: number
   }): Promise<RunResult> {
     const skillMode = task.skillMode ?? "inject"
-    let skillLoaded: boolean | undefined
+    let skillProvided: boolean | undefined
+    let skillObserved: boolean | undefined
     let prompt = `IMPORTANT: Do not ask clarifying questions. Proceed directly with implementation. Execute all steps immediately without waiting for user input.\n\n`
 
     // --- Skill handling ---
     if (task.skillContent) {
       if (skillMode === "inject") {
-        // Inject mode: prepend skill content to prompt
+        // Inject mode: prepend skill content to prompt. The content lands in
+        // the agent's context the moment we concat it, so skillProvided is
+        // already true — no behavioral inference required.
         prompt += task.skillContent + "\n\n---\n\n"
-        skillLoaded = false
+        skillProvided = true
       } else {
         const skillName = task.skillMeta?.name ?? "bench-skill"
         if (!this.hermesHome) {
@@ -362,7 +365,9 @@ export class HermesAdapter implements AgentAdapter {
         const skillDir = path.join(this.hermesHome, "skills", skillName)
         await mkdir(skillDir, { recursive: true })
         await Bun.write(path.join(skillDir, "SKILL.md"), task.skillContent)
-        skillLoaded = false
+        // Discover mode: hermes registers SKILL.md via the -s flag below, so
+        // structurally the skill is wired up before the agent runs.
+        skillProvided = true
       }
     }
 
@@ -532,30 +537,28 @@ export class HermesAdapter implements AgentAdapter {
     // --- Save conv log (export JSON is richer than raw stdout when available) ---
     await saveConvLog(exportResult.exitCode === 0 ? exportResult.stdout : stdout)
 
-    // --- Verify skill loaded ---
-    if (task.skillContent && skillLoaded === false) {
+    // --- Skill observation (behavioral) ---
+    // skillProvided is already set at the inject/register step above. The only
+    // thing the post-run scan can add is behavioral evidence: did the agent
+    // actually use the skill? We look for echoed snippets in step text.
+    if (task.skillContent && skillProvided) {
       const skillSnippet = task.skillContent.replace(/^#.*\n/m, "").trim().slice(0, 60)
-
-      if (skillMode === "inject") {
-        // Inject: if agent produced any tool calls or steps, skill was loaded (it's in the prompt)
-        if (result.steps.length > 0) {
-          skillLoaded = true
-        }
-      }
-
-      // Check if any assistant text references skill content
-      if (!skillLoaded && skillSnippet.length > 20) {
+      if (skillSnippet.length > 20) {
         for (const step of result.steps) {
           if (step.text?.includes(skillSnippet)) {
-            skillLoaded = true
+            skillObserved = true
             break
           }
         }
       }
     }
 
-    if (skillLoaded !== undefined) {
-      result.skillLoaded = skillLoaded
+    if (task.skillContent) {
+      result.skillProvided = skillProvided ?? false
+      if (skillObserved !== undefined) result.skillObserved = skillObserved
+      result.skillMode = skillMode
+      // Deprecated mirror — kept for one release while consumers migrate.
+      result.skillLoaded = skillProvided ?? false
     }
     // Subprocess-level failure overrides whatever the parse path decided.
     // Rare on this branch (we already got a session_id) but possible if the