-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathcodec_copilot_test.go
More file actions
483 lines (456 loc) · 19 KB
/
Copy pathcodec_copilot_test.go
File metadata and controls
483 lines (456 loc) · 19 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
package agenthooks
import (
"context"
"encoding/json"
"testing"
)
// Copilot omits the event name from every payload but permissionRequest and
// notification, so the shape reconstruction is the load-bearing part of the
// decoder: get it wrong and events are mislabelled in telemetry or, worse,
// gated with the wrong capability set.
func TestCopilotEventNamesFromShape(t *testing.T) {
cases := map[string]struct {
fixture string
kind EventKind
}{
"sessionStart": {"copilot/session_start.json", KindSessionStart},
"sessionEnd": {"copilot/session_end.json", KindSessionEnd},
"userPromptSubmitted": {"copilot/user_prompt_submitted.json", KindPromptSubmitted},
"preToolUse": {"copilot/pre_tool_use.json", KindToolPre},
"postToolUse": {"copilot/post_tool_use.json", KindToolPost},
"postToolUseFailure": {"copilot/post_tool_use_failure.json", KindToolError},
"permissionRequest": {"copilot/permission_request.json", KindPermission},
"agentStop": {"copilot/agent_stop.json", KindStop},
"subagentStart": {"copilot/subagent_start.json", KindSubagentStart},
"subagentStop": {"copilot/subagent_stop.json", KindSubagentStop},
"preCompact": {"copilot/pre_compact.json", KindCompactPre},
"notification": {"copilot/notification.json", KindNotification},
}
for want, tc := range cases {
typed, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, tc.fixture))
if err != nil {
t.Fatalf("%s: %v", want, err)
}
ev := eventOf(typed)
if ev.NativeName != want || ev.Kind != tc.kind {
t.Errorf("%s decoded as native=%q kind=%q, want kind=%q", want, ev.NativeName, ev.Kind, tc.kind)
}
if ev.Session.ID != "sess-copilot-1" {
t.Errorf("%s session id = %q", want, ev.Session.ID)
}
}
}
// toolArgs is a JSON-encoded string on pre/postToolUse and a plain object on
// permissionRequest; both must normalize to an object.
func TestCopilotToolArgsNormalize(t *testing.T) {
typed, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, "copilot/pre_tool_use.json"))
if err != nil {
t.Fatal(err)
}
pre, ok := typed.(*ToolPreEvent)
if !ok {
t.Fatalf("decoded %T, want *ToolPreEvent", typed)
}
var args struct {
Command string `json:"command"`
}
if err := json.Unmarshal(pre.Tool.Input, &args); err != nil {
t.Fatalf("tool input is not an object: %s", pre.Tool.Input)
}
if args.Command != "echo hello-from-gram" {
t.Errorf("command = %q", args.Command)
}
if pre.Tool.Canonical != ToolShell || !pre.Tool.Synthesized {
t.Errorf("tool = %+v; copilot ships no call id, so it must be synthesized", pre.Tool)
}
typed, err = decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, "copilot/permission_request.json"))
if err != nil {
t.Fatal(err)
}
perm, ok := typed.(*PermissionEvent)
if !ok {
t.Fatalf("decoded %T, want *PermissionEvent", typed)
}
if err := json.Unmarshal(perm.Tool.Input, &args); err != nil || args.Command != "echo hello-from-gram" {
t.Errorf("permission tool input = %s (%v)", perm.Tool.Input, err)
}
}
// The rule the whole codec is built around: Copilot denies a tool call on ANY
// non-zero exit from a preToolUse command hook, so a deny must ride stdout
// with exit 0 — and so must a fail-closed handler failure.
func TestCopilotDenyExitsZero(t *testing.T) {
typed, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, "copilot/pre_tool_use.json"))
if err != nil {
t.Fatal(err)
}
base := eventOf(typed)
wire, err := encodeCopilot(base, decisionCore{kind: DecisionDeny, reason: "blocked by policy"})
if err != nil {
t.Fatal(err)
}
if wire.ExitCode != 0 {
t.Fatalf("deny exit code = %d, want 0 (non-zero denies unconditionally and loses the reason)", wire.ExitCode)
}
var out map[string]any
if err := json.Unmarshal(wire.Stdout, &out); err != nil {
t.Fatalf("deny stdout %q: %v", wire.Stdout, err)
}
if out["permissionDecision"] != "deny" || out["permissionDecisionReason"] != "blocked by policy" {
t.Errorf("deny body = %s", wire.Stdout)
}
// failCore under FailClosed is the credential-ratchet path; it must also
// leave the exit code alone.
fail := failCore(Policy{Fail: FailClosed}, base)
if fail.kind != DecisionDeny {
t.Fatalf("fail-closed core = %s, want deny", fail.kind)
}
wire, err = encodeCopilot(base, fail)
if err != nil || wire.ExitCode != 0 {
t.Fatalf("fail-closed wire = %+v (%v), want exit 0", wire, err)
}
}
// permissionRequest speaks behavior/message, not permissionDecision, and the
// prompt event can express nothing at all (Copilot drops command-hook output
// for userPromptSubmitted).
func TestCopilotPerEventOutputSchemas(t *testing.T) {
perm, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, "copilot/permission_request.json"))
if err != nil {
t.Fatal(err)
}
wire, err := encodeCopilot(eventOf(perm), decisionCore{kind: DecisionDeny, reason: "nope"})
if err != nil {
t.Fatal(err)
}
var out map[string]any
if err := json.Unmarshal(wire.Stdout, &out); err != nil {
t.Fatal(err)
}
if out["behavior"] != "deny" || out["message"] != "nope" {
t.Errorf("permissionRequest body = %s", wire.Stdout)
}
if Capabilities(ProviderCopilot, VariantUnknown, KindPromptSubmitted).Has(CapDeny) {
t.Error("prompt.submitted must not claim deny: copilot drops command-hook output for it")
}
stop, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, "copilot/agent_stop.json"))
if err != nil {
t.Fatal(err)
}
wire, err = encodeCopilot(eventOf(stop), decisionCore{kind: DecisionContinue, instruction: "keep going"})
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(wire.Stdout, &out); err != nil {
t.Fatal(err)
}
if out["decision"] != "block" || out["reason"] != "keep going" {
t.Errorf("agentStop body = %s", wire.Stdout)
}
}
// preToolUse is the only event Copilot lets a hook steer, and it steers it
// through one string field. A typo in the value ("allowed", "Ask") is not a
// parse error on Copilot's side — it is an unrecognized decision, i.e. a
// silent fall-through to the normal permission flow. Pin the exact strings.
func TestCopilotToolPreDecisionValues(t *testing.T) {
typed, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, "copilot/pre_tool_use.json"))
if err != nil {
t.Fatal(err)
}
base := eventOf(typed)
for _, tc := range []struct {
name string
core decisionCore
want string
}{
{"allow", decisionCore{kind: DecisionAllow, reason: "on the allowlist"}, "allow"},
{"ask", decisionCore{kind: DecisionAsk, reason: "confirm?"}, "ask"},
} {
wire, err := encodeCopilot(base, tc.core)
if err != nil {
t.Fatalf("%s: %v", tc.name, err)
}
if wire.ExitCode != 0 {
t.Fatalf("%s exit code = %d, want 0 (any non-zero preToolUse exit denies unconditionally)", tc.name, wire.ExitCode)
}
var out map[string]any
if err := json.Unmarshal(wire.Stdout, &out); err != nil {
t.Fatalf("%s stdout %q: %v", tc.name, wire.Stdout, err)
}
if out["permissionDecision"] != tc.want || out["permissionDecisionReason"] != tc.core.reason {
t.Errorf("%s body = %s", tc.name, wire.Stdout)
}
}
}
// The highest-risk field in the codec: Copilot ignores an unknown key without
// complaint, so a rename of modifiedArgs — or wrapping the args in an extra
// envelope, or re-stringifying them the way toolArgs arrives on input — runs
// the ORIGINAL command while the handler believes it sanitized it. Assert the
// key name and that the value is the args object itself.
func TestCopilotToolPreModifiedArgs(t *testing.T) {
typed, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, "copilot/pre_tool_use.json"))
if err != nil {
t.Fatal(err)
}
wire, err := encodeCopilot(eventOf(typed), decisionCore{
kind: DecisionAllow,
hasUpdatedInput: true,
updatedInput: map[string]any{"command": "echo sanitized"},
})
if err != nil {
t.Fatal(err)
}
if wire.ExitCode != 0 {
t.Fatalf("modified-args exit code = %d, want 0", wire.ExitCode)
}
var out struct {
Decision string `json:"permissionDecision"`
Modified *struct {
Command string `json:"command"`
} `json:"modifiedArgs"`
}
if err := json.Unmarshal(wire.Stdout, &out); err != nil {
t.Fatalf("stdout %q: %v", wire.Stdout, err)
}
if out.Modified == nil {
t.Fatalf("no modifiedArgs key in %s; an input rewrite under any other name is a silent no-op", wire.Stdout)
}
if out.Modified.Command != "echo sanitized" {
t.Errorf("modifiedArgs.command = %q, want the rewritten args as a plain object; body = %s", out.Modified.Command, wire.Stdout)
}
if out.Decision != "allow" {
t.Errorf("permissionDecision = %q; the rewrite must not displace the verdict", out.Decision)
}
}
// permissionRequest answers on behavior/message, and allow here is the field
// that suppresses a user prompt — emitting it under preToolUse's
// permissionDecision name would leave the user staring at a prompt the
// handler already answered.
func TestCopilotPermissionAllow(t *testing.T) {
typed, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, "copilot/permission_request.json"))
if err != nil {
t.Fatal(err)
}
wire, err := encodeCopilot(eventOf(typed), decisionCore{kind: DecisionAllow, reason: "pre-approved"})
if err != nil {
t.Fatal(err)
}
if wire.ExitCode != 0 {
t.Fatalf("permission allow exit code = %d, want 0", wire.ExitCode)
}
var out map[string]any
if err := json.Unmarshal(wire.Stdout, &out); err != nil {
t.Fatal(err)
}
if out["behavior"] != "allow" || out["message"] != "pre-approved" || out["permissionDecision"] != nil {
t.Errorf("permission allow body = %s", wire.Stdout)
}
}
// subagentStop shares agentStop's schema; if it ever forked to its own case,
// a subagent continuation would stop silently while the parent's kept working
// — the kind of asymmetry that only shows up under a delegating agent.
func TestCopilotSubagentStopContinue(t *testing.T) {
typed, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, "copilot/subagent_stop.json"))
if err != nil {
t.Fatal(err)
}
wire, err := encodeCopilot(eventOf(typed), decisionCore{kind: DecisionContinue, instruction: "finish the migration"})
if err != nil {
t.Fatal(err)
}
if wire.ExitCode != 0 {
t.Fatalf("subagent continue exit code = %d, want 0", wire.ExitCode)
}
var out map[string]any
if err := json.Unmarshal(wire.Stdout, &out); err != nil {
t.Fatal(err)
}
if out["decision"] != "block" || out["reason"] != "finish the migration" {
t.Errorf("subagentStop body = %s", wire.Stdout)
}
}
// sessionStart is the only place Copilot accepts injected context, and it
// arrives under additionalContext — not the reason/message names every other
// copilot event uses. Wrong key means the context silently never reaches the
// model, which reads as "the agent ignored my instructions".
func TestCopilotSessionStartAdditionalContext(t *testing.T) {
typed, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, "copilot/session_start.json"))
if err != nil {
t.Fatal(err)
}
wire, err := encodeCopilot(eventOf(typed), decisionCore{
kind: DecisionContinueSession,
context: []string{"repo is frozen for release"},
})
if err != nil {
t.Fatal(err)
}
if wire.ExitCode != 0 {
t.Fatalf("session start exit code = %d, want 0", wire.ExitCode)
}
var out map[string]any
if err := json.Unmarshal(wire.Stdout, &out); err != nil {
t.Fatal(err)
}
if out["additionalContext"] != "repo is frozen for release" {
t.Errorf("sessionStart body = %s", wire.Stdout)
}
}
// Degradation is enforced generically in applyPolicy, not in this codec — so
// the guarantee only holds end to end through the runner. It matters here
// because encodeCopilot has no case for these: an ask on permissionRequest or
// a replace-output on postToolUse that slipped past the policy layer would
// serialize to {}, i.e. no opinion, which on permissionRequest silently means
// "whatever the user clicks" rather than the block the handler asked for.
// Every path must still exit 0: a non-zero exit propagates to preToolUse
// semantics as an unconditional deny.
func TestCopilotDegradesUnsupportedDecisions(t *testing.T) {
copilotArgs := []string{"agenthooks", "run", "--provider=copilot"}
// permissionRequest declares deny+allow but no ask. FallbackDeny must
// harden to a real behavior:deny, not fall through to the empty body.
deny := quietRunner(WithPolicy(Policy{AskFallback: FallbackDeny}))
deny.OnPermission(func(ctx context.Context, e *PermissionEvent) (ToolPreDecision, error) {
if e.Can(CapAsk) {
t.Error("copilot permissionRequest must not report CapAsk")
}
return AskUser("confirm?"), nil
})
out, code := runWith(t, deny, copilotArgs, fixture(t, "copilot/permission_request.json"))
if out != `{"behavior":"deny","message":"confirm?"}` || code != 0 {
t.Errorf("ask on permissionRequest = %q (exit %d), want a hardened deny at exit 0", out, code)
}
// The default fallback is honest instead: no opinion, empty body.
noop := quietRunner()
noop.OnPermission(func(ctx context.Context, e *PermissionEvent) (ToolPreDecision, error) {
return AskUser("confirm?"), nil
})
out, code = runWith(t, noop, copilotArgs, fixture(t, "copilot/permission_request.json"))
if out != "{}" || code != 0 {
t.Errorf("ask under FallbackNoDecision = %q (exit %d), want {} at exit 0", out, code)
}
// postToolUse has no capset at all on Copilot: a rewritten tool output
// must degrade to observation, never to a body that looks like a verdict.
post := quietRunner()
post.OnToolPost(func(ctx context.Context, e *ToolPostEvent) (ToolPostDecision, error) {
if e.Can(CapReplaceOutput) {
t.Error("copilot tool.post must not report CapReplaceOutput")
}
return ReplaceOutput("scrubbed"), nil
})
out, code = runWith(t, post, copilotArgs, fixture(t, "copilot/post_tool_use.json"))
if out != "{}" || code != 0 {
t.Errorf("replace-output on postToolUse = %q (exit %d), want {} at exit 0", out, code)
}
}
func TestCopilotDetection(t *testing.T) {
inv, err := parseArgs([]string{"agenthooks", "run", "--provider=copilot"})
if err != nil || inv.provider != ProviderCopilot {
t.Fatalf("--provider=copilot → %q (%v)", inv.provider, err)
}
t.Setenv("COPILOT_CLI", "1")
t.Setenv("CLAUDE_PLUGIN_ROOT", "/tmp/plugin")
if p, ok := detectFromEnv(); !ok || p != ProviderCopilot {
t.Errorf("env detection = %q; copilot cross-sets CLAUDE_PLUGIN_ROOT and must win", p)
}
if p, ok := detectFromShape(fixture(t, "copilot/pre_tool_use.json")); !ok || p != ProviderCopilot {
t.Errorf("shape detection = %q", p)
}
}
// A --provider=copilot registration receives the Claude-shaped snake_case
// payload from two directions — the CLI running the PascalCase compat file
// this library installs for VS Code, and VS Code discovering a camelCase CLI
// file, because both runtimes glob both hook directories. copilotEventName has
// no camelCase shape to reconstruct from there, so before the fallthrough
// every one of these landed on KindOther with the tool fields empty: hooks
// that look installed and healthy while reporting nothing useful.
func TestCopilotClaudeShapedFallthrough(t *testing.T) {
for _, tc := range []struct {
fixture, native string
kind EventKind
}{
{"claude/pre_tool_use.json", "PreToolUse", KindToolPre},
{"claude/user_prompt_submit.json", "UserPromptSubmit", KindPromptSubmitted},
{"claude/post_tool_use.json", "PostToolUse", KindToolPost},
{"claude/session_start.json", "SessionStart", KindSessionStart},
{"claude/stop.json", "Stop", KindStop},
} {
typed, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, tc.fixture))
if err != nil {
t.Fatalf("%s: %v", tc.fixture, err)
}
ev := eventOf(typed)
if ev.NativeName != tc.native || ev.Kind != tc.kind {
t.Errorf("%s decoded as native=%q kind=%q, want %q/%q", tc.fixture, ev.NativeName, ev.Kind, tc.native, tc.kind)
}
// The label must stay ProviderCopilot: it is what selects the CLI's
// flat response schema downstream. decodeClaude hardcodes
// claude-code, so the relabel in decodeClaudeAs is load-bearing.
if ev.Provider != ProviderCopilot {
t.Errorf("%s provider = %q, want %q", tc.fixture, ev.Provider, ProviderCopilot)
}
if ev.Session.ID != "sess-claude-1" {
t.Errorf("%s session id = %q", tc.fixture, ev.Session.ID)
}
}
// The bug this fixes, stated as the assertion: tool arguments survive.
typed, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, "claude/pre_tool_use.json"))
if err != nil {
t.Fatal(err)
}
pre, ok := typed.(*ToolPreEvent)
if !ok {
t.Fatalf("decoded %T, want *ToolPreEvent", typed)
}
if pre.Tool.Name != "Bash" || pre.Tool.Canonical != ToolShell {
t.Errorf("tool = %+v; the whole point of the fallthrough is that these are populated", pre.Tool)
}
// The camelCase corpus must be untouched. copilot/notification.json is the
// trap: it is the one native Copilot event that ships hook_event_name, so
// only the sessionId half of the discriminator keeps it on this path.
for name, want := range map[string]string{
"copilot/notification.json": "notification",
"copilot/pre_tool_use.json": "preToolUse",
"copilot/agent_stop.json": "agentStop",
"copilot/session_start.json": "sessionStart",
} {
typed, err := decodeCopilot(VariantUnknown, DetectionConfig, testNow, fixture(t, name))
if err != nil {
t.Fatalf("%s: %v", name, err)
}
if ev := eventOf(typed); ev.NativeName != want {
t.Errorf("%s decoded as native=%q, want %q; the fallthrough stole a camelCase payload", name, ev.NativeName, want)
}
}
}
// End to end for the shared file: ONE installed agenthooks-vscode.json, run by
// both runtimes. The Copilot CLI's own env demotes the --provider flag, and
// everything downstream follows from the provider constant — so the same
// PascalCase input produces the CLI's FLAT body here and VS Code's nested one
// without it. A deny answered in the wrong placement is accepted and ignored
// by either runtime, which is why this is asserted rather than reasoned about.
func TestCopilotPascalCaseSharedFile(t *testing.T) {
vscodeArgs := []string{"agenthooks", "run", "--provider=vscode-copilot"}
denier := func() *Runner {
r := quietRunner()
r.OnToolPre(func(ctx context.Context, e *ToolPreEvent) (ToolPreDecision, error) {
return Deny("blocked by policy"), nil
})
return r
}
t.Setenv("COPILOT_CLI", "1")
out, code := runWith(t, denier(), vscodeArgs, fixture(t, "claude/pre_tool_use.json"))
if out != `{"permissionDecision":"deny","permissionDecisionReason":"blocked by policy"}` || code != 0 {
t.Errorf("CLI session = %q (exit %d), want copilot's flat deny at exit 0", out, code)
}
t.Setenv("COPILOT_CLI", "")
out, code = runWith(t, denier(), vscodeArgs, fixture(t, "claude/pre_tool_use.json"))
var body struct {
HSO struct {
PermissionDecision string `json:"permissionDecision"`
} `json:"hookSpecificOutput"`
}
if err := json.Unmarshal([]byte(out), &body); err != nil {
t.Fatalf("VS Code session stdout %q: %v", out, err)
}
if body.HSO.PermissionDecision != "deny" || code != 0 {
t.Errorf("VS Code session = %q (exit %d), want a nested deny at exit 0", out, code)
}
}