Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions services/slackbotv2/src/message-overrides-strategy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
extractMessageOverrides,
validateStrategyOverrides
} from './overrides'
import { extractNaturalLanguageOverrides } from './natural-language-overrides'
import type { JsonObject, MessageOverridesStrategy } from './types'
import { errorMessage, isJsonObject } from './utils'

Expand Down Expand Up @@ -119,6 +120,17 @@ export function createOpenAiMessageOverridesStrategy(
return { cleanedText, overrides: explicitOverrides }
}

const naturalLanguageOverrides = extractNaturalLanguageOverrides(text)
if (naturalLanguageOverrides) {
options.logger?.info('slackbotv2_message_overrides_strategy_deterministic_match', {
harness: naturalLanguageOverrides.harnessType,
model: naturalLanguageOverrides.model,
provider: naturalLanguageOverrides.provider,
reasoning: naturalLanguageOverrides.reasoning
})
return { overrides: naturalLanguageOverrides }
}

const controller = new AbortController()
const timeout = setTimeout(() => controller.abort(), timeoutMs)
try {
Expand Down
121 changes: 121 additions & 0 deletions services/slackbotv2/src/natural-language-overrides.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import { validateStrategyOverrides } from './overrides'
import type { HarnessOverrides } from './overrides'

type OverrideField = 'harness' | 'model' | 'provider'
type Alias = { field: OverrideField; value: string }

const ALIASES: Record<string, Alias> = {
amp: { field: 'harness', value: 'amp' },
bedrock: { field: 'provider', value: 'amazon-bedrock' },
claude: { field: 'harness', value: 'claudecode' },
claudecode: { field: 'harness', value: 'claudecode' },
codex: { field: 'harness', value: 'codex' },
fable: { field: 'model', value: 'claude-fable-5' },
haiku: { field: 'model', value: 'claude-haiku-4-5' },
luna: { field: 'model', value: 'gpt-5.6-luna' },
meta: { field: 'provider', value: 'responses' },
nanocodex: { field: 'harness', value: 'nanocodex' },
openrouter: { field: 'provider', value: 'openrouter' },
opus: { field: 'model', value: 'claude-opus-4-8' },
sol: { field: 'model', value: 'gpt-5.6-sol' },
sonnet: { field: 'model', value: 'claude-sonnet-4-6' },
terra: { field: 'model', value: 'gpt-5.6-terra' }
}

const CLAUSE_PATTERN = new RegExp(
String.raw`\b(?:use|using|select|choose|pick|switch(?:ing)?(?:\s+me)?\s+to|run(?:ning)?(?:\s+this)?\s+(?:with|on))\b([^.!?\n]{0,120})`,
'gi'
)
const TOKEN_PATTERN = /[a-z0-9]+(?:[.-][a-z0-9]+)*/g
const REASONING_PATTERN =
/\b(none|minimal|low|medium|high|xhigh|max)\s+(?:reasoning|effort)\b|\b(?:reasoning|effort)\s+(?:to\s+)?(none|minimal|low|medium|high|xhigh|max)\b/i
const AMP_MODEL_PATTERN = /\b(deep|fast)\s+(?:model|mode)\b/i

/**
* Resolve common natural-language model selections without a network call.
* Matching is limited to explicit selection clauses, and edit-distance
* matching is reserved for names of at least five characters to avoid
* treating ordinary short words as selectors.
*/
export function extractNaturalLanguageOverrides(text: string): HarnessOverrides | undefined {
const raw: Record<OverrideField | 'reasoning', string | undefined> = {
harness: undefined,
model: undefined,
provider: undefined,
reasoning: undefined
}
let matched = false

for (const clauseMatch of text.matchAll(CLAUSE_PATTERN)) {
const clause = clauseMatch[1] ?? ''
const tokens = clause.toLowerCase().match(TOKEN_PATTERN) ?? []
for (const [index, token] of tokens.entries()) {
const alias = resolveAlias(token)
if (!alias || !isSelectorPosition(tokens, index)) continue
raw[alias.field] = alias.value
matched = true
}

const reasoningMatch = REASONING_PATTERN.exec(clause)
const reasoning = reasoningMatch?.[1] ?? reasoningMatch?.[2]
if (reasoning) {
raw.reasoning = reasoning.toLowerCase()
matched = true
}

const ampModelMatch = AMP_MODEL_PATTERN.exec(clause)
if (ampModelMatch) {
raw.model = ampModelMatch[1]!.toLowerCase()
matched = true
}
}

if (!matched) return undefined
const overrides = validateStrategyOverrides(raw)
return Object.values(overrides).some(value => value !== undefined) ? overrides : undefined
}

function isSelectorPosition(tokens: string[], index: number): boolean {
if (index <= 2) return true
return ['agent', 'harness', 'mode', 'model', 'provider'].includes(tokens[index + 1] ?? '')
}

function resolveAlias(token: string): Alias | undefined {
const exact = ALIASES[token]
if (exact) return exact
if (token.length < 5) return undefined

const matches = Object.entries(ALIASES).filter(
([name]) => name.length >= 5 && editDistanceAtMostOne(token, name)
)
return matches.length === 1 ? matches[0]![1] : undefined
}

function editDistanceAtMostOne(left: string, right: string): boolean {
if (Math.abs(left.length - right.length) > 1) return false
if (left === right) return true

if (left.length === right.length) {
let differences = 0
for (let index = 0; index < left.length; index += 1) {
if (left[index] !== right[index] && ++differences > 1) return false
}
return true
}

const [shorter, longer] = left.length < right.length ? [left, right] : [right, left]
let shortIndex = 0
let longIndex = 0
let skipped = false
while (shortIndex < shorter.length && longIndex < longer.length) {
if (shorter[shortIndex] === longer[longIndex]) {
shortIndex += 1
longIndex += 1
continue
}
if (skipped) return false
skipped = true
longIndex += 1
}
return true
}
103 changes: 96 additions & 7 deletions services/slackbotv2/test/overrides.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -537,24 +537,113 @@ describe('messageOverridesForText strategy invocation', () => {
})
})

test('falls back when the OpenAI strategy request fails', async () => {
test('resolves a natural-language alias before a failing OpenAI request', async () => {
let requestCount = 0
await expect(
messageOverridesForText(
slackOptions({
messageOverridesStrategy: createOpenAiMessageOverridesStrategy({
apiKey: 'test-key',
fetch: (async () =>
new Response('secret-token=do-not-log', {
fetch: (async () => {
requestCount += 1
return new Response('secret-token=do-not-log', {
status: 503,
statusText: 'Service Unavailable'
})) as unknown as typeof fetch,
})
}) as unknown as typeof fetch,
model: 'gpt-5.4-nano'
})
}),
'use sol',
trace
)
).resolves.toEqual({ overrides: {} })
).resolves.toEqual({
overrides: {
harnessType: 'codex',
model: 'gpt-5.6-sol',
provider: undefined,
reasoning: undefined
}
})
expect(requestCount).toBe(0)
})

test('fuzzy-matches a typo inside an explicit selection clause', async () => {
let requestCount = 0
const strategy = createOpenAiMessageOverridesStrategy({
apiKey: 'test-key',
fetch: (async () => {
requestCount += 1
throw new Error('a deterministic selection must not call the strategy model')
}) as unknown as typeof fetch,
model: 'gpt-5.4-nano'
})

await expect(strategy({ text: 'switch to sonet for this review' })).resolves.toEqual({
overrides: {
harnessType: 'claudecode',
model: 'claude-sonnet-4-6',
provider: undefined,
reasoning: undefined
}
})
expect(requestCount).toBe(0)
})

test('does not fuzzy-match model-like words without selection intent', async () => {
let requestCount = 0
const strategy = createOpenAiMessageOverridesStrategy({
apiKey: 'test-key',
fetch: (async () => {
requestCount += 1
return Response.json({
output: [
{
content: [
{
text: JSON.stringify({
harness: null,
model: null,
provider: null,
reasoning: null
})
}
]
}
]
})
}) as unknown as typeof fetch,
model: 'gpt-5.4-nano'
})

await expect(
strategy({ text: 'compare the Sonet protocol with Claude docs' })
).resolves.toEqual({
overrides: {
harnessType: undefined,
model: undefined,
provider: undefined,
reasoning: undefined
}
})
expect(requestCount).toBe(1)
})

test('does not select a model mentioned later in a task clause', async () => {
let requestCount = 0
const strategy = createOpenAiMessageOverridesStrategy({
apiKey: 'test-key',
fetch: (async () => {
requestCount += 1
return new Response('unavailable', { status: 503 })
}) as unknown as typeof fetch,
model: 'gpt-5.4-nano'
})

await expect(strategy({ text: 'use this tool to compare Claude documentation' })).resolves.toEqual(
{ overrides: {} }
)
expect(requestCount).toBe(1)
})

test('handles --nanocodex deterministically before the OpenAI strategy', async () => {
Expand All @@ -580,7 +669,7 @@ describe('messageOverridesForText strategy invocation', () => {
expect(requestCount).toBe(0)
})

test('allows the OpenAI strategy to select nanocodex from natural language', async () => {
test('selects nanocodex from natural language without an OpenAI request', async () => {
let requestBody: Record<string, unknown> | undefined
const strategy = createOpenAiMessageOverridesStrategy({
apiKey: 'test-key',
Expand Down Expand Up @@ -614,7 +703,7 @@ describe('messageOverridesForText strategy invocation', () => {
reasoning: undefined
}
})
expect(JSON.stringify(requestBody)).toContain('nanocodex')
expect(requestBody).toBeUndefined()
})
})

Expand Down