feat(function): Add tool streaming, XML Tool Call Parsing Support #7865
Conversation
✅ Deploy Preview for localai ready!
To edit notification comments on pull requests, go to your Netlify project configuration. |
Extend the function parsing system in LocalAI to support XML-style tool calls, similar to how JSON tool calls are currently parsed. This will allow models that return XML format (like <tool_call><function=name><parameter=key>value</parameter></function></tool_call>) to be properly parsed alongside text content. Signed-off-by: Ettore Di Giacinto <[email protected]>
mudler
force-pushed
the
feat/xml-tool-call
branch
from
e800bbe to
e4ea2f1
Compare
… no tools Signed-off-by: Ettore Di Giacinto <[email protected]>
Signed-off-by: Ettore Di Giacinto <[email protected]>
github-actions
bot
added
the
kind/documentation
![github-advanced-security[bot]](https://avatars.githubusercontent.com/in/57789?s=60&v=4){kind=small}
| returnError := func(err error, canRecover bool) (bool, error) { | ||
| xlog.Debug("Failed to parse XML tool call", "error", err, "position", p.pos) | ||
| if canRecover && recovery { | ||
| p.MoveTo(startPos) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| valEndSize := len(format.ValEnd) | ||
|
|
||
| if format.LastValEnd != nil { | ||
| p.MoveTo(savedPos) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| p.MoveTo(savedPos) | ||
| tc2 := p.tryFind2LiteralSplitBySpaces(*format.LastValEnd, format.ToolEnd) | ||
| if format.LastToolEnd != nil { | ||
| p.MoveTo(savedPos) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| if tc.Groups[0].Begin+len(*format.LastValEnd) < len(p.input) { | ||
| tc.Groups[0].End = tc.Groups[0].Begin + len(*format.LastValEnd) | ||
| } | ||
| p.MoveTo(tc.Groups[0].End) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| // Try to find first literal | ||
| tc1 := p.TryFindLiteral(literal1) | ||
| if tc1 == nil { | ||
| p.MoveTo(savedPos) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| // Try to find second literal | ||
| tc2 := p.TryFindLiteral(literal2) | ||
| if tc2 == nil { | ||
| p.MoveTo(savedPos) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| for _, fmtPreset := range formats { | ||
| if fmtPreset.format != nil { | ||
| // Try parsing with this format | ||
| parser.MoveTo(0) |
Check warning
Code scanning / gosec
Errors unhandled Warning
Signed-off-by: Ettore Di Giacinto <[email protected]>
mudler
force-pushed
the
feat/xml-tool-call
branch
from
fdaf6a0 to
335a23d
Compare
| p.MoveTo(tc.Groups[0].End) | ||
| toolEndSize = len(*format.LastToolEnd) | ||
| } else { | ||
| p.MoveTo(savedPos) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| } | ||
| if !AllSpace(tc.Prelude) { | ||
| // Non-whitespace before scope_start, stop parsing | ||
| p.MoveTo(tc.Groups[0].Begin - len(tc.Prelude)) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| testInput := p.input[testPos:] | ||
| if strings.HasPrefix(testInput, format.ScopeStart) { | ||
| // It's another scope_start, break to continue outer loop | ||
| p.MoveTo(testPos) |
Check warning
Code scanning / gosec
Errors unhandled Warning
Signed-off-by: Ettore Di Giacinto <[email protected]>
|
|
||
| if !AllSpace(tc.Prelude) { | ||
| // Non-whitespace before tool_start, stop parsing | ||
| p.MoveTo(tc.Groups[0].Begin - len(tc.Prelude)) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| (format.LastToolEnd != nil && strings.Contains(functionNamePrelude, *format.LastToolEnd)) { | ||
| // Empty tool call - function name is empty, tool_end is in the prelude | ||
| // Move back to start of tool_start and find tool_end | ||
| p.MoveTo(tc.Groups[0].Begin) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| if AllSpace(format.ToolSep) { | ||
| // GLM 4.5 format: function name is on a separate line after tool_start, before key_start | ||
| // The prelude contains the function name | ||
| p.MoveTo(funcName.Groups[0].Begin) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| p.MoveTo(funcName.Groups[0].Begin) | ||
| } else { | ||
| // Standard format: function name is before tool_sep | ||
| p.MoveTo(funcName.Groups[0].End) |
Check warning
Code scanning / gosec
Errors unhandled Warning
|
|
||
| if !AllSpace(keyStart.Prelude) { | ||
| // Non-whitespace before key_start, stop parsing parameters | ||
| p.MoveTo(keyStart.Groups[0].Begin - len(keyStart.Prelude)) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| jsonParsed = true | ||
| } else { | ||
| // Reset position if JSON parsing failed | ||
| p.MoveTo(valStart) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| } | ||
|
|
||
| // Rewind to json_end and check if val_end follows | ||
| p.MoveTo(jsonEnd) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| } | ||
| } else { | ||
| // val_end doesn't follow - rewind and parse as text | ||
| p.MoveTo(valStart) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| } | ||
|
|
||
| // Parse tool calls | ||
| p.MoveTo(tc.Groups[0].Begin) |
Check warning
Code scanning / gosec
Errors unhandled Warning
| endOfTool := p.pos | ||
| p.ConsumeSpaces() | ||
| if p.pos != len(p.input) { | ||
| p.MoveTo(endOfTool) |
Check warning
Code scanning / gosec
Errors unhandled Warning
Signed-off-by: Ettore Di Giacinto <[email protected]>
Signed-off-by: Ettore Di Giacinto <[email protected]>
mudler
changed the title
Signed-off-by: Ettore Di Giacinto <[email protected]>
Signed-off-by: Ettore Di Giacinto <[email protected]>
| // generateHealingMarker generates a unique marker that doesn't appear in the input | ||
| func generateHealingMarker(input string) string { | ||
| for { | ||
| id := fmt.Sprintf("%d", rand.Int63()) |
Check failure
Code scanning / gosec
Use of weak random number generator (math/rand or math/rand/v2 instead of crypto/rand) Error
mudler
added
enhancement
Signed-off-by: Ettore Di Giacinto <[email protected]>
github-actions
bot
added
the
kind/documentation
Extend the function parsing system in LocalAI to support tools streaming, more robust parsing, XML-style tool calls, similar to how JSON tool calls are currently parsed. This will allow models that return XML format (like <tool_call><function=name><parameter=key>value</tool_call>) to be properly parsed alongside text content. The implementation mimics the great work on llama.cpp and extends it across all backends. This also makes it easier to maintain long-term.