feat(vllm-cpp): add GLiNER2.5 NER via TokenClassify

backend/backend.proto

@@ -144,6 +144,12 @@ message TokenClassifyRequest {
   // PredictOptions.ModelIdentity for the full rationale. Empty means "no
   // identity supplied" and backends MUST skip the check.
   string ModelIdentity = 3;
+  // Labels overrides the backend's configured entity labels for this
+  // request. Empty means "use the model's configured labels" (the
+  // default for PII detection, where labels are fixed at load time).
+  // Non-empty enables zero-shot per-request label selection (kev /
+  // SystemOne: each question type supplies its own labels).
+  repeated string labels = 4;
 }
 
 // TokenClassifyEntity is one detected entity span. Byte offsets are

backend/go/vllm-cpp/Makefile

@@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
 
 # vllm.cpp version
 VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
-VLLM_CPP_VERSION?=ea8c83d75f461a520e41328c44bde6c949453fa6
+VLLM_CPP_VERSION?=5058268d7c6308d4b3bca731b79fce0399c1672e
 
 # MLX GEMM provider (darwin/metal only; see the metal branch below for why).
 # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun

backend/go/vllm-cpp/backend.go

@@ -10,6 +10,7 @@ package main
 // backend embeds base.Base and not base.SingleThread).
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -275,6 +276,67 @@ func (v *VllmCpp) Predict(opts *pb.PredictOptions) (string, error) {
 	return text, nil
 }
 
+// defaultNerLabels is the general-purpose entity type set used when the model
+// config does not supply ner_labels. These cover the most common NER use cases
+// and match the categories the GLiNER2.5 model card demonstrates.
+var defaultNerLabels = []string{
+	"person", "organization", "location",
+	"date", "time", "money", "quantity",
+}
+
+// TokenClassify runs zero-shot NER on the loaded GLiNER2.5 engine via the
+// vllm_gliner_ner C ABI (ABI v27). The engine refuses non-BoundaryExtractor
+// architectures, so a model loaded for chat or embeddings returns an error
+// here rather than silent garbage.
+func (v *VllmCpp) TokenClassify(_ context.Context, in *pb.TokenClassifyRequest) (*pb.TokenClassifyResponse, error) {
+	if v.engine == 0 {
+		return nil, fmt.Errorf("vllm-cpp: model not loaded")
+	}
+	labels := v.opts.nerLabels
+	if len(in.Labels) > 0 {
+		labels = in.Labels
+	}
+	if len(labels) == 0 {
+		labels = defaultNerLabels
+	}
+	threshold := v.opts.nerThreshold
+	if in.Threshold > 0 {
+		threshold = in.Threshold
+	}
+	maxWidth := v.opts.nerMaxWidth
+
+	labelPtrs, labelBacking := cStringArray(labels)
+	if len(labelPtrs) == 0 {
+		return nil, fmt.Errorf("vllm-cpp: no NER labels configured")
+	}
+	labelsPtr := uintptr(unsafe.Pointer(&labelPtrs[0])) // #nosec G103 -- borrowed by C for the call only
+
+	var out cNerResult
+	rc := vllmGlinerNer(v.engine, in.Text, labelsPtr, int32(len(labelPtrs)), threshold, maxWidth, unsafe.Pointer(&out)) // #nosec G103 -- POD in/out params
+	runtime.KeepAlive(labelBacking)
+	if rc != vllmOK {
+		return nil, fmt.Errorf("vllm-cpp: NER failed: %s", vllmLastError())
+	}
+	defer vllmNerResultFree(unsafe.Pointer(&out)) // #nosec G103 -- frees C-owned members
+
+	entities := make([]*pb.TokenClassifyEntity, 0, out.nEntities)
+	if out.nEntities > 0 && out.entities != 0 {
+		//nolint:govet // C-owned array, valid for this call before vllmNerResultFree
+		cents := unsafe.Slice((*cNerEntity)(unsafe.Pointer(out.entities)), int(out.nEntities)) // #nosec G103 -- C-owned, copied out immediately
+		for i := range cents {
+			e := &cents[i]
+			entities = append(entities, &pb.TokenClassifyEntity{
+				EntityGroup: goString(e.label),
+				Start:       e.charStart,
+				End:         e.charEnd,
+				Score:       e.confidence,
+				Text:        goString(e.text),
+			})
+		}
+	}
+	return &pb.TokenClassifyResponse{Entities: entities}, nil
+}
+
 func (v *VllmCpp) PredictStream(opts *pb.PredictOptions, results chan string) error {
 	if v.engine == 0 {
 		close(results)

backend/go/vllm-cpp/govllmcpp.go

@@ -21,7 +21,7 @@ import (
 // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks
 // the two against each other, because a mismatch is only caught at runtime by
 // registerLib, where it takes the backend down on every load (issue #11379).
-const abiVersion = 26
+const abiVersion = 27
 
 // The ABI's tri-state toggles (enable_prefix_caching ABI v7,
 // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is
@@ -252,8 +252,30 @@ var (
 	vllmVideoResultFree func(out unsafe.Pointer)
 	vllmVideoMuxArgv    func(params, outArgv, outArgc unsafe.Pointer) int32
 	vllmVideoMuxArgvFre func(argv uintptr, argc int32)
+
+	// Zero-shot NER (ABI v27, GLiNER2.5).
+	vllmGlinerNer     func(engine uintptr, text string, labels uintptr, nLabels int32, threshold float32, maxWidth int32, out unsafe.Pointer) int32
+	vllmNerResultFree func(out unsafe.Pointer)
 )
 
+// cNerEntity mirrors vllm_ner_entity. Layout matches the C struct on LP64:
+// two pointer-width fields, four int32, one float, padded to 40 bytes.
+type cNerEntity struct {
+	label      uintptr // char*
+	text       uintptr // char*
+	charStart  int32
+	charEnd    int32
+	tokenStart int32
+	tokenEnd   int32
+	confidence float32
+}
+
+// cNerResult mirrors vllm_ner_result.
+type cNerResult struct {
+	entities  uintptr // vllm_ner_entity*
+	nEntities int32
+}
+
 type libFunc struct {
 	ptr  any
 	name string
@@ -285,6 +307,8 @@ func registerLib(libName string) error {
 		{&vllmVideoResultFree, "vllm_video_result_free"},
 		{&vllmVideoMuxArgv, "vllm_video_mux_argv"},
 		{&vllmVideoMuxArgvFre, "vllm_video_mux_argv_free"},
+		{&vllmGlinerNer, "vllm_gliner_ner"},
+		{&vllmNerResultFree, "vllm_ner_result_free"},
 	} {
 		purego.RegisterLibFunc(lf.ptr, lib, lf.name)
 	}

backend/go/vllm-cpp/options.go

@@ -65,6 +65,15 @@ type loadOptions struct {
 	// MiniMax-H3 video+audio generation (ABI v12). Present only when the config
 	// carries at least one of its keys; see videoOptions.engaged.
 	video videoOptions
+	// Zero-shot NER labels (ABI v27, GLiNER2.5). GLiNER2.5 is truly zero-shot:
+	// the model ships no default labels, so the entity types to extract are
+	// supplied here from engine_args.ner_labels. When empty, a general-purpose
+	// default set is used.
+	nerLabels []string
+	// nerThreshold is the default sigmoid floor (0 = model default 0.5).
+	nerThreshold float32
+	// nerMaxWidth is the maximum span width in tokens (0 = engine default 12).
+	nerMaxWidth int32
 }
 
 // videoOptions is the MiniMax-H3 checkpoint SET plus its generation defaults.
@@ -342,6 +351,22 @@ func applyEngineArgs(lo *loadOptions, engineArgs string) {
 			if b, ok := v.(bool); ok {
 				lo.enableJumpForward = boolTriState(b)
 			}
+		case "ner_labels":
+			if arr, ok := v.([]any); ok {
+				for _, e := range arr {
+					if s, ok := e.(string); ok && s != "" {
+						lo.nerLabels = append(lo.nerLabels, s)
+					}
+				}
+			}
+		case "ner_threshold":
+			if f, ok := v.(float64); ok {
+				lo.nerThreshold = float32(f)
+			}
+		case "ner_max_width":
+			if f, ok := v.(float64); ok {
+				lo.nerMaxWidth = int32(f)
+			}
 		default:
 			if s, ok := videoScalarString(v); ok && applyVideoOption(&lo.video, k, s) {
 				continue

backend/go/vllm-cpp/vllmcpp_test.go

@@ -16,15 +16,15 @@ func TestVllmCpp(t *testing.T) {
 	RunSpecs(t, "vllm-cpp suite")
 }
 
-// The Go POD mirrors must match the C struct layout of vllm.h (ABI v26)
+// The Go POD mirrors must match the C struct layout of vllm.h (ABI v27)
 // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin
 // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h.
 var _ = Describe("C ABI struct mirrors", func() {
 	It("declares the ABI version the pinned engine reports", func() {
 		// VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile).
 		// Moving the pin past this without growing the mirrors below ships a
 		// backend that refuses every load at startup (issue #11379).
-		Expect(abiVersion).To(Equal(26))
+		Expect(abiVersion).To(Equal(27))
 	})
 
 	It("cModelParams matches vllm_model_params", func() {
@@ -92,6 +92,25 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(c.CompletionTokens)).To(Equal(uintptr(20)))
 		Expect(unsafe.Sizeof(c)).To(Equal(uintptr(24)))
 	})
+
+	It("cNerEntity matches vllm_ner_entity (ABI v27)", func() {
+		var e cNerEntity
+		Expect(unsafe.Offsetof(e.label)).To(Equal(uintptr(0)))
+		Expect(unsafe.Offsetof(e.text)).To(Equal(uintptr(8)))
+		Expect(unsafe.Offsetof(e.charStart)).To(Equal(uintptr(16)))
+		Expect(unsafe.Offsetof(e.charEnd)).To(Equal(uintptr(20)))
+		Expect(unsafe.Offsetof(e.tokenStart)).To(Equal(uintptr(24)))
+		Expect(unsafe.Offsetof(e.tokenEnd)).To(Equal(uintptr(28)))
+		Expect(unsafe.Offsetof(e.confidence)).To(Equal(uintptr(32)))
+		Expect(unsafe.Sizeof(e)).To(Equal(uintptr(40)))
+	})
+
+	It("cNerResult matches vllm_ner_result (ABI v27)", func() {
+		var r cNerResult
+		Expect(unsafe.Offsetof(r.entities)).To(Equal(uintptr(0)))
+		Expect(unsafe.Offsetof(r.nEntities)).To(Equal(uintptr(8)))
+		Expect(unsafe.Sizeof(r)).To(Equal(uintptr(16)))
+	})
 })
 
 // Pin/mirror skew is the failure mode this backend is most exposed to: the Go

core/backend/token_classify.go

@@ -30,6 +30,11 @@ type TokenClassifyOptions struct {
 	// callers (e.g. the PII redactor's MinScore) can still filter
 	// further once they know the per-request policy.
 	Threshold float32
+	// Labels overrides the backend's configured entity labels for this
+	// request. Empty means "use the model's configured labels" (the PII
+	// default). Non-empty enables zero-shot per-request label selection
+	// (kev / SystemOne questions).
+	Labels []string
 }
 
 // TokenClassifier runs a token-classification model over text and
@@ -39,6 +44,9 @@ type TokenClassifyOptions struct {
 // core/services/routing/piidetector).
 type TokenClassifier interface {
 	TokenClassify(ctx context.Context, text string) ([]TokenEntity, error)
+	// TokenClassifyWithLabels runs NER with the given labels, overriding
+	// the model's configured labels for this call.
+	TokenClassifyWithLabels(ctx context.Context, text string, labels []string) ([]TokenEntity, error)
 }
 
 // NewTokenClassifier binds (loader, modelConfig, appConfig) into a
@@ -63,6 +71,19 @@ func (m *modelTokenClassifier) TokenClassify(ctx context.Context, text string) (
 	return fn(ctx)
 }
 
+// TokenClassifyWithLabels runs NER with the given labels, overriding the
+// model's configured labels for this call. Used by the SystemOne endpoints
+// where each question supplies its own labels.
+func (m *modelTokenClassifier) TokenClassifyWithLabels(ctx context.Context, text string, labels []string) ([]TokenEntity, error) {
+	opts := m.opts
+	opts.Labels = labels
+	fn, err := ModelTokenClassify(text, opts, m.loader, m.modelConfig, m.appConfig)
+	if err != nil {
+		return nil, err
+	}
+	return fn(ctx)
+}
+
 // ModelTokenClassify loads the backend for modelConfig and returns a
 // closure that classifies `text`. Mirrors ModelScore: the closure is
 // bound to the loaded model so a caller can reuse it within a request
@@ -98,6 +119,7 @@ func ModelTokenClassify(text string, opts TokenClassifyOptions, loader *model.Mo
 			ModelIdentity: modelConfig.Model,
 			Text:          text,
 			Threshold:     opts.Threshold,
+			Labels:        opts.Labels,
 		})
 		entities := tokenClassifyResponseToEntities(resp)
 		if appConfig.EnableTracing {

core/config/backend_capabilities.go

@@ -342,12 +342,17 @@ var BackendCapabilities = map[string]BackendCapability{
 	//
 	// AcceptsImages is the fl2va keyframe (start_image/end_image), the same
 	// reason longcat-video declares it; the text path takes no image input.
+	//
+	// TokenClassify is possible (GLiNER2.5 zero-shot NER via vllm_gliner_ner,
+	// ABI v27), declared explicitly via known_usecases: [token_classify]. The
+	// engine refuses non-BoundaryExtractor architectures, so a chat or embedding
+	// model returns an error rather than silent garbage.
 	"vllm-cpp": {
-		GRPCMethods:      []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateVideo},
-		PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseVideo},
+		GRPCMethods:      []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateVideo, MethodTokenClassify},
+		PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseVideo, UsecaseTokenClassify},
 		DefaultUsecases:  []string{UsecaseChat},
 		AcceptsImages:    true,
-		Description:      "vllm.cpp — the LocalAI team's C++20 port of vLLM; text generation plus MiniMax-H3 video+audio generation",
+		Description:      "vllm.cpp — the LocalAI team's C++20 port of vLLM; text generation, MiniMax-H3 video+audio generation, and GLiNER2.5 zero-shot NER",
 	},
 	"vllm-omni": {
 		GRPCMethods:      []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateImage, MethodGenerateVideo, MethodTTS},

core/http/app.go

@@ -491,6 +491,7 @@ func API(application *application.Application) (*echo.Echo, error) {
 	// mode by attributing requests to the synthetic "local" user.
 	routes.RegisterUsageRoutes(e, application)
 	routes.RegisterPIIRoutes(e, application)
+	routes.RegisterSystemOneRoutes(e, application)
 	routes.RegisterMiddlewareRoutes(e, application)
 
 	routes.RegisterElevenLabsRoutes(e, requestExtractor, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig())