Merge pull request #188 from splunk/codex/ray-2.56-disable-biencoder

fix(ai-tier): pin Ray 2.56 and disable BiEncoder

Author: vavarshn

config/configs/model-scale.yaml

@@ -1,6 +1,6 @@
 applicationScale:
   AllMinilmL6V2: 1
-  BiEncoder: 1
+  BiEncoder: 0
   CrossEncoder: 1
   E5LanguageClassifier: 1
   Entrypoint: 1

docs/deployment/k0s-quick-reference.md

@@ -39,7 +39,7 @@ standalone via `stage-artifacts`), it additionally needs:
 
 | Resource | Minimum | Why |
 |---|---|---|
-| Disk (free) | 250 GB | >120 GB for 11 models + buffer for download staging and upload temp files |
+| Disk (free) | 250 GB | >120 GB for 10 models + buffer for download staging and upload temp files |
 | RAM | 16 GB | Scripts stream large files; less RAM causes swapping and slow uploads |
 | CPU | 4 cores | Parallel upload to MinIO/SeaweedFS/S3 |
 | Internet | Stable broadband | Downloads >120 GB from HuggingFace; safe to re-run — already-staged models are skipped |
@@ -87,50 +87,60 @@ in the cluster config to those mirrored paths (required for both paths —
 air-gap additionally needs every node to resolve that registry with no outbound
 internet):
 
-The default Ray and SAIA images currently use the `preview` tag — set `TAG`
-to match whatever tag you're mirroring:
+The example below mirrors the SAIA, Ray, and SLIM release images with the common
+tag `v1.0`, and the Splunk AI Operator with `v2.8`. Set the corresponding image
+fields in the cluster config to the mirrored paths.
 
 With `crane` (works on Ubuntu and RHEL 9, no Docker daemon required):
 
 ```bash
-TAG="preview"
+TAG="v1.0"
 for repo in \
   splunk/ai-tier-saia-data-loader \
   splunk/ai-tier-saia-api-v2 \
   splunk/ai-tier-saia-api \
   splunk/ai-tier-ray-head \
-  splunk/ai-tier-ray-worker; do
+  splunk/ai-tier-ray-worker \
+  splunk/ai-tier-slim-service; do
   crane copy "docker.io/${repo}:${TAG}" "<your-registry>/${repo}:${TAG}"
 done
+
+crane copy \
+  "docker.io/kpratyush775/splunk-ai-operator:v2.8" \
+  "<your-registry>/splunk/splunk-ai-operator:v2.8"
 ```
 
 With Docker instead:
 
 ```bash
-TAG="preview"
+TAG="v1.0"
 for repo in \
   splunk/ai-tier-saia-data-loader \
   splunk/ai-tier-saia-api-v2 \
   splunk/ai-tier-saia-api \
   splunk/ai-tier-ray-head \
-  splunk/ai-tier-ray-worker; do
+  splunk/ai-tier-ray-worker \
+  splunk/ai-tier-slim-service; do
   docker pull "docker.io/${repo}:${TAG}"
   docker tag "docker.io/${repo}:${TAG}" "<your-registry>/${repo}:${TAG}"
   docker push "<your-registry>/${repo}:${TAG}"
 done
+
+docker pull "docker.io/kpratyush775/splunk-ai-operator:v2.8"
+docker tag "docker.io/kpratyush775/splunk-ai-operator:v2.8" \
+  "<your-registry>/splunk/splunk-ai-operator:v2.8"
+docker push "<your-registry>/splunk/splunk-ai-operator:v2.8"
 ```
 
-After mirroring, replace the five fully qualified `images.ray.*` and
-`images.saia.*` values in the cluster config with the corresponding
-`<your-registry>/splunk/...:preview` paths. Mirror the Slim and operator images
-separately when those components are enabled, using the tags configured for
-your release. For the complete air-gap image list and the bulk `crane copy`
-alternative, see
+After mirroring, replace the fully qualified `images.ray.*`, `images.saia.*`,
+`images.slim.apiImage`, and `images.operator.image` values in the cluster
+config with the corresponding `<your-registry>/...` paths. For the complete
+air-gap image list and the bulk `crane copy` alternative, see
 [DEPLOYMENT_GUIDE.md — Phase 2: Mirror Container Images](../../tools/cluster_setup/DEPLOYMENT_GUIDE.md#phase-2--mirror-container-images).
 
-> `preview` is a mutable tag and the workloads use `imagePullPolicy:
-> IfNotPresent`. Use a new immutable tag or digest for controlled upgrades;
-> rerunning the installer with the same tag may keep the cached image.
+> Image tags can be mutable and the workloads use `imagePullPolicy:
+> IfNotPresent`. Use an immutable digest for controlled upgrades; rerunning the
+> installer with the same tag may keep the cached image.
 
 ---
 
@@ -169,7 +179,7 @@ node count (2 minimum) to get the cluster total.
 
 | Data | Minimum | Notes |
 |---|---|---|
-| Model weights | 250 GB | >120 GB for 11 models + re-staging headroom |
+| Model weights | 250 GB | >120 GB for 10 models + re-staging headroom |
 | Runtime data | 100 GB | Grows with usage |
 | **Total bucket** | **500 GB+** | Sufficient for now |
 
@@ -240,7 +250,7 @@ fully automatic, no manual steps needed. Full commands and details:
 
 ### Model Setup (Standard Path)
 
-Model weights (>120 GB, 11 models) must land in your object store before the
+Model weights (>120 GB, 10 models) must land in your object store before the
 AI platform can serve inference.
 
 - **Full (interactive) install** — the installer always prompts whether to
@@ -358,7 +368,7 @@ MinIO/SeaweedFS/S3) — can be the same machine that runs the installer:
 
 | Resource | Minimum | Why |
 |---|---|---|
-| Disk (free) | 250 GB | >120 GB for 11 models + buffer for download staging and upload temp files |
+| Disk (free) | 250 GB | >120 GB for 10 models + buffer for download staging and upload temp files |
 | RAM | 16 GB | Scripts stream large files; less RAM causes swapping and slow uploads |
 | CPU | 4 cores | Parallel upload to MinIO/SeaweedFS/S3 |
 | Internet | Stable broadband | Downloads >120 GB from HuggingFace; safe to re-run — already-staged models are skipped |

internal/controller/aiservice_controller.go

@@ -202,6 +202,19 @@ func (r *AIServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	return ctrl.Result{}, nil
 }
 
+// aiServiceEventFilter is the controller-wide event filter. Owned-resource
+// predicates are combined with this filter, so ConfigMap data changes must be
+// admitted here as well as by the ConfigMap-specific Owns predicate. This is
+// required for values snapshotted into pod templates, such as EMBEDDING_MODEL.
+func aiServiceEventFilter() predicate.Predicate {
+	return predicate.Or(
+		common.GenerationChangedPredicate(),
+		common.AnnotationChangedPredicate(),
+		common.LabelChangedPredicate(),
+		common.ConfigMapChangedPredicate(),
+	)
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *AIServiceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
@@ -226,11 +239,7 @@ func (r *AIServiceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 			)),
 		).
 		// Add predicates to filter events and avoid unnecessary reconciliations
-		WithEventFilter(predicate.Or(
-			common.GenerationChangedPredicate(),
-			common.AnnotationChangedPredicate(),
-			common.LabelChangedPredicate(),
-		)).
+		WithEventFilter(aiServiceEventFilter()).
 		// Configure concurrency control
 		WithOptions(controller.Options{
 			MaxConcurrentReconciles: aiv1.TotalWorker,

internal/controller/aiservice_controller_test.go

@@ -19,6 +19,7 @@ package controller
 import (
 	"context"
 	"os"
+	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -34,6 +35,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
@@ -415,3 +417,39 @@ var _ = Describe("AIService Controller", func() {
 		})
 	})
 })
+
+func TestAIServiceEventFilter(t *testing.T) {
+	oldConfig := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-saia-config", Namespace: "default", ResourceVersion: "1"},
+		Data:       map[string]string{"EMBEDDING_MODEL": "uae_large"},
+	}
+
+	t.Run("allows data-only ConfigMap updates through the global filter", func(t *testing.T) {
+		newConfig := oldConfig.DeepCopy()
+		newConfig.ResourceVersion = "2"
+		newConfig.Data["EMBEDDING_MODEL"] = "custom_encoder"
+
+		if !aiServiceEventFilter().Update(event.UpdateEvent{ObjectOld: oldConfig, ObjectNew: newConfig}) {
+			t.Fatal("data-only ConfigMap update was filtered out")
+		}
+	})
+
+	t.Run("filters ConfigMap updates that do not change data or watched metadata", func(t *testing.T) {
+		newConfig := oldConfig.DeepCopy()
+		newConfig.ResourceVersion = "2"
+
+		if aiServiceEventFilter().Update(event.UpdateEvent{ObjectOld: oldConfig, ObjectNew: newConfig}) {
+			t.Fatal("resourceVersion-only ConfigMap update was admitted")
+		}
+	})
+
+	t.Run("preserves generation-change reconciliation for AIService resources", func(t *testing.T) {
+		oldService := &aiv1.AIService{ObjectMeta: metav1.ObjectMeta{Generation: 1}}
+		newService := oldService.DeepCopy()
+		newService.Generation = 2
+
+		if !aiServiceEventFilter().Update(event.UpdateEvent{ObjectOld: oldService, ObjectNew: newService}) {
+			t.Fatal("AIService generation update was filtered out")
+		}
+	})
+}

pkg/ai/features/saia/impl.go

@@ -969,6 +969,40 @@ func saiaEnvFrom(ai *aiv1.AIService) []corev1.EnvFromSource {
 	}
 }
 
+const (
+	defaultV2EmbeddingModel     = "uae_large"
+	unsupportedV2EmbeddingModel = "bi_encoder"
+)
+
+// resolveV2EmbeddingModel reads the optional user override from the shared
+// SAIA ConfigMap and returns the model that the v2 API and worker must use.
+//
+// EMBEDDING_MODEL is added to the containers' explicit env list rather than
+// relying on envFrom. Kubernetes gives explicit env entries precedence over
+// envFrom entries with the same name, so a legacy bi_encoder value cannot make
+// SAIA v2 call the unsupported AI-tier endpoint. Other configured models are
+// preserved. SAIA v1 continues to consume the ConfigMap exactly as before.
+func (r *SaiaReconciler) resolveV2EmbeddingModel(ctx context.Context, ai *aiv1.AIService) (string, error) {
+	cm := &corev1.ConfigMap{}
+	err := r.Get(ctx, types.NamespacedName{
+		Name:      fmt.Sprintf("%s-saia-config", ai.Name),
+		Namespace: ai.Namespace,
+	}, cm)
+	if apierrors.IsNotFound(err) {
+		return defaultV2EmbeddingModel, nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("fetching SAIA ConfigMap for v2 embedding model: %w", err)
+	}
+
+	configured := cm.Data["EMBEDDING_MODEL"]
+	trimmed := strings.TrimSpace(configured)
+	if trimmed == "" || strings.EqualFold(trimmed, unsupportedV2EmbeddingModel) {
+		return defaultV2EmbeddingModel, nil
+	}
+	return configured, nil
+}
+
 // saiaVolumes returns the standard config volume and mount for SAIA pods.
 func saiaVolumes(ai *aiv1.AIService) ([]corev1.Volume, []corev1.VolumeMount) {
 	featureConfigName := fmt.Sprintf("splunk-%s-feature-config", ai.Name)
@@ -1193,6 +1227,11 @@ func (r *SaiaReconciler) reconcileSAIAv2Deployment(
 	ctx context.Context,
 	ai *aiv1.AIService,
 ) error {
+	embeddingModel, err := r.resolveV2EmbeddingModel(ctx, ai)
+	if err != nil {
+		return err
+	}
+
 	volumes, mounts := saiaVolumes(ai)
 	ports := []corev1.ContainerPort{
 		{Name: "http", ContainerPort: 8000},
@@ -1201,7 +1240,10 @@ func (r *SaiaReconciler) reconcileSAIAv2Deployment(
 
 	env := buildSAIABaseEnv(ai)
 	env = append(env, buildV2ExtraEnv(ai)...)
-	env = append(env, corev1.EnvVar{Name: "VAULT_TEMPLATE_DISABLED", Value: "true"})
+	env = append(env,
+		corev1.EnvVar{Name: "EMBEDDING_MODEL", Value: embeddingModel},
+		corev1.EnvVar{Name: "VAULT_TEMPLATE_DISABLED", Value: "true"},
+	)
 	env, volumes, mounts, ports = buildSAIATLSEnv(ai, env, volumes, mounts, ports)
 	sort.Slice(env, func(i, j int) bool { return env[i].Name < env[j].Name })
 
@@ -1311,6 +1353,11 @@ func (r *SaiaReconciler) reconcileSAIAv2Worker(
 	ctx context.Context,
 	ai *aiv1.AIService,
 ) error {
+	embeddingModel, err := r.resolveV2EmbeddingModel(ctx, ai)
+	if err != nil {
+		return err
+	}
+
 	volumes, mounts := saiaVolumes(ai)
 	ports := []corev1.ContainerPort{
 		{Name: "metrics", ContainerPort: 8088},
@@ -1331,6 +1378,7 @@ func (r *SaiaReconciler) reconcileSAIAv2Worker(
 	// conflate with the v1 worker APScheduler cron (which uses 600s for weekly
 	// jobs); v2 reuses the same env name for a different purpose.
 	env = append(env,
+		corev1.EnvVar{Name: "EMBEDDING_MODEL", Value: embeddingModel},
 		corev1.EnvVar{Name: "RUN_TASKS_DELAY_S", Value: "600"},
 		corev1.EnvVar{Name: "VAULT_TEMPLATE_DISABLED", Value: "true"},
 		corev1.EnvVar{Name: "WORKER_HEARTBEAT_PATH", Value: "/tmp/ingestion_worker_heartbeat"},