feat(embeddings): make OpenAI-compatible batch size configurable (#1142) (#1143)

OpenAIEmbeddings hardcoded batch_size=100 is incompatible with some
OpenAI-compatible providers that enforce smaller per-request limits
(e.g. DashScope / Aliyun Tongyi caps at 10). Without an override,
retain paths that extract > 10 facts fail with 400 errors.

Expose HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE (default 100) and
propagate it to both the 'openai' and 'openrouter' providers, which
share the same OpenAIEmbeddings client. Values <= 0 or non-integer
are rejected at config load time (_parse_positive_int) to fail fast
instead of triggering infinite loops or zero-step range() calls.

The new HindsightConfig field has a dataclass default so existing
direct constructors (tests, external integrations) keep working.

Fixes #1142.

Author: r266-tech

hindsight-api-slim/hindsight_api/config.py

@@ -177,6 +177,7 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]:
 ENV_EMBEDDINGS_OPENAI_API_KEY = "HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY"
 ENV_EMBEDDINGS_OPENAI_MODEL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL"
 ENV_EMBEDDINGS_OPENAI_BASE_URL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL"
+ENV_EMBEDDINGS_OPENAI_BATCH_SIZE = "HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"
 
 # Gemini/Vertex AI embeddings configuration
 ENV_EMBEDDINGS_GEMINI_API_KEY = "HINDSIGHT_API_EMBEDDINGS_GEMINI_API_KEY"
@@ -468,6 +469,7 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]:
 DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU = False  # Force CPU mode for local embeddings (avoids MPS/XPC issues on macOS)
 DEFAULT_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE = False  # Security: disabled by default, required for some models
 DEFAULT_EMBEDDINGS_OPENAI_MODEL = "text-embedding-3-small"
+DEFAULT_EMBEDDINGS_OPENAI_BATCH_SIZE = 100
 DEFAULT_EMBEDDINGS_GEMINI_MODEL = "gemini-embedding-001"
 DEFAULT_EMBEDDINGS_GEMINI_OUTPUT_DIMENSIONALITY = 768
 DEFAULT_EMBEDDING_DIMENSION = 384
@@ -725,6 +727,25 @@ def _parse_str_list(value: str) -> list[str]:
     return [v.strip() for v in value.split(",") if v.strip()]
 
 
+def _parse_positive_int(name: str, raw: str | None, default: int) -> int:
+    """
+    Parse an env var that must be a positive integer (>= 1).
+
+    Falls back to ``default`` when unset/empty. Raises ValueError on non-integer
+    or non-positive values so misconfiguration fails fast instead of triggering
+    infinite loops or zero-step range() calls downstream.
+    """
+    if raw is None or raw == "":
+        return default
+    try:
+        parsed = int(raw)
+    except ValueError as e:
+        raise ValueError(f"{name} must be an integer, got {raw!r}") from e
+    if parsed < 1:
+        raise ValueError(f"{name} must be >= 1, got {parsed}")
+    return parsed
+
+
 def _validate_extraction_mode(mode: str) -> str:
     """Validate and normalize extraction mode."""
     mode_lower = mode.lower()
@@ -1068,6 +1089,10 @@ class HindsightConfig:
     webhook_event_types: list[str]  # Event types to deliver globally
     webhook_delivery_poll_interval_seconds: int  # How often the delivery worker polls
 
+    # Defaulted fields (source-compatible additions — existing direct constructor callers keep working).
+    # Keep at the end of the dataclass; Python forbids non-default fields after default fields.
+    embeddings_openai_batch_size: int = DEFAULT_EMBEDDINGS_OPENAI_BATCH_SIZE
+
     # Class-level sets for configuration categorization
 
     # CREDENTIAL_FIELDS: Never exposed via API, never configurable per-tenant/bank
@@ -1376,6 +1401,11 @@ def from_env(cls) -> "HindsightConfig":
             in ("true", "1"),
             embeddings_tei_url=os.getenv(ENV_EMBEDDINGS_TEI_URL),
             embeddings_openai_base_url=os.getenv(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None,
+            embeddings_openai_batch_size=_parse_positive_int(
+                ENV_EMBEDDINGS_OPENAI_BATCH_SIZE,
+                os.getenv(ENV_EMBEDDINGS_OPENAI_BATCH_SIZE),
+                DEFAULT_EMBEDDINGS_OPENAI_BATCH_SIZE,
+            ),
             # Cohere embeddings (with backward-compatible fallback to shared API key)
             embeddings_cohere_api_key=os.getenv(ENV_EMBEDDINGS_COHERE_API_KEY) or os.getenv(ENV_COHERE_API_KEY),
             embeddings_cohere_model=os.getenv(ENV_EMBEDDINGS_COHERE_MODEL, DEFAULT_EMBEDDINGS_COHERE_MODEL),

hindsight-api-slim/hindsight_api/engine/embeddings.py

@@ -1100,7 +1100,12 @@ def create_embeddings_from_env() -> Embeddings:
             )
         model = os.environ.get(ENV_EMBEDDINGS_OPENAI_MODEL, DEFAULT_EMBEDDINGS_OPENAI_MODEL)
         base_url = os.environ.get(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None
-        return OpenAIEmbeddings(api_key=api_key, model=model, base_url=base_url)
+        return OpenAIEmbeddings(
+            api_key=api_key,
+            model=model,
+            base_url=base_url,
+            batch_size=config.embeddings_openai_batch_size,
+        )
     elif provider == "openrouter":
         api_key = config.embeddings_openrouter_api_key
         if not api_key:
@@ -1112,6 +1117,7 @@ def create_embeddings_from_env() -> Embeddings:
             api_key=api_key,
             model=config.embeddings_openrouter_model,
             base_url="https://openrouter.ai/api/v1",
+            batch_size=config.embeddings_openai_batch_size,
         )
     elif provider == "cohere":
         api_key = config.embeddings_cohere_api_key

hindsight-api-slim/tests/test_embeddings_openai_batch_size.py

@@ -0,0 +1,150 @@
+"""
+Tests for HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE config wiring.
+
+Regression test for issue #1142: `OpenAIEmbeddings` hardcoded `batch_size=100` is
+incompatible with OpenAI-compatible providers that enforce stricter per-request
+limits (e.g. DashScope / Aliyun Tongyi cap at 10). Users must be able to override
+the batch size via env var so `encode()` splits into smaller chunks.
+"""
+
+import os
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def setup_test_env():
+    """Save/restore env vars touched by these tests."""
+    from hindsight_api.config import clear_config_cache
+
+    env_vars_to_save = [
+        "HINDSIGHT_API_EMBEDDINGS_PROVIDER",
+        "HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY",
+        "HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL",
+        "HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE",
+        "HINDSIGHT_API_EMBEDDINGS_OPENROUTER_API_KEY",
+        "HINDSIGHT_API_LLM_API_KEY",
+        "HINDSIGHT_API_LLM_PROVIDER",
+    ]
+
+    original_values = {key: os.environ.get(key) for key in env_vars_to_save}
+
+    clear_config_cache()
+
+    yield
+
+    for key, original_value in original_values.items():
+        if original_value is None:
+            os.environ.pop(key, None)
+        else:
+            os.environ[key] = original_value
+
+    clear_config_cache()
+
+
+def test_default_openai_batch_size_is_100():
+    """Default batch size is 100 when env var unset (preserves legacy behavior)."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ.pop("HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE", None)
+
+    config = HindsightConfig.from_env()
+    assert config.embeddings_openai_batch_size == 100
+
+
+def test_openai_batch_size_env_var_is_read():
+    """HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE overrides the default."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "10"
+
+    config = HindsightConfig.from_env()
+    assert config.embeddings_openai_batch_size == 10
+
+
+def test_openai_embeddings_provider_uses_configured_batch_size():
+    """create_embeddings_from_env() propagates config to OpenAIEmbeddings for 'openai' provider."""
+    from hindsight_api.engine.embeddings import OpenAIEmbeddings, create_embeddings_from_env
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_PROVIDER"] = "openai"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY"] = "sk-test"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "10"
+
+    embeddings = create_embeddings_from_env()
+    assert isinstance(embeddings, OpenAIEmbeddings)
+    assert embeddings.batch_size == 10
+
+
+def test_openrouter_provider_uses_configured_batch_size():
+    """'openrouter' provider also honors the shared batch-size config (both paths use OpenAIEmbeddings)."""
+    from hindsight_api.engine.embeddings import OpenAIEmbeddings, create_embeddings_from_env
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_PROVIDER"] = "openrouter"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENROUTER_API_KEY"] = "sk-or-test"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "8"
+
+    embeddings = create_embeddings_from_env()
+    assert isinstance(embeddings, OpenAIEmbeddings)
+    assert embeddings.batch_size == 8
+
+
+def test_zero_batch_size_is_rejected():
+    """Zero would cause `range(0, N, 0)` to crash at runtime — fail fast at config load."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "0"
+
+    with pytest.raises(ValueError, match="must be >= 1"):
+        HindsightConfig.from_env()
+
+
+def test_negative_batch_size_is_rejected():
+    """Negative values would silently skip batching — reject at config load."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "-5"
+
+    with pytest.raises(ValueError, match="must be >= 1"):
+        HindsightConfig.from_env()
+
+
+def test_non_numeric_batch_size_is_rejected():
+    """Non-integer strings are rejected with a clear error pointing at the env var name."""
+    from hindsight_api.config import HindsightConfig
+
+    os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "mock"
+    os.environ["HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"] = "not-a-number"
+
+    with pytest.raises(ValueError, match="HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE"):
+        HindsightConfig.from_env()
+
+
+def test_openai_encode_splits_on_configured_batch_size(monkeypatch):
+    """encode() sends multiple upstream requests when len(texts) > batch_size."""
+    from types import SimpleNamespace
+
+    from hindsight_api.engine.embeddings import OpenAIEmbeddings
+
+    emb = OpenAIEmbeddings(api_key="sk-test", model="text-embedding-3-small", batch_size=10)
+
+    calls: list[int] = []
+
+    def fake_create(*, model, input):
+        calls.append(len(input))
+        return SimpleNamespace(data=[SimpleNamespace(index=i, embedding=[0.0] * 1536) for i in range(len(input))])
+
+    emb._client = SimpleNamespace(embeddings=SimpleNamespace(create=fake_create))
+    emb._dimension = 1536
+
+    vectors = emb.encode(["x"] * 25)
+
+    assert len(vectors) == 25
+    assert calls == [10, 10, 5], (
+        f"Expected upstream calls of size 10, 10, 5 when batch_size=10 and 25 inputs, got {calls}"
+    )

hindsight-docs/docs/developer/configuration.md

@@ -401,6 +401,7 @@ export HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF=120.0    # Cap at 2min instead of 1m
 | `HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY` | OpenAI API key (falls back to `HINDSIGHT_API_LLM_API_KEY`) | - |
 | `HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL` | OpenAI embedding model | `text-embedding-3-small` |
 | `HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL` | Custom base URL for OpenAI-compatible API (e.g., Azure OpenAI) | - |
+| `HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE` | Max inputs per `embeddings.create` call for `openai`/`openrouter` providers — lower this when the upstream endpoint enforces stricter limits (e.g. DashScope caps at 10) | `100` |
 | `HINDSIGHT_API_EMBEDDINGS_OPENROUTER_API_KEY` | OpenRouter API key for embeddings (falls back to `HINDSIGHT_API_OPENROUTER_API_KEY`, then `HINDSIGHT_API_LLM_API_KEY`) | - |
 | `HINDSIGHT_API_EMBEDDINGS_OPENROUTER_MODEL` | OpenRouter embedding model | `perplexity/pplx-embed-v1-0.6b` |
 | `HINDSIGHT_API_EMBEDDINGS_COHERE_API_KEY` | Cohere API key for embeddings | - |

skills/hindsight-docs/references/developer/configuration.md

@@ -401,6 +401,7 @@ export HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF=120.0    # Cap at 2min instead of 1m
 | `HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY` | OpenAI API key (falls back to `HINDSIGHT_API_LLM_API_KEY`) | - |
 | `HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL` | OpenAI embedding model | `text-embedding-3-small` |
 | `HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL` | Custom base URL for OpenAI-compatible API (e.g., Azure OpenAI) | - |
+| `HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE` | Max inputs per `embeddings.create` call for `openai`/`openrouter` providers — lower this when the upstream endpoint enforces stricter limits (e.g. DashScope caps at 10) | `100` |
 | `HINDSIGHT_API_EMBEDDINGS_OPENROUTER_API_KEY` | OpenRouter API key for embeddings (falls back to `HINDSIGHT_API_OPENROUTER_API_KEY`, then `HINDSIGHT_API_LLM_API_KEY`) | - |
 | `HINDSIGHT_API_EMBEDDINGS_OPENROUTER_MODEL` | OpenRouter embedding model | `perplexity/pplx-embed-v1-0.6b` |
 | `HINDSIGHT_API_EMBEDDINGS_COHERE_API_KEY` | Cohere API key for embeddings | - |