LLM - Raise or remove LLM timeout when local endpoint detected #4225 (#4254)

Author: dgtlmoon

changedetectionio/blueprint/settings/llm.py

@@ -189,17 +189,19 @@ def llm_test():
             # stay on a small base cap (matching upstream's pre-existing behavior) and only
             # reasoning-capable endpoints (Ollama, openai_compatible) opt into the extra
             # headroom needed for chain-of-thought to complete.
-            # Timeout: omit the override so the test inherits DEFAULT_TIMEOUT (60s, tunable
-            # via LLM_TIMEOUT). A shorter test-only timeout falsely fails on cold-starting
-            # cloud reasoning models (e.g. ollama.com hosting qwen3.5:397b takes ~60s on
-            # first hit) even though the same call succeeds in production.
-            from changedetectionio.llm.evaluator import apply_local_token_multiplier, get_llm_settings
+            # Timeout: resolve it the same way production calls do — cloud gets
+            # DEFAULT_TIMEOUT (300s, tunable via LLM_TIMEOUT), and local/self-hosted
+            # endpoints (IANA-restricted api_base) get the relaxed 1800s local cap so
+            # a cold-starting or slow-prefilling local model doesn't falsely fail the
+            # test even though the same call would succeed in production (issue #4225).
+            from changedetectionio.llm.evaluator import apply_local_token_multiplier, get_llm_settings, resolve_llm_timeout
             text, total_tokens, input_tokens, output_tokens = completion(
                 model=model,
                 messages=[{'role': 'user', 'content':
                     'Respond with just the word: ready'}],
                 api_key=llm_cfg.get('api_key') or None,
                 api_base=api_base or None,
+                timeout=resolve_llm_timeout(llm_cfg),
                 max_tokens=apply_local_token_multiplier(200, llm_cfg),
                 debug=get_llm_settings(datastore).debug,
             )

changedetectionio/llm/client.py

@@ -14,7 +14,16 @@
 # _summary_max_tokens() and are NOT subject to this cap.
 _MAX_COMPLETION_TOKENS = 400
 
-DEFAULT_TIMEOUT = int(os.getenv('LLM_TIMEOUT', 60))
+# Default request timeout (seconds). Raised from 60 to 300 because even cloud
+# reasoning models can be slow on the first hit (issue #4225). Overridable via
+# LLM_TIMEOUT.
+DEFAULT_TIMEOUT = int(os.getenv('LLM_TIMEOUT', 300))
+# Relaxed timeout for local / self-hosted endpoints (Ollama, vLLM, LM Studio,
+# llama.cpp on localhost or a LAN address). These run on modest hardware and can
+# spend many minutes on prompt prefill before the first token, so they get a much
+# longer deadline (Hermes-style, 30 min). Overridable via LLM_LOCAL_TIMEOUT; see
+# evaluator.resolve_llm_timeout() for how the endpoint is classified.
+DEFAULT_LOCAL_TIMEOUT = int(os.getenv('LLM_LOCAL_TIMEOUT', 1800))
 DEFAULT_RETRIES = 3
 
 
@@ -63,6 +72,9 @@ def completion(model: str, messages: list, api_key: str = None,
     Retries up to DEFAULT_RETRIES times on timeout or connection errors.
     Token counts are 0 if the provider doesn't return usage data.
     Raises on network/auth errors — callers handle gracefully.
+
+    timeout: seconds for the request. Local endpoints get a longer value than cloud —
+    see evaluator.resolve_llm_timeout().
     """
     try:
         import litellm

changedetectionio/llm/evaluator.py

@@ -12,6 +12,11 @@
   LLM_MODEL    — model string (e.g. "gpt-4o-mini", "ollama/llama3.2")
   LLM_API_KEY  — API key for cloud providers
   LLM_API_BASE — base URL for local/custom endpoints (e.g. http://localhost:11434)
+  LLM_TIMEOUT       — per-request timeout in seconds (default 300). When set, it
+                      applies to every endpoint (a single hard ceiling).
+  LLM_LOCAL_TIMEOUT — timeout in seconds for local/self-hosted endpoints (api_base
+                      on a private/LAN address); default 1800. Applied automatically
+                      unless LLM_TIMEOUT is set. See resolve_llm_timeout().
 """
 
 import hashlib
@@ -184,6 +189,50 @@ def apply_local_token_multiplier(base_max_tokens: int, llm_cfg: dict) -> int:
     return base_max_tokens * multiplier
 
 
+def _is_local_llm_endpoint(llm_cfg: dict) -> bool:
+    """
+    True when the configured `api_base` points at an IANA-restricted host
+    (private / loopback / link-local / reserved) — i.e. a local or LAN
+    self-hosted LLM (Ollama, vLLM, LM Studio, llama.cpp, ...).
+
+    Detection is purely by the api_base host, reusing the same IANA check the
+    SSRF guard uses (is_private_hostname). Because it resolves DNS, docker
+    service names (`http://ollama:11434`), `host.docker.internal`, and bare LAN
+    IPs are all recognised. No api_base (cloud providers) → False.
+    """
+    api_base = ((llm_cfg or {}).get('api_base') or '').strip()
+    if not api_base:
+        return False
+    try:
+        from urllib.parse import urlparse
+        from changedetectionio.validate_url import is_private_hostname
+        host = urlparse(api_base).hostname
+        return bool(host) and is_private_hostname(host)
+    except Exception:
+        # Never let timeout resolution break an LLM call — fall back to "not local".
+        return False
+
+
+def resolve_llm_timeout(llm_cfg: dict) -> int:
+    """
+    Per-request timeout (seconds) for an LLM call.
+
+    Cloud providers get client.DEFAULT_TIMEOUT (300s, tunable via LLM_TIMEOUT).
+    Local / self-hosted endpoints run on modest hardware and can spend many minutes
+    on prompt prefill before the first token, so a 300s cap trips prematurely
+    (issue #4225). When the api_base host is IANA-restricted (see
+    _is_local_llm_endpoint) we grant client.DEFAULT_LOCAL_TIMEOUT (1800s, tunable
+    via LLM_LOCAL_TIMEOUT) — mirroring how Hermes relaxes its timeouts for local
+    endpoints. An explicit LLM_TIMEOUT always wins, even for local endpoints, for
+    operators who want a single hard ceiling regardless.
+    """
+    if os.getenv('LLM_TIMEOUT', '').strip():
+        return llm_client.DEFAULT_TIMEOUT
+    if _is_local_llm_endpoint(llm_cfg):
+        return llm_client.DEFAULT_LOCAL_TIMEOUT
+    return llm_client.DEFAULT_TIMEOUT
+
+
 # ---------------------------------------------------------------------------
 # Intent resolution
 # ---------------------------------------------------------------------------
@@ -469,6 +518,7 @@ def run_setup(watch, datastore, snapshot_text: str) -> None:
             ],
             api_key=cfg.get('api_key'),
             api_base=cfg.get('api_base'),
+            timeout=resolve_llm_timeout(cfg),
             max_tokens=apply_local_token_multiplier(JSON_RESPONSE_MAX_TOKENS, cfg),
             extra_body=_thinking_extra_body(cfg['model'], settings.thinking_budget),
             debug=settings.debug,
@@ -617,6 +667,7 @@ def summarise_change(watch, datastore, diff: str, current_snapshot: str = '') ->
             ],
             api_key=cfg.get('api_key'),
             api_base=cfg.get('api_base'),
+            timeout=resolve_llm_timeout(cfg),
             max_tokens=apply_local_token_multiplier(
                 _summary_max_tokens(diff, max_cap=settings.max_summary_tokens),
                 cfg,
@@ -684,6 +735,7 @@ def preview_extract(watch, datastore, content: str) -> dict | None:
             ],
             api_key=cfg.get('api_key'),
             api_base=cfg.get('api_base'),
+            timeout=resolve_llm_timeout(cfg),
             max_tokens=apply_local_token_multiplier(JSON_RESPONSE_MAX_TOKENS, cfg),
             extra_body=_thinking_extra_body(cfg['model'], settings.thinking_budget),
             debug=settings.debug,
@@ -770,6 +822,7 @@ def evaluate_change(watch, datastore, diff: str, current_snapshot: str = '') ->
             ],
             api_key=cfg.get('api_key'),
             api_base=cfg.get('api_base'),
+            timeout=resolve_llm_timeout(cfg),
             max_tokens=apply_local_token_multiplier(JSON_RESPONSE_MAX_TOKENS, cfg),
             extra_body=_thinking_extra_body(cfg['model'], settings.thinking_budget),
             debug=settings.debug,

changedetectionio/processors/restock_diff/plugins/llm_restock.py

@@ -197,7 +197,7 @@ def get_itemprop_availability_override(content, fetcher_name, fetcher_instance,
         return None
 
     try:
-        from changedetectionio.llm.evaluator import _runtime_llm_config, accumulate_global_tokens, get_llm_settings
+        from changedetectionio.llm.evaluator import _runtime_llm_config, accumulate_global_tokens, get_llm_settings, resolve_llm_timeout
         from changedetectionio.llm import client as llm_client
     except ImportError as e:
         logger.debug(f"LLM restock fallback: LLM libraries not available ({e})")
@@ -236,6 +236,7 @@ def get_itemprop_availability_override(content, fetcher_name, fetcher_instance,
             ],
             api_key=llm_cfg.get('api_key'),
             api_base=llm_cfg.get('api_base'),
+            timeout=resolve_llm_timeout(llm_cfg),
             # 80 fits a {price, currency, availability} JSON answer comfortably for cloud
             # models. Local reasoning models burn most of that on chain-of-thought before
             # the JSON lands — the multiplier scales it up only when provider_kind says so.

changedetectionio/tests/llm/test_llm_restock_plugin.py

@@ -213,7 +213,7 @@ def test_llm_intent_appended_to_user_prompt(self):
         llm_restock.datastore = ds
 
         captured = {}
-        def fake_completion(model, messages, api_key, api_base, max_tokens):
+        def fake_completion(model, messages, api_key, api_base, max_tokens, timeout=None):
             captured['messages'] = messages
             return ('{"price": 299.0, "currency": "USD", "availability": "instock"}', 50, 40, 10)
 
@@ -237,7 +237,7 @@ def test_no_intent_prompt_unchanged(self):
         llm_restock.datastore = ds
 
         captured = {}
-        def fake_completion(model, messages, api_key, api_base, max_tokens):
+        def fake_completion(model, messages, api_key, api_base, max_tokens, timeout=None):
             captured['messages'] = messages
             return ('{"price": 9.99, "currency": "USD", "availability": "instock"}', 20, 15, 5)