diff --git a/openhands-sdk/openhands/sdk/llm/utils/unverified_models.py b/openhands-sdk/openhands/sdk/llm/utils/unverified_models.py index fc6c029cb0..271ece651e 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/unverified_models.py +++ b/openhands-sdk/openhands/sdk/llm/utils/unverified_models.py @@ -88,10 +88,40 @@ def _split_is_actually_version(split: list[str]) -> bool: ) +def _get_litellm_provider_names() -> set[str]: + try: + provider_list = litellm.provider_list + except AttributeError: + return set() + + result: set[str] = set() + + # In LiteLLM 1.80.10, this is `list(LlmProviders)` i.e. enum members. + for p in provider_list: + if isinstance(p, str): + if p: + result.add(p) + continue + + result.add(p.value) + + return result + + +_LITELLM_PROVIDER_NAMES = _get_litellm_provider_names() + + def _extract_model_and_provider(model: str) -> tuple[str, str, str]: + """Extract provider and model information from a model identifier. + + This is intentionally conservative: + - Only treat the prefix as a provider if it is a known LiteLLM provider. + - Otherwise, return empty provider (caller will bucket it under "other"). + + This prevents bogus providers like "us", "eu", "low", "1024-x-1024" from + leaking into downstream UIs. """ - Extract provider and model information from a model identifier. - """ + separator = "/" split = model.split(separator) @@ -116,6 +146,10 @@ def _extract_model_and_provider(model: str) -> tuple[str, str, str]: provider = split[0] model_id = separator.join(split[1:]) + + if provider not in _LITELLM_PROVIDER_NAMES: + return "", model, "" + return provider, model_id, separator diff --git a/tests/sdk/llm/test_model_list.py b/tests/sdk/llm/test_model_list.py index c11457e734..d568e669a2 100644 --- a/tests/sdk/llm/test_model_list.py +++ b/tests/sdk/llm/test_model_list.py @@ -17,7 +17,9 @@ def test_organize_models_and_providers(): "mistral/devstral-small-2505", "anthropic.claude-3-5", # Ignore dot separator for anthropic "unknown-model", - "custom-provider/custom-model", + "custom-provider/custom-model", # invalid provider -> bucketed under "other" + "us.anthropic.claude-3-5-sonnet-20241022-v2:0", # invalid provider prefix + "1024-x-1024/gpt-image-1.5", # invalid provider prefix "openai/another-model", ] @@ -35,8 +37,40 @@ def test_organize_models_and_providers(): assert len(result["openai"]) == 1 assert "another-model" in result["openai"] - assert len(result["other"]) == 1 + assert len(result["other"]) == 4 assert "unknown-model" in result["other"] + assert "custom-provider/custom-model" in result["other"] + assert "us.anthropic.claude-3-5-sonnet-20241022-v2:0" in result["other"] + assert "1024-x-1024/gpt-image-1.5" in result["other"] + + +def test_unverified_models_fallback_when_no_provider_list(): + models = [ + "openai/gpt-4o", # treated as unverified (provider validation disabled) + "anthropic/claude-sonnet-4-20250514", # treated as unverified + "o3", # openhands model -> excluded + "custom-provider/custom-model", # invalid provider -> bucketed under "other" + "us.anthropic.claude-3-5-sonnet-20241022-v2:0", # invalid provider prefix + "1024-x-1024/gpt-image-1.5", # invalid provider prefix + ] + + with ( + patch( + "openhands.sdk.llm.utils.unverified_models.get_supported_llm_models", + return_value=models, + ), + patch( + "openhands.sdk.llm.utils.unverified_models._LITELLM_PROVIDER_NAMES", + set(), + ), + ): + result = get_unverified_models() + + assert "openai" not in result + assert "anthropic" not in result + assert result == { + "other": ["openai/gpt-4o", "anthropic/claude-sonnet-4-20250514"] + models[3:] + } def test_list_bedrock_models_without_boto3(monkeypatch):