Add ovis models support with default buckets #846

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

testdig wants to merge 23 commits into vllm-project:main from testdig:ovis

+21 −0

vllm_gaudi/extension/bucketing/vision.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -11,6 +11,10 @@ @@
         },
         # Pixel-based models
+        'ovis': {
+            'is_batch_based': False,
+            'buckets': [1600, 3136, 4096, 6400]
+        },
         'ovis2.5': {
             'is_batch_based': False,
             'buckets': [784, 1600, 3136, 4096, 6400, 7744, 9216, 12544]
@@ Expand Down @@

vllm_gaudi/models/__init__.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -15,3 +15,6 @@ def register_model(): @@
         from vllm_gaudi.models.qwen3_vl import HpuQwen3_VLForConditionalGeneration  # noqa: F401
         ModelRegistry.register_model("Qwen3VLForConditionalGeneration",
                                      "vllm_gaudi.models.qwen3_vl:HpuQwen3_VLForConditionalGeneration")
+        from vllm_gaudi.models.ovis import HpuOvis  # noqa: F401
+        ModelRegistry.register_model("Ovis", "vllm_gaudi.models.ovis:HpuOvis")

vllm_gaudi/models/ovis.py

-Original file line number
+Diff line change
@@ -0,0 +1,14 @@
+    # SPDX-License-Identifier: Apache-2.0
+    from vllm.config import VllmConfig
+    from vllm.model_executor.models.ovis import (Ovis, OvisMultiModalProcessor, OvisProcessingInfo, OvisDummyInputsBuilder)
+    from vllm.multimodal import MULTIMODAL_REGISTRY
+    @MULTIMODAL_REGISTRY.register_processor(OvisMultiModalProcessor,
+                                            info=OvisProcessingInfo,
+                                            dummy_inputs=OvisDummyInputsBuilder)
+    class HpuOvis(Ovis):
+        def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+            super().__init__(vllm_config=vllm_config, prefix=prefix)

Provide feedback