Skip to content

Commit 6df6403

Browse files
[vLLM] Add Omni token counting and text replacement offsets for the Transformers backend (#49435)
* init: Add all files Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * refactor: Split out the modeling-file changes Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * fix: Count Qwen2.5-Omni audio tokens from padded feature frames Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * refactor: Build Gemma3n processor inputs through the ProcessorMixin hooks like Gemma 3 Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * refactor: Move the Gemma3n text type check into validate_inputs Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * refactor: Move the Qwen-Omni processors onto the OmniModalProcessorMixin hooks Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * fix: Process audio as usual when load_audio_from_video gets no videos Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * refactor: Address the round 3 review on the Qwen-Omni processors Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * refactor: Address the round 4 review on the Qwen-Omni processors Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * refactor: Address the round 5 review on the Omni processors Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * test: Move the audio token count test into the Mixin Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * refactor: Address the round 7 review on the Omni processors Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * test: Seed the Cohere ASR double-shift test before the model init Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * fix: Count only standalone audio against the placeholders in the Omni mixin Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> * fix: Return a list of prompts from the Gemma3n inputs layout for audio without text Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> --------- Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com>
1 parent 85db0e0 commit 6df6403

13 files changed

Lines changed: 643 additions & 487 deletions

‎src/transformers/models/embedding_gemma2/processing_embedding_gemma2.py‎

Lines changed: 3 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/transformers/models/gemma3n/processing_gemma3n.py‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
from ...feature_extraction_utils import BatchFeature
1919
from ...image_utils import ImageInput, make_nested_list_of_images
20-
from ...processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
20+
from ...processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack
2121
from ...tokenization_utils_base import PreTokenizedInput, TextInput
2222
from ...utils import auto_docstring
2323

@@ -98,9 +98,38 @@ def prepare_inputs_layout(self, images=None, text=None, audio=None, **kwargs):
9898
text = [" ".join([self.image_token] * len(image_list)) for image_list in images]
9999

100100
if audio is not None and not text:
101-
text = " ".join([self.audio_token] * len(audio))
101+
text = [self.audio_token] * len(audio)
102102
return images, text, None, audio
103103

104+
def _get_num_multimodal_tokens(self, image_sizes=None, audio_lengths=None, **kwargs):
105+
"""
106+
Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.
107+
108+
Args:
109+
image_sizes (`list[list[int]]`, *optional*):
110+
The input sizes formatted as (height, width) per each image.
111+
audio_lengths (`list[int]`, *optional*):
112+
The lengths of audio inputs in number of samples.
113+
114+
Returns:
115+
`MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
116+
input modalities, along with other useful data.
117+
"""
118+
119+
vision_data = {}
120+
if image_sizes is not None:
121+
# NOTE: No image cropping supported yet
122+
num_image_tokens = [self.image_seq_length] * len(image_sizes)
123+
num_image_patches = [1] * len(image_sizes)
124+
125+
vision_data.update({"num_image_tokens": num_image_tokens, "num_image_patches": num_image_patches})
126+
127+
if audio_lengths is not None:
128+
# NOTE: Every audio is padded or truncated to a fixed window
129+
vision_data["num_audio_tokens"] = [self.audio_seq_length] * len(audio_lengths)
130+
131+
return MultiModalData(**vision_data)
132+
104133
@property
105134
def model_input_names(self):
106135
tokenizer_input_names = self.tokenizer.model_input_names + ["token_type_ids"]

‎src/transformers/models/gemma4/processing_gemma4.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -259,8 +259,8 @@ def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, audio_l
259259
input modalities, along with other useful data.
260260
"""
261261

262-
images_kwargs = Gemma4ProcessorKwargs._defaults.get("images_kwargs", {})
263-
images_kwargs.update(kwargs)
262+
merged_kwargs = self._merge_kwargs(self.valid_processor_kwargs, **kwargs)
263+
images_kwargs = merged_kwargs["images_kwargs"]
264264
patch_size = images_kwargs.get("patch_size", None) or self.image_processor.patch_size
265265
pooling_kernel_size = (
266266
images_kwargs.get("pooling_kernel_size", None) or self.image_processor.pooling_kernel_size
@@ -288,8 +288,7 @@ def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, audio_l
288288
vision_data.update({"num_image_tokens": num_image_tokens, "num_image_patches": num_image_patches})
289289

290290
if video_sizes is not None:
291-
videos_kwargs = Gemma4ProcessorKwargs._defaults.get("videos_kwargs", {})
292-
videos_kwargs.update(kwargs)
291+
videos_kwargs = merged_kwargs["videos_kwargs"]
293292
patch_size = videos_kwargs.get("patch_size", None) or self.video_processor.patch_size
294293
pooling_kernel_size = (
295294
videos_kwargs.get("pooling_kernel_size", None) or self.video_processor.pooling_kernel_size

‎src/transformers/models/gemma4_unified/processing_gemma4_unified.py‎

Lines changed: 3 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)