Skip to content

Error fix #629

@QuantumSorcerer02

Description

@QuantumSorcerer02

--- FIX FOR gemma/gm/text/_tokenizer.py ---

class Gemma4Tokenizer(Tokenizer):
"""Tokenizer for Gemma 4 models."""

# ADDITION: Masking multimodal and internal reasoning tokens
FORBIDDEN_TOKENS = (
    special_tokens.IMAGE_PLACEHOLDER,
    special_tokens.START_OF_IMAGE,
    special_tokens.END_OF_IMAGE,
    special_tokens.AUDIO_PLACEHOLDER,
    special_tokens.START_OF_AUDIO,
    special_tokens.END_OF_AUDIO,
    special_tokens.THINKING_START,
    special_tokens.THINKING_END,
)

def __init__(self, ...):
    # existing init logic

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type
    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions