# ADDITION: Masking multimodal and internal reasoning tokens
FORBIDDEN_TOKENS = (
special_tokens.IMAGE_PLACEHOLDER,
special_tokens.START_OF_IMAGE,
special_tokens.END_OF_IMAGE,
special_tokens.AUDIO_PLACEHOLDER,
special_tokens.START_OF_AUDIO,
special_tokens.END_OF_AUDIO,
special_tokens.THINKING_START,
special_tokens.THINKING_END,
)
def __init__(self, ...):
# existing init logic
--- FIX FOR gemma/gm/text/_tokenizer.py ---
class Gemma4Tokenizer(Tokenizer):
"""Tokenizer for Gemma 4 models."""