From fbe29f9e6dfc8901bb551333f6ea9186d878604a Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:52:06 +0000 Subject: [PATCH 01/17] Add InternLM3, IQuestCoder and TeleChat3 Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 6 + docs/source/en/_toctree.yml | 6 + docs/source/en/model_doc/internlm3.md | 57 ++ docs/source/en/model_doc/iquestcoder.md | 74 +++ docs/source/en/model_doc/telechat3.md | 59 +++ src/transformers/models/__init__.py | 3 + src/transformers/models/auto/auto_mappings.py | 3 + src/transformers/models/auto/modeling_auto.py | 9 + src/transformers/models/internlm3/__init__.py | 33 ++ .../internlm3/configuration_internlm3.py | 106 ++++ .../models/internlm3/modeling_internlm3.py | 465 ++++++++++++++++ .../models/internlm3/modular_internlm3.py | 120 +++++ .../models/iquestcoder/__init__.py | 33 ++ .../iquestcoder/configuration_iquestcoder.py | 106 ++++ .../iquestcoder/modeling_iquestcoder.py | 498 ++++++++++++++++++ .../models/iquestcoder/modular_iquestcoder.py | 115 ++++ src/transformers/models/telechat3/__init__.py | 33 ++ .../telechat3/configuration_telechat3.py | 138 +++++ .../models/telechat3/modeling_telechat3.py | 474 +++++++++++++++++ .../models/telechat3/modular_telechat3.py | 126 +++++ tests/models/internlm3/__init__.py | 0 .../internlm3/test_modeling_internlm3.py | 75 +++ tests/models/iquestcoder/__init__.py | 0 .../iquestcoder/test_modeling_iquestcoder.py | 75 +++ tests/models/telechat3/__init__.py | 0 .../telechat3/test_modeling_telechat3.py | 102 ++++ 26 files changed, 2716 insertions(+) create mode 100644 docs/source/en/model_doc/internlm3.md create mode 100644 docs/source/en/model_doc/iquestcoder.md create mode 100644 docs/source/en/model_doc/telechat3.md create mode 100644 src/transformers/models/internlm3/__init__.py create mode 100644 src/transformers/models/internlm3/configuration_internlm3.py create mode 100644 src/transformers/models/internlm3/modeling_internlm3.py create mode 100644 src/transformers/models/internlm3/modular_internlm3.py create mode 100644 src/transformers/models/iquestcoder/__init__.py create mode 100644 src/transformers/models/iquestcoder/configuration_iquestcoder.py create mode 100644 src/transformers/models/iquestcoder/modeling_iquestcoder.py create mode 100644 src/transformers/models/iquestcoder/modular_iquestcoder.py create mode 100644 src/transformers/models/telechat3/__init__.py create mode 100644 src/transformers/models/telechat3/configuration_telechat3.py create mode 100644 src/transformers/models/telechat3/modeling_telechat3.py create mode 100644 src/transformers/models/telechat3/modular_telechat3.py create mode 100644 tests/models/internlm3/__init__.py create mode 100644 tests/models/internlm3/test_modeling_internlm3.py create mode 100644 tests/models/iquestcoder/__init__.py create mode 100644 tests/models/iquestcoder/test_modeling_iquestcoder.py create mode 100644 tests/models/telechat3/__init__.py create mode 100644 tests/models/telechat3/test_modeling_telechat3.py diff --git a/.gitattributes b/.gitattributes index ce469199d40d..d93fe2d3a168 100644 --- a/.gitattributes +++ b/.gitattributes @@ -267,7 +267,11 @@ src/transformers/models/inkling/configuration_*.py linguist-generated=true src/transformers/models/inkling/modeling_*.py linguist-generated=true src/transformers/models/instructblipvideo/configuration_*.py linguist-generated=true src/transformers/models/instructblipvideo/modeling_*.py linguist-generated=true +src/transformers/models/internlm3/configuration_*.py linguist-generated=true +src/transformers/models/internlm3/modeling_*.py linguist-generated=true src/transformers/models/internvl/modeling_*.py linguist-generated=true +src/transformers/models/iquestcoder/configuration_*.py linguist-generated=true +src/transformers/models/iquestcoder/modeling_*.py linguist-generated=true src/transformers/models/jais2/configuration_*.py linguist-generated=true src/transformers/models/jais2/modeling_*.py linguist-generated=true src/transformers/models/jamba/modeling_*.py linguist-generated=true @@ -568,6 +572,8 @@ src/transformers/models/t5gemma/configuration_*.py linguist-generated=true src/transformers/models/t5gemma/modeling_*.py linguist-generated=true src/transformers/models/t5gemma2/configuration_*.py linguist-generated=true src/transformers/models/t5gemma2/modeling_*.py linguist-generated=true +src/transformers/models/telechat3/configuration_*.py linguist-generated=true +src/transformers/models/telechat3/modeling_*.py linguist-generated=true src/transformers/models/timesfm/modeling_*.py linguist-generated=true src/transformers/models/timesfm2_5/configuration_*.py linguist-generated=true src/transformers/models/timesfm2_5/modeling_*.py linguist-generated=true diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 0906b5ef3bfe..bf62c2e73aef 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -745,6 +745,10 @@ title: HYV4 - local: model_doc/ibert title: I-BERT + - local: model_doc/internlm3 + title: InternLM3 + - local: model_doc/iquestcoder + title: IQuestCoder - local: model_doc/jais2 title: Jais2 - local: model_doc/jamba @@ -955,6 +959,8 @@ title: T5Gemma2 - local: model_doc/t5v1.1 title: T5v1.1 + - local: model_doc/telechat3 + title: TeleChat3 - local: model_doc/ul2 title: UL2 - local: model_doc/ultrabert diff --git a/docs/source/en/model_doc/internlm3.md b/docs/source/en/model_doc/internlm3.md new file mode 100644 index 000000000000..c71a87236135 --- /dev/null +++ b/docs/source/en/model_doc/internlm3.md @@ -0,0 +1,57 @@ + +*This model was contributed to Hugging Face Transformers on 2026-11-08.* +
+
+ FlashAttention + SDPA +
+
+ +# InternLM3 + +[InternLM3](https://huggingface.co/internlm/internlm3-8b-instruct) is an 8B instruction-tuned language model developed by [Shanghai AI Laboratory](https://huggingface.co/internlm). Its architecture is the same as [Llama](./llama), except that the biases of the query, key and value projections (`qkv_bias`) are configured separately from the biases of the attention output projection and MLP layers (`bias`). + +The checkpoints ship a custom tokenizer, so loading the tokenizer requires `trust_remote_code=True`. + +The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + + +tokenizer = AutoTokenizer.from_pretrained("internlm/internlm3-8b-instruct", trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto") + +messages = [{"role": "user", "content": "Explain the Pythagorean theorem in one sentence."}] +inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True) +output = model.generate(**inputs.to(model.device), max_new_tokens=64) +print(tokenizer.decode(output[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True)) +``` + +## InternLM3Config + +[[autodoc]] InternLM3Config + +## InternLM3Model + +[[autodoc]] InternLM3Model + - forward + +## InternLM3ForCausalLM + +[[autodoc]] InternLM3ForCausalLM + - forward diff --git a/docs/source/en/model_doc/iquestcoder.md b/docs/source/en/model_doc/iquestcoder.md new file mode 100644 index 000000000000..fbbe9ac37208 --- /dev/null +++ b/docs/source/en/model_doc/iquestcoder.md @@ -0,0 +1,74 @@ + +*This model was contributed to Hugging Face Transformers on 2026-11-08.* +
+
+ FlashAttention + SDPA +
+
+ +# IQuestCoder + +[IQuest-Coder-V1](https://huggingface.co/IQuestLab/IQuest-Coder-V1-40B-Instruct) is a family of code language models developed by [IQuest Lab](https://huggingface.co/IQuestLab), available in 7B, 14B and 40B sizes with base, instruct and thinking variants. Its architecture is the same as [Llama](./llama). + +The looped `IQuest-Coder-V1-*-Loop-*` checkpoints use a different architecture and aren't supported by this model. + +The checkpoints ship a custom tokenizer, so loading the tokenizer requires `trust_remote_code=True`. + +The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + + +tokenizer = AutoTokenizer.from_pretrained("IQuestLab/IQuest-Coder-V1-7B-Instruct", trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained("IQuestLab/IQuest-Coder-V1-7B-Instruct", device_map="auto") + +messages = [{"role": "user", "content": "Write a Python function that checks whether a number is prime."}] +inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True) +output = model.generate(**inputs.to(model.device), max_new_tokens=128) +print(tokenizer.decode(output[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True)) +``` + +## IQuestCoderConfig + +[[autodoc]] IQuestCoderConfig + +## IQuestCoderModel + +[[autodoc]] IQuestCoderModel + - forward + +## IQuestCoderForCausalLM + +[[autodoc]] IQuestCoderForCausalLM + - forward + +## IQuestCoderForSequenceClassification + +[[autodoc]] IQuestCoderForSequenceClassification + - forward + +## IQuestCoderForQuestionAnswering + +[[autodoc]] IQuestCoderForQuestionAnswering + - forward + +## IQuestCoderForTokenClassification + +[[autodoc]] IQuestCoderForTokenClassification + - forward diff --git a/docs/source/en/model_doc/telechat3.md b/docs/source/en/model_doc/telechat3.md new file mode 100644 index 000000000000..053be5d56a2d --- /dev/null +++ b/docs/source/en/model_doc/telechat3.md @@ -0,0 +1,59 @@ + +*This model was contributed to Hugging Face Transformers on 2026-11-08.* +
+
+ FlashAttention + SDPA +
+
+ +# TeleChat3 + +[TeleChat3](https://huggingface.co/Tele-AI/TeleChat3-36B-Thinking) is a family of 36B reasoning language models developed by [China Telecom Artificial Intelligence Technology](https://huggingface.co/Tele-AI), including the general-purpose TeleChat3-36B-Thinking and the code-focused TeleChat3-Coder-36B-Thinking. Its architecture is the same as [Llama](./llama). + +The original checkpoints use a custom `"telechat3-yarn"` RoPE type, which is YaRN with 0.07 instead of 0.1 as the coefficient of its default attention factor. [`TeleChat3Config`] converts it to `"yarn"` with an explicit `attention_factor` when the config is loaded. + +The checkpoints ship a custom tokenizer, so loading the tokenizer requires `trust_remote_code=True`. + +The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + + +tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat3-36B-Thinking", trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained("Tele-AI/TeleChat3-36B-Thinking", device_map="auto") + +messages = [{"role": "user", "content": "Explain the Pythagorean theorem in one sentence."}] +inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True) +output = model.generate(**inputs.to(model.device), max_new_tokens=256) +print(tokenizer.decode(output[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True)) +``` + +## TeleChat3Config + +[[autodoc]] TeleChat3Config + +## TeleChat3Model + +[[autodoc]] TeleChat3Model + - forward + +## TeleChat3ForCausalLM + +[[autodoc]] TeleChat3ForCausalLM + - forward diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index fe23a347ee02..9293bee0ecbc 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -243,7 +243,9 @@ from .inkling import * from .instructblip import * from .instructblipvideo import * + from .internlm3 import * from .internvl import * + from .iquestcoder import * from .jais2 import * from .jamba import * from .janus import * @@ -492,6 +494,7 @@ from .t5gemma2 import * from .table_transformer import * from .tapas import * + from .telechat3 import * from .textnet import * from .time_series_transformer import * from .timesfm import * diff --git a/src/transformers/models/auto/auto_mappings.py b/src/transformers/models/auto/auto_mappings.py index 40fe03609692..bed47ad55cda 100644 --- a/src/transformers/models/auto/auto_mappings.py +++ b/src/transformers/models/auto/auto_mappings.py @@ -339,8 +339,10 @@ ("instructblipvideo", "InstructBlipVideoConfig"), ("instructblipvideo_qformer", "InstructBlipVideoQFormerConfig"), ("instructblipvideo_vision_model", "InstructBlipVideoVisionConfig"), + ("internlm3", "InternLM3Config"), ("internvl", "InternVLConfig"), ("internvl_vision", "InternVLVisionConfig"), + ("iquestcoder", "IQuestCoderConfig"), ("jais2", "Jais2Config"), ("jamba", "JambaConfig"), ("janus", "JanusConfig"), @@ -717,6 +719,7 @@ ("t5gemma2_text", "T5Gemma2TextConfig"), ("table-transformer", "TableTransformerConfig"), ("tapas", "TapasConfig"), + ("telechat3", "TeleChat3Config"), ("textnet", "TextNetConfig"), ("time_series_transformer", "TimeSeriesTransformerConfig"), ("timesfm", "TimesFmConfig"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 11ad1fac05f8..ef36eb98458a 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -283,8 +283,10 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("inkling_text", "InklingTextModel"), ("instructblip", "InstructBlipModel"), ("instructblipvideo", "InstructBlipVideoModel"), + ("internlm3", "InternLM3Model"), ("internvl", "InternVLModel"), ("internvl_vision", "InternVLVisionModel"), + ("iquestcoder", "IQuestCoderModel"), ("jais2", "Jais2Model"), ("jamba", "JambaModel"), ("janus", "JanusModel"), @@ -566,6 +568,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("t5gemma2_encoder", "T5Gemma2Encoder"), ("table-transformer", "TableTransformerModel"), ("tapas", "TapasModel"), + ("telechat3", "TeleChat3Model"), ("textnet", "TextNetModel"), ("time_series_transformer", "TimeSeriesTransformerModel"), ("timesfm", "TimesFmModel"), @@ -839,6 +842,8 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("hy_v4", "HYV4ForCausalLM"), ("hyperclovax", "HyperCLOVAXForCausalLM"), ("inkling_text", "InklingForCausalLM"), + ("internlm3", "InternLM3ForCausalLM"), + ("iquestcoder", "IQuestCoderForCausalLM"), ("jais2", "Jais2ForCausalLM"), ("jamba", "JambaForCausalLM"), ("jetmoe", "JetMoeForCausalLM"), @@ -917,6 +922,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("solar_open", "SolarOpenForCausalLM"), ("stablelm", "StableLmForCausalLM"), ("starcoder2", "Starcoder2ForCausalLM"), + ("telechat3", "TeleChat3ForCausalLM"), ("trocr", "TrOCRForCausalLM"), ("vaultgemma", "VaultGemmaForCausalLM"), ("whisper", "WhisperForCausalLM"), @@ -1512,6 +1518,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("hunyuan_v1_dense", "HunYuanDenseV1ForSequenceClassification"), ("hunyuan_v1_moe", "HunYuanMoEV1ForSequenceClassification"), ("ibert", "IBertForSequenceClassification"), + ("iquestcoder", "IQuestCoderForSequenceClassification"), ("jamba", "JambaForSequenceClassification"), ("jetmoe", "JetMoeForSequenceClassification"), ("jina_embeddings_v3", "JinaEmbeddingsV3ForSequenceClassification"), @@ -1621,6 +1628,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("gpt_neox", "GPTNeoXForQuestionAnswering"), ("gptj", "GPTJForQuestionAnswering"), ("ibert", "IBertForQuestionAnswering"), + ("iquestcoder", "IQuestCoderForQuestionAnswering"), ("jina_embeddings_v3", "JinaEmbeddingsV3ForQuestionAnswering"), ("layoutlmv2", "LayoutLMv2ForQuestionAnswering"), ("layoutlmv3", "LayoutLMv3ForQuestionAnswering"), @@ -1743,6 +1751,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("gte", "GteForTokenClassification"), ("helium", "HeliumForTokenClassification"), ("ibert", "IBertForTokenClassification"), + ("iquestcoder", "IQuestCoderForTokenClassification"), ("jina_embeddings_v3", "JinaEmbeddingsV3ForTokenClassification"), ("layoutlm", "LayoutLMForTokenClassification"), ("layoutlmv2", "LayoutLMv2ForTokenClassification"), diff --git a/src/transformers/models/internlm3/__init__.py b/src/transformers/models/internlm3/__init__.py new file mode 100644 index 000000000000..0638ebaf2c91 --- /dev/null +++ b/src/transformers/models/internlm3/__init__.py @@ -0,0 +1,33 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from the modules in `src/transformers/models/internlm3`. +# Do NOT edit this file manually as any edits will be overwritten by auto-generation of the file. +# A module is picked up once it exports a name through `__all__`. +# Regenerate the file with: `python utils/check_inits.py --fix_and_overwrite` +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_internlm3 import * + from .modeling_internlm3 import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/src/transformers/models/internlm3/configuration_internlm3.py b/src/transformers/models/internlm3/configuration_internlm3.py new file mode 100644 index 000000000000..e64fb8145ba7 --- /dev/null +++ b/src/transformers/models/internlm3/configuration_internlm3.py @@ -0,0 +1,106 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/internlm3/modular_internlm3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_internlm3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from huggingface_hub.dataclasses import strict + +from ...configuration_utils import PreTrainedConfig +from ...modeling_rope_utils import RopeParameters +from ...utils import auto_docstring + + +@auto_docstring(checkpoint="internlm/internlm3-8b-instruct") +@strict +class InternLM3Config(PreTrainedConfig): + r""" + qkv_bias (`bool`, *optional*, defaults to `False`): + Whether to use a bias in the query, key and value projection layers. + bias (`bool`, *optional*, defaults to `False`): + Whether to use a bias in the attention output projection and MLP layers. + + ```python + >>> from transformers import InternLM3Model, InternLM3Config + + >>> # Initializing an InternLM3-8B style configuration + >>> configuration = InternLM3Config() + + >>> # Initializing a model from the InternLM3-8B style configuration + >>> model = InternLM3Model(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "internlm3" + keys_to_ignore_at_inference = ["past_key_values"] + # Default tensor parallel plan for base model `InternLM3Model` + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.k_proj": "colwise", + "layers.*.self_attn.v_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + + vocab_size: int = 128512 + hidden_size: int = 4096 + intermediate_size: int = 10240 + num_hidden_layers: int = 48 + num_attention_heads: int = 32 + num_key_value_heads: int | None = 2 + hidden_act: str = "silu" + max_position_embeddings: int = 32768 + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = 2 + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + tie_word_embeddings: bool = False + rope_parameters: RopeParameters | dict | None = None + attention_dropout: float | int = 0.0 + head_dim: int | None = 128 + qkv_bias: bool = False + bias: bool = False + + def __post_init__(self, **kwargs): + if self.head_dim is None: + self.head_dim = self.hidden_size // self.num_attention_heads + if self.num_key_value_heads is None: + self.num_key_value_heads = self.num_attention_heads + + super().__post_init__(**kwargs) + + def validate_architecture(self): + """Part of `@strict`-powered validation. Validates the architecture of the config.""" + if self.hidden_size % self.num_attention_heads != 0: + raise ValueError( + f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention " + f"heads ({self.num_attention_heads})." + ) + + +__all__ = ["InternLM3Config"] diff --git a/src/transformers/models/internlm3/modeling_internlm3.py b/src/transformers/models/internlm3/modeling_internlm3.py new file mode 100644 index 000000000000..3f5f055f77ac --- /dev/null +++ b/src/transformers/models/internlm3/modeling_internlm3.py @@ -0,0 +1,465 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/internlm3/modular_internlm3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_internlm3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable + +import torch +from torch import nn + +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub, use_kernelized_func +from ...masking_utils import create_causal_mask +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import merge_with_config_defaults +from ...utils.output_capturing import capture_outputs +from .configuration_internlm3 import InternLM3Config + + +class InternLM3MLP(nn.Module): + def __init__(self, config: InternLM3Config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.bias) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.bias) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.bias) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +@use_kernel_forward_from_hub("rotary_pos_emb") +def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + scaling: float, + dropout: float = 0.0, + **kwargs: Unpack[TransformersKwargs], +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +@use_kernelized_func(apply_rotary_pos_emb) +class InternLM3Attention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: InternLM3Config, layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads + self.scaling = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + self.is_causal = True + self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.qkv_bias) + self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.qkv_bias) + self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.qkv_bias) + self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.bias) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + attention_mask: torch.Tensor | None = None, + past_key_values: Cache | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple[torch.Tensor, torch.Tensor]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_values is not None: + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +@use_kernel_forward_from_hub("RMSNorm") +class InternLM3RMSNorm(nn.Module): + def __init__(self, hidden_size, eps: float = 1e-6) -> None: + """ + InternLM3RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class InternLM3DecoderLayer(GradientCheckpointingLayer): + def __init__(self, config: InternLM3Config, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + + self.self_attn = InternLM3Attention(config=config, layer_idx=layer_idx) + + self.mlp = InternLM3MLP(config) + self.input_layernorm = InternLM3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = InternLM3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + use_cache: bool | None = False, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + # Self Attention + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +@auto_docstring +class InternLM3PreTrainedModel(PreTrainedModel): + config: InternLM3Config + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["InternLM3DecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn = True + _supports_sdpa = True + _supports_flex_attn = True + + _can_compile_fullgraph = True + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": InternLM3DecoderLayer, + "attentions": InternLM3Attention, + } + + +class InternLM3RotaryEmbedding(nn.Module): + def __init__(self, config: InternLM3Config): + super().__init__() + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_default_rope_parameters + if self.rope_type != "default": + rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + inv_freq, self.attention_scaling = rope_init_fn(self.config) + + self.inv_freq = nn.Buffer(inv_freq, persistent=False) + self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False) + + @staticmethod + def compute_default_rope_parameters(config: InternLM3Config, **kwargs) -> tuple[torch.Tensor, float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + + attention_factor = 1.0 # Unused in this type of RoPE + # Compute the inverse frequencies + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + return inv_freq, attention_factor + + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + freqs = position_ids[..., None].float() * self.inv_freq.to(device=x.device, dtype=torch.float) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +@auto_docstring +class InternLM3Model(InternLM3PreTrainedModel): + def __init__(self, config: InternLM3Config): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [InternLM3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = InternLM3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = InternLM3RotaryEmbedding(config=config) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + @merge_with_config_defaults + @capture_outputs + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.Tensor | None = None, + use_cache: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithPast: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds: torch.Tensor = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if position_ids is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) + + causal_mask = create_causal_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=past_key_values, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) + + for decoder_layer in self.layers[: self.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_embeddings=position_embeddings, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + **kwargs, + ) + + hidden_states = self.norm(hidden_states) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + +@auto_docstring(checkpoint="internlm/internlm3-8b-instruct") +class InternLM3ForCausalLM(InternLM3PreTrainedModel, GenerationMixin): + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + _tp_plan = {"lm_head": "colwise_gather_output"} + _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + _fsdp_plan = {"lm_head": "keep_full_weight"} + + def __init__(self, config: InternLM3Config): + super().__init__(config) + self.model = InternLM3Model(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.Tensor | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + logits_to_keep: int | torch.Tensor = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> CausalLMOutputWithPast: + r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, InternLM3ForCausalLM + + >>> model = InternLM3ForCausalLM.from_pretrained("meta-internlm3/InternLM3-2-7b-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("meta-internlm3/InternLM3-2-7b-hf") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + outputs: BaseModelOutputWithPast = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep) + logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +__all__ = ["InternLM3ForCausalLM", "InternLM3Model", "InternLM3PreTrainedModel"] diff --git a/src/transformers/models/internlm3/modular_internlm3.py b/src/transformers/models/internlm3/modular_internlm3.py new file mode 100644 index 000000000000..c63fb3d66fef --- /dev/null +++ b/src/transformers/models/internlm3/modular_internlm3.py @@ -0,0 +1,120 @@ +# Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from huggingface_hub.dataclasses import strict +from torch import nn + +from ...modeling_rope_utils import RopeParameters +from ...utils import auto_docstring +from ..llama.configuration_llama import LlamaConfig +from ..llama.modeling_llama import ( + LlamaAttention, + LlamaDecoderLayer, + LlamaForCausalLM, + LlamaMLP, + LlamaModel, + LlamaPreTrainedModel, +) + + +@auto_docstring(checkpoint="internlm/internlm3-8b-instruct") +@strict +class InternLM3Config(LlamaConfig): + r""" + qkv_bias (`bool`, *optional*, defaults to `False`): + Whether to use a bias in the query, key and value projection layers. + bias (`bool`, *optional*, defaults to `False`): + Whether to use a bias in the attention output projection and MLP layers. + + ```python + >>> from transformers import InternLM3Model, InternLM3Config + + >>> # Initializing an InternLM3-8B style configuration + >>> configuration = InternLM3Config() + + >>> # Initializing a model from the InternLM3-8B style configuration + >>> model = InternLM3Model(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "internlm3" + + vocab_size: int = 128512 + hidden_size: int = 4096 + intermediate_size: int = 10240 + num_hidden_layers: int = 48 + num_attention_heads: int = 32 + num_key_value_heads: int | None = 2 + hidden_act: str = "silu" + max_position_embeddings: int = 32768 + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = 2 + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + tie_word_embeddings: bool = False + rope_parameters: RopeParameters | dict | None = None + attention_dropout: float | int = 0.0 + head_dim: int | None = 128 + qkv_bias: bool = False + bias: bool = False + + attention_bias = AttributeError() + mlp_bias = AttributeError() + pretraining_tp = AttributeError() + + +class InternLM3MLP(LlamaMLP): + def __init__(self, config: InternLM3Config): + super().__init__(config) + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.bias) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.bias) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.bias) + + +class InternLM3Attention(LlamaAttention): + def __init__(self, config: InternLM3Config, layer_idx: int): + super().__init__(config, layer_idx) + self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.qkv_bias) + self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.qkv_bias) + self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.qkv_bias) + self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.bias) + + +class InternLM3DecoderLayer(LlamaDecoderLayer): + pass + + +class InternLM3PreTrainedModel(LlamaPreTrainedModel): + pass + + +class InternLM3Model(LlamaModel): + pass + + +@auto_docstring(checkpoint="internlm/internlm3-8b-instruct") +class InternLM3ForCausalLM(LlamaForCausalLM): + pass + + +__all__ = [ + "InternLM3Config", + "InternLM3ForCausalLM", + "InternLM3Model", + "InternLM3PreTrainedModel", +] diff --git a/src/transformers/models/iquestcoder/__init__.py b/src/transformers/models/iquestcoder/__init__.py new file mode 100644 index 000000000000..0f0e6330adf3 --- /dev/null +++ b/src/transformers/models/iquestcoder/__init__.py @@ -0,0 +1,33 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from the modules in `src/transformers/models/iquestcoder`. +# Do NOT edit this file manually as any edits will be overwritten by auto-generation of the file. +# A module is picked up once it exports a name through `__all__`. +# Regenerate the file with: `python utils/check_inits.py --fix_and_overwrite` +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_iquestcoder import * + from .modeling_iquestcoder import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/src/transformers/models/iquestcoder/configuration_iquestcoder.py b/src/transformers/models/iquestcoder/configuration_iquestcoder.py new file mode 100644 index 000000000000..b485d37b3935 --- /dev/null +++ b/src/transformers/models/iquestcoder/configuration_iquestcoder.py @@ -0,0 +1,106 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/iquestcoder/modular_iquestcoder.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_iquestcoder.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from huggingface_hub.dataclasses import strict + +from ...configuration_utils import PreTrainedConfig +from ...modeling_rope_utils import RopeParameters +from ...utils import auto_docstring + + +@auto_docstring(checkpoint="IQuestLab/IQuest-Coder-V1-40B-Instruct") +@strict +class IQuestCoderConfig(PreTrainedConfig): + r""" + ```python + >>> from transformers import IQuestCoderModel, IQuestCoderConfig + + >>> # Initializing an IQuest-Coder-V1-40B style configuration + >>> configuration = IQuestCoderConfig() + + >>> # Initializing a model from the IQuest-Coder-V1-40B style configuration + >>> model = IQuestCoderModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "iquestcoder" + keys_to_ignore_at_inference = ["past_key_values"] + # Default tensor parallel plan for base model `IQuestCoderModel` + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.k_proj": "colwise", + "layers.*.self_attn.v_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + + vocab_size: int = 76800 + hidden_size: int = 5120 + intermediate_size: int = 27648 + num_hidden_layers: int = 80 + num_attention_heads: int = 40 + num_key_value_heads: int | None = 8 + hidden_act: str = "silu" + max_position_embeddings: int = 131072 + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = None + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + tie_word_embeddings: bool = False + rope_parameters: RopeParameters | dict | None = None + attention_bias: bool = False + attention_dropout: float | int = 0.0 + mlp_bias: bool = False + head_dim: int | None = 128 + + def __post_init__(self, **kwargs): + if self.head_dim is None: + self.head_dim = self.hidden_size // self.num_attention_heads + if self.num_key_value_heads is None: + self.num_key_value_heads = self.num_attention_heads + + super().__post_init__(**kwargs) + + def validate_architecture(self): + """Part of `@strict`-powered validation. Validates the architecture of the config.""" + if self.hidden_size % self.num_attention_heads != 0: + raise ValueError( + f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention " + f"heads ({self.num_attention_heads})." + ) + # The original code supports these options, but no released checkpoint enables them + if getattr(self, "clip_qkv", None) is not None: + raise ValueError("IQuestCoder does not support `clip_qkv`.") + if getattr(self, "use_sliding_window", False): + raise ValueError("IQuestCoder does not support `use_sliding_window`.") + + +__all__ = ["IQuestCoderConfig"] diff --git a/src/transformers/models/iquestcoder/modeling_iquestcoder.py b/src/transformers/models/iquestcoder/modeling_iquestcoder.py new file mode 100644 index 000000000000..ab75e70df189 --- /dev/null +++ b/src/transformers/models/iquestcoder/modeling_iquestcoder.py @@ -0,0 +1,498 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/iquestcoder/modular_iquestcoder.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_iquestcoder.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable + +import torch +from torch import nn + +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub, use_kernelized_func +from ...masking_utils import create_causal_mask +from ...modeling_layers import ( + GenericForQuestionAnswering, + GenericForSequenceClassification, + GenericForTokenClassification, + GradientCheckpointingLayer, +) +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import merge_with_config_defaults +from ...utils.output_capturing import capture_outputs +from .configuration_iquestcoder import IQuestCoderConfig + + +@use_kernel_forward_from_hub("RMSNorm") +class IQuestCoderRMSNorm(nn.Module): + def __init__(self, hidden_size, eps: float = 1e-6) -> None: + """ + IQuestCoderRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class IQuestCoderMLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +@use_kernel_forward_from_hub("rotary_pos_emb") +def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + scaling: float, + dropout: float = 0.0, + **kwargs: Unpack[TransformersKwargs], +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +@use_kernelized_func(apply_rotary_pos_emb) +class IQuestCoderAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: IQuestCoderConfig, layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads + self.scaling = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + self.is_causal = True + + self.q_proj = nn.Linear( + config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias + ) + self.k_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.v_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.o_proj = nn.Linear( + config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias + ) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + attention_mask: torch.Tensor | None = None, + past_key_values: Cache | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple[torch.Tensor, torch.Tensor]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_values is not None: + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class IQuestCoderDecoderLayer(GradientCheckpointingLayer): + def __init__(self, config: IQuestCoderConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + + self.self_attn = IQuestCoderAttention(config=config, layer_idx=layer_idx) + + self.mlp = IQuestCoderMLP(config) + self.input_layernorm = IQuestCoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = IQuestCoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + use_cache: bool | None = False, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + # Self Attention + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +@auto_docstring +class IQuestCoderPreTrainedModel(PreTrainedModel): + config: IQuestCoderConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["IQuestCoderDecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn = True + _supports_sdpa = True + _supports_flex_attn = True + + _can_compile_fullgraph = True + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": IQuestCoderDecoderLayer, + "attentions": IQuestCoderAttention, + } + + +class IQuestCoderRotaryEmbedding(nn.Module): + def __init__(self, config: IQuestCoderConfig): + super().__init__() + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_default_rope_parameters + if self.rope_type != "default": + rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + inv_freq, self.attention_scaling = rope_init_fn(self.config) + + self.inv_freq = nn.Buffer(inv_freq, persistent=False) + self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False) + + @staticmethod + def compute_default_rope_parameters(config: IQuestCoderConfig, **kwargs) -> tuple[torch.Tensor, float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + + attention_factor = 1.0 # Unused in this type of RoPE + # Compute the inverse frequencies + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + return inv_freq, attention_factor + + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + freqs = position_ids[..., None].float() * self.inv_freq.to(device=x.device, dtype=torch.float) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +@auto_docstring +class IQuestCoderModel(IQuestCoderPreTrainedModel): + def __init__(self, config: IQuestCoderConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [IQuestCoderDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = IQuestCoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = IQuestCoderRotaryEmbedding(config=config) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + @merge_with_config_defaults + @capture_outputs + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.Tensor | None = None, + use_cache: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithPast: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds: torch.Tensor = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if position_ids is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) + + causal_mask = create_causal_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=past_key_values, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) + + for decoder_layer in self.layers[: self.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_embeddings=position_embeddings, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + **kwargs, + ) + + hidden_states = self.norm(hidden_states) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + +@auto_docstring(checkpoint="IQuestLab/IQuest-Coder-V1-40B-Instruct") +class IQuestCoderForCausalLM(IQuestCoderPreTrainedModel, GenerationMixin): + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + _tp_plan = {"lm_head": "colwise_gather_output"} + _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + _fsdp_plan = {"lm_head": "keep_full_weight"} + + def __init__(self, config: IQuestCoderConfig): + super().__init__(config) + self.model = IQuestCoderModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.Tensor | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + logits_to_keep: int | torch.Tensor = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> CausalLMOutputWithPast: + r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, IQuestCoderForCausalLM + + >>> model = IQuestCoderForCausalLM.from_pretrained("meta-iquestcoder/IQuestCoder-2-7b-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("meta-iquestcoder/IQuestCoder-2-7b-hf") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + outputs: BaseModelOutputWithPast = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep) + logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class IQuestCoderForSequenceClassification(GenericForSequenceClassification, IQuestCoderPreTrainedModel): + pass + + +class IQuestCoderForQuestionAnswering(GenericForQuestionAnswering, IQuestCoderPreTrainedModel): + base_model_prefix = "transformer" # For BC, where `transformer` was used instead of `model` + + +class IQuestCoderForTokenClassification(GenericForTokenClassification, IQuestCoderPreTrainedModel): + pass + + +__all__ = [ + "IQuestCoderForCausalLM", + "IQuestCoderForQuestionAnswering", + "IQuestCoderForSequenceClassification", + "IQuestCoderForTokenClassification", + "IQuestCoderModel", + "IQuestCoderPreTrainedModel", +] diff --git a/src/transformers/models/iquestcoder/modular_iquestcoder.py b/src/transformers/models/iquestcoder/modular_iquestcoder.py new file mode 100644 index 000000000000..48dbb2b70399 --- /dev/null +++ b/src/transformers/models/iquestcoder/modular_iquestcoder.py @@ -0,0 +1,115 @@ +# Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from huggingface_hub.dataclasses import strict + +from ...modeling_rope_utils import RopeParameters +from ...utils import auto_docstring +from ..llama.configuration_llama import LlamaConfig +from ..llama.modeling_llama import ( + LlamaForCausalLM, + LlamaForQuestionAnswering, + LlamaForSequenceClassification, + LlamaForTokenClassification, + LlamaModel, + LlamaPreTrainedModel, +) + + +@auto_docstring(checkpoint="IQuestLab/IQuest-Coder-V1-40B-Instruct") +@strict +class IQuestCoderConfig(LlamaConfig): + r""" + ```python + >>> from transformers import IQuestCoderModel, IQuestCoderConfig + + >>> # Initializing an IQuest-Coder-V1-40B style configuration + >>> configuration = IQuestCoderConfig() + + >>> # Initializing a model from the IQuest-Coder-V1-40B style configuration + >>> model = IQuestCoderModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "iquestcoder" + + vocab_size: int = 76800 + hidden_size: int = 5120 + intermediate_size: int = 27648 + num_hidden_layers: int = 80 + num_attention_heads: int = 40 + num_key_value_heads: int | None = 8 + hidden_act: str = "silu" + max_position_embeddings: int = 131072 + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = None + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + tie_word_embeddings: bool = False + rope_parameters: RopeParameters | dict | None = None + attention_bias: bool = False + attention_dropout: float | int = 0.0 + mlp_bias: bool = False + head_dim: int | None = 128 + + pretraining_tp = AttributeError() + + def validate_architecture(self): + """Part of `@strict`-powered validation. Validates the architecture of the config.""" + super().validate_architecture() + # The original code supports these options, but no released checkpoint enables them + if getattr(self, "clip_qkv", None) is not None: + raise ValueError("IQuestCoder does not support `clip_qkv`.") + if getattr(self, "use_sliding_window", False): + raise ValueError("IQuestCoder does not support `use_sliding_window`.") + + +class IQuestCoderPreTrainedModel(LlamaPreTrainedModel): + pass + + +class IQuestCoderModel(LlamaModel): + pass + + +@auto_docstring(checkpoint="IQuestLab/IQuest-Coder-V1-40B-Instruct") +class IQuestCoderForCausalLM(LlamaForCausalLM): + pass + + +class IQuestCoderForSequenceClassification(LlamaForSequenceClassification): + pass + + +class IQuestCoderForQuestionAnswering(LlamaForQuestionAnswering): + pass + + +class IQuestCoderForTokenClassification(LlamaForTokenClassification): + pass + + +__all__ = [ + "IQuestCoderConfig", + "IQuestCoderForCausalLM", + "IQuestCoderForQuestionAnswering", + "IQuestCoderForSequenceClassification", + "IQuestCoderForTokenClassification", + "IQuestCoderModel", + "IQuestCoderPreTrainedModel", +] diff --git a/src/transformers/models/telechat3/__init__.py b/src/transformers/models/telechat3/__init__.py new file mode 100644 index 000000000000..cafe21913613 --- /dev/null +++ b/src/transformers/models/telechat3/__init__.py @@ -0,0 +1,33 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from the modules in `src/transformers/models/telechat3`. +# Do NOT edit this file manually as any edits will be overwritten by auto-generation of the file. +# A module is picked up once it exports a name through `__all__`. +# Regenerate the file with: `python utils/check_inits.py --fix_and_overwrite` +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_telechat3 import * + from .modeling_telechat3 import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/src/transformers/models/telechat3/configuration_telechat3.py b/src/transformers/models/telechat3/configuration_telechat3.py new file mode 100644 index 000000000000..d53ca6012710 --- /dev/null +++ b/src/transformers/models/telechat3/configuration_telechat3.py @@ -0,0 +1,138 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/telechat3/modular_telechat3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_telechat3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +from huggingface_hub.dataclasses import strict + +from ...configuration_utils import PreTrainedConfig +from ...modeling_rope_utils import RopeParameters +from ...utils import auto_docstring, logging + + +logger = logging.get_logger(__name__) + + +@auto_docstring(checkpoint="Tele-AI/TeleChat3-36B-Thinking") +@strict +class TeleChat3Config(PreTrainedConfig): + r""" + ```python + >>> from transformers import TeleChat3Model, TeleChat3Config + + >>> # Initializing a TeleChat3-36B style configuration + >>> configuration = TeleChat3Config() + + >>> # Initializing a model from the TeleChat3-36B style configuration + >>> model = TeleChat3Model(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "telechat3" + keys_to_ignore_at_inference = ["past_key_values"] + # Default tensor parallel plan for base model `TeleChat3Model` + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.k_proj": "colwise", + "layers.*.self_attn.v_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + + vocab_size: int = 131072 + hidden_size: int = 6144 + intermediate_size: int = 24576 + num_hidden_layers: int = 64 + num_attention_heads: int = 48 + num_key_value_heads: int | None = 8 + hidden_act: str = "silu" + max_position_embeddings: int = 32768 + initializer_range: float = 0.0048 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = 3 + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + tie_word_embeddings: bool = False + rope_parameters: RopeParameters | dict | None = None + attention_bias: bool = False + attention_dropout: float | int = 0.0 + mlp_bias: bool = False + head_dim: int | None = 128 + + def __post_init__(self, **kwargs): + if self.head_dim is None: + self.head_dim = self.hidden_size // self.num_attention_heads + if self.num_key_value_heads is None: + self.num_key_value_heads = self.num_attention_heads + + super().__post_init__(**kwargs) + + def validate_architecture(self): + """Part of `@strict`-powered validation. Validates the architecture of the config.""" + if self.hidden_size % self.num_attention_heads != 0: + raise ValueError( + f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention " + f"heads ({self.num_attention_heads})." + ) + + def convert_rope_params_to_dict(self, **kwargs): + # The original checkpoints use a custom `"telechat3-yarn"` RoPE type, which is YaRN with 0.07 instead of 0.1 + # as the coefficient of its default attention factor. Convert it to plain YaRN with an explicit attention factor + rope_parameters = kwargs.get("rope_scaling") or self.rope_parameters + if rope_parameters is not None and "telechat3-yarn" in ( + rope_parameters.get("rope_type"), + rope_parameters.get("type"), + ): + rope_parameters = {k: v for k, v in rope_parameters.items() if k != "type"} + rope_parameters["rope_type"] = "yarn" + if "original_max_position_embeddings" in rope_parameters: + # The original code derives the scaling factor from the context lengths, ignoring `factor` + rope_parameters["factor"] = ( + self.max_position_embeddings / rope_parameters["original_max_position_embeddings"] + ) + if rope_parameters.get("attention_factor") is None: + factor = rope_parameters["factor"] + mscale = rope_parameters.get("mscale") + mscale_all_dim = rope_parameters.get("mscale_all_dim") + + def get_mscale(scale, mscale=1.0): + return 1.0 if scale <= 1 else 0.07 * mscale * math.log(scale) + 1.0 + + if mscale and mscale_all_dim: + attention_factor = get_mscale(factor, mscale) / get_mscale(factor, mscale_all_dim) + else: + attention_factor = get_mscale(factor) + rope_parameters["attention_factor"] = attention_factor + logger.info("Converted rope_type 'telechat3-yarn' to 'yarn' with an explicit `attention_factor`.") + kwargs["rope_scaling"] = rope_parameters + return super().convert_rope_params_to_dict(**kwargs) + + +__all__ = ["TeleChat3Config"] diff --git a/src/transformers/models/telechat3/modeling_telechat3.py b/src/transformers/models/telechat3/modeling_telechat3.py new file mode 100644 index 000000000000..6f0d8284450a --- /dev/null +++ b/src/transformers/models/telechat3/modeling_telechat3.py @@ -0,0 +1,474 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/telechat3/modular_telechat3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_telechat3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable + +import torch +from torch import nn + +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub, use_kernelized_func +from ...masking_utils import create_causal_mask +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import merge_with_config_defaults +from ...utils.output_capturing import capture_outputs +from .configuration_telechat3 import TeleChat3Config + + +@use_kernel_forward_from_hub("RMSNorm") +class TeleChat3RMSNorm(nn.Module): + def __init__(self, hidden_size, eps: float = 1e-6) -> None: + """ + TeleChat3RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class TeleChat3MLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +@use_kernel_forward_from_hub("rotary_pos_emb") +def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + scaling: float, + dropout: float = 0.0, + **kwargs: Unpack[TransformersKwargs], +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +@use_kernelized_func(apply_rotary_pos_emb) +class TeleChat3Attention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: TeleChat3Config, layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads + self.scaling = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + self.is_causal = True + + self.q_proj = nn.Linear( + config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias + ) + self.k_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.v_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.o_proj = nn.Linear( + config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias + ) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + attention_mask: torch.Tensor | None = None, + past_key_values: Cache | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple[torch.Tensor, torch.Tensor]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_values is not None: + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class TeleChat3DecoderLayer(GradientCheckpointingLayer): + def __init__(self, config: TeleChat3Config, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + + self.self_attn = TeleChat3Attention(config=config, layer_idx=layer_idx) + + self.mlp = TeleChat3MLP(config) + self.input_layernorm = TeleChat3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = TeleChat3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + use_cache: bool | None = False, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + # Self Attention + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +@auto_docstring +class TeleChat3PreTrainedModel(PreTrainedModel): + config: TeleChat3Config + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["TeleChat3DecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn = True + _supports_sdpa = True + _supports_flex_attn = True + + _can_compile_fullgraph = True + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": TeleChat3DecoderLayer, + "attentions": TeleChat3Attention, + } + + +class TeleChat3RotaryEmbedding(nn.Module): + def __init__(self, config: TeleChat3Config): + super().__init__() + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_default_rope_parameters + if self.rope_type != "default": + rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + inv_freq, self.attention_scaling = rope_init_fn(self.config) + + self.inv_freq = nn.Buffer(inv_freq, persistent=False) + self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False) + + @staticmethod + def compute_default_rope_parameters(config: TeleChat3Config, **kwargs) -> tuple[torch.Tensor, float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + + attention_factor = 1.0 # Unused in this type of RoPE + # Compute the inverse frequencies + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + return inv_freq, attention_factor + + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + freqs = position_ids[..., None].float() * self.inv_freq.to(device=x.device, dtype=torch.float) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +@auto_docstring +class TeleChat3Model(TeleChat3PreTrainedModel): + def __init__(self, config: TeleChat3Config): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [TeleChat3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = TeleChat3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = TeleChat3RotaryEmbedding(config=config) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + @merge_with_config_defaults + @capture_outputs + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.Tensor | None = None, + use_cache: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithPast: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds: torch.Tensor = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if position_ids is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) + + causal_mask = create_causal_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=past_key_values, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) + + for decoder_layer in self.layers[: self.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_embeddings=position_embeddings, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + **kwargs, + ) + + hidden_states = self.norm(hidden_states) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + +@auto_docstring(checkpoint="Tele-AI/TeleChat3-36B-Thinking") +class TeleChat3ForCausalLM(TeleChat3PreTrainedModel, GenerationMixin): + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + _tp_plan = {"lm_head": "colwise_gather_output"} + _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + _fsdp_plan = {"lm_head": "keep_full_weight"} + + def __init__(self, config: TeleChat3Config): + super().__init__(config) + self.model = TeleChat3Model(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.Tensor | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + logits_to_keep: int | torch.Tensor = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> CausalLMOutputWithPast: + r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, TeleChat3ForCausalLM + + >>> model = TeleChat3ForCausalLM.from_pretrained("meta-telechat3/TeleChat3-2-7b-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("meta-telechat3/TeleChat3-2-7b-hf") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + outputs: BaseModelOutputWithPast = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep) + logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +__all__ = ["TeleChat3ForCausalLM", "TeleChat3Model", "TeleChat3PreTrainedModel"] diff --git a/src/transformers/models/telechat3/modular_telechat3.py b/src/transformers/models/telechat3/modular_telechat3.py new file mode 100644 index 000000000000..998c40d5bcfb --- /dev/null +++ b/src/transformers/models/telechat3/modular_telechat3.py @@ -0,0 +1,126 @@ +# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +from huggingface_hub.dataclasses import strict + +from ...configuration_utils import PreTrainedConfig +from ...modeling_rope_utils import RopeParameters +from ...utils import auto_docstring, logging +from ..llama.configuration_llama import LlamaConfig +from ..llama.modeling_llama import ( + LlamaForCausalLM, + LlamaModel, + LlamaPreTrainedModel, +) + + +logger = logging.get_logger(__name__) + + +@auto_docstring(checkpoint="Tele-AI/TeleChat3-36B-Thinking") +@strict +class TeleChat3Config(LlamaConfig): + r""" + ```python + >>> from transformers import TeleChat3Model, TeleChat3Config + + >>> # Initializing a TeleChat3-36B style configuration + >>> configuration = TeleChat3Config() + + >>> # Initializing a model from the TeleChat3-36B style configuration + >>> model = TeleChat3Model(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "telechat3" + + vocab_size: int = 131072 + hidden_size: int = 6144 + intermediate_size: int = 24576 + num_hidden_layers: int = 64 + num_attention_heads: int = 48 + num_key_value_heads: int | None = 8 + hidden_act: str = "silu" + max_position_embeddings: int = 32768 + initializer_range: float = 0.0048 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = 3 + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + tie_word_embeddings: bool = False + rope_parameters: RopeParameters | dict | None = None + attention_bias: bool = False + attention_dropout: float | int = 0.0 + mlp_bias: bool = False + head_dim: int | None = 128 + + pretraining_tp = AttributeError() + + def convert_rope_params_to_dict(self, **kwargs): + # The original checkpoints use a custom `"telechat3-yarn"` RoPE type, which is YaRN with 0.07 instead of 0.1 + # as the coefficient of its default attention factor. Convert it to plain YaRN with an explicit attention factor + rope_parameters = kwargs.get("rope_scaling") or self.rope_parameters + if rope_parameters is not None and "telechat3-yarn" in ( + rope_parameters.get("rope_type"), + rope_parameters.get("type"), + ): + rope_parameters = {k: v for k, v in rope_parameters.items() if k != "type"} + rope_parameters["rope_type"] = "yarn" + if "original_max_position_embeddings" in rope_parameters: + # The original code derives the scaling factor from the context lengths, ignoring `factor` + rope_parameters["factor"] = ( + self.max_position_embeddings / rope_parameters["original_max_position_embeddings"] + ) + if rope_parameters.get("attention_factor") is None: + factor = rope_parameters["factor"] + mscale = rope_parameters.get("mscale") + mscale_all_dim = rope_parameters.get("mscale_all_dim") + + def get_mscale(scale, mscale=1.0): + return 1.0 if scale <= 1 else 0.07 * mscale * math.log(scale) + 1.0 + + if mscale and mscale_all_dim: + attention_factor = get_mscale(factor, mscale) / get_mscale(factor, mscale_all_dim) + else: + attention_factor = get_mscale(factor) + rope_parameters["attention_factor"] = attention_factor + logger.info("Converted rope_type 'telechat3-yarn' to 'yarn' with an explicit `attention_factor`.") + kwargs["rope_scaling"] = rope_parameters + return PreTrainedConfig.convert_rope_params_to_dict(self, **kwargs) + + +class TeleChat3PreTrainedModel(LlamaPreTrainedModel): + pass + + +class TeleChat3Model(LlamaModel): + pass + + +@auto_docstring(checkpoint="Tele-AI/TeleChat3-36B-Thinking") +class TeleChat3ForCausalLM(LlamaForCausalLM): + pass + + +__all__ = [ + "TeleChat3Config", + "TeleChat3ForCausalLM", + "TeleChat3Model", + "TeleChat3PreTrainedModel", +] diff --git a/tests/models/internlm3/__init__.py b/tests/models/internlm3/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/internlm3/test_modeling_internlm3.py b/tests/models/internlm3/test_modeling_internlm3.py new file mode 100644 index 000000000000..a237954bbef8 --- /dev/null +++ b/tests/models/internlm3/test_modeling_internlm3.py @@ -0,0 +1,75 @@ +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Testing suite for the PyTorch InternLM3 model.""" + +import unittest + +from transformers import AutoTokenizer, is_torch_available +from transformers.testing_utils import cleanup, require_torch, require_torch_accelerator, slow, torch_device + +from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester + + +if is_torch_available(): + import torch + + from transformers import InternLM3ForCausalLM, InternLM3Model + + +class InternLM3ModelTester(CausalLMModelTester): + if is_torch_available(): + base_model_class = InternLM3Model + + +@require_torch +class InternLM3ModelTest(CausalLMModelTest, unittest.TestCase): + model_tester_class = InternLM3ModelTester + + def test_qkv_bias_is_independent_of_bias(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + config.qkv_bias = True + config.bias = False + layer = InternLM3Model(config).layers[0] + for proj in (layer.self_attn.q_proj, layer.self_attn.k_proj, layer.self_attn.v_proj): + self.assertIsNotNone(proj.bias) + for proj in (layer.self_attn.o_proj, layer.mlp.gate_proj, layer.mlp.up_proj, layer.mlp.down_proj): + self.assertIsNone(proj.bias) + + +@slow +@require_torch_accelerator +class InternLM3IntegrationTest(unittest.TestCase): + model_id = "internlm/internlm3-8b-instruct" + + def tearDown(self): + cleanup(torch_device, gc_collect=True) + + def test_model_8b_logits(self): + model = InternLM3ForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) + with torch.no_grad(): + logits = model(input_ids).logits.float().cpu() + + EXPECTED_MEAN = torch.tensor(EXPECTED_MEAN_PLACEHOLDER) + EXPECTED_SLICE = torch.tensor(EXPECTED_SLICE_PLACEHOLDER) + torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) + + def test_model_8b_generation(self): + tokenizer = AutoTokenizer.from_pretrained(self.model_id, trust_remote_code=True) + model = InternLM3ForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) + output = model.generate(**inputs, max_new_tokens=16, do_sample=False) + text = tokenizer.decode(output[0], skip_special_tokens=True) + self.assertEqual(text, EXPECTED_TEXT_PLACEHOLDER) diff --git a/tests/models/iquestcoder/__init__.py b/tests/models/iquestcoder/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/iquestcoder/test_modeling_iquestcoder.py b/tests/models/iquestcoder/test_modeling_iquestcoder.py new file mode 100644 index 000000000000..f0ffad236655 --- /dev/null +++ b/tests/models/iquestcoder/test_modeling_iquestcoder.py @@ -0,0 +1,75 @@ +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Testing suite for the PyTorch IQuestCoder model.""" + +import unittest + +from huggingface_hub.errors import StrictDataclassClassValidationError + +from transformers import AutoTokenizer, IQuestCoderConfig, is_torch_available +from transformers.testing_utils import cleanup, require_torch, require_torch_accelerator, slow, torch_device + +from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester + + +if is_torch_available(): + import torch + + from transformers import IQuestCoderForCausalLM, IQuestCoderModel + + +class IQuestCoderModelTester(CausalLMModelTester): + if is_torch_available(): + base_model_class = IQuestCoderModel + + +@require_torch +class IQuestCoderModelTest(CausalLMModelTest, unittest.TestCase): + model_tester_class = IQuestCoderModelTester + + def test_unsupported_options_are_rejected(self): + # Released checkpoints set these to their disabled values, which must keep loading + IQuestCoderConfig(clip_qkv=None, use_sliding_window=False, sliding_window=None, max_window_layers=0) + with self.assertRaises(StrictDataclassClassValidationError): + IQuestCoderConfig(clip_qkv=8.0) + with self.assertRaises(StrictDataclassClassValidationError): + IQuestCoderConfig(use_sliding_window=True, sliding_window=4096) + + +@slow +@require_torch_accelerator +class IQuestCoderIntegrationTest(unittest.TestCase): + model_id = "IQuestLab/IQuest-Coder-V1-7B-Instruct" + + def tearDown(self): + cleanup(torch_device, gc_collect=True) + + def test_model_7b_logits(self): + model = IQuestCoderForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) + with torch.no_grad(): + logits = model(input_ids).logits.float().cpu() + + EXPECTED_MEAN = torch.tensor(EXPECTED_MEAN_PLACEHOLDER) + EXPECTED_SLICE = torch.tensor(EXPECTED_SLICE_PLACEHOLDER) + torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) + + def test_model_7b_generation(self): + tokenizer = AutoTokenizer.from_pretrained(self.model_id, trust_remote_code=True) + model = IQuestCoderForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + inputs = tokenizer("def fibonacci(n):", return_tensors="pt").to(model.device) + output = model.generate(**inputs, max_new_tokens=16, do_sample=False) + text = tokenizer.decode(output[0], skip_special_tokens=True) + self.assertEqual(text, EXPECTED_TEXT_PLACEHOLDER) diff --git a/tests/models/telechat3/__init__.py b/tests/models/telechat3/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/telechat3/test_modeling_telechat3.py b/tests/models/telechat3/test_modeling_telechat3.py new file mode 100644 index 000000000000..2bec8bd1d81d --- /dev/null +++ b/tests/models/telechat3/test_modeling_telechat3.py @@ -0,0 +1,102 @@ +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Testing suite for the PyTorch TeleChat3 model.""" + +import math +import unittest + +from transformers import AutoTokenizer, TeleChat3Config, is_torch_available +from transformers.testing_utils import cleanup, require_torch, require_torch_accelerator, slow, torch_device + +from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester + + +if is_torch_available(): + import torch + + from transformers import TeleChat3ForCausalLM, TeleChat3Model + + +class TeleChat3ModelTester(CausalLMModelTester): + if is_torch_available(): + base_model_class = TeleChat3Model + + +@require_torch +class TeleChat3ModelTest(CausalLMModelTest, unittest.TestCase): + model_tester_class = TeleChat3ModelTester + + +class TeleChat3ConfigTest(unittest.TestCase): + def test_telechat3_yarn_is_converted_to_yarn(self): + # `rope_scaling` as found in the released checkpoints + config = TeleChat3Config( + max_position_embeddings=32768, + rope_theta=1000000.0, + rope_scaling={ + "beta_fast": 32.0, + "beta_slow": 1.0, + "factor": 4.0, + "original_max_position_embeddings": 8192, + "rope_type": "telechat3-yarn", + "type": "telechat3-yarn", + }, + ) + self.assertEqual(config.rope_parameters["rope_type"], "yarn") + self.assertNotIn("type", config.rope_parameters) + self.assertEqual(config.rope_parameters["factor"], 4.0) + self.assertAlmostEqual(config.rope_parameters["attention_factor"], 0.07 * math.log(4.0) + 1.0) + + def test_factor_follows_context_lengths(self): + # The original code ignores `factor` when `original_max_position_embeddings` is set + config = TeleChat3Config( + max_position_embeddings=16384, + rope_scaling={"factor": 4.0, "original_max_position_embeddings": 8192, "rope_type": "telechat3-yarn"}, + ) + self.assertEqual(config.rope_parameters["factor"], 2.0) + self.assertAlmostEqual(config.rope_parameters["attention_factor"], 0.07 * math.log(2.0) + 1.0) + + def test_explicit_attention_factor_is_kept(self): + config = TeleChat3Config( + rope_scaling={"factor": 4.0, "attention_factor": 1.5, "rope_type": "telechat3-yarn"}, + ) + self.assertEqual(config.rope_parameters["attention_factor"], 1.5) + + +@slow +@require_torch_accelerator +class TeleChat3IntegrationTest(unittest.TestCase): + model_id = "Tele-AI/TeleChat3-36B-Thinking" + + def tearDown(self): + cleanup(torch_device, gc_collect=True) + + def test_model_36b_logits(self): + model = TeleChat3ForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) + with torch.no_grad(): + logits = model(input_ids).logits.float().cpu() + + EXPECTED_MEAN = torch.tensor(EXPECTED_MEAN_PLACEHOLDER) + EXPECTED_SLICE = torch.tensor(EXPECTED_SLICE_PLACEHOLDER) + torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) + + def test_model_36b_generation(self): + tokenizer = AutoTokenizer.from_pretrained(self.model_id, trust_remote_code=True) + model = TeleChat3ForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) + output = model.generate(**inputs, max_new_tokens=16, do_sample=False) + text = tokenizer.decode(output[0], skip_special_tokens=True) + self.assertEqual(text, EXPECTED_TEXT_PLACEHOLDER) From c76bc6d93ee0f841e34c077e30f66b6120111e82 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:56:39 +0000 Subject: [PATCH 02/17] Fill integration test expectations from B200 runs Co-Authored-By: Claude Opus 5.5 --- tests/models/internlm3/test_modeling_internlm3.py | 9 ++++++--- .../models/iquestcoder/test_modeling_iquestcoder.py | 12 ++++++++---- tests/models/telechat3/test_modeling_telechat3.py | 9 ++++++--- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/models/internlm3/test_modeling_internlm3.py b/tests/models/internlm3/test_modeling_internlm3.py index a237954bbef8..8964424fe0f1 100644 --- a/tests/models/internlm3/test_modeling_internlm3.py +++ b/tests/models/internlm3/test_modeling_internlm3.py @@ -61,8 +61,10 @@ def test_model_8b_logits(self): with torch.no_grad(): logits = model(input_ids).logits.float().cpu() - EXPECTED_MEAN = torch.tensor(EXPECTED_MEAN_PLACEHOLDER) - EXPECTED_SLICE = torch.tensor(EXPECTED_SLICE_PLACEHOLDER) + EXPECTED_MEAN = torch.tensor([-0.2702, 1.8108, 2.4337, 4.6791, 3.8414, 4.021, 4.3054, 4.6507, 4.841, 4.5961]) + EXPECTED_SLICE = torch.tensor( + [-1.7656, -0.5547, 3.5625, -0.6719, 1.9141, 1.5781, 1.2344, 0.7188, 0.3398, 0.5039] + ) torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN, rtol=1e-2, atol=1e-2) torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) @@ -72,4 +74,5 @@ def test_model_8b_generation(self): inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) text = tokenizer.decode(output[0], skip_special_tokens=True) - self.assertEqual(text, EXPECTED_TEXT_PLACEHOLDER) + EXPECTED_TEXT = "The capital of France is Paris.Paris is the capital of France. It is a city that is known" + self.assertEqual(text, EXPECTED_TEXT) diff --git a/tests/models/iquestcoder/test_modeling_iquestcoder.py b/tests/models/iquestcoder/test_modeling_iquestcoder.py index f0ffad236655..d3b86e1d0642 100644 --- a/tests/models/iquestcoder/test_modeling_iquestcoder.py +++ b/tests/models/iquestcoder/test_modeling_iquestcoder.py @@ -61,15 +61,19 @@ def test_model_7b_logits(self): with torch.no_grad(): logits = model(input_ids).logits.float().cpu() - EXPECTED_MEAN = torch.tensor(EXPECTED_MEAN_PLACEHOLDER) - EXPECTED_SLICE = torch.tensor(EXPECTED_SLICE_PLACEHOLDER) + EXPECTED_MEAN = torch.tensor( + [23.4439, -0.9136, 6.2595, 9.6258, 8.6033, 13.4631, 6.3165, 8.7364, 11.1808, 6.8638] + ) + EXPECTED_SLICE = torch.tensor([25.75, 30.75, 26.625, 28.625, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0]) torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN, rtol=1e-2, atol=1e-2) torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) def test_model_7b_generation(self): tokenizer = AutoTokenizer.from_pretrained(self.model_id, trust_remote_code=True) model = IQuestCoderForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) - inputs = tokenizer("def fibonacci(n):", return_tensors="pt").to(model.device) + # The checkpoints are not meant to get a BOS token (`add_bos_token=False` in their tokenizer config) + inputs = tokenizer("def fibonacci(n):", return_tensors="pt", add_special_tokens=False).to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) text = tokenizer.decode(output[0], skip_special_tokens=True) - self.assertEqual(text, EXPECTED_TEXT_PLACEHOLDER) + EXPECTED_TEXT = "def fibonacci(n):\n if n <= 0:\n " + self.assertEqual(text, EXPECTED_TEXT) diff --git a/tests/models/telechat3/test_modeling_telechat3.py b/tests/models/telechat3/test_modeling_telechat3.py index 2bec8bd1d81d..119eea238612 100644 --- a/tests/models/telechat3/test_modeling_telechat3.py +++ b/tests/models/telechat3/test_modeling_telechat3.py @@ -88,8 +88,10 @@ def test_model_36b_logits(self): with torch.no_grad(): logits = model(input_ids).logits.float().cpu() - EXPECTED_MEAN = torch.tensor(EXPECTED_MEAN_PLACEHOLDER) - EXPECTED_SLICE = torch.tensor(EXPECTED_SLICE_PLACEHOLDER) + EXPECTED_MEAN = torch.tensor( + [-4.6161, -2.5073, -1.4698, -2.3869, -2.4526, -4.3019, -2.0553, -3.1628, -2.0378, -1.7738] + ) + EXPECTED_SLICE = torch.tensor([-8.75, -10.75, -7.875, -6.7188, -8.8125, -8.75, -8.6875, -8.75, -8.75, -7.1562]) torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN, rtol=1e-2, atol=1e-2) torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) @@ -99,4 +101,5 @@ def test_model_36b_generation(self): inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) text = tokenizer.decode(output[0], skip_special_tokens=True) - self.assertEqual(text, EXPECTED_TEXT_PLACEHOLDER) + EXPECTED_TEXT = "The capital of France is Paris.\n\n\nOkay, let's see. The user wants me" + self.assertEqual(text, EXPECTED_TEXT) From b91d47e68e3887341b00b02b2bc398cd8b96fc82 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:41:02 +0000 Subject: [PATCH 03/17] Add TeleChat (v1, TeleChat2 and TeleChat2.5) Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 2 + docs/source/en/_toctree.yml | 2 + docs/source/en/model_doc/internlm3.md | 2 +- docs/source/en/model_doc/iquestcoder.md | 2 +- docs/source/en/model_doc/telechat.md | 65 +++ docs/source/en/model_doc/telechat3.md | 2 +- src/transformers/conversion_mapping.py | 11 + src/transformers/models/__init__.py | 1 + src/transformers/models/auto/auto_mappings.py | 1 + src/transformers/models/auto/modeling_auto.py | 2 + src/transformers/models/telechat/__init__.py | 33 ++ .../models/telechat/configuration_telechat.py | 131 +++++ .../models/telechat/modeling_telechat.py | 482 ++++++++++++++++++ .../models/telechat/modular_telechat.py | 243 +++++++++ tests/models/telechat/__init__.py | 0 .../models/telechat/test_modeling_telechat.py | 171 +++++++ 16 files changed, 1147 insertions(+), 3 deletions(-) create mode 100644 docs/source/en/model_doc/telechat.md create mode 100644 src/transformers/models/telechat/__init__.py create mode 100644 src/transformers/models/telechat/configuration_telechat.py create mode 100644 src/transformers/models/telechat/modeling_telechat.py create mode 100644 src/transformers/models/telechat/modular_telechat.py create mode 100644 tests/models/telechat/__init__.py create mode 100644 tests/models/telechat/test_modeling_telechat.py diff --git a/.gitattributes b/.gitattributes index d93fe2d3a168..82324e230c85 100644 --- a/.gitattributes +++ b/.gitattributes @@ -572,6 +572,8 @@ src/transformers/models/t5gemma/configuration_*.py linguist-generated=true src/transformers/models/t5gemma/modeling_*.py linguist-generated=true src/transformers/models/t5gemma2/configuration_*.py linguist-generated=true src/transformers/models/t5gemma2/modeling_*.py linguist-generated=true +src/transformers/models/telechat/configuration_*.py linguist-generated=true +src/transformers/models/telechat/modeling_*.py linguist-generated=true src/transformers/models/telechat3/configuration_*.py linguist-generated=true src/transformers/models/telechat3/modeling_*.py linguist-generated=true src/transformers/models/timesfm/modeling_*.py linguist-generated=true diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index bf62c2e73aef..cbd1ce206418 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -959,6 +959,8 @@ title: T5Gemma2 - local: model_doc/t5v1.1 title: T5v1.1 + - local: model_doc/telechat + title: TeleChat - local: model_doc/telechat3 title: TeleChat3 - local: model_doc/ul2 diff --git a/docs/source/en/model_doc/internlm3.md b/docs/source/en/model_doc/internlm3.md index c71a87236135..49335446cd9d 100644 --- a/docs/source/en/model_doc/internlm3.md +++ b/docs/source/en/model_doc/internlm3.md @@ -25,7 +25,7 @@ rendered properly in your Markdown viewer. [InternLM3](https://huggingface.co/internlm/internlm3-8b-instruct) is an 8B instruction-tuned language model developed by [Shanghai AI Laboratory](https://huggingface.co/internlm). Its architecture is the same as [Llama](./llama), except that the biases of the query, key and value projections (`qkv_bias`) are configured separately from the biases of the attention output projection and MLP layers (`bias`). -The checkpoints ship a custom tokenizer, so loading the tokenizer requires `trust_remote_code=True`. +The tokenizer class named in the checkpoints' `tokenizer_config.json` only exists in their remote code, so loading the tokenizer requires `trust_remote_code=True`. The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. diff --git a/docs/source/en/model_doc/iquestcoder.md b/docs/source/en/model_doc/iquestcoder.md index fbbe9ac37208..9eebdaae8042 100644 --- a/docs/source/en/model_doc/iquestcoder.md +++ b/docs/source/en/model_doc/iquestcoder.md @@ -27,7 +27,7 @@ rendered properly in your Markdown viewer. The looped `IQuest-Coder-V1-*-Loop-*` checkpoints use a different architecture and aren't supported by this model. -The checkpoints ship a custom tokenizer, so loading the tokenizer requires `trust_remote_code=True`. +The tokenizer class named in the checkpoints' `tokenizer_config.json` only exists in their remote code, so loading the tokenizer requires `trust_remote_code=True`. The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. diff --git a/docs/source/en/model_doc/telechat.md b/docs/source/en/model_doc/telechat.md new file mode 100644 index 000000000000..05a0811ba5ff --- /dev/null +++ b/docs/source/en/model_doc/telechat.md @@ -0,0 +1,65 @@ + +*This model was contributed to Hugging Face Transformers on 2026-11-08.* +
+
+ FlashAttention + SDPA +
+
+ +# TeleChat + +[TeleChat](https://huggingface.co/Tele-AI) is a family of language models developed by [China Telecom Artificial Intelligence Technology](https://huggingface.co/Tele-AI), covering TeleChat (1B to 12B), TeleChat2 (3B to 115B) and TeleChat2.5 (35B and 115B). TeleChat3 is a separate model, see [TeleChat3](./telechat3). + +Its architecture is the same as [Llama](./llama), except that: + +- keys and values come from a single fused projection, laid out per head; +- only the attention output projection and the down projection of the MLP have biases; +- once the sequence grows beyond `training_seqlen`, RoPE is rescaled with a stepwise NTK scaling of its base and the cos/sin are scaled by `0.1 * ln(seq_len / training_seqlen) + 1`. + +The original checkpoints use different parameter names, which are converted when the model is loaded. The tokenizer class named in the checkpoints' `tokenizer_config.json` only exists in their remote code, so loading the tokenizer requires `trust_remote_code=True`. + +TeleChat-52B uses a different architecture and isn't supported by this model. + +The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + + +tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat2-3B", trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained("Tele-AI/TeleChat2-3B", device_map="auto") + +messages = [{"role": "user", "content": "Explain the Pythagorean theorem in one sentence."}] +inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True) +output = model.generate(**inputs.to(model.device), max_new_tokens=64) +print(tokenizer.decode(output[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True)) +``` + +## TeleChatConfig + +[[autodoc]] TeleChatConfig + +## TeleChatModel + +[[autodoc]] TeleChatModel + - forward + +## TeleChatForCausalLM + +[[autodoc]] TeleChatForCausalLM + - forward diff --git a/docs/source/en/model_doc/telechat3.md b/docs/source/en/model_doc/telechat3.md index 053be5d56a2d..28ce539b252a 100644 --- a/docs/source/en/model_doc/telechat3.md +++ b/docs/source/en/model_doc/telechat3.md @@ -27,7 +27,7 @@ rendered properly in your Markdown viewer. The original checkpoints use a custom `"telechat3-yarn"` RoPE type, which is YaRN with 0.07 instead of 0.1 as the coefficient of its default attention factor. [`TeleChat3Config`] converts it to `"yarn"` with an explicit `attention_factor` when the config is loaded. -The checkpoints ship a custom tokenizer, so loading the tokenizer requires `trust_remote_code=True`. +The tokenizer class named in the checkpoints' `tokenizer_config.json` only exists in their remote code, so loading the tokenizer requires `trust_remote_code=True`. The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. diff --git a/src/transformers/conversion_mapping.py b/src/transformers/conversion_mapping.py index e333db77b8f7..eb828acc62e1 100755 --- a/src/transformers/conversion_mapping.py +++ b/src/transformers/conversion_mapping.py @@ -1633,6 +1633,17 @@ def _build_checkpoint_conversion_mapping(): target_patterns="LayerNorm.bias", ), ], + "telechat": [ + # Targets are escaped and guarded so that, reversed for saving, they only match whole names. `(?>> from transformers import TeleChatModel, TeleChatConfig + + >>> # Initializing a TeleChat2-3B style configuration + >>> configuration = TeleChatConfig() + + >>> # Initializing a model from the TeleChat2-3B style configuration + >>> model = TeleChatModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "telechat" + keys_to_ignore_at_inference = ["past_key_values"] + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.kv_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + + vocab_size: int = 131072 + hidden_size: int = 3072 + intermediate_size: int = 6144 + num_hidden_layers: int = 24 + num_attention_heads: int = 24 + num_key_value_heads: int | None = None + hidden_act: str = "silu" + max_position_embeddings: int | None = None + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = 3 + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + # Checkpoints which predate `tie_word_embeddings` relied on the old default of tying them + tie_word_embeddings: bool = True + rope_parameters: RopeParameters | dict | None = None + attention_dropout: float | int = 0.0 + head_dim: int | None = None + attribute_map = { + "n_layer": "num_hidden_layers", + "n_head": "num_attention_heads", + "ffn_hidden_size": "intermediate_size", + "layer_norm_epsilon": "rms_norm_eps", + } + training_seqlen: int = 8192 + base_seqlen: int = 4096 + + def __post_init__(self, **kwargs): + # Resolve the aliases now, otherwise `head_dim` and `num_key_value_heads` get derived from the defaults + for alias, attribute in self.attribute_map.items(): + if alias in kwargs: + setattr(self, attribute, kwargs.pop(alias)) + if self.max_position_embeddings is None: + self.max_position_embeddings = kwargs.get("seq_length") or self.training_seqlen + if self.head_dim is None: + self.head_dim = self.hidden_size // self.num_attention_heads + if self.num_key_value_heads is None: + self.num_key_value_heads = self.num_attention_heads + + super().__post_init__(**kwargs) + + def validate_architecture(self): + """Part of `@strict`-powered validation. Validates the architecture of the config.""" + if self.hidden_size % self.num_attention_heads != 0: + raise ValueError( + f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention " + f"heads ({self.num_attention_heads})." + ) + if getattr(self, "n_embd", None) is not None: + raise ValueError( + "This config uses the TeleChat-52B architecture, which is not supported by `TeleChatConfig`. " + "Load it with `trust_remote_code=True` instead." + ) + # The original code supports these options, but no released checkpoint enables them + for option in ("embed_layernorm", "apply_residual_connection_post_layernorm"): + if getattr(self, option, False): + raise ValueError(f"TeleChat does not support `{option}`.") + + def convert_rope_params_to_dict(self, **kwargs): + # The original code ignores `rope_scaling` and always applies its own stepwise NTK scaling instead + kwargs.pop("rope_scaling", None) + return super().convert_rope_params_to_dict(**kwargs) + + +__all__ = ["TeleChatConfig"] diff --git a/src/transformers/models/telechat/modeling_telechat.py b/src/transformers/models/telechat/modeling_telechat.py new file mode 100644 index 000000000000..06fef3df0ce3 --- /dev/null +++ b/src/transformers/models/telechat/modeling_telechat.py @@ -0,0 +1,482 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/telechat/modular_telechat.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_telechat.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable + +import torch +from torch import nn + +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub, use_kernelized_func +from ...masking_utils import create_causal_mask +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import merge_with_config_defaults +from ...utils.output_capturing import capture_outputs +from .configuration_telechat import TeleChatConfig + + +class TeleChatRotaryEmbedding(nn.Module): + def __init__(self, config: TeleChatConfig): + super().__init__() + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_default_rope_parameters + if self.rope_type != "default": + rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + inv_freq, self.attention_scaling = rope_init_fn(self.config) + + self.inv_freq = nn.Buffer(inv_freq, persistent=False) + self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False) + + @staticmethod + def compute_default_rope_parameters(config: TeleChatConfig, **kwargs) -> tuple[torch.Tensor, float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + + attention_factor = 1.0 # Unused in this type of RoPE + # Compute the inverse frequencies + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + return inv_freq, attention_factor + + @torch.no_grad() + def forward(self, x, position_ids): + inv_freq, attention_scaling = self.compute_ntk_parameters(self.config, position_ids.max() + 1) + freqs = position_ids[..., None].float() * inv_freq.to(x.device) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * attention_scaling + sin = emb.sin() * attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + @staticmethod + def compute_ntk_parameters(config: TeleChatConfig, seq_len: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """ + Computes the inverse frequencies and attention scaling for the stepwise NTK scaling of the original code, + given the current total sequence length `seq_len`. + """ + base = config.rope_parameters["rope_theta"] + dim = config.head_dim + seq_len = seq_len.clamp(min=config.training_seqlen).float() + # Equal to `2 ** ceil(log2(seq_len / base_seqlen) + 1) - 1` from the original code, but exact for powers of 2 + doublings = torch.ceil(torch.log2(torch.ceil(seq_len / config.base_seqlen))) + ntk_alpha = (2 ** (doublings + 1) - 1).clamp(min=1) + base = base * ntk_alpha ** (dim / (dim - 2)) + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float, device=seq_len.device) / dim)) + attention_scaling = 0.1 * torch.log(seq_len / config.training_seqlen) + 1.0 + return inv_freq, attention_scaling + + +class TeleChatMLP(nn.Module): + def __init__(self, config: TeleChatConfig): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=True) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +@use_kernel_forward_from_hub("rotary_pos_emb") +def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + scaling: float, + dropout: float = 0.0, + **kwargs: Unpack[TransformersKwargs], +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +@use_kernelized_func(apply_rotary_pos_emb) +class TeleChatAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: TeleChatConfig, layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads + self.scaling = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + self.is_causal = True + self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=True) + # Keys and values are fused and laid out per head, i.e. `[k_0, v_0, k_1, v_1, ...]` + self.kv_proj = nn.Linear(config.hidden_size, 2 * config.num_key_value_heads * self.head_dim, bias=False) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + attention_mask: torch.Tensor | None = None, + past_key_values: Cache | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple[torch.Tensor, torch.Tensor]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + key_value_states = self.kv_proj(hidden_states).view(*input_shape, -1, 2 * self.head_dim) + key_states, value_states = key_value_states.transpose(1, 2).chunk(2, dim=-1) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_values is not None: + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +@use_kernel_forward_from_hub("RMSNorm") +class TeleChatRMSNorm(nn.Module): + def __init__(self, hidden_size, eps: float = 1e-6) -> None: + """ + TeleChatRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class TeleChatDecoderLayer(GradientCheckpointingLayer): + def __init__(self, config: TeleChatConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + + self.self_attn = TeleChatAttention(config=config, layer_idx=layer_idx) + + self.mlp = TeleChatMLP(config) + self.input_layernorm = TeleChatRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = TeleChatRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + use_cache: bool | None = False, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + # Self Attention + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +@auto_docstring +class TeleChatPreTrainedModel(PreTrainedModel): + config: TeleChatConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["TeleChatDecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn = True + _supports_sdpa = True + _supports_flex_attn = True + + _can_compile_fullgraph = True + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": TeleChatDecoderLayer, + "attentions": TeleChatAttention, + } + + +@auto_docstring +class TeleChatModel(TeleChatPreTrainedModel): + def __init__(self, config: TeleChatConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [TeleChatDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = TeleChatRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = TeleChatRotaryEmbedding(config=config) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + @merge_with_config_defaults + @capture_outputs + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.Tensor | None = None, + use_cache: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithPast: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds: torch.Tensor = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if position_ids is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) + + causal_mask = create_causal_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=past_key_values, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) + + for decoder_layer in self.layers[: self.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_embeddings=position_embeddings, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + **kwargs, + ) + + hidden_states = self.norm(hidden_states) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + +@auto_docstring(checkpoint="Tele-AI/TeleChat2-3B") +class TeleChatForCausalLM(TeleChatPreTrainedModel, GenerationMixin): + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + _tp_plan = {"lm_head": "colwise_gather_output"} + _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + _fsdp_plan = {"lm_head": "keep_full_weight"} + + def __init__(self, config: TeleChatConfig): + super().__init__(config) + self.model = TeleChatModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.Tensor | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + logits_to_keep: int | torch.Tensor = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> CausalLMOutputWithPast: + r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, TeleChatForCausalLM + + >>> model = TeleChatForCausalLM.from_pretrained("meta-telechat/TeleChat-2-7b-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("meta-telechat/TeleChat-2-7b-hf") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + outputs: BaseModelOutputWithPast = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep) + logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +__all__ = ["TeleChatForCausalLM", "TeleChatModel", "TeleChatPreTrainedModel"] diff --git a/src/transformers/models/telechat/modular_telechat.py b/src/transformers/models/telechat/modular_telechat.py new file mode 100644 index 000000000000..5103d1c4bb3d --- /dev/null +++ b/src/transformers/models/telechat/modular_telechat.py @@ -0,0 +1,243 @@ +# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable + +import torch +from huggingface_hub.dataclasses import strict +from torch import nn + +from ...cache_utils import Cache +from ...configuration_utils import PreTrainedConfig +from ...modeling_rope_utils import RopeParameters +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring +from ..llama.configuration_llama import LlamaConfig +from ..llama.modeling_llama import ( + LlamaAttention, + LlamaDecoderLayer, + LlamaForCausalLM, + LlamaMLP, + LlamaModel, + LlamaPreTrainedModel, + LlamaRotaryEmbedding, + apply_rotary_pos_emb, + eager_attention_forward, +) + + +@auto_docstring(checkpoint="Tele-AI/TeleChat2-3B") +@strict +class TeleChatConfig(LlamaConfig): + r""" + training_seqlen (`int`, *optional*, defaults to 8192): + Sequence length the model was trained with. Beyond it, RoPE is rescaled with stepwise NTK scaling. + base_seqlen (`int`, *optional*, defaults to 4096): + Sequence length the stepwise NTK scaling factor is computed relative to. + + ```python + >>> from transformers import TeleChatModel, TeleChatConfig + + >>> # Initializing a TeleChat2-3B style configuration + >>> configuration = TeleChatConfig() + + >>> # Initializing a model from the TeleChat2-3B style configuration + >>> model = TeleChatModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "telechat" + attribute_map = { + "n_layer": "num_hidden_layers", + "n_head": "num_attention_heads", + "ffn_hidden_size": "intermediate_size", + "layer_norm_epsilon": "rms_norm_eps", + } + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.kv_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + + vocab_size: int = 131072 + hidden_size: int = 3072 + intermediate_size: int = 6144 + num_hidden_layers: int = 24 + num_attention_heads: int = 24 + num_key_value_heads: int | None = None + hidden_act: str = "silu" + max_position_embeddings: int | None = None + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = 3 + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + # Checkpoints which predate `tie_word_embeddings` relied on the old default of tying them + tie_word_embeddings: bool = True + rope_parameters: RopeParameters | dict | None = None + attention_dropout: float | int = 0.0 + head_dim: int | None = None + training_seqlen: int = 8192 + base_seqlen: int = 4096 + + attention_bias = AttributeError() + mlp_bias = AttributeError() + pretraining_tp = AttributeError() + + def __post_init__(self, **kwargs): + # Resolve the aliases now, otherwise `head_dim` and `num_key_value_heads` get derived from the defaults + for alias, attribute in self.attribute_map.items(): + if alias in kwargs: + setattr(self, attribute, kwargs.pop(alias)) + if self.max_position_embeddings is None: + self.max_position_embeddings = kwargs.get("seq_length") or self.training_seqlen + super().__post_init__(**kwargs) + + def convert_rope_params_to_dict(self, **kwargs): + # The original code ignores `rope_scaling` and always applies its own stepwise NTK scaling instead + kwargs.pop("rope_scaling", None) + return PreTrainedConfig.convert_rope_params_to_dict(self, **kwargs) + + def validate_architecture(self): + """Part of `@strict`-powered validation. Validates the architecture of the config.""" + super().validate_architecture() + if getattr(self, "n_embd", None) is not None: + raise ValueError( + "This config uses the TeleChat-52B architecture, which is not supported by `TeleChatConfig`. " + "Load it with `trust_remote_code=True` instead." + ) + # The original code supports these options, but no released checkpoint enables them + for option in ("embed_layernorm", "apply_residual_connection_post_layernorm"): + if getattr(self, option, False): + raise ValueError(f"TeleChat does not support `{option}`.") + + +class TeleChatRotaryEmbedding(LlamaRotaryEmbedding): + @staticmethod + def compute_ntk_parameters(config: TeleChatConfig, seq_len: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """ + Computes the inverse frequencies and attention scaling for the stepwise NTK scaling of the original code, + given the current total sequence length `seq_len`. + """ + base = config.rope_parameters["rope_theta"] + dim = config.head_dim + seq_len = seq_len.clamp(min=config.training_seqlen).float() + # Equal to `2 ** ceil(log2(seq_len / base_seqlen) + 1) - 1` from the original code, but exact for powers of 2 + doublings = torch.ceil(torch.log2(torch.ceil(seq_len / config.base_seqlen))) + ntk_alpha = (2 ** (doublings + 1) - 1).clamp(min=1) + base = base * ntk_alpha ** (dim / (dim - 2)) + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float, device=seq_len.device) / dim)) + attention_scaling = 0.1 * torch.log(seq_len / config.training_seqlen) + 1.0 + return inv_freq, attention_scaling + + @torch.no_grad() + def forward(self, x, position_ids): + inv_freq, attention_scaling = self.compute_ntk_parameters(self.config, position_ids.max() + 1) + freqs = position_ids[..., None].float() * inv_freq.to(x.device) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * attention_scaling + sin = emb.sin() * attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +class TeleChatMLP(LlamaMLP): + def __init__(self, config: TeleChatConfig): + super().__init__(config) + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=True) + + +class TeleChatAttention(LlamaAttention): + def __init__(self, config: TeleChatConfig, layer_idx: int): + super().__init__(config, layer_idx) + self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False) + # Keys and values are fused and laid out per head, i.e. `[k_0, v_0, k_1, v_1, ...]` + self.kv_proj = nn.Linear(config.hidden_size, 2 * config.num_key_value_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=True) + del self.k_proj + del self.v_proj + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + attention_mask: torch.Tensor | None = None, + past_key_values: Cache | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple[torch.Tensor, torch.Tensor]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + key_value_states = self.kv_proj(hidden_states).view(*input_shape, -1, 2 * self.head_dim) + key_states, value_states = key_value_states.transpose(1, 2).chunk(2, dim=-1) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_values is not None: + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class TeleChatDecoderLayer(LlamaDecoderLayer): + pass + + +class TeleChatPreTrainedModel(LlamaPreTrainedModel): + pass + + +class TeleChatModel(LlamaModel): + pass + + +@auto_docstring(checkpoint="Tele-AI/TeleChat2-3B") +class TeleChatForCausalLM(LlamaForCausalLM): + pass + + +__all__ = [ + "TeleChatConfig", + "TeleChatForCausalLM", + "TeleChatModel", + "TeleChatPreTrainedModel", +] diff --git a/tests/models/telechat/__init__.py b/tests/models/telechat/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/telechat/test_modeling_telechat.py b/tests/models/telechat/test_modeling_telechat.py new file mode 100644 index 000000000000..1bbc59c81c90 --- /dev/null +++ b/tests/models/telechat/test_modeling_telechat.py @@ -0,0 +1,171 @@ +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Testing suite for the PyTorch TeleChat model.""" + +import math +import tempfile +import unittest + +from huggingface_hub.errors import StrictDataclassClassValidationError +from parameterized import parameterized + +from transformers import AutoTokenizer, TeleChatConfig, is_torch_available +from transformers.testing_utils import cleanup, require_torch, require_torch_accelerator, slow, torch_device + +from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester + + +if is_torch_available(): + import torch + from safetensors.torch import load_file + + from transformers import TeleChatForCausalLM, TeleChatModel + from transformers.models.telechat.modeling_telechat import TeleChatRotaryEmbedding + + +def original_ntk_parameters(config, seq_len): + """The stepwise NTK scaling of the original code.""" + seq_len = max(seq_len, config.training_seqlen) + context_value = math.log2(seq_len / config.base_seqlen) + 1 + ntk_alpha = max(2 ** math.ceil(context_value) - 1, 1) + scale = seq_len / config.training_seqlen + mscale = 1.0 if scale <= 1 else 0.1 * math.log(scale) + 1.0 + dim = config.head_dim + base = config.rope_parameters["rope_theta"] * ntk_alpha ** (dim / (dim - 2)) + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim)) + return inv_freq, mscale + + +class TeleChatModelTester(CausalLMModelTester): + if is_torch_available(): + base_model_class = TeleChatModel + + +@require_torch +class TeleChatModelTest(CausalLMModelTest, unittest.TestCase): + model_tester_class = TeleChatModelTester + + def test_reverse_loading_mapping(self): + # The `transformer.` prefix of the original checkpoints only exists on models with a head + super().test_reverse_loading_mapping(skip_base_model=True) + + @unittest.skip("TeleChat ignores `rope_scaling` and always applies its own stepwise NTK scaling") + def test_model_rope_scaling_frequencies(self): + pass + + @parameterized.expand([("linear",), ("dynamic",), ("yarn",)]) + @unittest.skip("TeleChat ignores `rope_scaling` and always applies its own stepwise NTK scaling") + def test_model_rope_scaling_from_config(self, scaling_type): + pass + + @parameterized.expand([(4096, 8192), (8192, 8192), (32768, 32768)]) + def test_ntk_scaling_matches_original(self, base_seqlen, training_seqlen): + config = TeleChatConfig(base_seqlen=base_seqlen, training_seqlen=training_seqlen) + for seq_len in [ + 1, + training_seqlen - 1, + training_seqlen, + training_seqlen + 1, + 2 * training_seqlen, + 3 * training_seqlen + 7, + ]: + expected_inv_freq, expected_mscale = original_ntk_parameters(config, seq_len) + inv_freq, mscale = TeleChatRotaryEmbedding.compute_ntk_parameters(config, torch.tensor(seq_len)) + torch.testing.assert_close(inv_freq, expected_inv_freq) + self.assertAlmostEqual(mscale.item(), expected_mscale, places=5) + + def test_original_checkpoint_round_trip(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + model = TeleChatForCausalLM(config).to(torch_device).eval() + # Randomise everything, norms included, so a weight that fails to load can't go unnoticed + for param in model.parameters(): + torch.nn.init.normal_(param) + with tempfile.TemporaryDirectory() as tmpdir: + model.save_pretrained(tmpdir) + saved = set(load_file(f"{tmpdir}/model.safetensors")) + # Saved in the layout of the original checkpoints + for key in [ + "transformer.word_embeddings.weight", + "transformer.h.0.input_layernorm.weight", + "transformer.h.0.self_attention.query.weight", + "transformer.h.0.self_attention.key_value.weight", + "transformer.h.0.self_attention.dense.bias", + "transformer.h.0.post_attention_layernorm.weight", + "transformer.h.0.mlp.down_proj.bias", + "transformer.ln_f.weight", + ]: + self.assertIn(key, saved) + reloaded = TeleChatForCausalLM.from_pretrained(tmpdir).to(torch_device) + for (name, param), (_, reloaded_param) in zip(model.named_parameters(), reloaded.named_parameters()): + torch.testing.assert_close(param, reloaded_param, msg=name) + + def test_original_config_keys(self): + config = TeleChatConfig( + n_layer=3, n_head=8, hidden_size=64, ffn_hidden_size=96, layer_norm_epsilon=1e-4, seq_length=1024 + ) + self.assertEqual(config.num_hidden_layers, 3) + self.assertEqual(config.num_attention_heads, 8) + self.assertEqual(config.num_key_value_heads, 8) + self.assertEqual(config.head_dim, 8) + self.assertEqual(config.intermediate_size, 96) + self.assertEqual(config.rms_norm_eps, 1e-4) + self.assertEqual(config.max_position_embeddings, 1024) + + def test_rope_scaling_is_ignored(self): + # The original code never reads `rope_scaling`, so neither do we + config = TeleChatConfig(rope_theta=1e6, rope_scaling={"factor": 1.0, "rope_type": "dynamic"}) + self.assertEqual(config.rope_parameters, {"rope_type": "default", "rope_theta": 1e6}) + + def test_unsupported_configs_are_rejected(self): + with self.assertRaises(StrictDataclassClassValidationError): + TeleChatConfig(embed_layernorm=True) + with self.assertRaises(StrictDataclassClassValidationError): + TeleChatConfig(apply_residual_connection_post_layernorm=True) + with self.assertRaises(StrictDataclassClassValidationError): + # TeleChat-52B + TeleChatConfig(n_embd=8192, use_mup=True) + + +@slow +@require_torch_accelerator +class TeleChatIntegrationTest(unittest.TestCase): + # The original code was written for fp16 + dtype = torch.float16 if is_torch_available() else None + + def tearDown(self): + cleanup(torch_device, gc_collect=True) + + def test_model_3b_logits(self): + model = TeleChatForCausalLM.from_pretrained("Tele-AI/TeleChat2-3B", device_map="auto", dtype=self.dtype) + input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) + with torch.no_grad(): + logits = model(input_ids).logits.float().cpu() + + EXPECTED_MEAN = torch.tensor( + [-4.919, -4.2666, -1.901, -2.9477, -1.5264, -2.575, -0.5755, -1.9635, -2.781, -1.7785] + ) + EXPECTED_SLICE = torch.tensor( + [-9.1016, -9.8984, -1.8105, -9.1875, -9.1797, -9.1797, -9.1797, -9.1797, -9.1797, -6.3086] + ) + torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) + + def test_model_3b_generation(self): + tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat2-3B", trust_remote_code=True) + model = TeleChatForCausalLM.from_pretrained("Tele-AI/TeleChat2-3B", device_map="auto", dtype=self.dtype) + inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) + output = model.generate(**inputs, max_new_tokens=16, do_sample=False) + text = tokenizer.decode(output[0], skip_special_tokens=True) + EXPECTED_TEXT = "The capital of France is known for its rich history, culture, and stunning architecture. The capital city," + self.assertEqual(text, EXPECTED_TEXT) From 0c3262b422f7c7653f5ddd562eecc12d3e6e88d0 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:47:53 +0000 Subject: [PATCH 04/17] Load InternLM3, IQuestCoder and TeleChat tokenizers without remote code The checkpoints' tokenizer_config.json names tokenizer classes that only exist in their remote code. They are thin SentencePiece wrappers, so route these model types to TokenizersBackend and make the generic conversion from a bare tokenizer.model follow the SentencePiece model's own settings: - extra whitespace removal, dummy prefix and whitespace escaping from the normalizer spec - BPE merges ranked by piece score - user-defined pieces emitted whole and never merged - added tokens matched on the raw text when there is a dummy prefix - added_tokens_decoder may swap two pieces' ids Co-Authored-By: Claude Opus 5.5 --- docs/source/en/model_doc/internlm3.md | 4 +- docs/source/en/model_doc/iquestcoder.md | 4 +- docs/source/en/model_doc/telechat.md | 4 +- docs/source/en/model_doc/telechat3.md | 4 +- src/transformers/convert_slow_tokenizer.py | 70 ++++++++++++--- .../models/auto/tokenization_auto.py | 4 + .../tokenization_utils_tokenizers.py | 71 +++++++++++---- .../internlm3/test_modeling_internlm3.py | 2 +- .../iquestcoder/test_modeling_iquestcoder.py | 5 +- .../models/telechat/test_modeling_telechat.py | 2 +- .../telechat3/test_modeling_telechat3.py | 2 +- tests/utils/test_convert_slow_tokenizer.py | 86 ++++++++++++++++++- 12 files changed, 214 insertions(+), 44 deletions(-) diff --git a/docs/source/en/model_doc/internlm3.md b/docs/source/en/model_doc/internlm3.md index 49335446cd9d..ffa53b1be7c7 100644 --- a/docs/source/en/model_doc/internlm3.md +++ b/docs/source/en/model_doc/internlm3.md @@ -25,15 +25,13 @@ rendered properly in your Markdown viewer. [InternLM3](https://huggingface.co/internlm/internlm3-8b-instruct) is an 8B instruction-tuned language model developed by [Shanghai AI Laboratory](https://huggingface.co/internlm). Its architecture is the same as [Llama](./llama), except that the biases of the query, key and value projections (`qkv_bias`) are configured separately from the biases of the attention output projection and MLP layers (`bias`). -The tokenizer class named in the checkpoints' `tokenizer_config.json` only exists in their remote code, so loading the tokenizer requires `trust_remote_code=True`. - The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. ```python from transformers import AutoModelForCausalLM, AutoTokenizer -tokenizer = AutoTokenizer.from_pretrained("internlm/internlm3-8b-instruct", trust_remote_code=True) +tokenizer = AutoTokenizer.from_pretrained("internlm/internlm3-8b-instruct") model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto") messages = [{"role": "user", "content": "Explain the Pythagorean theorem in one sentence."}] diff --git a/docs/source/en/model_doc/iquestcoder.md b/docs/source/en/model_doc/iquestcoder.md index 9eebdaae8042..63c914b12195 100644 --- a/docs/source/en/model_doc/iquestcoder.md +++ b/docs/source/en/model_doc/iquestcoder.md @@ -27,7 +27,7 @@ rendered properly in your Markdown viewer. The looped `IQuest-Coder-V1-*-Loop-*` checkpoints use a different architecture and aren't supported by this model. -The tokenizer class named in the checkpoints' `tokenizer_config.json` only exists in their remote code, so loading the tokenizer requires `trust_remote_code=True`. +The tokenizer is built from the checkpoints' SentencePiece model. Many of its pieces share the same score, and SentencePiece breaks those ties by position, which the `tokenizers` library can't reproduce, so a small fraction of texts tokenize differently from the original remote code. The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. @@ -35,7 +35,7 @@ The example below demonstrates how to generate text with the [`AutoModelForCausa from transformers import AutoModelForCausalLM, AutoTokenizer -tokenizer = AutoTokenizer.from_pretrained("IQuestLab/IQuest-Coder-V1-7B-Instruct", trust_remote_code=True) +tokenizer = AutoTokenizer.from_pretrained("IQuestLab/IQuest-Coder-V1-7B-Instruct") model = AutoModelForCausalLM.from_pretrained("IQuestLab/IQuest-Coder-V1-7B-Instruct", device_map="auto") messages = [{"role": "user", "content": "Write a Python function that checks whether a number is prime."}] diff --git a/docs/source/en/model_doc/telechat.md b/docs/source/en/model_doc/telechat.md index 05a0811ba5ff..96e50a35f31d 100644 --- a/docs/source/en/model_doc/telechat.md +++ b/docs/source/en/model_doc/telechat.md @@ -31,7 +31,7 @@ Its architecture is the same as [Llama](./llama), except that: - only the attention output projection and the down projection of the MLP have biases; - once the sequence grows beyond `training_seqlen`, RoPE is rescaled with a stepwise NTK scaling of its base and the cos/sin are scaled by `0.1 * ln(seq_len / training_seqlen) + 1`. -The original checkpoints use different parameter names, which are converted when the model is loaded. The tokenizer class named in the checkpoints' `tokenizer_config.json` only exists in their remote code, so loading the tokenizer requires `trust_remote_code=True`. +The original checkpoints use different parameter names, which are converted when the model is loaded. TeleChat-52B uses a different architecture and isn't supported by this model. @@ -41,7 +41,7 @@ The example below demonstrates how to generate text with the [`AutoModelForCausa from transformers import AutoModelForCausalLM, AutoTokenizer -tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat2-3B", trust_remote_code=True) +tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat2-3B") model = AutoModelForCausalLM.from_pretrained("Tele-AI/TeleChat2-3B", device_map="auto") messages = [{"role": "user", "content": "Explain the Pythagorean theorem in one sentence."}] diff --git a/docs/source/en/model_doc/telechat3.md b/docs/source/en/model_doc/telechat3.md index 28ce539b252a..96c7472f3b1f 100644 --- a/docs/source/en/model_doc/telechat3.md +++ b/docs/source/en/model_doc/telechat3.md @@ -27,15 +27,13 @@ rendered properly in your Markdown viewer. The original checkpoints use a custom `"telechat3-yarn"` RoPE type, which is YaRN with 0.07 instead of 0.1 as the coefficient of its default attention factor. [`TeleChat3Config`] converts it to `"yarn"` with an explicit `attention_factor` when the config is loaded. -The tokenizer class named in the checkpoints' `tokenizer_config.json` only exists in their remote code, so loading the tokenizer requires `trust_remote_code=True`. - The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. ```python from transformers import AutoModelForCausalLM, AutoTokenizer -tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat3-36B-Thinking", trust_remote_code=True) +tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat3-36B-Thinking") model = AutoModelForCausalLM.from_pretrained("Tele-AI/TeleChat3-36B-Thinking", device_map="auto") messages = [{"role": "user", "content": "Explain the Pythagorean theorem in one sentence."}] diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py index 6e988ffd56e6..99b31b393458 100644 --- a/src/transformers/convert_slow_tokenizer.py +++ b/src/transformers/convert_slow_tokenizer.py @@ -19,6 +19,7 @@ """ import os +import re import warnings from collections.abc import Collection @@ -639,22 +640,55 @@ class SpmConverter(Converter): @staticmethod def build_tokenizer_from_spm_proto(proto, vocab, merges=None): """ - Similar to convert_from_spm method, but used only when there is no `model_type` class, i.e. there is no matching class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracting stuff from the sentencepiece file + Similar to convert_from_spm method, but used only when there is no `model_type` class, i.e. there is no matching + class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracting stuff from the sentencepiece + file. + + Follows the SentencePiece model's own settings, so the result encodes like `SentencePieceProcessor.encode`: + the normalizer spec (extra whitespace removal, dummy prefix, whitespace escaping), BPE merges ranked by piece + score, and user-defined pieces, which SentencePiece always emits whole and never merges. When `merges` is + `None`, they are generated from `vocab` and the piece scores. """ byte_fallback = proto.trainer_spec.byte_fallback unk_piece = proto.trainer_spec.unk_piece - precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap + normalizer_spec = proto.normalizer_spec + precompiled_charsmap = normalizer_spec.precompiled_charsmap + # `vocab` may rename pieces (from `added_tokens_decoder`), so look pieces up by id + if isinstance(vocab, dict): + id_to_token = {token_id: token for token, token_id in vocab.items()} + else: + id_to_token = {token_id: item[0] for token_id, item in enumerate(vocab or [])} + user_defined = [id_to_token[i] for i, piece in enumerate(proto.pieces) if piece.type == 4 and i in id_to_token] + # A normalized added token's content is normalized too, so with a dummy prefix user-defined pieces would only + # match at the start of the text. Isolate them in the pre-tokenizer instead, after the prefix is added. + isolate_user_defined = bool(user_defined) and normalizer_spec.add_dummy_prefix # model if isinstance(vocab, dict): + if merges is None: + # Only normal pieces (type 1) are merged + normal = { + id_to_token[i]: (piece.score, i) + for i, piece in enumerate(proto.pieces) + if piece.type == 1 and i in id_to_token + } + ranked = [] + for token, rank in normal.items(): + for split in range(1, len(token)): + left, right = token[:split], token[split:] + if left in normal and right in normal: + ranked.append((-rank[0], rank[1], left, right)) + merges = [(left, right) for _, _, left, right in sorted(ranked)] tokenizer = Tokenizer( BPE( vocab=vocab, - merges=merges or [], + merges=merges, unk_token=unk_piece, fuse_unk=True, byte_fallback=byte_fallback, dropout=None, + # Isolated user-defined pieces aren't reachable through merges, so look whole words up first + ignore_merges=isolate_user_defined, ) ) elif isinstance(vocab, list) and vocab and isinstance(vocab[0], tuple | list): @@ -668,19 +702,33 @@ def build_tokenizer_from_spm_proto(proto, vocab, merges=None): else: return None - # normalizer - _normalizers = [normalizers.Replace(" ", "▁")] + # normalizer, in SentencePiece's order + _normalizers = [] if precompiled_charsmap: - _normalizers.insert(0, normalizers.Precompiled(precompiled_charsmap)) + _normalizers.append(normalizers.Precompiled(precompiled_charsmap)) + if normalizer_spec.remove_extra_whitespaces: + # Only spaces count: SentencePiece leaves other whitespace, such as newlines, alone + _normalizers += [normalizers.Replace(Regex(r"\A +| +\z"), ""), normalizers.Replace(Regex(" {2,}"), " ")] + if normalizer_spec.add_dummy_prefix: + _normalizers.append(normalizers.Prepend("▁")) + if normalizer_spec.escape_whitespaces: + _normalizers.append(normalizers.Replace(" ", "▁")) tokenizer.normalizer = normalizers.Sequence(_normalizers) + # user-defined pieces + if isolate_user_defined: + pattern = "|".join(re.escape(piece) for piece in sorted(user_defined, key=len, reverse=True)) + tokenizer.pre_tokenizer = pre_tokenizers.Split(Regex(pattern), behavior="isolated") + elif user_defined: + tokenizer.add_tokens([AddedToken(piece, normalized=True, special=False) for piece in user_defined]) + # decoder + _decoders = [decoders.Replace("▁", " ")] if byte_fallback: - tokenizer.decoder = decoders.Sequence( - [decoders.Replace("▁", " "), decoders.ByteFallback(), decoders.Fuse()] - ) - else: - tokenizer.decoder = decoders.Sequence([decoders.Replace("▁", " ")]) + _decoders += [decoders.ByteFallback(), decoders.Fuse()] + if normalizer_spec.add_dummy_prefix: + _decoders.append(decoders.Strip(content=" ", left=1)) + tokenizer.decoder = decoders.Sequence(_decoders) return tokenizer diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 5b8e67209521..26a08377bf88 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -397,6 +397,8 @@ "hyperclovax_vlm", "hyperclovax_vision_v2", "internlm2", + "internlm3", + "iquestcoder", "jamba", "llava", "llava_next", @@ -421,6 +423,8 @@ "qwen3_5_moe_text", "qwen4_exp", "step3_vl", + "telechat", + "telechat3", "vipllava", "cohere_asr", "camembertv2-base", diff --git a/src/transformers/tokenization_utils_tokenizers.py b/src/transformers/tokenization_utils_tokenizers.py index cbed619bb200..c9d3c7356e63 100644 --- a/src/transformers/tokenization_utils_tokenizers.py +++ b/src/transformers/tokenization_utils_tokenizers.py @@ -331,25 +331,66 @@ def _convert_from_sentencepiece(cls, vocab_file: str, local_kwargs: dict[str, An # from the proto so normalizer/decoder are configured correctly. if "tokenizer_object" not in local_kwargs and (cls is TokenizersBackend or "__init__" not in cls.__dict__): vocab = local_kwargs.pop("vocab", None) - merges = local_kwargs.pop("merges", None) + # Regenerated from the piece scores by `build_tokenizer_from_spm_proto` + local_kwargs.pop("merges", None) # Replace placeholder tokens as specified in added_tokens_decoder added_tokens_decoder = local_kwargs.get("added_tokens_decoder") or {} - if vocab is not None and added_tokens_decoder: + if isinstance(vocab, dict) and added_tokens_decoder: id_to_token = {token_id: token for token, token_id in vocab.items()} - for token_id, new_token in added_tokens_decoder.items(): - token_id = int(token_id) - new_token = str(new_token) - current_token = id_to_token.get(token_id) - if current_token and current_token != new_token and new_token not in vocab: - vocab[new_token] = vocab.pop(current_token) - id_to_token[token_id] = new_token - - tokenizer_object = SpmConverter.build_tokenizer_from_spm_proto( - proto=extractor.proto, - vocab=vocab, - merges=merges, - ) + renames = { + int(token_id): str(new_token) + for token_id, new_token in added_tokens_decoder.items() + if id_to_token.get(int(token_id)) not in (None, str(new_token)) + } + # A new token may already be in the vocab only if its id is renamed too (e.g. two pieces swapped) + renames = { + token_id: new_token + for token_id, new_token in renames.items() + if new_token not in vocab or vocab[new_token] in renames + } + for token_id in renames: + del vocab[id_to_token[token_id]] + for token_id, new_token in renames.items(): + vocab[new_token] = token_id + + # User-defined pieces are handled by the backend tokenizer, not as special tokens + user_defined = {piece.piece for piece in extractor.proto.pieces if piece.type == 4} + if user_defined and local_kwargs.get("additional_special_tokens"): + local_kwargs["additional_special_tokens"] = [ + token for token in local_kwargs["additional_special_tokens"] if str(token) not in user_defined + ] + + # The dummy prefix is added by the normalizer, which would also prefix a normalized added token's + # content so that it only matches at the start of the text. Match added tokens on the raw text instead, + # as SentencePiece-based tokenizers do. + if extractor.proto.normalizer_spec.add_dummy_prefix: + + def _unnormalized(token): + if not isinstance(token, AddedToken) or not token.normalized: + return token + return AddedToken( + token.content, + single_word=token.single_word, + lstrip=token.lstrip, + rstrip=token.rstrip, + normalized=False, + special=token.special, + ) + + if added_tokens_decoder: + local_kwargs["added_tokens_decoder"] = { + token_id: _unnormalized(token) for token_id, token in added_tokens_decoder.items() + } + for key in ("bos_token", "eos_token", "unk_token", "pad_token"): + if key in local_kwargs: + local_kwargs[key] = _unnormalized(local_kwargs[key]) + if local_kwargs.get("additional_special_tokens"): + local_kwargs["additional_special_tokens"] = [ + _unnormalized(token) for token in local_kwargs["additional_special_tokens"] + ] + + tokenizer_object = SpmConverter.build_tokenizer_from_spm_proto(proto=extractor.proto, vocab=vocab) if tokenizer_object is not None: local_kwargs["tokenizer_object"] = tokenizer_object # Set bos/eos tokens from proto spec if available. This is needed when diff --git a/tests/models/internlm3/test_modeling_internlm3.py b/tests/models/internlm3/test_modeling_internlm3.py index 8964424fe0f1..17653e8bf9da 100644 --- a/tests/models/internlm3/test_modeling_internlm3.py +++ b/tests/models/internlm3/test_modeling_internlm3.py @@ -69,7 +69,7 @@ def test_model_8b_logits(self): torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) def test_model_8b_generation(self): - tokenizer = AutoTokenizer.from_pretrained(self.model_id, trust_remote_code=True) + tokenizer = AutoTokenizer.from_pretrained(self.model_id) model = InternLM3ForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) diff --git a/tests/models/iquestcoder/test_modeling_iquestcoder.py b/tests/models/iquestcoder/test_modeling_iquestcoder.py index d3b86e1d0642..adcbffd3bb83 100644 --- a/tests/models/iquestcoder/test_modeling_iquestcoder.py +++ b/tests/models/iquestcoder/test_modeling_iquestcoder.py @@ -69,10 +69,9 @@ def test_model_7b_logits(self): torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) def test_model_7b_generation(self): - tokenizer = AutoTokenizer.from_pretrained(self.model_id, trust_remote_code=True) + tokenizer = AutoTokenizer.from_pretrained(self.model_id) model = IQuestCoderForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) - # The checkpoints are not meant to get a BOS token (`add_bos_token=False` in their tokenizer config) - inputs = tokenizer("def fibonacci(n):", return_tensors="pt", add_special_tokens=False).to(model.device) + inputs = tokenizer("def fibonacci(n):", return_tensors="pt").to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) text = tokenizer.decode(output[0], skip_special_tokens=True) EXPECTED_TEXT = "def fibonacci(n):\n if n <= 0:\n " diff --git a/tests/models/telechat/test_modeling_telechat.py b/tests/models/telechat/test_modeling_telechat.py index 1bbc59c81c90..6d158d8a6aa3 100644 --- a/tests/models/telechat/test_modeling_telechat.py +++ b/tests/models/telechat/test_modeling_telechat.py @@ -162,7 +162,7 @@ def test_model_3b_logits(self): torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) def test_model_3b_generation(self): - tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat2-3B", trust_remote_code=True) + tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat2-3B") model = TeleChatForCausalLM.from_pretrained("Tele-AI/TeleChat2-3B", device_map="auto", dtype=self.dtype) inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) diff --git a/tests/models/telechat3/test_modeling_telechat3.py b/tests/models/telechat3/test_modeling_telechat3.py index 119eea238612..2ec9a239d455 100644 --- a/tests/models/telechat3/test_modeling_telechat3.py +++ b/tests/models/telechat3/test_modeling_telechat3.py @@ -96,7 +96,7 @@ def test_model_36b_logits(self): torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) def test_model_36b_generation(self): - tokenizer = AutoTokenizer.from_pretrained(self.model_id, trust_remote_code=True) + tokenizer = AutoTokenizer.from_pretrained(self.model_id) model = TeleChat3ForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) diff --git a/tests/utils/test_convert_slow_tokenizer.py b/tests/utils/test_convert_slow_tokenizer.py index f0043522e30e..f8ca6aaca3f3 100644 --- a/tests/utils/test_convert_slow_tokenizer.py +++ b/tests/utils/test_convert_slow_tokenizer.py @@ -4,9 +4,14 @@ import warnings from dataclasses import dataclass -from transformers import AutoTokenizer +from transformers import AutoTokenizer, TokenizersBackend from transformers.convert_slow_tokenizer import SentencePieceExtractor, SpmConverter -from transformers.testing_utils import get_tests_dir +from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers +from transformers.utils import is_sentencepiece_available + + +if is_sentencepiece_available(): + import sentencepiece as spm @dataclass @@ -57,3 +62,80 @@ def test_spm_precompiled_charsmap_empty_is_none(self): tokenizer = AutoTokenizer.from_pretrained(tmp_dir) self.assertGreater(len(tokenizer("Hello, world!")["input_ids"]), 1) + + +@require_sentencepiece +@require_tokenizers +class SentencePieceModelConversionTest(unittest.TestCase): + """`TokenizersBackend` built from a bare `tokenizer.model` should encode and decode like SentencePiece.""" + + corpus = [ + "Hello world, this is a test of the tokenizer.", + " def f(x):\n return x + 1 # comment", + "数字 12345 和中文文本，还有更多的中文。", + "foo_bar__baz qux", + ] * 50 + texts = [ + "Hello world\n__x 123", + " leading space", + "trailing ", + "\n\n indented\tcode", + "中文 text 123", + "__init__", + "🤗 unseen characters", + "", + ] + + def _train(self, tmp_dir, **kwargs): + spm.SentencePieceTrainer.train( + sentence_iterator=iter(self.corpus), + model_prefix=f"{tmp_dir}/tokenizer", + vocab_size=400, + model_type="bpe", + byte_fallback=True, + normalization_rule_name="identity", + minloglevel=2, + **kwargs, + ) + return spm.SentencePieceProcessor(model_file=f"{tmp_dir}/tokenizer.model") + + def test_matches_sentencepiece(self): + configs = { + # User-defined pieces are never merged, and with a dummy prefix every chunk starts with "▁" + "dummy_prefix_user_defined": { + "add_dummy_prefix": True, + "remove_extra_whitespaces": False, + "allow_whitespace_only_pieces": True, + "split_digits": True, + "user_defined_symbols": ["__", "", "\n"], + }, + "no_dummy_prefix_user_defined": { + "add_dummy_prefix": False, + "remove_extra_whitespaces": False, + "user_defined_symbols": ["__", "", "\n"], + }, + "remove_extra_whitespaces": {"add_dummy_prefix": True, "remove_extra_whitespaces": True}, + } + for name, kwargs in configs.items(): + with self.subTest(name), tempfile.TemporaryDirectory() as tmp_dir: + sp = self._train(tmp_dir, **kwargs) + tokenizer = TokenizersBackend.from_pretrained(tmp_dir) + for text in self.texts: + ids = sp.encode(text) + self.assertEqual(tokenizer.encode(text, add_special_tokens=False), ids, text) + self.assertEqual(tokenizer.decode(ids), sp.decode(ids), text) + + def test_added_tokens_decoder_can_swap_pieces(self): + with tempfile.TemporaryDirectory() as tmp_dir: + sp = self._train(tmp_dir, control_symbols=["", ""]) + a_id, b_id = sp.piece_to_id(""), sp.piece_to_id("") + added_tokens_decoder = { + str(a_id): {"content": "", "special": True}, + str(b_id): {"content": "", "special": True}, + } + with open(f"{tmp_dir}/tokenizer_config.json", "w", encoding="utf-8") as f: + json.dump({"added_tokens_decoder": added_tokens_decoder}, f) + + tokenizer = TokenizersBackend.from_pretrained(tmp_dir) + self.assertEqual(tokenizer.convert_tokens_to_ids(["", ""]), [b_id, a_id]) + self.assertEqual(tokenizer.encode("x", add_special_tokens=False)[::2], [b_id, a_id]) From d01eabc9c851b79f52bdc4cc9815724867d0d4ca Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:43:24 +0000 Subject: [PATCH 05/17] Fix SentencePiece decoding without byte fallback and test offsets against the tokenized text The dummy-prefix `Strip` decoder applies per token, so without byte fallback (and its `Fuse`) it stripped the space from every word. The replacement offsets test decoded the input ids to recover the expanded text, which only works for tokenizers that round-trip their input; SentencePiece-based ones don't around added tokens. Co-Authored-By: Claude Opus 5.5 --- src/transformers/convert_slow_tokenizer.py | 7 ++-- tests/test_processing_common.py | 41 +++++++++++++++------- tests/utils/test_convert_slow_tokenizer.py | 8 +++-- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py index 99b31b393458..550a94481091 100644 --- a/src/transformers/convert_slow_tokenizer.py +++ b/src/transformers/convert_slow_tokenizer.py @@ -725,9 +725,12 @@ class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracti # decoder _decoders = [decoders.Replace("▁", " ")] if byte_fallback: - _decoders += [decoders.ByteFallback(), decoders.Fuse()] + _decoders.append(decoders.ByteFallback()) if normalizer_spec.add_dummy_prefix: - _decoders.append(decoders.Strip(content=" ", left=1)) + # `Strip` applies to each token, so fuse them first to strip only the start of the text + _decoders += [decoders.Fuse(), decoders.Strip(content=" ", left=1)] + elif byte_fallback: + _decoders.append(decoders.Fuse()) tokenizer.decoder = decoders.Sequence(_decoders) return tokenizer diff --git a/tests/test_processing_common.py b/tests/test_processing_common.py index 3529dea56f9d..d4f20fe5d95c 100644 --- a/tests/test_processing_common.py +++ b/tests/test_processing_common.py @@ -2004,23 +2004,38 @@ def test_replacement_offsets(self): else: processor_inputs[modality] = prepare_method() + # The offsets index the text after replacement, so capture the text the processor tokenizes + expanded_texts = [] + get_text_with_replacements = processor.get_text_with_replacements + + def capture_text_with_replacements(*args, **kwargs): + text, offsets = get_text_with_replacements(*args, **kwargs) + expanded_texts.append(text) + return text, offsets + # Test combined processing - model_inputs = processor( - **processor_inputs, - add_special_tokens=False, - padding=False, - truncation=False, - max_length=None, - return_text_replacement_offsets=True, - **processing_kwargs, - ) - detokenized_text = processor.tokenizer.batch_decode( - model_inputs["input_ids"], skip_special_tokens=False, clean_up_tokenization_spaces=True - ) + with patch.object(processor, "get_text_with_replacements", side_effect=capture_text_with_replacements): + model_inputs = processor( + **processor_inputs, + add_special_tokens=False, + padding=False, + truncation=False, + max_length=None, + return_text_replacement_offsets=True, + **processing_kwargs, + ) + if expanded_texts: + expanded_text = expanded_texts[-1] + expanded_text = [expanded_text] if isinstance(expanded_text, str) else expanded_text + else: + # Processors that don't expand the text with `get_text_with_replacements` + expanded_text = processor.tokenizer.batch_decode( + model_inputs["input_ids"], skip_special_tokens=False, clean_up_tokenization_spaces=True + ) self.assertIn("text_replacement_offsets", model_inputs) for i, sample_offsets in enumerate(model_inputs["text_replacement_offsets"]): for curr_dict in sample_offsets: self.assertIn(curr_dict["type"], modalities) start, end = curr_dict["new_span"] - self.assertEqual(detokenized_text[i][start:end], curr_dict["replacement"]) + self.assertEqual(expanded_text[i][start:end], curr_dict["replacement"]) diff --git a/tests/utils/test_convert_slow_tokenizer.py b/tests/utils/test_convert_slow_tokenizer.py index f8ca6aaca3f3..44aea1986e9e 100644 --- a/tests/utils/test_convert_slow_tokenizer.py +++ b/tests/utils/test_convert_slow_tokenizer.py @@ -87,12 +87,12 @@ class SentencePieceModelConversionTest(unittest.TestCase): ] def _train(self, tmp_dir, **kwargs): + kwargs.setdefault("byte_fallback", True) + kwargs.setdefault("vocab_size", 400) spm.SentencePieceTrainer.train( sentence_iterator=iter(self.corpus), model_prefix=f"{tmp_dir}/tokenizer", - vocab_size=400, model_type="bpe", - byte_fallback=True, normalization_rule_name="identity", minloglevel=2, **kwargs, @@ -115,6 +115,8 @@ def test_matches_sentencepiece(self): "user_defined_symbols": ["__", "", "\n"], }, "remove_extra_whitespaces": {"add_dummy_prefix": True, "remove_extra_whitespaces": True}, + # Without byte fallback the decoder has no `Fuse`, so only the start of the text may lose its space + "no_byte_fallback": {"add_dummy_prefix": True, "byte_fallback": False, "vocab_size": 200}, } for name, kwargs in configs.items(): with self.subTest(name), tempfile.TemporaryDirectory() as tmp_dir: @@ -122,6 +124,8 @@ def test_matches_sentencepiece(self): tokenizer = TokenizersBackend.from_pretrained(tmp_dir) for text in self.texts: ids = sp.encode(text) + if sp.unk_id() in ids: # unseen characters without byte fallback decode differently + continue self.assertEqual(tokenizer.encode(text, add_special_tokens=False), ids, text) self.assertEqual(tokenizer.decode(ids), sp.decode(ids), text) From 1c05d24ec19bae95ff0c18e1da14f9dd6cd9c83d Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:55:12 +0000 Subject: [PATCH 06/17] Move the SentencePiece conversion changes to #50555 Co-Authored-By: Claude Opus 5.5 --- src/transformers/convert_slow_tokenizer.py | 73 +++------------ .../tokenization_utils_tokenizers.py | 71 ++++----------- tests/test_processing_common.py | 41 +++------ tests/utils/test_convert_slow_tokenizer.py | 90 +------------------ 4 files changed, 41 insertions(+), 234 deletions(-) diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py index 550a94481091..6e988ffd56e6 100644 --- a/src/transformers/convert_slow_tokenizer.py +++ b/src/transformers/convert_slow_tokenizer.py @@ -19,7 +19,6 @@ """ import os -import re import warnings from collections.abc import Collection @@ -640,55 +639,22 @@ class SpmConverter(Converter): @staticmethod def build_tokenizer_from_spm_proto(proto, vocab, merges=None): """ - Similar to convert_from_spm method, but used only when there is no `model_type` class, i.e. there is no matching - class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracting stuff from the sentencepiece - file. - - Follows the SentencePiece model's own settings, so the result encodes like `SentencePieceProcessor.encode`: - the normalizer spec (extra whitespace removal, dummy prefix, whitespace escaping), BPE merges ranked by piece - score, and user-defined pieces, which SentencePiece always emits whole and never merges. When `merges` is - `None`, they are generated from `vocab` and the piece scores. + Similar to convert_from_spm method, but used only when there is no `model_type` class, i.e. there is no matching class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracting stuff from the sentencepiece file """ byte_fallback = proto.trainer_spec.byte_fallback unk_piece = proto.trainer_spec.unk_piece - normalizer_spec = proto.normalizer_spec - precompiled_charsmap = normalizer_spec.precompiled_charsmap - # `vocab` may rename pieces (from `added_tokens_decoder`), so look pieces up by id - if isinstance(vocab, dict): - id_to_token = {token_id: token for token, token_id in vocab.items()} - else: - id_to_token = {token_id: item[0] for token_id, item in enumerate(vocab or [])} - user_defined = [id_to_token[i] for i, piece in enumerate(proto.pieces) if piece.type == 4 and i in id_to_token] - # A normalized added token's content is normalized too, so with a dummy prefix user-defined pieces would only - # match at the start of the text. Isolate them in the pre-tokenizer instead, after the prefix is added. - isolate_user_defined = bool(user_defined) and normalizer_spec.add_dummy_prefix + precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap # model if isinstance(vocab, dict): - if merges is None: - # Only normal pieces (type 1) are merged - normal = { - id_to_token[i]: (piece.score, i) - for i, piece in enumerate(proto.pieces) - if piece.type == 1 and i in id_to_token - } - ranked = [] - for token, rank in normal.items(): - for split in range(1, len(token)): - left, right = token[:split], token[split:] - if left in normal and right in normal: - ranked.append((-rank[0], rank[1], left, right)) - merges = [(left, right) for _, _, left, right in sorted(ranked)] tokenizer = Tokenizer( BPE( vocab=vocab, - merges=merges, + merges=merges or [], unk_token=unk_piece, fuse_unk=True, byte_fallback=byte_fallback, dropout=None, - # Isolated user-defined pieces aren't reachable through merges, so look whole words up first - ignore_merges=isolate_user_defined, ) ) elif isinstance(vocab, list) and vocab and isinstance(vocab[0], tuple | list): @@ -702,36 +668,19 @@ class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracti else: return None - # normalizer, in SentencePiece's order - _normalizers = [] + # normalizer + _normalizers = [normalizers.Replace(" ", "▁")] if precompiled_charsmap: - _normalizers.append(normalizers.Precompiled(precompiled_charsmap)) - if normalizer_spec.remove_extra_whitespaces: - # Only spaces count: SentencePiece leaves other whitespace, such as newlines, alone - _normalizers += [normalizers.Replace(Regex(r"\A +| +\z"), ""), normalizers.Replace(Regex(" {2,}"), " ")] - if normalizer_spec.add_dummy_prefix: - _normalizers.append(normalizers.Prepend("▁")) - if normalizer_spec.escape_whitespaces: - _normalizers.append(normalizers.Replace(" ", "▁")) + _normalizers.insert(0, normalizers.Precompiled(precompiled_charsmap)) tokenizer.normalizer = normalizers.Sequence(_normalizers) - # user-defined pieces - if isolate_user_defined: - pattern = "|".join(re.escape(piece) for piece in sorted(user_defined, key=len, reverse=True)) - tokenizer.pre_tokenizer = pre_tokenizers.Split(Regex(pattern), behavior="isolated") - elif user_defined: - tokenizer.add_tokens([AddedToken(piece, normalized=True, special=False) for piece in user_defined]) - # decoder - _decoders = [decoders.Replace("▁", " ")] if byte_fallback: - _decoders.append(decoders.ByteFallback()) - if normalizer_spec.add_dummy_prefix: - # `Strip` applies to each token, so fuse them first to strip only the start of the text - _decoders += [decoders.Fuse(), decoders.Strip(content=" ", left=1)] - elif byte_fallback: - _decoders.append(decoders.Fuse()) - tokenizer.decoder = decoders.Sequence(_decoders) + tokenizer.decoder = decoders.Sequence( + [decoders.Replace("▁", " "), decoders.ByteFallback(), decoders.Fuse()] + ) + else: + tokenizer.decoder = decoders.Sequence([decoders.Replace("▁", " ")]) return tokenizer diff --git a/src/transformers/tokenization_utils_tokenizers.py b/src/transformers/tokenization_utils_tokenizers.py index c9d3c7356e63..cbed619bb200 100644 --- a/src/transformers/tokenization_utils_tokenizers.py +++ b/src/transformers/tokenization_utils_tokenizers.py @@ -331,66 +331,25 @@ def _convert_from_sentencepiece(cls, vocab_file: str, local_kwargs: dict[str, An # from the proto so normalizer/decoder are configured correctly. if "tokenizer_object" not in local_kwargs and (cls is TokenizersBackend or "__init__" not in cls.__dict__): vocab = local_kwargs.pop("vocab", None) - # Regenerated from the piece scores by `build_tokenizer_from_spm_proto` - local_kwargs.pop("merges", None) + merges = local_kwargs.pop("merges", None) # Replace placeholder tokens as specified in added_tokens_decoder added_tokens_decoder = local_kwargs.get("added_tokens_decoder") or {} - if isinstance(vocab, dict) and added_tokens_decoder: + if vocab is not None and added_tokens_decoder: id_to_token = {token_id: token for token, token_id in vocab.items()} - renames = { - int(token_id): str(new_token) - for token_id, new_token in added_tokens_decoder.items() - if id_to_token.get(int(token_id)) not in (None, str(new_token)) - } - # A new token may already be in the vocab only if its id is renamed too (e.g. two pieces swapped) - renames = { - token_id: new_token - for token_id, new_token in renames.items() - if new_token not in vocab or vocab[new_token] in renames - } - for token_id in renames: - del vocab[id_to_token[token_id]] - for token_id, new_token in renames.items(): - vocab[new_token] = token_id - - # User-defined pieces are handled by the backend tokenizer, not as special tokens - user_defined = {piece.piece for piece in extractor.proto.pieces if piece.type == 4} - if user_defined and local_kwargs.get("additional_special_tokens"): - local_kwargs["additional_special_tokens"] = [ - token for token in local_kwargs["additional_special_tokens"] if str(token) not in user_defined - ] - - # The dummy prefix is added by the normalizer, which would also prefix a normalized added token's - # content so that it only matches at the start of the text. Match added tokens on the raw text instead, - # as SentencePiece-based tokenizers do. - if extractor.proto.normalizer_spec.add_dummy_prefix: - - def _unnormalized(token): - if not isinstance(token, AddedToken) or not token.normalized: - return token - return AddedToken( - token.content, - single_word=token.single_word, - lstrip=token.lstrip, - rstrip=token.rstrip, - normalized=False, - special=token.special, - ) - - if added_tokens_decoder: - local_kwargs["added_tokens_decoder"] = { - token_id: _unnormalized(token) for token_id, token in added_tokens_decoder.items() - } - for key in ("bos_token", "eos_token", "unk_token", "pad_token"): - if key in local_kwargs: - local_kwargs[key] = _unnormalized(local_kwargs[key]) - if local_kwargs.get("additional_special_tokens"): - local_kwargs["additional_special_tokens"] = [ - _unnormalized(token) for token in local_kwargs["additional_special_tokens"] - ] - - tokenizer_object = SpmConverter.build_tokenizer_from_spm_proto(proto=extractor.proto, vocab=vocab) + for token_id, new_token in added_tokens_decoder.items(): + token_id = int(token_id) + new_token = str(new_token) + current_token = id_to_token.get(token_id) + if current_token and current_token != new_token and new_token not in vocab: + vocab[new_token] = vocab.pop(current_token) + id_to_token[token_id] = new_token + + tokenizer_object = SpmConverter.build_tokenizer_from_spm_proto( + proto=extractor.proto, + vocab=vocab, + merges=merges, + ) if tokenizer_object is not None: local_kwargs["tokenizer_object"] = tokenizer_object # Set bos/eos tokens from proto spec if available. This is needed when diff --git a/tests/test_processing_common.py b/tests/test_processing_common.py index d4f20fe5d95c..3529dea56f9d 100644 --- a/tests/test_processing_common.py +++ b/tests/test_processing_common.py @@ -2004,38 +2004,23 @@ def test_replacement_offsets(self): else: processor_inputs[modality] = prepare_method() - # The offsets index the text after replacement, so capture the text the processor tokenizes - expanded_texts = [] - get_text_with_replacements = processor.get_text_with_replacements - - def capture_text_with_replacements(*args, **kwargs): - text, offsets = get_text_with_replacements(*args, **kwargs) - expanded_texts.append(text) - return text, offsets - # Test combined processing - with patch.object(processor, "get_text_with_replacements", side_effect=capture_text_with_replacements): - model_inputs = processor( - **processor_inputs, - add_special_tokens=False, - padding=False, - truncation=False, - max_length=None, - return_text_replacement_offsets=True, - **processing_kwargs, - ) - if expanded_texts: - expanded_text = expanded_texts[-1] - expanded_text = [expanded_text] if isinstance(expanded_text, str) else expanded_text - else: - # Processors that don't expand the text with `get_text_with_replacements` - expanded_text = processor.tokenizer.batch_decode( - model_inputs["input_ids"], skip_special_tokens=False, clean_up_tokenization_spaces=True - ) + model_inputs = processor( + **processor_inputs, + add_special_tokens=False, + padding=False, + truncation=False, + max_length=None, + return_text_replacement_offsets=True, + **processing_kwargs, + ) + detokenized_text = processor.tokenizer.batch_decode( + model_inputs["input_ids"], skip_special_tokens=False, clean_up_tokenization_spaces=True + ) self.assertIn("text_replacement_offsets", model_inputs) for i, sample_offsets in enumerate(model_inputs["text_replacement_offsets"]): for curr_dict in sample_offsets: self.assertIn(curr_dict["type"], modalities) start, end = curr_dict["new_span"] - self.assertEqual(expanded_text[i][start:end], curr_dict["replacement"]) + self.assertEqual(detokenized_text[i][start:end], curr_dict["replacement"]) diff --git a/tests/utils/test_convert_slow_tokenizer.py b/tests/utils/test_convert_slow_tokenizer.py index 44aea1986e9e..f0043522e30e 100644 --- a/tests/utils/test_convert_slow_tokenizer.py +++ b/tests/utils/test_convert_slow_tokenizer.py @@ -4,14 +4,9 @@ import warnings from dataclasses import dataclass -from transformers import AutoTokenizer, TokenizersBackend +from transformers import AutoTokenizer from transformers.convert_slow_tokenizer import SentencePieceExtractor, SpmConverter -from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers -from transformers.utils import is_sentencepiece_available - - -if is_sentencepiece_available(): - import sentencepiece as spm +from transformers.testing_utils import get_tests_dir @dataclass @@ -62,84 +57,3 @@ def test_spm_precompiled_charsmap_empty_is_none(self): tokenizer = AutoTokenizer.from_pretrained(tmp_dir) self.assertGreater(len(tokenizer("Hello, world!")["input_ids"]), 1) - - -@require_sentencepiece -@require_tokenizers -class SentencePieceModelConversionTest(unittest.TestCase): - """`TokenizersBackend` built from a bare `tokenizer.model` should encode and decode like SentencePiece.""" - - corpus = [ - "Hello world, this is a test of the tokenizer.", - " def f(x):\n return x + 1 # comment", - "数字 12345 和中文文本，还有更多的中文。", - "foo_bar__baz qux", - ] * 50 - texts = [ - "Hello world\n__x 123", - " leading space", - "trailing ", - "\n\n indented\tcode", - "中文 text 123", - "__init__", - "🤗 unseen characters", - "", - ] - - def _train(self, tmp_dir, **kwargs): - kwargs.setdefault("byte_fallback", True) - kwargs.setdefault("vocab_size", 400) - spm.SentencePieceTrainer.train( - sentence_iterator=iter(self.corpus), - model_prefix=f"{tmp_dir}/tokenizer", - model_type="bpe", - normalization_rule_name="identity", - minloglevel=2, - **kwargs, - ) - return spm.SentencePieceProcessor(model_file=f"{tmp_dir}/tokenizer.model") - - def test_matches_sentencepiece(self): - configs = { - # User-defined pieces are never merged, and with a dummy prefix every chunk starts with "▁" - "dummy_prefix_user_defined": { - "add_dummy_prefix": True, - "remove_extra_whitespaces": False, - "allow_whitespace_only_pieces": True, - "split_digits": True, - "user_defined_symbols": ["__", "", "\n"], - }, - "no_dummy_prefix_user_defined": { - "add_dummy_prefix": False, - "remove_extra_whitespaces": False, - "user_defined_symbols": ["__", "", "\n"], - }, - "remove_extra_whitespaces": {"add_dummy_prefix": True, "remove_extra_whitespaces": True}, - # Without byte fallback the decoder has no `Fuse`, so only the start of the text may lose its space - "no_byte_fallback": {"add_dummy_prefix": True, "byte_fallback": False, "vocab_size": 200}, - } - for name, kwargs in configs.items(): - with self.subTest(name), tempfile.TemporaryDirectory() as tmp_dir: - sp = self._train(tmp_dir, **kwargs) - tokenizer = TokenizersBackend.from_pretrained(tmp_dir) - for text in self.texts: - ids = sp.encode(text) - if sp.unk_id() in ids: # unseen characters without byte fallback decode differently - continue - self.assertEqual(tokenizer.encode(text, add_special_tokens=False), ids, text) - self.assertEqual(tokenizer.decode(ids), sp.decode(ids), text) - - def test_added_tokens_decoder_can_swap_pieces(self): - with tempfile.TemporaryDirectory() as tmp_dir: - sp = self._train(tmp_dir, control_symbols=["", ""]) - a_id, b_id = sp.piece_to_id(""), sp.piece_to_id("") - added_tokens_decoder = { - str(a_id): {"content": "", "special": True}, - str(b_id): {"content": "", "special": True}, - } - with open(f"{tmp_dir}/tokenizer_config.json", "w", encoding="utf-8") as f: - json.dump({"added_tokens_decoder": added_tokens_decoder}, f) - - tokenizer = TokenizersBackend.from_pretrained(tmp_dir) - self.assertEqual(tokenizer.convert_tokens_to_ids(["", ""]), [b_id, a_id]) - self.assertEqual(tokenizer.encode("x", add_special_tokens=False)[::2], [b_id, a_id]) From a2281c222bab907a41b2718f72872cb9291cdcd1 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:55:12 +0000 Subject: [PATCH 07/17] Use the Llama implementation for InternLM3, IQuestCoder and TeleChat3 They only differ from Llama in their configs, so keep only the configs and point the auto mappings at the Llama classes. InternLM3's `qkv_bias` and `bias` are loaded as `attention_bias` and `mlp_bias`. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 3 - docs/source/en/model_doc/internlm3.md | 16 +- docs/source/en/model_doc/iquestcoder.md | 29 +- docs/source/en/model_doc/telechat.md | 2 +- docs/source/en/model_doc/telechat3.md | 14 +- src/transformers/models/auto/modeling_auto.py | 24 +- src/transformers/models/internlm3/__init__.py | 1 - .../internlm3/configuration_internlm3.py | 21 +- .../models/internlm3/modeling_internlm3.py | 465 ---------------- .../models/internlm3/modular_internlm3.py | 75 +-- .../models/iquestcoder/__init__.py | 1 - .../iquestcoder/configuration_iquestcoder.py | 6 +- .../iquestcoder/modeling_iquestcoder.py | 498 ------------------ .../models/iquestcoder/modular_iquestcoder.py | 49 +- src/transformers/models/telechat3/__init__.py | 1 - .../telechat3/configuration_telechat3.py | 6 +- .../models/telechat3/modeling_telechat3.py | 474 ----------------- .../models/telechat3/modular_telechat3.py | 31 +- .../internlm3/test_modeling_internlm3.py | 31 +- .../iquestcoder/test_modeling_iquestcoder.py | 13 +- .../telechat3/test_modeling_telechat3.py | 13 +- utils/check_config_attributes.py | 4 + 22 files changed, 100 insertions(+), 1677 deletions(-) delete mode 100644 src/transformers/models/internlm3/modeling_internlm3.py delete mode 100644 src/transformers/models/iquestcoder/modeling_iquestcoder.py delete mode 100644 src/transformers/models/telechat3/modeling_telechat3.py diff --git a/.gitattributes b/.gitattributes index 82324e230c85..15b21dc8dfcd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -268,10 +268,8 @@ src/transformers/models/inkling/modeling_*.py linguist-generated=true src/transformers/models/instructblipvideo/configuration_*.py linguist-generated=true src/transformers/models/instructblipvideo/modeling_*.py linguist-generated=true src/transformers/models/internlm3/configuration_*.py linguist-generated=true -src/transformers/models/internlm3/modeling_*.py linguist-generated=true src/transformers/models/internvl/modeling_*.py linguist-generated=true src/transformers/models/iquestcoder/configuration_*.py linguist-generated=true -src/transformers/models/iquestcoder/modeling_*.py linguist-generated=true src/transformers/models/jais2/configuration_*.py linguist-generated=true src/transformers/models/jais2/modeling_*.py linguist-generated=true src/transformers/models/jamba/modeling_*.py linguist-generated=true @@ -575,7 +573,6 @@ src/transformers/models/t5gemma2/modeling_*.py linguist-generated=true src/transformers/models/telechat/configuration_*.py linguist-generated=true src/transformers/models/telechat/modeling_*.py linguist-generated=true src/transformers/models/telechat3/configuration_*.py linguist-generated=true -src/transformers/models/telechat3/modeling_*.py linguist-generated=true src/transformers/models/timesfm/modeling_*.py linguist-generated=true src/transformers/models/timesfm2_5/configuration_*.py linguist-generated=true src/transformers/models/timesfm2_5/modeling_*.py linguist-generated=true diff --git a/docs/source/en/model_doc/internlm3.md b/docs/source/en/model_doc/internlm3.md index ffa53b1be7c7..3b9534314d8d 100644 --- a/docs/source/en/model_doc/internlm3.md +++ b/docs/source/en/model_doc/internlm3.md @@ -13,7 +13,7 @@ specific language governing permissions and limitations under the License. rendered properly in your Markdown viewer. --> -*This model was contributed to Hugging Face Transformers on 2026-11-08.* +*This model was contributed to Hugging Face Transformers on 2026-11-10.*
FlashAttention @@ -23,7 +23,9 @@ rendered properly in your Markdown viewer. # InternLM3 -[InternLM3](https://huggingface.co/internlm/internlm3-8b-instruct) is an 8B instruction-tuned language model developed by [Shanghai AI Laboratory](https://huggingface.co/internlm). Its architecture is the same as [Llama](./llama), except that the biases of the query, key and value projections (`qkv_bias`) are configured separately from the biases of the attention output projection and MLP layers (`bias`). +[InternLM3](https://huggingface.co/internlm/internlm3-8b-instruct) is an 8B instruction-tuned language model developed by [Shanghai AI Laboratory](https://huggingface.co/internlm). Its architecture is the same as [Llama](./llama). Its config names the biases of the query, key and value projections `qkv_bias` and those of the attention output projection and MLP layers `bias`; [`InternLM3Config`] loads them as `attention_bias` and `mlp_bias`, and requires them to be equal, as they are in the released checkpoints. + +The auto classes load it with the Llama implementation ([`LlamaModel`], [`LlamaForCausalLM`], [`LlamaForSequenceClassification`], [`LlamaForQuestionAnswering`] and [`LlamaForTokenClassification`]), so only [`InternLM3Config`] is specific to InternLM3. The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. @@ -43,13 +45,3 @@ print(tokenizer.decode(output[0][inputs["input_ids"].shape[1] :], skip_special_t ## InternLM3Config [[autodoc]] InternLM3Config - -## InternLM3Model - -[[autodoc]] InternLM3Model - - forward - -## InternLM3ForCausalLM - -[[autodoc]] InternLM3ForCausalLM - - forward diff --git a/docs/source/en/model_doc/iquestcoder.md b/docs/source/en/model_doc/iquestcoder.md index 63c914b12195..6009b0479a20 100644 --- a/docs/source/en/model_doc/iquestcoder.md +++ b/docs/source/en/model_doc/iquestcoder.md @@ -13,7 +13,7 @@ specific language governing permissions and limitations under the License. rendered properly in your Markdown viewer. --> -*This model was contributed to Hugging Face Transformers on 2026-11-08.* +*This model was contributed to Hugging Face Transformers on 2026-11-10.*
FlashAttention @@ -29,6 +29,8 @@ The looped `IQuest-Coder-V1-*-Loop-*` checkpoints use a different architecture a The tokenizer is built from the checkpoints' SentencePiece model. Many of its pieces share the same score, and SentencePiece breaks those ties by position, which the `tokenizers` library can't reproduce, so a small fraction of texts tokenize differently from the original remote code. +The auto classes load it with the Llama implementation ([`LlamaModel`], [`LlamaForCausalLM`], [`LlamaForSequenceClassification`], [`LlamaForQuestionAnswering`] and [`LlamaForTokenClassification`]), so only [`IQuestCoderConfig`] is specific to IQuestCoder. + The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. ```python @@ -47,28 +49,3 @@ print(tokenizer.decode(output[0][inputs["input_ids"].shape[1] :], skip_special_t ## IQuestCoderConfig [[autodoc]] IQuestCoderConfig - -## IQuestCoderModel - -[[autodoc]] IQuestCoderModel - - forward - -## IQuestCoderForCausalLM - -[[autodoc]] IQuestCoderForCausalLM - - forward - -## IQuestCoderForSequenceClassification - -[[autodoc]] IQuestCoderForSequenceClassification - - forward - -## IQuestCoderForQuestionAnswering - -[[autodoc]] IQuestCoderForQuestionAnswering - - forward - -## IQuestCoderForTokenClassification - -[[autodoc]] IQuestCoderForTokenClassification - - forward diff --git a/docs/source/en/model_doc/telechat.md b/docs/source/en/model_doc/telechat.md index 96e50a35f31d..88a7c1785bb8 100644 --- a/docs/source/en/model_doc/telechat.md +++ b/docs/source/en/model_doc/telechat.md @@ -13,7 +13,7 @@ specific language governing permissions and limitations under the License. rendered properly in your Markdown viewer. --> -*This model was contributed to Hugging Face Transformers on 2026-11-08.* +*This model was contributed to Hugging Face Transformers on 2026-11-10.*
FlashAttention diff --git a/docs/source/en/model_doc/telechat3.md b/docs/source/en/model_doc/telechat3.md index 96c7472f3b1f..9f1cf6f99706 100644 --- a/docs/source/en/model_doc/telechat3.md +++ b/docs/source/en/model_doc/telechat3.md @@ -13,7 +13,7 @@ specific language governing permissions and limitations under the License. rendered properly in your Markdown viewer. --> -*This model was contributed to Hugging Face Transformers on 2026-11-08.* +*This model was contributed to Hugging Face Transformers on 2026-11-10.*
FlashAttention @@ -27,6 +27,8 @@ rendered properly in your Markdown viewer. The original checkpoints use a custom `"telechat3-yarn"` RoPE type, which is YaRN with 0.07 instead of 0.1 as the coefficient of its default attention factor. [`TeleChat3Config`] converts it to `"yarn"` with an explicit `attention_factor` when the config is loaded. +The auto classes load it with the Llama implementation ([`LlamaModel`], [`LlamaForCausalLM`], [`LlamaForSequenceClassification`], [`LlamaForQuestionAnswering`] and [`LlamaForTokenClassification`]), so only [`TeleChat3Config`] is specific to TeleChat3. + The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. ```python @@ -45,13 +47,3 @@ print(tokenizer.decode(output[0][inputs["input_ids"].shape[1] :], skip_special_t ## TeleChat3Config [[autodoc]] TeleChat3Config - -## TeleChat3Model - -[[autodoc]] TeleChat3Model - - forward - -## TeleChat3ForCausalLM - -[[autodoc]] TeleChat3ForCausalLM - - forward diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index a69e800dae33..799e8f523668 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -283,10 +283,10 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("inkling_text", "InklingTextModel"), ("instructblip", "InstructBlipModel"), ("instructblipvideo", "InstructBlipVideoModel"), - ("internlm3", "InternLM3Model"), + ("internlm3", "LlamaModel"), ("internvl", "InternVLModel"), ("internvl_vision", "InternVLVisionModel"), - ("iquestcoder", "IQuestCoderModel"), + ("iquestcoder", "LlamaModel"), ("jais2", "Jais2Model"), ("jamba", "JambaModel"), ("janus", "JanusModel"), @@ -569,7 +569,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("table-transformer", "TableTransformerModel"), ("tapas", "TapasModel"), ("telechat", "TeleChatModel"), - ("telechat3", "TeleChat3Model"), + ("telechat3", "LlamaModel"), ("textnet", "TextNetModel"), ("time_series_transformer", "TimeSeriesTransformerModel"), ("timesfm", "TimesFmModel"), @@ -843,8 +843,8 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("hy_v4", "HYV4ForCausalLM"), ("hyperclovax", "HyperCLOVAXForCausalLM"), ("inkling_text", "InklingForCausalLM"), - ("internlm3", "InternLM3ForCausalLM"), - ("iquestcoder", "IQuestCoderForCausalLM"), + ("internlm3", "LlamaForCausalLM"), + ("iquestcoder", "LlamaForCausalLM"), ("jais2", "Jais2ForCausalLM"), ("jamba", "JambaForCausalLM"), ("jetmoe", "JetMoeForCausalLM"), @@ -924,7 +924,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("stablelm", "StableLmForCausalLM"), ("starcoder2", "Starcoder2ForCausalLM"), ("telechat", "TeleChatForCausalLM"), - ("telechat3", "TeleChat3ForCausalLM"), + ("telechat3", "LlamaForCausalLM"), ("trocr", "TrOCRForCausalLM"), ("vaultgemma", "VaultGemmaForCausalLM"), ("whisper", "WhisperForCausalLM"), @@ -1520,7 +1520,8 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("hunyuan_v1_dense", "HunYuanDenseV1ForSequenceClassification"), ("hunyuan_v1_moe", "HunYuanMoEV1ForSequenceClassification"), ("ibert", "IBertForSequenceClassification"), - ("iquestcoder", "IQuestCoderForSequenceClassification"), + ("internlm3", "LlamaForSequenceClassification"), + ("iquestcoder", "LlamaForSequenceClassification"), ("jamba", "JambaForSequenceClassification"), ("jetmoe", "JetMoeForSequenceClassification"), ("jina_embeddings_v3", "JinaEmbeddingsV3ForSequenceClassification"), @@ -1586,6 +1587,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("t5gemma", "T5GemmaForSequenceClassification"), ("t5gemma2", "T5Gemma2ForSequenceClassification"), ("tapas", "TapasForSequenceClassification"), + ("telechat3", "LlamaForSequenceClassification"), ("ultrabert", "UltrabertForSequenceClassification"), ("ultravbert", "UltraVBertForSequenceClassification"), ("umt5", "UMT5ForSequenceClassification"), @@ -1630,7 +1632,8 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("gpt_neox", "GPTNeoXForQuestionAnswering"), ("gptj", "GPTJForQuestionAnswering"), ("ibert", "IBertForQuestionAnswering"), - ("iquestcoder", "IQuestCoderForQuestionAnswering"), + ("internlm3", "LlamaForQuestionAnswering"), + ("iquestcoder", "LlamaForQuestionAnswering"), ("jina_embeddings_v3", "JinaEmbeddingsV3ForQuestionAnswering"), ("layoutlmv2", "LayoutLMv2ForQuestionAnswering"), ("layoutlmv3", "LayoutLMv3ForQuestionAnswering"), @@ -1674,6 +1677,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("splinter", "SplinterForQuestionAnswering"), ("squeezebert", "SqueezeBertForQuestionAnswering"), ("t5", "T5ForQuestionAnswering"), + ("telechat3", "LlamaForQuestionAnswering"), ("ultrabert", "UltrabertForQuestionAnswering"), ("umt5", "UMT5ForQuestionAnswering"), ("xlm", "XLMForQuestionAnsweringSimple"), @@ -1753,7 +1757,8 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("gte", "GteForTokenClassification"), ("helium", "HeliumForTokenClassification"), ("ibert", "IBertForTokenClassification"), - ("iquestcoder", "IQuestCoderForTokenClassification"), + ("internlm3", "LlamaForTokenClassification"), + ("iquestcoder", "LlamaForTokenClassification"), ("jina_embeddings_v3", "JinaEmbeddingsV3ForTokenClassification"), ("layoutlm", "LayoutLMForTokenClassification"), ("layoutlmv2", "LayoutLMv2ForTokenClassification"), @@ -1804,6 +1809,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("t5", "T5ForTokenClassification"), ("t5gemma", "T5GemmaForTokenClassification"), ("t5gemma2", "T5Gemma2ForTokenClassification"), + ("telechat3", "LlamaForTokenClassification"), ("ultrabert", "UltrabertForTokenClassification"), ("ultravbert", "UltraVBertForTokenClassification"), ("umt5", "UMT5ForTokenClassification"), diff --git a/src/transformers/models/internlm3/__init__.py b/src/transformers/models/internlm3/__init__.py index 0638ebaf2c91..6951a55f338d 100644 --- a/src/transformers/models/internlm3/__init__.py +++ b/src/transformers/models/internlm3/__init__.py @@ -25,7 +25,6 @@ if TYPE_CHECKING: from .configuration_internlm3 import * - from .modeling_internlm3 import * else: import sys diff --git a/src/transformers/models/internlm3/configuration_internlm3.py b/src/transformers/models/internlm3/configuration_internlm3.py index e64fb8145ba7..2d0a4c4b7e5e 100644 --- a/src/transformers/models/internlm3/configuration_internlm3.py +++ b/src/transformers/models/internlm3/configuration_internlm3.py @@ -29,19 +29,14 @@ @strict class InternLM3Config(PreTrainedConfig): r""" - qkv_bias (`bool`, *optional*, defaults to `False`): - Whether to use a bias in the query, key and value projection layers. - bias (`bool`, *optional*, defaults to `False`): - Whether to use a bias in the attention output projection and MLP layers. - ```python - >>> from transformers import InternLM3Model, InternLM3Config + >>> from transformers import AutoModelForCausalLM, InternLM3Config >>> # Initializing an InternLM3-8B style configuration >>> configuration = InternLM3Config() - >>> # Initializing a model from the InternLM3-8B style configuration - >>> model = InternLM3Model(configuration) + >>> # Initializing a model (which uses the Llama implementation) from that configuration + >>> model = AutoModelForCausalLM.from_config(configuration) >>> # Accessing the model configuration >>> configuration = model.config @@ -81,10 +76,11 @@ class InternLM3Config(PreTrainedConfig): eos_token_id: int | list[int] | None = 2 tie_word_embeddings: bool = False rope_parameters: RopeParameters | dict | None = None + attention_bias: bool = False attention_dropout: float | int = 0.0 + mlp_bias: bool = False head_dim: int | None = 128 - qkv_bias: bool = False - bias: bool = False + attribute_map = {"qkv_bias": "attention_bias", "bias": "mlp_bias"} def __post_init__(self, **kwargs): if self.head_dim is None: @@ -101,6 +97,11 @@ def validate_architecture(self): f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention " f"heads ({self.num_attention_heads})." ) + if self.attention_bias != self.mlp_bias: + raise ValueError( + "InternLM3 with `qkv_bias` different from `bias` is not supported, because the Llama implementation " + "uses the same bias setting for the query, key, value and output projections." + ) __all__ = ["InternLM3Config"] diff --git a/src/transformers/models/internlm3/modeling_internlm3.py b/src/transformers/models/internlm3/modeling_internlm3.py deleted file mode 100644 index 3f5f055f77ac..000000000000 --- a/src/transformers/models/internlm3/modeling_internlm3.py +++ /dev/null @@ -1,465 +0,0 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from src/transformers/models/internlm3/modular_internlm3.py. -# Do NOT edit this file manually as any edits will be overwritten by the generation of -# the file from the modular. If any change should be done, please apply the change to the -# modular_internlm3.py file directly. One of our CI enforces this. -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections.abc import Callable - -import torch -from torch import nn - -from ...activations import ACT2FN -from ...cache_utils import Cache, DynamicCache -from ...generation import GenerationMixin -from ...integrations import use_kernel_forward_from_hub, use_kernelized_func -from ...masking_utils import create_causal_mask -from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast -from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update -from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep -from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple -from ...utils.generic import merge_with_config_defaults -from ...utils.output_capturing import capture_outputs -from .configuration_internlm3 import InternLM3Config - - -class InternLM3MLP(nn.Module): - def __init__(self, config: InternLM3Config): - super().__init__() - self.config = config - self.hidden_size = config.hidden_size - self.intermediate_size = config.intermediate_size - self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.bias) - self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.bias) - self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.bias) - self.act_fn = ACT2FN[config.hidden_act] - - def forward(self, x): - down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) - return down_proj - - -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -@use_kernel_forward_from_hub("rotary_pos_emb") -def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): - """Applies Rotary Position Embedding to the query and key tensors. - - Args: - q (`torch.Tensor`): The query tensor. - k (`torch.Tensor`): The key tensor. - cos (`torch.Tensor`): The cosine part of the rotary embedding. - sin (`torch.Tensor`): The sine part of the rotary embedding. - unsqueeze_dim (`int`, *optional*, defaults to 1): - The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - Returns: - `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. - """ - cos = cos.unsqueeze(unsqueeze_dim) - sin = sin.unsqueeze(unsqueeze_dim) - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed, k_embed - - -def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """ - This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, - num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) - """ - batch, num_key_value_heads, slen, head_dim = hidden_states.shape - if n_rep == 1: - return hidden_states - hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) - return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) - - -def eager_attention_forward( - module: nn.Module, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - attention_mask: torch.Tensor | None, - scaling: float, - dropout: float = 0.0, - **kwargs: Unpack[TransformersKwargs], -): - key_states = repeat_kv(key, module.num_key_value_groups) - value_states = repeat_kv(value, module.num_key_value_groups) - - attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling - if attention_mask is not None: - attn_weights = attn_weights + attention_mask - - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) - attn_output = torch.matmul(attn_weights, value_states) - attn_output = attn_output.transpose(1, 2).contiguous() - - return attn_output, attn_weights - - -@use_kernelized_func(apply_rotary_pos_emb) -class InternLM3Attention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config: InternLM3Config, layer_idx: int): - super().__init__() - self.config = config - self.layer_idx = layer_idx - self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) - self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads - self.scaling = self.head_dim**-0.5 - self.attention_dropout = config.attention_dropout - self.is_causal = True - self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.qkv_bias) - self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.qkv_bias) - self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.qkv_bias) - self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.bias) - - def forward( - self, - hidden_states: torch.Tensor, - position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, - attention_mask: torch.Tensor | None = None, - past_key_values: Cache | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> tuple[torch.Tensor, torch.Tensor]: - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - - query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) - - if past_key_values is not None: - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) - - attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, eager_attention_forward - ) - - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask, - dropout=0.0 if not self.training else self.attention_dropout, - scaling=self.scaling, - **kwargs, - ) - - attn_output = attn_output.reshape(*input_shape, -1).contiguous() - attn_output = self.o_proj(attn_output) - return attn_output, attn_weights - - -@use_kernel_forward_from_hub("RMSNorm") -class InternLM3RMSNorm(nn.Module): - def __init__(self, hidden_size, eps: float = 1e-6) -> None: - """ - InternLM3RMSNorm is equivalent to T5LayerNorm - """ - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden_size)) - self.variance_epsilon = eps - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - input_dtype = hidden_states.dtype - hidden_states = hidden_states.to(torch.float32) - variance = hidden_states.pow(2).mean(-1, keepdim=True) - hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) - return self.weight * hidden_states.to(input_dtype) - - def extra_repr(self): - return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" - - -class InternLM3DecoderLayer(GradientCheckpointingLayer): - def __init__(self, config: InternLM3Config, layer_idx: int): - super().__init__() - self.hidden_size = config.hidden_size - - self.self_attn = InternLM3Attention(config=config, layer_idx=layer_idx) - - self.mlp = InternLM3MLP(config) - self.input_layernorm = InternLM3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = InternLM3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: Cache | None = None, - use_cache: bool | None = False, - position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> torch.Tensor: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - # Self Attention - hidden_states, _ = self.self_attn( - hidden_states=hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - use_cache=use_cache, - position_embeddings=position_embeddings, - **kwargs, - ) - hidden_states = residual + hidden_states - - # Fully Connected - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states - return hidden_states - - -@auto_docstring -class InternLM3PreTrainedModel(PreTrainedModel): - config: InternLM3Config - base_model_prefix = "model" - supports_gradient_checkpointing = True - _no_split_modules = ["InternLM3DecoderLayer"] - _skip_keys_device_placement = ["past_key_values"] - _supports_flash_attn = True - _supports_sdpa = True - _supports_flex_attn = True - - _can_compile_fullgraph = True - _supports_attention_backend = True - _can_record_outputs = { - "hidden_states": InternLM3DecoderLayer, - "attentions": InternLM3Attention, - } - - -class InternLM3RotaryEmbedding(nn.Module): - def __init__(self, config: InternLM3Config): - super().__init__() - self.max_seq_len_cached = config.max_position_embeddings - self.original_max_seq_len = config.max_position_embeddings - - self.config = config - - self.rope_type = self.config.rope_parameters["rope_type"] - rope_init_fn: Callable = self.compute_default_rope_parameters - if self.rope_type != "default": - rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] - inv_freq, self.attention_scaling = rope_init_fn(self.config) - - self.inv_freq = nn.Buffer(inv_freq, persistent=False) - self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False) - - @staticmethod - def compute_default_rope_parameters(config: InternLM3Config, **kwargs) -> tuple[torch.Tensor, float]: - """ - Computes the inverse frequencies according to the original RoPE implementation - Args: - config ([`~transformers.PreTrainedConfig`]): - The model configuration. - Returns: - Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the - post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). - """ - base = config.rope_parameters["rope_theta"] - dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads - - attention_factor = 1.0 # Unused in this type of RoPE - # Compute the inverse frequencies - inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) - return inv_freq, attention_factor - - @torch.no_grad() - @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) - def forward(self, x, position_ids): - freqs = position_ids[..., None].float() * self.inv_freq.to(device=x.device, dtype=torch.float) - emb = torch.cat((freqs, freqs), dim=-1) - cos = emb.cos() * self.attention_scaling - sin = emb.sin() * self.attention_scaling - - return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) - - -@auto_docstring -class InternLM3Model(InternLM3PreTrainedModel): - def __init__(self, config: InternLM3Config): - super().__init__(config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - - self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) - self.layers = nn.ModuleList( - [InternLM3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] - ) - self.norm = InternLM3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.rotary_emb = InternLM3RotaryEmbedding(config=config) - self.gradient_checkpointing = False - - # Initialize weights and apply final processing - self.post_init() - - @merge_with_config_defaults - @capture_outputs - @auto_docstring - def forward( - self, - input_ids: torch.LongTensor | None = None, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: Cache | None = None, - inputs_embeds: torch.Tensor | None = None, - use_cache: bool | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPast: - if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - - if inputs_embeds is None: - inputs_embeds: torch.Tensor = self.embed_tokens(input_ids) - - if use_cache and past_key_values is None: - past_key_values = DynamicCache(config=self.config) - - if position_ids is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - position_ids = position_ids.unsqueeze(0) - - causal_mask = create_causal_mask( - config=self.config, - inputs_embeds=inputs_embeds, - attention_mask=attention_mask, - past_key_values=past_key_values, - position_ids=position_ids, - ) - - hidden_states = inputs_embeds - position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) - - for decoder_layer in self.layers[: self.config.num_hidden_layers]: - hidden_states = decoder_layer( - hidden_states, - attention_mask=causal_mask, - position_embeddings=position_embeddings, - position_ids=position_ids, - past_key_values=past_key_values, - use_cache=use_cache, - **kwargs, - ) - - hidden_states = self.norm(hidden_states) - return BaseModelOutputWithPast( - last_hidden_state=hidden_states, - past_key_values=past_key_values, - ) - - -@auto_docstring(checkpoint="internlm/internlm3-8b-instruct") -class InternLM3ForCausalLM(InternLM3PreTrainedModel, GenerationMixin): - _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} - _tp_plan = {"lm_head": "colwise_gather_output"} - _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} - _fsdp_plan = {"lm_head": "keep_full_weight"} - - def __init__(self, config: InternLM3Config): - super().__init__(config) - self.model = InternLM3Model(config) - self.vocab_size = config.vocab_size - self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - - # Initialize weights and apply final processing - self.post_init() - - @can_return_tuple - @auto_docstring - def forward( - self, - input_ids: torch.LongTensor | None = None, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: Cache | None = None, - inputs_embeds: torch.Tensor | None = None, - labels: torch.LongTensor | None = None, - use_cache: bool | None = None, - logits_to_keep: int | torch.Tensor = 0, - **kwargs: Unpack[TransformersKwargs], - ) -> CausalLMOutputWithPast: - r""" - Example: - - ```python - >>> from transformers import AutoTokenizer, InternLM3ForCausalLM - - >>> model = InternLM3ForCausalLM.from_pretrained("meta-internlm3/InternLM3-2-7b-hf") - >>> tokenizer = AutoTokenizer.from_pretrained("meta-internlm3/InternLM3-2-7b-hf") - - >>> prompt = "Hey, are you conscious? Can you talk to me?" - >>> inputs = tokenizer(prompt, return_tensors="pt") - - >>> # Generate - >>> generate_ids = model.generate(inputs.input_ids, max_length=30) - >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." - ```""" - outputs: BaseModelOutputWithPast = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - **kwargs, - ) - - hidden_states = outputs.last_hidden_state - hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep) - logits = self.lm_head(hidden_states) - - loss = None - if labels is not None: - loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) - - return CausalLMOutputWithPast( - loss=loss, - logits=logits, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - ) - - -__all__ = ["InternLM3ForCausalLM", "InternLM3Model", "InternLM3PreTrainedModel"] diff --git a/src/transformers/models/internlm3/modular_internlm3.py b/src/transformers/models/internlm3/modular_internlm3.py index c63fb3d66fef..d0609ae5bdee 100644 --- a/src/transformers/models/internlm3/modular_internlm3.py +++ b/src/transformers/models/internlm3/modular_internlm3.py @@ -13,44 +13,31 @@ # limitations under the License. from huggingface_hub.dataclasses import strict -from torch import nn from ...modeling_rope_utils import RopeParameters from ...utils import auto_docstring from ..llama.configuration_llama import LlamaConfig -from ..llama.modeling_llama import ( - LlamaAttention, - LlamaDecoderLayer, - LlamaForCausalLM, - LlamaMLP, - LlamaModel, - LlamaPreTrainedModel, -) @auto_docstring(checkpoint="internlm/internlm3-8b-instruct") @strict class InternLM3Config(LlamaConfig): r""" - qkv_bias (`bool`, *optional*, defaults to `False`): - Whether to use a bias in the query, key and value projection layers. - bias (`bool`, *optional*, defaults to `False`): - Whether to use a bias in the attention output projection and MLP layers. - ```python - >>> from transformers import InternLM3Model, InternLM3Config + >>> from transformers import AutoModelForCausalLM, InternLM3Config >>> # Initializing an InternLM3-8B style configuration >>> configuration = InternLM3Config() - >>> # Initializing a model from the InternLM3-8B style configuration - >>> model = InternLM3Model(configuration) + >>> # Initializing a model (which uses the Llama implementation) from that configuration + >>> model = AutoModelForCausalLM.from_config(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```""" model_type = "internlm3" + attribute_map = {"qkv_bias": "attention_bias", "bias": "mlp_bias"} vocab_size: int = 128512 hidden_size: int = 4096 @@ -68,53 +55,21 @@ class InternLM3Config(LlamaConfig): eos_token_id: int | list[int] | None = 2 tie_word_embeddings: bool = False rope_parameters: RopeParameters | dict | None = None + attention_bias: bool = False attention_dropout: float | int = 0.0 + mlp_bias: bool = False head_dim: int | None = 128 - qkv_bias: bool = False - bias: bool = False - attention_bias = AttributeError() - mlp_bias = AttributeError() pretraining_tp = AttributeError() - -class InternLM3MLP(LlamaMLP): - def __init__(self, config: InternLM3Config): - super().__init__(config) - self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.bias) - self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.bias) - self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.bias) - - -class InternLM3Attention(LlamaAttention): - def __init__(self, config: InternLM3Config, layer_idx: int): - super().__init__(config, layer_idx) - self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.qkv_bias) - self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.qkv_bias) - self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.qkv_bias) - self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.bias) - - -class InternLM3DecoderLayer(LlamaDecoderLayer): - pass - - -class InternLM3PreTrainedModel(LlamaPreTrainedModel): - pass - - -class InternLM3Model(LlamaModel): - pass - - -@auto_docstring(checkpoint="internlm/internlm3-8b-instruct") -class InternLM3ForCausalLM(LlamaForCausalLM): - pass + def validate_architecture(self): + """Part of `@strict`-powered validation. Validates the architecture of the config.""" + super().validate_architecture() + if self.attention_bias != self.mlp_bias: + raise ValueError( + "InternLM3 with `qkv_bias` different from `bias` is not supported, because the Llama implementation " + "uses the same bias setting for the query, key, value and output projections." + ) -__all__ = [ - "InternLM3Config", - "InternLM3ForCausalLM", - "InternLM3Model", - "InternLM3PreTrainedModel", -] +__all__ = ["InternLM3Config"] diff --git a/src/transformers/models/iquestcoder/__init__.py b/src/transformers/models/iquestcoder/__init__.py index 0f0e6330adf3..07332cb6f1ff 100644 --- a/src/transformers/models/iquestcoder/__init__.py +++ b/src/transformers/models/iquestcoder/__init__.py @@ -25,7 +25,6 @@ if TYPE_CHECKING: from .configuration_iquestcoder import * - from .modeling_iquestcoder import * else: import sys diff --git a/src/transformers/models/iquestcoder/configuration_iquestcoder.py b/src/transformers/models/iquestcoder/configuration_iquestcoder.py index b485d37b3935..18549aa0ee43 100644 --- a/src/transformers/models/iquestcoder/configuration_iquestcoder.py +++ b/src/transformers/models/iquestcoder/configuration_iquestcoder.py @@ -30,13 +30,13 @@ class IQuestCoderConfig(PreTrainedConfig): r""" ```python - >>> from transformers import IQuestCoderModel, IQuestCoderConfig + >>> from transformers import AutoModelForCausalLM, IQuestCoderConfig >>> # Initializing an IQuest-Coder-V1-40B style configuration >>> configuration = IQuestCoderConfig() - >>> # Initializing a model from the IQuest-Coder-V1-40B style configuration - >>> model = IQuestCoderModel(configuration) + >>> # Initializing a model (which uses the Llama implementation) from that configuration + >>> model = AutoModelForCausalLM.from_config(configuration) >>> # Accessing the model configuration >>> configuration = model.config diff --git a/src/transformers/models/iquestcoder/modeling_iquestcoder.py b/src/transformers/models/iquestcoder/modeling_iquestcoder.py deleted file mode 100644 index ab75e70df189..000000000000 --- a/src/transformers/models/iquestcoder/modeling_iquestcoder.py +++ /dev/null @@ -1,498 +0,0 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from src/transformers/models/iquestcoder/modular_iquestcoder.py. -# Do NOT edit this file manually as any edits will be overwritten by the generation of -# the file from the modular. If any change should be done, please apply the change to the -# modular_iquestcoder.py file directly. One of our CI enforces this. -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections.abc import Callable - -import torch -from torch import nn - -from ...activations import ACT2FN -from ...cache_utils import Cache, DynamicCache -from ...generation import GenerationMixin -from ...integrations import use_kernel_forward_from_hub, use_kernelized_func -from ...masking_utils import create_causal_mask -from ...modeling_layers import ( - GenericForQuestionAnswering, - GenericForSequenceClassification, - GenericForTokenClassification, - GradientCheckpointingLayer, -) -from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast -from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update -from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep -from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple -from ...utils.generic import merge_with_config_defaults -from ...utils.output_capturing import capture_outputs -from .configuration_iquestcoder import IQuestCoderConfig - - -@use_kernel_forward_from_hub("RMSNorm") -class IQuestCoderRMSNorm(nn.Module): - def __init__(self, hidden_size, eps: float = 1e-6) -> None: - """ - IQuestCoderRMSNorm is equivalent to T5LayerNorm - """ - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden_size)) - self.variance_epsilon = eps - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - input_dtype = hidden_states.dtype - hidden_states = hidden_states.to(torch.float32) - variance = hidden_states.pow(2).mean(-1, keepdim=True) - hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) - return self.weight * hidden_states.to(input_dtype) - - def extra_repr(self): - return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" - - -class IQuestCoderMLP(nn.Module): - def __init__(self, config): - super().__init__() - self.config = config - self.hidden_size = config.hidden_size - self.intermediate_size = config.intermediate_size - self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) - self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) - self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias) - self.act_fn = ACT2FN[config.hidden_act] - - def forward(self, x): - down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) - return down_proj - - -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -@use_kernel_forward_from_hub("rotary_pos_emb") -def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): - """Applies Rotary Position Embedding to the query and key tensors. - - Args: - q (`torch.Tensor`): The query tensor. - k (`torch.Tensor`): The key tensor. - cos (`torch.Tensor`): The cosine part of the rotary embedding. - sin (`torch.Tensor`): The sine part of the rotary embedding. - unsqueeze_dim (`int`, *optional*, defaults to 1): - The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - Returns: - `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. - """ - cos = cos.unsqueeze(unsqueeze_dim) - sin = sin.unsqueeze(unsqueeze_dim) - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed, k_embed - - -def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """ - This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, - num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) - """ - batch, num_key_value_heads, slen, head_dim = hidden_states.shape - if n_rep == 1: - return hidden_states - hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) - return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) - - -def eager_attention_forward( - module: nn.Module, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - attention_mask: torch.Tensor | None, - scaling: float, - dropout: float = 0.0, - **kwargs: Unpack[TransformersKwargs], -): - key_states = repeat_kv(key, module.num_key_value_groups) - value_states = repeat_kv(value, module.num_key_value_groups) - - attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling - if attention_mask is not None: - attn_weights = attn_weights + attention_mask - - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) - attn_output = torch.matmul(attn_weights, value_states) - attn_output = attn_output.transpose(1, 2).contiguous() - - return attn_output, attn_weights - - -@use_kernelized_func(apply_rotary_pos_emb) -class IQuestCoderAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config: IQuestCoderConfig, layer_idx: int): - super().__init__() - self.config = config - self.layer_idx = layer_idx - self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) - self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads - self.scaling = self.head_dim**-0.5 - self.attention_dropout = config.attention_dropout - self.is_causal = True - - self.q_proj = nn.Linear( - config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias - ) - self.k_proj = nn.Linear( - config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias - ) - self.v_proj = nn.Linear( - config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias - ) - self.o_proj = nn.Linear( - config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias - ) - - def forward( - self, - hidden_states: torch.Tensor, - position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, - attention_mask: torch.Tensor | None = None, - past_key_values: Cache | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> tuple[torch.Tensor, torch.Tensor]: - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - - query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) - - if past_key_values is not None: - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) - - attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, eager_attention_forward - ) - - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask, - dropout=0.0 if not self.training else self.attention_dropout, - scaling=self.scaling, - **kwargs, - ) - - attn_output = attn_output.reshape(*input_shape, -1).contiguous() - attn_output = self.o_proj(attn_output) - return attn_output, attn_weights - - -class IQuestCoderDecoderLayer(GradientCheckpointingLayer): - def __init__(self, config: IQuestCoderConfig, layer_idx: int): - super().__init__() - self.hidden_size = config.hidden_size - - self.self_attn = IQuestCoderAttention(config=config, layer_idx=layer_idx) - - self.mlp = IQuestCoderMLP(config) - self.input_layernorm = IQuestCoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = IQuestCoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: Cache | None = None, - use_cache: bool | None = False, - position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> torch.Tensor: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - # Self Attention - hidden_states, _ = self.self_attn( - hidden_states=hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - use_cache=use_cache, - position_embeddings=position_embeddings, - **kwargs, - ) - hidden_states = residual + hidden_states - - # Fully Connected - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states - return hidden_states - - -@auto_docstring -class IQuestCoderPreTrainedModel(PreTrainedModel): - config: IQuestCoderConfig - base_model_prefix = "model" - supports_gradient_checkpointing = True - _no_split_modules = ["IQuestCoderDecoderLayer"] - _skip_keys_device_placement = ["past_key_values"] - _supports_flash_attn = True - _supports_sdpa = True - _supports_flex_attn = True - - _can_compile_fullgraph = True - _supports_attention_backend = True - _can_record_outputs = { - "hidden_states": IQuestCoderDecoderLayer, - "attentions": IQuestCoderAttention, - } - - -class IQuestCoderRotaryEmbedding(nn.Module): - def __init__(self, config: IQuestCoderConfig): - super().__init__() - self.max_seq_len_cached = config.max_position_embeddings - self.original_max_seq_len = config.max_position_embeddings - - self.config = config - - self.rope_type = self.config.rope_parameters["rope_type"] - rope_init_fn: Callable = self.compute_default_rope_parameters - if self.rope_type != "default": - rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] - inv_freq, self.attention_scaling = rope_init_fn(self.config) - - self.inv_freq = nn.Buffer(inv_freq, persistent=False) - self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False) - - @staticmethod - def compute_default_rope_parameters(config: IQuestCoderConfig, **kwargs) -> tuple[torch.Tensor, float]: - """ - Computes the inverse frequencies according to the original RoPE implementation - Args: - config ([`~transformers.PreTrainedConfig`]): - The model configuration. - Returns: - Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the - post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). - """ - base = config.rope_parameters["rope_theta"] - dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads - - attention_factor = 1.0 # Unused in this type of RoPE - # Compute the inverse frequencies - inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) - return inv_freq, attention_factor - - @torch.no_grad() - @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) - def forward(self, x, position_ids): - freqs = position_ids[..., None].float() * self.inv_freq.to(device=x.device, dtype=torch.float) - emb = torch.cat((freqs, freqs), dim=-1) - cos = emb.cos() * self.attention_scaling - sin = emb.sin() * self.attention_scaling - - return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) - - -@auto_docstring -class IQuestCoderModel(IQuestCoderPreTrainedModel): - def __init__(self, config: IQuestCoderConfig): - super().__init__(config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - - self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) - self.layers = nn.ModuleList( - [IQuestCoderDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] - ) - self.norm = IQuestCoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.rotary_emb = IQuestCoderRotaryEmbedding(config=config) - self.gradient_checkpointing = False - - # Initialize weights and apply final processing - self.post_init() - - @merge_with_config_defaults - @capture_outputs - @auto_docstring - def forward( - self, - input_ids: torch.LongTensor | None = None, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: Cache | None = None, - inputs_embeds: torch.Tensor | None = None, - use_cache: bool | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPast: - if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - - if inputs_embeds is None: - inputs_embeds: torch.Tensor = self.embed_tokens(input_ids) - - if use_cache and past_key_values is None: - past_key_values = DynamicCache(config=self.config) - - if position_ids is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - position_ids = position_ids.unsqueeze(0) - - causal_mask = create_causal_mask( - config=self.config, - inputs_embeds=inputs_embeds, - attention_mask=attention_mask, - past_key_values=past_key_values, - position_ids=position_ids, - ) - - hidden_states = inputs_embeds - position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) - - for decoder_layer in self.layers[: self.config.num_hidden_layers]: - hidden_states = decoder_layer( - hidden_states, - attention_mask=causal_mask, - position_embeddings=position_embeddings, - position_ids=position_ids, - past_key_values=past_key_values, - use_cache=use_cache, - **kwargs, - ) - - hidden_states = self.norm(hidden_states) - return BaseModelOutputWithPast( - last_hidden_state=hidden_states, - past_key_values=past_key_values, - ) - - -@auto_docstring(checkpoint="IQuestLab/IQuest-Coder-V1-40B-Instruct") -class IQuestCoderForCausalLM(IQuestCoderPreTrainedModel, GenerationMixin): - _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} - _tp_plan = {"lm_head": "colwise_gather_output"} - _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} - _fsdp_plan = {"lm_head": "keep_full_weight"} - - def __init__(self, config: IQuestCoderConfig): - super().__init__(config) - self.model = IQuestCoderModel(config) - self.vocab_size = config.vocab_size - self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - - # Initialize weights and apply final processing - self.post_init() - - @can_return_tuple - @auto_docstring - def forward( - self, - input_ids: torch.LongTensor | None = None, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: Cache | None = None, - inputs_embeds: torch.Tensor | None = None, - labels: torch.LongTensor | None = None, - use_cache: bool | None = None, - logits_to_keep: int | torch.Tensor = 0, - **kwargs: Unpack[TransformersKwargs], - ) -> CausalLMOutputWithPast: - r""" - Example: - - ```python - >>> from transformers import AutoTokenizer, IQuestCoderForCausalLM - - >>> model = IQuestCoderForCausalLM.from_pretrained("meta-iquestcoder/IQuestCoder-2-7b-hf") - >>> tokenizer = AutoTokenizer.from_pretrained("meta-iquestcoder/IQuestCoder-2-7b-hf") - - >>> prompt = "Hey, are you conscious? Can you talk to me?" - >>> inputs = tokenizer(prompt, return_tensors="pt") - - >>> # Generate - >>> generate_ids = model.generate(inputs.input_ids, max_length=30) - >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." - ```""" - outputs: BaseModelOutputWithPast = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - **kwargs, - ) - - hidden_states = outputs.last_hidden_state - hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep) - logits = self.lm_head(hidden_states) - - loss = None - if labels is not None: - loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) - - return CausalLMOutputWithPast( - loss=loss, - logits=logits, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - ) - - -class IQuestCoderForSequenceClassification(GenericForSequenceClassification, IQuestCoderPreTrainedModel): - pass - - -class IQuestCoderForQuestionAnswering(GenericForQuestionAnswering, IQuestCoderPreTrainedModel): - base_model_prefix = "transformer" # For BC, where `transformer` was used instead of `model` - - -class IQuestCoderForTokenClassification(GenericForTokenClassification, IQuestCoderPreTrainedModel): - pass - - -__all__ = [ - "IQuestCoderForCausalLM", - "IQuestCoderForQuestionAnswering", - "IQuestCoderForSequenceClassification", - "IQuestCoderForTokenClassification", - "IQuestCoderModel", - "IQuestCoderPreTrainedModel", -] diff --git a/src/transformers/models/iquestcoder/modular_iquestcoder.py b/src/transformers/models/iquestcoder/modular_iquestcoder.py index 48dbb2b70399..cf3bb03e5d4e 100644 --- a/src/transformers/models/iquestcoder/modular_iquestcoder.py +++ b/src/transformers/models/iquestcoder/modular_iquestcoder.py @@ -17,14 +17,6 @@ from ...modeling_rope_utils import RopeParameters from ...utils import auto_docstring from ..llama.configuration_llama import LlamaConfig -from ..llama.modeling_llama import ( - LlamaForCausalLM, - LlamaForQuestionAnswering, - LlamaForSequenceClassification, - LlamaForTokenClassification, - LlamaModel, - LlamaPreTrainedModel, -) @auto_docstring(checkpoint="IQuestLab/IQuest-Coder-V1-40B-Instruct") @@ -32,13 +24,13 @@ class IQuestCoderConfig(LlamaConfig): r""" ```python - >>> from transformers import IQuestCoderModel, IQuestCoderConfig + >>> from transformers import AutoModelForCausalLM, IQuestCoderConfig >>> # Initializing an IQuest-Coder-V1-40B style configuration >>> configuration = IQuestCoderConfig() - >>> # Initializing a model from the IQuest-Coder-V1-40B style configuration - >>> model = IQuestCoderModel(configuration) + >>> # Initializing a model (which uses the Llama implementation) from that configuration + >>> model = AutoModelForCausalLM.from_config(configuration) >>> # Accessing the model configuration >>> configuration = model.config @@ -79,37 +71,4 @@ def validate_architecture(self): raise ValueError("IQuestCoder does not support `use_sliding_window`.") -class IQuestCoderPreTrainedModel(LlamaPreTrainedModel): - pass - - -class IQuestCoderModel(LlamaModel): - pass - - -@auto_docstring(checkpoint="IQuestLab/IQuest-Coder-V1-40B-Instruct") -class IQuestCoderForCausalLM(LlamaForCausalLM): - pass - - -class IQuestCoderForSequenceClassification(LlamaForSequenceClassification): - pass - - -class IQuestCoderForQuestionAnswering(LlamaForQuestionAnswering): - pass - - -class IQuestCoderForTokenClassification(LlamaForTokenClassification): - pass - - -__all__ = [ - "IQuestCoderConfig", - "IQuestCoderForCausalLM", - "IQuestCoderForQuestionAnswering", - "IQuestCoderForSequenceClassification", - "IQuestCoderForTokenClassification", - "IQuestCoderModel", - "IQuestCoderPreTrainedModel", -] +__all__ = ["IQuestCoderConfig"] diff --git a/src/transformers/models/telechat3/__init__.py b/src/transformers/models/telechat3/__init__.py index cafe21913613..fa02dee59e82 100644 --- a/src/transformers/models/telechat3/__init__.py +++ b/src/transformers/models/telechat3/__init__.py @@ -25,7 +25,6 @@ if TYPE_CHECKING: from .configuration_telechat3 import * - from .modeling_telechat3 import * else: import sys diff --git a/src/transformers/models/telechat3/configuration_telechat3.py b/src/transformers/models/telechat3/configuration_telechat3.py index d53ca6012710..30820617b3d2 100644 --- a/src/transformers/models/telechat3/configuration_telechat3.py +++ b/src/transformers/models/telechat3/configuration_telechat3.py @@ -35,13 +35,13 @@ class TeleChat3Config(PreTrainedConfig): r""" ```python - >>> from transformers import TeleChat3Model, TeleChat3Config + >>> from transformers import AutoModelForCausalLM, TeleChat3Config >>> # Initializing a TeleChat3-36B style configuration >>> configuration = TeleChat3Config() - >>> # Initializing a model from the TeleChat3-36B style configuration - >>> model = TeleChat3Model(configuration) + >>> # Initializing a model (which uses the Llama implementation) from that configuration + >>> model = AutoModelForCausalLM.from_config(configuration) >>> # Accessing the model configuration >>> configuration = model.config diff --git a/src/transformers/models/telechat3/modeling_telechat3.py b/src/transformers/models/telechat3/modeling_telechat3.py deleted file mode 100644 index 6f0d8284450a..000000000000 --- a/src/transformers/models/telechat3/modeling_telechat3.py +++ /dev/null @@ -1,474 +0,0 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from src/transformers/models/telechat3/modular_telechat3.py. -# Do NOT edit this file manually as any edits will be overwritten by the generation of -# the file from the modular. If any change should be done, please apply the change to the -# modular_telechat3.py file directly. One of our CI enforces this. -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections.abc import Callable - -import torch -from torch import nn - -from ...activations import ACT2FN -from ...cache_utils import Cache, DynamicCache -from ...generation import GenerationMixin -from ...integrations import use_kernel_forward_from_hub, use_kernelized_func -from ...masking_utils import create_causal_mask -from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast -from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update -from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep -from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple -from ...utils.generic import merge_with_config_defaults -from ...utils.output_capturing import capture_outputs -from .configuration_telechat3 import TeleChat3Config - - -@use_kernel_forward_from_hub("RMSNorm") -class TeleChat3RMSNorm(nn.Module): - def __init__(self, hidden_size, eps: float = 1e-6) -> None: - """ - TeleChat3RMSNorm is equivalent to T5LayerNorm - """ - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden_size)) - self.variance_epsilon = eps - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - input_dtype = hidden_states.dtype - hidden_states = hidden_states.to(torch.float32) - variance = hidden_states.pow(2).mean(-1, keepdim=True) - hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) - return self.weight * hidden_states.to(input_dtype) - - def extra_repr(self): - return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" - - -class TeleChat3MLP(nn.Module): - def __init__(self, config): - super().__init__() - self.config = config - self.hidden_size = config.hidden_size - self.intermediate_size = config.intermediate_size - self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) - self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) - self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias) - self.act_fn = ACT2FN[config.hidden_act] - - def forward(self, x): - down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) - return down_proj - - -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -@use_kernel_forward_from_hub("rotary_pos_emb") -def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): - """Applies Rotary Position Embedding to the query and key tensors. - - Args: - q (`torch.Tensor`): The query tensor. - k (`torch.Tensor`): The key tensor. - cos (`torch.Tensor`): The cosine part of the rotary embedding. - sin (`torch.Tensor`): The sine part of the rotary embedding. - unsqueeze_dim (`int`, *optional*, defaults to 1): - The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - Returns: - `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. - """ - cos = cos.unsqueeze(unsqueeze_dim) - sin = sin.unsqueeze(unsqueeze_dim) - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed, k_embed - - -def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """ - This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, - num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) - """ - batch, num_key_value_heads, slen, head_dim = hidden_states.shape - if n_rep == 1: - return hidden_states - hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) - return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) - - -def eager_attention_forward( - module: nn.Module, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - attention_mask: torch.Tensor | None, - scaling: float, - dropout: float = 0.0, - **kwargs: Unpack[TransformersKwargs], -): - key_states = repeat_kv(key, module.num_key_value_groups) - value_states = repeat_kv(value, module.num_key_value_groups) - - attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling - if attention_mask is not None: - attn_weights = attn_weights + attention_mask - - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) - attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) - attn_output = torch.matmul(attn_weights, value_states) - attn_output = attn_output.transpose(1, 2).contiguous() - - return attn_output, attn_weights - - -@use_kernelized_func(apply_rotary_pos_emb) -class TeleChat3Attention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config: TeleChat3Config, layer_idx: int): - super().__init__() - self.config = config - self.layer_idx = layer_idx - self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) - self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads - self.scaling = self.head_dim**-0.5 - self.attention_dropout = config.attention_dropout - self.is_causal = True - - self.q_proj = nn.Linear( - config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias - ) - self.k_proj = nn.Linear( - config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias - ) - self.v_proj = nn.Linear( - config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias - ) - self.o_proj = nn.Linear( - config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias - ) - - def forward( - self, - hidden_states: torch.Tensor, - position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, - attention_mask: torch.Tensor | None = None, - past_key_values: Cache | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> tuple[torch.Tensor, torch.Tensor]: - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - - query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) - - if past_key_values is not None: - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) - - attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, eager_attention_forward - ) - - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask, - dropout=0.0 if not self.training else self.attention_dropout, - scaling=self.scaling, - **kwargs, - ) - - attn_output = attn_output.reshape(*input_shape, -1).contiguous() - attn_output = self.o_proj(attn_output) - return attn_output, attn_weights - - -class TeleChat3DecoderLayer(GradientCheckpointingLayer): - def __init__(self, config: TeleChat3Config, layer_idx: int): - super().__init__() - self.hidden_size = config.hidden_size - - self.self_attn = TeleChat3Attention(config=config, layer_idx=layer_idx) - - self.mlp = TeleChat3MLP(config) - self.input_layernorm = TeleChat3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = TeleChat3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: Cache | None = None, - use_cache: bool | None = False, - position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> torch.Tensor: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - # Self Attention - hidden_states, _ = self.self_attn( - hidden_states=hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - use_cache=use_cache, - position_embeddings=position_embeddings, - **kwargs, - ) - hidden_states = residual + hidden_states - - # Fully Connected - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states - return hidden_states - - -@auto_docstring -class TeleChat3PreTrainedModel(PreTrainedModel): - config: TeleChat3Config - base_model_prefix = "model" - supports_gradient_checkpointing = True - _no_split_modules = ["TeleChat3DecoderLayer"] - _skip_keys_device_placement = ["past_key_values"] - _supports_flash_attn = True - _supports_sdpa = True - _supports_flex_attn = True - - _can_compile_fullgraph = True - _supports_attention_backend = True - _can_record_outputs = { - "hidden_states": TeleChat3DecoderLayer, - "attentions": TeleChat3Attention, - } - - -class TeleChat3RotaryEmbedding(nn.Module): - def __init__(self, config: TeleChat3Config): - super().__init__() - self.max_seq_len_cached = config.max_position_embeddings - self.original_max_seq_len = config.max_position_embeddings - - self.config = config - - self.rope_type = self.config.rope_parameters["rope_type"] - rope_init_fn: Callable = self.compute_default_rope_parameters - if self.rope_type != "default": - rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] - inv_freq, self.attention_scaling = rope_init_fn(self.config) - - self.inv_freq = nn.Buffer(inv_freq, persistent=False) - self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False) - - @staticmethod - def compute_default_rope_parameters(config: TeleChat3Config, **kwargs) -> tuple[torch.Tensor, float]: - """ - Computes the inverse frequencies according to the original RoPE implementation - Args: - config ([`~transformers.PreTrainedConfig`]): - The model configuration. - Returns: - Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the - post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). - """ - base = config.rope_parameters["rope_theta"] - dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads - - attention_factor = 1.0 # Unused in this type of RoPE - # Compute the inverse frequencies - inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) - return inv_freq, attention_factor - - @torch.no_grad() - @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) - def forward(self, x, position_ids): - freqs = position_ids[..., None].float() * self.inv_freq.to(device=x.device, dtype=torch.float) - emb = torch.cat((freqs, freqs), dim=-1) - cos = emb.cos() * self.attention_scaling - sin = emb.sin() * self.attention_scaling - - return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) - - -@auto_docstring -class TeleChat3Model(TeleChat3PreTrainedModel): - def __init__(self, config: TeleChat3Config): - super().__init__(config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - - self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) - self.layers = nn.ModuleList( - [TeleChat3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] - ) - self.norm = TeleChat3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.rotary_emb = TeleChat3RotaryEmbedding(config=config) - self.gradient_checkpointing = False - - # Initialize weights and apply final processing - self.post_init() - - @merge_with_config_defaults - @capture_outputs - @auto_docstring - def forward( - self, - input_ids: torch.LongTensor | None = None, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: Cache | None = None, - inputs_embeds: torch.Tensor | None = None, - use_cache: bool | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPast: - if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - - if inputs_embeds is None: - inputs_embeds: torch.Tensor = self.embed_tokens(input_ids) - - if use_cache and past_key_values is None: - past_key_values = DynamicCache(config=self.config) - - if position_ids is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - position_ids = position_ids.unsqueeze(0) - - causal_mask = create_causal_mask( - config=self.config, - inputs_embeds=inputs_embeds, - attention_mask=attention_mask, - past_key_values=past_key_values, - position_ids=position_ids, - ) - - hidden_states = inputs_embeds - position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) - - for decoder_layer in self.layers[: self.config.num_hidden_layers]: - hidden_states = decoder_layer( - hidden_states, - attention_mask=causal_mask, - position_embeddings=position_embeddings, - position_ids=position_ids, - past_key_values=past_key_values, - use_cache=use_cache, - **kwargs, - ) - - hidden_states = self.norm(hidden_states) - return BaseModelOutputWithPast( - last_hidden_state=hidden_states, - past_key_values=past_key_values, - ) - - -@auto_docstring(checkpoint="Tele-AI/TeleChat3-36B-Thinking") -class TeleChat3ForCausalLM(TeleChat3PreTrainedModel, GenerationMixin): - _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} - _tp_plan = {"lm_head": "colwise_gather_output"} - _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} - _fsdp_plan = {"lm_head": "keep_full_weight"} - - def __init__(self, config: TeleChat3Config): - super().__init__(config) - self.model = TeleChat3Model(config) - self.vocab_size = config.vocab_size - self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) - - # Initialize weights and apply final processing - self.post_init() - - @can_return_tuple - @auto_docstring - def forward( - self, - input_ids: torch.LongTensor | None = None, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: Cache | None = None, - inputs_embeds: torch.Tensor | None = None, - labels: torch.LongTensor | None = None, - use_cache: bool | None = None, - logits_to_keep: int | torch.Tensor = 0, - **kwargs: Unpack[TransformersKwargs], - ) -> CausalLMOutputWithPast: - r""" - Example: - - ```python - >>> from transformers import AutoTokenizer, TeleChat3ForCausalLM - - >>> model = TeleChat3ForCausalLM.from_pretrained("meta-telechat3/TeleChat3-2-7b-hf") - >>> tokenizer = AutoTokenizer.from_pretrained("meta-telechat3/TeleChat3-2-7b-hf") - - >>> prompt = "Hey, are you conscious? Can you talk to me?" - >>> inputs = tokenizer(prompt, return_tensors="pt") - - >>> # Generate - >>> generate_ids = model.generate(inputs.input_ids, max_length=30) - >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." - ```""" - outputs: BaseModelOutputWithPast = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - **kwargs, - ) - - hidden_states = outputs.last_hidden_state - hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep) - logits = self.lm_head(hidden_states) - - loss = None - if labels is not None: - loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) - - return CausalLMOutputWithPast( - loss=loss, - logits=logits, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - ) - - -__all__ = ["TeleChat3ForCausalLM", "TeleChat3Model", "TeleChat3PreTrainedModel"] diff --git a/src/transformers/models/telechat3/modular_telechat3.py b/src/transformers/models/telechat3/modular_telechat3.py index 998c40d5bcfb..614eaa9c8aaa 100644 --- a/src/transformers/models/telechat3/modular_telechat3.py +++ b/src/transformers/models/telechat3/modular_telechat3.py @@ -20,11 +20,6 @@ from ...modeling_rope_utils import RopeParameters from ...utils import auto_docstring, logging from ..llama.configuration_llama import LlamaConfig -from ..llama.modeling_llama import ( - LlamaForCausalLM, - LlamaModel, - LlamaPreTrainedModel, -) logger = logging.get_logger(__name__) @@ -35,13 +30,13 @@ class TeleChat3Config(LlamaConfig): r""" ```python - >>> from transformers import TeleChat3Model, TeleChat3Config + >>> from transformers import AutoModelForCausalLM, TeleChat3Config >>> # Initializing a TeleChat3-36B style configuration >>> configuration = TeleChat3Config() - >>> # Initializing a model from the TeleChat3-36B style configuration - >>> model = TeleChat3Model(configuration) + >>> # Initializing a model (which uses the Llama implementation) from that configuration + >>> model = AutoModelForCausalLM.from_config(configuration) >>> # Accessing the model configuration >>> configuration = model.config @@ -105,22 +100,4 @@ def get_mscale(scale, mscale=1.0): return PreTrainedConfig.convert_rope_params_to_dict(self, **kwargs) -class TeleChat3PreTrainedModel(LlamaPreTrainedModel): - pass - - -class TeleChat3Model(LlamaModel): - pass - - -@auto_docstring(checkpoint="Tele-AI/TeleChat3-36B-Thinking") -class TeleChat3ForCausalLM(LlamaForCausalLM): - pass - - -__all__ = [ - "TeleChat3Config", - "TeleChat3ForCausalLM", - "TeleChat3Model", - "TeleChat3PreTrainedModel", -] +__all__ = ["TeleChat3Config"] diff --git a/tests/models/internlm3/test_modeling_internlm3.py b/tests/models/internlm3/test_modeling_internlm3.py index 17653e8bf9da..2f81d341f083 100644 --- a/tests/models/internlm3/test_modeling_internlm3.py +++ b/tests/models/internlm3/test_modeling_internlm3.py @@ -11,11 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Testing suite for the PyTorch InternLM3 model.""" +"""Testing suite for InternLM3, which uses the Llama implementation.""" import unittest -from transformers import AutoTokenizer, is_torch_available +from huggingface_hub.errors import StrictDataclassClassValidationError + +from transformers import AutoModelForCausalLM, AutoTokenizer, InternLM3Config, is_torch_available from transformers.testing_utils import cleanup, require_torch, require_torch_accelerator, slow, torch_device from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester @@ -24,27 +26,26 @@ if is_torch_available(): import torch - from transformers import InternLM3ForCausalLM, InternLM3Model + from transformers import LlamaModel class InternLM3ModelTester(CausalLMModelTester): + config_class = InternLM3Config if is_torch_available(): - base_model_class = InternLM3Model + base_model_class = LlamaModel @require_torch class InternLM3ModelTest(CausalLMModelTest, unittest.TestCase): model_tester_class = InternLM3ModelTester - def test_qkv_bias_is_independent_of_bias(self): - config, _ = self.model_tester.prepare_config_and_inputs_for_common() - config.qkv_bias = True - config.bias = False - layer = InternLM3Model(config).layers[0] - for proj in (layer.self_attn.q_proj, layer.self_attn.k_proj, layer.self_attn.v_proj): - self.assertIsNotNone(proj.bias) - for proj in (layer.self_attn.o_proj, layer.mlp.gate_proj, layer.mlp.up_proj, layer.mlp.down_proj): - self.assertIsNone(proj.bias) + def test_original_bias_names(self): + # The original config names, as found in the released checkpoints + config = InternLM3Config(qkv_bias=True, bias=True) + self.assertTrue(config.attention_bias) + self.assertTrue(config.mlp_bias) + with self.assertRaises(StrictDataclassClassValidationError): + InternLM3Config(qkv_bias=True, bias=False) @slow @@ -56,7 +57,7 @@ def tearDown(self): cleanup(torch_device, gc_collect=True) def test_model_8b_logits(self): - model = InternLM3ForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) with torch.no_grad(): logits = model(input_ids).logits.float().cpu() @@ -70,7 +71,7 @@ def test_model_8b_logits(self): def test_model_8b_generation(self): tokenizer = AutoTokenizer.from_pretrained(self.model_id) - model = InternLM3ForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) text = tokenizer.decode(output[0], skip_special_tokens=True) diff --git a/tests/models/iquestcoder/test_modeling_iquestcoder.py b/tests/models/iquestcoder/test_modeling_iquestcoder.py index adcbffd3bb83..e8474001f329 100644 --- a/tests/models/iquestcoder/test_modeling_iquestcoder.py +++ b/tests/models/iquestcoder/test_modeling_iquestcoder.py @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Testing suite for the PyTorch IQuestCoder model.""" +"""Testing suite for IQuestCoder, which uses the Llama implementation.""" import unittest from huggingface_hub.errors import StrictDataclassClassValidationError -from transformers import AutoTokenizer, IQuestCoderConfig, is_torch_available +from transformers import AutoModelForCausalLM, AutoTokenizer, IQuestCoderConfig, is_torch_available from transformers.testing_utils import cleanup, require_torch, require_torch_accelerator, slow, torch_device from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester @@ -26,12 +26,13 @@ if is_torch_available(): import torch - from transformers import IQuestCoderForCausalLM, IQuestCoderModel + from transformers import LlamaModel class IQuestCoderModelTester(CausalLMModelTester): + config_class = IQuestCoderConfig if is_torch_available(): - base_model_class = IQuestCoderModel + base_model_class = LlamaModel @require_torch @@ -56,7 +57,7 @@ def tearDown(self): cleanup(torch_device, gc_collect=True) def test_model_7b_logits(self): - model = IQuestCoderForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) with torch.no_grad(): logits = model(input_ids).logits.float().cpu() @@ -70,7 +71,7 @@ def test_model_7b_logits(self): def test_model_7b_generation(self): tokenizer = AutoTokenizer.from_pretrained(self.model_id) - model = IQuestCoderForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) inputs = tokenizer("def fibonacci(n):", return_tensors="pt").to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) text = tokenizer.decode(output[0], skip_special_tokens=True) diff --git a/tests/models/telechat3/test_modeling_telechat3.py b/tests/models/telechat3/test_modeling_telechat3.py index 2ec9a239d455..dfc56cffe29e 100644 --- a/tests/models/telechat3/test_modeling_telechat3.py +++ b/tests/models/telechat3/test_modeling_telechat3.py @@ -11,12 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Testing suite for the PyTorch TeleChat3 model.""" +"""Testing suite for TeleChat3, which uses the Llama implementation.""" import math import unittest -from transformers import AutoTokenizer, TeleChat3Config, is_torch_available +from transformers import AutoModelForCausalLM, AutoTokenizer, TeleChat3Config, is_torch_available from transformers.testing_utils import cleanup, require_torch, require_torch_accelerator, slow, torch_device from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester @@ -25,12 +25,13 @@ if is_torch_available(): import torch - from transformers import TeleChat3ForCausalLM, TeleChat3Model + from transformers import LlamaModel class TeleChat3ModelTester(CausalLMModelTester): + config_class = TeleChat3Config if is_torch_available(): - base_model_class = TeleChat3Model + base_model_class = LlamaModel @require_torch @@ -83,7 +84,7 @@ def tearDown(self): cleanup(torch_device, gc_collect=True) def test_model_36b_logits(self): - model = TeleChat3ForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) with torch.no_grad(): logits = model(input_ids).logits.float().cpu() @@ -97,7 +98,7 @@ def test_model_36b_logits(self): def test_model_36b_generation(self): tokenizer = AutoTokenizer.from_pretrained(self.model_id) - model = TeleChat3ForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) + model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) text = tokenizer.decode(output[0], skip_special_tokens=True) diff --git a/utils/check_config_attributes.py b/utils/check_config_attributes.py index 5dafe6bc8b97..b6e7af8daccd 100644 --- a/utils/check_config_attributes.py +++ b/utils/check_config_attributes.py @@ -321,6 +321,10 @@ "PPChart2TableConfig": True, "PPChart2TableVisionConfig": True, "GlmgaConfig": ["vision_config"], + # Use the Llama implementation, which the auto mappings point to + "InternLM3Config": True, + "IQuestCoderConfig": True, + "TeleChat3Config": True, "Sapiens2Config": [ "num_first_full_attention_layers", # builder attr consumed in __post_init__ to compute num_key_value_heads_per_layer "num_key_value_attention_heads", # builder attr consumed in __post_init__ to compute num_key_value_heads_per_layer From cb9084cdd372a0213d9170fecff467ffb8f24f15 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:03:06 +0000 Subject: [PATCH 08/17] Drop the redundant checkpoint from TeleChatForCausalLM's auto_docstring Co-Authored-By: Claude Opus 5.5 --- src/transformers/models/telechat/modeling_telechat.py | 2 +- src/transformers/models/telechat/modular_telechat.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/telechat/modeling_telechat.py b/src/transformers/models/telechat/modeling_telechat.py index 06fef3df0ce3..5006a1b6086d 100644 --- a/src/transformers/models/telechat/modeling_telechat.py +++ b/src/transformers/models/telechat/modeling_telechat.py @@ -405,7 +405,7 @@ def forward( ) -@auto_docstring(checkpoint="Tele-AI/TeleChat2-3B") +@auto_docstring class TeleChatForCausalLM(TeleChatPreTrainedModel, GenerationMixin): _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} _tp_plan = {"lm_head": "colwise_gather_output"} diff --git a/src/transformers/models/telechat/modular_telechat.py b/src/transformers/models/telechat/modular_telechat.py index 5103d1c4bb3d..842aaa58e541 100644 --- a/src/transformers/models/telechat/modular_telechat.py +++ b/src/transformers/models/telechat/modular_telechat.py @@ -230,7 +230,7 @@ class TeleChatModel(LlamaModel): pass -@auto_docstring(checkpoint="Tele-AI/TeleChat2-3B") +@auto_docstring class TeleChatForCausalLM(LlamaForCausalLM): pass From 88d04c34daeb8f37afa1838aab74e83d395faf3f Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:10:54 +0000 Subject: [PATCH 09/17] Split TeleChat's fused key/value projection with the conversion mapping The original checkpoints fuse keys and values per head (`[k_0, v_0, k_1, v_1, ...]`). Split them into `k_proj` and `v_proj` on load, with `Interleave` now taking a block size, so TeleChat uses Llama's attention forward. Co-Authored-By: Claude Opus 5.5 --- docs/source/en/model_doc/telechat.md | 3 +- src/transformers/conversion_mapping.py | 7 ++- src/transformers/core_model_loading.py | 20 ++++-- .../models/telechat/configuration_telechat.py | 5 +- .../models/telechat/modeling_telechat.py | 9 +-- .../models/telechat/modular_telechat.py | 62 +------------------ .../models/telechat/test_modeling_telechat.py | 12 ++++ tests/utils/test_core_model_loading.py | 21 +++++++ 8 files changed, 67 insertions(+), 72 deletions(-) diff --git a/docs/source/en/model_doc/telechat.md b/docs/source/en/model_doc/telechat.md index 88a7c1785bb8..eb7d87c6f579 100644 --- a/docs/source/en/model_doc/telechat.md +++ b/docs/source/en/model_doc/telechat.md @@ -27,11 +27,10 @@ rendered properly in your Markdown viewer. Its architecture is the same as [Llama](./llama), except that: -- keys and values come from a single fused projection, laid out per head; - only the attention output projection and the down projection of the MLP have biases; - once the sequence grows beyond `training_seqlen`, RoPE is rescaled with a stepwise NTK scaling of its base and the cos/sin are scaled by `0.1 * ln(seq_len / training_seqlen) + 1`. -The original checkpoints use different parameter names, which are converted when the model is loaded. +The original checkpoints use different parameter names and fuse the key and value projections per head (`[k_0, v_0, k_1, v_1, ...]`). They are converted when the model is loaded, and `save_pretrained` writes the original layout back. TeleChat-52B uses a different architecture and isn't supported by this model. diff --git a/src/transformers/conversion_mapping.py b/src/transformers/conversion_mapping.py index eb828acc62e1..e7d8eef374ad 100755 --- a/src/transformers/conversion_mapping.py +++ b/src/transformers/conversion_mapping.py @@ -1641,7 +1641,12 @@ def _build_checkpoint_conversion_mapping(): WeightRenaming(r"(? ConversionOps: class Interleave(ConversionOps): - """Deinterleaves a tensor along `dim` by splitting in two and transposing. Reshapes param back to its original size.""" + """Deinterleaves a tensor along `dim` by splitting in two and transposing. Reshapes param back to its original size. - def __init__(self, dim: int = 0, inverse: bool = False): + By default the two halves alternate element by element. With `block_size_attribute`, a config field, they alternate + in blocks of that size instead, e.g. `[k_0, v_0, k_1, v_1, ...]` for keys and values fused per head of `head_dim`. + """ + + def __init__(self, dim: int = 0, inverse: bool = False, block_size_attribute: str | None = None): self.dim = dim self.inverse = inverse + self.block_size_attribute = block_size_attribute def convert(self, input_dict, source_patterns, target_patterns, **kwargs): tensor = next(iter(input_dict.values())) tensor = tensor[0] if isinstance(tensor, list) else tensor + block_size = 1 + if self.block_size_attribute is not None: + block_size = getattr(kwargs["config"].get_text_config(), self.block_size_attribute) + # Split into two in given dim and transpose to interleave along it shape = list(tensor.shape) + num_blocks = shape[self.dim] // (2 * block_size) if self.inverse: - shape[self.dim : self.dim + 1] = [2, shape[self.dim] // 2] + shape[self.dim : self.dim + 1] = [2, num_blocks, block_size] else: - shape[self.dim : self.dim + 1] = [shape[self.dim] // 2, 2] + shape[self.dim : self.dim + 1] = [num_blocks, 2, block_size] tensor = tensor.reshape(shape).transpose(self.dim, self.dim + 1).reshape(tensor.shape).contiguous() return {target_patterns[0]: tensor} @@ -249,7 +259,7 @@ def convert(self, input_dict, source_patterns, target_patterns, **kwargs): @property def reverse_op(self) -> ConversionOps: # can use the same dim and it will inverse it back - return Interleave(self.dim, inverse=not self.inverse) + return Interleave(self.dim, inverse=not self.inverse, block_size_attribute=self.block_size_attribute) class MergeModulelist(ConversionOps): diff --git a/src/transformers/models/telechat/configuration_telechat.py b/src/transformers/models/telechat/configuration_telechat.py index 1e616140fd52..42dfd4f6c316 100644 --- a/src/transformers/models/telechat/configuration_telechat.py +++ b/src/transformers/models/telechat/configuration_telechat.py @@ -18,6 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from huggingface_hub.dataclasses import strict from ...configuration_utils import PreTrainedConfig @@ -49,9 +50,11 @@ class TeleChatConfig(PreTrainedConfig): model_type = "telechat" keys_to_ignore_at_inference = ["past_key_values"] + # Default tensor parallel plan for base model `TeleChatModel` base_model_tp_plan = { "layers.*.self_attn.q_proj": "colwise", - "layers.*.self_attn.kv_proj": "colwise", + "layers.*.self_attn.k_proj": "colwise", + "layers.*.self_attn.v_proj": "colwise", "layers.*.self_attn.o_proj": "rowwise", "layers.*.mlp.gate_proj": "colwise", "layers.*.mlp.up_proj": "colwise", diff --git a/src/transformers/models/telechat/modeling_telechat.py b/src/transformers/models/telechat/modeling_telechat.py index 5006a1b6086d..c2fc0d531bfb 100644 --- a/src/transformers/models/telechat/modeling_telechat.py +++ b/src/transformers/models/telechat/modeling_telechat.py @@ -18,6 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from collections.abc import Callable import torch @@ -203,9 +204,9 @@ def __init__(self, config: TeleChatConfig, layer_idx: int): self.attention_dropout = config.attention_dropout self.is_causal = True self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False) self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=True) - # Keys and values are fused and laid out per head, i.e. `[k_0, v_0, k_1, v_1, ...]` - self.kv_proj = nn.Linear(config.hidden_size, 2 * config.num_key_value_heads * self.head_dim, bias=False) def forward( self, @@ -219,8 +220,8 @@ def forward( hidden_shape = (*input_shape, -1, self.head_dim) query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - key_value_states = self.kv_proj(hidden_states).view(*input_shape, -1, 2 * self.head_dim) - key_states, value_states = key_value_states.transpose(1, 2).chunk(2, dim=-1) + key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) diff --git a/src/transformers/models/telechat/modular_telechat.py b/src/transformers/models/telechat/modular_telechat.py index 842aaa58e541..c0ee93370d40 100644 --- a/src/transformers/models/telechat/modular_telechat.py +++ b/src/transformers/models/telechat/modular_telechat.py @@ -12,18 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections.abc import Callable import torch from huggingface_hub.dataclasses import strict from torch import nn -from ...cache_utils import Cache from ...configuration_utils import PreTrainedConfig from ...modeling_rope_utils import RopeParameters -from ...modeling_utils import ALL_ATTENTION_FUNCTIONS -from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring +from ...utils import auto_docstring from ..llama.configuration_llama import LlamaConfig from ..llama.modeling_llama import ( LlamaAttention, @@ -33,8 +29,6 @@ LlamaModel, LlamaPreTrainedModel, LlamaRotaryEmbedding, - apply_rotary_pos_emb, - eager_attention_forward, ) @@ -67,14 +61,6 @@ class TeleChatConfig(LlamaConfig): "ffn_hidden_size": "intermediate_size", "layer_norm_epsilon": "rms_norm_eps", } - base_model_tp_plan = { - "layers.*.self_attn.q_proj": "colwise", - "layers.*.self_attn.kv_proj": "colwise", - "layers.*.self_attn.o_proj": "rowwise", - "layers.*.mlp.gate_proj": "colwise", - "layers.*.mlp.up_proj": "colwise", - "layers.*.mlp.down_proj": "rowwise", - } vocab_size: int = 131072 hidden_size: int = 3072 @@ -171,51 +157,9 @@ class TeleChatAttention(LlamaAttention): def __init__(self, config: TeleChatConfig, layer_idx: int): super().__init__(config, layer_idx) self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False) - # Keys and values are fused and laid out per head, i.e. `[k_0, v_0, k_1, v_1, ...]` - self.kv_proj = nn.Linear(config.hidden_size, 2 * config.num_key_value_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False) self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=True) - del self.k_proj - del self.v_proj - - def forward( - self, - hidden_states: torch.Tensor, - position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, - attention_mask: torch.Tensor | None = None, - past_key_values: Cache | None = None, - **kwargs: Unpack[TransformersKwargs], - ) -> tuple[torch.Tensor, torch.Tensor]: - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - - query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - key_value_states = self.kv_proj(hidden_states).view(*input_shape, -1, 2 * self.head_dim) - key_states, value_states = key_value_states.transpose(1, 2).chunk(2, dim=-1) - - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) - - if past_key_values is not None: - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) - - attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, eager_attention_forward - ) - - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask, - dropout=0.0 if not self.training else self.attention_dropout, - scaling=self.scaling, - **kwargs, - ) - - attn_output = attn_output.reshape(*input_shape, -1).contiguous() - attn_output = self.o_proj(attn_output) - return attn_output, attn_weights class TeleChatDecoderLayer(LlamaDecoderLayer): diff --git a/tests/models/telechat/test_modeling_telechat.py b/tests/models/telechat/test_modeling_telechat.py index 6d158d8a6aa3..dd5ca41fa8cd 100644 --- a/tests/models/telechat/test_modeling_telechat.py +++ b/tests/models/telechat/test_modeling_telechat.py @@ -106,6 +106,18 @@ def test_original_checkpoint_round_trip(self): "transformer.ln_f.weight", ]: self.assertIn(key, saved) + # Keys and values are fused per head, i.e. `[k_0, v_0, k_1, v_1, ...]` + attn = model.model.layers[0].self_attn + head_dim = config.head_dim + expected = torch.cat( + [ + attn.k_proj.weight.view(-1, head_dim, config.hidden_size), + attn.v_proj.weight.view(-1, head_dim, config.hidden_size), + ], + dim=1, + ).reshape(-1, config.hidden_size) + key_value = load_file(f"{tmpdir}/model.safetensors")["transformer.h.0.self_attention.key_value.weight"] + torch.testing.assert_close(key_value.to(torch_device), expected) reloaded = TeleChatForCausalLM.from_pretrained(tmpdir).to(torch_device) for (name, param), (_, reloaded_param) in zip(model.named_parameters(), reloaded.named_parameters()): torch.testing.assert_close(param, reloaded_param, msg=name) diff --git a/tests/utils/test_core_model_loading.py b/tests/utils/test_core_model_loading.py index 32355f9a2fe6..679ab89b18c7 100644 --- a/tests/utils/test_core_model_loading.py +++ b/tests/utils/test_core_model_loading.py @@ -31,6 +31,7 @@ Conv3dToLinear, ErnieFuseAndSplitTextVisionExperts, GroupWeightRename, + Interleave, LinearToConv3d, MergeModulelist, PermuteForRope, @@ -1382,6 +1383,26 @@ def test_reverse_renames_the_converter_outputs_one_to_many(self): self.assertTrue(compare_state_dicts(saved_qkv, state_dict)) +class TestInterleave(unittest.TestCase): + def test_block_size_splits_fused_heads(self): + # Keys and values fused per head of size 2: `[k_0, v_0, k_1, v_1]` + k = torch.arange(4, dtype=torch.float32).reshape(4, 1) + v = 10 + k + fused = torch.cat([k.view(2, 2, 1), v.view(2, 2, 1)], dim=1).reshape(8, 1) + config = SimpleNamespace(get_text_config=lambda: SimpleNamespace(head_dim=2)) + targets = ["k_proj.weight", "v_proj.weight"] + + interleave = Interleave(dim=0, block_size_attribute="head_dim") + deinterleaved = interleave.convert({"kv": fused}, ["kv"], targets, config=config) + split = Chunk(dim=0).convert(deinterleaved, ["kv"], targets) + torch.testing.assert_close(split["k_proj.weight"], k) + torch.testing.assert_close(split["v_proj.weight"], v) + + merged = Concatenate(dim=0).convert(split, targets, ["kv"]) + restored = interleave.reverse_op.convert(merged, targets, ["kv"], config=config) + torch.testing.assert_close(restored["kv"], fused) + + class TestConversionMapping(unittest.TestCase): def test_group_weight_rename(self): model = DummyModelWithNorm(PreTrainedConfig()) From 8e5053016469dfec76d90982ccaa17801098d254 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:14:07 +0000 Subject: [PATCH 10/17] Drop the modular files of the config-only models Their configs are plain `PreTrainedConfig`s, so modular adds nothing. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 3 - .../internlm3/configuration_internlm3.py | 6 - .../models/internlm3/modular_internlm3.py | 75 ------------- .../iquestcoder/configuration_iquestcoder.py | 6 - .../models/iquestcoder/modular_iquestcoder.py | 74 ------------- .../telechat3/configuration_telechat3.py | 6 - .../models/telechat3/modular_telechat3.py | 103 ------------------ 7 files changed, 273 deletions(-) delete mode 100644 src/transformers/models/internlm3/modular_internlm3.py delete mode 100644 src/transformers/models/iquestcoder/modular_iquestcoder.py delete mode 100644 src/transformers/models/telechat3/modular_telechat3.py diff --git a/.gitattributes b/.gitattributes index 15b21dc8dfcd..384f802b32ea 100644 --- a/.gitattributes +++ b/.gitattributes @@ -267,9 +267,7 @@ src/transformers/models/inkling/configuration_*.py linguist-generated=true src/transformers/models/inkling/modeling_*.py linguist-generated=true src/transformers/models/instructblipvideo/configuration_*.py linguist-generated=true src/transformers/models/instructblipvideo/modeling_*.py linguist-generated=true -src/transformers/models/internlm3/configuration_*.py linguist-generated=true src/transformers/models/internvl/modeling_*.py linguist-generated=true -src/transformers/models/iquestcoder/configuration_*.py linguist-generated=true src/transformers/models/jais2/configuration_*.py linguist-generated=true src/transformers/models/jais2/modeling_*.py linguist-generated=true src/transformers/models/jamba/modeling_*.py linguist-generated=true @@ -572,7 +570,6 @@ src/transformers/models/t5gemma2/configuration_*.py linguist-generated=true src/transformers/models/t5gemma2/modeling_*.py linguist-generated=true src/transformers/models/telechat/configuration_*.py linguist-generated=true src/transformers/models/telechat/modeling_*.py linguist-generated=true -src/transformers/models/telechat3/configuration_*.py linguist-generated=true src/transformers/models/timesfm/modeling_*.py linguist-generated=true src/transformers/models/timesfm2_5/configuration_*.py linguist-generated=true src/transformers/models/timesfm2_5/modeling_*.py linguist-generated=true diff --git a/src/transformers/models/internlm3/configuration_internlm3.py b/src/transformers/models/internlm3/configuration_internlm3.py index 2d0a4c4b7e5e..ee80f2f0be0b 100644 --- a/src/transformers/models/internlm3/configuration_internlm3.py +++ b/src/transformers/models/internlm3/configuration_internlm3.py @@ -1,9 +1,3 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from src/transformers/models/internlm3/modular_internlm3.py. -# Do NOT edit this file manually as any edits will be overwritten by the generation of -# the file from the modular. If any change should be done, please apply the change to the -# modular_internlm3.py file directly. One of our CI enforces this. -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/transformers/models/internlm3/modular_internlm3.py b/src/transformers/models/internlm3/modular_internlm3.py deleted file mode 100644 index d0609ae5bdee..000000000000 --- a/src/transformers/models/internlm3/modular_internlm3.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from huggingface_hub.dataclasses import strict - -from ...modeling_rope_utils import RopeParameters -from ...utils import auto_docstring -from ..llama.configuration_llama import LlamaConfig - - -@auto_docstring(checkpoint="internlm/internlm3-8b-instruct") -@strict -class InternLM3Config(LlamaConfig): - r""" - ```python - >>> from transformers import AutoModelForCausalLM, InternLM3Config - - >>> # Initializing an InternLM3-8B style configuration - >>> configuration = InternLM3Config() - - >>> # Initializing a model (which uses the Llama implementation) from that configuration - >>> model = AutoModelForCausalLM.from_config(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "internlm3" - attribute_map = {"qkv_bias": "attention_bias", "bias": "mlp_bias"} - - vocab_size: int = 128512 - hidden_size: int = 4096 - intermediate_size: int = 10240 - num_hidden_layers: int = 48 - num_attention_heads: int = 32 - num_key_value_heads: int | None = 2 - hidden_act: str = "silu" - max_position_embeddings: int = 32768 - initializer_range: float = 0.02 - rms_norm_eps: float = 1e-5 - use_cache: bool = True - pad_token_id: int | None = 2 - bos_token_id: int | None = 1 - eos_token_id: int | list[int] | None = 2 - tie_word_embeddings: bool = False - rope_parameters: RopeParameters | dict | None = None - attention_bias: bool = False - attention_dropout: float | int = 0.0 - mlp_bias: bool = False - head_dim: int | None = 128 - - pretraining_tp = AttributeError() - - def validate_architecture(self): - """Part of `@strict`-powered validation. Validates the architecture of the config.""" - super().validate_architecture() - if self.attention_bias != self.mlp_bias: - raise ValueError( - "InternLM3 with `qkv_bias` different from `bias` is not supported, because the Llama implementation " - "uses the same bias setting for the query, key, value and output projections." - ) - - -__all__ = ["InternLM3Config"] diff --git a/src/transformers/models/iquestcoder/configuration_iquestcoder.py b/src/transformers/models/iquestcoder/configuration_iquestcoder.py index 18549aa0ee43..aedb01164df2 100644 --- a/src/transformers/models/iquestcoder/configuration_iquestcoder.py +++ b/src/transformers/models/iquestcoder/configuration_iquestcoder.py @@ -1,9 +1,3 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from src/transformers/models/iquestcoder/modular_iquestcoder.py. -# Do NOT edit this file manually as any edits will be overwritten by the generation of -# the file from the modular. If any change should be done, please apply the change to the -# modular_iquestcoder.py file directly. One of our CI enforces this. -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/transformers/models/iquestcoder/modular_iquestcoder.py b/src/transformers/models/iquestcoder/modular_iquestcoder.py deleted file mode 100644 index cf3bb03e5d4e..000000000000 --- a/src/transformers/models/iquestcoder/modular_iquestcoder.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from huggingface_hub.dataclasses import strict - -from ...modeling_rope_utils import RopeParameters -from ...utils import auto_docstring -from ..llama.configuration_llama import LlamaConfig - - -@auto_docstring(checkpoint="IQuestLab/IQuest-Coder-V1-40B-Instruct") -@strict -class IQuestCoderConfig(LlamaConfig): - r""" - ```python - >>> from transformers import AutoModelForCausalLM, IQuestCoderConfig - - >>> # Initializing an IQuest-Coder-V1-40B style configuration - >>> configuration = IQuestCoderConfig() - - >>> # Initializing a model (which uses the Llama implementation) from that configuration - >>> model = AutoModelForCausalLM.from_config(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "iquestcoder" - - vocab_size: int = 76800 - hidden_size: int = 5120 - intermediate_size: int = 27648 - num_hidden_layers: int = 80 - num_attention_heads: int = 40 - num_key_value_heads: int | None = 8 - hidden_act: str = "silu" - max_position_embeddings: int = 131072 - initializer_range: float = 0.02 - rms_norm_eps: float = 1e-5 - use_cache: bool = True - pad_token_id: int | None = None - bos_token_id: int | None = 1 - eos_token_id: int | list[int] | None = 2 - tie_word_embeddings: bool = False - rope_parameters: RopeParameters | dict | None = None - attention_bias: bool = False - attention_dropout: float | int = 0.0 - mlp_bias: bool = False - head_dim: int | None = 128 - - pretraining_tp = AttributeError() - - def validate_architecture(self): - """Part of `@strict`-powered validation. Validates the architecture of the config.""" - super().validate_architecture() - # The original code supports these options, but no released checkpoint enables them - if getattr(self, "clip_qkv", None) is not None: - raise ValueError("IQuestCoder does not support `clip_qkv`.") - if getattr(self, "use_sliding_window", False): - raise ValueError("IQuestCoder does not support `use_sliding_window`.") - - -__all__ = ["IQuestCoderConfig"] diff --git a/src/transformers/models/telechat3/configuration_telechat3.py b/src/transformers/models/telechat3/configuration_telechat3.py index 30820617b3d2..8d30d069fa56 100644 --- a/src/transformers/models/telechat3/configuration_telechat3.py +++ b/src/transformers/models/telechat3/configuration_telechat3.py @@ -1,9 +1,3 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from src/transformers/models/telechat3/modular_telechat3.py. -# Do NOT edit this file manually as any edits will be overwritten by the generation of -# the file from the modular. If any change should be done, please apply the change to the -# modular_telechat3.py file directly. One of our CI enforces this. -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/transformers/models/telechat3/modular_telechat3.py b/src/transformers/models/telechat3/modular_telechat3.py deleted file mode 100644 index 614eaa9c8aaa..000000000000 --- a/src/transformers/models/telechat3/modular_telechat3.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import math - -from huggingface_hub.dataclasses import strict - -from ...configuration_utils import PreTrainedConfig -from ...modeling_rope_utils import RopeParameters -from ...utils import auto_docstring, logging -from ..llama.configuration_llama import LlamaConfig - - -logger = logging.get_logger(__name__) - - -@auto_docstring(checkpoint="Tele-AI/TeleChat3-36B-Thinking") -@strict -class TeleChat3Config(LlamaConfig): - r""" - ```python - >>> from transformers import AutoModelForCausalLM, TeleChat3Config - - >>> # Initializing a TeleChat3-36B style configuration - >>> configuration = TeleChat3Config() - - >>> # Initializing a model (which uses the Llama implementation) from that configuration - >>> model = AutoModelForCausalLM.from_config(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "telechat3" - - vocab_size: int = 131072 - hidden_size: int = 6144 - intermediate_size: int = 24576 - num_hidden_layers: int = 64 - num_attention_heads: int = 48 - num_key_value_heads: int | None = 8 - hidden_act: str = "silu" - max_position_embeddings: int = 32768 - initializer_range: float = 0.0048 - rms_norm_eps: float = 1e-5 - use_cache: bool = True - pad_token_id: int | None = 3 - bos_token_id: int | None = 1 - eos_token_id: int | list[int] | None = 2 - tie_word_embeddings: bool = False - rope_parameters: RopeParameters | dict | None = None - attention_bias: bool = False - attention_dropout: float | int = 0.0 - mlp_bias: bool = False - head_dim: int | None = 128 - - pretraining_tp = AttributeError() - - def convert_rope_params_to_dict(self, **kwargs): - # The original checkpoints use a custom `"telechat3-yarn"` RoPE type, which is YaRN with 0.07 instead of 0.1 - # as the coefficient of its default attention factor. Convert it to plain YaRN with an explicit attention factor - rope_parameters = kwargs.get("rope_scaling") or self.rope_parameters - if rope_parameters is not None and "telechat3-yarn" in ( - rope_parameters.get("rope_type"), - rope_parameters.get("type"), - ): - rope_parameters = {k: v for k, v in rope_parameters.items() if k != "type"} - rope_parameters["rope_type"] = "yarn" - if "original_max_position_embeddings" in rope_parameters: - # The original code derives the scaling factor from the context lengths, ignoring `factor` - rope_parameters["factor"] = ( - self.max_position_embeddings / rope_parameters["original_max_position_embeddings"] - ) - if rope_parameters.get("attention_factor") is None: - factor = rope_parameters["factor"] - mscale = rope_parameters.get("mscale") - mscale_all_dim = rope_parameters.get("mscale_all_dim") - - def get_mscale(scale, mscale=1.0): - return 1.0 if scale <= 1 else 0.07 * mscale * math.log(scale) + 1.0 - - if mscale and mscale_all_dim: - attention_factor = get_mscale(factor, mscale) / get_mscale(factor, mscale_all_dim) - else: - attention_factor = get_mscale(factor) - rope_parameters["attention_factor"] = attention_factor - logger.info("Converted rope_type 'telechat3-yarn' to 'yarn' with an explicit `attention_factor`.") - kwargs["rope_scaling"] = rope_parameters - return PreTrainedConfig.convert_rope_params_to_dict(self, **kwargs) - - -__all__ = ["TeleChat3Config"] From 53a3868e9c72b0eb03f792bdcc5dbd32b35721f4 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:16:11 +0000 Subject: [PATCH 11/17] Drop the integration tests of the config-only models They run the Llama implementation, which Llama's own tests cover. The config-specific behaviour is covered by the fast tests. Co-Authored-By: Claude Opus 5.5 --- .../internlm3/test_modeling_internlm3.py | 37 +------------------ .../iquestcoder/test_modeling_iquestcoder.py | 37 +------------------ .../telechat3/test_modeling_telechat3.py | 37 +------------------ 3 files changed, 6 insertions(+), 105 deletions(-) diff --git a/tests/models/internlm3/test_modeling_internlm3.py b/tests/models/internlm3/test_modeling_internlm3.py index 2f81d341f083..a05733c98c4f 100644 --- a/tests/models/internlm3/test_modeling_internlm3.py +++ b/tests/models/internlm3/test_modeling_internlm3.py @@ -17,15 +17,13 @@ from huggingface_hub.errors import StrictDataclassClassValidationError -from transformers import AutoModelForCausalLM, AutoTokenizer, InternLM3Config, is_torch_available -from transformers.testing_utils import cleanup, require_torch, require_torch_accelerator, slow, torch_device +from transformers import InternLM3Config, is_torch_available +from transformers.testing_utils import require_torch from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester if is_torch_available(): - import torch - from transformers import LlamaModel @@ -46,34 +44,3 @@ def test_original_bias_names(self): self.assertTrue(config.mlp_bias) with self.assertRaises(StrictDataclassClassValidationError): InternLM3Config(qkv_bias=True, bias=False) - - -@slow -@require_torch_accelerator -class InternLM3IntegrationTest(unittest.TestCase): - model_id = "internlm/internlm3-8b-instruct" - - def tearDown(self): - cleanup(torch_device, gc_collect=True) - - def test_model_8b_logits(self): - model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) - input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) - with torch.no_grad(): - logits = model(input_ids).logits.float().cpu() - - EXPECTED_MEAN = torch.tensor([-0.2702, 1.8108, 2.4337, 4.6791, 3.8414, 4.021, 4.3054, 4.6507, 4.841, 4.5961]) - EXPECTED_SLICE = torch.tensor( - [-1.7656, -0.5547, 3.5625, -0.6719, 1.9141, 1.5781, 1.2344, 0.7188, 0.3398, 0.5039] - ) - torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN, rtol=1e-2, atol=1e-2) - torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) - - def test_model_8b_generation(self): - tokenizer = AutoTokenizer.from_pretrained(self.model_id) - model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) - inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) - output = model.generate(**inputs, max_new_tokens=16, do_sample=False) - text = tokenizer.decode(output[0], skip_special_tokens=True) - EXPECTED_TEXT = "The capital of France is Paris.Paris is the capital of France. It is a city that is known" - self.assertEqual(text, EXPECTED_TEXT) diff --git a/tests/models/iquestcoder/test_modeling_iquestcoder.py b/tests/models/iquestcoder/test_modeling_iquestcoder.py index e8474001f329..0d7766739c89 100644 --- a/tests/models/iquestcoder/test_modeling_iquestcoder.py +++ b/tests/models/iquestcoder/test_modeling_iquestcoder.py @@ -17,15 +17,13 @@ from huggingface_hub.errors import StrictDataclassClassValidationError -from transformers import AutoModelForCausalLM, AutoTokenizer, IQuestCoderConfig, is_torch_available -from transformers.testing_utils import cleanup, require_torch, require_torch_accelerator, slow, torch_device +from transformers import IQuestCoderConfig, is_torch_available +from transformers.testing_utils import require_torch from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester if is_torch_available(): - import torch - from transformers import LlamaModel @@ -46,34 +44,3 @@ def test_unsupported_options_are_rejected(self): IQuestCoderConfig(clip_qkv=8.0) with self.assertRaises(StrictDataclassClassValidationError): IQuestCoderConfig(use_sliding_window=True, sliding_window=4096) - - -@slow -@require_torch_accelerator -class IQuestCoderIntegrationTest(unittest.TestCase): - model_id = "IQuestLab/IQuest-Coder-V1-7B-Instruct" - - def tearDown(self): - cleanup(torch_device, gc_collect=True) - - def test_model_7b_logits(self): - model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) - input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) - with torch.no_grad(): - logits = model(input_ids).logits.float().cpu() - - EXPECTED_MEAN = torch.tensor( - [23.4439, -0.9136, 6.2595, 9.6258, 8.6033, 13.4631, 6.3165, 8.7364, 11.1808, 6.8638] - ) - EXPECTED_SLICE = torch.tensor([25.75, 30.75, 26.625, 28.625, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0]) - torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN, rtol=1e-2, atol=1e-2) - torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) - - def test_model_7b_generation(self): - tokenizer = AutoTokenizer.from_pretrained(self.model_id) - model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) - inputs = tokenizer("def fibonacci(n):", return_tensors="pt").to(model.device) - output = model.generate(**inputs, max_new_tokens=16, do_sample=False) - text = tokenizer.decode(output[0], skip_special_tokens=True) - EXPECTED_TEXT = "def fibonacci(n):\n if n <= 0:\n " - self.assertEqual(text, EXPECTED_TEXT) diff --git a/tests/models/telechat3/test_modeling_telechat3.py b/tests/models/telechat3/test_modeling_telechat3.py index dfc56cffe29e..74dde06be1ea 100644 --- a/tests/models/telechat3/test_modeling_telechat3.py +++ b/tests/models/telechat3/test_modeling_telechat3.py @@ -16,15 +16,13 @@ import math import unittest -from transformers import AutoModelForCausalLM, AutoTokenizer, TeleChat3Config, is_torch_available -from transformers.testing_utils import cleanup, require_torch, require_torch_accelerator, slow, torch_device +from transformers import TeleChat3Config, is_torch_available +from transformers.testing_utils import require_torch from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester if is_torch_available(): - import torch - from transformers import LlamaModel @@ -73,34 +71,3 @@ def test_explicit_attention_factor_is_kept(self): rope_scaling={"factor": 4.0, "attention_factor": 1.5, "rope_type": "telechat3-yarn"}, ) self.assertEqual(config.rope_parameters["attention_factor"], 1.5) - - -@slow -@require_torch_accelerator -class TeleChat3IntegrationTest(unittest.TestCase): - model_id = "Tele-AI/TeleChat3-36B-Thinking" - - def tearDown(self): - cleanup(torch_device, gc_collect=True) - - def test_model_36b_logits(self): - model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) - input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) - with torch.no_grad(): - logits = model(input_ids).logits.float().cpu() - - EXPECTED_MEAN = torch.tensor( - [-4.6161, -2.5073, -1.4698, -2.3869, -2.4526, -4.3019, -2.0553, -3.1628, -2.0378, -1.7738] - ) - EXPECTED_SLICE = torch.tensor([-8.75, -10.75, -7.875, -6.7188, -8.8125, -8.75, -8.6875, -8.75, -8.75, -7.1562]) - torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN, rtol=1e-2, atol=1e-2) - torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) - - def test_model_36b_generation(self): - tokenizer = AutoTokenizer.from_pretrained(self.model_id) - model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) - inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) - output = model.generate(**inputs, max_new_tokens=16, do_sample=False) - text = tokenizer.decode(output[0], skip_special_tokens=True) - EXPECTED_TEXT = "The capital of France is Paris.\n\n\nOkay, let's see. The user wants me" - self.assertEqual(text, EXPECTED_TEXT) From 344b8d735227c08171f7a2c1cee85cdeb496006c Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:05:38 +0000 Subject: [PATCH 12/17] Keep only the TeleChat3 config tests for the config-only models The common tests re-ran Llama's suite with Llama-equivalent configs. TeleChat3's RoPE conversion is the only config logic worth a test. Co-Authored-By: Claude Opus 5.5 --- tests/models/internlm3/__init__.py | 0 .../internlm3/test_modeling_internlm3.py | 46 ------------------- tests/models/iquestcoder/__init__.py | 0 .../iquestcoder/test_modeling_iquestcoder.py | 46 ------------------- ...at3.py => test_configuration_telechat3.py} | 22 +-------- 5 files changed, 2 insertions(+), 112 deletions(-) delete mode 100644 tests/models/internlm3/__init__.py delete mode 100644 tests/models/internlm3/test_modeling_internlm3.py delete mode 100644 tests/models/iquestcoder/__init__.py delete mode 100644 tests/models/iquestcoder/test_modeling_iquestcoder.py rename tests/models/telechat3/{test_modeling_telechat3.py => test_configuration_telechat3.py} (79%) diff --git a/tests/models/internlm3/__init__.py b/tests/models/internlm3/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/models/internlm3/test_modeling_internlm3.py b/tests/models/internlm3/test_modeling_internlm3.py deleted file mode 100644 index a05733c98c4f..000000000000 --- a/tests/models/internlm3/test_modeling_internlm3.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2026 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Testing suite for InternLM3, which uses the Llama implementation.""" - -import unittest - -from huggingface_hub.errors import StrictDataclassClassValidationError - -from transformers import InternLM3Config, is_torch_available -from transformers.testing_utils import require_torch - -from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester - - -if is_torch_available(): - from transformers import LlamaModel - - -class InternLM3ModelTester(CausalLMModelTester): - config_class = InternLM3Config - if is_torch_available(): - base_model_class = LlamaModel - - -@require_torch -class InternLM3ModelTest(CausalLMModelTest, unittest.TestCase): - model_tester_class = InternLM3ModelTester - - def test_original_bias_names(self): - # The original config names, as found in the released checkpoints - config = InternLM3Config(qkv_bias=True, bias=True) - self.assertTrue(config.attention_bias) - self.assertTrue(config.mlp_bias) - with self.assertRaises(StrictDataclassClassValidationError): - InternLM3Config(qkv_bias=True, bias=False) diff --git a/tests/models/iquestcoder/__init__.py b/tests/models/iquestcoder/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/models/iquestcoder/test_modeling_iquestcoder.py b/tests/models/iquestcoder/test_modeling_iquestcoder.py deleted file mode 100644 index 0d7766739c89..000000000000 --- a/tests/models/iquestcoder/test_modeling_iquestcoder.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2026 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Testing suite for IQuestCoder, which uses the Llama implementation.""" - -import unittest - -from huggingface_hub.errors import StrictDataclassClassValidationError - -from transformers import IQuestCoderConfig, is_torch_available -from transformers.testing_utils import require_torch - -from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester - - -if is_torch_available(): - from transformers import LlamaModel - - -class IQuestCoderModelTester(CausalLMModelTester): - config_class = IQuestCoderConfig - if is_torch_available(): - base_model_class = LlamaModel - - -@require_torch -class IQuestCoderModelTest(CausalLMModelTest, unittest.TestCase): - model_tester_class = IQuestCoderModelTester - - def test_unsupported_options_are_rejected(self): - # Released checkpoints set these to their disabled values, which must keep loading - IQuestCoderConfig(clip_qkv=None, use_sliding_window=False, sliding_window=None, max_window_layers=0) - with self.assertRaises(StrictDataclassClassValidationError): - IQuestCoderConfig(clip_qkv=8.0) - with self.assertRaises(StrictDataclassClassValidationError): - IQuestCoderConfig(use_sliding_window=True, sliding_window=4096) diff --git a/tests/models/telechat3/test_modeling_telechat3.py b/tests/models/telechat3/test_configuration_telechat3.py similarity index 79% rename from tests/models/telechat3/test_modeling_telechat3.py rename to tests/models/telechat3/test_configuration_telechat3.py index 74dde06be1ea..bfa1f49b645e 100644 --- a/tests/models/telechat3/test_modeling_telechat3.py +++ b/tests/models/telechat3/test_configuration_telechat3.py @@ -11,30 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Testing suite for TeleChat3, which uses the Llama implementation.""" +"""Tests for the TeleChat3 config, which converts the original RoPE parameters for the Llama implementation.""" import math import unittest -from transformers import TeleChat3Config, is_torch_available -from transformers.testing_utils import require_torch - -from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester - - -if is_torch_available(): - from transformers import LlamaModel - - -class TeleChat3ModelTester(CausalLMModelTester): - config_class = TeleChat3Config - if is_torch_available(): - base_model_class = LlamaModel - - -@require_torch -class TeleChat3ModelTest(CausalLMModelTest, unittest.TestCase): - model_tester_class = TeleChat3ModelTester +from transformers import TeleChat3Config class TeleChat3ConfigTest(unittest.TestCase): From f0f228b0eea72b75a3f949fc9a3e427fd4dcf4c2 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:58:40 +0000 Subject: [PATCH 13/17] Address review: restore config modular files, TeleChat-only KV op, cached TeleChat RoPE - Restore the modular files of InternLM3, IQuestCoder and TeleChat3, inheriting from `LlamaConfig` - Revert the `Interleave` changes and add a TeleChat-only `TeleChatInterleaveKeyValue` op - Map TeleChat's original config keys explicitly instead of via `attribute_map` - Move `training_seqlen` and `base_seqlen` into `rope_parameters` (`"rope_type": "telechat_ntk"`) - Cache the TeleChat RoPE frequencies, recomputing them only when the length beyond training changes - Trim the TeleChat tests, use `Expectations` and `MemoryCleanupMixin`, drop the TeleChat3 config tests - Use note/warning callouts in the docs and drop the conversion paragraph from the TeleChat docs Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 3 + docs/source/en/model_doc/iquestcoder.md | 6 +- docs/source/en/model_doc/telechat.md | 7 +- src/transformers/conversion_mapping.py | 3 +- src/transformers/core_model_loading.py | 47 ++++--- .../internlm3/configuration_internlm3.py | 6 + .../models/internlm3/modular_internlm3.py | 75 +++++++++++ .../iquestcoder/configuration_iquestcoder.py | 6 + .../models/iquestcoder/modular_iquestcoder.py | 74 +++++++++++ .../models/telechat/configuration_telechat.py | 50 +++++--- .../models/telechat/modeling_telechat.py | 96 +++++++------- .../models/telechat/modular_telechat.py | 119 +++++++++++++----- .../telechat3/configuration_telechat3.py | 6 + .../models/telechat3/modular_telechat3.py | 103 +++++++++++++++ .../models/telechat/test_modeling_telechat.py | 112 ++++------------- tests/models/telechat3/__init__.py | 0 .../telechat3/test_configuration_telechat3.py | 55 -------- tests/utils/test_core_model_loading.py | 8 +- 18 files changed, 515 insertions(+), 261 deletions(-) create mode 100644 src/transformers/models/internlm3/modular_internlm3.py create mode 100644 src/transformers/models/iquestcoder/modular_iquestcoder.py create mode 100644 src/transformers/models/telechat3/modular_telechat3.py delete mode 100644 tests/models/telechat3/__init__.py delete mode 100644 tests/models/telechat3/test_configuration_telechat3.py diff --git a/.gitattributes b/.gitattributes index 384f802b32ea..15b21dc8dfcd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -267,7 +267,9 @@ src/transformers/models/inkling/configuration_*.py linguist-generated=true src/transformers/models/inkling/modeling_*.py linguist-generated=true src/transformers/models/instructblipvideo/configuration_*.py linguist-generated=true src/transformers/models/instructblipvideo/modeling_*.py linguist-generated=true +src/transformers/models/internlm3/configuration_*.py linguist-generated=true src/transformers/models/internvl/modeling_*.py linguist-generated=true +src/transformers/models/iquestcoder/configuration_*.py linguist-generated=true src/transformers/models/jais2/configuration_*.py linguist-generated=true src/transformers/models/jais2/modeling_*.py linguist-generated=true src/transformers/models/jamba/modeling_*.py linguist-generated=true @@ -570,6 +572,7 @@ src/transformers/models/t5gemma2/configuration_*.py linguist-generated=true src/transformers/models/t5gemma2/modeling_*.py linguist-generated=true src/transformers/models/telechat/configuration_*.py linguist-generated=true src/transformers/models/telechat/modeling_*.py linguist-generated=true +src/transformers/models/telechat3/configuration_*.py linguist-generated=true src/transformers/models/timesfm/modeling_*.py linguist-generated=true src/transformers/models/timesfm2_5/configuration_*.py linguist-generated=true src/transformers/models/timesfm2_5/modeling_*.py linguist-generated=true diff --git a/docs/source/en/model_doc/iquestcoder.md b/docs/source/en/model_doc/iquestcoder.md index 6009b0479a20..e9bc4717e1d7 100644 --- a/docs/source/en/model_doc/iquestcoder.md +++ b/docs/source/en/model_doc/iquestcoder.md @@ -25,9 +25,11 @@ rendered properly in your Markdown viewer. [IQuest-Coder-V1](https://huggingface.co/IQuestLab/IQuest-Coder-V1-40B-Instruct) is a family of code language models developed by [IQuest Lab](https://huggingface.co/IQuestLab), available in 7B, 14B and 40B sizes with base, instruct and thinking variants. Its architecture is the same as [Llama](./llama). -The looped `IQuest-Coder-V1-*-Loop-*` checkpoints use a different architecture and aren't supported by this model. +> [!NOTE] +> The looped `IQuest-Coder-V1-*-Loop-*` checkpoints use a different architecture and aren't supported by this model. -The tokenizer is built from the checkpoints' SentencePiece model. Many of its pieces share the same score, and SentencePiece breaks those ties by position, which the `tokenizers` library can't reproduce, so a small fraction of texts tokenize differently from the original remote code. +> [!WARNING] +> The tokenizer is built from the checkpoints' SentencePiece model. Many of its pieces share the same score, and SentencePiece breaks those ties by position, which the `tokenizers` library can't reproduce, so a small fraction of texts tokenize differently from the original remote code. The auto classes load it with the Llama implementation ([`LlamaModel`], [`LlamaForCausalLM`], [`LlamaForSequenceClassification`], [`LlamaForQuestionAnswering`] and [`LlamaForTokenClassification`]), so only [`IQuestCoderConfig`] is specific to IQuestCoder. diff --git a/docs/source/en/model_doc/telechat.md b/docs/source/en/model_doc/telechat.md index eb7d87c6f579..91371b135045 100644 --- a/docs/source/en/model_doc/telechat.md +++ b/docs/source/en/model_doc/telechat.md @@ -28,11 +28,10 @@ rendered properly in your Markdown viewer. Its architecture is the same as [Llama](./llama), except that: - only the attention output projection and the down projection of the MLP have biases; -- once the sequence grows beyond `training_seqlen`, RoPE is rescaled with a stepwise NTK scaling of its base and the cos/sin are scaled by `0.1 * ln(seq_len / training_seqlen) + 1`. +- once the sequence grows beyond the training length (`original_max_position_embeddings` in `rope_parameters`), RoPE is rescaled with a stepwise NTK scaling of its base and the cos/sin are scaled by `0.1 * ln(seq_len / original_max_position_embeddings) + 1`. -The original checkpoints use different parameter names and fuse the key and value projections per head (`[k_0, v_0, k_1, v_1, ...]`). They are converted when the model is loaded, and `save_pretrained` writes the original layout back. - -TeleChat-52B uses a different architecture and isn't supported by this model. +> [!NOTE] +> TeleChat-52B uses a different architecture and isn't supported by this model. The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. diff --git a/src/transformers/conversion_mapping.py b/src/transformers/conversion_mapping.py index e7d8eef374ad..9be89824ecf9 100755 --- a/src/transformers/conversion_mapping.py +++ b/src/transformers/conversion_mapping.py @@ -28,6 +28,7 @@ PermuteForRope, PrefixChange, Split, + TeleChatInterleaveKeyValue, Transpose, WeightConverter, WeightRenaming, @@ -1645,7 +1646,7 @@ def _build_checkpoint_conversion_mapping(): WeightConverter( "self_attention.key_value.weight", ["self_attn.k_proj.weight", "self_attn.v_proj.weight"], - operations=[Interleave(dim=0, block_size_attribute="head_dim"), Chunk(dim=0)], + operations=[TeleChatInterleaveKeyValue(), Chunk(dim=0)], ), WeightRenaming(r"self_attention\.dense\.", r"self_attn\.o_proj\."), ], diff --git a/src/transformers/core_model_loading.py b/src/transformers/core_model_loading.py index 97c8f7e80b83..5e3987fa4a6d 100644 --- a/src/transformers/core_model_loading.py +++ b/src/transformers/core_model_loading.py @@ -226,32 +226,22 @@ def reverse_op(self) -> ConversionOps: class Interleave(ConversionOps): - """Deinterleaves a tensor along `dim` by splitting in two and transposing. Reshapes param back to its original size. + """Deinterleaves a tensor along `dim` by splitting in two and transposing. Reshapes param back to its original size.""" - By default the two halves alternate element by element. With `block_size_attribute`, a config field, they alternate - in blocks of that size instead, e.g. `[k_0, v_0, k_1, v_1, ...]` for keys and values fused per head of `head_dim`. - """ - - def __init__(self, dim: int = 0, inverse: bool = False, block_size_attribute: str | None = None): + def __init__(self, dim: int = 0, inverse: bool = False): self.dim = dim self.inverse = inverse - self.block_size_attribute = block_size_attribute def convert(self, input_dict, source_patterns, target_patterns, **kwargs): tensor = next(iter(input_dict.values())) tensor = tensor[0] if isinstance(tensor, list) else tensor - block_size = 1 - if self.block_size_attribute is not None: - block_size = getattr(kwargs["config"].get_text_config(), self.block_size_attribute) - # Split into two in given dim and transpose to interleave along it shape = list(tensor.shape) - num_blocks = shape[self.dim] // (2 * block_size) if self.inverse: - shape[self.dim : self.dim + 1] = [2, num_blocks, block_size] + shape[self.dim : self.dim + 1] = [2, shape[self.dim] // 2] else: - shape[self.dim : self.dim + 1] = [num_blocks, 2, block_size] + shape[self.dim : self.dim + 1] = [shape[self.dim] // 2, 2] tensor = tensor.reshape(shape).transpose(self.dim, self.dim + 1).reshape(tensor.shape).contiguous() return {target_patterns[0]: tensor} @@ -259,7 +249,34 @@ def convert(self, input_dict, source_patterns, target_patterns, **kwargs): @property def reverse_op(self) -> ConversionOps: # can use the same dim and it will inverse it back - return Interleave(self.dim, inverse=not self.inverse, block_size_attribute=self.block_size_attribute) + return Interleave(self.dim, inverse=not self.inverse) + + +class TeleChatInterleaveKeyValue(ConversionOps): + """ + TeleChat fuses its key and value projections per head, i.e. `[k_0, v_0, k_1, v_1, ...]` along dim 0 with heads of + `head_dim`. This regroups them as `[k_0, k_1, ..., v_0, v_1, ...]` so that `Chunk` can split keys from values, or + does the opposite when `inverse` is set. + """ + + def __init__(self, inverse: bool = False): + self.inverse = inverse + + @torch.no_grad + def convert(self, input_dict, source_patterns, target_patterns, config, **kwargs): + tensor = next(iter(input_dict.values())) + tensor = tensor[0] if isinstance(tensor, list) else tensor + head_dim = config.get_text_config().head_dim + + rest = list(tensor.shape[1:]) + num_heads = tensor.shape[0] // (2 * head_dim) + shape = [2, num_heads, head_dim] if self.inverse else [num_heads, 2, head_dim] + tensor = tensor.reshape(shape + rest).transpose(0, 1).reshape(tensor.shape).contiguous() + return {target_patterns[0]: tensor} + + @property + def reverse_op(self) -> ConversionOps: + return TeleChatInterleaveKeyValue(inverse=not self.inverse) class MergeModulelist(ConversionOps): diff --git a/src/transformers/models/internlm3/configuration_internlm3.py b/src/transformers/models/internlm3/configuration_internlm3.py index ee80f2f0be0b..2d0a4c4b7e5e 100644 --- a/src/transformers/models/internlm3/configuration_internlm3.py +++ b/src/transformers/models/internlm3/configuration_internlm3.py @@ -1,3 +1,9 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/internlm3/modular_internlm3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_internlm3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/transformers/models/internlm3/modular_internlm3.py b/src/transformers/models/internlm3/modular_internlm3.py new file mode 100644 index 000000000000..d0609ae5bdee --- /dev/null +++ b/src/transformers/models/internlm3/modular_internlm3.py @@ -0,0 +1,75 @@ +# Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from huggingface_hub.dataclasses import strict + +from ...modeling_rope_utils import RopeParameters +from ...utils import auto_docstring +from ..llama.configuration_llama import LlamaConfig + + +@auto_docstring(checkpoint="internlm/internlm3-8b-instruct") +@strict +class InternLM3Config(LlamaConfig): + r""" + ```python + >>> from transformers import AutoModelForCausalLM, InternLM3Config + + >>> # Initializing an InternLM3-8B style configuration + >>> configuration = InternLM3Config() + + >>> # Initializing a model (which uses the Llama implementation) from that configuration + >>> model = AutoModelForCausalLM.from_config(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "internlm3" + attribute_map = {"qkv_bias": "attention_bias", "bias": "mlp_bias"} + + vocab_size: int = 128512 + hidden_size: int = 4096 + intermediate_size: int = 10240 + num_hidden_layers: int = 48 + num_attention_heads: int = 32 + num_key_value_heads: int | None = 2 + hidden_act: str = "silu" + max_position_embeddings: int = 32768 + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = 2 + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + tie_word_embeddings: bool = False + rope_parameters: RopeParameters | dict | None = None + attention_bias: bool = False + attention_dropout: float | int = 0.0 + mlp_bias: bool = False + head_dim: int | None = 128 + + pretraining_tp = AttributeError() + + def validate_architecture(self): + """Part of `@strict`-powered validation. Validates the architecture of the config.""" + super().validate_architecture() + if self.attention_bias != self.mlp_bias: + raise ValueError( + "InternLM3 with `qkv_bias` different from `bias` is not supported, because the Llama implementation " + "uses the same bias setting for the query, key, value and output projections." + ) + + +__all__ = ["InternLM3Config"] diff --git a/src/transformers/models/iquestcoder/configuration_iquestcoder.py b/src/transformers/models/iquestcoder/configuration_iquestcoder.py index aedb01164df2..18549aa0ee43 100644 --- a/src/transformers/models/iquestcoder/configuration_iquestcoder.py +++ b/src/transformers/models/iquestcoder/configuration_iquestcoder.py @@ -1,3 +1,9 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/iquestcoder/modular_iquestcoder.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_iquestcoder.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/transformers/models/iquestcoder/modular_iquestcoder.py b/src/transformers/models/iquestcoder/modular_iquestcoder.py new file mode 100644 index 000000000000..cf3bb03e5d4e --- /dev/null +++ b/src/transformers/models/iquestcoder/modular_iquestcoder.py @@ -0,0 +1,74 @@ +# Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from huggingface_hub.dataclasses import strict + +from ...modeling_rope_utils import RopeParameters +from ...utils import auto_docstring +from ..llama.configuration_llama import LlamaConfig + + +@auto_docstring(checkpoint="IQuestLab/IQuest-Coder-V1-40B-Instruct") +@strict +class IQuestCoderConfig(LlamaConfig): + r""" + ```python + >>> from transformers import AutoModelForCausalLM, IQuestCoderConfig + + >>> # Initializing an IQuest-Coder-V1-40B style configuration + >>> configuration = IQuestCoderConfig() + + >>> # Initializing a model (which uses the Llama implementation) from that configuration + >>> model = AutoModelForCausalLM.from_config(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "iquestcoder" + + vocab_size: int = 76800 + hidden_size: int = 5120 + intermediate_size: int = 27648 + num_hidden_layers: int = 80 + num_attention_heads: int = 40 + num_key_value_heads: int | None = 8 + hidden_act: str = "silu" + max_position_embeddings: int = 131072 + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = None + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + tie_word_embeddings: bool = False + rope_parameters: RopeParameters | dict | None = None + attention_bias: bool = False + attention_dropout: float | int = 0.0 + mlp_bias: bool = False + head_dim: int | None = 128 + + pretraining_tp = AttributeError() + + def validate_architecture(self): + """Part of `@strict`-powered validation. Validates the architecture of the config.""" + super().validate_architecture() + # The original code supports these options, but no released checkpoint enables them + if getattr(self, "clip_qkv", None) is not None: + raise ValueError("IQuestCoder does not support `clip_qkv`.") + if getattr(self, "use_sliding_window", False): + raise ValueError("IQuestCoder does not support `use_sliding_window`.") + + +__all__ = ["IQuestCoderConfig"] diff --git a/src/transformers/models/telechat/configuration_telechat.py b/src/transformers/models/telechat/configuration_telechat.py index 42dfd4f6c316..0956e6a271ed 100644 --- a/src/transformers/models/telechat/configuration_telechat.py +++ b/src/transformers/models/telechat/configuration_telechat.py @@ -30,10 +30,11 @@ @strict class TeleChatConfig(PreTrainedConfig): r""" - training_seqlen (`int`, *optional*, defaults to 8192): - Sequence length the model was trained with. Beyond it, RoPE is rescaled with stepwise NTK scaling. - base_seqlen (`int`, *optional*, defaults to 4096): - Sequence length the stepwise NTK scaling factor is computed relative to. + TeleChat applies its own stepwise NTK scaling of RoPE, described by `rope_parameters` with `"rope_type": "telechat_ntk"`: + + - `original_max_position_embeddings` (`training_seqlen` in the original configs, defaults to 8192): the sequence + length the model was trained with. Beyond it, the RoPE base and the cos/sin are rescaled. + - `base_seqlen` (defaults to 4096): the sequence length the NTK scaling factor of the base is computed relative to. ```python >>> from transformers import TeleChatModel, TeleChatConfig @@ -85,22 +86,22 @@ class TeleChatConfig(PreTrainedConfig): rope_parameters: RopeParameters | dict | None = None attention_dropout: float | int = 0.0 head_dim: int | None = None - attribute_map = { - "n_layer": "num_hidden_layers", - "n_head": "num_attention_heads", - "ffn_hidden_size": "intermediate_size", - "layer_norm_epsilon": "rms_norm_eps", - } - training_seqlen: int = 8192 - base_seqlen: int = 4096 def __post_init__(self, **kwargs): - # Resolve the aliases now, otherwise `head_dim` and `num_key_value_heads` get derived from the defaults - for alias, attribute in self.attribute_map.items(): - if alias in kwargs: - setattr(self, attribute, kwargs.pop(alias)) + # The original configs use different names for these. Resolve them now, otherwise `head_dim` and + # `num_key_value_heads` get derived from the defaults + if "n_layer" in kwargs: + self.num_hidden_layers = kwargs.pop("n_layer") + if "n_head" in kwargs: + self.num_attention_heads = kwargs.pop("n_head") + if "ffn_hidden_size" in kwargs: + self.intermediate_size = kwargs.pop("ffn_hidden_size") + if "layer_norm_epsilon" in kwargs: + self.rms_norm_eps = kwargs.pop("layer_norm_epsilon") + if "seq_length" in kwargs: + self.max_position_embeddings = kwargs.pop("seq_length") if self.max_position_embeddings is None: - self.max_position_embeddings = kwargs.get("seq_length") or self.training_seqlen + self.max_position_embeddings = kwargs.get("training_seqlen", 8192) if self.head_dim is None: self.head_dim = self.hidden_size // self.num_attention_heads if self.num_key_value_heads is None: @@ -128,7 +129,22 @@ def validate_architecture(self): def convert_rope_params_to_dict(self, **kwargs): # The original code ignores `rope_scaling` and always applies its own stepwise NTK scaling instead kwargs.pop("rope_scaling", None) + rope_parameters = self.rope_parameters or {} + self.rope_parameters = { + "rope_type": "telechat_ntk", + "rope_theta": rope_parameters.get("rope_theta", kwargs.pop("rope_theta", self.default_theta)), + "original_max_position_embeddings": rope_parameters.get( + "original_max_position_embeddings", kwargs.pop("training_seqlen", 8192) + ), + "base_seqlen": rope_parameters.get("base_seqlen", kwargs.pop("base_seqlen", 4096)), + } + kwargs.pop("training_seqlen", None) + kwargs.pop("base_seqlen", None) return super().convert_rope_params_to_dict(**kwargs) + def _validate_telechat_ntk_rope_parameters(self, rope_parameters: dict, ignore_keys: set[str] | None = None): + required_keys = {"rope_type", "rope_theta", "original_max_position_embeddings", "base_seqlen"} + self._check_received_keys("telechat_ntk", set(rope_parameters), required_keys, ignore_keys=ignore_keys) + __all__ = ["TeleChatConfig"] diff --git a/src/transformers/models/telechat/modeling_telechat.py b/src/transformers/models/telechat/modeling_telechat.py index c2fc0d531bfb..46e4436a0540 100644 --- a/src/transformers/models/telechat/modeling_telechat.py +++ b/src/transformers/models/telechat/modeling_telechat.py @@ -31,78 +31,86 @@ from ...masking_utils import create_causal_mask from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast -from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep from ...processing_utils import Unpack -from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling from ...utils.generic import merge_with_config_defaults from ...utils.output_capturing import capture_outputs from .configuration_telechat import TeleChatConfig class TeleChatRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + def __init__(self, config: TeleChatConfig): super().__init__() - self.max_seq_len_cached = config.max_position_embeddings - self.original_max_seq_len = config.max_position_embeddings - self.config = config - - self.rope_type = self.config.rope_parameters["rope_type"] - rope_init_fn: Callable = self.compute_default_rope_parameters - if self.rope_type != "default": - rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] - inv_freq, self.attention_scaling = rope_init_fn(self.config) + # Within the training length the frequencies are fixed, beyond it they depend on the sequence length + self.original_max_seq_len = config.rope_parameters["original_max_position_embeddings"] + self.max_seq_len_cached = self.original_max_seq_len + # `_init_weights` recomputes the buffers with `compute_default_rope_parameters` + self.rope_type = "default" + inv_freq, _ = self.compute_default_rope_parameters(config) + # The cos/sin are only scaled beyond the training length + self.attention_scaling = 1.0 self.inv_freq = nn.Buffer(inv_freq, persistent=False) self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False) @staticmethod - def compute_default_rope_parameters(config: TeleChatConfig, **kwargs) -> tuple[torch.Tensor, float]: + def compute_default_rope_parameters( + config: TeleChatConfig, device: torch.device | None = None, seq_len: int | torch.Tensor | None = None + ) -> tuple[torch.Tensor, torch.Tensor]: """ - Computes the inverse frequencies according to the original RoPE implementation - Args: - config ([`~transformers.PreTrainedConfig`]): - The model configuration. - Returns: - Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the - post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + Computes the inverse frequencies and attention scaling for the stepwise NTK scaling of the original code, + given the current total sequence length `seq_len` (the training length if `None`). """ - base = config.rope_parameters["rope_theta"] - dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + rope_parameters = config.rope_parameters + training_seqlen = rope_parameters["original_max_position_embeddings"] + dim = config.head_dim + seq_len = training_seqlen if seq_len is None else seq_len + seq_len = torch.as_tensor(seq_len, device=device).clamp(min=training_seqlen).float() + # Equal to `2 ** ceil(log2(seq_len / base_seqlen) + 1) - 1` from the original code, but exact for powers of 2 + doublings = torch.ceil(torch.log2(torch.ceil(seq_len / rope_parameters["base_seqlen"]))) + ntk_alpha = (2 ** (doublings + 1) - 1).clamp(min=1) + base = rope_parameters["rope_theta"] * ntk_alpha ** (dim / (dim - 2)) + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float, device=device) / dim)) + attention_scaling = 0.1 * torch.log(seq_len / training_seqlen) + 1.0 + return inv_freq, attention_scaling - attention_factor = 1.0 # Unused in this type of RoPE - # Compute the inverse frequencies - inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) - return inv_freq, attention_factor + def ntk_frequency_update(self, position_ids: torch.Tensor, device: torch.device): + """ + Like `dynamic_rope_update`, but the original code rescales for the exact sequence length rather than the + longest one seen so far, so the frequencies are recomputed whenever it changes beyond the training length. + """ + seq_len = max(int(position_ids.max()) + 1, self.original_max_seq_len) + if seq_len == self.max_seq_len_cached: + return + if seq_len == self.original_max_seq_len: + # The buffer is moved with the model, but not the original copy + inv_freq, self.attention_scaling = self.original_inv_freq.to(device), 1.0 + else: + inv_freq, self.attention_scaling = self.compute_default_rope_parameters(self.config, device, seq_len) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.max_seq_len_cached = seq_len @torch.no_grad() def forward(self, x, position_ids): - inv_freq, attention_scaling = self.compute_ntk_parameters(self.config, position_ids.max() + 1) - freqs = position_ids[..., None].float() * inv_freq.to(x.device) + if is_torchdynamo_compiling(): + # The cache lookup depends on the data, which can't be traced, so recompute the parameters instead + inv_freq, attention_scaling = self.compute_default_rope_parameters( + self.config, x.device, position_ids.max() + 1 + ) + else: + self.ntk_frequency_update(position_ids, x.device) + inv_freq, attention_scaling = self.inv_freq, self.attention_scaling + freqs = position_ids[..., None].float() * inv_freq.to(device=x.device, dtype=torch.float) emb = torch.cat((freqs, freqs), dim=-1) cos = emb.cos() * attention_scaling sin = emb.sin() * attention_scaling return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) - @staticmethod - def compute_ntk_parameters(config: TeleChatConfig, seq_len: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - """ - Computes the inverse frequencies and attention scaling for the stepwise NTK scaling of the original code, - given the current total sequence length `seq_len`. - """ - base = config.rope_parameters["rope_theta"] - dim = config.head_dim - seq_len = seq_len.clamp(min=config.training_seqlen).float() - # Equal to `2 ** ceil(log2(seq_len / base_seqlen) + 1) - 1` from the original code, but exact for powers of 2 - doublings = torch.ceil(torch.log2(torch.ceil(seq_len / config.base_seqlen))) - ntk_alpha = (2 ** (doublings + 1) - 1).clamp(min=1) - base = base * ntk_alpha ** (dim / (dim - 2)) - inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float, device=seq_len.device) / dim)) - attention_scaling = 0.1 * torch.log(seq_len / config.training_seqlen) + 1.0 - return inv_freq, attention_scaling - class TeleChatMLP(nn.Module): def __init__(self, config: TeleChatConfig): diff --git a/src/transformers/models/telechat/modular_telechat.py b/src/transformers/models/telechat/modular_telechat.py index c0ee93370d40..215e99f9a954 100644 --- a/src/transformers/models/telechat/modular_telechat.py +++ b/src/transformers/models/telechat/modular_telechat.py @@ -19,7 +19,7 @@ from ...configuration_utils import PreTrainedConfig from ...modeling_rope_utils import RopeParameters -from ...utils import auto_docstring +from ...utils import auto_docstring, is_torchdynamo_compiling from ..llama.configuration_llama import LlamaConfig from ..llama.modeling_llama import ( LlamaAttention, @@ -28,7 +28,6 @@ LlamaMLP, LlamaModel, LlamaPreTrainedModel, - LlamaRotaryEmbedding, ) @@ -36,10 +35,11 @@ @strict class TeleChatConfig(LlamaConfig): r""" - training_seqlen (`int`, *optional*, defaults to 8192): - Sequence length the model was trained with. Beyond it, RoPE is rescaled with stepwise NTK scaling. - base_seqlen (`int`, *optional*, defaults to 4096): - Sequence length the stepwise NTK scaling factor is computed relative to. + TeleChat applies its own stepwise NTK scaling of RoPE, described by `rope_parameters` with `"rope_type": "telechat_ntk"`: + + - `original_max_position_embeddings` (`training_seqlen` in the original configs, defaults to 8192): the sequence + length the model was trained with. Beyond it, the RoPE base and the cos/sin are rescaled. + - `base_seqlen` (defaults to 4096): the sequence length the NTK scaling factor of the base is computed relative to. ```python >>> from transformers import TeleChatModel, TeleChatConfig @@ -55,12 +55,6 @@ class TeleChatConfig(LlamaConfig): ```""" model_type = "telechat" - attribute_map = { - "n_layer": "num_hidden_layers", - "n_head": "num_attention_heads", - "ffn_hidden_size": "intermediate_size", - "layer_norm_epsilon": "rms_norm_eps", - } vocab_size: int = 131072 hidden_size: int = 3072 @@ -81,27 +75,48 @@ class TeleChatConfig(LlamaConfig): rope_parameters: RopeParameters | dict | None = None attention_dropout: float | int = 0.0 head_dim: int | None = None - training_seqlen: int = 8192 - base_seqlen: int = 4096 attention_bias = AttributeError() mlp_bias = AttributeError() pretraining_tp = AttributeError() def __post_init__(self, **kwargs): - # Resolve the aliases now, otherwise `head_dim` and `num_key_value_heads` get derived from the defaults - for alias, attribute in self.attribute_map.items(): - if alias in kwargs: - setattr(self, attribute, kwargs.pop(alias)) + # The original configs use different names for these. Resolve them now, otherwise `head_dim` and + # `num_key_value_heads` get derived from the defaults + if "n_layer" in kwargs: + self.num_hidden_layers = kwargs.pop("n_layer") + if "n_head" in kwargs: + self.num_attention_heads = kwargs.pop("n_head") + if "ffn_hidden_size" in kwargs: + self.intermediate_size = kwargs.pop("ffn_hidden_size") + if "layer_norm_epsilon" in kwargs: + self.rms_norm_eps = kwargs.pop("layer_norm_epsilon") + if "seq_length" in kwargs: + self.max_position_embeddings = kwargs.pop("seq_length") if self.max_position_embeddings is None: - self.max_position_embeddings = kwargs.get("seq_length") or self.training_seqlen + self.max_position_embeddings = kwargs.get("training_seqlen", 8192) super().__post_init__(**kwargs) def convert_rope_params_to_dict(self, **kwargs): # The original code ignores `rope_scaling` and always applies its own stepwise NTK scaling instead kwargs.pop("rope_scaling", None) + rope_parameters = self.rope_parameters or {} + self.rope_parameters = { + "rope_type": "telechat_ntk", + "rope_theta": rope_parameters.get("rope_theta", kwargs.pop("rope_theta", self.default_theta)), + "original_max_position_embeddings": rope_parameters.get( + "original_max_position_embeddings", kwargs.pop("training_seqlen", 8192) + ), + "base_seqlen": rope_parameters.get("base_seqlen", kwargs.pop("base_seqlen", 4096)), + } + kwargs.pop("training_seqlen", None) + kwargs.pop("base_seqlen", None) return PreTrainedConfig.convert_rope_params_to_dict(self, **kwargs) + def _validate_telechat_ntk_rope_parameters(self, rope_parameters: dict, ignore_keys: set[str] | None = None): + required_keys = {"rope_type", "rope_theta", "original_max_position_embeddings", "base_seqlen"} + self._check_received_keys("telechat_ntk", set(rope_parameters), required_keys, ignore_keys=ignore_keys) + def validate_architecture(self): """Part of `@strict`-powered validation. Validates the architecture of the config.""" super().validate_architecture() @@ -116,28 +131,72 @@ def validate_architecture(self): raise ValueError(f"TeleChat does not support `{option}`.") -class TeleChatRotaryEmbedding(LlamaRotaryEmbedding): +class TeleChatRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + def __init__(self, config: TeleChatConfig): + super().__init__() + self.config = config + # Within the training length the frequencies are fixed, beyond it they depend on the sequence length + self.original_max_seq_len = config.rope_parameters["original_max_position_embeddings"] + self.max_seq_len_cached = self.original_max_seq_len + # `_init_weights` recomputes the buffers with `compute_default_rope_parameters` + self.rope_type = "default" + inv_freq, _ = self.compute_default_rope_parameters(config) + # The cos/sin are only scaled beyond the training length + self.attention_scaling = 1.0 + + self.inv_freq = nn.Buffer(inv_freq, persistent=False) + self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False) + @staticmethod - def compute_ntk_parameters(config: TeleChatConfig, seq_len: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + def compute_default_rope_parameters( + config: TeleChatConfig, device: torch.device | None = None, seq_len: int | torch.Tensor | None = None + ) -> tuple[torch.Tensor, torch.Tensor]: """ Computes the inverse frequencies and attention scaling for the stepwise NTK scaling of the original code, - given the current total sequence length `seq_len`. + given the current total sequence length `seq_len` (the training length if `None`). """ - base = config.rope_parameters["rope_theta"] + rope_parameters = config.rope_parameters + training_seqlen = rope_parameters["original_max_position_embeddings"] dim = config.head_dim - seq_len = seq_len.clamp(min=config.training_seqlen).float() + seq_len = training_seqlen if seq_len is None else seq_len + seq_len = torch.as_tensor(seq_len, device=device).clamp(min=training_seqlen).float() # Equal to `2 ** ceil(log2(seq_len / base_seqlen) + 1) - 1` from the original code, but exact for powers of 2 - doublings = torch.ceil(torch.log2(torch.ceil(seq_len / config.base_seqlen))) + doublings = torch.ceil(torch.log2(torch.ceil(seq_len / rope_parameters["base_seqlen"]))) ntk_alpha = (2 ** (doublings + 1) - 1).clamp(min=1) - base = base * ntk_alpha ** (dim / (dim - 2)) - inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float, device=seq_len.device) / dim)) - attention_scaling = 0.1 * torch.log(seq_len / config.training_seqlen) + 1.0 + base = rope_parameters["rope_theta"] * ntk_alpha ** (dim / (dim - 2)) + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float, device=device) / dim)) + attention_scaling = 0.1 * torch.log(seq_len / training_seqlen) + 1.0 return inv_freq, attention_scaling + def ntk_frequency_update(self, position_ids: torch.Tensor, device: torch.device): + """ + Like `dynamic_rope_update`, but the original code rescales for the exact sequence length rather than the + longest one seen so far, so the frequencies are recomputed whenever it changes beyond the training length. + """ + seq_len = max(int(position_ids.max()) + 1, self.original_max_seq_len) + if seq_len == self.max_seq_len_cached: + return + if seq_len == self.original_max_seq_len: + # The buffer is moved with the model, but not the original copy + inv_freq, self.attention_scaling = self.original_inv_freq.to(device), 1.0 + else: + inv_freq, self.attention_scaling = self.compute_default_rope_parameters(self.config, device, seq_len) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.max_seq_len_cached = seq_len + @torch.no_grad() def forward(self, x, position_ids): - inv_freq, attention_scaling = self.compute_ntk_parameters(self.config, position_ids.max() + 1) - freqs = position_ids[..., None].float() * inv_freq.to(x.device) + if is_torchdynamo_compiling(): + # The cache lookup depends on the data, which can't be traced, so recompute the parameters instead + inv_freq, attention_scaling = self.compute_default_rope_parameters( + self.config, x.device, position_ids.max() + 1 + ) + else: + self.ntk_frequency_update(position_ids, x.device) + inv_freq, attention_scaling = self.inv_freq, self.attention_scaling + freqs = position_ids[..., None].float() * inv_freq.to(device=x.device, dtype=torch.float) emb = torch.cat((freqs, freqs), dim=-1) cos = emb.cos() * attention_scaling sin = emb.sin() * attention_scaling diff --git a/src/transformers/models/telechat3/configuration_telechat3.py b/src/transformers/models/telechat3/configuration_telechat3.py index 8d30d069fa56..30820617b3d2 100644 --- a/src/transformers/models/telechat3/configuration_telechat3.py +++ b/src/transformers/models/telechat3/configuration_telechat3.py @@ -1,3 +1,9 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/telechat3/modular_telechat3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_telechat3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/transformers/models/telechat3/modular_telechat3.py b/src/transformers/models/telechat3/modular_telechat3.py new file mode 100644 index 000000000000..614eaa9c8aaa --- /dev/null +++ b/src/transformers/models/telechat3/modular_telechat3.py @@ -0,0 +1,103 @@ +# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +from huggingface_hub.dataclasses import strict + +from ...configuration_utils import PreTrainedConfig +from ...modeling_rope_utils import RopeParameters +from ...utils import auto_docstring, logging +from ..llama.configuration_llama import LlamaConfig + + +logger = logging.get_logger(__name__) + + +@auto_docstring(checkpoint="Tele-AI/TeleChat3-36B-Thinking") +@strict +class TeleChat3Config(LlamaConfig): + r""" + ```python + >>> from transformers import AutoModelForCausalLM, TeleChat3Config + + >>> # Initializing a TeleChat3-36B style configuration + >>> configuration = TeleChat3Config() + + >>> # Initializing a model (which uses the Llama implementation) from that configuration + >>> model = AutoModelForCausalLM.from_config(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "telechat3" + + vocab_size: int = 131072 + hidden_size: int = 6144 + intermediate_size: int = 24576 + num_hidden_layers: int = 64 + num_attention_heads: int = 48 + num_key_value_heads: int | None = 8 + hidden_act: str = "silu" + max_position_embeddings: int = 32768 + initializer_range: float = 0.0048 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = 3 + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + tie_word_embeddings: bool = False + rope_parameters: RopeParameters | dict | None = None + attention_bias: bool = False + attention_dropout: float | int = 0.0 + mlp_bias: bool = False + head_dim: int | None = 128 + + pretraining_tp = AttributeError() + + def convert_rope_params_to_dict(self, **kwargs): + # The original checkpoints use a custom `"telechat3-yarn"` RoPE type, which is YaRN with 0.07 instead of 0.1 + # as the coefficient of its default attention factor. Convert it to plain YaRN with an explicit attention factor + rope_parameters = kwargs.get("rope_scaling") or self.rope_parameters + if rope_parameters is not None and "telechat3-yarn" in ( + rope_parameters.get("rope_type"), + rope_parameters.get("type"), + ): + rope_parameters = {k: v for k, v in rope_parameters.items() if k != "type"} + rope_parameters["rope_type"] = "yarn" + if "original_max_position_embeddings" in rope_parameters: + # The original code derives the scaling factor from the context lengths, ignoring `factor` + rope_parameters["factor"] = ( + self.max_position_embeddings / rope_parameters["original_max_position_embeddings"] + ) + if rope_parameters.get("attention_factor") is None: + factor = rope_parameters["factor"] + mscale = rope_parameters.get("mscale") + mscale_all_dim = rope_parameters.get("mscale_all_dim") + + def get_mscale(scale, mscale=1.0): + return 1.0 if scale <= 1 else 0.07 * mscale * math.log(scale) + 1.0 + + if mscale and mscale_all_dim: + attention_factor = get_mscale(factor, mscale) / get_mscale(factor, mscale_all_dim) + else: + attention_factor = get_mscale(factor) + rope_parameters["attention_factor"] = attention_factor + logger.info("Converted rope_type 'telechat3-yarn' to 'yarn' with an explicit `attention_factor`.") + kwargs["rope_scaling"] = rope_parameters + return PreTrainedConfig.convert_rope_params_to_dict(self, **kwargs) + + +__all__ = ["TeleChat3Config"] diff --git a/tests/models/telechat/test_modeling_telechat.py b/tests/models/telechat/test_modeling_telechat.py index dd5ca41fa8cd..4270c828f480 100644 --- a/tests/models/telechat/test_modeling_telechat.py +++ b/tests/models/telechat/test_modeling_telechat.py @@ -14,21 +14,19 @@ """Testing suite for the PyTorch TeleChat model.""" import math -import tempfile import unittest -from huggingface_hub.errors import StrictDataclassClassValidationError from parameterized import parameterized from transformers import AutoTokenizer, TeleChatConfig, is_torch_available -from transformers.testing_utils import cleanup, require_torch, require_torch_accelerator, slow, torch_device +from transformers.testing_utils import Expectations, require_torch, require_torch_accelerator, slow from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester +from ...test_memory_cleanup_mixin import MemoryCleanupMixin if is_torch_available(): import torch - from safetensors.torch import load_file from transformers import TeleChatForCausalLM, TeleChatModel from transformers.models.telechat.modeling_telechat import TeleChatRotaryEmbedding @@ -36,10 +34,11 @@ def original_ntk_parameters(config, seq_len): """The stepwise NTK scaling of the original code.""" - seq_len = max(seq_len, config.training_seqlen) - context_value = math.log2(seq_len / config.base_seqlen) + 1 + training_seqlen = config.rope_parameters["original_max_position_embeddings"] + seq_len = max(seq_len, training_seqlen) + context_value = math.log2(seq_len / config.rope_parameters["base_seqlen"]) + 1 ntk_alpha = max(2 ** math.ceil(context_value) - 1, 1) - scale = seq_len / config.training_seqlen + scale = seq_len / training_seqlen mscale = 1.0 if scale <= 1 else 0.1 * math.log(scale) + 1.0 dim = config.head_dim base = config.rope_parameters["rope_theta"] * ntk_alpha ** (dim / (dim - 2)) @@ -81,97 +80,30 @@ def test_ntk_scaling_matches_original(self, base_seqlen, training_seqlen): 3 * training_seqlen + 7, ]: expected_inv_freq, expected_mscale = original_ntk_parameters(config, seq_len) - inv_freq, mscale = TeleChatRotaryEmbedding.compute_ntk_parameters(config, torch.tensor(seq_len)) + inv_freq, mscale = TeleChatRotaryEmbedding.compute_default_rope_parameters(config, seq_len=seq_len) torch.testing.assert_close(inv_freq, expected_inv_freq) self.assertAlmostEqual(mscale.item(), expected_mscale, places=5) - def test_original_checkpoint_round_trip(self): - config, _ = self.model_tester.prepare_config_and_inputs_for_common() - model = TeleChatForCausalLM(config).to(torch_device).eval() - # Randomise everything, norms included, so a weight that fails to load can't go unnoticed - for param in model.parameters(): - torch.nn.init.normal_(param) - with tempfile.TemporaryDirectory() as tmpdir: - model.save_pretrained(tmpdir) - saved = set(load_file(f"{tmpdir}/model.safetensors")) - # Saved in the layout of the original checkpoints - for key in [ - "transformer.word_embeddings.weight", - "transformer.h.0.input_layernorm.weight", - "transformer.h.0.self_attention.query.weight", - "transformer.h.0.self_attention.key_value.weight", - "transformer.h.0.self_attention.dense.bias", - "transformer.h.0.post_attention_layernorm.weight", - "transformer.h.0.mlp.down_proj.bias", - "transformer.ln_f.weight", - ]: - self.assertIn(key, saved) - # Keys and values are fused per head, i.e. `[k_0, v_0, k_1, v_1, ...]` - attn = model.model.layers[0].self_attn - head_dim = config.head_dim - expected = torch.cat( - [ - attn.k_proj.weight.view(-1, head_dim, config.hidden_size), - attn.v_proj.weight.view(-1, head_dim, config.hidden_size), - ], - dim=1, - ).reshape(-1, config.hidden_size) - key_value = load_file(f"{tmpdir}/model.safetensors")["transformer.h.0.self_attention.key_value.weight"] - torch.testing.assert_close(key_value.to(torch_device), expected) - reloaded = TeleChatForCausalLM.from_pretrained(tmpdir).to(torch_device) - for (name, param), (_, reloaded_param) in zip(model.named_parameters(), reloaded.named_parameters()): - torch.testing.assert_close(param, reloaded_param, msg=name) - - def test_original_config_keys(self): - config = TeleChatConfig( - n_layer=3, n_head=8, hidden_size=64, ffn_hidden_size=96, layer_norm_epsilon=1e-4, seq_length=1024 - ) - self.assertEqual(config.num_hidden_layers, 3) - self.assertEqual(config.num_attention_heads, 8) - self.assertEqual(config.num_key_value_heads, 8) - self.assertEqual(config.head_dim, 8) - self.assertEqual(config.intermediate_size, 96) - self.assertEqual(config.rms_norm_eps, 1e-4) - self.assertEqual(config.max_position_embeddings, 1024) - - def test_rope_scaling_is_ignored(self): - # The original code never reads `rope_scaling`, so neither do we - config = TeleChatConfig(rope_theta=1e6, rope_scaling={"factor": 1.0, "rope_type": "dynamic"}) - self.assertEqual(config.rope_parameters, {"rope_type": "default", "rope_theta": 1e6}) - - def test_unsupported_configs_are_rejected(self): - with self.assertRaises(StrictDataclassClassValidationError): - TeleChatConfig(embed_layernorm=True) - with self.assertRaises(StrictDataclassClassValidationError): - TeleChatConfig(apply_residual_connection_post_layernorm=True) - with self.assertRaises(StrictDataclassClassValidationError): - # TeleChat-52B - TeleChatConfig(n_embd=8192, use_mup=True) - @slow @require_torch_accelerator -class TeleChatIntegrationTest(unittest.TestCase): - # The original code was written for fp16 +class TeleChatIntegrationTest(MemoryCleanupMixin, unittest.TestCase): + # The checkpoints' configs don't set a dtype, so `dtype="auto"` would load them in fp32. The original code uses fp16 dtype = torch.float16 if is_torch_available() else None - def tearDown(self): - cleanup(torch_device, gc_collect=True) - def test_model_3b_logits(self): model = TeleChatForCausalLM.from_pretrained("Tele-AI/TeleChat2-3B", device_map="auto", dtype=self.dtype) input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) - with torch.no_grad(): - logits = model(input_ids).logits.float().cpu() - - EXPECTED_MEAN = torch.tensor( - [-4.919, -4.2666, -1.901, -2.9477, -1.5264, -2.575, -0.5755, -1.9635, -2.781, -1.7785] - ) - EXPECTED_SLICE = torch.tensor( - [-9.1016, -9.8984, -1.8105, -9.1875, -9.1797, -9.1797, -9.1797, -9.1797, -9.1797, -6.3086] - ) - torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN, rtol=1e-2, atol=1e-2) - torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE, rtol=1e-2, atol=1e-2) + logits = model(input_ids).logits.float().cpu() + + EXPECTED_MEAN = Expectations({ + ("cuda", 10): torch.tensor([-4.919, -4.2666, -1.901, -2.9477, -1.5264, -2.575, -0.5755, -1.9635, -2.781, -1.7785]), + }) # fmt: skip + EXPECTED_SLICE = Expectations({ + ("cuda", 10): torch.tensor([-9.1016, -9.8984, -1.8105, -9.1875, -9.1797, -9.1797, -9.1797, -9.1797, -9.1797, -6.3086]), + }) # fmt: skip + torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN.get_expectation(), rtol=1e-2, atol=1e-2) + torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE.get_expectation(), rtol=1e-2, atol=1e-2) def test_model_3b_generation(self): tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat2-3B") @@ -179,5 +111,7 @@ def test_model_3b_generation(self): inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) text = tokenizer.decode(output[0], skip_special_tokens=True) - EXPECTED_TEXT = "The capital of France is known for its rich history, culture, and stunning architecture. The capital city," - self.assertEqual(text, EXPECTED_TEXT) + EXPECTED_TEXT = Expectations({ + ("cuda", 10): "The capital of France is known for its rich history, culture, and stunning architecture. The capital city,", + }) # fmt: skip + self.assertEqual(text, EXPECTED_TEXT.get_expectation()) diff --git a/tests/models/telechat3/__init__.py b/tests/models/telechat3/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/models/telechat3/test_configuration_telechat3.py b/tests/models/telechat3/test_configuration_telechat3.py deleted file mode 100644 index bfa1f49b645e..000000000000 --- a/tests/models/telechat3/test_configuration_telechat3.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2026 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for the TeleChat3 config, which converts the original RoPE parameters for the Llama implementation.""" - -import math -import unittest - -from transformers import TeleChat3Config - - -class TeleChat3ConfigTest(unittest.TestCase): - def test_telechat3_yarn_is_converted_to_yarn(self): - # `rope_scaling` as found in the released checkpoints - config = TeleChat3Config( - max_position_embeddings=32768, - rope_theta=1000000.0, - rope_scaling={ - "beta_fast": 32.0, - "beta_slow": 1.0, - "factor": 4.0, - "original_max_position_embeddings": 8192, - "rope_type": "telechat3-yarn", - "type": "telechat3-yarn", - }, - ) - self.assertEqual(config.rope_parameters["rope_type"], "yarn") - self.assertNotIn("type", config.rope_parameters) - self.assertEqual(config.rope_parameters["factor"], 4.0) - self.assertAlmostEqual(config.rope_parameters["attention_factor"], 0.07 * math.log(4.0) + 1.0) - - def test_factor_follows_context_lengths(self): - # The original code ignores `factor` when `original_max_position_embeddings` is set - config = TeleChat3Config( - max_position_embeddings=16384, - rope_scaling={"factor": 4.0, "original_max_position_embeddings": 8192, "rope_type": "telechat3-yarn"}, - ) - self.assertEqual(config.rope_parameters["factor"], 2.0) - self.assertAlmostEqual(config.rope_parameters["attention_factor"], 0.07 * math.log(2.0) + 1.0) - - def test_explicit_attention_factor_is_kept(self): - config = TeleChat3Config( - rope_scaling={"factor": 4.0, "attention_factor": 1.5, "rope_type": "telechat3-yarn"}, - ) - self.assertEqual(config.rope_parameters["attention_factor"], 1.5) diff --git a/tests/utils/test_core_model_loading.py b/tests/utils/test_core_model_loading.py index 679ab89b18c7..f6efb40e3533 100644 --- a/tests/utils/test_core_model_loading.py +++ b/tests/utils/test_core_model_loading.py @@ -31,12 +31,12 @@ Conv3dToLinear, ErnieFuseAndSplitTextVisionExperts, GroupWeightRename, - Interleave, LinearToConv3d, MergeModulelist, PermuteForRope, PrefixChange, Split, + TeleChatInterleaveKeyValue, VisionFuseAndPermuteForRope, VisionUnfuseAndPermuteForRope, WeightConverter, @@ -1383,8 +1383,8 @@ def test_reverse_renames_the_converter_outputs_one_to_many(self): self.assertTrue(compare_state_dicts(saved_qkv, state_dict)) -class TestInterleave(unittest.TestCase): - def test_block_size_splits_fused_heads(self): +class TestTeleChatInterleaveKeyValue(unittest.TestCase): + def test_splits_fused_heads(self): # Keys and values fused per head of size 2: `[k_0, v_0, k_1, v_1]` k = torch.arange(4, dtype=torch.float32).reshape(4, 1) v = 10 + k @@ -1392,7 +1392,7 @@ def test_block_size_splits_fused_heads(self): config = SimpleNamespace(get_text_config=lambda: SimpleNamespace(head_dim=2)) targets = ["k_proj.weight", "v_proj.weight"] - interleave = Interleave(dim=0, block_size_attribute="head_dim") + interleave = TeleChatInterleaveKeyValue() deinterleaved = interleave.convert({"kv": fused}, ["kv"], targets, config=config) split = Chunk(dim=0).convert(deinterleaved, ["kv"], targets) torch.testing.assert_close(split["k_proj.weight"], k) From 4b911349291e27d0697d23fa01e7c4b129c7d99d Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:03:26 +0000 Subject: [PATCH 14/17] Split InternLM3, IQuestCoder and TeleChat3 into their own PRs This PR now only adds TeleChat. Also document TeleChat's RoPE parameters under `rope_parameters`, since the docstring checker drops free text from config docstrings. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 3 - docs/source/en/_toctree.yml | 6 - docs/source/en/model_doc/internlm3.md | 47 ------ docs/source/en/model_doc/iquestcoder.md | 53 ------- docs/source/en/model_doc/telechat.md | 2 +- docs/source/en/model_doc/telechat3.md | 49 ------- src/transformers/models/__init__.py | 3 - src/transformers/models/auto/auto_mappings.py | 3 - src/transformers/models/auto/modeling_auto.py | 15 -- .../models/auto/tokenization_auto.py | 3 - src/transformers/models/internlm3/__init__.py | 32 ---- .../internlm3/configuration_internlm3.py | 107 -------------- .../models/internlm3/modular_internlm3.py | 75 ---------- .../models/iquestcoder/__init__.py | 32 ---- .../iquestcoder/configuration_iquestcoder.py | 106 -------------- .../models/iquestcoder/modular_iquestcoder.py | 74 ---------- .../models/telechat/configuration_telechat.py | 10 +- .../models/telechat/modular_telechat.py | 10 +- src/transformers/models/telechat3/__init__.py | 32 ---- .../telechat3/configuration_telechat3.py | 138 ------------------ .../models/telechat3/modular_telechat3.py | 103 ------------- utils/check_config_attributes.py | 4 - 22 files changed, 11 insertions(+), 896 deletions(-) delete mode 100644 docs/source/en/model_doc/internlm3.md delete mode 100644 docs/source/en/model_doc/iquestcoder.md delete mode 100644 docs/source/en/model_doc/telechat3.md delete mode 100644 src/transformers/models/internlm3/__init__.py delete mode 100644 src/transformers/models/internlm3/configuration_internlm3.py delete mode 100644 src/transformers/models/internlm3/modular_internlm3.py delete mode 100644 src/transformers/models/iquestcoder/__init__.py delete mode 100644 src/transformers/models/iquestcoder/configuration_iquestcoder.py delete mode 100644 src/transformers/models/iquestcoder/modular_iquestcoder.py delete mode 100644 src/transformers/models/telechat3/__init__.py delete mode 100644 src/transformers/models/telechat3/configuration_telechat3.py delete mode 100644 src/transformers/models/telechat3/modular_telechat3.py diff --git a/.gitattributes b/.gitattributes index 15b21dc8dfcd..384f802b32ea 100644 --- a/.gitattributes +++ b/.gitattributes @@ -267,9 +267,7 @@ src/transformers/models/inkling/configuration_*.py linguist-generated=true src/transformers/models/inkling/modeling_*.py linguist-generated=true src/transformers/models/instructblipvideo/configuration_*.py linguist-generated=true src/transformers/models/instructblipvideo/modeling_*.py linguist-generated=true -src/transformers/models/internlm3/configuration_*.py linguist-generated=true src/transformers/models/internvl/modeling_*.py linguist-generated=true -src/transformers/models/iquestcoder/configuration_*.py linguist-generated=true src/transformers/models/jais2/configuration_*.py linguist-generated=true src/transformers/models/jais2/modeling_*.py linguist-generated=true src/transformers/models/jamba/modeling_*.py linguist-generated=true @@ -572,7 +570,6 @@ src/transformers/models/t5gemma2/configuration_*.py linguist-generated=true src/transformers/models/t5gemma2/modeling_*.py linguist-generated=true src/transformers/models/telechat/configuration_*.py linguist-generated=true src/transformers/models/telechat/modeling_*.py linguist-generated=true -src/transformers/models/telechat3/configuration_*.py linguist-generated=true src/transformers/models/timesfm/modeling_*.py linguist-generated=true src/transformers/models/timesfm2_5/configuration_*.py linguist-generated=true src/transformers/models/timesfm2_5/modeling_*.py linguist-generated=true diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index cbd1ce206418..1b0f7dbd6e39 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -745,10 +745,6 @@ title: HYV4 - local: model_doc/ibert title: I-BERT - - local: model_doc/internlm3 - title: InternLM3 - - local: model_doc/iquestcoder - title: IQuestCoder - local: model_doc/jais2 title: Jais2 - local: model_doc/jamba @@ -961,8 +957,6 @@ title: T5v1.1 - local: model_doc/telechat title: TeleChat - - local: model_doc/telechat3 - title: TeleChat3 - local: model_doc/ul2 title: UL2 - local: model_doc/ultrabert diff --git a/docs/source/en/model_doc/internlm3.md b/docs/source/en/model_doc/internlm3.md deleted file mode 100644 index 3b9534314d8d..000000000000 --- a/docs/source/en/model_doc/internlm3.md +++ /dev/null @@ -1,47 +0,0 @@ - -*This model was contributed to Hugging Face Transformers on 2026-11-10.* -
-
- FlashAttention - SDPA -
-
- -# InternLM3 - -[InternLM3](https://huggingface.co/internlm/internlm3-8b-instruct) is an 8B instruction-tuned language model developed by [Shanghai AI Laboratory](https://huggingface.co/internlm). Its architecture is the same as [Llama](./llama). Its config names the biases of the query, key and value projections `qkv_bias` and those of the attention output projection and MLP layers `bias`; [`InternLM3Config`] loads them as `attention_bias` and `mlp_bias`, and requires them to be equal, as they are in the released checkpoints. - -The auto classes load it with the Llama implementation ([`LlamaModel`], [`LlamaForCausalLM`], [`LlamaForSequenceClassification`], [`LlamaForQuestionAnswering`] and [`LlamaForTokenClassification`]), so only [`InternLM3Config`] is specific to InternLM3. - -The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. - -```python -from transformers import AutoModelForCausalLM, AutoTokenizer - - -tokenizer = AutoTokenizer.from_pretrained("internlm/internlm3-8b-instruct") -model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto") - -messages = [{"role": "user", "content": "Explain the Pythagorean theorem in one sentence."}] -inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True) -output = model.generate(**inputs.to(model.device), max_new_tokens=64) -print(tokenizer.decode(output[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True)) -``` - -## InternLM3Config - -[[autodoc]] InternLM3Config diff --git a/docs/source/en/model_doc/iquestcoder.md b/docs/source/en/model_doc/iquestcoder.md deleted file mode 100644 index e9bc4717e1d7..000000000000 --- a/docs/source/en/model_doc/iquestcoder.md +++ /dev/null @@ -1,53 +0,0 @@ - -*This model was contributed to Hugging Face Transformers on 2026-11-10.* -
-
- FlashAttention - SDPA -
-
- -# IQuestCoder - -[IQuest-Coder-V1](https://huggingface.co/IQuestLab/IQuest-Coder-V1-40B-Instruct) is a family of code language models developed by [IQuest Lab](https://huggingface.co/IQuestLab), available in 7B, 14B and 40B sizes with base, instruct and thinking variants. Its architecture is the same as [Llama](./llama). - -> [!NOTE] -> The looped `IQuest-Coder-V1-*-Loop-*` checkpoints use a different architecture and aren't supported by this model. - -> [!WARNING] -> The tokenizer is built from the checkpoints' SentencePiece model. Many of its pieces share the same score, and SentencePiece breaks those ties by position, which the `tokenizers` library can't reproduce, so a small fraction of texts tokenize differently from the original remote code. - -The auto classes load it with the Llama implementation ([`LlamaModel`], [`LlamaForCausalLM`], [`LlamaForSequenceClassification`], [`LlamaForQuestionAnswering`] and [`LlamaForTokenClassification`]), so only [`IQuestCoderConfig`] is specific to IQuestCoder. - -The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. - -```python -from transformers import AutoModelForCausalLM, AutoTokenizer - - -tokenizer = AutoTokenizer.from_pretrained("IQuestLab/IQuest-Coder-V1-7B-Instruct") -model = AutoModelForCausalLM.from_pretrained("IQuestLab/IQuest-Coder-V1-7B-Instruct", device_map="auto") - -messages = [{"role": "user", "content": "Write a Python function that checks whether a number is prime."}] -inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True) -output = model.generate(**inputs.to(model.device), max_new_tokens=128) -print(tokenizer.decode(output[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True)) -``` - -## IQuestCoderConfig - -[[autodoc]] IQuestCoderConfig diff --git a/docs/source/en/model_doc/telechat.md b/docs/source/en/model_doc/telechat.md index 91371b135045..93cca254f5e1 100644 --- a/docs/source/en/model_doc/telechat.md +++ b/docs/source/en/model_doc/telechat.md @@ -23,7 +23,7 @@ rendered properly in your Markdown viewer. # TeleChat -[TeleChat](https://huggingface.co/Tele-AI) is a family of language models developed by [China Telecom Artificial Intelligence Technology](https://huggingface.co/Tele-AI), covering TeleChat (1B to 12B), TeleChat2 (3B to 115B) and TeleChat2.5 (35B and 115B). TeleChat3 is a separate model, see [TeleChat3](./telechat3). +[TeleChat](https://huggingface.co/Tele-AI) is a family of language models developed by [China Telecom Artificial Intelligence Technology](https://huggingface.co/Tele-AI), covering TeleChat (1B to 12B), TeleChat2 (3B to 115B) and TeleChat2.5 (35B and 115B). TeleChat3 is a separate model. Its architecture is the same as [Llama](./llama), except that: diff --git a/docs/source/en/model_doc/telechat3.md b/docs/source/en/model_doc/telechat3.md deleted file mode 100644 index 9f1cf6f99706..000000000000 --- a/docs/source/en/model_doc/telechat3.md +++ /dev/null @@ -1,49 +0,0 @@ - -*This model was contributed to Hugging Face Transformers on 2026-11-10.* -
-
- FlashAttention - SDPA -
-
- -# TeleChat3 - -[TeleChat3](https://huggingface.co/Tele-AI/TeleChat3-36B-Thinking) is a family of 36B reasoning language models developed by [China Telecom Artificial Intelligence Technology](https://huggingface.co/Tele-AI), including the general-purpose TeleChat3-36B-Thinking and the code-focused TeleChat3-Coder-36B-Thinking. Its architecture is the same as [Llama](./llama). - -The original checkpoints use a custom `"telechat3-yarn"` RoPE type, which is YaRN with 0.07 instead of 0.1 as the coefficient of its default attention factor. [`TeleChat3Config`] converts it to `"yarn"` with an explicit `attention_factor` when the config is loaded. - -The auto classes load it with the Llama implementation ([`LlamaModel`], [`LlamaForCausalLM`], [`LlamaForSequenceClassification`], [`LlamaForQuestionAnswering`] and [`LlamaForTokenClassification`]), so only [`TeleChat3Config`] is specific to TeleChat3. - -The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class. - -```python -from transformers import AutoModelForCausalLM, AutoTokenizer - - -tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat3-36B-Thinking") -model = AutoModelForCausalLM.from_pretrained("Tele-AI/TeleChat3-36B-Thinking", device_map="auto") - -messages = [{"role": "user", "content": "Explain the Pythagorean theorem in one sentence."}] -inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True) -output = model.generate(**inputs.to(model.device), max_new_tokens=256) -print(tokenizer.decode(output[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True)) -``` - -## TeleChat3Config - -[[autodoc]] TeleChat3Config diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index c87140579368..08948c840252 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -243,9 +243,7 @@ from .inkling import * from .instructblip import * from .instructblipvideo import * - from .internlm3 import * from .internvl import * - from .iquestcoder import * from .jais2 import * from .jamba import * from .janus import * @@ -495,7 +493,6 @@ from .table_transformer import * from .tapas import * from .telechat import * - from .telechat3 import * from .textnet import * from .time_series_transformer import * from .timesfm import * diff --git a/src/transformers/models/auto/auto_mappings.py b/src/transformers/models/auto/auto_mappings.py index 714ff97c04fd..514bd9cf67b7 100644 --- a/src/transformers/models/auto/auto_mappings.py +++ b/src/transformers/models/auto/auto_mappings.py @@ -339,10 +339,8 @@ ("instructblipvideo", "InstructBlipVideoConfig"), ("instructblipvideo_qformer", "InstructBlipVideoQFormerConfig"), ("instructblipvideo_vision_model", "InstructBlipVideoVisionConfig"), - ("internlm3", "InternLM3Config"), ("internvl", "InternVLConfig"), ("internvl_vision", "InternVLVisionConfig"), - ("iquestcoder", "IQuestCoderConfig"), ("jais2", "Jais2Config"), ("jamba", "JambaConfig"), ("janus", "JanusConfig"), @@ -720,7 +718,6 @@ ("table-transformer", "TableTransformerConfig"), ("tapas", "TapasConfig"), ("telechat", "TeleChatConfig"), - ("telechat3", "TeleChat3Config"), ("textnet", "TextNetConfig"), ("time_series_transformer", "TimeSeriesTransformerConfig"), ("timesfm", "TimesFmConfig"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 799e8f523668..569ecca91dc1 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -283,10 +283,8 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("inkling_text", "InklingTextModel"), ("instructblip", "InstructBlipModel"), ("instructblipvideo", "InstructBlipVideoModel"), - ("internlm3", "LlamaModel"), ("internvl", "InternVLModel"), ("internvl_vision", "InternVLVisionModel"), - ("iquestcoder", "LlamaModel"), ("jais2", "Jais2Model"), ("jamba", "JambaModel"), ("janus", "JanusModel"), @@ -569,7 +567,6 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("table-transformer", "TableTransformerModel"), ("tapas", "TapasModel"), ("telechat", "TeleChatModel"), - ("telechat3", "LlamaModel"), ("textnet", "TextNetModel"), ("time_series_transformer", "TimeSeriesTransformerModel"), ("timesfm", "TimesFmModel"), @@ -843,8 +840,6 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("hy_v4", "HYV4ForCausalLM"), ("hyperclovax", "HyperCLOVAXForCausalLM"), ("inkling_text", "InklingForCausalLM"), - ("internlm3", "LlamaForCausalLM"), - ("iquestcoder", "LlamaForCausalLM"), ("jais2", "Jais2ForCausalLM"), ("jamba", "JambaForCausalLM"), ("jetmoe", "JetMoeForCausalLM"), @@ -924,7 +919,6 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("stablelm", "StableLmForCausalLM"), ("starcoder2", "Starcoder2ForCausalLM"), ("telechat", "TeleChatForCausalLM"), - ("telechat3", "LlamaForCausalLM"), ("trocr", "TrOCRForCausalLM"), ("vaultgemma", "VaultGemmaForCausalLM"), ("whisper", "WhisperForCausalLM"), @@ -1520,8 +1514,6 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("hunyuan_v1_dense", "HunYuanDenseV1ForSequenceClassification"), ("hunyuan_v1_moe", "HunYuanMoEV1ForSequenceClassification"), ("ibert", "IBertForSequenceClassification"), - ("internlm3", "LlamaForSequenceClassification"), - ("iquestcoder", "LlamaForSequenceClassification"), ("jamba", "JambaForSequenceClassification"), ("jetmoe", "JetMoeForSequenceClassification"), ("jina_embeddings_v3", "JinaEmbeddingsV3ForSequenceClassification"), @@ -1587,7 +1579,6 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("t5gemma", "T5GemmaForSequenceClassification"), ("t5gemma2", "T5Gemma2ForSequenceClassification"), ("tapas", "TapasForSequenceClassification"), - ("telechat3", "LlamaForSequenceClassification"), ("ultrabert", "UltrabertForSequenceClassification"), ("ultravbert", "UltraVBertForSequenceClassification"), ("umt5", "UMT5ForSequenceClassification"), @@ -1632,8 +1623,6 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("gpt_neox", "GPTNeoXForQuestionAnswering"), ("gptj", "GPTJForQuestionAnswering"), ("ibert", "IBertForQuestionAnswering"), - ("internlm3", "LlamaForQuestionAnswering"), - ("iquestcoder", "LlamaForQuestionAnswering"), ("jina_embeddings_v3", "JinaEmbeddingsV3ForQuestionAnswering"), ("layoutlmv2", "LayoutLMv2ForQuestionAnswering"), ("layoutlmv3", "LayoutLMv3ForQuestionAnswering"), @@ -1677,7 +1666,6 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("splinter", "SplinterForQuestionAnswering"), ("squeezebert", "SqueezeBertForQuestionAnswering"), ("t5", "T5ForQuestionAnswering"), - ("telechat3", "LlamaForQuestionAnswering"), ("ultrabert", "UltrabertForQuestionAnswering"), ("umt5", "UMT5ForQuestionAnswering"), ("xlm", "XLMForQuestionAnsweringSimple"), @@ -1757,8 +1745,6 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("gte", "GteForTokenClassification"), ("helium", "HeliumForTokenClassification"), ("ibert", "IBertForTokenClassification"), - ("internlm3", "LlamaForTokenClassification"), - ("iquestcoder", "LlamaForTokenClassification"), ("jina_embeddings_v3", "JinaEmbeddingsV3ForTokenClassification"), ("layoutlm", "LayoutLMForTokenClassification"), ("layoutlmv2", "LayoutLMv2ForTokenClassification"), @@ -1809,7 +1795,6 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("t5", "T5ForTokenClassification"), ("t5gemma", "T5GemmaForTokenClassification"), ("t5gemma2", "T5Gemma2ForTokenClassification"), - ("telechat3", "LlamaForTokenClassification"), ("ultrabert", "UltrabertForTokenClassification"), ("ultravbert", "UltraVBertForTokenClassification"), ("umt5", "UMT5ForTokenClassification"), diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 26a08377bf88..1578c088d544 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -397,8 +397,6 @@ "hyperclovax_vlm", "hyperclovax_vision_v2", "internlm2", - "internlm3", - "iquestcoder", "jamba", "llava", "llava_next", @@ -424,7 +422,6 @@ "qwen4_exp", "step3_vl", "telechat", - "telechat3", "vipllava", "cohere_asr", "camembertv2-base", diff --git a/src/transformers/models/internlm3/__init__.py b/src/transformers/models/internlm3/__init__.py deleted file mode 100644 index 6951a55f338d..000000000000 --- a/src/transformers/models/internlm3/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from the modules in `src/transformers/models/internlm3`. -# Do NOT edit this file manually as any edits will be overwritten by auto-generation of the file. -# A module is picked up once it exports a name through `__all__`. -# Regenerate the file with: `python utils/check_inits.py --fix_and_overwrite` -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import TYPE_CHECKING - -from ...utils import _LazyModule -from ...utils.import_utils import define_import_structure - - -if TYPE_CHECKING: - from .configuration_internlm3 import * -else: - import sys - - _file = globals()["__file__"] - sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/src/transformers/models/internlm3/configuration_internlm3.py b/src/transformers/models/internlm3/configuration_internlm3.py deleted file mode 100644 index 2d0a4c4b7e5e..000000000000 --- a/src/transformers/models/internlm3/configuration_internlm3.py +++ /dev/null @@ -1,107 +0,0 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from src/transformers/models/internlm3/modular_internlm3.py. -# Do NOT edit this file manually as any edits will be overwritten by the generation of -# the file from the modular. If any change should be done, please apply the change to the -# modular_internlm3.py file directly. One of our CI enforces this. -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from huggingface_hub.dataclasses import strict - -from ...configuration_utils import PreTrainedConfig -from ...modeling_rope_utils import RopeParameters -from ...utils import auto_docstring - - -@auto_docstring(checkpoint="internlm/internlm3-8b-instruct") -@strict -class InternLM3Config(PreTrainedConfig): - r""" - ```python - >>> from transformers import AutoModelForCausalLM, InternLM3Config - - >>> # Initializing an InternLM3-8B style configuration - >>> configuration = InternLM3Config() - - >>> # Initializing a model (which uses the Llama implementation) from that configuration - >>> model = AutoModelForCausalLM.from_config(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "internlm3" - keys_to_ignore_at_inference = ["past_key_values"] - # Default tensor parallel plan for base model `InternLM3Model` - base_model_tp_plan = { - "layers.*.self_attn.q_proj": "colwise", - "layers.*.self_attn.k_proj": "colwise", - "layers.*.self_attn.v_proj": "colwise", - "layers.*.self_attn.o_proj": "rowwise", - "layers.*.mlp.gate_proj": "colwise", - "layers.*.mlp.up_proj": "colwise", - "layers.*.mlp.down_proj": "rowwise", - } - base_model_pp_plan = { - "embed_tokens": (["input_ids"], ["inputs_embeds"]), - "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), - "norm": (["hidden_states"], ["hidden_states"]), - } - - vocab_size: int = 128512 - hidden_size: int = 4096 - intermediate_size: int = 10240 - num_hidden_layers: int = 48 - num_attention_heads: int = 32 - num_key_value_heads: int | None = 2 - hidden_act: str = "silu" - max_position_embeddings: int = 32768 - initializer_range: float = 0.02 - rms_norm_eps: float = 1e-5 - use_cache: bool = True - pad_token_id: int | None = 2 - bos_token_id: int | None = 1 - eos_token_id: int | list[int] | None = 2 - tie_word_embeddings: bool = False - rope_parameters: RopeParameters | dict | None = None - attention_bias: bool = False - attention_dropout: float | int = 0.0 - mlp_bias: bool = False - head_dim: int | None = 128 - attribute_map = {"qkv_bias": "attention_bias", "bias": "mlp_bias"} - - def __post_init__(self, **kwargs): - if self.head_dim is None: - self.head_dim = self.hidden_size // self.num_attention_heads - if self.num_key_value_heads is None: - self.num_key_value_heads = self.num_attention_heads - - super().__post_init__(**kwargs) - - def validate_architecture(self): - """Part of `@strict`-powered validation. Validates the architecture of the config.""" - if self.hidden_size % self.num_attention_heads != 0: - raise ValueError( - f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention " - f"heads ({self.num_attention_heads})." - ) - if self.attention_bias != self.mlp_bias: - raise ValueError( - "InternLM3 with `qkv_bias` different from `bias` is not supported, because the Llama implementation " - "uses the same bias setting for the query, key, value and output projections." - ) - - -__all__ = ["InternLM3Config"] diff --git a/src/transformers/models/internlm3/modular_internlm3.py b/src/transformers/models/internlm3/modular_internlm3.py deleted file mode 100644 index d0609ae5bdee..000000000000 --- a/src/transformers/models/internlm3/modular_internlm3.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2026 Shanghai AI Laboratory and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from huggingface_hub.dataclasses import strict - -from ...modeling_rope_utils import RopeParameters -from ...utils import auto_docstring -from ..llama.configuration_llama import LlamaConfig - - -@auto_docstring(checkpoint="internlm/internlm3-8b-instruct") -@strict -class InternLM3Config(LlamaConfig): - r""" - ```python - >>> from transformers import AutoModelForCausalLM, InternLM3Config - - >>> # Initializing an InternLM3-8B style configuration - >>> configuration = InternLM3Config() - - >>> # Initializing a model (which uses the Llama implementation) from that configuration - >>> model = AutoModelForCausalLM.from_config(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "internlm3" - attribute_map = {"qkv_bias": "attention_bias", "bias": "mlp_bias"} - - vocab_size: int = 128512 - hidden_size: int = 4096 - intermediate_size: int = 10240 - num_hidden_layers: int = 48 - num_attention_heads: int = 32 - num_key_value_heads: int | None = 2 - hidden_act: str = "silu" - max_position_embeddings: int = 32768 - initializer_range: float = 0.02 - rms_norm_eps: float = 1e-5 - use_cache: bool = True - pad_token_id: int | None = 2 - bos_token_id: int | None = 1 - eos_token_id: int | list[int] | None = 2 - tie_word_embeddings: bool = False - rope_parameters: RopeParameters | dict | None = None - attention_bias: bool = False - attention_dropout: float | int = 0.0 - mlp_bias: bool = False - head_dim: int | None = 128 - - pretraining_tp = AttributeError() - - def validate_architecture(self): - """Part of `@strict`-powered validation. Validates the architecture of the config.""" - super().validate_architecture() - if self.attention_bias != self.mlp_bias: - raise ValueError( - "InternLM3 with `qkv_bias` different from `bias` is not supported, because the Llama implementation " - "uses the same bias setting for the query, key, value and output projections." - ) - - -__all__ = ["InternLM3Config"] diff --git a/src/transformers/models/iquestcoder/__init__.py b/src/transformers/models/iquestcoder/__init__.py deleted file mode 100644 index 07332cb6f1ff..000000000000 --- a/src/transformers/models/iquestcoder/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from the modules in `src/transformers/models/iquestcoder`. -# Do NOT edit this file manually as any edits will be overwritten by auto-generation of the file. -# A module is picked up once it exports a name through `__all__`. -# Regenerate the file with: `python utils/check_inits.py --fix_and_overwrite` -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import TYPE_CHECKING - -from ...utils import _LazyModule -from ...utils.import_utils import define_import_structure - - -if TYPE_CHECKING: - from .configuration_iquestcoder import * -else: - import sys - - _file = globals()["__file__"] - sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/src/transformers/models/iquestcoder/configuration_iquestcoder.py b/src/transformers/models/iquestcoder/configuration_iquestcoder.py deleted file mode 100644 index 18549aa0ee43..000000000000 --- a/src/transformers/models/iquestcoder/configuration_iquestcoder.py +++ /dev/null @@ -1,106 +0,0 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from src/transformers/models/iquestcoder/modular_iquestcoder.py. -# Do NOT edit this file manually as any edits will be overwritten by the generation of -# the file from the modular. If any change should be done, please apply the change to the -# modular_iquestcoder.py file directly. One of our CI enforces this. -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from huggingface_hub.dataclasses import strict - -from ...configuration_utils import PreTrainedConfig -from ...modeling_rope_utils import RopeParameters -from ...utils import auto_docstring - - -@auto_docstring(checkpoint="IQuestLab/IQuest-Coder-V1-40B-Instruct") -@strict -class IQuestCoderConfig(PreTrainedConfig): - r""" - ```python - >>> from transformers import AutoModelForCausalLM, IQuestCoderConfig - - >>> # Initializing an IQuest-Coder-V1-40B style configuration - >>> configuration = IQuestCoderConfig() - - >>> # Initializing a model (which uses the Llama implementation) from that configuration - >>> model = AutoModelForCausalLM.from_config(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "iquestcoder" - keys_to_ignore_at_inference = ["past_key_values"] - # Default tensor parallel plan for base model `IQuestCoderModel` - base_model_tp_plan = { - "layers.*.self_attn.q_proj": "colwise", - "layers.*.self_attn.k_proj": "colwise", - "layers.*.self_attn.v_proj": "colwise", - "layers.*.self_attn.o_proj": "rowwise", - "layers.*.mlp.gate_proj": "colwise", - "layers.*.mlp.up_proj": "colwise", - "layers.*.mlp.down_proj": "rowwise", - } - base_model_pp_plan = { - "embed_tokens": (["input_ids"], ["inputs_embeds"]), - "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), - "norm": (["hidden_states"], ["hidden_states"]), - } - - vocab_size: int = 76800 - hidden_size: int = 5120 - intermediate_size: int = 27648 - num_hidden_layers: int = 80 - num_attention_heads: int = 40 - num_key_value_heads: int | None = 8 - hidden_act: str = "silu" - max_position_embeddings: int = 131072 - initializer_range: float = 0.02 - rms_norm_eps: float = 1e-5 - use_cache: bool = True - pad_token_id: int | None = None - bos_token_id: int | None = 1 - eos_token_id: int | list[int] | None = 2 - tie_word_embeddings: bool = False - rope_parameters: RopeParameters | dict | None = None - attention_bias: bool = False - attention_dropout: float | int = 0.0 - mlp_bias: bool = False - head_dim: int | None = 128 - - def __post_init__(self, **kwargs): - if self.head_dim is None: - self.head_dim = self.hidden_size // self.num_attention_heads - if self.num_key_value_heads is None: - self.num_key_value_heads = self.num_attention_heads - - super().__post_init__(**kwargs) - - def validate_architecture(self): - """Part of `@strict`-powered validation. Validates the architecture of the config.""" - if self.hidden_size % self.num_attention_heads != 0: - raise ValueError( - f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention " - f"heads ({self.num_attention_heads})." - ) - # The original code supports these options, but no released checkpoint enables them - if getattr(self, "clip_qkv", None) is not None: - raise ValueError("IQuestCoder does not support `clip_qkv`.") - if getattr(self, "use_sliding_window", False): - raise ValueError("IQuestCoder does not support `use_sliding_window`.") - - -__all__ = ["IQuestCoderConfig"] diff --git a/src/transformers/models/iquestcoder/modular_iquestcoder.py b/src/transformers/models/iquestcoder/modular_iquestcoder.py deleted file mode 100644 index cf3bb03e5d4e..000000000000 --- a/src/transformers/models/iquestcoder/modular_iquestcoder.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2026 IQuest Lab and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from huggingface_hub.dataclasses import strict - -from ...modeling_rope_utils import RopeParameters -from ...utils import auto_docstring -from ..llama.configuration_llama import LlamaConfig - - -@auto_docstring(checkpoint="IQuestLab/IQuest-Coder-V1-40B-Instruct") -@strict -class IQuestCoderConfig(LlamaConfig): - r""" - ```python - >>> from transformers import AutoModelForCausalLM, IQuestCoderConfig - - >>> # Initializing an IQuest-Coder-V1-40B style configuration - >>> configuration = IQuestCoderConfig() - - >>> # Initializing a model (which uses the Llama implementation) from that configuration - >>> model = AutoModelForCausalLM.from_config(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "iquestcoder" - - vocab_size: int = 76800 - hidden_size: int = 5120 - intermediate_size: int = 27648 - num_hidden_layers: int = 80 - num_attention_heads: int = 40 - num_key_value_heads: int | None = 8 - hidden_act: str = "silu" - max_position_embeddings: int = 131072 - initializer_range: float = 0.02 - rms_norm_eps: float = 1e-5 - use_cache: bool = True - pad_token_id: int | None = None - bos_token_id: int | None = 1 - eos_token_id: int | list[int] | None = 2 - tie_word_embeddings: bool = False - rope_parameters: RopeParameters | dict | None = None - attention_bias: bool = False - attention_dropout: float | int = 0.0 - mlp_bias: bool = False - head_dim: int | None = 128 - - pretraining_tp = AttributeError() - - def validate_architecture(self): - """Part of `@strict`-powered validation. Validates the architecture of the config.""" - super().validate_architecture() - # The original code supports these options, but no released checkpoint enables them - if getattr(self, "clip_qkv", None) is not None: - raise ValueError("IQuestCoder does not support `clip_qkv`.") - if getattr(self, "use_sliding_window", False): - raise ValueError("IQuestCoder does not support `use_sliding_window`.") - - -__all__ = ["IQuestCoderConfig"] diff --git a/src/transformers/models/telechat/configuration_telechat.py b/src/transformers/models/telechat/configuration_telechat.py index 0956e6a271ed..72bea7afa8fc 100644 --- a/src/transformers/models/telechat/configuration_telechat.py +++ b/src/transformers/models/telechat/configuration_telechat.py @@ -30,11 +30,11 @@ @strict class TeleChatConfig(PreTrainedConfig): r""" - TeleChat applies its own stepwise NTK scaling of RoPE, described by `rope_parameters` with `"rope_type": "telechat_ntk"`: - - - `original_max_position_embeddings` (`training_seqlen` in the original configs, defaults to 8192): the sequence - length the model was trained with. Beyond it, the RoPE base and the cos/sin are rescaled. - - `base_seqlen` (defaults to 4096): the sequence length the NTK scaling factor of the base is computed relative to. + rope_parameters (`RopeParameters`, *optional*): + TeleChat always applies its own stepwise NTK scaling, `"rope_type": "telechat_ntk"`, with `rope_theta`, + `original_max_position_embeddings` (`training_seqlen` in the original configs, defaults to 8192), the length + beyond which the RoPE base and the cos/sin are rescaled, and `base_seqlen` (defaults to 4096), the length the + scaling factor of the base is computed relative to. ```python >>> from transformers import TeleChatModel, TeleChatConfig diff --git a/src/transformers/models/telechat/modular_telechat.py b/src/transformers/models/telechat/modular_telechat.py index 215e99f9a954..5a570fdd702e 100644 --- a/src/transformers/models/telechat/modular_telechat.py +++ b/src/transformers/models/telechat/modular_telechat.py @@ -35,11 +35,11 @@ @strict class TeleChatConfig(LlamaConfig): r""" - TeleChat applies its own stepwise NTK scaling of RoPE, described by `rope_parameters` with `"rope_type": "telechat_ntk"`: - - - `original_max_position_embeddings` (`training_seqlen` in the original configs, defaults to 8192): the sequence - length the model was trained with. Beyond it, the RoPE base and the cos/sin are rescaled. - - `base_seqlen` (defaults to 4096): the sequence length the NTK scaling factor of the base is computed relative to. + rope_parameters (`RopeParameters`, *optional*): + TeleChat always applies its own stepwise NTK scaling, `"rope_type": "telechat_ntk"`, with `rope_theta`, + `original_max_position_embeddings` (`training_seqlen` in the original configs, defaults to 8192), the length + beyond which the RoPE base and the cos/sin are rescaled, and `base_seqlen` (defaults to 4096), the length the + scaling factor of the base is computed relative to. ```python >>> from transformers import TeleChatModel, TeleChatConfig diff --git a/src/transformers/models/telechat3/__init__.py b/src/transformers/models/telechat3/__init__.py deleted file mode 100644 index fa02dee59e82..000000000000 --- a/src/transformers/models/telechat3/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from the modules in `src/transformers/models/telechat3`. -# Do NOT edit this file manually as any edits will be overwritten by auto-generation of the file. -# A module is picked up once it exports a name through `__all__`. -# Regenerate the file with: `python utils/check_inits.py --fix_and_overwrite` -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import TYPE_CHECKING - -from ...utils import _LazyModule -from ...utils.import_utils import define_import_structure - - -if TYPE_CHECKING: - from .configuration_telechat3 import * -else: - import sys - - _file = globals()["__file__"] - sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/src/transformers/models/telechat3/configuration_telechat3.py b/src/transformers/models/telechat3/configuration_telechat3.py deleted file mode 100644 index 30820617b3d2..000000000000 --- a/src/transformers/models/telechat3/configuration_telechat3.py +++ /dev/null @@ -1,138 +0,0 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from src/transformers/models/telechat3/modular_telechat3.py. -# Do NOT edit this file manually as any edits will be overwritten by the generation of -# the file from the modular. If any change should be done, please apply the change to the -# modular_telechat3.py file directly. One of our CI enforces this. -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import math - -from huggingface_hub.dataclasses import strict - -from ...configuration_utils import PreTrainedConfig -from ...modeling_rope_utils import RopeParameters -from ...utils import auto_docstring, logging - - -logger = logging.get_logger(__name__) - - -@auto_docstring(checkpoint="Tele-AI/TeleChat3-36B-Thinking") -@strict -class TeleChat3Config(PreTrainedConfig): - r""" - ```python - >>> from transformers import AutoModelForCausalLM, TeleChat3Config - - >>> # Initializing a TeleChat3-36B style configuration - >>> configuration = TeleChat3Config() - - >>> # Initializing a model (which uses the Llama implementation) from that configuration - >>> model = AutoModelForCausalLM.from_config(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "telechat3" - keys_to_ignore_at_inference = ["past_key_values"] - # Default tensor parallel plan for base model `TeleChat3Model` - base_model_tp_plan = { - "layers.*.self_attn.q_proj": "colwise", - "layers.*.self_attn.k_proj": "colwise", - "layers.*.self_attn.v_proj": "colwise", - "layers.*.self_attn.o_proj": "rowwise", - "layers.*.mlp.gate_proj": "colwise", - "layers.*.mlp.up_proj": "colwise", - "layers.*.mlp.down_proj": "rowwise", - } - base_model_pp_plan = { - "embed_tokens": (["input_ids"], ["inputs_embeds"]), - "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), - "norm": (["hidden_states"], ["hidden_states"]), - } - - vocab_size: int = 131072 - hidden_size: int = 6144 - intermediate_size: int = 24576 - num_hidden_layers: int = 64 - num_attention_heads: int = 48 - num_key_value_heads: int | None = 8 - hidden_act: str = "silu" - max_position_embeddings: int = 32768 - initializer_range: float = 0.0048 - rms_norm_eps: float = 1e-5 - use_cache: bool = True - pad_token_id: int | None = 3 - bos_token_id: int | None = 1 - eos_token_id: int | list[int] | None = 2 - tie_word_embeddings: bool = False - rope_parameters: RopeParameters | dict | None = None - attention_bias: bool = False - attention_dropout: float | int = 0.0 - mlp_bias: bool = False - head_dim: int | None = 128 - - def __post_init__(self, **kwargs): - if self.head_dim is None: - self.head_dim = self.hidden_size // self.num_attention_heads - if self.num_key_value_heads is None: - self.num_key_value_heads = self.num_attention_heads - - super().__post_init__(**kwargs) - - def validate_architecture(self): - """Part of `@strict`-powered validation. Validates the architecture of the config.""" - if self.hidden_size % self.num_attention_heads != 0: - raise ValueError( - f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention " - f"heads ({self.num_attention_heads})." - ) - - def convert_rope_params_to_dict(self, **kwargs): - # The original checkpoints use a custom `"telechat3-yarn"` RoPE type, which is YaRN with 0.07 instead of 0.1 - # as the coefficient of its default attention factor. Convert it to plain YaRN with an explicit attention factor - rope_parameters = kwargs.get("rope_scaling") or self.rope_parameters - if rope_parameters is not None and "telechat3-yarn" in ( - rope_parameters.get("rope_type"), - rope_parameters.get("type"), - ): - rope_parameters = {k: v for k, v in rope_parameters.items() if k != "type"} - rope_parameters["rope_type"] = "yarn" - if "original_max_position_embeddings" in rope_parameters: - # The original code derives the scaling factor from the context lengths, ignoring `factor` - rope_parameters["factor"] = ( - self.max_position_embeddings / rope_parameters["original_max_position_embeddings"] - ) - if rope_parameters.get("attention_factor") is None: - factor = rope_parameters["factor"] - mscale = rope_parameters.get("mscale") - mscale_all_dim = rope_parameters.get("mscale_all_dim") - - def get_mscale(scale, mscale=1.0): - return 1.0 if scale <= 1 else 0.07 * mscale * math.log(scale) + 1.0 - - if mscale and mscale_all_dim: - attention_factor = get_mscale(factor, mscale) / get_mscale(factor, mscale_all_dim) - else: - attention_factor = get_mscale(factor) - rope_parameters["attention_factor"] = attention_factor - logger.info("Converted rope_type 'telechat3-yarn' to 'yarn' with an explicit `attention_factor`.") - kwargs["rope_scaling"] = rope_parameters - return super().convert_rope_params_to_dict(**kwargs) - - -__all__ = ["TeleChat3Config"] diff --git a/src/transformers/models/telechat3/modular_telechat3.py b/src/transformers/models/telechat3/modular_telechat3.py deleted file mode 100644 index 614eaa9c8aaa..000000000000 --- a/src/transformers/models/telechat3/modular_telechat3.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2026 China Telecom Artificial Intelligence Technology Co., Ltd. and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import math - -from huggingface_hub.dataclasses import strict - -from ...configuration_utils import PreTrainedConfig -from ...modeling_rope_utils import RopeParameters -from ...utils import auto_docstring, logging -from ..llama.configuration_llama import LlamaConfig - - -logger = logging.get_logger(__name__) - - -@auto_docstring(checkpoint="Tele-AI/TeleChat3-36B-Thinking") -@strict -class TeleChat3Config(LlamaConfig): - r""" - ```python - >>> from transformers import AutoModelForCausalLM, TeleChat3Config - - >>> # Initializing a TeleChat3-36B style configuration - >>> configuration = TeleChat3Config() - - >>> # Initializing a model (which uses the Llama implementation) from that configuration - >>> model = AutoModelForCausalLM.from_config(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "telechat3" - - vocab_size: int = 131072 - hidden_size: int = 6144 - intermediate_size: int = 24576 - num_hidden_layers: int = 64 - num_attention_heads: int = 48 - num_key_value_heads: int | None = 8 - hidden_act: str = "silu" - max_position_embeddings: int = 32768 - initializer_range: float = 0.0048 - rms_norm_eps: float = 1e-5 - use_cache: bool = True - pad_token_id: int | None = 3 - bos_token_id: int | None = 1 - eos_token_id: int | list[int] | None = 2 - tie_word_embeddings: bool = False - rope_parameters: RopeParameters | dict | None = None - attention_bias: bool = False - attention_dropout: float | int = 0.0 - mlp_bias: bool = False - head_dim: int | None = 128 - - pretraining_tp = AttributeError() - - def convert_rope_params_to_dict(self, **kwargs): - # The original checkpoints use a custom `"telechat3-yarn"` RoPE type, which is YaRN with 0.07 instead of 0.1 - # as the coefficient of its default attention factor. Convert it to plain YaRN with an explicit attention factor - rope_parameters = kwargs.get("rope_scaling") or self.rope_parameters - if rope_parameters is not None and "telechat3-yarn" in ( - rope_parameters.get("rope_type"), - rope_parameters.get("type"), - ): - rope_parameters = {k: v for k, v in rope_parameters.items() if k != "type"} - rope_parameters["rope_type"] = "yarn" - if "original_max_position_embeddings" in rope_parameters: - # The original code derives the scaling factor from the context lengths, ignoring `factor` - rope_parameters["factor"] = ( - self.max_position_embeddings / rope_parameters["original_max_position_embeddings"] - ) - if rope_parameters.get("attention_factor") is None: - factor = rope_parameters["factor"] - mscale = rope_parameters.get("mscale") - mscale_all_dim = rope_parameters.get("mscale_all_dim") - - def get_mscale(scale, mscale=1.0): - return 1.0 if scale <= 1 else 0.07 * mscale * math.log(scale) + 1.0 - - if mscale and mscale_all_dim: - attention_factor = get_mscale(factor, mscale) / get_mscale(factor, mscale_all_dim) - else: - attention_factor = get_mscale(factor) - rope_parameters["attention_factor"] = attention_factor - logger.info("Converted rope_type 'telechat3-yarn' to 'yarn' with an explicit `attention_factor`.") - kwargs["rope_scaling"] = rope_parameters - return PreTrainedConfig.convert_rope_params_to_dict(self, **kwargs) - - -__all__ = ["TeleChat3Config"] diff --git a/utils/check_config_attributes.py b/utils/check_config_attributes.py index b6e7af8daccd..5dafe6bc8b97 100644 --- a/utils/check_config_attributes.py +++ b/utils/check_config_attributes.py @@ -321,10 +321,6 @@ "PPChart2TableConfig": True, "PPChart2TableVisionConfig": True, "GlmgaConfig": ["vision_config"], - # Use the Llama implementation, which the auto mappings point to - "InternLM3Config": True, - "IQuestCoderConfig": True, - "TeleChat3Config": True, "Sapiens2Config": [ "num_first_full_attention_layers", # builder attr consumed in __post_init__ to compute num_key_value_heads_per_layer "num_key_value_attention_heads", # builder attr consumed in __post_init__ to compute num_key_value_heads_per_layer From df06e02dcf77e66e27e46125aa0b65064717c0f9 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:48:10 +0000 Subject: [PATCH 15/17] Run the tensor parallel tests on TeleChat Its fused key/value conversion runs on each rank's shard, which holds whole per-head key/value pairs, so the TP tests check that it shards correctly. Co-Authored-By: Claude Opus 5.5 --- src/transformers/core_model_loading.py | 3 +++ tests/test_tensor_parallel_mixin.py | 1 + 2 files changed, 4 insertions(+) diff --git a/src/transformers/core_model_loading.py b/src/transformers/core_model_loading.py index 5e3987fa4a6d..e0b5c58e80b7 100644 --- a/src/transformers/core_model_loading.py +++ b/src/transformers/core_model_loading.py @@ -257,6 +257,9 @@ class TeleChatInterleaveKeyValue(ConversionOps): TeleChat fuses its key and value projections per head, i.e. `[k_0, v_0, k_1, v_1, ...]` along dim 0 with heads of `head_dim`. This regroups them as `[k_0, k_1, ..., v_0, v_1, ...]` so that `Chunk` can split keys from values, or does the opposite when `inverse` is set. + + TP-safe: with tensor parallelism it receives a contiguous shard of whole `[k_h, v_h]` pairs, which it regroups into + that rank's colwise shards of `k_proj` and `v_proj`. """ def __init__(self, inverse: bool = False): diff --git a/tests/test_tensor_parallel_mixin.py b/tests/test_tensor_parallel_mixin.py index 95e0b623c3b4..35a91ffcd4d6 100644 --- a/tests/test_tensor_parallel_mixin.py +++ b/tests/test_tensor_parallel_mixin.py @@ -54,6 +54,7 @@ "qwen3", "qwen2", "granite_swa", + "telechat", # MoE "qwen4_exp_text", "qwen3_moe", From 011bfbdf26bfe3567b9bf65e0f3fc072481518dd Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:53:59 +0000 Subject: [PATCH 16/17] Load TeleChat2-3B with the default dtype in the integration tests Its config has no dtype, but its weights are stored in bf16, so `dtype="auto"` loads them in bf16. The expected logits are re-recorded on B200; the generated text is unchanged. Co-Authored-By: Claude Opus 5.5 --- tests/models/telechat/test_modeling_telechat.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/models/telechat/test_modeling_telechat.py b/tests/models/telechat/test_modeling_telechat.py index 4270c828f480..42ce8b0743e6 100644 --- a/tests/models/telechat/test_modeling_telechat.py +++ b/tests/models/telechat/test_modeling_telechat.py @@ -88,26 +88,23 @@ def test_ntk_scaling_matches_original(self, base_seqlen, training_seqlen): @slow @require_torch_accelerator class TeleChatIntegrationTest(MemoryCleanupMixin, unittest.TestCase): - # The checkpoints' configs don't set a dtype, so `dtype="auto"` would load them in fp32. The original code uses fp16 - dtype = torch.float16 if is_torch_available() else None - def test_model_3b_logits(self): - model = TeleChatForCausalLM.from_pretrained("Tele-AI/TeleChat2-3B", device_map="auto", dtype=self.dtype) + model = TeleChatForCausalLM.from_pretrained("Tele-AI/TeleChat2-3B", device_map="auto") input_ids = torch.tensor([[1, 1404, 1435, 1272, 1363, 1446, 334, 1279, 1306, 395]], device=model.device) logits = model(input_ids).logits.float().cpu() EXPECTED_MEAN = Expectations({ - ("cuda", 10): torch.tensor([-4.919, -4.2666, -1.901, -2.9477, -1.5264, -2.575, -0.5755, -1.9635, -2.781, -1.7785]), + ("cuda", 10): torch.tensor([-4.9348, -4.2666, -1.8802, -2.955, -1.5048, -2.6213, -0.5889, -1.9853, -2.7734, -1.7984]), }) # fmt: skip EXPECTED_SLICE = Expectations({ - ("cuda", 10): torch.tensor([-9.1016, -9.8984, -1.8105, -9.1875, -9.1797, -9.1797, -9.1797, -9.1797, -9.1797, -6.3086]), + ("cuda", 10): torch.tensor([-9.125, -9.9375, -1.7422, -9.25, -9.25, -9.25, -9.25, -9.25, -9.25, -6.2812]), }) # fmt: skip torch.testing.assert_close(logits.mean(-1)[0], EXPECTED_MEAN.get_expectation(), rtol=1e-2, atol=1e-2) torch.testing.assert_close(logits[0, 0, :10], EXPECTED_SLICE.get_expectation(), rtol=1e-2, atol=1e-2) def test_model_3b_generation(self): tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat2-3B") - model = TeleChatForCausalLM.from_pretrained("Tele-AI/TeleChat2-3B", device_map="auto", dtype=self.dtype) + model = TeleChatForCausalLM.from_pretrained("Tele-AI/TeleChat2-3B", device_map="auto") inputs = tokenizer("The capital of France is", return_tensors="pt").to(model.device) output = model.generate(**inputs, max_new_tokens=16, do_sample=False) text = tokenizer.decode(output[0], skip_special_tokens=True) From 2dda807a0e0e50e71439b0a80edeecacecbcfc97 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:15:26 +0000 Subject: [PATCH 17/17] Pop TeleChat's original RoPE keys up front Python evaluates `dict.get`'s default before the lookup, so the pops nested in the defaults already ran and the trailing pops were dead. Pop the original keys once as defaults and build `rope_parameters` from them. Co-Authored-By: Claude Opus 5.5 --- .../models/telechat/configuration_telechat.py | 17 ++++++++--------- .../models/telechat/modular_telechat.py | 17 ++++++++--------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/transformers/models/telechat/configuration_telechat.py b/src/transformers/models/telechat/configuration_telechat.py index 72bea7afa8fc..5cc808adda5d 100644 --- a/src/transformers/models/telechat/configuration_telechat.py +++ b/src/transformers/models/telechat/configuration_telechat.py @@ -129,17 +129,16 @@ def validate_architecture(self): def convert_rope_params_to_dict(self, **kwargs): # The original code ignores `rope_scaling` and always applies its own stepwise NTK scaling instead kwargs.pop("rope_scaling", None) + # Defaults from the original config keys, for configs whose `rope_parameters` don't set them yet + defaults = { + "rope_theta": kwargs.pop("rope_theta", self.default_theta), + "original_max_position_embeddings": kwargs.pop("training_seqlen", 8192), + "base_seqlen": kwargs.pop("base_seqlen", 4096), + } rope_parameters = self.rope_parameters or {} - self.rope_parameters = { - "rope_type": "telechat_ntk", - "rope_theta": rope_parameters.get("rope_theta", kwargs.pop("rope_theta", self.default_theta)), - "original_max_position_embeddings": rope_parameters.get( - "original_max_position_embeddings", kwargs.pop("training_seqlen", 8192) - ), - "base_seqlen": rope_parameters.get("base_seqlen", kwargs.pop("base_seqlen", 4096)), + self.rope_parameters = {"rope_type": "telechat_ntk"} | { + k: rope_parameters.get(k, v) for k, v in defaults.items() } - kwargs.pop("training_seqlen", None) - kwargs.pop("base_seqlen", None) return super().convert_rope_params_to_dict(**kwargs) def _validate_telechat_ntk_rope_parameters(self, rope_parameters: dict, ignore_keys: set[str] | None = None): diff --git a/src/transformers/models/telechat/modular_telechat.py b/src/transformers/models/telechat/modular_telechat.py index 5a570fdd702e..665a2c5cdaa0 100644 --- a/src/transformers/models/telechat/modular_telechat.py +++ b/src/transformers/models/telechat/modular_telechat.py @@ -100,17 +100,16 @@ def __post_init__(self, **kwargs): def convert_rope_params_to_dict(self, **kwargs): # The original code ignores `rope_scaling` and always applies its own stepwise NTK scaling instead kwargs.pop("rope_scaling", None) + # Defaults from the original config keys, for configs whose `rope_parameters` don't set them yet + defaults = { + "rope_theta": kwargs.pop("rope_theta", self.default_theta), + "original_max_position_embeddings": kwargs.pop("training_seqlen", 8192), + "base_seqlen": kwargs.pop("base_seqlen", 4096), + } rope_parameters = self.rope_parameters or {} - self.rope_parameters = { - "rope_type": "telechat_ntk", - "rope_theta": rope_parameters.get("rope_theta", kwargs.pop("rope_theta", self.default_theta)), - "original_max_position_embeddings": rope_parameters.get( - "original_max_position_embeddings", kwargs.pop("training_seqlen", 8192) - ), - "base_seqlen": rope_parameters.get("base_seqlen", kwargs.pop("base_seqlen", 4096)), + self.rope_parameters = {"rope_type": "telechat_ntk"} | { + k: rope_parameters.get(k, v) for k, v in defaults.items() } - kwargs.pop("training_seqlen", None) - kwargs.pop("base_seqlen", None) return PreTrainedConfig.convert_rope_params_to_dict(self, **kwargs) def _validate_telechat_ntk_rope_parameters(self, rope_parameters: dict, ignore_keys: set[str] | None = None):