Add TeleChat

.gitattributes

@@ -568,6 +568,8 @@ src/transformers/models/t5gemma/configuration_*.py linguist-generated=true
 src/transformers/models/t5gemma/modeling_*.py linguist-generated=true
 src/transformers/models/t5gemma2/configuration_*.py linguist-generated=true
 src/transformers/models/t5gemma2/modeling_*.py linguist-generated=true
+src/transformers/models/telechat/configuration_*.py linguist-generated=true
+src/transformers/models/telechat/modeling_*.py linguist-generated=true
 src/transformers/models/timesfm/modeling_*.py linguist-generated=true
 src/transformers/models/timesfm2_5/configuration_*.py linguist-generated=true
 src/transformers/models/timesfm2_5/modeling_*.py linguist-generated=true

docs/source/en/_toctree.yml

@@ -955,6 +955,8 @@
         title: T5Gemma2
       - local: model_doc/t5v1.1
         title: T5v1.1
+      - local: model_doc/telechat
+        title: TeleChat
       - local: model_doc/ul2
         title: UL2
       - local: model_doc/ultrabert

docs/source/en/model_doc/telechat.md

@@ -0,0 +1,63 @@
+<!--Copyright 2026 The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+⚠️ Note that this file is in Markdown but contains specific syntax for our doc-builder (similar to MDX) that may not be
+rendered properly in your Markdown viewer.
+
+-->
+*This model was contributed to Hugging Face Transformers on 2026-11-10.*
+<div style="float: right;">
+    <div class="flex flex-wrap space-x-1">
+        <img alt="FlashAttention" src="https://img.shields.io/badge/%E2%9A%A1%EF%B8%8E%20FlashAttention-eae0c8?style=flat">
+        <img alt="SDPA" src="https://img.shields.io/badge/SDPA-DE3412?style=flat&logo=pytorch&logoColor=white">
+    </div>
+</div>
+
+# TeleChat
+
+[TeleChat](https://huggingface.co/Tele-AI) is a family of language models developed by [China Telecom Artificial Intelligence Technology](https://huggingface.co/Tele-AI), covering TeleChat (1B to 12B), TeleChat2 (3B to 115B) and TeleChat2.5 (35B and 115B). TeleChat3 is a separate model.
+
+Its architecture is the same as [Llama](./llama), except that:
+
+- only the attention output projection and the down projection of the MLP have biases;
+- once the sequence grows beyond the training length (`original_max_position_embeddings` in `rope_parameters`), RoPE is rescaled with a stepwise NTK scaling of its base and the cos/sin are scaled by `0.1 * ln(seq_len / original_max_position_embeddings) + 1`.
+
+> [!NOTE]
+> TeleChat-52B uses a different architecture and isn't supported by this model.
+
+The example below demonstrates how to generate text with the [`AutoModelForCausalLM`] class.
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+
+tokenizer = AutoTokenizer.from_pretrained("Tele-AI/TeleChat2-3B")
+model = AutoModelForCausalLM.from_pretrained("Tele-AI/TeleChat2-3B", device_map="auto")
+
+messages = [{"role": "user", "content": "Explain the Pythagorean theorem in one sentence."}]
+inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True)
+output = model.generate(**inputs.to(model.device), max_new_tokens=64)
+print(tokenizer.decode(output[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True))
+```
+
+## TeleChatConfig
+
+[[autodoc]] TeleChatConfig
+
+## TeleChatModel
+
+[[autodoc]] TeleChatModel
+    - forward
+
+## TeleChatForCausalLM
+
+[[autodoc]] TeleChatForCausalLM
+    - forward

src/transformers/conversion_mapping.py

@@ -28,6 +28,7 @@
     PermuteForRope,
     PrefixChange,
     Split,
+    TeleChatInterleaveKeyValue,
     Transpose,
     WeightConverter,
     WeightRenaming,
@@ -1633,6 +1634,22 @@ def _build_checkpoint_conversion_mapping():
                 target_patterns="LayerNorm.bias",
             ),
         ],
+        "telechat": [
+            # Targets are escaped and guarded so that, reversed for saving, they only match whole names. `(?<![^.])` matches
+            # at the start of a key or after a dot, so these also apply to base-model-only checkpoints (Tele-AI/telechat-7B)
+            WeightRenaming(r"^transformer\.", r"model\."),
+            WeightRenaming(r"(?<![^.])h\.", r"(?<![^.])layers\."),
+            WeightRenaming(r"(?<![^.])word_embeddings\.", r"(?<![^.])embed_tokens\."),
+            WeightRenaming(r"(?<![^.])ln_f\.", r"(?<![^.])norm\."),
+            WeightRenaming(r"self_attention\.query\.", r"self_attn\.q_proj\."),
+            # Keys and values are fused per head, i.e. `[k_0, v_0, k_1, v_1, ...]`
+            WeightConverter(
+                "self_attention.key_value.weight",
+                ["self_attn.k_proj.weight", "self_attn.v_proj.weight"],
+                operations=[TeleChatInterleaveKeyValue(), Chunk(dim=0)],
+            ),
+            WeightRenaming(r"self_attention\.dense\.", r"self_attn\.o_proj\."),
+        ],
         "sapiens2": [
             WeightRenaming(r"^cls_token", r"embeddings.cls_token"),
             WeightRenaming(r"^storage_tokens", r"embeddings.register_tokens"),

src/transformers/core_model_loading.py

@@ -252,6 +252,36 @@ def reverse_op(self) -> ConversionOps:
         return Interleave(self.dim, inverse=not self.inverse)
 
 
+class TeleChatInterleaveKeyValue(ConversionOps):
+    """
+    TeleChat fuses its key and value projections per head, i.e. `[k_0, v_0, k_1, v_1, ...]` along dim 0 with heads of
+    `head_dim`. This regroups them as `[k_0, k_1, ..., v_0, v_1, ...]` so that `Chunk` can split keys from values, or
+    does the opposite when `inverse` is set.
+
+    TP-safe: with tensor parallelism it receives a contiguous shard of whole `[k_h, v_h]` pairs, which it regroups into
+    that rank's colwise shards of `k_proj` and `v_proj`.
+    """
+
+    def __init__(self, inverse: bool = False):
+        self.inverse = inverse
+
+    @torch.no_grad
+    def convert(self, input_dict, source_patterns, target_patterns, config, **kwargs):
+        tensor = next(iter(input_dict.values()))
+        tensor = tensor[0] if isinstance(tensor, list) else tensor
+        head_dim = config.get_text_config().head_dim
+
+        rest = list(tensor.shape[1:])
+        num_heads = tensor.shape[0] // (2 * head_dim)
+        shape = [2, num_heads, head_dim] if self.inverse else [num_heads, 2, head_dim]
+        tensor = tensor.reshape(shape + rest).transpose(0, 1).reshape(tensor.shape).contiguous()
+        return {target_patterns[0]: tensor}
+
+    @property
+    def reverse_op(self) -> ConversionOps:
+        return TeleChatInterleaveKeyValue(inverse=not self.inverse)
+
+
 class MergeModulelist(ConversionOps):
     """
     Merge a list of tensors into a single tensor along the first dimension.

src/transformers/models/auto/modeling_auto.py

@@ -566,6 +566,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("t5gemma2_encoder", "T5Gemma2Encoder"),
         ("table-transformer", "TableTransformerModel"),
         ("tapas", "TapasModel"),
+        ("telechat", "TeleChatModel"),
         ("textnet", "TextNetModel"),
         ("time_series_transformer", "TimeSeriesTransformerModel"),
         ("timesfm", "TimesFmModel"),
@@ -917,6 +918,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("solar_open", "SolarOpenForCausalLM"),
         ("stablelm", "StableLmForCausalLM"),
         ("starcoder2", "Starcoder2ForCausalLM"),
+        ("telechat", "TeleChatForCausalLM"),
         ("trocr", "TrOCRForCausalLM"),
         ("vaultgemma", "VaultGemmaForCausalLM"),
         ("whisper", "WhisperForCausalLM"),

src/transformers/models/auto/tokenization_auto.py

@@ -421,6 +421,7 @@
     "qwen3_5_moe_text",
     "qwen4_exp",
     "step3_vl",
+    "telechat",
     "vipllava",
     "cohere_asr",
     "camembertv2-base",