🚨 Slim down [audio] to torchaudio + soundfile + soxr (drop librosa and phonemizer)

docker/consistency.dockerfile

@@ -3,7 +3,7 @@ ARG REF=main
 RUN apt-get update && apt-get install -y time git g++ cmake pkg-config make git-lfs
 RUN uv pip install --no-cache-dir GitPython
 RUN uv pip install --no-cache-dir --upgrade 'torch<=2.15.0' 'torchaudio' 'torchvision' --index-url https://download.pytorch.org/whl/cpu
-RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[quality,testing,torch-speech,vision]"
+RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[quality,testing,vision]"
 RUN git lfs install
 
 # Use a custom patched pytest to force exit the process at the end, to avoid `Too long with no output (exceeded 10m0s): context deadline exceeded` (#40201)

docs/source/en/model_doc/qwen2_audio.md

@@ -45,20 +45,16 @@ The abstract from the paper is the following:
 ### Inference
 
 ```python
-from io import BytesIO
-from urllib.request import urlopen
-
-import librosa
-
 from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration
+from transformers.audio_utils import load_audio
 
 
 model = Qwen2AudioForConditionalGeneration.from_pretrained("Qwen/Qwen2-Audio-7B", trust_remote_code=True, device_map="auto")
 processor = AutoProcessor.from_pretrained("Qwen/Qwen2-Audio-7B", trust_remote_code=True)
 
 prompt = "<|audio_bos|><|AUDIO|><|audio_eos|>Generate the caption in English:"
 url = "https://huggingface.co/datasets/hf-internal-testing/transformers-synthetic-assets/resolve/main/audio/glass_breaking.mp3"
-audio, sr = librosa.load(BytesIO(urlopen(url).read()), sr=processor.feature_extractor.sampling_rate)
+audio = load_audio(url, sampling_rate=processor.feature_extractor.sampling_rate)
 inputs = processor(text=prompt, audio=audio, return_tensors="pt").to(model.device)
 
 generate_ids = model.generate(**inputs, max_length=256)
@@ -83,12 +79,8 @@ In the following, we demonstrate how to use `Qwen2-Audio-7B-Instruct` for the in
 In the voice chat mode, users can freely engage in voice interactions with Qwen2-Audio without text input:
 
 ```python
-from io import BytesIO
-from urllib.request import urlopen
-
-import librosa
-
 from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration
+from transformers.audio_utils import load_audio
 
 
 processor = AutoProcessor.from_pretrained("Qwen/Qwen2-Audio-7B-Instruct")
@@ -109,10 +101,7 @@ for message in conversation:
     if isinstance(message["content"], list):
         for ele in message["content"]:
             if ele["type"] == "audio":
-                audios.append(librosa.load(
-                    BytesIO(urlopen(ele['audio_url']).read()),
-                    sr=processor.feature_extractor.sampling_rate)[0]
-                )
+                audios.append(load_audio(ele['audio_url'], sampling_rate=processor.feature_extractor.sampling_rate))
 
 inputs = processor(text=text, audio=audios, return_tensors="pt", padding=True).to(model.device)
 inputs.input_ids = inputs.input_ids.to(model.device)
@@ -128,12 +117,8 @@ response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_
 In the audio analysis, users could provide both audio and text instructions for analysis:
 
 ```python
-from io import BytesIO
-from urllib.request import urlopen
-
-import librosa
-
 from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration
+from transformers.audio_utils import load_audio
 
 
 processor = AutoProcessor.from_pretrained("Qwen/Qwen2-Audio-7B-Instruct")
@@ -161,11 +146,7 @@ for message in conversation:
     if isinstance(message["content"], list):
         for ele in message["content"]:
             if ele["type"] == "audio":
-                audios.append(
-                    librosa.load(
-                        BytesIO(urlopen(ele['audio_url']).read()),
-                        sr=processor.feature_extractor.sampling_rate)[0]
-                )
+                audios.append(load_audio(ele['audio_url'], sampling_rate=processor.feature_extractor.sampling_rate))
 
 inputs = processor(text=text, audio=audios, return_tensors="pt", padding=True).to(model.device)
 inputs.input_ids = inputs.input_ids.to(model.device)
@@ -181,12 +162,8 @@ response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_
 We also support batch inference:
 
 ```python
-from io import BytesIO
-from urllib.request import urlopen
-
-import librosa
-
 from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration
+from transformers.audio_utils import load_audio
 
 
 processor = AutoProcessor.from_pretrained("Qwen/Qwen2-Audio-7B-Instruct")
@@ -221,11 +198,7 @@ for conversation in conversations:
         if isinstance(message["content"], list):
             for ele in message["content"]:
                 if ele["type"] == "audio":
-                    audios.append(
-                        librosa.load(
-                            BytesIO(urlopen(ele['audio_url']).read()),
-                            sr=processor.feature_extractor.sampling_rate)[0]
-                    )
+                    audios.append(load_audio(ele['audio_url'], sampling_rate=processor.feature_extractor.sampling_rate))
 
 inputs = processor(text=text, audio=audios, return_tensors="pt", padding=True).to(model.device)
 inputs['input_ids'] = inputs['input_ids'].to(model.device)

docs/source/en/model_doc/vibevoice_acoustic_tokenizer.md

@@ -45,7 +45,7 @@ import torch
 from scipy.io import wavfile
 
 from transformers import AutoFeatureExtractor, VibeVoiceAcousticTokenizerModel
-from transformers.audio_utils import load_audio_librosa
+from transformers.audio_utils import load_audio
 
 
 model_id = "microsoft/VibeVoice-AcousticTokenizer"
@@ -57,7 +57,7 @@ print("Model loaded on device:", model.device)
 print("Model dtype:", model.dtype)
 
 # load audio
-audio = load_audio_librosa(
+audio = load_audio(
     "https://huggingface.co/datasets/bezzam/vibevoice_samples/resolve/main/voices/en-Alice_woman.wav",
     sampling_rate=feature_extractor.sampling_rate,
 )
@@ -96,7 +96,7 @@ import torch
 from scipy.io import wavfile
 
 from transformers import AutoFeatureExtractor, VibeVoiceAcousticTokenizerModel
-from transformers.audio_utils import load_audio_librosa
+from transformers.audio_utils import load_audio
 
 
 model_id = "microsoft/VibeVoice-AcousticTokenizer"
@@ -108,7 +108,7 @@ print("Model loaded on device:", model.device)
 print("Model dtype:", model.dtype)
 
 # load audio
-audio = load_audio_librosa(
+audio = load_audio(
     "https://huggingface.co/datasets/bezzam/vibevoice_samples/resolve/main/voices/en-Alice_woman.wav",
     sampling_rate=feature_extractor.sampling_rate,
 )

docs/source/en/tasks/asr.md

@@ -36,7 +36,7 @@ To see all architectures and checkpoints compatible with this task, we recommend
 Before you begin, make sure you have all the necessary libraries installed:
 
 ```bash
-pip install transformers datasets evaluate jiwer soundfile librosa torchcodec
+pip install transformers datasets evaluate jiwer soundfile torchcodec
 ```
 
 We encourage you to login to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to login:

docs/source/en/tasks/audio_classification.md

@@ -36,7 +36,7 @@ To see all architectures and checkpoints compatible with this task, we recommend
 Before you begin, make sure you have all the necessary libraries installed:
 
 ```bash
-pip install transformers datasets evaluate soundfile librosa torchcodec
+pip install transformers datasets evaluate soundfile torchcodec
 ```
 
 We encourage you to login to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to login:

docs/source/en/tasks/text-to-speech.md

@@ -90,7 +90,7 @@ The remainder of this guide illustrates how to:
 Before you begin, make sure you have all the necessary libraries installed:
 
 ```bash
-pip install transformers datasets soundfile librosa torchcodec speechbrain accelerate
+pip install transformers datasets soundfile torchcodec speechbrain accelerate
 ```
 
 Install 🤗Transformers from source as not all the SpeechT5 features have been merged into an official release yet:

setup.py

@@ -130,6 +130,8 @@
     "scikit-learn",
     "scipy",
     "sentencepiece>=0.1.91,!=0.1.92",
+    "soundfile>=0.13.0",  # first version with wheels on every supported platform and Python
+    "soxr>=0.5.0",  # first version with wheels on every supported platform and Python
     "starlette",
     "sudachipy>=0.6.6",
     "sudachidict_core>=20220729",
@@ -167,7 +169,7 @@ def deps_list(*pkgs):
 
 extras["torch"] = deps_list("torch", "accelerate")
 extras["vision"] = deps_list("torchvision", "Pillow")
-extras["audio"] = deps_list("torchaudio", "librosa", "phonemizer")
+extras["audio"] = deps_list("torchaudio", "soundfile", "soxr")
 extras["video"] = deps_list("av")
 extras["timm"] = deps_list("timm")
 extras["quality"] = deps_list(
@@ -213,6 +215,8 @@ def deps_list(*pkgs):
         "parameterized",
         "psutil",
         "diffusers",  # Needed for VibeVoice TTS integration tests
+        "librosa",  # reference implementation in audio tests
+        "phonemizer",  # Wav2Vec2Phoneme / VITS tokenizer tests (also needs the system `espeak-ng`)
         "nltk",
         "sacremoses",
         "rjieba",

src/transformers/audio_utils.py

@@ -18,6 +18,7 @@
 
 import base64
 import io
+import math
 import os
 import warnings
 from collections.abc import Sequence
@@ -29,9 +30,9 @@
 from huggingface_hub.utils import httpx
 
 from .utils import (
-    is_librosa_available,
     is_numpy_array,
     is_soundfile_available,
+    is_soxr_available,
     is_torch_tensor,
     is_torchaudio_available,
     is_torchcodec_greater_or_equal,
@@ -47,10 +48,7 @@
 if is_soundfile_available():
     import soundfile as sf
 
-if is_librosa_available():
-    import librosa
-
-    # TODO: @eustlb, we actually don't need librosa but soxr is installed with librosa
+if is_soxr_available():
     import soxr
 
 if is_torchaudio_available():
@@ -195,6 +193,22 @@ def _resolve_audio_source(audio: str, timeout: float | None = None) -> "str | by
         )
 
 
+def _load_audio_soundfile(source: "str | bytes", sampling_rate: int) -> np.ndarray:
+    """Decode with soundfile and resample with soxr: the same steps as `librosa.load`, so the output is identical
+    (downmix by averaging, `soxr_hq` resampling, padding/trimming to `ceil(num_samples * sampling_rate / src_rate)`)."""
+    requires_backends(load_audio, ["audio"])
+    with sf.SoundFile(BytesIO(source) if isinstance(source, bytes) else source) as f:
+        src_sampling_rate = f.samplerate
+        waveform = f.read(dtype="float32", always_2d=True).T
+    waveform = waveform.mean(axis=0) if waveform.shape[0] > 1 else waveform[0]
+
+    if src_sampling_rate != sampling_rate:
+        num_samples = math.ceil(waveform.shape[-1] * sampling_rate / src_sampling_rate)
+        waveform = soxr.resample(waveform, src_sampling_rate, sampling_rate, quality="soxr_hq")
+        waveform = np.pad(waveform, (0, max(0, num_samples - len(waveform))))[:num_samples]
+    return np.ascontiguousarray(waveform, dtype=np.float32)
+
+
 def load_audio(audio: str | np.ndarray, sampling_rate=16000, timeout=None, backend: str = "auto") -> np.ndarray:
     """
     Loads `audio` to an np.ndarray object.
@@ -211,9 +225,11 @@ def load_audio(audio: str | np.ndarray, sampling_rate=16000, timeout=None, backe
             The timeout value in seconds for the URL request.
         backend (`str`, *optional*, defaults to `"auto"`):
             Decoding backend: `"auto"` uses torchcodec when available (>=0.3.0) and falls back to
-            librosa; `"torchcodec"`, `"librosa"` or `"torchaudio"` force that backend (and error if it
-            is missing). `"torchaudio"` decodes with `torchaudio.load` and resamples with
+            soundfile; `"torchcodec"`, `"soundfile"` or `"torchaudio"` force that backend (and error if it
+            is missing). `"soundfile"` decodes with `soundfile` and resamples with `soxr` (identical to
+            `librosa.load`). `"torchaudio"` decodes with `torchaudio.load` and resamples with
             `torchaudio.functional.resample` (matches serving stacks such as sglang bit-for-bit).
+            `"librosa"` is a deprecated alias of `"soundfile"`.
 
     Returns:
         `np.ndarray`: A numpy array representing the audio.
@@ -226,29 +242,36 @@ def load_audio(audio: str | np.ndarray, sampling_rate=16000, timeout=None, backe
             f"a local file path, or a base64-encoded string (optionally wrapped as a `data:...` URI) but got {type(audio)}."
         )
 
-    # torchcodec handles audio/video; librosa only plain audio. `backend` lets callers pin one.
+    # torchcodec handles audio/video; soundfile only plain audio. `backend` lets callers pin one.
+    if backend == "librosa":
+        warnings.warn(
+            '`backend="librosa"` is deprecated and will be removed in Transformers v6. It now decodes with soundfile '
+            'and resamples with soxr, with output identical to `librosa.load`. Use `backend="soundfile"` instead.',
+            FutureWarning,
+        )
+        backend = "soundfile"
     if backend == "auto":
         resolved_backend = (
             "torchcodec"
             if is_torchcodec_greater_or_equal("0.3.0") and is_torchcodec_usable(decode="audio")
-            else "librosa"
+            else "soundfile"
         )
-    elif backend in ("torchcodec", "librosa", "torchaudio"):
+    elif backend in ("torchcodec", "soundfile", "torchaudio"):
         resolved_backend = backend
     else:
-        raise ValueError(f"Unknown backend {backend!r}; expected 'auto', 'torchcodec', 'librosa', or 'torchaudio'.")
-    # soundfile-based backends (librosa / torchaudio) cannot decode the video-ish formats below.
+        raise ValueError(f"Unknown backend {backend!r}; expected 'auto', 'torchcodec', 'soundfile', or 'torchaudio'.")
+    # soundfile-based backends (soundfile / torchaudio) cannot decode the video-ish formats below.
     use_torchcodec = resolved_backend == "torchcodec"
 
     # 1. Identify the format from the source string (extension / `data:` media type), without fetching.
     filetype = _format_from_source(audio)
-    # 2. With librosa as the only backend, fail fast and clearly on a format it cannot decode.
+    # 2. Without torchcodec, fail fast and clearly on a format soundfile cannot decode.
     if not use_torchcodec and filetype in TORCHCODEC_ONLY_FILETYPES:
         raise RuntimeError(
-            f"The audio source is a '{filetype}' file, which librosa cannot decode. {_NEEDS_TORCHCODEC}"
+            f"The audio source is a '{filetype}' file, which soundfile cannot decode. {_NEEDS_TORCHCODEC}"
         )
 
-    # 3. Resolve to local path or bytes; sniff format for raw base64 payloads before passing to librosa.
+    # 3. Resolve to local path or bytes; sniff format for raw base64 payloads before passing to soundfile.
     source = _resolve_audio_source(audio, timeout=timeout)
     if not use_torchcodec and filetype is None and isinstance(source, bytes):
         try:
@@ -257,15 +280,15 @@ def load_audio(audio: str | np.ndarray, sampling_rate=16000, timeout=None, backe
             filetype = None
         if filetype in TORCHCODEC_ONLY_FILETYPES:
             raise RuntimeError(
-                f"The audio source is a '{filetype}' file, which librosa cannot decode. {_NEEDS_TORCHCODEC}"
+                f"The audio source is a '{filetype}' file, which soundfile cannot decode. {_NEEDS_TORCHCODEC}"
             )
 
     # 4. Decode with the selected backend (`requires_backends` raises a clear error if it is missing).
     if use_torchcodec:
         requires_backends(load_audio, ["torchcodec"])
         from torchcodec.decoders import AudioDecoder
 
-        # `num_channels=1` matches what most models expect and librosa's default.
+        # `num_channels=1` matches what most models expect and the soundfile backend.
         return AudioDecoder(source, sample_rate=sampling_rate, num_channels=1).get_all_samples().data[0].numpy()
 
     if resolved_backend == "torchaudio":
@@ -277,8 +300,7 @@ def load_audio(audio: str | np.ndarray, sampling_rate=16000, timeout=None, backe
             waveform = torchaudio.functional.resample(waveform, orig_freq=src_sampling_rate, new_freq=sampling_rate)
         return waveform.numpy().astype(np.float32)
 
-    requires_backends(load_audio, ["librosa"])
-    return librosa.load(BytesIO(source) if isinstance(source, bytes) else source, sr=sampling_rate)[0]
+    return _load_audio_soundfile(source, sampling_rate)
 
 
 def load_audio_torchcodec(audio: str | np.ndarray, sampling_rate=16000, timeout=None) -> np.ndarray:
@@ -292,13 +314,13 @@ def load_audio_torchcodec(audio: str | np.ndarray, sampling_rate=16000, timeout=
 
 
 def load_audio_librosa(audio: str | np.ndarray, sampling_rate=16000, timeout=None) -> np.ndarray:
-    """Deprecated. Use [`load_audio`] instead (equivalent to `backend="librosa"`)."""
+    """Deprecated. Use [`load_audio`] instead (equivalent to `backend="soundfile"`)."""
     warnings.warn(
         "`load_audio_librosa` is deprecated and will be removed in a future version. "
-        'Use `load_audio(..., backend="librosa")` instead.',
+        'Use `load_audio(..., backend="soundfile")` instead.',
         FutureWarning,
     )
-    return load_audio(audio, sampling_rate=sampling_rate, timeout=timeout, backend="librosa")
+    return load_audio(audio, sampling_rate=sampling_rate, timeout=timeout, backend="soundfile")
 
 
 def load_audio_as(
@@ -327,7 +349,9 @@ def load_audio_as(
             - `dict`: Dictionary with 'data' (base64 encoded audio data) and 'format' keys (if return_format="dict")
             - `io.BytesIO`: BytesIO object containing audio data (if return_format="buffer")
     """
-    requires_backends(load_audio_as, ["librosa"])
+    requires_backends(load_audio_as, ["soundfile"])
+    if sampling_rate is not None:
+        requires_backends(load_audio_as, ["soxr"])  # only needed to resample
 
     if return_format not in ["base64", "dict", "buffer"]:
         raise ValueError(f"Invalid return_format: {return_format}. Must be 'base64', 'dict', or 'buffer'")
@@ -752,6 +776,7 @@ def mel_filter_bank(
     norm: str | None = None,
     mel_scale: str = "htk",
     triangularize_in_mel_space: bool = False,
+    dtype: str | np.dtype = np.float64,
 ) -> np.ndarray:
     """
     Creates a frequency bin conversion matrix used to obtain a mel spectrogram. This is called a *mel filter bank*, and
@@ -791,6 +816,10 @@ def mel_filter_bank(
         triangularize_in_mel_space (`bool`, *optional*, defaults to `False`):
             If this option is enabled, the triangular filter is applied in mel space rather than frequency space. This
             should be set to `true` in order to get the same results as `torchaudio` when computing mel filters.
+        dtype (`str` or `np.dtype`, *optional*, defaults to `np.float64`):
+            Data type of the returned filters. The triangular filters are computed in float64 and cast to `dtype`
+            before the `"slaney"` normalization. With `norm="slaney"`, `mel_scale="slaney"` and `dtype=np.float32`,
+            the result is bit-exact with `librosa.filters.mel(..., norm="slaney")` (transposed).
 
     Returns:
         `np.ndarray` of shape (`num_frequency_bins`, `num_mel_filters`): Triangular filter bank matrix. This is a
@@ -820,7 +849,7 @@ def mel_filter_bank(
         # frequencies of FFT bins in Hz
         fft_freqs = np.linspace(0, sampling_rate // 2, num_frequency_bins)
 
-    mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs)
+    mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs).astype(dtype, copy=False)
 
     if norm is not None and norm == "slaney":
         # Slaney-style mel is scaled to be approx constant energy per channel