Skip to content

Commit 31d360b

Browse files
tarekziadetarekziade
authored
Fix image backend equivalence tests for Pillow 13 (#50105)
1 parent 73bfa52 commit 31d360b

9 files changed

Lines changed: 103 additions & 52 deletions

‎tests/models/chinese_clip/test_image_processing_chinese_clip.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ def __init__(self, **kwargs):
3333
@require_vision
3434
class ChineseCLIPImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
3535
image_processor_tester_class = ChineseCLIPImageProcessingTester
36+
# Pillow >= 13 can reverse bicubic resize passes, increasing isolated pixel differences.
37+
backend_equivalence_atol = 0.5
3638

3739
@unittest.skip(
3840
reason="ChineseCLIPImageProcessor doesn't treat 4 channel PIL and numpy consistently yet"
@@ -45,6 +47,8 @@ def test_call_numpy_4_channels(self):
4547
@require_vision
4648
class ChineseCLIPImageProcessingTestFourChannels(ImageProcessingTestMixin, unittest.TestCase):
4749
image_processor_tester_class = ChineseCLIPImageProcessingTester
50+
# Pillow >= 13 can reverse bicubic resize passes, increasing isolated pixel differences.
51+
backend_equivalence_atol = 0.5
4852

4953
def setUp(self):
5054
super().setUp()

‎tests/models/cohere_compass/test_image_processing_cohere_compass.py‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,9 @@ def prepare_video_inputs(self, equal_resolution=False, numpify=False, torchify=F
9595
@require_torch
9696
@require_vision
9797
class CohereCompassImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
98+
# Pillow >= 13 can reverse bicubic resize passes, increasing isolated pixel differences.
99+
backend_equivalence_atol = 0.5
100+
98101
image_processor_tester_class = CohereCompassImageProcessingTester
99102

100103
def test_image_processor_to_json_string(self):
@@ -269,10 +272,12 @@ def test_backends_equivalence(self):
269272
reference_backend = backend_names[0]
270273
reference_encoding = encodings[reference_backend]
271274
for backend_name in backend_names[1:]:
272-
self._assert_tensors_equivalence(reference_encoding.pixel_values, encodings[backend_name].pixel_values)
273-
self.assertEqual(reference_encoding.image_grid_thw.dtype, encodings[backend_name].image_grid_thw.dtype)
274-
self._assert_tensors_equivalence(
275-
reference_encoding.image_grid_thw.float(), encodings[backend_name].image_grid_thw.float()
275+
self._assert_encodings_equivalence(
276+
reference_encoding,
277+
encodings[backend_name],
278+
reference_backend,
279+
backend_name,
280+
atol=self.backend_equivalence_atol,
276281
)
277282

278283
@require_vision
@@ -294,10 +299,12 @@ def test_backends_equivalence_batched(self):
294299
reference_backend = backend_names[0]
295300
reference_encoding = encodings[reference_backend]
296301
for backend_name in backend_names[1:]:
297-
self._assert_tensors_equivalence(reference_encoding.pixel_values, encodings[backend_name].pixel_values)
298-
self.assertEqual(reference_encoding.image_grid_thw.dtype, encodings[backend_name].image_grid_thw.dtype)
299-
self._assert_tensors_equivalence(
300-
reference_encoding.image_grid_thw.float(), encodings[backend_name].image_grid_thw.float()
302+
self._assert_encodings_equivalence(
303+
reference_encoding,
304+
encodings[backend_name],
305+
reference_backend,
306+
backend_name,
307+
atol=self.backend_equivalence_atol,
301308
)
302309

303310
def test_get_num_patches_without_images(self):

‎tests/models/ernie4_5_vl_moe/test_image_processing_ernie4_5_vl_moe.py‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ def prepare_image_inputs(self, equal_resolution=False, numpify=False, torchify=F
6161
@require_torch
6262
@require_vision
6363
class Ernie4_5_VLMoeImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
64+
# Pillow >= 13 can reverse bicubic resize passes, increasing isolated pixel differences.
65+
backend_equivalence_atol = 0.5
66+
6467
image_processor_tester_class = Ernie4_5_VLMoeImageProcessingTester
6568

6669
def test_select_best_resolution(self):
@@ -229,10 +232,8 @@ def test_backends_equivalence(self):
229232
encoding_pil = image_processor_pil(dummy_image, return_tensors="pt")
230233
encoding_torchvision = image_processor_torchvision(dummy_image, return_tensors="pt")
231234

232-
self._assert_tensors_equivalence(encoding_pil.pixel_values, encoding_torchvision.pixel_values)
233-
self.assertEqual(encoding_pil.image_grid_thw.dtype, encoding_torchvision.image_grid_thw.dtype)
234-
self._assert_tensors_equivalence(
235-
encoding_pil.image_grid_thw.float(), encoding_torchvision.image_grid_thw.float()
235+
self._assert_encodings_equivalence(
236+
encoding_pil, encoding_torchvision, "pil", "torchvision", atol=self.backend_equivalence_atol
236237
)
237238

238239
@require_vision
@@ -254,10 +255,8 @@ def test_backends_equivalence_batched(self):
254255
encoding_pil = image_processor_pil(dummy_images, return_tensors="pt")
255256
encoding_torchvision = image_processor_torchvision(dummy_images, return_tensors="pt")
256257

257-
self._assert_tensors_equivalence(encoding_pil.pixel_values, encoding_torchvision.pixel_values)
258-
self.assertEqual(encoding_pil.image_grid_thw.dtype, encoding_torchvision.image_grid_thw.dtype)
259-
self._assert_tensors_equivalence(
260-
encoding_pil.image_grid_thw.float(), encoding_torchvision.image_grid_thw.float()
258+
self._assert_encodings_equivalence(
259+
encoding_pil, encoding_torchvision, "pil", "torchvision", atol=self.backend_equivalence_atol
261260
)
262261

263262
def test_get_num_patches_without_images(self):

‎tests/models/flava/test_image_processing_flava.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ class FlavaImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
6767
maxDiff = None
6868

6969
image_processor_tester_class = FlavaImageProcessingTester
70+
# Pillow >= 13 can reverse bicubic resize passes, increasing isolated pixel differences.
71+
backend_equivalence_atol = 0.5
7072

7173
def test_from_dict_with_codebook_size_overrides(self):
7274
for image_processing_class in self.image_processing_classes.values():

‎tests/models/ovis2/test_image_processing_ovis2.py‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,10 @@
1616

1717
from transformers.image_utils import SizeDict
1818
from transformers.testing_utils import require_torch, require_vision
19-
from transformers.utils import is_torch_available
2019

2120
from ...test_image_processing_common import ImageProcessingTester, ImageProcessingTestMixin
2221

2322

24-
if is_torch_available():
25-
import torch
26-
27-
2823
class Ovis2ImageProcessingTester(ImageProcessingTester):
2924
def __init__(self, **kwargs):
3025
# Image processor init kwargs
@@ -38,6 +33,8 @@ def __init__(self, **kwargs):
3833
@require_vision
3934
class Ovis2ProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
4035
image_processor_tester_class = Ovis2ImageProcessingTester
36+
# Pillow >= 13 can reverse bicubic resize passes, increasing isolated pixel differences.
37+
backend_equivalence_atol = 0.5
4138

4239
def test_backends_equivalence_crop_to_patches(self):
4340
"""Test equivalence between backends when cropping to patches."""
@@ -52,11 +49,17 @@ def test_backends_equivalence_crop_to_patches(self):
5249
encodings[backend_name] = image_processor(dummy_image, return_tensors="pt")
5350

5451
backend_names = list(encodings.keys())
55-
reference_encoding = encodings[backend_names[0]].pixel_values
52+
reference_backend = backend_names[0]
53+
reference_encoding = encodings[reference_backend]
5654
for backend_name in backend_names[1:]:
57-
self.assertTrue(torch.allclose(reference_encoding, encodings[backend_name].pixel_values, atol=1e-1))
58-
self.assertLessEqual(
59-
torch.mean(torch.abs(reference_encoding - encodings[backend_name].pixel_values)).item(), 1e-3
55+
# Use the shared mean-error limit for patches resized in a different pass order.
56+
self._assert_encodings_equivalence(
57+
reference_encoding,
58+
encodings[backend_name],
59+
reference_backend,
60+
backend_name,
61+
atol=self.backend_equivalence_atol,
62+
rtol=1e-5,
6063
)
6164

6265
def test_backends_equivalence_batched_crop_to_patches(self):
@@ -76,11 +79,17 @@ def test_backends_equivalence_batched_crop_to_patches(self):
7679
encodings[backend_name] = image_processor(dummy_images, return_tensors="pt")
7780

7881
backend_names = list(encodings.keys())
79-
reference_encoding = encodings[backend_names[0]].pixel_values
82+
reference_backend = backend_names[0]
83+
reference_encoding = encodings[reference_backend]
8084
for backend_name in backend_names[1:]:
81-
self.assertTrue(torch.allclose(reference_encoding, encodings[backend_name].pixel_values, atol=1e-1))
82-
self.assertLessEqual(
83-
torch.mean(torch.abs(reference_encoding - encodings[backend_name].pixel_values)).item(), 1e-3
85+
# Use the shared mean-error limit for patches resized in a different pass order.
86+
self._assert_encodings_equivalence(
87+
reference_encoding,
88+
encodings[backend_name],
89+
reference_backend,
90+
backend_name,
91+
atol=self.backend_equivalence_atol,
92+
rtol=1e-5,
8493
)
8594

8695
def test_crop_to_patches(self):

‎tests/models/paddleocr_vl/test_image_processing_paddleocr_vl.py‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@ def expected_output_image_shape(self, images):
8686
@require_torch
8787
@require_vision
8888
class PaddleOCRVLImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
89+
# Pillow >= 13 can reverse bicubic resize passes, increasing isolated pixel differences.
90+
backend_equivalence_atol = 0.5
91+
8992
image_processor_tester_class = PaddleOCRVLImageProcessingTester
9093

9194
def test_image_processor_to_json_string(self):
@@ -234,10 +237,12 @@ def test_backends_equivalence(self):
234237
reference_backend = backend_names[0]
235238
reference_encoding = encodings[reference_backend]
236239
for backend_name in backend_names[1:]:
237-
self._assert_tensors_equivalence(reference_encoding.pixel_values, encodings[backend_name].pixel_values)
238-
self.assertEqual(reference_encoding.image_grid_thw.dtype, encodings[backend_name].image_grid_thw.dtype)
239-
self._assert_tensors_equivalence(
240-
reference_encoding.image_grid_thw.float(), encodings[backend_name].image_grid_thw.float()
240+
self._assert_encodings_equivalence(
241+
reference_encoding,
242+
encodings[backend_name],
243+
reference_backend,
244+
backend_name,
245+
atol=self.backend_equivalence_atol,
241246
)
242247

243248
@require_vision
@@ -257,9 +262,12 @@ def test_backends_equivalence_batched(self):
257262
reference_backend = backend_names[0]
258263
reference_encoding = encodings[reference_backend]
259264
for backend_name in backend_names[1:]:
260-
self._assert_tensors_equivalence(reference_encoding.pixel_values, encodings[backend_name].pixel_values)
261-
self._assert_tensors_equivalence(
262-
reference_encoding.image_grid_thw.float(), encodings[backend_name].image_grid_thw.float()
265+
self._assert_encodings_equivalence(
266+
reference_encoding,
267+
encodings[backend_name],
268+
reference_backend,
269+
backend_name,
270+
atol=self.backend_equivalence_atol,
263271
)
264272

265273
def test_get_num_patches_without_images(self):

‎tests/models/qwen2_vl/test_image_processing_qwen2_vl.py‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ def prepare_video_inputs(self, equal_resolution=False, numpify=False, torchify=F
8080
@require_torch
8181
@require_vision
8282
class Qwen2VLImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
83+
# Pillow >= 13 can reverse bicubic resize passes, increasing isolated pixel differences.
84+
backend_equivalence_atol = 0.5
85+
8386
image_processor_tester_class = Qwen2VLImageProcessingTester
8487

8588
def test_image_processor_to_json_string(self):
@@ -278,10 +281,12 @@ def test_backends_equivalence(self):
278281
reference_backend = backend_names[0]
279282
reference_encoding = encodings[reference_backend]
280283
for backend_name in backend_names[1:]:
281-
self._assert_tensors_equivalence(reference_encoding.pixel_values, encodings[backend_name].pixel_values)
282-
self.assertEqual(reference_encoding.image_grid_thw.dtype, encodings[backend_name].image_grid_thw.dtype)
283-
self._assert_tensors_equivalence(
284-
reference_encoding.image_grid_thw.float(), encodings[backend_name].image_grid_thw.float()
284+
self._assert_encodings_equivalence(
285+
reference_encoding,
286+
encodings[backend_name],
287+
reference_backend,
288+
backend_name,
289+
atol=self.backend_equivalence_atol,
285290
)
286291

287292
@require_vision
@@ -308,10 +313,12 @@ def test_backends_equivalence_batched(self):
308313
reference_backend = backend_names[0]
309314
reference_encoding = encodings[reference_backend]
310315
for backend_name in backend_names[1:]:
311-
self._assert_tensors_equivalence(reference_encoding.pixel_values, encodings[backend_name].pixel_values)
312-
self.assertEqual(reference_encoding.image_grid_thw.dtype, encodings[backend_name].image_grid_thw.dtype)
313-
self._assert_tensors_equivalence(
314-
reference_encoding.image_grid_thw.float(), encodings[backend_name].image_grid_thw.float()
316+
self._assert_encodings_equivalence(
317+
reference_encoding,
318+
encodings[backend_name],
319+
reference_backend,
320+
backend_name,
321+
atol=self.backend_equivalence_atol,
315322
)
316323

317324
def test_get_num_patches_without_images(self):

‎tests/models/video_llama_3/test_image_processing_video_llama_3.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ def prepare_video_inputs(self, equal_resolution=False, numpify=False, torchify=F
8080
@require_torch
8181
@require_vision
8282
class VideoLlama3ImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
83+
# Pillow >= 13 can reverse bicubic resize passes, increasing isolated pixel differences.
84+
backend_equivalence_atol = 0.3
85+
8386
image_processor_tester_class = VideoLlama3ImageProcessingTester
8487

8588
def test_image_processor_to_json_string(self):
@@ -260,9 +263,9 @@ def test_backends_equivalence(self):
260263
reference = encodings[reference_backend]
261264
for backend_name in backend_names[1:]:
262265
encoding = encodings[backend_name]
263-
self._assert_tensors_equivalence(reference.pixel_values, encoding.pixel_values)
264-
self.assertEqual(reference.image_grid_thw.dtype, encoding.image_grid_thw.dtype)
265-
self._assert_tensors_equivalence(reference.image_grid_thw.float(), encoding.image_grid_thw.float())
266+
self._assert_encodings_equivalence(
267+
reference, encoding, reference_backend, backend_name, atol=self.backend_equivalence_atol
268+
)
266269

267270
@require_vision
268271
@require_torch
@@ -288,9 +291,9 @@ def test_backends_equivalence_batched(self):
288291
reference = encodings[reference_backend]
289292
for backend_name in backend_names[1:]:
290293
encoding = encodings[backend_name]
291-
self._assert_tensors_equivalence(reference.pixel_values, encoding.pixel_values)
292-
self.assertEqual(reference.image_grid_thw.dtype, encoding.image_grid_thw.dtype)
293-
self._assert_tensors_equivalence(reference.image_grid_thw.float(), encoding.image_grid_thw.float())
294+
self._assert_encodings_equivalence(
295+
reference, encoding, reference_backend, backend_name, atol=self.backend_equivalence_atol
296+
)
294297

295298
def test_get_num_patches_without_images(self):
296299
for image_processing_class in self.image_processing_classes.values():

‎tests/test_image_processing_common.py‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,10 @@ class ImageProcessingTestMixin:
340340
# Must be set by subclass
341341
image_processor_tester_class = None
342342

343+
# Models can allow larger individual pixel differences between image backends without
344+
# relaxing the mean-error limit or exact comparisons of integer outputs.
345+
backend_equivalence_atol = 1e-1
346+
343347
test_cast_dtype = None
344348

345349
def setUp(self):
@@ -430,7 +434,11 @@ def test_backends_equivalence(self):
430434
reference_encoding = encodings[reference_backend]
431435
for backend_name in backend_names[1:]:
432436
self._assert_encodings_equivalence(
433-
reference_encoding, encodings[backend_name], reference_backend, backend_name
437+
reference_encoding,
438+
encodings[backend_name],
439+
reference_backend,
440+
backend_name,
441+
atol=self.backend_equivalence_atol,
434442
)
435443

436444
@require_vision
@@ -453,7 +461,11 @@ def test_backends_equivalence_batched(self):
453461
reference_encoding = encodings[reference_backend]
454462
for backend_name in backend_names[1:]:
455463
self._assert_encodings_equivalence(
456-
reference_encoding, encodings[backend_name], reference_backend, backend_name
464+
reference_encoding,
465+
encodings[backend_name],
466+
reference_backend,
467+
backend_name,
468+
atol=self.backend_equivalence_atol,
457469
)
458470

459471
def _assert_has_attributes(self, processor, expected_attributes: dict[str, Any]) -> None:

0 commit comments

Comments
 (0)