From c147b4b41b5f210299a30e5acd60c8a20b3ad370 Mon Sep 17 00:00:00 2001 From: remi-or Date: Mon, 2 Jun 2025 19:04:27 -0500 Subject: [PATCH 1/9] Added a compile-friendly versiom of resize to BaseImgProcessorFast --- .../image_processing_utils_fast.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/transformers/image_processing_utils_fast.py b/src/transformers/image_processing_utils_fast.py index b1e261412203..ed2c32076db2 100644 --- a/src/transformers/image_processing_utils_fast.py +++ b/src/transformers/image_processing_utils_fast.py @@ -279,8 +279,34 @@ def resize( "Size must contain 'height' and 'width' keys, or 'max_height' and 'max_width', or 'shortest_edge' key. Got" f" {size}." ) + # This is a workaround to avoid a bug in torch.compile when dealing with uint8 + # TODO: remove this once the bug is fixed (detected with torch==2.7.0+git1fee196, torchvision==0.22.0+9eb57cd) + if torch.compiler.is_compiling(): + return self.compile_friendly_resize(image, new_size, interpolation, antialias) return F.resize(image, new_size, interpolation=interpolation, antialias=antialias) + @staticmethod + def compile_friendly_resize( + image: "torch.Tensor", + new_size: tuple[int, int], + interpolation: Optional["F.InterpolationMode"] = None, + antialias: bool = True, + ) -> "torch.Tensor": + """ + A wrapper around `F.resize` so that it is compatible with torch.compile when the image is a uint8 tensor. + """ + if image.dtype == torch.uint8: + image = image.float() / 256 + image = F.resize(image, new_size, interpolation=interpolation, antialias=antialias) + image = image * 256 + image = image.masked_fill(image > 255, 255) + image = image.masked_fill(image < 0, 0) + image = image.round().to(torch.uint8) + else: + image = F.resize(image, new_size, interpolation=interpolation, antialias=antialias) + return image + + def rescale( self, image: "torch.Tensor", From 907b63de00c4ebe502a8f5d46dcd2362ed757197 Mon Sep 17 00:00:00 2001 From: remi-or Date: Mon, 2 Jun 2025 19:05:17 -0500 Subject: [PATCH 2/9] Changed qwen2 processor to use its parent class .resize --- .../models/qwen2_vl/video_processing_qwen2_vl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py b/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py index adaf369473b2..ee8077acb2de 100644 --- a/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py @@ -143,8 +143,10 @@ def _preprocess( min_pixels=min_pixels, max_pixels=max_pixels, ) - stacked_videos = F.resize( - stacked_videos, size=(resized_height, resized_width), interpolation=interpolation + stacked_videos = self.resize( + image=stacked_videos, + size=SizeDict(height=resized_height, width=resized_width), + interpolation=interpolation, ) resized_videos_grouped[shape] = stacked_videos resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index) From cb36ed80641bc267ce16489334cedf6079063144 Mon Sep 17 00:00:00 2001 From: remi-or Date: Mon, 2 Jun 2025 19:13:48 -0500 Subject: [PATCH 3/9] Style --- src/transformers/image_processing_utils_fast.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/image_processing_utils_fast.py b/src/transformers/image_processing_utils_fast.py index ed2c32076db2..95ec7a261918 100644 --- a/src/transformers/image_processing_utils_fast.py +++ b/src/transformers/image_processing_utils_fast.py @@ -306,7 +306,6 @@ def compile_friendly_resize( image = F.resize(image, new_size, interpolation=interpolation, antialias=antialias) return image - def rescale( self, image: "torch.Tensor", From 18d48cbf3ca0f53ad63dbd5270068a0d07ce5f7d Mon Sep 17 00:00:00 2001 From: remi-or Date: Thu, 5 Jun 2025 05:40:11 -0500 Subject: [PATCH 4/9] underlined issue only happens on AMD w/ comment and bool check --- src/transformers/image_processing_utils_fast.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/image_processing_utils_fast.py b/src/transformers/image_processing_utils_fast.py index 95ec7a261918..a373a82c81a5 100644 --- a/src/transformers/image_processing_utils_fast.py +++ b/src/transformers/image_processing_utils_fast.py @@ -49,6 +49,7 @@ is_vision_available, logging, ) +from .utils.import_utils import is_rocm_platform if is_vision_available(): @@ -279,9 +280,9 @@ def resize( "Size must contain 'height' and 'width' keys, or 'max_height' and 'max_width', or 'shortest_edge' key. Got" f" {size}." ) - # This is a workaround to avoid a bug in torch.compile when dealing with uint8 + # This is a workaround to avoid a bug in torch.compile when dealing with uint8 on AMD MI3XX GPUs # TODO: remove this once the bug is fixed (detected with torch==2.7.0+git1fee196, torchvision==0.22.0+9eb57cd) - if torch.compiler.is_compiling(): + if torch.compiler.is_compiling() and is_rocm_platform(): return self.compile_friendly_resize(image, new_size, interpolation, antialias) return F.resize(image, new_size, interpolation=interpolation, antialias=antialias) From 7a7aff2d017419c045d8f85c01f6be20c2865d74 Mon Sep 17 00:00:00 2001 From: remi-or Date: Thu, 5 Jun 2025 06:00:16 -0500 Subject: [PATCH 5/9] Fixed some utils functions --- src/transformers/utils/import_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/utils/import_utils.py b/src/transformers/utils/import_utils.py index ff097849f971..58d4165c24e0 100644 --- a/src/transformers/utils/import_utils.py +++ b/src/transformers/utils/import_utils.py @@ -481,7 +481,7 @@ def is_cuda_platform(): if is_torch_available(): import torch - torch.version.cuda is not None + return torch.version.cuda is not None else: return False @@ -490,7 +490,7 @@ def is_rocm_platform(): if is_torch_available(): import torch - torch.version.hip is not None + return torch.version.hip is not None else: return False From 7769cc004fea24b0550858370434cd1f82f35854 Mon Sep 17 00:00:00 2001 From: remi-or Date: Thu, 5 Jun 2025 06:16:24 -0500 Subject: [PATCH 6/9] Fixed the same issue for bridgetower --- .../bridgetower/image_processing_bridgetower_fast.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/bridgetower/image_processing_bridgetower_fast.py b/src/transformers/models/bridgetower/image_processing_bridgetower_fast.py index 68d5b2c0f899..01df06c52934 100644 --- a/src/transformers/models/bridgetower/image_processing_bridgetower_fast.py +++ b/src/transformers/models/bridgetower/image_processing_bridgetower_fast.py @@ -164,13 +164,18 @@ def resize( raise ValueError(f"The `size` dictionary must contain the key `shortest_edge`. Got {size.keys()}") shorter = size.shortest_edge longer = int(1333 / 800 * shorter) - output_size = get_resize_output_image_size( + output_height, output_width = get_resize_output_image_size( image, shorter=shorter, longer=longer, size_divisor=size_divisor, ) - return F.resize(image, output_size, interpolation=interpolation, antialias=antialias) + return super().resize( + image=image, + size=SizeDict(height=output_height, width=output_width), + interpolation=interpolation, + antialias=antialias, + ) def center_crop( self, From e4bae98cb77e26b393452dcd7ff20080cfd35c5b Mon Sep 17 00:00:00 2001 From: remi-or Date: Thu, 5 Jun 2025 06:16:53 -0500 Subject: [PATCH 7/9] Fixed the same issue for llava_next --- .../models/llava_next/image_processing_llava_next_fast.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/llava_next/image_processing_llava_next_fast.py b/src/transformers/models/llava_next/image_processing_llava_next_fast.py index ac90290cef48..2f7a398479b0 100644 --- a/src/transformers/models/llava_next/image_processing_llava_next_fast.py +++ b/src/transformers/models/llava_next/image_processing_llava_next_fast.py @@ -137,7 +137,11 @@ def _resize_for_patching( new_height, new_width = get_patch_output_size(image, target_resolution, input_data_format) # Resize the image - resized_image = F.resize(image, (new_height, new_width), interpolation=interpolation) + resized_image = self.resize( + image=image, + size=SizeDict(height=new_height, width=new_width), + interpolation=interpolation, + ) return resized_image From 54135b3312811eb5d5738af1a0f429945df0ccdb Mon Sep 17 00:00:00 2001 From: remi-or Date: Thu, 5 Jun 2025 07:40:27 -0500 Subject: [PATCH 8/9] Repo consistency for llava onevision --- .../image_processing_llava_onevision_fast.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/llava_onevision/image_processing_llava_onevision_fast.py b/src/transformers/models/llava_onevision/image_processing_llava_onevision_fast.py index a29631fcb6af..cf5483e226b9 100644 --- a/src/transformers/models/llava_onevision/image_processing_llava_onevision_fast.py +++ b/src/transformers/models/llava_onevision/image_processing_llava_onevision_fast.py @@ -142,7 +142,11 @@ def _resize_for_patching( new_height, new_width = get_patch_output_size(image, target_resolution, input_data_format) # Resize the image - resized_image = F.resize(image, (new_height, new_width), interpolation=interpolation) + resized_image = self.resize( + image=image, + size=SizeDict(height=new_height, width=new_width), + interpolation=interpolation, + ) return resized_image From 14775c0b3efb7a5997ae71f12644370746fbd0ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Ouazan?= <83456801+remi-or@users.noreply.github.com> Date: Thu, 5 Jun 2025 16:18:32 +0200 Subject: [PATCH 9/9] Update src/transformers/image_processing_utils_fast.py Co-authored-by: Mohit Sharma --- src/transformers/image_processing_utils_fast.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/image_processing_utils_fast.py b/src/transformers/image_processing_utils_fast.py index a373a82c81a5..a0e78e824a71 100644 --- a/src/transformers/image_processing_utils_fast.py +++ b/src/transformers/image_processing_utils_fast.py @@ -281,6 +281,7 @@ def resize( f" {size}." ) # This is a workaround to avoid a bug in torch.compile when dealing with uint8 on AMD MI3XX GPUs + # Tracked in PyTorch issue: https://github.com/pytorch/pytorch/issues/155209 # TODO: remove this once the bug is fixed (detected with torch==2.7.0+git1fee196, torchvision==0.22.0+9eb57cd) if torch.compiler.is_compiling() and is_rocm_platform(): return self.compile_friendly_resize(image, new_size, interpolation, antialias)