From 204d6431cead2ddc190525592930354691bf4e4c Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Wed, 26 Mar 2025 16:13:18 +0100 Subject: [PATCH 01/18] adding fast processor for beit --- docs/source/en/model_doc/beit.md | 5 ++ docs/source/ja/model_doc/beit.md | 5 ++ src/transformers/__init__.py | 2 + .../models/auto/image_processing_auto.py | 4 +- src/transformers/models/beit/__init__.py | 1 + .../models/beit/image_processing_beit_fast.py | 51 +++++++++++++++++++ .../utils/dummy_torchvision_objects.py | 7 +++ .../models/beit/test_image_processing_beit.py | 6 ++- 8 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 src/transformers/models/beit/image_processing_beit_fast.py diff --git a/docs/source/en/model_doc/beit.md b/docs/source/en/model_doc/beit.md index 24dfabf682b6..c9206717e54a 100644 --- a/docs/source/en/model_doc/beit.md +++ b/docs/source/en/model_doc/beit.md @@ -149,6 +149,11 @@ If you're interested in submitting a resource to be included here, please feel f [[autodoc]] BeitImageProcessor - preprocess + +## BeitImageProcessorFast + +[[autodoc]] BeitImageProcessorFast + - preprocess - post_process_semantic_segmentation diff --git a/docs/source/ja/model_doc/beit.md b/docs/source/ja/model_doc/beit.md index 45eb1efa5dd8..e0b94693a36f 100644 --- a/docs/source/ja/model_doc/beit.md +++ b/docs/source/ja/model_doc/beit.md @@ -105,6 +105,11 @@ BEiT の使用を開始するのに役立つ公式 Hugging Face およびコミ [[autodoc]] BeitImageProcessor - preprocess + +## BeitImageProcessorFast + +[[autodoc]] BeitImageProcessorFast + - preprocess - post_process_semantic_segmentation ## BeitModel diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index e8da536747d4..ab2293d3d250 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -1343,6 +1343,7 @@ ] else: _import_structure["image_processing_utils_fast"] = ["BaseImageProcessorFast"] + _import_structure["models.beit"].append("BeitImageProcessorFast") _import_structure["models.blip"].append("BlipImageProcessorFast") _import_structure["models.clip"].append("CLIPImageProcessorFast") _import_structure["models.convnext"].append("ConvNextImageProcessorFast") @@ -6601,6 +6602,7 @@ from .utils.dummy_torchvision_objects import * else: from .image_processing_utils_fast import BaseImageProcessorFast + from .models.beit import BeitImageProcessorFast from .models.blip import BlipImageProcessorFast from .models.clip import CLIPImageProcessorFast from .models.convnext import ConvNextImageProcessorFast diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index 77b9734189b3..068a51dba451 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -57,7 +57,7 @@ [ ("align", ("EfficientNetImageProcessor",)), ("aria", ("AriaImageProcessor")), - ("beit", ("BeitImageProcessor",)), + ("beit", ("BeitImageProcessor", "BeitImageProcessorFast")), ("bit", ("BitImageProcessor",)), ("blip", ("BlipImageProcessor", "BlipImageProcessorFast")), ("blip-2", ("BlipImageProcessor", "BlipImageProcessorFast")), @@ -70,7 +70,7 @@ ("convnext", ("ConvNextImageProcessor", "ConvNextImageProcessorFast")), ("convnextv2", ("ConvNextImageProcessor", "ConvNextImageProcessorFast")), ("cvt", ("ConvNextImageProcessor", "ConvNextImageProcessorFast")), - ("data2vec-vision", ("BeitImageProcessor",)), + ("data2vec-vision", ("BeitImageProcessor", "BeitImageProcessorFast")), ("deformable_detr", ("DeformableDetrImageProcessor", "DeformableDetrImageProcessorFast")), ("deit", ("DeiTImageProcessor", "DeiTImageProcessorFast")), ("depth_anything", ("DPTImageProcessor",)), diff --git a/src/transformers/models/beit/__init__.py b/src/transformers/models/beit/__init__.py index 44f838a9a5d9..3f412a350068 100644 --- a/src/transformers/models/beit/__init__.py +++ b/src/transformers/models/beit/__init__.py @@ -21,6 +21,7 @@ from .configuration_beit import * from .feature_extraction_beit import * from .image_processing_beit import * + from .image_processing_beit_fast import * from .modeling_beit import * from .modeling_flax_beit import * else: diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py new file mode 100644 index 000000000000..e1ecced2037c --- /dev/null +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -0,0 +1,51 @@ +# coding=utf-8 +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fast Image processor class for Beit.""" + +from ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast +from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, PILImageResampling +from ...utils import add_start_docstrings + + +@add_start_docstrings( + "Constructs a fast Beit image processor.", + BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, +) +class BeitImageProcessorFast(BaseImageProcessorFast): + # This generated class can be used as a starting point for the fast image processor. + # if the image processor is only used for simple augmentations, such as resizing, center cropping, rescaling, or normalizing, + # only the default values should be set in the class. + # If the image processor requires more complex augmentations, methods from BaseImageProcessorFast can be overridden. + # In most cases, only the `_preprocess` method should be overridden. + + # For an example of a fast image processor requiring more complex augmentations, see `LlavaNextImageProcessorFast`. + + # Default values should be checked against the slow image processor + # None values left after checking can be removed + # resample = PILImageResampling.BILINEAR + image_mean = IMAGENET_STANDARD_MEAN + image_std = IMAGENET_STANDARD_STD + size = {"height": 224, "width": 224} + default_to_square = True + crop_size = {"height": 224, "width": 224} + do_resize = True + do_center_crop = False + do_rescale = True + do_normalize = True + do_convert_rgb = None + do_reduce_labels = False + + +__all__ = ["BeitImageProcessorFast"] diff --git a/src/transformers/utils/dummy_torchvision_objects.py b/src/transformers/utils/dummy_torchvision_objects.py index 50314fc55e9e..b30772752185 100644 --- a/src/transformers/utils/dummy_torchvision_objects.py +++ b/src/transformers/utils/dummy_torchvision_objects.py @@ -9,6 +9,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torchvision"]) +class BeitImageProcessorFast(metaclass=DummyObject): + _backends = ["torchvision"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torchvision"]) + + class BlipImageProcessorFast(metaclass=DummyObject): _backends = ["torchvision"] diff --git a/tests/models/beit/test_image_processing_beit.py b/tests/models/beit/test_image_processing_beit.py index 58175c6fe18c..95ecbb11caa5 100644 --- a/tests/models/beit/test_image_processing_beit.py +++ b/tests/models/beit/test_image_processing_beit.py @@ -19,7 +19,7 @@ from datasets import load_dataset from transformers.testing_utils import require_torch, require_vision -from transformers.utils import is_torch_available, is_vision_available +from transformers.utils import is_torch_available, is_torchvision_available, is_vision_available from ...test_image_processing_common import ImageProcessingTestMixin, prepare_image_inputs @@ -32,6 +32,9 @@ from transformers import BeitImageProcessor + if is_torchvision_available(): + from transformers import BeitImageProcessorFast + class BeitImageProcessingTester: def __init__( @@ -119,6 +122,7 @@ def prepare_semantic_batch_inputs(): @require_vision class BeitImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase): image_processing_class = BeitImageProcessor if is_vision_available() else None + fast_image_processing_class = BeitImageProcessorFast if is_torchvision_available() else None def setUp(self): super().setUp() From d266e74a5bfe1e3c21dd7a2f7422110d25e03d39 Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Wed, 26 Mar 2025 16:23:14 +0100 Subject: [PATCH 02/18] adding resample --- src/transformers/models/beit/image_processing_beit_fast.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index e1ecced2037c..5c5bb8208209 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -35,6 +35,7 @@ class BeitImageProcessorFast(BaseImageProcessorFast): # Default values should be checked against the slow image processor # None values left after checking can be removed # resample = PILImageResampling.BILINEAR + resample = PILImageResampling.BICUBIC image_mean = IMAGENET_STANDARD_MEAN image_std = IMAGENET_STANDARD_STD size = {"height": 224, "width": 224} From 9510e0fe8d720a0009dc3cd6cf32207b47120423 Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Thu, 27 Mar 2025 14:02:05 +0100 Subject: [PATCH 03/18] address review issues and add segmentation maps logic --- .../models/beit/image_processing_beit_fast.py | 201 ++++++++++++++++-- .../models/beit/test_image_processing_beit.py | 19 +- 2 files changed, 196 insertions(+), 24 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index 5c5bb8208209..3c077d3d9d7d 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -14,27 +14,47 @@ # limitations under the License. """Fast Image processor class for Beit.""" -from ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast -from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, PILImageResampling -from ...utils import add_start_docstrings +import torch +from typing import Optional, Union, Dict + +from ...image_processing_utils import BatchFeature +from ...image_processing_utils_fast import ( + BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, + BaseImageProcessorFast, + DefaultFastImageProcessorKwargs, + BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, + group_images_by_shape, + reorder_images, +) +from ...processing_utils import Unpack +from ...image_utils import ( + IMAGENET_STANDARD_MEAN, + IMAGENET_STANDARD_STD, + PILImageResampling, + ImageInput, + SizeDict, + validate_kwargs, + pil_torch_interpolation_mapping, + ChannelDimension +) +from ...utils import add_start_docstrings, TensorType + + +class BeitFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + do_reduce_labels: Optional[bool] @add_start_docstrings( "Constructs a fast Beit image processor.", BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, + """ + do_reduce_labels (`bool`, *optional*, defaults to `self.do_reduce_labels`): + Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 + is used for background, and background itself is not included in all classes of a dataset (e.g. + ADE20k). The background label will be replaced by 255. + """, ) class BeitImageProcessorFast(BaseImageProcessorFast): - # This generated class can be used as a starting point for the fast image processor. - # if the image processor is only used for simple augmentations, such as resizing, center cropping, rescaling, or normalizing, - # only the default values should be set in the class. - # If the image processor requires more complex augmentations, methods from BaseImageProcessorFast can be overridden. - # In most cases, only the `_preprocess` method should be overridden. - - # For an example of a fast image processor requiring more complex augmentations, see `LlavaNextImageProcessorFast`. - - # Default values should be checked against the slow image processor - # None values left after checking can be removed - # resample = PILImageResampling.BILINEAR resample = PILImageResampling.BICUBIC image_mean = IMAGENET_STANDARD_MEAN image_std = IMAGENET_STANDARD_STD @@ -45,8 +65,159 @@ class BeitImageProcessorFast(BaseImageProcessorFast): do_center_crop = False do_rescale = True do_normalize = True - do_convert_rgb = None do_reduce_labels = False + valid_kwargs = BeitFastImageProcessorKwargs + + def __init__(self, **kwargs: Unpack[BeitFastImageProcessorKwargs]): + super().__init__(**kwargs) + + def reduce_label(self, label: list["torch.Tensor"]): + label = torch.where(label == 0, torch.tensor(255, dtype=label.dtype), label) + label = label - 1 + label = torch.where(label == 254, torch.tensor(255, dtype=label.dtype), label) + + return label + + def _preprocess( + self, + images: list["torch.Tensor"], + do_reduce_labels: bool, + do_resize: bool, + size: SizeDict, + interpolation: Optional["F.InterpolationMode"], + do_center_crop: bool, + crop_size: SizeDict, + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: Optional[Union[float, list[float]]], + image_std: Optional[Union[float, list[float]]], + return_tensors: Optional[Union[str, TensorType]], + **kwargs, + ) -> BatchFeature: + if do_reduce_labels: + images = self.reduce_label(images) + + # Group images by size for batched resizing + grouped_images, grouped_images_index = group_images_by_shape(images) + resized_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + if do_resize: + stacked_images = self.resize(image=stacked_images, size=size, interpolation=interpolation) + resized_images_grouped[shape] = stacked_images + resized_images = reorder_images(resized_images_grouped, grouped_images_index) + + # Group images by size for further processing + # Needed in case do_resize is False, or resize returns images with different sizes + grouped_images, grouped_images_index = group_images_by_shape(resized_images) + processed_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + if do_center_crop: + stacked_images = self.center_crop(stacked_images, crop_size) + # Fused rescale and normalize + stacked_images = self.rescale_and_normalize( + stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std + ) + processed_images_grouped[shape] = stacked_images + + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images + return processed_images + + def _preprocess_images( + self, + images: ImageInput, + **kwargs + ): + return self._preprocess( + images, + **kwargs, + ) + + def _preprocess_segmentation_maps( + self, + segmentation_maps, + **kwargs, + ): + """Preprocesses a single segmentation map.""" + # Add an axis to the segmentation maps for transformations. + kwargs["do_normalize"] = False + kwargs["do_rescale"] = False + kwargs["input_data_format"] = ChannelDimension.FIRST + segmentation_maps = self._preprocess( + images=segmentation_maps, + **kwargs + ) + segmentation_maps = [seg_map.to(torch.uint64) for seg_map in segmentation_maps] + return segmentation_maps + + def __call__(self, images, segmentation_maps=None, **kwargs): + # Overrides the `__call__` method of the `Preprocessor` class such that the images and segmentation maps can both + # be passed in as positional arguments. + return super().__call__(images, segmentation_maps=segmentation_maps, **kwargs) + + @add_start_docstrings( + "Constructs a fast Beit image processor.", + BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, + """ + do_reduce_labels (`bool`, *optional*, defaults to `self.do_reduce_labels`): + Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 + is used for background, and background itself is not included in all classes of a dataset (e.g. + ADE20k). The background label will be replaced by 255. + """, + ) + def preprocess(self, images: ImageInput, segmentation_maps: Optional[ImageInput] = None, **kwargs: Unpack[DefaultFastImageProcessorKwargs]) -> BatchFeature: + validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self.valid_kwargs.__annotations__.keys()) + # Set default kwargs from self. This ensures that if a kwarg is not provided + # by the user, it gets its default value from the instance, or is set to None. + for kwarg_name in self.valid_kwargs.__annotations__: + kwargs.setdefault(kwarg_name, getattr(self, kwarg_name, None)) + + # Extract parameters that are only used for preparing the input images + do_convert_rgb = kwargs.pop("do_convert_rgb") + input_data_format = kwargs.pop("input_data_format") + device = kwargs.pop("device") + # Prepare input images + images = self._prepare_input_images( + images=images, do_convert_rgb=do_convert_rgb, input_data_format=input_data_format, device=device + ) + + # Prepare segmentation maps + if segmentation_maps: + segmentation_maps = self._prepare_input_images( + images=segmentation_maps, do_convert_rgb=do_convert_rgb, input_data_format=input_data_format, device=device + ) + + # Update kwargs that need further processing before being validated + kwargs = self._further_process_kwargs(**kwargs) + + # Validate kwargs + self._validate_preprocess_kwargs(**kwargs) + + # torch resize uses interpolation instead of resample + resample = kwargs.pop("resample") + kwargs["interpolation"] = ( + pil_torch_interpolation_mapping[resample] if isinstance(resample, (PILImageResampling, int)) else resample + ) + + # Pop kwargs that are not needed in _preprocess + kwargs.pop("default_to_square") + kwargs.pop("data_format") + + images = self._preprocess_images( + images=images, + **kwargs, + ) + data = {"pixel_values": images} + + if segmentation_maps: + segmentation_maps = self._preprocess_segmentation_maps( + segmentation_maps=segmentation_maps, + **kwargs, + ) + data["labels"] = segmentation_maps + + return BatchFeature(data=data, tensor_type=kwargs["return_tensors"]) __all__ = ["BeitImageProcessorFast"] diff --git a/tests/models/beit/test_image_processing_beit.py b/tests/models/beit/test_image_processing_beit.py index 95ecbb11caa5..db0ff30ece62 100644 --- a/tests/models/beit/test_image_processing_beit.py +++ b/tests/models/beit/test_image_processing_beit.py @@ -133,15 +133,16 @@ def image_processor_dict(self): return self.image_processor_tester.prepare_image_processor_dict() def test_image_processor_properties(self): - image_processing = self.image_processing_class(**self.image_processor_dict) - self.assertTrue(hasattr(image_processing, "do_resize")) - self.assertTrue(hasattr(image_processing, "size")) - self.assertTrue(hasattr(image_processing, "do_center_crop")) - self.assertTrue(hasattr(image_processing, "center_crop")) - self.assertTrue(hasattr(image_processing, "do_normalize")) - self.assertTrue(hasattr(image_processing, "image_mean")) - self.assertTrue(hasattr(image_processing, "image_std")) - self.assertTrue(hasattr(image_processing, "do_reduce_labels")) + for image_processing_class in self.image_processor_list: + image_processing = self.image_processing_class(**self.image_processor_dict) + self.assertTrue(hasattr(image_processing, "do_resize")) + self.assertTrue(hasattr(image_processing, "size")) + self.assertTrue(hasattr(image_processing, "do_center_crop")) + self.assertTrue(hasattr(image_processing, "center_crop")) + self.assertTrue(hasattr(image_processing, "do_normalize")) + self.assertTrue(hasattr(image_processing, "image_mean")) + self.assertTrue(hasattr(image_processing, "image_std")) + self.assertTrue(hasattr(image_processing, "do_reduce_labels")) def test_image_processor_from_dict_with_kwargs(self): image_processor = self.image_processing_class.from_dict(self.image_processor_dict) From a09b91cd3bee4c1269675bcfaf44f2e6c1806180 Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Thu, 27 Mar 2025 14:15:19 +0100 Subject: [PATCH 04/18] style --- .../models/beit/image_processing_beit_fast.py | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index 3c077d3d9d7d..18935959d188 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -14,30 +14,34 @@ # limitations under the License. """Fast Image processor class for Beit.""" +from typing import Optional, Union + import torch -from typing import Optional, Union, Dict +from torchvision.transforms import functional as F from ...image_processing_utils import BatchFeature from ...image_processing_utils_fast import ( BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, group_images_by_shape, reorder_images, ) -from ...processing_utils import Unpack from ...image_utils import ( IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, - PILImageResampling, + ChannelDimension, ImageInput, + PILImageResampling, SizeDict, - validate_kwargs, pil_torch_interpolation_mapping, - ChannelDimension + validate_kwargs, +) +from ...processing_utils import Unpack +from ...utils import ( + TensorType, + add_start_docstrings, ) -from ...utils import add_start_docstrings, TensorType class BeitFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): @@ -70,7 +74,7 @@ class BeitImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[BeitFastImageProcessorKwargs]): super().__init__(**kwargs) - + def reduce_label(self, label: list["torch.Tensor"]): label = torch.where(label == 0, torch.tensor(255, dtype=label.dtype), label) label = label - 1 @@ -124,11 +128,7 @@ def _preprocess( processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images return processed_images - def _preprocess_images( - self, - images: ImageInput, - **kwargs - ): + def _preprocess_images(self, images: ImageInput, **kwargs): return self._preprocess( images, **kwargs, @@ -144,10 +144,7 @@ def _preprocess_segmentation_maps( kwargs["do_normalize"] = False kwargs["do_rescale"] = False kwargs["input_data_format"] = ChannelDimension.FIRST - segmentation_maps = self._preprocess( - images=segmentation_maps, - **kwargs - ) + segmentation_maps = self._preprocess(images=segmentation_maps, **kwargs) segmentation_maps = [seg_map.to(torch.uint64) for seg_map in segmentation_maps] return segmentation_maps @@ -166,7 +163,12 @@ def __call__(self, images, segmentation_maps=None, **kwargs): ADE20k). The background label will be replaced by 255. """, ) - def preprocess(self, images: ImageInput, segmentation_maps: Optional[ImageInput] = None, **kwargs: Unpack[DefaultFastImageProcessorKwargs]) -> BatchFeature: + def preprocess( + self, + images: ImageInput, + segmentation_maps: Optional[ImageInput] = None, + **kwargs: Unpack[DefaultFastImageProcessorKwargs], + ) -> BatchFeature: validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self.valid_kwargs.__annotations__.keys()) # Set default kwargs from self. This ensures that if a kwarg is not provided # by the user, it gets its default value from the instance, or is set to None. @@ -185,7 +187,10 @@ def preprocess(self, images: ImageInput, segmentation_maps: Optional[ImageInput] # Prepare segmentation maps if segmentation_maps: segmentation_maps = self._prepare_input_images( - images=segmentation_maps, do_convert_rgb=do_convert_rgb, input_data_format=input_data_format, device=device + images=segmentation_maps, + do_convert_rgb=do_convert_rgb, + input_data_format=input_data_format, + device=device, ) # Update kwargs that need further processing before being validated @@ -216,7 +221,7 @@ def preprocess(self, images: ImageInput, segmentation_maps: Optional[ImageInput] **kwargs, ) data["labels"] = segmentation_maps - + return BatchFeature(data=data, tensor_type=kwargs["return_tensors"]) From 85ced8fbd6fd46f028d2842252ad89fe57f4fb0f Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Thu, 27 Mar 2025 17:29:29 +0100 Subject: [PATCH 05/18] chore: adding tests --- .../models/beit/image_processing_beit_fast.py | 63 +++- .../models/beit/test_image_processing_beit.py | 294 ++++++++++-------- 2 files changed, 211 insertions(+), 146 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index 18935959d188..a0fd655abef1 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -14,7 +14,7 @@ # limitations under the License. """Fast Image processor class for Beit.""" -from typing import Optional, Union +from typing import List, Optional, Tuple, Union import torch from torchvision.transforms import functional as F @@ -34,6 +34,7 @@ ImageInput, PILImageResampling, SizeDict, + is_torch_tensor, pil_torch_interpolation_mapping, validate_kwargs, ) @@ -128,12 +129,6 @@ def _preprocess( processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images return processed_images - def _preprocess_images(self, images: ImageInput, **kwargs): - return self._preprocess( - images, - **kwargs, - ) - def _preprocess_segmentation_maps( self, segmentation_maps, @@ -144,8 +139,7 @@ def _preprocess_segmentation_maps( kwargs["do_normalize"] = False kwargs["do_rescale"] = False kwargs["input_data_format"] = ChannelDimension.FIRST - segmentation_maps = self._preprocess(images=segmentation_maps, **kwargs) - segmentation_maps = [seg_map.to(torch.uint64) for seg_map in segmentation_maps] + segmentation_maps = self._preprocess(images=segmentation_maps, **kwargs).to(torch.int64) return segmentation_maps def __call__(self, images, segmentation_maps=None, **kwargs): @@ -185,7 +179,7 @@ def preprocess( ) # Prepare segmentation maps - if segmentation_maps: + if segmentation_maps is not None: segmentation_maps = self._prepare_input_images( images=segmentation_maps, do_convert_rgb=do_convert_rgb, @@ -209,20 +203,63 @@ def preprocess( kwargs.pop("default_to_square") kwargs.pop("data_format") - images = self._preprocess_images( + images = self._preprocess( images=images, **kwargs, ) data = {"pixel_values": images} - if segmentation_maps: + if segmentation_maps is not None: segmentation_maps = self._preprocess_segmentation_maps( segmentation_maps=segmentation_maps, **kwargs, ) data["labels"] = segmentation_maps - return BatchFeature(data=data, tensor_type=kwargs["return_tensors"]) + return BatchFeature(data=data) + + def post_process_semantic_segmentation(self, outputs, target_sizes: List[Tuple] = None): + """ + Converts the output of [`BeitForSemanticSegmentation`] into semantic segmentation maps. Only supports PyTorch. + + Args: + outputs ([`BeitForSemanticSegmentation`]): + Raw outputs of the model. + target_sizes (`List[Tuple]` of length `batch_size`, *optional*): + List of tuples corresponding to the requested final size (height, width) of each prediction. If unset, + predictions will not be resized. + + Returns: + semantic_segmentation: `List[torch.Tensor]` of length `batch_size`, where each item is a semantic + segmentation map of shape (height, width) corresponding to the target_sizes entry (if `target_sizes` is + specified). Each entry of each `torch.Tensor` correspond to a semantic class id. + """ + # TODO: add support for other frameworks + logits = outputs.logits + + # Resize logits and compute semantic segmentation maps + if target_sizes is not None: + if len(logits) != len(target_sizes): + raise ValueError( + "Make sure that you pass in as many target sizes as the batch dimension of the logits" + ) + + if is_torch_tensor(target_sizes): + target_sizes = target_sizes.numpy() + + semantic_segmentation = [] + + for idx in range(len(logits)): + resized_logits = torch.nn.functional.interpolate( + logits[idx].unsqueeze(dim=0), size=target_sizes[idx], mode="bilinear", align_corners=False + ) + semantic_map = resized_logits[0].argmax(dim=0) + semantic_segmentation.append(semantic_map) + else: + semantic_segmentation = logits.argmax(dim=1) + semantic_segmentation = [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0])] + + return semantic_segmentation __all__ = ["BeitImageProcessorFast"] diff --git a/tests/models/beit/test_image_processing_beit.py b/tests/models/beit/test_image_processing_beit.py index db0ff30ece62..d77b80d3d9d6 100644 --- a/tests/models/beit/test_image_processing_beit.py +++ b/tests/models/beit/test_image_processing_beit.py @@ -134,7 +134,7 @@ def image_processor_dict(self): def test_image_processor_properties(self): for image_processing_class in self.image_processor_list: - image_processing = self.image_processing_class(**self.image_processor_dict) + image_processing = image_processing_class(**self.image_processor_dict) self.assertTrue(hasattr(image_processing, "do_resize")) self.assertTrue(hasattr(image_processing, "size")) self.assertTrue(hasattr(image_processing, "do_center_crop")) @@ -145,148 +145,176 @@ def test_image_processor_properties(self): self.assertTrue(hasattr(image_processing, "do_reduce_labels")) def test_image_processor_from_dict_with_kwargs(self): - image_processor = self.image_processing_class.from_dict(self.image_processor_dict) - self.assertEqual(image_processor.size, {"height": 20, "width": 20}) - self.assertEqual(image_processor.crop_size, {"height": 18, "width": 18}) - self.assertEqual(image_processor.do_reduce_labels, False) - - image_processor = self.image_processing_class.from_dict( - self.image_processor_dict, size=42, crop_size=84, do_reduce_labels=True - ) - self.assertEqual(image_processor.size, {"height": 42, "width": 42}) - self.assertEqual(image_processor.crop_size, {"height": 84, "width": 84}) - self.assertEqual(image_processor.do_reduce_labels, True) + for image_processing_class in self.image_processor_list: + image_processor = image_processing_class.from_dict(self.image_processor_dict) + self.assertEqual(image_processor.size, {"height": 20, "width": 20}) + self.assertEqual(image_processor.crop_size, {"height": 18, "width": 18}) + self.assertEqual(image_processor.do_reduce_labels, False) + + image_processor = image_processing_class.from_dict( + self.image_processor_dict, size=42, crop_size=84, do_reduce_labels=True + ) + self.assertEqual(image_processor.size, {"height": 42, "width": 42}) + self.assertEqual(image_processor.crop_size, {"height": 84, "width": 84}) + self.assertEqual(image_processor.do_reduce_labels, True) def test_call_segmentation_maps(self): - # Initialize image_processing - image_processing = self.image_processing_class(**self.image_processor_dict) - # create random PyTorch tensors - image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, torchify=True) - maps = [] - for image in image_inputs: - self.assertIsInstance(image, torch.Tensor) - maps.append(torch.zeros(image.shape[-2:]).long()) - - # Test not batched input - encoding = image_processing(image_inputs[0], maps[0], return_tensors="pt") - self.assertEqual( - encoding["pixel_values"].shape, - ( - 1, - self.image_processor_tester.num_channels, - self.image_processor_tester.crop_size["height"], - self.image_processor_tester.crop_size["width"], - ), - ) - self.assertEqual( - encoding["labels"].shape, - ( - 1, - self.image_processor_tester.crop_size["height"], - self.image_processor_tester.crop_size["width"], - ), - ) - self.assertEqual(encoding["labels"].dtype, torch.long) - self.assertTrue(encoding["labels"].min().item() >= 0) - self.assertTrue(encoding["labels"].max().item() <= 255) - - # Test batched - encoding = image_processing(image_inputs, maps, return_tensors="pt") - self.assertEqual( - encoding["pixel_values"].shape, - ( - self.image_processor_tester.batch_size, - self.image_processor_tester.num_channels, - self.image_processor_tester.crop_size["height"], - self.image_processor_tester.crop_size["width"], - ), - ) - self.assertEqual( - encoding["labels"].shape, - ( - self.image_processor_tester.batch_size, - self.image_processor_tester.crop_size["height"], - self.image_processor_tester.crop_size["width"], - ), - ) - self.assertEqual(encoding["labels"].dtype, torch.long) - self.assertTrue(encoding["labels"].min().item() >= 0) - self.assertTrue(encoding["labels"].max().item() <= 255) - - # Test not batched input (PIL images) - image, segmentation_map = prepare_semantic_single_inputs() - - encoding = image_processing(image, segmentation_map, return_tensors="pt") - self.assertEqual( - encoding["pixel_values"].shape, - ( - 1, - self.image_processor_tester.num_channels, - self.image_processor_tester.crop_size["height"], - self.image_processor_tester.crop_size["width"], - ), - ) - self.assertEqual( - encoding["labels"].shape, - ( - 1, - self.image_processor_tester.crop_size["height"], - self.image_processor_tester.crop_size["width"], - ), - ) - self.assertEqual(encoding["labels"].dtype, torch.long) - self.assertTrue(encoding["labels"].min().item() >= 0) - self.assertTrue(encoding["labels"].max().item() <= 255) - - # Test batched input (PIL images) - images, segmentation_maps = prepare_semantic_batch_inputs() - - encoding = image_processing(images, segmentation_maps, return_tensors="pt") - self.assertEqual( - encoding["pixel_values"].shape, - ( - 2, - self.image_processor_tester.num_channels, - self.image_processor_tester.crop_size["height"], - self.image_processor_tester.crop_size["width"], - ), - ) - self.assertEqual( - encoding["labels"].shape, - ( - 2, - self.image_processor_tester.crop_size["height"], - self.image_processor_tester.crop_size["width"], - ), - ) - self.assertEqual(encoding["labels"].dtype, torch.long) - self.assertTrue(encoding["labels"].min().item() >= 0) - self.assertTrue(encoding["labels"].max().item() <= 255) + for image_processing_class in self.image_processor_list: + # Initialize image_processing + image_processing = image_processing_class(**self.image_processor_dict) + # create random PyTorch tensors + image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, torchify=True) + maps = [] + for image in image_inputs: + self.assertIsInstance(image, torch.Tensor) + maps.append(torch.zeros(image.shape[-2:]).long()) + + # Test not batched input + encoding = image_processing(image_inputs[0], maps[0], return_tensors="pt") + self.assertEqual( + encoding["pixel_values"].shape, + ( + 1, + self.image_processor_tester.num_channels, + self.image_processor_tester.crop_size["height"], + self.image_processor_tester.crop_size["width"], + ), + ) + self.assertEqual( + encoding["labels"].shape, + ( + 1, + self.image_processor_tester.crop_size["height"], + self.image_processor_tester.crop_size["width"], + ), + ) + self.assertEqual(encoding["labels"].dtype, torch.long) + self.assertTrue(encoding["labels"].min().item() >= 0) + self.assertTrue(encoding["labels"].max().item() <= 255) + + # Test batched + encoding = image_processing(image_inputs, maps, return_tensors="pt") + self.assertEqual( + encoding["pixel_values"].shape, + ( + self.image_processor_tester.batch_size, + self.image_processor_tester.num_channels, + self.image_processor_tester.crop_size["height"], + self.image_processor_tester.crop_size["width"], + ), + ) + self.assertEqual( + encoding["labels"].shape, + ( + self.image_processor_tester.batch_size, + self.image_processor_tester.crop_size["height"], + self.image_processor_tester.crop_size["width"], + ), + ) + self.assertEqual(encoding["labels"].dtype, torch.long) + self.assertTrue(encoding["labels"].min().item() >= 0) + self.assertTrue(encoding["labels"].max().item() <= 255) + + # Test not batched input (PIL images) + image, segmentation_map = prepare_semantic_single_inputs() + + encoding = image_processing(image, segmentation_map, return_tensors="pt") + self.assertEqual( + encoding["pixel_values"].shape, + ( + 1, + self.image_processor_tester.num_channels, + self.image_processor_tester.crop_size["height"], + self.image_processor_tester.crop_size["width"], + ), + ) + self.assertEqual( + encoding["labels"].shape, + ( + 1, + self.image_processor_tester.crop_size["height"], + self.image_processor_tester.crop_size["width"], + ), + ) + self.assertEqual(encoding["labels"].dtype, torch.long) + self.assertTrue(encoding["labels"].min().item() >= 0) + self.assertTrue(encoding["labels"].max().item() <= 255) + + # Test batched input (PIL images) + images, segmentation_maps = prepare_semantic_batch_inputs() + + encoding = image_processing(images, segmentation_maps, return_tensors="pt") + self.assertEqual( + encoding["pixel_values"].shape, + ( + 2, + self.image_processor_tester.num_channels, + self.image_processor_tester.crop_size["height"], + self.image_processor_tester.crop_size["width"], + ), + ) + self.assertEqual( + encoding["labels"].shape, + ( + 2, + self.image_processor_tester.crop_size["height"], + self.image_processor_tester.crop_size["width"], + ), + ) + self.assertEqual(encoding["labels"].dtype, torch.long) + self.assertTrue(encoding["labels"].min().item() >= 0) + self.assertTrue(encoding["labels"].max().item() <= 255) def test_reduce_labels(self): - # Initialize image_processing - image_processing = self.image_processing_class(**self.image_processor_dict) + for image_processing_class in self.image_processor_list: + # Initialize image_processing + image_processing = image_processing_class(**self.image_processor_dict) - # ADE20k has 150 classes, and the background is included, so labels should be between 0 and 150 - image, map = prepare_semantic_single_inputs() - encoding = image_processing(image, map, return_tensors="pt") - self.assertTrue(encoding["labels"].min().item() >= 0) - self.assertTrue(encoding["labels"].max().item() <= 150) + # ADE20k has 150 classes, and the background is included, so labels should be between 0 and 150 + image, map = prepare_semantic_single_inputs() + encoding = image_processing(image, map, return_tensors="pt") + self.assertTrue(encoding["labels"].min().item() >= 0) + self.assertTrue(encoding["labels"].max().item() <= 150) - image_processing.do_reduce_labels = True - encoding = image_processing(image, map, return_tensors="pt") - self.assertTrue(encoding["labels"].min().item() >= 0) - self.assertTrue(encoding["labels"].max().item() <= 255) + image_processing.do_reduce_labels = True + encoding = image_processing(image, map, return_tensors="pt") + self.assertTrue(encoding["labels"].min().item() >= 0) + self.assertTrue(encoding["labels"].max().item() <= 255) def test_removed_deprecated_kwargs(self): image_processor_dict = dict(self.image_processor_dict) image_processor_dict.pop("do_reduce_labels", None) image_processor_dict["reduce_labels"] = True - # test we are able to create the image processor with the deprecated kwargs - image_processor = self.image_processing_class(**image_processor_dict) - self.assertEqual(image_processor.do_reduce_labels, True) + for image_processing_class in self.image_processor_list: + # test we are able to create the image processor with the deprecated kwargs + image_processor = image_processing_class(**image_processor_dict) + self.assertEqual(image_processor.do_reduce_labels, True) + + # test we still support reduce_labels with config + image_processor = image_processing_class.from_dict(image_processor_dict) + self.assertEqual(image_processor.do_reduce_labels, True) + + def test_slow_fast_equivalence(self): + if not self.test_slow_image_processor or not self.test_fast_image_processor: + self.skipTest(reason="Skipping slow/fast equivalence test") + + if self.image_processing_class is None or self.fast_image_processing_class is None: + self.skipTest(reason="Skipping slow/fast equivalence test as one of the image processors is not defined") - # test we still support reduce_labels with config - image_processor = self.image_processing_class.from_dict(image_processor_dict) - self.assertEqual(image_processor.do_reduce_labels, True) + dataset = load_dataset("hf-internal-testing/fixtures_ade20k", split="test", trust_remote_code=True) + + dummy_image = Image.open(dataset[0]["file"]) + dummy_map = Image.open(dataset[1]["file"]) + + image_processor_slow = self.image_processing_class(**self.image_processor_dict) + image_processor_fast = self.fast_image_processing_class(**self.image_processor_dict) + + image_encoding_slow = image_processor_slow(dummy_image, segmentation_maps=dummy_map, return_tensors="pt") + image_encoding_fast = image_processor_fast(dummy_image, segmentation_maps=dummy_map, return_tensors="pt") + + self.assertTrue(torch.allclose(image_encoding_slow.pixel_values, image_encoding_fast.pixel_values, atol=1e-1)) + self.assertLessEqual( + torch.mean(torch.abs(image_encoding_slow.pixel_values - image_encoding_fast.pixel_values)).item(), 1e-3 + ) + self.assertTrue(torch.allclose(image_encoding_slow.labels, image_encoding_fast.labels, atol=1e-1)) From bb8174895ee347f70cf7ce6116a195a9d84d492b Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Thu, 27 Mar 2025 17:50:49 +0100 Subject: [PATCH 06/18] reduce label test --- .../models/beit/image_processing_beit_fast.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index a0fd655abef1..4720a80f65b0 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -76,10 +76,13 @@ class BeitImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[BeitFastImageProcessorKwargs]): super().__init__(**kwargs) - def reduce_label(self, label: list["torch.Tensor"]): - label = torch.where(label == 0, torch.tensor(255, dtype=label.dtype), label) - label = label - 1 - label = torch.where(label == 254, torch.tensor(255, dtype=label.dtype), label) + def reduce_label(self, labels: list["torch.Tensor"]): + for idx in range(len(labels)): + label = labels[idx] + label = torch.where(label == 0, torch.tensor(255, dtype=label.dtype), label) + label = label - 1 + label = torch.where(label == 254, torch.tensor(255, dtype=label.dtype), label) + labels[idx] = label return label From d8c4e1321fb3f690647b0a13c0f279df7a771cd8 Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Thu, 27 Mar 2025 18:29:06 +0100 Subject: [PATCH 07/18] adding batched tests --- .../models/beit/image_processing_beit_fast.py | 14 ++++++++- .../models/beit/test_image_processing_beit.py | 30 ++++++++++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index 4720a80f65b0..e8ad636db350 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -14,7 +14,7 @@ # limitations under the License. """Fast Image processor class for Beit.""" -from typing import List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import torch from torchvision.transforms import functional as F @@ -43,6 +43,7 @@ TensorType, add_start_docstrings, ) +from ...utils.deprecation import deprecate_kwarg class BeitFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): @@ -76,6 +77,16 @@ class BeitImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[BeitFastImageProcessorKwargs]): super().__init__(**kwargs) + @classmethod + def from_dict(cls, image_processor_dict: Dict[str, Any], **kwargs): + """ + Overrides the `from_dict` method from the base class to save support of deprecated `reduce_labels` in old configs + """ + image_processor_dict = image_processor_dict.copy() + if "reduce_labels" in image_processor_dict: + image_processor_dict["do_reduce_labels"] = image_processor_dict.pop("reduce_labels") + return super().from_dict(image_processor_dict, **kwargs) + def reduce_label(self, labels: list["torch.Tensor"]): for idx in range(len(labels)): label = labels[idx] @@ -150,6 +161,7 @@ def __call__(self, images, segmentation_maps=None, **kwargs): # be passed in as positional arguments. return super().__call__(images, segmentation_maps=segmentation_maps, **kwargs) + @deprecate_kwarg("reduce_labels", new_name="do_reduce_labels", version="4.41.0") @add_start_docstrings( "Constructs a fast Beit image processor.", BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, diff --git a/tests/models/beit/test_image_processing_beit.py b/tests/models/beit/test_image_processing_beit.py index d77b80d3d9d6..16ed1bfda96c 100644 --- a/tests/models/beit/test_image_processing_beit.py +++ b/tests/models/beit/test_image_processing_beit.py @@ -302,10 +302,7 @@ def test_slow_fast_equivalence(self): if self.image_processing_class is None or self.fast_image_processing_class is None: self.skipTest(reason="Skipping slow/fast equivalence test as one of the image processors is not defined") - dataset = load_dataset("hf-internal-testing/fixtures_ade20k", split="test", trust_remote_code=True) - - dummy_image = Image.open(dataset[0]["file"]) - dummy_map = Image.open(dataset[1]["file"]) + dummy_image, dummy_map = prepare_semantic_single_inputs() image_processor_slow = self.image_processing_class(**self.image_processor_dict) image_processor_fast = self.fast_image_processing_class(**self.image_processor_dict) @@ -318,3 +315,28 @@ def test_slow_fast_equivalence(self): torch.mean(torch.abs(image_encoding_slow.pixel_values - image_encoding_fast.pixel_values)).item(), 1e-3 ) self.assertTrue(torch.allclose(image_encoding_slow.labels, image_encoding_fast.labels, atol=1e-1)) + + def test_slow_fast_equivalence_batched(self): + if not self.test_slow_image_processor or not self.test_fast_image_processor: + self.skipTest(reason="Skipping slow/fast equivalence test") + + if self.image_processing_class is None or self.fast_image_processing_class is None: + self.skipTest(reason="Skipping slow/fast equivalence test as one of the image processors is not defined") + + if hasattr(self.image_processor_tester, "do_center_crop") and self.image_processor_tester.do_center_crop: + self.skipTest( + reason="Skipping as do_center_crop is True and center_crop functions are not equivalent for fast and slow processors" + ) + + dummy_images, dummy_maps = prepare_semantic_batch_inputs() + + image_processor_slow = self.image_processing_class(**self.image_processor_dict) + image_processor_fast = self.fast_image_processing_class(**self.image_processor_dict) + + encoding_slow = image_processor_slow(dummy_images, segmentation_maps=dummy_maps, return_tensors="pt") + encoding_fast = image_processor_fast(dummy_images, segmentation_maps=dummy_maps, return_tensors="pt") + + self.assertTrue(torch.allclose(encoding_slow.pixel_values, encoding_fast.pixel_values, atol=1e-1)) + self.assertLessEqual( + torch.mean(torch.abs(encoding_slow.pixel_values - encoding_fast.pixel_values)).item(), 1e-3 + ) From 77e3fcf1ff21853e1252f24999df403e2c135d2e Mon Sep 17 00:00:00 2001 From: Aritra Roy Gosthipaty Date: Tue, 8 Apr 2025 10:49:32 +0530 Subject: [PATCH 08/18] Update src/transformers/models/beit/image_processing_beit_fast.py Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> --- src/transformers/models/beit/image_processing_beit_fast.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index e8ad636db350..cf8c6e7d8e9d 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -74,6 +74,8 @@ class BeitImageProcessorFast(BaseImageProcessorFast): do_reduce_labels = False valid_kwargs = BeitFastImageProcessorKwargs + @deprecate_kwarg("reduce_labels", new_name="do_reduce_labels", version="4.41.0") + @filter_out_non_signature_kwargs(extra=INIT_SERVICE_KWARGS) def __init__(self, **kwargs: Unpack[BeitFastImageProcessorKwargs]): super().__init__(**kwargs) From 4931ad715b194ba67c2f0b884ae525444a07af7f Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Tue, 8 Apr 2025 07:28:12 +0200 Subject: [PATCH 09/18] fix imports and make segmentation masks --- .../models/beit/image_processing_beit_fast.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index cf8c6e7d8e9d..a8d86d6e9daa 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -19,7 +19,7 @@ import torch from torchvision.transforms import functional as F -from ...image_processing_utils import BatchFeature +from ...image_processing_utils import BatchFeature, INIT_SERVICE_KWARGS from ...image_processing_utils_fast import ( BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast, @@ -37,11 +37,13 @@ is_torch_tensor, pil_torch_interpolation_mapping, validate_kwargs, + make_list_of_images, ) from ...processing_utils import Unpack from ...utils import ( TensorType, add_start_docstrings, + filter_out_non_signature_kwargs ) from ...utils.deprecation import deprecate_kwarg @@ -197,12 +199,7 @@ def preprocess( # Prepare segmentation maps if segmentation_maps is not None: - segmentation_maps = self._prepare_input_images( - images=segmentation_maps, - do_convert_rgb=do_convert_rgb, - input_data_format=input_data_format, - device=device, - ) + segmentation_maps = make_list_of_images(expected_ndims=1) # Update kwargs that need further processing before being validated kwargs = self._further_process_kwargs(**kwargs) From 6877afa8be392ed467cc0d6202e6acfad6225e67 Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Tue, 8 Apr 2025 08:50:50 +0200 Subject: [PATCH 10/18] fix tests --- .../models/beit/image_processing_beit_fast.py | 7 +------ tests/models/beit/test_image_processing_beit.py | 13 ------------- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index a8d86d6e9daa..21603997a03d 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -76,11 +76,6 @@ class BeitImageProcessorFast(BaseImageProcessorFast): do_reduce_labels = False valid_kwargs = BeitFastImageProcessorKwargs - @deprecate_kwarg("reduce_labels", new_name="do_reduce_labels", version="4.41.0") - @filter_out_non_signature_kwargs(extra=INIT_SERVICE_KWARGS) - def __init__(self, **kwargs: Unpack[BeitFastImageProcessorKwargs]): - super().__init__(**kwargs) - @classmethod def from_dict(cls, image_processor_dict: Dict[str, Any], **kwargs): """ @@ -199,7 +194,7 @@ def preprocess( # Prepare segmentation maps if segmentation_maps is not None: - segmentation_maps = make_list_of_images(expected_ndims=1) + segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=1) # Update kwargs that need further processing before being validated kwargs = self._further_process_kwargs(**kwargs) diff --git a/tests/models/beit/test_image_processing_beit.py b/tests/models/beit/test_image_processing_beit.py index 16ed1bfda96c..44559b33631e 100644 --- a/tests/models/beit/test_image_processing_beit.py +++ b/tests/models/beit/test_image_processing_beit.py @@ -281,19 +281,6 @@ def test_reduce_labels(self): self.assertTrue(encoding["labels"].min().item() >= 0) self.assertTrue(encoding["labels"].max().item() <= 255) - def test_removed_deprecated_kwargs(self): - image_processor_dict = dict(self.image_processor_dict) - image_processor_dict.pop("do_reduce_labels", None) - image_processor_dict["reduce_labels"] = True - - for image_processing_class in self.image_processor_list: - # test we are able to create the image processor with the deprecated kwargs - image_processor = image_processing_class(**image_processor_dict) - self.assertEqual(image_processor.do_reduce_labels, True) - - # test we still support reduce_labels with config - image_processor = image_processing_class.from_dict(image_processor_dict) - self.assertEqual(image_processor.do_reduce_labels, True) def test_slow_fast_equivalence(self): if not self.test_slow_image_processor or not self.test_fast_image_processor: From ce7e70482770dbad586ace2ced38d744b2bd0860 Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Tue, 8 Apr 2025 09:18:30 +0200 Subject: [PATCH 11/18] build segmentation maps --- src/transformers/models/beit/image_processing_beit_fast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index 21603997a03d..1bf0dbe3fecc 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -194,7 +194,7 @@ def preprocess( # Prepare segmentation maps if segmentation_maps is not None: - segmentation_maps = make_list_of_images(segmentation_maps, expected_ndims=1) + segmentation_maps = make_list_of_images(images=segmentation_maps, expected_ndims=2) # Update kwargs that need further processing before being validated kwargs = self._further_process_kwargs(**kwargs) From 0bd8b2d5fc3e60d0a72ef3465a6c5f13e44bcf9c Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Tue, 15 Apr 2025 17:39:24 +0200 Subject: [PATCH 12/18] all tests pass --- .../models/beit/image_processing_beit_fast.py | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index 1bf0dbe3fecc..92687b6ba0bb 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -38,6 +38,8 @@ pil_torch_interpolation_mapping, validate_kwargs, make_list_of_images, + to_numpy_array, + infer_channel_dimension_format, ) from ...processing_utils import Unpack from ...utils import ( @@ -142,18 +144,51 @@ def _preprocess( processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images return processed_images + # def _preprocess_segmentation_maps( + # self, + # segmentation_maps, + # **kwargs, + # ): + # """Preprocesses a single segmentation map.""" + # # Add an axis to the segmentation maps for transformations. + # kwargs["do_normalize"] = False + # kwargs["do_rescale"] = False + # kwargs["input_data_format"] = ChannelDimension.FIRST + # segmentation_maps = self._preprocess(images=segmentation_maps, **kwargs).to(torch.int64) + # return segmentation_maps def _preprocess_segmentation_maps( self, segmentation_maps, **kwargs, ): """Preprocesses a single segmentation map.""" - # Add an axis to the segmentation maps for transformations. + processed_segmentation_maps = list() + added_dimension = False # we will assume that the batch of maps will all either have added dims or not + for segmentation_map in segmentation_maps: + segmentation_map = to_numpy_array(segmentation_map) + # Add an axis to the segmentation maps for transformations. + if segmentation_map.ndim == 2: + segmentation_map = segmentation_map[None, ...] + added_dimension = True + input_data_format = ChannelDimension.FIRST + else: + added_dimension = False + if input_data_format is None: + input_data_format = infer_channel_dimension_format(segmentation_map, num_channels=1) + + processed_segmentation_maps.append(torch.tensor(segmentation_map)) + kwargs["do_normalize"] = False kwargs["do_rescale"] = False kwargs["input_data_format"] = ChannelDimension.FIRST - segmentation_maps = self._preprocess(images=segmentation_maps, **kwargs).to(torch.int64) - return segmentation_maps + processed_segmentation_maps = self._preprocess( + images=processed_segmentation_maps, + **kwargs + ) + if added_dimension: + processed_segmentation_maps = processed_segmentation_maps.squeeze(1) + processed_segmentation_maps = processed_segmentation_maps.to(torch.int64) + return processed_segmentation_maps def __call__(self, images, segmentation_maps=None, **kwargs): # Overrides the `__call__` method of the `Preprocessor` class such that the images and segmentation maps can both From 3383d8c18b0ce9df2c8ea6643de4b58acdc98e5d Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Tue, 15 Apr 2025 17:48:07 +0200 Subject: [PATCH 13/18] style --- .../models/beit/image_processing_beit_fast.py | 32 +++++-------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index 92687b6ba0bb..ae384d76f803 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -19,7 +19,7 @@ import torch from torchvision.transforms import functional as F -from ...image_processing_utils import BatchFeature, INIT_SERVICE_KWARGS +from ...image_processing_utils import BatchFeature from ...image_processing_utils_fast import ( BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast, @@ -34,19 +34,15 @@ ImageInput, PILImageResampling, SizeDict, + infer_channel_dimension_format, is_torch_tensor, - pil_torch_interpolation_mapping, - validate_kwargs, make_list_of_images, + pil_torch_interpolation_mapping, to_numpy_array, - infer_channel_dimension_format, + validate_kwargs, ) from ...processing_utils import Unpack -from ...utils import ( - TensorType, - add_start_docstrings, - filter_out_non_signature_kwargs -) +from ...utils import TensorType, add_start_docstrings from ...utils.deprecation import deprecate_kwarg @@ -144,18 +140,6 @@ def _preprocess( processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images return processed_images - # def _preprocess_segmentation_maps( - # self, - # segmentation_maps, - # **kwargs, - # ): - # """Preprocesses a single segmentation map.""" - # # Add an axis to the segmentation maps for transformations. - # kwargs["do_normalize"] = False - # kwargs["do_rescale"] = False - # kwargs["input_data_format"] = ChannelDimension.FIRST - # segmentation_maps = self._preprocess(images=segmentation_maps, **kwargs).to(torch.int64) - # return segmentation_maps def _preprocess_segmentation_maps( self, segmentation_maps, @@ -164,7 +148,7 @@ def _preprocess_segmentation_maps( """Preprocesses a single segmentation map.""" processed_segmentation_maps = list() added_dimension = False # we will assume that the batch of maps will all either have added dims or not - for segmentation_map in segmentation_maps: + for segmentation_map in segmentation_maps: segmentation_map = to_numpy_array(segmentation_map) # Add an axis to the segmentation maps for transformations. if segmentation_map.ndim == 2: @@ -175,9 +159,9 @@ def _preprocess_segmentation_maps( added_dimension = False if input_data_format is None: input_data_format = infer_channel_dimension_format(segmentation_map, num_channels=1) - + processed_segmentation_maps.append(torch.tensor(segmentation_map)) - + kwargs["do_normalize"] = False kwargs["do_rescale"] = False kwargs["input_data_format"] = ChannelDimension.FIRST From 49cc6bdc11b2a25eb580575a0e12b3b3e507627e Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Tue, 15 Apr 2025 17:51:57 +0200 Subject: [PATCH 14/18] style fix --- src/transformers/models/beit/image_processing_beit_fast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index ae384d76f803..e3588daa06c7 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -146,7 +146,7 @@ def _preprocess_segmentation_maps( **kwargs, ): """Preprocesses a single segmentation map.""" - processed_segmentation_maps = list() + processed_segmentation_maps = [] added_dimension = False # we will assume that the batch of maps will all either have added dims or not for segmentation_map in segmentation_maps: segmentation_map = to_numpy_array(segmentation_map) From 0a5e38c723a736abeb78bb41e09f93b6145f0de0 Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Thu, 17 Apr 2025 12:29:07 +0200 Subject: [PATCH 15/18] style --- src/transformers/models/beit/image_processing_beit_fast.py | 7 ++----- tests/models/beit/test_image_processing_beit.py | 1 - 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index e3588daa06c7..ac88a356cea5 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -147,7 +147,7 @@ def _preprocess_segmentation_maps( ): """Preprocesses a single segmentation map.""" processed_segmentation_maps = [] - added_dimension = False # we will assume that the batch of maps will all either have added dims or not + added_dimension = False # we will assume that the batch of maps will all either have added dims or not for segmentation_map in segmentation_maps: segmentation_map = to_numpy_array(segmentation_map) # Add an axis to the segmentation maps for transformations. @@ -165,10 +165,7 @@ def _preprocess_segmentation_maps( kwargs["do_normalize"] = False kwargs["do_rescale"] = False kwargs["input_data_format"] = ChannelDimension.FIRST - processed_segmentation_maps = self._preprocess( - images=processed_segmentation_maps, - **kwargs - ) + processed_segmentation_maps = self._preprocess(images=processed_segmentation_maps, **kwargs) if added_dimension: processed_segmentation_maps = processed_segmentation_maps.squeeze(1) processed_segmentation_maps = processed_segmentation_maps.to(torch.int64) diff --git a/tests/models/beit/test_image_processing_beit.py b/tests/models/beit/test_image_processing_beit.py index 97a216c79f67..5a46279a6829 100644 --- a/tests/models/beit/test_image_processing_beit.py +++ b/tests/models/beit/test_image_processing_beit.py @@ -280,7 +280,6 @@ def test_reduce_labels(self): self.assertTrue(encoding["labels"].min().item() >= 0) self.assertTrue(encoding["labels"].max().item() <= 255) - def test_slow_fast_equivalence(self): if not self.test_slow_image_processor or not self.test_fast_image_processor: self.skipTest(reason="Skipping slow/fast equivalence test") From 37e38277b6ed59989a473ca6e6fd7c6b5614b8f8 Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Tue, 29 Apr 2025 04:02:57 +0200 Subject: [PATCH 16/18] chore: delete demo.py file --- demo.py | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 demo.py diff --git a/demo.py b/demo.py deleted file mode 100644 index 4961cf27da01..000000000000 --- a/demo.py +++ /dev/null @@ -1,7 +0,0 @@ -from transformers import BeitImageProcessor, BeitImageProcessorFast - -im_pro = BeitImageProcessor(size={"height":20, "width":20}) -im_pro_fast = BeitImageProcessorFast(size={"height":20, "width": 20}) - -print(im_pro) -print(im_pro_fast) \ No newline at end of file From 4f5f002b6ebc04994db7a178c0752f3d7de7c32d Mon Sep 17 00:00:00 2001 From: ariG23498 Date: Tue, 29 Apr 2025 04:30:31 +0200 Subject: [PATCH 17/18] review suggestions --- .../models/beit/image_processing_beit_fast.py | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index ac88a356cea5..50dd6fe7f5bf 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -34,11 +34,9 @@ ImageInput, PILImageResampling, SizeDict, - infer_channel_dimension_format, is_torch_tensor, make_list_of_images, pil_torch_interpolation_mapping, - to_numpy_array, validate_kwargs, ) from ...processing_utils import Unpack @@ -147,27 +145,23 @@ def _preprocess_segmentation_maps( ): """Preprocesses a single segmentation map.""" processed_segmentation_maps = [] - added_dimension = False # we will assume that the batch of maps will all either have added dims or not for segmentation_map in segmentation_maps: - segmentation_map = to_numpy_array(segmentation_map) - # Add an axis to the segmentation maps for transformations. + segmentation_map = self._process_image( + segmentation_map, do_convert_rgb=False, input_data_format=ChannelDimension.FIRST + ) + if segmentation_map.ndim == 2: segmentation_map = segmentation_map[None, ...] - added_dimension = True - input_data_format = ChannelDimension.FIRST - else: - added_dimension = False - if input_data_format is None: - input_data_format = infer_channel_dimension_format(segmentation_map, num_channels=1) - processed_segmentation_maps.append(torch.tensor(segmentation_map)) + processed_segmentation_maps.append(segmentation_map) kwargs["do_normalize"] = False kwargs["do_rescale"] = False kwargs["input_data_format"] = ChannelDimension.FIRST processed_segmentation_maps = self._preprocess(images=processed_segmentation_maps, **kwargs) - if added_dimension: - processed_segmentation_maps = processed_segmentation_maps.squeeze(1) + + processed_segmentation_maps = processed_segmentation_maps.squeeze(1) + processed_segmentation_maps = processed_segmentation_maps.to(torch.int64) return processed_segmentation_maps From bb71922cff1b80e7214ba3461324ecff4178500c Mon Sep 17 00:00:00 2001 From: Aritra Roy Gosthipaty Date: Tue, 6 May 2025 21:59:47 +0530 Subject: [PATCH 18/18] Update docs/source/en/model_doc/beit.md Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> --- docs/source/en/model_doc/beit.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/model_doc/beit.md b/docs/source/en/model_doc/beit.md index c9206717e54a..e40fbdc9c889 100644 --- a/docs/source/en/model_doc/beit.md +++ b/docs/source/en/model_doc/beit.md @@ -149,7 +149,7 @@ If you're interested in submitting a resource to be included here, please feel f [[autodoc]] BeitImageProcessor - preprocess - + - post_process_semantic_segmentation ## BeitImageProcessorFast [[autodoc]] BeitImageProcessorFast