From 9b8536ab927824cfec5710b480954f7faf6dede6 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Sun, 4 Jan 2026 20:09:03 +0800 Subject: [PATCH 01/21] init --- src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 2 + .../models/auto/image_processing_auto.py | 1 + src/transformers/models/auto/modeling_auto.py | 1 + .../models/pp_doclayout_v3/__init__.py | 31 + .../configuration_pp_doclayout_v3.py | 352 +++ .../image_processing_pp_doclayout_v3.py | 316 +++ .../image_processing_pp_doclayout_v3_fast.py | 164 ++ .../modeling_pp_doclayout_v3.py | 2202 +++++++++++++++++ .../modular_pp_doclayout_v3.py | 1860 ++++++++++++++ utils/check_repo.py | 2 + 11 files changed, 4932 insertions(+) create mode 100644 src/transformers/models/pp_doclayout_v3/__init__.py create mode 100644 src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py create mode 100644 src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py create mode 100644 src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py create mode 100644 src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py create mode 100644 src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index 342f366f3bc8..7cbdc0ea6f7c 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -292,6 +292,7 @@ from .plbart import * from .poolformer import * from .pop2piano import * + from .pp_doclayout_v3 import * from .prompt_depth_anything import * from .prophetnet import * from .pvt import * diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 55dd1b820073..8aa508aa5657 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -331,6 +331,7 @@ ("plbart", "PLBartConfig"), ("poolformer", "PoolFormerConfig"), ("pop2piano", "Pop2PianoConfig"), + ("pp_doclayout_v3", "PPDocLayoutV3Config"), ("prompt_depth_anything", "PromptDepthAnythingConfig"), ("prophetnet", "ProphetNetConfig"), ("pvt", "PvtConfig"), @@ -799,6 +800,7 @@ ("plbart", "PLBart"), ("poolformer", "PoolFormer"), ("pop2piano", "Pop2Piano"), + ("pp_doclayout_v3", "PPDocLayoutV3"), ("prompt_depth_anything", "PromptDepthAnything"), ("prophetnet", "ProphetNet"), ("pvt", "PVT"), diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index 9d5b531def2a..c0ee68933f72 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -163,6 +163,7 @@ ("pixio", ("BitImageProcessor", "BitImageProcessorFast")), ("pixtral", ("PixtralImageProcessor", "PixtralImageProcessorFast")), ("poolformer", ("PoolFormerImageProcessor", "PoolFormerImageProcessorFast")), + ("pp_doclayout_v3", ("PPDocLayoutV3ImageProcessor", "PPDocLayoutV3ImageProcessorFast")), ("prompt_depth_anything", ("PromptDepthAnythingImageProcessor", "PromptDepthAnythingImageProcessorFast")), ("pvt", ("PvtImageProcessor", "PvtImageProcessorFast")), ("pvt_v2", ("PvtImageProcessor", "PvtImageProcessorFast")), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 8b151b68e1df..728a209b39e5 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -1140,6 +1140,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("dab-detr", "DabDetrForObjectDetection"), ("deformable_detr", "DeformableDetrForObjectDetection"), ("detr", "DetrForObjectDetection"), + ("pp_doclayout_v3", "PPDocLayoutV3ForObjectDetection"), ("rt_detr", "RTDetrForObjectDetection"), ("rt_detr_v2", "RTDetrV2ForObjectDetection"), ("table-transformer", "TableTransformerForObjectDetection"), diff --git a/src/transformers/models/pp_doclayout_v3/__init__.py b/src/transformers/models/pp_doclayout_v3/__init__.py new file mode 100644 index 000000000000..cb53946c81dc --- /dev/null +++ b/src/transformers/models/pp_doclayout_v3/__init__.py @@ -0,0 +1,31 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_pp_doclayout_v3 import * + from .image_processing_pp_doclayout_v3 import * + from .image_processing_pp_doclayout_v3_fast import * + from .modeling_pp_doclayout_v3 import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py new file mode 100644 index 000000000000..dfc26a6e36e8 --- /dev/null +++ b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py @@ -0,0 +1,352 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_pp_doclayout_v3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2025 The PaddlePaddle Team and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ...configuration_utils import PreTrainedConfig +from ...utils import logging +from ...utils.backbone_utils import verify_backbone_config_arguments +from ..auto import CONFIG_MAPPING, AutoConfig + + +logger = logging.get_logger(__name__) + + +def _default_id2label() -> dict[int, str]: + return { + 0: "abstract", + 1: "algorithm", + 2: "aside_text", + 3: "chart", + 4: "content", + 5: "formula", + 6: "doc_title", + 7: "figure_title", + 8: "footer", + 9: "footer", + 10: "footnote", + 11: "formula_number", + 12: "header", + 13: "header", + 14: "image", + 15: "formula", + 16: "number", + 17: "paragraph_title", + 18: "reference", + 19: "reference_content", + 20: "seal", + 21: "table", + 22: "text", + 23: "text", + 24: "vision_footnote", + } + + +class PPDocLayoutV3Config(PreTrainedConfig): + r""" + This is the configuration class to store the configuration of a [`PP-DocLayoutV3`]. It is used to instantiate a + RT-DETR model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the PP-DocLayoutV3 + [PaddlePaddle/PP-DocLayoutV3_safetensors](https://huggingface.co/PaddlePaddle/PP-DocLayoutV3_safetensors) architecture. + + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. + + Args: + initializer_range (`float`, *optional*, defaults to 0.01): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + initializer_bias_prior_prob (`float`, *optional*): + The prior probability used by the bias initializer to initialize biases for `enc_score_head` and `class_embed`. + If `None`, `prior_prob` computed as `prior_prob = 1 / (num_labels + 1)` while initializing model weights. + layer_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the layer normalization layers. + batch_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the batch normalization layers. + backbone_config (`Union[dict, "PreTrainedConfig"]`, *optional*, defaults to `RTDetrResNetConfig()`): + The configuration of the backbone model. + backbone (`str`, *optional*): + Name of backbone to use when `backbone_config` is `None`. If `use_pretrained_backbone` is `True`, this + will load the corresponding pretrained weights from the timm or transformers library. If `use_pretrained_backbone` + is `False`, this loads the backbone's config and uses that to initialize the backbone with random weights. + use_pretrained_backbone (`bool`, *optional*, defaults to `False`): + Whether to use pretrained weights for the backbone. + use_timm_backbone (`bool`, *optional*, defaults to `False`): + Whether to load `backbone` from the timm library. If `False`, the backbone is loaded from the transformers + library. + freeze_backbone_batch_norms (`bool`, *optional*, defaults to `True`): + Whether to freeze the batch normalization layers in the backbone. + backbone_kwargs (`dict`, *optional*): + Keyword arguments to be passed to AutoBackbone when loading from a checkpoint + e.g. `{'out_indices': (0, 1, 2, 3)}`. Cannot be specified if `backbone_config` is set. + encoder_hidden_dim (`int`, *optional*, defaults to 256): + Dimension of the layers in hybrid encoder. + encoder_in_channels (`list`, *optional*, defaults to `[512, 1024, 2048]`): + Multi level features input for encoder. + feat_strides (`list[int]`, *optional*, defaults to `[8, 16, 32]`): + Strides used in each feature map. + encoder_layers (`int`, *optional*, defaults to 1): + Total of layers to be used by the encoder. + encoder_ffn_dim (`int`, *optional*, defaults to 1024): + Dimension of the "intermediate" (often named feed-forward) layer in decoder. + encoder_attention_heads (`int`, *optional*, defaults to 8): + Number of attention heads for each attention layer in the Transformer encoder. + dropout (`float`, *optional*, defaults to 0.0): + The ratio for all dropout layers. + activation_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for activations inside the fully connected layer. + encode_proj_layers (`list[int]`, *optional*, defaults to `[2]`): + Indexes of the projected layers to be used in the encoder. + positional_encoding_temperature (`int`, *optional*, defaults to 10000): + The temperature parameter used to create the positional encodings. + encoder_activation_function (`str`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + activation_function (`str`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the general layer. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + eval_size (`tuple[int, int]`, *optional*): + Height and width used to computes the effective height and width of the position embeddings after taking + into account the stride. + normalize_before (`bool`, *optional*, defaults to `False`): + Determine whether to apply layer normalization in the transformer encoder layer before self-attention and + feed-forward modules. + hidden_expansion (`float`, *optional*, defaults to 1.0): + Expansion ratio to enlarge the dimension size of RepVGGBlock and CSPRepLayer. + mask_feat_channels (`list[int]`, *optional*, defaults to `[64, 64]): + The channels of the multi-level features for mask enhancement. + x4_feat_dim (`int`, *optional*, defaults to 128): + The dimension of the x4 feature map. + d_model (`int`, *optional*, defaults to 256): + Dimension of the layers exclude hybrid encoder. + num_queries (`int`, *optional*, defaults to 300): + Number of object queries. + decoder_in_channels (`list`, *optional*, defaults to `[256, 256, 256]`): + Multi level features dimension for decoder + decoder_ffn_dim (`int`, *optional*, defaults to 1024): + Dimension of the "intermediate" (often named feed-forward) layer in decoder. + num_feature_levels (`int`, *optional*, defaults to 3): + The number of input feature levels. + decoder_n_points (`int`, *optional*, defaults to 4): + The number of sampled keys in each feature level for each attention head in the decoder. + decoder_layers (`int`, *optional*, defaults to 6): + Number of decoder layers. + decoder_attention_heads (`int`, *optional*, defaults to 8): + Number of attention heads for each attention layer in the Transformer decoder. + decoder_activation_function (`str`, *optional*, defaults to `"relu"`): + The non-linear activation function (function or string) in the decoder. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + num_denoising (`int`, *optional*, defaults to 100): + The total number of denoising tasks or queries to be used for contrastive denoising. + label_noise_ratio (`float`, *optional*, defaults to 0.5): + The fraction of denoising labels to which random noise should be added. + box_noise_scale (`float`, *optional*, defaults to 1.0): + Scale or magnitude of noise to be added to the bounding boxes. + learn_initial_query (`bool`, *optional*, defaults to `False`): + Indicates whether the initial query embeddings for the decoder should be learned during training + mask_enhanced (`bool`, *optional*, defaults to `True`): + Whether to use enhanced masked attention. + anchor_image_size (`tuple[int, int]`, *optional*): + Height and width of the input image used during evaluation to generate the bounding box anchors. If None, automatic generate anchor is applied. + disable_custom_kernels (`bool`, *optional*, defaults to `True`): + Whether to disable custom kernels. + is_encoder_decoder (`bool`, *optional*, defaults to `True`): + Whether the architecture has an encoder decoder structure. + gp_head_size (`int`, *optional*, defaults to 64): + The size of the global pointer head. + tril_mask (`bool`, *optional*, defaults to `True`): + Whether to mask out the upper triangular part of the attention matrix. + id2label (`dict[int, str]`, *optional*): + Mapping from class id to class name. + + Examples: + + ```python + >>> from transformers import PPDocLayoutV3Config, PPDocLayoutV3ForObjectDetection + + >>> # Initializing a PP-DocLayoutV3 configuration + >>> configuration = PPDocLayoutV3Config() + + >>> # Initializing a model (with random weights) from the configuration + >>> model = PPDocLayoutV3ForObjectDetection(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "pp_doclayout_v2" + sub_configs = {"backbone_config": AutoConfig} + + layer_types = ("basic", "bottleneck") + attribute_map = { + "hidden_size": "d_model", + "num_attention_heads": "encoder_attention_heads", + } + + def __init__( + self, + initializer_range=0.01, + initializer_bias_prior_prob=None, + layer_norm_eps=1e-5, + batch_norm_eps=1e-5, + # backbone + backbone_config=None, + backbone=None, + use_pretrained_backbone=False, + use_timm_backbone=False, + freeze_backbone_batch_norms=True, + backbone_kwargs=None, + # encoder PPDocLayoutV3HybridEncoder + encoder_hidden_dim=256, + encoder_in_channels=[512, 1024, 2048], + feat_strides=[8, 16, 32], + encoder_layers=1, + encoder_ffn_dim=1024, + encoder_attention_heads=8, + dropout=0.0, + activation_dropout=0.0, + encode_proj_layers=[2], + positional_encoding_temperature=10000, + encoder_activation_function="gelu", + activation_function="silu", + eval_size=None, + normalize_before=False, + hidden_expansion=1.0, + mask_feat_channels=[64, 64], + x4_feat_dim=128, + # decoder PPDocLayoutV3Transformer + d_model=256, + num_prototypes=32, + label_noise_ratio=0.4, + box_noise_scale=0.4, + mask_enhanced=True, + num_queries=300, + decoder_in_channels=[256, 256, 256], + decoder_ffn_dim=1024, + num_feature_levels=3, + decoder_n_points=4, + decoder_layers=6, + decoder_attention_heads=8, + decoder_activation_function="relu", + attention_dropout=0.0, + num_denoising=100, + learn_initial_query=False, + anchor_image_size=None, + disable_custom_kernels=True, + is_encoder_decoder=True, + gp_head_size=64, + tril_mask=True, + # label + id2label=None, + **kwargs, + ): + self.initializer_range = initializer_range + self.initializer_bias_prior_prob = initializer_bias_prior_prob + self.layer_norm_eps = layer_norm_eps + self.batch_norm_eps = batch_norm_eps + + if backbone_config is None and backbone is None: + logger.info( + "`backbone_config` and `backbone` are `None`. Initializing the config with the default `HGNetV3` backbone." + ) + backbone_config = { + "model_type": "hgnet_v2", + "arch": "L", + "return_idx": [0, 1, 2, 3], + "freeze_stem_only": True, + "freeze_at": 0, + "freeze_norm": True, + "lr_mult_list": [0, 0.05, 0.05, 0.05, 0.05], + "out_features": ["stage1", "stage2", "stage3", "stage4"], + } + config_class = CONFIG_MAPPING["hgnet_v2"] + backbone_config = config_class.from_dict(backbone_config) + elif isinstance(backbone_config, dict): + backbone_model_type = backbone_config.get("model_type") + if backbone_model_type is None: + raise ValueError("`backbone_config` dict must contain key `model_type`.") + config_class = CONFIG_MAPPING[backbone_model_type] + backbone_config = config_class.from_dict(backbone_config) + + verify_backbone_config_arguments( + use_timm_backbone=use_timm_backbone, + use_pretrained_backbone=use_pretrained_backbone, + backbone=backbone, + backbone_config=backbone_config, + backbone_kwargs=backbone_kwargs, + ) + self.backbone_config = backbone_config + self.backbone = backbone + self.use_pretrained_backbone = use_pretrained_backbone + self.use_timm_backbone = use_timm_backbone + self.freeze_backbone_batch_norms = freeze_backbone_batch_norms + self.backbone_kwargs = dict(backbone_kwargs) if backbone_kwargs is not None else None + + # ---- encoder ---- + self.encoder_hidden_dim = encoder_hidden_dim + self.encoder_in_channels = list(encoder_in_channels) + self.feat_strides = list(feat_strides) + self.encoder_layers = encoder_layers + self.encoder_ffn_dim = encoder_ffn_dim + self.encoder_attention_heads = encoder_attention_heads + self.dropout = dropout + self.activation_dropout = activation_dropout + self.encode_proj_layers = list(encode_proj_layers) + self.positional_encoding_temperature = positional_encoding_temperature + self.encoder_activation_function = encoder_activation_function + self.activation_function = activation_function + self.eval_size = list(eval_size) if eval_size is not None else None + self.normalize_before = normalize_before + self.hidden_expansion = hidden_expansion + self.mask_feat_channels = mask_feat_channels + self.x4_feat_dim = x4_feat_dim + + # ---- decoder ---- + self.d_model = d_model + self.num_queries = num_queries + self.num_prototypes = num_prototypes + self.decoder_in_channels = list(decoder_in_channels) + self.decoder_ffn_dim = decoder_ffn_dim + self.num_feature_levels = num_feature_levels + self.decoder_n_points = decoder_n_points + self.decoder_layers = decoder_layers + self.decoder_attention_heads = decoder_attention_heads + self.decoder_activation_function = decoder_activation_function + self.attention_dropout = attention_dropout + self.num_denoising = num_denoising + self.label_noise_ratio = label_noise_ratio + self.mask_enhanced = mask_enhanced + self.box_noise_scale = box_noise_scale + self.learn_initial_query = learn_initial_query + self.anchor_image_size = list(anchor_image_size) if anchor_image_size is not None else None + self.disable_custom_kernels = disable_custom_kernels + self.gp_head_size = gp_head_size + self.tril_mask = tril_mask + + super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs) + + if kwargs.get("num_labels") and not id2label: + self.id2label = None + self.num_labels = kwargs["num_labels"] + else: + self.id2label = {int(k): v for k, v in id2label.items()} if id2label else _default_id2label() + self.num_labels = len(self.id2label) + + +__all__ = ["PPDocLayoutV3Config"] diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py new file mode 100644 index 000000000000..8b3bb2cc8751 --- /dev/null +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py @@ -0,0 +1,316 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_pp_doclayout_v3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2025 The PaddlePaddle Team and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Union + +import torch + +from ...feature_extraction_utils import BatchFeature +from ...image_processing_utils import BaseImageProcessor, get_size_dict +from ...image_transforms import resize, to_channel_dimension_format +from ...image_utils import ( + ChannelDimension, + ImageInput, + PILImageResampling, + infer_channel_dimension_format, + make_flat_list_of_images, + to_numpy_array, + valid_images, + validate_preprocess_arguments, +) +from ...utils import filter_out_non_signature_kwargs +from ...utils.generic import TensorType + + +def postprocess(outputs, threshold, target_sizes): + boxes = outputs.pred_boxes + logits = outputs.logits + order_logits = outputs.order_logits + + order_seqs = get_order(order_logits) + + cxcy, wh = torch.split(boxes, 2, dim=-1) + boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) + + if target_sizes is not None: + if len(logits) != len(target_sizes): + raise ValueError("Make sure that you pass in as many target sizes as the batch dimension of the logits") + if isinstance(target_sizes, list): + img_h, img_w = torch.as_tensor(target_sizes).unbind(1) + else: + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) + boxes = boxes * scale_fct[:, None, :] + + num_top_queries = logits.shape[1] + num_classes = logits.shape[2] + + scores = torch.nn.functional.sigmoid(logits) + scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) + labels = index % num_classes + index = index // num_classes + boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) + order_seqs = order_seqs.gather(dim=1, index=index) + + results = [] + for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): + order_seq = order_seq[score >= threshold] + order_seq, indices = torch.sort(order_seq) + results.append( + { + "scores": score[score >= threshold][indices], + "labels": label[score >= threshold][indices], + "boxes": box[score >= threshold][indices], + "order_seq": order_seq, + } + ) + + return results + + +def get_order(order_logits): + # order_logits: (B, N, N) upper-triangular meaningful + order_scores = torch.sigmoid(order_logits) + B, N, _ = order_scores.shape + + one = torch.ones((N, N), dtype=order_scores.dtype, device=order_scores.device) + upper = torch.triu(one, 1) + lower = torch.tril(one, -1) + + Q = order_scores * upper + (1.0 - order_scores.transpose(1, 2)) * lower + order_votes = Q.sum(dim=1) # (B, N) + + order_pointers = torch.argsort(order_votes, dim=1) # (B, N) + order_seq = torch.full_like(order_pointers, -1) + batch = torch.arange(B, device=order_pointers.device)[:, None] + order_seq[batch, order_pointers] = torch.arange(N, device=order_pointers.device)[None, :] + + return order_seq + + +class PPDocLayoutV3ImageProcessor(BaseImageProcessor): + r""" + Constructs a PPDocLayoutV3 image processor. + + Args: + do_resize (`bool`, *optional*, defaults to `True`): + Controls whether to resize the image's (height, width) dimensions to the specified `size`. Can be + overridden by the `do_resize` parameter in the `preprocess` method. + size (`dict[str, int]` *optional*, defaults to `{"height": 640, "width": 640}`): + Size of the image's `(height, width)` dimensions after resizing. Can be overridden by the `size` parameter + in the `preprocess` method. Available options are: + - `{"height": int, "width": int}`: The image will be resized to the exact size `(height, width)`. + Do NOT keep the aspect ratio. + - `{"shortest_edge": int, "longest_edge": int}`: The image will be resized to a maximum size respecting + the aspect ratio and keeping the shortest edge less or equal to `shortest_edge` and the longest edge + less or equal to `longest_edge`. + - `{"max_height": int, "max_width": int}`: The image will be resized to the maximum size respecting the + aspect ratio and keeping the height less or equal to `max_height` and the width less or equal to + `max_width`. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): + Resampling filter to use if resizing the image. + do_rescale (`bool`, *optional*, defaults to `True`): + Controls whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the + `do_rescale` parameter in the `preprocess` method. + rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): + Scale factor to use if rescaling the image. Can be overridden by the `rescale_factor` parameter in the + `preprocess` method. + Controls whether to normalize the image. Can be overridden by the `do_normalize` parameter in the + `preprocess` method. + do_normalize (`bool`, *optional*, defaults to `True`): + Whether to normalize the image. + image_mean (`float` or `list[float]`, *optional*, defaults to `[0, 0, 0]`): + Mean values to use when normalizing the image. Can be a single value or a list of values, one for each + channel. Can be overridden by the `image_mean` parameter in the `preprocess` method. + image_std (`float` or `list[float]`, *optional*, defaults to `[1, 1, 1]`): + Standard deviation values to use when normalizing the image. Can be a single value or a list of values, one + for each channel. Can be overridden by the `image_std` parameter in the `preprocess` method. + """ + + model_input_names = ["pixel_values"] + + def __init__( + self, + do_resize: bool = True, + size: Optional[dict[str, int]] = None, + resample: Optional[PILImageResampling] = PILImageResampling.BICUBIC, + do_rescale: bool = True, + rescale_factor: Union[int, float] = 1 / 255, + do_normalize: bool = True, + image_mean: Optional[Union[float, list[float]]] = [0, 0, 0], + image_std: Optional[Union[float, list[float]]] = [1, 1, 1], + **kwargs, + ) -> None: + size = size if size is not None else {"height": 800, "width": 800} + size = get_size_dict(size, default_to_square=False) + + super().__init__(**kwargs) + self.do_resize = do_resize + self.size = size + self.do_rescale = do_rescale + self.rescale_factor = rescale_factor + self.do_normalize = do_normalize + self.image_mean = image_mean + self.image_std = image_std + self.resample = resample + + @filter_out_non_signature_kwargs() + def preprocess( + self, + images: ImageInput, + do_resize: Optional[bool] = None, + size: Optional[dict[str, int]] = None, + resample: Optional[PILImageResampling] = None, + do_rescale: Optional[bool] = None, + rescale_factor: Optional[Union[int, float]] = None, + do_normalize: Optional[bool] = None, + image_mean: Optional[Union[float, list[float]]] = None, + image_std: Optional[Union[float, list[float]]] = None, + return_tensors: Optional[Union[TensorType, str]] = None, + data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + ) -> BatchFeature: + """ + Preprocess an image or a batch of images so that it can be used by the model. + + Args: + images (`ImageInput`): + Image or batch of images to preprocess. Expects a single or batch of images with pixel values ranging + from 0 to 255. If passing in images with pixel values between 0 and 1, set `do_rescale=False`. + do_resize (`bool`, *optional*, defaults to self.do_resize): + Whether to resize the image. + size (`dict[str, int]`, *optional*, defaults to self.size): + Size of the image's `(height, width)` dimensions after resizing. Available options are: + - `{"height": int, "width": int}`: The image will be resized to the exact size `(height, width)`. + Do NOT keep the aspect ratio. + - `{"shortest_edge": int, "longest_edge": int}`: The image will be resized to a maximum size respecting + the aspect ratio and keeping the shortest edge less or equal to `shortest_edge` and the longest edge + less or equal to `longest_edge`. + - `{"max_height": int, "max_width": int}`: The image will be resized to the maximum size respecting the + aspect ratio and keeping the height less or equal to `max_height` and the width less or equal to + `max_width`. + resample (`PILImageResampling`, *optional*, defaults to self.resample): + Resampling filter to use when resizing the image. + do_rescale (`bool`, *optional*, defaults to self.do_rescale): + Whether to rescale the image. + rescale_factor (`float`, *optional*, defaults to self.rescale_factor): + Rescale factor to use when rescaling the image. + do_normalize (`bool`, *optional*, defaults to self.do_normalize): + Whether to normalize the image. + image_mean (`float` or `list[float]`, *optional*, defaults to self.image_mean): + Mean to use when normalizing the image. + image_std (`float` or `list[float]`, *optional*, defaults to self.image_std): + Standard deviation to use when normalizing the image. + return_tensors (`str` or `TensorType`, *optional*, defaults to self.return_tensors): + Type of tensors to return. If `None`, will return the list of images. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + """ + do_resize = self.do_resize if do_resize is None else do_resize + size = self.size if size is None else size + size = get_size_dict(size=size, default_to_square=True) + resample = self.resample if resample is None else resample + do_rescale = self.do_rescale if do_rescale is None else do_rescale + rescale_factor = self.rescale_factor if rescale_factor is None else rescale_factor + do_normalize = self.do_normalize if do_normalize is None else do_normalize + image_mean = self.image_mean if image_mean is None else image_mean + image_std = self.image_std if image_std is None else image_std + + validate_preprocess_arguments( + do_rescale=do_rescale, + rescale_factor=rescale_factor, + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + do_resize=do_resize, + size=size, + resample=resample, + ) + + images = make_flat_list_of_images(images) + if not valid_images(images): + raise ValueError("Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, or torch.Tensor") + + # All transformations expect numpy arrays + images = [to_numpy_array(image) for image in images] + + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) + + # transformations + if do_resize: + images = [ + resize( + image, size=(size["height"], size["width"]), resample=resample, input_data_format=input_data_format + ) + for image in images + ] + + if do_rescale: + images = [self.rescale(image, rescale_factor, input_data_format=input_data_format) for image in images] + + if do_normalize: + images = [ + self.normalize(image, image_mean, image_std, input_data_format=input_data_format) for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] + encoded_inputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) + + return encoded_inputs + + def post_process_object_detection( + self, + outputs, + threshold: float = 0.5, + target_sizes: Optional[Union[TensorType, list[tuple]]] = None, + ): + """ + Converts the raw output of [`PPDocLayoutV3ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, + bottom_right_x, bottom_right_y) format. Only supports PyTorch. + + Args: + outputs ([`PPDocLayoutV3ObjectDetectionOutput`]): + Raw outputs of the model. + threshold (`float`, *optional*, defaults to 0.5): + Score threshold to keep object detection predictions. + target_sizes (`torch.Tensor` or `list[tuple[int, int]]`, *optional*): + Tensor of shape `(batch_size, 2)` or list of tuples (`tuple[int, int]`) containing the target size + `(height, width)` of each image in the batch. If unset, predictions will not be resized. + + Returns: + `list[Dict]`: An ordered list of dictionaries, each dictionary containing the scores, labels and boxes for an image + in the batch as predicted by the model. + """ + return postprocess(outputs=outputs, threshold=threshold, target_sizes=target_sizes) + + +__all__ = ["PPDocLayoutV3ImageProcessor"] diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py new file mode 100644 index 000000000000..e53f6d0fee25 --- /dev/null +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py @@ -0,0 +1,164 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_pp_doclayout_v3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2025 The PaddlePaddle Team and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Union + +import torch +from torchvision.transforms.v2.functional import InterpolationMode + +from ...feature_extraction_utils import BatchFeature +from ...image_processing_utils_fast import BaseImageProcessorFast, SizeDict +from ...image_utils import PILImageResampling +from ...utils.generic import TensorType + + +def postprocess(outputs, threshold, target_sizes): + boxes = outputs.pred_boxes + logits = outputs.logits + order_logits = outputs.order_logits + + order_seqs = get_order(order_logits) + + cxcy, wh = torch.split(boxes, 2, dim=-1) + boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) + + if target_sizes is not None: + if len(logits) != len(target_sizes): + raise ValueError("Make sure that you pass in as many target sizes as the batch dimension of the logits") + if isinstance(target_sizes, list): + img_h, img_w = torch.as_tensor(target_sizes).unbind(1) + else: + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) + boxes = boxes * scale_fct[:, None, :] + + num_top_queries = logits.shape[1] + num_classes = logits.shape[2] + + scores = torch.nn.functional.sigmoid(logits) + scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) + labels = index % num_classes + index = index // num_classes + boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) + order_seqs = order_seqs.gather(dim=1, index=index) + + results = [] + for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): + order_seq = order_seq[score >= threshold] + order_seq, indices = torch.sort(order_seq) + results.append( + { + "scores": score[score >= threshold][indices], + "labels": label[score >= threshold][indices], + "boxes": box[score >= threshold][indices], + "order_seq": order_seq, + } + ) + + return results + + +def get_order(order_logits): + # order_logits: (B, N, N) upper-triangular meaningful + order_scores = torch.sigmoid(order_logits) + B, N, _ = order_scores.shape + + one = torch.ones((N, N), dtype=order_scores.dtype, device=order_scores.device) + upper = torch.triu(one, 1) + lower = torch.tril(one, -1) + + Q = order_scores * upper + (1.0 - order_scores.transpose(1, 2)) * lower + order_votes = Q.sum(dim=1) # (B, N) + + order_pointers = torch.argsort(order_votes, dim=1) # (B, N) + order_seq = torch.full_like(order_pointers, -1) + batch = torch.arange(B, device=order_pointers.device)[:, None] + order_seq[batch, order_pointers] = torch.arange(N, device=order_pointers.device)[None, :] + + return order_seq + + +class PPDocLayoutV3ImageProcessorFast(BaseImageProcessorFast): + resample = PILImageResampling.BICUBIC + image_mean = [0, 0, 0] + image_std = [1, 1, 1] + size = {"height": 800, "width": 800} + do_resize = True + do_rescale = True + do_normalize = True + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + + def _preprocess( + self, + images: list[torch.Tensor], + do_resize: bool, + size: SizeDict, + interpolation: Optional[InterpolationMode], + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: Optional[Union[float, list[float]]], + image_std: Optional[Union[float, list[float]]], + return_tensors: Optional[Union[str, TensorType]], + **kwargs, + ) -> BatchFeature: + """ + Preprocess an image or a batch of images so that it can be used by the model. + """ + data = {} + processed_images = [] + for image in images: + if do_resize: + image = self.resize(image, size=size, interpolation=interpolation) + # Fused rescale and normalize + image = self.rescale_and_normalize(image, do_rescale, rescale_factor, do_normalize, image_mean, image_std) + + processed_images.append(image) + + images = processed_images + + data.update({"pixel_values": torch.stack(images, dim=0)}) + encoded_inputs = BatchFeature(data, tensor_type=return_tensors) + + return encoded_inputs + + def post_process_object_detection( + self, + outputs, + threshold: float = 0.5, + target_sizes: Optional[Union[TensorType, list[tuple]]] = None, + ): + """ + Converts the raw output of [`DetrForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, + bottom_right_x, bottom_right_y) format. Only supports PyTorch. + + Args: + outputs ([`DetrObjectDetectionOutput`]): + Raw outputs of the model. + Returns: + `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image + in the batch as predicted by the model. + """ + return postprocess(outputs=outputs, threshold=threshold, target_sizes=target_sizes) + + +__all__ = ["PPDocLayoutV3ImageProcessorFast"] diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py new file mode 100644 index 000000000000..5c09131d2118 --- /dev/null +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -0,0 +1,2202 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_pp_doclayout_v3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2025 The PaddlePaddle Team and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import warnings +from dataclasses import dataclass +from typing import Optional, Union + +import numpy as np +import torch +import torch.nn.functional as F +from torch import Tensor, nn + +from ... import initialization as init +from ...activations import ACT2CLS, ACT2FN +from ...image_transforms import center_to_corners_format, corners_to_center_format +from ...integrations import use_kernel_forward_from_hub +from ...modeling_outputs import BaseModelOutput +from ...modeling_utils import PreTrainedModel +from ...pytorch_utils import compile_compatible_method_lru_cache +from ...utils import ModelOutput, auto_docstring, torch_int +from ...utils.backbone_utils import load_backbone +from .configuration_pp_doclayout_v3 import PPDocLayoutV3Config + + +class GlobalPointer(nn.Module): + def __init__(self, config): + super().__init__() + self.head_size = config.gp_head_size + self.tril_mask = config.tril_mask + self.dense = nn.Linear(config.d_model, self.head_size * 2) + self.dropout = nn.Dropout(0.1) + + def forward(self, inputs): + B, N, _ = inputs.shape + proj = self.dense(inputs).reshape([B, N, 2, self.head_size]) + proj = self.dropout(proj) + qw, kw = proj[..., 0, :], proj[..., 1, :] + + logits = torch.einsum("bmd,bnd->bmn", qw, kw) / (self.head_size**0.5) # [B, N, N] + + if self.tril_mask: + lower = torch.tril(torch.ones([N, N], dtype=torch.float32, device=logits.device)) + logits = logits - lower.unsqueeze(0).to(logits.dtype) * 1e4 + + return logits + + +# TODO: Replace all occurrences of the checkpoint with the final one + + +@use_kernel_forward_from_hub("MultiScaleDeformableAttention") +class MultiScaleDeformableAttention(nn.Module): + def forward( + self, + value: Tensor, + value_spatial_shapes: Tensor, + value_spatial_shapes_list: list[tuple], + level_start_index: Tensor, + sampling_locations: Tensor, + attention_weights: Tensor, + im2col_step: int, + ): + batch_size, _, num_heads, hidden_dim = value.shape + _, num_queries, num_heads, num_levels, num_points, _ = sampling_locations.shape + value_list = value.split([height * width for height, width in value_spatial_shapes_list], dim=1) + sampling_grids = 2 * sampling_locations - 1 + sampling_value_list = [] + for level_id, (height, width) in enumerate(value_spatial_shapes_list): + # batch_size, height*width, num_heads, hidden_dim + # -> batch_size, height*width, num_heads*hidden_dim + # -> batch_size, num_heads*hidden_dim, height*width + # -> batch_size*num_heads, hidden_dim, height, width + value_l_ = ( + value_list[level_id] + .flatten(2) + .transpose(1, 2) + .reshape(batch_size * num_heads, hidden_dim, height, width) + ) + # batch_size, num_queries, num_heads, num_points, 2 + # -> batch_size, num_heads, num_queries, num_points, 2 + # -> batch_size*num_heads, num_queries, num_points, 2 + sampling_grid_l_ = sampling_grids[:, :, :, level_id].transpose(1, 2).flatten(0, 1) + # batch_size*num_heads, hidden_dim, num_queries, num_points + sampling_value_l_ = nn.functional.grid_sample( + value_l_, + sampling_grid_l_, + mode="bilinear", + padding_mode="zeros", + align_corners=False, + ) + sampling_value_list.append(sampling_value_l_) + # (batch_size, num_queries, num_heads, num_levels, num_points) + # -> (batch_size, num_heads, num_queries, num_levels, num_points) + # -> (batch_size, num_heads, 1, num_queries, num_levels*num_points) + attention_weights = attention_weights.transpose(1, 2).reshape( + batch_size * num_heads, 1, num_queries, num_levels * num_points + ) + output = ( + (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights) + .sum(-1) + .view(batch_size, num_heads * hidden_dim, num_queries) + ) + return output.transpose(1, 2).contiguous() + + +class PPDocLayoutV3MultiscaleDeformableAttention(nn.Module): + """ + Multiscale deformable attention as proposed in Deformable DETR. + """ + + def __init__(self, config: PPDocLayoutV3Config, num_heads: int, n_points: int): + super().__init__() + + self.attn = MultiScaleDeformableAttention() + + if config.d_model % num_heads != 0: + raise ValueError( + f"embed_dim (d_model) must be divisible by num_heads, but got {config.d_model} and {num_heads}" + ) + dim_per_head = config.d_model // num_heads + # check if dim_per_head is power of 2 + if not ((dim_per_head & (dim_per_head - 1) == 0) and dim_per_head != 0): + warnings.warn( + "You'd better set embed_dim (d_model) in PPDocLayoutV3MultiscaleDeformableAttention to make the" + " dimension of each attention head a power of 2 which is more efficient in the authors' CUDA" + " implementation." + ) + + self.im2col_step = 64 + + self.d_model = config.d_model + self.n_levels = config.num_feature_levels + self.n_heads = num_heads + self.n_points = n_points + + self.sampling_offsets = nn.Linear(config.d_model, num_heads * self.n_levels * n_points * 2) + self.attention_weights = nn.Linear(config.d_model, num_heads * self.n_levels * n_points) + self.value_proj = nn.Linear(config.d_model, config.d_model) + self.output_proj = nn.Linear(config.d_model, config.d_model) + + self.disable_custom_kernels = config.disable_custom_kernels + + def with_pos_embed(self, tensor: torch.Tensor, position_embeddings: Optional[Tensor]): + return tensor if position_embeddings is None else tensor + position_embeddings + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states=None, + encoder_attention_mask=None, + position_embeddings: Optional[torch.Tensor] = None, + reference_points=None, + spatial_shapes=None, + spatial_shapes_list=None, + level_start_index=None, + output_attentions: bool = False, + ): + # add position embeddings to the hidden states before projecting to queries and keys + if position_embeddings is not None: + hidden_states = self.with_pos_embed(hidden_states, position_embeddings) + + batch_size, num_queries, _ = hidden_states.shape + batch_size, sequence_length, _ = encoder_hidden_states.shape + total_elements = sum(height * width for height, width in spatial_shapes_list) + if total_elements != sequence_length: + raise ValueError( + "Make sure to align the spatial shapes with the sequence length of the encoder hidden states" + ) + + value = self.value_proj(encoder_hidden_states) + if attention_mask is not None: + # we invert the attention_mask + value = value.masked_fill(~attention_mask[..., None], float(0)) + value = value.view(batch_size, sequence_length, self.n_heads, self.d_model // self.n_heads) + sampling_offsets = self.sampling_offsets(hidden_states).view( + batch_size, num_queries, self.n_heads, self.n_levels, self.n_points, 2 + ) + attention_weights = self.attention_weights(hidden_states).view( + batch_size, num_queries, self.n_heads, self.n_levels * self.n_points + ) + attention_weights = F.softmax(attention_weights, -1).view( + batch_size, num_queries, self.n_heads, self.n_levels, self.n_points + ) + # batch_size, num_queries, n_heads, n_levels, n_points, 2 + num_coordinates = reference_points.shape[-1] + if num_coordinates == 2: + offset_normalizer = torch.stack([spatial_shapes[..., 1], spatial_shapes[..., 0]], -1) + sampling_locations = ( + reference_points[:, :, None, :, None, :] + + sampling_offsets / offset_normalizer[None, None, None, :, None, :] + ) + elif num_coordinates == 4: + sampling_locations = ( + reference_points[:, :, None, :, None, :2] + + sampling_offsets / self.n_points * reference_points[:, :, None, :, None, 2:] * 0.5 + ) + else: + raise ValueError(f"Last dim of reference_points must be 2 or 4, but got {reference_points.shape[-1]}") + + output = self.attn( + value, + spatial_shapes, + spatial_shapes_list, + level_start_index, + sampling_locations, + attention_weights, + self.im2col_step, + ) + + output = self.output_proj(output) + + return output, attention_weights + + +@auto_docstring +class PPDocLayoutV3PreTrainedModel(PreTrainedModel): + config: PPDocLayoutV3Config + base_model_prefix = "pp_doclayout_v3" + main_input_name = "pixel_values" + input_modalities = ("image",) + _no_split_modules = [r"PPDocLayoutV3HybridEncoder", r"PPDocLayoutV3DecoderLayer"] + + @torch.no_grad() + def _init_weights(self, module): + if isinstance(module, PPDocLayoutV3MultiscaleDeformableAttention): + init.constant_(module.sampling_offsets.weight, 0.0) + default_dtype = torch.get_default_dtype() + thetas = torch.arange(module.n_heads, dtype=torch.int64).to(default_dtype) * ( + 2.0 * math.pi / module.n_heads + ) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = ( + (grid_init / grid_init.abs().max(-1, keepdim=True)[0]) + .view(module.n_heads, 1, 1, 2) + .repeat(1, module.n_levels, module.n_points, 1) + ) + for i in range(module.n_points): + grid_init[:, :, i, :] *= i + 1 + + init.copy_(module.sampling_offsets.bias, grid_init.view(-1)) + init.constant_(module.attention_weights.weight, 0.0) + init.constant_(module.attention_weights.bias, 0.0) + init.xavier_uniform_(module.value_proj.weight) + init.constant_(module.value_proj.bias, 0.0) + init.xavier_uniform_(module.output_proj.weight) + init.constant_(module.output_proj.bias, 0.0) + + elif isinstance(module, PPDocLayoutV3Model): + prior_prob = self.config.initializer_bias_prior_prob or 1 / (self.config.num_labels + 1) + bias = float(-math.log((1 - prior_prob) / prior_prob)) + init.xavier_uniform_(module.enc_score_head.weight) + init.constant_(module.enc_score_head.bias, bias) + + elif isinstance(module, (nn.Linear, nn.Conv2d, nn.BatchNorm2d)): + init.normal_(module.weight, mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + init.zeros_(module.bias) + if getattr(module, "running_mean", None) is not None: + init.zeros_(module.running_mean) + init.ones_(module.running_var) + init.zeros_(module.num_batches_tracked) + + elif isinstance(module, nn.LayerNorm): + init.ones_(module.weight) + init.zeros_(module.bias) + + if isinstance(module, nn.Embedding): + init.normal_(module.weight, mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + init.zeros_(module.weight.data[module.padding_idx]) + + +@dataclass +class PPDocLayoutV3DecoderOutput(ModelOutput): + r""" + intermediate_hidden_states (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, hidden_size)`): + Stacked intermediate hidden states (output of each layer of the decoder). + intermediate_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, sequence_length, config.num_labels)`): + Stacked intermediate logits (logits of each layer of the decoder). + intermediate_reference_points (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, sequence_length, hidden_size)`): + Stacked intermediate reference points (reference points of each layer of the decoder). + intermediate_predicted_corners (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked intermediate predicted corners (predicted corners of each layer of the decoder). + initial_reference_points (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked initial reference points (initial reference points of each layer of the decoder). + cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` and `config.add_cross_attention=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax, + used to compute the weighted average in the cross-attention heads. + dec_out_order_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, config.num_queries)`): + Stacked order logits (order logits of each layer of the decoder). + dec_out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, 200, 200)`): + Stacked masks (masks of each layer of the decoder). + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + intermediate_hidden_states: Optional[torch.FloatTensor] = None + intermediate_logits: Optional[torch.FloatTensor] = None + intermediate_reference_points: Optional[torch.FloatTensor] = None + intermediate_predicted_corners: Optional[torch.FloatTensor] = None + initial_reference_points: Optional[torch.FloatTensor] = None + hidden_states: Optional[tuple[torch.FloatTensor]] = None + attentions: Optional[tuple[torch.FloatTensor]] = None + cross_attentions: Optional[tuple[torch.FloatTensor]] = None + dec_out_order_logits: Optional[torch.FloatTensor] = None + dec_out_masks: Optional[torch.FloatTensor] = None + + +@dataclass +@auto_docstring( + custom_intro=""" + Base class for outputs of the PP-DocLayoutV3 model. + """ +) +class PPDocLayoutV3ModelOutput(ModelOutput): + r""" + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the decoder of the model. + intermediate_hidden_states (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, hidden_size)`): + Stacked intermediate hidden states (output of each layer of the decoder). + intermediate_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, sequence_length, config.num_labels)`): + Stacked intermediate logits (logits of each layer of the decoder). + intermediate_reference_points (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked intermediate reference points (reference points of each layer of the decoder). + intermediate_predicted_corners (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked intermediate predicted corners (predicted corners of each layer of the decoder). + initial_reference_points (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): + Initial reference points used for the first decoder layer. + init_reference_points (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): + Initial reference points sent through the Transformer decoder. + enc_topk_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`): + Predicted bounding boxes scores where the top `config.two_stage_num_proposals` scoring bounding boxes are + picked as region proposals in the encoder stage. Output of bounding box binary classification (i.e. + foreground and background). + enc_topk_bboxes (`torch.FloatTensor` of shape `(batch_size, sequence_length, 4)`): + Logits of predicted bounding boxes coordinates in the encoder stage. + enc_outputs_class (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Predicted bounding boxes scores where the top `config.two_stage_num_proposals` scoring bounding boxes are + picked as region proposals in the first stage. Output of bounding box binary classification (i.e. + foreground and background). + enc_outputs_coord_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, 4)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Logits of predicted bounding boxes coordinates in the first stage. + denoising_meta_values (`dict`): + Extra dictionary for the denoising related values. + out_order_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, config.num_queries)`): + Stacked order logits (order logits of each layer of the decoder). + out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, 200, 200)`): + Stacked masks (masks of each layer of the decoder). + """ + + last_hidden_state: Optional[torch.FloatTensor] = None + intermediate_hidden_states: Optional[torch.FloatTensor] = None + intermediate_logits: Optional[torch.FloatTensor] = None + intermediate_reference_points: Optional[torch.FloatTensor] = None + intermediate_predicted_corners: Optional[torch.FloatTensor] = None + initial_reference_points: Optional[torch.FloatTensor] = None + decoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None + decoder_attentions: Optional[tuple[torch.FloatTensor]] = None + cross_attentions: Optional[tuple[torch.FloatTensor]] = None + encoder_last_hidden_state: Optional[torch.FloatTensor] = None + encoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None + encoder_attentions: Optional[tuple[torch.FloatTensor]] = None + init_reference_points: Optional[torch.FloatTensor] = None + enc_topk_logits: Optional[torch.FloatTensor] = None + enc_topk_bboxes: Optional[torch.FloatTensor] = None + enc_outputs_class: Optional[torch.FloatTensor] = None + enc_outputs_coord_logits: Optional[torch.FloatTensor] = None + denoising_meta_values: Optional[dict] = None + out_order_logits: Optional[torch.FloatTensor] = None + out_masks: Optional[torch.FloatTensor] = None + + +# taken from https://github.com/facebookresearch/detr/blob/master/models/detr.py +class PPDocLayoutV3MLPPredictionHead(nn.Module): + """ + Very simple multi-layer perceptron (MLP, also called FFN), used to predict the normalized center coordinates, + height and width of a bounding box w.r.t. an image. + + Copied from https://github.com/facebookresearch/detr/blob/master/models/detr.py + Origin from https://github.com/lyuwenyu/RT-DETR/blob/94f5e16708329d2f2716426868ec89aa774af016/PPDocLayoutV3_paddle/ppdet/modeling/transformers/utils.py#L453 + + """ + + def __init__(self, config, input_dim, d_model, output_dim, num_layers): + super().__init__() + self.num_layers = num_layers + h = [d_model] * (num_layers - 1) + self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) + + def forward(self, x): + for i, layer in enumerate(self.layers): + x = nn.functional.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + return x + + +class BaseConv(nn.Module): + def __init__( + self, + in_channels, + out_channels, + ksize, + stride, + groups=1, + bias=False, + ): + super().__init__() + self.conv = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=ksize, + stride=stride, + padding=(ksize - 1) // 2, + groups=groups, + bias=bias, + ) + self.bn = nn.BatchNorm2d(out_channels) + + def forward(self, x): + # use 'x * F.sigmoid(x)' replace 'silu' + x = self.bn(self.conv(x)) + y = x * F.sigmoid(x) + return y + + +class MaskFeatFPN(nn.Module): + def __init__( + self, + in_channels=[256, 256, 256], + fpn_strides=[32, 16, 8], + feat_channels=256, + dropout_ratio=0.0, + out_channels=256, + align_corners=False, + ): + super().__init__() + + assert len(in_channels) == len(fpn_strides), ( + f"Error: The lengths of 'in_channels' and 'fpn_strides' must be equal. " + f"Got len(in_channels)={len(in_channels)} and len(fpn_strides)={len(fpn_strides)}." + ) + + reorder_index = np.argsort(fpn_strides, axis=0) + in_channels = [in_channels[i] for i in reorder_index] + fpn_strides = [fpn_strides[i] for i in reorder_index] + + self.reorder_index = reorder_index + self.fpn_strides = fpn_strides + self.dropout_ratio = dropout_ratio + self.align_corners = align_corners + if self.dropout_ratio > 0: + self.dropout = nn.Dropout2D(dropout_ratio) + + self.scale_heads = nn.ModuleList() + for i in range(len(fpn_strides)): + head_length = max(1, int(np.log2(fpn_strides[i]) - np.log2(fpn_strides[0]))) + scale_head = [] + for k in range(head_length): + in_c = in_channels[i] if k == 0 else feat_channels + scale_head.append(nn.Sequential(BaseConv(in_c, feat_channels, 3, 1))) + if fpn_strides[i] != fpn_strides[0]: + scale_head.append(nn.Upsample(scale_factor=2, mode="bilinear", align_corners=align_corners)) + + self.scale_heads.append(nn.Sequential(*scale_head)) + + self.output_conv = BaseConv(feat_channels, out_channels, 3, 1) + + def forward(self, inputs): + x = [inputs[i] for i in self.reorder_index] + + output = self.scale_heads[0](x[0]) + for i in range(1, len(self.fpn_strides)): + output = output + F.interpolate( + self.scale_heads[i](x[i]), size=output.shape[2:], mode="bilinear", align_corners=self.align_corners + ) + + if self.dropout_ratio > 0: + output = self.dropout(output) + output = self.output_conv(output) + return output + + +class PPDocLayoutV3ConvNormLayer(nn.Module): + def __init__(self, config, in_channels, out_channels, kernel_size, stride, padding=None, activation=None): + super().__init__() + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size, + stride, + padding=(kernel_size - 1) // 2 if padding is None else padding, + bias=False, + ) + self.norm = nn.BatchNorm2d(out_channels, config.batch_norm_eps) + self.activation = nn.Identity() if activation is None else ACT2CLS[activation]() + + def forward(self, hidden_state): + hidden_state = self.conv(hidden_state) + hidden_state = self.norm(hidden_state) + hidden_state = self.activation(hidden_state) + return hidden_state + + +class PPDocLayoutV3MultiheadAttention(nn.Module): + """ + Multi-headed attention from 'Attention Is All You Need' paper. + + Here, we add position embeddings to the queries and keys (as explained in the Deformable DETR paper). + """ + + def __init__( + self, + embed_dim: int, + num_heads: int, + dropout: float = 0.0, + bias: bool = True, + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + if self.head_dim * num_heads != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" + f" {num_heads})." + ) + self.scaling = self.head_dim**-0.5 + + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + + def _reshape(self, tensor: torch.Tensor, seq_len: int, batch_size: int): + return tensor.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def with_pos_embed(self, tensor: torch.Tensor, position_embeddings: Optional[Tensor]): + return tensor if position_embeddings is None else tensor + position_embeddings + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_embeddings: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + batch_size, target_len, embed_dim = hidden_states.size() + # add position embeddings to the hidden states before projecting to queries and keys + if position_embeddings is not None: + hidden_states_original = hidden_states + hidden_states = self.with_pos_embed(hidden_states, position_embeddings) + + # get queries, keys and values + query_states = self.q_proj(hidden_states) * self.scaling + key_states = self._reshape(self.k_proj(hidden_states), -1, batch_size) + value_states = self._reshape(self.v_proj(hidden_states_original), -1, batch_size) + + proj_shape = (batch_size * self.num_heads, -1, self.head_dim) + query_states = self._reshape(query_states, target_len, batch_size).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + source_len = key_states.size(1) + + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (batch_size * self.num_heads, target_len, source_len): + raise ValueError( + f"Attention weights should be of size {(batch_size * self.num_heads, target_len, source_len)}, but is" + f" {attn_weights.size()}" + ) + + # expand attention_mask + if attention_mask is not None: + # [seq_len, seq_len] -> [batch_size, 1, target_seq_len, source_seq_len] + attention_mask = attention_mask.expand(batch_size, 1, *attention_mask.size()) + + if attention_mask is not None: + if attention_mask.size() != (batch_size, 1, target_len, source_len): + raise ValueError( + f"Attention mask should be of size {(batch_size, 1, target_len, source_len)}, but is" + f" {attention_mask.size()}" + ) + if attention_mask.dtype == torch.bool: + attention_mask = torch.zeros_like(attention_mask, dtype=attn_weights.dtype).masked_fill_( + attention_mask, -torch.inf + ) + attn_weights = attn_weights.view(batch_size, self.num_heads, target_len, source_len) + attention_mask + attn_weights = attn_weights.view(batch_size * self.num_heads, target_len, source_len) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if output_attentions: + # this operation is a bit awkward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(batch_size, self.num_heads, target_len, source_len) + attn_weights = attn_weights_reshaped.view(batch_size * self.num_heads, target_len, source_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (batch_size * self.num_heads, target_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(batch_size, self.num_heads, target_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.view(batch_size, self.num_heads, target_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(batch_size, target_len, embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped + + +class PPDocLayoutV3EncoderLayer(nn.Module): + def __init__(self, config: PPDocLayoutV3Config): + super().__init__() + self.normalize_before = config.normalize_before + + # self-attention + self.self_attn = PPDocLayoutV3MultiheadAttention( + embed_dim=config.encoder_hidden_dim, + num_heads=config.num_attention_heads, + dropout=config.dropout, + ) + self.self_attn_layer_norm = nn.LayerNorm(config.encoder_hidden_dim, eps=config.layer_norm_eps) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.encoder_activation_function] + self.activation_dropout = config.activation_dropout + self.fc1 = nn.Linear(config.encoder_hidden_dim, config.encoder_ffn_dim) + self.fc2 = nn.Linear(config.encoder_ffn_dim, config.encoder_hidden_dim) + self.final_layer_norm = nn.LayerNorm(config.encoder_hidden_dim, eps=config.layer_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + position_embeddings: Optional[torch.Tensor] = None, + output_attentions: bool = False, + **kwargs, + ): + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`): attention mask of size + `(batch, 1, target_len, source_len)` where padding elements are indicated by very large negative + values. + position_embeddings (`torch.FloatTensor`, *optional*): + Object queries (also called content embeddings), to be added to the hidden states. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + if self.normalize_before: + hidden_states = self.self_attn_layer_norm(hidden_states) + + hidden_states, attn_weights = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_embeddings=position_embeddings, + output_attentions=output_attentions, + ) + + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + if not self.normalize_before: + hidden_states = self.self_attn_layer_norm(hidden_states) + + if self.normalize_before: + hidden_states = self.final_layer_norm(hidden_states) + residual = hidden_states + + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) + + hidden_states = self.fc2(hidden_states) + + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + + hidden_states = residual + hidden_states + if not self.normalize_before: + hidden_states = self.final_layer_norm(hidden_states) + + if self.training: + if torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any(): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (attn_weights,) + + return outputs + + +class PPDocLayoutV3RepVggBlock(nn.Module): + """ + RepVGG architecture block introduced by the work "RepVGG: Making VGG-style ConvNets Great Again". + """ + + def __init__(self, config: PPDocLayoutV3Config): + super().__init__() + + activation = config.activation_function + hidden_channels = int(config.encoder_hidden_dim * config.hidden_expansion) + self.conv1 = PPDocLayoutV3ConvNormLayer(config, hidden_channels, hidden_channels, 3, 1, padding=1) + self.conv2 = PPDocLayoutV3ConvNormLayer(config, hidden_channels, hidden_channels, 1, 1, padding=0) + self.activation = nn.Identity() if activation is None else ACT2CLS[activation]() + + def forward(self, x): + y = self.conv1(x) + self.conv2(x) + return self.activation(y) + + +class PPDocLayoutV3CSPRepLayer(nn.Module): + """ + Cross Stage Partial (CSP) network layer with RepVGG blocks. + """ + + def __init__(self, config: PPDocLayoutV3Config): + super().__init__() + + in_channels = config.encoder_hidden_dim * 2 + out_channels = config.encoder_hidden_dim + num_blocks = 3 + activation = config.activation_function + + hidden_channels = int(out_channels * config.hidden_expansion) + self.conv1 = PPDocLayoutV3ConvNormLayer(config, in_channels, hidden_channels, 1, 1, activation=activation) + self.conv2 = PPDocLayoutV3ConvNormLayer(config, in_channels, hidden_channels, 1, 1, activation=activation) + self.bottlenecks = nn.Sequential(*[PPDocLayoutV3RepVggBlock(config) for _ in range(num_blocks)]) + if hidden_channels != out_channels: + self.conv3 = PPDocLayoutV3ConvNormLayer(config, hidden_channels, out_channels, 1, 1, activation=activation) + else: + self.conv3 = nn.Identity() + + def forward(self, hidden_state): + hidden_state_1 = self.conv1(hidden_state) + hidden_state_1 = self.bottlenecks(hidden_state_1) + hidden_state_2 = self.conv2(hidden_state) + return self.conv3(hidden_state_1 + hidden_state_2) + + +class PPDocLayoutV3Encoder(nn.Module): + def __init__(self, config: PPDocLayoutV3Config): + super().__init__() + + self.layers = nn.ModuleList([PPDocLayoutV3EncoderLayer(config) for _ in range(config.encoder_layers)]) + + def forward(self, src, src_mask=None, pos_embed=None, output_attentions: bool = False) -> torch.Tensor: + hidden_states = src + for layer in self.layers: + hidden_states = layer( + hidden_states, + attention_mask=src_mask, + position_embeddings=pos_embed, + output_attentions=output_attentions, + ) + return hidden_states + + +class PPDocLayoutV3HybridEncoder(nn.Module): + """ + Decoder consisting of a projection layer, a set of `PPDocLayoutV3Encoder`, a top-down Feature Pyramid Network + (FPN) and a bottom-up Path Aggregation Network (PAN). More details on the paper: https://huggingface.co/papers/2304.08069 + + Args: + config: PPDocLayoutV3Config + """ + + def __init__(self, config: PPDocLayoutV3Config): + super().__init__() + self.config = config + self.in_channels = config.encoder_in_channels + self.feat_strides = config.feat_strides + self.encoder_hidden_dim = config.encoder_hidden_dim + self.encode_proj_layers = config.encode_proj_layers + self.positional_encoding_temperature = config.positional_encoding_temperature + self.eval_size = config.eval_size + self.out_channels = [self.encoder_hidden_dim for _ in self.in_channels] + self.out_strides = self.feat_strides + self.num_fpn_stages = len(self.in_channels) - 1 + self.num_pan_stages = len(self.in_channels) - 1 + activation = config.activation_function + + # encoder transformer + self.encoder = nn.ModuleList([PPDocLayoutV3Encoder(config) for _ in range(len(self.encode_proj_layers))]) + + # top-down FPN + self.lateral_convs = nn.ModuleList() + self.fpn_blocks = nn.ModuleList() + for _ in range(self.num_fpn_stages): + lateral_conv = PPDocLayoutV3ConvNormLayer( + config, + in_channels=self.encoder_hidden_dim, + out_channels=self.encoder_hidden_dim, + kernel_size=1, + stride=1, + activation=activation, + ) + fpn_block = PPDocLayoutV3CSPRepLayer(config) + self.lateral_convs.append(lateral_conv) + self.fpn_blocks.append(fpn_block) + + # bottom-up PAN + self.downsample_convs = nn.ModuleList() + self.pan_blocks = nn.ModuleList() + for _ in range(self.num_pan_stages): + downsample_conv = PPDocLayoutV3ConvNormLayer( + config, + in_channels=self.encoder_hidden_dim, + out_channels=self.encoder_hidden_dim, + kernel_size=3, + stride=2, + activation=activation, + ) + pan_block = PPDocLayoutV3CSPRepLayer(config) + self.downsample_convs.append(downsample_conv) + self.pan_blocks.append(pan_block) + + feat_strides = config.feat_strides + mask_feat_channels = config.mask_feat_channels + self.mask_feat_head = MaskFeatFPN( + [self.encoder_hidden_dim] * len(feat_strides), + feat_strides, + feat_channels=mask_feat_channels[0], + out_channels=mask_feat_channels[1], + ) + self.enc_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1) + self.enc_mask_output = nn.Sequential( + BaseConv(mask_feat_channels[1], mask_feat_channels[1], 3, 1), + nn.Conv2d(in_channels=mask_feat_channels[1], out_channels=config.num_prototypes, kernel_size=1), + ) + + @staticmethod + def build_2d_sincos_position_embedding( + width, height, embed_dim=256, temperature=10000.0, device="cpu", dtype=torch.float32 + ): + grid_w = torch.arange(torch_int(width), device=device).to(dtype) + grid_h = torch.arange(torch_int(height), device=device).to(dtype) + grid_w, grid_h = torch.meshgrid(grid_w, grid_h, indexing="xy") + if embed_dim % 4 != 0: + raise ValueError("Embed dimension must be divisible by 4 for 2D sin-cos position embedding") + pos_dim = embed_dim // 4 + omega = torch.arange(pos_dim, device=device).to(dtype) / pos_dim + omega = 1.0 / (temperature**omega) + + out_w = grid_w.flatten()[..., None] @ omega[None] + out_h = grid_h.flatten()[..., None] @ omega[None] + + return torch.concat([out_h.sin(), out_h.cos(), out_w.sin(), out_w.cos()], dim=1)[None, :, :] + + def forward( + self, + inputs_embeds=None, + x4_feat=None, + attention_mask=None, + position_embeddings=None, + spatial_shapes=None, + level_start_index=None, + valid_ratios=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + Args: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Flattened feature map (output of the backbone + projection layer) that is passed to the encoder. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding pixel features. Mask values selected in `[0, 1]`: + - 1 for pixel features that are real (i.e. **not masked**), + - 0 for pixel features that are padding (i.e. **masked**). + [What are attention masks?](../glossary#attention-mask) + position_embeddings (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Position embeddings that are added to the queries and keys in each self-attention layer. + spatial_shapes (`torch.LongTensor` of shape `(num_feature_levels, 2)`): + Spatial shapes of each feature map. + level_start_index (`torch.LongTensor` of shape `(num_feature_levels)`): + Starting index of each feature map. + valid_ratios (`torch.FloatTensor` of shape `(batch_size, num_feature_levels, 2)`): + Ratio of valid area in each feature level. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + hidden_states = inputs_embeds + + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + # encoder + if self.config.encoder_layers > 0: + for i, enc_ind in enumerate(self.encode_proj_layers): + if output_hidden_states: + encoder_states = encoder_states + (hidden_states[enc_ind],) + height, width = hidden_states[enc_ind].shape[2:] + # flatten [batch, channel, height, width] to [batch, height*width, channel] + src_flatten = hidden_states[enc_ind].flatten(2).permute(0, 2, 1) + if self.training or self.eval_size is None: + pos_embed = self.build_2d_sincos_position_embedding( + width, + height, + self.encoder_hidden_dim, + self.positional_encoding_temperature, + device=src_flatten.device, + dtype=src_flatten.dtype, + ) + else: + pos_embed = None + + layer_outputs = self.encoder[i]( + src_flatten, + pos_embed=pos_embed, + output_attentions=output_attentions, + ) + hidden_states[enc_ind] = ( + layer_outputs[0].permute(0, 2, 1).reshape(-1, self.encoder_hidden_dim, height, width).contiguous() + ) + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states[enc_ind],) + + # top-down FPN + fpn_feature_maps = [hidden_states[-1]] + for idx, (lateral_conv, fpn_block) in enumerate(zip(self.lateral_convs, self.fpn_blocks)): + backbone_feature_map = hidden_states[self.num_fpn_stages - idx - 1] + top_fpn_feature_map = fpn_feature_maps[-1] + # apply lateral block + top_fpn_feature_map = lateral_conv(top_fpn_feature_map) + fpn_feature_maps[-1] = top_fpn_feature_map + # apply fpn block + top_fpn_feature_map = F.interpolate(top_fpn_feature_map, scale_factor=2.0, mode="nearest") + fused_feature_map = torch.concat([top_fpn_feature_map, backbone_feature_map], dim=1) + new_fpn_feature_map = fpn_block(fused_feature_map) + fpn_feature_maps.append(new_fpn_feature_map) + + fpn_feature_maps.reverse() + + # bottom-up PAN + pan_feature_maps = [fpn_feature_maps[0]] + for idx, (downsample_conv, pan_block) in enumerate(zip(self.downsample_convs, self.pan_blocks)): + top_pan_feature_map = pan_feature_maps[-1] + fpn_feature_map = fpn_feature_maps[idx + 1] + downsampled_feature_map = downsample_conv(top_pan_feature_map) + fused_feature_map = torch.concat([downsampled_feature_map, fpn_feature_map], dim=1) + new_pan_feature_map = pan_block(fused_feature_map) + pan_feature_maps.append(new_pan_feature_map) + + mask_feat = self.mask_feat_head(pan_feature_maps) + mask_feat = F.interpolate(mask_feat, scale_factor=2, mode="bilinear", align_corners=False) + mask_feat += self.enc_mask_lateral(x4_feat[0]) + mask_feat = self.enc_mask_output(mask_feat) + + if not return_dict: + return tuple(v for v in [pan_feature_maps, encoder_states, all_attentions, mask_feat] if v is not None) + + return PPDocLayoutV3HybridEncoderOutput( + last_hidden_state=pan_feature_maps, + hidden_states=encoder_states, + attentions=all_attentions, + mask_feat=mask_feat, + ) + + +class PPDocLayoutV3DecoderLayer(nn.Module): + def __init__(self, config: PPDocLayoutV3Config): + super().__init__() + # self-attention + self.self_attn = PPDocLayoutV3MultiheadAttention( + embed_dim=config.d_model, + num_heads=config.decoder_attention_heads, + dropout=config.attention_dropout, + ) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.decoder_activation_function] + self.activation_dropout = config.activation_dropout + + self.self_attn_layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) + # cross-attention + self.encoder_attn = PPDocLayoutV3MultiscaleDeformableAttention( + config, + num_heads=config.decoder_attention_heads, + n_points=config.decoder_n_points, + ) + self.encoder_attn_layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) + # feedforward neural networks + self.fc1 = nn.Linear(config.d_model, config.decoder_ffn_dim) + self.fc2 = nn.Linear(config.decoder_ffn_dim, config.d_model) + self.final_layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: Optional[torch.Tensor] = None, + reference_points=None, + spatial_shapes=None, + spatial_shapes_list=None, + level_start_index=None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + ): + """ + Args: + hidden_states (`torch.FloatTensor`): + Input to the layer of shape `(seq_len, batch, embed_dim)`. + position_embeddings (`torch.FloatTensor`, *optional*): + Position embeddings that are added to the queries and keys in the self-attention layer. + reference_points (`torch.FloatTensor`, *optional*): + Reference points. + spatial_shapes (`torch.LongTensor`, *optional*): + Spatial shapes. + level_start_index (`torch.LongTensor`, *optional*): + Level start index. + encoder_hidden_states (`torch.FloatTensor`): + cross attention input to the layer of shape `(seq_len, batch, embed_dim)` + encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size + `(batch, 1, target_len, source_len)` where padding elements are indicated by very large negative + values. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + + # Self Attention + hidden_states, self_attn_weights = self.self_attn( + hidden_states=hidden_states, + attention_mask=encoder_attention_mask, + position_embeddings=position_embeddings, + output_attentions=output_attentions, + ) + + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + second_residual = hidden_states + + # Cross-Attention + cross_attn_weights = None + hidden_states, cross_attn_weights = self.encoder_attn( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + position_embeddings=position_embeddings, + reference_points=reference_points, + spatial_shapes=spatial_shapes, + spatial_shapes_list=spatial_shapes_list, + level_start_index=level_start_index, + output_attentions=output_attentions, + ) + + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = second_residual + hidden_states + + hidden_states = self.encoder_attn_layer_norm(hidden_states) + + # Fully Connected + residual = hidden_states + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) + hidden_states = self.fc2(hidden_states) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + hidden_states = self.final_layer_norm(hidden_states) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights, cross_attn_weights) + + return outputs + + +def inverse_sigmoid(x, eps=1e-5): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1 / x2) + + +class PPDocLayoutV3Decoder(PPDocLayoutV3PreTrainedModel): + def __init__(self, config: PPDocLayoutV3Config): + super().__init__(config) + + self.dropout = config.dropout + self.layers = nn.ModuleList([PPDocLayoutV3DecoderLayer(config) for _ in range(config.decoder_layers)]) + self.query_pos_head = PPDocLayoutV3MLPPredictionHead( + config, 4, 2 * config.d_model, config.d_model, num_layers=2 + ) + + # hack implementation for iterative bounding box refinement and two-stage Deformable DETR + self.bbox_embed = None + self.class_embed = None + + self.num_queries = config.num_queries + + # Initialize weights and apply final processing + self.post_init() + + def forward( + self, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + position_embeddings=None, + reference_points=None, + spatial_shapes=None, + spatial_shapes_list=None, + level_start_index=None, + valid_ratios=None, + order_head=None, + global_pointer=None, + mask_query_head=None, + dec_norm=None, + mask_feat=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs, + ): + r""" + Args: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): + The query embeddings that are passed into the decoder. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + of the decoder. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing cross-attention on padding pixel_values of the encoder. Mask values selected + in `[0, 1]`: + - 1 for pixels that are real (i.e. **not masked**), + - 0 for pixels that are padding (i.e. **masked**). + position_embeddings (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Position embeddings that are added to the queries and keys in each self-attention layer. + reference_points (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)` is `as_two_stage` else `(batch_size, num_queries, 2)` or , *optional*): + Reference point in range `[0, 1]`, top-left (0,0), bottom-right (1, 1), including padding area. + spatial_shapes (`torch.FloatTensor` of shape `(num_feature_levels, 2)`): + Spatial shapes of the feature maps. + level_start_index (`torch.LongTensor` of shape `(num_feature_levels)`, *optional*): + Indexes for the start of each feature level. In range `[0, sequence_length]`. + valid_ratios (`torch.FloatTensor` of shape `(batch_size, num_feature_levels, 2)`, *optional*): + Ratio of valid area in each feature level. + + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if inputs_embeds is not None: + hidden_states = inputs_embeds + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None + intermediate = () + intermediate_reference_points = () + intermediate_logits = () + dec_out_order_logits = () + dec_out_masks = () + + reference_points = F.sigmoid(reference_points) + + # https://github.com/lyuwenyu/RT-DETR/blob/94f5e16708329d2f2716426868ec89aa774af016/rtdetr_pytorch/src/zoo/rtdetr/rtdetr_decoder.py#L252 + for idx, decoder_layer in enumerate(self.layers): + reference_points_input = reference_points.unsqueeze(2) + position_embeddings = self.query_pos_head(reference_points) + + if output_hidden_states: + all_hidden_states += (hidden_states,) + + layer_outputs = decoder_layer( + hidden_states, + position_embeddings=position_embeddings, + encoder_hidden_states=encoder_hidden_states, + reference_points=reference_points_input, + spatial_shapes=spatial_shapes, + spatial_shapes_list=spatial_shapes_list, + level_start_index=level_start_index, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + ) + + hidden_states = layer_outputs[0] + + # hack implementation for iterative bounding box refinement + if self.bbox_embed is not None: + predicted_corners = self.bbox_embed(hidden_states) + new_reference_points = F.sigmoid(predicted_corners + inverse_sigmoid(reference_points)) + reference_points = new_reference_points.detach() + + intermediate += (hidden_states,) + intermediate_reference_points += ( + (new_reference_points,) if self.bbox_embed is not None else (reference_points,) + ) + + # get_pred_class_order_and_mask + out_query = dec_norm(hidden_states) + mask_query_embed = mask_query_head(out_query) + batch_size, mask_dim, _ = mask_query_embed.shape + _, _, mask_h, mask_w = mask_feat.shape + out_mask = torch.bmm(mask_query_embed, mask_feat.flatten(start_dim=2)).reshape( + batch_size, mask_dim, mask_h, mask_w + ) + dec_out_masks += (out_mask,) + + if self.class_embed is not None: + logits = self.class_embed(out_query) + intermediate_logits += (logits,) + + if order_head is not None and global_pointer is not None: + valid_query = out_query[:, -self.num_queries :] if self.num_queries is not None else out_query + order_logits = global_pointer(order_head(valid_query)) + dec_out_order_logits += (order_logits,) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + if encoder_hidden_states is not None: + all_cross_attentions += (layer_outputs[2],) + + # Keep batch_size as first dimension + intermediate = torch.stack(intermediate, dim=1) + intermediate_reference_points = torch.stack(intermediate_reference_points, dim=1) + if self.class_embed is not None: + intermediate_logits = torch.stack(intermediate_logits, dim=1) + if order_head is not None and global_pointer is not None: + dec_out_order_logits = torch.stack(dec_out_order_logits, dim=1) + dec_out_masks = torch.stack(dec_out_masks, dim=1) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if not return_dict: + return tuple( + v + for v in [ + hidden_states, + intermediate, + intermediate_logits, + intermediate_reference_points, + dec_out_order_logits, + dec_out_masks, + all_hidden_states, + all_self_attns, + all_cross_attentions, + ] + if v is not None + ) + return PPDocLayoutV3DecoderOutput( + last_hidden_state=hidden_states, + intermediate_hidden_states=intermediate, + intermediate_logits=intermediate_logits, + intermediate_reference_points=intermediate_reference_points, + dec_out_order_logits=dec_out_order_logits, + dec_out_masks=dec_out_masks, + hidden_states=all_hidden_states, + attentions=all_self_attns, + cross_attentions=all_cross_attentions, + ) + + +class PPDocLayoutV3FrozenBatchNorm2d(nn.Module): + """ + BatchNorm2d where the batch statistics and the affine parameters are fixed. + + Copy-paste from torchvision.misc.ops with added eps before rqsrt, without which any other models than + torchvision.models.resnet[18,34,50,101] produce nans. + """ + + def __init__(self, n): + super().__init__() + self.register_buffer("weight", torch.ones(n)) + self.register_buffer("bias", torch.zeros(n)) + self.register_buffer("running_mean", torch.zeros(n)) + self.register_buffer("running_var", torch.ones(n)) + + def _load_from_state_dict( + self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ): + num_batches_tracked_key = prefix + "num_batches_tracked" + if num_batches_tracked_key in state_dict: + del state_dict[num_batches_tracked_key] + + super()._load_from_state_dict( + state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ) + + def forward(self, x): + # move reshapes to the beginning + # to make it user-friendly + weight = self.weight.reshape(1, -1, 1, 1) + bias = self.bias.reshape(1, -1, 1, 1) + running_var = self.running_var.reshape(1, -1, 1, 1) + running_mean = self.running_mean.reshape(1, -1, 1, 1) + epsilon = 1e-5 + scale = weight * (running_var + epsilon).rsqrt() + bias = bias - running_mean * scale + return x * scale + bias + + +def replace_batch_norm(model): + r""" + Recursively replace all `torch.nn.BatchNorm2d` with `PPDocLayoutV3FrozenBatchNorm2d`. + + Args: + model (torch.nn.Module): + input model + """ + for name, module in model.named_children(): + if isinstance(module, nn.BatchNorm2d): + new_module = PPDocLayoutV3FrozenBatchNorm2d(module.num_features) + + if module.weight.device != torch.device("meta"): + new_module.weight.copy_(module.weight) + new_module.bias.copy_(module.bias) + new_module.running_mean.copy_(module.running_mean) + new_module.running_var.copy_(module.running_var) + + model._modules[name] = new_module + + if len(list(module.children())) > 0: + replace_batch_norm(module) + + +class PPDocLayoutV3ConvEncoder(nn.Module): + """ + Convolutional backbone using the modeling_pp_doclayout_v3_resnet.py. + + nn.BatchNorm2d layers are replaced by PPDocLayoutV3FrozenBatchNorm2d as defined above. + https://github.com/lyuwenyu/RT-DETR/blob/main/PPDocLayoutV3_pytorch/src/nn/backbone/presnet.py#L142 + """ + + def __init__(self, config): + super().__init__() + + backbone = load_backbone(config) + + if config.freeze_backbone_batch_norms: + # replace batch norm by frozen batch norm + with torch.no_grad(): + replace_batch_norm(backbone) + self.model = backbone + self.intermediate_channel_sizes = self.model.channels + + def forward(self, pixel_values: torch.Tensor, pixel_mask: torch.Tensor): + # send pixel_values through the model to get list of feature maps + features = self.model(pixel_values).feature_maps + + out = [] + for feature_map in features: + # downsample pixel_mask to match shape of corresponding feature_map + mask = nn.functional.interpolate(pixel_mask[None].float(), size=feature_map.shape[-2:]).to(torch.bool)[0] + out.append((feature_map, mask)) + return out + + +def get_contrastive_denoising_training_group( + targets, + num_classes, + num_queries, + class_embed, + num_denoising_queries=100, + label_noise_ratio=0.5, + box_noise_scale=1.0, +): + """ + Creates a contrastive denoising training group using ground-truth samples. It adds noise to labels and boxes. + + Args: + targets (`list[dict]`): + The target objects, each containing 'class_labels' and 'boxes' for objects in an image. + num_classes (`int`): + Total number of classes in the dataset. + num_queries (`int`): + Number of query slots in the transformer. + class_embed (`callable`): + A function or a model layer to embed class labels. + num_denoising_queries (`int`, *optional*, defaults to 100): + Number of denoising queries. + label_noise_ratio (`float`, *optional*, defaults to 0.5): + Ratio of noise applied to labels. + box_noise_scale (`float`, *optional*, defaults to 1.0): + Scale of noise applied to bounding boxes. + Returns: + `tuple` comprising various elements: + - **input_query_class** (`torch.FloatTensor`) -- + Class queries with applied label noise. + - **input_query_bbox** (`torch.FloatTensor`) -- + Bounding box queries with applied box noise. + - **attn_mask** (`torch.FloatTensor`) -- + Attention mask for separating denoising and reconstruction queries. + - **denoising_meta_values** (`dict`) -- + Metadata including denoising positive indices, number of groups, and split sizes. + """ + + if num_denoising_queries <= 0: + return None, None, None, None + + num_ground_truths = [len(t["class_labels"]) for t in targets] + device = targets[0]["class_labels"].device + + max_gt_num = max(num_ground_truths) + if max_gt_num == 0: + return None, None, None, None + + num_groups_denoising_queries = num_denoising_queries // max_gt_num + num_groups_denoising_queries = 1 if num_groups_denoising_queries == 0 else num_groups_denoising_queries + # pad gt to max_num of a batch + batch_size = len(num_ground_truths) + + input_query_class = torch.full([batch_size, max_gt_num], num_classes, dtype=torch.int32, device=device) + input_query_bbox = torch.zeros([batch_size, max_gt_num, 4], device=device) + pad_gt_mask = torch.zeros([batch_size, max_gt_num], dtype=torch.bool, device=device) + + for i in range(batch_size): + num_gt = num_ground_truths[i] + if num_gt > 0: + input_query_class[i, :num_gt] = targets[i]["class_labels"] + input_query_bbox[i, :num_gt] = targets[i]["boxes"] + pad_gt_mask[i, :num_gt] = 1 + # each group has positive and negative queries. + input_query_class = input_query_class.tile([1, 2 * num_groups_denoising_queries]) + input_query_bbox = input_query_bbox.tile([1, 2 * num_groups_denoising_queries, 1]) + pad_gt_mask = pad_gt_mask.tile([1, 2 * num_groups_denoising_queries]) + # positive and negative mask + negative_gt_mask = torch.zeros([batch_size, max_gt_num * 2, 1], device=device) + negative_gt_mask[:, max_gt_num:] = 1 + negative_gt_mask = negative_gt_mask.tile([1, num_groups_denoising_queries, 1]) + positive_gt_mask = 1 - negative_gt_mask + # contrastive denoising training positive index + positive_gt_mask = positive_gt_mask.squeeze(-1) * pad_gt_mask + denoise_positive_idx = torch.nonzero(positive_gt_mask)[:, 1] + denoise_positive_idx = torch.split( + denoise_positive_idx, [n * num_groups_denoising_queries for n in num_ground_truths] + ) + # total denoising queries + num_denoising_queries = torch_int(max_gt_num * 2 * num_groups_denoising_queries) + + if label_noise_ratio > 0: + mask = torch.rand_like(input_query_class, dtype=torch.float) < (label_noise_ratio * 0.5) + # randomly put a new one here + new_label = torch.randint_like(mask, 0, num_classes, dtype=input_query_class.dtype) + input_query_class = torch.where(mask & pad_gt_mask, new_label, input_query_class) + + if box_noise_scale > 0: + known_bbox = center_to_corners_format(input_query_bbox) + diff = torch.tile(input_query_bbox[..., 2:] * 0.5, [1, 1, 2]) * box_noise_scale + rand_sign = torch.randint_like(input_query_bbox, 0, 2) * 2.0 - 1.0 + rand_part = torch.rand_like(input_query_bbox) + rand_part = (rand_part + 1.0) * negative_gt_mask + rand_part * (1 - negative_gt_mask) + rand_part *= rand_sign + known_bbox += rand_part * diff + known_bbox.clip_(min=0.0, max=1.0) + input_query_bbox = corners_to_center_format(known_bbox) + input_query_bbox = inverse_sigmoid(input_query_bbox) + + input_query_class = class_embed(input_query_class) + + target_size = num_denoising_queries + num_queries + attn_mask = torch.full([target_size, target_size], 0, dtype=torch.float, device=device) + # match query cannot see the reconstruction + attn_mask[num_denoising_queries:, :num_denoising_queries] = -torch.inf + + # reconstructions cannot see each other + for i in range(num_groups_denoising_queries): + idx_block_start = max_gt_num * 2 * i + idx_block_end = max_gt_num * 2 * (i + 1) + attn_mask[idx_block_start:idx_block_end, :idx_block_start] = -torch.inf + attn_mask[idx_block_start:idx_block_end, idx_block_end:num_denoising_queries] = -torch.inf + + denoising_meta_values = { + "dn_positive_idx": denoise_positive_idx, + "dn_num_group": num_groups_denoising_queries, + "dn_num_split": [num_denoising_queries, num_queries], + } + + return input_query_class, input_query_bbox, attn_mask, denoising_meta_values + + +def bbox_xyxy_to_cxcywh(x): + x1 = x[..., 0] + y1 = x[..., 1] + x2 = x[..., 2] + y2 = x[..., 3] + return torch.stack([(x1 + x2) / 2, (y1 + y2) / 2, (x2 - x1), (y2 - y1)], dim=-1) + + +def mask_to_box_coordinate(mask, normalize=False, format="xyxy", dtype=torch.float32): + assert mask.ndim == 4, f"Error: The 'mask' variable must be a 4-dimensional array, but got {mask.ndim} dimensions." + assert format in ["xyxy", "xywh"], ( + f"Error: The 'format' variable must be either 'xyxy' or 'xywh', but got '{format}'." + ) + + mask = mask.bool() + + h, w = mask.shape[-2:] + y, x = torch.meshgrid(torch.arange(h, device=mask.device), torch.arange(w, device=mask.device), indexing="ij") + x = x.to(dtype) + y = y.to(dtype) + + x_mask = x * mask + x_max = x_mask.flatten(start_dim=-2).max(dim=-1).values + 1 + x_min = ( + torch.where(mask, x_mask, torch.tensor(1e8, device=mask.device, dtype=dtype)) + .flatten(start_dim=-2) + .min(dim=-1) + .values + ) + + y_mask = y * mask + y_max = y_mask.flatten(start_dim=-2).max(dim=-1).values + 1 + y_min = ( + torch.where(mask, y_mask, torch.tensor(1e8, device=mask.device, dtype=dtype)) + .flatten(start_dim=-2) + .min(dim=-1) + .values + ) + + out_bbox = torch.stack([x_min, y_min, x_max, y_max], dim=-1) + mask_any = torch.any(mask, dim=(-2, -1)).unsqueeze(-1) + out_bbox = out_bbox * mask_any + + if normalize: + norm_tensor = torch.tensor([w, h, w, h], device=mask.device, dtype=dtype) + out_bbox /= norm_tensor + + return out_bbox if format == "xyxy" else bbox_xyxy_to_cxcywh(out_bbox) + + +@auto_docstring( + custom_intro=""" + RT-DETR Model (consisting of a backbone and encoder-decoder) outputting raw hidden states without any head on top. + """ +) +class PPDocLayoutV3Model(PPDocLayoutV3PreTrainedModel): + def __init__(self, config: PPDocLayoutV3Config): + super().__init__(config) + + # Create backbone + self.backbone = PPDocLayoutV3ConvEncoder(config) + intermediate_channel_sizes = self.backbone.intermediate_channel_sizes + + # Create encoder input projection layers + # https://github.com/lyuwenyu/RT-DETR/blob/94f5e16708329d2f2716426868ec89aa774af016/PPDocLayoutV3_pytorch/src/zoo/PPDocLayoutV3/hybrid_encoder.py#L212 + num_backbone_outs = len(intermediate_channel_sizes) + encoder_input_proj_list = [] + for _ in range(num_backbone_outs): + in_channels = intermediate_channel_sizes[_] + encoder_input_proj_list.append( + nn.Sequential( + nn.Conv2d(in_channels, config.encoder_hidden_dim, kernel_size=1, bias=False), + nn.BatchNorm2d(config.encoder_hidden_dim), + ) + ) + + self.encoder_input_proj = nn.ModuleList(encoder_input_proj_list[1:]) # noqa + + # Create encoder + self.encoder = PPDocLayoutV3HybridEncoder(config) + + # denoising part + if config.num_denoising > 0: + self.denoising_class_embed = nn.Embedding( + config.num_labels + 1, config.d_model, padding_idx=config.num_labels + ) + + # decoder embedding + if config.learn_initial_query: + self.weight_embedding = nn.Embedding(config.num_queries, config.d_model) + + # encoder head + self.enc_output = nn.Sequential( + nn.Linear(config.d_model, config.d_model), + nn.LayerNorm(config.d_model, eps=config.layer_norm_eps), + ) + self.enc_score_head = nn.Linear(config.d_model, config.num_labels) + self.enc_bbox_head = PPDocLayoutV3MLPPredictionHead(config, config.d_model, config.d_model, 4, num_layers=3) + + # init encoder output anchors and valid_mask + if config.anchor_image_size: + self.anchors, self.valid_mask = self.generate_anchors(dtype=self.dtype) + + # Create decoder input projection layers + # https://github.com/lyuwenyu/RT-DETR/blob/94f5e16708329d2f2716426868ec89aa774af016/PPDocLayoutV3_pytorch/src/zoo/PPDocLayoutV3/PPDocLayoutV3_decoder.py#L412 + num_backbone_outs = len(config.decoder_in_channels) + decoder_input_proj_list = [] + for _ in range(num_backbone_outs): + in_channels = config.decoder_in_channels[_] + decoder_input_proj_list.append( + nn.Sequential( + nn.Conv2d(in_channels, config.d_model, kernel_size=1, bias=False), + nn.BatchNorm2d(config.d_model, config.batch_norm_eps), + ) + ) + for _ in range(config.num_feature_levels - num_backbone_outs): + decoder_input_proj_list.append( + nn.Sequential( + nn.Conv2d(in_channels, config.d_model, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(config.d_model, config.batch_norm_eps), + ) + ) + in_channels = config.d_model + self.decoder_input_proj = nn.ModuleList(decoder_input_proj_list) + self.decoder = PPDocLayoutV3Decoder(config) + + self.dec_order_head = nn.Linear(config.d_model, config.d_model) + self.dec_global_pointer = GlobalPointer(config) + self.dec_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) + self.decoder.class_embed = self.enc_score_head + self.decoder.bbox_embed = self.enc_bbox_head + + self.mask_enhanced = config.mask_enhanced + self.mask_query_head = PPDocLayoutV3MLPPredictionHead( + config, config.d_model, config.d_model, config.num_prototypes, num_layers=3 + ) + + self.post_init() + + def freeze_backbone(self): + for param in self.backbone.parameters(): + param.requires_grad_(False) + + def unfreeze_backbone(self): + for param in self.backbone.parameters(): + param.requires_grad_(True) + + @compile_compatible_method_lru_cache(maxsize=32) + def generate_anchors(self, spatial_shapes=None, grid_size=0.05, device="cpu", dtype=torch.float32): + if spatial_shapes is None: + spatial_shapes = [ + [int(self.config.anchor_image_size[0] / s), int(self.config.anchor_image_size[1] / s)] + for s in self.config.feat_strides + ] + anchors = [] + for level, (height, width) in enumerate(spatial_shapes): + grid_y, grid_x = torch.meshgrid( + torch.arange(end=height, device=device).to(dtype), + torch.arange(end=width, device=device).to(dtype), + indexing="ij", + ) + grid_xy = torch.stack([grid_x, grid_y], -1) + grid_xy = grid_xy.unsqueeze(0) + 0.5 + grid_xy[..., 0] /= width + grid_xy[..., 1] /= height + wh = torch.ones_like(grid_xy) * grid_size * (2.0**level) + anchors.append(torch.concat([grid_xy, wh], -1).reshape(-1, height * width, 4)) + # define the valid range for anchor coordinates + eps = 1e-2 + anchors = torch.concat(anchors, 1) + valid_mask = ((anchors > eps) * (anchors < 1 - eps)).all(-1, keepdim=True) + anchors = torch.log(anchors / (1 - anchors)) + anchors = torch.where(valid_mask, anchors, torch.tensor(torch.finfo(dtype).max, dtype=dtype, device=device)) + + return anchors, valid_mask + + @auto_docstring + def forward( + self, + pixel_values: torch.FloatTensor, + pixel_mask: Optional[torch.LongTensor] = None, + encoder_outputs: Optional[torch.FloatTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[list[dict]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **kwargs, + ) -> Union[tuple[torch.FloatTensor], PPDocLayoutV3ModelOutput]: + r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. + labels (`list[Dict]` of len `(batch_size,)`, *optional*): + Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the + following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch + respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes + in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. + + Examples: + + ```python + >>> from transformers import AutoImageProcessor, PPDocLayoutV2Model + >>> from PIL import Image + >>> import requests + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> image_processor = AutoImageProcessor.from_pretrained("PekingU/PPDocLayoutV2_r50vd") + >>> model = PPDocLayoutV2Model.from_pretrained("PekingU/PPDocLayoutV2_r50vd") + + >>> inputs = image_processor(images=image, return_tensors="pt") + + >>> outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + [1, 300, 256] + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + batch_size, num_channels, height, width = pixel_values.shape + device = pixel_values.device + + if pixel_mask is None: + pixel_mask = torch.ones(((batch_size, height, width)), device=device) + + features = self.backbone(pixel_values, pixel_mask) + x4_feat = features.pop(0) + proj_feats = [self.encoder_input_proj[level](source) for level, (source, mask) in enumerate(features)] + + if encoder_outputs is None: + encoder_outputs = self.encoder( + proj_feats, + x4_feat, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + # If the user passed a tuple for encoder_outputs, we wrap it in a PPDocLayoutV3HybridEncoderOutput when return_dict=True + elif return_dict and not isinstance(encoder_outputs, PPDocLayoutV3HybridEncoderOutput): + encoder_outputs = PPDocLayoutV3HybridEncoderOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] if output_hidden_states else None, + attentions=encoder_outputs[2] + if len(encoder_outputs) > 2 + else encoder_outputs[1] + if output_attentions + else None, + mask_feat=encoder_outputs[-1], + ) + + mask_feat = ( + encoder_outputs.mask_feat + if isinstance(encoder_outputs, PPDocLayoutV3HybridEncoderOutput) + else encoder_outputs[-1] + ) + + # Equivalent to def _get_encoder_input + # https://github.com/lyuwenyu/RT-DETR/blob/94f5e16708329d2f2716426868ec89aa774af016/rtdetr_pytorch/src/zoo/rtdetr/rtdetr_decoder.py#L412 + sources = [] + for level, source in enumerate(encoder_outputs[0]): + sources.append(self.decoder_input_proj[level](source)) + + # Lowest resolution feature maps are obtained via 3x3 stride 2 convolutions on the final stage + if self.config.num_feature_levels > len(sources): + _len_sources = len(sources) + sources.append(self.decoder_input_proj[_len_sources](encoder_outputs[0])[-1]) + for i in range(_len_sources + 1, self.config.num_feature_levels): + sources.append(self.decoder_input_proj[i](encoder_outputs[0][-1])) + + # Prepare encoder inputs (by flattening) + source_flatten = [] + spatial_shapes_list = [] + spatial_shapes = torch.empty((len(sources), 2), device=device, dtype=torch.long) + for level, source in enumerate(sources): + height, width = source.shape[-2:] + spatial_shapes[level, 0] = height + spatial_shapes[level, 1] = width + spatial_shapes_list.append((height, width)) + source = source.flatten(2).transpose(1, 2) + source_flatten.append(source) + source_flatten = torch.cat(source_flatten, 1) + level_start_index = torch.cat((spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1])) + + # prepare denoising training + if self.training and self.config.num_denoising > 0 and labels is not None: + ( + denoising_class, + denoising_bbox_unact, + attention_mask, + denoising_meta_values, + ) = get_contrastive_denoising_training_group( + targets=labels, + num_classes=self.config.num_labels, + num_queries=self.config.num_queries, + class_embed=self.denoising_class_embed, + num_denoising_queries=self.config.num_denoising, + label_noise_ratio=self.config.label_noise_ratio, + box_noise_scale=self.config.box_noise_scale, + ) + else: + denoising_class, denoising_bbox_unact, attention_mask, denoising_meta_values = None, None, None, None + + batch_size = len(source_flatten) + device = source_flatten.device + dtype = source_flatten.dtype + + # prepare input for decoder + if self.training or self.config.anchor_image_size is None: + # Pass spatial_shapes as tuple to make it hashable and make sure + # lru_cache is working for generate_anchors() + spatial_shapes_tuple = tuple(spatial_shapes_list) + anchors, valid_mask = self.generate_anchors(spatial_shapes_tuple, device=device, dtype=dtype) + else: + anchors, valid_mask = self.anchors, self.valid_mask + anchors, valid_mask = anchors.to(device, dtype), valid_mask.to(device, dtype) + + # use the valid_mask to selectively retain values in the feature map where the mask is `True` + memory = valid_mask.to(source_flatten.dtype) * source_flatten + + output_memory = self.enc_output(memory) + + enc_outputs_class = self.enc_score_head(output_memory) + enc_outputs_coord_logits = self.enc_bbox_head(output_memory) + anchors + + _, topk_ind = torch.topk(enc_outputs_class.max(-1).values, self.config.num_queries, dim=1) + + reference_points_unact = enc_outputs_coord_logits.gather( + dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, enc_outputs_coord_logits.shape[-1]) + ) + + # _get_pred_class_and_mask + batch_ind = torch.arange(memory.shape[0], device=output_memory.device).unsqueeze(1) + target = output_memory[batch_ind, topk_ind] + out_query = self.dec_norm(target) + mask_query_embed = self.mask_query_head(out_query) + batch_size, mask_dim, _ = mask_query_embed.shape + _, _, mask_h, mask_w = mask_feat.shape + enc_out_masks = torch.bmm(mask_query_embed, mask_feat.flatten(start_dim=2)).reshape( + batch_size, mask_dim, mask_h, mask_w + ) + + enc_topk_bboxes = F.sigmoid(reference_points_unact) + + enc_topk_logits = enc_outputs_class.gather( + dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, enc_outputs_class.shape[-1]) + ) + + # extract region features + if self.config.learn_initial_query: + target = self.weight_embedding.tile([batch_size, 1, 1]) + else: + target = output_memory.gather(dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, output_memory.shape[-1])) + target = target.detach() + + if denoising_class is not None: + target = torch.concat([denoising_class, target], 1) + + if self.mask_enhanced: + reference_points = mask_to_box_coordinate(enc_out_masks > 0, normalize=True, format="xywh") + reference_points_unact = inverse_sigmoid(reference_points) + + if denoising_bbox_unact is not None: + reference_points_unact = torch.concat([denoising_bbox_unact, reference_points_unact], 1) + + init_reference_points = reference_points_unact.detach() + + # decoder + decoder_outputs = self.decoder( + inputs_embeds=target, + encoder_hidden_states=source_flatten, + encoder_attention_mask=attention_mask, + reference_points=init_reference_points, + spatial_shapes=spatial_shapes, + spatial_shapes_list=spatial_shapes_list, + level_start_index=level_start_index, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + order_head=self.dec_order_head, + global_pointer=self.dec_global_pointer, + mask_query_head=self.mask_query_head, + dec_norm=self.dec_norm, + mask_feat=mask_feat, + ) + + if not return_dict: + enc_outputs = tuple( + value + for value in [enc_topk_logits, enc_topk_bboxes, enc_outputs_class, enc_outputs_coord_logits] + if value is not None + ) + dn_outputs = tuple(value if value is not None else None for value in [denoising_meta_values]) + tuple_outputs = ( + decoder_outputs + encoder_outputs[:-1] + (init_reference_points,) + enc_outputs + dn_outputs + ) + + return tuple_outputs + + return PPDocLayoutV3ModelOutput( + last_hidden_state=decoder_outputs.last_hidden_state, + intermediate_hidden_states=decoder_outputs.intermediate_hidden_states, + intermediate_logits=decoder_outputs.intermediate_logits, + intermediate_reference_points=decoder_outputs.intermediate_reference_points, + intermediate_predicted_corners=decoder_outputs.intermediate_predicted_corners, + initial_reference_points=decoder_outputs.initial_reference_points, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + out_order_logits=decoder_outputs.dec_out_order_logits, + out_masks=decoder_outputs.dec_out_masks, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + init_reference_points=init_reference_points, + enc_topk_logits=enc_topk_logits, + enc_topk_bboxes=enc_topk_bboxes, + enc_outputs_class=enc_outputs_class, + enc_outputs_coord_logits=enc_outputs_coord_logits, + denoising_meta_values=denoising_meta_values, + ) + + +@dataclass +class PPDocLayoutV3HybridEncoderOutput(BaseModelOutput): + mask_feat: torch.FloatTensor = None + + +@dataclass +@auto_docstring +class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): + r""" + logits (`torch.FloatTensor` of shape `(batch_size, num_queries, num_classes + 1)`): + Classification logits (including no-object) for all queries. + order_logits (`tuple` of `torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): + Order logits of the final layer of the decoder. + out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, height, width)`): + Masks of the final layer of the decoder. + pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): + Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These + values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding + possible padding). You can use [`~RTDetrImageProcessor.post_process_object_detection`] to retrieve the + unnormalized (absolute) bounding boxes. + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the decoder of the model. + intermediate_hidden_states (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, hidden_size)`): + Stacked intermediate hidden states (output of each layer of the decoder). + intermediate_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, config.num_labels)`): + Stacked intermediate logits (logits of each layer of the decoder). + intermediate_reference_points (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked intermediate reference points (reference points of each layer of the decoder). + intermediate_predicted_corners (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked intermediate predicted corners (predicted corners of each layer of the decoder). + initial_reference_points (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked initial reference points (initial reference points of each layer of the decoder). + init_reference_points (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): + Initial reference points sent through the Transformer decoder. + enc_topk_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Logits of predicted bounding boxes coordinates in the encoder. + enc_topk_bboxes (`torch.FloatTensor` of shape `(batch_size, sequence_length, 4)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Logits of predicted bounding boxes coordinates in the encoder. + enc_outputs_class (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Predicted bounding boxes scores where the top `config.two_stage_num_proposals` scoring bounding boxes are + picked as region proposals in the first stage. Output of bounding box binary classification (i.e. + foreground and background). + enc_outputs_coord_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, 4)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Logits of predicted bounding boxes coordinates in the first stage. + denoising_meta_values (`dict`): + Extra dictionary for the denoising related values + """ + + logits: Optional[torch.FloatTensor] = None + pred_boxes: Optional[torch.FloatTensor] = None + order_logits: Optional[torch.FloatTensor] = None + out_masks: Optional[torch.FloatTensor] = None + last_hidden_state: Optional[torch.FloatTensor] = None + intermediate_hidden_states: Optional[torch.FloatTensor] = None + intermediate_logits: Optional[torch.FloatTensor] = None + intermediate_reference_points: Optional[torch.FloatTensor] = None + intermediate_predicted_corners: Optional[torch.FloatTensor] = None + initial_reference_points: Optional[torch.FloatTensor] = None + decoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None + decoder_attentions: Optional[tuple[torch.FloatTensor]] = None + cross_attentions: Optional[tuple[torch.FloatTensor]] = None + encoder_last_hidden_state: Optional[torch.FloatTensor] = None + encoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None + encoder_attentions: Optional[tuple[torch.FloatTensor]] = None + init_reference_points: Optional[tuple[torch.FloatTensor]] = None + enc_topk_logits: Optional[torch.FloatTensor] = None + enc_topk_bboxes: Optional[torch.FloatTensor] = None + enc_outputs_class: Optional[torch.FloatTensor] = None + enc_outputs_coord_logits: Optional[torch.FloatTensor] = None + denoising_meta_values: Optional[dict] = None + + +@auto_docstring( + custom_intro=""" + RT-DETR Model (consisting of a backbone and encoder-decoder) outputting bounding boxes and logits to be further + decoded into scores and classes. + """ +) +class PPDocLayoutV3ForObjectDetection(PPDocLayoutV3PreTrainedModel): + # When using clones, all layers > 0 will be clones, but layer 0 *is* required + # We can't initialize the model on meta device as some weights are modified during the initialization + _no_split_modules = None + _keys_to_ignore_on_load_missing = ["num_batches_tracked", "rel_pos_y_bias", "rel_pos_x_bias"] + _tied_weights_keys = { + "model.decoder.class_embed": "model.enc_score_head", + "model.decoder.bbox_embed": "model.enc_bbox_head", + } + + def __init__(self, config: PPDocLayoutV3Config): + super().__init__(config) + self.model = PPDocLayoutV3Model(config) + + self.model.denoising_class_embed = nn.Embedding(config.num_labels, config.d_model) + self.num_queries = config.num_queries + # if two-stage, the last class_embed and bbox_embed is for region proposal generation + self.post_init() + + def _set_aux_loss(self, outputs_class, outputs_coord): + return [{"logits": a, "pred_boxes": b} for a, b in zip(outputs_class, outputs_coord)] + + @auto_docstring + def forward( + self, + pixel_values: torch.FloatTensor, + pixel_mask: Optional[torch.LongTensor] = None, + encoder_outputs: Optional[torch.FloatTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[list[dict]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **kwargs, + ) -> Union[tuple[torch.FloatTensor], PPDocLayoutV3ForObjectDetectionOutput]: + r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. + labels (`list[Dict]` of len `(batch_size,)`, *optional*): + Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the + following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch + respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes + in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. + + Examples: + + ```python + >>> from transformers import AutoModelForObjectDetection, AutoImageProcessor + >>> from PIL import Image + >>> import requests + >>> import torch + + >>> url = "https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/layout_demo.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> model_path = "PaddlePaddle/PP-DocLayoutV3_safetensors" + >>> image_processor = AutoImageProcessor.from_pretrained(model_path) + >>> model = AutoModelForObjectDetection.from_pretrained(model_path) + + >>> # prepare image for the model + >>> inputs = image_processor(images=[image], return_tensors="pt") + + >>> # forward pass + >>> outputs = model(**inputs) + + >>> # convert outputs (bounding boxes and class logits) to Pascal VOC format (xmin, ymin, xmax, ymax) + >>> results = image_processor.post_process_object_detection(outputs, target_sizes=torch.tensor([image.size[::-1]])) + + >>> # print outputs + >>> for result in results: + ... for idx, (score, label_id, box) in enumerate(zip(result["scores"], result["labels"], result["boxes"])): + ... score, label = score.item(), label_id.item() + ... box = [round(i, 2) for i in box.tolist()] + ... print(f"Order {idx + 1}: {model.config.id2label[label]}: {score:.2f} {box}") + Order 1: text: 0.99 [334.95, 184.78, 897.25, 654.83] + Order 2: paragraph_title: 0.97 [337.28, 683.92, 869.16, 798.35] + Order 3: text: 0.99 [335.75, 842.82, 892.13, 1454.32] + Order 4: text: 0.99 [920.18, 185.28, 1476.38, 464.49] + Order 5: text: 0.98 [920.47, 483.68, 1480.63, 765.72] + Order 6: text: 0.98 [920.62, 846.8, 1482.09, 1220.67] + Order 7: text: 0.97 [920.92, 1239.41, 1469.55, 1378.02] + Order 8: footnote: 0.86 [335.03, 1614.68, 1483.33, 1731.73] + Order 9: footnote: 0.83 [334.64, 1756.74, 1471.78, 1845.69] + Order 10: text: 0.81 [336.8, 1910.52, 661.64, 1939.92] + Order 11: footnote: 0.96 [336.24, 2114.42, 1450.14, 2172.12] + Order 12: number: 0.88 [106.0, 2257.5, 135.84, 2282.18] + Order 13: footer: 0.93 [338.4, 2255.52, 986.15, 2284.37] + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.model( + pixel_values, + pixel_mask=pixel_mask, + encoder_outputs=encoder_outputs, + inputs_embeds=inputs_embeds, + decoder_inputs_embeds=decoder_inputs_embeds, + labels=labels, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + intermediate_logits = outputs.intermediate_logits if return_dict else outputs[2] + intermediate_reference_points = outputs.intermediate_reference_points if return_dict else outputs[3] + order_logits = outputs.out_order_logits if return_dict else outputs[4] + out_masks = outputs.out_masks if return_dict else outputs[5] + + pred_boxes = intermediate_reference_points[:, -1] + logits = intermediate_logits[:, -1] + order_logits = order_logits[:, -1] + out_masks = out_masks[:, -1] + + if labels is not None: + raise ValueError("PPDocLayoutV3ForObjectDetection does not support training") + + if not return_dict: + return (logits, pred_boxes, order_logits, out_masks) + outputs[:4] + outputs[6:] + + return PPDocLayoutV3ForObjectDetectionOutput( + logits=logits, + pred_boxes=pred_boxes, + order_logits=order_logits, + out_masks=out_masks, + last_hidden_state=outputs.last_hidden_state, + intermediate_hidden_states=outputs.intermediate_hidden_states, + intermediate_logits=outputs.intermediate_logits, + intermediate_reference_points=outputs.intermediate_reference_points, + intermediate_predicted_corners=outputs.intermediate_predicted_corners, + initial_reference_points=outputs.initial_reference_points, + decoder_hidden_states=outputs.decoder_hidden_states, + decoder_attentions=outputs.decoder_attentions, + cross_attentions=outputs.cross_attentions, + encoder_last_hidden_state=outputs.encoder_last_hidden_state, + encoder_hidden_states=outputs.encoder_hidden_states, + encoder_attentions=outputs.encoder_attentions, + init_reference_points=outputs.init_reference_points, + enc_topk_logits=outputs.enc_topk_logits, + enc_topk_bboxes=outputs.enc_topk_bboxes, + enc_outputs_class=outputs.enc_outputs_class, + enc_outputs_coord_logits=outputs.enc_outputs_coord_logits, + denoising_meta_values=outputs.denoising_meta_values, + ) + + +__all__ = ["PPDocLayoutV3ForObjectDetection", "PPDocLayoutV3Model", "PPDocLayoutV3PreTrainedModel"] diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py new file mode 100644 index 000000000000..9c0f397b0b2d --- /dev/null +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -0,0 +1,1860 @@ +# Copyright 2025 The PaddlePaddle Team and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from dataclasses import dataclass +from typing import Optional, Union + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn +from torchvision.transforms.v2.functional import InterpolationMode + +from ... import initialization as init +from ...configuration_utils import PreTrainedConfig +from ...feature_extraction_utils import BatchFeature +from ...image_processing_utils import BaseImageProcessor, get_size_dict +from ...image_processing_utils_fast import ( + BaseImageProcessorFast, + SizeDict, +) +from ...image_transforms import resize, to_channel_dimension_format +from ...image_utils import ( + ChannelDimension, + ImageInput, + PILImageResampling, + infer_channel_dimension_format, + make_flat_list_of_images, + to_numpy_array, + valid_images, + validate_preprocess_arguments, +) +from ...modeling_outputs import BaseModelOutput +from ...modeling_utils import PreTrainedModel +from ...utils import ( + ModelOutput, + auto_docstring, + filter_out_non_signature_kwargs, + logging, +) +from ...utils.backbone_utils import verify_backbone_config_arguments +from ...utils.generic import TensorType +from ..auto import CONFIG_MAPPING, AutoConfig +from ..rt_detr.modeling_rt_detr import ( + RTDetrDecoder, + RTDetrDecoderOutput, + RTDetrForObjectDetection, + RTDetrHybridEncoder, + RTDetrMLPPredictionHead, + RTDetrModel, + RTDetrModelOutput, + RTDetrMultiscaleDeformableAttention, + get_contrastive_denoising_training_group, + inverse_sigmoid, +) + + +logger = logging.get_logger(__name__) + + +def _default_id2label() -> dict[int, str]: + return { + 0: "abstract", + 1: "algorithm", + 2: "aside_text", + 3: "chart", + 4: "content", + 5: "formula", + 6: "doc_title", + 7: "figure_title", + 8: "footer", + 9: "footer", + 10: "footnote", + 11: "formula_number", + 12: "header", + 13: "header", + 14: "image", + 15: "formula", + 16: "number", + 17: "paragraph_title", + 18: "reference", + 19: "reference_content", + 20: "seal", + 21: "table", + 22: "text", + 23: "text", + 24: "vision_footnote", + } + + +class PPDocLayoutV3Config(PreTrainedConfig): + r""" + This is the configuration class to store the configuration of a [`PP-DocLayoutV3`]. It is used to instantiate a + RT-DETR model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the PP-DocLayoutV3 + [PaddlePaddle/PP-DocLayoutV3_safetensors](https://huggingface.co/PaddlePaddle/PP-DocLayoutV3_safetensors) architecture. + + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. + + Args: + initializer_range (`float`, *optional*, defaults to 0.01): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + initializer_bias_prior_prob (`float`, *optional*): + The prior probability used by the bias initializer to initialize biases for `enc_score_head` and `class_embed`. + If `None`, `prior_prob` computed as `prior_prob = 1 / (num_labels + 1)` while initializing model weights. + layer_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the layer normalization layers. + batch_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the batch normalization layers. + backbone_config (`Union[dict, "PreTrainedConfig"]`, *optional*, defaults to `RTDetrResNetConfig()`): + The configuration of the backbone model. + backbone (`str`, *optional*): + Name of backbone to use when `backbone_config` is `None`. If `use_pretrained_backbone` is `True`, this + will load the corresponding pretrained weights from the timm or transformers library. If `use_pretrained_backbone` + is `False`, this loads the backbone's config and uses that to initialize the backbone with random weights. + use_pretrained_backbone (`bool`, *optional*, defaults to `False`): + Whether to use pretrained weights for the backbone. + use_timm_backbone (`bool`, *optional*, defaults to `False`): + Whether to load `backbone` from the timm library. If `False`, the backbone is loaded from the transformers + library. + freeze_backbone_batch_norms (`bool`, *optional*, defaults to `True`): + Whether to freeze the batch normalization layers in the backbone. + backbone_kwargs (`dict`, *optional*): + Keyword arguments to be passed to AutoBackbone when loading from a checkpoint + e.g. `{'out_indices': (0, 1, 2, 3)}`. Cannot be specified if `backbone_config` is set. + encoder_hidden_dim (`int`, *optional*, defaults to 256): + Dimension of the layers in hybrid encoder. + encoder_in_channels (`list`, *optional*, defaults to `[512, 1024, 2048]`): + Multi level features input for encoder. + feat_strides (`list[int]`, *optional*, defaults to `[8, 16, 32]`): + Strides used in each feature map. + encoder_layers (`int`, *optional*, defaults to 1): + Total of layers to be used by the encoder. + encoder_ffn_dim (`int`, *optional*, defaults to 1024): + Dimension of the "intermediate" (often named feed-forward) layer in decoder. + encoder_attention_heads (`int`, *optional*, defaults to 8): + Number of attention heads for each attention layer in the Transformer encoder. + dropout (`float`, *optional*, defaults to 0.0): + The ratio for all dropout layers. + activation_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for activations inside the fully connected layer. + encode_proj_layers (`list[int]`, *optional*, defaults to `[2]`): + Indexes of the projected layers to be used in the encoder. + positional_encoding_temperature (`int`, *optional*, defaults to 10000): + The temperature parameter used to create the positional encodings. + encoder_activation_function (`str`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + activation_function (`str`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the general layer. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + eval_size (`tuple[int, int]`, *optional*): + Height and width used to computes the effective height and width of the position embeddings after taking + into account the stride. + normalize_before (`bool`, *optional*, defaults to `False`): + Determine whether to apply layer normalization in the transformer encoder layer before self-attention and + feed-forward modules. + hidden_expansion (`float`, *optional*, defaults to 1.0): + Expansion ratio to enlarge the dimension size of RepVGGBlock and CSPRepLayer. + mask_feat_channels (`list[int]`, *optional*, defaults to `[64, 64]): + The channels of the multi-level features for mask enhancement. + x4_feat_dim (`int`, *optional*, defaults to 128): + The dimension of the x4 feature map. + d_model (`int`, *optional*, defaults to 256): + Dimension of the layers exclude hybrid encoder. + num_queries (`int`, *optional*, defaults to 300): + Number of object queries. + decoder_in_channels (`list`, *optional*, defaults to `[256, 256, 256]`): + Multi level features dimension for decoder + decoder_ffn_dim (`int`, *optional*, defaults to 1024): + Dimension of the "intermediate" (often named feed-forward) layer in decoder. + num_feature_levels (`int`, *optional*, defaults to 3): + The number of input feature levels. + decoder_n_points (`int`, *optional*, defaults to 4): + The number of sampled keys in each feature level for each attention head in the decoder. + decoder_layers (`int`, *optional*, defaults to 6): + Number of decoder layers. + decoder_attention_heads (`int`, *optional*, defaults to 8): + Number of attention heads for each attention layer in the Transformer decoder. + decoder_activation_function (`str`, *optional*, defaults to `"relu"`): + The non-linear activation function (function or string) in the decoder. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + num_denoising (`int`, *optional*, defaults to 100): + The total number of denoising tasks or queries to be used for contrastive denoising. + label_noise_ratio (`float`, *optional*, defaults to 0.5): + The fraction of denoising labels to which random noise should be added. + box_noise_scale (`float`, *optional*, defaults to 1.0): + Scale or magnitude of noise to be added to the bounding boxes. + learn_initial_query (`bool`, *optional*, defaults to `False`): + Indicates whether the initial query embeddings for the decoder should be learned during training + mask_enhanced (`bool`, *optional*, defaults to `True`): + Whether to use enhanced masked attention. + anchor_image_size (`tuple[int, int]`, *optional*): + Height and width of the input image used during evaluation to generate the bounding box anchors. If None, automatic generate anchor is applied. + disable_custom_kernels (`bool`, *optional*, defaults to `True`): + Whether to disable custom kernels. + is_encoder_decoder (`bool`, *optional*, defaults to `True`): + Whether the architecture has an encoder decoder structure. + gp_head_size (`int`, *optional*, defaults to 64): + The size of the global pointer head. + tril_mask (`bool`, *optional*, defaults to `True`): + Whether to mask out the upper triangular part of the attention matrix. + id2label (`dict[int, str]`, *optional*): + Mapping from class id to class name. + + Examples: + + ```python + >>> from transformers import PPDocLayoutV3Config, PPDocLayoutV3ForObjectDetection + + >>> # Initializing a PP-DocLayoutV3 configuration + >>> configuration = PPDocLayoutV3Config() + + >>> # Initializing a model (with random weights) from the configuration + >>> model = PPDocLayoutV3ForObjectDetection(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "pp_doclayout_v2" + sub_configs = {"backbone_config": AutoConfig} + + layer_types = ("basic", "bottleneck") + attribute_map = { + "hidden_size": "d_model", + "num_attention_heads": "encoder_attention_heads", + } + + def __init__( + self, + initializer_range=0.01, + initializer_bias_prior_prob=None, + layer_norm_eps=1e-5, + batch_norm_eps=1e-5, + # backbone + backbone_config=None, + backbone=None, + use_pretrained_backbone=False, + use_timm_backbone=False, + freeze_backbone_batch_norms=True, + backbone_kwargs=None, + # encoder PPDocLayoutV3HybridEncoder + encoder_hidden_dim=256, + encoder_in_channels=[512, 1024, 2048], + feat_strides=[8, 16, 32], + encoder_layers=1, + encoder_ffn_dim=1024, + encoder_attention_heads=8, + dropout=0.0, + activation_dropout=0.0, + encode_proj_layers=[2], + positional_encoding_temperature=10000, + encoder_activation_function="gelu", + activation_function="silu", + eval_size=None, + normalize_before=False, + hidden_expansion=1.0, + mask_feat_channels=[64, 64], + x4_feat_dim=128, + # decoder PPDocLayoutV3Transformer + d_model=256, + num_prototypes=32, + label_noise_ratio=0.4, + box_noise_scale=0.4, + mask_enhanced=True, + num_queries=300, + decoder_in_channels=[256, 256, 256], + decoder_ffn_dim=1024, + num_feature_levels=3, + decoder_n_points=4, + decoder_layers=6, + decoder_attention_heads=8, + decoder_activation_function="relu", + attention_dropout=0.0, + num_denoising=100, + learn_initial_query=False, + anchor_image_size=None, + disable_custom_kernels=True, + is_encoder_decoder=True, + gp_head_size=64, + tril_mask=True, + # label + id2label=None, + **kwargs, + ): + self.initializer_range = initializer_range + self.initializer_bias_prior_prob = initializer_bias_prior_prob + self.layer_norm_eps = layer_norm_eps + self.batch_norm_eps = batch_norm_eps + + if backbone_config is None and backbone is None: + logger.info( + "`backbone_config` and `backbone` are `None`. Initializing the config with the default `HGNetV3` backbone." + ) + backbone_config = { + "model_type": "hgnet_v2", + "arch": "L", + "return_idx": [0, 1, 2, 3], + "freeze_stem_only": True, + "freeze_at": 0, + "freeze_norm": True, + "lr_mult_list": [0, 0.05, 0.05, 0.05, 0.05], + "out_features": ["stage1", "stage2", "stage3", "stage4"], + } + config_class = CONFIG_MAPPING["hgnet_v2"] + backbone_config = config_class.from_dict(backbone_config) + elif isinstance(backbone_config, dict): + backbone_model_type = backbone_config.get("model_type") + if backbone_model_type is None: + raise ValueError("`backbone_config` dict must contain key `model_type`.") + config_class = CONFIG_MAPPING[backbone_model_type] + backbone_config = config_class.from_dict(backbone_config) + + verify_backbone_config_arguments( + use_timm_backbone=use_timm_backbone, + use_pretrained_backbone=use_pretrained_backbone, + backbone=backbone, + backbone_config=backbone_config, + backbone_kwargs=backbone_kwargs, + ) + self.backbone_config = backbone_config + self.backbone = backbone + self.use_pretrained_backbone = use_pretrained_backbone + self.use_timm_backbone = use_timm_backbone + self.freeze_backbone_batch_norms = freeze_backbone_batch_norms + self.backbone_kwargs = dict(backbone_kwargs) if backbone_kwargs is not None else None + + # ---- encoder ---- + self.encoder_hidden_dim = encoder_hidden_dim + self.encoder_in_channels = list(encoder_in_channels) + self.feat_strides = list(feat_strides) + self.encoder_layers = encoder_layers + self.encoder_ffn_dim = encoder_ffn_dim + self.encoder_attention_heads = encoder_attention_heads + self.dropout = dropout + self.activation_dropout = activation_dropout + self.encode_proj_layers = list(encode_proj_layers) + self.positional_encoding_temperature = positional_encoding_temperature + self.encoder_activation_function = encoder_activation_function + self.activation_function = activation_function + self.eval_size = list(eval_size) if eval_size is not None else None + self.normalize_before = normalize_before + self.hidden_expansion = hidden_expansion + self.mask_feat_channels = mask_feat_channels + self.x4_feat_dim = x4_feat_dim + + # ---- decoder ---- + self.d_model = d_model + self.num_queries = num_queries + self.num_prototypes = num_prototypes + self.decoder_in_channels = list(decoder_in_channels) + self.decoder_ffn_dim = decoder_ffn_dim + self.num_feature_levels = num_feature_levels + self.decoder_n_points = decoder_n_points + self.decoder_layers = decoder_layers + self.decoder_attention_heads = decoder_attention_heads + self.decoder_activation_function = decoder_activation_function + self.attention_dropout = attention_dropout + self.num_denoising = num_denoising + self.label_noise_ratio = label_noise_ratio + self.mask_enhanced = mask_enhanced + self.box_noise_scale = box_noise_scale + self.learn_initial_query = learn_initial_query + self.anchor_image_size = list(anchor_image_size) if anchor_image_size is not None else None + self.disable_custom_kernels = disable_custom_kernels + self.gp_head_size = gp_head_size + self.tril_mask = tril_mask + + super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs) + + if kwargs.get("num_labels") and not id2label: + self.id2label = None + self.num_labels = kwargs["num_labels"] + else: + self.id2label = {int(k): v for k, v in id2label.items()} if id2label else _default_id2label() + self.num_labels = len(self.id2label) + + +def postprocess(outputs, threshold, target_sizes): + boxes = outputs.pred_boxes + logits = outputs.logits + order_logits = outputs.order_logits + + order_seqs = get_order(order_logits) + + cxcy, wh = torch.split(boxes, 2, dim=-1) + boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) + + if target_sizes is not None: + if len(logits) != len(target_sizes): + raise ValueError("Make sure that you pass in as many target sizes as the batch dimension of the logits") + if isinstance(target_sizes, list): + img_h, img_w = torch.as_tensor(target_sizes).unbind(1) + else: + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) + boxes = boxes * scale_fct[:, None, :] + + num_top_queries = logits.shape[1] + num_classes = logits.shape[2] + + scores = torch.nn.functional.sigmoid(logits) + scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) + labels = index % num_classes + index = index // num_classes + boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) + order_seqs = order_seqs.gather(dim=1, index=index) + + results = [] + for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): + order_seq = order_seq[score >= threshold] + order_seq, indices = torch.sort(order_seq) + results.append( + { + "scores": score[score >= threshold][indices], + "labels": label[score >= threshold][indices], + "boxes": box[score >= threshold][indices], + "order_seq": order_seq, + } + ) + + return results + + +def get_order(order_logits): + # order_logits: (B, N, N) upper-triangular meaningful + order_scores = torch.sigmoid(order_logits) + B, N, _ = order_scores.shape + + one = torch.ones((N, N), dtype=order_scores.dtype, device=order_scores.device) + upper = torch.triu(one, 1) + lower = torch.tril(one, -1) + + Q = order_scores * upper + (1.0 - order_scores.transpose(1, 2)) * lower + order_votes = Q.sum(dim=1) # (B, N) + + order_pointers = torch.argsort(order_votes, dim=1) # (B, N) + order_seq = torch.full_like(order_pointers, -1) + batch = torch.arange(B, device=order_pointers.device)[:, None] + order_seq[batch, order_pointers] = torch.arange(N, device=order_pointers.device)[None, :] + + return order_seq + + +class PPDocLayoutV3ImageProcessor(BaseImageProcessor): + r""" + Constructs a PPDocLayoutV3 image processor. + + Args: + do_resize (`bool`, *optional*, defaults to `True`): + Controls whether to resize the image's (height, width) dimensions to the specified `size`. Can be + overridden by the `do_resize` parameter in the `preprocess` method. + size (`dict[str, int]` *optional*, defaults to `{"height": 640, "width": 640}`): + Size of the image's `(height, width)` dimensions after resizing. Can be overridden by the `size` parameter + in the `preprocess` method. Available options are: + - `{"height": int, "width": int}`: The image will be resized to the exact size `(height, width)`. + Do NOT keep the aspect ratio. + - `{"shortest_edge": int, "longest_edge": int}`: The image will be resized to a maximum size respecting + the aspect ratio and keeping the shortest edge less or equal to `shortest_edge` and the longest edge + less or equal to `longest_edge`. + - `{"max_height": int, "max_width": int}`: The image will be resized to the maximum size respecting the + aspect ratio and keeping the height less or equal to `max_height` and the width less or equal to + `max_width`. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): + Resampling filter to use if resizing the image. + do_rescale (`bool`, *optional*, defaults to `True`): + Controls whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the + `do_rescale` parameter in the `preprocess` method. + rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): + Scale factor to use if rescaling the image. Can be overridden by the `rescale_factor` parameter in the + `preprocess` method. + Controls whether to normalize the image. Can be overridden by the `do_normalize` parameter in the + `preprocess` method. + do_normalize (`bool`, *optional*, defaults to `True`): + Whether to normalize the image. + image_mean (`float` or `list[float]`, *optional*, defaults to `[0, 0, 0]`): + Mean values to use when normalizing the image. Can be a single value or a list of values, one for each + channel. Can be overridden by the `image_mean` parameter in the `preprocess` method. + image_std (`float` or `list[float]`, *optional*, defaults to `[1, 1, 1]`): + Standard deviation values to use when normalizing the image. Can be a single value or a list of values, one + for each channel. Can be overridden by the `image_std` parameter in the `preprocess` method. + """ + + model_input_names = ["pixel_values"] + + def __init__( + self, + do_resize: bool = True, + size: Optional[dict[str, int]] = None, + resample: Optional[PILImageResampling] = PILImageResampling.BICUBIC, + do_rescale: bool = True, + rescale_factor: Union[int, float] = 1 / 255, + do_normalize: bool = True, + image_mean: Optional[Union[float, list[float]]] = [0, 0, 0], + image_std: Optional[Union[float, list[float]]] = [1, 1, 1], + **kwargs, + ) -> None: + size = size if size is not None else {"height": 800, "width": 800} + size = get_size_dict(size, default_to_square=False) + + super().__init__(**kwargs) + self.do_resize = do_resize + self.size = size + self.do_rescale = do_rescale + self.rescale_factor = rescale_factor + self.do_normalize = do_normalize + self.image_mean = image_mean + self.image_std = image_std + self.resample = resample + + @filter_out_non_signature_kwargs() + def preprocess( + self, + images: ImageInput, + do_resize: Optional[bool] = None, + size: Optional[dict[str, int]] = None, + resample: Optional[PILImageResampling] = None, + do_rescale: Optional[bool] = None, + rescale_factor: Optional[Union[int, float]] = None, + do_normalize: Optional[bool] = None, + image_mean: Optional[Union[float, list[float]]] = None, + image_std: Optional[Union[float, list[float]]] = None, + return_tensors: Optional[Union[TensorType, str]] = None, + data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + ) -> BatchFeature: + """ + Preprocess an image or a batch of images so that it can be used by the model. + + Args: + images (`ImageInput`): + Image or batch of images to preprocess. Expects a single or batch of images with pixel values ranging + from 0 to 255. If passing in images with pixel values between 0 and 1, set `do_rescale=False`. + do_resize (`bool`, *optional*, defaults to self.do_resize): + Whether to resize the image. + size (`dict[str, int]`, *optional*, defaults to self.size): + Size of the image's `(height, width)` dimensions after resizing. Available options are: + - `{"height": int, "width": int}`: The image will be resized to the exact size `(height, width)`. + Do NOT keep the aspect ratio. + - `{"shortest_edge": int, "longest_edge": int}`: The image will be resized to a maximum size respecting + the aspect ratio and keeping the shortest edge less or equal to `shortest_edge` and the longest edge + less or equal to `longest_edge`. + - `{"max_height": int, "max_width": int}`: The image will be resized to the maximum size respecting the + aspect ratio and keeping the height less or equal to `max_height` and the width less or equal to + `max_width`. + resample (`PILImageResampling`, *optional*, defaults to self.resample): + Resampling filter to use when resizing the image. + do_rescale (`bool`, *optional*, defaults to self.do_rescale): + Whether to rescale the image. + rescale_factor (`float`, *optional*, defaults to self.rescale_factor): + Rescale factor to use when rescaling the image. + do_normalize (`bool`, *optional*, defaults to self.do_normalize): + Whether to normalize the image. + image_mean (`float` or `list[float]`, *optional*, defaults to self.image_mean): + Mean to use when normalizing the image. + image_std (`float` or `list[float]`, *optional*, defaults to self.image_std): + Standard deviation to use when normalizing the image. + return_tensors (`str` or `TensorType`, *optional*, defaults to self.return_tensors): + Type of tensors to return. If `None`, will return the list of images. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + """ + do_resize = self.do_resize if do_resize is None else do_resize + size = self.size if size is None else size + size = get_size_dict(size=size, default_to_square=True) + resample = self.resample if resample is None else resample + do_rescale = self.do_rescale if do_rescale is None else do_rescale + rescale_factor = self.rescale_factor if rescale_factor is None else rescale_factor + do_normalize = self.do_normalize if do_normalize is None else do_normalize + image_mean = self.image_mean if image_mean is None else image_mean + image_std = self.image_std if image_std is None else image_std + + validate_preprocess_arguments( + do_rescale=do_rescale, + rescale_factor=rescale_factor, + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + do_resize=do_resize, + size=size, + resample=resample, + ) + + images = make_flat_list_of_images(images) + if not valid_images(images): + raise ValueError("Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, or torch.Tensor") + + # All transformations expect numpy arrays + images = [to_numpy_array(image) for image in images] + + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) + + # transformations + if do_resize: + images = [ + resize( + image, size=(size["height"], size["width"]), resample=resample, input_data_format=input_data_format + ) + for image in images + ] + + if do_rescale: + images = [self.rescale(image, rescale_factor, input_data_format=input_data_format) for image in images] + + if do_normalize: + images = [ + self.normalize(image, image_mean, image_std, input_data_format=input_data_format) for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] + encoded_inputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) + + return encoded_inputs + + def post_process_object_detection( + self, + outputs, + threshold: float = 0.5, + target_sizes: Optional[Union[TensorType, list[tuple]]] = None, + ): + """ + Converts the raw output of [`PPDocLayoutV3ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, + bottom_right_x, bottom_right_y) format. Only supports PyTorch. + + Args: + outputs ([`PPDocLayoutV3ObjectDetectionOutput`]): + Raw outputs of the model. + threshold (`float`, *optional*, defaults to 0.5): + Score threshold to keep object detection predictions. + target_sizes (`torch.Tensor` or `list[tuple[int, int]]`, *optional*): + Tensor of shape `(batch_size, 2)` or list of tuples (`tuple[int, int]`) containing the target size + `(height, width)` of each image in the batch. If unset, predictions will not be resized. + + Returns: + `list[Dict]`: An ordered list of dictionaries, each dictionary containing the scores, labels and boxes for an image + in the batch as predicted by the model. + """ + return postprocess(outputs=outputs, threshold=threshold, target_sizes=target_sizes) + + +class PPDocLayoutV3ImageProcessorFast(BaseImageProcessorFast): + resample = PILImageResampling.BICUBIC + image_mean = [0, 0, 0] + image_std = [1, 1, 1] + size = {"height": 800, "width": 800} + do_resize = True + do_rescale = True + do_normalize = True + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + + def _preprocess( + self, + images: list[torch.Tensor], + do_resize: bool, + size: SizeDict, + interpolation: Optional[InterpolationMode], + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: Optional[Union[float, list[float]]], + image_std: Optional[Union[float, list[float]]], + return_tensors: Optional[Union[str, TensorType]], + **kwargs, + ) -> BatchFeature: + """ + Preprocess an image or a batch of images so that it can be used by the model. + """ + data = {} + processed_images = [] + for image in images: + if do_resize: + image = self.resize(image, size=size, interpolation=interpolation) + # Fused rescale and normalize + image = self.rescale_and_normalize(image, do_rescale, rescale_factor, do_normalize, image_mean, image_std) + + processed_images.append(image) + + images = processed_images + + data.update({"pixel_values": torch.stack(images, dim=0)}) + encoded_inputs = BatchFeature(data, tensor_type=return_tensors) + + return encoded_inputs + + def post_process_object_detection( + self, + outputs, + threshold: float = 0.5, + target_sizes: Optional[Union[TensorType, list[tuple]]] = None, + ): + """ + Converts the raw output of [`DetrForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, + bottom_right_x, bottom_right_y) format. Only supports PyTorch. + + Args: + outputs ([`DetrObjectDetectionOutput`]): + Raw outputs of the model. + Returns: + `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image + in the batch as predicted by the model. + """ + return postprocess(outputs=outputs, threshold=threshold, target_sizes=target_sizes) + + +class GlobalPointer(nn.Module): + def __init__(self, config): + super().__init__() + self.head_size = config.gp_head_size + self.tril_mask = config.tril_mask + self.dense = nn.Linear(config.d_model, self.head_size * 2) + self.dropout = nn.Dropout(0.1) + + def forward(self, inputs): + B, N, _ = inputs.shape + proj = self.dense(inputs).reshape([B, N, 2, self.head_size]) + proj = self.dropout(proj) + qw, kw = proj[..., 0, :], proj[..., 1, :] + + logits = torch.einsum("bmd,bnd->bmn", qw, kw) / (self.head_size**0.5) # [B, N, N] + + if self.tril_mask: + lower = torch.tril(torch.ones([N, N], dtype=torch.float32, device=logits.device)) + logits = logits - lower.unsqueeze(0).to(logits.dtype) * 1e4 + + return logits + + +class PPDocLayoutV3MultiscaleDeformableAttention(RTDetrMultiscaleDeformableAttention): + pass + + +@auto_docstring +class PPDocLayoutV3PreTrainedModel(PreTrainedModel): + config: PPDocLayoutV3Config + base_model_prefix = "pp_doclayout_v3" + main_input_name = "pixel_values" + input_modalities = ("image",) + _no_split_modules = [r"PPDocLayoutV3HybridEncoder", r"PPDocLayoutV3DecoderLayer"] + + @torch.no_grad() + def _init_weights(self, module): + if isinstance(module, PPDocLayoutV3MultiscaleDeformableAttention): + init.constant_(module.sampling_offsets.weight, 0.0) + default_dtype = torch.get_default_dtype() + thetas = torch.arange(module.n_heads, dtype=torch.int64).to(default_dtype) * ( + 2.0 * math.pi / module.n_heads + ) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = ( + (grid_init / grid_init.abs().max(-1, keepdim=True)[0]) + .view(module.n_heads, 1, 1, 2) + .repeat(1, module.n_levels, module.n_points, 1) + ) + for i in range(module.n_points): + grid_init[:, :, i, :] *= i + 1 + + init.copy_(module.sampling_offsets.bias, grid_init.view(-1)) + init.constant_(module.attention_weights.weight, 0.0) + init.constant_(module.attention_weights.bias, 0.0) + init.xavier_uniform_(module.value_proj.weight) + init.constant_(module.value_proj.bias, 0.0) + init.xavier_uniform_(module.output_proj.weight) + init.constant_(module.output_proj.bias, 0.0) + + elif isinstance(module, PPDocLayoutV3Model): + prior_prob = self.config.initializer_bias_prior_prob or 1 / (self.config.num_labels + 1) + bias = float(-math.log((1 - prior_prob) / prior_prob)) + init.xavier_uniform_(module.enc_score_head.weight) + init.constant_(module.enc_score_head.bias, bias) + + elif isinstance(module, (nn.Linear, nn.Conv2d, nn.BatchNorm2d)): + init.normal_(module.weight, mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + init.zeros_(module.bias) + if getattr(module, "running_mean", None) is not None: + init.zeros_(module.running_mean) + init.ones_(module.running_var) + init.zeros_(module.num_batches_tracked) + + elif isinstance(module, nn.LayerNorm): + init.ones_(module.weight) + init.zeros_(module.bias) + + if isinstance(module, nn.Embedding): + init.normal_(module.weight, mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + init.zeros_(module.weight.data[module.padding_idx]) + + +def bbox_xyxy_to_cxcywh(x): + x1 = x[..., 0] + y1 = x[..., 1] + x2 = x[..., 2] + y2 = x[..., 3] + return torch.stack([(x1 + x2) / 2, (y1 + y2) / 2, (x2 - x1), (y2 - y1)], dim=-1) + + +def mask_to_box_coordinate(mask, normalize=False, format="xyxy", dtype=torch.float32): + assert mask.ndim == 4, f"Error: The 'mask' variable must be a 4-dimensional array, but got {mask.ndim} dimensions." + assert format in ["xyxy", "xywh"], ( + f"Error: The 'format' variable must be either 'xyxy' or 'xywh', but got '{format}'." + ) + + mask = mask.bool() + + h, w = mask.shape[-2:] + y, x = torch.meshgrid(torch.arange(h, device=mask.device), torch.arange(w, device=mask.device), indexing="ij") + x = x.to(dtype) + y = y.to(dtype) + + x_mask = x * mask + x_max = x_mask.flatten(start_dim=-2).max(dim=-1).values + 1 + x_min = ( + torch.where(mask, x_mask, torch.tensor(1e8, device=mask.device, dtype=dtype)) + .flatten(start_dim=-2) + .min(dim=-1) + .values + ) + + y_mask = y * mask + y_max = y_mask.flatten(start_dim=-2).max(dim=-1).values + 1 + y_min = ( + torch.where(mask, y_mask, torch.tensor(1e8, device=mask.device, dtype=dtype)) + .flatten(start_dim=-2) + .min(dim=-1) + .values + ) + + out_bbox = torch.stack([x_min, y_min, x_max, y_max], dim=-1) + mask_any = torch.any(mask, dim=(-2, -1)).unsqueeze(-1) + out_bbox = out_bbox * mask_any + + if normalize: + norm_tensor = torch.tensor([w, h, w, h], device=mask.device, dtype=dtype) + out_bbox /= norm_tensor + + return out_bbox if format == "xyxy" else bbox_xyxy_to_cxcywh(out_bbox) + + +@dataclass +class PPDocLayoutV3DecoderOutput(RTDetrDecoderOutput): + r""" + intermediate_hidden_states (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, hidden_size)`): + Stacked intermediate hidden states (output of each layer of the decoder). + intermediate_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, sequence_length, config.num_labels)`): + Stacked intermediate logits (logits of each layer of the decoder). + intermediate_reference_points (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, sequence_length, hidden_size)`): + Stacked intermediate reference points (reference points of each layer of the decoder). + intermediate_predicted_corners (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked intermediate predicted corners (predicted corners of each layer of the decoder). + initial_reference_points (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked initial reference points (initial reference points of each layer of the decoder). + cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` and `config.add_cross_attention=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax, + used to compute the weighted average in the cross-attention heads. + dec_out_order_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, config.num_queries)`): + Stacked order logits (order logits of each layer of the decoder). + dec_out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, 200, 200)`): + Stacked masks (masks of each layer of the decoder). + """ + + dec_out_order_logits: Optional[torch.FloatTensor] = None + dec_out_masks: Optional[torch.FloatTensor] = None + + +@dataclass +@auto_docstring( + custom_intro=""" + Base class for outputs of the PP-DocLayoutV3 model. + """ +) +class PPDocLayoutV3ModelOutput(RTDetrModelOutput): + r""" + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the decoder of the model. + intermediate_hidden_states (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, hidden_size)`): + Stacked intermediate hidden states (output of each layer of the decoder). + intermediate_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, sequence_length, config.num_labels)`): + Stacked intermediate logits (logits of each layer of the decoder). + intermediate_reference_points (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked intermediate reference points (reference points of each layer of the decoder). + intermediate_predicted_corners (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked intermediate predicted corners (predicted corners of each layer of the decoder). + initial_reference_points (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): + Initial reference points used for the first decoder layer. + init_reference_points (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): + Initial reference points sent through the Transformer decoder. + enc_topk_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`): + Predicted bounding boxes scores where the top `config.two_stage_num_proposals` scoring bounding boxes are + picked as region proposals in the encoder stage. Output of bounding box binary classification (i.e. + foreground and background). + enc_topk_bboxes (`torch.FloatTensor` of shape `(batch_size, sequence_length, 4)`): + Logits of predicted bounding boxes coordinates in the encoder stage. + enc_outputs_class (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Predicted bounding boxes scores where the top `config.two_stage_num_proposals` scoring bounding boxes are + picked as region proposals in the first stage. Output of bounding box binary classification (i.e. + foreground and background). + enc_outputs_coord_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, 4)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Logits of predicted bounding boxes coordinates in the first stage. + denoising_meta_values (`dict`): + Extra dictionary for the denoising related values. + out_order_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, config.num_queries)`): + Stacked order logits (order logits of each layer of the decoder). + out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, 200, 200)`): + Stacked masks (masks of each layer of the decoder). + """ + + out_order_logits: Optional[torch.FloatTensor] = None + out_masks: Optional[torch.FloatTensor] = None + + +class PPDocLayoutV3MLPPredictionHead(RTDetrMLPPredictionHead): + pass + + +class BaseConv(nn.Module): + def __init__( + self, + in_channels, + out_channels, + ksize, + stride, + groups=1, + bias=False, + ): + super().__init__() + self.conv = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=ksize, + stride=stride, + padding=(ksize - 1) // 2, + groups=groups, + bias=bias, + ) + self.bn = nn.BatchNorm2d(out_channels) + + def forward(self, x): + # use 'x * F.sigmoid(x)' replace 'silu' + x = self.bn(self.conv(x)) + y = x * F.sigmoid(x) + return y + + +class MaskFeatFPN(nn.Module): + def __init__( + self, + in_channels=[256, 256, 256], + fpn_strides=[32, 16, 8], + feat_channels=256, + dropout_ratio=0.0, + out_channels=256, + align_corners=False, + ): + super().__init__() + + assert len(in_channels) == len(fpn_strides), ( + f"Error: The lengths of 'in_channels' and 'fpn_strides' must be equal. " + f"Got len(in_channels)={len(in_channels)} and len(fpn_strides)={len(fpn_strides)}." + ) + + reorder_index = np.argsort(fpn_strides, axis=0) + in_channels = [in_channels[i] for i in reorder_index] + fpn_strides = [fpn_strides[i] for i in reorder_index] + + self.reorder_index = reorder_index + self.fpn_strides = fpn_strides + self.dropout_ratio = dropout_ratio + self.align_corners = align_corners + if self.dropout_ratio > 0: + self.dropout = nn.Dropout2D(dropout_ratio) + + self.scale_heads = nn.ModuleList() + for i in range(len(fpn_strides)): + head_length = max(1, int(np.log2(fpn_strides[i]) - np.log2(fpn_strides[0]))) + scale_head = [] + for k in range(head_length): + in_c = in_channels[i] if k == 0 else feat_channels + scale_head.append(nn.Sequential(BaseConv(in_c, feat_channels, 3, 1))) + if fpn_strides[i] != fpn_strides[0]: + scale_head.append(nn.Upsample(scale_factor=2, mode="bilinear", align_corners=align_corners)) + + self.scale_heads.append(nn.Sequential(*scale_head)) + + self.output_conv = BaseConv(feat_channels, out_channels, 3, 1) + + def forward(self, inputs): + x = [inputs[i] for i in self.reorder_index] + + output = self.scale_heads[0](x[0]) + for i in range(1, len(self.fpn_strides)): + output = output + F.interpolate( + self.scale_heads[i](x[i]), size=output.shape[2:], mode="bilinear", align_corners=self.align_corners + ) + + if self.dropout_ratio > 0: + output = self.dropout(output) + output = self.output_conv(output) + return output + + +class PPDocLayoutV3HybridEncoder(RTDetrHybridEncoder): + def __init__(self, config: PPDocLayoutV3Config): + super().__init__() + + feat_strides = config.feat_strides + mask_feat_channels = config.mask_feat_channels + self.mask_feat_head = MaskFeatFPN( + [self.encoder_hidden_dim] * len(feat_strides), + feat_strides, + feat_channels=mask_feat_channels[0], + out_channels=mask_feat_channels[1], + ) + self.enc_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1) + self.enc_mask_output = nn.Sequential( + BaseConv(mask_feat_channels[1], mask_feat_channels[1], 3, 1), + nn.Conv2d(in_channels=mask_feat_channels[1], out_channels=config.num_prototypes, kernel_size=1), + ) + + def forward( + self, + inputs_embeds=None, + x4_feat=None, + attention_mask=None, + position_embeddings=None, + spatial_shapes=None, + level_start_index=None, + valid_ratios=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + Args: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Flattened feature map (output of the backbone + projection layer) that is passed to the encoder. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding pixel features. Mask values selected in `[0, 1]`: + - 1 for pixel features that are real (i.e. **not masked**), + - 0 for pixel features that are padding (i.e. **masked**). + [What are attention masks?](../glossary#attention-mask) + position_embeddings (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Position embeddings that are added to the queries and keys in each self-attention layer. + spatial_shapes (`torch.LongTensor` of shape `(num_feature_levels, 2)`): + Spatial shapes of each feature map. + level_start_index (`torch.LongTensor` of shape `(num_feature_levels)`): + Starting index of each feature map. + valid_ratios (`torch.FloatTensor` of shape `(batch_size, num_feature_levels, 2)`): + Ratio of valid area in each feature level. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + hidden_states = inputs_embeds + + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + # encoder + if self.config.encoder_layers > 0: + for i, enc_ind in enumerate(self.encode_proj_layers): + if output_hidden_states: + encoder_states = encoder_states + (hidden_states[enc_ind],) + height, width = hidden_states[enc_ind].shape[2:] + # flatten [batch, channel, height, width] to [batch, height*width, channel] + src_flatten = hidden_states[enc_ind].flatten(2).permute(0, 2, 1) + if self.training or self.eval_size is None: + pos_embed = self.build_2d_sincos_position_embedding( + width, + height, + self.encoder_hidden_dim, + self.positional_encoding_temperature, + device=src_flatten.device, + dtype=src_flatten.dtype, + ) + else: + pos_embed = None + + layer_outputs = self.encoder[i]( + src_flatten, + pos_embed=pos_embed, + output_attentions=output_attentions, + ) + hidden_states[enc_ind] = ( + layer_outputs[0].permute(0, 2, 1).reshape(-1, self.encoder_hidden_dim, height, width).contiguous() + ) + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states[enc_ind],) + + # top-down FPN + fpn_feature_maps = [hidden_states[-1]] + for idx, (lateral_conv, fpn_block) in enumerate(zip(self.lateral_convs, self.fpn_blocks)): + backbone_feature_map = hidden_states[self.num_fpn_stages - idx - 1] + top_fpn_feature_map = fpn_feature_maps[-1] + # apply lateral block + top_fpn_feature_map = lateral_conv(top_fpn_feature_map) + fpn_feature_maps[-1] = top_fpn_feature_map + # apply fpn block + top_fpn_feature_map = F.interpolate(top_fpn_feature_map, scale_factor=2.0, mode="nearest") + fused_feature_map = torch.concat([top_fpn_feature_map, backbone_feature_map], dim=1) + new_fpn_feature_map = fpn_block(fused_feature_map) + fpn_feature_maps.append(new_fpn_feature_map) + + fpn_feature_maps.reverse() + + # bottom-up PAN + pan_feature_maps = [fpn_feature_maps[0]] + for idx, (downsample_conv, pan_block) in enumerate(zip(self.downsample_convs, self.pan_blocks)): + top_pan_feature_map = pan_feature_maps[-1] + fpn_feature_map = fpn_feature_maps[idx + 1] + downsampled_feature_map = downsample_conv(top_pan_feature_map) + fused_feature_map = torch.concat([downsampled_feature_map, fpn_feature_map], dim=1) + new_pan_feature_map = pan_block(fused_feature_map) + pan_feature_maps.append(new_pan_feature_map) + + mask_feat = self.mask_feat_head(pan_feature_maps) + mask_feat = F.interpolate(mask_feat, scale_factor=2, mode="bilinear", align_corners=False) + mask_feat += self.enc_mask_lateral(x4_feat[0]) + mask_feat = self.enc_mask_output(mask_feat) + + if not return_dict: + return tuple(v for v in [pan_feature_maps, encoder_states, all_attentions, mask_feat] if v is not None) + + return PPDocLayoutV3HybridEncoderOutput( + last_hidden_state=pan_feature_maps, + hidden_states=encoder_states, + attentions=all_attentions, + mask_feat=mask_feat, + ) + + +class PPDocLayoutV3Decoder(RTDetrDecoder): + def __init__(self, config: PPDocLayoutV3Config): + super().__init__() + + self.num_queries = config.num_queries + + def forward( + self, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + position_embeddings=None, + reference_points=None, + spatial_shapes=None, + spatial_shapes_list=None, + level_start_index=None, + valid_ratios=None, + order_head=None, + global_pointer=None, + mask_query_head=None, + dec_norm=None, + mask_feat=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs, + ): + r""" + Args: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): + The query embeddings that are passed into the decoder. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + of the decoder. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing cross-attention on padding pixel_values of the encoder. Mask values selected + in `[0, 1]`: + - 1 for pixels that are real (i.e. **not masked**), + - 0 for pixels that are padding (i.e. **masked**). + position_embeddings (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Position embeddings that are added to the queries and keys in each self-attention layer. + reference_points (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)` is `as_two_stage` else `(batch_size, num_queries, 2)` or , *optional*): + Reference point in range `[0, 1]`, top-left (0,0), bottom-right (1, 1), including padding area. + spatial_shapes (`torch.FloatTensor` of shape `(num_feature_levels, 2)`): + Spatial shapes of the feature maps. + level_start_index (`torch.LongTensor` of shape `(num_feature_levels)`, *optional*): + Indexes for the start of each feature level. In range `[0, sequence_length]`. + valid_ratios (`torch.FloatTensor` of shape `(batch_size, num_feature_levels, 2)`, *optional*): + Ratio of valid area in each feature level. + + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if inputs_embeds is not None: + hidden_states = inputs_embeds + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None + intermediate = () + intermediate_reference_points = () + intermediate_logits = () + dec_out_order_logits = () + dec_out_masks = () + + reference_points = F.sigmoid(reference_points) + + # https://github.com/lyuwenyu/RT-DETR/blob/94f5e16708329d2f2716426868ec89aa774af016/rtdetr_pytorch/src/zoo/rtdetr/rtdetr_decoder.py#L252 + for idx, decoder_layer in enumerate(self.layers): + reference_points_input = reference_points.unsqueeze(2) + position_embeddings = self.query_pos_head(reference_points) + + if output_hidden_states: + all_hidden_states += (hidden_states,) + + layer_outputs = decoder_layer( + hidden_states, + position_embeddings=position_embeddings, + encoder_hidden_states=encoder_hidden_states, + reference_points=reference_points_input, + spatial_shapes=spatial_shapes, + spatial_shapes_list=spatial_shapes_list, + level_start_index=level_start_index, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + ) + + hidden_states = layer_outputs[0] + + # hack implementation for iterative bounding box refinement + if self.bbox_embed is not None: + predicted_corners = self.bbox_embed(hidden_states) + new_reference_points = F.sigmoid(predicted_corners + inverse_sigmoid(reference_points)) + reference_points = new_reference_points.detach() + + intermediate += (hidden_states,) + intermediate_reference_points += ( + (new_reference_points,) if self.bbox_embed is not None else (reference_points,) + ) + + # get_pred_class_order_and_mask + out_query = dec_norm(hidden_states) + mask_query_embed = mask_query_head(out_query) + batch_size, mask_dim, _ = mask_query_embed.shape + _, _, mask_h, mask_w = mask_feat.shape + out_mask = torch.bmm(mask_query_embed, mask_feat.flatten(start_dim=2)).reshape( + batch_size, mask_dim, mask_h, mask_w + ) + dec_out_masks += (out_mask,) + + if self.class_embed is not None: + logits = self.class_embed(out_query) + intermediate_logits += (logits,) + + if order_head is not None and global_pointer is not None: + valid_query = out_query[:, -self.num_queries :] if self.num_queries is not None else out_query + order_logits = global_pointer(order_head(valid_query)) + dec_out_order_logits += (order_logits,) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + if encoder_hidden_states is not None: + all_cross_attentions += (layer_outputs[2],) + + # Keep batch_size as first dimension + intermediate = torch.stack(intermediate, dim=1) + intermediate_reference_points = torch.stack(intermediate_reference_points, dim=1) + if self.class_embed is not None: + intermediate_logits = torch.stack(intermediate_logits, dim=1) + if order_head is not None and global_pointer is not None: + dec_out_order_logits = torch.stack(dec_out_order_logits, dim=1) + dec_out_masks = torch.stack(dec_out_masks, dim=1) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if not return_dict: + return tuple( + v + for v in [ + hidden_states, + intermediate, + intermediate_logits, + intermediate_reference_points, + dec_out_order_logits, + dec_out_masks, + all_hidden_states, + all_self_attns, + all_cross_attentions, + ] + if v is not None + ) + return PPDocLayoutV3DecoderOutput( + last_hidden_state=hidden_states, + intermediate_hidden_states=intermediate, + intermediate_logits=intermediate_logits, + intermediate_reference_points=intermediate_reference_points, + dec_out_order_logits=dec_out_order_logits, + dec_out_masks=dec_out_masks, + hidden_states=all_hidden_states, + attentions=all_self_attns, + cross_attentions=all_cross_attentions, + ) + + +class PPDocLayoutV3Model(RTDetrModel): + def __init__(self, config: PPDocLayoutV3Config): + super().__init__(config) + + self.encoder_input_proj = nn.ModuleList(encoder_input_proj_list[1:]) # noqa + + self.dec_order_head = nn.Linear(config.d_model, config.d_model) + self.dec_global_pointer = GlobalPointer(config) + self.dec_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) + self.decoder = PPDocLayoutV3Decoder(config) + self.decoder.class_embed = self.enc_score_head + self.decoder.bbox_embed = self.enc_bbox_head + + self.mask_enhanced = config.mask_enhanced + self.mask_query_head = PPDocLayoutV3MLPPredictionHead( + config, config.d_model, config.d_model, config.num_prototypes, num_layers=3 + ) + + def forward( + self, + pixel_values: torch.FloatTensor, + pixel_mask: Optional[torch.LongTensor] = None, + encoder_outputs: Optional[torch.FloatTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[list[dict]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **kwargs, + ) -> Union[tuple[torch.FloatTensor], PPDocLayoutV3ModelOutput]: + r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. + labels (`list[Dict]` of len `(batch_size,)`, *optional*): + Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the + following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch + respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes + in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. + + Examples: + + ```python + >>> from transformers import AutoImageProcessor, PPDocLayoutV2Model + >>> from PIL import Image + >>> import requests + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> image_processor = AutoImageProcessor.from_pretrained("PekingU/PPDocLayoutV2_r50vd") + >>> model = PPDocLayoutV2Model.from_pretrained("PekingU/PPDocLayoutV2_r50vd") + + >>> inputs = image_processor(images=image, return_tensors="pt") + + >>> outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + [1, 300, 256] + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + batch_size, num_channels, height, width = pixel_values.shape + device = pixel_values.device + + if pixel_mask is None: + pixel_mask = torch.ones(((batch_size, height, width)), device=device) + + features = self.backbone(pixel_values, pixel_mask) + x4_feat = features.pop(0) + proj_feats = [self.encoder_input_proj[level](source) for level, (source, mask) in enumerate(features)] + + if encoder_outputs is None: + encoder_outputs = self.encoder( + proj_feats, + x4_feat, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + # If the user passed a tuple for encoder_outputs, we wrap it in a PPDocLayoutV3HybridEncoderOutput when return_dict=True + elif return_dict and not isinstance(encoder_outputs, PPDocLayoutV3HybridEncoderOutput): + encoder_outputs = PPDocLayoutV3HybridEncoderOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] if output_hidden_states else None, + attentions=encoder_outputs[2] + if len(encoder_outputs) > 2 + else encoder_outputs[1] + if output_attentions + else None, + mask_feat=encoder_outputs[-1], + ) + + mask_feat = ( + encoder_outputs.mask_feat + if isinstance(encoder_outputs, PPDocLayoutV3HybridEncoderOutput) + else encoder_outputs[-1] + ) + + # Equivalent to def _get_encoder_input + # https://github.com/lyuwenyu/RT-DETR/blob/94f5e16708329d2f2716426868ec89aa774af016/rtdetr_pytorch/src/zoo/rtdetr/rtdetr_decoder.py#L412 + sources = [] + for level, source in enumerate(encoder_outputs[0]): + sources.append(self.decoder_input_proj[level](source)) + + # Lowest resolution feature maps are obtained via 3x3 stride 2 convolutions on the final stage + if self.config.num_feature_levels > len(sources): + _len_sources = len(sources) + sources.append(self.decoder_input_proj[_len_sources](encoder_outputs[0])[-1]) + for i in range(_len_sources + 1, self.config.num_feature_levels): + sources.append(self.decoder_input_proj[i](encoder_outputs[0][-1])) + + # Prepare encoder inputs (by flattening) + source_flatten = [] + spatial_shapes_list = [] + spatial_shapes = torch.empty((len(sources), 2), device=device, dtype=torch.long) + for level, source in enumerate(sources): + height, width = source.shape[-2:] + spatial_shapes[level, 0] = height + spatial_shapes[level, 1] = width + spatial_shapes_list.append((height, width)) + source = source.flatten(2).transpose(1, 2) + source_flatten.append(source) + source_flatten = torch.cat(source_flatten, 1) + level_start_index = torch.cat((spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1])) + + # prepare denoising training + if self.training and self.config.num_denoising > 0 and labels is not None: + ( + denoising_class, + denoising_bbox_unact, + attention_mask, + denoising_meta_values, + ) = get_contrastive_denoising_training_group( + targets=labels, + num_classes=self.config.num_labels, + num_queries=self.config.num_queries, + class_embed=self.denoising_class_embed, + num_denoising_queries=self.config.num_denoising, + label_noise_ratio=self.config.label_noise_ratio, + box_noise_scale=self.config.box_noise_scale, + ) + else: + denoising_class, denoising_bbox_unact, attention_mask, denoising_meta_values = None, None, None, None + + batch_size = len(source_flatten) + device = source_flatten.device + dtype = source_flatten.dtype + + # prepare input for decoder + if self.training or self.config.anchor_image_size is None: + # Pass spatial_shapes as tuple to make it hashable and make sure + # lru_cache is working for generate_anchors() + spatial_shapes_tuple = tuple(spatial_shapes_list) + anchors, valid_mask = self.generate_anchors(spatial_shapes_tuple, device=device, dtype=dtype) + else: + anchors, valid_mask = self.anchors, self.valid_mask + anchors, valid_mask = anchors.to(device, dtype), valid_mask.to(device, dtype) + + # use the valid_mask to selectively retain values in the feature map where the mask is `True` + memory = valid_mask.to(source_flatten.dtype) * source_flatten + + output_memory = self.enc_output(memory) + + enc_outputs_class = self.enc_score_head(output_memory) + enc_outputs_coord_logits = self.enc_bbox_head(output_memory) + anchors + + _, topk_ind = torch.topk(enc_outputs_class.max(-1).values, self.config.num_queries, dim=1) + + reference_points_unact = enc_outputs_coord_logits.gather( + dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, enc_outputs_coord_logits.shape[-1]) + ) + + # _get_pred_class_and_mask + batch_ind = torch.arange(memory.shape[0], device=output_memory.device).unsqueeze(1) + target = output_memory[batch_ind, topk_ind] + out_query = self.dec_norm(target) + mask_query_embed = self.mask_query_head(out_query) + batch_size, mask_dim, _ = mask_query_embed.shape + _, _, mask_h, mask_w = mask_feat.shape + enc_out_masks = torch.bmm(mask_query_embed, mask_feat.flatten(start_dim=2)).reshape( + batch_size, mask_dim, mask_h, mask_w + ) + + enc_topk_bboxes = F.sigmoid(reference_points_unact) + + enc_topk_logits = enc_outputs_class.gather( + dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, enc_outputs_class.shape[-1]) + ) + + # extract region features + if self.config.learn_initial_query: + target = self.weight_embedding.tile([batch_size, 1, 1]) + else: + target = output_memory.gather(dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, output_memory.shape[-1])) + target = target.detach() + + if denoising_class is not None: + target = torch.concat([denoising_class, target], 1) + + if self.mask_enhanced: + reference_points = mask_to_box_coordinate(enc_out_masks > 0, normalize=True, format="xywh") + reference_points_unact = inverse_sigmoid(reference_points) + + if denoising_bbox_unact is not None: + reference_points_unact = torch.concat([denoising_bbox_unact, reference_points_unact], 1) + + init_reference_points = reference_points_unact.detach() + + # decoder + decoder_outputs = self.decoder( + inputs_embeds=target, + encoder_hidden_states=source_flatten, + encoder_attention_mask=attention_mask, + reference_points=init_reference_points, + spatial_shapes=spatial_shapes, + spatial_shapes_list=spatial_shapes_list, + level_start_index=level_start_index, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + order_head=self.dec_order_head, + global_pointer=self.dec_global_pointer, + mask_query_head=self.mask_query_head, + dec_norm=self.dec_norm, + mask_feat=mask_feat, + ) + + if not return_dict: + enc_outputs = tuple( + value + for value in [enc_topk_logits, enc_topk_bboxes, enc_outputs_class, enc_outputs_coord_logits] + if value is not None + ) + dn_outputs = tuple(value if value is not None else None for value in [denoising_meta_values]) + tuple_outputs = ( + decoder_outputs + encoder_outputs[:-1] + (init_reference_points,) + enc_outputs + dn_outputs + ) + + return tuple_outputs + + return PPDocLayoutV3ModelOutput( + last_hidden_state=decoder_outputs.last_hidden_state, + intermediate_hidden_states=decoder_outputs.intermediate_hidden_states, + intermediate_logits=decoder_outputs.intermediate_logits, + intermediate_reference_points=decoder_outputs.intermediate_reference_points, + intermediate_predicted_corners=decoder_outputs.intermediate_predicted_corners, + initial_reference_points=decoder_outputs.initial_reference_points, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + out_order_logits=decoder_outputs.dec_out_order_logits, + out_masks=decoder_outputs.dec_out_masks, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + init_reference_points=init_reference_points, + enc_topk_logits=enc_topk_logits, + enc_topk_bboxes=enc_topk_bboxes, + enc_outputs_class=enc_outputs_class, + enc_outputs_coord_logits=enc_outputs_coord_logits, + denoising_meta_values=denoising_meta_values, + ) + + +@dataclass +class PPDocLayoutV3HybridEncoderOutput(BaseModelOutput): + mask_feat: torch.FloatTensor = None + + +@dataclass +@auto_docstring +class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): + r""" + logits (`torch.FloatTensor` of shape `(batch_size, num_queries, num_classes + 1)`): + Classification logits (including no-object) for all queries. + order_logits (`tuple` of `torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): + Order logits of the final layer of the decoder. + out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, height, width)`): + Masks of the final layer of the decoder. + pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): + Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These + values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding + possible padding). You can use [`~RTDetrImageProcessor.post_process_object_detection`] to retrieve the + unnormalized (absolute) bounding boxes. + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the decoder of the model. + intermediate_hidden_states (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, hidden_size)`): + Stacked intermediate hidden states (output of each layer of the decoder). + intermediate_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, config.num_labels)`): + Stacked intermediate logits (logits of each layer of the decoder). + intermediate_reference_points (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked intermediate reference points (reference points of each layer of the decoder). + intermediate_predicted_corners (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked intermediate predicted corners (predicted corners of each layer of the decoder). + initial_reference_points (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, 4)`): + Stacked initial reference points (initial reference points of each layer of the decoder). + init_reference_points (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): + Initial reference points sent through the Transformer decoder. + enc_topk_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Logits of predicted bounding boxes coordinates in the encoder. + enc_topk_bboxes (`torch.FloatTensor` of shape `(batch_size, sequence_length, 4)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Logits of predicted bounding boxes coordinates in the encoder. + enc_outputs_class (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Predicted bounding boxes scores where the top `config.two_stage_num_proposals` scoring bounding boxes are + picked as region proposals in the first stage. Output of bounding box binary classification (i.e. + foreground and background). + enc_outputs_coord_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, 4)`, *optional*, returned when `config.with_box_refine=True` and `config.two_stage=True`): + Logits of predicted bounding boxes coordinates in the first stage. + denoising_meta_values (`dict`): + Extra dictionary for the denoising related values + """ + + logits: Optional[torch.FloatTensor] = None + pred_boxes: Optional[torch.FloatTensor] = None + order_logits: Optional[torch.FloatTensor] = None + out_masks: Optional[torch.FloatTensor] = None + last_hidden_state: Optional[torch.FloatTensor] = None + intermediate_hidden_states: Optional[torch.FloatTensor] = None + intermediate_logits: Optional[torch.FloatTensor] = None + intermediate_reference_points: Optional[torch.FloatTensor] = None + intermediate_predicted_corners: Optional[torch.FloatTensor] = None + initial_reference_points: Optional[torch.FloatTensor] = None + decoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None + decoder_attentions: Optional[tuple[torch.FloatTensor]] = None + cross_attentions: Optional[tuple[torch.FloatTensor]] = None + encoder_last_hidden_state: Optional[torch.FloatTensor] = None + encoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None + encoder_attentions: Optional[tuple[torch.FloatTensor]] = None + init_reference_points: Optional[tuple[torch.FloatTensor]] = None + enc_topk_logits: Optional[torch.FloatTensor] = None + enc_topk_bboxes: Optional[torch.FloatTensor] = None + enc_outputs_class: Optional[torch.FloatTensor] = None + enc_outputs_coord_logits: Optional[torch.FloatTensor] = None + denoising_meta_values: Optional[dict] = None + + +class PPDocLayoutV3ForObjectDetection(RTDetrForObjectDetection, PPDocLayoutV3PreTrainedModel): + _keys_to_ignore_on_load_missing = ["num_batches_tracked", "rel_pos_y_bias", "rel_pos_x_bias"] + _tied_weights_keys = { + "model.decoder.class_embed": "model.enc_score_head", + "model.decoder.bbox_embed": "model.enc_bbox_head", + } + + def __init__(self, config: PPDocLayoutV3Config): + super().__init__(config) + + del self.model.decoder.class_embed + del self.model.decoder.bbox_embed + del num_pred # noqa + + self.model.denoising_class_embed = nn.Embedding(config.num_labels, config.d_model) + self.num_queries = config.num_queries + + self.post_init() + + @auto_docstring + def forward( + self, + pixel_values: torch.FloatTensor, + pixel_mask: Optional[torch.LongTensor] = None, + encoder_outputs: Optional[torch.FloatTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[list[dict]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **kwargs, + ) -> Union[tuple[torch.FloatTensor], PPDocLayoutV3ForObjectDetectionOutput]: + r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. + labels (`list[Dict]` of len `(batch_size,)`, *optional*): + Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the + following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch + respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes + in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. + + Examples: + + ```python + >>> from transformers import AutoModelForObjectDetection, AutoImageProcessor + >>> from PIL import Image + >>> import requests + >>> import torch + + >>> url = "https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/layout_demo.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> model_path = "PaddlePaddle/PP-DocLayoutV3_safetensors" + >>> image_processor = AutoImageProcessor.from_pretrained(model_path) + >>> model = AutoModelForObjectDetection.from_pretrained(model_path) + + >>> # prepare image for the model + >>> inputs = image_processor(images=[image], return_tensors="pt") + + >>> # forward pass + >>> outputs = model(**inputs) + + >>> # convert outputs (bounding boxes and class logits) to Pascal VOC format (xmin, ymin, xmax, ymax) + >>> results = image_processor.post_process_object_detection(outputs, target_sizes=torch.tensor([image.size[::-1]])) + + >>> # print outputs + >>> for result in results: + ... for idx, (score, label_id, box) in enumerate(zip(result["scores"], result["labels"], result["boxes"])): + ... score, label = score.item(), label_id.item() + ... box = [round(i, 2) for i in box.tolist()] + ... print(f"Order {idx + 1}: {model.config.id2label[label]}: {score:.2f} {box}") + Order 1: text: 0.99 [334.95, 184.78, 897.25, 654.83] + Order 2: paragraph_title: 0.97 [337.28, 683.92, 869.16, 798.35] + Order 3: text: 0.99 [335.75, 842.82, 892.13, 1454.32] + Order 4: text: 0.99 [920.18, 185.28, 1476.38, 464.49] + Order 5: text: 0.98 [920.47, 483.68, 1480.63, 765.72] + Order 6: text: 0.98 [920.62, 846.8, 1482.09, 1220.67] + Order 7: text: 0.97 [920.92, 1239.41, 1469.55, 1378.02] + Order 8: footnote: 0.86 [335.03, 1614.68, 1483.33, 1731.73] + Order 9: footnote: 0.83 [334.64, 1756.74, 1471.78, 1845.69] + Order 10: text: 0.81 [336.8, 1910.52, 661.64, 1939.92] + Order 11: footnote: 0.96 [336.24, 2114.42, 1450.14, 2172.12] + Order 12: number: 0.88 [106.0, 2257.5, 135.84, 2282.18] + Order 13: footer: 0.93 [338.4, 2255.52, 986.15, 2284.37] + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.model( + pixel_values, + pixel_mask=pixel_mask, + encoder_outputs=encoder_outputs, + inputs_embeds=inputs_embeds, + decoder_inputs_embeds=decoder_inputs_embeds, + labels=labels, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + intermediate_logits = outputs.intermediate_logits if return_dict else outputs[2] + intermediate_reference_points = outputs.intermediate_reference_points if return_dict else outputs[3] + order_logits = outputs.out_order_logits if return_dict else outputs[4] + out_masks = outputs.out_masks if return_dict else outputs[5] + + pred_boxes = intermediate_reference_points[:, -1] + logits = intermediate_logits[:, -1] + order_logits = order_logits[:, -1] + out_masks = out_masks[:, -1] + + if labels is not None: + raise ValueError("PPDocLayoutV3ForObjectDetection does not support training") + + if not return_dict: + return (logits, pred_boxes, order_logits, out_masks) + outputs[:4] + outputs[6:] + + return PPDocLayoutV3ForObjectDetectionOutput( + logits=logits, + pred_boxes=pred_boxes, + order_logits=order_logits, + out_masks=out_masks, + last_hidden_state=outputs.last_hidden_state, + intermediate_hidden_states=outputs.intermediate_hidden_states, + intermediate_logits=outputs.intermediate_logits, + intermediate_reference_points=outputs.intermediate_reference_points, + intermediate_predicted_corners=outputs.intermediate_predicted_corners, + initial_reference_points=outputs.initial_reference_points, + decoder_hidden_states=outputs.decoder_hidden_states, + decoder_attentions=outputs.decoder_attentions, + cross_attentions=outputs.cross_attentions, + encoder_last_hidden_state=outputs.encoder_last_hidden_state, + encoder_hidden_states=outputs.encoder_hidden_states, + encoder_attentions=outputs.encoder_attentions, + init_reference_points=outputs.init_reference_points, + enc_topk_logits=outputs.enc_topk_logits, + enc_topk_bboxes=outputs.enc_topk_bboxes, + enc_outputs_class=outputs.enc_outputs_class, + enc_outputs_coord_logits=outputs.enc_outputs_coord_logits, + denoising_meta_values=outputs.denoising_meta_values, + ) + + +__all__ = [ + "PPDocLayoutV3ForObjectDetection", + "PPDocLayoutV3ImageProcessor", + "PPDocLayoutV3ImageProcessorFast", + "PPDocLayoutV3Config", + "PPDocLayoutV3Model", + "PPDocLayoutV3PreTrainedModel", +] diff --git a/utils/check_repo.py b/utils/check_repo.py index f36cda07dc51..5e77fbcb2b00 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -158,6 +158,7 @@ "SeamlessM4TCodeHifiGan", # Building part of bigger (tested) model. "SeamlessM4TTextToUnitForConditionalGeneration", # Building part of bigger (tested) model. "ChameleonVQVAE", # VQVAE here is used only for encoding (discretizing) and is tested as part of bigger model + "PPDocLayoutV3Model", # Building part of bigger (tested) model. Tested implicitly through PPDocLayoutV3ForObjectDetection. "PaddleOCRVLModel", # Building part of bigger (tested) model. Tested implicitly through PaddleOCRVLForConditionalGeneration. "PaddleOCRVisionModel", # Building part of bigger (tested) model. Tested implicitly through PaddleOCRVLForConditionalGeneration. "PaddleOCRVisionTransformer", # Building part of bigger (tested) model. Tested implicitly through PaddleOCRVLForConditionalGeneration. @@ -197,6 +198,7 @@ "Qwen2_5_VLTextModel", # Building part of bigger (tested) model "InternVLVisionModel", # Building part of bigger (tested) model "JanusVisionModel", # Building part of bigger (tested) model + "PPDocLayoutV3Model", # Building part of bigger (tested) model "TimesFmModel", # Building part of bigger (tested) model "CsmDepthDecoderForCausalLM", # Building part of bigger (tested) model. Tested implicitly through CsmForConditionalGenerationIntegrationTest. "CsmDepthDecoderModel", # Building part of bigger (tested) model. Tested implicitly through CsmForConditionalGenerationIntegrationTest. From 358db8484e180304771fd56e94913fd7f7aea477 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Sun, 4 Jan 2026 20:13:36 +0800 Subject: [PATCH 02/21] add tests --- tests/models/pp_doclayout_v3/__init__.py | 0 .../test_modeling_pp_doclayout_v3.py | 501 ++++++++++++++++++ 2 files changed, 501 insertions(+) create mode 100644 tests/models/pp_doclayout_v3/__init__.py create mode 100644 tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py diff --git a/tests/models/pp_doclayout_v3/__init__.py b/tests/models/pp_doclayout_v3/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py new file mode 100644 index 000000000000..294d203eb5ef --- /dev/null +++ b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py @@ -0,0 +1,501 @@ +# coding = utf-8 +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Testing suite for the PP-DocLayoutV3 model.""" + +import inspect +import math +import tempfile +import unittest + +from parameterized import parameterized + +from transformers import ( + PPDocLayoutV3Config, + PPDocLayoutV3ForObjectDetection, + is_torch_available, + is_vision_available, +) +from transformers.testing_utils import ( + require_torch, + require_torch_accelerator, + slow, + torch_device, +) + +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ModelTesterMixin, floats_tensor +from ...test_pipeline_mixin import PipelineTesterMixin + + +if is_torch_available(): + import torch + +if is_vision_available(): + pass + + +class PPDocLayoutV3ModelTester: + def __init__( + self, + parent, + batch_size=3, + is_training=False, + n_targets=3, + num_labels=25, + initializer_range=0.01, + layer_norm_eps=1e-5, + batch_norm_eps=1e-5, + # backbone + backbone_config=None, + # encoder HybridEncoder + encoder_hidden_dim=32, + encoder_in_channels=[128, 256, 512], + feat_strides=[8, 16, 32], + encoder_layers=1, + encoder_ffn_dim=64, + encoder_attention_heads=2, + dropout=0.0, + activation_dropout=0.0, + encode_proj_layers=[2], + positional_encoding_temperature=10000, + encoder_activation_function="gelu", + activation_function="silu", + eval_size=None, + normalize_before=False, + # decoder PPDocLayoutV3Transformer + d_model=256, + num_queries=30, + decoder_in_channels=[32, 32, 32], + decoder_ffn_dim=64, + num_feature_levels=3, + decoder_n_points=4, + decoder_layers=2, + decoder_attention_heads=2, + decoder_activation_function="relu", + attention_dropout=0.0, + num_denoising=0, + label_noise_ratio=0.5, + box_noise_scale=1.0, + learn_initial_query=False, + anchor_image_size=None, + image_size=128, + disable_custom_kernels=True, + ): + self.parent = parent + self.batch_size = batch_size + self.num_channels = 3 + self.is_training = is_training + self.n_targets = n_targets + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.batch_norm_eps = batch_norm_eps + self.backbone_config = backbone_config + self.encoder_hidden_dim = encoder_hidden_dim + self.encoder_in_channels = encoder_in_channels + self.feat_strides = feat_strides + self.num_labels = num_labels + self.encoder_layers = encoder_layers + self.encoder_ffn_dim = encoder_ffn_dim + self.encoder_attention_heads = encoder_attention_heads + self.dropout = dropout + self.activation_dropout = activation_dropout + self.encode_proj_layers = encode_proj_layers + self.positional_encoding_temperature = positional_encoding_temperature + self.encoder_activation_function = encoder_activation_function + self.activation_function = activation_function + self.eval_size = eval_size + self.normalize_before = normalize_before + self.d_model = d_model + self.num_queries = num_queries + self.decoder_in_channels = decoder_in_channels + self.decoder_ffn_dim = decoder_ffn_dim + self.num_feature_levels = num_feature_levels + self.decoder_n_points = decoder_n_points + self.decoder_layers = decoder_layers + self.decoder_attention_heads = decoder_attention_heads + self.decoder_activation_function = decoder_activation_function + self.attention_dropout = attention_dropout + self.num_denoising = num_denoising + self.label_noise_ratio = label_noise_ratio + self.box_noise_scale = box_noise_scale + self.learn_initial_query = learn_initial_query + self.anchor_image_size = anchor_image_size + self.image_size = image_size + self.disable_custom_kernels = disable_custom_kernels + + self.encoder_seq_length = math.ceil(self.image_size / 32) * math.ceil(self.image_size / 32) + + def prepare_config_and_inputs(self): + pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) + config = self.get_config() + # config.num_labels = self.num_labels + + return config, pixel_values + + def get_config(self): + hidden_sizes = [10, 20, 30, 40] + backbone_config = { + "model_type": "hgnet_v2", + "arch": "L", + "return_idx": [0, 1, 2, 3], + "freeze_stem_only": True, + "freeze_at": 0, + "freeze_norm": True, + "lr_mult_list": [0, 0.05, 0.05, 0.05, 0.05], + "out_features": ["stage1", "stage2", "stage3", "stage4"], + } + return PPDocLayoutV3Config( + backbone_config=backbone_config, + encoder_hidden_dim=self.encoder_hidden_dim, + encoder_in_channels=hidden_sizes[1:], + feat_strides=self.feat_strides, + encoder_layers=self.encoder_layers, + encoder_ffn_dim=self.encoder_ffn_dim, + encoder_attention_heads=self.encoder_attention_heads, + dropout=self.dropout, + activation_dropout=self.activation_dropout, + encode_proj_layers=self.encode_proj_layers, + positional_encoding_temperature=self.positional_encoding_temperature, + encoder_activation_function=self.encoder_activation_function, + activation_function=self.activation_function, + eval_size=self.eval_size, + normalize_before=self.normalize_before, + d_model=self.d_model, + num_queries=self.num_queries, + decoder_in_channels=self.decoder_in_channels, + decoder_ffn_dim=self.decoder_ffn_dim, + num_feature_levels=self.num_feature_levels, + decoder_n_points=self.decoder_n_points, + decoder_layers=self.decoder_layers, + decoder_attention_heads=self.decoder_attention_heads, + decoder_activation_function=self.decoder_activation_function, + attention_dropout=self.attention_dropout, + num_denoising=self.num_denoising, + label_noise_ratio=self.label_noise_ratio, + box_noise_scale=self.box_noise_scale, + learn_initial_query=self.learn_initial_query, + anchor_image_size=self.anchor_image_size, + image_size=self.image_size, + disable_custom_kernels=self.disable_custom_kernels, + ) + + def prepare_config_and_inputs_for_common(self): + config, pixel_values = self.prepare_config_and_inputs() + inputs_dict = {"pixel_values": pixel_values} + return config, inputs_dict + + def create_and_check_pp_doclayout_v2_object_detection_head_model(self, config, pixel_values): + model = PPDocLayoutV3ForObjectDetection(config=config) + model.to(torch_device) + model.eval() + + result = model(pixel_values) + + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_queries, self.num_labels)) + self.parent.assertEqual(result.pred_boxes.shape, (self.batch_size, self.num_queries, 4)) + + +@require_torch +class PPDocLayoutV3ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): + all_model_classes = (PPDocLayoutV3ForObjectDetection,) if is_torch_available() else () + pipeline_model_mapping = {"object-detection": PPDocLayoutV3ForObjectDetection} if is_torch_available() else {} + is_encoder_decoder = True + + test_missing_keys = False + test_torch_exportable = True + + def setUp(self): + self.model_tester = PPDocLayoutV3ModelTester(self) + self.config_tester = ConfigTester( + self, + config_class=PPDocLayoutV3Config, + has_text_modality=False, + ) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_pp_doclayout_v2_object_detection_head_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_pp_doclayout_v2_object_detection_head_model(*config_and_inputs) + + @unittest.skip(reason="PPDocLayoutV3 has tied weights.") + def test_load_save_without_tied_weights(self): + pass + + @unittest.skip(reason="PPDocLayoutV3 does not use inputs_embeds") + def test_inputs_embeds(self): + pass + + @unittest.skip(reason="PPDocLayoutV3 does not use test_inputs_embeds_matches_input_ids") + def test_inputs_embeds_matches_input_ids(self): + pass + + @unittest.skip(reason="PPDocLayoutV3 does not support input and output embeddings") + def test_model_get_set_embeddings(self): + pass + + @unittest.skip(reason="PPDocLayoutV3 does not support input and output embeddings") + def test_model_common_attributes(self): + pass + + @unittest.skip(reason="PPDocLayoutV3 does not use token embeddings") + def test_resize_tokens_embeddings(self): + pass + + @unittest.skip(reason="Feed forward chunking is not implemented") + def test_feed_forward_chunking(self): + pass + + @unittest.skip(reason="PPDocLayoutV3 does not support this test") + def test_model_is_small(self): + pass + + @unittest.skip(reason="PPDocLayoutV3 does not support training") + def test_retain_grad_hidden_states_attentions(self): + pass + + def test_forward_signature(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + signature = inspect.signature(model.forward) + arg_names = [*signature.parameters.keys()] + expected_arg_names = ["pixel_values"] + self.assertListEqual(arg_names[:1], expected_arg_names) + + @parameterized.expand(["float32", "float16", "bfloat16"]) + @require_torch_accelerator + @slow + def test_inference_with_different_dtypes(self, dtype_str): + dtype = { + "float32": torch.float32, + "float16": torch.float16, + "bfloat16": torch.bfloat16, + }[dtype_str] + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + model.to(torch_device).to(dtype) + model.eval() + for key, tensor in inputs_dict.items(): + if tensor.dtype == torch.float32: + inputs_dict[key] = tensor.to(dtype) + with torch.no_grad(): + _ = model(**self._prepare_for_class(inputs_dict, model_class)) + + @parameterized.expand(["float32", "float16", "bfloat16"]) + @require_torch_accelerator + @slow + def test_inference_equivalence_for_static_and_dynamic_anchors(self, dtype_str): + dtype = { + "float32": torch.float32, + "float16": torch.float16, + "bfloat16": torch.bfloat16, + }[dtype_str] + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + h, w = inputs_dict["pixel_values"].shape[-2:] + + # convert inputs to the desired dtype + for key, tensor in inputs_dict.items(): + if tensor.dtype == torch.float32: + inputs_dict[key] = tensor.to(dtype) + + for model_class in self.all_model_classes: + with tempfile.TemporaryDirectory() as tmpdirname: + model_class(config).save_pretrained(tmpdirname) + model_static = model_class.from_pretrained( + tmpdirname, anchor_image_size=[h, w], device_map=torch_device, dtype=dtype + ).eval() + model_dynamic = model_class.from_pretrained( + tmpdirname, anchor_image_size=None, device_map=torch_device, dtype=dtype + ).eval() + + self.assertIsNotNone(model_static.config.anchor_image_size) + self.assertIsNone(model_dynamic.config.anchor_image_size) + + with torch.no_grad(): + outputs_static = model_static(**self._prepare_for_class(inputs_dict, model_class)) + outputs_dynamic = model_dynamic(**self._prepare_for_class(inputs_dict, model_class)) + + self.assertTrue( + torch.allclose(outputs_static.logits, outputs_dynamic.logits, rtol=1e-4, atol=1e-4), + f"Max diff: {(outputs_static.logits - outputs_dynamic.logits).abs().max()}", + ) + + def test_hidden_states_output(self): + def check_hidden_states_output(inputs_dict, config, model_class): + model = model_class(config) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model(**self._prepare_for_class(inputs_dict, model_class)) + + hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states + + expected_num_layers = getattr( + self.model_tester, "expected_num_hidden_layers", len(self.model_tester.encoder_in_channels) - 1 + ) + self.assertEqual(len(hidden_states), expected_num_layers) + + self.assertListEqual( + list(hidden_states[1].shape[-2:]), + [ + self.model_tester.image_size // self.model_tester.feat_strides[-1], + self.model_tester.image_size // self.model_tester.feat_strides[-1], + ], + ) + + if config.is_encoder_decoder: + hidden_states = outputs.decoder_hidden_states + + expected_num_layers = getattr( + self.model_tester, "expected_num_hidden_layers", self.model_tester.decoder_layers + 1 + ) + + self.assertIsInstance(hidden_states, (list, tuple)) + self.assertEqual(len(hidden_states), expected_num_layers) + + self.assertListEqual( + list(hidden_states[0].shape[-2:]), + [self.model_tester.num_queries, self.model_tester.d_model], + ) + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + inputs_dict["output_hidden_states"] = True + check_hidden_states_output(inputs_dict, config, model_class) + + # check that output_hidden_states also work using config + del inputs_dict["output_hidden_states"] + config.output_hidden_states = True + + check_hidden_states_output(inputs_dict, config, model_class) + + def test_attention_outputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.return_dict = True + + for model_class in self.all_model_classes: + inputs_dict["output_attentions"] = True + inputs_dict["output_hidden_states"] = False + config.return_dict = True + model = model_class._from_config(config, attn_implementation="eager") + config = model.config + model.to(torch_device) + model.eval() + with torch.no_grad(): + outputs = model(**self._prepare_for_class(inputs_dict, model_class)) + attentions = outputs.encoder_attentions + self.assertEqual(len(attentions), self.model_tester.encoder_layers) + + # check that output_attentions also work using config + del inputs_dict["output_attentions"] + config.output_attentions = True + model = model_class(config) + model.to(torch_device) + model.eval() + with torch.no_grad(): + outputs = model(**self._prepare_for_class(inputs_dict, model_class)) + attentions = outputs.encoder_attentions + self.assertEqual(len(attentions), self.model_tester.encoder_layers) + + self.assertListEqual( + list(attentions[0].shape[-3:]), + [ + self.model_tester.encoder_attention_heads, + self.model_tester.encoder_seq_length, + self.model_tester.encoder_seq_length, + ], + ) + out_len = len(outputs) + + correct_outlen = 14 + + # loss is at first position + if "labels" in inputs_dict: + correct_outlen += 1 # loss is added to beginning + # Object Detection model returns pred_logits and pred_boxes + if model_class.__name__ == "PPDocLayoutV3ForObjectDetection": + correct_outlen += 3 + + self.assertEqual(out_len, correct_outlen) + + # decoder attentions + decoder_attentions = outputs.decoder_attentions + self.assertIsInstance(decoder_attentions, (list, tuple)) + self.assertEqual(len(decoder_attentions), self.model_tester.decoder_layers) + self.assertListEqual( + list(decoder_attentions[0].shape[-3:]), + [ + self.model_tester.decoder_attention_heads, + self.model_tester.num_queries, + self.model_tester.num_queries, + ], + ) + + # cross attentions + cross_attentions = outputs.cross_attentions + self.assertIsInstance(cross_attentions, (list, tuple)) + self.assertEqual(len(cross_attentions), self.model_tester.decoder_layers) + self.assertListEqual( + list(cross_attentions[0].shape[-3:]), + [ + self.model_tester.decoder_attention_heads, + self.model_tester.num_feature_levels, + self.model_tester.decoder_n_points, + ], + ) + + # Check attention is always last and order is fine + inputs_dict["output_attentions"] = True + inputs_dict["output_hidden_states"] = True + model = model_class(config) + model.to(torch_device) + model.eval() + with torch.no_grad(): + outputs = model(**self._prepare_for_class(inputs_dict, model_class)) + + if hasattr(self.model_tester, "num_hidden_states_types"): + added_hidden_states = self.model_tester.num_hidden_states_types + else: + # RTDetr should maintin encoder_hidden_states output + added_hidden_states = 2 + self.assertEqual(out_len + added_hidden_states, len(outputs)) + + self_attentions = outputs.encoder_attentions + + self.assertEqual(len(self_attentions), self.model_tester.encoder_layers) + self.assertListEqual( + list(self_attentions[0].shape[-3:]), + [ + self.model_tester.encoder_attention_heads, + self.model_tester.encoder_seq_length, + self.model_tester.encoder_seq_length, + ], + ) + + +# TODO: +# @require_torch +# @require_vision +# @slow +# class PPDocLayoutV3ModelIntegrationTest(unittest.TestCase): From 4f19273924d95cd86df3da94f3f8c998a7fc19ed Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Wed, 7 Jan 2026 16:36:22 +0800 Subject: [PATCH 03/21] update --- .../configuration_pp_doclayout_v3.py | 41 --- .../image_processing_pp_doclayout_v3.py | 107 ++++--- .../image_processing_pp_doclayout_v3_fast.py | 107 ++++--- .../modeling_pp_doclayout_v3.py | 67 +++-- .../modular_pp_doclayout_v3.py | 260 +++++++++--------- 5 files changed, 263 insertions(+), 319 deletions(-) diff --git a/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py index dfc26a6e36e8..d6ff1359d6ed 100644 --- a/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py @@ -27,36 +27,6 @@ logger = logging.get_logger(__name__) -def _default_id2label() -> dict[int, str]: - return { - 0: "abstract", - 1: "algorithm", - 2: "aside_text", - 3: "chart", - 4: "content", - 5: "formula", - 6: "doc_title", - 7: "figure_title", - 8: "footer", - 9: "footer", - 10: "footnote", - 11: "formula_number", - 12: "header", - 13: "header", - 14: "image", - 15: "formula", - 16: "number", - 17: "paragraph_title", - 18: "reference", - 19: "reference_content", - 20: "seal", - 21: "table", - 22: "text", - 23: "text", - 24: "vision_footnote", - } - - class PPDocLayoutV3Config(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`PP-DocLayoutV3`]. It is used to instantiate a @@ -170,8 +140,6 @@ class PPDocLayoutV3Config(PreTrainedConfig): Whether the architecture has an encoder decoder structure. gp_head_size (`int`, *optional*, defaults to 64): The size of the global pointer head. - tril_mask (`bool`, *optional*, defaults to `True`): - Whether to mask out the upper triangular part of the attention matrix. id2label (`dict[int, str]`, *optional*): Mapping from class id to class name. @@ -251,7 +219,6 @@ def __init__( disable_custom_kernels=True, is_encoder_decoder=True, gp_head_size=64, - tril_mask=True, # label id2label=None, **kwargs, @@ -337,16 +304,8 @@ def __init__( self.anchor_image_size = list(anchor_image_size) if anchor_image_size is not None else None self.disable_custom_kernels = disable_custom_kernels self.gp_head_size = gp_head_size - self.tril_mask = tril_mask super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs) - if kwargs.get("num_labels") and not id2label: - self.id2label = None - self.num_labels = kwargs["num_labels"] - else: - self.id2label = {int(k): v for k, v in id2label.items()} if id2label else _default_id2label() - self.num_labels = len(self.id2label) - __all__ = ["PPDocLayoutV3Config"] diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py index 8b3bb2cc8751..ad8760145335 100644 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py @@ -39,68 +39,21 @@ from ...utils.generic import TensorType -def postprocess(outputs, threshold, target_sizes): - boxes = outputs.pred_boxes - logits = outputs.logits - order_logits = outputs.order_logits - - order_seqs = get_order(order_logits) - - cxcy, wh = torch.split(boxes, 2, dim=-1) - boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) - - if target_sizes is not None: - if len(logits) != len(target_sizes): - raise ValueError("Make sure that you pass in as many target sizes as the batch dimension of the logits") - if isinstance(target_sizes, list): - img_h, img_w = torch.as_tensor(target_sizes).unbind(1) - else: - img_h, img_w = target_sizes.unbind(1) - scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) - boxes = boxes * scale_fct[:, None, :] - - num_top_queries = logits.shape[1] - num_classes = logits.shape[2] - - scores = torch.nn.functional.sigmoid(logits) - scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) - labels = index % num_classes - index = index // num_classes - boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) - order_seqs = order_seqs.gather(dim=1, index=index) - - results = [] - for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): - order_seq = order_seq[score >= threshold] - order_seq, indices = torch.sort(order_seq) - results.append( - { - "scores": score[score >= threshold][indices], - "labels": label[score >= threshold][indices], - "boxes": box[score >= threshold][indices], - "order_seq": order_seq, - } - ) - - return results - - -def get_order(order_logits): - # order_logits: (B, N, N) upper-triangular meaningful +def get_order_seqs(order_logits): order_scores = torch.sigmoid(order_logits) - B, N, _ = order_scores.shape + batch_size, sequence_length, _ = order_scores.shape - one = torch.ones((N, N), dtype=order_scores.dtype, device=order_scores.device) + one = torch.ones((sequence_length, sequence_length), dtype=order_scores.dtype, device=order_scores.device) upper = torch.triu(one, 1) lower = torch.tril(one, -1) Q = order_scores * upper + (1.0 - order_scores.transpose(1, 2)) * lower - order_votes = Q.sum(dim=1) # (B, N) + order_votes = Q.sum(dim=1) - order_pointers = torch.argsort(order_votes, dim=1) # (B, N) + order_pointers = torch.argsort(order_votes, dim=1) order_seq = torch.full_like(order_pointers, -1) - batch = torch.arange(B, device=order_pointers.device)[:, None] - order_seq[batch, order_pointers] = torch.arange(N, device=order_pointers.device)[None, :] + batch = torch.arange(batch_size, device=order_pointers.device)[:, None] + order_seq[batch, order_pointers] = torch.arange(sequence_length, device=order_pointers.device)[None, :] return order_seq @@ -310,7 +263,51 @@ def post_process_object_detection( `list[Dict]`: An ordered list of dictionaries, each dictionary containing the scores, labels and boxes for an image in the batch as predicted by the model. """ - return postprocess(outputs=outputs, threshold=threshold, target_sizes=target_sizes) + boxes = outputs.pred_boxes + logits = outputs.logits + order_logits = outputs.order_logits + + order_seqs = get_order_seqs(order_logits) + + cxcy, wh = torch.split(boxes, 2, dim=-1) + boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) + + if target_sizes is not None: + if len(logits) != len(target_sizes): + raise ValueError( + "Make sure that you pass in as many target sizes as the batch dimension of the logits" + ) + if isinstance(target_sizes, list): + img_h, img_w = torch.as_tensor(target_sizes).unbind(1) + else: + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) + boxes = boxes * scale_fct[:, None, :] + + num_top_queries = logits.shape[1] + num_classes = logits.shape[2] + + scores = torch.nn.functional.sigmoid(logits) + scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) + labels = index % num_classes + index = index // num_classes + boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) + order_seqs = order_seqs.gather(dim=1, index=index) + + results = [] + for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): + order_seq = order_seq[score >= threshold] + order_seq, indices = torch.sort(order_seq) + results.append( + { + "scores": score[score >= threshold][indices], + "labels": label[score >= threshold][indices], + "boxes": box[score >= threshold][indices], + "order_seq": order_seq, + } + ) + + return results __all__ = ["PPDocLayoutV3ImageProcessor"] diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py index e53f6d0fee25..6c31da2ff1c2 100644 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py @@ -29,68 +29,21 @@ from ...utils.generic import TensorType -def postprocess(outputs, threshold, target_sizes): - boxes = outputs.pred_boxes - logits = outputs.logits - order_logits = outputs.order_logits - - order_seqs = get_order(order_logits) - - cxcy, wh = torch.split(boxes, 2, dim=-1) - boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) - - if target_sizes is not None: - if len(logits) != len(target_sizes): - raise ValueError("Make sure that you pass in as many target sizes as the batch dimension of the logits") - if isinstance(target_sizes, list): - img_h, img_w = torch.as_tensor(target_sizes).unbind(1) - else: - img_h, img_w = target_sizes.unbind(1) - scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) - boxes = boxes * scale_fct[:, None, :] - - num_top_queries = logits.shape[1] - num_classes = logits.shape[2] - - scores = torch.nn.functional.sigmoid(logits) - scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) - labels = index % num_classes - index = index // num_classes - boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) - order_seqs = order_seqs.gather(dim=1, index=index) - - results = [] - for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): - order_seq = order_seq[score >= threshold] - order_seq, indices = torch.sort(order_seq) - results.append( - { - "scores": score[score >= threshold][indices], - "labels": label[score >= threshold][indices], - "boxes": box[score >= threshold][indices], - "order_seq": order_seq, - } - ) - - return results - - -def get_order(order_logits): - # order_logits: (B, N, N) upper-triangular meaningful +def get_order_seqs(order_logits): order_scores = torch.sigmoid(order_logits) - B, N, _ = order_scores.shape + batch_size, sequence_length, _ = order_scores.shape - one = torch.ones((N, N), dtype=order_scores.dtype, device=order_scores.device) + one = torch.ones((sequence_length, sequence_length), dtype=order_scores.dtype, device=order_scores.device) upper = torch.triu(one, 1) lower = torch.tril(one, -1) Q = order_scores * upper + (1.0 - order_scores.transpose(1, 2)) * lower - order_votes = Q.sum(dim=1) # (B, N) + order_votes = Q.sum(dim=1) - order_pointers = torch.argsort(order_votes, dim=1) # (B, N) + order_pointers = torch.argsort(order_votes, dim=1) order_seq = torch.full_like(order_pointers, -1) - batch = torch.arange(B, device=order_pointers.device)[:, None] - order_seq[batch, order_pointers] = torch.arange(N, device=order_pointers.device)[None, :] + batch = torch.arange(batch_size, device=order_pointers.device)[:, None] + order_seq[batch, order_pointers] = torch.arange(sequence_length, device=order_pointers.device)[None, :] return order_seq @@ -158,7 +111,51 @@ def post_process_object_detection( `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image in the batch as predicted by the model. """ - return postprocess(outputs=outputs, threshold=threshold, target_sizes=target_sizes) + boxes = outputs.pred_boxes + logits = outputs.logits + order_logits = outputs.order_logits + + order_seqs = get_order_seqs(order_logits) + + cxcy, wh = torch.split(boxes, 2, dim=-1) + boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) + + if target_sizes is not None: + if len(logits) != len(target_sizes): + raise ValueError( + "Make sure that you pass in as many target sizes as the batch dimension of the logits" + ) + if isinstance(target_sizes, list): + img_h, img_w = torch.as_tensor(target_sizes).unbind(1) + else: + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) + boxes = boxes * scale_fct[:, None, :] + + num_top_queries = logits.shape[1] + num_classes = logits.shape[2] + + scores = torch.nn.functional.sigmoid(logits) + scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) + labels = index % num_classes + index = index // num_classes + boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) + order_seqs = order_seqs.gather(dim=1, index=index) + + results = [] + for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): + order_seq = order_seq[score >= threshold] + order_seq, indices = torch.sort(order_seq) + results.append( + { + "scores": score[score >= threshold][indices], + "labels": label[score >= threshold][indices], + "boxes": box[score >= threshold][indices], + "order_seq": order_seq, + } + ) + + return results __all__ = ["PPDocLayoutV3ImageProcessorFast"] diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index 5c09131d2118..645379acc88d 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -44,7 +44,6 @@ class GlobalPointer(nn.Module): def __init__(self, config): super().__init__() self.head_size = config.gp_head_size - self.tril_mask = config.tril_mask self.dense = nn.Linear(config.d_model, self.head_size * 2) self.dropout = nn.Dropout(0.1) @@ -56,9 +55,8 @@ def forward(self, inputs): logits = torch.einsum("bmd,bnd->bmn", qw, kw) / (self.head_size**0.5) # [B, N, N] - if self.tril_mask: - lower = torch.tril(torch.ones([N, N], dtype=torch.float32, device=logits.device)) - logits = logits - lower.unsqueeze(0).to(logits.dtype) * 1e4 + lower = torch.tril(torch.ones([N, N], dtype=torch.float32, device=logits.device)) + logits = logits - lower.unsqueeze(0).to(logits.dtype) * 1e4 return logits @@ -321,6 +319,7 @@ class PPDocLayoutV3DecoderOutput(ModelOutput): hidden_states: Optional[tuple[torch.FloatTensor]] = None attentions: Optional[tuple[torch.FloatTensor]] = None cross_attentions: Optional[tuple[torch.FloatTensor]] = None + dec_out_order_logits: Optional[torch.FloatTensor] = None dec_out_masks: Optional[torch.FloatTensor] = None @@ -385,6 +384,7 @@ class PPDocLayoutV3ModelOutput(ModelOutput): enc_outputs_class: Optional[torch.FloatTensor] = None enc_outputs_coord_logits: Optional[torch.FloatTensor] = None denoising_meta_values: Optional[dict] = None + out_order_logits: Optional[torch.FloatTensor] = None out_masks: Optional[torch.FloatTensor] = None @@ -1535,54 +1535,51 @@ def get_contrastive_denoising_training_group( return input_query_class, input_query_bbox, attn_mask, denoising_meta_values -def bbox_xyxy_to_cxcywh(x): - x1 = x[..., 0] - y1 = x[..., 1] - x2 = x[..., 2] - y2 = x[..., 3] - return torch.stack([(x1 + x2) / 2, (y1 + y2) / 2, (x2 - x1), (y2 - y1)], dim=-1) +def mask_to_box_coordinate(mask, dtype=torch.float32): + mask = mask.bool() + height, width = mask.shape[-2:] -def mask_to_box_coordinate(mask, normalize=False, format="xyxy", dtype=torch.float32): - assert mask.ndim == 4, f"Error: The 'mask' variable must be a 4-dimensional array, but got {mask.ndim} dimensions." - assert format in ["xyxy", "xywh"], ( - f"Error: The 'format' variable must be either 'xyxy' or 'xywh', but got '{format}'." + y_coords, x_coords = torch.meshgrid( + torch.arange(height, device=mask.device), torch.arange(width, device=mask.device), indexing="ij" ) + x_coords = x_coords.to(dtype) + y_coords = y_coords.to(dtype) - mask = mask.bool() - - h, w = mask.shape[-2:] - y, x = torch.meshgrid(torch.arange(h, device=mask.device), torch.arange(w, device=mask.device), indexing="ij") - x = x.to(dtype) - y = y.to(dtype) - - x_mask = x * mask - x_max = x_mask.flatten(start_dim=-2).max(dim=-1).values + 1 + x_coords_masked = x_coords * mask + x_max = x_coords_masked.flatten(start_dim=-2).max(dim=-1).values + 1 x_min = ( - torch.where(mask, x_mask, torch.tensor(1e8, device=mask.device, dtype=dtype)) + torch.where(mask, x_coords_masked, torch.tensor(1e8, device=mask.device, dtype=dtype)) .flatten(start_dim=-2) .min(dim=-1) .values ) - y_mask = y * mask - y_max = y_mask.flatten(start_dim=-2).max(dim=-1).values + 1 + y_coords_masked = y_coords * mask + y_max = y_coords_masked.flatten(start_dim=-2).max(dim=-1).values + 1 y_min = ( - torch.where(mask, y_mask, torch.tensor(1e8, device=mask.device, dtype=dtype)) + torch.where(mask, y_coords_masked, torch.tensor(1e8, device=mask.device, dtype=dtype)) .flatten(start_dim=-2) .min(dim=-1) .values ) - out_bbox = torch.stack([x_min, y_min, x_max, y_max], dim=-1) - mask_any = torch.any(mask, dim=(-2, -1)).unsqueeze(-1) - out_bbox = out_bbox * mask_any + unnormalized_bbox = torch.stack([x_min, y_min, x_max, y_max], dim=-1) + + is_mask_non_empty = torch.any(mask, dim=(-2, -1)).unsqueeze(-1) + unnormalized_bbox = unnormalized_bbox * is_mask_non_empty + + norm_tensor = torch.tensor([width, height, width, height], device=mask.device, dtype=dtype) + normalized_bbox_xyxy = unnormalized_bbox / norm_tensor + + x_min_norm, y_min_norm, x_max_norm, y_max_norm = normalized_bbox_xyxy.unbind(dim=-1) - if normalize: - norm_tensor = torch.tensor([w, h, w, h], device=mask.device, dtype=dtype) - out_bbox /= norm_tensor + center_x = (x_min_norm + x_max_norm) / 2 + center_y = (y_min_norm + y_max_norm) / 2 + box_width = x_max_norm - x_min_norm + box_height = y_max_norm - y_min_norm - return out_bbox if format == "xyxy" else bbox_xyxy_to_cxcywh(out_bbox) + return torch.stack([center_x, center_y, box_width, box_height], dim=-1) @auto_docstring( @@ -1904,7 +1901,7 @@ def forward( target = torch.concat([denoising_class, target], 1) if self.mask_enhanced: - reference_points = mask_to_box_coordinate(enc_out_masks > 0, normalize=True, format="xywh") + reference_points = mask_to_box_coordinate(enc_out_masks > 0) reference_points_unact = inverse_sigmoid(reference_points) if denoising_bbox_unact is not None: diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index 9c0f397b0b2d..a4b9f22cd6e0 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -69,36 +69,6 @@ logger = logging.get_logger(__name__) -def _default_id2label() -> dict[int, str]: - return { - 0: "abstract", - 1: "algorithm", - 2: "aside_text", - 3: "chart", - 4: "content", - 5: "formula", - 6: "doc_title", - 7: "figure_title", - 8: "footer", - 9: "footer", - 10: "footnote", - 11: "formula_number", - 12: "header", - 13: "header", - 14: "image", - 15: "formula", - 16: "number", - 17: "paragraph_title", - 18: "reference", - 19: "reference_content", - 20: "seal", - 21: "table", - 22: "text", - 23: "text", - 24: "vision_footnote", - } - - class PPDocLayoutV3Config(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`PP-DocLayoutV3`]. It is used to instantiate a @@ -212,8 +182,6 @@ class PPDocLayoutV3Config(PreTrainedConfig): Whether the architecture has an encoder decoder structure. gp_head_size (`int`, *optional*, defaults to 64): The size of the global pointer head. - tril_mask (`bool`, *optional*, defaults to `True`): - Whether to mask out the upper triangular part of the attention matrix. id2label (`dict[int, str]`, *optional*): Mapping from class id to class name. @@ -293,7 +261,6 @@ def __init__( disable_custom_kernels=True, is_encoder_decoder=True, gp_head_size=64, - tril_mask=True, # label id2label=None, **kwargs, @@ -379,80 +346,25 @@ def __init__( self.anchor_image_size = list(anchor_image_size) if anchor_image_size is not None else None self.disable_custom_kernels = disable_custom_kernels self.gp_head_size = gp_head_size - self.tril_mask = tril_mask super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs) - if kwargs.get("num_labels") and not id2label: - self.id2label = None - self.num_labels = kwargs["num_labels"] - else: - self.id2label = {int(k): v for k, v in id2label.items()} if id2label else _default_id2label() - self.num_labels = len(self.id2label) - -def postprocess(outputs, threshold, target_sizes): - boxes = outputs.pred_boxes - logits = outputs.logits - order_logits = outputs.order_logits - - order_seqs = get_order(order_logits) - - cxcy, wh = torch.split(boxes, 2, dim=-1) - boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) - - if target_sizes is not None: - if len(logits) != len(target_sizes): - raise ValueError("Make sure that you pass in as many target sizes as the batch dimension of the logits") - if isinstance(target_sizes, list): - img_h, img_w = torch.as_tensor(target_sizes).unbind(1) - else: - img_h, img_w = target_sizes.unbind(1) - scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) - boxes = boxes * scale_fct[:, None, :] - - num_top_queries = logits.shape[1] - num_classes = logits.shape[2] - - scores = torch.nn.functional.sigmoid(logits) - scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) - labels = index % num_classes - index = index // num_classes - boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) - order_seqs = order_seqs.gather(dim=1, index=index) - - results = [] - for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): - order_seq = order_seq[score >= threshold] - order_seq, indices = torch.sort(order_seq) - results.append( - { - "scores": score[score >= threshold][indices], - "labels": label[score >= threshold][indices], - "boxes": box[score >= threshold][indices], - "order_seq": order_seq, - } - ) - - return results - - -def get_order(order_logits): - # order_logits: (B, N, N) upper-triangular meaningful +def get_order_seqs(order_logits): order_scores = torch.sigmoid(order_logits) - B, N, _ = order_scores.shape + batch_size, sequence_length, _ = order_scores.shape - one = torch.ones((N, N), dtype=order_scores.dtype, device=order_scores.device) + one = torch.ones((sequence_length, sequence_length), dtype=order_scores.dtype, device=order_scores.device) upper = torch.triu(one, 1) lower = torch.tril(one, -1) Q = order_scores * upper + (1.0 - order_scores.transpose(1, 2)) * lower - order_votes = Q.sum(dim=1) # (B, N) + order_votes = Q.sum(dim=1) - order_pointers = torch.argsort(order_votes, dim=1) # (B, N) + order_pointers = torch.argsort(order_votes, dim=1) order_seq = torch.full_like(order_pointers, -1) - batch = torch.arange(B, device=order_pointers.device)[:, None] - order_seq[batch, order_pointers] = torch.arange(N, device=order_pointers.device)[None, :] + batch = torch.arange(batch_size, device=order_pointers.device)[:, None] + order_seq[batch, order_pointers] = torch.arange(sequence_length, device=order_pointers.device)[None, :] return order_seq @@ -662,7 +574,51 @@ def post_process_object_detection( `list[Dict]`: An ordered list of dictionaries, each dictionary containing the scores, labels and boxes for an image in the batch as predicted by the model. """ - return postprocess(outputs=outputs, threshold=threshold, target_sizes=target_sizes) + boxes = outputs.pred_boxes + logits = outputs.logits + order_logits = outputs.order_logits + + order_seqs = get_order_seqs(order_logits) + + cxcy, wh = torch.split(boxes, 2, dim=-1) + boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) + + if target_sizes is not None: + if len(logits) != len(target_sizes): + raise ValueError( + "Make sure that you pass in as many target sizes as the batch dimension of the logits" + ) + if isinstance(target_sizes, list): + img_h, img_w = torch.as_tensor(target_sizes).unbind(1) + else: + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) + boxes = boxes * scale_fct[:, None, :] + + num_top_queries = logits.shape[1] + num_classes = logits.shape[2] + + scores = torch.nn.functional.sigmoid(logits) + scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) + labels = index % num_classes + index = index // num_classes + boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) + order_seqs = order_seqs.gather(dim=1, index=index) + + results = [] + for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): + order_seq = order_seq[score >= threshold] + order_seq, indices = torch.sort(order_seq) + results.append( + { + "scores": score[score >= threshold][indices], + "labels": label[score >= threshold][indices], + "boxes": box[score >= threshold][indices], + "order_seq": order_seq, + } + ) + + return results class PPDocLayoutV3ImageProcessorFast(BaseImageProcessorFast): @@ -728,14 +684,57 @@ def post_process_object_detection( `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image in the batch as predicted by the model. """ - return postprocess(outputs=outputs, threshold=threshold, target_sizes=target_sizes) + boxes = outputs.pred_boxes + logits = outputs.logits + order_logits = outputs.order_logits + + order_seqs = get_order_seqs(order_logits) + + cxcy, wh = torch.split(boxes, 2, dim=-1) + boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) + + if target_sizes is not None: + if len(logits) != len(target_sizes): + raise ValueError( + "Make sure that you pass in as many target sizes as the batch dimension of the logits" + ) + if isinstance(target_sizes, list): + img_h, img_w = torch.as_tensor(target_sizes).unbind(1) + else: + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) + boxes = boxes * scale_fct[:, None, :] + + num_top_queries = logits.shape[1] + num_classes = logits.shape[2] + + scores = torch.nn.functional.sigmoid(logits) + scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) + labels = index % num_classes + index = index // num_classes + boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) + order_seqs = order_seqs.gather(dim=1, index=index) + + results = [] + for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): + order_seq = order_seq[score >= threshold] + order_seq, indices = torch.sort(order_seq) + results.append( + { + "scores": score[score >= threshold][indices], + "labels": label[score >= threshold][indices], + "boxes": box[score >= threshold][indices], + "order_seq": order_seq, + } + ) + + return results class GlobalPointer(nn.Module): def __init__(self, config): super().__init__() self.head_size = config.gp_head_size - self.tril_mask = config.tril_mask self.dense = nn.Linear(config.d_model, self.head_size * 2) self.dropout = nn.Dropout(0.1) @@ -747,9 +746,8 @@ def forward(self, inputs): logits = torch.einsum("bmd,bnd->bmn", qw, kw) / (self.head_size**0.5) # [B, N, N] - if self.tril_mask: - lower = torch.tril(torch.ones([N, N], dtype=torch.float32, device=logits.device)) - logits = logits - lower.unsqueeze(0).to(logits.dtype) * 1e4 + lower = torch.tril(torch.ones([N, N], dtype=torch.float32, device=logits.device)) + logits = logits - lower.unsqueeze(0).to(logits.dtype) * 1e4 return logits @@ -816,54 +814,51 @@ def _init_weights(self, module): init.zeros_(module.weight.data[module.padding_idx]) -def bbox_xyxy_to_cxcywh(x): - x1 = x[..., 0] - y1 = x[..., 1] - x2 = x[..., 2] - y2 = x[..., 3] - return torch.stack([(x1 + x2) / 2, (y1 + y2) / 2, (x2 - x1), (y2 - y1)], dim=-1) +def mask_to_box_coordinate(mask, dtype=torch.float32): + mask = mask.bool() + height, width = mask.shape[-2:] -def mask_to_box_coordinate(mask, normalize=False, format="xyxy", dtype=torch.float32): - assert mask.ndim == 4, f"Error: The 'mask' variable must be a 4-dimensional array, but got {mask.ndim} dimensions." - assert format in ["xyxy", "xywh"], ( - f"Error: The 'format' variable must be either 'xyxy' or 'xywh', but got '{format}'." + y_coords, x_coords = torch.meshgrid( + torch.arange(height, device=mask.device), torch.arange(width, device=mask.device), indexing="ij" ) + x_coords = x_coords.to(dtype) + y_coords = y_coords.to(dtype) - mask = mask.bool() - - h, w = mask.shape[-2:] - y, x = torch.meshgrid(torch.arange(h, device=mask.device), torch.arange(w, device=mask.device), indexing="ij") - x = x.to(dtype) - y = y.to(dtype) - - x_mask = x * mask - x_max = x_mask.flatten(start_dim=-2).max(dim=-1).values + 1 + x_coords_masked = x_coords * mask + x_max = x_coords_masked.flatten(start_dim=-2).max(dim=-1).values + 1 x_min = ( - torch.where(mask, x_mask, torch.tensor(1e8, device=mask.device, dtype=dtype)) + torch.where(mask, x_coords_masked, torch.tensor(1e8, device=mask.device, dtype=dtype)) .flatten(start_dim=-2) .min(dim=-1) .values ) - y_mask = y * mask - y_max = y_mask.flatten(start_dim=-2).max(dim=-1).values + 1 + y_coords_masked = y_coords * mask + y_max = y_coords_masked.flatten(start_dim=-2).max(dim=-1).values + 1 y_min = ( - torch.where(mask, y_mask, torch.tensor(1e8, device=mask.device, dtype=dtype)) + torch.where(mask, y_coords_masked, torch.tensor(1e8, device=mask.device, dtype=dtype)) .flatten(start_dim=-2) .min(dim=-1) .values ) - out_bbox = torch.stack([x_min, y_min, x_max, y_max], dim=-1) - mask_any = torch.any(mask, dim=(-2, -1)).unsqueeze(-1) - out_bbox = out_bbox * mask_any + unnormalized_bbox = torch.stack([x_min, y_min, x_max, y_max], dim=-1) - if normalize: - norm_tensor = torch.tensor([w, h, w, h], device=mask.device, dtype=dtype) - out_bbox /= norm_tensor + is_mask_non_empty = torch.any(mask, dim=(-2, -1)).unsqueeze(-1) + unnormalized_bbox = unnormalized_bbox * is_mask_non_empty - return out_bbox if format == "xyxy" else bbox_xyxy_to_cxcywh(out_bbox) + norm_tensor = torch.tensor([width, height, width, height], device=mask.device, dtype=dtype) + normalized_bbox_xyxy = unnormalized_bbox / norm_tensor + + x_min_norm, y_min_norm, x_max_norm, y_max_norm = normalized_bbox_xyxy.unbind(dim=-1) + + center_x = (x_min_norm + x_max_norm) / 2 + center_y = (y_min_norm + y_max_norm) / 2 + box_width = x_max_norm - x_min_norm + box_height = y_max_norm - y_min_norm + + return torch.stack([center_x, center_y, box_width, box_height], dim=-1) @dataclass @@ -1564,7 +1559,7 @@ def forward( target = torch.concat([denoising_class, target], 1) if self.mask_enhanced: - reference_points = mask_to_box_coordinate(enc_out_masks > 0, normalize=True, format="xywh") + reference_points = mask_to_box_coordinate(enc_out_masks > 0) reference_points_unact = inverse_sigmoid(reference_points) if denoising_bbox_unact is not None: @@ -1675,7 +1670,6 @@ class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): denoising_meta_values (`dict`): Extra dictionary for the denoising related values """ - logits: Optional[torch.FloatTensor] = None pred_boxes: Optional[torch.FloatTensor] = None order_logits: Optional[torch.FloatTensor] = None From 43d1cbfc33e65680c8564b4cb8168f835cf6e5b4 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Tue, 13 Jan 2026 20:33:06 +0800 Subject: [PATCH 04/21] update --- docs/source/en/_toctree.yml | 2 + src/transformers/models/auto/modeling_auto.py | 1 + .../configuration_pp_doclayout_v3.py | 22 +-- .../image_processing_pp_doclayout_v3.py | 44 ++--- .../image_processing_pp_doclayout_v3_fast.py | 46 ++--- .../modeling_pp_doclayout_v3.py | 81 ++++---- .../modular_pp_doclayout_v3.py | 179 +++++++++--------- .../test_modeling_pp_doclayout_v3.py | 55 ------ 8 files changed, 180 insertions(+), 250 deletions(-) diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 670854e4895d..cc31fd4c1b76 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -1151,6 +1151,8 @@ title: Pix2Struct - local: model_doc/pixtral title: Pixtral + - local: model_doc/pp_doclayout_v3 + title: PP-DocLayoutV3 - local: model_doc/qwen2_5_omni title: Qwen2.5-Omni - local: model_doc/qwen2_5_vl diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 728a209b39e5..54ba7085c01f 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -327,6 +327,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("pixtral", "PixtralVisionModel"), ("plbart", "PLBartModel"), ("poolformer", "PoolFormerModel"), + ("pp_doclayout_v3", "PPDocLayoutV3Model"), ("prophetnet", "ProphetNetModel"), ("pvt", "PvtModel"), ("pvt_v2", "PvtV2Model"), diff --git a/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py index d6ff1359d6ed..dce57bfa3540 100644 --- a/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py @@ -97,12 +97,20 @@ class PPDocLayoutV3Config(PreTrainedConfig): feed-forward modules. hidden_expansion (`float`, *optional*, defaults to 1.0): Expansion ratio to enlarge the dimension size of RepVGGBlock and CSPRepLayer. - mask_feat_channels (`list[int]`, *optional*, defaults to `[64, 64]): + mask_feat_channels (`list[int]`, *optional*, defaults to `[64, 64]`): The channels of the multi-level features for mask enhancement. x4_feat_dim (`int`, *optional*, defaults to 128): The dimension of the x4 feature map. d_model (`int`, *optional*, defaults to 256): Dimension of the layers exclude hybrid encoder. + num_prototypes (`int`, *optional*, defaults to 32): + Dimension of the layers exclude mask query head. + label_noise_ratio (`float`, *optional*, defaults to 0.4): + The fraction of denoising labels to which random noise should be added. + box_noise_scale (`float`, *optional*, defaults to 0.4): + Scale or magnitude of noise to be added to the bounding boxes. + mask_enhanced (`bool`, *optional*, defaults to `True`): + Whether to use enhanced masked attention. num_queries (`int`, *optional*, defaults to 300): Number of object queries. decoder_in_channels (`list`, *optional*, defaults to `[256, 256, 256]`): @@ -124,14 +132,8 @@ class PPDocLayoutV3Config(PreTrainedConfig): The dropout ratio for the attention probabilities. num_denoising (`int`, *optional*, defaults to 100): The total number of denoising tasks or queries to be used for contrastive denoising. - label_noise_ratio (`float`, *optional*, defaults to 0.5): - The fraction of denoising labels to which random noise should be added. - box_noise_scale (`float`, *optional*, defaults to 1.0): - Scale or magnitude of noise to be added to the bounding boxes. learn_initial_query (`bool`, *optional*, defaults to `False`): Indicates whether the initial query embeddings for the decoder should be learned during training - mask_enhanced (`bool`, *optional*, defaults to `True`): - Whether to use enhanced masked attention. anchor_image_size (`tuple[int, int]`, *optional*): Height and width of the input image used during evaluation to generate the bounding box anchors. If None, automatic generate anchor is applied. disable_custom_kernels (`bool`, *optional*, defaults to `True`): @@ -140,8 +142,6 @@ class PPDocLayoutV3Config(PreTrainedConfig): Whether the architecture has an encoder decoder structure. gp_head_size (`int`, *optional*, defaults to 64): The size of the global pointer head. - id2label (`dict[int, str]`, *optional*): - Mapping from class id to class name. Examples: @@ -158,7 +158,7 @@ class PPDocLayoutV3Config(PreTrainedConfig): >>> configuration = model.config ```""" - model_type = "pp_doclayout_v2" + model_type = "pp_doclayout_v3" sub_configs = {"backbone_config": AutoConfig} layer_types = ("basic", "bottleneck") @@ -219,8 +219,6 @@ def __init__( disable_custom_kernels=True, is_encoder_decoder=True, gp_head_size=64, - # label - id2label=None, **kwargs, ): self.initializer_range = initializer_range diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py index ad8760145335..39bad505ac3d 100644 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py @@ -39,25 +39,6 @@ from ...utils.generic import TensorType -def get_order_seqs(order_logits): - order_scores = torch.sigmoid(order_logits) - batch_size, sequence_length, _ = order_scores.shape - - one = torch.ones((sequence_length, sequence_length), dtype=order_scores.dtype, device=order_scores.device) - upper = torch.triu(one, 1) - lower = torch.tril(one, -1) - - Q = order_scores * upper + (1.0 - order_scores.transpose(1, 2)) * lower - order_votes = Q.sum(dim=1) - - order_pointers = torch.argsort(order_votes, dim=1) - order_seq = torch.full_like(order_pointers, -1) - batch = torch.arange(batch_size, device=order_pointers.device)[:, None] - order_seq[batch, order_pointers] = torch.arange(sequence_length, device=order_pointers.device)[None, :] - - return order_seq - - class PPDocLayoutV3ImageProcessor(BaseImageProcessor): r""" Constructs a PPDocLayoutV3 image processor. @@ -124,6 +105,23 @@ def __init__( self.image_std = image_std self.resample = resample + def _get_order_seqs(self, order_logits): + order_scores = torch.sigmoid(order_logits) + batch_size, sequence_length, _ = order_scores.shape + + order_votes = order_scores.triu(diagonal=1).sum(dim=1) + (1.0 - order_scores.transpose(1, 2)).tril( + diagonal=-1 + ).sum(dim=1) + + order_pointers = torch.argsort(order_votes, dim=1) + order_seq = torch.empty_like(order_pointers) + ranks = torch.arange(sequence_length, device=order_pointers.device, dtype=order_pointers.dtype).expand( + batch_size, -1 + ) + order_seq.scatter_(1, order_pointers, ranks) + + return order_seq + @filter_out_non_signature_kwargs() def preprocess( self, @@ -267,10 +265,12 @@ def post_process_object_detection( logits = outputs.logits order_logits = outputs.order_logits - order_seqs = get_order_seqs(order_logits) + order_seqs = self._get_order_seqs(order_logits) - cxcy, wh = torch.split(boxes, 2, dim=-1) - boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) + box_centers, box_dims = torch.split(boxes, 2, dim=-1) + top_left_coords = box_centers - 0.5 * box_dims + bottom_right_coords = box_centers + 0.5 * box_dims + boxes = torch.cat([top_left_coords, bottom_right_coords], dim=-1) if target_sizes is not None: if len(logits) != len(target_sizes): diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py index 6c31da2ff1c2..76f54f5997f1 100644 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py @@ -29,25 +29,6 @@ from ...utils.generic import TensorType -def get_order_seqs(order_logits): - order_scores = torch.sigmoid(order_logits) - batch_size, sequence_length, _ = order_scores.shape - - one = torch.ones((sequence_length, sequence_length), dtype=order_scores.dtype, device=order_scores.device) - upper = torch.triu(one, 1) - lower = torch.tril(one, -1) - - Q = order_scores * upper + (1.0 - order_scores.transpose(1, 2)) * lower - order_votes = Q.sum(dim=1) - - order_pointers = torch.argsort(order_votes, dim=1) - order_seq = torch.full_like(order_pointers, -1) - batch = torch.arange(batch_size, device=order_pointers.device)[:, None] - order_seq[batch, order_pointers] = torch.arange(sequence_length, device=order_pointers.device)[None, :] - - return order_seq - - class PPDocLayoutV3ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = [0, 0, 0] @@ -60,6 +41,23 @@ class PPDocLayoutV3ImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) + def _get_order_seqs(self, order_logits): + order_scores = torch.sigmoid(order_logits) + batch_size, sequence_length, _ = order_scores.shape + + order_votes = order_scores.triu(diagonal=1).sum(dim=1) + (1.0 - order_scores.transpose(1, 2)).tril( + diagonal=-1 + ).sum(dim=1) + + order_pointers = torch.argsort(order_votes, dim=1) + order_seq = torch.empty_like(order_pointers) + ranks = torch.arange(sequence_length, device=order_pointers.device, dtype=order_pointers.dtype).expand( + batch_size, -1 + ) + order_seq.scatter_(1, order_pointers, ranks) + + return order_seq + def _preprocess( self, images: list[torch.Tensor], @@ -101,7 +99,7 @@ def post_process_object_detection( target_sizes: Optional[Union[TensorType, list[tuple]]] = None, ): """ - Converts the raw output of [`DetrForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, + Converts the raw output of [`PPDocLayoutV3ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, bottom_right_x, bottom_right_y) format. Only supports PyTorch. Args: @@ -115,10 +113,12 @@ def post_process_object_detection( logits = outputs.logits order_logits = outputs.order_logits - order_seqs = get_order_seqs(order_logits) + order_seqs = self._get_order_seqs(order_logits) - cxcy, wh = torch.split(boxes, 2, dim=-1) - boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) + box_centers, box_dims = torch.split(boxes, 2, dim=-1) + top_left_coords = box_centers - 0.5 * box_dims + bottom_right_coords = box_centers + 0.5 * box_dims + boxes = torch.cat([top_left_coords, bottom_right_coords], dim=-1) if target_sizes is not None: if len(logits) != len(target_sizes): diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index 645379acc88d..7e4dc16bf5ad 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -48,15 +48,15 @@ def __init__(self, config): self.dropout = nn.Dropout(0.1) def forward(self, inputs): - B, N, _ = inputs.shape - proj = self.dense(inputs).reshape([B, N, 2, self.head_size]) - proj = self.dropout(proj) - qw, kw = proj[..., 0, :], proj[..., 1, :] + batch_size, sequence_length, _ = inputs.shape + query_key_projection = self.dense(inputs).reshape(batch_size, sequence_length, 2, self.head_size) + query_key_projection = self.dropout(query_key_projection) + queries = query_key_projection[:, :, 0, :] + keys = query_key_projection[:, :, 1, :] - logits = torch.einsum("bmd,bnd->bmn", qw, kw) / (self.head_size**0.5) # [B, N, N] - - lower = torch.tril(torch.ones([N, N], dtype=torch.float32, device=logits.device)) - logits = logits - lower.unsqueeze(0).to(logits.dtype) * 1e4 + logits = (queries @ keys.transpose(-2, -1)) / (self.head_size**0.5) + lower = torch.tril(torch.ones([sequence_length, sequence_length], dtype=logits.dtype, device=logits.device)) + logits = logits - lower.unsqueeze(0) * 1e4 return logits @@ -414,31 +414,20 @@ def forward(self, x): class BaseConv(nn.Module): def __init__( - self, - in_channels, - out_channels, - ksize, - stride, - groups=1, - bias=False, + self, in_channels: int, out_channels: int, kernel_size: int = 3, stride: int = 1, activation: str = "relu" ): super().__init__() - self.conv = nn.Conv2d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=ksize, - stride=stride, - padding=(ksize - 1) // 2, - groups=groups, - bias=bias, + self.convolution = nn.Conv2d( + in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=kernel_size // 2, bias=False ) - self.bn = nn.BatchNorm2d(out_channels) + self.normalization = nn.BatchNorm2d(out_channels) + self.activation = ACT2FN[activation] if activation is not None else nn.Identity() - def forward(self, x): - # use 'x * F.sigmoid(x)' replace 'silu' - x = self.bn(self.conv(x)) - y = x * F.sigmoid(x) - return y + def forward(self, input: Tensor) -> Tensor: + hidden_state = self.convolution(input) + hidden_state = self.normalization(hidden_state) + hidden_state = self.activation(hidden_state) + return hidden_state class MaskFeatFPN(nn.Module): @@ -458,7 +447,7 @@ def __init__( f"Got len(in_channels)={len(in_channels)} and len(fpn_strides)={len(fpn_strides)}." ) - reorder_index = np.argsort(fpn_strides, axis=0) + reorder_index = np.argsort(fpn_strides, axis=0).tolist() in_channels = [in_channels[i] for i in reorder_index] fpn_strides = [fpn_strides[i] for i in reorder_index] @@ -467,7 +456,7 @@ def __init__( self.dropout_ratio = dropout_ratio self.align_corners = align_corners if self.dropout_ratio > 0: - self.dropout = nn.Dropout2D(dropout_ratio) + self.dropout = nn.Dropout2d(dropout_ratio) self.scale_heads = nn.ModuleList() for i in range(len(fpn_strides)): @@ -475,13 +464,13 @@ def __init__( scale_head = [] for k in range(head_length): in_c = in_channels[i] if k == 0 else feat_channels - scale_head.append(nn.Sequential(BaseConv(in_c, feat_channels, 3, 1))) + scale_head.append(nn.Sequential(BaseConv(in_c, feat_channels, 3, 1, "silu"))) if fpn_strides[i] != fpn_strides[0]: scale_head.append(nn.Upsample(scale_factor=2, mode="bilinear", align_corners=align_corners)) self.scale_heads.append(nn.Sequential(*scale_head)) - self.output_conv = BaseConv(feat_channels, out_channels, 3, 1) + self.output_conv = BaseConv(feat_channels, out_channels, 3, 1, "silu") def forward(self, inputs): x = [inputs[i] for i in self.reorder_index] @@ -856,9 +845,9 @@ def __init__(self, config: PPDocLayoutV3Config): feat_channels=mask_feat_channels[0], out_channels=mask_feat_channels[1], ) - self.enc_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1) + self.enc_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1, "silu") self.enc_mask_output = nn.Sequential( - BaseConv(mask_feat_channels[1], mask_feat_channels[1], 3, 1), + BaseConv(mask_feat_channels[1], mask_feat_channels[1], 3, 1, "silu"), nn.Conv2d(in_channels=mask_feat_channels[1], out_channels=config.num_prototypes, kernel_size=1), ) @@ -1535,7 +1524,7 @@ def get_contrastive_denoising_training_group( return input_query_class, input_query_bbox, attn_mask, denoising_meta_values -def mask_to_box_coordinate(mask, dtype=torch.float32): +def mask_to_box_coordinate(mask, dtype): mask = mask.bool() height, width = mask.shape[-2:] @@ -1584,7 +1573,7 @@ def mask_to_box_coordinate(mask, dtype=torch.float32): @auto_docstring( custom_intro=""" - RT-DETR Model (consisting of a backbone and encoder-decoder) outputting raw hidden states without any head on top. + PP-DocLayoutV3 Model (consisting of a backbone and encoder-decoder) outputting raw hidden states without any head on top. """ ) class PPDocLayoutV3Model(PPDocLayoutV3PreTrainedModel): @@ -1598,6 +1587,7 @@ def __init__(self, config: PPDocLayoutV3Config): # Create encoder input projection layers # https://github.com/lyuwenyu/RT-DETR/blob/94f5e16708329d2f2716426868ec89aa774af016/PPDocLayoutV3_pytorch/src/zoo/PPDocLayoutV3/hybrid_encoder.py#L212 num_backbone_outs = len(intermediate_channel_sizes) + encoder_input_proj_list = [] for _ in range(num_backbone_outs): in_channels = intermediate_channel_sizes[_] @@ -1607,8 +1597,7 @@ def __init__(self, config: PPDocLayoutV3Config): nn.BatchNorm2d(config.encoder_hidden_dim), ) ) - - self.encoder_input_proj = nn.ModuleList(encoder_input_proj_list[1:]) # noqa + self.encoder_input_proj = nn.ModuleList(encoder_input_proj_list[1:]) # Create encoder self.encoder = PPDocLayoutV3HybridEncoder(config) @@ -1714,8 +1703,6 @@ def forward( pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.LongTensor] = None, encoder_outputs: Optional[torch.FloatTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - decoder_inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[list[dict]] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, @@ -1808,7 +1795,7 @@ def forward( # Lowest resolution feature maps are obtained via 3x3 stride 2 convolutions on the final stage if self.config.num_feature_levels > len(sources): _len_sources = len(sources) - sources.append(self.decoder_input_proj[_len_sources](encoder_outputs[0])[-1]) + sources.append(self.decoder_input_proj[_len_sources](encoder_outputs[0][-1])) for i in range(_len_sources + 1, self.config.num_feature_levels): sources.append(self.decoder_input_proj[i](encoder_outputs[0][-1])) @@ -1901,7 +1888,7 @@ def forward( target = torch.concat([denoising_class, target], 1) if self.mask_enhanced: - reference_points = mask_to_box_coordinate(enc_out_masks > 0) + reference_points = mask_to_box_coordinate(enc_out_masks > 0, dtype=reference_points_unact.dtype) reference_points_unact = inverse_sigmoid(reference_points) if denoising_bbox_unact is not None: @@ -2039,8 +2026,8 @@ class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): @auto_docstring( custom_intro=""" - RT-DETR Model (consisting of a backbone and encoder-decoder) outputting bounding boxes and logits to be further - decoded into scores and classes. + PP-DocLayoutV3 Model (consisting of a backbone and encoder-decoder) outputs bounding boxes and logits sorted according to reading order, + which are further decoded into scores and classes. """ ) class PPDocLayoutV3ForObjectDetection(PPDocLayoutV3PreTrainedModel): @@ -2071,8 +2058,6 @@ def forward( pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.LongTensor] = None, encoder_outputs: Optional[torch.FloatTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - decoder_inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[list[dict]] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, @@ -2146,8 +2131,6 @@ def forward( pixel_values, pixel_mask=pixel_mask, encoder_outputs=encoder_outputs, - inputs_embeds=inputs_embeds, - decoder_inputs_embeds=decoder_inputs_embeds, labels=labels, output_attentions=output_attentions, output_hidden_states=output_hidden_states, diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index a4b9f22cd6e0..d09392346598 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -52,6 +52,7 @@ from ...utils.backbone_utils import verify_backbone_config_arguments from ...utils.generic import TensorType from ..auto import CONFIG_MAPPING, AutoConfig +from ..resnet.modeling_resnet import ResNetConvLayer from ..rt_detr.modeling_rt_detr import ( RTDetrDecoder, RTDetrDecoderOutput, @@ -139,12 +140,20 @@ class PPDocLayoutV3Config(PreTrainedConfig): feed-forward modules. hidden_expansion (`float`, *optional*, defaults to 1.0): Expansion ratio to enlarge the dimension size of RepVGGBlock and CSPRepLayer. - mask_feat_channels (`list[int]`, *optional*, defaults to `[64, 64]): + mask_feat_channels (`list[int]`, *optional*, defaults to `[64, 64]`): The channels of the multi-level features for mask enhancement. x4_feat_dim (`int`, *optional*, defaults to 128): The dimension of the x4 feature map. d_model (`int`, *optional*, defaults to 256): Dimension of the layers exclude hybrid encoder. + num_prototypes (`int`, *optional*, defaults to 32): + Dimension of the layers exclude mask query head. + label_noise_ratio (`float`, *optional*, defaults to 0.4): + The fraction of denoising labels to which random noise should be added. + box_noise_scale (`float`, *optional*, defaults to 0.4): + Scale or magnitude of noise to be added to the bounding boxes. + mask_enhanced (`bool`, *optional*, defaults to `True`): + Whether to use enhanced masked attention. num_queries (`int`, *optional*, defaults to 300): Number of object queries. decoder_in_channels (`list`, *optional*, defaults to `[256, 256, 256]`): @@ -166,14 +175,8 @@ class PPDocLayoutV3Config(PreTrainedConfig): The dropout ratio for the attention probabilities. num_denoising (`int`, *optional*, defaults to 100): The total number of denoising tasks or queries to be used for contrastive denoising. - label_noise_ratio (`float`, *optional*, defaults to 0.5): - The fraction of denoising labels to which random noise should be added. - box_noise_scale (`float`, *optional*, defaults to 1.0): - Scale or magnitude of noise to be added to the bounding boxes. learn_initial_query (`bool`, *optional*, defaults to `False`): Indicates whether the initial query embeddings for the decoder should be learned during training - mask_enhanced (`bool`, *optional*, defaults to `True`): - Whether to use enhanced masked attention. anchor_image_size (`tuple[int, int]`, *optional*): Height and width of the input image used during evaluation to generate the bounding box anchors. If None, automatic generate anchor is applied. disable_custom_kernels (`bool`, *optional*, defaults to `True`): @@ -182,8 +185,6 @@ class PPDocLayoutV3Config(PreTrainedConfig): Whether the architecture has an encoder decoder structure. gp_head_size (`int`, *optional*, defaults to 64): The size of the global pointer head. - id2label (`dict[int, str]`, *optional*): - Mapping from class id to class name. Examples: @@ -200,7 +201,7 @@ class PPDocLayoutV3Config(PreTrainedConfig): >>> configuration = model.config ```""" - model_type = "pp_doclayout_v2" + model_type = "pp_doclayout_v3" sub_configs = {"backbone_config": AutoConfig} layer_types = ("basic", "bottleneck") @@ -261,8 +262,6 @@ def __init__( disable_custom_kernels=True, is_encoder_decoder=True, gp_head_size=64, - # label - id2label=None, **kwargs, ): self.initializer_range = initializer_range @@ -350,25 +349,6 @@ def __init__( super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs) -def get_order_seqs(order_logits): - order_scores = torch.sigmoid(order_logits) - batch_size, sequence_length, _ = order_scores.shape - - one = torch.ones((sequence_length, sequence_length), dtype=order_scores.dtype, device=order_scores.device) - upper = torch.triu(one, 1) - lower = torch.tril(one, -1) - - Q = order_scores * upper + (1.0 - order_scores.transpose(1, 2)) * lower - order_votes = Q.sum(dim=1) - - order_pointers = torch.argsort(order_votes, dim=1) - order_seq = torch.full_like(order_pointers, -1) - batch = torch.arange(batch_size, device=order_pointers.device)[:, None] - order_seq[batch, order_pointers] = torch.arange(sequence_length, device=order_pointers.device)[None, :] - - return order_seq - - class PPDocLayoutV3ImageProcessor(BaseImageProcessor): r""" Constructs a PPDocLayoutV3 image processor. @@ -435,6 +415,23 @@ def __init__( self.image_std = image_std self.resample = resample + def _get_order_seqs(self, order_logits): + order_scores = torch.sigmoid(order_logits) + batch_size, sequence_length, _ = order_scores.shape + + order_votes = order_scores.triu(diagonal=1).sum(dim=1) + (1.0 - order_scores.transpose(1, 2)).tril( + diagonal=-1 + ).sum(dim=1) + + order_pointers = torch.argsort(order_votes, dim=1) + order_seq = torch.empty_like(order_pointers) + ranks = torch.arange(sequence_length, device=order_pointers.device, dtype=order_pointers.dtype).expand( + batch_size, -1 + ) + order_seq.scatter_(1, order_pointers, ranks) + + return order_seq + @filter_out_non_signature_kwargs() def preprocess( self, @@ -578,10 +575,12 @@ def post_process_object_detection( logits = outputs.logits order_logits = outputs.order_logits - order_seqs = get_order_seqs(order_logits) + order_seqs = self._get_order_seqs(order_logits) - cxcy, wh = torch.split(boxes, 2, dim=-1) - boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) + box_centers, box_dims = torch.split(boxes, 2, dim=-1) + top_left_coords = box_centers - 0.5 * box_dims + bottom_right_coords = box_centers + 0.5 * box_dims + boxes = torch.cat([top_left_coords, bottom_right_coords], dim=-1) if target_sizes is not None: if len(logits) != len(target_sizes): @@ -633,6 +632,23 @@ class PPDocLayoutV3ImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) + def _get_order_seqs(self, order_logits): + order_scores = torch.sigmoid(order_logits) + batch_size, sequence_length, _ = order_scores.shape + + order_votes = order_scores.triu(diagonal=1).sum(dim=1) + (1.0 - order_scores.transpose(1, 2)).tril( + diagonal=-1 + ).sum(dim=1) + + order_pointers = torch.argsort(order_votes, dim=1) + order_seq = torch.empty_like(order_pointers) + ranks = torch.arange(sequence_length, device=order_pointers.device, dtype=order_pointers.dtype).expand( + batch_size, -1 + ) + order_seq.scatter_(1, order_pointers, ranks) + + return order_seq + def _preprocess( self, images: list[torch.Tensor], @@ -674,7 +690,7 @@ def post_process_object_detection( target_sizes: Optional[Union[TensorType, list[tuple]]] = None, ): """ - Converts the raw output of [`DetrForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, + Converts the raw output of [`PPDocLayoutV3ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, bottom_right_x, bottom_right_y) format. Only supports PyTorch. Args: @@ -688,10 +704,12 @@ def post_process_object_detection( logits = outputs.logits order_logits = outputs.order_logits - order_seqs = get_order_seqs(order_logits) + order_seqs = self._get_order_seqs(order_logits) - cxcy, wh = torch.split(boxes, 2, dim=-1) - boxes = torch.cat([cxcy - 0.5 * wh, cxcy + 0.5 * wh], dim=-1) + box_centers, box_dims = torch.split(boxes, 2, dim=-1) + top_left_coords = box_centers - 0.5 * box_dims + bottom_right_coords = box_centers + 0.5 * box_dims + boxes = torch.cat([top_left_coords, bottom_right_coords], dim=-1) if target_sizes is not None: if len(logits) != len(target_sizes): @@ -739,15 +757,15 @@ def __init__(self, config): self.dropout = nn.Dropout(0.1) def forward(self, inputs): - B, N, _ = inputs.shape - proj = self.dense(inputs).reshape([B, N, 2, self.head_size]) - proj = self.dropout(proj) - qw, kw = proj[..., 0, :], proj[..., 1, :] + batch_size, sequence_length, _ = inputs.shape + query_key_projection = self.dense(inputs).reshape(batch_size, sequence_length, 2, self.head_size) + query_key_projection = self.dropout(query_key_projection) + queries = query_key_projection[:, :, 0, :] + keys = query_key_projection[:, :, 1, :] - logits = torch.einsum("bmd,bnd->bmn", qw, kw) / (self.head_size**0.5) # [B, N, N] - - lower = torch.tril(torch.ones([N, N], dtype=torch.float32, device=logits.device)) - logits = logits - lower.unsqueeze(0).to(logits.dtype) * 1e4 + logits = (queries @ keys.transpose(-2, -1)) / (self.head_size**0.5) + lower = torch.tril(torch.ones([sequence_length, sequence_length], dtype=logits.dtype, device=logits.device)) + logits = logits - lower.unsqueeze(0) * 1e4 return logits @@ -814,7 +832,7 @@ def _init_weights(self, module): init.zeros_(module.weight.data[module.padding_idx]) -def mask_to_box_coordinate(mask, dtype=torch.float32): +def mask_to_box_coordinate(mask, dtype): mask = mask.bool() height, width = mask.shape[-2:] @@ -938,33 +956,8 @@ class PPDocLayoutV3MLPPredictionHead(RTDetrMLPPredictionHead): pass -class BaseConv(nn.Module): - def __init__( - self, - in_channels, - out_channels, - ksize, - stride, - groups=1, - bias=False, - ): - super().__init__() - self.conv = nn.Conv2d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=ksize, - stride=stride, - padding=(ksize - 1) // 2, - groups=groups, - bias=bias, - ) - self.bn = nn.BatchNorm2d(out_channels) - - def forward(self, x): - # use 'x * F.sigmoid(x)' replace 'silu' - x = self.bn(self.conv(x)) - y = x * F.sigmoid(x) - return y +class BaseConv(ResNetConvLayer): + pass class MaskFeatFPN(nn.Module): @@ -984,7 +977,7 @@ def __init__( f"Got len(in_channels)={len(in_channels)} and len(fpn_strides)={len(fpn_strides)}." ) - reorder_index = np.argsort(fpn_strides, axis=0) + reorder_index = np.argsort(fpn_strides, axis=0).tolist() in_channels = [in_channels[i] for i in reorder_index] fpn_strides = [fpn_strides[i] for i in reorder_index] @@ -993,7 +986,7 @@ def __init__( self.dropout_ratio = dropout_ratio self.align_corners = align_corners if self.dropout_ratio > 0: - self.dropout = nn.Dropout2D(dropout_ratio) + self.dropout = nn.Dropout2d(dropout_ratio) self.scale_heads = nn.ModuleList() for i in range(len(fpn_strides)): @@ -1001,13 +994,13 @@ def __init__( scale_head = [] for k in range(head_length): in_c = in_channels[i] if k == 0 else feat_channels - scale_head.append(nn.Sequential(BaseConv(in_c, feat_channels, 3, 1))) + scale_head.append(nn.Sequential(BaseConv(in_c, feat_channels, 3, 1, "silu"))) if fpn_strides[i] != fpn_strides[0]: scale_head.append(nn.Upsample(scale_factor=2, mode="bilinear", align_corners=align_corners)) self.scale_heads.append(nn.Sequential(*scale_head)) - self.output_conv = BaseConv(feat_channels, out_channels, 3, 1) + self.output_conv = BaseConv(feat_channels, out_channels, 3, 1, "silu") def forward(self, inputs): x = [inputs[i] for i in self.reorder_index] @@ -1036,9 +1029,9 @@ def __init__(self, config: PPDocLayoutV3Config): feat_channels=mask_feat_channels[0], out_channels=mask_feat_channels[1], ) - self.enc_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1) + self.enc_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1, "silu") self.enc_mask_output = nn.Sequential( - BaseConv(mask_feat_channels[1], mask_feat_channels[1], 3, 1), + BaseConv(mask_feat_channels[1], mask_feat_channels[1], 3, 1, "silu"), nn.Conv2d(in_channels=mask_feat_channels[1], out_channels=config.num_prototypes, kernel_size=1), ) @@ -1349,11 +1342,17 @@ def forward( ) +@auto_docstring( + custom_intro=""" + PP-DocLayoutV3 Model (consisting of a backbone and encoder-decoder) outputting raw hidden states without any head on top. + """ +) class PPDocLayoutV3Model(RTDetrModel): def __init__(self, config: PPDocLayoutV3Config): super().__init__(config) - self.encoder_input_proj = nn.ModuleList(encoder_input_proj_list[1:]) # noqa + encoder_input_proj_list = [] + self.encoder_input_proj = nn.ModuleList(encoder_input_proj_list[1:]) self.dec_order_head = nn.Linear(config.d_model, config.d_model) self.dec_global_pointer = GlobalPointer(config) @@ -1367,13 +1366,12 @@ def __init__(self, config: PPDocLayoutV3Config): config, config.d_model, config.d_model, config.num_prototypes, num_layers=3 ) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.LongTensor] = None, encoder_outputs: Optional[torch.FloatTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - decoder_inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[list[dict]] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, @@ -1466,7 +1464,7 @@ def forward( # Lowest resolution feature maps are obtained via 3x3 stride 2 convolutions on the final stage if self.config.num_feature_levels > len(sources): _len_sources = len(sources) - sources.append(self.decoder_input_proj[_len_sources](encoder_outputs[0])[-1]) + sources.append(self.decoder_input_proj[_len_sources](encoder_outputs[0][-1])) for i in range(_len_sources + 1, self.config.num_feature_levels): sources.append(self.decoder_input_proj[i](encoder_outputs[0][-1])) @@ -1559,7 +1557,7 @@ def forward( target = torch.concat([denoising_class, target], 1) if self.mask_enhanced: - reference_points = mask_to_box_coordinate(enc_out_masks > 0) + reference_points = mask_to_box_coordinate(enc_out_masks > 0, dtype=reference_points_unact.dtype) reference_points_unact = inverse_sigmoid(reference_points) if denoising_bbox_unact is not None: @@ -1670,6 +1668,7 @@ class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): denoising_meta_values (`dict`): Extra dictionary for the denoising related values """ + logits: Optional[torch.FloatTensor] = None pred_boxes: Optional[torch.FloatTensor] = None order_logits: Optional[torch.FloatTensor] = None @@ -1694,6 +1693,12 @@ class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): denoising_meta_values: Optional[dict] = None +@auto_docstring( + custom_intro=""" + PP-DocLayoutV3 Model (consisting of a backbone and encoder-decoder) outputs bounding boxes and logits sorted according to reading order, + which are further decoded into scores and classes. + """ +) class PPDocLayoutV3ForObjectDetection(RTDetrForObjectDetection, PPDocLayoutV3PreTrainedModel): _keys_to_ignore_on_load_missing = ["num_batches_tracked", "rel_pos_y_bias", "rel_pos_x_bias"] _tied_weights_keys = { @@ -1719,8 +1724,6 @@ def forward( pixel_values: torch.FloatTensor, pixel_mask: Optional[torch.LongTensor] = None, encoder_outputs: Optional[torch.FloatTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - decoder_inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[list[dict]] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, @@ -1794,8 +1797,6 @@ def forward( pixel_values, pixel_mask=pixel_mask, encoder_outputs=encoder_outputs, - inputs_embeds=inputs_embeds, - decoder_inputs_embeds=decoder_inputs_embeds, labels=labels, output_attentions=output_attentions, output_hidden_states=output_hidden_states, diff --git a/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py index 294d203eb5ef..afd4399e3d1e 100644 --- a/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py +++ b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py @@ -16,7 +16,6 @@ import inspect import math -import tempfile import unittest from parameterized import parameterized @@ -196,16 +195,6 @@ def prepare_config_and_inputs_for_common(self): inputs_dict = {"pixel_values": pixel_values} return config, inputs_dict - def create_and_check_pp_doclayout_v2_object_detection_head_model(self, config, pixel_values): - model = PPDocLayoutV3ForObjectDetection(config=config) - model.to(torch_device) - model.eval() - - result = model(pixel_values) - - self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_queries, self.num_labels)) - self.parent.assertEqual(result.pred_boxes.shape, (self.batch_size, self.num_queries, 4)) - @require_torch class PPDocLayoutV3ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): @@ -227,10 +216,6 @@ def setUp(self): def test_config(self): self.config_tester.run_common_tests() - def test_pp_doclayout_v2_object_detection_head_model(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_pp_doclayout_v2_object_detection_head_model(*config_and_inputs) - @unittest.skip(reason="PPDocLayoutV3 has tied weights.") def test_load_save_without_tied_weights(self): pass @@ -299,46 +284,6 @@ def test_inference_with_different_dtypes(self, dtype_str): with torch.no_grad(): _ = model(**self._prepare_for_class(inputs_dict, model_class)) - @parameterized.expand(["float32", "float16", "bfloat16"]) - @require_torch_accelerator - @slow - def test_inference_equivalence_for_static_and_dynamic_anchors(self, dtype_str): - dtype = { - "float32": torch.float32, - "float16": torch.float16, - "bfloat16": torch.bfloat16, - }[dtype_str] - - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - h, w = inputs_dict["pixel_values"].shape[-2:] - - # convert inputs to the desired dtype - for key, tensor in inputs_dict.items(): - if tensor.dtype == torch.float32: - inputs_dict[key] = tensor.to(dtype) - - for model_class in self.all_model_classes: - with tempfile.TemporaryDirectory() as tmpdirname: - model_class(config).save_pretrained(tmpdirname) - model_static = model_class.from_pretrained( - tmpdirname, anchor_image_size=[h, w], device_map=torch_device, dtype=dtype - ).eval() - model_dynamic = model_class.from_pretrained( - tmpdirname, anchor_image_size=None, device_map=torch_device, dtype=dtype - ).eval() - - self.assertIsNotNone(model_static.config.anchor_image_size) - self.assertIsNone(model_dynamic.config.anchor_image_size) - - with torch.no_grad(): - outputs_static = model_static(**self._prepare_for_class(inputs_dict, model_class)) - outputs_dynamic = model_dynamic(**self._prepare_for_class(inputs_dict, model_class)) - - self.assertTrue( - torch.allclose(outputs_static.logits, outputs_dynamic.logits, rtol=1e-4, atol=1e-4), - f"Max diff: {(outputs_static.logits - outputs_dynamic.logits).abs().max()}", - ) - def test_hidden_states_output(self): def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(config) From 1eef91afffc6d07b68c16c962234bc683f33e854 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Tue, 13 Jan 2026 20:35:20 +0800 Subject: [PATCH 05/21] add model_doc --- docs/source/en/model_doc/pp_doclayout_v3.md | 47 +++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 docs/source/en/model_doc/pp_doclayout_v3.md diff --git a/docs/source/en/model_doc/pp_doclayout_v3.md b/docs/source/en/model_doc/pp_doclayout_v3.md new file mode 100644 index 000000000000..12e51cc4510a --- /dev/null +++ b/docs/source/en/model_doc/pp_doclayout_v3.md @@ -0,0 +1,47 @@ + +*This model was released on 2026-x-x and added to Hugging Face Transformers on 2026-x-x.* + +# PP-DocLayoutV3 + +
+PyTorch +
+ +## Overview + +TBD. + +## PPDocLayoutV3ForObjectDetection + +[[autodoc]] PPDocLayoutV3ForObjectDetection + - forward + +## PPDocLayoutV3Model + +[[autodoc]] PPDocLayoutV3Model + +## PPDocLayoutV3Config + +[[autodoc]] PPDocLayoutV3Config + +## PPDocLayoutV3ImageProcessor + +[[autodoc]] PPDocLayoutV3ImageProcessor + +## PPDocLayoutV3ImageProcessorFast + +[[autodoc]] PPDocLayoutV3ImageProcessorFast From 8da28dbf6f810817f199477128c3ac3722ec2e29 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Wed, 14 Jan 2026 18:02:00 +0800 Subject: [PATCH 06/21] update --- docs/source/en/model_doc/pp_doclayout_v3.md | 102 +++++++++++++++++- .../configuration_pp_doclayout_v3.py | 4 +- .../image_processing_pp_doclayout_v3.py | 19 +++- .../image_processing_pp_doclayout_v3_fast.py | 18 +++- .../modeling_pp_doclayout_v3.py | 8 +- .../modular_pp_doclayout_v3.py | 41 ++++++- .../test_modeling_pp_doclayout_v3.py | 7 +- 7 files changed, 178 insertions(+), 21 deletions(-) diff --git a/docs/source/en/model_doc/pp_doclayout_v3.md b/docs/source/en/model_doc/pp_doclayout_v3.md index 12e51cc4510a..0d995b1f7108 100644 --- a/docs/source/en/model_doc/pp_doclayout_v3.md +++ b/docs/source/en/model_doc/pp_doclayout_v3.md @@ -1,4 +1,4 @@ - -*This model was released on 2026-x-x and added to Hugging Face Transformers on 2026-x-x.* +*This model was released on 2026-01-22 and added to Hugging Face Transformers on 2026-01-22.* # PP-DocLayoutV3 @@ -138,10 +138,6 @@ for result in results: [[autodoc]] PPDocLayoutV3Config -## PPDocLayoutV3ImageProcessor - -[[autodoc]] PPDocLayoutV3ImageProcessor - ## PPDocLayoutV3ImageProcessorFast [[autodoc]] PPDocLayoutV3ImageProcessorFast diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index c0ee68933f72..186e4880d715 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -163,7 +163,7 @@ ("pixio", ("BitImageProcessor", "BitImageProcessorFast")), ("pixtral", ("PixtralImageProcessor", "PixtralImageProcessorFast")), ("poolformer", ("PoolFormerImageProcessor", "PoolFormerImageProcessorFast")), - ("pp_doclayout_v3", ("PPDocLayoutV3ImageProcessor", "PPDocLayoutV3ImageProcessorFast")), + ("pp_doclayout_v3", (None, "PPDocLayoutV3ImageProcessorFast")), ("prompt_depth_anything", ("PromptDepthAnythingImageProcessor", "PromptDepthAnythingImageProcessorFast")), ("pvt", ("PvtImageProcessor", "PvtImageProcessorFast")), ("pvt_v2", ("PvtImageProcessor", "PvtImageProcessorFast")), diff --git a/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py index 8ec6092f844f..95de9840e7cf 100644 --- a/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py @@ -30,7 +30,7 @@ class PPDocLayoutV3Config(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`PP-DocLayoutV3`]. It is used to instantiate a - RT-DETR model according to the specified arguments, defining the model architecture. Instantiating a configuration + PP-DocLayoutV3 model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the PP-DocLayoutV3 [PaddlePaddle/PP-DocLayoutV3_safetensors](https://huggingface.co/PaddlePaddle/PP-DocLayoutV3_safetensors) architecture. @@ -142,7 +142,8 @@ class PPDocLayoutV3Config(PreTrainedConfig): Whether the architecture has an encoder decoder structure. gp_head_size (`int`, *optional*, defaults to 64): The size of the global pointer head. - + gp_dropout_value (`float`, *optional*, defaults to 0.1): + The dropout probability in the global pointer head. Examples: ```python @@ -219,6 +220,7 @@ def __init__( disable_custom_kernels=True, is_encoder_decoder=True, gp_head_size=64, + gp_dropout_value=0.1, **kwargs, ): self.initializer_range = initializer_range @@ -302,6 +304,7 @@ def __init__( self.anchor_image_size = list(anchor_image_size) if anchor_image_size is not None else None self.disable_custom_kernels = disable_custom_kernels self.gp_head_size = gp_head_size + self.gp_dropout_value = gp_dropout_value super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs) diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py deleted file mode 100644 index 7d4fe9677c84..000000000000 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3.py +++ /dev/null @@ -1,328 +0,0 @@ -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# This file was automatically generated from src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py. -# Do NOT edit this file manually as any edits will be overwritten by the generation of -# the file from the modular. If any change should be done, please apply the change to the -# modular_pp_doclayout_v3.py file directly. One of our CI enforces this. -# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -# Copyright 2026 The PaddlePaddle Team and The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Union - -import torch - -from ...feature_extraction_utils import BatchFeature -from ...image_processing_utils import BaseImageProcessor, get_size_dict -from ...image_transforms import resize, to_channel_dimension_format -from ...image_utils import ( - ChannelDimension, - ImageInput, - PILImageResampling, - infer_channel_dimension_format, - make_flat_list_of_images, - to_numpy_array, - valid_images, - validate_preprocess_arguments, -) -from ...utils import filter_out_non_signature_kwargs, requires_backends -from ...utils.generic import TensorType - - -class PPDocLayoutV3ImageProcessor(BaseImageProcessor): - r""" - Constructs a PPDocLayoutV3 image processor. - - Args: - do_resize (`bool`, *optional*, defaults to `True`): - Controls whether to resize the image's (height, width) dimensions to the specified `size`. Can be - overridden by the `do_resize` parameter in the `preprocess` method. - size (`dict[str, int]` *optional*, defaults to `{"height": 640, "width": 640}`): - Size of the image's `(height, width)` dimensions after resizing. Can be overridden by the `size` parameter - in the `preprocess` method. Available options are: - - `{"height": int, "width": int}`: The image will be resized to the exact size `(height, width)`. - Do NOT keep the aspect ratio. - - `{"shortest_edge": int, "longest_edge": int}`: The image will be resized to a maximum size respecting - the aspect ratio and keeping the shortest edge less or equal to `shortest_edge` and the longest edge - less or equal to `longest_edge`. - - `{"max_height": int, "max_width": int}`: The image will be resized to the maximum size respecting the - aspect ratio and keeping the height less or equal to `max_height` and the width less or equal to - `max_width`. - resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): - Resampling filter to use if resizing the image. - do_rescale (`bool`, *optional*, defaults to `True`): - Controls whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the - `do_rescale` parameter in the `preprocess` method. - rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): - Scale factor to use if rescaling the image. Can be overridden by the `rescale_factor` parameter in the - `preprocess` method. - Controls whether to normalize the image. Can be overridden by the `do_normalize` parameter in the - `preprocess` method. - do_normalize (`bool`, *optional*, defaults to `True`): - Whether to normalize the image. - image_mean (`float` or `list[float]`, *optional*, defaults to `[0, 0, 0]`): - Mean values to use when normalizing the image. Can be a single value or a list of values, one for each - channel. Can be overridden by the `image_mean` parameter in the `preprocess` method. - image_std (`float` or `list[float]`, *optional*, defaults to `[1, 1, 1]`): - Standard deviation values to use when normalizing the image. Can be a single value or a list of values, one - for each channel. Can be overridden by the `image_std` parameter in the `preprocess` method. - """ - - model_input_names = ["pixel_values"] - - def __init__( - self, - do_resize: bool = True, - size: Optional[dict[str, int]] = None, - resample: Optional[PILImageResampling] = PILImageResampling.BICUBIC, - do_rescale: bool = True, - rescale_factor: Union[int, float] = 1 / 255, - do_normalize: bool = True, - image_mean: Optional[Union[float, list[float]]] = [0, 0, 0], - image_std: Optional[Union[float, list[float]]] = [1, 1, 1], - **kwargs, - ) -> None: - size = size if size is not None else {"height": 800, "width": 800} - size = get_size_dict(size, default_to_square=False) - - super().__init__(**kwargs) - self.do_resize = do_resize - self.size = size - self.do_rescale = do_rescale - self.rescale_factor = rescale_factor - self.do_normalize = do_normalize - self.image_mean = image_mean - self.image_std = image_std - self.resample = resample - - def _get_order_seqs(self, order_logits): - """ - Computes the order sequences for a batch of inputs based on logits. - - This function takes in the order logits, calculates order scores using a sigmoid activation, - and determines the order sequences by ranking the votes derived from the scores. - - Args: - order_logits (`torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): - Stacked order logits. - - Returns: - torch.Tensor: A tensor of shape `(batch_size, num_queries)`: - Containing the computed order sequences for each input in the batch. Each row represents the ranked order of elements for the corresponding input in the batch. - """ - order_scores = torch.sigmoid(order_logits) - batch_size, sequence_length, _ = order_scores.shape - - order_votes = order_scores.triu(diagonal=1).sum(dim=1) + (1.0 - order_scores.transpose(1, 2)).tril( - diagonal=-1 - ).sum(dim=1) - - order_pointers = torch.argsort(order_votes, dim=1) - order_seq = torch.empty_like(order_pointers) - ranks = torch.arange(sequence_length, device=order_pointers.device, dtype=order_pointers.dtype).expand( - batch_size, -1 - ) - order_seq.scatter_(1, order_pointers, ranks) - - return order_seq - - @filter_out_non_signature_kwargs() - def preprocess( - self, - images: ImageInput, - do_resize: Optional[bool] = None, - size: Optional[dict[str, int]] = None, - resample: Optional[PILImageResampling] = None, - do_rescale: Optional[bool] = None, - rescale_factor: Optional[Union[int, float]] = None, - do_normalize: Optional[bool] = None, - image_mean: Optional[Union[float, list[float]]] = None, - image_std: Optional[Union[float, list[float]]] = None, - return_tensors: Optional[Union[TensorType, str]] = None, - data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, - input_data_format: Optional[Union[str, ChannelDimension]] = None, - ) -> BatchFeature: - """ - Preprocess an image or a batch of images so that it can be used by the model. - - Args: - images (`ImageInput`): - Image or batch of images to preprocess. Expects a single or batch of images with pixel values ranging - from 0 to 255. If passing in images with pixel values between 0 and 1, set `do_rescale=False`. - do_resize (`bool`, *optional*, defaults to self.do_resize): - Whether to resize the image. - size (`dict[str, int]`, *optional*, defaults to self.size): - Size of the image's `(height, width)` dimensions after resizing. Available options are: - - `{"height": int, "width": int}`: The image will be resized to the exact size `(height, width)`. - Do NOT keep the aspect ratio. - - `{"shortest_edge": int, "longest_edge": int}`: The image will be resized to a maximum size respecting - the aspect ratio and keeping the shortest edge less or equal to `shortest_edge` and the longest edge - less or equal to `longest_edge`. - - `{"max_height": int, "max_width": int}`: The image will be resized to the maximum size respecting the - aspect ratio and keeping the height less or equal to `max_height` and the width less or equal to - `max_width`. - resample (`PILImageResampling`, *optional*, defaults to self.resample): - Resampling filter to use when resizing the image. - do_rescale (`bool`, *optional*, defaults to self.do_rescale): - Whether to rescale the image. - rescale_factor (`float`, *optional*, defaults to self.rescale_factor): - Rescale factor to use when rescaling the image. - do_normalize (`bool`, *optional*, defaults to self.do_normalize): - Whether to normalize the image. - image_mean (`float` or `list[float]`, *optional*, defaults to self.image_mean): - Mean to use when normalizing the image. - image_std (`float` or `list[float]`, *optional*, defaults to self.image_std): - Standard deviation to use when normalizing the image. - return_tensors (`str` or `TensorType`, *optional*, defaults to self.return_tensors): - Type of tensors to return. If `None`, will return the list of images. - data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): - The channel dimension format for the output image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Unset: Use the channel dimension format of the input image. - input_data_format (`ChannelDimension` or `str`, *optional*): - The channel dimension format for the input image. If unset, the channel dimension format is inferred - from the input image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - """ - do_resize = self.do_resize if do_resize is None else do_resize - size = self.size if size is None else size - size = get_size_dict(size=size, default_to_square=True) - resample = self.resample if resample is None else resample - do_rescale = self.do_rescale if do_rescale is None else do_rescale - rescale_factor = self.rescale_factor if rescale_factor is None else rescale_factor - do_normalize = self.do_normalize if do_normalize is None else do_normalize - image_mean = self.image_mean if image_mean is None else image_mean - image_std = self.image_std if image_std is None else image_std - - validate_preprocess_arguments( - do_rescale=do_rescale, - rescale_factor=rescale_factor, - do_normalize=do_normalize, - image_mean=image_mean, - image_std=image_std, - do_resize=do_resize, - size=size, - resample=resample, - ) - - images = make_flat_list_of_images(images) - if not valid_images(images): - raise ValueError("Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, or torch.Tensor") - - # All transformations expect numpy arrays - images = [to_numpy_array(image) for image in images] - - if input_data_format is None: - input_data_format = infer_channel_dimension_format(images[0]) - - # transformations - if do_resize: - images = [ - resize( - image, size=(size["height"], size["width"]), resample=resample, input_data_format=input_data_format - ) - for image in images - ] - - if do_rescale: - images = [self.rescale(image, rescale_factor, input_data_format=input_data_format) for image in images] - - if do_normalize: - images = [ - self.normalize(image, image_mean, image_std, input_data_format=input_data_format) for image in images - ] - - images = [ - to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images - ] - encoded_inputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) - - return encoded_inputs - - def post_process_object_detection( - self, - outputs, - threshold: float = 0.5, - target_sizes: Optional[Union[TensorType, list[tuple]]] = None, - ): - """ - Converts the raw output of [`PPDocLayoutV3ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, - bottom_right_x, bottom_right_y) format. Only supports PyTorch. - - Args: - outputs ([`PPDocLayoutV3ObjectDetectionOutput`]): - Raw outputs of the model. - threshold (`float`, *optional*, defaults to 0.5): - Score threshold to keep object detection predictions. - target_sizes (`torch.Tensor` or `list[tuple[int, int]]`, *optional*): - Tensor of shape `(batch_size, 2)` or list of tuples (`tuple[int, int]`) containing the target size - `(height, width)` of each image in the batch. If unset, predictions will not be resized. - - Returns: - `list[Dict]`: An ordered list of dictionaries, each dictionary containing the scores, labels and boxes for an image - in the batch as predicted by the model. - """ - requires_backends(self, ["torch"]) - boxes = outputs.pred_boxes - logits = outputs.logits - order_logits = outputs.order_logits - - order_seqs = self._get_order_seqs(order_logits) - - box_centers, box_dims = torch.split(boxes, 2, dim=-1) - top_left_coords = box_centers - 0.5 * box_dims - bottom_right_coords = box_centers + 0.5 * box_dims - boxes = torch.cat([top_left_coords, bottom_right_coords], dim=-1) - - if target_sizes is not None: - if len(logits) != len(target_sizes): - raise ValueError( - "Make sure that you pass in as many target sizes as the batch dimension of the logits" - ) - if isinstance(target_sizes, list): - img_h, img_w = torch.as_tensor(target_sizes).unbind(1) - else: - img_h, img_w = target_sizes.unbind(1) - scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) - boxes = boxes * scale_fct[:, None, :] - - num_top_queries = logits.shape[1] - num_classes = logits.shape[2] - - scores = torch.nn.functional.sigmoid(logits) - scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) - labels = index % num_classes - index = index // num_classes - boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) - order_seqs = order_seqs.gather(dim=1, index=index) - - results = [] - for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): - order_seq = order_seq[score >= threshold] - order_seq, indices = torch.sort(order_seq) - results.append( - { - "scores": score[score >= threshold][indices], - "labels": label[score >= threshold][indices], - "boxes": box[score >= threshold][indices], - "order_seq": order_seq, - } - ) - - return results - - -__all__ = ["PPDocLayoutV3ImageProcessor"] diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py index 44232f857f78..49ed5a2f04c5 100644 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py @@ -21,15 +21,14 @@ from typing import Optional, Union import torch -from torchvision.transforms.v2.functional import InterpolationMode -from ...feature_extraction_utils import BatchFeature -from ...image_processing_utils_fast import BaseImageProcessorFast, SizeDict +from ...image_processing_utils_fast import BaseImageProcessorFast from ...image_utils import PILImageResampling -from ...utils import requires_backends +from ...utils import auto_docstring, requires_backends from ...utils.generic import TensorType +@auto_docstring class PPDocLayoutV3ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = [0, 0, 0] @@ -73,40 +72,6 @@ def _get_order_seqs(self, order_logits): return order_seq - def _preprocess( - self, - images: list[torch.Tensor], - do_resize: bool, - size: SizeDict, - interpolation: Optional[InterpolationMode], - do_rescale: bool, - rescale_factor: float, - do_normalize: bool, - image_mean: Optional[Union[float, list[float]]], - image_std: Optional[Union[float, list[float]]], - return_tensors: Optional[Union[str, TensorType]], - **kwargs, - ) -> BatchFeature: - """ - Preprocess an image or a batch of images so that it can be used by the model. - """ - data = {} - processed_images = [] - for image in images: - if do_resize: - image = self.resize(image, size=size, interpolation=interpolation) - # Fused rescale and normalize - image = self.rescale_and_normalize(image, do_rescale, rescale_factor, do_normalize, image_mean, image_std) - - processed_images.append(image) - - images = processed_images - - data.update({"pixel_values": torch.stack(images, dim=0)}) - encoded_inputs = BatchFeature(data, tensor_type=return_tensors) - - return encoded_inputs - def post_process_object_detection( self, outputs, diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index b10752eacb83..e6565a13c850 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -45,14 +45,13 @@ def __init__(self, config): super().__init__() self.head_size = config.gp_head_size self.dense = nn.Linear(config.d_model, self.head_size * 2) - self.dropout = nn.Dropout(0.1) + self.dropout = nn.Dropout(config.gp_dropout_value) def forward(self, inputs): batch_size, sequence_length, _ = inputs.shape query_key_projection = self.dense(inputs).reshape(batch_size, sequence_length, 2, self.head_size) query_key_projection = self.dropout(query_key_projection) - queries = query_key_projection[:, :, 0, :] - keys = query_key_projection[:, :, 1, :] + queries, keys = torch.unbind(query_key_projection, dim=2) logits = (queries @ keys.transpose(-2, -1)) / (self.head_size**0.5) mask = torch.tril(torch.ones(sequence_length, sequence_length, device=logits.device)).bool() @@ -239,7 +238,21 @@ class PPDocLayoutV3PreTrainedModel(PreTrainedModel): @torch.no_grad() def _init_weights(self, module): - if isinstance(module, PPDocLayoutV3MultiscaleDeformableAttention): + """Initialize the weights""" + if isinstance(module, PPDocLayoutV3ForObjectDetection): + if module.model.decoder.class_embed is not None: + layer = module.model.decoder.class_embed + prior_prob = self.config.initializer_bias_prior_prob or 1 / (self.config.num_labels + 1) + bias = float(-math.log((1 - prior_prob) / prior_prob)) + init.xavier_uniform_(layer.weight) + init.constant_(layer.bias, bias) + + if module.model.decoder.bbox_embed is not None: + layer = module.model.decoder.bbox_embed + init.constant_(layer.layers[-1].weight, 0) + init.constant_(layer.layers[-1].bias, 0) + + elif isinstance(module, PPDocLayoutV3MultiscaleDeformableAttention): init.constant_(module.sampling_offsets.weight, 0.0) default_dtype = torch.get_default_dtype() thetas = torch.arange(module.n_heads, dtype=torch.int64).to(default_dtype) * ( @@ -442,10 +455,11 @@ def __init__( ): super().__init__() - assert len(in_channels) == len(fpn_strides), ( - f"Error: The lengths of 'in_channels' and 'fpn_strides' must be equal. " - f"Got len(in_channels)={len(in_channels)} and len(fpn_strides)={len(fpn_strides)}." - ) + if len(in_channels) != len(fpn_strides): + raise ValueError( + f"The lengths of 'in_channels' and 'fpn_strides' must be equal. " + f"Got len(in_channels)={len(in_channels)} and len(fpn_strides)={len(fpn_strides)}." + ) reorder_index = np.argsort(fpn_strides, axis=0).tolist() in_channels = [in_channels[i] for i in reorder_index] @@ -780,11 +794,9 @@ def forward(self, src, src_mask=None, pos_embed=None, output_attentions: bool = class PPDocLayoutV3HybridEncoder(nn.Module): """ - Decoder consisting of a projection layer, a set of `PPDocLayoutV3Encoder`, a top-down Feature Pyramid Network - (FPN) and a bottom-up Path Aggregation Network (PAN). More details on the paper: https://huggingface.co/papers/2304.08069 - - Args: - config: PPDocLayoutV3Config + Main difference to `RTDetrHybridEncoder`: + 1. Mask Feature Head: Added `MaskFeatFPN` module (`self.mask_feat_head`) for document - specific mask feature generation. + 2. Extra Conv Layers: Introduced `self.encoder_mask_lateral` and `self.encoder_mask_output` for mask feature processing and output. """ def __init__(self, config: PPDocLayoutV3Config): @@ -845,8 +857,8 @@ def __init__(self, config: PPDocLayoutV3Config): feat_channels=mask_feat_channels[0], out_channels=mask_feat_channels[1], ) - self.enc_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1, "silu") - self.enc_mask_output = nn.Sequential( + self.encoder_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1, "silu") + self.encoder_mask_output = nn.Sequential( BaseConv(mask_feat_channels[1], mask_feat_channels[1], 3, 1, "silu"), nn.Conv2d(in_channels=mask_feat_channels[1], out_channels=config.num_prototypes, kernel_size=1), ) @@ -982,8 +994,8 @@ def forward( mask_feat = self.mask_feat_head(pan_feature_maps) mask_feat = F.interpolate(mask_feat, scale_factor=2, mode="bilinear", align_corners=False) - mask_feat += self.enc_mask_lateral(x4_feat[0]) - mask_feat = self.enc_mask_output(mask_feat) + mask_feat += self.encoder_mask_lateral(x4_feat[0]) + mask_feat = self.encoder_mask_output(mask_feat) if not return_dict: return tuple(v for v in [pan_feature_maps, encoder_states, all_attentions, mask_feat] if v is not None) @@ -1114,6 +1126,11 @@ def inverse_sigmoid(x, eps=1e-5): class PPDocLayoutV3Decoder(PPDocLayoutV3PreTrainedModel): + """ + Main difference to `RTDetrDecoder`: + A new mask generation process is introduced at each decoder layer. + """ + def __init__(self, config: PPDocLayoutV3Config): super().__init__(config) @@ -1146,7 +1163,7 @@ def forward( order_head=None, global_pointer=None, mask_query_head=None, - dec_norm=None, + norm=None, mask_feat=None, output_attentions=None, output_hidden_states=None, @@ -1240,7 +1257,7 @@ def forward( ) # get_pred_class_order_and_mask - out_query = dec_norm(hidden_states) + out_query = norm(hidden_states) mask_query_embed = mask_query_head(out_query) batch_size, mask_dim, _ = mask_query_embed.shape _, _, mask_h, mask_w = mask_feat.shape @@ -1647,9 +1664,9 @@ def __init__(self, config: PPDocLayoutV3Config): self.decoder_input_proj = nn.ModuleList(decoder_input_proj_list) self.decoder = PPDocLayoutV3Decoder(config) - self.dec_order_head = nn.Linear(config.d_model, config.d_model) - self.dec_global_pointer = GlobalPointer(config) - self.dec_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) + self.decoder_order_head = nn.Linear(config.d_model, config.d_model) + self.decoder_global_pointer = GlobalPointer(config) + self.decoder_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) self.decoder.class_embed = self.enc_score_head self.decoder.bbox_embed = self.enc_bbox_head @@ -1863,7 +1880,7 @@ def forward( # _get_pred_class_and_mask batch_ind = torch.arange(memory.shape[0], device=output_memory.device).unsqueeze(1) target = output_memory[batch_ind, topk_ind] - out_query = self.dec_norm(target) + out_query = self.decoder_norm(target) mask_query_embed = self.mask_query_head(out_query) batch_size, mask_dim, _ = mask_query_embed.shape _, _, mask_h, mask_w = mask_feat.shape @@ -1908,10 +1925,10 @@ def forward( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - order_head=self.dec_order_head, - global_pointer=self.dec_global_pointer, + order_head=self.decoder_order_head, + global_pointer=self.decoder_global_pointer, mask_query_head=self.mask_query_head, - dec_norm=self.dec_norm, + norm=self.decoder_norm, mask_feat=mask_feat, ) @@ -1954,6 +1971,11 @@ def forward( @dataclass class PPDocLayoutV3HybridEncoderOutput(BaseModelOutput): + r""" + mask_feat (`torch.FloatTensor` of shape `(batch_size, config.num_queries, 200, 200)`): + Mask features for each query in the batch. + """ + mask_feat: torch.FloatTensor = None @@ -1970,7 +1992,7 @@ class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding - possible padding). You can use [`~PPDocLayoutV3ImageProcessor.post_process_object_detection`] to retrieve the + possible padding). You can use [`~PPDocLayoutV3ImageProcessorFast.post_process_object_detection`] to retrieve the unnormalized (absolute) bounding boxes. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): Sequence of hidden-states at the output of the last layer of the decoder of the model. diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index 0c81acb01c8e..887e4c3fafa0 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -20,33 +20,19 @@ import torch import torch.nn.functional as F from torch import nn -from torchvision.transforms.v2.functional import InterpolationMode from ... import initialization as init from ...configuration_utils import PreTrainedConfig -from ...feature_extraction_utils import BatchFeature -from ...image_processing_utils import BaseImageProcessor, get_size_dict from ...image_processing_utils_fast import ( BaseImageProcessorFast, - SizeDict, ) -from ...image_transforms import resize, to_channel_dimension_format from ...image_utils import ( - ChannelDimension, - ImageInput, PILImageResampling, - infer_channel_dimension_format, - make_flat_list_of_images, - to_numpy_array, - valid_images, - validate_preprocess_arguments, ) from ...modeling_outputs import BaseModelOutput -from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, auto_docstring, - filter_out_non_signature_kwargs, logging, requires_backends, ) @@ -63,6 +49,7 @@ RTDetrModel, RTDetrModelOutput, RTDetrMultiscaleDeformableAttention, + RTDetrPreTrainedModel, get_contrastive_denoising_training_group, inverse_sigmoid, ) @@ -74,7 +61,7 @@ class PPDocLayoutV3Config(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`PP-DocLayoutV3`]. It is used to instantiate a - RT-DETR model according to the specified arguments, defining the model architecture. Instantiating a configuration + PP-DocLayoutV3 model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the PP-DocLayoutV3 [PaddlePaddle/PP-DocLayoutV3_safetensors](https://huggingface.co/PaddlePaddle/PP-DocLayoutV3_safetensors) architecture. @@ -186,7 +173,8 @@ class PPDocLayoutV3Config(PreTrainedConfig): Whether the architecture has an encoder decoder structure. gp_head_size (`int`, *optional*, defaults to 64): The size of the global pointer head. - + gp_dropout_value (`float`, *optional*, defaults to 0.1): + The dropout probability in the global pointer head. Examples: ```python @@ -263,6 +251,7 @@ def __init__( disable_custom_kernels=True, is_encoder_decoder=True, gp_head_size=64, + gp_dropout_value=0.1, **kwargs, ): self.initializer_range = initializer_range @@ -346,296 +335,12 @@ def __init__( self.anchor_image_size = list(anchor_image_size) if anchor_image_size is not None else None self.disable_custom_kernels = disable_custom_kernels self.gp_head_size = gp_head_size + self.gp_dropout_value = gp_dropout_value super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs) -class PPDocLayoutV3ImageProcessor(BaseImageProcessor): - r""" - Constructs a PPDocLayoutV3 image processor. - - Args: - do_resize (`bool`, *optional*, defaults to `True`): - Controls whether to resize the image's (height, width) dimensions to the specified `size`. Can be - overridden by the `do_resize` parameter in the `preprocess` method. - size (`dict[str, int]` *optional*, defaults to `{"height": 640, "width": 640}`): - Size of the image's `(height, width)` dimensions after resizing. Can be overridden by the `size` parameter - in the `preprocess` method. Available options are: - - `{"height": int, "width": int}`: The image will be resized to the exact size `(height, width)`. - Do NOT keep the aspect ratio. - - `{"shortest_edge": int, "longest_edge": int}`: The image will be resized to a maximum size respecting - the aspect ratio and keeping the shortest edge less or equal to `shortest_edge` and the longest edge - less or equal to `longest_edge`. - - `{"max_height": int, "max_width": int}`: The image will be resized to the maximum size respecting the - aspect ratio and keeping the height less or equal to `max_height` and the width less or equal to - `max_width`. - resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): - Resampling filter to use if resizing the image. - do_rescale (`bool`, *optional*, defaults to `True`): - Controls whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the - `do_rescale` parameter in the `preprocess` method. - rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): - Scale factor to use if rescaling the image. Can be overridden by the `rescale_factor` parameter in the - `preprocess` method. - Controls whether to normalize the image. Can be overridden by the `do_normalize` parameter in the - `preprocess` method. - do_normalize (`bool`, *optional*, defaults to `True`): - Whether to normalize the image. - image_mean (`float` or `list[float]`, *optional*, defaults to `[0, 0, 0]`): - Mean values to use when normalizing the image. Can be a single value or a list of values, one for each - channel. Can be overridden by the `image_mean` parameter in the `preprocess` method. - image_std (`float` or `list[float]`, *optional*, defaults to `[1, 1, 1]`): - Standard deviation values to use when normalizing the image. Can be a single value or a list of values, one - for each channel. Can be overridden by the `image_std` parameter in the `preprocess` method. - """ - - model_input_names = ["pixel_values"] - - def __init__( - self, - do_resize: bool = True, - size: Optional[dict[str, int]] = None, - resample: Optional[PILImageResampling] = PILImageResampling.BICUBIC, - do_rescale: bool = True, - rescale_factor: Union[int, float] = 1 / 255, - do_normalize: bool = True, - image_mean: Optional[Union[float, list[float]]] = [0, 0, 0], - image_std: Optional[Union[float, list[float]]] = [1, 1, 1], - **kwargs, - ) -> None: - size = size if size is not None else {"height": 800, "width": 800} - size = get_size_dict(size, default_to_square=False) - - super().__init__(**kwargs) - self.do_resize = do_resize - self.size = size - self.do_rescale = do_rescale - self.rescale_factor = rescale_factor - self.do_normalize = do_normalize - self.image_mean = image_mean - self.image_std = image_std - self.resample = resample - - def _get_order_seqs(self, order_logits): - """ - Computes the order sequences for a batch of inputs based on logits. - - This function takes in the order logits, calculates order scores using a sigmoid activation, - and determines the order sequences by ranking the votes derived from the scores. - - Args: - order_logits (`torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): - Stacked order logits. - - Returns: - torch.Tensor: A tensor of shape `(batch_size, num_queries)`: - Containing the computed order sequences for each input in the batch. Each row represents the ranked order of elements for the corresponding input in the batch. - """ - order_scores = torch.sigmoid(order_logits) - batch_size, sequence_length, _ = order_scores.shape - - order_votes = order_scores.triu(diagonal=1).sum(dim=1) + (1.0 - order_scores.transpose(1, 2)).tril( - diagonal=-1 - ).sum(dim=1) - - order_pointers = torch.argsort(order_votes, dim=1) - order_seq = torch.empty_like(order_pointers) - ranks = torch.arange(sequence_length, device=order_pointers.device, dtype=order_pointers.dtype).expand( - batch_size, -1 - ) - order_seq.scatter_(1, order_pointers, ranks) - - return order_seq - - @filter_out_non_signature_kwargs() - def preprocess( - self, - images: ImageInput, - do_resize: Optional[bool] = None, - size: Optional[dict[str, int]] = None, - resample: Optional[PILImageResampling] = None, - do_rescale: Optional[bool] = None, - rescale_factor: Optional[Union[int, float]] = None, - do_normalize: Optional[bool] = None, - image_mean: Optional[Union[float, list[float]]] = None, - image_std: Optional[Union[float, list[float]]] = None, - return_tensors: Optional[Union[TensorType, str]] = None, - data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, - input_data_format: Optional[Union[str, ChannelDimension]] = None, - ) -> BatchFeature: - """ - Preprocess an image or a batch of images so that it can be used by the model. - - Args: - images (`ImageInput`): - Image or batch of images to preprocess. Expects a single or batch of images with pixel values ranging - from 0 to 255. If passing in images with pixel values between 0 and 1, set `do_rescale=False`. - do_resize (`bool`, *optional*, defaults to self.do_resize): - Whether to resize the image. - size (`dict[str, int]`, *optional*, defaults to self.size): - Size of the image's `(height, width)` dimensions after resizing. Available options are: - - `{"height": int, "width": int}`: The image will be resized to the exact size `(height, width)`. - Do NOT keep the aspect ratio. - - `{"shortest_edge": int, "longest_edge": int}`: The image will be resized to a maximum size respecting - the aspect ratio and keeping the shortest edge less or equal to `shortest_edge` and the longest edge - less or equal to `longest_edge`. - - `{"max_height": int, "max_width": int}`: The image will be resized to the maximum size respecting the - aspect ratio and keeping the height less or equal to `max_height` and the width less or equal to - `max_width`. - resample (`PILImageResampling`, *optional*, defaults to self.resample): - Resampling filter to use when resizing the image. - do_rescale (`bool`, *optional*, defaults to self.do_rescale): - Whether to rescale the image. - rescale_factor (`float`, *optional*, defaults to self.rescale_factor): - Rescale factor to use when rescaling the image. - do_normalize (`bool`, *optional*, defaults to self.do_normalize): - Whether to normalize the image. - image_mean (`float` or `list[float]`, *optional*, defaults to self.image_mean): - Mean to use when normalizing the image. - image_std (`float` or `list[float]`, *optional*, defaults to self.image_std): - Standard deviation to use when normalizing the image. - return_tensors (`str` or `TensorType`, *optional*, defaults to self.return_tensors): - Type of tensors to return. If `None`, will return the list of images. - data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): - The channel dimension format for the output image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Unset: Use the channel dimension format of the input image. - input_data_format (`ChannelDimension` or `str`, *optional*): - The channel dimension format for the input image. If unset, the channel dimension format is inferred - from the input image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - """ - do_resize = self.do_resize if do_resize is None else do_resize - size = self.size if size is None else size - size = get_size_dict(size=size, default_to_square=True) - resample = self.resample if resample is None else resample - do_rescale = self.do_rescale if do_rescale is None else do_rescale - rescale_factor = self.rescale_factor if rescale_factor is None else rescale_factor - do_normalize = self.do_normalize if do_normalize is None else do_normalize - image_mean = self.image_mean if image_mean is None else image_mean - image_std = self.image_std if image_std is None else image_std - - validate_preprocess_arguments( - do_rescale=do_rescale, - rescale_factor=rescale_factor, - do_normalize=do_normalize, - image_mean=image_mean, - image_std=image_std, - do_resize=do_resize, - size=size, - resample=resample, - ) - - images = make_flat_list_of_images(images) - if not valid_images(images): - raise ValueError("Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, or torch.Tensor") - - # All transformations expect numpy arrays - images = [to_numpy_array(image) for image in images] - - if input_data_format is None: - input_data_format = infer_channel_dimension_format(images[0]) - - # transformations - if do_resize: - images = [ - resize( - image, size=(size["height"], size["width"]), resample=resample, input_data_format=input_data_format - ) - for image in images - ] - - if do_rescale: - images = [self.rescale(image, rescale_factor, input_data_format=input_data_format) for image in images] - - if do_normalize: - images = [ - self.normalize(image, image_mean, image_std, input_data_format=input_data_format) for image in images - ] - - images = [ - to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images - ] - encoded_inputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) - - return encoded_inputs - - def post_process_object_detection( - self, - outputs, - threshold: float = 0.5, - target_sizes: Optional[Union[TensorType, list[tuple]]] = None, - ): - """ - Converts the raw output of [`PPDocLayoutV3ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, - bottom_right_x, bottom_right_y) format. Only supports PyTorch. - - Args: - outputs ([`PPDocLayoutV3ObjectDetectionOutput`]): - Raw outputs of the model. - threshold (`float`, *optional*, defaults to 0.5): - Score threshold to keep object detection predictions. - target_sizes (`torch.Tensor` or `list[tuple[int, int]]`, *optional*): - Tensor of shape `(batch_size, 2)` or list of tuples (`tuple[int, int]`) containing the target size - `(height, width)` of each image in the batch. If unset, predictions will not be resized. - - Returns: - `list[Dict]`: An ordered list of dictionaries, each dictionary containing the scores, labels and boxes for an image - in the batch as predicted by the model. - """ - requires_backends(self, ["torch"]) - boxes = outputs.pred_boxes - logits = outputs.logits - order_logits = outputs.order_logits - - order_seqs = self._get_order_seqs(order_logits) - - box_centers, box_dims = torch.split(boxes, 2, dim=-1) - top_left_coords = box_centers - 0.5 * box_dims - bottom_right_coords = box_centers + 0.5 * box_dims - boxes = torch.cat([top_left_coords, bottom_right_coords], dim=-1) - - if target_sizes is not None: - if len(logits) != len(target_sizes): - raise ValueError( - "Make sure that you pass in as many target sizes as the batch dimension of the logits" - ) - if isinstance(target_sizes, list): - img_h, img_w = torch.as_tensor(target_sizes).unbind(1) - else: - img_h, img_w = target_sizes.unbind(1) - scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) - boxes = boxes * scale_fct[:, None, :] - - num_top_queries = logits.shape[1] - num_classes = logits.shape[2] - - scores = torch.nn.functional.sigmoid(logits) - scores, index = torch.topk(scores.flatten(1), num_top_queries, dim=-1) - labels = index % num_classes - index = index // num_classes - boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) - order_seqs = order_seqs.gather(dim=1, index=index) - - results = [] - for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): - order_seq = order_seq[score >= threshold] - order_seq, indices = torch.sort(order_seq) - results.append( - { - "scores": score[score >= threshold][indices], - "labels": label[score >= threshold][indices], - "boxes": box[score >= threshold][indices], - "order_seq": order_seq, - } - ) - - return results - - +@auto_docstring class PPDocLayoutV3ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = [0, 0, 0] @@ -679,40 +384,6 @@ def _get_order_seqs(self, order_logits): return order_seq - def _preprocess( - self, - images: list[torch.Tensor], - do_resize: bool, - size: SizeDict, - interpolation: Optional[InterpolationMode], - do_rescale: bool, - rescale_factor: float, - do_normalize: bool, - image_mean: Optional[Union[float, list[float]]], - image_std: Optional[Union[float, list[float]]], - return_tensors: Optional[Union[str, TensorType]], - **kwargs, - ) -> BatchFeature: - """ - Preprocess an image or a batch of images so that it can be used by the model. - """ - data = {} - processed_images = [] - for image in images: - if do_resize: - image = self.resize(image, size=size, interpolation=interpolation) - # Fused rescale and normalize - image = self.rescale_and_normalize(image, do_rescale, rescale_factor, do_normalize, image_mean, image_std) - - processed_images.append(image) - - images = processed_images - - data.update({"pixel_values": torch.stack(images, dim=0)}) - encoded_inputs = BatchFeature(data, tensor_type=return_tensors) - - return encoded_inputs - def post_process_object_detection( self, outputs, @@ -785,14 +456,13 @@ def __init__(self, config): super().__init__() self.head_size = config.gp_head_size self.dense = nn.Linear(config.d_model, self.head_size * 2) - self.dropout = nn.Dropout(0.1) + self.dropout = nn.Dropout(config.gp_dropout_value) def forward(self, inputs): batch_size, sequence_length, _ = inputs.shape query_key_projection = self.dense(inputs).reshape(batch_size, sequence_length, 2, self.head_size) query_key_projection = self.dropout(query_key_projection) - queries = query_key_projection[:, :, 0, :] - keys = query_key_projection[:, :, 1, :] + queries, keys = torch.unbind(query_key_projection, dim=2) logits = (queries @ keys.transpose(-2, -1)) / (self.head_size**0.5) mask = torch.tril(torch.ones(sequence_length, sequence_length, device=logits.device)).bool() @@ -806,7 +476,7 @@ class PPDocLayoutV3MultiscaleDeformableAttention(RTDetrMultiscaleDeformableAtten @auto_docstring -class PPDocLayoutV3PreTrainedModel(PreTrainedModel): +class PPDocLayoutV3PreTrainedModel(RTDetrPreTrainedModel): config: PPDocLayoutV3Config base_model_prefix = "pp_doclayout_v3" main_input_name = "pixel_values" @@ -815,7 +485,20 @@ class PPDocLayoutV3PreTrainedModel(PreTrainedModel): @torch.no_grad() def _init_weights(self, module): - if isinstance(module, PPDocLayoutV3MultiscaleDeformableAttention): + if isinstance(module, PPDocLayoutV3ForObjectDetection): + if module.model.decoder.class_embed is not None: + layer = module.model.decoder.class_embed + prior_prob = self.config.initializer_bias_prior_prob or 1 / (self.config.num_labels + 1) + bias = float(-math.log((1 - prior_prob) / prior_prob)) + init.xavier_uniform_(layer.weight) + init.constant_(layer.bias, bias) + + if module.model.decoder.bbox_embed is not None: + layer = module.model.decoder.bbox_embed + init.constant_(layer.layers[-1].weight, 0) + init.constant_(layer.layers[-1].bias, 0) + + elif isinstance(module, PPDocLayoutV3MultiscaleDeformableAttention): init.constant_(module.sampling_offsets.weight, 0.0) default_dtype = torch.get_default_dtype() thetas = torch.arange(module.n_heads, dtype=torch.int64).to(default_dtype) * ( @@ -1003,10 +686,11 @@ def __init__( ): super().__init__() - assert len(in_channels) == len(fpn_strides), ( - f"Error: The lengths of 'in_channels' and 'fpn_strides' must be equal. " - f"Got len(in_channels)={len(in_channels)} and len(fpn_strides)={len(fpn_strides)}." - ) + if len(in_channels) != len(fpn_strides): + raise ValueError( + f"The lengths of 'in_channels' and 'fpn_strides' must be equal. " + f"Got len(in_channels)={len(in_channels)} and len(fpn_strides)={len(fpn_strides)}." + ) reorder_index = np.argsort(fpn_strides, axis=0).tolist() in_channels = [in_channels[i] for i in reorder_index] @@ -1049,6 +733,12 @@ def forward(self, inputs): class PPDocLayoutV3HybridEncoder(RTDetrHybridEncoder): + """ + Main difference to `RTDetrHybridEncoder`: + 1. Mask Feature Head: Added `MaskFeatFPN` module (`self.mask_feat_head`) for document - specific mask feature generation. + 2. Extra Conv Layers: Introduced `self.encoder_mask_lateral` and `self.encoder_mask_output` for mask feature processing and output. + """ + def __init__(self, config: PPDocLayoutV3Config): super().__init__() @@ -1060,8 +750,8 @@ def __init__(self, config: PPDocLayoutV3Config): feat_channels=mask_feat_channels[0], out_channels=mask_feat_channels[1], ) - self.enc_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1, "silu") - self.enc_mask_output = nn.Sequential( + self.encoder_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1, "silu") + self.encoder_mask_output = nn.Sequential( BaseConv(mask_feat_channels[1], mask_feat_channels[1], 3, 1, "silu"), nn.Conv2d(in_channels=mask_feat_channels[1], out_channels=config.num_prototypes, kernel_size=1), ) @@ -1179,8 +869,8 @@ def forward( mask_feat = self.mask_feat_head(pan_feature_maps) mask_feat = F.interpolate(mask_feat, scale_factor=2, mode="bilinear", align_corners=False) - mask_feat += self.enc_mask_lateral(x4_feat[0]) - mask_feat = self.enc_mask_output(mask_feat) + mask_feat += self.encoder_mask_lateral(x4_feat[0]) + mask_feat = self.encoder_mask_output(mask_feat) if not return_dict: return tuple(v for v in [pan_feature_maps, encoder_states, all_attentions, mask_feat] if v is not None) @@ -1194,6 +884,11 @@ def forward( class PPDocLayoutV3Decoder(RTDetrDecoder): + """ + Main difference to `RTDetrDecoder`: + A new mask generation process is introduced at each decoder layer. + """ + def __init__(self, config: PPDocLayoutV3Config): super().__init__() @@ -1213,7 +908,7 @@ def forward( order_head=None, global_pointer=None, mask_query_head=None, - dec_norm=None, + norm=None, mask_feat=None, output_attentions=None, output_hidden_states=None, @@ -1307,7 +1002,7 @@ def forward( ) # get_pred_class_order_and_mask - out_query = dec_norm(hidden_states) + out_query = norm(hidden_states) mask_query_embed = mask_query_head(out_query) batch_size, mask_dim, _ = mask_query_embed.shape _, _, mask_h, mask_w = mask_feat.shape @@ -1385,9 +1080,9 @@ def __init__(self, config: PPDocLayoutV3Config): encoder_input_proj_list = [] self.encoder_input_proj = nn.ModuleList(encoder_input_proj_list[1:]) - self.dec_order_head = nn.Linear(config.d_model, config.d_model) - self.dec_global_pointer = GlobalPointer(config) - self.dec_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) + self.decoder_order_head = nn.Linear(config.d_model, config.d_model) + self.decoder_global_pointer = GlobalPointer(config) + self.decoder_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) self.decoder = PPDocLayoutV3Decoder(config) self.decoder.class_embed = self.enc_score_head self.decoder.bbox_embed = self.enc_bbox_head @@ -1563,7 +1258,7 @@ def forward( # _get_pred_class_and_mask batch_ind = torch.arange(memory.shape[0], device=output_memory.device).unsqueeze(1) target = output_memory[batch_ind, topk_ind] - out_query = self.dec_norm(target) + out_query = self.decoder_norm(target) mask_query_embed = self.mask_query_head(out_query) batch_size, mask_dim, _ = mask_query_embed.shape _, _, mask_h, mask_w = mask_feat.shape @@ -1608,10 +1303,10 @@ def forward( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - order_head=self.dec_order_head, - global_pointer=self.dec_global_pointer, + order_head=self.decoder_order_head, + global_pointer=self.decoder_global_pointer, mask_query_head=self.mask_query_head, - dec_norm=self.dec_norm, + norm=self.decoder_norm, mask_feat=mask_feat, ) @@ -1654,6 +1349,11 @@ def forward( @dataclass class PPDocLayoutV3HybridEncoderOutput(BaseModelOutput): + r""" + mask_feat (`torch.FloatTensor` of shape `(batch_size, config.num_queries, 200, 200)`): + Mask features for each query in the batch. + """ + mask_feat: torch.FloatTensor = None @@ -1670,7 +1370,7 @@ class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding - possible padding). You can use [`~PPDocLayoutV3ImageProcessor.post_process_object_detection`] to retrieve the + possible padding). You can use [`~PPDocLayoutV3ImageProcessorFast.post_process_object_detection`] to retrieve the unnormalized (absolute) bounding boxes. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): Sequence of hidden-states at the output of the last layer of the decoder of the model. @@ -1878,7 +1578,6 @@ def forward( __all__ = [ "PPDocLayoutV3ForObjectDetection", - "PPDocLayoutV3ImageProcessor", "PPDocLayoutV3ImageProcessorFast", "PPDocLayoutV3Config", "PPDocLayoutV3Model", diff --git a/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py index 87b3718c3638..da2e43913f03 100644 --- a/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py +++ b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py @@ -18,16 +18,20 @@ import math import unittest +import requests from parameterized import parameterized from transformers import ( PPDocLayoutV3Config, PPDocLayoutV3ForObjectDetection, + PPDocLayoutV3ImageProcessor, is_torch_available, + is_vision_available, ) from transformers.testing_utils import ( require_torch, require_torch_accelerator, + require_vision, slow, torch_device, ) @@ -40,6 +44,9 @@ if is_torch_available(): import torch +if is_vision_available(): + from PIL import Image + class PPDocLayoutV3ModelTester: def __init__( @@ -56,7 +63,7 @@ def __init__( backbone_config=None, # encoder HybridEncoder encoder_hidden_dim=32, - encoder_in_channels=[128, 256, 512], + encoder_in_channels=[32, 32, 32], feat_strides=[8, 16, 32], encoder_layers=1, encoder_ffn_dim=64, @@ -69,11 +76,13 @@ def __init__( activation_function="silu", eval_size=None, normalize_before=False, + mask_feat_channels=[32, 32], + x4_feat_dim=32, # decoder PPDocLayoutV3Transformer - d_model=256, + d_model=32, num_queries=30, decoder_in_channels=[32, 32, 32], - decoder_ffn_dim=64, + decoder_ffn_dim=8, num_feature_levels=3, decoder_n_points=4, decoder_layers=2, @@ -112,6 +121,8 @@ def __init__( self.activation_function = activation_function self.eval_size = eval_size self.normalize_before = normalize_before + self.mask_feat_channels = mask_feat_channels + self.x4_feat_dim = x4_feat_dim self.d_model = d_model self.num_queries = num_queries self.decoder_in_channels = decoder_in_channels @@ -145,12 +156,25 @@ def get_config(self): "model_type": "hgnet_v2", "arch": "L", "return_idx": [0, 1, 2, 3], + "hidden_sizes": [32, 32, 32, 32], + "stem_channels": [3, 32, 32], + "stage_in_channels": [32, 32, 32, 32], + "stage_mid_channels": [32, 32, 32, 32], + "stage_out_channels": [32, 32, 32, 32], "freeze_stem_only": True, "freeze_at": 0, "freeze_norm": True, "lr_mult_list": [0, 0.05, 0.05, 0.05, 0.05], "out_features": ["stage1", "stage2", "stage3", "stage4"], } + # depths=[3, 4, 6, 3], + # hidden_sizes=[256, 512, 1024, 2048], + # stem_channels=[3, 32, 48], + # stage_in_channels=[48, 128, 512, 1024], + # stage_mid_channels=[48, 96, 192, 384], + # stage_out_channels=[128, 512, 1024, 2048], + # stage_num_blocks=[1, 1, 3, 1], + return PPDocLayoutV3Config( backbone_config=backbone_config, encoder_hidden_dim=self.encoder_hidden_dim, @@ -167,6 +191,8 @@ def get_config(self): activation_function=self.activation_function, eval_size=self.eval_size, normalize_before=self.normalize_before, + mask_feat_channels=self.mask_feat_channels, + x4_feat_dim=self.x4_feat_dim, d_model=self.d_model, num_queries=self.num_queries, decoder_in_channels=self.decoder_in_channels, @@ -240,9 +266,9 @@ def test_resize_tokens_embeddings(self): def test_feed_forward_chunking(self): pass - @unittest.skip(reason="PPDocLayoutV3 does not support this test") - def test_model_is_small(self): - pass + # @unittest.skip(reason="PPDocLayoutV3 does not support this test") + # def test_model_is_small(self): + # pass @unittest.skip(reason="PPDocLayoutV3 does not support training") def test_retain_grad_hidden_states_attentions(self): @@ -279,6 +305,7 @@ def test_inference_with_different_dtypes(self, dtype_str): with torch.no_grad(): _ = model(**self._prepare_for_class(inputs_dict, model_class)) + # We have not `num_hidden_layers`, use `encoder_in_channels` instead def test_hidden_states_output(self): def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(config) @@ -417,7 +444,6 @@ def test_attention_outputs(self): if hasattr(self.model_tester, "num_hidden_states_types"): added_hidden_states = self.model_tester.num_hidden_states_types else: - # RTDetr should maintin encoder_hidden_states output added_hidden_states = 2 self.assertEqual(out_len + added_hidden_states, len(outputs)) @@ -435,7 +461,70 @@ def test_attention_outputs(self): # TODO: -# @require_torch -# @require_vision -# @slow -# class PPDocLayoutV3ModelIntegrationTest(unittest.TestCase): +# Later, we will determine these values based on the latest weights. +@require_torch +@require_vision +@slow +class PPDocLayoutV3ModelIntegrationTest(unittest.TestCase): + def setUp(self): + model_path = "PaddlePaddle/PP-DocLayoutV3_safetensors" + self.model = PPDocLayoutV3ForObjectDetection.from_pretrained(model_path).to(torch_device) + self.image_processor = ( + PPDocLayoutV3ImageProcessor.from_pretrained(model_path) if is_vision_available() else None + ) + url = "https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/layout_demo.jpg" + self.image = Image.open(requests.get(url, stream=True).raw) + + def test_inference_object_detection_head(self): + inputs = self.image_processor(images=self.image, return_tensors="pt").to(torch_device) + + with torch.no_grad(): + outputs = self.model(**inputs) + + expected_shape_logits = torch.Size((1, 300, self.model.config.num_labels)) + expected_logits = torch.tensor( + [[-5.6224, -6.5667, -4.9352], [-5.7931, -5.5543, -5.6476], [-4.5742, -5.0603, -7.2864]] + ).to(torch_device) + self.assertEqual(outputs.logits.shape, expected_shape_logits) + torch.testing.assert_close(outputs.logits[0, :3, :3], expected_logits, rtol=2e-4, atol=2e-4) + + expected_shape_boxes = torch.Size((1, 300, 4)) + expected_boxes = torch.tensor( + [[0.4000, 0.9702, 0.3897], [0.3642, 0.3164, 0.3212], [0.3716, 0.1786, 0.3386]] + ).to(torch_device) + self.assertEqual(outputs.pred_boxes.shape, expected_shape_boxes) + torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_boxes, rtol=2e-4, atol=2e-4) + + expected_shape_order_logits = torch.Size((1, 300, 300)) + expected_order_logits = torch.tensor( + [ + [-10000.0000, -937.0416, -1045.3816], + [-10000.0000, -10000.0000, -343.8752], + [-10000.0000, -10000.0000, -10000.0000], + ] + ).to(torch_device) + self.assertEqual(outputs.order_logits.shape, expected_shape_order_logits) + torch.testing.assert_close(outputs.order_logits[0, :3, :3], expected_order_logits, rtol=2e-4, atol=2e-4) + + # verify postprocessing + results = self.image_processor.post_process_object_detection( + outputs, threshold=0.5, target_sizes=[self.image.size[::-1]] + )[0] + + expected_scores = torch.tensor( + [0.9834, 0.9485, 0.9837, 0.9728, 0.9741, 0.9770, 0.9508, 0.9390, 0.9482, 0.8391, 0.9358, 0.8249, 0.9095] + ).to(torch_device) + torch.testing.assert_close(results["scores"], expected_scores, rtol=2e-4, atol=2e-4) + + expected_labels = [22, 17, 22, 22, 22, 22, 22, 10, 10, 22, 10, 16, 8] + self.assertSequenceEqual(results["labels"].tolist(), expected_labels) + + expected_slice_boxes = torch.tensor( + [ + [334.5682, 182.9777, 894.6927, 652.4594], + [336.7216, 683.4235, 867.9361, 796.9210], + [335.2677, 841.1227, 891.1608, 1453.0148], + [919.8677, 183.4835, 1475.8800, 463.4977], + ] + ).to(torch_device) + torch.testing.assert_close(results["boxes"][:4], expected_slice_boxes, rtol=2e-4, atol=2e-4) From dbfc8edf6188b68f1654fc0003baed24e2dc3c4f Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Thu, 22 Jan 2026 18:04:40 +0800 Subject: [PATCH 08/21] update --- docs/source/en/model_doc/pp_doclayout_v3.md | 4 + .../configuration_pp_doclayout_v3.py | 12 +- .../modeling_pp_doclayout_v3.py | 129 +++++++-------- .../modular_pp_doclayout_v3.py | 147 +++++++++--------- .../test_modeling_pp_doclayout_v3.py | 25 +-- 5 files changed, 156 insertions(+), 161 deletions(-) diff --git a/docs/source/en/model_doc/pp_doclayout_v3.md b/docs/source/en/model_doc/pp_doclayout_v3.md index 6ef5194dad7d..05097b6f9a71 100644 --- a/docs/source/en/model_doc/pp_doclayout_v3.md +++ b/docs/source/en/model_doc/pp_doclayout_v3.md @@ -141,3 +141,7 @@ for result in results: ## PPDocLayoutV3ImageProcessorFast [[autodoc]] PPDocLayoutV3ImageProcessorFast + +## PPDocLayoutV3HybridEncoderOutput + +[[autodoc]] PPDocLayoutV3HybridEncoderOutput diff --git a/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py index 95de9840e7cf..f2c783aa498a 100644 --- a/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py @@ -97,7 +97,7 @@ class PPDocLayoutV3Config(PreTrainedConfig): feed-forward modules. hidden_expansion (`float`, *optional*, defaults to 1.0): Expansion ratio to enlarge the dimension size of RepVGGBlock and CSPRepLayer. - mask_feat_channels (`list[int]`, *optional*, defaults to `[64, 64]`): + mask_feature_channels (`list[int]`, *optional*, defaults to `[64, 64]`): The channels of the multi-level features for mask enhancement. x4_feat_dim (`int`, *optional*, defaults to 128): The dimension of the x4 feature map. @@ -140,7 +140,7 @@ class PPDocLayoutV3Config(PreTrainedConfig): Whether to disable custom kernels. is_encoder_decoder (`bool`, *optional*, defaults to `True`): Whether the architecture has an encoder decoder structure. - gp_head_size (`int`, *optional*, defaults to 64): + global_pointer_head_size (`int`, *optional*, defaults to 64): The size of the global pointer head. gp_dropout_value (`float`, *optional*, defaults to 0.1): The dropout probability in the global pointer head. @@ -197,7 +197,7 @@ def __init__( eval_size=None, normalize_before=False, hidden_expansion=1.0, - mask_feat_channels=[64, 64], + mask_feature_channels=[64, 64], x4_feat_dim=128, # decoder PPDocLayoutV3Transformer d_model=256, @@ -219,7 +219,7 @@ def __init__( anchor_image_size=None, disable_custom_kernels=True, is_encoder_decoder=True, - gp_head_size=64, + global_pointer_head_size=64, gp_dropout_value=0.1, **kwargs, ): @@ -281,7 +281,7 @@ def __init__( self.eval_size = list(eval_size) if eval_size is not None else None self.normalize_before = normalize_before self.hidden_expansion = hidden_expansion - self.mask_feat_channels = mask_feat_channels + self.mask_feature_channels = mask_feature_channels self.x4_feat_dim = x4_feat_dim # ---- decoder ---- @@ -303,7 +303,7 @@ def __init__( self.learn_initial_query = learn_initial_query self.anchor_image_size = list(anchor_image_size) if anchor_image_size is not None else None self.disable_custom_kernels = disable_custom_kernels - self.gp_head_size = gp_head_size + self.global_pointer_head_size = global_pointer_head_size self.gp_dropout_value = gp_dropout_value super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs) diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index e6565a13c850..490993024221 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -43,7 +43,7 @@ class GlobalPointer(nn.Module): def __init__(self, config): super().__init__() - self.head_size = config.gp_head_size + self.head_size = config.global_pointer_head_size self.dense = nn.Linear(config.d_model, self.head_size * 2) self.dropout = nn.Dropout(config.gp_dropout_value) @@ -317,9 +317,9 @@ class PPDocLayoutV3DecoderOutput(ModelOutput): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the weighted average in the cross-attention heads. - dec_out_order_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, config.num_queries)`): + decoder_out_order_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, config.num_queries)`): Stacked order logits (order logits of each layer of the decoder). - dec_out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, 200, 200)`): + decoder_out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, 200, 200)`): Stacked masks (masks of each layer of the decoder). """ @@ -333,8 +333,8 @@ class PPDocLayoutV3DecoderOutput(ModelOutput): attentions: Optional[tuple[torch.FloatTensor]] = None cross_attentions: Optional[tuple[torch.FloatTensor]] = None - dec_out_order_logits: Optional[torch.FloatTensor] = None - dec_out_masks: Optional[torch.FloatTensor] = None + decoder_out_order_logits: Optional[torch.FloatTensor] = None + decoder_out_masks: Optional[torch.FloatTensor] = None @dataclass @@ -443,24 +443,35 @@ def forward(self, input: Tensor) -> Tensor: return hidden_state +class ScaleHead(nn.Module): + def __init__(self, in_channels, feature_channels, fpn_stride, base_stride, align_corners=False): + super().__init__() + head_length = max(1, int(np.log2(fpn_stride) - np.log2(base_stride))) + self.layers = nn.ModuleList() + for k in range(head_length): + in_c = in_channels if k == 0 else feature_channels + self.layers.append(BaseConv(in_c, feature_channels, 3, 1, "silu")) + if fpn_stride != base_stride: + self.layers.append(nn.Upsample(scale_factor=2, mode="bilinear", align_corners=align_corners)) + + def forward(self, x): + for layer in self.layers: + x = layer(x) + return x + + class MaskFeatFPN(nn.Module): def __init__( self, in_channels=[256, 256, 256], fpn_strides=[32, 16, 8], - feat_channels=256, + feature_channels=256, dropout_ratio=0.0, out_channels=256, align_corners=False, ): super().__init__() - if len(in_channels) != len(fpn_strides): - raise ValueError( - f"The lengths of 'in_channels' and 'fpn_strides' must be equal. " - f"Got len(in_channels)={len(in_channels)} and len(fpn_strides)={len(fpn_strides)}." - ) - reorder_index = np.argsort(fpn_strides, axis=0).tolist() in_channels = [in_channels[i] for i in reorder_index] fpn_strides = [fpn_strides[i] for i in reorder_index] @@ -474,17 +485,16 @@ def __init__( self.scale_heads = nn.ModuleList() for i in range(len(fpn_strides)): - head_length = max(1, int(np.log2(fpn_strides[i]) - np.log2(fpn_strides[0]))) - scale_head = [] - for k in range(head_length): - in_c = in_channels[i] if k == 0 else feat_channels - scale_head.append(nn.Sequential(BaseConv(in_c, feat_channels, 3, 1, "silu"))) - if fpn_strides[i] != fpn_strides[0]: - scale_head.append(nn.Upsample(scale_factor=2, mode="bilinear", align_corners=align_corners)) - - self.scale_heads.append(nn.Sequential(*scale_head)) - - self.output_conv = BaseConv(feat_channels, out_channels, 3, 1, "silu") + self.scale_heads.append( + ScaleHead( + in_channels=in_channels[i], + feature_channels=feature_channels, + fpn_stride=fpn_strides[i], + base_stride=fpn_strides[0], + align_corners=align_corners, + ) + ) + self.output_conv = BaseConv(feature_channels, out_channels, 3, 1, "silu") def forward(self, inputs): x = [inputs[i] for i in self.reorder_index] @@ -501,6 +511,18 @@ def forward(self, inputs): return output +class EncoderMaskOutput(nn.Module): + def __init__(self, in_channels, num_prototypes): + super().__init__() + self.base_conv = BaseConv(in_channels, in_channels, 3, 1, "silu") + self.conv = nn.Conv2d(in_channels, num_prototypes, kernel_size=1) + + def forward(self, x): + x = self.base_conv(x) + x = self.conv(x) + return x + + class PPDocLayoutV3ConvNormLayer(nn.Module): def __init__(self, config, in_channels, out_channels, kernel_size, stride, padding=None, activation=None): super().__init__() @@ -795,7 +817,7 @@ def forward(self, src, src_mask=None, pos_embed=None, output_attentions: bool = class PPDocLayoutV3HybridEncoder(nn.Module): """ Main difference to `RTDetrHybridEncoder`: - 1. Mask Feature Head: Added `MaskFeatFPN` module (`self.mask_feat_head`) for document - specific mask feature generation. + 1. Mask Feature Head: Added `MaskFeatFPN` module (`self.mask_feature_head`) for document - specific mask feature generation. 2. Extra Conv Layers: Introduced `self.encoder_mask_lateral` and `self.encoder_mask_output` for mask feature processing and output. """ @@ -850,17 +872,16 @@ def __init__(self, config: PPDocLayoutV3Config): self.pan_blocks.append(pan_block) feat_strides = config.feat_strides - mask_feat_channels = config.mask_feat_channels - self.mask_feat_head = MaskFeatFPN( + mask_feature_channels = config.mask_feature_channels + self.mask_feature_head = MaskFeatFPN( [self.encoder_hidden_dim] * len(feat_strides), feat_strides, - feat_channels=mask_feat_channels[0], - out_channels=mask_feat_channels[1], + feature_channels=mask_feature_channels[0], + out_channels=mask_feature_channels[1], ) - self.encoder_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1, "silu") - self.encoder_mask_output = nn.Sequential( - BaseConv(mask_feat_channels[1], mask_feat_channels[1], 3, 1, "silu"), - nn.Conv2d(in_channels=mask_feat_channels[1], out_channels=config.num_prototypes, kernel_size=1), + self.encoder_mask_lateral = BaseConv(config.x4_feat_dim, mask_feature_channels[1], 3, 1, "silu") + self.encoder_mask_output = EncoderMaskOutput( + in_channels=mask_feature_channels[1], num_prototypes=config.num_prototypes ) @staticmethod @@ -911,14 +932,6 @@ def forward( Starting index of each feature map. valid_ratios (`torch.FloatTensor` of shape `(batch_size, num_feature_levels, 2)`): Ratio of valid area in each feature level. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -992,7 +1005,7 @@ def forward( new_pan_feature_map = pan_block(fused_feature_map) pan_feature_maps.append(new_pan_feature_map) - mask_feat = self.mask_feat_head(pan_feature_maps) + mask_feat = self.mask_feature_head(pan_feature_maps) mask_feat = F.interpolate(mask_feat, scale_factor=2, mode="bilinear", align_corners=False) mask_feat += self.encoder_mask_lateral(x4_feat[0]) mask_feat = self.encoder_mask_output(mask_feat) @@ -1192,15 +1205,6 @@ def forward( Indexes for the start of each feature level. In range `[0, sequence_length]`. valid_ratios (`torch.FloatTensor` of shape `(batch_size, num_feature_levels, 2)`, *optional*): Ratio of valid area in each feature level. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -1218,8 +1222,8 @@ def forward( intermediate = () intermediate_reference_points = () intermediate_logits = () - dec_out_order_logits = () - dec_out_masks = () + decoder_out_order_logits = () + decoder_out_masks = () reference_points = F.sigmoid(reference_points) @@ -1264,7 +1268,7 @@ def forward( out_mask = torch.bmm(mask_query_embed, mask_feat.flatten(start_dim=2)).reshape( batch_size, mask_dim, mask_h, mask_w ) - dec_out_masks += (out_mask,) + decoder_out_masks += (out_mask,) if self.class_embed is not None: logits = self.class_embed(out_query) @@ -1273,7 +1277,7 @@ def forward( if order_head is not None and global_pointer is not None: valid_query = out_query[:, -self.num_queries :] if self.num_queries is not None else out_query order_logits = global_pointer(order_head(valid_query)) - dec_out_order_logits += (order_logits,) + decoder_out_order_logits += (order_logits,) if output_attentions: all_self_attns += (layer_outputs[1],) @@ -1287,8 +1291,8 @@ def forward( if self.class_embed is not None: intermediate_logits = torch.stack(intermediate_logits, dim=1) if order_head is not None and global_pointer is not None: - dec_out_order_logits = torch.stack(dec_out_order_logits, dim=1) - dec_out_masks = torch.stack(dec_out_masks, dim=1) + decoder_out_order_logits = torch.stack(decoder_out_order_logits, dim=1) + decoder_out_masks = torch.stack(decoder_out_masks, dim=1) # add hidden states from the last decoder layer if output_hidden_states: @@ -1302,8 +1306,8 @@ def forward( intermediate, intermediate_logits, intermediate_reference_points, - dec_out_order_logits, - dec_out_masks, + decoder_out_order_logits, + decoder_out_masks, all_hidden_states, all_self_attns, all_cross_attentions, @@ -1315,8 +1319,8 @@ def forward( intermediate_hidden_states=intermediate, intermediate_logits=intermediate_logits, intermediate_reference_points=intermediate_reference_points, - dec_out_order_logits=dec_out_order_logits, - dec_out_masks=dec_out_masks, + decoder_out_order_logits=decoder_out_order_logits, + decoder_out_masks=decoder_out_masks, hidden_states=all_hidden_states, attentions=all_self_attns, cross_attentions=all_cross_attentions, @@ -1955,8 +1959,8 @@ def forward( decoder_hidden_states=decoder_outputs.hidden_states, decoder_attentions=decoder_outputs.attentions, cross_attentions=decoder_outputs.cross_attentions, - out_order_logits=decoder_outputs.dec_out_order_logits, - out_masks=decoder_outputs.dec_out_masks, + out_order_logits=decoder_outputs.decoder_out_order_logits, + out_masks=decoder_outputs.decoder_out_masks, encoder_last_hidden_state=encoder_outputs.last_hidden_state, encoder_hidden_states=encoder_outputs.hidden_states, encoder_attentions=encoder_outputs.attentions, @@ -1970,6 +1974,7 @@ def forward( @dataclass +@auto_docstring class PPDocLayoutV3HybridEncoderOutput(BaseModelOutput): r""" mask_feat (`torch.FloatTensor` of shape `(batch_size, config.num_queries, 200, 200)`): diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index 887e4c3fafa0..ada65e485169 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -128,7 +128,7 @@ class PPDocLayoutV3Config(PreTrainedConfig): feed-forward modules. hidden_expansion (`float`, *optional*, defaults to 1.0): Expansion ratio to enlarge the dimension size of RepVGGBlock and CSPRepLayer. - mask_feat_channels (`list[int]`, *optional*, defaults to `[64, 64]`): + mask_feature_channels (`list[int]`, *optional*, defaults to `[64, 64]`): The channels of the multi-level features for mask enhancement. x4_feat_dim (`int`, *optional*, defaults to 128): The dimension of the x4 feature map. @@ -171,7 +171,7 @@ class PPDocLayoutV3Config(PreTrainedConfig): Whether to disable custom kernels. is_encoder_decoder (`bool`, *optional*, defaults to `True`): Whether the architecture has an encoder decoder structure. - gp_head_size (`int`, *optional*, defaults to 64): + global_pointer_head_size (`int`, *optional*, defaults to 64): The size of the global pointer head. gp_dropout_value (`float`, *optional*, defaults to 0.1): The dropout probability in the global pointer head. @@ -228,7 +228,7 @@ def __init__( eval_size=None, normalize_before=False, hidden_expansion=1.0, - mask_feat_channels=[64, 64], + mask_feature_channels=[64, 64], x4_feat_dim=128, # decoder PPDocLayoutV3Transformer d_model=256, @@ -250,7 +250,7 @@ def __init__( anchor_image_size=None, disable_custom_kernels=True, is_encoder_decoder=True, - gp_head_size=64, + global_pointer_head_size=64, gp_dropout_value=0.1, **kwargs, ): @@ -312,7 +312,7 @@ def __init__( self.eval_size = list(eval_size) if eval_size is not None else None self.normalize_before = normalize_before self.hidden_expansion = hidden_expansion - self.mask_feat_channels = mask_feat_channels + self.mask_feature_channels = mask_feature_channels self.x4_feat_dim = x4_feat_dim # ---- decoder ---- @@ -334,7 +334,7 @@ def __init__( self.learn_initial_query = learn_initial_query self.anchor_image_size = list(anchor_image_size) if anchor_image_size is not None else None self.disable_custom_kernels = disable_custom_kernels - self.gp_head_size = gp_head_size + self.global_pointer_head_size = global_pointer_head_size self.gp_dropout_value = gp_dropout_value super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs) @@ -454,7 +454,7 @@ def post_process_object_detection( class GlobalPointer(nn.Module): def __init__(self, config): super().__init__() - self.head_size = config.gp_head_size + self.head_size = config.global_pointer_head_size self.dense = nn.Linear(config.d_model, self.head_size * 2) self.dropout = nn.Dropout(config.gp_dropout_value) @@ -477,12 +477,6 @@ class PPDocLayoutV3MultiscaleDeformableAttention(RTDetrMultiscaleDeformableAtten @auto_docstring class PPDocLayoutV3PreTrainedModel(RTDetrPreTrainedModel): - config: PPDocLayoutV3Config - base_model_prefix = "pp_doclayout_v3" - main_input_name = "pixel_values" - input_modalities = ("image",) - _no_split_modules = [r"PPDocLayoutV3HybridEncoder", r"PPDocLayoutV3DecoderLayer"] - @torch.no_grad() def _init_weights(self, module): if isinstance(module, PPDocLayoutV3ForObjectDetection): @@ -610,14 +604,14 @@ class PPDocLayoutV3DecoderOutput(RTDetrDecoderOutput): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the weighted average in the cross-attention heads. - dec_out_order_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, config.num_queries)`): + decoder_out_order_logits (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, config.num_queries)`): Stacked order logits (order logits of each layer of the decoder). - dec_out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, 200, 200)`): + decoder_out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, config.num_queries, 200, 200)`): Stacked masks (masks of each layer of the decoder). """ - dec_out_order_logits: Optional[torch.FloatTensor] = None - dec_out_masks: Optional[torch.FloatTensor] = None + decoder_out_order_logits: Optional[torch.FloatTensor] = None + decoder_out_masks: Optional[torch.FloatTensor] = None @dataclass @@ -674,24 +668,35 @@ class BaseConv(ResNetConvLayer): pass +class ScaleHead(nn.Module): + def __init__(self, in_channels, feature_channels, fpn_stride, base_stride, align_corners=False): + super().__init__() + head_length = max(1, int(np.log2(fpn_stride) - np.log2(base_stride))) + self.layers = nn.ModuleList() + for k in range(head_length): + in_c = in_channels if k == 0 else feature_channels + self.layers.append(BaseConv(in_c, feature_channels, 3, 1, "silu")) + if fpn_stride != base_stride: + self.layers.append(nn.Upsample(scale_factor=2, mode="bilinear", align_corners=align_corners)) + + def forward(self, x): + for layer in self.layers: + x = layer(x) + return x + + class MaskFeatFPN(nn.Module): def __init__( self, in_channels=[256, 256, 256], fpn_strides=[32, 16, 8], - feat_channels=256, + feature_channels=256, dropout_ratio=0.0, out_channels=256, align_corners=False, ): super().__init__() - if len(in_channels) != len(fpn_strides): - raise ValueError( - f"The lengths of 'in_channels' and 'fpn_strides' must be equal. " - f"Got len(in_channels)={len(in_channels)} and len(fpn_strides)={len(fpn_strides)}." - ) - reorder_index = np.argsort(fpn_strides, axis=0).tolist() in_channels = [in_channels[i] for i in reorder_index] fpn_strides = [fpn_strides[i] for i in reorder_index] @@ -705,17 +710,16 @@ def __init__( self.scale_heads = nn.ModuleList() for i in range(len(fpn_strides)): - head_length = max(1, int(np.log2(fpn_strides[i]) - np.log2(fpn_strides[0]))) - scale_head = [] - for k in range(head_length): - in_c = in_channels[i] if k == 0 else feat_channels - scale_head.append(nn.Sequential(BaseConv(in_c, feat_channels, 3, 1, "silu"))) - if fpn_strides[i] != fpn_strides[0]: - scale_head.append(nn.Upsample(scale_factor=2, mode="bilinear", align_corners=align_corners)) - - self.scale_heads.append(nn.Sequential(*scale_head)) - - self.output_conv = BaseConv(feat_channels, out_channels, 3, 1, "silu") + self.scale_heads.append( + ScaleHead( + in_channels=in_channels[i], + feature_channels=feature_channels, + fpn_stride=fpn_strides[i], + base_stride=fpn_strides[0], + align_corners=align_corners, + ) + ) + self.output_conv = BaseConv(feature_channels, out_channels, 3, 1, "silu") def forward(self, inputs): x = [inputs[i] for i in self.reorder_index] @@ -732,10 +736,22 @@ def forward(self, inputs): return output +class EncoderMaskOutput(nn.Module): + def __init__(self, in_channels, num_prototypes): + super().__init__() + self.base_conv = BaseConv(in_channels, in_channels, 3, 1, "silu") + self.conv = nn.Conv2d(in_channels, num_prototypes, kernel_size=1) + + def forward(self, x): + x = self.base_conv(x) + x = self.conv(x) + return x + + class PPDocLayoutV3HybridEncoder(RTDetrHybridEncoder): """ Main difference to `RTDetrHybridEncoder`: - 1. Mask Feature Head: Added `MaskFeatFPN` module (`self.mask_feat_head`) for document - specific mask feature generation. + 1. Mask Feature Head: Added `MaskFeatFPN` module (`self.mask_feature_head`) for document - specific mask feature generation. 2. Extra Conv Layers: Introduced `self.encoder_mask_lateral` and `self.encoder_mask_output` for mask feature processing and output. """ @@ -743,17 +759,16 @@ def __init__(self, config: PPDocLayoutV3Config): super().__init__() feat_strides = config.feat_strides - mask_feat_channels = config.mask_feat_channels - self.mask_feat_head = MaskFeatFPN( + mask_feature_channels = config.mask_feature_channels + self.mask_feature_head = MaskFeatFPN( [self.encoder_hidden_dim] * len(feat_strides), feat_strides, - feat_channels=mask_feat_channels[0], - out_channels=mask_feat_channels[1], + feature_channels=mask_feature_channels[0], + out_channels=mask_feature_channels[1], ) - self.encoder_mask_lateral = BaseConv(config.x4_feat_dim, mask_feat_channels[1], 3, 1, "silu") - self.encoder_mask_output = nn.Sequential( - BaseConv(mask_feat_channels[1], mask_feat_channels[1], 3, 1, "silu"), - nn.Conv2d(in_channels=mask_feat_channels[1], out_channels=config.num_prototypes, kernel_size=1), + self.encoder_mask_lateral = BaseConv(config.x4_feat_dim, mask_feature_channels[1], 3, 1, "silu") + self.encoder_mask_output = EncoderMaskOutput( + in_channels=mask_feature_channels[1], num_prototypes=config.num_prototypes ) def forward( @@ -786,14 +801,6 @@ def forward( Starting index of each feature map. valid_ratios (`torch.FloatTensor` of shape `(batch_size, num_feature_levels, 2)`): Ratio of valid area in each feature level. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -867,7 +874,7 @@ def forward( new_pan_feature_map = pan_block(fused_feature_map) pan_feature_maps.append(new_pan_feature_map) - mask_feat = self.mask_feat_head(pan_feature_maps) + mask_feat = self.mask_feature_head(pan_feature_maps) mask_feat = F.interpolate(mask_feat, scale_factor=2, mode="bilinear", align_corners=False) mask_feat += self.encoder_mask_lateral(x4_feat[0]) mask_feat = self.encoder_mask_output(mask_feat) @@ -937,15 +944,6 @@ def forward( Indexes for the start of each feature level. In range `[0, sequence_length]`. valid_ratios (`torch.FloatTensor` of shape `(batch_size, num_feature_levels, 2)`, *optional*): Ratio of valid area in each feature level. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -963,8 +961,8 @@ def forward( intermediate = () intermediate_reference_points = () intermediate_logits = () - dec_out_order_logits = () - dec_out_masks = () + decoder_out_order_logits = () + decoder_out_masks = () reference_points = F.sigmoid(reference_points) @@ -1009,7 +1007,7 @@ def forward( out_mask = torch.bmm(mask_query_embed, mask_feat.flatten(start_dim=2)).reshape( batch_size, mask_dim, mask_h, mask_w ) - dec_out_masks += (out_mask,) + decoder_out_masks += (out_mask,) if self.class_embed is not None: logits = self.class_embed(out_query) @@ -1018,7 +1016,7 @@ def forward( if order_head is not None and global_pointer is not None: valid_query = out_query[:, -self.num_queries :] if self.num_queries is not None else out_query order_logits = global_pointer(order_head(valid_query)) - dec_out_order_logits += (order_logits,) + decoder_out_order_logits += (order_logits,) if output_attentions: all_self_attns += (layer_outputs[1],) @@ -1032,8 +1030,8 @@ def forward( if self.class_embed is not None: intermediate_logits = torch.stack(intermediate_logits, dim=1) if order_head is not None and global_pointer is not None: - dec_out_order_logits = torch.stack(dec_out_order_logits, dim=1) - dec_out_masks = torch.stack(dec_out_masks, dim=1) + decoder_out_order_logits = torch.stack(decoder_out_order_logits, dim=1) + decoder_out_masks = torch.stack(decoder_out_masks, dim=1) # add hidden states from the last decoder layer if output_hidden_states: @@ -1047,8 +1045,8 @@ def forward( intermediate, intermediate_logits, intermediate_reference_points, - dec_out_order_logits, - dec_out_masks, + decoder_out_order_logits, + decoder_out_masks, all_hidden_states, all_self_attns, all_cross_attentions, @@ -1060,8 +1058,8 @@ def forward( intermediate_hidden_states=intermediate, intermediate_logits=intermediate_logits, intermediate_reference_points=intermediate_reference_points, - dec_out_order_logits=dec_out_order_logits, - dec_out_masks=dec_out_masks, + decoder_out_order_logits=decoder_out_order_logits, + decoder_out_masks=decoder_out_masks, hidden_states=all_hidden_states, attentions=all_self_attns, cross_attentions=all_cross_attentions, @@ -1333,8 +1331,8 @@ def forward( decoder_hidden_states=decoder_outputs.hidden_states, decoder_attentions=decoder_outputs.attentions, cross_attentions=decoder_outputs.cross_attentions, - out_order_logits=decoder_outputs.dec_out_order_logits, - out_masks=decoder_outputs.dec_out_masks, + out_order_logits=decoder_outputs.decoder_out_order_logits, + out_masks=decoder_outputs.decoder_out_masks, encoder_last_hidden_state=encoder_outputs.last_hidden_state, encoder_hidden_states=encoder_outputs.hidden_states, encoder_attentions=encoder_outputs.attentions, @@ -1348,6 +1346,7 @@ def forward( @dataclass +@auto_docstring class PPDocLayoutV3HybridEncoderOutput(BaseModelOutput): r""" mask_feat (`torch.FloatTensor` of shape `(batch_size, config.num_queries, 200, 200)`): diff --git a/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py index da2e43913f03..f64a04154406 100644 --- a/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py +++ b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py @@ -1,5 +1,5 @@ # coding = utf-8 -# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ from transformers import ( PPDocLayoutV3Config, PPDocLayoutV3ForObjectDetection, - PPDocLayoutV3ImageProcessor, + PPDocLayoutV3ImageProcessorFast, is_torch_available, is_vision_available, ) @@ -76,7 +76,7 @@ def __init__( activation_function="silu", eval_size=None, normalize_before=False, - mask_feat_channels=[32, 32], + mask_feature_channels=[32, 32], x4_feat_dim=32, # decoder PPDocLayoutV3Transformer d_model=32, @@ -121,7 +121,7 @@ def __init__( self.activation_function = activation_function self.eval_size = eval_size self.normalize_before = normalize_before - self.mask_feat_channels = mask_feat_channels + self.mask_feature_channels = mask_feature_channels self.x4_feat_dim = x4_feat_dim self.d_model = d_model self.num_queries = num_queries @@ -146,7 +146,6 @@ def __init__( def prepare_config_and_inputs(self): pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) config = self.get_config() - # config.num_labels = self.num_labels return config, pixel_values @@ -167,14 +166,6 @@ def get_config(self): "lr_mult_list": [0, 0.05, 0.05, 0.05, 0.05], "out_features": ["stage1", "stage2", "stage3", "stage4"], } - # depths=[3, 4, 6, 3], - # hidden_sizes=[256, 512, 1024, 2048], - # stem_channels=[3, 32, 48], - # stage_in_channels=[48, 128, 512, 1024], - # stage_mid_channels=[48, 96, 192, 384], - # stage_out_channels=[128, 512, 1024, 2048], - # stage_num_blocks=[1, 1, 3, 1], - return PPDocLayoutV3Config( backbone_config=backbone_config, encoder_hidden_dim=self.encoder_hidden_dim, @@ -191,7 +182,7 @@ def get_config(self): activation_function=self.activation_function, eval_size=self.eval_size, normalize_before=self.normalize_before, - mask_feat_channels=self.mask_feat_channels, + mask_feature_channels=self.mask_feature_channels, x4_feat_dim=self.x4_feat_dim, d_model=self.d_model, num_queries=self.num_queries, @@ -266,10 +257,6 @@ def test_resize_tokens_embeddings(self): def test_feed_forward_chunking(self): pass - # @unittest.skip(reason="PPDocLayoutV3 does not support this test") - # def test_model_is_small(self): - # pass - @unittest.skip(reason="PPDocLayoutV3 does not support training") def test_retain_grad_hidden_states_attentions(self): pass @@ -470,7 +457,7 @@ def setUp(self): model_path = "PaddlePaddle/PP-DocLayoutV3_safetensors" self.model = PPDocLayoutV3ForObjectDetection.from_pretrained(model_path).to(torch_device) self.image_processor = ( - PPDocLayoutV3ImageProcessor.from_pretrained(model_path) if is_vision_available() else None + PPDocLayoutV3ImageProcessorFast.from_pretrained(model_path) if is_vision_available() else None ) url = "https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/layout_demo.jpg" self.image = Image.open(requests.get(url, stream=True).raw) From 036d47868960bba98172dce1aabf4274efbdfd6b Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Thu, 22 Jan 2026 18:20:08 +0800 Subject: [PATCH 09/21] Update to adapt to the latest check_code_quality. --- src/transformers/models/auto/modeling_auto.py | 2 +- .../image_processing_pp_doclayout_v3_fast.py | 4 +- .../modeling_pp_doclayout_v3.py | 171 +++++++++--------- .../modular_pp_doclayout_v3.py | 83 +++++---- 4 files changed, 128 insertions(+), 132 deletions(-) diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 37b73a929880..6424bd7399f5 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -1030,8 +1030,8 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("dab-detr", "DabDetrForObjectDetection"), ("deformable_detr", "DeformableDetrForObjectDetection"), ("detr", "DetrForObjectDetection"), - ("pp_doclayout_v3", "PPDocLayoutV3ForObjectDetection"), ("lw_detr", "LwDetrForObjectDetection"), + ("pp_doclayout_v3", "PPDocLayoutV3ForObjectDetection"), ("rt_detr", "RTDetrForObjectDetection"), ("rt_detr_v2", "RTDetrV2ForObjectDetection"), ("table-transformer", "TableTransformerForObjectDetection"), diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py index 49ed5a2f04c5..176d8de293de 100644 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py @@ -18,8 +18,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Union - import torch from ...image_processing_utils_fast import BaseImageProcessorFast @@ -76,7 +74,7 @@ def post_process_object_detection( self, outputs, threshold: float = 0.5, - target_sizes: Optional[Union[TensorType, list[tuple]]] = None, + target_sizes: TensorType | list[tuple] | None = None, ): """ Converts the raw output of [`PPDocLayoutV3ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index 490993024221..bab4584cc761 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -21,7 +21,6 @@ import math import warnings from dataclasses import dataclass -from typing import Optional, Union import numpy as np import torch @@ -35,7 +34,7 @@ from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import compile_compatible_method_lru_cache -from ...utils import ModelOutput, auto_docstring, torch_int +from ...utils import ModelOutput, auto_docstring, torch_compilable_check, torch_int from ...utils.backbone_utils import load_backbone from .configuration_pp_doclayout_v3 import PPDocLayoutV3Config @@ -155,16 +154,16 @@ def __init__(self, config: PPDocLayoutV3Config, num_heads: int, n_points: int): self.disable_custom_kernels = config.disable_custom_kernels - def with_pos_embed(self, tensor: torch.Tensor, position_embeddings: Optional[Tensor]): + def with_pos_embed(self, tensor: torch.Tensor, position_embeddings: Tensor | None): return tensor if position_embeddings is None else tensor + position_embeddings def forward( self, hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, + attention_mask: torch.Tensor | None = None, encoder_hidden_states=None, encoder_attention_mask=None, - position_embeddings: Optional[torch.Tensor] = None, + position_embeddings: torch.Tensor | None = None, reference_points=None, spatial_shapes=None, spatial_shapes_list=None, @@ -178,10 +177,10 @@ def forward( batch_size, num_queries, _ = hidden_states.shape batch_size, sequence_length, _ = encoder_hidden_states.shape total_elements = sum(height * width for height, width in spatial_shapes_list) - if total_elements != sequence_length: - raise ValueError( - "Make sure to align the spatial shapes with the sequence length of the encoder hidden states" - ) + torch_compilable_check( + total_elements == sequence_length, + "Make sure to align the spatial shapes with the sequence length of the encoder hidden states", + ) value = self.value_proj(encoder_hidden_states) if attention_mask is not None: @@ -323,18 +322,18 @@ class PPDocLayoutV3DecoderOutput(ModelOutput): Stacked masks (masks of each layer of the decoder). """ - last_hidden_state: Optional[torch.FloatTensor] = None - intermediate_hidden_states: Optional[torch.FloatTensor] = None - intermediate_logits: Optional[torch.FloatTensor] = None - intermediate_reference_points: Optional[torch.FloatTensor] = None - intermediate_predicted_corners: Optional[torch.FloatTensor] = None - initial_reference_points: Optional[torch.FloatTensor] = None - hidden_states: Optional[tuple[torch.FloatTensor]] = None - attentions: Optional[tuple[torch.FloatTensor]] = None - cross_attentions: Optional[tuple[torch.FloatTensor]] = None + last_hidden_state: torch.FloatTensor | None = None + intermediate_hidden_states: torch.FloatTensor | None = None + intermediate_logits: torch.FloatTensor | None = None + intermediate_reference_points: torch.FloatTensor | None = None + intermediate_predicted_corners: torch.FloatTensor | None = None + initial_reference_points: torch.FloatTensor | None = None + hidden_states: tuple[torch.FloatTensor] | None = None + attentions: tuple[torch.FloatTensor] | None = None + cross_attentions: tuple[torch.FloatTensor] | None = None - decoder_out_order_logits: Optional[torch.FloatTensor] = None - decoder_out_masks: Optional[torch.FloatTensor] = None + decoder_out_order_logits: torch.FloatTensor | None = None + decoder_out_masks: torch.FloatTensor | None = None @dataclass @@ -379,27 +378,27 @@ class PPDocLayoutV3ModelOutput(ModelOutput): Stacked masks (masks of each layer of the decoder). """ - last_hidden_state: Optional[torch.FloatTensor] = None - intermediate_hidden_states: Optional[torch.FloatTensor] = None - intermediate_logits: Optional[torch.FloatTensor] = None - intermediate_reference_points: Optional[torch.FloatTensor] = None - intermediate_predicted_corners: Optional[torch.FloatTensor] = None - initial_reference_points: Optional[torch.FloatTensor] = None - decoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None - decoder_attentions: Optional[tuple[torch.FloatTensor]] = None - cross_attentions: Optional[tuple[torch.FloatTensor]] = None - encoder_last_hidden_state: Optional[torch.FloatTensor] = None - encoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None - encoder_attentions: Optional[tuple[torch.FloatTensor]] = None - init_reference_points: Optional[torch.FloatTensor] = None - enc_topk_logits: Optional[torch.FloatTensor] = None - enc_topk_bboxes: Optional[torch.FloatTensor] = None - enc_outputs_class: Optional[torch.FloatTensor] = None - enc_outputs_coord_logits: Optional[torch.FloatTensor] = None - denoising_meta_values: Optional[dict] = None - - out_order_logits: Optional[torch.FloatTensor] = None - out_masks: Optional[torch.FloatTensor] = None + last_hidden_state: torch.FloatTensor | None = None + intermediate_hidden_states: torch.FloatTensor | None = None + intermediate_logits: torch.FloatTensor | None = None + intermediate_reference_points: torch.FloatTensor | None = None + intermediate_predicted_corners: torch.FloatTensor | None = None + initial_reference_points: torch.FloatTensor | None = None + decoder_hidden_states: tuple[torch.FloatTensor] | None = None + decoder_attentions: tuple[torch.FloatTensor] | None = None + cross_attentions: tuple[torch.FloatTensor] | None = None + encoder_last_hidden_state: torch.FloatTensor | None = None + encoder_hidden_states: tuple[torch.FloatTensor] | None = None + encoder_attentions: tuple[torch.FloatTensor] | None = None + init_reference_points: torch.FloatTensor | None = None + enc_topk_logits: torch.FloatTensor | None = None + enc_topk_bboxes: torch.FloatTensor | None = None + enc_outputs_class: torch.FloatTensor | None = None + enc_outputs_coord_logits: torch.FloatTensor | None = None + denoising_meta_values: dict | None = None + + out_order_logits: torch.FloatTensor | None = None + out_masks: torch.FloatTensor | None = None # taken from https://github.com/facebookresearch/detr/blob/master/models/detr.py @@ -578,16 +577,16 @@ def __init__( def _reshape(self, tensor: torch.Tensor, seq_len: int, batch_size: int): return tensor.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - def with_pos_embed(self, tensor: torch.Tensor, position_embeddings: Optional[Tensor]): + def with_pos_embed(self, tensor: torch.Tensor, position_embeddings: Tensor | None): return tensor if position_embeddings is None else tensor + position_embeddings def forward( self, hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_embeddings: Optional[torch.Tensor] = None, + attention_mask: torch.Tensor | None = None, + position_embeddings: torch.Tensor | None = None, output_attentions: bool = False, - ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: """Input shape: Batch x Time x Channel""" batch_size, target_len, embed_dim = hidden_states.size() @@ -688,7 +687,7 @@ def forward( self, hidden_states: torch.Tensor, attention_mask: torch.Tensor, - position_embeddings: Optional[torch.Tensor] = None, + position_embeddings: torch.Tensor | None = None, output_attentions: bool = False, **kwargs, ): @@ -1050,14 +1049,14 @@ def __init__(self, config: PPDocLayoutV3Config): def forward( self, hidden_states: torch.Tensor, - position_embeddings: Optional[torch.Tensor] = None, + position_embeddings: torch.Tensor | None = None, reference_points=None, spatial_shapes=None, spatial_shapes_list=None, level_start_index=None, - encoder_hidden_states: Optional[torch.Tensor] = None, - encoder_attention_mask: Optional[torch.Tensor] = None, - output_attentions: Optional[bool] = False, + encoder_hidden_states: torch.Tensor | None = None, + encoder_attention_mask: torch.Tensor | None = None, + output_attentions: bool | None = False, ): """ Args: @@ -1722,14 +1721,14 @@ def generate_anchors(self, spatial_shapes=None, grid_size=0.05, device="cpu", dt def forward( self, pixel_values: torch.FloatTensor, - pixel_mask: Optional[torch.LongTensor] = None, - encoder_outputs: Optional[torch.FloatTensor] = None, - labels: Optional[list[dict]] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, + pixel_mask: torch.LongTensor | None = None, + encoder_outputs: torch.FloatTensor | None = None, + labels: list[dict] | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, **kwargs, - ) -> Union[tuple[torch.FloatTensor], PPDocLayoutV3ModelOutput]: + ) -> tuple[torch.FloatTensor] | PPDocLayoutV3ModelOutput: r""" inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you @@ -2027,28 +2026,28 @@ class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): Extra dictionary for the denoising related values """ - logits: Optional[torch.FloatTensor] = None - pred_boxes: Optional[torch.FloatTensor] = None - order_logits: Optional[torch.FloatTensor] = None - out_masks: Optional[torch.FloatTensor] = None - last_hidden_state: Optional[torch.FloatTensor] = None - intermediate_hidden_states: Optional[torch.FloatTensor] = None - intermediate_logits: Optional[torch.FloatTensor] = None - intermediate_reference_points: Optional[torch.FloatTensor] = None - intermediate_predicted_corners: Optional[torch.FloatTensor] = None - initial_reference_points: Optional[torch.FloatTensor] = None - decoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None - decoder_attentions: Optional[tuple[torch.FloatTensor]] = None - cross_attentions: Optional[tuple[torch.FloatTensor]] = None - encoder_last_hidden_state: Optional[torch.FloatTensor] = None - encoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None - encoder_attentions: Optional[tuple[torch.FloatTensor]] = None - init_reference_points: Optional[tuple[torch.FloatTensor]] = None - enc_topk_logits: Optional[torch.FloatTensor] = None - enc_topk_bboxes: Optional[torch.FloatTensor] = None - enc_outputs_class: Optional[torch.FloatTensor] = None - enc_outputs_coord_logits: Optional[torch.FloatTensor] = None - denoising_meta_values: Optional[dict] = None + logits: torch.FloatTensor | None = None + pred_boxes: torch.FloatTensor | None = None + order_logits: torch.FloatTensor | None = None + out_masks: torch.FloatTensor | None = None + last_hidden_state: torch.FloatTensor | None = None + intermediate_hidden_states: torch.FloatTensor | None = None + intermediate_logits: torch.FloatTensor | None = None + intermediate_reference_points: torch.FloatTensor | None = None + intermediate_predicted_corners: torch.FloatTensor | None = None + initial_reference_points: torch.FloatTensor | None = None + decoder_hidden_states: tuple[torch.FloatTensor] | None = None + decoder_attentions: tuple[torch.FloatTensor] | None = None + cross_attentions: tuple[torch.FloatTensor] | None = None + encoder_last_hidden_state: torch.FloatTensor | None = None + encoder_hidden_states: tuple[torch.FloatTensor] | None = None + encoder_attentions: tuple[torch.FloatTensor] | None = None + init_reference_points: tuple[torch.FloatTensor] | None = None + enc_topk_logits: torch.FloatTensor | None = None + enc_topk_bboxes: torch.FloatTensor | None = None + enc_outputs_class: torch.FloatTensor | None = None + enc_outputs_coord_logits: torch.FloatTensor | None = None + denoising_meta_values: dict | None = None @auto_docstring( @@ -2083,14 +2082,14 @@ def _set_aux_loss(self, outputs_class, outputs_coord): def forward( self, pixel_values: torch.FloatTensor, - pixel_mask: Optional[torch.LongTensor] = None, - encoder_outputs: Optional[torch.FloatTensor] = None, - labels: Optional[list[dict]] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, + pixel_mask: torch.LongTensor | None = None, + encoder_outputs: torch.FloatTensor | None = None, + labels: list[dict] | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, **kwargs, - ) -> Union[tuple[torch.FloatTensor], PPDocLayoutV3ForObjectDetectionOutput]: + ) -> tuple[torch.FloatTensor] | PPDocLayoutV3ForObjectDetectionOutput: r""" inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index ada65e485169..498aa7e454c8 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -14,7 +14,6 @@ import math from dataclasses import dataclass -from typing import Optional, Union import numpy as np import torch @@ -388,7 +387,7 @@ def post_process_object_detection( self, outputs, threshold: float = 0.5, - target_sizes: Optional[Union[TensorType, list[tuple]]] = None, + target_sizes: TensorType | list[tuple] | None = None, ): """ Converts the raw output of [`PPDocLayoutV3ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y, @@ -610,8 +609,8 @@ class PPDocLayoutV3DecoderOutput(RTDetrDecoderOutput): Stacked masks (masks of each layer of the decoder). """ - decoder_out_order_logits: Optional[torch.FloatTensor] = None - decoder_out_masks: Optional[torch.FloatTensor] = None + decoder_out_order_logits: torch.FloatTensor | None = None + decoder_out_masks: torch.FloatTensor | None = None @dataclass @@ -656,8 +655,8 @@ class PPDocLayoutV3ModelOutput(RTDetrModelOutput): Stacked masks (masks of each layer of the decoder). """ - out_order_logits: Optional[torch.FloatTensor] = None - out_masks: Optional[torch.FloatTensor] = None + out_order_logits: torch.FloatTensor | None = None + out_masks: torch.FloatTensor | None = None class PPDocLayoutV3MLPPredictionHead(RTDetrMLPPredictionHead): @@ -1094,14 +1093,14 @@ def __init__(self, config: PPDocLayoutV3Config): def forward( self, pixel_values: torch.FloatTensor, - pixel_mask: Optional[torch.LongTensor] = None, - encoder_outputs: Optional[torch.FloatTensor] = None, - labels: Optional[list[dict]] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, + pixel_mask: torch.LongTensor | None = None, + encoder_outputs: torch.FloatTensor | None = None, + labels: list[dict] | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, **kwargs, - ) -> Union[tuple[torch.FloatTensor], PPDocLayoutV3ModelOutput]: + ) -> tuple[torch.FloatTensor] | PPDocLayoutV3ModelOutput: r""" inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you @@ -1399,28 +1398,28 @@ class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): Extra dictionary for the denoising related values """ - logits: Optional[torch.FloatTensor] = None - pred_boxes: Optional[torch.FloatTensor] = None - order_logits: Optional[torch.FloatTensor] = None - out_masks: Optional[torch.FloatTensor] = None - last_hidden_state: Optional[torch.FloatTensor] = None - intermediate_hidden_states: Optional[torch.FloatTensor] = None - intermediate_logits: Optional[torch.FloatTensor] = None - intermediate_reference_points: Optional[torch.FloatTensor] = None - intermediate_predicted_corners: Optional[torch.FloatTensor] = None - initial_reference_points: Optional[torch.FloatTensor] = None - decoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None - decoder_attentions: Optional[tuple[torch.FloatTensor]] = None - cross_attentions: Optional[tuple[torch.FloatTensor]] = None - encoder_last_hidden_state: Optional[torch.FloatTensor] = None - encoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None - encoder_attentions: Optional[tuple[torch.FloatTensor]] = None - init_reference_points: Optional[tuple[torch.FloatTensor]] = None - enc_topk_logits: Optional[torch.FloatTensor] = None - enc_topk_bboxes: Optional[torch.FloatTensor] = None - enc_outputs_class: Optional[torch.FloatTensor] = None - enc_outputs_coord_logits: Optional[torch.FloatTensor] = None - denoising_meta_values: Optional[dict] = None + logits: torch.FloatTensor | None = None + pred_boxes: torch.FloatTensor | None = None + order_logits: torch.FloatTensor | None = None + out_masks: torch.FloatTensor | None = None + last_hidden_state: torch.FloatTensor | None = None + intermediate_hidden_states: torch.FloatTensor | None = None + intermediate_logits: torch.FloatTensor | None = None + intermediate_reference_points: torch.FloatTensor | None = None + intermediate_predicted_corners: torch.FloatTensor | None = None + initial_reference_points: torch.FloatTensor | None = None + decoder_hidden_states: tuple[torch.FloatTensor] | None = None + decoder_attentions: tuple[torch.FloatTensor] | None = None + cross_attentions: tuple[torch.FloatTensor] | None = None + encoder_last_hidden_state: torch.FloatTensor | None = None + encoder_hidden_states: tuple[torch.FloatTensor] | None = None + encoder_attentions: tuple[torch.FloatTensor] | None = None + init_reference_points: tuple[torch.FloatTensor] | None = None + enc_topk_logits: torch.FloatTensor | None = None + enc_topk_bboxes: torch.FloatTensor | None = None + enc_outputs_class: torch.FloatTensor | None = None + enc_outputs_coord_logits: torch.FloatTensor | None = None + denoising_meta_values: dict | None = None @auto_docstring( @@ -1452,14 +1451,14 @@ def __init__(self, config: PPDocLayoutV3Config): def forward( self, pixel_values: torch.FloatTensor, - pixel_mask: Optional[torch.LongTensor] = None, - encoder_outputs: Optional[torch.FloatTensor] = None, - labels: Optional[list[dict]] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, + pixel_mask: torch.LongTensor | None = None, + encoder_outputs: torch.FloatTensor | None = None, + labels: list[dict] | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, **kwargs, - ) -> Union[tuple[torch.FloatTensor], PPDocLayoutV3ForObjectDetectionOutput]: + ) -> tuple[torch.FloatTensor] | PPDocLayoutV3ForObjectDetectionOutput: r""" inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you From 0d6c65d6bdfa94526a235a624695229045c91162 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Thu, 22 Jan 2026 19:26:15 +0800 Subject: [PATCH 10/21] Set to use --- .../models/pp_doclayout_v3/configuration_pp_doclayout_v3.py | 4 ++++ .../models/pp_doclayout_v3/modular_pp_doclayout_v3.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py index f2c783aa498a..077842e92770 100644 --- a/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/configuration_pp_doclayout_v3.py @@ -47,6 +47,8 @@ class PPDocLayoutV3Config(PreTrainedConfig): The epsilon used by the layer normalization layers. batch_norm_eps (`float`, *optional*, defaults to 1e-05): The epsilon used by the batch normalization layers. + tie_word_embeddings (`bool`, *optional*, defaults to `True`): + Whether the model's input and output word embeddings should be tied. backbone_config (`Union[dict, "PreTrainedConfig"]`, *optional*): The configuration of the backbone model. backbone (`str`, *optional*): @@ -174,6 +176,7 @@ def __init__( initializer_bias_prior_prob=None, layer_norm_eps=1e-5, batch_norm_eps=1e-5, + tie_word_embeddings=True, # backbone backbone_config=None, backbone=None, @@ -227,6 +230,7 @@ def __init__( self.initializer_bias_prior_prob = initializer_bias_prior_prob self.layer_norm_eps = layer_norm_eps self.batch_norm_eps = batch_norm_eps + self.tie_word_embeddings = tie_word_embeddings if backbone_config is None and backbone is None: logger.info( diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index 498aa7e454c8..bcc6edde5ebf 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -77,6 +77,8 @@ class PPDocLayoutV3Config(PreTrainedConfig): The epsilon used by the layer normalization layers. batch_norm_eps (`float`, *optional*, defaults to 1e-05): The epsilon used by the batch normalization layers. + tie_word_embeddings (`bool`, *optional*, defaults to `True`): + Whether the model's input and output word embeddings should be tied. backbone_config (`Union[dict, "PreTrainedConfig"]`, *optional*): The configuration of the backbone model. backbone (`str`, *optional*): @@ -204,6 +206,7 @@ def __init__( initializer_bias_prior_prob=None, layer_norm_eps=1e-5, batch_norm_eps=1e-5, + tie_word_embeddings=True, # backbone backbone_config=None, backbone=None, @@ -257,6 +260,7 @@ def __init__( self.initializer_bias_prior_prob = initializer_bias_prior_prob self.layer_norm_eps = layer_norm_eps self.batch_norm_eps = batch_norm_eps + self.tie_word_embeddings = tie_word_embeddings if backbone_config is None and backbone is None: logger.info( From 20ad4561b64c98a96c4309a9c371e3ae23735534 Mon Sep 17 00:00:00 2001 From: vasqu Date: Thu, 22 Jan 2026 13:23:58 +0100 Subject: [PATCH 11/21] mark as xfail --- tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py index f64a04154406..33ecb5cd172e 100644 --- a/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py +++ b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py @@ -18,6 +18,7 @@ import math import unittest +import pytest import requests from parameterized import parameterized @@ -452,6 +453,7 @@ def test_attention_outputs(self): @require_torch @require_vision @slow +@pytest.mark.xfail(reason="Weigths will determine the values of these tests") class PPDocLayoutV3ModelIntegrationTest(unittest.TestCase): def setUp(self): model_path = "PaddlePaddle/PP-DocLayoutV3_safetensors" From cd97341e259b91fe45d655f835518bf8917afbe0 Mon Sep 17 00:00:00 2001 From: vasqu Date: Thu, 22 Jan 2026 13:34:43 +0100 Subject: [PATCH 12/21] fix wrong doc link --- docs/source/en/model_doc/pp_doclayout_v3.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/source/en/model_doc/pp_doclayout_v3.md b/docs/source/en/model_doc/pp_doclayout_v3.md index 05097b6f9a71..6ef5194dad7d 100644 --- a/docs/source/en/model_doc/pp_doclayout_v3.md +++ b/docs/source/en/model_doc/pp_doclayout_v3.md @@ -141,7 +141,3 @@ for result in results: ## PPDocLayoutV3ImageProcessorFast [[autodoc]] PPDocLayoutV3ImageProcessorFast - -## PPDocLayoutV3HybridEncoderOutput - -[[autodoc]] PPDocLayoutV3HybridEncoderOutput From 6cd1b46124ae4164a9b4f1a065b461818d5b31f5 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Thu, 22 Jan 2026 21:10:48 +0800 Subject: [PATCH 13/21] fix nits --- docs/source/en/model_doc/pp_doclayout_v3.md | 4 ++-- src/transformers/models/pp_doclayout_v3/__init__.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/source/en/model_doc/pp_doclayout_v3.md b/docs/source/en/model_doc/pp_doclayout_v3.md index 6ef5194dad7d..a9bdaeeb5b42 100644 --- a/docs/source/en/model_doc/pp_doclayout_v3.md +++ b/docs/source/en/model_doc/pp_doclayout_v3.md @@ -29,7 +29,7 @@ TBD. ### Single input inference -The example below demonstrates how to generate text with PaddleOCRVL using [`Pipeline`] or the [`AutoModel`]. +The example below demonstrates how to generate text with PP-DocLayoutV3 using [`Pipeline`] or the [`AutoModel`]. @@ -76,7 +76,7 @@ for result in results: ### Batched inference -PaddleOCRVL also supports batched inference. We advise users to use `padding_side="left"` when computing batched generation as it leads to more accurate results. Here is how you can do it with PaddleOCRVL using [`Pipeline`] or the [`AutoModel`]: +PP-DocLayoutV3 also supports batched inference. We advise users to use `padding_side="left"` when computing batched generation as it leads to more accurate results. Here is how you can do it with PP-DocLayoutV3 using [`Pipeline`] or the [`AutoModel`]: diff --git a/src/transformers/models/pp_doclayout_v3/__init__.py b/src/transformers/models/pp_doclayout_v3/__init__.py index cb53946c81dc..b0e74b2e8ef9 100644 --- a/src/transformers/models/pp_doclayout_v3/__init__.py +++ b/src/transformers/models/pp_doclayout_v3/__init__.py @@ -21,7 +21,6 @@ if TYPE_CHECKING: from .configuration_pp_doclayout_v3 import * - from .image_processing_pp_doclayout_v3 import * from .image_processing_pp_doclayout_v3_fast import * from .modeling_pp_doclayout_v3 import * else: From 7f25cd5a3c79e7041b203cc73cf550175c6d32a2 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Thu, 22 Jan 2026 21:15:17 +0800 Subject: [PATCH 14/21] remove padding_left --- docs/source/en/model_doc/pp_doclayout_v3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/model_doc/pp_doclayout_v3.md b/docs/source/en/model_doc/pp_doclayout_v3.md index a9bdaeeb5b42..36a18d5ffe20 100644 --- a/docs/source/en/model_doc/pp_doclayout_v3.md +++ b/docs/source/en/model_doc/pp_doclayout_v3.md @@ -76,7 +76,7 @@ for result in results: ### Batched inference -PP-DocLayoutV3 also supports batched inference. We advise users to use `padding_side="left"` when computing batched generation as it leads to more accurate results. Here is how you can do it with PP-DocLayoutV3 using [`Pipeline`] or the [`AutoModel`]: +PP-DocLayoutV3 also supports batched inference. Here is how you can do it with PP-DocLayoutV3 using [`Pipeline`] or the [`AutoModel`]: From cbbc37905412321554e84d4487b017fac6be7504 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Tue, 27 Jan 2026 22:38:25 +0800 Subject: [PATCH 15/21] update --- .../image_processing_pp_doclayout_v3_fast.py | 182 ++++++++++++++++- .../modeling_pp_doclayout_v3.py | 10 +- .../modular_pp_doclayout_v3.py | 192 +++++++++++++++++- 3 files changed, 368 insertions(+), 16 deletions(-) diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py index 176d8de293de..652689770f2c 100644 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py @@ -18,10 +18,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Optional + +import numpy as np import torch +import torchvision.transforms.v2.functional as tvF -from ...image_processing_utils_fast import BaseImageProcessorFast -from ...image_utils import PILImageResampling +from ...image_processing_utils_fast import BaseImageProcessorFast, BatchFeature +from ...image_transforms import group_images_by_shape, reorder_images +from ...image_utils import PILImageResampling, SizeDict from ...utils import auto_docstring, requires_backends from ...utils.generic import TensorType @@ -39,6 +44,56 @@ class PPDocLayoutV3ImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) + # We require `self.resize(..., antialias=False)` to approximate the output of `cv2.resize` + def _preprocess( + self, + images: list["torch.Tensor"], + do_resize: bool, + size: SizeDict, + interpolation: Optional["tvF.InterpolationMode"], + do_center_crop: bool, + crop_size: SizeDict, + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: float | list[float] | None, + image_std: float | list[float] | None, + do_pad: bool | None, + pad_size: SizeDict | None, + disable_grouping: bool | None, + return_tensors: str | TensorType | None, + **kwargs, + ) -> BatchFeature: + # Group images by size for batched resizing + grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping) + resized_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + if do_resize: + stacked_images = self.resize( + image=stacked_images, size=size, interpolation=interpolation, antialias=False + ) + resized_images_grouped[shape] = stacked_images + resized_images = reorder_images(resized_images_grouped, grouped_images_index) + + # Group images by size for further processing + # Needed in case do_resize is False, or resize returns images with different sizes + grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping) + processed_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + if do_center_crop: + stacked_images = self.center_crop(stacked_images, crop_size) + # Fused rescale and normalize + stacked_images = self.rescale_and_normalize( + stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std + ) + processed_images_grouped[shape] = stacked_images + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + + if do_pad: + processed_images = self.pad(processed_images, pad_size=pad_size, disable_grouping=disable_grouping) + + return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors) + def _get_order_seqs(self, order_logits): """ Computes the order sequences for a batch of inputs based on logits. @@ -70,6 +125,112 @@ def _get_order_seqs(self, order_logits): return order_seq + def angle_between_vectors(self, v1, v2): + unit_v1 = v1 / np.linalg.norm(v1) + unit_v2 = v2 / np.linalg.norm(v2) + dot_prod = np.clip(np.dot(unit_v1, unit_v2), -1.0, 1.0) + angle_rad = np.arccos(dot_prod) + return np.degrees(angle_rad) + + def is_convex(self, p_prev, p_curr, p_next): + v1 = p_curr - p_prev + v2 = p_next - p_curr + cross = v1[0] * v2[1] - v1[1] * v2[0] + return cross < 0 + + def extract_custom_vertices(self, polygon, sharp_angle_thresh=45): + poly = np.array(polygon) + n = len(poly) + res = [] + i = 0 + while i < n: + p_prev = poly[(i - 1) % n] + p_curr = poly[i] + p_next = poly[(i + 1) % n] + v1 = p_prev - p_curr + v2 = p_next - p_curr + angle = self.angle_between_vectors(v1, v2) + if self.is_convex(p_prev, p_curr, p_next): + if abs(angle - sharp_angle_thresh) < 1: + # Calculate the new point based on the direction of two vectors. + dir_vec = v1 / np.linalg.norm(v1) + v2 / np.linalg.norm(v2) + dir_vec = dir_vec / np.linalg.norm(dir_vec) + d = (np.linalg.norm(v1) + np.linalg.norm(v2)) / 2 + p_new = p_curr + dir_vec * d + res.append(tuple(p_new)) + else: + res.append(tuple(p_curr)) + i += 1 + return res + + def _mask2polygon(self, mask, epsilon_ratio=0.004): + """ + Postprocess mask by removing small noise. + Args: + mask (ndarray): The input mask of shape [H, W]. + epsilon_ratio (float): The ratio of epsilon. + Returns: + ndarray: The output mask after postprocessing. + """ + requires_backends(self._mask2polygon, ["cv2"]) + import cv2 + + cnts, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + if not cnts: + return None + + cnt = max(cnts, key=cv2.contourArea) + epsilon = epsilon_ratio * cv2.arcLength(cnt, True) + approx_cnt = cv2.approxPolyDP(cnt, epsilon, True) + polygon_points = approx_cnt.squeeze() + polygon_points = np.atleast_2d(polygon_points) + + polygon_points = self.extract_custom_vertices(polygon_points) + + return polygon_points + + def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): + requires_backends(self._extract_polygon_points_by_masks, ["cv2"]) + import cv2 + + scale_w, scale_h = scale_ratio[0] / 4, scale_ratio[1] / 4 + mask_height, mask_width = masks.shape[1:] + polygon_points = [] + + for i in range(len(boxes)): + x_min, y_min, x_max, y_max = boxes[i].astype(np.int32) + box_w, box_h = x_max - x_min, y_max - y_min + + # default rect + rect = np.array( + [[x_min, y_min], [x_max, y_min], [x_max, y_max], [x_min, y_max]], + dtype=np.float32, + ) + + if box_w <= 0 or box_h <= 0: + polygon_points.append(rect) + continue + + # crop mask + x_s = np.clip([int(round(x_min * scale_w)), int(round(x_max * scale_w))], 0, mask_width) + y_s = np.clip([int(round(y_min * scale_h)), int(round(y_max * scale_h))], 0, mask_height) + cropped = masks[i, y_s[0] : y_s[1], x_s[0] : x_s[1]] + + # resize mask to match box size + resized_mask = cv2.resize(cropped.astype(np.uint8), (box_w, box_h), interpolation=cv2.INTER_NEAREST) + + polygon = self._mask2polygon(resized_mask) + if polygon is not None and len(polygon) < 4: + polygon_points.append(rect) + continue + if polygon is not None and len(polygon) > 0: + polygon = polygon + np.array([x_min, y_min]) + + polygon_points.append(polygon) + + return polygon_points + def post_process_object_detection( self, outputs, @@ -84,13 +245,14 @@ def post_process_object_detection( outputs ([`DetrObjectDetectionOutput`]): Raw outputs of the model. Returns: - `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image + `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels, boxes and polygon_points for an image in the batch as predicted by the model. """ requires_backends(self, ["torch"]) boxes = outputs.pred_boxes logits = outputs.logits order_logits = outputs.order_logits + masks = outputs.out_masks order_seqs = self._get_order_seqs(order_logits) @@ -119,17 +281,29 @@ def post_process_object_detection( labels = index % num_classes index = index // num_classes boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) + masks = masks.gather( + dim=1, index=index.unsqueeze(-1).unsqueeze(-1).repeat(1, 1, masks.shape[-2], masks.shape[-1]) + ) + masks = (masks.sigmoid() > threshold).int() order_seqs = order_seqs.gather(dim=1, index=index) results = [] - for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): + for score, label, box, order_seq, target_size, mask in zip( + scores, labels, boxes, order_seqs, target_sizes, masks + ): order_seq = order_seq[score >= threshold] order_seq, indices = torch.sort(order_seq) + polygon_points = self._extract_polygon_points_by_masks( + box[score >= threshold][indices].detach().numpy(), + mask[score >= threshold][indices].detach().numpy(), + [self.size["width"] / target_size[1], self.size["height"] / target_size[0]], + ) results.append( { "scores": score[score >= threshold][indices], "labels": label[score >= threshold][indices], "boxes": box[score >= threshold][indices], + "polygon_points": polygon_points, "order_seq": order_seq, } ) diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index bab4584cc761..28a3c0a8bc85 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -39,7 +39,7 @@ from .configuration_pp_doclayout_v3 import PPDocLayoutV3Config -class GlobalPointer(nn.Module): +class PPDocLayoutV3GlobalPointer(nn.Module): def __init__(self, config): super().__init__() self.head_size = config.global_pointer_head_size @@ -1275,7 +1275,7 @@ def forward( if order_head is not None and global_pointer is not None: valid_query = out_query[:, -self.num_queries :] if self.num_queries is not None else out_query - order_logits = global_pointer(order_head(valid_query)) + order_logits = global_pointer(order_head[idx](valid_query)) decoder_out_order_logits += (order_logits,) if output_attentions: @@ -1667,8 +1667,10 @@ def __init__(self, config: PPDocLayoutV3Config): self.decoder_input_proj = nn.ModuleList(decoder_input_proj_list) self.decoder = PPDocLayoutV3Decoder(config) - self.decoder_order_head = nn.Linear(config.d_model, config.d_model) - self.decoder_global_pointer = GlobalPointer(config) + self.decoder_order_head = nn.ModuleList( + [nn.Linear(config.d_model, config.d_model) for _ in range(config.decoder_layers)] + ) + self.decoder_global_pointer = PPDocLayoutV3GlobalPointer(config) self.decoder_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) self.decoder.class_embed = self.enc_score_head self.decoder.bbox_embed = self.enc_bbox_head diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index bcc6edde5ebf..f13dd7656f45 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -14,20 +14,25 @@ import math from dataclasses import dataclass +from typing import Optional import numpy as np import torch import torch.nn.functional as F +import torchvision.transforms.v2.functional as tvF from torch import nn from ... import initialization as init from ...configuration_utils import PreTrainedConfig from ...image_processing_utils_fast import ( BaseImageProcessorFast, + BatchFeature, ) -from ...image_utils import ( - PILImageResampling, +from ...image_transforms import ( + group_images_by_shape, + reorder_images, ) +from ...image_utils import PILImageResampling, SizeDict from ...modeling_outputs import BaseModelOutput from ...utils import ( ModelOutput, @@ -356,6 +361,56 @@ class PPDocLayoutV3ImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) + # We require `self.resize(..., antialias=False)` to approximate the output of `cv2.resize` + def _preprocess( + self, + images: list["torch.Tensor"], + do_resize: bool, + size: SizeDict, + interpolation: Optional["tvF.InterpolationMode"], + do_center_crop: bool, + crop_size: SizeDict, + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: float | list[float] | None, + image_std: float | list[float] | None, + do_pad: bool | None, + pad_size: SizeDict | None, + disable_grouping: bool | None, + return_tensors: str | TensorType | None, + **kwargs, + ) -> BatchFeature: + # Group images by size for batched resizing + grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping) + resized_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + if do_resize: + stacked_images = self.resize( + image=stacked_images, size=size, interpolation=interpolation, antialias=False + ) + resized_images_grouped[shape] = stacked_images + resized_images = reorder_images(resized_images_grouped, grouped_images_index) + + # Group images by size for further processing + # Needed in case do_resize is False, or resize returns images with different sizes + grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping) + processed_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + if do_center_crop: + stacked_images = self.center_crop(stacked_images, crop_size) + # Fused rescale and normalize + stacked_images = self.rescale_and_normalize( + stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std + ) + processed_images_grouped[shape] = stacked_images + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + + if do_pad: + processed_images = self.pad(processed_images, pad_size=pad_size, disable_grouping=disable_grouping) + + return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors) + def _get_order_seqs(self, order_logits): """ Computes the order sequences for a batch of inputs based on logits. @@ -387,6 +442,112 @@ def _get_order_seqs(self, order_logits): return order_seq + def angle_between_vectors(self, v1, v2): + unit_v1 = v1 / np.linalg.norm(v1) + unit_v2 = v2 / np.linalg.norm(v2) + dot_prod = np.clip(np.dot(unit_v1, unit_v2), -1.0, 1.0) + angle_rad = np.arccos(dot_prod) + return np.degrees(angle_rad) + + def is_convex(self, p_prev, p_curr, p_next): + v1 = p_curr - p_prev + v2 = p_next - p_curr + cross = v1[0] * v2[1] - v1[1] * v2[0] + return cross < 0 + + def extract_custom_vertices(self, polygon, sharp_angle_thresh=45): + poly = np.array(polygon) + n = len(poly) + res = [] + i = 0 + while i < n: + p_prev = poly[(i - 1) % n] + p_curr = poly[i] + p_next = poly[(i + 1) % n] + v1 = p_prev - p_curr + v2 = p_next - p_curr + angle = self.angle_between_vectors(v1, v2) + if self.is_convex(p_prev, p_curr, p_next): + if abs(angle - sharp_angle_thresh) < 1: + # Calculate the new point based on the direction of two vectors. + dir_vec = v1 / np.linalg.norm(v1) + v2 / np.linalg.norm(v2) + dir_vec = dir_vec / np.linalg.norm(dir_vec) + d = (np.linalg.norm(v1) + np.linalg.norm(v2)) / 2 + p_new = p_curr + dir_vec * d + res.append(tuple(p_new)) + else: + res.append(tuple(p_curr)) + i += 1 + return res + + def _mask2polygon(self, mask, epsilon_ratio=0.004): + """ + Postprocess mask by removing small noise. + Args: + mask (ndarray): The input mask of shape [H, W]. + epsilon_ratio (float): The ratio of epsilon. + Returns: + ndarray: The output mask after postprocessing. + """ + requires_backends(self._mask2polygon, ["cv2"]) + import cv2 + + cnts, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + if not cnts: + return None + + cnt = max(cnts, key=cv2.contourArea) + epsilon = epsilon_ratio * cv2.arcLength(cnt, True) + approx_cnt = cv2.approxPolyDP(cnt, epsilon, True) + polygon_points = approx_cnt.squeeze() + polygon_points = np.atleast_2d(polygon_points) + + polygon_points = self.extract_custom_vertices(polygon_points) + + return polygon_points + + def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): + requires_backends(self._extract_polygon_points_by_masks, ["cv2"]) + import cv2 + + scale_w, scale_h = scale_ratio[0] / 4, scale_ratio[1] / 4 + mask_height, mask_width = masks.shape[1:] + polygon_points = [] + + for i in range(len(boxes)): + x_min, y_min, x_max, y_max = boxes[i].astype(np.int32) + box_w, box_h = x_max - x_min, y_max - y_min + + # default rect + rect = np.array( + [[x_min, y_min], [x_max, y_min], [x_max, y_max], [x_min, y_max]], + dtype=np.float32, + ) + + if box_w <= 0 or box_h <= 0: + polygon_points.append(rect) + continue + + # crop mask + x_s = np.clip([int(round(x_min * scale_w)), int(round(x_max * scale_w))], 0, mask_width) + y_s = np.clip([int(round(y_min * scale_h)), int(round(y_max * scale_h))], 0, mask_height) + cropped = masks[i, y_s[0] : y_s[1], x_s[0] : x_s[1]] + + # resize mask to match box size + resized_mask = cv2.resize(cropped.astype(np.uint8), (box_w, box_h), interpolation=cv2.INTER_NEAREST) + + polygon = self._mask2polygon(resized_mask) + if polygon is not None and len(polygon) < 4: + polygon_points.append(rect) + continue + if polygon is not None and len(polygon) > 0: + polygon = polygon + np.array([x_min, y_min]) + + polygon_points.append(polygon) + + return polygon_points + def post_process_object_detection( self, outputs, @@ -401,13 +562,14 @@ def post_process_object_detection( outputs ([`DetrObjectDetectionOutput`]): Raw outputs of the model. Returns: - `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image + `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels, boxes and polygon_points for an image in the batch as predicted by the model. """ requires_backends(self, ["torch"]) boxes = outputs.pred_boxes logits = outputs.logits order_logits = outputs.order_logits + masks = outputs.out_masks order_seqs = self._get_order_seqs(order_logits) @@ -436,17 +598,29 @@ def post_process_object_detection( labels = index % num_classes index = index // num_classes boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) + masks = masks.gather( + dim=1, index=index.unsqueeze(-1).unsqueeze(-1).repeat(1, 1, masks.shape[-2], masks.shape[-1]) + ) + masks = (masks.sigmoid() > threshold).int() order_seqs = order_seqs.gather(dim=1, index=index) results = [] - for score, label, box, order_seq in zip(scores, labels, boxes, order_seqs): + for score, label, box, order_seq, target_size, mask in zip( + scores, labels, boxes, order_seqs, target_sizes, masks + ): order_seq = order_seq[score >= threshold] order_seq, indices = torch.sort(order_seq) + polygon_points = self._extract_polygon_points_by_masks( + box[score >= threshold][indices].detach().numpy(), + mask[score >= threshold][indices].detach().numpy(), + [self.size["width"] / target_size[1], self.size["height"] / target_size[0]], + ) results.append( { "scores": score[score >= threshold][indices], "labels": label[score >= threshold][indices], "boxes": box[score >= threshold][indices], + "polygon_points": polygon_points, "order_seq": order_seq, } ) @@ -454,7 +628,7 @@ def post_process_object_detection( return results -class GlobalPointer(nn.Module): +class PPDocLayoutV3GlobalPointer(nn.Module): def __init__(self, config): super().__init__() self.head_size = config.global_pointer_head_size @@ -1018,7 +1192,7 @@ def forward( if order_head is not None and global_pointer is not None: valid_query = out_query[:, -self.num_queries :] if self.num_queries is not None else out_query - order_logits = global_pointer(order_head(valid_query)) + order_logits = global_pointer(order_head[idx](valid_query)) decoder_out_order_logits += (order_logits,) if output_attentions: @@ -1081,8 +1255,10 @@ def __init__(self, config: PPDocLayoutV3Config): encoder_input_proj_list = [] self.encoder_input_proj = nn.ModuleList(encoder_input_proj_list[1:]) - self.decoder_order_head = nn.Linear(config.d_model, config.d_model) - self.decoder_global_pointer = GlobalPointer(config) + self.decoder_order_head = nn.ModuleList( + [nn.Linear(config.d_model, config.d_model) for _ in range(config.decoder_layers)] + ) + self.decoder_global_pointer = PPDocLayoutV3GlobalPointer(config) self.decoder_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) self.decoder = PPDocLayoutV3Decoder(config) self.decoder.class_embed = self.enc_score_head From bc2b9bd84fbd6ac4fa443e4d6dee9aac24398a28 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Wed, 28 Jan 2026 00:59:22 +0800 Subject: [PATCH 16/21] update --- docs/source/en/model_doc/pp_doclayout_v3.md | 2 +- .../image_processing_pp_doclayout_v3_fast.py | 58 ++++++-------- .../modeling_pp_doclayout_v3.py | 20 ++--- .../modular_pp_doclayout_v3.py | 78 +++++++++---------- 4 files changed, 71 insertions(+), 87 deletions(-) diff --git a/docs/source/en/model_doc/pp_doclayout_v3.md b/docs/source/en/model_doc/pp_doclayout_v3.md index 36a18d5ffe20..947c628aacc1 100644 --- a/docs/source/en/model_doc/pp_doclayout_v3.md +++ b/docs/source/en/model_doc/pp_doclayout_v3.md @@ -13,7 +13,7 @@ specific language governing permissions and limitations under the License. rendered properly in your Markdown viewer. --> -*This model was released on 2026-01-22 and added to Hugging Face Transformers on 2026-01-22.* +*This model was released on 2026-01-28 and added to Hugging Face Transformers on 2026-01-28.* # PP-DocLayoutV3 diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py index 652689770f2c..0785fdfe0075 100644 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py @@ -125,41 +125,32 @@ def _get_order_seqs(self, order_logits): return order_seq - def angle_between_vectors(self, v1, v2): - unit_v1 = v1 / np.linalg.norm(v1) - unit_v2 = v2 / np.linalg.norm(v2) - dot_prod = np.clip(np.dot(unit_v1, unit_v2), -1.0, 1.0) - angle_rad = np.arccos(dot_prod) - return np.degrees(angle_rad) - - def is_convex(self, p_prev, p_curr, p_next): - v1 = p_curr - p_prev - v2 = p_next - p_curr - cross = v1[0] * v2[1] - v1[1] * v2[0] - return cross < 0 - def extract_custom_vertices(self, polygon, sharp_angle_thresh=45): poly = np.array(polygon) n = len(poly) res = [] i = 0 while i < n: - p_prev = poly[(i - 1) % n] - p_curr = poly[i] - p_next = poly[(i + 1) % n] - v1 = p_prev - p_curr - v2 = p_next - p_curr - angle = self.angle_between_vectors(v1, v2) - if self.is_convex(p_prev, p_curr, p_next): + previous_point = poly[(i - 1) % n] + current_point = poly[i] + next_point = poly[(i + 1) % n] + vector_1 = previous_point - current_point + vector_2 = next_point - current_point + cross_product_value = (vector_1[1] * vector_2[0]) - (vector_1[0] * vector_2[1]) + if cross_product_value < 0: + angle_cos = np.clip( + (vector_1 @ vector_2) / (np.linalg.norm(vector_1) * np.linalg.norm(vector_2)), -1.0, 1.0 + ) + angle = np.degrees(np.arccos(angle_cos)) if abs(angle - sharp_angle_thresh) < 1: # Calculate the new point based on the direction of two vectors. - dir_vec = v1 / np.linalg.norm(v1) + v2 / np.linalg.norm(v2) + dir_vec = vector_1 / np.linalg.norm(vector_1) + vector_2 / np.linalg.norm(vector_2) dir_vec = dir_vec / np.linalg.norm(dir_vec) - d = (np.linalg.norm(v1) + np.linalg.norm(v2)) / 2 - p_new = p_curr + dir_vec * d + d = (np.linalg.norm(vector_1) + np.linalg.norm(vector_2)) / 2 + p_new = current_point + dir_vec * d res.append(tuple(p_new)) else: - res.append(tuple(p_curr)) + res.append(tuple(current_point)) i += 1 return res @@ -172,15 +163,14 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): Returns: ndarray: The output mask after postprocessing. """ - requires_backends(self._mask2polygon, ["cv2"]) import cv2 - cnts, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - if not cnts: + if not contours: return None - cnt = max(cnts, key=cv2.contourArea) + cnt = max(contours, key=cv2.contourArea) epsilon = epsilon_ratio * cv2.arcLength(cnt, True) approx_cnt = cv2.approxPolyDP(cnt, epsilon, True) polygon_points = approx_cnt.squeeze() @@ -191,7 +181,7 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): return polygon_points def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): - requires_backends(self._extract_polygon_points_by_masks, ["cv2"]) + requires_backends(self, ["cv2"]) import cv2 scale_w, scale_h = scale_ratio[0] / 4, scale_ratio[1] / 4 @@ -213,12 +203,14 @@ def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): continue # crop mask - x_s = np.clip([int(round(x_min * scale_w)), int(round(x_max * scale_w))], 0, mask_width) - y_s = np.clip([int(round(y_min * scale_h)), int(round(y_max * scale_h))], 0, mask_height) - cropped = masks[i, y_s[0] : y_s[1], x_s[0] : x_s[1]] + x_coordinates = [int(round((x_min * scale_w).item())), int(round((x_max * scale_w).item()))] + x_start, x_end = np.clip(x_coordinates, 0, mask_width) + y_coordinates = [int(round((y_min * scale_h).item())), int(round((y_max * scale_h).item()))] + y_start, y_end = np.clip(y_coordinates, 0, mask_height) + cropped_mask = masks[i, y_start:y_end, x_start:x_end] # resize mask to match box size - resized_mask = cv2.resize(cropped.astype(np.uint8), (box_w, box_h), interpolation=cv2.INTER_NEAREST) + resized_mask = cv2.resize(cropped_mask.astype(np.uint8), (box_w, box_h), interpolation=cv2.INTER_NEAREST) polygon = self._mask2polygon(resized_mask) if polygon is not None and len(polygon) < 4: diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index 28a3c0a8bc85..837fc68430b4 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -424,7 +424,7 @@ def forward(self, x): return x -class BaseConv(nn.Module): +class PPDocLayoutV3ConvLayer(nn.Module): def __init__( self, in_channels: int, out_channels: int, kernel_size: int = 3, stride: int = 1, activation: str = "relu" ): @@ -442,14 +442,14 @@ def forward(self, input: Tensor) -> Tensor: return hidden_state -class ScaleHead(nn.Module): +class PPDocLayoutV3ScaleHead(nn.Module): def __init__(self, in_channels, feature_channels, fpn_stride, base_stride, align_corners=False): super().__init__() head_length = max(1, int(np.log2(fpn_stride) - np.log2(base_stride))) self.layers = nn.ModuleList() for k in range(head_length): in_c = in_channels if k == 0 else feature_channels - self.layers.append(BaseConv(in_c, feature_channels, 3, 1, "silu")) + self.layers.append(PPDocLayoutV3ConvLayer(in_c, feature_channels, 3, 1, "silu")) if fpn_stride != base_stride: self.layers.append(nn.Upsample(scale_factor=2, mode="bilinear", align_corners=align_corners)) @@ -459,7 +459,7 @@ def forward(self, x): return x -class MaskFeatFPN(nn.Module): +class PPDocLayoutV3MaskFeatFPN(nn.Module): def __init__( self, in_channels=[256, 256, 256], @@ -485,7 +485,7 @@ def __init__( self.scale_heads = nn.ModuleList() for i in range(len(fpn_strides)): self.scale_heads.append( - ScaleHead( + PPDocLayoutV3ScaleHead( in_channels=in_channels[i], feature_channels=feature_channels, fpn_stride=fpn_strides[i], @@ -493,7 +493,7 @@ def __init__( align_corners=align_corners, ) ) - self.output_conv = BaseConv(feature_channels, out_channels, 3, 1, "silu") + self.output_conv = PPDocLayoutV3ConvLayer(feature_channels, out_channels, 3, 1, "silu") def forward(self, inputs): x = [inputs[i] for i in self.reorder_index] @@ -513,7 +513,7 @@ def forward(self, inputs): class EncoderMaskOutput(nn.Module): def __init__(self, in_channels, num_prototypes): super().__init__() - self.base_conv = BaseConv(in_channels, in_channels, 3, 1, "silu") + self.base_conv = PPDocLayoutV3ConvLayer(in_channels, in_channels, 3, 1, "silu") self.conv = nn.Conv2d(in_channels, num_prototypes, kernel_size=1) def forward(self, x): @@ -816,7 +816,7 @@ def forward(self, src, src_mask=None, pos_embed=None, output_attentions: bool = class PPDocLayoutV3HybridEncoder(nn.Module): """ Main difference to `RTDetrHybridEncoder`: - 1. Mask Feature Head: Added `MaskFeatFPN` module (`self.mask_feature_head`) for document - specific mask feature generation. + 1. Mask Feature Head: Added `PPDocLayoutV3MaskFeatFPN` module (`self.mask_feature_head`) for document - specific mask feature generation. 2. Extra Conv Layers: Introduced `self.encoder_mask_lateral` and `self.encoder_mask_output` for mask feature processing and output. """ @@ -872,13 +872,13 @@ def __init__(self, config: PPDocLayoutV3Config): feat_strides = config.feat_strides mask_feature_channels = config.mask_feature_channels - self.mask_feature_head = MaskFeatFPN( + self.mask_feature_head = PPDocLayoutV3MaskFeatFPN( [self.encoder_hidden_dim] * len(feat_strides), feat_strides, feature_channels=mask_feature_channels[0], out_channels=mask_feature_channels[1], ) - self.encoder_mask_lateral = BaseConv(config.x4_feat_dim, mask_feature_channels[1], 3, 1, "silu") + self.encoder_mask_lateral = PPDocLayoutV3ConvLayer(config.x4_feat_dim, mask_feature_channels[1], 3, 1, "silu") self.encoder_mask_output = EncoderMaskOutput( in_channels=mask_feature_channels[1], num_prototypes=config.num_prototypes ) diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index f13dd7656f45..54c68b4f9c62 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -442,41 +442,32 @@ def _get_order_seqs(self, order_logits): return order_seq - def angle_between_vectors(self, v1, v2): - unit_v1 = v1 / np.linalg.norm(v1) - unit_v2 = v2 / np.linalg.norm(v2) - dot_prod = np.clip(np.dot(unit_v1, unit_v2), -1.0, 1.0) - angle_rad = np.arccos(dot_prod) - return np.degrees(angle_rad) - - def is_convex(self, p_prev, p_curr, p_next): - v1 = p_curr - p_prev - v2 = p_next - p_curr - cross = v1[0] * v2[1] - v1[1] * v2[0] - return cross < 0 - def extract_custom_vertices(self, polygon, sharp_angle_thresh=45): poly = np.array(polygon) n = len(poly) res = [] i = 0 while i < n: - p_prev = poly[(i - 1) % n] - p_curr = poly[i] - p_next = poly[(i + 1) % n] - v1 = p_prev - p_curr - v2 = p_next - p_curr - angle = self.angle_between_vectors(v1, v2) - if self.is_convex(p_prev, p_curr, p_next): + previous_point = poly[(i - 1) % n] + current_point = poly[i] + next_point = poly[(i + 1) % n] + vector_1 = previous_point - current_point + vector_2 = next_point - current_point + cross_product_value = (vector_1[1] * vector_2[0]) - (vector_1[0] * vector_2[1]) + if cross_product_value < 0: + angle_cos = np.clip( + (vector_1 @ vector_2) / (np.linalg.norm(vector_1) * np.linalg.norm(vector_2)), -1.0, 1.0 + ) + angle = np.degrees(np.arccos(angle_cos)) if abs(angle - sharp_angle_thresh) < 1: # Calculate the new point based on the direction of two vectors. - dir_vec = v1 / np.linalg.norm(v1) + v2 / np.linalg.norm(v2) + dir_vec = vector_1 / np.linalg.norm(vector_1) + vector_2 / np.linalg.norm(vector_2) dir_vec = dir_vec / np.linalg.norm(dir_vec) - d = (np.linalg.norm(v1) + np.linalg.norm(v2)) / 2 - p_new = p_curr + dir_vec * d + d = (np.linalg.norm(vector_1) + np.linalg.norm(vector_2)) / 2 + p_new = current_point + dir_vec * d res.append(tuple(p_new)) else: - res.append(tuple(p_curr)) + res.append(tuple(current_point)) i += 1 return res @@ -489,15 +480,14 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): Returns: ndarray: The output mask after postprocessing. """ - requires_backends(self._mask2polygon, ["cv2"]) import cv2 - cnts, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - if not cnts: + if not contours: return None - cnt = max(cnts, key=cv2.contourArea) + cnt = max(contours, key=cv2.contourArea) epsilon = epsilon_ratio * cv2.arcLength(cnt, True) approx_cnt = cv2.approxPolyDP(cnt, epsilon, True) polygon_points = approx_cnt.squeeze() @@ -508,7 +498,7 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): return polygon_points def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): - requires_backends(self._extract_polygon_points_by_masks, ["cv2"]) + requires_backends(self, ["cv2"]) import cv2 scale_w, scale_h = scale_ratio[0] / 4, scale_ratio[1] / 4 @@ -530,12 +520,14 @@ def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): continue # crop mask - x_s = np.clip([int(round(x_min * scale_w)), int(round(x_max * scale_w))], 0, mask_width) - y_s = np.clip([int(round(y_min * scale_h)), int(round(y_max * scale_h))], 0, mask_height) - cropped = masks[i, y_s[0] : y_s[1], x_s[0] : x_s[1]] + x_coordinates = [int(round((x_min * scale_w).item())), int(round((x_max * scale_w).item()))] + x_start, x_end = np.clip(x_coordinates, 0, mask_width) + y_coordinates = [int(round((y_min * scale_h).item())), int(round((y_max * scale_h).item()))] + y_start, y_end = np.clip(y_coordinates, 0, mask_height) + cropped_mask = masks[i, y_start:y_end, x_start:x_end] # resize mask to match box size - resized_mask = cv2.resize(cropped.astype(np.uint8), (box_w, box_h), interpolation=cv2.INTER_NEAREST) + resized_mask = cv2.resize(cropped_mask.astype(np.uint8), (box_w, box_h), interpolation=cv2.INTER_NEAREST) polygon = self._mask2polygon(resized_mask) if polygon is not None and len(polygon) < 4: @@ -841,18 +833,18 @@ class PPDocLayoutV3MLPPredictionHead(RTDetrMLPPredictionHead): pass -class BaseConv(ResNetConvLayer): +class PPDocLayoutV3ConvLayer(ResNetConvLayer): pass -class ScaleHead(nn.Module): +class PPDocLayoutV3ScaleHead(nn.Module): def __init__(self, in_channels, feature_channels, fpn_stride, base_stride, align_corners=False): super().__init__() head_length = max(1, int(np.log2(fpn_stride) - np.log2(base_stride))) self.layers = nn.ModuleList() for k in range(head_length): in_c = in_channels if k == 0 else feature_channels - self.layers.append(BaseConv(in_c, feature_channels, 3, 1, "silu")) + self.layers.append(PPDocLayoutV3ConvLayer(in_c, feature_channels, 3, 1, "silu")) if fpn_stride != base_stride: self.layers.append(nn.Upsample(scale_factor=2, mode="bilinear", align_corners=align_corners)) @@ -862,7 +854,7 @@ def forward(self, x): return x -class MaskFeatFPN(nn.Module): +class PPDocLayoutV3MaskFeatFPN(nn.Module): def __init__( self, in_channels=[256, 256, 256], @@ -888,7 +880,7 @@ def __init__( self.scale_heads = nn.ModuleList() for i in range(len(fpn_strides)): self.scale_heads.append( - ScaleHead( + PPDocLayoutV3ScaleHead( in_channels=in_channels[i], feature_channels=feature_channels, fpn_stride=fpn_strides[i], @@ -896,7 +888,7 @@ def __init__( align_corners=align_corners, ) ) - self.output_conv = BaseConv(feature_channels, out_channels, 3, 1, "silu") + self.output_conv = PPDocLayoutV3ConvLayer(feature_channels, out_channels, 3, 1, "silu") def forward(self, inputs): x = [inputs[i] for i in self.reorder_index] @@ -916,7 +908,7 @@ def forward(self, inputs): class EncoderMaskOutput(nn.Module): def __init__(self, in_channels, num_prototypes): super().__init__() - self.base_conv = BaseConv(in_channels, in_channels, 3, 1, "silu") + self.base_conv = PPDocLayoutV3ConvLayer(in_channels, in_channels, 3, 1, "silu") self.conv = nn.Conv2d(in_channels, num_prototypes, kernel_size=1) def forward(self, x): @@ -928,7 +920,7 @@ def forward(self, x): class PPDocLayoutV3HybridEncoder(RTDetrHybridEncoder): """ Main difference to `RTDetrHybridEncoder`: - 1. Mask Feature Head: Added `MaskFeatFPN` module (`self.mask_feature_head`) for document - specific mask feature generation. + 1. Mask Feature Head: Added `PPDocLayoutV3MaskFeatFPN` module (`self.mask_feature_head`) for document - specific mask feature generation. 2. Extra Conv Layers: Introduced `self.encoder_mask_lateral` and `self.encoder_mask_output` for mask feature processing and output. """ @@ -937,13 +929,13 @@ def __init__(self, config: PPDocLayoutV3Config): feat_strides = config.feat_strides mask_feature_channels = config.mask_feature_channels - self.mask_feature_head = MaskFeatFPN( + self.mask_feature_head = PPDocLayoutV3MaskFeatFPN( [self.encoder_hidden_dim] * len(feat_strides), feat_strides, feature_channels=mask_feature_channels[0], out_channels=mask_feature_channels[1], ) - self.encoder_mask_lateral = BaseConv(config.x4_feat_dim, mask_feature_channels[1], 3, 1, "silu") + self.encoder_mask_lateral = PPDocLayoutV3ConvLayer(config.x4_feat_dim, mask_feature_channels[1], 3, 1, "silu") self.encoder_mask_output = EncoderMaskOutput( in_channels=mask_feature_channels[1], num_prototypes=config.num_prototypes ) From 9f95f83650f15ab06a06c8ce0a5c34970dcd7263 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Wed, 28 Jan 2026 01:24:47 +0800 Subject: [PATCH 17/21] use is_cv2_available --- .../image_processing_pp_doclayout_v3_fast.py | 13 ++++++------- .../pp_doclayout_v3/modular_pp_doclayout_v3.py | 12 ++++++------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py index 0785fdfe0075..3c9260250888 100644 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py @@ -27,10 +27,14 @@ from ...image_processing_utils_fast import BaseImageProcessorFast, BatchFeature from ...image_transforms import group_images_by_shape, reorder_images from ...image_utils import PILImageResampling, SizeDict -from ...utils import auto_docstring, requires_backends +from ...utils import auto_docstring, is_cv2_available, requires_backends from ...utils.generic import TensorType +if is_cv2_available(): + import cv2 + + @auto_docstring class PPDocLayoutV3ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC @@ -163,8 +167,6 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): Returns: ndarray: The output mask after postprocessing. """ - import cv2 - contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: @@ -181,9 +183,6 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): return polygon_points def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): - requires_backends(self, ["cv2"]) - import cv2 - scale_w, scale_h = scale_ratio[0] / 4, scale_ratio[1] / 4 mask_height, mask_width = masks.shape[1:] polygon_points = [] @@ -240,7 +239,7 @@ def post_process_object_detection( `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels, boxes and polygon_points for an image in the batch as predicted by the model. """ - requires_backends(self, ["torch"]) + requires_backends(self, ["torch", "cv2"]) boxes = outputs.pred_boxes logits = outputs.logits order_logits = outputs.order_logits diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index 54c68b4f9c62..8b44469a7536 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -37,6 +37,7 @@ from ...utils import ( ModelOutput, auto_docstring, + is_cv2_available, logging, requires_backends, ) @@ -59,6 +60,10 @@ ) +if is_cv2_available(): + import cv2 + + logger = logging.get_logger(__name__) @@ -480,8 +485,6 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): Returns: ndarray: The output mask after postprocessing. """ - import cv2 - contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: @@ -498,9 +501,6 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): return polygon_points def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): - requires_backends(self, ["cv2"]) - import cv2 - scale_w, scale_h = scale_ratio[0] / 4, scale_ratio[1] / 4 mask_height, mask_width = masks.shape[1:] polygon_points = [] @@ -557,7 +557,7 @@ def post_process_object_detection( `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels, boxes and polygon_points for an image in the batch as predicted by the model. """ - requires_backends(self, ["torch"]) + requires_backends(self, ["torch", "cv2"]) boxes = outputs.pred_boxes logits = outputs.logits order_logits = outputs.order_logits From 3de4614f0b30c20a63acda3914e209a5c8caf522 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Wed, 28 Jan 2026 01:38:17 +0800 Subject: [PATCH 18/21] fix nits --- .../models/pp_doclayout_v3/__init__.py | 2 +- .../image_processing_pp_doclayout_v3_fast.py | 28 ++++++++-------- .../modeling_pp_doclayout_v3.py | 4 +-- .../modular_pp_doclayout_v3.py | 32 +++++++++---------- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/transformers/models/pp_doclayout_v3/__init__.py b/src/transformers/models/pp_doclayout_v3/__init__.py index b0e74b2e8ef9..55281a5ad722 100644 --- a/src/transformers/models/pp_doclayout_v3/__init__.py +++ b/src/transformers/models/pp_doclayout_v3/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 The HuggingFace Team. All rights reserved. +# Copyright 2026 The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py index 3c9260250888..565280100d45 100644 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py @@ -150,9 +150,9 @@ def extract_custom_vertices(self, polygon, sharp_angle_thresh=45): # Calculate the new point based on the direction of two vectors. dir_vec = vector_1 / np.linalg.norm(vector_1) + vector_2 / np.linalg.norm(vector_2) dir_vec = dir_vec / np.linalg.norm(dir_vec) - d = (np.linalg.norm(vector_1) + np.linalg.norm(vector_2)) / 2 - p_new = current_point + dir_vec * d - res.append(tuple(p_new)) + step_size = (np.linalg.norm(vector_1) + np.linalg.norm(vector_2)) / 2 + new_point = current_point + dir_vec * step_size + res.append(tuple(new_point)) else: res.append(tuple(current_point)) i += 1 @@ -172,10 +172,10 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): if not contours: return None - cnt = max(contours, key=cv2.contourArea) - epsilon = epsilon_ratio * cv2.arcLength(cnt, True) - approx_cnt = cv2.approxPolyDP(cnt, epsilon, True) - polygon_points = approx_cnt.squeeze() + contours = max(contours, key=cv2.contourArea) + epsilon = epsilon_ratio * cv2.arcLength(contours, True) + approx_contours = cv2.approxPolyDP(contours, epsilon, True) + polygon_points = approx_contours.squeeze() polygon_points = np.atleast_2d(polygon_points) polygon_points = self.extract_custom_vertices(polygon_points) @@ -183,7 +183,7 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): return polygon_points def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): - scale_w, scale_h = scale_ratio[0] / 4, scale_ratio[1] / 4 + scale_width, scale_height = scale_ratio[0] / 4, scale_ratio[1] / 4 mask_height, mask_width = masks.shape[1:] polygon_points = [] @@ -202,9 +202,9 @@ def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): continue # crop mask - x_coordinates = [int(round((x_min * scale_w).item())), int(round((x_max * scale_w).item()))] + x_coordinates = [int(round((x_min * scale_width).item())), int(round((x_max * scale_width).item()))] x_start, x_end = np.clip(x_coordinates, 0, mask_width) - y_coordinates = [int(round((y_min * scale_h).item())), int(round((y_max * scale_h).item()))] + y_coordinates = [int(round((y_min * scale_height).item())), int(round((y_max * scale_height).item()))] y_start, y_end = np.clip(y_coordinates, 0, mask_height) cropped_mask = masks[i, y_start:y_end, x_start:x_end] @@ -258,11 +258,11 @@ def post_process_object_detection( "Make sure that you pass in as many target sizes as the batch dimension of the logits" ) if isinstance(target_sizes, list): - img_h, img_w = torch.as_tensor(target_sizes).unbind(1) + img_height, img_width = torch.as_tensor(target_sizes).unbind(1) else: - img_h, img_w = target_sizes.unbind(1) - scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) - boxes = boxes * scale_fct[:, None, :] + img_height, img_width = target_sizes.unbind(1) + scale_factor = torch.stack([img_width, img_height, img_width, img_height], dim=1).to(boxes.device) + boxes = boxes * scale_factor[:, None, :] num_top_queries = logits.shape[1] num_classes = logits.shape[2] diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index 837fc68430b4..1e483c28608c 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -510,7 +510,7 @@ def forward(self, inputs): return output -class EncoderMaskOutput(nn.Module): +class PPDocLayoutV3EncoderMaskOutput(nn.Module): def __init__(self, in_channels, num_prototypes): super().__init__() self.base_conv = PPDocLayoutV3ConvLayer(in_channels, in_channels, 3, 1, "silu") @@ -879,7 +879,7 @@ def __init__(self, config: PPDocLayoutV3Config): out_channels=mask_feature_channels[1], ) self.encoder_mask_lateral = PPDocLayoutV3ConvLayer(config.x4_feat_dim, mask_feature_channels[1], 3, 1, "silu") - self.encoder_mask_output = EncoderMaskOutput( + self.encoder_mask_output = PPDocLayoutV3EncoderMaskOutput( in_channels=mask_feature_channels[1], num_prototypes=config.num_prototypes ) diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index 8b44469a7536..32886942bec8 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -468,9 +468,9 @@ def extract_custom_vertices(self, polygon, sharp_angle_thresh=45): # Calculate the new point based on the direction of two vectors. dir_vec = vector_1 / np.linalg.norm(vector_1) + vector_2 / np.linalg.norm(vector_2) dir_vec = dir_vec / np.linalg.norm(dir_vec) - d = (np.linalg.norm(vector_1) + np.linalg.norm(vector_2)) / 2 - p_new = current_point + dir_vec * d - res.append(tuple(p_new)) + step_size = (np.linalg.norm(vector_1) + np.linalg.norm(vector_2)) / 2 + new_point = current_point + dir_vec * step_size + res.append(tuple(new_point)) else: res.append(tuple(current_point)) i += 1 @@ -490,10 +490,10 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): if not contours: return None - cnt = max(contours, key=cv2.contourArea) - epsilon = epsilon_ratio * cv2.arcLength(cnt, True) - approx_cnt = cv2.approxPolyDP(cnt, epsilon, True) - polygon_points = approx_cnt.squeeze() + contours = max(contours, key=cv2.contourArea) + epsilon = epsilon_ratio * cv2.arcLength(contours, True) + approx_contours = cv2.approxPolyDP(contours, epsilon, True) + polygon_points = approx_contours.squeeze() polygon_points = np.atleast_2d(polygon_points) polygon_points = self.extract_custom_vertices(polygon_points) @@ -501,7 +501,7 @@ def _mask2polygon(self, mask, epsilon_ratio=0.004): return polygon_points def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): - scale_w, scale_h = scale_ratio[0] / 4, scale_ratio[1] / 4 + scale_width, scale_height = scale_ratio[0] / 4, scale_ratio[1] / 4 mask_height, mask_width = masks.shape[1:] polygon_points = [] @@ -520,9 +520,9 @@ def _extract_polygon_points_by_masks(self, boxes, masks, scale_ratio): continue # crop mask - x_coordinates = [int(round((x_min * scale_w).item())), int(round((x_max * scale_w).item()))] + x_coordinates = [int(round((x_min * scale_width).item())), int(round((x_max * scale_width).item()))] x_start, x_end = np.clip(x_coordinates, 0, mask_width) - y_coordinates = [int(round((y_min * scale_h).item())), int(round((y_max * scale_h).item()))] + y_coordinates = [int(round((y_min * scale_height).item())), int(round((y_max * scale_height).item()))] y_start, y_end = np.clip(y_coordinates, 0, mask_height) cropped_mask = masks[i, y_start:y_end, x_start:x_end] @@ -576,11 +576,11 @@ def post_process_object_detection( "Make sure that you pass in as many target sizes as the batch dimension of the logits" ) if isinstance(target_sizes, list): - img_h, img_w = torch.as_tensor(target_sizes).unbind(1) + img_height, img_width = torch.as_tensor(target_sizes).unbind(1) else: - img_h, img_w = target_sizes.unbind(1) - scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device) - boxes = boxes * scale_fct[:, None, :] + img_height, img_width = target_sizes.unbind(1) + scale_factor = torch.stack([img_width, img_height, img_width, img_height], dim=1).to(boxes.device) + boxes = boxes * scale_factor[:, None, :] num_top_queries = logits.shape[1] num_classes = logits.shape[2] @@ -905,7 +905,7 @@ def forward(self, inputs): return output -class EncoderMaskOutput(nn.Module): +class PPDocLayoutV3EncoderMaskOutput(nn.Module): def __init__(self, in_channels, num_prototypes): super().__init__() self.base_conv = PPDocLayoutV3ConvLayer(in_channels, in_channels, 3, 1, "silu") @@ -936,7 +936,7 @@ def __init__(self, config: PPDocLayoutV3Config): out_channels=mask_feature_channels[1], ) self.encoder_mask_lateral = PPDocLayoutV3ConvLayer(config.x4_feat_dim, mask_feature_channels[1], 3, 1, "silu") - self.encoder_mask_output = EncoderMaskOutput( + self.encoder_mask_output = PPDocLayoutV3EncoderMaskOutput( in_channels=mask_feature_channels[1], num_prototypes=config.num_prototypes ) From 5b9046bb1c70693377340959c74ad53d2512872d Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Wed, 28 Jan 2026 01:51:55 +0800 Subject: [PATCH 19/21] move _tied_weights_keys --- .../models/pp_doclayout_v3/modeling_pp_doclayout_v3.py | 9 +++++---- .../models/pp_doclayout_v3/modular_pp_doclayout_v3.py | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index 1e483c28608c..b166619ea9a4 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -1597,6 +1597,11 @@ def mask_to_box_coordinate(mask, dtype): """ ) class PPDocLayoutV3Model(PPDocLayoutV3PreTrainedModel): + _tied_weights_keys = { + "decoder.class_embed": "enc_score_head", + "decoder.bbox_embed": "enc_bbox_head", + } + def __init__(self, config: PPDocLayoutV3Config): super().__init__(config) @@ -2063,10 +2068,6 @@ class PPDocLayoutV3ForObjectDetection(PPDocLayoutV3PreTrainedModel): # We can't initialize the model on meta device as some weights are modified during the initialization _no_split_modules = None _keys_to_ignore_on_load_missing = ["num_batches_tracked", "rel_pos_y_bias", "rel_pos_x_bias"] - _tied_weights_keys = { - "model.decoder.class_embed": "model.enc_score_head", - "model.decoder.bbox_embed": "model.enc_bbox_head", - } def __init__(self, config: PPDocLayoutV3Config): super().__init__(config) diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index 32886942bec8..ca736052791f 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -1241,6 +1241,11 @@ def forward( """ ) class PPDocLayoutV3Model(RTDetrModel): + _tied_weights_keys = { + "decoder.class_embed": "enc_score_head", + "decoder.bbox_embed": "enc_bbox_head", + } + def __init__(self, config: PPDocLayoutV3Config): super().__init__(config) @@ -1602,10 +1607,6 @@ class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): ) class PPDocLayoutV3ForObjectDetection(RTDetrForObjectDetection, PPDocLayoutV3PreTrainedModel): _keys_to_ignore_on_load_missing = ["num_batches_tracked", "rel_pos_y_bias", "rel_pos_x_bias"] - _tied_weights_keys = { - "model.decoder.class_embed": "model.enc_score_head", - "model.decoder.bbox_embed": "model.enc_bbox_head", - } def __init__(self, config: PPDocLayoutV3Config): super().__init__(config) From 1d5bfab473003579920b6b0f85d51ca75e9f3f1b Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Thu, 29 Jan 2026 02:00:31 +0800 Subject: [PATCH 20/21] update --- docs/source/en/model_doc/pp_doclayout_v3.md | 18 +++++++++----- .../modeling_pp_doclayout_v3.py | 23 ++++++------------ .../modular_pp_doclayout_v3.py | 24 +++++++------------ 3 files changed, 27 insertions(+), 38 deletions(-) diff --git a/docs/source/en/model_doc/pp_doclayout_v3.md b/docs/source/en/model_doc/pp_doclayout_v3.md index 947c628aacc1..cd771f75d3cc 100644 --- a/docs/source/en/model_doc/pp_doclayout_v3.md +++ b/docs/source/en/model_doc/pp_doclayout_v3.md @@ -13,7 +13,7 @@ specific language governing permissions and limitations under the License. rendered properly in your Markdown viewer. --> -*This model was released on 2026-01-28 and added to Hugging Face Transformers on 2026-01-28.* +*This model was released on 2026-01-29 and added to Hugging Face Transformers on 2026-01-29.* # PP-DocLayoutV3 @@ -23,7 +23,13 @@ rendered properly in your Markdown viewer. ## Overview -TBD. +**PP-DocLayoutV3** is a unified and high-efficiency model designed for comprehensive layout analysis. It addresses the challenges of complex physical distortionsβ€”such as skewing, curving, and adverse lightingβ€”by integrating instance segmentation and reading order prediction into a single, end-to-end framework. + +## Model Architecture + +PP-DocLayoutV3 evolves from a traditional detection-based approach to a robust instance segmentation architecture built upon the RT-DETR framework. Instead of simple bounding boxes, it utilizes a mask-based detection head to predict pixel-accurate segments for layout elements. + +Unlike its predecessor, PP-DocLayoutV3 eliminates decoupled stages by embedding a Global Pointer Mechanism directly within the Transformer decoder layers. This allows the model to concurrently output classification labels, precise masks, and logical reading orders in a single forward pass, significantly reducing latency while enhancing parsing precision on complex document layouts. ## Usage @@ -65,10 +71,10 @@ inputs = image_processor(images=image, return_tensors="pt") outputs = model(**inputs) results = image_processor.post_process_object_detection(outputs, target_sizes=[image.size[::-1]]) for result in results: - for idx, (score, label_id, box) in enumerate(zip(result["scores"], result["labels"], result["boxes"])): + for idx, (score, label_id, box, polygon_points) in enumerate(zip(result["scores"], result["labels"], result["boxes"], result["polygon_points"])): score, label = score.item(), label_id.item() box = [round(i, 2) for i in box.tolist()] - print(f"Order {idx + 1}: {model.config.id2label[label]}: {score:.2f} {box}") + print(f"Order {idx + 1}: {model.config.id2label[label]}, score: {score:.2f}, box: {box}, polygon_points: {polygon_points}") ``` @@ -116,10 +122,10 @@ outputs = model(**inputs) results = image_processor.post_process_object_detection(outputs, target_sizes=target_sizes) for result in results: print("result:") - for idx, (score, label_id, box) in enumerate(zip(result["scores"], result["labels"], result["boxes"])): + for idx, (score, label_id, box, polygon_points) in enumerate(zip(result["scores"], result["labels"], result["boxes"], result["polygon_points"])): score, label = score.item(), label_id.item() box = [round(i, 2) for i in box.tolist()] - print(f"Order {idx + 1}: {model.config.id2label[label]}: {score:.2f} {box}") + print(f"Order {idx + 1}: {model.config.id2label[label]}, score: {score:.2f}, box: {box}, polygon_points: {polygon_points}") ``` diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index b166619ea9a4..3891a23dfe89 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -238,20 +238,7 @@ class PPDocLayoutV3PreTrainedModel(PreTrainedModel): @torch.no_grad() def _init_weights(self, module): """Initialize the weights""" - if isinstance(module, PPDocLayoutV3ForObjectDetection): - if module.model.decoder.class_embed is not None: - layer = module.model.decoder.class_embed - prior_prob = self.config.initializer_bias_prior_prob or 1 / (self.config.num_labels + 1) - bias = float(-math.log((1 - prior_prob) / prior_prob)) - init.xavier_uniform_(layer.weight) - init.constant_(layer.bias, bias) - - if module.model.decoder.bbox_embed is not None: - layer = module.model.decoder.bbox_embed - init.constant_(layer.layers[-1].weight, 0) - init.constant_(layer.layers[-1].bias, 0) - - elif isinstance(module, PPDocLayoutV3MultiscaleDeformableAttention): + if isinstance(module, PPDocLayoutV3MultiscaleDeformableAttention): init.constant_(module.sampling_offsets.weight, 0.0) default_dtype = torch.get_default_dtype() thetas = torch.arange(module.n_heads, dtype=torch.int64).to(default_dtype) * ( @@ -279,6 +266,8 @@ def _init_weights(self, module): bias = float(-math.log((1 - prior_prob) / prior_prob)) init.xavier_uniform_(module.enc_score_head.weight) init.constant_(module.enc_score_head.bias, bias) + init.xavier_uniform_(module.decoder.class_embed.weight) + init.constant_(module.decoder.class_embed.bias, bias) elif isinstance(module, (nn.Linear, nn.Conv2d, nn.BatchNorm2d)): init.normal_(module.weight, mean=0.0, std=self.config.initializer_range) @@ -1677,8 +1666,10 @@ def __init__(self, config: PPDocLayoutV3Config): ) self.decoder_global_pointer = PPDocLayoutV3GlobalPointer(config) self.decoder_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) - self.decoder.class_embed = self.enc_score_head - self.decoder.bbox_embed = self.enc_bbox_head + self.decoder.class_embed = nn.Linear(config.d_model, config.num_labels) + self.decoder.bbox_embed = PPDocLayoutV3MLPPredictionHead( + config, config.d_model, config.d_model, 4, num_layers=3 + ) self.mask_enhanced = config.mask_enhanced self.mask_query_head = PPDocLayoutV3MLPPredictionHead( diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index ca736052791f..78f566c28c77 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -648,20 +648,8 @@ class PPDocLayoutV3MultiscaleDeformableAttention(RTDetrMultiscaleDeformableAtten class PPDocLayoutV3PreTrainedModel(RTDetrPreTrainedModel): @torch.no_grad() def _init_weights(self, module): - if isinstance(module, PPDocLayoutV3ForObjectDetection): - if module.model.decoder.class_embed is not None: - layer = module.model.decoder.class_embed - prior_prob = self.config.initializer_bias_prior_prob or 1 / (self.config.num_labels + 1) - bias = float(-math.log((1 - prior_prob) / prior_prob)) - init.xavier_uniform_(layer.weight) - init.constant_(layer.bias, bias) - - if module.model.decoder.bbox_embed is not None: - layer = module.model.decoder.bbox_embed - init.constant_(layer.layers[-1].weight, 0) - init.constant_(layer.layers[-1].bias, 0) - - elif isinstance(module, PPDocLayoutV3MultiscaleDeformableAttention): + """Initialize the weights""" + if isinstance(module, PPDocLayoutV3MultiscaleDeformableAttention): init.constant_(module.sampling_offsets.weight, 0.0) default_dtype = torch.get_default_dtype() thetas = torch.arange(module.n_heads, dtype=torch.int64).to(default_dtype) * ( @@ -689,6 +677,8 @@ def _init_weights(self, module): bias = float(-math.log((1 - prior_prob) / prior_prob)) init.xavier_uniform_(module.enc_score_head.weight) init.constant_(module.enc_score_head.bias, bias) + init.xavier_uniform_(module.decoder.class_embed.weight) + init.constant_(module.decoder.class_embed.bias, bias) elif isinstance(module, (nn.Linear, nn.Conv2d, nn.BatchNorm2d)): init.normal_(module.weight, mean=0.0, std=self.config.initializer_range) @@ -1258,8 +1248,10 @@ def __init__(self, config: PPDocLayoutV3Config): self.decoder_global_pointer = PPDocLayoutV3GlobalPointer(config) self.decoder_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps) self.decoder = PPDocLayoutV3Decoder(config) - self.decoder.class_embed = self.enc_score_head - self.decoder.bbox_embed = self.enc_bbox_head + self.decoder.class_embed = nn.Linear(config.d_model, config.num_labels) + self.decoder.bbox_embed = PPDocLayoutV3MLPPredictionHead( + config, config.d_model, config.d_model, 4, num_layers=3 + ) self.mask_enhanced = config.mask_enhanced self.mask_query_head = PPDocLayoutV3MLPPredictionHead( From 4f4f99fedd5bb09f37404cd1786d3fa776eeff88 Mon Sep 17 00:00:00 2001 From: zhangyue66 Date: Thu, 29 Jan 2026 17:39:16 +0800 Subject: [PATCH 21/21] update --- .../image_processing_pp_doclayout_v3_fast.py | 4 +- .../modular_pp_doclayout_v3.py | 4 +- .../test_modeling_pp_doclayout_v3.py | 44 +++++++++++-------- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py index 565280100d45..b98876d8ee47 100644 --- a/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py +++ b/src/transformers/models/pp_doclayout_v3/image_processing_pp_doclayout_v3_fast.py @@ -285,8 +285,8 @@ def post_process_object_detection( order_seq = order_seq[score >= threshold] order_seq, indices = torch.sort(order_seq) polygon_points = self._extract_polygon_points_by_masks( - box[score >= threshold][indices].detach().numpy(), - mask[score >= threshold][indices].detach().numpy(), + box[score >= threshold][indices].detach().cpu().numpy(), + mask[score >= threshold][indices].detach().cpu().numpy(), [self.size["width"] / target_size[1], self.size["height"] / target_size[0]], ) results.append( diff --git a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py index 78f566c28c77..38ea39c0201a 100644 --- a/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modular_pp_doclayout_v3.py @@ -603,8 +603,8 @@ def post_process_object_detection( order_seq = order_seq[score >= threshold] order_seq, indices = torch.sort(order_seq) polygon_points = self._extract_polygon_points_by_masks( - box[score >= threshold][indices].detach().numpy(), - mask[score >= threshold][indices].detach().numpy(), + box[score >= threshold][indices].detach().cpu().numpy(), + mask[score >= threshold][indices].detach().cpu().numpy(), [self.size["width"] / target_size[1], self.size["height"] / target_size[0]], ) results.append( diff --git a/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py index 33ecb5cd172e..604f5e755a62 100644 --- a/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py +++ b/tests/models/pp_doclayout_v3/test_modeling_pp_doclayout_v3.py @@ -18,7 +18,6 @@ import math import unittest -import pytest import requests from parameterized import parameterized @@ -448,12 +447,9 @@ def test_attention_outputs(self): ) -# TODO: -# Later, we will determine these values based on the latest weights. @require_torch @require_vision @slow -@pytest.mark.xfail(reason="Weigths will determine the values of these tests") class PPDocLayoutV3ModelIntegrationTest(unittest.TestCase): def setUp(self): model_path = "PaddlePaddle/PP-DocLayoutV3_safetensors" @@ -472,28 +468,28 @@ def test_inference_object_detection_head(self): expected_shape_logits = torch.Size((1, 300, self.model.config.num_labels)) expected_logits = torch.tensor( - [[-5.6224, -6.5667, -4.9352], [-5.7931, -5.5543, -5.6476], [-4.5742, -5.0603, -7.2864]] + [[-4.7670, -6.2655, -6.3641], [-4.9534, -5.8549, -6.4938], [-5.1931, -6.2573, -6.6023]] ).to(torch_device) self.assertEqual(outputs.logits.shape, expected_shape_logits) - torch.testing.assert_close(outputs.logits[0, :3, :3], expected_logits, rtol=2e-4, atol=2e-4) + torch.testing.assert_close(outputs.logits[0, :3, :3], expected_logits, rtol=2e-4, atol=2e-2) expected_shape_boxes = torch.Size((1, 300, 4)) expected_boxes = torch.tensor( - [[0.4000, 0.9702, 0.3897], [0.3642, 0.3164, 0.3212], [0.3716, 0.1786, 0.3386]] + [[0.3725, 0.1789, 0.3373], [0.7256, 0.2672, 0.3378], [0.7247, 0.1389, 0.3352]] ).to(torch_device) self.assertEqual(outputs.pred_boxes.shape, expected_shape_boxes) - torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_boxes, rtol=2e-4, atol=2e-4) + torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_boxes, rtol=2e-4, atol=2e-2) expected_shape_order_logits = torch.Size((1, 300, 300)) expected_order_logits = torch.tensor( [ - [-10000.0000, -937.0416, -1045.3816], - [-10000.0000, -10000.0000, -343.8752], + [-10000.0000, 2333.5664, 1632.4893], + [-10000.0000, -10000.0000, -1068.3279], [-10000.0000, -10000.0000, -10000.0000], ] ).to(torch_device) self.assertEqual(outputs.order_logits.shape, expected_shape_order_logits) - torch.testing.assert_close(outputs.order_logits[0, :3, :3], expected_order_logits, rtol=2e-4, atol=2e-4) + torch.testing.assert_close(outputs.order_logits[0, :3, :3], expected_order_logits, rtol=2e-2, atol=2e-2) # verify postprocessing results = self.image_processor.post_process_object_detection( @@ -501,19 +497,29 @@ def test_inference_object_detection_head(self): )[0] expected_scores = torch.tensor( - [0.9834, 0.9485, 0.9837, 0.9728, 0.9741, 0.9770, 0.9508, 0.9390, 0.9482, 0.8391, 0.9358, 0.8249, 0.9095] + [0.9605, 0.9050, 0.9517, 0.9482, 0.9640, 0.9519, 0.9216, 0.7799, 0.7979, 0.5582, 0.7412, 0.7018, 0.8377] ).to(torch_device) - torch.testing.assert_close(results["scores"], expected_scores, rtol=2e-4, atol=2e-4) + torch.testing.assert_close(results["scores"], expected_scores, rtol=2e-2, atol=2e-2) - expected_labels = [22, 17, 22, 22, 22, 22, 22, 10, 10, 22, 10, 16, 8] + expected_labels = [22, 17, 22, 22, 22, 22, 22, 10, 10, 10, 10, 16, 8] self.assertSequenceEqual(results["labels"].tolist(), expected_labels) expected_slice_boxes = torch.tensor( [ - [334.5682, 182.9777, 894.6927, 652.4594], - [336.7216, 683.4235, 867.9361, 796.9210], - [335.2677, 841.1227, 891.1608, 1453.0148], - [919.8677, 183.4835, 1475.8800, 463.4977], + [337.0705, 183.0614, 895.0403, 653.6794], + [337.8179, 684.5647, 868.7692, 798.1080], + [921.4486, 185.6825, 1475.8827, 464.3206], + [920.6929, 484.8696, 1479.4470, 765.1530], ] ).to(torch_device) - torch.testing.assert_close(results["boxes"][:4], expected_slice_boxes, rtol=2e-4, atol=2e-4) + torch.testing.assert_close(results["boxes"][:4], expected_slice_boxes, rtol=2e-2, atol=2e-2) + + expected_slice_polygon_points = torch.tensor([[867, 684], [636, 684], [337, 696], [337, 797], [867, 797]]).to( + torch_device + ) + torch.testing.assert_close( + torch.tensor(results["polygon_points"][1], device=torch_device, dtype=expected_slice_polygon_points.dtype), + expected_slice_polygon_points, + rtol=0, + atol=2, + )