From 00099bb5a40a6ae35766f330fc9b333b8c8e80ec Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 8 Aug 2024 12:36:21 +0200 Subject: [PATCH 1/2] fix llava-next-video generation --- .../models/llava_next_video/modeling_llava_next_video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index e5583d782d88..afed3a8248e0 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -853,7 +853,7 @@ def forward( inputs_embeds = self.get_input_embeddings()(input_ids) # Merge text and images in prefill stage - if past_key_values is None: + if past_key_values is None or past_key_values.get_seq_length() == 0: # First merge image tokens if there are any if pixel_values is not None and pixel_values.size(0) > 0: image_features = self._get_image_features(pixel_values, image_sizes) From 7c70acbf7e10c1f49b95700afe5b18999159f602 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 8 Aug 2024 15:23:52 +0200 Subject: [PATCH 2/2] add tests --- tests/models/blip/test_modeling_blip.py | 12 +++++++++++ tests/models/blip_2/test_modeling_blip_2.py | 16 +++++++++++++-- .../test_modeling_instructblip.py | 18 ++++++++++++++--- .../test_modeling_instructblipvideo.py | 20 ++++++++++++++----- tests/models/kosmos2/test_modeling_kosmos2.py | 11 ++++++++++ tests/models/llava/test_modeling_llava.py | 12 +++++++++++ .../llava_next/test_modeling_llava_next.py | 15 ++++++++++++-- .../test_modeling_llava_next_video.py | 15 ++++++++++++-- .../paligemma/test_modeling_paligemma.py | 12 +++++++++++ .../video_llava/test_modeling_video_llava.py | 15 ++++++++++++-- .../models/vipllava/test_modeling_vipllava.py | 12 +++++++++++ 11 files changed, 142 insertions(+), 16 deletions(-) diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 2f8ee3229ff2..7b953c7f46b9 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -1106,6 +1106,7 @@ def test_model_from_pretrained(self): @require_torch class BlipTextImageModelTest(ModelTesterMixin, unittest.TestCase): all_model_classes = (BlipForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (BlipForConditionalGeneration,) if is_torch_available() else () fx_compatible = False test_head_masking = False test_pruning = False @@ -1116,6 +1117,17 @@ class BlipTextImageModelTest(ModelTesterMixin, unittest.TestCase): def setUp(self): self.model_tester = BlipTextImageModelsModelTester(self) + def test_greedy_generation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_generative_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + out = model.generate(**inputs_dict, min_new_tokens=20, max_new_tokens=20) + self.assertTrue(out.shape[1] == inputs_dict["input_ids"].shape[1] + 19) + def test_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_model(*config_and_inputs) diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 28ed3a79cae5..52a2e219cd4a 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -314,7 +314,7 @@ def __init__( hidden_act="gelu", hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, - max_position_embeddings=20, + max_position_embeddings=256, eos_token_id=2, pad_token_id=1, bos_token_id=0, @@ -436,8 +436,9 @@ def prepare_config_and_inputs_for_common(self): @require_torch -class Blip2ForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): +class Blip2ForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, unittest.TestCase): all_model_classes = (Blip2ForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (Blip2ForConditionalGeneration,) if is_torch_available() else () fx_compatible = False test_head_masking = False test_pruning = False @@ -448,6 +449,17 @@ class Blip2ForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, GenerationT def setUp(self): self.model_tester = Blip2ForConditionalGenerationDecoderOnlyModelTester(self) + def test_greedy_generation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_generative_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + out = model.generate(**inputs_dict, min_new_tokens=20, max_new_tokens=20) + self.assertTrue(out.shape[1] == 21) # BLIP is special, so should be 21 + def test_for_conditional_generation(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_for_conditional_generation(*config_and_inputs) diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index 1aaa8e1a8b68..1b7740e125fc 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -38,7 +38,6 @@ ) from transformers.utils import is_torch_available, is_vision_available -from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, @@ -319,7 +318,7 @@ def __init__( hidden_act="gelu", hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, - max_position_embeddings=20, + max_position_embeddings=256, eos_token_id=2, pad_token_id=1, bos_token_id=0, @@ -452,8 +451,9 @@ def prepare_config_and_inputs_for_common(self): @require_torch -class InstructBlipForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): +class InstructBlipForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, unittest.TestCase): all_model_classes = (InstructBlipForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (InstructBlipForConditionalGeneration,) if is_torch_available() else () fx_compatible = False test_head_masking = False test_pruning = False @@ -464,6 +464,18 @@ class InstructBlipForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, Gene def setUp(self): self.model_tester = InstructBlipForConditionalGenerationDecoderOnlyModelTester(self) + def test_greedy_generation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_generative_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + model.config.text_config.architectures = ["OptForCausalLM"] + + out = model.generate(**inputs_dict, min_new_tokens=20, max_new_tokens=20) + self.assertTrue(out.shape[1] == 21) # BLIP is special, therefore 21 + def test_for_conditional_generation(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_for_conditional_generation(*config_and_inputs) diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index 1265db3a2a2e..9de6cdecde3a 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -38,7 +38,6 @@ ) from transformers.utils import is_torch_available, is_vision_available -from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, @@ -333,7 +332,7 @@ def __init__( hidden_act="gelu", hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, - max_position_embeddings=100, + max_position_embeddings=256, eos_token_id=2, pad_token_id=1, bos_token_id=0, @@ -471,10 +470,9 @@ def prepare_config_and_inputs_for_common(self): @require_torch -class InstructBlipVideoForConditionalGenerationDecoderOnlyTest( - ModelTesterMixin, GenerationTesterMixin, unittest.TestCase -): +class InstructBlipVideoForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, unittest.TestCase): all_model_classes = (InstructBlipVideoForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (InstructBlipVideoForConditionalGeneration,) if is_torch_available() else () fx_compatible = False test_head_masking = False test_pruning = False @@ -485,6 +483,18 @@ class InstructBlipVideoForConditionalGenerationDecoderOnlyTest( def setUp(self): self.model_tester = InstructBlipVideoForConditionalGenerationDecoderOnlyModelTester(self) + def test_greedy_generation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_generative_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + model.config.text_config.architectures = ["OptForCausalLM"] + + out = model.generate(**inputs_dict, min_new_tokens=20, max_new_tokens=20) + self.assertTrue(out.shape[1] == 21) # BLIP is special, therefore 21 + def test_for_conditional_generation(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_for_conditional_generation(*config_and_inputs) diff --git a/tests/models/kosmos2/test_modeling_kosmos2.py b/tests/models/kosmos2/test_modeling_kosmos2.py index 6f34689004ef..639ac2eb4358 100644 --- a/tests/models/kosmos2/test_modeling_kosmos2.py +++ b/tests/models/kosmos2/test_modeling_kosmos2.py @@ -281,6 +281,17 @@ def setUp(self): self.model_tester = Kosmos2ModelTester(self) self.config_tester = ConfigTester(self, config_class=Kosmos2Config, hidden_size=37) + def test_greedy_generation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_generative_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + out = model.generate(**inputs_dict, min_new_tokens=20, max_new_tokens=20) + self.assertTrue(out.shape[1] == inputs_dict["input_ids"].shape[1] + 20) + # overwrite from common to skip `image_to_text_projection.latent_query` def test_initialization(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/models/llava/test_modeling_llava.py b/tests/models/llava/test_modeling_llava.py index ce13ab6738af..05beb1f83709 100644 --- a/tests/models/llava/test_modeling_llava.py +++ b/tests/models/llava/test_modeling_llava.py @@ -178,6 +178,7 @@ class LlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase """ all_model_classes = (LlavaForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (LlavaForConditionalGeneration,) if is_torch_available() else () pipeline_model_mapping = {"image-to-text": LlavaForConditionalGeneration} if is_torch_available() else {} test_pruning = False test_head_masking = False @@ -186,6 +187,17 @@ def setUp(self): self.model_tester = LlavaVisionText2TextModelTester(self) self.config_tester = ConfigTester(self, config_class=LlavaConfig, has_text_modality=False) + def test_greedy_generation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_generative_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + out = model.generate(**inputs_dict, min_new_tokens=20, max_new_tokens=20) + self.assertTrue(out.shape[1] == inputs_dict["input_ids"].shape[1] + 20) + @unittest.skip( reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124" ) diff --git a/tests/models/llava_next/test_modeling_llava_next.py b/tests/models/llava_next/test_modeling_llava_next.py index 70d91002a91b..1a9e587e7704 100644 --- a/tests/models/llava_next/test_modeling_llava_next.py +++ b/tests/models/llava_next/test_modeling_llava_next.py @@ -34,7 +34,6 @@ torch_device, ) -from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, @@ -208,12 +207,13 @@ def create_and_check_llava_next_model_fp16_autocast_forward( @require_torch -class LlavaNextForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): +class LlavaNextForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase): """ Model tester for `LlavaNextForConditionalGeneration`. """ all_model_classes = (LlavaNextForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (LlavaNextForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False @@ -237,6 +237,17 @@ def test_initialization(self): msg=f"Parameter {name} of model {model_class} seems not properly initialized", ) + def test_greedy_generation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_generative_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + out = model.generate(**inputs_dict, min_new_tokens=20, max_new_tokens=20) + self.assertTrue(out.shape[1] == inputs_dict["input_ids"].shape[1] + 20) + @unittest.skip( reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124" ) diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index 9ba7ef869ddf..64e7c303c511 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -34,7 +34,6 @@ torch_device, ) -from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, @@ -223,12 +222,13 @@ def create_and_check_llava_next_video_model_fp16_autocast_forward( @require_torch -class LlavaNextVideoForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): +class LlavaNextVideoForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase): """ Model tester for `LlavaNextVideoForConditionalGeneration`. """ all_model_classes = (LlavaNextVideoForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (LlavaNextVideoForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False @@ -274,6 +274,17 @@ def test_inputs_embeds(self): with torch.no_grad(): model(**inputs) + def test_greedy_generation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_generative_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + out = model.generate(**inputs_dict, min_new_tokens=20, max_new_tokens=20) + self.assertTrue(out.shape[1] == inputs_dict["input_ids"].shape[1] + 20) + @unittest.skip( reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124" ) diff --git a/tests/models/paligemma/test_modeling_paligemma.py b/tests/models/paligemma/test_modeling_paligemma.py index 7753ae073dd3..f459549d75ae 100644 --- a/tests/models/paligemma/test_modeling_paligemma.py +++ b/tests/models/paligemma/test_modeling_paligemma.py @@ -176,6 +176,7 @@ class PaliGemmaForConditionalGenerationModelTest(ModelTesterMixin, unittest.Test """ all_model_classes = (PaliGemmaForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (PaliGemmaForConditionalGeneration,) if is_torch_available() else () fx_compatible = False test_pruning = False test_torchscript = False @@ -185,6 +186,17 @@ def setUp(self): self.model_tester = PaliGemmaVisionText2TextModelTester(self) self.config_tester = ConfigTester(self, config_class=PaliGemmaConfig, has_text_modality=False) + def test_greedy_generation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_generative_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + out = model.generate(**inputs_dict, min_new_tokens=20, max_new_tokens=20) + self.assertTrue(out.shape[1] == inputs_dict["input_ids"].shape[1] + 20) + @unittest.skip( reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124" ) diff --git a/tests/models/video_llava/test_modeling_video_llava.py b/tests/models/video_llava/test_modeling_video_llava.py index fe3eea97dcf3..6a2c30157654 100644 --- a/tests/models/video_llava/test_modeling_video_llava.py +++ b/tests/models/video_llava/test_modeling_video_llava.py @@ -30,7 +30,6 @@ ) from transformers.testing_utils import require_bitsandbytes, require_torch, require_torch_gpu, slow, torch_device -from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor @@ -190,12 +189,13 @@ def prepare_config_and_inputs_for_batched_test(self): @require_torch -class VideoLlavaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): +class VideoLlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase): """ Model tester for `VideoLlavaForConditionalGeneration`. """ all_model_classes = (VideoLlavaForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (VideoLlavaForConditionalGeneration,) if is_torch_available() else () fx_compatible = False test_pruning = False test_resize_embeddings = True @@ -205,6 +205,17 @@ def setUp(self): self.model_tester = VideoLlavaVisionText2TextModelTester(self) self.config_tester = ConfigTester(self, config_class=VideoLlavaConfig, has_text_modality=False) + def test_greedy_generation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_generative_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + out = model.generate(**inputs_dict, min_new_tokens=20, max_new_tokens=20) + self.assertTrue(out.shape[1] == inputs_dict["input_ids"].shape[1] + 20) + @unittest.skip( reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124" ) diff --git a/tests/models/vipllava/test_modeling_vipllava.py b/tests/models/vipllava/test_modeling_vipllava.py index a4e89d3f9ddf..29615f00caac 100644 --- a/tests/models/vipllava/test_modeling_vipllava.py +++ b/tests/models/vipllava/test_modeling_vipllava.py @@ -158,6 +158,7 @@ class VipLlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestC """ all_model_classes = (VipLlavaForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (VipLlavaForConditionalGeneration,) if is_torch_available() else () fx_compatible = False test_pruning = False test_resize_embeddings = True @@ -167,6 +168,17 @@ def setUp(self): self.model_tester = VipLlavaVisionText2TextModelTester(self) self.config_tester = ConfigTester(self, config_class=VipLlavaConfig, has_text_modality=False) + def test_greedy_generation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_generative_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + out = model.generate(**inputs_dict, min_new_tokens=20, max_new_tokens=20) + self.assertTrue(out.shape[1] == inputs_dict["input_ids"].shape[1] + 20) + @unittest.skip( reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124" )