Enforce manual seed to reduce flakiness

tests/causal_lm_tester.py

@@ -604,6 +604,8 @@ def test_flash_attn_2_equivalence(self):
             if not model_class._supports_flash_attn:
                 self.skipTest(reason="Model does not support Flash Attention 2")
 
+            # Set seed for deterministic test - ensures reproducible model initialization and inputs
+            set_seed(42)
             config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
             model = model_class(config)
 

tests/cli/conftest.py

@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import sys
+
 import pytest
 from typer.testing import CliRunner
 
@@ -21,6 +23,19 @@
 def cli():
     def _cli_invoke(*args):
         runner = CliRunner()
-        return runner.invoke(transformers.cli.transformers.app, list(args), catch_exceptions=False)
+
+        old_out_close = sys.stdout.close
+        old_err_close = sys.stderr.close
+
+        def _noop(*a, **k):
+            return None
+
+        sys.stdout.close = _noop
+        sys.stderr.close = _noop
+        try:
+            return runner.invoke(transformers.cli.transformers.app, list(args), catch_exceptions=False)
+        finally:
+            sys.stdout.close = old_out_close
+            sys.stderr.close = old_err_close
 
     return _cli_invoke

tests/generation/test_utils.py

@@ -36,6 +36,7 @@
     is_torch_available,
     logging,
     pipeline,
+    set_seed,
 )
 from transformers.testing_utils import (
     CaptureLogger,
@@ -237,7 +238,7 @@ def _sample_generate(
         return_dict_in_generate=False,
         use_cache=True,
     ):
-        torch.manual_seed(0)
+        set_seed(42)
         logits_processor_kwargs = self._get_logits_processor_kwargs(do_sample=True, config=model.config)
         output_generate = model.generate(
             do_sample=True,
@@ -299,7 +300,7 @@ def _beam_sample_generate(
         return_dict_in_generate=False,
         use_cache=True,
     ):
-        torch.manual_seed(0)
+        set_seed(42)
         logits_processor_kwargs = self._get_logits_processor_kwargs(do_sample=True, config=model.config)
         output_generate = model.generate(
             do_sample=True,
@@ -662,6 +663,8 @@ def test_assisted_decoding_matches_greedy_search(self, assistant_type):
             ):
                 self.skipTest(reason="May fix in the future: need model-specific fixes")
 
+            # Set seed for deterministic test - ensures reproducible model initialization and inputs
+            set_seed(42)
             # enable cache
             config, inputs_dict = self.prepare_config_and_inputs_for_generate(batch_size=1)
             set_config_for_less_flaky_test(config)
@@ -760,6 +763,8 @@ def test_prompt_lookup_decoding_matches_greedy_search(self):
             ):
                 self.skipTest(reason="May fix in the future: need model-specific fixes")
 
+            # Set seed for deterministic test - ensures reproducible model initialization and inputs
+            set_seed(42)
             # enable cache
             config, inputs_dict = self.prepare_config_and_inputs_for_generate(batch_size=1)
 
@@ -1118,6 +1123,8 @@ def test_generate_from_inputs_embeds(self, _, num_beams):
         # When supported, tests that the decoder model can generate from `inputs_embeds` instead of `input_ids`
         # if fails, you should probably update the `prepare_inputs_for_generation` function
         for model_class in self.all_generative_model_classes:
+            # Set seed for deterministic test - ensures reproducible model initialization and inputs
+            set_seed(42)
             config, inputs_dict = self.prepare_config_and_inputs_for_generate()
 
             # This test is for decoder-only models (encoder-decoder models have native input embeddings support in the
@@ -1250,6 +1257,8 @@ def test_generate_continue_from_past_key_values(self):
             if any(model_name in model_class.__name__.lower() for model_name in ["umt5"]):
                 self.skipTest(reason="TODO: needs modeling or test input preparation fixes for compatibility")
 
+            # Set seed for deterministic test - ensures reproducible model initialization and inputs
+            set_seed(42)
             config, inputs = self.model_tester.prepare_config_and_inputs_for_common()
 
             if not hasattr(config.get_text_config(), "use_cache"):
@@ -1416,6 +1425,8 @@ def test_generate_with_static_cache(self):
             if not model_class._can_compile_fullgraph:
                 self.skipTest(reason="This model does not support the static cache format")
 
+            # Set seed for deterministic test - ensures reproducible model initialization and inputs
+            set_seed(42)
             config, inputs_dict = self.prepare_config_and_inputs_for_generate()
             set_config_for_less_flaky_test(config)
             main_input = inputs_dict[model_class.main_input_name]
@@ -3095,7 +3106,7 @@ def test_TopH_example_integration(self):
         encoder_input_str = "Tell me a joke about a monkey."
         input_ids = tokenizer(encoder_input_str, return_tensors="pt")
 
-        torch.manual_seed(0)
+        set_seed(42)
 
         outputs = model.generate(
             **input_ids,
@@ -4085,7 +4096,7 @@ def test_max_time(self):
         model = GPT2LMHeadModel.from_pretrained("openai-community/gpt2")
         model.to(torch_device)
 
-        torch.manual_seed(0)
+        set_seed(42)
         tokenized = tokenizer("Today is a nice day and", return_tensors="pt", return_token_type_ids=True)
         input_ids = tokenized.input_ids.to(torch_device)
 
@@ -4884,7 +4895,7 @@ def test_hub_gen_strategies(self, custom_generate, extra_kwargs):
             "custom_generate": custom_generate,
         }
         generation_kwargs.update(extra_kwargs)
-        torch.manual_seed(0)
+        set_seed(42)
         output = model.generate(**generation_kwargs, **model_inputs)
         self.assertEqual(output.sequences.shape, (1, 9))
 

tests/models/doge/test_modeling_doge.py

@@ -375,7 +375,7 @@ def test_Doge_20M_hard(self):
         tokenizer = AutoTokenizer.from_pretrained("SmallDoge/Doge-20M")
         model = DogeForCausalLM.from_pretrained("SmallDoge/Doge-20M", device_map="auto", dtype=torch.bfloat16)
         input_text = ["Here's everything I know about dogs. Dogs is the best animal in the"]
-        set_seed(0)
+        set_seed(42)
         model_inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
 
         generated_ids = model.generate(**model_inputs, max_new_tokens=20, do_sample=False)

tests/models/donut/test_image_processing_donut.py

@@ -17,6 +17,7 @@
 
 import numpy as np
 
+from transformers import set_seed
 from transformers.testing_utils import is_flaky, require_torch, require_vision
 from transformers.utils import is_torch_available, is_torchvision_available, is_vision_available
 
@@ -157,6 +158,8 @@ def test_image_processor_preprocess_with_kwargs(self):
     @is_flaky()
     def test_call_pil(self):
         for image_processing_class in self.image_processor_list:
+            # Set seed for deterministic test - ensures reproducible image generation
+            set_seed(42)
             # Initialize image_processing
             image_processing = image_processing_class(**self.image_processor_dict)
             # create random PIL images
@@ -191,6 +194,8 @@ def test_call_pil(self):
     @is_flaky()
     def test_call_numpy(self):
         for image_processing_class in self.image_processor_list:
+            # Set seed for deterministic test - ensures reproducible image generation
+            set_seed(42)
             # Initialize image_processing
             image_processing = image_processing_class(**self.image_processor_dict)
             # create random numpy tensors
@@ -225,6 +230,8 @@ def test_call_numpy(self):
     @is_flaky()
     def test_call_pytorch(self):
         for image_processing_class in self.image_processor_list:
+            # Set seed for deterministic test - ensures reproducible image generation
+            set_seed(42)
             # Initialize image_processing
             image_processing = image_processing_class(**self.image_processor_dict)
             # create random PyTorch tensors

tests/models/esm/test_modeling_esm.py

@@ -18,7 +18,7 @@
 
 import pytest
 
-from transformers import BitsAndBytesConfig, EsmConfig, is_torch_available
+from transformers import BitsAndBytesConfig, EsmConfig, is_torch_available, set_seed
 from transformers.testing_utils import (
     TestCasePlus,
     is_flaky,
@@ -315,6 +315,8 @@ def test_flash_attn_2_equivalence(self):
             if not model_class._supports_flash_attn:
                 self.skipTest(reason="Model does not support Flash Attention 2")
 
+            # Set seed for deterministic test - ensures reproducible model initialization and inputs
+            set_seed(42)
             config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
             model = model_class(config)
 

tests/models/fastspeech2_conformer/test_modeling_fastspeech2_conformer.py

@@ -401,7 +401,7 @@ def test_training_integration(self):
         model.to(torch_device)
         # Set self.training manually to keep deterministic but run the training path
         model.training = True
-        set_seed(0)
+        set_seed(42)
 
         tokenizer = FastSpeech2ConformerTokenizer.from_pretrained("espnet/fastspeech2_conformer")
         text = "Test that this generates speech"

tests/models/mimi/test_modeling_mimi.py

@@ -22,7 +22,7 @@
 from datasets import Audio, load_dataset
 from pytest import mark
 
-from transformers import AutoFeatureExtractor, MimiConfig
+from transformers import AutoFeatureExtractor, MimiConfig, set_seed
 from transformers.testing_utils import (
     is_flaky,
     is_torch_available,
@@ -298,6 +298,8 @@ def test_identity_shortcut(self):
     @is_flaky()
     def test_flash_attn_2_inference_equivalence(self):
         for model_class in self.all_model_classes:
+            # Set seed for deterministic test - ensures reproducible model initialization and inputs
+            set_seed(42)
             config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
             model = model_class(config)
 

tests/models/minimax_m2/test_modeling_minimax_m2.py

@@ -15,7 +15,7 @@
 
 import unittest
 
-from transformers import AutoTokenizer, is_torch_available
+from transformers import AutoTokenizer, is_torch_available, set_seed
 from transformers.testing_utils import (
     Expectations,
     cleanup,
@@ -52,6 +52,8 @@ def test_load_balancing_loss(self):
         r"""
         Let's make sure we can actually compute the loss and do a backward on it.
         """
+        # Set seed for deterministic test - ensures reproducible model initialization and inputs
+        set_seed(42)
         config, input_dict = self.model_tester.prepare_config_and_inputs_for_common()
         config.num_labels = 3
         config.num_experts = 3

tests/models/mistral/test_modeling_mistral.py

@@ -228,7 +228,7 @@ def test_speculative_generation(self):
         input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.model.embed_tokens.weight.device)
 
         # greedy generation outputs
-        set_seed(0)
+        set_seed(42)
         generated_ids = model.generate(
             input_ids, max_new_tokens=20, do_sample=True, temperature=0.3, assistant_model=model
         )

tests/models/mixtral/test_modeling_mixtral.py

@@ -17,7 +17,7 @@
 
 import pytest
 
-from transformers import is_torch_available
+from transformers import is_torch_available, set_seed
 from transformers.testing_utils import (
     Expectations,
     is_flaky,
@@ -74,6 +74,8 @@ def test_load_balancing_loss(self):
         r"""
         Let's make sure we can actually compute the loss and do a backward on it.
         """
+        # Set seed for deterministic test - ensures reproducible model initialization and inputs
+        set_seed(42)
         config, input_dict = self.model_tester.prepare_config_and_inputs_for_common()
         config.num_labels = 3
         config.num_local_experts = 3

tests/models/musicgen/test_modeling_musicgen.py

@@ -1160,7 +1160,7 @@ def test_generate_unconditional_sampling(self):
         unconditional_inputs = model.get_unconditional_inputs(num_samples=2)
         unconditional_inputs = place_dict_on_device(unconditional_inputs, device=torch_device)
 
-        set_seed(0)
+        set_seed(42)
         output_values = model.generate(**unconditional_inputs, do_sample=True, max_new_tokens=10)
 
         # fmt: off
@@ -1241,7 +1241,7 @@ def test_generate_text_prompt_sampling(self):
         input_ids = inputs.input_ids.to(torch_device)
         attention_mask = inputs.attention_mask.to(torch_device)
 
-        set_seed(0)
+        set_seed(42)
         output_values = model.generate(
             input_ids, attention_mask=attention_mask, do_sample=True, guidance_scale=None, max_new_tokens=10
         )

tests/models/musicgen_melody/test_modeling_musicgen_melody.py

@@ -1170,7 +1170,7 @@ def test_generate_unconditional_sampling(self):
         # for stochastic sampling we can generate multiple outputs
         unconditional_inputs = self.processor.get_unconditional_inputs(num_samples=2).to(torch_device)
 
-        set_seed(0)
+        set_seed(42)
 
         output_values = model.generate(
             **unconditional_inputs, do_sample=True, max_new_tokens=10, guidance_scale=1.0, temperature=1.0, top_k=250
@@ -1254,7 +1254,7 @@ def test_generate_text_prompt_sampling(self):
         input_ids = inputs.input_ids.to(torch_device)
         attention_mask = inputs.attention_mask.to(torch_device)
 
-        set_seed(0)
+        set_seed(42)
         output_values = model.generate(
             input_ids,
             attention_mask=attention_mask,