diff --git a/src/transformers/integrations/ggml.py b/src/transformers/integrations/ggml.py index c9ba021c54db..4bdf0ae60b26 100644 --- a/src/transformers/integrations/ggml.py +++ b/src/transformers/integrations/ggml.py @@ -144,6 +144,42 @@ "expert_count": "num_experts", "expert_used_count": "num_experts_per_tok", }, + "qwen3_5_moe_text": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + # Non-MoE layers in the hybrid stack still use a regular MLP whose + # size comes from feed_forward_length. + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + "rope.dimension_count": None, + "rope.freq_base": "rope_theta", + "attention.key_length": "head_dim", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_rms_epsilon": "rms_norm_eps", + "vocab_size": "vocab_size", + "expert_count": "num_experts", + "expert_used_count": "num_experts_per_tok", + "expert_feed_forward_length": "moe_intermediate_size", + "expert_shared_feed_forward_length": "shared_expert_intermediate_size", + # Hybrid layer pattern: convert_hf_to_gguf emits full_attention_interval; + # Qwen3_5MoeTextConfig.__post_init__ pops this kwarg to build layer_types. + "full_attention_interval": "full_attention_interval", + # GatedDeltaNet (linear-attention) shape parameters. The writer reuses + # the SSM key namespace; the mapping is: + # ssm.conv_kernel -> linear_conv_kernel_dim + # ssm.state_size -> linear_key_head_dim + # ssm.group_count -> linear_num_key_heads + # ssm.time_step_rank -> linear_num_value_heads + # ssm.inner_size is derived (linear_value_head_dim * linear_num_value_heads) + # and has no direct config field; ignored here so linear_value_head_dim + # falls back to its config default. + "ssm.conv_kernel": "linear_conv_kernel_dim", + "ssm.state_size": "linear_key_head_dim", + "ssm.group_count": "linear_num_key_heads", + "ssm.time_step_rank": "linear_num_value_heads", + "ssm.inner_size": None, + }, "falcon": { "context_length": "max_position_embeddings", "block_count": "num_hidden_layers", @@ -353,6 +389,11 @@ # (the parameter right after LLM_FFN_SILU corresponds to norm_topk_prob) "norm_topk_prob": True, }, + "qwen3_5_moe_text": { + # Same as qwen3_moe — llama.cpp's qwen35moe.cpp normalizes routed + # expert weights, so override the HF default to match. + "norm_topk_prob": True, + }, "minimax_m2": { # MiniMax-M2 uses routing bias (e_score_correction_bias) for MoE expert selection, # but this is not stored in GGUF metadata. Set it as default so the model weights @@ -791,6 +832,7 @@ def converted(self) -> Tokenizer: "qwen2_moe": GGUFQwen2Converter, "qwen3": GGUFQwen2Converter, "qwen3_moe": GGUFQwen2Converter, + "qwen3_5_moe_text": GGUFQwen2Converter, "phi3": GGUFPhi3Converter, "bloom": GGUFGPTConverter, "falcon": GGUFGPTConverter, diff --git a/src/transformers/modeling_gguf_pytorch_utils.py b/src/transformers/modeling_gguf_pytorch_utils.py index 2de6cc13fc85..ed74bcd426dd 100644 --- a/src/transformers/modeling_gguf_pytorch_utils.py +++ b/src/transformers/modeling_gguf_pytorch_utils.py @@ -458,6 +458,8 @@ def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[st "qwen2moe": Qwen2MoeTensorProcessor, "gpt_oss": GptOssTensorProcessor, "qwen3moe": Qwen2MoeTensorProcessor, + # Qwen3.5 MoE reuses the qwen2/qwen3 fused 3-D ffn_*_exps layout. + "qwen35moe": Qwen2MoeTensorProcessor, "bloom": BloomTensorProcessor, "t5": T5TensorProcessor, "t5encoder": T5TensorProcessor, @@ -512,6 +514,8 @@ def get_gguf_hf_weights_map( model_type = "qwen2moe" elif model_type == "qwen3_moe": model_type = "qwen3moe" + elif model_type == "qwen3_5_moe_text": + model_type = "qwen35moe" elif model_type == "gemma3_text": model_type = "gemma3" elif model_type == "umt5": @@ -630,6 +634,12 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo updated_architecture = "gpt_oss" elif "qwen3moe" in architecture: updated_architecture = "qwen3_moe" + elif "qwen35moe" in architecture: + # GGUF identifies Qwen3.5 MoE as "qwen35moe". Route to the + # text-only qwen3_5_moe_text config rather than the multimodal + # qwen3_5_moe wrapper so Qwen3_5MoeForCausalLM gets the matching + # Qwen3_5MoeTextConfig. + updated_architecture = "qwen3_5_moe_text" elif "minimax-m2" in architecture: updated_architecture = "minimax_m2" @@ -715,6 +725,21 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo i for i, num_kv_heads in enumerate(gguf_num_key_value_heads) if num_kv_heads > 0 ] + if updated_architecture == "qwen3_5_moe_text": + # GatedDeltaNet's value head dim isn't emitted as its own GGUF key — + # the writer only emits ssm.inner_size (= linear_value_head_dim * + # linear_num_value_heads). Recover it here so the config matches the + # checkpoint instead of silently falling back to the class default. + ssm_inner_key = f"{architecture}.ssm.inner_size" + n_v_heads = parsed_parameters["config"].get("linear_num_value_heads") + if ssm_inner_key in reader.fields and n_v_heads: + ssm_inner = _gguf_parse_value( + reader.fields[ssm_inner_key].parts[reader.fields[ssm_inner_key].data[0]], + reader.fields[ssm_inner_key].types, + ) + if ssm_inner % n_v_heads == 0: + parsed_parameters["config"]["linear_value_head_dim"] = ssm_inner // n_v_heads + if updated_architecture == "gpt_oss": # Helper to read keys with the correct prefix def read_gpt_key(reader, suffix, default=None): diff --git a/tests/quantization/ggml/test_ggml.py b/tests/quantization/ggml/test_ggml.py index aa5cdbc7adc6..9fe7f8045f40 100644 --- a/tests/quantization/ggml/test_ggml.py +++ b/tests/quantization/ggml/test_ggml.py @@ -309,6 +309,7 @@ class GgufModelTests(unittest.TestCase): gemma3_vision_model_id = "unsloth/gemma-3-4b-it-GGUF" qwen3_model_id = "Qwen/Qwen3-0.6B-GGUF" qwen3moe_model_id = "Qwen/Qwen3-30B-A3B-GGUF" + qwen35moe_model_id = "unsloth/Qwen3.6-35B-A3B-GGUF" umt5_encoder_model_id = "city96/umt5-xxl-encoder-gguf" lfm2_model_id = "LiquidAI/LFM2-1.2B-GGUF" @@ -349,6 +350,7 @@ class GgufModelTests(unittest.TestCase): fp16_deci_model_id = "decilm-7b-uniform-gqa-f16.gguf" q8_0_qwen3_model_id = "Qwen3-0.6B-Q8_0.gguf" q4_k_m_qwen3moe_model_id = "Qwen3-30B-A3B-Q4_K_M.gguf" + iq3_s_qwen35moe_model_id = "Qwen3.6-35B-A3B-UD-IQ3_S.gguf" q8_0_umt5_encoder_model_id = "umt5-xxl-encoder-Q8_0.gguf" q4_k_m_lfm2_model_id = "LFM2-1.2B-Q4_K_M.gguf" gpt_oss_model_id = "unsloth/gpt-oss-20b-GGUF" @@ -1095,6 +1097,22 @@ def test_qwen3moe_q4_k_m(self): EXPECTED_TEXT = "Hello, I am a 20 year old male" self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT) + @unittest.skip("Heavyweight: ~12.7 GB GGUF download. Run manually.") + def test_qwen35moe_iq3_s(self): + # Smoke test for Qwen3.5/3.6 MoE GGUF support: tokenizer + model + # both load without error and the model produces non-empty output. + # A smaller fixture would be preferable; none was available at the + # time this test was added. + tokenizer = AutoTokenizer.from_pretrained(self.qwen35moe_model_id, gguf_file=self.iq3_s_qwen35moe_model_id) + model = AutoModelForCausalLM.from_pretrained( + self.qwen35moe_model_id, + gguf_file=self.iq3_s_qwen35moe_model_id, + dtype=torch.float16, + ) + text = tokenizer(self.example_text, return_tensors="pt") + out = model.generate(**text, max_new_tokens=4) + self.assertGreater(len(tokenizer.decode(out[0], skip_special_tokens=True)), 0) + def test_umt5_encoder_q8_0(self): """ Verifies that a UMT5 encoder loads directly from a GGUF file using