[GGUF] Add support for Qwen3.5 MoE (qwen35moe arch)

src/transformers/integrations/ggml.py

@@ -144,6 +144,42 @@
         "expert_count": "num_experts",
         "expert_used_count": "num_experts_per_tok",
     },
+    "qwen3_5_moe_text": {
+        "context_length": "max_position_embeddings",
+        "block_count": "num_hidden_layers",
+        # Non-MoE layers in the hybrid stack still use a regular MLP whose
+        # size comes from feed_forward_length.
+        "feed_forward_length": "intermediate_size",
+        "embedding_length": "hidden_size",
+        "rope.dimension_count": None,
+        "rope.freq_base": "rope_theta",
+        "attention.key_length": "head_dim",
+        "attention.head_count": "num_attention_heads",
+        "attention.head_count_kv": "num_key_value_heads",
+        "attention.layer_norm_rms_epsilon": "rms_norm_eps",
+        "vocab_size": "vocab_size",
+        "expert_count": "num_experts",
+        "expert_used_count": "num_experts_per_tok",
+        "expert_feed_forward_length": "moe_intermediate_size",
+        "expert_shared_feed_forward_length": "shared_expert_intermediate_size",
+        # Hybrid layer pattern: convert_hf_to_gguf emits full_attention_interval;
+        # Qwen3_5MoeTextConfig.__post_init__ pops this kwarg to build layer_types.
+        "full_attention_interval": "full_attention_interval",
+        # GatedDeltaNet (linear-attention) shape parameters. The writer reuses
+        # the SSM key namespace; the mapping is:
+        #   ssm.conv_kernel    -> linear_conv_kernel_dim
+        #   ssm.state_size     -> linear_key_head_dim
+        #   ssm.group_count    -> linear_num_key_heads
+        #   ssm.time_step_rank -> linear_num_value_heads
+        # ssm.inner_size is derived (linear_value_head_dim * linear_num_value_heads)
+        # and has no direct config field; ignored here so linear_value_head_dim
+        # falls back to its config default.
+        "ssm.conv_kernel": "linear_conv_kernel_dim",
+        "ssm.state_size": "linear_key_head_dim",
+        "ssm.group_count": "linear_num_key_heads",
+        "ssm.time_step_rank": "linear_num_value_heads",
+        "ssm.inner_size": None,
+    },
     "falcon": {
         "context_length": "max_position_embeddings",
         "block_count": "num_hidden_layers",
@@ -353,6 +389,11 @@
         #      (the parameter right after LLM_FFN_SILU corresponds to norm_topk_prob)
         "norm_topk_prob": True,
     },
+    "qwen3_5_moe_text": {
+        # Same as qwen3_moe — llama.cpp's qwen35moe.cpp normalizes routed
+        # expert weights, so override the HF default to match.
+        "norm_topk_prob": True,
+    },
     "minimax_m2": {
         # MiniMax-M2 uses routing bias (e_score_correction_bias) for MoE expert selection,
         # but this is not stored in GGUF metadata. Set it as default so the model weights
@@ -791,6 +832,7 @@ def converted(self) -> Tokenizer:
     "qwen2_moe": GGUFQwen2Converter,
     "qwen3": GGUFQwen2Converter,
     "qwen3_moe": GGUFQwen2Converter,
+    "qwen3_5_moe_text": GGUFQwen2Converter,
     "phi3": GGUFPhi3Converter,
     "bloom": GGUFGPTConverter,
     "falcon": GGUFGPTConverter,

src/transformers/modeling_gguf_pytorch_utils.py

@@ -458,6 +458,8 @@ def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[st
     "qwen2moe": Qwen2MoeTensorProcessor,
     "gpt_oss": GptOssTensorProcessor,
     "qwen3moe": Qwen2MoeTensorProcessor,
+    # Qwen3.5 MoE reuses the qwen2/qwen3 fused 3-D ffn_*_exps layout.
+    "qwen35moe": Qwen2MoeTensorProcessor,
     "bloom": BloomTensorProcessor,
     "t5": T5TensorProcessor,
     "t5encoder": T5TensorProcessor,
@@ -512,6 +514,8 @@ def get_gguf_hf_weights_map(
         model_type = "qwen2moe"
     elif model_type == "qwen3_moe":
         model_type = "qwen3moe"
+    elif model_type == "qwen3_5_moe_text":
+        model_type = "qwen35moe"
     elif model_type == "gemma3_text":
         model_type = "gemma3"
     elif model_type == "umt5":
@@ -630,6 +634,12 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo
         updated_architecture = "gpt_oss"
     elif "qwen3moe" in architecture:
         updated_architecture = "qwen3_moe"
+    elif "qwen35moe" in architecture:
+        # GGUF identifies Qwen3.5 MoE as "qwen35moe". Route to the
+        # text-only qwen3_5_moe_text config rather than the multimodal
+        # qwen3_5_moe wrapper so Qwen3_5MoeForCausalLM gets the matching
+        # Qwen3_5MoeTextConfig.
+        updated_architecture = "qwen3_5_moe_text"
     elif "minimax-m2" in architecture:
         updated_architecture = "minimax_m2"
 
@@ -715,6 +725,21 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo
             i for i, num_kv_heads in enumerate(gguf_num_key_value_heads) if num_kv_heads > 0
         ]
 
+    if updated_architecture == "qwen3_5_moe_text":
+        # GatedDeltaNet's value head dim isn't emitted as its own GGUF key —
+        # the writer only emits ssm.inner_size (= linear_value_head_dim *
+        # linear_num_value_heads). Recover it here so the config matches the
+        # checkpoint instead of silently falling back to the class default.
+        ssm_inner_key = f"{architecture}.ssm.inner_size"
+        n_v_heads = parsed_parameters["config"].get("linear_num_value_heads")
+        if ssm_inner_key in reader.fields and n_v_heads:
+            ssm_inner = _gguf_parse_value(
+                reader.fields[ssm_inner_key].parts[reader.fields[ssm_inner_key].data[0]],
+                reader.fields[ssm_inner_key].types,
+            )
+            if ssm_inner % n_v_heads == 0:
+                parsed_parameters["config"]["linear_value_head_dim"] = ssm_inner // n_v_heads
+
     if updated_architecture == "gpt_oss":
         # Helper to read keys with the correct prefix
         def read_gpt_key(reader, suffix, default=None):

tests/quantization/ggml/test_ggml.py

@@ -309,6 +309,7 @@ class GgufModelTests(unittest.TestCase):
     gemma3_vision_model_id = "unsloth/gemma-3-4b-it-GGUF"
     qwen3_model_id = "Qwen/Qwen3-0.6B-GGUF"
     qwen3moe_model_id = "Qwen/Qwen3-30B-A3B-GGUF"
+    qwen35moe_model_id = "unsloth/Qwen3.6-35B-A3B-GGUF"
     umt5_encoder_model_id = "city96/umt5-xxl-encoder-gguf"
     lfm2_model_id = "LiquidAI/LFM2-1.2B-GGUF"
 
@@ -349,6 +350,7 @@ class GgufModelTests(unittest.TestCase):
     fp16_deci_model_id = "decilm-7b-uniform-gqa-f16.gguf"
     q8_0_qwen3_model_id = "Qwen3-0.6B-Q8_0.gguf"
     q4_k_m_qwen3moe_model_id = "Qwen3-30B-A3B-Q4_K_M.gguf"
+    iq3_s_qwen35moe_model_id = "Qwen3.6-35B-A3B-UD-IQ3_S.gguf"
     q8_0_umt5_encoder_model_id = "umt5-xxl-encoder-Q8_0.gguf"
     q4_k_m_lfm2_model_id = "LFM2-1.2B-Q4_K_M.gguf"
     gpt_oss_model_id = "unsloth/gpt-oss-20b-GGUF"
@@ -1095,6 +1097,22 @@ def test_qwen3moe_q4_k_m(self):
         EXPECTED_TEXT = "Hello, I am a 20 year old male"
         self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
 
+    @unittest.skip("Heavyweight: ~12.7 GB GGUF download. Run manually.")
+    def test_qwen35moe_iq3_s(self):
+        # Smoke test for Qwen3.5/3.6 MoE GGUF support: tokenizer + model
+        # both load without error and the model produces non-empty output.
+        # A smaller fixture would be preferable; none was available at the
+        # time this test was added.
+        tokenizer = AutoTokenizer.from_pretrained(self.qwen35moe_model_id, gguf_file=self.iq3_s_qwen35moe_model_id)
+        model = AutoModelForCausalLM.from_pretrained(
+            self.qwen35moe_model_id,
+            gguf_file=self.iq3_s_qwen35moe_model_id,
+            dtype=torch.float16,
+        )
+        text = tokenizer(self.example_text, return_tensors="pt")
+        out = model.generate(**text, max_new_tokens=4)
+        self.assertGreater(len(tokenizer.decode(out[0], skip_special_tokens=True)), 0)
+
     def test_umt5_encoder_q8_0(self):
         """
         Verifies that a UMT5 encoder loads directly from a GGUF file using