diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index ea9191873282..9f02eb1e5712 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -3613,34 +3613,159 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std: return supported; } +// A specific pipeline's configuration +struct PipelineConfigParameter { + uint32_t subgroup_size = 0; + // Calculate specialization constants used for a specific pipeline. + // If empty we use the default. + // Some kernels must calculate specialization constants + // based on subgroup size so we have an interface to override the default here. + std::vector (*calc_specialization_constants)(const PipelineConfigParameter &, const std::vector &) = nullptr; +}; + +// Pipeline configuration for a target GPU. +// This may contain a group of piplines struct GpuPipelineConfig { // GPU architecture identifier. // Example: vk_device_architecture::AMD_GCN vk_device_architecture arch; - // Mapping of pipeline names to their specific subgroup sizes. - // Example: {"soft_max_f32", 64} - std::unordered_map pipelines; + // Mapping of pipeline names to their specific configuration parameters. + // Example: {"soft_max_f32", {64}} + std::unordered_map pipelines; // Default subgroup size for this GPU. // Defaults to 0 if not explicitly provided. uint32_t default_subgroup_size = 0; + + // Backend-specific policy for updating subgroup/specialization outputs. + void (*update_subgroup_params)( + bool pipeline_param_found, + const PipelineConfigParameter & pipeline_param, + const std::vector & specialization_constants, + bool require_full_subgroups, + uint32_t required_subgroup_size, + uint32_t default_subgroup_size, + uint32_t & final_required_subgroup_size, + std::vector & final_specialization_constant) = nullptr; }; // Pipeline configuration for RDNA1 GPUs. -static const std::unordered_map rdna1_pipelines = { - {"soft_max", 64}, {"im2col", 64}, - {"argmax", 64}, {"mul_mat_vec", 64}, - {"mul_mat_vec_f16", 32}, {"mul_mat_vec_f32_f16", 32} +static const std::unordered_map rdna1_pipelines = { + {"soft_max", {64}}, + {"im2col", {64}}, + {"argmax", {64}}, + {"mul_mat_vec", {64}}, + {"mul_mat_vec_f16", {32}}, + {"mul_mat_vec_f32_f16", {32}}, }; // Pipeline configuration for RDNA2 GPUs. -static const std::unordered_map rdna2_pipelines = { - {"soft_max", 64}, {"im2col", 64}, +static const std::unordered_map rdna2_pipelines = { + {"soft_max", {64}}, + {"im2col", {64}}, }; static constexpr uint32_t RDNA_DEFAULT_SUBGROUP_SIZE = 32; +static std::vector calc_specialization_constant_intel_warptile(const PipelineConfigParameter& config, const std::vector& current) { + GGML_ASSERT(current.size() == 12); // assuming *_warptile constants + std::vector output = current; + // replacing subgroup_size_8 with current subgroup size + output[4] = config.subgroup_size; // WM + output[10] = config.subgroup_size; // WARP + // Recalculate BLOCK_SIZE to maintain NUM_WARPS = (BM/WM) * (BN/WN). + const uint32_t BM = current[1]; + const uint32_t BN = current[2]; + const uint32_t WN = current[5]; + const uint32_t num_warps = (BM / output[4]) * (BN / WN); + output[0] = num_warps * output[10]; + return output; +} + +// Xe2+ GPU targeted pipelines +static const std::unordered_map xe2_pipelines = { + {"aligned_m", {16, calc_specialization_constant_intel_warptile}}, + {"aligned_s", {16, calc_specialization_constant_intel_warptile}}, +}; + +static const std::unordered_map xe1_pipelines = { + // Following are same pipelines as integrated + {"matmul_id_f16_f32_f16acc_aligned_s", {8, calc_specialization_constant_intel_warptile}}, + {"matmul_id_f32_f32_aligned_m", {8, calc_specialization_constant_intel_warptile}}, + {"matmul_id_f32_f32_aligned_s", {8, calc_specialization_constant_intel_warptile}}, + {"matmul_id_iq2_xs_f32_f16acc_aligned_s", {8, calc_specialization_constant_intel_warptile}}, + {"matmul_id_q4_0_q8_1_s", {8, calc_specialization_constant_intel_warptile}}, + {"matmul_id_q4_k_q8_1_s", {8, calc_specialization_constant_intel_warptile}}, + {"matmul_id_q6_k_q8_1_s", {8, calc_specialization_constant_intel_warptile}}, + {"matmul_id_q8_0_q8_1_s", {8, calc_specialization_constant_intel_warptile}}, + {"mul_mat_vec_iq2_s_f32_f32", {8}}, + {"mul_mat_vec_iq2_xs_f32_f32", {8}}, + {"mul_mat_vec_iq2_xxs_f32_f32", {8}}, + {"mul_mat_vec_iq3_s_f32_f32", {8}}, + {"mul_mat_vec_iq3_xxs_f32_f32", {8}}, + {"mul_mat_vec_iq4_nl_f32_f32", {8}}, + {"mul_mat_vec_iq4_xs_f32_f32", {8}}, + {"mul_mat_vec_nvfp4_f32_f32", {8}}, + {"mul_mat_vec_q2_k_f32_f32", {8}}, + {"mul_mat_vec_q3_k_f32_f32", {8}}, + {"mul_mat_vec_q6_k_f32_f32", {8}}, + // Following are pipelines only for discrete + // {"matmul_id_q4_k_q8_1_m", {8, calc_specialization_constant_intel_warptile}}, + // {"matmul_id_q6_k_q8_1_m", {8, calc_specialization_constant_intel_warptile}}, + // {"matmul_id_f16_f32_f16acc_aligned_m", {8, calc_specialization_constant_intel_warptile}}, +}; + + +static void update_subgroup_params_intel( + bool pipeline_param_found, + const PipelineConfigParameter & pipeline_param, + const std::vector & specialization_constants, + bool require_full_subgroups, + uint32_t required_subgroup_size, + uint32_t default_subgroup_size, + uint32_t & final_required_subgroup_size, + std::vector & final_specialization_constant) { + GGML_UNUSED(require_full_subgroups); + GGML_UNUSED(default_subgroup_size); + if (pipeline_param_found) { + // We have a GPU configuration and a specific parameter for this pipeline. + // We overwrite all valid parameters assuming the setting creator knows what they are doing. + if (pipeline_param.subgroup_size) { + final_required_subgroup_size = pipeline_param.subgroup_size; + } + if (pipeline_param.calc_specialization_constants) { + final_specialization_constant = pipeline_param.calc_specialization_constants(pipeline_param, specialization_constants); + } + } +} + +static void update_subgroup_params_amd( + bool pipeline_param_found, + const PipelineConfigParameter & pipeline_param, + const std::vector & specialization_constants, + bool require_full_subgroups, + uint32_t required_subgroup_size, + uint32_t default_subgroup_size, + uint32_t & final_required_subgroup_size, + std::vector & final_specialization_constant) { + GGML_UNUSED(specialization_constants); + GGML_UNUSED(final_specialization_constant); + + if (!require_full_subgroups && required_subgroup_size == 0) { + if (pipeline_param_found) { + final_required_subgroup_size = pipeline_param.subgroup_size; + } else { + // If no pipeline setting exists we use the GPU config default + final_required_subgroup_size = default_subgroup_size; + } + } +} + +// Intel GPU can use subgroup 8, 16, or 32 depending on architeture. +// Pre-Xe2 is 8, 16, or 32. Xe2 onward is 16 or 32. 32 is the default if nothing is specified. +static constexpr uint32_t INTEL_DEFAULT_SUBGROUP_SIZE = 32; + // Define configurations for different GPUs. static std::vector gpu_pipeline_configs = { { @@ -3648,36 +3773,61 @@ static std::vector gpu_pipeline_configs = { { rdna1_pipelines, }, - RDNA_DEFAULT_SUBGROUP_SIZE + RDNA_DEFAULT_SUBGROUP_SIZE, + update_subgroup_params_amd }, { vk_device_architecture::AMD_RDNA2, { rdna2_pipelines, }, - RDNA_DEFAULT_SUBGROUP_SIZE + RDNA_DEFAULT_SUBGROUP_SIZE, + update_subgroup_params_amd + }, + { + vk_device_architecture::INTEL_XE1, + { + xe1_pipelines, + }, + INTEL_DEFAULT_SUBGROUP_SIZE, + update_subgroup_params_intel + }, + { + vk_device_architecture::INTEL_XE2, + { + xe2_pipelines, + }, + INTEL_DEFAULT_SUBGROUP_SIZE, + update_subgroup_params_intel }, }; -static uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch) { - for (const auto &config : gpu_pipeline_configs) { +static bool get_gpu_pipeline_config(GpuPipelineConfig* output, const vk_device_architecture& arch) { + for (const auto & config : gpu_pipeline_configs) { if (config.arch == arch) { - auto pipIt = config.pipelines.find(pipeline_name); - if (pipIt != config.pipelines.end()) { - return pipIt->second; - } - std::vector> sorted_pipelines(config.pipelines.begin(), config.pipelines.end()); - std::sort(sorted_pipelines.begin(), sorted_pipelines.end(), - [](const auto &a, const auto &b) { return a.first.size() > b.first.size(); }); - for (const auto &entry : sorted_pipelines) { - if (pipeline_name.find(entry.first) != std::string::npos) { - return entry.second; - } - } - return config.default_subgroup_size; + *output = config; + return true; + } + } + return false; +} + +static bool get_pipeline_config_parameter(PipelineConfigParameter* output, const GpuPipelineConfig& config, const std::string &pipeline_name) { + auto pipIt = config.pipelines.find(pipeline_name); + if (pipIt != config.pipelines.end()) { + *output = pipIt->second; + return true; + } + std::vector> sorted_pipelines(config.pipelines.begin(), config.pipelines.end()); + std::sort(sorted_pipelines.begin(), sorted_pipelines.end(), + [](const auto &a, const auto &b) { return a.first.size() > b.first.size(); }); + for (const auto &entry : sorted_pipelines) { + if (pipeline_name.find(entry.first) != std::string::npos) { + *output = entry.second; + return true; } } - return 0; // If no matching configuration is found + return false; } // Whether scalar flash attention will use the MMQ path for the given k_type. @@ -3941,9 +4091,29 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { auto const &ggml_vk_create_pipeline = [&](vk_device& device, vk_pipeline& base_pipeline, const char *name, size_t spv_size, const void* spv_data, const char *entrypoint, uint32_t parameter_count, uint32_t push_constant_size, std::array wg_denoms, const std::vector& specialization_constants, uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) { - - if (!require_full_subgroups && required_subgroup_size == 0) { - required_subgroup_size = get_subgroup_size(name, device->architecture); + // Override subgroup size and specialization constant based on pipeline name + GpuPipelineConfig gpu_config = {}; + PipelineConfigParameter pipeline_param = {}; + bool pipeline_param_found = false; + auto gpu_config_found = get_gpu_pipeline_config(&gpu_config, device->architecture); + if (gpu_config_found) { + pipeline_param_found = get_pipeline_config_parameter(&pipeline_param, gpu_config, std::string(name)); + } + + // If we have a config for this GPU we update the specialization constant and required subgroup size + // based on custom logic for each GPU + std::vector final_specialization_constant = specialization_constants; + uint32_t final_required_subgroup_size = required_subgroup_size; + if (gpu_config_found && gpu_config.update_subgroup_params) { + gpu_config.update_subgroup_params( + pipeline_param_found, + pipeline_param, + specialization_constants, + require_full_subgroups, + required_subgroup_size, + gpu_config.default_subgroup_size, + final_required_subgroup_size, + final_specialization_constant); } vk_pipeline *ptr = &base_pipeline; @@ -3992,10 +4162,10 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { claimed_task.entrypoint = entrypoint; claimed_task.parameter_count = parameter_count; claimed_task.wg_denoms = wg_denoms; - claimed_task.specialization_constants = specialization_constants; + claimed_task.specialization_constants = final_specialization_constant; claimed_task.disable_robustness = disable_robustness; claimed_task.require_full_subgroups = require_full_subgroups; - claimed_task.required_subgroup_size = required_subgroup_size; + claimed_task.required_subgroup_size = final_required_subgroup_size; has_claimed_task = true; } } @@ -4142,6 +4312,54 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { }; const int mul_mat_id_param_count = 5; + struct subgroup_kernel_choice { + bool use_subgroup_16 = false; + uint32_t required_subgroup_size = 0; + }; + + GpuPipelineConfig matmul_id_gpu_config = {}; + const bool matmul_id_gpu_config_found = get_gpu_pipeline_config(&matmul_id_gpu_config, device->architecture); + const auto choose_mulmat_id_kernel = [&](const char * mulmat_id_kernel_name) { + subgroup_kernel_choice choice {}; + if (!(device->subgroup_ballot && device->subgroup_require_full_support)) { + return choice; + } + + if (matmul_id_gpu_config_found) { + PipelineConfigParameter pipeline_param = {}; + if (get_pipeline_config_parameter(&pipeline_param, matmul_id_gpu_config, std::string(mulmat_id_kernel_name)) && pipeline_param.subgroup_size) { + choice.required_subgroup_size = pipeline_param.subgroup_size; + } + } + + choice.use_subgroup_16 = (choice.required_subgroup_size == 0 && subgroup_min_size_16) || + (choice.required_subgroup_size == 16); + return choice; + }; + + const auto choose_mulmat_id_variant = [&](const std::string & subgroup_kernel_name, + const std::string & base_kernel_name, + uint32_t subgroup_default_required_size, + uint32_t base_default_required_size, + const std::function & emit_base, + const std::function & emit_subgroup) { + PipelineConfigParameter pipeline_param = {}; + if (matmul_id_gpu_config_found && + get_pipeline_config_parameter(&pipeline_param, matmul_id_gpu_config, base_kernel_name) && + pipeline_param.subgroup_size) { + emit_base(pipeline_param.subgroup_size); + return; + } + + const subgroup_kernel_choice subgroup_choice = choose_mulmat_id_kernel(subgroup_kernel_name.c_str()); + if (subgroup_choice.use_subgroup_16) { + const uint32_t required_subgroup_size = subgroup_choice.required_subgroup_size > 0 ? subgroup_choice.required_subgroup_size : subgroup_default_required_size; + emit_subgroup(required_subgroup_size); + return; + } + + emit_base(base_default_required_size); + }; #if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) if (device->coopmat2) { @@ -4389,16 +4607,31 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { if (device->mul_mat ## ID ## _s[TYPE]) \ ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ +#define CREATE_MMQ_VARIANT(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE, SZ, SLOT) \ + if (device->mul_mat ## ID ## _ ## SZ ## _int[TYPE]) { \ + ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc-> SLOT, #NAMELC "_" #SZ, NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), SZ ## _ ## WG_DENOMS, SZ ## _ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ + } + #define CREATE_MMQ(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->l, #NAMELC "_l", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, l_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ - if (device->mul_mat ## ID ## _m_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->m, #NAMELC "_m", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, m_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ - if (device->mul_mat ## ID ## _s_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->s, #NAMELC "_s", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, s_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ + CREATE_MMQ_VARIANT(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE, l, l) \ + CREATE_MMQ_VARIANT(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE, m, m) \ + CREATE_MMQ_VARIANT(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE, s, s) \ + +#define CREATE_MM_VARIANT_UNALIGNED(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE, SZ, SLOT) \ + if (device->mul_mat ## ID ## _ ## SZ [TYPE]) \ + ggml_vk_create_pipeline(device, device-> PIPELINE_NAME -> SLOT, #NAMELC #F16ACC "_" #SZ, (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), SZ ## _ ## WG_DENOMS, ggml_vk_mul_mm_spec(SZ ## _ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ + +#define CREATE_MM_VARIANT_ALIGNED(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE, SZ, SLOT, ALIGNV) \ + if (device->mul_mat ## ID ## _ ## SZ [TYPE]) \ + ggml_vk_create_pipeline(device, device-> PIPELINE_NAME -> SLOT, #NAMELC #F16ACC "_aligned_" #SZ, (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), SZ ## _ ## WG_DENOMS, ggml_vk_mul_mm_spec(SZ ## _ ## WARPTILE, true), ALIGNV, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ + +#define CREATE_MM_VARIANT_UNALIGNED_NODOT2(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE, SZ, SLOT) \ + if (device->mul_mat ## ID ## _ ## SZ [TYPE]) \ + ggml_vk_create_pipeline(device, device-> PIPELINE_NAME -> SLOT, #NAMELC #F16ACC "_" #SZ, NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), SZ ## _ ## WG_DENOMS, ggml_vk_mul_mm_spec(SZ ## _ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ + +#define CREATE_MM_VARIANT_ALIGNED_NODOT2(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE, SZ, SLOT, ALIGNV) \ + if (device->mul_mat ## ID ## _ ## SZ [TYPE]) \ + ggml_vk_create_pipeline(device, device-> PIPELINE_NAME -> SLOT, #NAMELC #F16ACC "_aligned_" #SZ, NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), SZ ## _ ## WG_DENOMS, ggml_vk_mul_mm_spec(SZ ## _ ## WARPTILE, true), ALIGNV, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ // Create 2 variants, {f16,f32} accumulator #define CREATE_MM2(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ @@ -4453,97 +4686,122 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } #endif - if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_subgroup_f16_f32, wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); +#define CREATE_MM_ID_AUTO(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, BASE_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE) \ + do { \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_l"), std::string(#BASE_NAMELC #F16ACC "_l"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, l); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, l); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_m"), std::string(#BASE_NAMELC #F16ACC "_m"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, m); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, m); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_s"), std::string(#BASE_NAMELC #F16ACC "_s"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, s); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, s); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_aligned_l"), std::string(#BASE_NAMELC #F16ACC "_aligned_l"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, a_l, l_align); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, a_l, l_align); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_aligned_m"), std::string(#BASE_NAMELC #F16ACC "_aligned_m"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, a_m, m_align); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, a_m, m_align); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_aligned_s"), std::string(#BASE_NAMELC #F16ACC "_aligned_s"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, a_s, s_align); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, a_s, s_align); }); \ + } while (false) + +#define CREATE_MM2_ID_AUTO(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, BASE_NAMELC, WG_DENOMS, SUB_WARPTILE, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE) \ + CREATE_MM_ID_AUTO(TYPE, PIPELINE_NAME . f16acc, SUBGROUP_NAMELC, BASE_NAMELC, _f16acc, WG_DENOMS, SUB_WARPTILE, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE); \ + CREATE_MM_ID_AUTO(TYPE, PIPELINE_NAME . f32acc, SUBGROUP_NAMELC, BASE_NAMELC, , WG_DENOMS, SUB_WARPTILE, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE) + +#define CREATE_MM_ID_AUTO_NODOT2(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, BASE_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE) \ + do { \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_l"), std::string(#BASE_NAMELC #F16ACC "_l"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_NODOT2(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, l); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_NODOT2(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, l); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_m"), std::string(#BASE_NAMELC #F16ACC "_m"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_NODOT2(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, m); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_NODOT2(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, m); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_s"), std::string(#BASE_NAMELC #F16ACC "_s"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_NODOT2(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, s); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_NODOT2(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, s); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_aligned_l"), std::string(#BASE_NAMELC #F16ACC "_aligned_l"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_NODOT2(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, a_l, l_align); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_NODOT2(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, a_l, l_align); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_aligned_m"), std::string(#BASE_NAMELC #F16ACC "_aligned_m"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_NODOT2(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, a_m, m_align); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_NODOT2(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, a_m, m_align); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC #F16ACC "_aligned_s"), std::string(#BASE_NAMELC #F16ACC "_aligned_s"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_NODOT2(TYPE, PIPELINE_NAME, BASE_NAMELC, F16ACC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, a_s, s_align); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_NODOT2(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, F16ACC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, a_s, s_align); }); \ + } while (false) + +#define CREATE_MMQ_ID_AUTO(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, BASE_NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE) \ + do { \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC "_l"), std::string(#BASE_NAMELC "_l"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MMQ_VARIANT(TYPE, PIPELINE_NAME, BASE_NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, l); }, \ + [&](uint32_t req) { CREATE_MMQ_VARIANT(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, l); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC "_m"), std::string(#BASE_NAMELC "_m"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MMQ_VARIANT(TYPE, PIPELINE_NAME, BASE_NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, m); }, \ + [&](uint32_t req) { CREATE_MMQ_VARIANT(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, m); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC "_s"), std::string(#BASE_NAMELC "_s"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MMQ_VARIANT(TYPE, PIPELINE_NAME, BASE_NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, s); }, \ + [&](uint32_t req) { CREATE_MMQ_VARIANT(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, s); }); \ + } while (false) + + CREATE_MM_ID_AUTO(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, matmul_id_f32_f32, , wg_denoms, warptile_id, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_subgroup_f16, matmul_id_f16, wg_denoms, warptile_id, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_subgroup_f16_f32, matmul_id_f16_f32, wg_denoms, warptile_id, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + CREATE_MM_ID_AUTO_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, matmul_id_bf16, , wg_denoms, warptile_id, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + + CREATE_MM2_ID_AUTO(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, matmul_id_q1_0_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, matmul_id_q4_0_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, matmul_id_q4_1_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f32, matmul_id_q5_0_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f32, matmul_id_q5_1_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f32, matmul_id_q8_0_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f32, matmul_id_q2_k_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f32, matmul_id_q3_k_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f32, matmul_id_q4_k_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f32, matmul_id_q5_k_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f32, matmul_id_q6_k_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f32, matmul_id_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f32, matmul_id_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f32, matmul_id_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f32, matmul_id_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f32, matmul_id_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f32, matmul_id_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f32, matmul_id_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f32, matmul_id_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f32, matmul_id_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32, matmul_id_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM2_ID_AUTO(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32, matmul_id_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - - CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - } -#endif - } else { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_f16_f32, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_q5_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_q5_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_q8_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_q2_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_q3_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_q4_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_q5_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_q6_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); + if (device->integer_dot_product) { + CREATE_MMQ_ID_AUTO(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_q8_1, matmul_id_q4_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MMQ_ID_AUTO(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_q8_1, matmul_id_q4_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MMQ_ID_AUTO(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_q8_1, matmul_id_q5_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MMQ_ID_AUTO(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_q8_1, matmul_id_q5_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MMQ_ID_AUTO(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_q8_1, matmul_id_q8_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_0], matmul_id_q4_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_1], matmul_id_q4_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_0], matmul_id_q5_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_1], matmul_id_q5_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q8_0], matmul_id_q8_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_MXFP4], matmul_id_mxfp4_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_K], matmul_id_q2_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q3_K], matmul_id_q3_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_K], matmul_id_q4_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_K], matmul_id_q5_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q6_K], matmul_id_q6_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - } -#endif + CREATE_MMQ_ID_AUTO(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_q8_1, matmul_id_mxfp4_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + + CREATE_MMQ_ID_AUTO(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_q8_1, matmul_id_q2_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + CREATE_MMQ_ID_AUTO(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_q8_1, matmul_id_q3_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + CREATE_MMQ_ID_AUTO(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_q8_1, matmul_id_q4_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + CREATE_MMQ_ID_AUTO(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_q8_1, matmul_id_q5_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + CREATE_MMQ_ID_AUTO(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_q8_1, matmul_id_q6_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); } +#endif + +#undef CREATE_MMQ_ID_AUTO +#undef CREATE_MM_ID_AUTO_NODOT2 +#undef CREATE_MM2_ID_AUTO +#undef CREATE_MM_ID_AUTO +#undef CREATE_MMQ_VARIANT +#undef CREATE_MM_VARIANT_ALIGNED_NODOT2 +#undef CREATE_MM_VARIANT_UNALIGNED_NODOT2 +#undef CREATE_MM_VARIANT_ALIGNED +#undef CREATE_MM_VARIANT_UNALIGNED #undef CREATE_MM2 #undef CREATE_MMQ #undef CREATE_MM @@ -4572,6 +4830,14 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { if (device->mul_mat ## ID ## _s_int[TYPE]) \ ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC "_s", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, s_ ## WARPTILE, 1); \ +#define CREATE_MM_VARIANT_UNALIGNED_FP32(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE, SZ, SLOT) \ + if (device->mul_mat ## ID ## _ ## SZ [TYPE]) \ + ggml_vk_create_pipeline(device, device-> PIPELINE_NAME -> SLOT, #NAMELC "_" #SZ, NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), SZ ## _ ## WG_DENOMS, SZ ## _ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ + +#define CREATE_MM_VARIANT_ALIGNED_FP32(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE, SZ, SLOT, ALIGNV) \ + if (device->mul_mat ## ID ## _ ## SZ [TYPE]) \ + ggml_vk_create_pipeline(device, device-> PIPELINE_NAME -> SLOT, #NAMELC "_aligned_" #SZ, NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), SZ ## _ ## WG_DENOMS, SZ ## _ ## WARPTILE, ALIGNV, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ + CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); CREATE_MM(GGML_TYPE_F16, pipeline_matmul_f16.f32acc, matmul_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); @@ -4619,63 +4885,59 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } #endif - if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16.f32acc, matmul_id_subgroup_f16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16_f32.f32acc, matmul_id_subgroup_f16_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - - CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_subgroup_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_subgroup_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_subgroup_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0].f32acc, matmul_id_subgroup_q5_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1].f32acc, matmul_id_subgroup_q5_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0].f32acc, matmul_id_subgroup_q8_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K].f32acc, matmul_id_subgroup_q2_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K].f32acc, matmul_id_subgroup_q3_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K].f32acc, matmul_id_subgroup_q4_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K].f32acc, matmul_id_subgroup_q5_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K].f32acc, matmul_id_subgroup_q6_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S].f32acc, matmul_id_subgroup_iq1_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M].f32acc, matmul_id_subgroup_iq1_m_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS].f32acc, matmul_id_subgroup_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS].f32acc, matmul_id_subgroup_iq2_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S].f32acc, matmul_id_subgroup_iq2_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS].f32acc, matmul_id_subgroup_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S].f32acc, matmul_id_subgroup_iq3_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS].f32acc, matmul_id_subgroup_iq4_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL].f32acc, matmul_id_subgroup_iq4_nl_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4].f32acc, matmul_id_subgroup_mxfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4].f32acc, matmul_id_subgroup_nvfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - } else { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16.f32acc, matmul_id_f16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16_f32.f32acc, matmul_id_f16_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0].f32acc, matmul_id_q5_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1].f32acc, matmul_id_q5_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0].f32acc, matmul_id_q8_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K].f32acc, matmul_id_q2_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K].f32acc, matmul_id_q3_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K].f32acc, matmul_id_q4_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K].f32acc, matmul_id_q5_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K].f32acc, matmul_id_q6_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S].f32acc, matmul_id_iq1_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M].f32acc, matmul_id_iq1_m_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS].f32acc, matmul_id_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS].f32acc, matmul_id_iq2_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S].f32acc, matmul_id_iq2_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS].f32acc, matmul_id_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S].f32acc, matmul_id_iq3_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS].f32acc, matmul_id_iq4_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL].f32acc, matmul_id_iq4_nl_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4].f32acc, matmul_id_mxfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4].f32acc, matmul_id_nvfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - } +#define CREATE_MM_ID_AUTO_FP32(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, BASE_NAMELC, WG_DENOMS, SUB_WARPTILE, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE) \ + do { \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC "_l"), std::string(#BASE_NAMELC "_l"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_FP32(TYPE, PIPELINE_NAME, BASE_NAMELC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, l); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_FP32(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, l); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC "_m"), std::string(#BASE_NAMELC "_m"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_FP32(TYPE, PIPELINE_NAME, BASE_NAMELC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, m); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_FP32(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, m); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC "_s"), std::string(#BASE_NAMELC "_s"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_FP32(TYPE, PIPELINE_NAME, BASE_NAMELC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, s); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_UNALIGNED_FP32(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, s); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC "_aligned_l"), std::string(#BASE_NAMELC "_aligned_l"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_FP32(TYPE, PIPELINE_NAME, BASE_NAMELC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, a_l, l_align); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_FP32(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, l, a_l, l_align); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC "_aligned_m"), std::string(#BASE_NAMELC "_aligned_m"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_FP32(TYPE, PIPELINE_NAME, BASE_NAMELC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, a_m, m_align); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_FP32(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, m, a_m, m_align); }); \ + choose_mulmat_id_variant(std::string(#SUBGROUP_NAMELC "_aligned_s"), std::string(#BASE_NAMELC "_aligned_s"), SUB_REQSUBGROUPSIZE, BASE_REQSUBGROUPSIZE, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_FP32(TYPE, PIPELINE_NAME, BASE_NAMELC, WG_DENOMS, BASE_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, a_s, s_align); }, \ + [&](uint32_t req) { CREATE_MM_VARIANT_ALIGNED_FP32(TYPE, PIPELINE_NAME, SUBGROUP_NAMELC, WG_DENOMS, SUB_WARPTILE, PUSHCONST, PARAMCOUNT, ID, req, s, a_s, s_align); }); \ + } while (false) + + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, matmul_id_f32_f32, wg_denoms, warptile_id, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_F16, pipeline_matmul_id_f16.f32acc, matmul_id_subgroup_f16, matmul_id_f16, wg_denoms, warptile_id, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_F16, pipeline_matmul_id_f16_f32.f32acc, matmul_id_subgroup_f16_f32, matmul_id_f16_f32, wg_denoms, warptile_id, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, matmul_id_bf16, wg_denoms, warptile_id, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16, 0); + + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_subgroup_q1_0_f32, matmul_id_q1_0_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_subgroup_q4_0_f32, matmul_id_q4_0_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_subgroup_q4_1_f32, matmul_id_q4_1_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0].f32acc, matmul_id_subgroup_q5_0_f32, matmul_id_q5_0_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1].f32acc, matmul_id_subgroup_q5_1_f32, matmul_id_q5_1_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0].f32acc, matmul_id_subgroup_q8_0_f32, matmul_id_q8_0_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K].f32acc, matmul_id_subgroup_q2_k_f32, matmul_id_q2_k_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K].f32acc, matmul_id_subgroup_q3_k_f32, matmul_id_q3_k_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K].f32acc, matmul_id_subgroup_q4_k_f32, matmul_id_q4_k_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K].f32acc, matmul_id_subgroup_q5_k_f32, matmul_id_q5_k_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K].f32acc, matmul_id_subgroup_q6_k_f32, matmul_id_q6_k_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S].f32acc, matmul_id_subgroup_iq1_s_f32, matmul_id_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M].f32acc, matmul_id_subgroup_iq1_m_f32, matmul_id_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS].f32acc, matmul_id_subgroup_iq2_xxs_f32, matmul_id_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS].f32acc, matmul_id_subgroup_iq2_xs_f32, matmul_id_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S].f32acc, matmul_id_subgroup_iq2_s_f32, matmul_id_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS].f32acc, matmul_id_subgroup_iq3_xxs_f32, matmul_id_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S].f32acc, matmul_id_subgroup_iq3_s_f32, matmul_id_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS].f32acc, matmul_id_subgroup_iq4_xs_f32, matmul_id_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL].f32acc, matmul_id_subgroup_iq4_nl_f32, matmul_id_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4].f32acc, matmul_id_subgroup_mxfp4_f32, matmul_id_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + CREATE_MM_ID_AUTO_FP32(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4].f32acc, matmul_id_subgroup_nvfp4_f32, matmul_id_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size, 0); + +#undef CREATE_MM_ID_AUTO_FP32 +#undef CREATE_MM_VARIANT_ALIGNED_FP32 +#undef CREATE_MM_VARIANT_UNALIGNED_FP32 } // reusing CREATE_MM from the fp32 path if ((device->coopmat2 || device->coopmat_support) @@ -4727,9 +4989,10 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { const uint32_t subgroup_size16 = std::max(subgroup_size, 16u); const uint32_t force_subgroup_size = use_subgroups ? subgroup_size : 0; - const uint32_t force_subgroup_size16 = use_subgroups16 ? subgroup_size16 : 0; static constexpr uint32_t mul_mat_vec_num_bindings = 5; static constexpr uint32_t mul_mat_vec_id_num_bindings = 6; + GpuPipelineConfig mul_mat_vec_gpu_config = {}; + const bool mul_mat_vec_gpu_config_found = get_gpu_pipeline_config(&mul_mat_vec_gpu_config, device->architecture); for (uint32_t w = 0; w < DMMV_WG_SIZE_COUNT; ++w) { const uint32_t wg_size_subgroup = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size : (subgroup_size * 4); @@ -4739,11 +5002,26 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { (use_subgroups && w == DMMV_WG_SIZE_LARGE) ? SHADER_REDUCTION_MODE_HYBRID : SHADER_REDUCTION_MODE_SHMEM; - const shader_reduction_mode reduc16 = (use_subgroups16 && w == DMMV_WG_SIZE_SUBGROUP) ? SHADER_REDUCTION_MODE_SUBGROUP : - (use_subgroups16 && w == DMMV_WG_SIZE_LARGE) ? SHADER_REDUCTION_MODE_HYBRID : - SHADER_REDUCTION_MODE_SHMEM; - for (uint32_t i = 0; i < mul_mat_vec_max_cols; ++i) { + const auto create_reduc16_pipeline = [&](vk_pipeline & pipeline, const char * name, const auto & shader_lens, const auto & shader_data, std::array wg_denoms, std::initializer_list specialization_constants) { + uint32_t required_subgroup_size = 0; + + if (mul_mat_vec_gpu_config_found) { + PipelineConfigParameter pipeline_param = {}; + if (get_pipeline_config_parameter(&pipeline_param, mul_mat_vec_gpu_config, std::string(name)) && pipeline_param.subgroup_size) { + required_subgroup_size = pipeline_param.subgroup_size; + } + } + + // We use subgroup 16 only when no override setting exists and prefers to 16, or override setting is set to 16 + const bool pipeline_use_subgroups16 = use_subgroups && ((required_subgroup_size == 0 && subgroup_min_size_16) || (required_subgroup_size == 16)); + const shader_reduction_mode pipeline_reduc16 = (pipeline_use_subgroups16 && w == DMMV_WG_SIZE_SUBGROUP) ? SHADER_REDUCTION_MODE_SUBGROUP : + (pipeline_use_subgroups16 && w == DMMV_WG_SIZE_LARGE) ? SHADER_REDUCTION_MODE_HYBRID : + SHADER_REDUCTION_MODE_SHMEM; + + ggml_vk_create_pipeline(device, pipeline, name, shader_lens[pipeline_reduc16], shader_data[pipeline_reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), wg_denoms, std::vector(specialization_constants), 1, true, pipeline_use_subgroups16, pipeline_use_subgroups16 ? 16 : required_subgroup_size); + }; + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f32_f32", arr_dmmv_f32_f32_f32_len[reduc], arr_dmmv_f32_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1, 1, 1}, {wg_size_subgroup, 1, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f32_f32", arr_dmmv_f16_f32_f32_len[reduc], arr_dmmv_f16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f32_f32", arr_dmmv_bf16_f32_f32_len[reduc], arr_dmmv_bf16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); @@ -4753,22 +5031,22 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f32_f32", arr_dmmv_q5_0_f32_f32_len[reduc], arr_dmmv_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f32_f32", arr_dmmv_q5_1_f32_f32_len[reduc], arr_dmmv_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f32_f32", arr_dmmv_q8_0_f32_f32_len[reduc], arr_dmmv_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f32_f32", arr_dmmv_q2_k_f32_f32_len[reduc16], arr_dmmv_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f32_f32", arr_dmmv_q3_k_f32_f32_len[reduc16], arr_dmmv_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f32_f32", arr_dmmv_q4_k_f32_f32_len[reduc16], arr_dmmv_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f32_f32", arr_dmmv_q5_k_f32_f32_len[reduc16], arr_dmmv_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f32_f32", arr_dmmv_q6_k_f32_f32_len[reduc16], arr_dmmv_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f32_f32", arr_dmmv_iq1_s_f32_f32_len[reduc16], arr_dmmv_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f32_f32", arr_dmmv_iq1_m_f32_f32_len[reduc16], arr_dmmv_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f32_f32", arr_dmmv_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f32_f32", arr_dmmv_iq2_xs_f32_f32_len[reduc16], arr_dmmv_iq2_xs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f32_f32", arr_dmmv_iq2_s_f32_f32_len[reduc16], arr_dmmv_iq2_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f32_f32", arr_dmmv_iq3_xxs_f32_f32_len[reduc16], arr_dmmv_iq3_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f32_f32", arr_dmmv_iq3_s_f32_f32_len[reduc16], arr_dmmv_iq3_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f32_f32", arr_dmmv_iq4_xs_f32_f32_len[reduc16], arr_dmmv_iq4_xs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f32_f32", arr_dmmv_iq4_nl_f32_f32_len[reduc16], arr_dmmv_iq4_nl_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f32_f32", arr_dmmv_mxfp4_f32_f32_len[reduc16], arr_dmmv_mxfp4_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f32_f32", arr_dmmv_nvfp4_f32_f32_len[reduc16], arr_dmmv_nvfp4_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f32_f32", arr_dmmv_q2_k_f32_f32_len, arr_dmmv_q2_k_f32_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f32_f32", arr_dmmv_q3_k_f32_f32_len, arr_dmmv_q3_k_f32_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f32_f32", arr_dmmv_q4_k_f32_f32_len, arr_dmmv_q4_k_f32_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f32_f32", arr_dmmv_q5_k_f32_f32_len, arr_dmmv_q5_k_f32_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f32_f32", arr_dmmv_q6_k_f32_f32_len, arr_dmmv_q6_k_f32_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f32_f32", arr_dmmv_iq1_s_f32_f32_len, arr_dmmv_iq1_s_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f32_f32", arr_dmmv_iq1_m_f32_f32_len, arr_dmmv_iq1_m_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f32_f32", arr_dmmv_iq2_xxs_f32_f32_len, arr_dmmv_iq2_xxs_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f32_f32", arr_dmmv_iq2_xs_f32_f32_len, arr_dmmv_iq2_xs_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f32_f32", arr_dmmv_iq2_s_f32_f32_len, arr_dmmv_iq2_s_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f32_f32", arr_dmmv_iq3_xxs_f32_f32_len, arr_dmmv_iq3_xxs_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f32_f32", arr_dmmv_iq3_s_f32_f32_len, arr_dmmv_iq3_s_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f32_f32", arr_dmmv_iq4_xs_f32_f32_len, arr_dmmv_iq4_xs_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f32_f32", arr_dmmv_iq4_nl_f32_f32_len, arr_dmmv_iq4_nl_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f32_f32", arr_dmmv_mxfp4_f32_f32_len, arr_dmmv_mxfp4_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f32_f32", arr_dmmv_nvfp4_f32_f32_len, arr_dmmv_nvfp4_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f16_f32", arr_dmmv_f32_f16_f32_len[reduc], arr_dmmv_f32_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1, 1, 1}, {wg_size_subgroup, 1, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f16_f32", arr_dmmv_f16_f16_f32_len[reduc], arr_dmmv_f16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); @@ -4779,22 +5057,22 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f16_f32", arr_dmmv_q5_0_f16_f32_len[reduc], arr_dmmv_q5_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f16_f32", arr_dmmv_q5_1_f16_f32_len[reduc], arr_dmmv_q5_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f16_f32", arr_dmmv_q8_0_f16_f32_len[reduc], arr_dmmv_q8_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f16_f32", arr_dmmv_q2_k_f16_f32_len[reduc16], arr_dmmv_q2_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f16_f32", arr_dmmv_q3_k_f16_f32_len[reduc16], arr_dmmv_q3_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f16_f32", arr_dmmv_q4_k_f16_f32_len[reduc16], arr_dmmv_q4_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f16_f32", arr_dmmv_q5_k_f16_f32_len[reduc16], arr_dmmv_q5_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f16_f32", arr_dmmv_q6_k_f16_f32_len[reduc16], arr_dmmv_q6_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f16_f32", arr_dmmv_iq1_s_f16_f32_len[reduc16], arr_dmmv_iq1_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f16_f32", arr_dmmv_iq1_m_f16_f32_len[reduc16], arr_dmmv_iq1_m_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f16_f32", arr_dmmv_iq2_xxs_f16_f32_len[reduc16], arr_dmmv_iq2_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f16_f32", arr_dmmv_iq2_xs_f16_f32_len[reduc16], arr_dmmv_iq2_xs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f16_f32", arr_dmmv_iq2_s_f16_f32_len[reduc16], arr_dmmv_iq2_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f16_f32", arr_dmmv_iq3_xxs_f16_f32_len[reduc16], arr_dmmv_iq3_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f16_f32", arr_dmmv_iq3_s_f16_f32_len[reduc16], arr_dmmv_iq3_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f16_f32", arr_dmmv_iq4_xs_f16_f32_len[reduc16], arr_dmmv_iq4_xs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f16_f32", arr_dmmv_iq4_nl_f16_f32_len[reduc16], arr_dmmv_iq4_nl_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f16_f32", arr_dmmv_mxfp4_f16_f32_len[reduc16], arr_dmmv_mxfp4_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f16_f32", arr_dmmv_nvfp4_f16_f32_len[reduc16], arr_dmmv_nvfp4_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f16_f32", arr_dmmv_q2_k_f16_f32_len, arr_dmmv_q2_k_f16_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f16_f32", arr_dmmv_q3_k_f16_f32_len, arr_dmmv_q3_k_f16_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f16_f32", arr_dmmv_q4_k_f16_f32_len, arr_dmmv_q4_k_f16_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f16_f32", arr_dmmv_q5_k_f16_f32_len, arr_dmmv_q5_k_f16_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f16_f32", arr_dmmv_q6_k_f16_f32_len, arr_dmmv_q6_k_f16_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f16_f32", arr_dmmv_iq1_s_f16_f32_len, arr_dmmv_iq1_s_f16_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f16_f32", arr_dmmv_iq1_m_f16_f32_len, arr_dmmv_iq1_m_f16_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f16_f32", arr_dmmv_iq2_xxs_f16_f32_len, arr_dmmv_iq2_xxs_f16_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f16_f32", arr_dmmv_iq2_xs_f16_f32_len, arr_dmmv_iq2_xs_f16_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f16_f32", arr_dmmv_iq2_s_f16_f32_len, arr_dmmv_iq2_s_f16_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f16_f32", arr_dmmv_iq3_xxs_f16_f32_len, arr_dmmv_iq3_xxs_f16_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f16_f32", arr_dmmv_iq3_s_f16_f32_len, arr_dmmv_iq3_s_f16_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f16_f32", arr_dmmv_iq4_xs_f16_f32_len, arr_dmmv_iq4_xs_f16_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f16_f32", arr_dmmv_iq4_nl_f16_f32_len, arr_dmmv_iq4_nl_f16_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f16_f32", arr_dmmv_mxfp4_f16_f32_len, arr_dmmv_mxfp4_f16_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); + create_reduc16_pipeline(device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f16_f32", arr_dmmv_nvfp4_f16_f32_len, arr_dmmv_nvfp4_f16_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}); #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { @@ -4822,6 +5100,25 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #endif // GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT } + const auto create_reduc16_id_pipeline = [&](vk_pipeline & pipeline, const char * name, const auto & shader_lens, const auto & shader_data, std::array wg_denoms, std::initializer_list specialization_constants) { + uint32_t required_subgroup_size = 0; + + if (mul_mat_vec_gpu_config_found) { + PipelineConfigParameter pipeline_param = {}; + if (get_pipeline_config_parameter(&pipeline_param, mul_mat_vec_gpu_config, std::string(name)) && pipeline_param.subgroup_size) { + required_subgroup_size = pipeline_param.subgroup_size; + } + } + + // We use subgroup 16 only when no override setting exists and prefers to 16, or override setting is set to 16 + const bool pipeline_use_subgroups16 = use_subgroups && ((required_subgroup_size == 0 && subgroup_min_size_16) || (required_subgroup_size == 16)); + const shader_reduction_mode pipeline_reduc16 = (pipeline_use_subgroups16 && w == DMMV_WG_SIZE_SUBGROUP) ? SHADER_REDUCTION_MODE_SUBGROUP : + (pipeline_use_subgroups16 && w == DMMV_WG_SIZE_LARGE) ? SHADER_REDUCTION_MODE_HYBRID : + SHADER_REDUCTION_MODE_SHMEM; + + ggml_vk_create_pipeline(device, pipeline, name, shader_lens[pipeline_reduc16], shader_data[pipeline_reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), wg_denoms, std::vector(specialization_constants), 1, true, pipeline_use_subgroups16, pipeline_use_subgroups16 ? 16 : required_subgroup_size); + }; + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F32 ], "mul_mat_vec_id_f32_f32", arr_dmmv_id_f32_f32_f32_len[reduc], arr_dmmv_id_f32_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1, 1, 1}, {wg_size_subgroup, 1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F16 ], "mul_mat_vec_id_f16_f32", arr_dmmv_id_f16_f32_f32_len[reduc], arr_dmmv_id_f16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_BF16], "mul_mat_vec_id_bf16_f32", arr_dmmv_id_bf16_f32_f32_len[reduc], arr_dmmv_id_bf16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); @@ -4831,22 +5128,22 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_0], "mul_mat_vec_id_q5_0_f32", arr_dmmv_id_q5_0_f32_f32_len[reduc], arr_dmmv_id_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_1], "mul_mat_vec_id_q5_1_f32", arr_dmmv_id_q5_1_f32_f32_len[reduc], arr_dmmv_id_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q8_0], "mul_mat_vec_id_q8_0_f32", arr_dmmv_id_q8_0_f32_f32_len[reduc], arr_dmmv_id_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_K], "mul_mat_vec_id_q2_k_f32", arr_dmmv_id_q2_k_f32_f32_len[reduc16], arr_dmmv_id_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q3_K], "mul_mat_vec_id_q3_k_f32", arr_dmmv_id_q3_k_f32_f32_len[reduc16], arr_dmmv_id_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_K], "mul_mat_vec_id_q4_k_f32", arr_dmmv_id_q4_k_f32_f32_len[reduc16], arr_dmmv_id_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_K], "mul_mat_vec_id_q5_k_f32", arr_dmmv_id_q5_k_f32_f32_len[reduc16], arr_dmmv_id_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q6_K], "mul_mat_vec_id_q6_k_f32", arr_dmmv_id_q6_k_f32_f32_len[reduc16], arr_dmmv_id_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_f32", arr_dmmv_id_iq1_s_f32_f32_len[reduc16], arr_dmmv_id_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_f32", arr_dmmv_id_iq1_m_f32_f32_len[reduc16], arr_dmmv_id_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XXS], "mul_mat_vec_id_iq2_xxs_f32", arr_dmmv_id_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XS], "mul_mat_vec_id_iq2_xs_f32", arr_dmmv_id_iq2_xs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_S], "mul_mat_vec_id_iq2_s_f32", arr_dmmv_id_iq2_s_f32_f32_len[reduc16], arr_dmmv_id_iq2_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_XXS], "mul_mat_vec_id_iq3_xxs_f32", arr_dmmv_id_iq3_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq3_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_S], "mul_mat_vec_id_iq3_s_f32", arr_dmmv_id_iq3_s_f32_f32_len[reduc16], arr_dmmv_id_iq3_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_XS], "mul_mat_vec_id_iq4_xs_f32", arr_dmmv_id_iq4_xs_f32_f32_len[reduc16], arr_dmmv_id_iq4_xs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_NL], "mul_mat_vec_id_iq4_nl_f32", arr_dmmv_id_iq4_nl_f32_f32_len[reduc16], arr_dmmv_id_iq4_nl_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_MXFP4], "mul_mat_vec_id_mxfp4_f32", arr_dmmv_id_mxfp4_f32_f32_len[reduc16], arr_dmmv_id_mxfp4_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_NVFP4], "mul_mat_vec_id_nvfp4_f32", arr_dmmv_id_nvfp4_f32_f32_len[reduc16], arr_dmmv_id_nvfp4_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_K], "mul_mat_vec_id_q2_k_f32", arr_dmmv_id_q2_k_f32_f32_len, arr_dmmv_id_q2_k_f32_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q3_K], "mul_mat_vec_id_q3_k_f32", arr_dmmv_id_q3_k_f32_f32_len, arr_dmmv_id_q3_k_f32_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_K], "mul_mat_vec_id_q4_k_f32", arr_dmmv_id_q4_k_f32_f32_len, arr_dmmv_id_q4_k_f32_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_K], "mul_mat_vec_id_q5_k_f32", arr_dmmv_id_q5_k_f32_f32_len, arr_dmmv_id_q5_k_f32_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q6_K], "mul_mat_vec_id_q6_k_f32", arr_dmmv_id_q6_k_f32_f32_len, arr_dmmv_id_q6_k_f32_f32_data, {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_f32", arr_dmmv_id_iq1_s_f32_f32_len, arr_dmmv_id_iq1_s_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_f32", arr_dmmv_id_iq1_m_f32_f32_len, arr_dmmv_id_iq1_m_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XXS], "mul_mat_vec_id_iq2_xxs_f32", arr_dmmv_id_iq2_xxs_f32_f32_len, arr_dmmv_id_iq2_xxs_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XS], "mul_mat_vec_id_iq2_xs_f32", arr_dmmv_id_iq2_xs_f32_f32_len, arr_dmmv_id_iq2_xs_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_S], "mul_mat_vec_id_iq2_s_f32", arr_dmmv_id_iq2_s_f32_f32_len, arr_dmmv_id_iq2_s_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_XXS], "mul_mat_vec_id_iq3_xxs_f32", arr_dmmv_id_iq3_xxs_f32_f32_len, arr_dmmv_id_iq3_xxs_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_S], "mul_mat_vec_id_iq3_s_f32", arr_dmmv_id_iq3_s_f32_f32_len, arr_dmmv_id_iq3_s_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_XS], "mul_mat_vec_id_iq4_xs_f32", arr_dmmv_id_iq4_xs_f32_f32_len, arr_dmmv_id_iq4_xs_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_NL], "mul_mat_vec_id_iq4_nl_f32", arr_dmmv_id_iq4_nl_f32_f32_len, arr_dmmv_id_iq4_nl_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_MXFP4], "mul_mat_vec_id_mxfp4_f32", arr_dmmv_id_mxfp4_f32_f32_len, arr_dmmv_id_mxfp4_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}); + create_reduc16_id_pipeline(device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_NVFP4], "mul_mat_vec_id_nvfp4_f32", arr_dmmv_id_nvfp4_f32_f32_len, arr_dmmv_id_nvfp4_f32_f32_data, {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}); #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { @@ -6652,7 +6949,12 @@ static void ggml_vk_print_gpu_info(size_t idx) { bool bf16 = false; #endif - uint32_t default_subgroup_size = get_subgroup_size("", device_architecture); + uint32_t default_subgroup_size = 0; + GpuPipelineConfig gpu_config = {}; + auto config_found = get_gpu_pipeline_config(&gpu_config, device_architecture); + if (config_found) { + default_subgroup_size = gpu_config.default_subgroup_size; + } const size_t subgroup_size = (default_subgroup_size != 0) ? default_subgroup_size : subgroup_props.subgroupSize; const bool uma = props2.properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 15b50209c850..60b246228099 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9424,7 +9424,7 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_cumsum(GGML_TYPE_F32, { 2048, 16, 5, 4 })); test_cases.emplace_back(new test_cumsum(GGML_TYPE_F32, { 20000, 10, 4, 1 })); - for (int bs : {1, 2, 3, 4, 5, 8, 512}) { + for (int bs : {1, 2, 3, 4, 5, 8, 32, 64, 512}) { for (ggml_type type_a : all_types) { for (ggml_type type_b : {GGML_TYPE_F32}) { test_cases.emplace_back(new test_mul_mat(type_a, type_b, 4096, bs, 14336, {1, 1}, {1, 1}));