diff --git a/.github/workflows/linux_cuda_no_cudnn.yml b/.github/workflows/linux_cuda_no_cudnn.yml
index 3194c3d2581f9..0359d022e40cf 100644
--- a/.github/workflows/linux_cuda_no_cudnn.yml
+++ b/.github/workflows/linux_cuda_no_cudnn.yml
@@ -96,14 +96,15 @@ jobs:
done < perms.txt
fi
- - name: Verify CUDA provider has no direct cuDNN dependency
+ - name: Verify CUDA provider has no direct cuDNN or cuFFT dependency
run: |
docker run --rm --gpus all \
-v "${{ runner.temp }}/Release:/build/Release" \
"${{ steps.build_docker_image_step.outputs.full-image-name }}" \
bash -lc 'set -euo pipefail
ldd /build/Release/Release/libonnxruntime_providers_cuda.so | tee /tmp/ldd.txt
- ! grep -i cudnn /tmp/ldd.txt'
+ ! grep -i cudnn /tmp/ldd.txt
+ ! grep -i cufft /tmp/ldd.txt'
- name: Run no-cuDNN CUDA EP op smoke test
run: |
@@ -136,6 +137,24 @@ jobs:
exit 1
fi
+ # Likewise remove cuFFT so the smoke test proves the provider loads and runs
+ # non-FFT operators when cuFFT is physically unavailable (it is loaded lazily and
+ # only needed by the Rfft/Irfft contrib ops).
+ for root in /usr /opt /lib /lib64; do
+ if [ -e "$root" ]; then
+ find "$root" -name "libcufft*.so*" -print -delete 2>/dev/null || true
+ fi
+ done
+ ldconfig
+ if ldconfig -p | grep -qi cufft; then
+ echo "cuFFT remains available in the dynamic linker cache" >&2
+ exit 1
+ fi
+ if find /usr /opt /lib /lib64 -name "libcufft*.so*" -print -quit 2>/dev/null | grep -q .; then
+ echo "cuFFT runtime libraries remain in the no-cuDNN test container" >&2
+ exit 1
+ fi
+
WHEEL_PATH=$(find /build/Release/Release/dist -type f -name "onnxruntime_gpu-*.whl" | head -n 1)
if [ -z "$WHEEL_PATH" ]; then
echo "No built onnxruntime GPU wheel found under /build/Release/Release/dist" >&2
@@ -254,5 +273,32 @@ jobs:
np.argmin(data, axis=1).reshape(2, 1).astype(np.int64),
)
- print("CUDA no-cuDNN op smoke tests passed")
+ # Rfft requires cuFFT, which was physically removed above. With CPU fallback
+ # disabled the node must run on CUDA and fail cleanly with NOT_IMPLEMENTED
+ # (proving cuFFT is optional and its absence does not crash the provider).
+ rfft_model = helper.make_model(
+ helper.make_graph(
+ [helper.make_node("Rfft", ["x"], ["y"], domain="com.microsoft", signal_ndim=1, onesided=1)],
+ "no_cufft_rfft",
+ [helper.make_tensor_value_info("x", TensorProto.FLOAT, [2, 4])],
+ [helper.make_tensor_value_info("y", TensorProto.FLOAT, [2, 3, 2])],
+ ),
+ opset_imports=[helper.make_opsetid("", 17), helper.make_opsetid("com.microsoft", 1)],
+ )
+ rfft_model.ir_version = 10
+ rfft_options = ort.SessionOptions()
+ rfft_options.add_session_config_entry("session.disable_cpu_ep_fallback", "1")
+ try:
+ rfft_sess = ort.InferenceSession(
+ rfft_model.SerializeToString(), sess_options=rfft_options, providers=providers
+ )
+ rfft_sess.run(None, {"x": np.random.rand(2, 4).astype(np.float32)})
+ except Exception as exc: # noqa: BLE001
+ message = str(exc).lower()
+ assert "cufft" in message, "Rfft without cuFFT raised an unexpected error: " + str(exc)
+ print("[no-cuFFT] Rfft correctly failed: " + str(exc).splitlines()[0])
+ else:
+ raise AssertionError("Rfft unexpectedly succeeded while cuFFT was unavailable")
+
+ print("CUDA no-cuDNN/no-cuFFT op smoke tests passed")
PY'
diff --git a/.github/workflows/windows_cuda_no_cudnn.yml b/.github/workflows/windows_cuda_no_cudnn.yml
index b5d81b3c412ee..b6863fc891d86 100644
--- a/.github/workflows/windows_cuda_no_cudnn.yml
+++ b/.github/workflows/windows_cuda_no_cudnn.yml
@@ -203,10 +203,20 @@ jobs:
dir
shell: pwsh
+ - name: Remove cuFFT runtime DLLs
+ shell: pwsh
+ run: |
+ $cudaRoot = Join-Path $env:RUNNER_TEMP "v13.0"
+ Get-ChildItem -Path $cudaRoot -Recurse -Include "cufft*.dll" | Remove-Item -Force
+ if (Get-ChildItem -Path $cudaRoot -Recurse -Include "cufft*.dll" -ErrorAction SilentlyContinue) {
+ Write-Error "cuFFT runtime DLLs must not be present in the no-cuDNN test environment"
+ exit 1
+ }
+
- name: Add CUDA to PATH
shell: pwsh
run: |
- Write-Host "Adding CUDA to PATH without adding any cuDNN directory"
+ Write-Host "Adding CUDA to PATH without cuDNN or cuFFT runtime DLLs"
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v13.0\bin"
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v13.0\bin\x64"
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v13.0\extras\CUPTI\lib64"
@@ -226,7 +236,7 @@ jobs:
shell: pwsh
run: nvidia-smi
- - name: Verify CUDA plugin has no direct cuDNN dependency
+ - name: Verify CUDA plugin has no direct cuDNN or cuFFT dependency
shell: pwsh
run: |
$pluginPath = "${{ runner.temp }}\build\Release\Release\onnxruntime_providers_cuda.dll"
@@ -240,6 +250,10 @@ jobs:
Write-Error "CUDA plugin EP has a direct cuDNN dependency"
exit 1
}
+ if (Select-String -Path $env:RUNNER_TEMP\cuda_plugin_dependents.txt -Pattern "cufft" -SimpleMatch -Quiet) {
+ Write-Error "CUDA plugin EP has a direct cuFFT dependency"
+ exit 1
+ }
- name: Run CUDA Plugin EP Python Tests without cuDNN
working-directory: ${{ github.workspace }}\onnxruntime\test\python\transformers
diff --git a/cmake/onnxruntime_providers_cuda.cmake b/cmake/onnxruntime_providers_cuda.cmake
index e7b403300e268..c5fc0fd53ece1 100644
--- a/cmake/onnxruntime_providers_cuda.cmake
+++ b/cmake/onnxruntime_providers_cuda.cmake
@@ -402,7 +402,9 @@
endif()
target_compile_definitions(${target} PRIVATE NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING)
target_include_directories(${target} PRIVATE ${CUDNN_INCLUDE_DIR})
- target_link_libraries(${target} PRIVATE CUDA::cublasLt CUDA::cublas cudnn_frontend CUDA::curand CUDA::cufft CUDA::cudart CUDA::cuda_driver
+ # cuDNN and cuFFT are loaded dynamically at runtime (see cudnn_stub.cc / cufft_stub.cc), so they are
+ # intentionally not linked here to avoid a hard runtime dependency when the related ops are not used.
+ target_link_libraries(${target} PRIVATE CUDA::cublasLt CUDA::cublas cudnn_frontend CUDA::curand CUDA::cudart CUDA::cuda_driver
${ABSEIL_LIBS} ${ONNXRUNTIME_PROVIDERS_SHARED} Boost::mp11 safeint_interface)
endif()
diff --git a/cmake/onnxruntime_providers_cuda_plugin.cmake b/cmake/onnxruntime_providers_cuda_plugin.cmake
index a1a2930999ada..bd126f92807b4 100644
--- a/cmake/onnxruntime_providers_cuda_plugin.cmake
+++ b/cmake/onnxruntime_providers_cuda_plugin.cmake
@@ -393,11 +393,13 @@ onnxruntime_add_include_to_target(
add_dependencies(onnxruntime_providers_cuda_plugin ${onnxruntime_EXTERNAL_DEPENDENCIES})
# Link libraries
+# cuDNN and cuFFT are loaded dynamically at runtime (see cudnn_stub.cc / cufft_stub.cc, which are
+# picked up by the GLOB above), so they are intentionally not linked here to avoid a hard runtime
+# dependency when the related ops are not used.
target_link_libraries(onnxruntime_providers_cuda_plugin PRIVATE
CUDA::cudart
CUDA::cublas
CUDA::cublasLt
- CUDA::cufft
CUDA::cuda_driver
cudnn_frontend
Boost::mp11
diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake
index 53950fc8feecd..a6bf825e572eb 100644
--- a/cmake/onnxruntime_unittests.cmake
+++ b/cmake/onnxruntime_unittests.cmake
@@ -1051,6 +1051,7 @@ if (onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS AND onnxruntime_BUILD_CUDA_EP_AS_P
"${ONNXRUNTIME_ROOT}/core/providers/cuda/cuda_utils.cu"
"${ONNXRUNTIME_ROOT}/core/providers/cuda/cudnn_common.cc"
"${ONNXRUNTIME_ROOT}/core/providers/cuda/cudnn_loader.cc"
+ "${ONNXRUNTIME_ROOT}/core/providers/cuda/cufft_loader.cc"
"${ONNXRUNTIME_ROOT}/core/providers/cuda/reduction/reduction_functions.cc"
"${ONNXRUNTIME_ROOT}/core/providers/cuda/reduction/reduction_functions.cu"
"${TEST_SRC_DIR}/providers/cuda/test_cases/cuda_plugin_test_shims.cc"
@@ -1408,7 +1409,6 @@ block()
CUDA::cublas
CUDA::cublasLt
CUDA::curand
- CUDA::cufft
CUDA::cuda_driver
CUDNN::cudnn
cudnn_frontend)
diff --git a/csharp/ApiDocs/ApiDocs.csproj b/csharp/ApiDocs/ApiDocs.csproj
index eba7895a6b83d..85440714d5417 100644
--- a/csharp/ApiDocs/ApiDocs.csproj
+++ b/csharp/ApiDocs/ApiDocs.csproj
@@ -6,12 +6,15 @@
true
-
-
- all
- runtime; build; native; contentfiles; analyzers; buildtransitive
-
-
+
diff --git a/docs/CUDA_cuDNN_Optional_Design.md b/docs/CUDA_cuDNN_Optional_Design.md
index 8d24b8db30186..1d761d31b72b0 100644
--- a/docs/CUDA_cuDNN_Optional_Design.md
+++ b/docs/CUDA_cuDNN_Optional_Design.md
@@ -358,6 +358,40 @@ TensorRT / NV‑RTX EPs are untouched and continue to link cuDNN as before. (If
and the shimmed CUDA EP are in the same process, symbol collision must be avoided — see
§8 Risks.)
+### 3.6 Applying the same pattern to cuFFT
+
+The identical shim + lazy‑loader technique is used to drop the hard dependency on **cuFFT**
+(`libcufft.so.*` / `cufft64_*.dll`). cuFFT is only needed by the FFT contrib operators
+(`Rfft` / `Irfft` in `contrib_ops/cuda/math`), which use a small, enumerable set of entry
+points: `cufftCreate`, `cufftDestroy`, `cufftSetStream`, `cufftXtMakePlanMany`, and
+`cufftXtExec`.
+
+Implementation:
+
+- **Loader** — `onnxruntime/core/providers/cuda/cufft_loader.{h,cc}` defines a
+ `CufftLibrary` singleton that mirrors `CudnnLibrary`: it lazily `dlopen`/`LoadLibrary`s the
+ cuFFT runtime, resolves symbols on demand, caches them, and exposes `Available()` / `Error()`.
+ It uses the same security‑conscious search behavior as the cuDNN loader (Windows
+ `LOAD_LIBRARY_SEARCH_DEFAULT_DIRS` plus a PATH fallback that never loads from the current
+ working directory). The candidate library name is selected at **compile time** from the
+ `CUDA_VERSION` macro (cuFFT 12 for CUDA 13.x, cuFFT 11 for CUDA 12.x), because cuFFT's
+ SONAME/DLL version tracks the CUDA major version. Unsupported CUDA major versions fail at
+ compile time until their cuFFT library mapping is added explicitly.
+- **Shim** — `onnxruntime/core/providers/cuda/cufft_stub.cc` defines the five entry points
+ above as trampolines that forward to the resolved symbol (returning `CUFFT_INTERNAL_ERROR`
+ when the library is unavailable), so ORT's direct calls link against *our* definitions and
+ the final binary has **no import entry for libcufft**.
+- **Availability guard** — `CudaCall` (in `cuda_call.cc`) and the plugin's
+ `CUFFT_RETURN_IF_ERROR` macro check `CufftLibrary::Available()` and return a clear
+ `NOT_IMPLEMENTED` status ("cuFFT is unavailable …") when cuFFT is missing, exactly like the
+ cuDNN path.
+
+Unlike cuDNN, cuFFT needs **no provider option** (there is no `enable_cufft`): it is always
+loaded lazily on first FFT‑op use, and there is no `cufft_frontend` equivalent to configure.
+cuFFT headers (part of the CUDA toolkit) are still required at build time, but `CUDA::cufft`
+is no longer linked in `onnxruntime_providers_cuda.cmake`,
+`onnxruntime_providers_cuda_plugin.cmake`, or the CUDA unit‑test target.
+
---
## 4. Phase 1 — Make cuDNN optional (no new kernels)
diff --git a/docs/contrib_ops/cuda/matmul_nbits.md b/docs/contrib_ops/cuda/matmul_nbits.md
index 73b3371be8841..4f620126f504d 100644
--- a/docs/contrib_ops/cuda/matmul_nbits.md
+++ b/docs/contrib_ops/cuda/matmul_nbits.md
@@ -307,6 +307,19 @@ present. `ComputeInternal` then:
## 9. Testing
+- CUDA EP internal tests run through `CUDA_EP_Unittest` in
+ [onnxruntime/test/providers/cuda/cuda_provider_test.cc](../../../onnxruntime/test/providers/cuda/cuda_provider_test.cc).
+ Run them from `onnxruntime_provider_test` with:
+
+ ```bash
+ ./onnxruntime_provider_test --gtest_filter=CUDA_EP_Unittest.*
+ ```
+
+ This wrapper executes the internal CUDA-UT shared library and covers the
+ fpA_intB / MatMulNBits groupwise GEMM tests under
+ [onnxruntime/test/contrib_ops/cuda_kernels/fpA_intB_gemm_kernel_test.cc](../../../onnxruntime/test/contrib_ops/cuda_kernels/fpA_intB_gemm_kernel_test.cc)
+ as well as the SM90 validation tests in
+ [onnxruntime/test/contrib_ops/cuda_kernels/matmul_nbits_sm90_validation_test.cc](../../../onnxruntime/test/contrib_ops/cuda_kernels/matmul_nbits_sm90_validation_test.cc).
- Python operator tests: `onnxruntime/test/python/transformers` (see the QMoE /
GEMV profiling helpers, e.g. `profile_qmoe_gemv.sh`).
- CUDA prepacked-weight parity tests:
diff --git a/docs/cuda_plugin_ep/cuda_plugin_ep_design.md b/docs/cuda_plugin_ep/cuda_plugin_ep_design.md
index 7dfa2bd2d8667..a4c67342194e2 100644
--- a/docs/cuda_plugin_ep/cuda_plugin_ep_design.md
+++ b/docs/cuda_plugin_ep/cuda_plugin_ep_design.md
@@ -43,6 +43,12 @@ There is intentionally no provider option for a custom cuDNN DLL/SO path. Provid
No-cuDNN plugin validation runs `test_cuda_plugin_ep.py` with `ORT_TEST_CUDA_PLUGIN_EP=1` and `ORT_TEST_CUDA_PLUGIN_NO_CUDNN=1`. That mode passes `enable_cudnn=0` to plugin sessions and skips tests for operators that still require cuDNN in the current implementation. Non-cuDNN operator coverage, plugin registration, device enumeration, graph assignment, CUDA graph, I/O binding, and profiling tests continue to run.
+### 2.1.2 Optional cuFFT Runtime Dependency
+
+The CUDA Plugin EP applies the same optional-dependency model to **cuFFT**. cuFFT is only used by the FFT contrib operators (`Rfft` / `Irfft`), so the plugin shared library must not link directly to cuFFT or contain a cuFFT DLL/SO in its dynamic dependency table. cuFFT is loaded lazily through the ORT cuFFT loader (`CufftLibrary`, `core/providers/cuda/cufft_loader.{h,cc}`) on first FFT-op use; the entry points ORT calls are provided by `cufft_stub.cc` trampolines that forward to the resolved symbols.
+
+Unlike cuDNN, there is **no provider option** for cuFFT (no `enable_cufft`) and no frontend library to configure. When cuFFT is unavailable, `CUFFT_RETURN_IF_ERROR` (and `CudaCall`) return `NOT_IMPLEMENTED` with an actionable message, so FFT ops fail cleanly while all other operators keep working. cuFFT headers (part of the CUDA toolkit) are still required at build time, but `CUDA::cufft` is no longer linked. The loader picks the cuFFT library name at compile time from the `CUDA_VERSION` macro (cuFFT 12 for CUDA 13.x, cuFFT 11 for CUDA 12.x); unsupported CUDA major versions fail at compile time until their mapping is added.
+
### 2.2 Preprocessor Defines
Each build target uses different preprocessor defines that control how framework types are resolved:
@@ -651,7 +657,7 @@ Use `onnxruntime_BUILD_CUDA_EP_AS_PLUGIN=OFF` when you need to build the origina
### 9.3 Plugin Independence
-The plugin build's `libonnxruntime_providers_cuda.so` is **fully self-contained**. It does not depend on `libonnxruntime_providers_shared.so` at load time. It statically links against `onnxruntime_framework`, `onnxruntime_graph`, `onnxruntime_common`, `onnxruntime_mlas`, `onnxruntime_flatbuffers`, and links dynamically against CUDA (`cudart`, `cublas`, `cublasLt`, `cufft`) and protobuf. cuDNN is loaded lazily only when enabled and available at runtime. Communication with the ORT runtime happens exclusively through the C API (`OrtApi`/`OrtEpApi`) passed at load time.
+The plugin build's `libonnxruntime_providers_cuda.so` is **fully self-contained**. It does not depend on `libonnxruntime_providers_shared.so` at load time. It statically links against `onnxruntime_framework`, `onnxruntime_graph`, `onnxruntime_common`, `onnxruntime_mlas`, `onnxruntime_flatbuffers`, and links dynamically against CUDA (`cudart`, `cublas`, `cublasLt`) and protobuf. cuDNN and cuFFT are loaded lazily only when available at runtime (cuDNN additionally gated by `enable_cudnn`), so neither appears in the library's dynamic dependency table. Communication with the ORT runtime happens exclusively through the C API (`OrtApi`/`OrtEpApi`) passed at load time.
### 9.4 Build Outputs
diff --git a/onnxruntime/contrib_ops/cuda/quantization/matmul_nbits.cc b/onnxruntime/contrib_ops/cuda/quantization/matmul_nbits.cc
index 2630d76b6e4dc..2fcdef9ffbdd7 100644
--- a/onnxruntime/contrib_ops/cuda/quantization/matmul_nbits.cc
+++ b/onnxruntime/contrib_ops/cuda/quantization/matmul_nbits.cc
@@ -18,6 +18,7 @@
#include "contrib_ops/cuda/llm/fpA_intB_gemm_adaptor.h"
#include "contrib_ops/cuda/llm/fpA_intB_gemm_preprocessors.h"
#include "contrib_ops/cuda/llm/common/cuda_runtime_utils.h"
+#include "contrib_ops/cuda/quantization/matmul_nbits_sm90_validation.h"
#endif
#include "contrib_ops/cuda/llm/common/logger.h"
#include "contrib_ops/cpu/quantization/matmul_nbits_helper.h"
@@ -56,6 +57,38 @@ int64_t MatMulNBits::RequiredWeightPrepackedFormat() const {
return FpAIntBPackingSmForKernel() == 90 ? kMatMulNBitsWeightPrepackedSm90 : kMatMulNBitsWeightPrepackedSm80;
}
+// See matmul_nbits_sm90_validation.h for why these two functions are declared there but defined
+// here (non-inline, single definition, inside the CUDA EP translation unit that observes
+// COMPILE_HOPPER_TMA_GEMMS).
+bool IsNativeSm90FpAIntBGemmCompiled() {
+#if defined(COMPILE_HOPPER_TMA_GEMMS)
+ return true;
+#else
+ return false;
+#endif
+}
+
+void ValidateSm90PrepackedWeightSupport(int sm, int64_t block_size) {
+ // The native SM90 (Hopper TMA/WGMMA) mixed-GEMM kernel requires a compute-capability 9.0
+ // device and a block_size that is a multiple of the Hopper K tile (128 / sizeof(half) = 64).
+ // block_size=32 is only supported by the SM80/Ampere-class kernel + GEMV path.
+ ORT_ENFORCE(sm == 90,
+ "weight_prepacked=2 (SM90 layout) requires a compute capability 9.0 (Hopper) device, but got sm ", sm);
+#if !defined(COMPILE_HOPPER_TMA_GEMMS)
+ // The native SM90 (Hopper) fpA_intB TMA/WGMMA kernel is not compiled in this build (for
+ // example Windows/MSVC, where CUDA 13 NVCC host stubs hit MSVC C2719 with over-aligned TMA
+ // parameters; see docs/contrib_ops/cuda/moe_qmoe.md section 14.1). The SM90 weight layout
+ // cannot be consumed by the SM80 kernel, so fail early here with a clear message instead of
+ // dispatching to the throwing launcher stub during tactic profiling.
+ ORT_THROW(
+ "weight_prepacked=2 (SM90 layout) is not supported by this ONNX Runtime build "
+ "(the native SM90 Hopper fpA_intB kernel is unavailable, e.g. on Windows/MSVC). "
+ "Re-export the model with weight_prepacked=0 or 1 to use the SM80-compatible fpA_intB layout.");
+#endif
+ ORT_ENFORCE(block_size == 64 || block_size == 128,
+ "weight_prepacked=2 (SM90 layout) supports block_size 64 or 128 only, but got ", block_size);
+}
+
template
void MatMulNBits::InitGemmProfiler(int sm) {
gemmProfiler_ = s_profilerManager.createGemmPluginProfiler(/*inference*/ false);
diff --git a/onnxruntime/contrib_ops/cuda/quantization/matmul_nbits.h b/onnxruntime/contrib_ops/cuda/quantization/matmul_nbits.h
index e15af07b82ae8..cc638a987a935 100644
--- a/onnxruntime/contrib_ops/cuda/quantization/matmul_nbits.h
+++ b/onnxruntime/contrib_ops/cuda/quantization/matmul_nbits.h
@@ -14,6 +14,7 @@
#include "core/providers/cuda/cuda_kernel.h"
#include "core/providers/cuda/shared_inc/fpgeneric.h"
#include "contrib_ops/cuda/llm/fpA_intB_gemm_profiler.h"
+#include "contrib_ops/cuda/quantization/matmul_nbits_sm90_validation.h"
#include "core/platform/env_var_utils.h"
namespace onnxruntime {
@@ -129,24 +130,9 @@ class MatMulNBits final : public CudaKernel {
"weight_prepacked must be 0 (not prepacked), 1 (SM80 layout), or 2 (SM90 layout), but got ",
weight_prepacked_);
if (weight_prepacked_ == kMatMulNBitsWeightPrepackedSm90) {
-#if !defined(COMPILE_HOPPER_TMA_GEMMS)
- // The native SM90 (Hopper) fpA_intB TMA/WGMMA kernel is not compiled in this build (for
- // example Windows/MSVC, where CUDA 13 NVCC host stubs hit MSVC C2719 with over-aligned TMA
- // parameters; see docs/contrib_ops/cuda/moe_qmoe.md section 14.1). The SM90 weight layout
- // cannot be consumed by the SM80 kernel, so fail early here with a clear message instead of
- // dispatching to the throwing launcher stub during tactic profiling.
- ORT_THROW(
- "weight_prepacked=2 (native SM90 layout) is not supported by this ONNX Runtime build "
- "(the native SM90 Hopper fpA_intB kernel is unavailable, e.g. on Windows/MSVC). "
- "Re-export the model with weight_prepacked=0 or 1 to use the SM80-compatible fpA_intB layout.");
-#endif
- // The native SM90 (Hopper TMA/WGMMA) mixed-GEMM kernel requires a compute-capability 9.0
- // device and a block_size that is a multiple of the Hopper K tile (128 / sizeof(half) = 64).
- // block_size=32 is only supported by the SM80/Ampere-class kernel + GEMV path.
- ORT_ENFORCE(sm_ == 90,
- "weight_prepacked=2 (SM90 layout) requires a compute capability 9.0 (Hopper) device, but got sm ", sm_);
- ORT_ENFORCE(block_size_ == 64 || block_size_ == 128,
- "weight_prepacked=2 (SM90 layout) supports block_size 64 or 128 only, but got ", block_size_);
+ // See matmul_nbits_sm90_validation.h / matmul_nbits.cc for the validation logic (extracted
+ // into a pure function of (sm, block_size) so it can be unit-tested without a Hopper GPU).
+ ValidateSm90PrepackedWeightSupport(sm_, block_size_);
}
if constexpr (std::is_same::value || std::is_same::value) {
diff --git a/onnxruntime/contrib_ops/cuda/quantization/matmul_nbits_sm90_validation.h b/onnxruntime/contrib_ops/cuda/quantization/matmul_nbits_sm90_validation.h
new file mode 100644
index 0000000000000..60fa055474095
--- /dev/null
+++ b/onnxruntime/contrib_ops/cuda/quantization/matmul_nbits_sm90_validation.h
@@ -0,0 +1,49 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+//
+// Pure, host-only (no GPU required) helpers for validating a MatMulNBits node that requests
+// weight_prepacked=2 (the native SM90/Hopper mixed-GEMM weight layout). These are declared here,
+// separate from matmul_nbits.h, so that unit tests can exercise the validation logic with synthetic
+// (sm, block_size) values -- e.g. a Hopper `sm` on a machine/build that has no GPU at all -- which is
+// not otherwise possible since MatMulNBits normally reads `sm` from the real device properties.
+//
+// Both functions are DEFINED (non-inline) in matmul_nbits.cc, NOT in this header. matmul_nbits.cc is
+// compiled as part of the CUDA execution provider target (onnxruntime_providers_cuda /
+// onnxruntime_providers_cuda_plugin), which is the only place where the COMPILE_HOPPER_TMA_GEMMS
+// macro -- recording whether this build actually compiles the native SM90 (Hopper TMA/WGMMA)
+// fpA_intB kernel; it is left undefined on Windows/MSVC, see cmake/onnxruntime_providers_cuda.cmake
+// -- is defined consistently. If IsNativeSm90FpAIntBGemmCompiled() were instead defined inline in
+// this header, each translation unit that includes it (including one that does not receive that
+// target-scoped compile definition) could observe a different answer for whether the native SM90
+// kernel is available, silently producing an ODR violation / macro-skew bug. Keeping a single
+// non-inline definition inside matmul_nbits.cc avoids that hazard.
+//
+#pragma once
+
+#include
+
+#if USE_FPA_INTB_GEMM
+
+namespace onnxruntime {
+namespace contrib {
+namespace cuda {
+
+// Returns true iff this build compiled the native SM90 (Hopper TMA/WGMMA) fpA_intB mixed-GEMM
+// kernel, i.e. iff COMPILE_HOPPER_TMA_GEMMS was defined when matmul_nbits.cc was compiled.
+// Pure/host-only: does not touch the GPU and can be called without a CUDA device present.
+bool IsNativeSm90FpAIntBGemmCompiled();
+
+// Validates that a MatMulNBits node requesting weight_prepacked=2 (the native SM90 weight layout)
+// can actually be served, given the device compute capability `sm` (e.g. as computed from
+// GetDeviceProp().major*10+minor) and the node's `block_size` attribute. Throws (ORT_ENFORCE /
+// ORT_THROW) with a diagnostic message if the request cannot be served; returns normally otherwise.
+// Pure/host-only: `sm` and `block_size` are plain parameters (not read from real hardware), so this
+// can be unit-tested with synthetic values on any machine, without a GPU.
+void ValidateSm90PrepackedWeightSupport(int sm, int64_t block_size);
+
+} // namespace cuda
+} // namespace contrib
+} // namespace onnxruntime
+
+#endif // USE_FPA_INTB_GEMM
diff --git a/onnxruntime/core/providers/cuda/cuda_call.cc b/onnxruntime/core/providers/cuda/cuda_call.cc
index d27dbf266d183..23143141f32c1 100644
--- a/onnxruntime/core/providers/cuda/cuda_call.cc
+++ b/onnxruntime/core/providers/cuda/cuda_call.cc
@@ -11,6 +11,7 @@
#endif
#ifndef USE_CUDA_MINIMAL
#include "core/providers/cuda/cudnn_loader.h"
+#include "core/providers/cuda/cufft_loader.h"
#endif
#include
@@ -107,6 +108,18 @@ std::conditional_t CudaCall(
}
}
}
+ if constexpr (std::is_same_v) {
+ if (!cuda::CufftLibrary::Get().Available()) {
+ auto status = ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED,
+ "cuFFT is unavailable for CUDA Execution Provider: ",
+ cuda::CufftLibrary::Get().Error());
+ if constexpr (THRW) {
+ ORT_THROW(status.ErrorMessage());
+ } else {
+ return status;
+ }
+ }
+ }
#endif
try {
#ifdef _WIN32
diff --git a/onnxruntime/core/providers/cuda/cufft_loader.cc b/onnxruntime/core/providers/cuda/cufft_loader.cc
new file mode 100644
index 0000000000000..6c9ed625e1e94
--- /dev/null
+++ b/onnxruntime/core/providers/cuda/cufft_loader.cc
@@ -0,0 +1,209 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/providers/cuda/cufft_loader.h"
+
+#ifndef USE_CUDA_MINIMAL
+
+#include
+
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include
+#else
+#include
+#endif
+
+namespace {
+
+// cuFFT's SONAME/DLL version tracks the CUDA major version it ships with:
+// CUDA 12.x -> cuFFT 11, CUDA 13.x -> cuFFT 12. Select the name that matches the
+// CUDA toolkit this library was built against (CUDA_VERSION comes from cuda_pch.h).
+#if CUDA_VERSION >= 13000 && CUDA_VERSION < 14000
+#ifdef _WIN32
+constexpr const char* kCufftLibraryName = "cufft64_12.dll";
+#else
+constexpr const char* kCufftLibraryName = "libcufft.so.12";
+#endif
+#elif CUDA_VERSION >= 12000 && CUDA_VERSION < 13000
+#ifdef _WIN32
+constexpr const char* kCufftLibraryName = "cufft64_11.dll";
+#else
+constexpr const char* kCufftLibraryName = "libcufft.so.11";
+#endif
+#else
+#error Unsupported CUDA_VERSION for dynamic cuFFT loading
+#endif
+
+#ifdef _WIN32
+// Search the directories listed in the PATH environment variable for the given
+// library and, if found, load it by its full path. Loading by full path (rather
+// than letting the loader search) preserves the historical PATH-based cuFFT
+// discovery without ever loading from the current working directory, which the
+// LOAD_LIBRARY_SEARCH_DEFAULT_DIRS-only search excludes for security reasons.
+HMODULE LoadLibraryFromPathEnv(const std::string& candidate) {
+ DWORD length = GetEnvironmentVariableA("PATH", nullptr, 0);
+ if (length == 0) {
+ return nullptr;
+ }
+
+ std::string path_value(length, '\0');
+ length = GetEnvironmentVariableA("PATH", path_value.data(), length);
+ if (length == 0) {
+ return nullptr;
+ }
+ path_value.resize(length);
+
+ size_t start = 0;
+ while (start <= path_value.size()) {
+ size_t end = path_value.find(';', start);
+ if (end == std::string::npos) {
+ end = path_value.size();
+ }
+
+ std::string dir = path_value.substr(start, end - start);
+ start = end + 1;
+ if (dir.empty()) {
+ continue;
+ }
+
+ if (dir.back() != '\\' && dir.back() != '/') {
+ dir.push_back('\\');
+ }
+ std::string full_path = dir + candidate;
+
+ if (GetFileAttributesA(full_path.c_str()) == INVALID_FILE_ATTRIBUTES) {
+ continue;
+ }
+
+ HMODULE handle = LoadLibraryExA(full_path.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+ if (handle != nullptr) {
+ return handle;
+ }
+ }
+
+ return nullptr;
+}
+#endif
+
+void* LoadLibraryCandidate(const std::string& candidate, std::string& error) {
+#ifdef _WIN32
+ // Use LOAD_LIBRARY_SEARCH_DEFAULT_DIRS so cuFFT is resolved only from the
+ // application directory, %WINDIR%\System32, and directories added via
+ // AddDllDirectory/SetDefaultDllDirectories. This deliberately excludes the
+ // current working directory from the search order to avoid loading an
+ // attacker-controlled DLL from the process CWD.
+ HMODULE handle = LoadLibraryExA(candidate.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+ if (handle == nullptr) {
+ // Fall back to searching the directories listed in PATH (loading by full
+ // path), matching the pre-existing OS-loader behavior when cuFFT was a
+ // direct import dependency. The current working directory is never searched.
+ handle = LoadLibraryFromPathEnv(candidate);
+ }
+ if (handle == nullptr) {
+ error = "LoadLibrary failed for " + candidate + " with error " + std::to_string(GetLastError());
+ }
+ return reinterpret_cast(handle);
+#else
+ dlerror();
+ void* handle = dlopen(candidate.c_str(), RTLD_NOW | RTLD_LOCAL);
+ if (handle == nullptr) {
+ const char* dl_error = dlerror();
+ error = "dlopen failed for " + candidate + ": " + (dl_error != nullptr ? dl_error : "unknown error");
+ }
+ return handle;
+#endif
+}
+
+void* GetLibrarySymbol(void* handle, const char* symbol, std::string& error) {
+#ifdef _WIN32
+ void* address = reinterpret_cast(GetProcAddress(reinterpret_cast(handle), symbol));
+ if (address == nullptr) {
+ error = "GetProcAddress failed for " + std::string(symbol) + " with error " + std::to_string(GetLastError());
+ }
+ return address;
+#else
+ dlerror();
+ void* address = dlsym(handle, symbol);
+ const char* dl_error = dlerror();
+ if (address == nullptr || dl_error != nullptr) {
+ error = "dlsym failed for " + std::string(symbol) + ": " + (dl_error != nullptr ? dl_error : "unknown error");
+ }
+ return address;
+#endif
+}
+
+} // namespace
+
+namespace onnxruntime::cuda {
+
+CufftLibrary& CufftLibrary::Get() {
+ static CufftLibrary library;
+ return library;
+}
+
+bool CufftLibrary::Available() {
+ return EnsureLoaded();
+}
+
+std::string CufftLibrary::Error() const {
+ std::lock_guard lock(mutex_);
+ return error_.empty() ? CufftUnavailableErrorString() : error_;
+}
+
+bool CufftLibrary::EnsureLoaded() {
+ std::lock_guard lock(mutex_);
+ if (load_attempted_) {
+ return available_;
+ }
+
+ load_attempted_ = true;
+ handle_ = LoadLibraryCandidate(kCufftLibraryName, error_);
+ if (handle_ != nullptr) {
+ available_ = true;
+ error_.clear();
+ return true;
+ }
+
+ available_ = false;
+ if (error_.empty()) {
+ error_ = "cuFFT library was not found";
+ }
+ return false;
+}
+
+void* CufftLibrary::ResolveSymbol(const char* symbol) {
+ {
+ std::lock_guard lock(mutex_);
+ auto it = symbols_.find(symbol);
+ if (it != symbols_.end()) {
+ return it->second;
+ }
+ }
+
+ if (!EnsureLoaded()) {
+ return nullptr;
+ }
+
+ std::lock_guard lock(mutex_);
+ std::string symbol_error;
+ void* address = GetLibrarySymbol(handle_, symbol, symbol_error);
+ if (address == nullptr) {
+ available_ = false;
+ error_ = symbol_error;
+ return nullptr;
+ }
+
+ symbols_.emplace(symbol, address);
+ return address;
+}
+
+const char* CufftUnavailableErrorString() {
+ return "cuFFT is not available. Install cuFFT or update the system library search path to enable FFT (Rfft/Irfft) operators.";
+}
+
+} // namespace onnxruntime::cuda
+
+#endif // USE_CUDA_MINIMAL
diff --git a/onnxruntime/core/providers/cuda/cufft_loader.h b/onnxruntime/core/providers/cuda/cufft_loader.h
new file mode 100644
index 0000000000000..f7be4b4357470
--- /dev/null
+++ b/onnxruntime/core/providers/cuda/cufft_loader.h
@@ -0,0 +1,51 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#ifndef USE_CUDA_MINIMAL
+
+#include
+#include
+#include
+
+#include "core/providers/cuda/cuda_pch.h"
+
+namespace onnxruntime::cuda {
+
+// Lazily loads the cuFFT runtime library and resolves its symbols on demand so
+// that the CUDA Execution Provider does not carry a hard link-time dependency on
+// cuFFT. cuFFT is only required by the FFT contrib ops (Rfft/Irfft); when the
+// library is not present those ops fail with NOT_IMPLEMENTED while the rest of
+// the provider keeps working.
+class CufftLibrary {
+ public:
+ static CufftLibrary& Get();
+
+ bool Available();
+ std::string Error() const;
+
+ template
+ T Resolve(const char* symbol) {
+ return reinterpret_cast(ResolveSymbol(symbol));
+ }
+
+ private:
+ CufftLibrary() = default;
+
+ bool EnsureLoaded();
+ void* ResolveSymbol(const char* symbol);
+
+ mutable std::mutex mutex_;
+ bool load_attempted_{false};
+ bool available_{false};
+ std::string error_;
+ void* handle_{nullptr};
+ std::unordered_map symbols_;
+};
+
+const char* CufftUnavailableErrorString();
+
+} // namespace onnxruntime::cuda
+
+#endif // USE_CUDA_MINIMAL
diff --git a/onnxruntime/core/providers/cuda/cufft_stub.cc b/onnxruntime/core/providers/cuda/cufft_stub.cc
new file mode 100644
index 0000000000000..a1cc195375556
--- /dev/null
+++ b/onnxruntime/core/providers/cuda/cufft_stub.cc
@@ -0,0 +1,54 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// Provides definitions for the small set of cuFFT entry points used by the CUDA
+// Execution Provider. Each stub forwards to the matching symbol resolved from the
+// dynamically loaded cuFFT runtime library. Linking against these stubs (instead
+// of the cuFFT import library) removes the hard runtime dependency on cuFFT so
+// that models without FFT (Rfft/Irfft) ops can run without cuFFT installed.
+
+#include "core/providers/cuda/cufft_loader.h"
+
+#ifndef USE_CUDA_MINIMAL
+
+#include "cufft.h"
+#include "cufftXt.h"
+
+#define ORT_CUFFT_FORWARD_RESULT(name, ...) \
+ using Fn = decltype(&name); \
+ auto fn = onnxruntime::cuda::CufftLibrary::Get().Resolve(#name); \
+ return fn != nullptr ? fn(__VA_ARGS__) : CUFFT_INTERNAL_ERROR
+
+extern "C" {
+
+cufftResult CUFFTAPI cufftCreate(cufftHandle* handle) {
+ ORT_CUFFT_FORWARD_RESULT(cufftCreate, handle);
+}
+
+cufftResult CUFFTAPI cufftDestroy(cufftHandle plan) {
+ ORT_CUFFT_FORWARD_RESULT(cufftDestroy, plan);
+}
+
+cufftResult CUFFTAPI cufftSetStream(cufftHandle plan, cudaStream_t stream) {
+ ORT_CUFFT_FORWARD_RESULT(cufftSetStream, plan, stream);
+}
+
+cufftResult CUFFTAPI cufftXtMakePlanMany(cufftHandle plan, int rank, long long int* n,
+ long long int* inembed, long long int istride, long long int idist,
+ cudaDataType inputtype,
+ long long int* onembed, long long int ostride, long long int odist,
+ cudaDataType outputtype,
+ long long int batch, size_t* workSize, cudaDataType executiontype) {
+ ORT_CUFFT_FORWARD_RESULT(cufftXtMakePlanMany, plan, rank, n, inembed, istride, idist, inputtype, onembed, ostride,
+ odist, outputtype, batch, workSize, executiontype);
+}
+
+cufftResult CUFFTAPI cufftXtExec(cufftHandle plan, void* input, void* output, int direction) {
+ ORT_CUFFT_FORWARD_RESULT(cufftXtExec, plan, input, output, direction);
+}
+
+} // extern "C"
+
+#undef ORT_CUFFT_FORWARD_RESULT
+
+#endif // USE_CUDA_MINIMAL
diff --git a/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h b/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h
index 7c6cdd0e4dc80..ed698aaf74eb1 100644
--- a/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h
+++ b/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h
@@ -34,6 +34,7 @@
#include
#include "core/providers/cuda/shared_inc/cuda_call.h"
#include "core/providers/cuda/cudnn_loader.h"
+#include "core/providers/cuda/cufft_loader.h"
#include "contrib_ops/cuda/bert/attention_kernel_options.h"
#ifdef __CUDACC__
@@ -279,6 +280,11 @@ using ::onnxruntime::HandleNegativeAxis;
{ \
cufftResult _status = (expr); \
if (_status != CUFFT_SUCCESS) { \
+ if (!onnxruntime::cuda::CufftLibrary::Get().Available()) { \
+ return onnxruntime::common::Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::NOT_IMPLEMENTED, \
+ std::string("cuFFT is unavailable for CUDA Plugin Execution Provider: ") + \
+ onnxruntime::cuda::CufftLibrary::Get().Error()); \
+ } \
return onnxruntime::common::Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::FAIL, std::string("cuFFT error: ") + std::to_string((int)_status)); \
} \
}
diff --git a/onnxruntime/test/contrib_ops/cuda_kernels/fpA_intB_gemm_kernel_test.cc b/onnxruntime/test/contrib_ops/cuda_kernels/fpA_intB_gemm_kernel_test.cc
index 533277be48ec3..99c2ef8ce76c5 100644
--- a/onnxruntime/test/contrib_ops/cuda_kernels/fpA_intB_gemm_kernel_test.cc
+++ b/onnxruntime/test/contrib_ops/cuda_kernels/fpA_intB_gemm_kernel_test.cc
@@ -400,6 +400,9 @@ class KernelTestFixture : public ::testing::Test {
using WType = typename cutlassTypeMapper::WType;
using onnxruntime::llm::kernels::cutlass_kernels::CutlassFpAIntBGemmRunner;
auto runner = std::make_shared::QuantOp>>();
+ if (onnxruntime::llm::common::getSMVersion() == 90) {
+ runner->setUseSm90Native(true);
+ }
auto& gemm_runner = *runner;
int ws_bytes = gemm_runner.getWorkspaceSize(m_, n_, k_);
CudaBuffer ws_buffer(ws_bytes);
@@ -565,8 +568,6 @@ TEST_F(Fp16Int8GroupwiseTest, Fp16_Int8_Gemm_CudaKernel) {
for (const auto& [n, k] : get_n_k_list()) {
InitBuffers(m, n, k, 64);
EXPECT_TRUE(BenchmarkAndVerifyKernel());
- InitBuffers(m, n, k, 128);
- EXPECT_TRUE(BenchmarkAndVerifyKernel());
}
}
}
@@ -582,8 +583,6 @@ TEST_F(Fp16Int4GroupwiseTest, Fp16_Int4_Gemm_CudaKernel) {
for (const auto& [n, k] : get_n_k_list()) {
InitBuffers(m, n, k, 64);
EXPECT_TRUE(BenchmarkAndVerifyKernel());
- InitBuffers(m, n, k, 128);
- EXPECT_TRUE(BenchmarkAndVerifyKernel());
}
}
}
@@ -599,8 +598,6 @@ TEST_F(Bf16Int8GroupwiseTest, BF16_Int8_Gemm_CudaKernel) {
for (const auto& [n, k] : get_n_k_list()) {
InitBuffers(m, n, k, 64);
EXPECT_TRUE(BenchmarkAndVerifyKernel());
- InitBuffers(m, n, k, 128);
- EXPECT_TRUE(BenchmarkAndVerifyKernel());
}
}
}
@@ -616,8 +613,6 @@ TEST_F(Bf16Int4GroupwiseTest, BF16_Int4_Gemm_CudaKernel) {
for (const auto& [n, k] : get_n_k_list()) {
InitBuffers(m, n, k, 64);
EXPECT_TRUE(BenchmarkAndVerifyKernel());
- InitBuffers(m, n, k, 128);
- EXPECT_TRUE(BenchmarkAndVerifyKernel());
}
}
}
diff --git a/onnxruntime/test/contrib_ops/cuda_kernels/matmul_nbits_sm90_validation_test.cc b/onnxruntime/test/contrib_ops/cuda_kernels/matmul_nbits_sm90_validation_test.cc
new file mode 100644
index 0000000000000..9617e33117d48
--- /dev/null
+++ b/onnxruntime/test/contrib_ops/cuda_kernels/matmul_nbits_sm90_validation_test.cc
@@ -0,0 +1,93 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// GPU-free coverage for the SM90 (Hopper) weight_prepacked=2 validation logic used by
+// contrib_ops/cuda/quantization/matmul_nbits.h's MatMulNBits constructor.
+//
+// MatMulNBits normally reads the device compute capability from real hardware
+// (GetDeviceProp()), so the "native SM90 kernel is not compiled in this build" throw (e.g. on
+// Windows/MSVC, see the COMPILE_HOPPER_TMA_GEMMS macro in cmake/onnxruntime_providers_cuda.cmake) is
+// only reachable on an actual Hopper GPU that was built without the native kernel -- a combination
+// existing tests cannot produce, since they all GTEST_SKIP() when no CUDA device is present (see
+// e.g. MatMulNBits.Fp16_Int4_PrepackedSm90BlockSize32Rejected in test/contrib_ops/matmul_4bits_test.cc).
+// ValidateSm90PrepackedWeightSupport() takes sm/block_size as plain parameters instead of reading
+// real hardware, so it can be exercised here with synthetic values and no GPU at all.
+//
+// This lives alongside the other CUDA-EP-internal tests under contrib_ops/cuda_kernels/ (rather than
+// in test/contrib_ops/matmul_4bits_test.cc) because onnxruntime_providers_cuda is a runtime-loaded
+// shared library (see core/providers/cuda/symbols.def, which exports only the small provider-bridge
+// interface): matmul_nbits.cc's free functions are not exported from that DLL/so, so a normal
+// provider-test binary cannot call them directly (it never links onnxruntime_providers_cuda at
+// compile time -- see AddTest()/DEPENDS in cmake/onnxruntime_unittests.cmake). Files under
+// contrib_ops/cuda_kernels/ are instead compiled directly into the onnxruntime_providers_cuda_ut
+// module together with the CUDA EP's own object files whenever
+// onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS is set (as it is in the windows_cuda.yml / linux_cuda_ci.yml
+// CI legs), so this test both links successfully and observes COMPILE_HOPPER_TMA_GEMMS exactly as the
+// real onnxruntime_providers_cuda target does -- no ODR/macro-skew risk.
+//
+// Test can be run like the following:
+// ./onnxruntime_provider_test --gtest_filter=CUDA_EP_Unittest.*
+#if USE_FPA_INTB_GEMM
+#include
+#include
+
+#include
+
+#include "core/common/common.h"
+#include "contrib_ops/cuda/quantization/matmul_nbits_sm90_validation.h"
+
+namespace onnxruntime {
+namespace test {
+
+namespace {
+// Runs `fn` and returns the message of any thrown exception, or "" if it did not throw.
+template
+std::string CaughtMessage(Fn&& fn) {
+ std::string message;
+ ORT_TRY {
+ fn();
+ }
+ ORT_CATCH(const std::exception& ex) {
+ ORT_HANDLE_EXCEPTION([&]() { message = ex.what(); });
+ }
+ return message;
+}
+} // namespace
+
+// Non-Hopper compute capability is always rejected, regardless of whether this build compiled the
+// native SM90 kernel. This assertion is build-independent.
+TEST(MatMulNBitsSm90ValidationTest, RejectsNonHopperComputeCapability) {
+ const std::string message = CaughtMessage([]() {
+ onnxruntime::contrib::cuda::ValidateSm90PrepackedWeightSupport(/*sm=*/80, /*block_size=*/64);
+ });
+ EXPECT_THAT(message, ::testing::HasSubstr("weight_prepacked=2 (SM90 layout) requires a compute capability 9.0"));
+}
+
+// A Hopper (sm=90) request is validated differently depending on whether this build actually
+// compiled the native SM90 (Hopper TMA/WGMMA) fpA_intB kernel.
+TEST(MatMulNBitsSm90ValidationTest, Sm90SupportMatchesBuildCapability) {
+ using onnxruntime::contrib::cuda::IsNativeSm90FpAIntBGemmCompiled;
+ using onnxruntime::contrib::cuda::ValidateSm90PrepackedWeightSupport;
+
+ if (IsNativeSm90FpAIntBGemmCompiled()) {
+ // The native SM90 kernel is available: a supported block_size (64 or 128) must be accepted ...
+ EXPECT_NO_THROW(ValidateSm90PrepackedWeightSupport(/*sm=*/90, /*block_size=*/64));
+
+ // ... but block_size=32 is SM80/GEMV-only and must still be rejected.
+ const std::string message = CaughtMessage([]() {
+ ValidateSm90PrepackedWeightSupport(/*sm=*/90, /*block_size=*/32);
+ });
+ EXPECT_THAT(message, ::testing::HasSubstr("supports block_size 64 or 128 only"));
+ } else {
+ // The native SM90 kernel is not compiled in this build (e.g. Windows/MSVC): even a Hopper
+ // device with an otherwise-supported block_size must be rejected with a build-support message.
+ const std::string message = CaughtMessage([]() {
+ ValidateSm90PrepackedWeightSupport(/*sm=*/90, /*block_size=*/64);
+ });
+ EXPECT_THAT(message, ::testing::HasSubstr("is not supported by this ONNX Runtime build"));
+ }
+}
+
+} // namespace test
+} // namespace onnxruntime
+#endif // USE_FPA_INTB_GEMM
diff --git a/onnxruntime/test/contrib_ops/matmul_4bits_test.cc b/onnxruntime/test/contrib_ops/matmul_4bits_test.cc
index 2927477a66244..e932cf8fc2d38 100644
--- a/onnxruntime/test/contrib_ops/matmul_4bits_test.cc
+++ b/onnxruntime/test/contrib_ops/matmul_4bits_test.cc
@@ -1011,10 +1011,11 @@ TEST(MatMulNBits, Fp16_Int4_PrepackedWeightRejectedWhenFpAIntBUnsupported) {
// weight_prepacked=2 selects the native SM90 (Hopper) mixed-GEMM layout. It is rejected up front
// unless the device is SM90 and block_size is 64 or 128 (the SM90 TMA kernel requires group_size to
// be a multiple of the 64-element Hopper K tile, so block_size=32 is SM80-only). When the fpA_intB
-// path is compiled in, both rejection messages begin with "weight_prepacked=2 (SM90 layout)", so the
-// check is device-independent: non-Hopper hits the compute-capability guard, Hopper hits the
-// block_size guard. In a build without onnxruntime_USE_FPA_INTB_GEMM the kernel rejects any
-// weight_prepacked!=0 up front with a different ("weight_prepacked requires ...") message.
+// path is compiled in, all rejection messages begin with "weight_prepacked=2 (SM90 layout)", so the
+// assertion is stable across machine/build combinations: non-Hopper hits the compute-capability
+// guard, SM90 without native TMA support hits the build-support guard, and SM90 with native TMA
+// support hits the block_size guard. In a build without onnxruntime_USE_FPA_INTB_GEMM the kernel
+// rejects any weight_prepacked!=0 up front with a different ("weight_prepacked requires ...") message.
TEST(MatMulNBits, Fp16_Int4_PrepackedSm90BlockSize32Rejected) {
ScopedEnvironmentVariables scoped_env_vars{EnvVarMap{{"ORT_FPA_INTB_GEMM", "1"}}};
@@ -1038,6 +1039,12 @@ TEST(MatMulNBits, Fp16_Int4_PrepackedSm90BlockSize32Rejected) {
RunTest(opts, std::move(eps));
}
+// This GTEST_SKIPs without a CUDA device, so it cannot cover the "build without the native SM90
+// kernel" throw on a real Hopper GPU (the only hardware/build combination that reaches it). That
+// throw is instead covered GPU-free, with synthetic (sm, block_size) values, by
+// MatMulNBitsSm90ValidationTest in test/contrib_ops/cuda_kernels/matmul_nbits_sm90_validation_test.cc
+// (see the comment there for why that coverage lives in a separate translation unit).
+
// Exercises the CUDA small-M batched GEMV tiles: CtaM in {2,4,8,16} (with M values that are not a
// multiple of CtaM so the row-skip path runs) and CtaN in {1,2} (N divisible / not divisible by 16).
TEST(MatMulNBits, Fp16_Int4_SmallMBatchedTiles) {
diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml
index bd5290e8f792c..a4bef2dd16831 100644
--- a/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml
@@ -95,6 +95,7 @@ stages:
DockerBuildArgs: "
--build-arg TRT_VERSION=${{ variables.linux_trt_version }}
--build-arg BUILD_UID=$( id -u )
+ --secret id=PIP_INDEX_URL
"
Repository: onnxruntimecuda${{ variables.CUDA_VERSION_MAJOR }}xtrt86build
- template: ../templates/set-version-number-variables-step.yml
diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml
index 10e5aa34727ed..b238fe0a68222 100644
--- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml
@@ -451,6 +451,29 @@ stages:
FolderPath: '$(Build.ArtifactStagingDirectory)'
DoEsrp: true
+ # Validate that the C# API documentation still generates, using the maintained
+ # 'docfx' .NET global tool. This replaces the deprecated 'docfx.console' NuGet
+ # package (last release 2.59.4), which used to run as an MSBuild target during
+ # the 'Build C# bindings' step and broke against current MSBuild/VS. Mirrors
+ # .github/workflows/publish-csharp-apidocs.yml. Runs last because it re-restores
+ # Microsoft.ML.OnnxRuntime.csproj with IncludeMobileTargets=false, which must not
+ # disturb the multi-targeted restore used by the NuGet packaging steps above.
+ - task: PowerShell@2
+ displayName: 'Generate C# API docs (docfx global tool)'
+ inputs:
+ targetType: 'inline'
+ script: |
+ dotnet tool update -g docfx
+ $env:PATH = "$env:USERPROFILE\.dotnet\tools;$env:PATH"
+ # docfx cannot correctly restore the multi-targeted (mobile) project itself,
+ # so restore it explicitly with the mobile targets disabled first.
+ dotnet restore csharp\ApiDocs\ApiDocs.csproj
+ dotnet restore -p:IncludeMobileTargets=false csharp\src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj
+ docfx metadata csharp\ApiDocs\docfx.json
+ dotnet build csharp\ApiDocs\ApiDocs.csproj --no-restore
+ docfx build csharp\ApiDocs\docfx.json
+ workingDirectory: '$(Build.SourcesDirectory)'
+
- template: ../stages/nodejs-npm-packaging-stage.yml
parameters:
diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml
index a19a92e40aa72..97fac845977b1 100644
--- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml
@@ -57,7 +57,7 @@ jobs:
parameters:
Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu
- DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
+ DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --secret id=PIP_INDEX_URL"
Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging
- ${{ if eq(parameters.OnnxruntimeArch, 'aarch64') }}:
@@ -65,7 +65,7 @@ jobs:
parameters:
Dockerfile: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile
Context: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu
- DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
+ DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --secret id=PIP_INDEX_URL"
Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging
- task: CmdLine@2
diff --git a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml
index e41f51bb8ec08..da4892786c32b 100644
--- a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml
@@ -65,6 +65,8 @@ steps:
--container-registry onnxruntimebuildcache \
--repository "${{ parameters.Repository }}"
displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}"
+ env:
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
ContainerRegistry: onnxruntimebuildcache
- ${{ if eq(parameters.UseImageCacheContainerRegistry, false) }}:
# the difference is no --container-registry
@@ -78,6 +80,8 @@ steps:
--docker-build-args "${{ parameters.DockerBuildArgs }}" \
--repository "${{ parameters.Repository }}"
displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}"
+ env:
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
ContainerRegistry: onnxruntimebuildcache
- script: |
@@ -86,4 +90,3 @@ steps:
docker system df
df -h
displayName: Check Docker Images
-
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml
index 1226be6ca9083..e74dea27c38fb 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml
@@ -59,6 +59,8 @@ jobs:
artifactName: drop-linux-cpu-${{ parameters.arch }}-${{ parameters.ep }}
variables:
+ - name: SYSTEM_ACCESSTOKEN
+ value: $(System.AccessToken)
- name: extra_build_args
${{ if ne(parameters.extra_build_arg, '') }}:
value: '-x "${{ parameters.extra_build_arg }}"'
@@ -84,7 +86,7 @@ jobs:
parameters:
Dockerfile: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu/Dockerfile
Context: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu
- DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
+ DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --secret id=PIP_INDEX_URL --secret id=CARGO_CONFIG,src=$HOME/.cargo/config.toml --secret id=SYSTEM_ACCESSTOKEN"
Repository: onnxruntimecpubuildpython${{ parameters.arch }}
- task: Bash@3
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml
index a3d2eba50be99..ca83d4da8f8fa 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml
@@ -37,6 +37,7 @@ jobs:
timeoutInMinutes: ${{ parameters.timeout }}
variables:
skipComponentGovernanceDetection: true
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
workspace:
clean: all
pool: ${{ parameters.machine_pool }}
@@ -64,7 +65,7 @@ jobs:
parameters:
Dockerfile: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu/Dockerfile
Context: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu
- DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
+ DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --secret id=PIP_INDEX_URL --secret id=CARGO_CONFIG,src=$HOME/.cargo/config.toml --secret id=SYSTEM_ACCESSTOKEN"
Repository: onnxruntimecpubuildpython${{ parameters.arch }}
- task: Bash@3
diff --git a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/Dockerfile
index d39e39a5a429d..491bb814cdd00 100644
--- a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/Dockerfile
+++ b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/Dockerfile
@@ -2,7 +2,13 @@ ARG BASEIMAGE=onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/buil
FROM $BASEIMAGE
ADD scripts /tmp/scripts
-RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
+RUN --mount=type=secret,id=PIP_INDEX_URL,required=false \
+ --mount=type=secret,id=CARGO_CONFIG,required=false \
+ --mount=type=secret,id=SYSTEM_ACCESSTOKEN,required=false \
+ if [ -f /run/secrets/PIP_INDEX_URL ]; then export PIP_INDEX_URL=$(cat /run/secrets/PIP_INDEX_URL); fi && \
+ if [ -f /run/secrets/CARGO_CONFIG ]; then mkdir -p /.cargo && cp /run/secrets/CARGO_CONFIG /.cargo/config.toml; fi && \
+ if [ -f /run/secrets/SYSTEM_ACCESSTOKEN ]; then export CARGO_REGISTRIES_ONNXRUNTIME_TOKEN="Basic $(printf ':%s' "$(cat /run/secrets/SYSTEM_ACCESSTOKEN)" | base64 -w0)"; fi && \
+ cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
ARG BUILD_UID=1001
ARG BUILD_USER=onnxruntimedev
diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
index d2d5ae684e10e..62c4aa11f5ac7 100644
--- a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
+++ b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
@@ -8,7 +8,10 @@ FROM $BASEIMAGE
ENV LANG=en_US.UTF-8
ENV LC_ALL=en_US.UTF-8
-RUN python3 -m pip install flatbuffers
+ADD scripts /tmp/scripts
+RUN --mount=type=secret,id=PIP_INDEX_URL,required=false \
+ if [ -f /run/secrets/PIP_INDEX_URL ]; then export PIP_INDEX_URL=$(cat /run/secrets/PIP_INDEX_URL); fi && \
+ python3 -m pip install flatbuffers
ARG BUILD_UID=1001
ARG BUILD_USER=onnxruntimedev