diff --git a/.github/workflows/windows_webgpu.yml b/.github/workflows/windows_webgpu.yml index 872d3b182c310..e67eda41d2e0e 100644 --- a/.github/workflows/windows_webgpu.yml +++ b/.github/workflows/windows_webgpu.yml @@ -155,6 +155,86 @@ jobs: working-directory: ${{ github.workspace }}\csharp continue-on-error: true + webgpu_plugin_build_x64_RelWithDebInfo: + runs-on: [ + "self-hosted", + "1ES.Pool=onnxruntime-github-Win2022-GPU-A10", + "JobId=webgpu_plugin_build_x64_RelWithDebInfo-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + ] + timeout-minutes: 300 + env: + OnnxRuntimeBuildDirectory: ${{ github.workspace }} + setVcvars: true + ALLOW_RELEASED_ONNX_OPSET_ONLY: "0" + DocUpdateNeeded: false + NVIDIA_TF32_OVERRIDE: "0" + ONNXRUNTIME_TEST_GPU_DEVICE_ID: "0" + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: none + + - name: Setup Python 3.12 + uses: actions/setup-python@v6 + with: + python-version: "3.12" + architecture: x64 + + - name: Locate vcvarsall and Setup Env + uses: ./.github/actions/locate-vcvarsall-and-setup-env + with: + architecture: x64 + + - name: Install python modules + run: python -m pip install -r tools\ci_build\github\windows\python\requirements.txt + shell: cmd + working-directory: ${{ github.workspace }} + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "20.x" + + - uses: actions/cache@v5 + id: onnx-node-tests-cache + with: + path: ${{ github.workspace }}/js/test/ + key: onnxnodetests-${{ hashFiles('js/scripts/prepare-onnx-node-tests.ts') }} + + - name: Build and Test + shell: pwsh + run: | + python.exe ${{ github.workspace }}\tools\ci_build\build.py ` + --config RelWithDebInfo ` + --build_dir ${{ github.workspace }} ` + --skip_submodule_sync ` + --parallel ` + --use_binskim_compliant_compile_flags ` + --cmake_generator "Visual Studio 17 2022" ` + --enable_onnx_tests ` + --use_webgpu shared_lib ` + --wgsl_template static ` + --use_vcpkg --use_vcpkg_ms_internal_asset_cache ` + --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=ON onnxruntime_ENABLE_DAWN_BACKEND_D3D12=1 onnxruntime_ENABLE_DAWN_BACKEND_VULKAN=1 ` + --disable_rtti ` + --enable_lto + + if ($lastExitCode -ne 0) { + exit $lastExitCode + } + + - name: Publish artifacts + uses: actions/upload-artifact@v4 + with: + name: webgpu-plugin-binaries + path: | + ${{ github.workspace }}/RelWithDebInfo/RelWithDebInfo/onnxruntime_providers_webgpu.dll + ${{ github.workspace }}/RelWithDebInfo/RelWithDebInfo/onnxruntime_providers_webgpu.pdb + ${{ github.workspace }}/RelWithDebInfo/RelWithDebInfo/dxcompiler.dll + ${{ github.workspace }}/RelWithDebInfo/RelWithDebInfo/dxil.dll + webgpu_external_dawn_build_x64_RelWithDebInfo: runs-on: [ "self-hosted", diff --git a/cmake/onnxruntime_providers_webgpu.cmake b/cmake/onnxruntime_providers_webgpu.cmake index be7f2613a6272..cd29e4dad0a17 100644 --- a/cmake/onnxruntime_providers_webgpu.cmake +++ b/cmake/onnxruntime_providers_webgpu.cmake @@ -56,7 +56,7 @@ endif() source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_providers_webgpu_cc_srcs}) - onnxruntime_add_shared_library(onnxruntime_providers_webgpu ${onnxruntime_providers_webgpu_cc_srcs}) + onnxruntime_add_shared_library_module(onnxruntime_providers_webgpu ${onnxruntime_providers_webgpu_cc_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers_webgpu ${REPO_ROOT}/include/onnxruntime/core/session onnxruntime_common @@ -119,6 +119,12 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") message(FATAL_ERROR "WebGPU EP shared library build is not supported on Emscripten. Please use static library build.") endif() + + # Configure precompiled headers for shared library build + # PCH ensures ep/adapters.h is included first and improves compilation speed + target_precompile_headers(onnxruntime_providers_webgpu PRIVATE + "${REPO_ROOT}/include/onnxruntime/ep/adapters.h" + ) endif() set_target_properties(onnxruntime_providers_webgpu PROPERTIES CXX_STANDARD_REQUIRED ON) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 9ae3e79d86443..8137f8b3a2529 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1042,6 +1042,18 @@ function(onnxruntime_apply_test_target_workarounds target) endif() endfunction() +# Set environment variables for plugin EP tests when run via CTest. +function(onnxruntime_set_plugin_ep_test_environment target) + if(onnxruntime_USE_WEBGPU AND onnxruntime_USE_EP_API_ADAPTERS) + set(ORT_PLUGIN_EP_JSON_CONFIG "{\"ep_library_registration_name\": \"WebGPU_PluginEP\", \"ep_library_path\": \"$\", \"selected_ep_name\": \"WebGpuExecutionProvider\"}") + set_tests_properties(${target} PROPERTIES + ENVIRONMENT "ORT_UNIT_TEST_MAIN_DYNAMIC_PLUGIN_EP_CONFIG_JSON=${ORT_PLUGIN_EP_JSON_CONFIG}" + ) + # TODO: add for other plugin EPs if needed + # elseif() + endif() +endfunction() + function(onnxruntime_apply_emscripten_test_link_settings target) if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set_target_properties(${target} PROPERTIES LINK_DEPENDS ${TEST_SRC_DIR}/wasm/onnxruntime_test_adapter.js) @@ -1250,6 +1262,7 @@ block() ) onnxruntime_apply_test_target_workarounds(onnxruntime_provider_test) + onnxruntime_set_plugin_ep_test_environment(onnxruntime_provider_test) # Expose QNN SDK headers to unit tests via an interface target if(onnxruntime_USE_QNN) diff --git a/include/onnxruntime/ep/adapter/allocator.h b/include/onnxruntime/ep/adapter/allocator.h index 2765069ebf336..4f107ae72c0e9 100644 --- a/include/onnxruntime/ep/adapter/allocator.h +++ b/include/onnxruntime/ep/adapter/allocator.h @@ -18,23 +18,50 @@ namespace adapter { /// class Allocator : public OrtAllocator { public: + /** + * Create from an existing AllocatorPtr. + */ explicit Allocator(const OrtMemoryInfo* memory_info, AllocatorPtr impl) - : OrtAllocator{}, memory_info_(memory_info), impl_(impl) { + : Allocator{memory_info} { + ORT_ENFORCE(impl != nullptr, "Allocator implementation cannot be null."); + impl_ = impl; + } + + using AllocatorFactory = AllocatorPtr (*)(const OrtMemoryInfo& memory_info); + + /** + * Create from an AllocatorFactory, which will be called lazily when the first allocation is made. + */ + explicit Allocator(const OrtMemoryInfo* memory_info, AllocatorFactory get_allocator_impl) + : Allocator{memory_info} { + get_allocator_impl_ = get_allocator_impl; + } + + private: + explicit Allocator(const OrtMemoryInfo* memory_info) + : OrtAllocator{}, memory_info_(memory_info) { version = ORT_API_VERSION; Alloc = AllocImpl; Free = FreeImpl; Info = InfoImpl; } + AllocatorPtr GetImpl() { + if (!impl_) { + std::call_once(init_flag_, [this]() { + impl_ = get_allocator_impl_(*memory_info_); + }); + } + return impl_; + } - private: static void* ORT_API_CALL AllocImpl(OrtAllocator* this_ptr, size_t size) noexcept { auto* allocator = static_cast(this_ptr); - return allocator->impl_->Alloc(size); + return allocator->GetImpl()->Alloc(size); } static void ORT_API_CALL FreeImpl(OrtAllocator* this_ptr, void* p) noexcept { auto* allocator = static_cast(this_ptr); - allocator->impl_->Free(p); + allocator->GetImpl()->Free(p); } static const OrtMemoryInfo* ORT_API_CALL InfoImpl(const OrtAllocator* this_ptr) noexcept { @@ -44,6 +71,8 @@ class Allocator : public OrtAllocator { const OrtMemoryInfo* memory_info_; AllocatorPtr impl_; + AllocatorFactory get_allocator_impl_; + std::once_flag init_flag_; }; } // namespace adapter diff --git a/include/onnxruntime/ep/adapter/ep.h b/include/onnxruntime/ep/adapter/ep.h index 34fc7682a8138..ca0a8c9599eda 100644 --- a/include/onnxruntime/ep/adapter/ep.h +++ b/include/onnxruntime/ep/adapter/ep.h @@ -27,6 +27,7 @@ class Ep : public OrtEp { profiler_{impl_->GetProfiler()}, temp_space_cpu_allocator_{temp_space_cpu_allocator}, temp_space_allocator_{temp_space_allocator} { + ort_version_supported = ORT_API_VERSION; } public: diff --git a/include/onnxruntime/ep/adapter/op_kernel_info.h b/include/onnxruntime/ep/adapter/op_kernel_info.h index bd6172a668e33..644cb30788ec6 100644 --- a/include/onnxruntime/ep/adapter/op_kernel_info.h +++ b/include/onnxruntime/ep/adapter/op_kernel_info.h @@ -19,7 +19,7 @@ #include "tensor_helper.h" namespace onnxruntime { -struct DataTransferManager; +class DataTransferManager; struct IExecutionProvider; } // namespace onnxruntime diff --git a/include/onnxruntime/ep/common.h b/include/onnxruntime/ep/common.h index 03cd571461755..0e779ba3d4081 100644 --- a/include/onnxruntime/ep/common.h +++ b/include/onnxruntime/ep/common.h @@ -41,3 +41,24 @@ OrtStatus* _status = (status_expr); \ Ort::Status _ignored{_status}; \ } while (false) + +// Helper macros to convert exceptions to OrtStatus* return values. +// Usage: +// EXCEPTION_TO_RETURNED_STATUS_BEGIN +// ... code that may throw ... +// EXCEPTION_TO_RETURNED_STATUS_END +#define EXCEPTION_TO_RETURNED_STATUS_BEGIN try { +#define EXCEPTION_TO_RETURNED_STATUS_END \ + } \ + catch (const Ort::Exception& ex) { \ + Ort::Status status(ex); \ + return status.release(); \ + } \ + catch (const std::exception& ex) { \ + Ort::Status status(ex.what(), ORT_EP_FAIL); \ + return status.release(); \ + } \ + catch (...) { \ + Ort::Status status("Unknown exception", ORT_EP_FAIL); \ + return status.release(); \ + } diff --git a/onnxruntime/core/providers/cpu/tensor/upsamplebase.h b/onnxruntime/core/providers/cpu/tensor/upsamplebase.h index 7dcf88133e967..ff5498c0b4644 100644 --- a/onnxruntime/core/providers/cpu/tensor/upsamplebase.h +++ b/onnxruntime/core/providers/cpu/tensor/upsamplebase.h @@ -265,8 +265,17 @@ class UpsampleBase { if (scales_input_idx_ > 0) { const Tensor* scale; bool get_scale = info.TryGetConstantInput(scales_input_idx_, &scale); - auto x_shape = node.InputDefs()[0]->Shape(); - int64_t rank = x_shape ? x_shape->dim_size() : -1; + int64_t rank = -1; + if constexpr (std::is_same_v) { + auto x_shape = node.InputDefs()[0]->Shape(); + if (x_shape != nullptr) { + rank = x_shape->dim_size(); + } + } else { + auto type_info = info.GetKernelInfo().GetInputTypeInfo(0); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + rank = static_cast(tensor_info.GetDimensionsCount()); + } if (get_scale && scale->Shape().Size() > 0 && ((opset < 18) || (rank > 0 && opset >= 18))) { ORT_THROW_IF_ERROR(ParseScalesData(scale, scales_, rank)); scales_cached_ = true; diff --git a/onnxruntime/core/providers/webgpu/compute_context.h b/onnxruntime/core/providers/webgpu/compute_context.h index 5277d64ad3611..38848e98509ba 100644 --- a/onnxruntime/core/providers/webgpu/compute_context.h +++ b/onnxruntime/core/providers/webgpu/compute_context.h @@ -107,7 +107,11 @@ class ComputeContextBase { // Get the logger. // inline const logging::Logger& Logger() const { +#if defined(ORT_USE_EP_API_ADAPTERS) + return ep_.GetEpLogger(); +#else return *ep_.GetLogger(); +#endif } // diff --git a/onnxruntime/core/providers/webgpu/controlflow/if.cc b/onnxruntime/core/providers/webgpu/controlflow/if.cc index 233d1f760383f..29b5e66d5075a 100644 --- a/onnxruntime/core/providers/webgpu/controlflow/if.cc +++ b/onnxruntime/core/providers/webgpu/controlflow/if.cc @@ -3,6 +3,10 @@ #include "core/providers/webgpu/controlflow/if.h" +#if defined(ORT_USE_EP_API_ADAPTERS) +#include "core/framework/error_code_helper.h" +#endif + using namespace ONNX_NAMESPACE; using namespace onnxruntime::common; @@ -68,10 +72,20 @@ ONNX_OPERATOR_KERNEL_EX(If, .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), If); +#if !defined(ORT_USE_EP_API_ADAPTERS) Status If::Compute(OpKernelContext* ctx) const { // call the base CPU version. return onnxruntime::If::Compute(ctx); } +#else +Status If::CreateControlFlowKernelImpl(const OrtKernelInfo* info, OrtKernelImpl** impl) { + return ToStatusAndRelease(ep::Api().ep.CreateIfKernel(info, impl)); +} + +Status If::Compute(OpKernelContext* /*ctx*/) const { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "If operator should be handled by ORT core."); +} +#endif } // namespace webgpu -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/controlflow/if.h b/onnxruntime/core/providers/webgpu/controlflow/if.h index 0755c5d33d7a3..0aa30e939eb1a 100644 --- a/onnxruntime/core/providers/webgpu/controlflow/if.h +++ b/onnxruntime/core/providers/webgpu/controlflow/if.h @@ -10,6 +10,8 @@ namespace onnxruntime { namespace webgpu { +#if !defined(ORT_USE_EP_API_ADAPTERS) + // Use the CPU implementation for the logic class If final : public onnxruntime::If { public: @@ -18,5 +20,16 @@ class If final : public onnxruntime::If { Status Compute(OpKernelContext* ctx) const override; }; +#else + +class If final : public OpKernel { + public: + If(const OpKernelInfo& info) : OpKernel(info) {} + + Status CreateControlFlowKernelImpl(const OrtKernelInfo* info, OrtKernelImpl** impl) override; + Status Compute(OpKernelContext* ctx) const override; +}; +#endif + } // namespace webgpu -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/data_transfer.cc b/onnxruntime/core/providers/webgpu/data_transfer.cc index 6d66a7308f1de..5f109bf73e3c5 100644 --- a/onnxruntime/core/providers/webgpu/data_transfer.cc +++ b/onnxruntime/core/providers/webgpu/data_transfer.cc @@ -7,38 +7,48 @@ namespace onnxruntime { namespace webgpu { -bool DataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const { - return (dst_device.Type() == OrtDevice::GPU && src_device.Type() == OrtDevice::CPU) || - (dst_device.Type() == OrtDevice::GPU && src_device.Type() == OrtDevice::GPU) || - (dst_device.Type() == OrtDevice::CPU && src_device.Type() == OrtDevice::GPU); -} - -common::Status DataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const { - size_t bytes = src.SizeInBytes(); +common::Status DataTransferImpl::CopyTensor(void const* src_data, + bool src_is_gpu, + void* dst_data, + bool dst_is_gpu, + size_t bytes) const { if (bytes > 0) { - void const* src_data = src.DataRaw(); - void* dst_data = dst.MutableDataRaw(); - - auto& src_device = src.Location().device; - auto& dst_device = dst.Location().device; - - if (dst_device.Type() == OrtDevice::GPU) { - if (src_device.Type() == OrtDevice::GPU) { + if (dst_is_gpu) { + if (src_is_gpu) { // copy from GPU to GPU buffer_manager_.MemCpy(static_cast(const_cast(src_data)), - static_cast(dst_data), bytes); + static_cast(dst_data), + bytes); } else { // copy from CPU to GPU - buffer_manager_.Upload(const_cast(src_data), static_cast(dst_data), bytes); + buffer_manager_.Upload(const_cast(src_data), + static_cast(dst_data), + bytes); } - } else /* if (src_device.Type() == OrtDevice::GPU) */ { + } else { // copy from GPU to CPU - buffer_manager_.Download(static_cast(const_cast(src_data)), dst_data, bytes); + buffer_manager_.Download(static_cast(const_cast(src_data)), + dst_data, + bytes); } } return Status::OK(); } +bool DataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const { + return (dst_device.Type() == OrtDevice::GPU && src_device.Type() == OrtDevice::CPU) || + (dst_device.Type() == OrtDevice::GPU && src_device.Type() == OrtDevice::GPU) || + (dst_device.Type() == OrtDevice::CPU && src_device.Type() == OrtDevice::GPU); +} + +common::Status DataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const { + return impl_.CopyTensor(src.DataRaw(), + src.Location().device.Type() == OrtDevice::GPU, + dst.MutableDataRaw(), + dst.Location().device.Type() == OrtDevice::GPU, + src.SizeInBytes()); +} + } // namespace webgpu } // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/data_transfer.h b/onnxruntime/core/providers/webgpu/data_transfer.h index 0adf380149acf..e6ce92a7ca7a6 100644 --- a/onnxruntime/core/providers/webgpu/data_transfer.h +++ b/onnxruntime/core/providers/webgpu/data_transfer.h @@ -3,6 +3,7 @@ #pragma once +#include "core/common/status.h" #include "core/framework/data_transfer.h" #include "core/framework/execution_provider.h" @@ -11,9 +12,25 @@ namespace webgpu { class BufferManager; +// Low-level data transfer implementation that operates on raw pointers. +// Used by both DataTransfer (IDataTransfer subclass) and the C API data transfer wrapper. +class DataTransferImpl { + public: + DataTransferImpl(const BufferManager& buffer_manager) : buffer_manager_{buffer_manager} {}; + + common::Status CopyTensor(void const* src_data, + bool src_is_gpu, + void* dst_data, + bool dst_is_gpu, + size_t bytes) const; + + private: + const BufferManager& buffer_manager_; +}; + class DataTransfer : public IDataTransfer { public: - DataTransfer(const BufferManager& buffer_manager) : buffer_manager_{buffer_manager} {}; + DataTransfer(const BufferManager& buffer_manager) : impl_{buffer_manager} {}; ~DataTransfer() {}; bool CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const override; @@ -21,7 +38,7 @@ class DataTransfer : public IDataTransfer { common::Status CopyTensor(const Tensor& src, Tensor& dst) const override; private: - const BufferManager& buffer_manager_; + DataTransferImpl impl_; }; } // namespace webgpu diff --git a/onnxruntime/core/providers/webgpu/ep/api.cc b/onnxruntime/core/providers/webgpu/ep/api.cc new file mode 100644 index 0000000000000..9eeb3d71df89f --- /dev/null +++ b/onnxruntime/core/providers/webgpu/ep/api.cc @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#define ORT_API_MANUAL_INIT +#include "onnxruntime_cxx_api.h" +#undef ORT_API_MANUAL_INIT + +#include + +#include "core/providers/webgpu/ep/factory.h" + +// To make symbols visible on macOS/iOS +#ifdef __APPLE__ +#define EXPORT_SYMBOL __attribute__((visibility("default"))) +#else +#define EXPORT_SYMBOL +#endif + +namespace onnxruntime { +namespace webgpu { +void CleanupWebGpuContexts(); +void CleanupKernelRegistries(); +} // namespace webgpu +} // namespace onnxruntime + +namespace google { +namespace protobuf { +void ShutdownProtobufLibrary(); +} // namespace protobuf +} // namespace google + +extern "C" { +// +// Public symbols +// +EXPORT_SYMBOL OrtStatus* CreateEpFactories(const char* /*registration_name*/, const OrtApiBase* ort_api_base, + const OrtLogger* default_logger, + OrtEpFactory** factories, size_t max_factories, size_t* num_factories) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + + // Manual init for the C++ API + onnxruntime::ep::ApiInit(ort_api_base); + + if (max_factories < 1) { + return onnxruntime::ep::Api().ort.CreateStatus(ORT_INVALID_ARGUMENT, + "Not enough space to return EP factory. Need at least one."); + } + + // Initialize the global default logger + ::onnxruntime::ep::adapter::LoggingManager::CreateDefaultLogger(default_logger); + + // Factory could use registration_name or define its own EP name. + std::unique_ptr factory = std::make_unique(); + + factories[0] = factory.release(); + *num_factories = 1; + + return nullptr; + + EXCEPTION_TO_RETURNED_STATUS_END +} + +EXPORT_SYMBOL OrtStatus* ReleaseEpFactory(OrtEpFactory* factory) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + // STEP.1 - Release the factory + delete static_cast(factory); + + // STEP.2 - Clean up cached kernel registries + onnxruntime::webgpu::CleanupKernelRegistries(); + + // STEP.3 - Clean up WebGPU contexts + onnxruntime::webgpu::CleanupWebGpuContexts(); + + // STEP.4 - Destroy the global default logger wrapper + ::onnxruntime::ep::adapter::LoggingManager::DestroyDefaultLogger(); + + // STEP.5 - Shutdown protobuf library + google::protobuf::ShutdownProtobufLibrary(); + + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END +} + +} // extern "C" diff --git a/onnxruntime/core/providers/webgpu/ep/ep.cc b/onnxruntime/core/providers/webgpu/ep/ep.cc new file mode 100644 index 0000000000000..6beb62b5cf074 --- /dev/null +++ b/onnxruntime/core/providers/webgpu/ep/ep.cc @@ -0,0 +1,273 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "ep.h" + +#include "factory.h" + +#include "core/framework/run_options.h" +#include "core/framework/kernel_registry.h" +#include "core/session/onnxruntime_run_options_config_keys.h" +#include "core/session/plugin_ep/ep_kernel_registration.h" +#include "core/providers/webgpu/webgpu_execution_provider.h" + +#include "ep/get_capability_utils.h" + +namespace onnxruntime { +namespace webgpu { +namespace ep { + +using onnxruntime::ep::Api; + +// Constructor +Ep::Ep(std::unique_ptr impl, Factory& factory, const OrtLogger& logger, const Config& config) + : onnxruntime::ep::adapter::Ep{std::move(impl), config.cpu_allocator, config.device_allocator}, + factory_{factory}, + logger_{logger}, + config_{config} { + // Initialize the execution provider's function table + GetName = GetNameImpl; + GetCapability = GetCapabilityImpl; + GetKernelRegistry = GetKernelRegistryImpl; + Compile = nullptr; // Per-kernel EP does not use Compile + ReleaseNodeComputeInfos = nullptr; + GetPreferredDataLayout = GetPreferredDataLayoutImpl; + ShouldConvertDataLayoutForOp = ShouldConvertDataLayoutForOpImpl; + SetDynamicOptions = nullptr; // Not implemented + OnRunStart = OnRunStartImpl; + OnRunEnd = OnRunEndImpl; + CreateAllocator = CreateAllocatorImpl; + CreateSyncStreamForDevice = nullptr; // Not stream aware + GetCompiledModelCompatibilityInfo = nullptr; // Not a compiled EP + IsConcurrentRunSupported = IsConcurrentRunSupportedImpl; +} + +// OrtEp interface implementations +const char* ORT_API_CALL Ep::GetNameImpl(const OrtEp* this_ptr) noexcept { + const auto* ep = static_cast(this_ptr); + return ep->factory_.GetName(&ep->factory_); +} + +OrtStatus* ORT_API_CALL Ep::GetCapabilityImpl(OrtEp* this_ptr, const OrtGraph* graph, + OrtEpGraphSupportInfo* graph_support_info) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + + auto& ep = *static_cast(static_cast(this_ptr)->EpImpl()); + Ort::ConstGraph ort_graph{graph}; + + // Get all nodes in the graph + std::vector all_nodes = ort_graph.GetNodes(); + + if (all_nodes.empty()) { + return nullptr; // No nodes to process + } + + std::vector candidate_nodes; + std::vector tentative_candidate_nodes; + + // For each node, check if we have a registered kernel for it + for (const auto& node : all_nodes) { + std::string ep_name = node.GetEpName(); + + if (ep_name == kWebGpuExecutionProvider) { + candidate_nodes.push_back(node); + continue; + } + + // Reject nodes already assigned to a different (non-CPU) EP + if (!ep_name.empty() && ep_name != kCpuExecutionProvider) { + continue; + } + + const OrtKernelDef* kernel_def = nullptr; + RETURN_IF_ERROR(Api().ep.EpGraphSupportInfo_LookUpKernel(graph_support_info, node, &kernel_def)); + + if (kernel_def == nullptr) { + LOGS(ep.GetEpLogger(), INFO) << "webgpu kernel not found in registries for Op type: " + << node.GetOperatorType() << " node name: " << node.GetName(); + continue; + } + + auto cpu_node_names = ep.GetForceCpuNodeNames(); + if (std::find(cpu_node_names.begin(), + cpu_node_names.end(), + node.GetName()) != cpu_node_names.end()) { + LOGS(ep.GetEpLogger(), INFO) << "Force CPU execution for node: " << node.GetName(); + continue; + } + + // + // The following code checks if the node is really supported by webgpu EP + // + +#define FALLBACK_TO_CPU_IF_EXIST_INPUT(idx) \ + if (inputs.size() > idx && inputs[idx] != nullptr) { \ + continue; \ + } + +#define FALLBACK_TO_CPU_IF_EXIST_OUTPUT(idx) \ + if (outputs.size() > idx && outputs[idx] != nullptr) { \ + continue; \ + } + + // Check for Attention + if (node.GetOperatorType() == "Attention" && node.GetDomain() == kMSDomain) { + const auto& inputs = node.GetInputs(); + const auto& outputs = node.GetOutputs(); + + // Current implementation does not support mask_index(input[3]), past(input[4]) and past_seq_len(input[6]) + FALLBACK_TO_CPU_IF_EXIST_INPUT(3); + FALLBACK_TO_CPU_IF_EXIST_INPUT(4); + FALLBACK_TO_CPU_IF_EXIST_INPUT(6); + + // Current implementation does not support present(output[1]) + FALLBACK_TO_CPU_IF_EXIST_OUTPUT(1); + + // If attribute past_present_share_buffer is set, fallback to CPU + bool has_past_present_share_buffer = false; + for (const auto& attr : node.GetAttributes()) { + if (attr.GetName() == "past_present_share_buffer") { + int64_t val = 0; + RETURN_IF_ERROR(attr.GetValue(val)); + if (val != 0) { + has_past_present_share_buffer = true; + } + break; + } + } + if (has_past_present_share_buffer) { + continue; + } + } + + candidate_nodes.push_back(node); + tentative_candidate_nodes.push_back(node); + } + + std::unordered_set cpu_preferred_nodes; + RETURN_IF_ERROR(onnxruntime::ep::GetCpuPreferredNodes(*ort_graph, + *graph_support_info, + static_cast(this_ptr)->GetOrtLogger(), + tentative_candidate_nodes, + cpu_preferred_nodes)); + + for (const auto& node : candidate_nodes) { + if (cpu_preferred_nodes.count(node) == 0) { + RETURN_IF_ERROR(Api().ep.EpGraphSupportInfo_AddSingleNode(graph_support_info, node)); + } + } + + return nullptr; + + EXCEPTION_TO_RETURNED_STATUS_END +} + +OrtStatus* ORT_API_CALL Ep::GetKernelRegistryImpl( + _In_ OrtEp* this_ptr, + _Outptr_result_maybenull_ const OrtKernelRegistry** kernel_registry) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + + *kernel_registry = nullptr; + + // For the WebGPU EP, delegate to the CreateKernelRegistry function + // which properly constructs a registry using only public APIs + auto* ep = static_cast(this_ptr); + + auto& webgpu_ep = *static_cast(ep->EpImpl()); + + *kernel_registry = *webgpu_ep.GetKernelRegistryImpl(); + return nullptr; + + EXCEPTION_TO_RETURNED_STATUS_END +} + +OrtStatus* ORT_API_CALL Ep::GetPreferredDataLayoutImpl(_In_ OrtEp* this_ptr, + _Out_ OrtEpDataLayout* preferred_data_layout) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + // Delegate to the underlying WebGPU EP's GetPreferredLayout() + // DataLayout enum values map 1:1 to OrtEpDataLayout (NCHW=0, NHWC=1) + auto* ep = static_cast(this_ptr); + *preferred_data_layout = static_cast(ep->EpImpl()->GetPreferredLayout()); + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END +} + +OrtStatus* ORT_API_CALL Ep::ShouldConvertDataLayoutForOpImpl(_In_ OrtEp* this_ptr, + _In_z_ const char* domain, + _In_z_ const char* op_type, + _In_ OrtEpDataLayout target_data_layout, + _Outptr_ int* should_convert) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + // DataLayout enum values map 1:1 to OrtEpDataLayout (NCHW=0, NHWC=1) + auto* ep = static_cast(this_ptr); + auto result = ep->EpImpl()->ShouldConvertDataLayoutForOp(domain, op_type, + static_cast(target_data_layout)); + if (result.has_value()) { + *should_convert = result.value() ? 1 : 0; + } else { + *should_convert = -1; + } + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END +} + +OrtStatus* ORT_API_CALL Ep::OnRunStartImpl(_In_ OrtEp* this_ptr, + _In_ const OrtRunOptions* run_options) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + onnxruntime::RunOptions options{}; + // currently only option "gpu_graph_id" is used + auto graph_annotation_str = Api().ort.GetRunConfigEntry(run_options, kOrtRunOptionsConfigCudaGraphAnnotation); + if (graph_annotation_str != nullptr) { + auto status = options.config_options.AddConfigEntry(kOrtRunOptionsConfigCudaGraphAnnotation, graph_annotation_str); + if (!status.IsOK()) { + return Api().ort.CreateStatus(static_cast(status.Code()), + status.ErrorMessage().c_str()); + } + } + auto* ep = static_cast(this_ptr); + auto status = ep->EpImpl()->OnRunStart(options); + if (!status.IsOK()) { + return Api().ort.CreateStatus(static_cast(status.Code()), + status.ErrorMessage().c_str()); + } + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END +} + +OrtStatus* ORT_API_CALL Ep::OnRunEndImpl(_In_ OrtEp* this_ptr, + _In_ const OrtRunOptions* /*run_options*/, + _In_ bool sync_stream) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + auto* ep = static_cast(this_ptr); + auto status = ep->EpImpl()->OnRunEnd(sync_stream, {}); + if (!status.IsOK()) { + return Api().ort.CreateStatus(static_cast(status.Code()), + status.ErrorMessage().c_str()); + } + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END +} + +OrtStatus* ORT_API_CALL Ep::IsConcurrentRunSupportedImpl(_In_ OrtEp* /*this_ptr*/, _Out_ bool* is_concurrent_run_supported) noexcept { + *is_concurrent_run_supported = false; + return nullptr; +} + +OrtStatus* ORT_API_CALL Ep::CreateAllocatorImpl(_In_ OrtEp* this_ptr, + _In_ const OrtMemoryInfo* memory_info, + _Outptr_result_maybenull_ OrtAllocator** allocator) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + auto* ep = static_cast(this_ptr); + Ort::ConstMemoryInfo ort_memory_info{memory_info}; + if (ort_memory_info.GetAllocatorType() == OrtReadOnlyAllocator) { + *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, ep->config_.initializer_allocator); + } else { + *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, ep->config_.device_allocator); + } + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END +} + +} // namespace ep +} // namespace webgpu +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/ep/ep.h b/onnxruntime/core/providers/webgpu/ep/ep.h new file mode 100644 index 0000000000000..815623025f8a4 --- /dev/null +++ b/onnxruntime/core/providers/webgpu/ep/ep.h @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "core/providers/webgpu/webgpu_execution_provider.h" + +namespace onnxruntime { +namespace webgpu { +namespace ep { + +class Factory; + +/// +/// A bridge class between the EP API and the WebGPU EP implementation. +/// +class Ep : public onnxruntime::ep::adapter::Ep { + public: + struct Config { + AllocatorPtr cpu_allocator; + AllocatorPtr device_allocator; + AllocatorPtr initializer_allocator; + }; + + Ep(std::unique_ptr impl, Factory& factory, const OrtLogger& logger, const Config& config); + + inline const OrtLogger& GetOrtLogger() const noexcept { + return logger_; + } + + private: + static const char* ORT_API_CALL GetNameImpl(const OrtEp* this_ptr) noexcept; + + static OrtStatus* ORT_API_CALL GetCapabilityImpl(OrtEp* this_ptr, const OrtGraph* graph, + OrtEpGraphSupportInfo* graph_support_info) noexcept; + + static OrtStatus* ORT_API_CALL GetKernelRegistryImpl( + _In_ OrtEp* this_ptr, + _Outptr_result_maybenull_ const OrtKernelRegistry** kernel_registry) noexcept; + + static OrtStatus* ORT_API_CALL CreateAllocatorImpl(_In_ OrtEp* this_ptr, + _In_ const OrtMemoryInfo* memory_info, + _Outptr_result_maybenull_ OrtAllocator** allocator) noexcept; + + static OrtStatus* ORT_API_CALL GetPreferredDataLayoutImpl(_In_ OrtEp* this_ptr, + _Out_ OrtEpDataLayout* preferred_data_layout) noexcept; + + static OrtStatus* ORT_API_CALL ShouldConvertDataLayoutForOpImpl(_In_ OrtEp* this_ptr, + _In_z_ const char* domain, + _In_z_ const char* op_type, + _In_ OrtEpDataLayout target_data_layout, + _Outptr_ int* should_convert) noexcept; + + static OrtStatus* ORT_API_CALL OnRunStartImpl(_In_ OrtEp* this_ptr, + _In_ const OrtRunOptions* run_options) noexcept; + + static OrtStatus* ORT_API_CALL OnRunEndImpl(_In_ OrtEp* this_ptr, + _In_ const OrtRunOptions* run_options, + _In_ bool sync_stream) noexcept; + + static OrtStatus* ORT_API_CALL IsConcurrentRunSupportedImpl(_In_ OrtEp* this_ptr, + _Out_ bool* is_concurrent_run_supported) noexcept; + + Factory& factory_; + const OrtLogger& logger_; + Config config_{}; +}; + +} // namespace ep +} // namespace webgpu +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/ep/factory.cc b/onnxruntime/core/providers/webgpu/ep/factory.cc new file mode 100644 index 0000000000000..99dd0c68f6954 --- /dev/null +++ b/onnxruntime/core/providers/webgpu/ep/factory.cc @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "factory.h" +#include "ep.h" + +#include "core/framework/error_code_helper.h" +#include "core/graph/constants.h" + +#include "core/framework/execution_provider.h" +#include "core/framework/config_options.h" +#include "core/providers/webgpu/webgpu_provider_factory_creator.h" +#include "core/providers/webgpu/webgpu_context.h" +#include "core/providers/webgpu/allocator.h" + +namespace onnxruntime { +namespace webgpu { +namespace ep { + +using onnxruntime::ep::Api; + +// Constructor +Factory::Factory() : OrtEpFactory{}, + default_memory_info_{WEBGPU_BUFFER, OrtMemoryInfoDeviceType_GPU, + 0, // vendor id + 0, // device id + OrtDeviceMemoryType_DEFAULT, + 0, // alignment + OrtDeviceAllocator}, + readonly_memory_info_{WEBGPU_BUFFER, OrtMemoryInfoDeviceType_GPU, + 0, // vendor id + 0, // device id + OrtDeviceMemoryType_DEFAULT, + 0, // alignment + OrtReadOnlyAllocator} { + ort_version_supported = ORT_API_VERSION; + + GetName = GetNameImpl; + GetVendor = GetVendorImpl; + GetVendorId = GetVendorIdImpl; + GetVersion = GetVersionImpl; + + GetSupportedDevices = GetSupportedDevicesImpl; + CreateEp = CreateEpImpl; + ReleaseEp = ReleaseEpImpl; + + CreateAllocator = CreateAllocatorImpl; + ReleaseAllocator = ReleaseAllocatorImpl; + CreateDataTransfer = CreateDataTransferImpl; + + IsStreamAware = IsStreamAwareImpl; +} + +// Static C API implementations + +const char* ORT_API_CALL Factory::GetNameImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return kWebGpuExecutionProvider; +} + +const char* ORT_API_CALL Factory::GetVendorImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return "Microsoft"; +} + +uint32_t ORT_API_CALL Factory::GetVendorIdImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return 0; +} + +const char* ORT_API_CALL Factory::GetVersionImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return "0.1.0"; +} + +OrtStatus* ORT_API_CALL Factory::GetSupportedDevicesImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* const* devices, + size_t num_devices, + OrtEpDevice** ep_devices, + size_t max_ep_devices, + size_t* p_num_ep_devices) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + auto factory = static_cast(this_ptr); + + size_t& num_ep_devices = *p_num_ep_devices; + num_ep_devices = 0; + + for (size_t i = 0; i < num_devices && num_ep_devices < max_ep_devices; ++i) { + const OrtHardwareDevice& device = *devices[i]; + if (Api().ort.HardwareDevice_Type(&device) == OrtHardwareDeviceType::OrtHardwareDeviceType_GPU) { + // TODO: any metadata or options to add? + OrtEpDevice* ep_device = nullptr; + ORT_API_RETURN_IF_ERROR(Api().ep.CreateEpDevice(this_ptr, + &device, nullptr, nullptr, + &ep_device)); + ORT_API_RETURN_IF_ERROR(Api().ep.EpDevice_AddAllocatorInfo(ep_device, factory->default_memory_info_)); + ORT_API_RETURN_IF_ERROR(Api().ep.EpDevice_AddAllocatorInfo(ep_device, factory->readonly_memory_info_)); + ep_devices[num_ep_devices++] = ep_device; + } + } + + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END +} + +OrtStatus* ORT_API_CALL Factory::CreateEpImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* const* /*devices*/, + const OrtKeyValuePairs* const* /*ep_metadata*/, + size_t num_devices, + const OrtSessionOptions* session_options, + const OrtLogger* logger, + OrtEp** ep) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + if (num_devices == 0) { + return Api().ort.CreateStatus(ORT_INVALID_ARGUMENT, "No hardware devices provided to create WebGPU EP."); + } + + OrtKeyValuePairs* session_config_entries = nullptr; + ORT_API_RETURN_IF_ERROR(Api().ort.GetSessionOptionsConfigEntries(session_options, &session_config_entries)); + Ort::KeyValuePairs session_config_entries_holder(session_config_entries); // allow automatic release + + auto config_options = ConfigOptions{}; + const char* const* keys = nullptr; + const char* const* values = nullptr; + size_t num_entries = 0; + Api().ort.GetKeyValuePairs(session_config_entries, &keys, &values, &num_entries); + for (size_t i = 0; i < num_entries; ++i) { + auto status = config_options.AddConfigEntry(keys[i], values[i]); + if (!status.IsOK()) { + return Api().ort.CreateStatus((OrtErrorCode)status.Code(), status.ErrorMessage().c_str()); + } + } + + auto webgpu_ep_factory = WebGpuProviderFactoryCreator::Create(config_options); + auto webgpu_ep = webgpu_ep_factory->CreateProvider(*session_options, *logger); + static_cast(webgpu_ep.get())->SetEpLogger(logger); + auto factory = static_cast(this_ptr); + const int context_id = webgpu_ep->GetDeviceId(); + Ep::Config webgpu_ep_config{ + CPUAllocator::DefaultInstance(), // CPU allocator + std::make_shared(WebGpuContextFactory::GetContext(context_id).BufferManager(), false), // default device allocator + std::make_shared(WebGpuContextFactory::GetContext(context_id).InitializerBufferManager(), true), // initializer device allocator + }; + *ep = new Ep(std::move(webgpu_ep), *factory, *logger, webgpu_ep_config); + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END +} + +void ORT_API_CALL Factory::ReleaseEpImpl(OrtEpFactory* /*this_ptr*/, OrtEp* ep) noexcept { + delete static_cast(ep); +} + +OrtStatus* ORT_API_CALL Factory::CreateAllocatorImpl( + OrtEpFactory* /*this_ptr*/, + const OrtMemoryInfo* memory_info, + const OrtKeyValuePairs* /*allocator_options*/, + OrtAllocator** allocator) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + Ort::ConstMemoryInfo ort_memory_info{memory_info}; + + if (ort_memory_info.GetAllocatorType() != OrtDeviceAllocator || + ort_memory_info.GetDeviceId() != 0 || + ort_memory_info.GetAllocatorName() != WEBGPU_BUFFER) { + return Api().ort.CreateStatus(ORT_INVALID_ARGUMENT, + "Unsupported memory info for shared allocator."); + } + + *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, + [](const OrtMemoryInfo&) -> AllocatorPtr { + return std::make_shared(WebGpuContextFactory::DefaultContext().BufferManager(), false); + }); + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END +} + +void ORT_API_CALL Factory::ReleaseAllocatorImpl(OrtEpFactory* /*this_ptr*/, OrtAllocator* allocator) noexcept { + onnxruntime::ep::adapter::Allocator* ptr = static_cast(allocator); + delete ptr; +} + +OrtStatus* ORT_API_CALL Factory::CreateDataTransferImpl( + OrtEpFactory* /*this_ptr*/, + OrtDataTransferImpl** data_transfer) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + *data_transfer = OrtWebGpuCreateDataTransfer(); // TODO(fs-eire): pass context id if needed + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END +} + +bool ORT_API_CALL Factory::IsStreamAwareImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return false; // Default: not stream aware +} + +OrtStatus* ORT_API_CALL Factory::CreateSyncStreamForDeviceImpl( + OrtEpFactory* /*this_ptr*/, + const OrtMemoryDevice* /*memory_device*/, + const OrtKeyValuePairs* /*stream_options*/, + OrtSyncStreamImpl** stream) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + *stream = nullptr; + return Api().ort.CreateStatus(ORT_NOT_IMPLEMENTED, + "CreateSyncStreamForDevice is not implemented for this EP factory."); + EXCEPTION_TO_RETURNED_STATUS_END +} + +} // namespace ep +} // namespace webgpu +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/ep/factory.h b/onnxruntime/core/providers/webgpu/ep/factory.h new file mode 100644 index 0000000000000..f23b3871ebc60 --- /dev/null +++ b/onnxruntime/core/providers/webgpu/ep/factory.h @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include "ep.h" + +namespace onnxruntime { +namespace webgpu { +namespace ep { + +/// +/// A bridge class between the EP API and the WebGPU EP Factory implementation. +/// +class Factory : public OrtEpFactory { + private: + // Static C API implementations + static const char* ORT_API_CALL GetNameImpl(const OrtEpFactory* this_ptr) noexcept; + static const char* ORT_API_CALL GetVendorImpl(const OrtEpFactory* this_ptr) noexcept; + static uint32_t ORT_API_CALL GetVendorIdImpl(const OrtEpFactory* this_ptr) noexcept; + static const char* ORT_API_CALL GetVersionImpl(const OrtEpFactory* this_ptr) noexcept; + + static OrtStatus* ORT_API_CALL GetSupportedDevicesImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* const* devices, + size_t num_devices, + OrtEpDevice** ep_devices, + size_t max_ep_devices, + size_t* p_num_ep_devices) noexcept; + + static OrtStatus* ORT_API_CALL CreateEpImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* const* devices, + const OrtKeyValuePairs* const* ep_metadata, + size_t num_devices, + const OrtSessionOptions* session_options, + const OrtLogger* logger, + OrtEp** ep) noexcept; + + static void ORT_API_CALL ReleaseEpImpl(OrtEpFactory* this_ptr, OrtEp* ep) noexcept; + + static OrtStatus* ORT_API_CALL CreateAllocatorImpl( + OrtEpFactory* this_ptr, + const OrtMemoryInfo* memory_info, + const OrtKeyValuePairs* allocator_options, + OrtAllocator** allocator) noexcept; + + static void ORT_API_CALL ReleaseAllocatorImpl(OrtEpFactory* this_ptr, OrtAllocator* allocator) noexcept; + + static OrtStatus* ORT_API_CALL CreateDataTransferImpl( + OrtEpFactory* this_ptr, + OrtDataTransferImpl** data_transfer) noexcept; + + static bool ORT_API_CALL IsStreamAwareImpl(const OrtEpFactory* this_ptr) noexcept; + + static OrtStatus* ORT_API_CALL CreateSyncStreamForDeviceImpl( + OrtEpFactory* this_ptr, + const OrtMemoryDevice* memory_device, + const OrtKeyValuePairs* stream_options, + OrtSyncStreamImpl** stream) noexcept; + + Ort::MemoryInfo default_memory_info_; + Ort::MemoryInfo readonly_memory_info_; // used for initializers + + public: + Factory(); + ~Factory() = default; +}; + +} // namespace ep +} // namespace webgpu +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/generator/range.cc b/onnxruntime/core/providers/webgpu/generator/range.cc index 3fa062f327ba2..a84660a020fed 100644 --- a/onnxruntime/core/providers/webgpu/generator/range.cc +++ b/onnxruntime/core/providers/webgpu/generator/range.cc @@ -79,36 +79,40 @@ template class Range; template class Range; template class Range; -void RegisterRangeKernels(KernelRegistry& kernel_registry, bool enable_int64) { - // Helper lambda to create kernel - auto create_range_kernel_info = [](auto type_tag) { - using T = decltype(type_tag); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { - out = std::make_unique>(info); - return Status::OK(); - }; - - return KernelCreateInfo( - KernelDefBuilder() - .SetName("Range") - .SetDomain(kOnnxDomain) - .SinceVersion(11) - .Provider(kWebGpuExecutionProvider) - .TypeConstraint("T", DataTypeImpl::GetTensorType()) - .InputMemoryType(OrtMemTypeCPU, 0) - .InputMemoryType(OrtMemTypeCPU, 1) - .InputMemoryType(OrtMemTypeCPU, 2) - .Build(), - kernel_create_fn); - }; +namespace { +template +Status CreateRangeKernel(FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) { + out = std::make_unique>(info); + return Status::OK(); +} + +template +KernelCreateInfo CreateRangeKernelInfo() { + return KernelCreateInfo( + KernelDefBuilder() + .SetName("Range") + .SetDomain(kOnnxDomain) + .SinceVersion(11) + .Provider(kWebGpuExecutionProvider) + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .InputMemoryType(OrtMemTypeCPU, 0) + .InputMemoryType(OrtMemTypeCPU, 1) + .InputMemoryType(OrtMemTypeCPU, 2) + .Build(), + CreateRangeKernel); +} + +} // namespace + +void RegisterRangeKernels(KernelRegistry& kernel_registry, bool enable_int64) { // Always register float and int32_t - ORT_THROW_IF_ERROR(kernel_registry.Register(create_range_kernel_info(float{}))); - ORT_THROW_IF_ERROR(kernel_registry.Register(create_range_kernel_info(int32_t{}))); + ORT_THROW_IF_ERROR(kernel_registry.Register(CreateRangeKernelInfo())); + ORT_THROW_IF_ERROR(kernel_registry.Register(CreateRangeKernelInfo())); // Register int64_t only if int64 support is enabled if (enable_int64) { - ORT_THROW_IF_ERROR(kernel_registry.Register(create_range_kernel_info(int64_t{}))); + ORT_THROW_IF_ERROR(kernel_registry.Register(CreateRangeKernelInfo())); } } diff --git a/onnxruntime/core/providers/webgpu/tensor/cast.cc b/onnxruntime/core/providers/webgpu/tensor/cast.cc index 6bb0f688bfdb7..2695c2800d37a 100644 --- a/onnxruntime/core/providers/webgpu/tensor/cast.cc +++ b/onnxruntime/core/providers/webgpu/tensor/cast.cc @@ -90,7 +90,7 @@ template KernelCreateInfo CreateCastKernelInfo(bool enable_int64) { const auto& type_constraints = GetOpTypeConstraints(enable_int64, true); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { + KernelCreatePtrFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { out = std::make_unique(info); return Status::OK(); }; diff --git a/onnxruntime/core/providers/webgpu/tensor/concat.cc b/onnxruntime/core/providers/webgpu/tensor/concat.cc index 75453b991a0cd..c6178d44dba75 100644 --- a/onnxruntime/core/providers/webgpu/tensor/concat.cc +++ b/onnxruntime/core/providers/webgpu/tensor/concat.cc @@ -98,7 +98,7 @@ Status Concat::ComputeInternal(ComputeContext& context) const { } Prepare prepare; - ORT_RETURN_IF_ERROR(PrepareForCompute(&context.KernelContext(), input_tensors, prepare)); + ORT_RETURN_IF_ERROR(PrepareForComputeImpl(&context.KernelContext(), input_tensors, prepare)); if (prepare.output_num_elements == 0) { return Status::OK(); } diff --git a/onnxruntime/core/providers/webgpu/tensor/expand.cc b/onnxruntime/core/providers/webgpu/tensor/expand.cc index 0dacd589cbba8..0d39b1ec9d35e 100644 --- a/onnxruntime/core/providers/webgpu/tensor/expand.cc +++ b/onnxruntime/core/providers/webgpu/tensor/expand.cc @@ -108,7 +108,7 @@ template KernelCreateInfo CreateExpandVersionedKernelInfo(bool enable_int64) { const auto& type_constraints = GetOpTypeConstraints(enable_int64, true); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { + KernelCreatePtrFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { out = std::make_unique(info); return Status::OK(); }; @@ -129,7 +129,7 @@ template KernelCreateInfo CreateExpandKernelInfo(bool enable_int64) { const auto& type_constraints = GetOpTypeConstraints(enable_int64, true); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { + KernelCreatePtrFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { out = std::make_unique(info); return Status::OK(); }; diff --git a/onnxruntime/core/providers/webgpu/tensor/gather.cc b/onnxruntime/core/providers/webgpu/tensor/gather.cc index b3e5c7b4e8310..970c2d6bed7a3 100644 --- a/onnxruntime/core/providers/webgpu/tensor/gather.cc +++ b/onnxruntime/core/providers/webgpu/tensor/gather.cc @@ -60,7 +60,7 @@ Status GatherProgram::GenerateShaderCode(ShaderHelper& shader) const { Status Gather::ComputeInternal(ComputeContext& context) const { Prepare p; - ORT_RETURN_IF_ERROR(PrepareForCompute(&context.KernelContext(), p)); + ORT_RETURN_IF_ERROR(PrepareForComputeImpl(&context.KernelContext(), p)); uint32_t data_size = onnxruntime::narrow(p.output_tensor->Shape().Size()); if (data_size == 0) { return Status::OK(); diff --git a/onnxruntime/core/providers/webgpu/tensor/pad.cc b/onnxruntime/core/providers/webgpu/tensor/pad.cc index 0e77ec46bbddb..7a576c4b53ecf 100644 --- a/onnxruntime/core/providers/webgpu/tensor/pad.cc +++ b/onnxruntime/core/providers/webgpu/tensor/pad.cc @@ -49,7 +49,7 @@ Status Pad::ComputeInternal(ComputeContext& context) const { const auto pads_data = pads_tensor->DataAsSpan(); // Compute Pads by applying axes if specified otherwise copy the supplied pads. - PadBase::ComputePads(context.KernelContext(), data_rank, pads_data, pads); + PadBase::ComputePadsImpl(context.KernelContext(), data_rank, pads_data, pads); // Separate out any negative pads into the slices array PadBase::SeparateNegativeToSlices(pads, slices); diff --git a/onnxruntime/core/providers/webgpu/tensor/shape_op.cc b/onnxruntime/core/providers/webgpu/tensor/shape_op.cc index b211d48dab1c9..09194aa9f4dbb 100644 --- a/onnxruntime/core/providers/webgpu/tensor/shape_op.cc +++ b/onnxruntime/core/providers/webgpu/tensor/shape_op.cc @@ -3,11 +3,73 @@ #include "core/providers/webgpu/webgpu_kernel.h" #include "core/providers/webgpu/webgpu_supported_types.h" -#include "core/providers/cpu/tensor/shape_op.h" namespace onnxruntime { namespace webgpu { +#ifndef SHARED_PROVIDER +#include "core/common/common.h" +#include "core/common/narrow.h" +#include "core/framework/op_kernel.h" +#endif + +#include +#include + +class Shape final : public OpKernel { + public: + Shape(const OpKernelInfo& info) : OpKernel(info) { + info.GetAttrOrDefault("start", &start_index_, 0); + + if (start_index_ != 0) { + // "start" is provided and is non-default (default is 0) + needs_slicing_ = true; + } + + if (info.GetAttr("end", &end_index_).IsOK()) { + needs_slicing_ = true; + } + } + + // Takes a tensor as input and outputs an 1D int64 tensor + // containing the shape of the input tensor. + Status Compute(OpKernelContext* context) const override { + const auto* input = context->Input(0); + const TensorShape& input_shape = input->Shape(); + + int64_t rank = gsl::narrow_cast(input_shape.NumDimensions()); + + if (!needs_slicing_) { // vanilla use of Shape (no slicing) + Tensor* output = context->Output(0, {rank}); + input_shape.CopyDims(output->MutableData(), static_cast(rank)); + } else { // slicing is needed + int64_t true_start = start_index_; + int64_t true_end = end_index_; + + // Deal with negative(s) and clamp + true_start = true_start < 0 ? true_start + rank : true_start; + true_start = true_start < 0 ? 0 : ((true_start > rank) ? rank : true_start); + + true_end = true_end < 0 ? true_end + rank : true_end; + true_end = true_end < 0 ? 0 : ((true_end > rank) ? rank : true_end); + + auto slice_length = true_end - true_start; + Tensor* output = context->Output(0, {slice_length < 0 ? 0 : slice_length}); + + if (slice_length > 0) { + input_shape.CopyDims(output->MutableData(), onnxruntime::narrow(true_start), onnxruntime::narrow(slice_length)); + } + } + + return Status::OK(); + } + + private: + bool needs_slicing_ = false; + int64_t start_index_ = 0; + int64_t end_index_ = std::numeric_limits::max(); +}; + ONNX_OPERATOR_VERSIONED_KERNEL_EX( Shape, kOnnxDomain, diff --git a/onnxruntime/core/providers/webgpu/tensor/unsqueeze.cc b/onnxruntime/core/providers/webgpu/tensor/unsqueeze.cc index 104fcf1812af8..3337448564bed 100644 --- a/onnxruntime/core/providers/webgpu/tensor/unsqueeze.cc +++ b/onnxruntime/core/providers/webgpu/tensor/unsqueeze.cc @@ -12,7 +12,7 @@ template KernelCreateInfo CreateUnsqueezeVersionedKernelInfo(bool enable_int64) { const auto& type_constraints = GetOpTypeConstraints(enable_int64, true); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { + KernelCreatePtrFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { out = std::make_unique(info); return Status::OK(); }; @@ -47,7 +47,7 @@ template KernelCreateInfo CreateUnsqueezeKernelInfo(bool enable_int64) { const auto& type_constraints = GetOpTypeConstraints(enable_int64, true); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { + KernelCreatePtrFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { out = std::make_unique(info); return Status::OK(); }; diff --git a/onnxruntime/core/providers/webgpu/tensor/upsample.cc b/onnxruntime/core/providers/webgpu/tensor/upsample.cc index fb406883ba4ba..8f51ed45004bf 100644 --- a/onnxruntime/core/providers/webgpu/tensor/upsample.cc +++ b/onnxruntime/core/providers/webgpu/tensor/upsample.cc @@ -90,7 +90,7 @@ Status Upsample::ComputeInternal(ComputeContext& context) const { InlinedVector scales_array(input_dims.size()); // opset < 10 - if (OpKernel::Node().InputDefs().size() == 1) { + if (OpKernel::Node().SinceVersion() < 10) { scales_array = scales_; // Compute output shape from scales attributes and input dims ComputeOutputShape(scales_array, input_dims, output_dims); diff --git a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc index c6255e6f352d9..b4d751ce3a2c0 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc +++ b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc @@ -3,6 +3,7 @@ #include "core/providers/webgpu/webgpu_execution_provider.h" +#include #include #include #include @@ -445,8 +446,8 @@ class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kWebGpuExecutionProvider, kOnnxD class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kWebGpuExecutionProvider, kOnnxDomain, 16, 17, ScatterElements); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kWebGpuExecutionProvider, kOnnxDomain, 18, ScatterElements); -std::unique_ptr RegisterKernels(bool enable_graph_capture = false, bool enable_int64 = false) { - auto kernel_registry = std::make_unique(); +std::unique_ptr RegisterKernels(bool enable_graph_capture, bool enable_int64) { + auto kernel_registry = std::make_unique(); static const BuildKernelCreateInfoFn function_table[] = { BuildKernelCreateInfo, // default entry to avoid the list becoming empty after ops-reducing @@ -837,6 +838,72 @@ std::unique_ptr RegisterKernels(bool enable_graph_capture = fals return kernel_registry; } +#if defined(ORT_USE_EP_API_ADAPTERS) + +namespace { +std::mutex g_kernel_registry_mutex; +std::shared_ptr g_kernel_registry; +std::shared_ptr g_graph_capture_kernel_registry; +std::shared_ptr g_int64_kernel_registry; +} // namespace + +void CleanupKernelRegistries() { + std::lock_guard lock(g_kernel_registry_mutex); + g_kernel_registry.reset(); + g_graph_capture_kernel_registry.reset(); + g_int64_kernel_registry.reset(); +} +#endif + +std::shared_ptr GetKernelRegistry(bool enable_graph_capture, bool enable_int64) { + // kernel registry variables are defined differently based on build configuration + // + // - When building as a static library, use static local variable. This is because + // we don't have a reliable way to explicitly destroy the kernel registry after + // use. + // + // - When building as a shared library, use global variables. The cleanup will be performed + // when `ReleaseEpFactory` is called. + // + // Graph capture mode needs a separate kernel registry because contrib kernel registration + // differs based on enable_graph_capture, and enable_int64 is always true when + // enable_graph_capture is true. + if (enable_graph_capture) { +#if !defined(ORT_USE_EP_API_ADAPTERS) + static std::shared_ptr registry = RegisterKernels(true, true); + return registry; +#else + std::lock_guard lock(g_kernel_registry_mutex); + if (g_graph_capture_kernel_registry == nullptr) { + g_graph_capture_kernel_registry = RegisterKernels(true, true); + } + return g_graph_capture_kernel_registry; +#endif + } else if (enable_int64) { +#if defined(ORT_USE_EP_API_ADAPTERS) + std::lock_guard lock(g_kernel_registry_mutex); + if (g_int64_kernel_registry == nullptr) { + g_int64_kernel_registry = RegisterKernels(false, true); + } + return g_int64_kernel_registry; +#else + static std::shared_ptr registry = RegisterKernels(false, true); + return registry; +#endif + } else { +#if defined(ORT_USE_EP_API_ADAPTERS) + std::lock_guard lock(g_kernel_registry_mutex); + if (g_kernel_registry == nullptr) { + g_kernel_registry = RegisterKernels(false, false); + } + return g_kernel_registry; +#else + static std::shared_ptr registry = RegisterKernels(false, false); + return registry; +#endif + } +} + } // namespace webgpu using namespace webgpu; @@ -850,6 +917,7 @@ WebGpuExecutionProvider::WebGpuExecutionProvider(int context_id, preferred_data_layout_{config.data_layout}, force_cpu_node_names_{std::move(config.force_cpu_node_names)}, enable_graph_capture_{config.enable_graph_capture}, + // enable_int64_ is always true when enable_graph_capture_ is true enable_int64_{config.enable_graph_capture || config.enable_int64}, multi_rotary_cache_concat_offset_{config.multi_rotary_cache_concat_offset}, prepack_allocator_{std::make_shared(context_.InitializerBufferManager(), false)} { @@ -882,6 +950,7 @@ std::vector WebGpuExecutionProvider::CreatePreferredAllocators() { }; } +#if !defined(ORT_USE_EP_API_ADAPTERS) std::vector> WebGpuExecutionProvider::GetCapability( const onnxruntime::GraphViewer& graph, const IKernelLookup& kernel_lookup, @@ -973,20 +1042,7 @@ std::vector> WebGpuExecutionProvider::GetCapa return result; } -std::shared_ptr WebGpuExecutionProvider::GetKernelRegistry() const { - // Cache registries based on enable_graph_capture_ and enable_int64_ flags - // Note: enable_int64_ is always true when enable_graph_capture_ is true - if (enable_graph_capture_) { - static std::shared_ptr registry = webgpu::RegisterKernels(true, true); - return registry; - } else if (enable_int64_) { - static std::shared_ptr registry = webgpu::RegisterKernels(false, true); - return registry; - } else { - static std::shared_ptr registry = webgpu::RegisterKernels(false, false); - return registry; - } -} +#endif // !defined(ORT_USE_EP_API_ADAPTERS) std::unique_ptr WebGpuExecutionProvider::GetDataTransfer() const { return std::make_unique(BufferManager()); diff --git a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.h b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.h index b5a6b5f167faf..b46d3f3cb45d2 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.h +++ b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.h @@ -4,6 +4,11 @@ #pragma once +#include +#include +#include +#include + #include "core/framework/execution_provider.h" #include "core/framework/session_options.h" #include "core/graph/constants.h" @@ -28,6 +33,9 @@ class GpuBufferAllocator; // Forward declare CapturedCommandInfo which is now defined in webgpu_context.h struct CapturedCommandInfo; + +// The actual implementation of kernel registration. +std::shared_ptr GetKernelRegistry(bool enable_graph_capture, bool enable_int64); } // namespace webgpu struct WebGpuExecutionProviderConfig { @@ -44,13 +52,21 @@ class WebGpuExecutionProvider : public IExecutionProvider { WebGpuExecutionProvider(int context_id, webgpu::WebGpuContext& context, WebGpuExecutionProviderConfig&& config); ~WebGpuExecutionProvider() override; + inline auto GetKernelRegistryImpl() const { + return webgpu::GetKernelRegistry(enable_graph_capture_, enable_int64_); + } + +#if !defined(ORT_USE_EP_API_ADAPTERS) std::vector> GetCapability( const onnxruntime::GraphViewer& graph_viewer, const IKernelLookup& /*kernel_lookup*/, const GraphOptimizerRegistry& /* graph_optimizer_registry */, IResourceAccountant* /* resource_accountant */) const override; - std::shared_ptr GetKernelRegistry() const override; + std::shared_ptr GetKernelRegistry() const override { + return GetKernelRegistryImpl(); + } +#endif std::unique_ptr GetDataTransfer() const override; #if defined(__wasm__) std::unique_ptr GetExternalDataLoader() const override; @@ -83,8 +99,18 @@ class WebGpuExecutionProvider : public IExecutionProvider { Status ReplayGraph(int graph_annotation_id) override; webgpu::BufferManager& BufferManager() const; AllocatorPtr PrepackAllocator() const { return prepack_allocator_; } + std::span GetForceCpuNodeNames() const { return force_cpu_node_names_; } uint32_t MultiRotaryCacheConcatOffset() const { return multi_rotary_cache_concat_offset_; } +#if defined(ORT_USE_EP_API_ADAPTERS) + inline onnxruntime::ep::adapter::Logger& GetEpLogger() const { + return *ep_logger_; + } + inline void SetEpLogger(const OrtLogger* logger) { + ep_logger_ = std::make_unique(logger); + } +#endif + private: bool IsGraphCaptureAllowed() const; void IncrementRegularRunCountBeforeGraphCapture(); @@ -114,6 +140,10 @@ class WebGpuExecutionProvider : public IExecutionProvider { // Allocator for prepacked weights (uses buffers without mapping) AllocatorPtr prepack_allocator_; + +#if defined(ORT_USE_EP_API_ADAPTERS) + std::unique_ptr ep_logger_; +#endif }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc b/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc index fc2496f0c7b68..16899370e47f1 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc +++ b/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc @@ -284,11 +284,11 @@ std::shared_ptr WebGpuProviderFactoryCreator::Create( // WebGPU DataTransfer implementation wrapper for the C API with lazy initialization struct WebGpuDataTransferImpl : OrtDataTransferImpl { - WebGpuDataTransferImpl(const OrtApi& ort_api_in) + WebGpuDataTransferImpl(const OrtApi& ort_api_in, int context_id) : ort_api{ort_api_in}, ep_api{*ort_api_in.GetEpApi()}, data_transfer_{nullptr}, - context_id_{0}, // Always use context 0 for Environment's data transfer + context_id_{context_id}, init_mutex_{} { ort_version_supported = ORT_API_VERSION; CanCopy = CanCopyImpl; // OrtDataTransferImpl::CanCopy callback @@ -327,9 +327,9 @@ struct WebGpuDataTransferImpl : OrtDataTransferImpl { // If both are GPU, they must have the same device ID if (src_type == OrtMemoryInfoDeviceType_GPU && dst_type == OrtMemoryInfoDeviceType_GPU) { - uint64_t src_device_id = impl.ep_api.MemoryDevice_GetDeviceId(src_memory_device); - uint64_t dst_device_id = impl.ep_api.MemoryDevice_GetDeviceId(dst_memory_device); - if (src_device_id != dst_device_id) { + int src_device_id = impl.ep_api.MemoryDevice_GetDeviceId(src_memory_device); + int dst_device_id = impl.ep_api.MemoryDevice_GetDeviceId(dst_memory_device); + if (src_device_id != impl.context_id_ || dst_device_id != impl.context_id_) { return false; // Cannot copy between different devices } } @@ -362,19 +362,40 @@ struct WebGpuDataTransferImpl : OrtDataTransferImpl { auto& context = WebGpuContextFactory::DefaultContext(); - // Create the DataTransfer instance - // Note: The DataTransfer holds a const reference to BufferManager. The BufferManager's lifecycle + // Create the DataTransferImpl instance + // Note: The DataTransferImpl holds a const reference to BufferManager. The BufferManager's lifecycle // is managed by the WebGpuContext, which is stored in a static WebGpuContextFactory and persists // for the lifetime of the application, ensuring the reference remains valid. - impl.data_transfer_ = std::make_unique(context.BufferManager()); + impl.data_transfer_ = std::make_unique(context.BufferManager()); } } // Now perform the actual tensor copy for (size_t idx = 0; idx < num_tensors; ++idx) { - const OrtValue* src_tensor = src_tensors[idx]; - OrtValue* dst_tensor = dst_tensors[idx]; - auto status = impl.data_transfer_->CopyTensor(src_tensor->Get(), *dst_tensor->GetMutable()); +#if defined(ORT_USE_EP_API_ADAPTERS) + Ort::ConstValue src_value{src_tensors[idx]}; + const void* src_data = src_value.GetTensorRawData(); + size_t size = src_value.GetTensorSizeInBytes(); + bool src_is_gpu = src_value.GetTensorMemoryInfo().GetDeviceType() == OrtMemoryInfoDeviceType_GPU; + + Ort::UnownedValue dst_value{dst_tensors[idx]}; + void* dst_data = dst_value.GetTensorMutableRawData(); + bool dst_is_gpu = dst_value.GetTensorMemoryInfo().GetDeviceType() == OrtMemoryInfoDeviceType_GPU; +#else + const Tensor& src_tensor = src_tensors[idx]->Get(); + const void* src_data = src_tensor.DataRaw(); + size_t size = src_tensor.SizeInBytes(); + bool src_is_gpu = src_tensor.Location().device.Type() == OrtDevice::GPU; + + Tensor& dst_tensor = *dst_tensors[idx]->GetMutable(); + void* dst_data = dst_tensor.MutableDataRaw(); + bool dst_is_gpu = dst_tensor.Location().device.Type() == OrtDevice::GPU; +#endif + auto status = impl.data_transfer_->CopyTensor(src_data, + src_is_gpu, + dst_data, + dst_is_gpu, + size); if (!status.IsOK()) { return OrtApis::CreateStatus(ORT_RUNTIME_EXCEPTION, status.ErrorMessage().c_str()); } @@ -398,19 +419,23 @@ struct WebGpuDataTransferImpl : OrtDataTransferImpl { const OrtApi& ort_api; const OrtEpApi& ep_api; - std::unique_ptr data_transfer_; // Lazy-initialized - int context_id_; // Track which context we're using - std::mutex init_mutex_; // Protects lazy initialization + std::unique_ptr data_transfer_; // Lazy-initialized + int context_id_; // Track which context we're using + std::mutex init_mutex_; // Protects lazy initialization }; -OrtDataTransferImpl* OrtWebGpuCreateDataTransfer() { +OrtDataTransferImpl* OrtWebGpuCreateDataTransfer(int context_id /* = 0 */) { +#if defined(ORT_USE_EP_API_ADAPTERS) + return new WebGpuDataTransferImpl(onnxruntime::ep::Api().ort, context_id); +#else // Validate API version is supported const OrtApi* api = OrtApis::GetApi(ORT_API_VERSION); if (!api) { // API version not supported - return nullptr to indicate failure return nullptr; } - return new WebGpuDataTransferImpl(*api); + return new WebGpuDataTransferImpl(*api, context_id); +#endif } } // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/webgpu_provider_factory_creator.h b/onnxruntime/core/providers/webgpu/webgpu_provider_factory_creator.h index 021e33ef25309..876a2e11d791a 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_provider_factory_creator.h +++ b/onnxruntime/core/providers/webgpu/webgpu_provider_factory_creator.h @@ -22,6 +22,6 @@ struct WebGpuProviderFactoryCreator { // C API to create data transfer for WebGPU EP with lazy initialization // Context will be determined from tensors during the first CopyTensors call // Caller takes ownership of the returned OrtDataTransferImpl* -OrtDataTransferImpl* OrtWebGpuCreateDataTransfer(); +OrtDataTransferImpl* OrtWebGpuCreateDataTransfer(int context_id = 0); } // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/matmul_2bits_test.cc b/onnxruntime/test/contrib_ops/matmul_2bits_test.cc index 00ea8947b9dd7..06ff495327ecd 100644 --- a/onnxruntime/test/contrib_ops/matmul_2bits_test.cc +++ b/onnxruntime/test/contrib_ops/matmul_2bits_test.cc @@ -451,7 +451,7 @@ TEST(MatMul2Bits, Float32_2b_Accuracy4) { TestMatMul2BitsTyped(); } -#ifdef USE_WEBGPU +#if defined(USE_WEBGPU) && !defined(ORT_USE_EP_API_ADAPTERS) namespace { @@ -594,7 +594,7 @@ TEST(MatMul2BitsWebGpu, Float32_ZeroPoint_DP4A) { RunTest2Bits(opts); } -#endif // USE_WEBGPU +#endif // defined(USE_WEBGPU) && !defined(ORT_USE_EP_API_ADAPTERS) } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/unittest_util/base_tester.cc b/onnxruntime/test/unittest_util/base_tester.cc index 18ead92ce3f18..1f744df14cfb8 100644 --- a/onnxruntime/test/unittest_util/base_tester.cc +++ b/onnxruntime/test/unittest_util/base_tester.cc @@ -749,10 +749,8 @@ void BaseTester::RunWithConfig(size_t* number_of_pre_packed_weights_counter, execution_provider = DefaultXnnpackExecutionProvider(); else if (provider_type == onnxruntime::kDmlExecutionProvider) execution_provider = DefaultDmlExecutionProvider(); -#if !defined(USE_WEBGPU) || !defined(ORT_USE_EP_API_ADAPTERS) else if (provider_type == onnxruntime::kWebGpuExecutionProvider) execution_provider = DefaultWebGpuExecutionProvider(); -#endif else if (provider_type == dynamic_plugin_ep_name) { execution_provider = dynamic_plugin_ep_infra::MakeEp(); } diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc index fd2cf2f712628..1f82e1f893eab 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc @@ -11,6 +11,7 @@ #include "nlohmann/json.hpp" #include "core/common/common.h" +#include "core/framework/config_options.h" #include "core/framework/execution_provider.h" #include "core/session/abi_session_options_impl.h" #include "core/session/ort_env.h" @@ -167,7 +168,7 @@ void Shutdown() { g_plugin_ep_infrastructure_state.reset(); } -std::unique_ptr MakeEp(const logging::Logger* logger) { +std::unique_ptr MakeEp(const logging::Logger* logger, const ConfigOptions* ep_options) { if (!IsInitialized()) { return nullptr; } @@ -182,6 +183,13 @@ std::unique_ptr MakeEp(const logging::Logger* logger) { StrMapToKeyValueCstrVectors(state.config.default_ep_options, default_ep_option_key_cstrs, default_ep_option_value_cstrs); + if (ep_options != nullptr) { + for (const auto& [key, value] : ep_options->configurations) { + default_ep_option_key_cstrs.push_back(key.c_str()); + default_ep_option_value_cstrs.push_back(value.c_str()); + } + } + OrtSessionOptions ort_session_options{}; ORT_THROW_IF_ERROR(AddEpOptionsToSessionOptions(state.selected_c_ep_devices, default_ep_option_key_cstrs, diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h index 680045be9330c..0962df8e35308 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h @@ -17,6 +17,7 @@ namespace onnxruntime { struct IExecutionProviderFactory; class IExecutionProvider; +struct ConfigOptions; namespace logging { class Logger; @@ -74,7 +75,8 @@ bool IsInitialized(); void Shutdown(); // Returns a dynamic plugin EP `IExecutionProvider` instance, or `nullptr` if uninitialized. -std::unique_ptr MakeEp(const logging::Logger* logger = nullptr); +// `ep_options` provides additional EP-specific option overrides (key-value pairs) on top of the defaults. +std::unique_ptr MakeEp(const logging::Logger* logger = nullptr, const ConfigOptions* ep_options = nullptr); // Gets the dynamic plugin EP name, or `std::nullopt` if uninitialized. std::optional GetEpName(); diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 6dc38f84c79d5..7e6bc6ae06020 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -14,8 +14,13 @@ #ifdef USE_CUDA #include "core/providers/cuda/cuda_provider_options.h" #endif +#if defined(USE_WEBGPU) +#include "core/graph/constants.h" +#include "core/session/abi_session_options_impl.h" +#endif #include "core/session/onnxruntime_cxx_api.h" #include "test/util/include/providers.h" +#include "test/unittest_util/test_dynamic_plugin_ep.h" namespace onnxruntime { @@ -273,19 +278,37 @@ std::unique_ptr DefaultXnnpackExecutionProvider() { } std::unique_ptr DefaultWebGpuExecutionProvider(bool is_nhwc) { -#if defined(USE_WEBGPU) && !defined(ORT_USE_EP_API_ADAPTERS) +#if defined(USE_WEBGPU) ConfigOptions config_options{}; + + // Helper to strip the EP prefix from config entry keys when building as a plugin EP. + // The full key is like "ep.webgpuexecutionprovider.storageBufferCacheMode", and the + // config entry expects just "storageBufferCacheMode" in the EP API build. + // Returns a pointer into the original string, so the result is valid as long as the input is. + auto strip_ep_prefix = [](const char* full_key) -> const char* { +#if defined(ORT_USE_EP_API_ADAPTERS) + std::string_view key{full_key}; + std::string_view prefix = OrtSessionOptions::GetProviderOptionPrefix(kWebGpuExecutionProvider); + ORT_ENFORCE(key.length() >= prefix.length() && key.substr(0, prefix.length()) == prefix, + "Config key \"", key, "\" does not start with expected prefix \"", prefix, "\""); + return full_key + prefix.length(); +#else + return full_key; +#endif + }; + // Disable storage buffer cache - ORT_ENFORCE(config_options.AddConfigEntry(webgpu::options::kStorageBufferCacheMode, + ORT_ENFORCE(config_options.AddConfigEntry(strip_ep_prefix(webgpu::options::kStorageBufferCacheMode), webgpu::options::kBufferCacheMode_Disabled) .IsOK()); if (!is_nhwc) { // Enable NCHW support - ORT_ENFORCE(config_options.AddConfigEntry(webgpu::options::kPreferredLayout, + ORT_ENFORCE(config_options.AddConfigEntry(strip_ep_prefix(webgpu::options::kPreferredLayout), webgpu::options::kPreferredLayout_NCHW) .IsOK()); } - return WebGpuProviderFactoryCreator::Create(config_options)->CreateProvider(); + + return WebGpuExecutionProviderWithOptions(config_options); #else ORT_UNUSED_PARAMETER(is_nhwc); return nullptr; @@ -293,8 +316,16 @@ std::unique_ptr DefaultWebGpuExecutionProvider(bool is_nhwc) } std::unique_ptr WebGpuExecutionProviderWithOptions(const ConfigOptions& config_options) { -#if defined(USE_WEBGPU) && !defined(ORT_USE_EP_API_ADAPTERS) +#if defined(USE_WEBGPU) +#if defined(ORT_USE_EP_API_ADAPTERS) + auto ep_name = dynamic_plugin_ep_infra::GetEpName(); + ORT_ENFORCE(ep_name == kWebGpuExecutionProvider, + "Dynamic plugin EP is not the WebGPU EP. Expected \"", kWebGpuExecutionProvider, + "\", got \"", ep_name.value_or(""), "\""); + return dynamic_plugin_ep_infra::MakeEp(nullptr, &config_options); +#else return WebGpuProviderFactoryCreator::Create(config_options)->CreateProvider(); +#endif #else ORT_UNUSED_PARAMETER(config_options); return nullptr;