From 3694c53b5d43c673545838d19ae6b7443b177aaf Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Tue, 3 Mar 2026 15:21:39 -0800 Subject: [PATCH 01/27] Implement EP API for WebGPU EP --- cmake/onnxruntime_providers_webgpu.cmake | 6 + cmake/onnxruntime_unittests.cmake | 13 + .../onnxruntime/ep/adapter/op_kernel_info.h | 2 +- .../core/providers/cpu/tensor/upsamplebase.h | 18 +- .../core/providers/webgpu/compute_context.h | 4 + .../core/providers/webgpu/controlflow/if.cc | 16 +- .../core/providers/webgpu/controlflow/if.h | 15 +- .../core/providers/webgpu/data_transfer.cc | 41 ++- .../core/providers/webgpu/data_transfer.h | 6 + onnxruntime/core/providers/webgpu/ep/api.cc | 78 ++++++ onnxruntime/core/providers/webgpu/ep/ep.cc | 263 ++++++++++++++++++ onnxruntime/core/providers/webgpu/ep/ep.h | 78 ++++++ .../core/providers/webgpu/ep/factory.cc | 213 ++++++++++++++ .../core/providers/webgpu/ep/factory.h | 88 ++++++ .../core/providers/webgpu/generator/range.cc | 54 ++-- .../core/providers/webgpu/tensor/cast.cc | 2 +- .../core/providers/webgpu/tensor/concat.cc | 2 +- .../core/providers/webgpu/tensor/expand.cc | 4 +- .../core/providers/webgpu/tensor/gather.cc | 2 +- .../core/providers/webgpu/tensor/pad.cc | 2 +- .../core/providers/webgpu/tensor/shape_op.cc | 65 ++++- .../core/providers/webgpu/tensor/unsqueeze.cc | 4 +- .../core/providers/webgpu/tensor/upsample.cc | 2 +- .../webgpu/webgpu_execution_provider.cc | 77 +++-- .../webgpu/webgpu_execution_provider.h | 32 ++- .../webgpu/webgpu_provider_factory.cc | 45 ++- .../webgpu/webgpu_provider_factory_creator.h | 2 +- .../test/contrib_ops/matmul_2bits_test.cc | 4 +- .../unittest_util/test_dynamic_plugin_ep.cc | 9 +- .../unittest_util/test_dynamic_plugin_ep.h | 3 +- onnxruntime/test/util/default_providers.cc | 28 +- 31 files changed, 1088 insertions(+), 90 deletions(-) create mode 100644 onnxruntime/core/providers/webgpu/ep/api.cc create mode 100644 onnxruntime/core/providers/webgpu/ep/ep.cc create mode 100644 onnxruntime/core/providers/webgpu/ep/ep.h create mode 100644 onnxruntime/core/providers/webgpu/ep/factory.cc create mode 100644 onnxruntime/core/providers/webgpu/ep/factory.h diff --git a/cmake/onnxruntime_providers_webgpu.cmake b/cmake/onnxruntime_providers_webgpu.cmake index be7f2613a6272..babf696df4915 100644 --- a/cmake/onnxruntime_providers_webgpu.cmake +++ b/cmake/onnxruntime_providers_webgpu.cmake @@ -119,6 +119,12 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") message(FATAL_ERROR "WebGPU EP shared library build is not supported on Emscripten. Please use static library build.") endif() + + # Configure precompiled headers for shared library build + # PCH ensures ep/adapters.h is included first and improves compilation speed + target_precompile_headers(onnxruntime_providers_webgpu PRIVATE + "${REPO_ROOT}/include/onnxruntime/ep/adapters.h" + ) endif() set_target_properties(onnxruntime_providers_webgpu PROPERTIES CXX_STANDARD_REQUIRED ON) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 9ae3e79d86443..442b556d825cb 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1042,6 +1042,18 @@ function(onnxruntime_apply_test_target_workarounds target) endif() endfunction() +# Set environment variables for plugin EP tests when run via CTest. +function(onnxruntime_set_plugin_ep_test_environment target) + if(onnxruntime_USE_WEBGPU AND onnxruntime_USE_EP_API_ADAPTERS) + set(ORT_PLUGIN_EP_JSON_CONFIG "{\"ep_library_registration_name\": \"WebGPU_PluginEP\", \"ep_library_path\": \"onnxruntime_providers_webgpu.dll\", \"selected_ep_name\": \"WebGpuExecutionProvider\"}") + set_tests_properties(${target} PROPERTIES + ENVIRONMENT "ORT_UNIT_TEST_MAIN_DYNAMIC_PLUGIN_EP_CONFIG_JSON=${ORT_PLUGIN_EP_JSON_CONFIG}" + ) + # TODO: add for other plugin EPs if needed + # elseif() + endif() +endfunction() + function(onnxruntime_apply_emscripten_test_link_settings target) if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set_target_properties(${target} PROPERTIES LINK_DEPENDS ${TEST_SRC_DIR}/wasm/onnxruntime_test_adapter.js) @@ -1250,6 +1262,7 @@ block() ) onnxruntime_apply_test_target_workarounds(onnxruntime_provider_test) + onnxruntime_set_plugin_ep_test_environment(onnxruntime_provider_test) # Expose QNN SDK headers to unit tests via an interface target if(onnxruntime_USE_QNN) diff --git a/include/onnxruntime/ep/adapter/op_kernel_info.h b/include/onnxruntime/ep/adapter/op_kernel_info.h index bd6172a668e33..644cb30788ec6 100644 --- a/include/onnxruntime/ep/adapter/op_kernel_info.h +++ b/include/onnxruntime/ep/adapter/op_kernel_info.h @@ -19,7 +19,7 @@ #include "tensor_helper.h" namespace onnxruntime { -struct DataTransferManager; +class DataTransferManager; struct IExecutionProvider; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/upsamplebase.h b/onnxruntime/core/providers/cpu/tensor/upsamplebase.h index 7dcf88133e967..42b22b02a1d67 100644 --- a/onnxruntime/core/providers/cpu/tensor/upsamplebase.h +++ b/onnxruntime/core/providers/cpu/tensor/upsamplebase.h @@ -265,8 +265,22 @@ class UpsampleBase { if (scales_input_idx_ > 0) { const Tensor* scale; bool get_scale = info.TryGetConstantInput(scales_input_idx_, &scale); - auto x_shape = node.InputDefs()[0]->Shape(); - int64_t rank = x_shape ? x_shape->dim_size() : -1; + int64_t rank = -1; + if constexpr (std::is_same_v) { + auto x_shape = node.InputDefs()[0]->Shape(); + if (x_shape != nullptr) { + rank = x_shape->dim_size(); + } + } else { + int is_const; + auto tensor = info.GetKernelInfo().GetTensorConstantInput(0, &is_const); + if (is_const) { + auto type_and_shape_info = tensor.GetTensorTypeAndShapeInfo(); + if (type_and_shape_info.HasShape()) { + rank = static_cast(type_and_shape_info.GetShape().size()); + } + } + } if (get_scale && scale->Shape().Size() > 0 && ((opset < 18) || (rank > 0 && opset >= 18))) { ORT_THROW_IF_ERROR(ParseScalesData(scale, scales_, rank)); scales_cached_ = true; diff --git a/onnxruntime/core/providers/webgpu/compute_context.h b/onnxruntime/core/providers/webgpu/compute_context.h index 5277d64ad3611..38848e98509ba 100644 --- a/onnxruntime/core/providers/webgpu/compute_context.h +++ b/onnxruntime/core/providers/webgpu/compute_context.h @@ -107,7 +107,11 @@ class ComputeContextBase { // Get the logger. // inline const logging::Logger& Logger() const { +#if defined(ORT_USE_EP_API_ADAPTERS) + return ep_.GetEpLogger(); +#else return *ep_.GetLogger(); +#endif } // diff --git a/onnxruntime/core/providers/webgpu/controlflow/if.cc b/onnxruntime/core/providers/webgpu/controlflow/if.cc index 233d1f760383f..29b5e66d5075a 100644 --- a/onnxruntime/core/providers/webgpu/controlflow/if.cc +++ b/onnxruntime/core/providers/webgpu/controlflow/if.cc @@ -3,6 +3,10 @@ #include "core/providers/webgpu/controlflow/if.h" +#if defined(ORT_USE_EP_API_ADAPTERS) +#include "core/framework/error_code_helper.h" +#endif + using namespace ONNX_NAMESPACE; using namespace onnxruntime::common; @@ -68,10 +72,20 @@ ONNX_OPERATOR_KERNEL_EX(If, .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), If); +#if !defined(ORT_USE_EP_API_ADAPTERS) Status If::Compute(OpKernelContext* ctx) const { // call the base CPU version. return onnxruntime::If::Compute(ctx); } +#else +Status If::CreateControlFlowKernelImpl(const OrtKernelInfo* info, OrtKernelImpl** impl) { + return ToStatusAndRelease(ep::Api().ep.CreateIfKernel(info, impl)); +} + +Status If::Compute(OpKernelContext* /*ctx*/) const { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "If operator should be handled by ORT core."); +} +#endif } // namespace webgpu -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/controlflow/if.h b/onnxruntime/core/providers/webgpu/controlflow/if.h index 0755c5d33d7a3..0aa30e939eb1a 100644 --- a/onnxruntime/core/providers/webgpu/controlflow/if.h +++ b/onnxruntime/core/providers/webgpu/controlflow/if.h @@ -10,6 +10,8 @@ namespace onnxruntime { namespace webgpu { +#if !defined(ORT_USE_EP_API_ADAPTERS) + // Use the CPU implementation for the logic class If final : public onnxruntime::If { public: @@ -18,5 +20,16 @@ class If final : public onnxruntime::If { Status Compute(OpKernelContext* ctx) const override; }; +#else + +class If final : public OpKernel { + public: + If(const OpKernelInfo& info) : OpKernel(info) {} + + Status CreateControlFlowKernelImpl(const OrtKernelInfo* info, OrtKernelImpl** impl) override; + Status Compute(OpKernelContext* ctx) const override; +}; +#endif + } // namespace webgpu -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/data_transfer.cc b/onnxruntime/core/providers/webgpu/data_transfer.cc index 6d66a7308f1de..792e6cbc05d3f 100644 --- a/onnxruntime/core/providers/webgpu/data_transfer.cc +++ b/onnxruntime/core/providers/webgpu/data_transfer.cc @@ -13,32 +13,45 @@ bool DataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_dev (dst_device.Type() == OrtDevice::CPU && src_device.Type() == OrtDevice::GPU); } -common::Status DataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const { - size_t bytes = src.SizeInBytes(); +common::Status DataTransfer::CopyTensorImpl(void const* src_data, + bool src_is_gpu, + void* dst_data, + bool dst_is_gpu, + size_t bytes) const { if (bytes > 0) { - void const* src_data = src.DataRaw(); - void* dst_data = dst.MutableDataRaw(); - - auto& src_device = src.Location().device; - auto& dst_device = dst.Location().device; - - if (dst_device.Type() == OrtDevice::GPU) { - if (src_device.Type() == OrtDevice::GPU) { + if (dst_is_gpu) { + if (src_is_gpu) { // copy from GPU to GPU buffer_manager_.MemCpy(static_cast(const_cast(src_data)), - static_cast(dst_data), bytes); + static_cast(dst_data), + bytes); } else { // copy from CPU to GPU - buffer_manager_.Upload(const_cast(src_data), static_cast(dst_data), bytes); + buffer_manager_.Upload(const_cast(src_data), + static_cast(dst_data), + bytes); } - } else /* if (src_device.Type() == OrtDevice::GPU) */ { + } else { // copy from GPU to CPU - buffer_manager_.Download(static_cast(const_cast(src_data)), dst_data, bytes); + buffer_manager_.Download(static_cast(const_cast(src_data)), + dst_data, + bytes); } } return Status::OK(); } +common::Status DataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const { + void const* src_data = src.DataRaw(); + void* dst_data = dst.MutableDataRaw(); + + return CopyTensorImpl(src_data, + src.Location().device.Type() == OrtDevice::GPU, + dst_data, + dst.Location().device.Type() == OrtDevice::GPU, + src.SizeInBytes()); +} + } // namespace webgpu } // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/data_transfer.h b/onnxruntime/core/providers/webgpu/data_transfer.h index 0adf380149acf..29f70341989f7 100644 --- a/onnxruntime/core/providers/webgpu/data_transfer.h +++ b/onnxruntime/core/providers/webgpu/data_transfer.h @@ -20,6 +20,12 @@ class DataTransfer : public IDataTransfer { common::Status CopyTensor(const Tensor& src, Tensor& dst) const override; + common::Status CopyTensorImpl(void const* src_data, + bool src_is_gpu, + void* dst_data, + bool dst_is_gpu, + size_t bytes) const; + private: const BufferManager& buffer_manager_; }; diff --git a/onnxruntime/core/providers/webgpu/ep/api.cc b/onnxruntime/core/providers/webgpu/ep/api.cc new file mode 100644 index 0000000000000..24276fbaf02e7 --- /dev/null +++ b/onnxruntime/core/providers/webgpu/ep/api.cc @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#define ORT_API_MANUAL_INIT +#include "onnxruntime_cxx_api.h" +#undef ORT_API_MANUAL_INIT + +#include + +#include "core/providers/webgpu/ep/factory.h" + +// To make symbols visible on macOS/iOS +#ifdef __APPLE__ +#define EXPORT_SYMBOL __attribute__((visibility("default"))) +#else +#define EXPORT_SYMBOL +#endif + +namespace onnxruntime { +namespace webgpu { +void CleanupWebGpuContexts(); +void CleanupKernelRegistries(); +} // namespace webgpu +} // namespace onnxruntime + +namespace google { +namespace protobuf { +void ShutdownProtobufLibrary(); +} // namespace protobuf +} // namespace google + +extern "C" { +// +// Public symbols +// +EXPORT_SYMBOL OrtStatus* CreateEpFactories(const char* /*registration_name*/, const OrtApiBase* ort_api_base, + const OrtLogger* default_logger, + OrtEpFactory** factories, size_t max_factories, size_t* num_factories) { + // Manual init for the C++ API + onnxruntime::ep::ApiInit(ort_api_base); + + if (max_factories < 1) { + return onnxruntime::ep::Api().ort.CreateStatus(ORT_INVALID_ARGUMENT, + "Not enough space to return EP factory. Need at least one."); + } + + // Initialize the global default logger + ::onnxruntime::ep::adapter::Logger::CreateDefaultLogger(default_logger); + + // Factory could use registration_name or define its own EP name. + std::unique_ptr factory = std::make_unique(); + + factories[0] = factory.release(); + *num_factories = 1; + + return nullptr; +} + +EXPORT_SYMBOL OrtStatus* ReleaseEpFactory(OrtEpFactory* factory) { + // STEP.1 - Release the factory + delete static_cast(factory); + + // STEP.2 - Clean up cached kernel registries + onnxruntime::webgpu::CleanupKernelRegistries(); + + // STEP.3 - Clean up WebGPU contexts + onnxruntime::webgpu::CleanupWebGpuContexts(); + + // STEP.4 - Destroy the global default logger wrapper + ::onnxruntime::ep::adapter::Logger::DestroyDefaultLogger(); + + // STEP.5 - Shutdown protobuf library + google::protobuf::ShutdownProtobufLibrary(); + + return nullptr; +} + +} // extern "C" diff --git a/onnxruntime/core/providers/webgpu/ep/ep.cc b/onnxruntime/core/providers/webgpu/ep/ep.cc new file mode 100644 index 0000000000000..308f4c001c50b --- /dev/null +++ b/onnxruntime/core/providers/webgpu/ep/ep.cc @@ -0,0 +1,263 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "ep.h" + +#include "factory.h" + +#include "core/framework/run_options.h" +#include "core/framework/kernel_registry.h" +#include "core/session/onnxruntime_run_options_config_keys.h" +#include "core/session/plugin_ep/ep_kernel_registration.h" +#include "core/providers/webgpu/webgpu_execution_provider.h" + +#include "ep/get_capability_utils.h" + +namespace onnxruntime { +namespace webgpu { +namespace ep { + +using onnxruntime::ep::Api; + +// Constructor +Ep::Ep(std::unique_ptr impl, Factory& factory, const OrtLogger& logger, const Config& config) + : onnxruntime::ep::adapter::Ep{std::move(impl), config.cpu_allocator, config.device_allocator}, + factory_{factory}, + logger_{logger}, + config_{config} { + ort_version_supported = ORT_API_VERSION; + + // Initialize the execution provider's function table + GetName = GetNameImpl; + GetCapability = GetCapabilityImpl; + GetKernelRegistry = GetKernelRegistryImpl; + Compile = nullptr; // Per-kernel EP does not use Compile + ReleaseNodeComputeInfos = nullptr; + GetPreferredDataLayout = GetPreferredDataLayoutImpl; + ShouldConvertDataLayoutForOp = ShouldConvertDataLayoutForOpImpl; + SetDynamicOptions = nullptr; // Not implemented + OnRunStart = OnRunStartImpl; + OnRunEnd = OnRunEndImpl; + CreateAllocator = CreateAllocatorImpl; + CreateSyncStreamForDevice = nullptr; // Not stream aware + GetCompiledModelCompatibilityInfo = nullptr; // Not a compiled EP + IsConcurrentRunSupported = IsConcurrentRunSupportedImpl; +} + +// OrtEp interface implementations +const char* ORT_API_CALL Ep::GetNameImpl(const OrtEp* this_ptr) noexcept { + const auto* ep = static_cast(this_ptr); + return ep->factory_.GetName(&ep->factory_); +} + +OrtStatus* ORT_API_CALL Ep::GetCapabilityImpl(OrtEp* this_ptr, const OrtGraph* graph, + OrtEpGraphSupportInfo* graph_support_info) noexcept { + try { + auto& ep = *static_cast(static_cast(this_ptr)->EpImpl()); + Ort::ConstGraph ort_graph{graph}; + + // Get all nodes in the graph + std::vector all_nodes = ort_graph.GetNodes(); + + if (all_nodes.empty()) { + return nullptr; // No nodes to process + } + + std::vector candidate_nodes; + + // For each node, check if we have a registered kernel for it + for (const auto& node : all_nodes) { + if (node.GetEpName() == kWebGpuExecutionProvider) { + candidate_nodes.push_back(node); + continue; + } + + const OrtKernelDef* kernel_def = nullptr; + RETURN_IF_ERROR(Api().ep.EpGraphSupportInfo_LookUpKernel(graph_support_info, node, &kernel_def)); + + if (kernel_def == nullptr) { + LOGS(ep.GetEpLogger(), INFO) << "webgpu kernel not found in registries for Op type: " + << node.GetOperatorType() << " node name: " << node.GetName(); + continue; + } + + auto cpu_node_names = ep.GetForceCpuNodeNames(); + if (std::find(cpu_node_names.begin(), + cpu_node_names.end(), + node.GetName()) != cpu_node_names.end()) { + LOGS(ep.GetEpLogger(), INFO) << "Force CPU execution for node: " << node.GetName(); + continue; + } + + // + // The following code checks if the node is really supported by webgpu EP + // + +#define FALLBACK_TO_CPU_IF_EXIST_INPUT(idx) \ + if (inputs.size() > idx && inputs[idx] != nullptr) { \ + continue; \ + } + +#define FALLBACK_TO_CPU_IF_EXIST_OUTPUT(idx) \ + if (outputs.size() > idx && outputs[idx] != nullptr) { \ + continue; \ + } + + // Check for Attention + if (node.GetOperatorType() == "Attention" && node.GetDomain() == kMSDomain) { + const auto& inputs = node.GetInputs(); + const auto& outputs = node.GetOutputs(); + + // Current implementation does not support mask_index(input[3]), past(input[4]) and past_seq_len(input[6]) + FALLBACK_TO_CPU_IF_EXIST_INPUT(3); + FALLBACK_TO_CPU_IF_EXIST_INPUT(4); + FALLBACK_TO_CPU_IF_EXIST_INPUT(6); + + // Current implementation does not support present(output[1]) + FALLBACK_TO_CPU_IF_EXIST_OUTPUT(1); + + // If attribute past_present_share_buffer is set, fallback to CPU + bool has_past_present_share_buffer = false; + for (const auto& attr : node.GetAttributes()) { + if (attr.GetName() == "past_present_share_buffer") { + int64_t val = 0; + RETURN_IF_ERROR(attr.GetValue(val)); + if (val != 0) { + has_past_present_share_buffer = true; + } + } + } + if (has_past_present_share_buffer) { + continue; + } + } + + candidate_nodes.push_back(node); + } + + std::unordered_set cpu_preferred_nodes; + RETURN_IF_ERROR(onnxruntime::ep::GetCpuPreferredNodes(*ort_graph, + *graph_support_info, + static_cast(this_ptr)->GetOrtLogger(), + candidate_nodes, + cpu_preferred_nodes)); + + for (const auto& node : candidate_nodes) { + if (cpu_preferred_nodes.count(node) == 0) { + RETURN_IF_ERROR(Api().ep.EpGraphSupportInfo_AddSingleNode(graph_support_info, node)); + } + } + } catch (const Ort::Exception& ex) { + Ort::Status status(ex); + return status.release(); + } catch (const std::exception& ex) { + Ort::Status status(ex.what(), ORT_EP_FAIL); + return status.release(); + } + + return nullptr; +} + +OrtStatus* ORT_API_CALL Ep::GetKernelRegistryImpl( + _In_ OrtEp* this_ptr, + _Outptr_result_maybenull_ const OrtKernelRegistry** kernel_registry) noexcept { + try { + *kernel_registry = nullptr; + + // For the WebGPU EP, delegate to the CreateKernelRegistry function + // which properly constructs a registry using only public APIs + auto* ep = static_cast(this_ptr); + + auto& webgpu_ep = *static_cast(ep->EpImpl()); + + *kernel_registry = *webgpu_ep.GetKernelRegistryImpl(); + return nullptr; + } catch (const Ort::Exception& ex) { + Ort::Status status(ex); + return status.release(); + } catch (const std::exception& ex) { + Ort::Status status(ex.what(), ORT_EP_FAIL); + return status.release(); + } +} + +OrtStatus* ORT_API_CALL Ep::GetPreferredDataLayoutImpl(_In_ OrtEp* this_ptr, + _Out_ OrtEpDataLayout* preferred_data_layout) noexcept { + // Delegate to the underlying WebGPU EP's GetPreferredLayout() + // DataLayout enum values map 1:1 to OrtEpDataLayout (NCHW=0, NHWC=1) + auto* ep = static_cast(this_ptr); + *preferred_data_layout = static_cast(ep->EpImpl()->GetPreferredLayout()); + return nullptr; +} + +OrtStatus* ORT_API_CALL Ep::ShouldConvertDataLayoutForOpImpl(_In_ OrtEp* this_ptr, + _In_z_ const char* domain, + _In_z_ const char* op_type, + _In_ OrtEpDataLayout target_data_layout, + _Outptr_ int* should_convert) noexcept { + // DataLayout enum values map 1:1 to OrtEpDataLayout (NCHW=0, NHWC=1) + auto* ep = static_cast(this_ptr); + auto result = ep->EpImpl()->ShouldConvertDataLayoutForOp(domain, op_type, + static_cast(target_data_layout)); + if (result.has_value()) { + *should_convert = result.value() ? 1 : 0; + } else { + *should_convert = -1; + } + return nullptr; +} + +OrtStatus* ORT_API_CALL Ep::OnRunStartImpl(_In_ OrtEp* this_ptr, + _In_ const OrtRunOptions* run_options) noexcept { + onnxruntime::RunOptions options{}; + // currently only option "gpu_graph_id" is used + auto graph_annotation_str = Api().ort.GetRunConfigEntry(run_options, kOrtRunOptionsConfigCudaGraphAnnotation); + if (graph_annotation_str != nullptr) { + auto status = options.config_options.AddConfigEntry(kOrtRunOptionsConfigCudaGraphAnnotation, graph_annotation_str); + if (!status.IsOK()) { + return Api().ort.CreateStatus(static_cast(status.Code()), + status.ErrorMessage().c_str()); + } + } + auto* ep = static_cast(this_ptr); + auto status = ep->EpImpl()->OnRunStart(options); + if (!status.IsOK()) { + return Api().ort.CreateStatus(static_cast(status.Code()), + status.ErrorMessage().c_str()); + } + return nullptr; +} + +OrtStatus* ORT_API_CALL Ep::OnRunEndImpl(_In_ OrtEp* this_ptr, + _In_ const OrtRunOptions* /*run_options*/, + _In_ bool sync_stream) noexcept { + auto* ep = static_cast(this_ptr); + auto status = ep->EpImpl()->OnRunEnd(sync_stream, {}); + if (!status.IsOK()) { + return Api().ort.CreateStatus(static_cast(status.Code()), + status.ErrorMessage().c_str()); + } + return nullptr; +} + +OrtStatus* ORT_API_CALL Ep::IsConcurrentRunSupportedImpl(_In_ OrtEp* /*this_ptr*/, _Out_ bool* is_concurrent_run_supported) noexcept { + *is_concurrent_run_supported = false; + return nullptr; +} + +OrtStatus* ORT_API_CALL Ep::CreateAllocatorImpl(_In_ OrtEp* this_ptr, + _In_ const OrtMemoryInfo* memory_info, + _Outptr_result_maybenull_ OrtAllocator** allocator) noexcept { + auto* ep = static_cast(this_ptr); + Ort::ConstMemoryInfo ort_memory_info{memory_info}; + if (ort_memory_info.GetAllocatorType() == OrtReadOnlyAllocator) { + *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, ep->config_.initializer_allocator); + } else { + *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, ep->config_.device_allocator); + } + return nullptr; +} + +} // namespace ep +} // namespace webgpu +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/ep/ep.h b/onnxruntime/core/providers/webgpu/ep/ep.h new file mode 100644 index 0000000000000..c867cea1ec5be --- /dev/null +++ b/onnxruntime/core/providers/webgpu/ep/ep.h @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "core/providers/webgpu/webgpu_execution_provider.h" + +namespace onnxruntime { +namespace webgpu { +namespace ep { + +class Factory; + +/// +/// A bridge class between the EP API and the WebGPU EP implementation. +/// +class Ep : public onnxruntime::ep::adapter::Ep { + public: + struct Config { + AllocatorPtr cpu_allocator; + AllocatorPtr device_allocator; + AllocatorPtr initializer_allocator; + }; + + // Do not use a std::unique_ptr for impl_ because this requires the actual type definition. + Ep(std::unique_ptr impl, Factory& factory, const OrtLogger& logger, const Config& config); + + inline const OrtLogger& GetOrtLogger() const noexcept { + return logger_; + } + + private: + static const char* ORT_API_CALL GetNameImpl(const OrtEp* this_ptr) noexcept; + + static OrtStatus* ORT_API_CALL GetCapabilityImpl(OrtEp* this_ptr, const OrtGraph* graph, + OrtEpGraphSupportInfo* graph_support_info) noexcept; + + static OrtStatus* ORT_API_CALL GetKernelRegistryImpl( + _In_ OrtEp* this_ptr, + _Outptr_result_maybenull_ const OrtKernelRegistry** kernel_registry) noexcept; + + static OrtStatus* ORT_API_CALL CreateAllocatorImpl(_In_ OrtEp* this_ptr, + _In_ const OrtMemoryInfo* memory_info, + _Outptr_result_maybenull_ OrtAllocator** allocator) noexcept; + + static OrtStatus* ORT_API_CALL GetPreferredDataLayoutImpl(_In_ OrtEp* this_ptr, + _Out_ OrtEpDataLayout* preferred_data_layout) noexcept; + + static OrtStatus* ORT_API_CALL ShouldConvertDataLayoutForOpImpl(_In_ OrtEp* this_ptr, + _In_z_ const char* domain, + _In_z_ const char* op_type, + _In_ OrtEpDataLayout target_data_layout, + _Outptr_ int* should_convert) noexcept; + + static OrtStatus* ORT_API_CALL OnRunStartImpl(_In_ OrtEp* this_ptr, + _In_ const OrtRunOptions* run_options) noexcept; + + static OrtStatus* ORT_API_CALL OnRunEndImpl(_In_ OrtEp* this_ptr, + _In_ const OrtRunOptions* run_options, + _In_ bool sync_stream) noexcept; + + static OrtStatus* ORT_API_CALL IsConcurrentRunSupportedImpl(_In_ OrtEp* this_ptr, + _Out_ bool* is_concurrent_run_supported) noexcept; + + Factory& factory_; + const OrtLogger& logger_; + Config config_{}; +}; + +} // namespace ep +} // namespace webgpu +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/ep/factory.cc b/onnxruntime/core/providers/webgpu/ep/factory.cc new file mode 100644 index 0000000000000..737b64d47b2bb --- /dev/null +++ b/onnxruntime/core/providers/webgpu/ep/factory.cc @@ -0,0 +1,213 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "factory.h" +#include "ep.h" + +#include "core/framework/error_code_helper.h" +#include "core/graph/constants.h" + +#include "core/framework/execution_provider.h" +#include "core/framework/config_options.h" +#include "core/providers/webgpu/webgpu_provider_factory_creator.h" +#include "core/providers/webgpu/webgpu_context.h" +#include "core/providers/webgpu/allocator.h" + +namespace onnxruntime { +namespace webgpu { +namespace ep { + +using onnxruntime::ep::Api; + +// Constructor +Factory::Factory() : OrtEpFactory{}, + config_{ + CPUAllocator::DefaultInstance(), // CPU allocator + std::make_shared(WebGpuContextFactory::DefaultContext().BufferManager(), false), // default device allocator + std::make_shared(WebGpuContextFactory::DefaultContext().InitializerBufferManager(), true), // initializer device allocator + }, + default_memory_info_{WEBGPU_BUFFER, OrtMemoryInfoDeviceType_GPU, + 0, // vendor id + 0, // device id + OrtDeviceMemoryType_DEFAULT, + 0, // alignment + OrtDeviceAllocator}, + readonly_memory_info_{WEBGPU_BUFFER, OrtMemoryInfoDeviceType_GPU, + 0, // vendor id + 0, // device id + OrtDeviceMemoryType_DEFAULT, + 0, // alignment + OrtReadOnlyAllocator} { + ort_version_supported = ORT_API_VERSION; + + GetName = GetNameImpl; + GetVendor = GetVendorImpl; + GetVendorId = GetVendorIdImpl; + GetVersion = GetVersionImpl; + + GetSupportedDevices = GetSupportedDevicesImpl; + CreateEp = CreateEpImpl; + ReleaseEp = ReleaseEpImpl; + + CreateAllocator = CreateAllocatorImpl; // TODO + ReleaseAllocator = ReleaseAllocatorImpl; // TODO + CreateDataTransfer = CreateDataTransferImpl; // TODO +} + +// Static C API implementations + +const char* ORT_API_CALL Factory::GetNameImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return kWebGpuExecutionProvider; +} + +const char* ORT_API_CALL Factory::GetVendorImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return "Microsoft"; +} + +uint32_t ORT_API_CALL Factory::GetVendorIdImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return 0; +} + +const char* ORT_API_CALL Factory::GetVersionImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return "0.1.0"; +} + +OrtStatus* ORT_API_CALL Factory::GetSupportedDevicesImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* const* devices, + size_t num_devices, + OrtEpDevice** ep_devices, + size_t max_ep_devices, + size_t* p_num_ep_devices) noexcept { + auto factory = static_cast(this_ptr); + + size_t& num_ep_devices = *p_num_ep_devices; + num_ep_devices = 0; + + for (size_t i = 0; i < num_devices && num_ep_devices < max_ep_devices; ++i) { + const OrtHardwareDevice& device = *devices[i]; + if (Api().ort.HardwareDevice_Type(&device) == OrtHardwareDeviceType::OrtHardwareDeviceType_GPU) { + // TODO: any metadata or options to add? + OrtEpDevice* ep_device = nullptr; + ORT_API_RETURN_IF_ERROR(Api().ep.CreateEpDevice(this_ptr, + &device, nullptr, nullptr, + &ep_device)); + ORT_API_RETURN_IF_ERROR(Api().ep.EpDevice_AddAllocatorInfo(ep_device, factory->default_memory_info_)); + ORT_API_RETURN_IF_ERROR(Api().ep.EpDevice_AddAllocatorInfo(ep_device, factory->readonly_memory_info_)); + ep_devices[num_ep_devices++] = ep_device; + } + } + + return nullptr; +} + +OrtStatus* ORT_API_CALL Factory::CreateEpImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* const* /*devices*/, + const OrtKeyValuePairs* const* /*ep_metadata*/, + size_t num_devices, + const OrtSessionOptions* session_options, + const OrtLogger* logger, + OrtEp** ep) noexcept { + if (num_devices == 0) { + return Api().ort.CreateStatus(ORT_INVALID_ARGUMENT, "No hardware devices provided to create WebGPU EP."); + } + + OrtKeyValuePairs* session_config_entries = nullptr; + ORT_API_RETURN_IF_ERROR(Api().ort.GetSessionOptionsConfigEntries(session_options, &session_config_entries)); + Ort::KeyValuePairs session_config_entries_holder(session_config_entries); // allow automatic release + + auto config_options = ConfigOptions{}; + const char* const* keys = nullptr; + const char* const* values = nullptr; + size_t num_entries = 0; + Api().ort.GetKeyValuePairs(session_config_entries, &keys, &values, &num_entries); + for (size_t i = 0; i < num_entries; ++i) { + auto status = config_options.AddConfigEntry(keys[i], values[i]); + if (!status.IsOK()) { + return Api().ort.CreateStatus((OrtErrorCode)status.Code(), status.ErrorMessage().c_str()); + } + } + + try { + auto webgpu_ep_factory = WebGpuProviderFactoryCreator::Create(config_options); + auto webgpu_ep = webgpu_ep_factory->CreateProvider(*session_options, *logger); + static_cast(webgpu_ep.get())->SetEpLogger(logger); + auto factory = static_cast(this_ptr); + *ep = new Ep(std::move(webgpu_ep), *factory, *logger, factory->config_); + return nullptr; + } catch (const Ort::Exception& ex) { + Ort::Status status(ex); + return status.release(); + } +} + +void ORT_API_CALL Factory::ReleaseEpImpl(OrtEpFactory* /*this_ptr*/, OrtEp* ep) noexcept { + delete static_cast(ep); +} + +OrtStatus* ORT_API_CALL Factory::CreateAllocatorImpl( + OrtEpFactory* this_ptr, + const OrtMemoryInfo* memory_info, + const OrtKeyValuePairs* /*allocator_options*/, + OrtAllocator** allocator) noexcept { + auto factory = static_cast(this_ptr); + Ort::ConstMemoryInfo ort_memory_info{memory_info}; + if (ort_memory_info.GetAllocatorType() == OrtReadOnlyAllocator) { + *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, factory->config_.initializer_allocator); + } else { + *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, factory->config_.device_allocator); + } + return nullptr; +} + +void ORT_API_CALL Factory::ReleaseAllocatorImpl(OrtEpFactory* /*this_ptr*/, OrtAllocator* allocator) noexcept { + onnxruntime::ep::adapter::Allocator* ptr = static_cast(allocator); + delete ptr; +} + +OrtStatus* ORT_API_CALL Factory::CreateDataTransferImpl( + OrtEpFactory* /*this_ptr*/, + OrtDataTransferImpl** data_transfer) noexcept { + try { + *data_transfer = OrtWebGpuCreateDataTransfer(); // TODO(fs-eire): pass context id if needed + return nullptr; + } catch (const Ort::Exception& ex) { + Ort::Status status(ex); + return status.release(); + } +} + +bool ORT_API_CALL Factory::IsStreamAwareImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return false; // Default: not stream aware +} + +OrtStatus* ORT_API_CALL Factory::CreateSyncStreamForDeviceImpl( + OrtEpFactory* /*this_ptr*/, + const OrtMemoryDevice* /*memory_device*/, + const OrtKeyValuePairs* /*stream_options*/, + OrtSyncStreamImpl** stream) noexcept { + *stream = nullptr; + return Api().ort.CreateStatus(ORT_NOT_IMPLEMENTED, + "CreateSyncStreamForDevice is not implemented for this EP factory."); +} + +OrtStatus* ORT_API_CALL Factory::ValidateCompiledModelCompatibilityInfoImpl( + OrtEpFactory* /*this_ptr*/, + const OrtHardwareDevice* const* /*devices*/, + size_t /*num_devices*/, + const char* /*compatibility_info*/, + OrtCompiledModelCompatibility* model_compatibility) noexcept { + *model_compatibility = OrtCompiledModelCompatibility_EP_NOT_APPLICABLE; + return nullptr; +} + +OrtStatus* ORT_API_CALL Factory::SetEnvironmentOptionsImpl( + OrtEpFactory* /*this_ptr*/, + const OrtKeyValuePairs* /*options*/) noexcept { + return nullptr; // Default implementation does nothing +} + +} // namespace ep +} // namespace webgpu +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/ep/factory.h b/onnxruntime/core/providers/webgpu/ep/factory.h new file mode 100644 index 0000000000000..cd886e19b56d7 --- /dev/null +++ b/onnxruntime/core/providers/webgpu/ep/factory.h @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include "ep.h" + +namespace onnxruntime { +namespace webgpu { +namespace ep { + +/// +/// A bridge class between the EP API and the WebGPU EP Factory implementation. +/// +class Factory : public OrtEpFactory { + private: + // Static C API implementations + static const char* ORT_API_CALL GetNameImpl(const OrtEpFactory* this_ptr) noexcept; + static const char* ORT_API_CALL GetVendorImpl(const OrtEpFactory* this_ptr) noexcept; + static uint32_t ORT_API_CALL GetVendorIdImpl(const OrtEpFactory* this_ptr) noexcept; + static const char* ORT_API_CALL GetVersionImpl(const OrtEpFactory* this_ptr) noexcept; + + static OrtStatus* ORT_API_CALL GetSupportedDevicesImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* const* devices, + size_t num_devices, + OrtEpDevice** ep_devices, + size_t max_ep_devices, + size_t* p_num_ep_devices) noexcept; + + static OrtStatus* ORT_API_CALL CreateEpImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* const* devices, + const OrtKeyValuePairs* const* ep_metadata, + size_t num_devices, + const OrtSessionOptions* session_options, + const OrtLogger* logger, + OrtEp** ep) noexcept; + + static void ORT_API_CALL ReleaseEpImpl(OrtEpFactory* this_ptr, OrtEp* ep) noexcept; + + static OrtStatus* ORT_API_CALL CreateAllocatorImpl( + OrtEpFactory* this_ptr, + const OrtMemoryInfo* memory_info, + const OrtKeyValuePairs* allocator_options, + OrtAllocator** allocator) noexcept; + + static void ORT_API_CALL ReleaseAllocatorImpl(OrtEpFactory* this_ptr, OrtAllocator* allocator) noexcept; + + static OrtStatus* ORT_API_CALL CreateDataTransferImpl( + OrtEpFactory* this_ptr, + OrtDataTransferImpl** data_transfer) noexcept; + + static bool ORT_API_CALL IsStreamAwareImpl(const OrtEpFactory* this_ptr) noexcept; + + static OrtStatus* ORT_API_CALL CreateSyncStreamForDeviceImpl( + OrtEpFactory* this_ptr, + const OrtMemoryDevice* memory_device, + const OrtKeyValuePairs* stream_options, + OrtSyncStreamImpl** stream) noexcept; + + static OrtStatus* ORT_API_CALL ValidateCompiledModelCompatibilityInfoImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* const* devices, + size_t num_devices, + const char* compatibility_info, + OrtCompiledModelCompatibility* model_compatibility) noexcept; + + static OrtStatus* ORT_API_CALL SetEnvironmentOptionsImpl( + OrtEpFactory* this_ptr, + const OrtKeyValuePairs* options) noexcept; + + Ep::Config config_; + Ort::MemoryInfo default_memory_info_; + Ort::MemoryInfo readonly_memory_info_; // used for initializers + + public: + Factory(); + ~Factory() = default; +}; + +} // namespace ep +} // namespace webgpu +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/generator/range.cc b/onnxruntime/core/providers/webgpu/generator/range.cc index 3fa062f327ba2..a84660a020fed 100644 --- a/onnxruntime/core/providers/webgpu/generator/range.cc +++ b/onnxruntime/core/providers/webgpu/generator/range.cc @@ -79,36 +79,40 @@ template class Range; template class Range; template class Range; -void RegisterRangeKernels(KernelRegistry& kernel_registry, bool enable_int64) { - // Helper lambda to create kernel - auto create_range_kernel_info = [](auto type_tag) { - using T = decltype(type_tag); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { - out = std::make_unique>(info); - return Status::OK(); - }; - - return KernelCreateInfo( - KernelDefBuilder() - .SetName("Range") - .SetDomain(kOnnxDomain) - .SinceVersion(11) - .Provider(kWebGpuExecutionProvider) - .TypeConstraint("T", DataTypeImpl::GetTensorType()) - .InputMemoryType(OrtMemTypeCPU, 0) - .InputMemoryType(OrtMemTypeCPU, 1) - .InputMemoryType(OrtMemTypeCPU, 2) - .Build(), - kernel_create_fn); - }; +namespace { +template +Status CreateRangeKernel(FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) { + out = std::make_unique>(info); + return Status::OK(); +} + +template +KernelCreateInfo CreateRangeKernelInfo() { + return KernelCreateInfo( + KernelDefBuilder() + .SetName("Range") + .SetDomain(kOnnxDomain) + .SinceVersion(11) + .Provider(kWebGpuExecutionProvider) + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .InputMemoryType(OrtMemTypeCPU, 0) + .InputMemoryType(OrtMemTypeCPU, 1) + .InputMemoryType(OrtMemTypeCPU, 2) + .Build(), + CreateRangeKernel); +} + +} // namespace + +void RegisterRangeKernels(KernelRegistry& kernel_registry, bool enable_int64) { // Always register float and int32_t - ORT_THROW_IF_ERROR(kernel_registry.Register(create_range_kernel_info(float{}))); - ORT_THROW_IF_ERROR(kernel_registry.Register(create_range_kernel_info(int32_t{}))); + ORT_THROW_IF_ERROR(kernel_registry.Register(CreateRangeKernelInfo())); + ORT_THROW_IF_ERROR(kernel_registry.Register(CreateRangeKernelInfo())); // Register int64_t only if int64 support is enabled if (enable_int64) { - ORT_THROW_IF_ERROR(kernel_registry.Register(create_range_kernel_info(int64_t{}))); + ORT_THROW_IF_ERROR(kernel_registry.Register(CreateRangeKernelInfo())); } } diff --git a/onnxruntime/core/providers/webgpu/tensor/cast.cc b/onnxruntime/core/providers/webgpu/tensor/cast.cc index 6bb0f688bfdb7..2695c2800d37a 100644 --- a/onnxruntime/core/providers/webgpu/tensor/cast.cc +++ b/onnxruntime/core/providers/webgpu/tensor/cast.cc @@ -90,7 +90,7 @@ template KernelCreateInfo CreateCastKernelInfo(bool enable_int64) { const auto& type_constraints = GetOpTypeConstraints(enable_int64, true); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { + KernelCreatePtrFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { out = std::make_unique(info); return Status::OK(); }; diff --git a/onnxruntime/core/providers/webgpu/tensor/concat.cc b/onnxruntime/core/providers/webgpu/tensor/concat.cc index 75453b991a0cd..c6178d44dba75 100644 --- a/onnxruntime/core/providers/webgpu/tensor/concat.cc +++ b/onnxruntime/core/providers/webgpu/tensor/concat.cc @@ -98,7 +98,7 @@ Status Concat::ComputeInternal(ComputeContext& context) const { } Prepare prepare; - ORT_RETURN_IF_ERROR(PrepareForCompute(&context.KernelContext(), input_tensors, prepare)); + ORT_RETURN_IF_ERROR(PrepareForComputeImpl(&context.KernelContext(), input_tensors, prepare)); if (prepare.output_num_elements == 0) { return Status::OK(); } diff --git a/onnxruntime/core/providers/webgpu/tensor/expand.cc b/onnxruntime/core/providers/webgpu/tensor/expand.cc index 0dacd589cbba8..0d39b1ec9d35e 100644 --- a/onnxruntime/core/providers/webgpu/tensor/expand.cc +++ b/onnxruntime/core/providers/webgpu/tensor/expand.cc @@ -108,7 +108,7 @@ template KernelCreateInfo CreateExpandVersionedKernelInfo(bool enable_int64) { const auto& type_constraints = GetOpTypeConstraints(enable_int64, true); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { + KernelCreatePtrFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { out = std::make_unique(info); return Status::OK(); }; @@ -129,7 +129,7 @@ template KernelCreateInfo CreateExpandKernelInfo(bool enable_int64) { const auto& type_constraints = GetOpTypeConstraints(enable_int64, true); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { + KernelCreatePtrFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { out = std::make_unique(info); return Status::OK(); }; diff --git a/onnxruntime/core/providers/webgpu/tensor/gather.cc b/onnxruntime/core/providers/webgpu/tensor/gather.cc index b3e5c7b4e8310..970c2d6bed7a3 100644 --- a/onnxruntime/core/providers/webgpu/tensor/gather.cc +++ b/onnxruntime/core/providers/webgpu/tensor/gather.cc @@ -60,7 +60,7 @@ Status GatherProgram::GenerateShaderCode(ShaderHelper& shader) const { Status Gather::ComputeInternal(ComputeContext& context) const { Prepare p; - ORT_RETURN_IF_ERROR(PrepareForCompute(&context.KernelContext(), p)); + ORT_RETURN_IF_ERROR(PrepareForComputeImpl(&context.KernelContext(), p)); uint32_t data_size = onnxruntime::narrow(p.output_tensor->Shape().Size()); if (data_size == 0) { return Status::OK(); diff --git a/onnxruntime/core/providers/webgpu/tensor/pad.cc b/onnxruntime/core/providers/webgpu/tensor/pad.cc index 0e77ec46bbddb..7a576c4b53ecf 100644 --- a/onnxruntime/core/providers/webgpu/tensor/pad.cc +++ b/onnxruntime/core/providers/webgpu/tensor/pad.cc @@ -49,7 +49,7 @@ Status Pad::ComputeInternal(ComputeContext& context) const { const auto pads_data = pads_tensor->DataAsSpan(); // Compute Pads by applying axes if specified otherwise copy the supplied pads. - PadBase::ComputePads(context.KernelContext(), data_rank, pads_data, pads); + PadBase::ComputePadsImpl(context.KernelContext(), data_rank, pads_data, pads); // Separate out any negative pads into the slices array PadBase::SeparateNegativeToSlices(pads, slices); diff --git a/onnxruntime/core/providers/webgpu/tensor/shape_op.cc b/onnxruntime/core/providers/webgpu/tensor/shape_op.cc index b211d48dab1c9..e5e6e4d61eed5 100644 --- a/onnxruntime/core/providers/webgpu/tensor/shape_op.cc +++ b/onnxruntime/core/providers/webgpu/tensor/shape_op.cc @@ -3,11 +3,74 @@ #include "core/providers/webgpu/webgpu_kernel.h" #include "core/providers/webgpu/webgpu_supported_types.h" -#include "core/providers/cpu/tensor/shape_op.h" +// #include "core/providers/cpu/tensor/shape_op.h" namespace onnxruntime { namespace webgpu { +#ifndef SHARED_PROVIDER +#include "core/common/common.h" +#include "core/common/narrow.h" +#include "core/framework/op_kernel.h" +#endif + +#include +#include + +class Shape final : public OpKernel { + public: + Shape(const OpKernelInfo& info) : OpKernel(info) { + info.GetAttrOrDefault("start", &start_index_, 0); + + if (start_index_ != 0) { + // "start" is provided and is non-default (default is 0) + needs_slicing_ = true; + } + + if (info.GetAttr("end", &end_index_).IsOK()) { + needs_slicing_ = true; + } + } + + // Takes a tensor as input and outputs an 1D int64 tensor + // containing the shape of the input tensor. + Status Compute(OpKernelContext* context) const override { + const auto* input = context->Input(0); + const TensorShape& input_shape = input->Shape(); + + int64_t rank = gsl::narrow_cast(input_shape.NumDimensions()); + + if (!needs_slicing_) { // vanilla use of Shape (no slicing) + Tensor* output = context->Output(0, {rank}); + input_shape.CopyDims(output->MutableData(), static_cast(rank)); + } else { // slicing is needed + int64_t true_start = start_index_; + int64_t true_end = end_index_; + + // Deal with negative(s) and clamp + true_start = true_start < 0 ? true_start + rank : true_start; + true_start = true_start < 0 ? 0 : ((true_start > rank) ? rank : true_start); + + true_end = true_end < 0 ? true_end + rank : true_end; + true_end = true_end < 0 ? 0 : ((true_end > rank) ? rank : true_end); + + auto slice_length = true_end - true_start; + Tensor* output = context->Output(0, {slice_length < 0 ? 0 : slice_length}); + + if (slice_length > 0) { + input_shape.CopyDims(output->MutableData(), onnxruntime::narrow(true_start), onnxruntime::narrow(slice_length)); + } + } + + return Status::OK(); + } + + private: + bool needs_slicing_ = false; + int64_t start_index_ = 0; + int64_t end_index_ = std::numeric_limits::max(); +}; + ONNX_OPERATOR_VERSIONED_KERNEL_EX( Shape, kOnnxDomain, diff --git a/onnxruntime/core/providers/webgpu/tensor/unsqueeze.cc b/onnxruntime/core/providers/webgpu/tensor/unsqueeze.cc index 104fcf1812af8..3337448564bed 100644 --- a/onnxruntime/core/providers/webgpu/tensor/unsqueeze.cc +++ b/onnxruntime/core/providers/webgpu/tensor/unsqueeze.cc @@ -12,7 +12,7 @@ template KernelCreateInfo CreateUnsqueezeVersionedKernelInfo(bool enable_int64) { const auto& type_constraints = GetOpTypeConstraints(enable_int64, true); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { + KernelCreatePtrFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { out = std::make_unique(info); return Status::OK(); }; @@ -47,7 +47,7 @@ template KernelCreateInfo CreateUnsqueezeKernelInfo(bool enable_int64) { const auto& type_constraints = GetOpTypeConstraints(enable_int64, true); - KernelCreateFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { + KernelCreatePtrFn kernel_create_fn = [](FuncManager&, const OpKernelInfo& info, std::unique_ptr& out) -> Status { out = std::make_unique(info); return Status::OK(); }; diff --git a/onnxruntime/core/providers/webgpu/tensor/upsample.cc b/onnxruntime/core/providers/webgpu/tensor/upsample.cc index fb406883ba4ba..8f51ed45004bf 100644 --- a/onnxruntime/core/providers/webgpu/tensor/upsample.cc +++ b/onnxruntime/core/providers/webgpu/tensor/upsample.cc @@ -90,7 +90,7 @@ Status Upsample::ComputeInternal(ComputeContext& context) const { InlinedVector scales_array(input_dims.size()); // opset < 10 - if (OpKernel::Node().InputDefs().size() == 1) { + if (OpKernel::Node().SinceVersion() < 10) { scales_array = scales_; // Compute output shape from scales attributes and input dims ComputeOutputShape(scales_array, input_dims, output_dims); diff --git a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc index c6255e6f352d9..b79454ff2769f 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc +++ b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc @@ -445,8 +445,8 @@ class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kWebGpuExecutionProvider, kOnnxD class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kWebGpuExecutionProvider, kOnnxDomain, 16, 17, ScatterElements); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kWebGpuExecutionProvider, kOnnxDomain, 18, ScatterElements); -std::unique_ptr RegisterKernels(bool enable_graph_capture = false, bool enable_int64 = false) { - auto kernel_registry = std::make_unique(); +std::unique_ptr RegisterKernels(bool enable_graph_capture, bool enable_int64) { + auto kernel_registry = std::make_unique(); static const BuildKernelCreateInfoFn function_table[] = { BuildKernelCreateInfo, // default entry to avoid the list becoming empty after ops-reducing @@ -837,6 +837,63 @@ std::unique_ptr RegisterKernels(bool enable_graph_capture = fals return kernel_registry; } +#if defined(ORT_USE_EP_API_ADAPTERS) + +namespace { +std::shared_ptr g_kernel_registry; +std::shared_ptr g_graph_capture_kernel_registry; +std::shared_ptr g_int64_kernel_registry; +} // namespace + +void CleanupKernelRegistries() { + g_kernel_registry.reset(); + g_graph_capture_kernel_registry.reset(); + g_int64_kernel_registry.reset(); +} +#endif + +std::shared_ptr GetKernelRegistry(bool enable_graph_capture, bool enable_int64) { + // kernel registry variables are defined differently based on build configuration + // + // - When building as a static library, use static local variable. This is because + // we don't have a reliable way to explicitly destroy the kernel registry after + // use. + // + // - When building as a shared library, use global variables. The cleanup will be performed + // when `ReleaseEpFactory` is called. + if (enable_graph_capture) { +#if !defined(ORT_USE_EP_API_ADAPTERS) + static std::shared_ptr registry = RegisterKernels(true, true); + return registry; +#else + if (g_graph_capture_kernel_registry == nullptr) { + g_graph_capture_kernel_registry = RegisterKernels(true, true); + } + return g_graph_capture_kernel_registry; +#endif + } else if (enable_int64) { +#if defined(ORT_USE_EP_API_ADAPTERS) + if (g_int64_kernel_registry == nullptr) { + g_int64_kernel_registry = RegisterKernels(false, true); + } + return g_int64_kernel_registry; +#else + static std::shared_ptr registry = RegisterKernels(false, true); + return registry; +#endif + } else { +#if defined(ORT_USE_EP_API_ADAPTERS) + if (g_kernel_registry == nullptr) { + g_kernel_registry = RegisterKernels(false, false); + } + return g_kernel_registry; +#else + static std::shared_ptr registry = RegisterKernels(false, false); + return registry; +#endif + } +} + } // namespace webgpu using namespace webgpu; @@ -882,6 +939,7 @@ std::vector WebGpuExecutionProvider::CreatePreferredAllocators() { }; } +#if !defined(ORT_USE_EP_API_ADAPTERS) std::vector> WebGpuExecutionProvider::GetCapability( const onnxruntime::GraphViewer& graph, const IKernelLookup& kernel_lookup, @@ -973,20 +1031,7 @@ std::vector> WebGpuExecutionProvider::GetCapa return result; } -std::shared_ptr WebGpuExecutionProvider::GetKernelRegistry() const { - // Cache registries based on enable_graph_capture_ and enable_int64_ flags - // Note: enable_int64_ is always true when enable_graph_capture_ is true - if (enable_graph_capture_) { - static std::shared_ptr registry = webgpu::RegisterKernels(true, true); - return registry; - } else if (enable_int64_) { - static std::shared_ptr registry = webgpu::RegisterKernels(false, true); - return registry; - } else { - static std::shared_ptr registry = webgpu::RegisterKernels(false, false); - return registry; - } -} +#endif // !defined(ORT_USE_EP_API_ADAPTERS) std::unique_ptr WebGpuExecutionProvider::GetDataTransfer() const { return std::make_unique(BufferManager()); diff --git a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.h b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.h index b5a6b5f167faf..b46d3f3cb45d2 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.h +++ b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.h @@ -4,6 +4,11 @@ #pragma once +#include +#include +#include +#include + #include "core/framework/execution_provider.h" #include "core/framework/session_options.h" #include "core/graph/constants.h" @@ -28,6 +33,9 @@ class GpuBufferAllocator; // Forward declare CapturedCommandInfo which is now defined in webgpu_context.h struct CapturedCommandInfo; + +// The actual implementation of kernel registration. +std::shared_ptr GetKernelRegistry(bool enable_graph_capture, bool enable_int64); } // namespace webgpu struct WebGpuExecutionProviderConfig { @@ -44,13 +52,21 @@ class WebGpuExecutionProvider : public IExecutionProvider { WebGpuExecutionProvider(int context_id, webgpu::WebGpuContext& context, WebGpuExecutionProviderConfig&& config); ~WebGpuExecutionProvider() override; + inline auto GetKernelRegistryImpl() const { + return webgpu::GetKernelRegistry(enable_graph_capture_, enable_int64_); + } + +#if !defined(ORT_USE_EP_API_ADAPTERS) std::vector> GetCapability( const onnxruntime::GraphViewer& graph_viewer, const IKernelLookup& /*kernel_lookup*/, const GraphOptimizerRegistry& /* graph_optimizer_registry */, IResourceAccountant* /* resource_accountant */) const override; - std::shared_ptr GetKernelRegistry() const override; + std::shared_ptr GetKernelRegistry() const override { + return GetKernelRegistryImpl(); + } +#endif std::unique_ptr GetDataTransfer() const override; #if defined(__wasm__) std::unique_ptr GetExternalDataLoader() const override; @@ -83,8 +99,18 @@ class WebGpuExecutionProvider : public IExecutionProvider { Status ReplayGraph(int graph_annotation_id) override; webgpu::BufferManager& BufferManager() const; AllocatorPtr PrepackAllocator() const { return prepack_allocator_; } + std::span GetForceCpuNodeNames() const { return force_cpu_node_names_; } uint32_t MultiRotaryCacheConcatOffset() const { return multi_rotary_cache_concat_offset_; } +#if defined(ORT_USE_EP_API_ADAPTERS) + inline onnxruntime::ep::adapter::Logger& GetEpLogger() const { + return *ep_logger_; + } + inline void SetEpLogger(const OrtLogger* logger) { + ep_logger_ = std::make_unique(logger); + } +#endif + private: bool IsGraphCaptureAllowed() const; void IncrementRegularRunCountBeforeGraphCapture(); @@ -114,6 +140,10 @@ class WebGpuExecutionProvider : public IExecutionProvider { // Allocator for prepacked weights (uses buffers without mapping) AllocatorPtr prepack_allocator_; + +#if defined(ORT_USE_EP_API_ADAPTERS) + std::unique_ptr ep_logger_; +#endif }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc b/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc index fc2496f0c7b68..c4efc690d80f5 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc +++ b/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc @@ -284,11 +284,11 @@ std::shared_ptr WebGpuProviderFactoryCreator::Create( // WebGPU DataTransfer implementation wrapper for the C API with lazy initialization struct WebGpuDataTransferImpl : OrtDataTransferImpl { - WebGpuDataTransferImpl(const OrtApi& ort_api_in) + WebGpuDataTransferImpl(const OrtApi& ort_api_in, int context_id) : ort_api{ort_api_in}, ep_api{*ort_api_in.GetEpApi()}, data_transfer_{nullptr}, - context_id_{0}, // Always use context 0 for Environment's data transfer + context_id_{context_id}, // Always use context 0 for Environment's data transfer init_mutex_{} { ort_version_supported = ORT_API_VERSION; CanCopy = CanCopyImpl; // OrtDataTransferImpl::CanCopy callback @@ -327,9 +327,9 @@ struct WebGpuDataTransferImpl : OrtDataTransferImpl { // If both are GPU, they must have the same device ID if (src_type == OrtMemoryInfoDeviceType_GPU && dst_type == OrtMemoryInfoDeviceType_GPU) { - uint64_t src_device_id = impl.ep_api.MemoryDevice_GetDeviceId(src_memory_device); - uint64_t dst_device_id = impl.ep_api.MemoryDevice_GetDeviceId(dst_memory_device); - if (src_device_id != dst_device_id) { + int src_device_id = impl.ep_api.MemoryDevice_GetDeviceId(src_memory_device); + int dst_device_id = impl.ep_api.MemoryDevice_GetDeviceId(dst_memory_device); + if (src_device_id != impl.context_id_ || dst_device_id != impl.context_id_) { return false; // Cannot copy between different devices } } @@ -372,9 +372,30 @@ struct WebGpuDataTransferImpl : OrtDataTransferImpl { // Now perform the actual tensor copy for (size_t idx = 0; idx < num_tensors; ++idx) { - const OrtValue* src_tensor = src_tensors[idx]; - OrtValue* dst_tensor = dst_tensors[idx]; - auto status = impl.data_transfer_->CopyTensor(src_tensor->Get(), *dst_tensor->GetMutable()); +#if defined(ORT_USE_EP_API_ADAPTERS) + Ort::ConstValue src_value{src_tensors[idx]}; + const void* src_data = src_value.GetTensorRawData(); + size_t size = src_value.GetTensorSizeInBytes(); + bool src_is_gpu = src_value.GetTensorMemoryInfo().GetDeviceType() == OrtMemoryInfoDeviceType_GPU; + + Ort::UnownedValue dst_value{dst_tensors[idx]}; + void* dst_data = dst_value.GetTensorMutableRawData(); + bool dst_is_gpu = dst_value.GetTensorMemoryInfo().GetDeviceType() == OrtMemoryInfoDeviceType_GPU; +#else + const Tensor& src_tensor = src_tensors[idx]->Get(); + const void* src_data = src_tensor.DataRaw(); + size_t size = src_tensor.SizeInBytes(); + bool src_is_gpu = src_tensor.Location().device.Type() == OrtDevice::GPU; + + Tensor& dst_tensor = *dst_tensors[idx]->GetMutable(); + void* dst_data = dst_tensor.MutableDataRaw(); + bool dst_is_gpu = dst_tensor.Location().device.Type() == OrtDevice::GPU; +#endif + auto status = impl.data_transfer_->CopyTensorImpl(src_data, + src_is_gpu, + dst_data, + dst_is_gpu, + size); if (!status.IsOK()) { return OrtApis::CreateStatus(ORT_RUNTIME_EXCEPTION, status.ErrorMessage().c_str()); } @@ -403,14 +424,18 @@ struct WebGpuDataTransferImpl : OrtDataTransferImpl { std::mutex init_mutex_; // Protects lazy initialization }; -OrtDataTransferImpl* OrtWebGpuCreateDataTransfer() { +OrtDataTransferImpl* OrtWebGpuCreateDataTransfer(int context_id /* = 0 */) { +#if defined(ORT_USE_EP_API_ADAPTERS) + return new WebGpuDataTransferImpl(onnxruntime::ep::Api().ort, context_id); +#else // Validate API version is supported const OrtApi* api = OrtApis::GetApi(ORT_API_VERSION); if (!api) { // API version not supported - return nullptr to indicate failure return nullptr; } - return new WebGpuDataTransferImpl(*api); + return new WebGpuDataTransferImpl(*api, context_id); +#endif } } // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/webgpu_provider_factory_creator.h b/onnxruntime/core/providers/webgpu/webgpu_provider_factory_creator.h index 021e33ef25309..876a2e11d791a 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_provider_factory_creator.h +++ b/onnxruntime/core/providers/webgpu/webgpu_provider_factory_creator.h @@ -22,6 +22,6 @@ struct WebGpuProviderFactoryCreator { // C API to create data transfer for WebGPU EP with lazy initialization // Context will be determined from tensors during the first CopyTensors call // Caller takes ownership of the returned OrtDataTransferImpl* -OrtDataTransferImpl* OrtWebGpuCreateDataTransfer(); +OrtDataTransferImpl* OrtWebGpuCreateDataTransfer(int context_id = 0); } // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/matmul_2bits_test.cc b/onnxruntime/test/contrib_ops/matmul_2bits_test.cc index 00ea8947b9dd7..06ff495327ecd 100644 --- a/onnxruntime/test/contrib_ops/matmul_2bits_test.cc +++ b/onnxruntime/test/contrib_ops/matmul_2bits_test.cc @@ -451,7 +451,7 @@ TEST(MatMul2Bits, Float32_2b_Accuracy4) { TestMatMul2BitsTyped(); } -#ifdef USE_WEBGPU +#if defined(USE_WEBGPU) && !defined(ORT_USE_EP_API_ADAPTERS) namespace { @@ -594,7 +594,7 @@ TEST(MatMul2BitsWebGpu, Float32_ZeroPoint_DP4A) { RunTest2Bits(opts); } -#endif // USE_WEBGPU +#endif // defined(USE_WEBGPU) && !defined(ORT_USE_EP_API_ADAPTERS) } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc index fd2cf2f712628..3257cfc99e245 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc @@ -167,7 +167,7 @@ void Shutdown() { g_plugin_ep_infrastructure_state.reset(); } -std::unique_ptr MakeEp(const logging::Logger* logger) { +std::unique_ptr MakeEp(const logging::Logger* logger, const ConfigOptions* config_options) { if (!IsInitialized()) { return nullptr; } @@ -182,6 +182,13 @@ std::unique_ptr MakeEp(const logging::Logger* logger) { StrMapToKeyValueCstrVectors(state.config.default_ep_options, default_ep_option_key_cstrs, default_ep_option_value_cstrs); + if (config_options != nullptr) { + for (const auto& [key, value] : config_options->configurations) { + default_ep_option_key_cstrs.push_back(key.c_str()); + default_ep_option_value_cstrs.push_back(value.c_str()); + } + } + OrtSessionOptions ort_session_options{}; ORT_THROW_IF_ERROR(AddEpOptionsToSessionOptions(state.selected_c_ep_devices, default_ep_option_key_cstrs, diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h index 680045be9330c..091946da8fc26 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h @@ -17,6 +17,7 @@ namespace onnxruntime { struct IExecutionProviderFactory; class IExecutionProvider; +struct ConfigOptions; namespace logging { class Logger; @@ -74,7 +75,7 @@ bool IsInitialized(); void Shutdown(); // Returns a dynamic plugin EP `IExecutionProvider` instance, or `nullptr` if uninitialized. -std::unique_ptr MakeEp(const logging::Logger* logger = nullptr); +std::unique_ptr MakeEp(const logging::Logger* logger = nullptr, const ConfigOptions* config_options = nullptr); // Gets the dynamic plugin EP name, or `std::nullopt` if uninitialized. std::optional GetEpName(); diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 6dc38f84c79d5..169c504b84e12 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -14,8 +14,13 @@ #ifdef USE_CUDA #include "core/providers/cuda/cuda_provider_options.h" #endif +#if defined(USE_WEBGPU) +#include "core/graph/constants.h" +#include "core/session/abi_session_options_impl.h" +#endif #include "core/session/onnxruntime_cxx_api.h" #include "test/util/include/providers.h" +#include "test/unittest_util/test_dynamic_plugin_ep.h" namespace onnxruntime { @@ -273,19 +278,30 @@ std::unique_ptr DefaultXnnpackExecutionProvider() { } std::unique_ptr DefaultWebGpuExecutionProvider(bool is_nhwc) { -#if defined(USE_WEBGPU) && !defined(ORT_USE_EP_API_ADAPTERS) +#if defined(USE_WEBGPU) ConfigOptions config_options{}; +#if defined(ORT_USE_EP_API_ADAPTERS) + // used to remove the EP prefix from the config entry keys + size_t config_entry_key_offset = OrtSessionOptions::GetProviderOptionPrefix(kWebGpuExecutionProvider).length(); +#else + size_t config_entry_key_offset = 0; +#endif + // Disable storage buffer cache - ORT_ENFORCE(config_options.AddConfigEntry(webgpu::options::kStorageBufferCacheMode, + ORT_ENFORCE(config_options.AddConfigEntry(webgpu::options::kStorageBufferCacheMode + config_entry_key_offset, webgpu::options::kBufferCacheMode_Disabled) .IsOK()); if (!is_nhwc) { // Enable NCHW support - ORT_ENFORCE(config_options.AddConfigEntry(webgpu::options::kPreferredLayout, + ORT_ENFORCE(config_options.AddConfigEntry(webgpu::options::kPreferredLayout + config_entry_key_offset, webgpu::options::kPreferredLayout_NCHW) .IsOK()); } +#if defined(ORT_USE_EP_API_ADAPTERS) + return dynamic_plugin_ep_infra::MakeEp(nullptr, &config_options); +#else return WebGpuProviderFactoryCreator::Create(config_options)->CreateProvider(); +#endif #else ORT_UNUSED_PARAMETER(is_nhwc); return nullptr; @@ -293,8 +309,12 @@ std::unique_ptr DefaultWebGpuExecutionProvider(bool is_nhwc) } std::unique_ptr WebGpuExecutionProviderWithOptions(const ConfigOptions& config_options) { -#if defined(USE_WEBGPU) && !defined(ORT_USE_EP_API_ADAPTERS) +#if defined(USE_WEBGPU) +#if defined(ORT_USE_EP_API_ADAPTERS) + return dynamic_plugin_ep_infra::MakeEp(nullptr, &config_options); +#else return WebGpuProviderFactoryCreator::Create(config_options)->CreateProvider(); +#endif #else ORT_UNUSED_PARAMETER(config_options); return nullptr; From 7f0b14376c0255db59a1537a51f9ed92d82cb9d8 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Sat, 7 Mar 2026 17:56:36 -0800 Subject: [PATCH 02/27] integrate with adapters change of LoggingManager --- onnxruntime/core/providers/webgpu/ep/api.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/webgpu/ep/api.cc b/onnxruntime/core/providers/webgpu/ep/api.cc index 24276fbaf02e7..04c5fa2fa9105 100644 --- a/onnxruntime/core/providers/webgpu/ep/api.cc +++ b/onnxruntime/core/providers/webgpu/ep/api.cc @@ -45,7 +45,7 @@ EXPORT_SYMBOL OrtStatus* CreateEpFactories(const char* /*registration_name*/, co } // Initialize the global default logger - ::onnxruntime::ep::adapter::Logger::CreateDefaultLogger(default_logger); + ::onnxruntime::ep::adapter::LoggingManager::CreateDefaultLogger(default_logger); // Factory could use registration_name or define its own EP name. std::unique_ptr factory = std::make_unique(); @@ -67,7 +67,7 @@ EXPORT_SYMBOL OrtStatus* ReleaseEpFactory(OrtEpFactory* factory) { onnxruntime::webgpu::CleanupWebGpuContexts(); // STEP.4 - Destroy the global default logger wrapper - ::onnxruntime::ep::adapter::Logger::DestroyDefaultLogger(); + ::onnxruntime::ep::adapter::LoggingManager::DestroyDefaultLogger(); // STEP.5 - Shutdown protobuf library google::protobuf::ShutdownProtobufLibrary(); From ebf1a1cf6e896e96c89e6e8bd854ddccb7085582 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Wed, 25 Feb 2026 23:43:13 -0800 Subject: [PATCH 03/27] add pipeline job for plugin EP build --- .github/workflows/windows_webgpu.yml | 101 +++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/.github/workflows/windows_webgpu.yml b/.github/workflows/windows_webgpu.yml index 872d3b182c310..7b242ca61d884 100644 --- a/.github/workflows/windows_webgpu.yml +++ b/.github/workflows/windows_webgpu.yml @@ -155,6 +155,107 @@ jobs: working-directory: ${{ github.workspace }}\csharp continue-on-error: true + webgpu_plugin_build_x64_RelWithDebInfo: + runs-on: [ + "self-hosted", + "1ES.Pool=onnxruntime-github-Win2022-GPU-A10", + "JobId=webgpu_plugin_build_x64_RelWithDebInfo-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + ] + timeout-minutes: 300 + env: + OrtPackageId: Microsoft.ML.OnnxRuntime + OnnxRuntimeBuildDirectory: ${{ github.workspace }} + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + setVcvars: true + ALLOW_RELEASED_ONNX_OPSET_ONLY: "0" + DocUpdateNeeded: false + NVIDIA_TF32_OVERRIDE: "0" + ONNXRUNTIME_TEST_GPU_DEVICE_ID: "0" + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: none + + - name: Setup Python 3.12 + uses: actions/setup-python@v6 + with: + python-version: "3.12" + architecture: x64 + + - name: Locate vcvarsall and Setup Env + uses: ./.github/actions/locate-vcvarsall-and-setup-env + with: + architecture: x64 + + - name: Install python modules + run: python -m pip install -r tools\ci_build\github\windows\python\requirements.txt + shell: cmd + working-directory: ${{ github.workspace }} + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "20.x" + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + env: + PROCESSOR_ARCHITECTURE: x64 + with: + dotnet-version: "8.x" + + - name: Use Nuget 6.x + uses: nuget/setup-nuget@v2 + with: + nuget-version: "6.x" + + - name: NuGet restore + run: | + nuget restore packages.config -ConfigFile NuGet.config -PackagesDirectory ${{ github.workspace }}\RelWithDebInfo + shell: cmd + working-directory: ${{ github.workspace }} + + - uses: actions/cache@v5 + id: onnx-node-tests-cache + with: + path: ${{ github.workspace }}/js/test/ + key: onnxnodetests-${{ hashFiles('js/scripts/prepare-onnx-node-tests.ts') }} + + - name: Build and Test + shell: pwsh + run: | + python.exe ${{ github.workspace }}\tools\ci_build\build.py ` + --config RelWithDebInfo ` + --build_dir ${{ github.workspace }} ` + --skip_submodule_sync ` + --parallel ` + --use_binskim_compliant_compile_flags ` + --cmake_generator "Visual Studio 17 2022" ` + --enable_onnx_tests ` + --use_webgpu shared_lib ` + --wgsl_template static ` + --use_vcpkg --use_vcpkg_ms_internal_asset_cache ` + --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=ON ` + --disable_rtti ` + --enable_lto + + if ($lastExitCode -ne 0) { + exit $lastExitCode + } + Remove-Item "${{ github.workspace }}\RelWithDebInfo" -Include "*.obj" -Recurse + + - name: Publish artifacts + uses: actions/upload-artifact@v4 + with: + name: webgpu-plugin-binaries + path: | + ${{ github.workspace }}/RelWithDebInfo/RelWithDebInfo/onnxruntime_providers_webgpu.dll + ${{ github.workspace }}/RelWithDebInfo/RelWithDebInfo/onnxruntime_providers_webgpu.pdb + ${{ github.workspace }}/RelWithDebInfo/RelWithDebInfo/dxcompiler.dll + ${{ github.workspace }}/RelWithDebInfo/RelWithDebInfo/dxil.dll + webgpu_external_dawn_build_x64_RelWithDebInfo: runs-on: [ "self-hosted", From 078cb4ae30e54542df492542cffdd40d8e051f33 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Wed, 11 Mar 2026 10:36:01 -0700 Subject: [PATCH 04/27] resolve comments - update upsamplebase.h --- onnxruntime/core/providers/cpu/tensor/upsamplebase.h | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/onnxruntime/core/providers/cpu/tensor/upsamplebase.h b/onnxruntime/core/providers/cpu/tensor/upsamplebase.h index 42b22b02a1d67..ff5498c0b4644 100644 --- a/onnxruntime/core/providers/cpu/tensor/upsamplebase.h +++ b/onnxruntime/core/providers/cpu/tensor/upsamplebase.h @@ -272,14 +272,9 @@ class UpsampleBase { rank = x_shape->dim_size(); } } else { - int is_const; - auto tensor = info.GetKernelInfo().GetTensorConstantInput(0, &is_const); - if (is_const) { - auto type_and_shape_info = tensor.GetTensorTypeAndShapeInfo(); - if (type_and_shape_info.HasShape()) { - rank = static_cast(type_and_shape_info.GetShape().size()); - } - } + auto type_info = info.GetKernelInfo().GetInputTypeInfo(0); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + rank = static_cast(tensor_info.GetDimensionsCount()); } if (get_scale && scale->Shape().Size() > 0 && ((opset < 18) || (rank > 0 && opset >= 18))) { ORT_THROW_IF_ERROR(ParseScalesData(scale, scales_, rank)); From 9336a6304425a7d96c3f7fcf478e949f04ae0a7f Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Wed, 11 Mar 2026 15:29:43 -0700 Subject: [PATCH 05/27] remove out-of-dated todos. --- onnxruntime/core/providers/webgpu/ep/factory.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/webgpu/ep/factory.cc b/onnxruntime/core/providers/webgpu/ep/factory.cc index 737b64d47b2bb..299d8643b8c41 100644 --- a/onnxruntime/core/providers/webgpu/ep/factory.cc +++ b/onnxruntime/core/providers/webgpu/ep/factory.cc @@ -49,9 +49,9 @@ Factory::Factory() : OrtEpFactory{}, CreateEp = CreateEpImpl; ReleaseEp = ReleaseEpImpl; - CreateAllocator = CreateAllocatorImpl; // TODO - ReleaseAllocator = ReleaseAllocatorImpl; // TODO - CreateDataTransfer = CreateDataTransferImpl; // TODO + CreateAllocator = CreateAllocatorImpl; + ReleaseAllocator = ReleaseAllocatorImpl; + CreateDataTransfer = CreateDataTransferImpl; } // Static C API implementations From 53c8942000f96795c447a6e495f2206f5ca805f2 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:54:26 -0700 Subject: [PATCH 06/27] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- onnxruntime/core/providers/webgpu/tensor/shape_op.cc | 1 - onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/webgpu/tensor/shape_op.cc b/onnxruntime/core/providers/webgpu/tensor/shape_op.cc index e5e6e4d61eed5..09194aa9f4dbb 100644 --- a/onnxruntime/core/providers/webgpu/tensor/shape_op.cc +++ b/onnxruntime/core/providers/webgpu/tensor/shape_op.cc @@ -3,7 +3,6 @@ #include "core/providers/webgpu/webgpu_kernel.h" #include "core/providers/webgpu/webgpu_supported_types.h" -// #include "core/providers/cpu/tensor/shape_op.h" namespace onnxruntime { namespace webgpu { diff --git a/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc b/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc index c4efc690d80f5..c62fb29a4cb22 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc +++ b/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc @@ -288,7 +288,7 @@ struct WebGpuDataTransferImpl : OrtDataTransferImpl { : ort_api{ort_api_in}, ep_api{*ort_api_in.GetEpApi()}, data_transfer_{nullptr}, - context_id_{context_id}, // Always use context 0 for Environment's data transfer + context_id_{context_id}, init_mutex_{} { ort_version_supported = ORT_API_VERSION; CanCopy = CanCopyImpl; // OrtDataTransferImpl::CanCopy callback From b5a90044d3a16fc460a62f24b786360ae3041b96 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:06:58 -0700 Subject: [PATCH 07/27] resolve comments - lock on kernel registry access --- .../core/providers/webgpu/webgpu_execution_provider.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc index b79454ff2769f..58124f1e9c7ab 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc +++ b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc @@ -3,6 +3,7 @@ #include "core/providers/webgpu/webgpu_execution_provider.h" +#include #include #include #include @@ -840,12 +841,14 @@ std::unique_ptr RegisterKernels(bool enable_graph_capture, bool #if defined(ORT_USE_EP_API_ADAPTERS) namespace { +std::mutex g_kernel_registry_mutex; std::shared_ptr g_kernel_registry; std::shared_ptr g_graph_capture_kernel_registry; std::shared_ptr g_int64_kernel_registry; } // namespace void CleanupKernelRegistries() { + std::lock_guard lock(g_kernel_registry_mutex); g_kernel_registry.reset(); g_graph_capture_kernel_registry.reset(); g_int64_kernel_registry.reset(); @@ -866,6 +869,7 @@ std::shared_ptr GetKernelRegistry(bool enable_graph_capture, boo static std::shared_ptr registry = RegisterKernels(true, true); return registry; #else + std::lock_guard lock(g_kernel_registry_mutex); if (g_graph_capture_kernel_registry == nullptr) { g_graph_capture_kernel_registry = RegisterKernels(true, true); } @@ -873,6 +877,7 @@ std::shared_ptr GetKernelRegistry(bool enable_graph_capture, boo #endif } else if (enable_int64) { #if defined(ORT_USE_EP_API_ADAPTERS) + std::lock_guard lock(g_kernel_registry_mutex); if (g_int64_kernel_registry == nullptr) { g_int64_kernel_registry = RegisterKernels(false, true); } @@ -883,6 +888,7 @@ std::shared_ptr GetKernelRegistry(bool enable_graph_capture, boo #endif } else { #if defined(ORT_USE_EP_API_ADAPTERS) + std::lock_guard lock(g_kernel_registry_mutex); if (g_kernel_registry == nullptr) { g_kernel_registry = RegisterKernels(false, false); } From 42e458ea4d30287acc9ccb7c958eb0d751a9aaf0 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:10:29 -0700 Subject: [PATCH 08/27] substr --- onnxruntime/test/util/default_providers.cc | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 169c504b84e12..e00dfa3d5b139 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -280,20 +280,30 @@ std::unique_ptr DefaultXnnpackExecutionProvider() { std::unique_ptr DefaultWebGpuExecutionProvider(bool is_nhwc) { #if defined(USE_WEBGPU) ConfigOptions config_options{}; + + // Helper to strip the EP prefix from config entry keys when building as a plugin EP. + // The full key is like "ep.webgpuexecutionprovider.storageBufferCacheMode", and the + // config entry expects just "storageBufferCacheMode" in the EP API build. + // Returns a pointer into the original string, so the result is valid as long as the input is. + auto strip_ep_prefix = [](const char* full_key) -> const char* { #if defined(ORT_USE_EP_API_ADAPTERS) - // used to remove the EP prefix from the config entry keys - size_t config_entry_key_offset = OrtSessionOptions::GetProviderOptionPrefix(kWebGpuExecutionProvider).length(); + std::string_view key{full_key}; + std::string_view prefix = OrtSessionOptions::GetProviderOptionPrefix(kWebGpuExecutionProvider); + ORT_ENFORCE(key.length() >= prefix.length() && key.substr(0, prefix.length()) == prefix, + "Config key \"", key, "\" does not start with expected prefix \"", prefix, "\""); + return full_key + prefix.length(); #else - size_t config_entry_key_offset = 0; + return full_key; #endif + }; // Disable storage buffer cache - ORT_ENFORCE(config_options.AddConfigEntry(webgpu::options::kStorageBufferCacheMode + config_entry_key_offset, + ORT_ENFORCE(config_options.AddConfigEntry(strip_ep_prefix(webgpu::options::kStorageBufferCacheMode), webgpu::options::kBufferCacheMode_Disabled) .IsOK()); if (!is_nhwc) { // Enable NCHW support - ORT_ENFORCE(config_options.AddConfigEntry(webgpu::options::kPreferredLayout + config_entry_key_offset, + ORT_ENFORCE(config_options.AddConfigEntry(strip_ep_prefix(webgpu::options::kPreferredLayout), webgpu::options::kPreferredLayout_NCHW) .IsOK()); } From 51edd0747cfe417ada87733c4df31d2551b12738 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:13:05 -0700 Subject: [PATCH 09/27] unittest --- cmake/onnxruntime_unittests.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 442b556d825cb..8137f8b3a2529 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1045,7 +1045,7 @@ endfunction() # Set environment variables for plugin EP tests when run via CTest. function(onnxruntime_set_plugin_ep_test_environment target) if(onnxruntime_USE_WEBGPU AND onnxruntime_USE_EP_API_ADAPTERS) - set(ORT_PLUGIN_EP_JSON_CONFIG "{\"ep_library_registration_name\": \"WebGPU_PluginEP\", \"ep_library_path\": \"onnxruntime_providers_webgpu.dll\", \"selected_ep_name\": \"WebGpuExecutionProvider\"}") + set(ORT_PLUGIN_EP_JSON_CONFIG "{\"ep_library_registration_name\": \"WebGPU_PluginEP\", \"ep_library_path\": \"$\", \"selected_ep_name\": \"WebGpuExecutionProvider\"}") set_tests_properties(${target} PROPERTIES ENVIRONMENT "ORT_UNIT_TEST_MAIN_DYNAMIC_PLUGIN_EP_CONFIG_JSON=${ORT_PLUGIN_EP_JSON_CONFIG}" ) From 8a51859825f2797ff6ee5f577822e194fcc6f398 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 13 Mar 2026 21:08:35 -0700 Subject: [PATCH 10/27] resolve comments (part.1) --- include/onnxruntime/ep/common.h | 21 ++ onnxruntime/core/providers/webgpu/ep/ep.cc | 179 +++++++++--------- .../core/providers/webgpu/ep/factory.cc | 48 ++--- .../core/providers/webgpu/ep/factory.h | 11 -- .../webgpu/webgpu_execution_provider.cc | 5 + 5 files changed, 126 insertions(+), 138 deletions(-) diff --git a/include/onnxruntime/ep/common.h b/include/onnxruntime/ep/common.h index 03cd571461755..0e779ba3d4081 100644 --- a/include/onnxruntime/ep/common.h +++ b/include/onnxruntime/ep/common.h @@ -41,3 +41,24 @@ OrtStatus* _status = (status_expr); \ Ort::Status _ignored{_status}; \ } while (false) + +// Helper macros to convert exceptions to OrtStatus* return values. +// Usage: +// EXCEPTION_TO_RETURNED_STATUS_BEGIN +// ... code that may throw ... +// EXCEPTION_TO_RETURNED_STATUS_END +#define EXCEPTION_TO_RETURNED_STATUS_BEGIN try { +#define EXCEPTION_TO_RETURNED_STATUS_END \ + } \ + catch (const Ort::Exception& ex) { \ + Ort::Status status(ex); \ + return status.release(); \ + } \ + catch (const std::exception& ex) { \ + Ort::Status status(ex.what(), ORT_EP_FAIL); \ + return status.release(); \ + } \ + catch (...) { \ + Ort::Status status("Unknown exception", ORT_EP_FAIL); \ + return status.release(); \ + } diff --git a/onnxruntime/core/providers/webgpu/ep/ep.cc b/onnxruntime/core/providers/webgpu/ep/ep.cc index 308f4c001c50b..8265e4ba60d3b 100644 --- a/onnxruntime/core/providers/webgpu/ep/ep.cc +++ b/onnxruntime/core/providers/webgpu/ep/ep.cc @@ -52,46 +52,47 @@ const char* ORT_API_CALL Ep::GetNameImpl(const OrtEp* this_ptr) noexcept { OrtStatus* ORT_API_CALL Ep::GetCapabilityImpl(OrtEp* this_ptr, const OrtGraph* graph, OrtEpGraphSupportInfo* graph_support_info) noexcept { - try { - auto& ep = *static_cast(static_cast(this_ptr)->EpImpl()); - Ort::ConstGraph ort_graph{graph}; + EXCEPTION_TO_RETURNED_STATUS_BEGIN - // Get all nodes in the graph - std::vector all_nodes = ort_graph.GetNodes(); + auto& ep = *static_cast(static_cast(this_ptr)->EpImpl()); + Ort::ConstGraph ort_graph{graph}; - if (all_nodes.empty()) { - return nullptr; // No nodes to process - } + // Get all nodes in the graph + std::vector all_nodes = ort_graph.GetNodes(); - std::vector candidate_nodes; + if (all_nodes.empty()) { + return nullptr; // No nodes to process + } - // For each node, check if we have a registered kernel for it - for (const auto& node : all_nodes) { - if (node.GetEpName() == kWebGpuExecutionProvider) { - candidate_nodes.push_back(node); - continue; - } + std::vector candidate_nodes; - const OrtKernelDef* kernel_def = nullptr; - RETURN_IF_ERROR(Api().ep.EpGraphSupportInfo_LookUpKernel(graph_support_info, node, &kernel_def)); + // For each node, check if we have a registered kernel for it + for (const auto& node : all_nodes) { + if (node.GetEpName() == kWebGpuExecutionProvider) { + candidate_nodes.push_back(node); + continue; + } - if (kernel_def == nullptr) { - LOGS(ep.GetEpLogger(), INFO) << "webgpu kernel not found in registries for Op type: " - << node.GetOperatorType() << " node name: " << node.GetName(); - continue; - } + const OrtKernelDef* kernel_def = nullptr; + RETURN_IF_ERROR(Api().ep.EpGraphSupportInfo_LookUpKernel(graph_support_info, node, &kernel_def)); - auto cpu_node_names = ep.GetForceCpuNodeNames(); - if (std::find(cpu_node_names.begin(), - cpu_node_names.end(), - node.GetName()) != cpu_node_names.end()) { - LOGS(ep.GetEpLogger(), INFO) << "Force CPU execution for node: " << node.GetName(); - continue; - } + if (kernel_def == nullptr) { + LOGS(ep.GetEpLogger(), INFO) << "webgpu kernel not found in registries for Op type: " + << node.GetOperatorType() << " node name: " << node.GetName(); + continue; + } - // - // The following code checks if the node is really supported by webgpu EP - // + auto cpu_node_names = ep.GetForceCpuNodeNames(); + if (std::find(cpu_node_names.begin(), + cpu_node_names.end(), + node.GetName()) != cpu_node_names.end()) { + LOGS(ep.GetEpLogger(), INFO) << "Force CPU execution for node: " << node.GetName(); + continue; + } + + // + // The following code checks if the node is really supported by webgpu EP + // #define FALLBACK_TO_CPU_IF_EXIST_INPUT(idx) \ if (inputs.size() > idx && inputs[idx] != nullptr) { \ @@ -103,82 +104,74 @@ OrtStatus* ORT_API_CALL Ep::GetCapabilityImpl(OrtEp* this_ptr, const OrtGraph* g continue; \ } - // Check for Attention - if (node.GetOperatorType() == "Attention" && node.GetDomain() == kMSDomain) { - const auto& inputs = node.GetInputs(); - const auto& outputs = node.GetOutputs(); - - // Current implementation does not support mask_index(input[3]), past(input[4]) and past_seq_len(input[6]) - FALLBACK_TO_CPU_IF_EXIST_INPUT(3); - FALLBACK_TO_CPU_IF_EXIST_INPUT(4); - FALLBACK_TO_CPU_IF_EXIST_INPUT(6); - - // Current implementation does not support present(output[1]) - FALLBACK_TO_CPU_IF_EXIST_OUTPUT(1); - - // If attribute past_present_share_buffer is set, fallback to CPU - bool has_past_present_share_buffer = false; - for (const auto& attr : node.GetAttributes()) { - if (attr.GetName() == "past_present_share_buffer") { - int64_t val = 0; - RETURN_IF_ERROR(attr.GetValue(val)); - if (val != 0) { - has_past_present_share_buffer = true; - } + // Check for Attention + if (node.GetOperatorType() == "Attention" && node.GetDomain() == kMSDomain) { + const auto& inputs = node.GetInputs(); + const auto& outputs = node.GetOutputs(); + + // Current implementation does not support mask_index(input[3]), past(input[4]) and past_seq_len(input[6]) + FALLBACK_TO_CPU_IF_EXIST_INPUT(3); + FALLBACK_TO_CPU_IF_EXIST_INPUT(4); + FALLBACK_TO_CPU_IF_EXIST_INPUT(6); + + // Current implementation does not support present(output[1]) + FALLBACK_TO_CPU_IF_EXIST_OUTPUT(1); + + // If attribute past_present_share_buffer is set, fallback to CPU + bool has_past_present_share_buffer = false; + for (const auto& attr : node.GetAttributes()) { + if (attr.GetName() == "past_present_share_buffer") { + int64_t val = 0; + RETURN_IF_ERROR(attr.GetValue(val)); + if (val != 0) { + has_past_present_share_buffer = true; } - } - if (has_past_present_share_buffer) { - continue; + break; } } - - candidate_nodes.push_back(node); + if (has_past_present_share_buffer) { + continue; + } } - std::unordered_set cpu_preferred_nodes; - RETURN_IF_ERROR(onnxruntime::ep::GetCpuPreferredNodes(*ort_graph, - *graph_support_info, - static_cast(this_ptr)->GetOrtLogger(), - candidate_nodes, - cpu_preferred_nodes)); + candidate_nodes.push_back(node); + } + + std::unordered_set cpu_preferred_nodes; + RETURN_IF_ERROR(onnxruntime::ep::GetCpuPreferredNodes(*ort_graph, + *graph_support_info, + static_cast(this_ptr)->GetOrtLogger(), + candidate_nodes, + cpu_preferred_nodes)); - for (const auto& node : candidate_nodes) { - if (cpu_preferred_nodes.count(node) == 0) { - RETURN_IF_ERROR(Api().ep.EpGraphSupportInfo_AddSingleNode(graph_support_info, node)); - } + for (const auto& node : candidate_nodes) { + if (cpu_preferred_nodes.count(node) == 0) { + RETURN_IF_ERROR(Api().ep.EpGraphSupportInfo_AddSingleNode(graph_support_info, node)); } - } catch (const Ort::Exception& ex) { - Ort::Status status(ex); - return status.release(); - } catch (const std::exception& ex) { - Ort::Status status(ex.what(), ORT_EP_FAIL); - return status.release(); } return nullptr; + + EXCEPTION_TO_RETURNED_STATUS_END } OrtStatus* ORT_API_CALL Ep::GetKernelRegistryImpl( _In_ OrtEp* this_ptr, _Outptr_result_maybenull_ const OrtKernelRegistry** kernel_registry) noexcept { - try { - *kernel_registry = nullptr; - - // For the WebGPU EP, delegate to the CreateKernelRegistry function - // which properly constructs a registry using only public APIs - auto* ep = static_cast(this_ptr); - - auto& webgpu_ep = *static_cast(ep->EpImpl()); - - *kernel_registry = *webgpu_ep.GetKernelRegistryImpl(); - return nullptr; - } catch (const Ort::Exception& ex) { - Ort::Status status(ex); - return status.release(); - } catch (const std::exception& ex) { - Ort::Status status(ex.what(), ORT_EP_FAIL); - return status.release(); - } + EXCEPTION_TO_RETURNED_STATUS_BEGIN + + *kernel_registry = nullptr; + + // For the WebGPU EP, delegate to the CreateKernelRegistry function + // which properly constructs a registry using only public APIs + auto* ep = static_cast(this_ptr); + + auto& webgpu_ep = *static_cast(ep->EpImpl()); + + *kernel_registry = *webgpu_ep.GetKernelRegistryImpl(); + return nullptr; + + EXCEPTION_TO_RETURNED_STATUS_END } OrtStatus* ORT_API_CALL Ep::GetPreferredDataLayoutImpl(_In_ OrtEp* this_ptr, diff --git a/onnxruntime/core/providers/webgpu/ep/factory.cc b/onnxruntime/core/providers/webgpu/ep/factory.cc index 299d8643b8c41..4451ec704448a 100644 --- a/onnxruntime/core/providers/webgpu/ep/factory.cc +++ b/onnxruntime/core/providers/webgpu/ep/factory.cc @@ -52,6 +52,8 @@ Factory::Factory() : OrtEpFactory{}, CreateAllocator = CreateAllocatorImpl; ReleaseAllocator = ReleaseAllocatorImpl; CreateDataTransfer = CreateDataTransferImpl; + + IsStreamAware = IsStreamAwareImpl; } // Static C API implementations @@ -129,17 +131,14 @@ OrtStatus* ORT_API_CALL Factory::CreateEpImpl( } } - try { - auto webgpu_ep_factory = WebGpuProviderFactoryCreator::Create(config_options); - auto webgpu_ep = webgpu_ep_factory->CreateProvider(*session_options, *logger); - static_cast(webgpu_ep.get())->SetEpLogger(logger); - auto factory = static_cast(this_ptr); - *ep = new Ep(std::move(webgpu_ep), *factory, *logger, factory->config_); - return nullptr; - } catch (const Ort::Exception& ex) { - Ort::Status status(ex); - return status.release(); - } + EXCEPTION_TO_RETURNED_STATUS_BEGIN + auto webgpu_ep_factory = WebGpuProviderFactoryCreator::Create(config_options); + auto webgpu_ep = webgpu_ep_factory->CreateProvider(*session_options, *logger); + static_cast(webgpu_ep.get())->SetEpLogger(logger); + auto factory = static_cast(this_ptr); + *ep = new Ep(std::move(webgpu_ep), *factory, *logger, factory->config_); + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END } void ORT_API_CALL Factory::ReleaseEpImpl(OrtEpFactory* /*this_ptr*/, OrtEp* ep) noexcept { @@ -169,13 +168,10 @@ void ORT_API_CALL Factory::ReleaseAllocatorImpl(OrtEpFactory* /*this_ptr*/, OrtA OrtStatus* ORT_API_CALL Factory::CreateDataTransferImpl( OrtEpFactory* /*this_ptr*/, OrtDataTransferImpl** data_transfer) noexcept { - try { - *data_transfer = OrtWebGpuCreateDataTransfer(); // TODO(fs-eire): pass context id if needed - return nullptr; - } catch (const Ort::Exception& ex) { - Ort::Status status(ex); - return status.release(); - } + EXCEPTION_TO_RETURNED_STATUS_BEGIN + *data_transfer = OrtWebGpuCreateDataTransfer(); // TODO(fs-eire): pass context id if needed + return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END } bool ORT_API_CALL Factory::IsStreamAwareImpl(const OrtEpFactory* /*this_ptr*/) noexcept { @@ -192,22 +188,6 @@ OrtStatus* ORT_API_CALL Factory::CreateSyncStreamForDeviceImpl( "CreateSyncStreamForDevice is not implemented for this EP factory."); } -OrtStatus* ORT_API_CALL Factory::ValidateCompiledModelCompatibilityInfoImpl( - OrtEpFactory* /*this_ptr*/, - const OrtHardwareDevice* const* /*devices*/, - size_t /*num_devices*/, - const char* /*compatibility_info*/, - OrtCompiledModelCompatibility* model_compatibility) noexcept { - *model_compatibility = OrtCompiledModelCompatibility_EP_NOT_APPLICABLE; - return nullptr; -} - -OrtStatus* ORT_API_CALL Factory::SetEnvironmentOptionsImpl( - OrtEpFactory* /*this_ptr*/, - const OrtKeyValuePairs* /*options*/) noexcept { - return nullptr; // Default implementation does nothing -} - } // namespace ep } // namespace webgpu } // namespace onnxruntime diff --git a/onnxruntime/core/providers/webgpu/ep/factory.h b/onnxruntime/core/providers/webgpu/ep/factory.h index cd886e19b56d7..86613b2a2ab11 100644 --- a/onnxruntime/core/providers/webgpu/ep/factory.h +++ b/onnxruntime/core/providers/webgpu/ep/factory.h @@ -63,17 +63,6 @@ class Factory : public OrtEpFactory { const OrtKeyValuePairs* stream_options, OrtSyncStreamImpl** stream) noexcept; - static OrtStatus* ORT_API_CALL ValidateCompiledModelCompatibilityInfoImpl( - OrtEpFactory* this_ptr, - const OrtHardwareDevice* const* devices, - size_t num_devices, - const char* compatibility_info, - OrtCompiledModelCompatibility* model_compatibility) noexcept; - - static OrtStatus* ORT_API_CALL SetEnvironmentOptionsImpl( - OrtEpFactory* this_ptr, - const OrtKeyValuePairs* options) noexcept; - Ep::Config config_; Ort::MemoryInfo default_memory_info_; Ort::MemoryInfo readonly_memory_info_; // used for initializers diff --git a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc index 58124f1e9c7ab..b4d751ce3a2c0 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc +++ b/onnxruntime/core/providers/webgpu/webgpu_execution_provider.cc @@ -864,6 +864,10 @@ std::shared_ptr GetKernelRegistry(bool enable_graph_capture, boo // // - When building as a shared library, use global variables. The cleanup will be performed // when `ReleaseEpFactory` is called. + // + // Graph capture mode needs a separate kernel registry because contrib kernel registration + // differs based on enable_graph_capture, and enable_int64 is always true when + // enable_graph_capture is true. if (enable_graph_capture) { #if !defined(ORT_USE_EP_API_ADAPTERS) static std::shared_ptr registry = RegisterKernels(true, true); @@ -913,6 +917,7 @@ WebGpuExecutionProvider::WebGpuExecutionProvider(int context_id, preferred_data_layout_{config.data_layout}, force_cpu_node_names_{std::move(config.force_cpu_node_names)}, enable_graph_capture_{config.enable_graph_capture}, + // enable_int64_ is always true when enable_graph_capture_ is true enable_int64_{config.enable_graph_capture || config.enable_int64}, multi_rotary_cache_concat_offset_{config.multi_rotary_cache_concat_offset}, prepack_allocator_{std::make_shared(context_.InitializerBufferManager(), false)} { From fd6bd724cbed94a8aff1a4cadf808d91bc040b4f Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 13 Mar 2026 21:20:57 -0700 Subject: [PATCH 11/27] resolve comments (part.2) --- .github/workflows/windows_webgpu.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.github/workflows/windows_webgpu.yml b/.github/workflows/windows_webgpu.yml index 7b242ca61d884..2ee68d74102e3 100644 --- a/.github/workflows/windows_webgpu.yml +++ b/.github/workflows/windows_webgpu.yml @@ -163,9 +163,7 @@ jobs: ] timeout-minutes: 300 env: - OrtPackageId: Microsoft.ML.OnnxRuntime OnnxRuntimeBuildDirectory: ${{ github.workspace }} - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true setVcvars: true ALLOW_RELEASED_ONNX_OPSET_ONLY: "0" DocUpdateNeeded: false @@ -199,24 +197,6 @@ jobs: with: node-version: "20.x" - - name: Setup .NET - uses: actions/setup-dotnet@v5 - env: - PROCESSOR_ARCHITECTURE: x64 - with: - dotnet-version: "8.x" - - - name: Use Nuget 6.x - uses: nuget/setup-nuget@v2 - with: - nuget-version: "6.x" - - - name: NuGet restore - run: | - nuget restore packages.config -ConfigFile NuGet.config -PackagesDirectory ${{ github.workspace }}\RelWithDebInfo - shell: cmd - working-directory: ${{ github.workspace }} - - uses: actions/cache@v5 id: onnx-node-tests-cache with: @@ -244,7 +224,6 @@ jobs: if ($lastExitCode -ne 0) { exit $lastExitCode } - Remove-Item "${{ github.workspace }}\RelWithDebInfo" -Include "*.obj" -Recurse - name: Publish artifacts uses: actions/upload-artifact@v4 From 17bcb6cb0a44265052f925f78703dd6e22d68e57 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 13 Mar 2026 21:53:31 -0700 Subject: [PATCH 12/27] resolve comments (part.3) --- onnxruntime/core/providers/webgpu/ep/ep.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/webgpu/ep/ep.cc b/onnxruntime/core/providers/webgpu/ep/ep.cc index 8265e4ba60d3b..0e6a59afe330d 100644 --- a/onnxruntime/core/providers/webgpu/ep/ep.cc +++ b/onnxruntime/core/providers/webgpu/ep/ep.cc @@ -68,11 +68,18 @@ OrtStatus* ORT_API_CALL Ep::GetCapabilityImpl(OrtEp* this_ptr, const OrtGraph* g // For each node, check if we have a registered kernel for it for (const auto& node : all_nodes) { - if (node.GetEpName() == kWebGpuExecutionProvider) { + std::string ep_name = node.GetEpName(); + + if (ep_name == kWebGpuExecutionProvider) { candidate_nodes.push_back(node); continue; } + // Reject nodes already assigned to a different (non-CPU) EP + if (!ep_name.empty() && ep_name != kCpuExecutionProvider) { + continue; + } + const OrtKernelDef* kernel_def = nullptr; RETURN_IF_ERROR(Api().ep.EpGraphSupportInfo_LookUpKernel(graph_support_info, node, &kernel_def)); From 569f048c096931668e24f90e84d86e5300d71f99 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 13 Mar 2026 22:20:07 -0700 Subject: [PATCH 13/27] resolve comments (part.4) --- include/onnxruntime/ep/adapter/ep.h | 1 + onnxruntime/core/providers/webgpu/ep/api.cc | 6 ++++++ onnxruntime/core/providers/webgpu/ep/ep.cc | 12 ++++++++++-- onnxruntime/core/providers/webgpu/ep/factory.cc | 8 +++++++- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/include/onnxruntime/ep/adapter/ep.h b/include/onnxruntime/ep/adapter/ep.h index 34fc7682a8138..ca0a8c9599eda 100644 --- a/include/onnxruntime/ep/adapter/ep.h +++ b/include/onnxruntime/ep/adapter/ep.h @@ -27,6 +27,7 @@ class Ep : public OrtEp { profiler_{impl_->GetProfiler()}, temp_space_cpu_allocator_{temp_space_cpu_allocator}, temp_space_allocator_{temp_space_allocator} { + ort_version_supported = ORT_API_VERSION; } public: diff --git a/onnxruntime/core/providers/webgpu/ep/api.cc b/onnxruntime/core/providers/webgpu/ep/api.cc index 04c5fa2fa9105..885d30a9728d8 100644 --- a/onnxruntime/core/providers/webgpu/ep/api.cc +++ b/onnxruntime/core/providers/webgpu/ep/api.cc @@ -39,6 +39,8 @@ EXPORT_SYMBOL OrtStatus* CreateEpFactories(const char* /*registration_name*/, co // Manual init for the C++ API onnxruntime::ep::ApiInit(ort_api_base); + EXCEPTION_TO_RETURNED_STATUS_BEGIN + if (max_factories < 1) { return onnxruntime::ep::Api().ort.CreateStatus(ORT_INVALID_ARGUMENT, "Not enough space to return EP factory. Need at least one."); @@ -54,9 +56,12 @@ EXPORT_SYMBOL OrtStatus* CreateEpFactories(const char* /*registration_name*/, co *num_factories = 1; return nullptr; + + EXCEPTION_TO_RETURNED_STATUS_END } EXPORT_SYMBOL OrtStatus* ReleaseEpFactory(OrtEpFactory* factory) { + EXCEPTION_TO_RETURNED_STATUS_BEGIN // STEP.1 - Release the factory delete static_cast(factory); @@ -73,6 +78,7 @@ EXPORT_SYMBOL OrtStatus* ReleaseEpFactory(OrtEpFactory* factory) { google::protobuf::ShutdownProtobufLibrary(); return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END } } // extern "C" diff --git a/onnxruntime/core/providers/webgpu/ep/ep.cc b/onnxruntime/core/providers/webgpu/ep/ep.cc index 0e6a59afe330d..285fdddf11ffd 100644 --- a/onnxruntime/core/providers/webgpu/ep/ep.cc +++ b/onnxruntime/core/providers/webgpu/ep/ep.cc @@ -25,8 +25,6 @@ Ep::Ep(std::unique_ptr impl, Factory& factory, const OrtLogg factory_{factory}, logger_{logger}, config_{config} { - ort_version_supported = ORT_API_VERSION; - // Initialize the execution provider's function table GetName = GetNameImpl; GetCapability = GetCapabilityImpl; @@ -183,11 +181,13 @@ OrtStatus* ORT_API_CALL Ep::GetKernelRegistryImpl( OrtStatus* ORT_API_CALL Ep::GetPreferredDataLayoutImpl(_In_ OrtEp* this_ptr, _Out_ OrtEpDataLayout* preferred_data_layout) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN // Delegate to the underlying WebGPU EP's GetPreferredLayout() // DataLayout enum values map 1:1 to OrtEpDataLayout (NCHW=0, NHWC=1) auto* ep = static_cast(this_ptr); *preferred_data_layout = static_cast(ep->EpImpl()->GetPreferredLayout()); return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END } OrtStatus* ORT_API_CALL Ep::ShouldConvertDataLayoutForOpImpl(_In_ OrtEp* this_ptr, @@ -195,6 +195,7 @@ OrtStatus* ORT_API_CALL Ep::ShouldConvertDataLayoutForOpImpl(_In_ OrtEp* this_pt _In_z_ const char* op_type, _In_ OrtEpDataLayout target_data_layout, _Outptr_ int* should_convert) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN // DataLayout enum values map 1:1 to OrtEpDataLayout (NCHW=0, NHWC=1) auto* ep = static_cast(this_ptr); auto result = ep->EpImpl()->ShouldConvertDataLayoutForOp(domain, op_type, @@ -205,10 +206,12 @@ OrtStatus* ORT_API_CALL Ep::ShouldConvertDataLayoutForOpImpl(_In_ OrtEp* this_pt *should_convert = -1; } return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END } OrtStatus* ORT_API_CALL Ep::OnRunStartImpl(_In_ OrtEp* this_ptr, _In_ const OrtRunOptions* run_options) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN onnxruntime::RunOptions options{}; // currently only option "gpu_graph_id" is used auto graph_annotation_str = Api().ort.GetRunConfigEntry(run_options, kOrtRunOptionsConfigCudaGraphAnnotation); @@ -226,11 +229,13 @@ OrtStatus* ORT_API_CALL Ep::OnRunStartImpl(_In_ OrtEp* this_ptr, status.ErrorMessage().c_str()); } return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END } OrtStatus* ORT_API_CALL Ep::OnRunEndImpl(_In_ OrtEp* this_ptr, _In_ const OrtRunOptions* /*run_options*/, _In_ bool sync_stream) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN auto* ep = static_cast(this_ptr); auto status = ep->EpImpl()->OnRunEnd(sync_stream, {}); if (!status.IsOK()) { @@ -238,6 +243,7 @@ OrtStatus* ORT_API_CALL Ep::OnRunEndImpl(_In_ OrtEp* this_ptr, status.ErrorMessage().c_str()); } return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END } OrtStatus* ORT_API_CALL Ep::IsConcurrentRunSupportedImpl(_In_ OrtEp* /*this_ptr*/, _Out_ bool* is_concurrent_run_supported) noexcept { @@ -248,6 +254,7 @@ OrtStatus* ORT_API_CALL Ep::IsConcurrentRunSupportedImpl(_In_ OrtEp* /*this_ptr* OrtStatus* ORT_API_CALL Ep::CreateAllocatorImpl(_In_ OrtEp* this_ptr, _In_ const OrtMemoryInfo* memory_info, _Outptr_result_maybenull_ OrtAllocator** allocator) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN auto* ep = static_cast(this_ptr); Ort::ConstMemoryInfo ort_memory_info{memory_info}; if (ort_memory_info.GetAllocatorType() == OrtReadOnlyAllocator) { @@ -256,6 +263,7 @@ OrtStatus* ORT_API_CALL Ep::CreateAllocatorImpl(_In_ OrtEp* this_ptr, *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, ep->config_.device_allocator); } return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END } } // namespace ep diff --git a/onnxruntime/core/providers/webgpu/ep/factory.cc b/onnxruntime/core/providers/webgpu/ep/factory.cc index 4451ec704448a..31fc2593ec10d 100644 --- a/onnxruntime/core/providers/webgpu/ep/factory.cc +++ b/onnxruntime/core/providers/webgpu/ep/factory.cc @@ -81,6 +81,7 @@ OrtStatus* ORT_API_CALL Factory::GetSupportedDevicesImpl( OrtEpDevice** ep_devices, size_t max_ep_devices, size_t* p_num_ep_devices) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN auto factory = static_cast(this_ptr); size_t& num_ep_devices = *p_num_ep_devices; @@ -101,6 +102,7 @@ OrtStatus* ORT_API_CALL Factory::GetSupportedDevicesImpl( } return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END } OrtStatus* ORT_API_CALL Factory::CreateEpImpl( @@ -111,6 +113,7 @@ OrtStatus* ORT_API_CALL Factory::CreateEpImpl( const OrtSessionOptions* session_options, const OrtLogger* logger, OrtEp** ep) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN if (num_devices == 0) { return Api().ort.CreateStatus(ORT_INVALID_ARGUMENT, "No hardware devices provided to create WebGPU EP."); } @@ -131,7 +134,6 @@ OrtStatus* ORT_API_CALL Factory::CreateEpImpl( } } - EXCEPTION_TO_RETURNED_STATUS_BEGIN auto webgpu_ep_factory = WebGpuProviderFactoryCreator::Create(config_options); auto webgpu_ep = webgpu_ep_factory->CreateProvider(*session_options, *logger); static_cast(webgpu_ep.get())->SetEpLogger(logger); @@ -150,6 +152,7 @@ OrtStatus* ORT_API_CALL Factory::CreateAllocatorImpl( const OrtMemoryInfo* memory_info, const OrtKeyValuePairs* /*allocator_options*/, OrtAllocator** allocator) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN auto factory = static_cast(this_ptr); Ort::ConstMemoryInfo ort_memory_info{memory_info}; if (ort_memory_info.GetAllocatorType() == OrtReadOnlyAllocator) { @@ -158,6 +161,7 @@ OrtStatus* ORT_API_CALL Factory::CreateAllocatorImpl( *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, factory->config_.device_allocator); } return nullptr; + EXCEPTION_TO_RETURNED_STATUS_END } void ORT_API_CALL Factory::ReleaseAllocatorImpl(OrtEpFactory* /*this_ptr*/, OrtAllocator* allocator) noexcept { @@ -183,9 +187,11 @@ OrtStatus* ORT_API_CALL Factory::CreateSyncStreamForDeviceImpl( const OrtMemoryDevice* /*memory_device*/, const OrtKeyValuePairs* /*stream_options*/, OrtSyncStreamImpl** stream) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN *stream = nullptr; return Api().ort.CreateStatus(ORT_NOT_IMPLEMENTED, "CreateSyncStreamForDevice is not implemented for this EP factory."); + EXCEPTION_TO_RETURNED_STATUS_END } } // namespace ep From ea5a74a6980f0461ecd47b0ff028775afd9a90af Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 13 Mar 2026 22:33:05 -0700 Subject: [PATCH 14/27] resolve comments (part.5) --- onnxruntime/core/providers/webgpu/ep/ep.h | 1 - 1 file changed, 1 deletion(-) diff --git a/onnxruntime/core/providers/webgpu/ep/ep.h b/onnxruntime/core/providers/webgpu/ep/ep.h index c867cea1ec5be..815623025f8a4 100644 --- a/onnxruntime/core/providers/webgpu/ep/ep.h +++ b/onnxruntime/core/providers/webgpu/ep/ep.h @@ -28,7 +28,6 @@ class Ep : public onnxruntime::ep::adapter::Ep { AllocatorPtr initializer_allocator; }; - // Do not use a std::unique_ptr for impl_ because this requires the actual type definition. Ep(std::unique_ptr impl, Factory& factory, const OrtLogger& logger, const Config& config); inline const OrtLogger& GetOrtLogger() const noexcept { From d11b669cd34788c199ff670be6086391a7035562 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 13 Mar 2026 23:01:14 -0700 Subject: [PATCH 15/27] use onnxruntime_add_shared_library_module --- cmake/onnxruntime_providers_webgpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/onnxruntime_providers_webgpu.cmake b/cmake/onnxruntime_providers_webgpu.cmake index babf696df4915..cd29e4dad0a17 100644 --- a/cmake/onnxruntime_providers_webgpu.cmake +++ b/cmake/onnxruntime_providers_webgpu.cmake @@ -56,7 +56,7 @@ endif() source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_providers_webgpu_cc_srcs}) - onnxruntime_add_shared_library(onnxruntime_providers_webgpu ${onnxruntime_providers_webgpu_cc_srcs}) + onnxruntime_add_shared_library_module(onnxruntime_providers_webgpu ${onnxruntime_providers_webgpu_cc_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers_webgpu ${REPO_ROOT}/include/onnxruntime/core/session onnxruntime_common From 16503cd4e388241ae769d4314c5730102986f1f1 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Mon, 16 Mar 2026 14:47:08 -0700 Subject: [PATCH 16/27] build vulkan also --- .github/workflows/windows_webgpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_webgpu.yml b/.github/workflows/windows_webgpu.yml index 2ee68d74102e3..e67eda41d2e0e 100644 --- a/.github/workflows/windows_webgpu.yml +++ b/.github/workflows/windows_webgpu.yml @@ -217,7 +217,7 @@ jobs: --use_webgpu shared_lib ` --wgsl_template static ` --use_vcpkg --use_vcpkg_ms_internal_asset_cache ` - --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=ON ` + --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=ON onnxruntime_ENABLE_DAWN_BACKEND_D3D12=1 onnxruntime_ENABLE_DAWN_BACKEND_VULKAN=1 ` --disable_rtti ` --enable_lto From f509662ee3d42438e2892d8d3614cf2392994bfb Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Wed, 18 Mar 2026 18:30:18 -0700 Subject: [PATCH 17/27] allow lazy initialized allocator --- include/onnxruntime/ep/adapter/allocator.h | 36 ++++++++++++++++--- .../core/providers/webgpu/ep/factory.cc | 31 +++++++++------- .../core/providers/webgpu/ep/factory.h | 1 - 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/include/onnxruntime/ep/adapter/allocator.h b/include/onnxruntime/ep/adapter/allocator.h index 2765069ebf336..fb62186a6efc4 100644 --- a/include/onnxruntime/ep/adapter/allocator.h +++ b/include/onnxruntime/ep/adapter/allocator.h @@ -18,23 +18,49 @@ namespace adapter { /// class Allocator : public OrtAllocator { public: + /** + * Create from an existing AllocatorPtr. + */ explicit Allocator(const OrtMemoryInfo* memory_info, AllocatorPtr impl) - : OrtAllocator{}, memory_info_(memory_info), impl_(impl) { + : Allocator{memory_info} { + impl_ = impl; + } + + using AllocatorFactory = AllocatorPtr (*)(const OrtMemoryInfo& memory_info); + + /** + * Create from an AllocatorFactory, which will be called lazily when the first allocation is made. + */ + explicit Allocator(const OrtMemoryInfo* memory_info, AllocatorFactory get_allocator_impl) + : Allocator{memory_info} { + get_allocator_impl_ = get_allocator_impl; + } + + private: + explicit Allocator(const OrtMemoryInfo* memory_info) + : OrtAllocator{}, memory_info_(memory_info) { version = ORT_API_VERSION; Alloc = AllocImpl; Free = FreeImpl; Info = InfoImpl; } + AllocatorPtr GetImpl() { + if (!impl_) { + std::call_once(init_flag_, [this]() { + impl_ = get_allocator_impl_(*memory_info_); + }); + } + return impl_; + } - private: static void* ORT_API_CALL AllocImpl(OrtAllocator* this_ptr, size_t size) noexcept { auto* allocator = static_cast(this_ptr); - return allocator->impl_->Alloc(size); + return allocator->GetImpl()->Alloc(size); } static void ORT_API_CALL FreeImpl(OrtAllocator* this_ptr, void* p) noexcept { auto* allocator = static_cast(this_ptr); - allocator->impl_->Free(p); + allocator->GetImpl()->Free(p); } static const OrtMemoryInfo* ORT_API_CALL InfoImpl(const OrtAllocator* this_ptr) noexcept { @@ -44,6 +70,8 @@ class Allocator : public OrtAllocator { const OrtMemoryInfo* memory_info_; AllocatorPtr impl_; + AllocatorFactory get_allocator_impl_; + std::once_flag init_flag_; }; } // namespace adapter diff --git a/onnxruntime/core/providers/webgpu/ep/factory.cc b/onnxruntime/core/providers/webgpu/ep/factory.cc index 31fc2593ec10d..99dd0c68f6954 100644 --- a/onnxruntime/core/providers/webgpu/ep/factory.cc +++ b/onnxruntime/core/providers/webgpu/ep/factory.cc @@ -21,11 +21,6 @@ using onnxruntime::ep::Api; // Constructor Factory::Factory() : OrtEpFactory{}, - config_{ - CPUAllocator::DefaultInstance(), // CPU allocator - std::make_shared(WebGpuContextFactory::DefaultContext().BufferManager(), false), // default device allocator - std::make_shared(WebGpuContextFactory::DefaultContext().InitializerBufferManager(), true), // initializer device allocator - }, default_memory_info_{WEBGPU_BUFFER, OrtMemoryInfoDeviceType_GPU, 0, // vendor id 0, // device id @@ -138,7 +133,13 @@ OrtStatus* ORT_API_CALL Factory::CreateEpImpl( auto webgpu_ep = webgpu_ep_factory->CreateProvider(*session_options, *logger); static_cast(webgpu_ep.get())->SetEpLogger(logger); auto factory = static_cast(this_ptr); - *ep = new Ep(std::move(webgpu_ep), *factory, *logger, factory->config_); + const int context_id = webgpu_ep->GetDeviceId(); + Ep::Config webgpu_ep_config{ + CPUAllocator::DefaultInstance(), // CPU allocator + std::make_shared(WebGpuContextFactory::GetContext(context_id).BufferManager(), false), // default device allocator + std::make_shared(WebGpuContextFactory::GetContext(context_id).InitializerBufferManager(), true), // initializer device allocator + }; + *ep = new Ep(std::move(webgpu_ep), *factory, *logger, webgpu_ep_config); return nullptr; EXCEPTION_TO_RETURNED_STATUS_END } @@ -148,18 +149,24 @@ void ORT_API_CALL Factory::ReleaseEpImpl(OrtEpFactory* /*this_ptr*/, OrtEp* ep) } OrtStatus* ORT_API_CALL Factory::CreateAllocatorImpl( - OrtEpFactory* this_ptr, + OrtEpFactory* /*this_ptr*/, const OrtMemoryInfo* memory_info, const OrtKeyValuePairs* /*allocator_options*/, OrtAllocator** allocator) noexcept { EXCEPTION_TO_RETURNED_STATUS_BEGIN - auto factory = static_cast(this_ptr); Ort::ConstMemoryInfo ort_memory_info{memory_info}; - if (ort_memory_info.GetAllocatorType() == OrtReadOnlyAllocator) { - *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, factory->config_.initializer_allocator); - } else { - *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, factory->config_.device_allocator); + + if (ort_memory_info.GetAllocatorType() != OrtDeviceAllocator || + ort_memory_info.GetDeviceId() != 0 || + ort_memory_info.GetAllocatorName() != WEBGPU_BUFFER) { + return Api().ort.CreateStatus(ORT_INVALID_ARGUMENT, + "Unsupported memory info for shared allocator."); } + + *allocator = new onnxruntime::ep::adapter::Allocator(memory_info, + [](const OrtMemoryInfo&) -> AllocatorPtr { + return std::make_shared(WebGpuContextFactory::DefaultContext().BufferManager(), false); + }); return nullptr; EXCEPTION_TO_RETURNED_STATUS_END } diff --git a/onnxruntime/core/providers/webgpu/ep/factory.h b/onnxruntime/core/providers/webgpu/ep/factory.h index 86613b2a2ab11..f23b3871ebc60 100644 --- a/onnxruntime/core/providers/webgpu/ep/factory.h +++ b/onnxruntime/core/providers/webgpu/ep/factory.h @@ -63,7 +63,6 @@ class Factory : public OrtEpFactory { const OrtKeyValuePairs* stream_options, OrtSyncStreamImpl** stream) noexcept; - Ep::Config config_; Ort::MemoryInfo default_memory_info_; Ort::MemoryInfo readonly_memory_info_; // used for initializers From 741e0835efaf4f714719ea96502d3a1dd9d98ff4 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Thu, 19 Mar 2026 12:00:27 -0700 Subject: [PATCH 18/27] resolve comments (MakeEP) --- onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc | 6 +++--- onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc index 3257cfc99e245..0410a62372176 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc @@ -167,7 +167,7 @@ void Shutdown() { g_plugin_ep_infrastructure_state.reset(); } -std::unique_ptr MakeEp(const logging::Logger* logger, const ConfigOptions* config_options) { +std::unique_ptr MakeEp(const logging::Logger* logger, const ConfigOptions* ep_options) { if (!IsInitialized()) { return nullptr; } @@ -182,8 +182,8 @@ std::unique_ptr MakeEp(const logging::Logger* logger, const StrMapToKeyValueCstrVectors(state.config.default_ep_options, default_ep_option_key_cstrs, default_ep_option_value_cstrs); - if (config_options != nullptr) { - for (const auto& [key, value] : config_options->configurations) { + if (ep_options != nullptr) { + for (const auto& [key, value] : ep_options->configurations) { default_ep_option_key_cstrs.push_back(key.c_str()); default_ep_option_value_cstrs.push_back(value.c_str()); } diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h index 091946da8fc26..0962df8e35308 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h @@ -75,7 +75,8 @@ bool IsInitialized(); void Shutdown(); // Returns a dynamic plugin EP `IExecutionProvider` instance, or `nullptr` if uninitialized. -std::unique_ptr MakeEp(const logging::Logger* logger = nullptr, const ConfigOptions* config_options = nullptr); +// `ep_options` provides additional EP-specific option overrides (key-value pairs) on top of the defaults. +std::unique_ptr MakeEp(const logging::Logger* logger = nullptr, const ConfigOptions* ep_options = nullptr); // Gets the dynamic plugin EP name, or `std::nullopt` if uninitialized. std::optional GetEpName(); From 666772c524fe1c68c7c54a5708f1e26309cb367a Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Thu, 19 Mar 2026 16:21:20 -0700 Subject: [PATCH 19/27] resolve comments (MakeEP, part.2) --- onnxruntime/test/util/default_providers.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index e00dfa3d5b139..e6af2e859a4e9 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -307,11 +307,8 @@ std::unique_ptr DefaultWebGpuExecutionProvider(bool is_nhwc) webgpu::options::kPreferredLayout_NCHW) .IsOK()); } -#if defined(ORT_USE_EP_API_ADAPTERS) - return dynamic_plugin_ep_infra::MakeEp(nullptr, &config_options); -#else - return WebGpuProviderFactoryCreator::Create(config_options)->CreateProvider(); -#endif + + return WebGpuExecutionProviderWithOptions(config_options); #else ORT_UNUSED_PARAMETER(is_nhwc); return nullptr; @@ -321,6 +318,10 @@ std::unique_ptr DefaultWebGpuExecutionProvider(bool is_nhwc) std::unique_ptr WebGpuExecutionProviderWithOptions(const ConfigOptions& config_options) { #if defined(USE_WEBGPU) #if defined(ORT_USE_EP_API_ADAPTERS) + auto ep_name = dynamic_plugin_ep_infra::GetEpName(); + ORT_ENFORCE(ep_name.has_value() && *ep_name == kWebGpuExecutionProvider, + "Dynamic plugin EP is not the WebGPU EP. Expected \"", kWebGpuExecutionProvider, + "\", got \"", ep_name.value_or(""), "\""); return dynamic_plugin_ep_infra::MakeEp(nullptr, &config_options); #else return WebGpuProviderFactoryCreator::Create(config_options)->CreateProvider(); From 399c4199766996f5ca508d97cc83319594e37caa Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:28:59 -0700 Subject: [PATCH 20/27] Update onnxruntime/test/util/default_providers.cc Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com> --- onnxruntime/test/util/default_providers.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index e6af2e859a4e9..7e6bc6ae06020 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -319,7 +319,7 @@ std::unique_ptr WebGpuExecutionProviderWithOptions(const Con #if defined(USE_WEBGPU) #if defined(ORT_USE_EP_API_ADAPTERS) auto ep_name = dynamic_plugin_ep_infra::GetEpName(); - ORT_ENFORCE(ep_name.has_value() && *ep_name == kWebGpuExecutionProvider, + ORT_ENFORCE(ep_name == kWebGpuExecutionProvider, "Dynamic plugin EP is not the WebGPU EP. Expected \"", kWebGpuExecutionProvider, "\", got \"", ep_name.value_or(""), "\""); return dynamic_plugin_ep_infra::MakeEp(nullptr, &config_options); From 76fa10b2514fa8aa04d935b5432a025e67a1b3f0 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:27:02 -0700 Subject: [PATCH 21/27] resolve comments (exceptions) --- onnxruntime/core/providers/webgpu/ep/api.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/providers/webgpu/ep/api.cc b/onnxruntime/core/providers/webgpu/ep/api.cc index 885d30a9728d8..9eeb3d71df89f 100644 --- a/onnxruntime/core/providers/webgpu/ep/api.cc +++ b/onnxruntime/core/providers/webgpu/ep/api.cc @@ -35,12 +35,12 @@ extern "C" { // EXPORT_SYMBOL OrtStatus* CreateEpFactories(const char* /*registration_name*/, const OrtApiBase* ort_api_base, const OrtLogger* default_logger, - OrtEpFactory** factories, size_t max_factories, size_t* num_factories) { + OrtEpFactory** factories, size_t max_factories, size_t* num_factories) noexcept { + EXCEPTION_TO_RETURNED_STATUS_BEGIN + // Manual init for the C++ API onnxruntime::ep::ApiInit(ort_api_base); - EXCEPTION_TO_RETURNED_STATUS_BEGIN - if (max_factories < 1) { return onnxruntime::ep::Api().ort.CreateStatus(ORT_INVALID_ARGUMENT, "Not enough space to return EP factory. Need at least one."); @@ -60,7 +60,7 @@ EXPORT_SYMBOL OrtStatus* CreateEpFactories(const char* /*registration_name*/, co EXCEPTION_TO_RETURNED_STATUS_END } -EXPORT_SYMBOL OrtStatus* ReleaseEpFactory(OrtEpFactory* factory) { +EXPORT_SYMBOL OrtStatus* ReleaseEpFactory(OrtEpFactory* factory) noexcept { EXCEPTION_TO_RETURNED_STATUS_BEGIN // STEP.1 - Release the factory delete static_cast(factory); From 8fa7314d23c97acdf9be28793e3160f3c0eed945 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:28:33 -0700 Subject: [PATCH 22/27] resolve comments (forward declaration) --- onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h | 1 + 1 file changed, 1 insertion(+) diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h index 0962df8e35308..f7cf9cc383627 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h @@ -20,6 +20,7 @@ class IExecutionProvider; struct ConfigOptions; namespace logging { +struct ConfigOptions; class Logger; } // namespace logging From cdabb41c3777e044e7affeb7938051f8d97bbf04 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 20 Mar 2026 17:05:09 -0700 Subject: [PATCH 23/27] resolve comments (candidate nodes) --- onnxruntime/core/providers/webgpu/ep/ep.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/webgpu/ep/ep.cc b/onnxruntime/core/providers/webgpu/ep/ep.cc index 285fdddf11ffd..6beb62b5cf074 100644 --- a/onnxruntime/core/providers/webgpu/ep/ep.cc +++ b/onnxruntime/core/providers/webgpu/ep/ep.cc @@ -63,6 +63,7 @@ OrtStatus* ORT_API_CALL Ep::GetCapabilityImpl(OrtEp* this_ptr, const OrtGraph* g } std::vector candidate_nodes; + std::vector tentative_candidate_nodes; // For each node, check if we have a registered kernel for it for (const auto& node : all_nodes) { @@ -140,13 +141,14 @@ OrtStatus* ORT_API_CALL Ep::GetCapabilityImpl(OrtEp* this_ptr, const OrtGraph* g } candidate_nodes.push_back(node); + tentative_candidate_nodes.push_back(node); } std::unordered_set cpu_preferred_nodes; RETURN_IF_ERROR(onnxruntime::ep::GetCpuPreferredNodes(*ort_graph, *graph_support_info, static_cast(this_ptr)->GetOrtLogger(), - candidate_nodes, + tentative_candidate_nodes, cpu_preferred_nodes)); for (const auto& node : candidate_nodes) { From 74606d9ac379444a7c6bfe52921597e85adc50a3 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 20 Mar 2026 19:37:57 -0700 Subject: [PATCH 24/27] Revert "resolve comments (forward declaration)" This reverts commit 5932371b4e8758008fec136c864a0cd4950448d9. --- onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h | 1 - 1 file changed, 1 deletion(-) diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h index f7cf9cc383627..0962df8e35308 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h @@ -20,7 +20,6 @@ class IExecutionProvider; struct ConfigOptions; namespace logging { -struct ConfigOptions; class Logger; } // namespace logging From 3ad62f55fc659d68ec88939aa110f2cb59dfc2f1 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Sat, 21 Mar 2026 16:49:12 -0700 Subject: [PATCH 25/27] resolve comments (CopyTensorImpl) --- .../core/providers/webgpu/data_transfer.cc | 27 +++++++++---------- .../core/providers/webgpu/data_transfer.h | 27 +++++++++++++------ .../webgpu/webgpu_provider_factory.cc | 22 +++++++-------- 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/onnxruntime/core/providers/webgpu/data_transfer.cc b/onnxruntime/core/providers/webgpu/data_transfer.cc index 792e6cbc05d3f..5f109bf73e3c5 100644 --- a/onnxruntime/core/providers/webgpu/data_transfer.cc +++ b/onnxruntime/core/providers/webgpu/data_transfer.cc @@ -7,13 +7,7 @@ namespace onnxruntime { namespace webgpu { -bool DataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const { - return (dst_device.Type() == OrtDevice::GPU && src_device.Type() == OrtDevice::CPU) || - (dst_device.Type() == OrtDevice::GPU && src_device.Type() == OrtDevice::GPU) || - (dst_device.Type() == OrtDevice::CPU && src_device.Type() == OrtDevice::GPU); -} - -common::Status DataTransfer::CopyTensorImpl(void const* src_data, +common::Status DataTransferImpl::CopyTensor(void const* src_data, bool src_is_gpu, void* dst_data, bool dst_is_gpu, @@ -42,15 +36,18 @@ common::Status DataTransfer::CopyTensorImpl(void const* src_data, return Status::OK(); } -common::Status DataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const { - void const* src_data = src.DataRaw(); - void* dst_data = dst.MutableDataRaw(); +bool DataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const { + return (dst_device.Type() == OrtDevice::GPU && src_device.Type() == OrtDevice::CPU) || + (dst_device.Type() == OrtDevice::GPU && src_device.Type() == OrtDevice::GPU) || + (dst_device.Type() == OrtDevice::CPU && src_device.Type() == OrtDevice::GPU); +} - return CopyTensorImpl(src_data, - src.Location().device.Type() == OrtDevice::GPU, - dst_data, - dst.Location().device.Type() == OrtDevice::GPU, - src.SizeInBytes()); +common::Status DataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const { + return impl_.CopyTensor(src.DataRaw(), + src.Location().device.Type() == OrtDevice::GPU, + dst.MutableDataRaw(), + dst.Location().device.Type() == OrtDevice::GPU, + src.SizeInBytes()); } } // namespace webgpu diff --git a/onnxruntime/core/providers/webgpu/data_transfer.h b/onnxruntime/core/providers/webgpu/data_transfer.h index 29f70341989f7..e6ce92a7ca7a6 100644 --- a/onnxruntime/core/providers/webgpu/data_transfer.h +++ b/onnxruntime/core/providers/webgpu/data_transfer.h @@ -3,6 +3,7 @@ #pragma once +#include "core/common/status.h" #include "core/framework/data_transfer.h" #include "core/framework/execution_provider.h" @@ -11,23 +12,33 @@ namespace webgpu { class BufferManager; +// Low-level data transfer implementation that operates on raw pointers. +// Used by both DataTransfer (IDataTransfer subclass) and the C API data transfer wrapper. +class DataTransferImpl { + public: + DataTransferImpl(const BufferManager& buffer_manager) : buffer_manager_{buffer_manager} {}; + + common::Status CopyTensor(void const* src_data, + bool src_is_gpu, + void* dst_data, + bool dst_is_gpu, + size_t bytes) const; + + private: + const BufferManager& buffer_manager_; +}; + class DataTransfer : public IDataTransfer { public: - DataTransfer(const BufferManager& buffer_manager) : buffer_manager_{buffer_manager} {}; + DataTransfer(const BufferManager& buffer_manager) : impl_{buffer_manager} {}; ~DataTransfer() {}; bool CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const override; common::Status CopyTensor(const Tensor& src, Tensor& dst) const override; - common::Status CopyTensorImpl(void const* src_data, - bool src_is_gpu, - void* dst_data, - bool dst_is_gpu, - size_t bytes) const; - private: - const BufferManager& buffer_manager_; + DataTransferImpl impl_; }; } // namespace webgpu diff --git a/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc b/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc index c62fb29a4cb22..16899370e47f1 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc +++ b/onnxruntime/core/providers/webgpu/webgpu_provider_factory.cc @@ -362,11 +362,11 @@ struct WebGpuDataTransferImpl : OrtDataTransferImpl { auto& context = WebGpuContextFactory::DefaultContext(); - // Create the DataTransfer instance - // Note: The DataTransfer holds a const reference to BufferManager. The BufferManager's lifecycle + // Create the DataTransferImpl instance + // Note: The DataTransferImpl holds a const reference to BufferManager. The BufferManager's lifecycle // is managed by the WebGpuContext, which is stored in a static WebGpuContextFactory and persists // for the lifetime of the application, ensuring the reference remains valid. - impl.data_transfer_ = std::make_unique(context.BufferManager()); + impl.data_transfer_ = std::make_unique(context.BufferManager()); } } @@ -391,11 +391,11 @@ struct WebGpuDataTransferImpl : OrtDataTransferImpl { void* dst_data = dst_tensor.MutableDataRaw(); bool dst_is_gpu = dst_tensor.Location().device.Type() == OrtDevice::GPU; #endif - auto status = impl.data_transfer_->CopyTensorImpl(src_data, - src_is_gpu, - dst_data, - dst_is_gpu, - size); + auto status = impl.data_transfer_->CopyTensor(src_data, + src_is_gpu, + dst_data, + dst_is_gpu, + size); if (!status.IsOK()) { return OrtApis::CreateStatus(ORT_RUNTIME_EXCEPTION, status.ErrorMessage().c_str()); } @@ -419,9 +419,9 @@ struct WebGpuDataTransferImpl : OrtDataTransferImpl { const OrtApi& ort_api; const OrtEpApi& ep_api; - std::unique_ptr data_transfer_; // Lazy-initialized - int context_id_; // Track which context we're using - std::mutex init_mutex_; // Protects lazy initialization + std::unique_ptr data_transfer_; // Lazy-initialized + int context_id_; // Track which context we're using + std::mutex init_mutex_; // Protects lazy initialization }; OrtDataTransferImpl* OrtWebGpuCreateDataTransfer(int context_id /* = 0 */) { From 2e6f36e53904a4425f75ae94cb6be7be4b3720b3 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Mon, 23 Mar 2026 14:24:35 -0700 Subject: [PATCH 26/27] resolve comments --- include/onnxruntime/ep/adapter/allocator.h | 1 + onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/include/onnxruntime/ep/adapter/allocator.h b/include/onnxruntime/ep/adapter/allocator.h index fb62186a6efc4..4f107ae72c0e9 100644 --- a/include/onnxruntime/ep/adapter/allocator.h +++ b/include/onnxruntime/ep/adapter/allocator.h @@ -23,6 +23,7 @@ class Allocator : public OrtAllocator { */ explicit Allocator(const OrtMemoryInfo* memory_info, AllocatorPtr impl) : Allocator{memory_info} { + ORT_ENFORCE(impl != nullptr, "Allocator implementation cannot be null."); impl_ = impl; } diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc index 0410a62372176..1f82e1f893eab 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc @@ -11,6 +11,7 @@ #include "nlohmann/json.hpp" #include "core/common/common.h" +#include "core/framework/config_options.h" #include "core/framework/execution_provider.h" #include "core/session/abi_session_options_impl.h" #include "core/session/ort_env.h" From c11dcb38266a30c716fd30b179264350ff7290bd Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Mon, 23 Mar 2026 14:28:51 -0700 Subject: [PATCH 27/27] resolve comments --- onnxruntime/test/unittest_util/base_tester.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/onnxruntime/test/unittest_util/base_tester.cc b/onnxruntime/test/unittest_util/base_tester.cc index 18ead92ce3f18..1f744df14cfb8 100644 --- a/onnxruntime/test/unittest_util/base_tester.cc +++ b/onnxruntime/test/unittest_util/base_tester.cc @@ -749,10 +749,8 @@ void BaseTester::RunWithConfig(size_t* number_of_pre_packed_weights_counter, execution_provider = DefaultXnnpackExecutionProvider(); else if (provider_type == onnxruntime::kDmlExecutionProvider) execution_provider = DefaultDmlExecutionProvider(); -#if !defined(USE_WEBGPU) || !defined(ORT_USE_EP_API_ADAPTERS) else if (provider_type == onnxruntime::kWebGpuExecutionProvider) execution_provider = DefaultWebGpuExecutionProvider(); -#endif else if (provider_type == dynamic_plugin_ep_name) { execution_provider = dynamic_plugin_ep_infra::MakeEp(); }