From ff2d8d52f8ee37847da0724a2a4da5616d904c69 Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Sun, 14 Dec 2025 16:50:38 -0800 Subject: [PATCH 01/17] [QNN-EP] Implement file mapped weights feature - Create file mapping callback interface class - Android expected to have support in the future - Implement Windows callbacks in WindowsFileMapper - New option disable_file_mapped_weights - Feature is enabled by default with retry logic --- .../qnn/builder/onnx_ctx_model_helper.cc | 14 + .../qnn/builder/qnn_backend_manager.cc | 303 ++++++++++++-- .../qnn/builder/qnn_backend_manager.h | 36 ++ .../core/providers/qnn/builder/qnn_def.h | 6 + .../qnn_file_mapping_callback_interface.h | 45 +++ .../qnn/builder/qnn_windows_file_mapper.cc | 379 ++++++++++++++++++ .../qnn/builder/qnn_windows_file_mapper.h | 74 ++++ .../providers/qnn/qnn_execution_provider.cc | 18 +- .../providers/qnn/qnn_execution_provider.h | 1 + .../core/providers/qnn/rpcmem_library.cc | 2 + .../core/providers/qnn/rpcmem_library.h | 15 + onnxruntime/test/perftest/ort_test_session.cc | 7 +- 12 files changed, 864 insertions(+), 36 deletions(-) create mode 100644 onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h create mode 100644 onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc create mode 100644 onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h diff --git a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc index d468894080b3d..0dfb7928e3adc 100644 --- a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc +++ b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc @@ -94,6 +94,7 @@ Status GetEpContextFromMainNode(const onnxruntime::Node& main_context_node, const std::string& context_binary = node_helper.Get(EP_CACHE_CONTEXT, ""); return qnn_backend_manager->LoadCachedQnnContextFromBuffer(const_cast(context_binary.c_str()), static_cast(context_binary.length()), + "", main_context_node.Name(), qnn_models, max_spill_fill_size); @@ -127,6 +128,18 @@ Status GetEpContextFromMainNode(const onnxruntime::Node& main_context_node, return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_GRAPH, "The file path in ep_cache_context does not exist or is not accessible."); } + std::string context_binary_path_str = context_binary_path.string(); +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED + if (qnn_backend_manager->FileMappingIsEnabled()) { + return qnn_backend_manager->LoadCachedQnnContextFromBuffer(nullptr, + 0, + context_binary_path_str, + main_context_node.Name(), + qnn_models, + max_spill_fill_size); + } +#endif + size_t buffer_size{0}; std::ifstream cache_file(context_binary_path.string().c_str(), std::ifstream::binary); ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to open cache file."); @@ -144,6 +157,7 @@ Status GetEpContextFromMainNode(const onnxruntime::Node& main_context_node, cache_file.close(); return qnn_backend_manager->LoadCachedQnnContextFromBuffer(buffer.get(), static_cast(buffer_size), + context_binary_path_str, main_context_node.Name(), qnn_models, max_spill_fill_size); diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index 164e4c3157f62..40180d8166ff5 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -29,6 +29,10 @@ #include "core/providers/qnn/builder/qnn_configs_helper.h" #include "core/providers/qnn/builder/qnn_utils.h" +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#include "core/providers/qnn/builder/qnn_windows_file_mapper.h" +#endif + // Flag to determine if Backend should do node validation for each opNode added #define DO_GRAPH_NODE_VALIDATIONS 1 @@ -770,22 +774,54 @@ Status SetQnnContextConfig(ContextPriority context_priority, QnnContext_Config_t return Status::OK(); } +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +// callback required for allocating file mapping resources +static Qnn_ErrorHandle_t DmaDataProvider(Qnn_ContextBinaryDataRequest_t request, + Qnn_ContextBinaryDmaDataResponse_t* response, void* notify_param) { + if (notify_param == nullptr) { + LOGS_DEFAULT(ERROR) << "DmaProvider: notify_param is null"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + auto pair = reinterpret_cast*>(notify_param); + + if (pair->first == nullptr) { + LOGS_DEFAULT(ERROR) << "DmaProvider: file mapper is null"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + }; + + return pair->first->MapDmaData(request, response, pair->second); +} + +// callback required for releasing file mapping resources +static Qnn_ErrorHandle_t DmaDataRelease(Qnn_ContextBinaryDmaDataMem_t data_mem, void* notify_param) { + if (notify_param == nullptr) { + LOGS_DEFAULT(ERROR) << "DmaRelease: notify_param is null"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + + auto pair = reinterpret_cast*>(notify_param); + + if (pair->first == nullptr) { + LOGS_DEFAULT(ERROR) << "DmaRelease: file mapper is null"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + }; + + return pair->first->ReleaseDmaData(data_mem, pair->second); +} +#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED + // callback required to add context handles to class list // when using contextCreateFromBinaryListAsync() -void ContextCreateAsyncCallback(Qnn_ContextHandle_t context, - Qnn_GraphHandle_t graph, - const char* graphName, - QnnContext_createFromBinaryAsyncNotifyType_t notifyType, - void* notifyParam, - Qnn_ErrorHandle_t status) { +static void ContextCreateAsyncCallback(Qnn_ContextHandle_t context, + Qnn_GraphHandle_t /* graph */, + const char* /* graph_name */, + QnnContext_createFromBinaryAsyncNotifyType_t /* notify_type */, + void* notify_param, + Qnn_ErrorHandle_t /* status */) { auto qnn_backend_manager = SharedContext::GetInstance().GetSharedQnnBackendManager(); if (context) { - qnn_backend_manager->ProcessContextFromBinListAsync(context, notifyParam); - } - - if (nullptr == graphName || graph || notifyType || status) { - // Avoid compilation unused var warning error + qnn_backend_manager->ProcessContextFromBinListAsync(context, notify_param); } } @@ -809,6 +845,32 @@ void QnnBackendManager::ProcessContextFromBinListAsync(Qnn_ContextHandle_t conte } } +Status QnnBackendManager::ReadContextBinIfValid(const std::string& context_bin_filepath, + BufferInfo_t& buffer_info, + bool read_file_contents) { + std::ifstream cache_file(context_bin_filepath.c_str(), std::ifstream::binary); + ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to retrieve context binary from: ", context_bin_filepath); + + cache_file.seekg(0, cache_file.end); + size_t buffer_size = static_cast(cache_file.tellg()); + ORT_RETURN_IF(0 == buffer_size, "Empty cache file encountered."); + + std::unique_ptr buffer; + if (read_file_contents) { + cache_file.seekg(0, cache_file.beg); + buffer = std::make_unique(buffer_size); + ORT_RETURN_IF(nullptr == buffer, "Failed to allocate memory for cache file."); + + const auto& read_result = cache_file.read(buffer.get(), buffer_size); + ORT_RETURN_IF(!read_result, "Failed to read contents from cached context file."); + } + + buffer_info.data = std::move(buffer); + buffer_info.size = buffer_size; + + return Status::OK(); +} + Status QnnBackendManager::CreateContextVtcmBackupBufferSharingEnabled(std::unordered_map>>& context_bin_map) { #if QNN_API_VERSION_MAJOR == 2 && (QNN_API_VERSION_MINOR >= 26) QnnContext_Config_t context_config_resource_sharing = QNN_CONTEXT_CONFIG_INIT; @@ -845,6 +907,26 @@ Status QnnBackendManager::CreateContextVtcmBackupBufferSharingEnabled(std::unord #endif nullptr}; +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED + if (file_mapped_weights_enabled_ && file_mapper_) { + // Retry logic -- if context creation failed with file mapped weights, then retry with feature disabled + if (CreateContextFromListAsyncV2(configs, context_bin_map) != Status::OK()) { + LOGS(*logger_, WARNING) << "Failed to create context with file mapping enabled. Retrying with feature disabled."; + + file_mapped_weights_enabled_ = false; + // Destruction of file_mapper_ to prevent resource leaks + file_mapper_.reset(); + } else { + return Status::OK(); + } + } +#endif + return CreateContextFromListAsyncV1(configs, context_bin_map); +} + +Status QnnBackendManager::CreateContextFromListAsyncV1(const QnnContext_Config_t** configs, + std::unordered_map>>& context_bin_map) { std::vector context_params_list; std::vector context_paramsv1_list; std::vector context_params_ptr_list; @@ -856,20 +938,12 @@ Status QnnBackendManager::CreateContextVtcmBackupBufferSharingEnabled(std::unord for (auto& it : context_bin_map) { auto context_bin_filepath = it.first; - std::ifstream cache_file(context_bin_filepath.c_str(), std::ifstream::binary); - ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to retrieve context binary from: ", context_bin_filepath); + BufferInfo_t buffer_info; + ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info, true)); - cache_file.seekg(0, cache_file.end); - size_t buffer_size = static_cast(cache_file.tellg()); - ORT_RETURN_IF(0 == buffer_size, "Empty cache file encountered."); + std::unique_ptr buffer = std::move(buffer_info.data); + size_t buffer_size = buffer_info.size; - cache_file.seekg(0, cache_file.beg); - std::unique_ptr buffer = std::make_unique(buffer_size); - ORT_RETURN_IF(nullptr == buffer, "Failed to allocate memory for cache file."); - const auto& read_result = cache_file.read(buffer.get(), buffer_size); - ORT_RETURN_IF(!read_result, "Failed to read contents from cached context file."); - - cache_file.close(); QnnContext_ParamsV1_t context_params_v1 = {nullptr, buffer.get(), buffer_size, @@ -901,6 +975,88 @@ Status QnnBackendManager::CreateContextVtcmBackupBufferSharingEnabled(std::unord return Status::OK(); } +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +Status QnnBackendManager::CreateContextFromListAsyncV2(const QnnContext_Config_t** configs, + std::unordered_map>>& context_bin_map) { + std::vector context_params_list; + std::vector context_paramsv2_list; + std::vector context_callbacks_list; + std::vector context_params_ptr_list; + + std::vector> file_mapping_notify_param_list; + std::vector buffer_list; + + context_params_list.reserve(context_bin_map.size()); + context_callbacks_list.reserve(context_bin_map.size()); + context_params_ptr_list.reserve(context_bin_map.size() + 1); + file_mapping_notify_param_list.reserve(context_bin_map.size()); + + for (auto& it : context_bin_map) { + auto context_bin_filepath = it.first; + + BufferInfo_t buffer_info; + ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info, false)); + + size_t buffer_size = buffer_info.size; + ORT_RETURN_IF(buffer_size == 0, "Context bin has a size of 0 bytes: ", context_bin_filepath); + + void* buffer = nullptr; + void* file_mapping_handle = nullptr; + ORT_RETURN_IF_ERROR(file_mapper_->MapContextBin(context_bin_filepath, &file_mapping_handle)); + ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappingPointer(context_bin_filepath, &buffer)); + + auto& notify_param = file_mapping_notify_params_.emplace_back(file_mapper_.get(), + file_mapping_handle); + + Qnn_ContextBinaryCallback_t context_file_map_callbacks; + context_file_map_callbacks.type = QNN_CONTEXT_CALLBACK_DMA_BUFFER; + context_file_map_callbacks.dmaBufferCallback.version = QNN_CONTEXT_CALLBACK_DMA_BUFFER_VERSION_1; + context_file_map_callbacks.dmaBufferCallback.v1.dataProvide = DmaDataProvider; + context_file_map_callbacks.dmaBufferCallback.v1.dataRelease = DmaDataRelease; + context_file_map_callbacks.dmaBufferCallback.v1.notifyParam = reinterpret_cast(¬ify_param); + + // Callbacks require QnnContext_ParamsV2_t which is new to QNN API 2.31 + QnnContext_ParamsV2_t context_params_v2 = {nullptr, + buffer, + buffer_size, + nullptr, + ContextCreateAsyncCallback, + it.second.get(), + &context_file_map_callbacks}; + + QnnContext_Params_t context_params = {QnnContext_ParamsVersion_t::QNN_CONTEXT_PARAMS_VERSION_2, + {}}; + context_params.v2 = &context_params_v2; + + buffer_list.push_back(buffer); + context_params_list.push_back(std::move(context_params)); + context_callbacks_list.push_back(std::move(context_file_map_callbacks)); + context_paramsv2_list.push_back(std::move(context_params_v2)); + context_params_ptr_list.push_back(&(context_params_list.back())); + file_mapping_notify_param_list.push_back(std::move(notify_param)); + } + context_params_ptr_list.push_back(nullptr); + auto result = qnn_interface_.contextCreateFromBinaryListAsync(backend_handle_, + device_handle_, + context_params_ptr_list.data(), + configs, + nullptr); + file_mapping_notify_param_list.clear(); + context_params_ptr_list.clear(); + context_callbacks_list.clear(); + context_paramsv2_list.clear(); + context_params_list.clear(); + + for (auto& buffer : buffer_list) { + ORT_RETURN_IF_ERROR(file_mapper_->FreeContextBinMappingPointer(buffer)); + } + buffer_list.clear(); + + ORT_RETURN_IF(QNN_CONTEXT_NO_ERROR != result, "Failed to create context. Error: ", QnnErrorHandleToString(result), ", Code:", result); + return Status::OK(); +} +#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED + Status QnnBackendManager::SetContextPriority(ContextPriority context_priority) { QnnContext_Config_t context_priority_config = QNN_CONTEXT_CONFIG_INIT; ORT_RETURN_IF_ERROR(SetQnnContextConfig(context_priority, context_priority_config)); @@ -1098,6 +1254,7 @@ Status QnnBackendManager::GetMaxSpillFillBufferSize(unsigned char* buffer, } Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t buffer_length, + const std::string& context_bin_filepath, std::string node_name, QnnModelLookupTable& qnn_models, int64_t max_spill_fill_size) { @@ -1106,6 +1263,28 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t nullptr == qnn_sys_interface_.systemContextFree; ORT_RETURN_IF(result, "Failed to get valid function pointer."); + void* bin_buffer = nullptr; +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED + void* file_mapping_handle; + if (buffer == nullptr) { + ORT_RETURN_IF(!file_mapped_weights_enabled_, "Attempting to load QNN context from buffer but buffer is null"); + ORT_RETURN_IF(!file_mapper_, "Attemping to use File Mapping feature but file_mapper_ is uninitialized"); + + BufferInfo_t buffer_info; + ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info, false)); + buffer_length = buffer_info.size; + + ORT_RETURN_IF(buffer_length == 0, "Context bin has a size of 0 bytes", context_bin_filepath); + ORT_RETURN_IF_ERROR(file_mapper_->MapContextBin(context_bin_filepath, &file_mapping_handle)); + ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappingPointer(context_bin_filepath, &bin_buffer)); + + } else { + bin_buffer = static_cast(buffer); + } +#else + bin_buffer = static_cast(buffer); +#endif + QnnSystemContext_Handle_t sys_ctx_handle = nullptr; auto rt = qnn_sys_interface_.systemContextCreate(&sys_ctx_handle); ORT_RETURN_IF(QNN_SUCCESS != rt, "Failed to create system handle."); @@ -1113,7 +1292,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t const QnnSystemContext_BinaryInfo_t* binary_info = nullptr; Qnn_ContextBinarySize_t binary_info_size{0}; rt = qnn_sys_interface_.systemContextGetBinaryInfo(sys_ctx_handle, - static_cast(buffer), + bin_buffer, buffer_length, &binary_info, &binary_info_size); @@ -1188,6 +1367,25 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t ORT_RETURN_IF(nullptr == qnn_interface_.contextCreateFromBinary, "Invalid function pointer for contextCreateFromBinary."); +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED + Qnn_ContextBinaryCallback_t callbacks; + if (file_mapped_weights_enabled_ && file_mapper_) { + ORT_RETURN_IF(nullptr == qnn_interface_.contextCreateFromBinaryWithCallback, + "Invalid function pointer for contextCreateFromBinaryWithCallback."); + + auto& notify_param = file_mapping_notify_params_.emplace_back(file_mapper_.get(), + file_mapping_handle); + + callbacks.type = QNN_CONTEXT_CALLBACK_DMA_BUFFER; + callbacks.dmaBufferCallback.version = QNN_CONTEXT_CALLBACK_DMA_BUFFER_VERSION_1; + callbacks.dmaBufferCallback.v1.dataProvide = DmaDataProvider; + callbacks.dmaBufferCallback.v1.dataRelease = DmaDataRelease; + callbacks.dmaBufferCallback.v1.notifyParam = reinterpret_cast(¬ify_param); + } +#else + ORT_UNUSED_PARAMETER(context_bin_filepath); +#endif + qnn::profile::ProfilingInfo profiling_info; #ifdef QNN_SYSTEM_PROFILE_API_ENABLED if (ProfilingEnabled()) { @@ -1195,13 +1393,47 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t } #endif - rt = qnn_interface_.contextCreateFromBinary(backend_handle_, - device_handle_, - context_configs, - static_cast(buffer), - buffer_length, - &context, - profile_backend_handle_); +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED + std::unique_ptr backup_buffer; + if (file_mapped_weights_enabled_ && file_mapper_) { + rt = qnn_interface_.contextCreateFromBinaryWithCallback(backend_handle_, + device_handle_, + context_configs, + &callbacks, + bin_buffer, + buffer_length, + &context, + profile_backend_handle_, + NULL); + + ORT_RETURN_IF_ERROR(file_mapper_->FreeContextBinMappingPointer(bin_buffer)); + + if (rt != QNN_SUCCESS) { + LOGS(*logger_, WARNING) << "Failed to create context with file mapping enabled. Retrying with feature disabled."; + + file_mapped_weights_enabled_ = false; + // Destruction of file_mapper_ to prevent resource leaks + file_mapper_.reset(); + + // Read context bin from file since file mapping has failed + BufferInfo_t buffer_info; + ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info, true)); + backup_buffer = std::move(buffer_info.data); + + bin_buffer = static_cast(backup_buffer.get()); + } + } +#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED + + if (!file_mapped_weights_enabled_ || rt != QNN_SUCCESS) { + rt = qnn_interface_.contextCreateFromBinary(backend_handle_, + device_handle_, + context_configs, + bin_buffer, + buffer_length, + &context, + profile_backend_handle_); + } #ifdef QNN_SYSTEM_PROFILE_API_ENABLED if (ProfilingEnabled()) { @@ -1249,6 +1481,7 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, bool need_load_system_lib, bool share_ep_contexts, bool enable_vtcm_backup_buffer_sharing, + bool enable_file_mapped_weights, std::unordered_map>>& context_bin_map) { std::lock_guard lock(logger_recursive_mutex_); if (backend_setup_completed_) { @@ -1280,6 +1513,7 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, return Status::OK(); } + file_mapped_weights_enabled_ = enable_file_mapped_weights; vtcm_backup_buffer_sharing_enabled_ = enable_vtcm_backup_buffer_sharing; Status status = Status::OK(); @@ -1335,6 +1569,12 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, LOGS(logger, VERBOSE) << "LoadOpPackage succeed."; } +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED + if (file_mapped_weights_enabled_ && !file_mapper_ && GetQnnBackendType() == QnnBackendType::HTP) { + file_mapper_ = std::make_shared(logger); + } +#endif + bool enable_htp_weight_sharing = false; if (share_ep_contexts && !load_from_cached_context) { #if defined(__aarch64__) || defined(_M_ARM64) @@ -1529,7 +1769,6 @@ void QnnBackendManager::ReleaseResources() { } backend_setup_completed_ = false; - return; } diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index f1c6c19bb1311..25a8e70420646 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -32,6 +32,10 @@ #include "core/providers/qnn/builder/qnn_profile_serializer.h" #include "core/providers/qnn/builder/qnn_node_group/qnn_node_group.h" +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#include "core/providers/qnn/builder/qnn_file_mapping_callback_interface.h" +#endif + namespace onnxruntime { namespace qnn { @@ -154,6 +158,7 @@ class QnnBackendManager : public std::enable_shared_from_this std::unique_ptr GetContextBinaryBuffer(uint64_t& written_buffer_size); Status LoadCachedQnnContextFromBuffer(char* buffer, uint64_t buffer_length, + const std::string& context_bin_filepath, std::string node_name, std::unordered_map>& qnn_models, int64_t max_spill_fill_size); @@ -163,6 +168,7 @@ class QnnBackendManager : public std::enable_shared_from_this Status SetupBackend(const logging::Logger& logger, bool load_from_cached_context, bool need_load_system_lib, bool share_ep_contexts, bool enable_vtcm_backup_buffer_sharing, + bool enable_file_mapped_weights, std::unordered_map>>& context_bin_map); Status CreateHtpPowerCfgId(uint32_t deviceId, uint32_t coreId, uint32_t& htp_power_config_id); @@ -246,7 +252,14 @@ class QnnBackendManager : public std::enable_shared_from_this bool ProfilingEnabled() { return profiling_enabled_; } #endif + bool FileMappingIsEnabled() { return file_mapped_weights_enabled_; }; + private: + typedef struct BufferInfo { + std::unique_ptr data; + size_t size; + } BufferInfo_t; + Status LoadBackend(); Status InitializeBackend(); @@ -263,9 +276,23 @@ class QnnBackendManager : public std::enable_shared_from_this Status CreateContext(bool enable_htp_weight_sharing); + Status ReadContextBinIfValid(const std::string& context_bin_filepath, + BufferInfo_t& buffer_info, + bool read_file_contents); + Status CreateContextVtcmBackupBufferSharingEnabled(std::unordered_map>>& context_bin_map); + Status CreateContextFromListAsyncV1(const QnnContext_Config_t** configs, + std::unordered_map>>& context_bin_map); + +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED + Status CreateContextFromListAsyncV2(const QnnContext_Config_t** configs, + std::unordered_map>>& context_bin_map); +#endif + Status ReleaseContext(); // Sets the ORT logger and creates a corresponding QNN logger with the same log level. @@ -451,6 +478,15 @@ class QnnBackendManager : public std::enable_shared_from_this bool context_created_ = false; bool backend_setup_completed_ = false; bool vtcm_backup_buffer_sharing_enabled_ = false; + bool file_mapped_weights_enabled_ = false; + +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED + std::shared_ptr file_mapper_ = nullptr; + // Notify params for file mapping must persist throughout lifetime of + // QnnBackendManager for release of DMA data callback on destruction + std::vector> file_mapping_notify_params_; +#endif + // NPU backend requires quantized model QnnBackendType qnn_backend_type_ = QnnBackendType::CPU; Qnn_ProfileHandle_t profile_backend_handle_ = nullptr; diff --git a/onnxruntime/core/providers/qnn/builder/qnn_def.h b/onnxruntime/core/providers/qnn/builder/qnn_def.h index 625166f62d166..8ddd8e6d65c25 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_def.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_def.h @@ -19,6 +19,12 @@ namespace qnn { #define QNN_SYSTEM_PROFILE_API_ENABLED #endif +#if defined(_WIN32) && (defined(__aarch64__) || defined(_M_ARM64)) +#if QNN_API_VERSION_MAJOR > 2 || ((QNN_API_VERSION_MAJOR) == 2 && (QNN_API_VERSION_MINOR >= 32)) +#define QNN_FILE_MAPPED_WEIGHTS_ENABLED +#endif +#endif + // QNN only support subset of POSIX of dlopen/dlsym/dladdr/dlerror/dlclose // except the following flags for dlopen, others should be done only // when we really need them diff --git a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h new file mode 100644 index 0000000000000..735995abfb6fa --- /dev/null +++ b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/qnn/builder/qnn_def.h" +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED + +#include + +#include + +#include "core/providers/qnn/ort_api.h" + +namespace onnxruntime { +namespace qnn { + +class FileMappingCallbackInterface { + public: + virtual ~FileMappingCallbackInterface() = default; + virtual Status MapContextBin(const std::string& bin_filepath, + void** notify_param) = 0; + virtual Status ReleaseContextBin(const std::string& model_name) = 0; + + virtual Status GetContextBinMappingPointer(const std::string& bin_filepath, void** mapping_ptr) = 0; + + virtual Status FreeContextBinMappingPointer(LPVOID bin_mapping_pointer) = 0; + + virtual Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, + Qnn_ContextBinaryDmaDataResponse_t* _response, + void* notify_param) = 0; + virtual Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, + void* notify_param) = 0; + + virtual Qnn_ErrorHandle_t MapRawData(Qnn_ContextBinaryDataRequest_t request, + Qnn_ContextBinaryRawDataResponse_t* _response, + void* notify_param) = 0; + virtual Qnn_ErrorHandle_t ReleaseRawData(Qnn_ContextBinaryRawDataMem_t data_mem, + void* notify_param) = 0; +}; + +} // namespace qnn +} // namespace onnxruntime + +#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED \ No newline at end of file diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc new file mode 100644 index 0000000000000..458c0e3eaaed3 --- /dev/null +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc @@ -0,0 +1,379 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/qnn/builder/qnn_windows_file_mapper.h" +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED + +#include + +#include + +#include "core/providers/qnn/ort_api.h" +#include "core/providers/qnn/rpcmem_library.h" + +namespace onnxruntime { +namespace qnn { + +WindowsFileMapper::WindowsFileMapper(const logging::Logger& logger) : logger_(&logger) { +} + +// Close all handles and registered buffers +// Use LOGS_DEFAULT here as this function will be called during destruction of QnnBackendManager +// At time of destruction. Usage of logger_ will not be available and will result in a seg fault +WindowsFileMapper::~WindowsFileMapper() { + std::lock_guard lock(map_mutex_); + + // Ideally, there should be nothing to clean up at this point + // but free any resources anyway if applicable + if (!mapping_handle_to_info_map_.empty() || !context_bin_map_view_pointers_.empty()) { + LOGS_DEFAULT(WARNING) << "File mapping resources still exist. Attempting to free all resources."; + } + + context_bin_to_mapping_handle_map_.clear(); + + for (auto& mapview_ptr : context_bin_map_view_pointers_) { + CleanUpDataMapping(mapview_ptr, nullptr, 0); + } + + for (auto& kv : mapping_handle_to_info_map_) { + HANDLE file_mapping_handle = kv.first; + auto& mapping_info = kv.second; + + CleanUpDataMappings(mapping_info.mapped_data); + CloseHandles(mapping_info.file_handle, file_mapping_handle); + } + mapping_handle_to_info_map_.clear(); +} + +Status WindowsFileMapper::MapContextBin(const std::string& bin_filepath, + void** notify_param) { + LOGS(*logger_, INFO) << "Creating context bin file mapping for " + << bin_filepath; + + ORT_RETURN_IF(bin_filepath.empty(), "Context bin file path is empty"); + + std::lock_guard lock(map_mutex_); + + HANDLE file_handle = CreateFileA(bin_filepath.c_str(), + GENERIC_READ, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + NULL); + ORT_RETURN_IF(file_handle == INVALID_HANDLE_VALUE, + "Failed to create file handle for context bin", + bin_filepath); + + LOGS(*logger_, VERBOSE) << "Created file handle (" << file_handle << ") for context bin: " + << bin_filepath; + + HANDLE file_mapping_handle = CreateFileMappingA(file_handle, NULL, PAGE_READONLY, 0x00, 0x00, NULL); + ORT_RETURN_IF(file_mapping_handle == INVALID_HANDLE_VALUE, + "Failed to create file mapping for context bin ", bin_filepath); + + LOGS(*logger_, INFO) << "Created file mapping with handle (" << file_mapping_handle << ") for context bin:" + << bin_filepath; + + auto inserted = context_bin_to_mapping_handle_map_[bin_filepath] = file_mapping_handle; + if (!inserted) { + CloseHandles(file_handle, file_mapping_handle); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to add file handle mapping for context bin: ", + bin_filepath); + } + mapping_handle_to_info_map_.insert({file_mapping_handle, {file_handle, {}}}); + + *notify_param = reinterpret_cast(file_mapping_handle); + return Status::OK(); +} + +Status WindowsFileMapper::ReleaseContextBin(const std::string& bin_filepath) { + LOGS(*logger_, INFO) << "Removing context bin file mapping for " + << bin_filepath; + std::lock_guard lock(map_mutex_); + auto status = Status::OK(); + + auto bin_map_it = std::find_if(context_bin_to_mapping_handle_map_.begin(), + context_bin_to_mapping_handle_map_.end(), + [&bin_filepath](const auto& kv) { + return kv.first == bin_filepath; + }); + + if (bin_map_it == context_bin_to_mapping_handle_map_.end()) { + LOGS(*logger_, VERBOSE) << "File handle does not exist for " << bin_filepath; + return status; + } + + HANDLE file_mapping_handle = bin_map_it->second; + auto mapping_it = std::find_if(mapping_handle_to_info_map_.begin(), + mapping_handle_to_info_map_.end(), + [file_mapping_handle](const auto& kv) { + return kv.first == file_mapping_handle; + }); + + HANDLE file_handle = nullptr; + auto it = mapping_handle_to_info_map_.find(file_mapping_handle); + if (it == mapping_handle_to_info_map_.end()) { + status = ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "File mapping information does not exist for file mapping handle: ", + file_mapping_handle, ", context bin: ", bin_filepath); + } else { + MappingInfo_t& mapping_info = it->second; + file_handle = mapping_info.file_handle; + auto mapped_data = mapping_info.mapped_data; + + if (!mapped_data.empty()) { + LOGS(*logger_, WARNING) << "Attemping to remove context bin: " << bin_filepath + << ", but data regions still need to be unmapped. " + << "Proceeding with unmapping."; + CleanUpDataMappings(mapped_data); + } + } + + // Will ignore handles that are null + CloseHandles(file_handle, file_mapping_handle); + ORT_UNUSED_PARAMETER(mapping_handle_to_info_map_.erase(file_mapping_handle)); + ORT_UNUSED_PARAMETER(context_bin_to_mapping_handle_map_.erase(bin_filepath)); + + return Status::OK(); +} + +Status WindowsFileMapper::GetContextBinMappingPointer(const std::string& bin_filepath, void** mapping_ptr) { + LOGS(*logger_, INFO) << "Creating mapping pointer for " << bin_filepath; + + std::lock_guard lock(map_mutex_); + auto it = std::find_if(context_bin_to_mapping_handle_map_.begin(), + context_bin_to_mapping_handle_map_.end(), + [&bin_filepath](const auto& kv) { + return kv.first == bin_filepath; + }); + + ORT_RETURN_IF(it == context_bin_to_mapping_handle_map_.end(), + "Failed to create mapping pointer: File mapping does not exist for ", + bin_filepath); + + HANDLE& file_mapping_handle = it->second; + + LPVOID mapview_ptr = MapViewOfFile(file_mapping_handle, + FILE_MAP_READ, + 0, 0, 0); + + ORT_RETURN_IF(mapview_ptr == nullptr, "Failed to create mapping pointer for ", bin_filepath); + + if (!context_bin_map_view_pointers_.insert(mapview_ptr).second) { + LOGS(*logger_, ERROR) << "Unable to insert mapping pointer " << mapview_ptr << " into set"; + if (!UnmapViewOfFile(mapview_ptr)) { + LOGS(*logger_, ERROR) << "Failed to unmap mapping pointer: " << mapview_ptr; + } + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create mapping pointer for ", bin_filepath); + } + + *mapping_ptr = mapview_ptr; + LOGS(*logger_, INFO) << "Created mapping pointer (" << mapping_ptr << ") for " << bin_filepath; + return Status::OK(); +} + +Status WindowsFileMapper::FreeContextBinMappingPointer(LPVOID bin_mapping_pointer) { + LOGS(*logger_, INFO) << "Releasing mapping pointer " << bin_mapping_pointer; + + std::lock_guard lock(map_mutex_); + auto it = std::find_if(context_bin_map_view_pointers_.begin(), + context_bin_map_view_pointers_.end(), + [bin_mapping_pointer](const auto& pointer) { + return pointer == bin_mapping_pointer; + }); + + ORT_RETURN_IF(it == context_bin_map_view_pointers_.end(), "Mapping pointer ", + bin_mapping_pointer, " cannot be found and is invalid"); + + ORT_RETURN_IF(!UnmapViewOfFile(bin_mapping_pointer), "Failed to free mapping pointer ", bin_mapping_pointer); + ORT_UNUSED_PARAMETER(context_bin_map_view_pointers_.erase(bin_mapping_pointer)); + return Status::OK(); +} + +Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t request, + Qnn_ContextBinaryDmaDataResponse_t* response, + void* notify_param) { + if (notify_param == nullptr) { + LOGS(*logger_, ERROR) << "Attempting to map DMA data for null mapping handle"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + + std::lock_guard lock(map_mutex_); + HANDLE file_mapping_handle = reinterpret_cast(notify_param); + LOGS(*logger_, INFO) << "Mapping DMA data for request: mapping handle(" + << file_mapping_handle << "), offset(" << request.offset + << "), size(" << request.size << "), isBackendMappingNeeded(" + << request.isBackendMappingNeeded << ")"; + + auto buffer_size = request.size; + if (buffer_size == 0 || !request.isBackendMappingNeeded) { + LOGS(*logger_, ERROR) << "Mapping request size must be > 0 with backend mapping required"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + + auto it = mapping_handle_to_info_map_.find(file_mapping_handle); + if (it == mapping_handle_to_info_map_.end()) { + LOGS(*logger_, ERROR) << "File mapping info not found for mapping handle: " << file_mapping_handle; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + MappingInfo_t& mapping_info = it->second; + + // Align to nearest granularity boundary + SYSTEM_INFO sys_info; + GetSystemInfo(&sys_info); + Qnn_ContextBinarySize_t granularity = sys_info.dwAllocationGranularity; + SIZE_T aligned_offset = request.offset & ~(granularity - 1); + SIZE_T delta = request.offset - aligned_offset; + + LPVOID aligned_data_ptr = MapViewOfFile(file_mapping_handle, + FILE_MAP_READ, + (aligned_offset >> 32), + (aligned_offset & 0xFFFFFFFF), + (buffer_size + delta)); + + if (aligned_data_ptr == nullptr) { + LOGS(*logger_, ERROR) << "Failed to map DMA data for file mapping handle "; + return QNN_COMMON_ERROR_SYSTEM; + } + + LPVOID unaligned_data_ptr = static_cast(aligned_data_ptr) + delta; + LOGS(*logger_, INFO) << "Created DMA data mapping with: address(" << aligned_data_ptr + << "), aligned offset(" << aligned_offset << "), delta(" << delta + << "), unaligned address(" << unaligned_data_ptr << ")"; + + rpcmem_lib_.Api().register_buf(unaligned_data_ptr, buffer_size, NULL, + rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); + + auto fd = rpcmem_lib_.Api().to_fd(unaligned_data_ptr); + if (fd == -1) { + LOGS(*logger_, ERROR) << "Failed to register DMA data mapping to RPCMEM"; + if (!UnmapViewOfFile(aligned_data_ptr)) { + LOGS(*logger_, ERROR) << "Failed to unmap DMA data with address: " << aligned_data_ptr; + } + return QNN_COMMON_ERROR_SYSTEM; + } + + mapping_info.mapped_data.insert({unaligned_data_ptr, {aligned_data_ptr, buffer_size}}); + response->dmaBuffer.fd = fd; + response->dmaBuffer.data = reinterpret_cast(unaligned_data_ptr); + response->dataStartOffset = 0; + response->alignedSize = buffer_size; + + return QNN_SUCCESS; +} + +// Use LOGS_DEFAULT here as this function will be called during destruction of QnnBackendManager +// At time of destruction. Usage of logger_ will not be available and will result in a seg fault +Qnn_ErrorHandle_t WindowsFileMapper::ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, + void* notify_param) { + if (notify_param == nullptr) { + LOGS_DEFAULT(ERROR) << "Attempting to release DMA data for null mapping handle"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + + std::lock_guard lock(map_mutex_); + HANDLE file_mapping_handle = static_cast(notify_param); + LOGS_DEFAULT(INFO) << "Releasing DMA data mapping for mapping handle(" << file_mapping_handle + << "), address(" << data_mem.dmaBuffer.data << "), size: (" + << data_mem.memSize << ")"; + + if (data_mem.dmaBuffer.data == nullptr || data_mem.memSize == 0) { + LOGS_DEFAULT(ERROR) << "Mapping release request address must not be null and size must be > 0"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + + auto mapping_info_it = mapping_handle_to_info_map_.find(file_mapping_handle); + if (mapping_info_it == mapping_handle_to_info_map_.end()) { + LOGS_DEFAULT(ERROR) << "File mapping info not found for mapping handle: " << file_mapping_handle; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + MappingInfo_t& mapping_info = mapping_info_it->second; + + LPVOID unaligned_data_ptr = reinterpret_cast(data_mem.dmaBuffer.data); + auto& mapped_data = mapping_info.mapped_data; + auto mapped_data_it = std::find_if(mapped_data.begin(), mapped_data.end(), + [unaligned_data_ptr](const auto& kv) { + return kv.first == unaligned_data_ptr; + }); + + if (mapped_data_it == mapped_data.end()) { + LOGS_DEFAULT(ERROR) << "Failed to find DMA data mapping for address: " << unaligned_data_ptr; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + + LPVOID aligned_data_ptr = mapped_data_it->second.aligned_data_ptr; + + CleanUpDataMapping(unaligned_data_ptr, aligned_data_ptr, data_mem.memSize); + if (!mapped_data.erase(unaligned_data_ptr)) { + LOGS_DEFAULT(WARNING) << "Possible leak: failed to remove unordered_map entry for DMA data address: " + << unaligned_data_ptr; + } + + return QNN_SUCCESS; +} + +Qnn_ErrorHandle_t WindowsFileMapper::MapRawData(Qnn_ContextBinaryDataRequest_t request, + Qnn_ContextBinaryRawDataResponse_t* response, + void* notify_param) { + ORT_UNUSED_PARAMETER(request); + ORT_UNUSED_PARAMETER(response); + ORT_UNUSED_PARAMETER(notify_param); + + LOGS(*logger_, ERROR) << "File mapping for raw binary data is unsupported on Windows"; + return QNN_CONTEXT_ERROR_UNSUPPORTED_FEATURE; +} + +// Use LOGS_DEFAULT for all clean up functions below as they will be called during destruction of +// QnnBackendManagerAt time of destruction. Usage of logger_ will not be available and will result +// in a seg fault +Qnn_ErrorHandle_t WindowsFileMapper::ReleaseRawData(Qnn_ContextBinaryRawDataMem_t data_mem, + void* notify_param) { + ORT_UNUSED_PARAMETER(data_mem); + ORT_UNUSED_PARAMETER(notify_param); + + LOGS_DEFAULT(ERROR) << "File mapping for raw binary data is unsupported on Windows"; + return QNN_CONTEXT_ERROR_UNSUPPORTED_FEATURE; +} + +void WindowsFileMapper::CleanUpDataMapping(LPVOID unaligned_data_ptr, LPVOID aligned_data_ptr, + size_t buffer_size) { + if (unaligned_data_ptr) { + // Set file descriptor to -1 to signal deregistration + rpcmem_lib_.Api().register_buf(unaligned_data_ptr, buffer_size, -1, + rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); + + auto fd = rpcmem_lib_.Api().to_fd(unaligned_data_ptr); + if (fd != -1) { + LOGS_DEFAULT(ERROR) << "Failed to deregister buffer from RPCMEM: " << unaligned_data_ptr; + } + } + + if (aligned_data_ptr && !UnmapViewOfFile(aligned_data_ptr)) { + LOGS_DEFAULT(ERROR) << "Failed to unmap view of pointer: " << aligned_data_ptr; + } +} + +void WindowsFileMapper::CleanUpDataMappings(const std::unordered_map& mapped_data) { + // Key is unaligned data pointer + for (const auto& kv : mapped_data) { + auto mapped_data_info = kv.second; + // Will handle null ptrs + CleanUpDataMapping(kv.first, mapped_data_info.aligned_data_ptr, + mapped_data_info.buffer_size); + } +} + +void WindowsFileMapper::CloseHandles(HANDLE file_handle, HANDLE file_mapping_handle) { + if (file_mapping_handle && !CloseHandle(file_mapping_handle)) { + LOGS_DEFAULT(ERROR) << "Failed to close file mapping handle: " << file_mapping_handle; + } + if (file_handle && !CloseHandle(file_handle)) { + LOGS_DEFAULT(ERROR) << "Failed to close file handle: " << file_handle; + } +} + +} // namespace qnn +} // namespace onnxruntime + +#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h new file mode 100644 index 0000000000000..7c19d5ff8a9a6 --- /dev/null +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/qnn/builder/qnn_file_mapping_callback_interface.h" +#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED + +#include + +#include + +#include "core/providers/qnn/rpcmem_library.h" +#include "core/providers/qnn/ort_api.h" + +namespace onnxruntime { +namespace qnn { + +class WindowsFileMapper : public FileMappingCallbackInterface { + public: + WindowsFileMapper(const logging::Logger& logger); + ~WindowsFileMapper() override; + Status MapContextBin(const std::string& bin_filepath, + void** notify_param) override; + Status ReleaseContextBin(const std::string& model_name) override; + + Status GetContextBinMappingPointer(const std::string& bin_filepath, void** mapping_ptr) override; + + Status FreeContextBinMappingPointer(LPVOID bin_mapping_pointer) override; + + Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, + Qnn_ContextBinaryDmaDataResponse_t* response, + void* notify_param) override; + Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, + void* notify_param) override; + + Qnn_ErrorHandle_t MapRawData(Qnn_ContextBinaryDataRequest_t request, + Qnn_ContextBinaryRawDataResponse_t* response, + void* notify_param) override; + Qnn_ErrorHandle_t ReleaseRawData(Qnn_ContextBinaryRawDataMem_t data_mem, + void* notify_param) override; + + private: + typedef struct MappedDataInfo { + LPVOID aligned_data_ptr = nullptr; + size_t buffer_size = 0; + } MappedDataInfo_t; + + typedef struct MappingInfo { + HANDLE file_handle; + + // Maps unaligned data pointers to aligned data pointers + std::unordered_map mapped_data; + } MappingInfo_t; + + void CleanUpDataMapping(LPVOID unaligned_data_ptr, LPVOID aligned_data_ptr, + size_t buffer_size); + void CleanUpDataMappings(const std::unordered_map& mapped_data); + void CloseHandles(HANDLE file_handle, HANDLE file_mapping_handle); + + std::mutex map_mutex_; // Applies to both unordered maps + std::unordered_map context_bin_to_mapping_handle_map_; + std::unordered_map mapping_handle_to_info_map_; + std::unordered_set context_bin_map_view_pointers_; + + const logging::Logger* logger_; + + RpcMemLibrary rpcmem_lib_; +}; + +} // namespace qnn +} // namespace onnxruntime + +#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index 737216b81139c..a6f3b2b8590bf 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -475,6 +475,21 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio #endif } + static const std::string DISABLE_FILE_MAPPED_WEIGHTS = "disable_file_mapped_weights"; + auto disable_file_mapped_weights_pos = provider_options_map.find(DISABLE_FILE_MAPPED_WEIGHTS); + if (disable_file_mapped_weights_pos != provider_options_map.end()) { + if ("1" == disable_file_mapped_weights_pos->second) { + enable_file_mapped_weights_ = false; + } + LOGS_DEFAULT(VERBOSE) << "User specified disable_file_mapped_weights: " << enable_file_mapped_weights_; + } + +#ifndef QNN_FILE_MAPPED_WEIGHTS_ENABLED + enable_file_mapped_weights_ = false; + LOGS_DEFAULT(WARNING) << "File mapped weights feature is only available on Windows arm64 devices for QNN API versions >= 2.32. " + << "Feature will be disabled by default"; +#endif + static const std::string QNN_DEVICE_ID = "device_id"; auto dev_id_pos = provider_options_map.find(QNN_DEVICE_ID); if (dev_id_pos != provider_options_map.end()) { @@ -926,7 +941,7 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer } std::unordered_map>> context_bin_map; - if (enable_vtcm_backup_buffer_sharing_) { + if (enable_vtcm_backup_buffer_sharing_ || enable_file_mapped_weights_) { std::unordered_set ep_ctx_nodes; GetMainEPCtxNodes(graph_viewer, ep_ctx_nodes, logger); @@ -956,6 +971,7 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer context_cache_enabled_ && enable_spill_fill_buffer_, share_ep_contexts_, enable_vtcm_backup_buffer_sharing_, + enable_file_mapped_weights_, context_bin_map); context_bin_map.clear(); diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.h b/onnxruntime/core/providers/qnn/qnn_execution_provider.h index dd301d7915935..05a73304806ec 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.h +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.h @@ -119,6 +119,7 @@ class QNNExecutionProvider : public IExecutionProvider { bool share_ep_contexts_ = false; bool stop_share_ep_contexts_ = false; bool enable_spill_fill_buffer_ = false; + bool enable_file_mapped_weights_ = true; #if defined(_WIN32) onnxruntime::logging::EtwRegistrationManager::EtwInternalCallback callback_ETWSink_provider_ = nullptr; #endif diff --git a/onnxruntime/core/providers/qnn/rpcmem_library.cc b/onnxruntime/core/providers/qnn/rpcmem_library.cc index 20918f8bc6de1..f89a15157ddf4 100644 --- a/onnxruntime/core/providers/qnn/rpcmem_library.cc +++ b/onnxruntime/core/providers/qnn/rpcmem_library.cc @@ -165,6 +165,8 @@ RpcMemApi CreateApi(void* library_handle) { ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(library_handle, "rpcmem_to_fd", (void**)&api.to_fd)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(library_handle, "remote_register_buf_attr2", (void**)&api.register_buf)); + return api; } diff --git a/onnxruntime/core/providers/qnn/rpcmem_library.h b/onnxruntime/core/providers/qnn/rpcmem_library.h index 2746e147373bb..0f4b5b5391f59 100644 --- a/onnxruntime/core/providers/qnn/rpcmem_library.h +++ b/onnxruntime/core/providers/qnn/rpcmem_library.h @@ -24,6 +24,9 @@ constexpr uint32_t RPCMEM_DEFAULT_FLAGS = 1; constexpr int RPCMEM_HEAP_ID_SYSTEM = 25; +constexpr int RPCMEM_ATTR_IMPORT_BUFFER = 256; +constexpr int RPCMEM_ATTR_READ_ONLY = 512; + /** * Allocate a zero-copy buffer for size upto 2 GB with the FastRPC framework. * Buffers larger than 2 GB must be allocated with rpcmem_alloc2 @@ -46,6 +49,17 @@ using FreeFnPtr = void (*)(void* po); */ using ToFdFnPtr = int (*)(void* po); +/** + * Registers and maps a CPU buffer to RPC memory space + * @param[in] buff Data pointer for a CPU-allocated buffer + * @param[in] size Size of the buffer in bytes + * @param[in] fd File descriptor for a CPU-allocated buffer + * Note: Can be NULL if N/A or -1 to signal deregistration + * @param[in] attr Specified attributes for the buffer + * @return Data pointer for an RPCMEM-allocated buffer + */ +using RegisterBufFnPtr = void (*)(void* buff, size_t size, int fd, int attr); + } // namespace rpcmem // RPCMEM API function pointers. @@ -53,6 +67,7 @@ struct RpcMemApi { rpcmem::AllocFnPtr alloc; rpcmem::FreeFnPtr free; rpcmem::ToFdFnPtr to_fd; + rpcmem::RegisterBufFnPtr register_buf; }; // Loads and provides access to the RPCMEM API functions from a dynamically loaded library. diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 3468e2e55c7b6..16bf08415fe33 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -352,7 +352,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device "qnn_saver_path", "htp_graph_finalization_optimization_mode", "qnn_context_priority", "htp_arch", "enable_htp_fp16_precision", "offload_graph_io_quantization", "enable_htp_spill_fill_buffer", "enable_htp_shared_memory_allocator", "dump_json_qnn_graph", - "json_qnn_graph_dir"}); + "json_qnn_graph_dir", "disable_file_mapped_weights"}); for (const auto& provider_option : provider_options) { const std::string& key = provider_option.first; const std::string& value = provider_option.second; @@ -404,7 +404,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device ORT_THROW("Supported qnn_context_priority: low, normal, normal_high, high"); } } else if (key == "htp_arch") { - std::set supported_htp_archs = {"0", "68", "69", "73", "75"}; + std::set supported_htp_archs = {"0", "68", "69", "73", "75", "81"}; if (supported_htp_archs.find(value) == supported_htp_archs.end()) { std::ostringstream str_stream; std::copy(supported_htp_archs.begin(), supported_htp_archs.end(), @@ -416,7 +416,8 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device key == "offload_graph_io_quantization" || key == "enable_htp_spill_fill_buffer" || key == "enable_htp_shared_memory_allocator" || - key == "dump_json_qnn_graph") { + key == "dump_json_qnn_graph" || + key == "disable_file_mapped_weights") { std::set supported_options = {"0", "1"}; if (supported_options.find(value) == supported_options.end()) { std::ostringstream str_stream; From 5a25c35dff78632aa312c2e8a99bdbea94071218 Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Tue, 13 Jan 2026 17:03:50 -0800 Subject: [PATCH 02/17] Address PR comments --- .../core/providers/qnn/builder/qnn_backend_manager.cc | 2 +- .../core/providers/qnn/builder/qnn_backend_manager.h | 2 +- .../qnn/builder/qnn_file_mapping_callback_interface.h | 2 +- .../core/providers/qnn/builder/qnn_windows_file_mapper.cc | 8 ++++---- .../core/providers/qnn/builder/qnn_windows_file_mapper.h | 3 ++- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index 40180d8166ff5..15f6b83aa4912 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -787,7 +787,7 @@ static Qnn_ErrorHandle_t DmaDataProvider(Qnn_ContextBinaryDataRequest_t request, if (pair->first == nullptr) { LOGS_DEFAULT(ERROR) << "DmaProvider: file mapper is null"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; - }; + } return pair->first->MapDmaData(request, response, pair->second); } diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index 25a8e70420646..b38c865797e5d 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -252,7 +252,7 @@ class QnnBackendManager : public std::enable_shared_from_this bool ProfilingEnabled() { return profiling_enabled_; } #endif - bool FileMappingIsEnabled() { return file_mapped_weights_enabled_; }; + bool FileMappingIsEnabled() { return file_mapped_weights_enabled_; } private: typedef struct BufferInfo { diff --git a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h index 735995abfb6fa..567ad9596408f 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h @@ -33,7 +33,7 @@ class FileMappingCallbackInterface { void* notify_param) = 0; virtual Qnn_ErrorHandle_t MapRawData(Qnn_ContextBinaryDataRequest_t request, - Qnn_ContextBinaryRawDataResponse_t* _response, + Qnn_ContextBinaryRawDataResponse_t* response, void* notify_param) = 0; virtual Qnn_ErrorHandle_t ReleaseRawData(Qnn_ContextBinaryRawDataMem_t data_mem, void* notify_param) = 0; diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc index 458c0e3eaaed3..d78e919968f78 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc @@ -75,8 +75,8 @@ Status WindowsFileMapper::MapContextBin(const std::string& bin_filepath, LOGS(*logger_, INFO) << "Created file mapping with handle (" << file_mapping_handle << ") for context bin:" << bin_filepath; - auto inserted = context_bin_to_mapping_handle_map_[bin_filepath] = file_mapping_handle; - if (!inserted) { + auto inserted = context_bin_to_mapping_handle_map_.insert({bin_filepath, file_mapping_handle}); + if (!inserted.second) { CloseHandles(file_handle, file_mapping_handle); return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to add file handle mapping for context bin: ", bin_filepath); @@ -123,7 +123,7 @@ Status WindowsFileMapper::ReleaseContextBin(const std::string& bin_filepath) { auto mapped_data = mapping_info.mapped_data; if (!mapped_data.empty()) { - LOGS(*logger_, WARNING) << "Attemping to remove context bin: " << bin_filepath + LOGS(*logger_, WARNING) << "Attempting to remove context bin: " << bin_filepath << ", but data regions still need to be unmapped. " << "Proceeding with unmapping."; CleanUpDataMappings(mapped_data); @@ -325,7 +325,7 @@ Qnn_ErrorHandle_t WindowsFileMapper::MapRawData(Qnn_ContextBinaryDataRequest_t r } // Use LOGS_DEFAULT for all clean up functions below as they will be called during destruction of -// QnnBackendManagerAt time of destruction. Usage of logger_ will not be available and will result +// QnnBackendManager at time of destruction. Usage of logger_ will not be available and will result // in a seg fault Qnn_ErrorHandle_t WindowsFileMapper::ReleaseRawData(Qnn_ContextBinaryRawDataMem_t data_mem, void* notify_param) { diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h index 7c19d5ff8a9a6..143946a443cd7 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h @@ -7,6 +7,7 @@ #ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED #include +#include #include @@ -18,7 +19,7 @@ namespace qnn { class WindowsFileMapper : public FileMappingCallbackInterface { public: - WindowsFileMapper(const logging::Logger& logger); + explicit WindowsFileMapper(const logging::Logger& logger); ~WindowsFileMapper() override; Status MapContextBin(const std::string& bin_filepath, void** notify_param) override; From 2e451aeed311a23b6541f042cf8274d8be299096 Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Wed, 14 Jan 2026 13:23:10 -0800 Subject: [PATCH 03/17] Remove unnecessary underscore --- .../providers/qnn/builder/qnn_file_mapping_callback_interface.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h index 567ad9596408f..457899dc2afe7 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h @@ -27,7 +27,7 @@ class FileMappingCallbackInterface { virtual Status FreeContextBinMappingPointer(LPVOID bin_mapping_pointer) = 0; virtual Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, - Qnn_ContextBinaryDmaDataResponse_t* _response, + Qnn_ContextBinaryDmaDataResponse_t* response, void* notify_param) = 0; virtual Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, void* notify_param) = 0; From 8bb49a31f7c6a53ea4eb44da2365837559fda02c Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Fri, 16 Jan 2026 17:49:34 -0800 Subject: [PATCH 04/17] Adderss PR comments --- .../qnn/builder/qnn_backend_manager.cc | 43 +++++----- .../qnn/builder/qnn_backend_manager.h | 2 +- .../qnn_file_mapping_callback_interface.h | 8 +- .../qnn/builder/qnn_windows_file_mapper.cc | 81 +++++++++---------- .../providers/qnn/qnn_execution_provider.cc | 1 - 5 files changed, 59 insertions(+), 76 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index 15f6b83aa4912..ac6f6db1d0a55 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -848,16 +848,25 @@ void QnnBackendManager::ProcessContextFromBinListAsync(Qnn_ContextHandle_t conte Status QnnBackendManager::ReadContextBinIfValid(const std::string& context_bin_filepath, BufferInfo_t& buffer_info, bool read_file_contents) { - std::ifstream cache_file(context_bin_filepath.c_str(), std::ifstream::binary); - ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to retrieve context binary from: ", context_bin_filepath); + std::error_code ec; + ORT_RETURN_IF(!std::filesystem::exists(context_bin_filepath, ec), "Context binary does not exist: ", context_bin_filepath); + ORT_RETURN_IF(ec, "Failed to read context binary: ", context_bin_filepath, + ", error: ", ec.message()); - cache_file.seekg(0, cache_file.end); - size_t buffer_size = static_cast(cache_file.tellg()); - ORT_RETURN_IF(0 == buffer_size, "Empty cache file encountered."); + auto file_size = std::filesystem::file_size(context_bin_filepath, ec); + ORT_RETURN_IF(ec, "Failed to retrieve size of context binary: ", context_bin_filepath, + ", error: ", ec.message()); + ORT_RETURN_IF(file_size == 0, "Context binary is empty: ", context_bin_filepath); + ORT_RETURN_IF(file_size > SIZE_MAX, "Context binary (", context_bin_filepath, ") file size (", file_size, + " bytes) exceeds maximum value of size_t for this platform (", SIZE_MAX, " bytes)."); + + size_t buffer_size = static_cast(file_size); std::unique_ptr buffer; if (read_file_contents) { - cache_file.seekg(0, cache_file.beg); + std::ifstream cache_file(context_bin_filepath.c_str(), std::ifstream::binary); + ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to read context binary from: ", context_bin_filepath); + buffer = std::make_unique(buffer_size); ORT_RETURN_IF(nullptr == buffer, "Failed to allocate memory for cache file."); @@ -966,11 +975,6 @@ Status QnnBackendManager::CreateContextFromListAsyncV1(const QnnContext_Config_t configs, nullptr); - context_params_ptr_list.clear(); - context_paramsv1_list.clear(); - context_params_list.clear(); - buffer_list.clear(); - ORT_RETURN_IF(QNN_CONTEXT_NO_ERROR != result, "Failed to create context. Error: ", QnnErrorHandleToString(result), ", Code:", result); return Status::OK(); } @@ -983,13 +987,11 @@ Status QnnBackendManager::CreateContextFromListAsyncV2(const QnnContext_Config_t std::vector context_callbacks_list; std::vector context_params_ptr_list; - std::vector> file_mapping_notify_param_list; std::vector buffer_list; context_params_list.reserve(context_bin_map.size()); context_callbacks_list.reserve(context_bin_map.size()); context_params_ptr_list.reserve(context_bin_map.size() + 1); - file_mapping_notify_param_list.reserve(context_bin_map.size()); for (auto& it : context_bin_map) { auto context_bin_filepath = it.first; @@ -998,7 +1000,6 @@ Status QnnBackendManager::CreateContextFromListAsyncV2(const QnnContext_Config_t ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info, false)); size_t buffer_size = buffer_info.size; - ORT_RETURN_IF(buffer_size == 0, "Context bin has a size of 0 bytes: ", context_bin_filepath); void* buffer = nullptr; void* file_mapping_handle = nullptr; @@ -1015,7 +1016,7 @@ Status QnnBackendManager::CreateContextFromListAsyncV2(const QnnContext_Config_t context_file_map_callbacks.dmaBufferCallback.v1.dataRelease = DmaDataRelease; context_file_map_callbacks.dmaBufferCallback.v1.notifyParam = reinterpret_cast(¬ify_param); - // Callbacks require QnnContext_ParamsV2_t which is new to QNN API 2.31 + // Callbacks require QnnContext_ParamsV2_t which is new to QNN API 2.32 QnnContext_ParamsV2_t context_params_v2 = {nullptr, buffer, buffer_size, @@ -1033,7 +1034,6 @@ Status QnnBackendManager::CreateContextFromListAsyncV2(const QnnContext_Config_t context_callbacks_list.push_back(std::move(context_file_map_callbacks)); context_paramsv2_list.push_back(std::move(context_params_v2)); context_params_ptr_list.push_back(&(context_params_list.back())); - file_mapping_notify_param_list.push_back(std::move(notify_param)); } context_params_ptr_list.push_back(nullptr); auto result = qnn_interface_.contextCreateFromBinaryListAsync(backend_handle_, @@ -1041,11 +1041,6 @@ Status QnnBackendManager::CreateContextFromListAsyncV2(const QnnContext_Config_t context_params_ptr_list.data(), configs, nullptr); - file_mapping_notify_param_list.clear(); - context_params_ptr_list.clear(); - context_callbacks_list.clear(); - context_paramsv2_list.clear(); - context_params_list.clear(); for (auto& buffer : buffer_list) { ORT_RETURN_IF_ERROR(file_mapper_->FreeContextBinMappingPointer(buffer)); @@ -1266,8 +1261,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t void* bin_buffer = nullptr; #ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED void* file_mapping_handle; - if (buffer == nullptr) { - ORT_RETURN_IF(!file_mapped_weights_enabled_, "Attempting to load QNN context from buffer but buffer is null"); + if (file_mapped_weights_enabled_) { ORT_RETURN_IF(!file_mapper_, "Attemping to use File Mapping feature but file_mapper_ is uninitialized"); BufferInfo_t buffer_info; @@ -1279,6 +1273,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappingPointer(context_bin_filepath, &bin_buffer)); } else { + ORT_RETURN_IF(buffer == nullptr, "Attempting to load QNN context from buffer but buffer is null"); bin_buffer = static_cast(buffer); } #else @@ -1571,7 +1566,7 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, #ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED if (file_mapped_weights_enabled_ && !file_mapper_ && GetQnnBackendType() == QnnBackendType::HTP) { - file_mapper_ = std::make_shared(logger); + file_mapper_ = std::make_unique(logger); } #endif diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index b38c865797e5d..7af505cdfaf0d 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -481,7 +481,7 @@ class QnnBackendManager : public std::enable_shared_from_this bool file_mapped_weights_enabled_ = false; #ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED - std::shared_ptr file_mapper_ = nullptr; + std::unique_ptr file_mapper_ = nullptr; // Notify params for file mapping must persist throughout lifetime of // QnnBackendManager for release of DMA data callback on destruction std::vector> file_mapping_notify_params_; diff --git a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h index 457899dc2afe7..4e7cf434873c7 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h @@ -3,14 +3,12 @@ #pragma once -#include "core/providers/qnn/builder/qnn_def.h" -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED - #include #include #include "core/providers/qnn/ort_api.h" +#include "core/providers/qnn/builder/qnn_def.h" namespace onnxruntime { namespace qnn { @@ -40,6 +38,4 @@ class FileMappingCallbackInterface { }; } // namespace qnn -} // namespace onnxruntime - -#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED \ No newline at end of file +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc index d78e919968f78..c985f5569cce1 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc @@ -21,8 +21,6 @@ WindowsFileMapper::WindowsFileMapper(const logging::Logger& logger) : logger_(&l // Use LOGS_DEFAULT here as this function will be called during destruction of QnnBackendManager // At time of destruction. Usage of logger_ will not be available and will result in a seg fault WindowsFileMapper::~WindowsFileMapper() { - std::lock_guard lock(map_mutex_); - // Ideally, there should be nothing to clean up at this point // but free any resources anyway if applicable if (!mapping_handle_to_info_map_.empty() || !context_bin_map_view_pointers_.empty()) { @@ -54,6 +52,14 @@ Status WindowsFileMapper::MapContextBin(const std::string& bin_filepath, std::lock_guard lock(map_mutex_); + auto file_mapping_it = context_bin_to_mapping_handle_map_.find(bin_filepath); + if (file_mapping_it != context_bin_to_mapping_handle_map_.end()) { + LOGS(*logger_, INFO) << "Context bin file mapping already exists for " + << bin_filepath; + *notify_param = reinterpret_cast(file_mapping_it->second); + return Status::OK(); + } + HANDLE file_handle = CreateFileA(bin_filepath.c_str(), GENERIC_READ, FILE_SHARE_READ, @@ -62,15 +68,18 @@ Status WindowsFileMapper::MapContextBin(const std::string& bin_filepath, FILE_ATTRIBUTE_NORMAL, NULL); ORT_RETURN_IF(file_handle == INVALID_HANDLE_VALUE, - "Failed to create file handle for context bin", - bin_filepath); + "Failed to create file handle with error code ", + GetLastError(), " for context bin", bin_filepath); LOGS(*logger_, VERBOSE) << "Created file handle (" << file_handle << ") for context bin: " << bin_filepath; HANDLE file_mapping_handle = CreateFileMappingA(file_handle, NULL, PAGE_READONLY, 0x00, 0x00, NULL); - ORT_RETURN_IF(file_mapping_handle == INVALID_HANDLE_VALUE, - "Failed to create file mapping for context bin ", bin_filepath); + if (file_mapping_handle == INVALID_HANDLE_VALUE) { + CloseHandles(file_handle, nullptr); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create file mapping with error code ", + GetLastError(), " for context bin ", bin_filepath); + } LOGS(*logger_, INFO) << "Created file mapping with handle (" << file_mapping_handle << ") for context bin:" << bin_filepath; @@ -93,11 +102,7 @@ Status WindowsFileMapper::ReleaseContextBin(const std::string& bin_filepath) { std::lock_guard lock(map_mutex_); auto status = Status::OK(); - auto bin_map_it = std::find_if(context_bin_to_mapping_handle_map_.begin(), - context_bin_to_mapping_handle_map_.end(), - [&bin_filepath](const auto& kv) { - return kv.first == bin_filepath; - }); + auto bin_map_it = context_bin_to_mapping_handle_map_.find(bin_filepath); if (bin_map_it == context_bin_to_mapping_handle_map_.end()) { LOGS(*logger_, VERBOSE) << "File handle does not exist for " << bin_filepath; @@ -105,11 +110,6 @@ Status WindowsFileMapper::ReleaseContextBin(const std::string& bin_filepath) { } HANDLE file_mapping_handle = bin_map_it->second; - auto mapping_it = std::find_if(mapping_handle_to_info_map_.begin(), - mapping_handle_to_info_map_.end(), - [file_mapping_handle](const auto& kv) { - return kv.first == file_mapping_handle; - }); HANDLE file_handle = nullptr; auto it = mapping_handle_to_info_map_.find(file_mapping_handle); @@ -142,11 +142,7 @@ Status WindowsFileMapper::GetContextBinMappingPointer(const std::string& bin_fil LOGS(*logger_, INFO) << "Creating mapping pointer for " << bin_filepath; std::lock_guard lock(map_mutex_); - auto it = std::find_if(context_bin_to_mapping_handle_map_.begin(), - context_bin_to_mapping_handle_map_.end(), - [&bin_filepath](const auto& kv) { - return kv.first == bin_filepath; - }); + auto it = context_bin_to_mapping_handle_map_.find(bin_filepath); ORT_RETURN_IF(it == context_bin_to_mapping_handle_map_.end(), "Failed to create mapping pointer: File mapping does not exist for ", @@ -160,14 +156,7 @@ Status WindowsFileMapper::GetContextBinMappingPointer(const std::string& bin_fil ORT_RETURN_IF(mapview_ptr == nullptr, "Failed to create mapping pointer for ", bin_filepath); - if (!context_bin_map_view_pointers_.insert(mapview_ptr).second) { - LOGS(*logger_, ERROR) << "Unable to insert mapping pointer " << mapview_ptr << " into set"; - if (!UnmapViewOfFile(mapview_ptr)) { - LOGS(*logger_, ERROR) << "Failed to unmap mapping pointer: " << mapview_ptr; - } - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create mapping pointer for ", bin_filepath); - } - + ORT_UNUSED_PARAMETER(context_bin_map_view_pointers_.insert(mapview_ptr)); *mapping_ptr = mapview_ptr; LOGS(*logger_, INFO) << "Created mapping pointer (" << mapping_ptr << ") for " << bin_filepath; return Status::OK(); @@ -177,16 +166,13 @@ Status WindowsFileMapper::FreeContextBinMappingPointer(LPVOID bin_mapping_pointe LOGS(*logger_, INFO) << "Releasing mapping pointer " << bin_mapping_pointer; std::lock_guard lock(map_mutex_); - auto it = std::find_if(context_bin_map_view_pointers_.begin(), - context_bin_map_view_pointers_.end(), - [bin_mapping_pointer](const auto& pointer) { - return pointer == bin_mapping_pointer; - }); + auto it = context_bin_map_view_pointers_.find(bin_mapping_pointer); ORT_RETURN_IF(it == context_bin_map_view_pointers_.end(), "Mapping pointer ", bin_mapping_pointer, " cannot be found and is invalid"); - ORT_RETURN_IF(!UnmapViewOfFile(bin_mapping_pointer), "Failed to free mapping pointer ", bin_mapping_pointer); + ORT_RETURN_IF(!UnmapViewOfFile(bin_mapping_pointer), "Failed to free mapping pointer ", bin_mapping_pointer, + " with error code ", GetLastError()); ORT_UNUSED_PARAMETER(context_bin_map_view_pointers_.erase(bin_mapping_pointer)); return Status::OK(); } @@ -233,7 +219,8 @@ Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t r (buffer_size + delta)); if (aligned_data_ptr == nullptr) { - LOGS(*logger_, ERROR) << "Failed to map DMA data for file mapping handle "; + LOGS(*logger_, ERROR) << "Failed to map DMA data with error code " << GetLastError() + << " for file mapping handle " << file_mapping_handle; return QNN_COMMON_ERROR_SYSTEM; } @@ -249,7 +236,8 @@ Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t r if (fd == -1) { LOGS(*logger_, ERROR) << "Failed to register DMA data mapping to RPCMEM"; if (!UnmapViewOfFile(aligned_data_ptr)) { - LOGS(*logger_, ERROR) << "Failed to unmap DMA data with address: " << aligned_data_ptr; + LOGS(*logger_, ERROR) << "Failed to unmap DMA data with error code " << GetLastError() + << " with address : " << aligned_data_ptr; } return QNN_COMMON_ERROR_SYSTEM; } @@ -292,10 +280,7 @@ Qnn_ErrorHandle_t WindowsFileMapper::ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_ LPVOID unaligned_data_ptr = reinterpret_cast(data_mem.dmaBuffer.data); auto& mapped_data = mapping_info.mapped_data; - auto mapped_data_it = std::find_if(mapped_data.begin(), mapped_data.end(), - [unaligned_data_ptr](const auto& kv) { - return kv.first == unaligned_data_ptr; - }); + auto mapped_data_it = mapped_data.find(unaligned_data_ptr); if (mapped_data_it == mapped_data.end()) { LOGS_DEFAULT(ERROR) << "Failed to find DMA data mapping for address: " << unaligned_data_ptr; @@ -310,6 +295,11 @@ Qnn_ErrorHandle_t WindowsFileMapper::ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_ << unaligned_data_ptr; } + if (mapped_data.empty()) { + CloseHandles(mapping_info.file_handle, file_mapping_handle); + ORT_UNUSED_PARAMETER(mapping_handle_to_info_map_.erase(mapping_info_it)); + } + return QNN_SUCCESS; } @@ -350,7 +340,8 @@ void WindowsFileMapper::CleanUpDataMapping(LPVOID unaligned_data_ptr, LPVOID ali } if (aligned_data_ptr && !UnmapViewOfFile(aligned_data_ptr)) { - LOGS_DEFAULT(ERROR) << "Failed to unmap view of pointer: " << aligned_data_ptr; + LOGS_DEFAULT(ERROR) << "Failed to unmap view of pointer: " << aligned_data_ptr + << ", error code: " << GetLastError(); } } @@ -366,10 +357,12 @@ void WindowsFileMapper::CleanUpDataMappings(const std::unordered_map>()); // Push context bin filepath for lookup between sessions From d606a95d8a2b60e906135caece0918dbf8b9050f Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Fri, 16 Jan 2026 18:26:09 -0800 Subject: [PATCH 05/17] Added unit tests with feature disabled --- .../test/providers/qnn/qnn_ep_context_test.cc | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc index a2f1b9b56538b..77d13742413b5 100644 --- a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc @@ -1897,6 +1897,114 @@ TEST_F(QnnHTPBackendTests, VTCMBackupBufferSharing) { std::remove(qnn_ctx_binary_file_name1.c_str()); } +TEST_F(QnnHTPBackendTests, FileMapping_Off) { + ProviderOptions provider_options; + provider_options["backend_type"] = "htp"; + provider_options["offload_graph_io_quantization"] = "0"; + provider_options["disable_file_mapped_weights"] = "1"; + + // Create QDQ models + std::vector onnx_model_paths{"./weight_share1.onnx", "./weight_share2.onnx"}; + // cleanup in case some failure test doesn't remove them + for (auto model_path : onnx_model_paths) { + std::remove(model_path.c_str()); + } + + std::vector ctx_model_paths; + for (auto model_path : onnx_model_paths) { + CreateQdqModel(model_path, DefaultLoggingManager().DefaultLogger()); + EXPECT_TRUE(std::filesystem::exists(model_path.c_str())); + auto pos = model_path.find_last_of("."); + if (pos != std::string::npos) { + model_path = model_path.substr(0, pos) + "_ctx.onnx"; + } else { + model_path = model_path + "_ctx.onnx"; + } + ctx_model_paths.push_back(model_path); + } + for (auto ctx_model_path : ctx_model_paths) { + std::remove(ctx_model_path.c_str()); + } + + DumpModelWithSharedCtx(provider_options, onnx_model_paths[0], onnx_model_paths[1]); + + std::string qnn_ctx_binary_file_name1; + GetContextBinaryFileName(ctx_model_paths[0], qnn_ctx_binary_file_name1, + DefaultLoggingManager().DefaultLogger()); + EXPECT_TRUE(!qnn_ctx_binary_file_name1.empty()); + + std::string qnn_ctx_binary_file_name2; + GetContextBinaryFileName(ctx_model_paths[1], qnn_ctx_binary_file_name2, + DefaultLoggingManager().DefaultLogger()); + EXPECT_TRUE(!qnn_ctx_binary_file_name2.empty()); + // 2 *_ctx.onn point to same .bin file + EXPECT_TRUE(qnn_ctx_binary_file_name1 == qnn_ctx_binary_file_name2); + auto file_size_1 = std::filesystem::file_size(qnn_ctx_binary_file_name1); + EXPECT_TRUE(file_size_1 > 0); + + // only load and run the session on real device +#if defined(__aarch64__) || defined(_M_ARM64) + Ort::SessionOptions so1; + so1.SetLogId("so1"); + so1.AddConfigEntry(kOrtSessionOptionShareEpContexts, "1"); + so1.AppendExecutionProvider("QNN", provider_options); + Ort::SessionOptions so2; + + // Test CreateFromBinaryListAsync path + provider_options["enable_vtcm_backup_buffer_sharing"] = "1"; + so2.SetLogId("so2"); + so2.AddConfigEntry(kOrtSessionOptionShareEpContexts, "1"); + so2.AppendExecutionProvider("QNN", provider_options); + + EXPECT_TRUE(2 == ctx_model_paths.size()); +#ifdef _WIN32 + std::wstring ctx_model_file1(ctx_model_paths[0].begin(), ctx_model_paths[0].end()); + std::wstring ctx_model_file2(ctx_model_paths[1].begin(), ctx_model_paths[1].end()); +#else + std::string ctx_model_file1(ctx_model_paths[0].begin(), ctx_model_paths[0].end()); + std::string ctx_model_file2(ctx_model_paths[1].begin(), ctx_model_paths[1].end()); +#endif + Ort::Session session1(*ort_env, ctx_model_file1.c_str(), so1); + Ort::Session session2(*ort_env, ctx_model_file2.c_str(), so2); + + std::vector input_names; + std::vector output_names; + GetModelInputNames(ctx_model_paths[1], input_names, output_names, + DefaultLoggingManager().DefaultLogger()); + + // Run sessions + // prepare input + std::vector input_dim{2, 3}; + std::vector input_value(2 * 3, 0.0f); + Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault); + std::vector ort_inputs; + std::vector input_names_c; + for (size_t i = 0; i < input_names.size(); ++i) { + auto input_tensor = Ort::Value::CreateTensor(info, input_value.data(), input_value.size(), + input_dim.data(), input_dim.size()); + ort_inputs.push_back(std::move(input_tensor)); + input_names_c.push_back(input_names[i].c_str()); + } + std::vector output_names_c; + for (size_t i = 0; i < output_names.size(); ++i) { + output_names_c.push_back(output_names[i].c_str()); + } + + auto ort_outputs1 = session1.Run(Ort::RunOptions{}, input_names_c.data(), ort_inputs.data(), ort_inputs.size(), + output_names_c.data(), 1); + auto ort_outputs2 = session2.Run(Ort::RunOptions{}, input_names_c.data(), ort_inputs.data(), ort_inputs.size(), + output_names_c.data(), 1); +#endif + + for (auto model_path : onnx_model_paths) { + std::remove(model_path.c_str()); + } + for (auto ctx_model_path : ctx_model_paths) { + std::remove(ctx_model_path.c_str()); + } + std::remove(qnn_ctx_binary_file_name1.c_str()); +} + // For Ort sessions to generate the context binary, with session option ep.share_ep_contexts enabled // Ort sessions will share the QnnBackendManager, so that all graphs from all models compile into the same Qnn context TEST_F(QnnHTPBackendTests, QnnContextGenWeightSharingSessionAPI) { From 4a9d1cbdc80cd598fe3bd7a43e302cac793d88d3 Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Wed, 21 Jan 2026 10:58:09 -0800 Subject: [PATCH 06/17] Use smart pointers and wil library to close file handles on destruction Reuse mapview pointer of context bin and aligned offsets to calculate pointers to weights --- .../qnn/builder/onnx_ctx_model_helper.cc | 2 +- .../qnn/builder/qnn_backend_manager.cc | 106 +++--- .../qnn/builder/qnn_backend_manager.h | 22 +- .../core/providers/qnn/builder/qnn_def.h | 2 +- .../qnn_file_mapping_callback_interface.h | 16 +- .../qnn/builder/qnn_windows_file_mapper.cc | 354 +++++------------- .../qnn/builder/qnn_windows_file_mapper.h | 51 +-- .../providers/qnn/qnn_execution_provider.cc | 5 +- onnxruntime/test/perftest/ort_test_session.cc | 5 +- 9 files changed, 197 insertions(+), 366 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc index 0dfb7928e3adc..0e49c0f897bea 100644 --- a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc +++ b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc @@ -129,7 +129,7 @@ Status GetEpContextFromMainNode(const onnxruntime::Node& main_context_node, } std::string context_binary_path_str = context_binary_path.string(); -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE if (qnn_backend_manager->FileMappingIsEnabled()) { return qnn_backend_manager->LoadCachedQnnContextFromBuffer(nullptr, 0, diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index ac6f6db1d0a55..71dd99adcf2dc 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -29,7 +29,7 @@ #include "core/providers/qnn/builder/qnn_configs_helper.h" #include "core/providers/qnn/builder/qnn_utils.h" -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE #include "core/providers/qnn/builder/qnn_windows_file_mapper.h" #endif @@ -774,10 +774,10 @@ Status SetQnnContextConfig(ContextPriority context_priority, QnnContext_Config_t return Status::OK(); } -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE // callback required for allocating file mapping resources -static Qnn_ErrorHandle_t DmaDataProvider(Qnn_ContextBinaryDataRequest_t request, - Qnn_ContextBinaryDmaDataResponse_t* response, void* notify_param) { +static Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, + Qnn_ContextBinaryDmaDataResponse_t* response, void* notify_param) { if (notify_param == nullptr) { LOGS_DEFAULT(ERROR) << "DmaProvider: notify_param is null"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; @@ -793,7 +793,7 @@ static Qnn_ErrorHandle_t DmaDataProvider(Qnn_ContextBinaryDataRequest_t request, } // callback required for releasing file mapping resources -static Qnn_ErrorHandle_t DmaDataRelease(Qnn_ContextBinaryDmaDataMem_t data_mem, void* notify_param) { +static Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, void* notify_param) { if (notify_param == nullptr) { LOGS_DEFAULT(ERROR) << "DmaRelease: notify_param is null"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; @@ -808,7 +808,7 @@ static Qnn_ErrorHandle_t DmaDataRelease(Qnn_ContextBinaryDmaDataMem_t data_mem, return pair->first->ReleaseDmaData(data_mem, pair->second); } -#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED +#endif // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE // callback required to add context handles to class list // when using contextCreateFromBinaryListAsync() @@ -916,11 +916,12 @@ Status QnnBackendManager::CreateContextVtcmBackupBufferSharingEnabled(std::unord #endif nullptr}; -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE if (file_mapped_weights_enabled_ && file_mapper_) { // Retry logic -- if context creation failed with file mapped weights, then retry with feature disabled - if (CreateContextFromListAsyncV2(configs, context_bin_map) != Status::OK()) { - LOGS(*logger_, WARNING) << "Failed to create context with file mapping enabled. Retrying with feature disabled."; + auto res = CreateContextFromListAsyncWithCallback(configs, context_bin_map); + if (!res.IsOK()) { + LOGS(*logger_, WARNING) << res.ErrorMessage() << ". Retrying with feature disabled."; file_mapped_weights_enabled_ = false; // Destruction of file_mapper_ to prevent resource leaks @@ -930,12 +931,12 @@ Status QnnBackendManager::CreateContextVtcmBackupBufferSharingEnabled(std::unord } } #endif - return CreateContextFromListAsyncV1(configs, context_bin_map); + return CreateContextFromListAsync(configs, context_bin_map); } -Status QnnBackendManager::CreateContextFromListAsyncV1(const QnnContext_Config_t** configs, - std::unordered_map>>& context_bin_map) { +Status QnnBackendManager::CreateContextFromListAsync(const QnnContext_Config_t** configs, + std::unordered_map>>& context_bin_map) { std::vector context_params_list; std::vector context_paramsv1_list; std::vector context_params_ptr_list; @@ -979,16 +980,15 @@ Status QnnBackendManager::CreateContextFromListAsyncV1(const QnnContext_Config_t return Status::OK(); } -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED -Status QnnBackendManager::CreateContextFromListAsyncV2(const QnnContext_Config_t** configs, - std::unordered_map>>& context_bin_map) { +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE +Status QnnBackendManager::CreateContextFromListAsyncWithCallback(const QnnContext_Config_t** configs, + std::unordered_map>>& context_bin_map) { std::vector context_params_list; std::vector context_paramsv2_list; std::vector context_callbacks_list; std::vector context_params_ptr_list; - std::vector buffer_list; - context_params_list.reserve(context_bin_map.size()); context_callbacks_list.reserve(context_bin_map.size()); context_params_ptr_list.reserve(context_bin_map.size() + 1); @@ -1001,20 +1001,21 @@ Status QnnBackendManager::CreateContextFromListAsyncV2(const QnnContext_Config_t size_t buffer_size = buffer_info.size; - void* buffer = nullptr; - void* file_mapping_handle = nullptr; - ORT_RETURN_IF_ERROR(file_mapper_->MapContextBin(context_bin_filepath, &file_mapping_handle)); - ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappingPointer(context_bin_filepath, &buffer)); + void* buffer; + ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappedMemoryPtr(context_bin_filepath, &buffer)); + - auto& notify_param = file_mapping_notify_params_.emplace_back(file_mapper_.get(), - file_mapping_handle); + auto notify_param_ptr = std::make_unique>(file_mapper_.get(), + buffer); Qnn_ContextBinaryCallback_t context_file_map_callbacks; context_file_map_callbacks.type = QNN_CONTEXT_CALLBACK_DMA_BUFFER; context_file_map_callbacks.dmaBufferCallback.version = QNN_CONTEXT_CALLBACK_DMA_BUFFER_VERSION_1; - context_file_map_callbacks.dmaBufferCallback.v1.dataProvide = DmaDataProvider; - context_file_map_callbacks.dmaBufferCallback.v1.dataRelease = DmaDataRelease; - context_file_map_callbacks.dmaBufferCallback.v1.notifyParam = reinterpret_cast(¬ify_param); + context_file_map_callbacks.dmaBufferCallback.v1.dataProvide = MapDmaData; + context_file_map_callbacks.dmaBufferCallback.v1.dataRelease = ReleaseDmaData; + context_file_map_callbacks.dmaBufferCallback.v1.notifyParam = reinterpret_cast(notify_param_ptr.get()); + + file_mapping_notify_params_.push_back(std::move(notify_param_ptr)); // Callbacks require QnnContext_ParamsV2_t which is new to QNN API 2.32 QnnContext_ParamsV2_t context_params_v2 = {nullptr, @@ -1028,8 +1029,6 @@ Status QnnBackendManager::CreateContextFromListAsyncV2(const QnnContext_Config_t QnnContext_Params_t context_params = {QnnContext_ParamsVersion_t::QNN_CONTEXT_PARAMS_VERSION_2, {}}; context_params.v2 = &context_params_v2; - - buffer_list.push_back(buffer); context_params_list.push_back(std::move(context_params)); context_callbacks_list.push_back(std::move(context_file_map_callbacks)); context_paramsv2_list.push_back(std::move(context_params_v2)); @@ -1042,15 +1041,11 @@ Status QnnBackendManager::CreateContextFromListAsyncV2(const QnnContext_Config_t configs, nullptr); - for (auto& buffer : buffer_list) { - ORT_RETURN_IF_ERROR(file_mapper_->FreeContextBinMappingPointer(buffer)); - } - buffer_list.clear(); - - ORT_RETURN_IF(QNN_CONTEXT_NO_ERROR != result, "Failed to create context. Error: ", QnnErrorHandleToString(result), ", Code:", result); + ORT_RETURN_IF(QNN_CONTEXT_NO_ERROR != result, "Failed to create context with file mapping enabled. Error: ", + QnnErrorHandleToString(result), ", Code:", result); return Status::OK(); } -#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED +#endif // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE Status QnnBackendManager::SetContextPriority(ContextPriority context_priority) { QnnContext_Config_t context_priority_config = QNN_CONTEXT_CONFIG_INIT; @@ -1259,8 +1254,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t ORT_RETURN_IF(result, "Failed to get valid function pointer."); void* bin_buffer = nullptr; -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED - void* file_mapping_handle; +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE if (file_mapped_weights_enabled_) { ORT_RETURN_IF(!file_mapper_, "Attemping to use File Mapping feature but file_mapper_ is uninitialized"); @@ -1269,8 +1263,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t buffer_length = buffer_info.size; ORT_RETURN_IF(buffer_length == 0, "Context bin has a size of 0 bytes", context_bin_filepath); - ORT_RETURN_IF_ERROR(file_mapper_->MapContextBin(context_bin_filepath, &file_mapping_handle)); - ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappingPointer(context_bin_filepath, &bin_buffer)); + ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappedMemoryPtr(context_bin_filepath, &bin_buffer)); } else { ORT_RETURN_IF(buffer == nullptr, "Attempting to load QNN context from buffer but buffer is null"); @@ -1362,20 +1355,23 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t ORT_RETURN_IF(nullptr == qnn_interface_.contextCreateFromBinary, "Invalid function pointer for contextCreateFromBinary."); -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE Qnn_ContextBinaryCallback_t callbacks; if (file_mapped_weights_enabled_ && file_mapper_) { ORT_RETURN_IF(nullptr == qnn_interface_.contextCreateFromBinaryWithCallback, "Invalid function pointer for contextCreateFromBinaryWithCallback."); - auto& notify_param = file_mapping_notify_params_.emplace_back(file_mapper_.get(), - file_mapping_handle); + + auto notify_param_ptr = std::make_unique>(file_mapper_.get(), + bin_buffer); callbacks.type = QNN_CONTEXT_CALLBACK_DMA_BUFFER; callbacks.dmaBufferCallback.version = QNN_CONTEXT_CALLBACK_DMA_BUFFER_VERSION_1; - callbacks.dmaBufferCallback.v1.dataProvide = DmaDataProvider; - callbacks.dmaBufferCallback.v1.dataRelease = DmaDataRelease; - callbacks.dmaBufferCallback.v1.notifyParam = reinterpret_cast(¬ify_param); + callbacks.dmaBufferCallback.v1.dataProvide = MapDmaData; + callbacks.dmaBufferCallback.v1.dataRelease = ReleaseDmaData; + callbacks.dmaBufferCallback.v1.notifyParam = reinterpret_cast(notify_param_ptr.get()); + + file_mapping_notify_params_.push_back(std::move(notify_param_ptr)); } #else ORT_UNUSED_PARAMETER(context_bin_filepath); @@ -1388,7 +1384,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t } #endif -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE std::unique_ptr backup_buffer; if (file_mapped_weights_enabled_ && file_mapper_) { rt = qnn_interface_.contextCreateFromBinaryWithCallback(backend_handle_, @@ -1401,10 +1397,11 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t profile_backend_handle_, NULL); - ORT_RETURN_IF_ERROR(file_mapper_->FreeContextBinMappingPointer(bin_buffer)); - if (rt != QNN_SUCCESS) { - LOGS(*logger_, WARNING) << "Failed to create context with file mapping enabled. Retrying with feature disabled."; + + LOGS(*logger_, WARNING) << "Failed to create context with file mapping enabled. Error: " + << QnnErrorHandleToString(result) << ", Code : " << result + << ". Retrying with feature disabled."; file_mapped_weights_enabled_ = false; // Destruction of file_mapper_ to prevent resource leaks @@ -1418,7 +1415,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t bin_buffer = static_cast(backup_buffer.get()); } } -#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED +#endif // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE if (!file_mapped_weights_enabled_ || rt != QNN_SUCCESS) { rt = qnn_interface_.contextCreateFromBinary(backend_handle_, @@ -1477,6 +1474,7 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, bool share_ep_contexts, bool enable_vtcm_backup_buffer_sharing, bool enable_file_mapped_weights, + std::shared_ptr rpcmem_library, std::unordered_map>>& context_bin_map) { std::lock_guard lock(logger_recursive_mutex_); if (backend_setup_completed_) { @@ -1564,10 +1562,12 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, LOGS(logger, VERBOSE) << "LoadOpPackage succeed."; } -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE if (file_mapped_weights_enabled_ && !file_mapper_ && GetQnnBackendType() == QnnBackendType::HTP) { - file_mapper_ = std::make_unique(logger); + file_mapper_ = std::make_unique(logger, rpcmem_library); } +#else + ORT_UNUSED_PARAMETER(rpcmem_library); #endif bool enable_htp_weight_sharing = false; diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index 0314f4c0c1c95..ff388ca2eba55 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -25,6 +25,7 @@ #include "System/QnnSystemInterface.h" #include "core/providers/qnn/ort_api.h" +#include "core/providers/qnn/rpcmem_library.h" #include "core/providers/qnn/builder/op_builder_factory.h" #include "core/providers/qnn/builder/qnn_context_mem_handle_manager.h" #include "core/providers/qnn/builder/qnn_def.h" @@ -32,7 +33,7 @@ #include "core/providers/qnn/builder/qnn_profile_serializer.h" #include "core/providers/qnn/builder/qnn_node_group/qnn_node_group.h" -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE #include "core/providers/qnn/builder/qnn_file_mapping_callback_interface.h" #endif @@ -169,6 +170,7 @@ class QnnBackendManager : public std::enable_shared_from_this bool need_load_system_lib, bool share_ep_contexts, bool enable_vtcm_backup_buffer_sharing, bool enable_file_mapped_weights, + std::shared_ptr rpcmem_library, std::unordered_map>>& context_bin_map); Status CreateHtpPowerCfgId(uint32_t deviceId, uint32_t coreId, uint32_t& htp_power_config_id); @@ -285,14 +287,14 @@ class QnnBackendManager : public std::enable_shared_from_this Status CreateContextVtcmBackupBufferSharingEnabled(std::unordered_map>>& context_bin_map); - Status CreateContextFromListAsyncV1(const QnnContext_Config_t** configs, - std::unordered_map>>& context_bin_map); + Status CreateContextFromListAsync(const QnnContext_Config_t** configs, + std::unordered_map>>& context_bin_map); -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED - Status CreateContextFromListAsyncV2(const QnnContext_Config_t** configs, - std::unordered_map>>& context_bin_map); +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE + Status CreateContextFromListAsyncWithCallback(const QnnContext_Config_t** configs, + std::unordered_map>>& context_bin_map); #endif Status ReleaseContext(); @@ -482,11 +484,11 @@ class QnnBackendManager : public std::enable_shared_from_this bool vtcm_backup_buffer_sharing_enabled_ = false; bool file_mapped_weights_enabled_ = false; -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE std::unique_ptr file_mapper_ = nullptr; // Notify params for file mapping must persist throughout lifetime of // QnnBackendManager for release of DMA data callback on destruction - std::vector> file_mapping_notify_params_; + std::vector>> file_mapping_notify_params_; #endif // NPU backend requires quantized model diff --git a/onnxruntime/core/providers/qnn/builder/qnn_def.h b/onnxruntime/core/providers/qnn/builder/qnn_def.h index 8ddd8e6d65c25..847de084c49f6 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_def.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_def.h @@ -21,7 +21,7 @@ namespace qnn { #if defined(_WIN32) && (defined(__aarch64__) || defined(_M_ARM64)) #if QNN_API_VERSION_MAJOR > 2 || ((QNN_API_VERSION_MAJOR) == 2 && (QNN_API_VERSION_MINOR >= 32)) -#define QNN_FILE_MAPPED_WEIGHTS_ENABLED +#define QNN_FILE_MAPPED_WEIGHTS_AVAILABLE #endif #endif diff --git a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h index 4e7cf434873c7..ad2b267ac7f87 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h @@ -16,25 +16,21 @@ namespace qnn { class FileMappingCallbackInterface { public: virtual ~FileMappingCallbackInterface() = default; - virtual Status MapContextBin(const std::string& bin_filepath, - void** notify_param) = 0; - virtual Status ReleaseContextBin(const std::string& model_name) = 0; - virtual Status GetContextBinMappingPointer(const std::string& bin_filepath, void** mapping_ptr) = 0; - - virtual Status FreeContextBinMappingPointer(LPVOID bin_mapping_pointer) = 0; + virtual Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath, + void** mapped_data_ptr) = 0; virtual Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, Qnn_ContextBinaryDmaDataResponse_t* response, - void* notify_param) = 0; + void* mapped_data_ptr) = 0; virtual Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, - void* notify_param) = 0; + void* mapped_data_ptr) = 0; virtual Qnn_ErrorHandle_t MapRawData(Qnn_ContextBinaryDataRequest_t request, Qnn_ContextBinaryRawDataResponse_t* response, - void* notify_param) = 0; + void* mapped_data_ptr) = 0; virtual Qnn_ErrorHandle_t ReleaseRawData(Qnn_ContextBinaryRawDataMem_t data_mem, - void* notify_param) = 0; + void* mapped_data_ptr) = 0; }; } // namespace qnn diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc index c985f5569cce1..f27bf468592d6 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc @@ -2,7 +2,7 @@ // Licensed under the MIT License. #include "core/providers/qnn/builder/qnn_windows_file_mapper.h" -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE #include @@ -11,184 +11,110 @@ #include "core/providers/qnn/ort_api.h" #include "core/providers/qnn/rpcmem_library.h" +#include + namespace onnxruntime { namespace qnn { -WindowsFileMapper::WindowsFileMapper(const logging::Logger& logger) : logger_(&logger) { +WindowsFileMapper::WindowsFileMapper(const logging::Logger& logger, + std::shared_ptr rpcmem_lib) + : logger_(&logger), + rpcmem_lib_(rpcmem_lib){ } // Close all handles and registered buffers // Use LOGS_DEFAULT here as this function will be called during destruction of QnnBackendManager // At time of destruction. Usage of logger_ will not be available and will result in a seg fault WindowsFileMapper::~WindowsFileMapper() { - // Ideally, there should be nothing to clean up at this point - // but free any resources anyway if applicable - if (!mapping_handle_to_info_map_.empty() || !context_bin_map_view_pointers_.empty()) { - LOGS_DEFAULT(WARNING) << "File mapping resources still exist. Attempting to free all resources."; - } - - context_bin_to_mapping_handle_map_.clear(); - - for (auto& mapview_ptr : context_bin_map_view_pointers_) { - CleanUpDataMapping(mapview_ptr, nullptr, 0); - } - - for (auto& kv : mapping_handle_to_info_map_) { - HANDLE file_mapping_handle = kv.first; - auto& mapping_info = kv.second; - - CleanUpDataMappings(mapping_info.mapped_data); - CloseHandles(mapping_info.file_handle, file_mapping_handle); - } - mapping_handle_to_info_map_.clear(); } -Status WindowsFileMapper::MapContextBin(const std::string& bin_filepath, - void** notify_param) { +Status WindowsFileMapper::GetContextBinMappedMemoryPtr(const std::string& bin_filepath, + void** mapped_data_ptr) { LOGS(*logger_, INFO) << "Creating context bin file mapping for " << bin_filepath; ORT_RETURN_IF(bin_filepath.empty(), "Context bin file path is empty"); - std::lock_guard lock(map_mutex_); - - auto file_mapping_it = context_bin_to_mapping_handle_map_.find(bin_filepath); - if (file_mapping_it != context_bin_to_mapping_handle_map_.end()) { - LOGS(*logger_, INFO) << "Context bin file mapping already exists for " - << bin_filepath; - *notify_param = reinterpret_cast(file_mapping_it->second); - return Status::OK(); - } - - HANDLE file_handle = CreateFileA(bin_filepath.c_str(), - GENERIC_READ, - FILE_SHARE_READ, - NULL, - OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, - NULL); - ORT_RETURN_IF(file_handle == INVALID_HANDLE_VALUE, - "Failed to create file handle with error code ", - GetLastError(), " for context bin", bin_filepath); - - LOGS(*logger_, VERBOSE) << "Created file handle (" << file_handle << ") for context bin: " + std::wstring bin_filepath_wstr(bin_filepath.begin(), bin_filepath.end()); + wil::unique_hfile file_handle{CreateFile2(bin_filepath_wstr.c_str(), + GENERIC_READ, + FILE_SHARE_READ, + OPEN_EXISTING, + NULL)}; + if (file_handle.get() == INVALID_HANDLE_VALUE) { + const auto error_code = GetLastError(); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Failed to create file handle for context bin", bin_filepath, + ". Error code: ", error_code, ", \"", + std::system_category().message(error_code), "\""); + } + + LOGS(*logger_, VERBOSE) << "Created file handle (" << file_handle.get() << ") for context bin: " << bin_filepath; - HANDLE file_mapping_handle = CreateFileMappingA(file_handle, NULL, PAGE_READONLY, 0x00, 0x00, NULL); - if (file_mapping_handle == INVALID_HANDLE_VALUE) { - CloseHandles(file_handle, nullptr); - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create file mapping with error code ", - GetLastError(), " for context bin ", bin_filepath); - } - - LOGS(*logger_, INFO) << "Created file mapping with handle (" << file_mapping_handle << ") for context bin:" - << bin_filepath; - - auto inserted = context_bin_to_mapping_handle_map_.insert({bin_filepath, file_mapping_handle}); - if (!inserted.second) { - CloseHandles(file_handle, file_mapping_handle); - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to add file handle mapping for context bin: ", - bin_filepath); - } - mapping_handle_to_info_map_.insert({file_mapping_handle, {file_handle, {}}}); - - *notify_param = reinterpret_cast(file_mapping_handle); + wil::unique_hfile file_mapping_handle{CreateFileMappingW(file_handle.get(), + nullptr, + PAGE_READONLY, + 0x00, + 0x00, + nullptr)}; + if (file_mapping_handle.get() == INVALID_HANDLE_VALUE) { + const auto error_code = GetLastError(); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Failed to create file mapping handle for context bin", + bin_filepath, ". Error code: ", error_code, ", \"", + std::system_category().message(error_code), "\""); + } + + LOGS(*logger_, VERBOSE) << "Created file mapping with handle (" << file_mapping_handle.get() + << ") for context bin:" << bin_filepath; + + void* const mapped_base_ptr = MapViewOfFile(file_mapping_handle.get(), + FILE_MAP_READ, + 0, 0, 0); + + if (mapped_base_ptr == nullptr) { + const auto error_code = GetLastError(); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Failed to retrieve mapview pointer for context bin", + bin_filepath, ". Error code: ", error_code, ", \"", + std::system_category().message(error_code), "\""); + } + + LOGS(*logger_, INFO) << "Created mapview pointer with address " << mapped_base_ptr + << " for context bin " << bin_filepath; + + onnxruntime::Env::MappedMemoryPtr mapped_memory_ptr{reinterpret_cast(mapped_base_ptr), + [mapped_base_ptr](void*) { + UnmapFile(mapped_base_ptr); + }}; + + *mapped_data_ptr = mapped_memory_ptr.get(); + mapped_memory_ptrs.push_back(std::move(mapped_memory_ptr)); + return Status::OK(); } -Status WindowsFileMapper::ReleaseContextBin(const std::string& bin_filepath) { - LOGS(*logger_, INFO) << "Removing context bin file mapping for " - << bin_filepath; - std::lock_guard lock(map_mutex_); - auto status = Status::OK(); - - auto bin_map_it = context_bin_to_mapping_handle_map_.find(bin_filepath); - - if (bin_map_it == context_bin_to_mapping_handle_map_.end()) { - LOGS(*logger_, VERBOSE) << "File handle does not exist for " << bin_filepath; - return status; +void WindowsFileMapper::UnmapFile(void* addr) noexcept { + bool successful = UnmapViewOfFile(addr); + if (!successful) { + const auto error_code = GetLastError(); + LOGS_DEFAULT(ERROR) << "Failed to unmap view of file with ptr: " << addr + << ", Error code: " << error_code << ", \"" + << std::system_category().message(error_code) << "\""; } - - HANDLE file_mapping_handle = bin_map_it->second; - - HANDLE file_handle = nullptr; - auto it = mapping_handle_to_info_map_.find(file_mapping_handle); - if (it == mapping_handle_to_info_map_.end()) { - status = ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "File mapping information does not exist for file mapping handle: ", - file_mapping_handle, ", context bin: ", bin_filepath); - } else { - MappingInfo_t& mapping_info = it->second; - file_handle = mapping_info.file_handle; - auto mapped_data = mapping_info.mapped_data; - - if (!mapped_data.empty()) { - LOGS(*logger_, WARNING) << "Attempting to remove context bin: " << bin_filepath - << ", but data regions still need to be unmapped. " - << "Proceeding with unmapping."; - CleanUpDataMappings(mapped_data); - } - } - - // Will ignore handles that are null - CloseHandles(file_handle, file_mapping_handle); - ORT_UNUSED_PARAMETER(mapping_handle_to_info_map_.erase(file_mapping_handle)); - ORT_UNUSED_PARAMETER(context_bin_to_mapping_handle_map_.erase(bin_filepath)); - - return Status::OK(); -} - -Status WindowsFileMapper::GetContextBinMappingPointer(const std::string& bin_filepath, void** mapping_ptr) { - LOGS(*logger_, INFO) << "Creating mapping pointer for " << bin_filepath; - - std::lock_guard lock(map_mutex_); - auto it = context_bin_to_mapping_handle_map_.find(bin_filepath); - - ORT_RETURN_IF(it == context_bin_to_mapping_handle_map_.end(), - "Failed to create mapping pointer: File mapping does not exist for ", - bin_filepath); - - HANDLE& file_mapping_handle = it->second; - - LPVOID mapview_ptr = MapViewOfFile(file_mapping_handle, - FILE_MAP_READ, - 0, 0, 0); - - ORT_RETURN_IF(mapview_ptr == nullptr, "Failed to create mapping pointer for ", bin_filepath); - - ORT_UNUSED_PARAMETER(context_bin_map_view_pointers_.insert(mapview_ptr)); - *mapping_ptr = mapview_ptr; - LOGS(*logger_, INFO) << "Created mapping pointer (" << mapping_ptr << ") for " << bin_filepath; - return Status::OK(); -} - -Status WindowsFileMapper::FreeContextBinMappingPointer(LPVOID bin_mapping_pointer) { - LOGS(*logger_, INFO) << "Releasing mapping pointer " << bin_mapping_pointer; - - std::lock_guard lock(map_mutex_); - auto it = context_bin_map_view_pointers_.find(bin_mapping_pointer); - - ORT_RETURN_IF(it == context_bin_map_view_pointers_.end(), "Mapping pointer ", - bin_mapping_pointer, " cannot be found and is invalid"); - - ORT_RETURN_IF(!UnmapViewOfFile(bin_mapping_pointer), "Failed to free mapping pointer ", bin_mapping_pointer, - " with error code ", GetLastError()); - ORT_UNUSED_PARAMETER(context_bin_map_view_pointers_.erase(bin_mapping_pointer)); - return Status::OK(); } Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t request, Qnn_ContextBinaryDmaDataResponse_t* response, - void* notify_param) { - if (notify_param == nullptr) { - LOGS(*logger_, ERROR) << "Attempting to map DMA data for null mapping handle"; + void* mapped_data_ptr) { + if (mapped_data_ptr == nullptr) { + LOGS(*logger_, ERROR) << "Attempting to map DMA data for null memory mapped pointer"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } - std::lock_guard lock(map_mutex_); - HANDLE file_mapping_handle = reinterpret_cast(notify_param); - LOGS(*logger_, INFO) << "Mapping DMA data for request: mapping handle(" - << file_mapping_handle << "), offset(" << request.offset + LOGS(*logger_, INFO) << "Mapping DMA data for request: memory mapped pointer(" + << mapped_data_ptr << "), offset(" << request.offset << "), size(" << request.size << "), isBackendMappingNeeded(" << request.isBackendMappingNeeded << ")"; @@ -198,13 +124,6 @@ Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t r return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } - auto it = mapping_handle_to_info_map_.find(file_mapping_handle); - if (it == mapping_handle_to_info_map_.end()) { - LOGS(*logger_, ERROR) << "File mapping info not found for mapping handle: " << file_mapping_handle; - return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; - } - MappingInfo_t& mapping_info = it->second; - // Align to nearest granularity boundary SYSTEM_INFO sys_info; GetSystemInfo(&sys_info); @@ -212,27 +131,22 @@ Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t r SIZE_T aligned_offset = request.offset & ~(granularity - 1); SIZE_T delta = request.offset - aligned_offset; - LPVOID aligned_data_ptr = MapViewOfFile(file_mapping_handle, - FILE_MAP_READ, - (aligned_offset >> 32), - (aligned_offset & 0xFFFFFFFF), - (buffer_size + delta)); - - if (aligned_data_ptr == nullptr) { - LOGS(*logger_, ERROR) << "Failed to map DMA data with error code " << GetLastError() - << " for file mapping handle " << file_mapping_handle; - return QNN_COMMON_ERROR_SYSTEM; - } + void* aligned_data_ptr = static_cast(mapped_data_ptr) + aligned_offset; - LPVOID unaligned_data_ptr = static_cast(aligned_data_ptr) + delta; + void* unaligned_data_ptr = static_cast(aligned_data_ptr) + delta; LOGS(*logger_, INFO) << "Created DMA data mapping with: address(" << aligned_data_ptr << "), aligned offset(" << aligned_offset << "), delta(" << delta << "), unaligned address(" << unaligned_data_ptr << ")"; - rpcmem_lib_.Api().register_buf(unaligned_data_ptr, buffer_size, NULL, + if (!rpcmem_lib_) { + LOGS(*logger_, ERROR) << "RPCMem Libs are not initialized"; + return QNN_COMMON_ERROR_RESOURCE_UNAVAILABLE; + } + + rpcmem_lib_->Api().register_buf(unaligned_data_ptr, buffer_size, NULL, rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); - auto fd = rpcmem_lib_.Api().to_fd(unaligned_data_ptr); + auto fd = rpcmem_lib_->Api().to_fd(unaligned_data_ptr); if (fd == -1) { LOGS(*logger_, ERROR) << "Failed to register DMA data mapping to RPCMEM"; if (!UnmapViewOfFile(aligned_data_ptr)) { @@ -242,7 +156,6 @@ Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t r return QNN_COMMON_ERROR_SYSTEM; } - mapping_info.mapped_data.insert({unaligned_data_ptr, {aligned_data_ptr, buffer_size}}); response->dmaBuffer.fd = fd; response->dmaBuffer.data = reinterpret_cast(unaligned_data_ptr); response->dataStartOffset = 0; @@ -254,50 +167,29 @@ Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t r // Use LOGS_DEFAULT here as this function will be called during destruction of QnnBackendManager // At time of destruction. Usage of logger_ will not be available and will result in a seg fault Qnn_ErrorHandle_t WindowsFileMapper::ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, - void* notify_param) { - if (notify_param == nullptr) { - LOGS_DEFAULT(ERROR) << "Attempting to release DMA data for null mapping handle"; + void* mapped_data_ptr) { + if (mapped_data_ptr == nullptr) { + LOGS_DEFAULT(ERROR) << "Attempting to release DMA data for null memory mapped pointer"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } - std::lock_guard lock(map_mutex_); - HANDLE file_mapping_handle = static_cast(notify_param); - LOGS_DEFAULT(INFO) << "Releasing DMA data mapping for mapping handle(" << file_mapping_handle - << "), address(" << data_mem.dmaBuffer.data << "), size: (" - << data_mem.memSize << ")"; + LOGS_DEFAULT(INFO) << "Releasing DMA data mapping for memory mapped pointer(" + << mapped_data_ptr << "), address(" << data_mem.dmaBuffer.data + << "), size: (" << data_mem.memSize << ")"; if (data_mem.dmaBuffer.data == nullptr || data_mem.memSize == 0) { LOGS_DEFAULT(ERROR) << "Mapping release request address must not be null and size must be > 0"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } - auto mapping_info_it = mapping_handle_to_info_map_.find(file_mapping_handle); - if (mapping_info_it == mapping_handle_to_info_map_.end()) { - LOGS_DEFAULT(ERROR) << "File mapping info not found for mapping handle: " << file_mapping_handle; - return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; - } - MappingInfo_t& mapping_info = mapping_info_it->second; - - LPVOID unaligned_data_ptr = reinterpret_cast(data_mem.dmaBuffer.data); - auto& mapped_data = mapping_info.mapped_data; - auto mapped_data_it = mapped_data.find(unaligned_data_ptr); - - if (mapped_data_it == mapped_data.end()) { - LOGS_DEFAULT(ERROR) << "Failed to find DMA data mapping for address: " << unaligned_data_ptr; - return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; - } - - LPVOID aligned_data_ptr = mapped_data_it->second.aligned_data_ptr; - - CleanUpDataMapping(unaligned_data_ptr, aligned_data_ptr, data_mem.memSize); - if (!mapped_data.erase(unaligned_data_ptr)) { - LOGS_DEFAULT(WARNING) << "Possible leak: failed to remove unordered_map entry for DMA data address: " - << unaligned_data_ptr; - } + void* unaligned_data_ptr = data_mem.dmaBuffer.data; + rpcmem_lib_->Api().register_buf(unaligned_data_ptr, data_mem.memSize, -1, + rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); - if (mapped_data.empty()) { - CloseHandles(mapping_info.file_handle, file_mapping_handle); - ORT_UNUSED_PARAMETER(mapping_handle_to_info_map_.erase(mapping_info_it)); + auto fd = rpcmem_lib_->Api().to_fd(unaligned_data_ptr); + if (fd != -1) { + LOGS_DEFAULT(ERROR) << "Failed to deregister buffer from RPCMEM: " << unaligned_data_ptr; + return QNN_CONTEXT_ERROR_MEM_ALLOC; } return QNN_SUCCESS; @@ -305,10 +197,10 @@ Qnn_ErrorHandle_t WindowsFileMapper::ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_ Qnn_ErrorHandle_t WindowsFileMapper::MapRawData(Qnn_ContextBinaryDataRequest_t request, Qnn_ContextBinaryRawDataResponse_t* response, - void* notify_param) { + void* mapped_data_ptr) { ORT_UNUSED_PARAMETER(request); ORT_UNUSED_PARAMETER(response); - ORT_UNUSED_PARAMETER(notify_param); + ORT_UNUSED_PARAMETER(mapped_data_ptr); LOGS(*logger_, ERROR) << "File mapping for raw binary data is unsupported on Windows"; return QNN_CONTEXT_ERROR_UNSUPPORTED_FEATURE; @@ -318,55 +210,15 @@ Qnn_ErrorHandle_t WindowsFileMapper::MapRawData(Qnn_ContextBinaryDataRequest_t r // QnnBackendManager at time of destruction. Usage of logger_ will not be available and will result // in a seg fault Qnn_ErrorHandle_t WindowsFileMapper::ReleaseRawData(Qnn_ContextBinaryRawDataMem_t data_mem, - void* notify_param) { + void* mapped_data_ptr) { ORT_UNUSED_PARAMETER(data_mem); - ORT_UNUSED_PARAMETER(notify_param); + ORT_UNUSED_PARAMETER(mapped_data_ptr); LOGS_DEFAULT(ERROR) << "File mapping for raw binary data is unsupported on Windows"; return QNN_CONTEXT_ERROR_UNSUPPORTED_FEATURE; } -void WindowsFileMapper::CleanUpDataMapping(LPVOID unaligned_data_ptr, LPVOID aligned_data_ptr, - size_t buffer_size) { - if (unaligned_data_ptr) { - // Set file descriptor to -1 to signal deregistration - rpcmem_lib_.Api().register_buf(unaligned_data_ptr, buffer_size, -1, - rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); - - auto fd = rpcmem_lib_.Api().to_fd(unaligned_data_ptr); - if (fd != -1) { - LOGS_DEFAULT(ERROR) << "Failed to deregister buffer from RPCMEM: " << unaligned_data_ptr; - } - } - - if (aligned_data_ptr && !UnmapViewOfFile(aligned_data_ptr)) { - LOGS_DEFAULT(ERROR) << "Failed to unmap view of pointer: " << aligned_data_ptr - << ", error code: " << GetLastError(); - } -} - -void WindowsFileMapper::CleanUpDataMappings(const std::unordered_map& mapped_data) { - // Key is unaligned data pointer - for (const auto& kv : mapped_data) { - auto mapped_data_info = kv.second; - // Will handle null ptrs - CleanUpDataMapping(kv.first, mapped_data_info.aligned_data_ptr, - mapped_data_info.buffer_size); - } -} - -void WindowsFileMapper::CloseHandles(HANDLE file_handle, HANDLE file_mapping_handle) { - if (file_mapping_handle && !CloseHandle(file_mapping_handle)) { - LOGS_DEFAULT(ERROR) << "Failed to close file mapping handle: " << file_mapping_handle - << ", error code: " << GetLastError(); - } - if (file_handle && !CloseHandle(file_handle)) { - LOGS_DEFAULT(ERROR) << "Failed to close file handle: " << file_handle - << ", error code: " << GetLastError(); - } -} - } // namespace qnn } // namespace onnxruntime -#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED +#endif // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h index 143946a443cd7..675c3020d13c9 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h @@ -4,13 +4,14 @@ #pragma once #include "core/providers/qnn/builder/qnn_file_mapping_callback_interface.h" -#ifdef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE #include #include #include +#include "core/platform/env.h" #include "core/providers/qnn/rpcmem_library.h" #include "core/providers/qnn/ort_api.h" @@ -19,57 +20,35 @@ namespace qnn { class WindowsFileMapper : public FileMappingCallbackInterface { public: - explicit WindowsFileMapper(const logging::Logger& logger); + explicit WindowsFileMapper(const logging::Logger& logger, + std::shared_ptr rpcmem_lib); ~WindowsFileMapper() override; - Status MapContextBin(const std::string& bin_filepath, - void** notify_param) override; - Status ReleaseContextBin(const std::string& model_name) override; - Status GetContextBinMappingPointer(const std::string& bin_filepath, void** mapping_ptr) override; + Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath, + void** mapped_data_ptr) override; - Status FreeContextBinMappingPointer(LPVOID bin_mapping_pointer) override; + static void UnmapFile(void* addr) noexcept; Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, Qnn_ContextBinaryDmaDataResponse_t* response, - void* notify_param) override; + void* mapped_data_ptr) override; Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, - void* notify_param) override; + void* mapped_data_ptr) override; Qnn_ErrorHandle_t MapRawData(Qnn_ContextBinaryDataRequest_t request, Qnn_ContextBinaryRawDataResponse_t* response, - void* notify_param) override; + void* mapped_data_ptr) override; Qnn_ErrorHandle_t ReleaseRawData(Qnn_ContextBinaryRawDataMem_t data_mem, - void* notify_param) override; + void* mapped_data_ptr) override; private: - typedef struct MappedDataInfo { - LPVOID aligned_data_ptr = nullptr; - size_t buffer_size = 0; - } MappedDataInfo_t; - - typedef struct MappingInfo { - HANDLE file_handle; - - // Maps unaligned data pointers to aligned data pointers - std::unordered_map mapped_data; - } MappingInfo_t; - - void CleanUpDataMapping(LPVOID unaligned_data_ptr, LPVOID aligned_data_ptr, - size_t buffer_size); - void CleanUpDataMappings(const std::unordered_map& mapped_data); - void CloseHandles(HANDLE file_handle, HANDLE file_mapping_handle); - - std::mutex map_mutex_; // Applies to both unordered maps - std::unordered_map context_bin_to_mapping_handle_map_; - std::unordered_map mapping_handle_to_info_map_; - std::unordered_set context_bin_map_view_pointers_; - + // A container of smart pointers of mapview memory pointers to mapped context bins + std::vector mapped_memory_ptrs; const logging::Logger* logger_; - - RpcMemLibrary rpcmem_lib_; + std::shared_ptr rpcmem_lib_; }; } // namespace qnn } // namespace onnxruntime -#endif // QNN_FILE_MAPPED_WEIGHTS_ENABLED +#endif // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index d07b16dbf0e49..94537a7327a67 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -484,7 +484,7 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio LOGS_DEFAULT(VERBOSE) << "User specified disable_file_mapped_weights: " << enable_file_mapped_weights_; } -#ifndef QNN_FILE_MAPPED_WEIGHTS_ENABLED +#ifndef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE enable_file_mapped_weights_ = false; LOGS_DEFAULT(WARNING) << "File mapped weights feature is only available on Windows arm64 devices for QNN API versions >= 2.32. " << "Feature will be disabled by default"; @@ -567,7 +567,7 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio } static const std::string QNN_HTP_SHARED_MEMORY_ALLOCATOR_ENABLED = "enable_htp_shared_memory_allocator"; - if (ParseBoolOption(QNN_HTP_SHARED_MEMORY_ALLOCATOR_ENABLED, false, provider_options_map)) { + if (ParseBoolOption(QNN_HTP_SHARED_MEMORY_ALLOCATOR_ENABLED, false, provider_options_map) || enable_file_mapped_weights_) { // Initialize rpcmem_library_. // This is necessary for HtpSharedMemoryAllocator to function and also indicates that the allocator is available. rpcmem_library_ = std::make_shared(); @@ -992,6 +992,7 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer share_ep_contexts_, enable_vtcm_backup_buffer_sharing_, enable_file_mapped_weights_, + rpcmem_library_, context_bin_map); context_bin_map.clear(); diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 5b365c6867a7a..bc7519d331be3 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -357,7 +357,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device "qnn_saver_path", "htp_graph_finalization_optimization_mode", "qnn_context_priority", "htp_arch", "enable_htp_fp16_precision", "offload_graph_io_quantization", "enable_htp_spill_fill_buffer", "enable_htp_shared_memory_allocator", "dump_json_qnn_graph", - "json_qnn_graph_dir", "disable_file_mapped_weights", "htp_bf16_enable"}); + "json_qnn_graph_dir", "disable_file_mapped_weights", "htp_bf16_enable", "enable_vtcm_backup_buffer_sharing"}); for (const auto& provider_option : provider_options) { const std::string& key = provider_option.first; const std::string& value = provider_option.second; @@ -422,7 +422,8 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device key == "enable_htp_spill_fill_buffer" || key == "enable_htp_shared_memory_allocator" || key == "dump_json_qnn_graph" || - key == "disable_file_mapped_weights") { + key == "disable_file_mapped_weights" || + key == "enable_vtcm_backup_buffer_sharing") { std::set supported_options = {"0", "1"}; if (supported_options.find(value) == supported_options.end()) { std::ostringstream str_stream; From b59b045e2502eb1791bcc2089b237dd02e30b58d Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Wed, 21 Jan 2026 19:08:51 -0800 Subject: [PATCH 07/17] ran lintrunner, separated retrieval of file size from reading context bin into a buffer --- .../qnn/builder/qnn_backend_manager.cc | 64 +++++++++---------- .../qnn/builder/qnn_backend_manager.h | 5 +- .../qnn/builder/qnn_windows_file_mapper.cc | 20 +++--- 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index 71dd99adcf2dc..f04fa8bf9429c 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -845,34 +845,39 @@ void QnnBackendManager::ProcessContextFromBinListAsync(Qnn_ContextHandle_t conte } } -Status QnnBackendManager::ReadContextBinIfValid(const std::string& context_bin_filepath, - BufferInfo_t& buffer_info, - bool read_file_contents) { +Status QnnBackendManager::GetFileSizeIfValid(const std::string& filepath, + size_t& file_size) { std::error_code ec; - ORT_RETURN_IF(!std::filesystem::exists(context_bin_filepath, ec), "Context binary does not exist: ", context_bin_filepath); - ORT_RETURN_IF(ec, "Failed to read context binary: ", context_bin_filepath, + ORT_RETURN_IF(!std::filesystem::exists(filepath, ec), "Context binary does not exist: ", filepath); + ORT_RETURN_IF(ec, "Failed to read file: ", filepath, ", error: ", ec.message()); - auto file_size = std::filesystem::file_size(context_bin_filepath, ec); - ORT_RETURN_IF(ec, "Failed to retrieve size of context binary: ", context_bin_filepath, + auto size = std::filesystem::file_size(filepath, ec); + ORT_RETURN_IF(ec, "Failed to retrieve size of file: ", filepath, ", error: ", ec.message()); - ORT_RETURN_IF(file_size == 0, "Context binary is empty: ", context_bin_filepath); - ORT_RETURN_IF(file_size > SIZE_MAX, "Context binary (", context_bin_filepath, ") file size (", file_size, + + ORT_RETURN_IF(size == 0, "File is empty: ", filepath); + ORT_RETURN_IF(size > SIZE_MAX, "File (", filepath, ") file size (", size, " bytes) exceeds maximum value of size_t for this platform (", SIZE_MAX, " bytes)."); - size_t buffer_size = static_cast(file_size); + file_size = static_cast(size); + return Status::OK(); +} + +Status QnnBackendManager::ReadContextBinIfValid(const std::string& context_bin_filepath, + BufferInfo_t& buffer_info) { + size_t buffer_size; + ORT_RETURN_IF_ERROR(GetFileSizeIfValid(context_bin_filepath, buffer_size)); std::unique_ptr buffer; - if (read_file_contents) { - std::ifstream cache_file(context_bin_filepath.c_str(), std::ifstream::binary); - ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to read context binary from: ", context_bin_filepath); + std::ifstream cache_file(context_bin_filepath.c_str(), std::ifstream::binary); + ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to read context binary from: ", context_bin_filepath); - buffer = std::make_unique(buffer_size); - ORT_RETURN_IF(nullptr == buffer, "Failed to allocate memory for cache file."); + buffer = std::make_unique(buffer_size); + ORT_RETURN_IF(nullptr == buffer, "Failed to allocate memory for cache file."); - const auto& read_result = cache_file.read(buffer.get(), buffer_size); - ORT_RETURN_IF(!read_result, "Failed to read contents from cached context file."); - } + const auto& read_result = cache_file.read(buffer.get(), buffer_size); + ORT_RETURN_IF(!read_result, "Failed to read contents from cached context file."); buffer_info.data = std::move(buffer); buffer_info.size = buffer_size; @@ -949,7 +954,7 @@ Status QnnBackendManager::CreateContextFromListAsync(const QnnContext_Config_t** auto context_bin_filepath = it.first; BufferInfo_t buffer_info; - ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info, true)); + ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info)); std::unique_ptr buffer = std::move(buffer_info.data); size_t buffer_size = buffer_info.size; @@ -982,7 +987,7 @@ Status QnnBackendManager::CreateContextFromListAsync(const QnnContext_Config_t** #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE Status QnnBackendManager::CreateContextFromListAsyncWithCallback(const QnnContext_Config_t** configs, - std::unordered_map>>& context_bin_map) { std::vector context_params_list; std::vector context_paramsv2_list; @@ -996,17 +1001,14 @@ Status QnnBackendManager::CreateContextFromListAsyncWithCallback(const QnnContex for (auto& it : context_bin_map) { auto context_bin_filepath = it.first; - BufferInfo_t buffer_info; - ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info, false)); - - size_t buffer_size = buffer_info.size; + size_t buffer_size; + ORT_RETURN_IF_ERROR(GetFileSizeIfValid(context_bin_filepath, buffer_size)); void* buffer; ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappedMemoryPtr(context_bin_filepath, &buffer)); - auto notify_param_ptr = std::make_unique>(file_mapper_.get(), - buffer); + buffer); Qnn_ContextBinaryCallback_t context_file_map_callbacks; context_file_map_callbacks.type = QNN_CONTEXT_CALLBACK_DMA_BUFFER; @@ -1042,7 +1044,7 @@ Status QnnBackendManager::CreateContextFromListAsyncWithCallback(const QnnContex nullptr); ORT_RETURN_IF(QNN_CONTEXT_NO_ERROR != result, "Failed to create context with file mapping enabled. Error: ", - QnnErrorHandleToString(result), ", Code:", result); + QnnErrorHandleToString(result), ", Code:", result); return Status::OK(); } #endif // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE @@ -1258,9 +1260,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t if (file_mapped_weights_enabled_) { ORT_RETURN_IF(!file_mapper_, "Attemping to use File Mapping feature but file_mapper_ is uninitialized"); - BufferInfo_t buffer_info; - ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info, false)); - buffer_length = buffer_info.size; + ORT_RETURN_IF_ERROR(GetFileSizeIfValid(context_bin_filepath, buffer_length)); ORT_RETURN_IF(buffer_length == 0, "Context bin has a size of 0 bytes", context_bin_filepath); ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappedMemoryPtr(context_bin_filepath, &bin_buffer)); @@ -1361,7 +1361,6 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t ORT_RETURN_IF(nullptr == qnn_interface_.contextCreateFromBinaryWithCallback, "Invalid function pointer for contextCreateFromBinaryWithCallback."); - auto notify_param_ptr = std::make_unique>(file_mapper_.get(), bin_buffer); @@ -1398,7 +1397,6 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t NULL); if (rt != QNN_SUCCESS) { - LOGS(*logger_, WARNING) << "Failed to create context with file mapping enabled. Error: " << QnnErrorHandleToString(result) << ", Code : " << result << ". Retrying with feature disabled."; @@ -1409,7 +1407,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t // Read context bin from file since file mapping has failed BufferInfo_t buffer_info; - ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info, true)); + ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info)); backup_buffer = std::move(buffer_info.data); bin_buffer = static_cast(backup_buffer.get()); diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index ff388ca2eba55..cd02d1f855556 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -280,9 +280,10 @@ class QnnBackendManager : public std::enable_shared_from_this Status CreateContext(bool enable_htp_weight_sharing); + Status GetFileSizeIfValid(const std::string& filepath, size_t& file_size); + Status ReadContextBinIfValid(const std::string& context_bin_filepath, - BufferInfo_t& buffer_info, - bool read_file_contents); + BufferInfo_t& buffer_info); Status CreateContextVtcmBackupBufferSharingEnabled(std::unordered_map>>& context_bin_map); diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc index f27bf468592d6..0fccf8fe1fdf2 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc @@ -19,7 +19,7 @@ namespace qnn { WindowsFileMapper::WindowsFileMapper(const logging::Logger& logger, std::shared_ptr rpcmem_lib) : logger_(&logger), - rpcmem_lib_(rpcmem_lib){ + rpcmem_lib_(rpcmem_lib) { } // Close all handles and registered buffers @@ -53,11 +53,11 @@ Status WindowsFileMapper::GetContextBinMappedMemoryPtr(const std::string& bin_fi << bin_filepath; wil::unique_hfile file_mapping_handle{CreateFileMappingW(file_handle.get(), - nullptr, - PAGE_READONLY, - 0x00, - 0x00, - nullptr)}; + nullptr, + PAGE_READONLY, + 0x00, + 0x00, + nullptr)}; if (file_mapping_handle.get() == INVALID_HANDLE_VALUE) { const auto error_code = GetLastError(); return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, @@ -91,7 +91,7 @@ Status WindowsFileMapper::GetContextBinMappedMemoryPtr(const std::string& bin_fi *mapped_data_ptr = mapped_memory_ptr.get(); mapped_memory_ptrs.push_back(std::move(mapped_memory_ptr)); - + return Status::OK(); } @@ -144,7 +144,7 @@ Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t r } rpcmem_lib_->Api().register_buf(unaligned_data_ptr, buffer_size, NULL, - rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); + rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); auto fd = rpcmem_lib_->Api().to_fd(unaligned_data_ptr); if (fd == -1) { @@ -174,7 +174,7 @@ Qnn_ErrorHandle_t WindowsFileMapper::ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_ } LOGS_DEFAULT(INFO) << "Releasing DMA data mapping for memory mapped pointer(" - << mapped_data_ptr << "), address(" << data_mem.dmaBuffer.data + << mapped_data_ptr << "), address(" << data_mem.dmaBuffer.data << "), size: (" << data_mem.memSize << ")"; if (data_mem.dmaBuffer.data == nullptr || data_mem.memSize == 0) { @@ -184,7 +184,7 @@ Qnn_ErrorHandle_t WindowsFileMapper::ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_ void* unaligned_data_ptr = data_mem.dmaBuffer.data; rpcmem_lib_->Api().register_buf(unaligned_data_ptr, data_mem.memSize, -1, - rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); + rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); auto fd = rpcmem_lib_->Api().to_fd(unaligned_data_ptr); if (fd != -1) { From 716b02d0d3b1c32146c0c0c15d2ae14f4d4dd237 Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Thu, 22 Jan 2026 15:06:03 -0800 Subject: [PATCH 08/17] Return empty result on GetCapability() call if rpc mem library is required but not loaded --- .../qnn/builder/qnn_windows_file_mapper.cc | 1 + .../providers/qnn/qnn_execution_provider.cc | 20 +++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc index 0fccf8fe1fdf2..afd90f91f37c2 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc @@ -20,6 +20,7 @@ WindowsFileMapper::WindowsFileMapper(const logging::Logger& logger, std::shared_ptr rpcmem_lib) : logger_(&logger), rpcmem_lib_(rpcmem_lib) { + ORT_ENFORCE(rpcmem_lib); } // Close all handles and registered buffers diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index 94537a7327a67..a54f9899567f4 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -567,11 +567,18 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio } static const std::string QNN_HTP_SHARED_MEMORY_ALLOCATOR_ENABLED = "enable_htp_shared_memory_allocator"; - if (ParseBoolOption(QNN_HTP_SHARED_MEMORY_ALLOCATOR_ENABLED, false, provider_options_map) || enable_file_mapped_weights_) { + bool enable_htp_shared_mem_allocator = ParseBoolOption(QNN_HTP_SHARED_MEMORY_ALLOCATOR_ENABLED, false, provider_options_map); + if (enable_htp_shared_mem_allocator || enable_file_mapped_weights_) { // Initialize rpcmem_library_. - // This is necessary for HtpSharedMemoryAllocator to function and also indicates that the allocator is available. - rpcmem_library_ = std::make_shared(); - model_settings_.htp_shared_memory = true; + // This is necessary for HtpSharedMemoryAllocator and file mapped weights to function and + // also indicates that the allocator is available. + try { + rpcmem_library_ = std::make_shared(); + } catch (std::exception e) { + // If unable to load, return empty capability on GetCapability() call + LOGS_DEFAULT(ERROR) << "Unable to load RPCMEM Library: " << e.what(); + } + model_settings_.htp_shared_memory = enable_htp_shared_mem_allocator; } dump_json_qnn_graph_ = ParseBoolOption("dump_json_qnn_graph", false, provider_options_map); @@ -924,6 +931,11 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer const auto& logger = *GetLogger(); + if (enable_file_mapped_weights_ && !rpcmem_library_) { + LOGS(logger, ERROR) << "RPCMEM Library is required for file mapped weights but is not loaded."; + return result; + } + // Check BF16 compatibility early if (model_settings_.htp_bf16_enable) { // Check SoC model From ed85fa598265fe754dd434735c16688602f44eda Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Thu, 22 Jan 2026 19:04:45 -0800 Subject: [PATCH 09/17] Update logic surrounding rpc mem library initialization Add QnnHtpSharedAllocator to HTP check during unit testing to cover cases where RPCMEM is not available --- .../qnn/builder/qnn_backend_manager.cc | 5 ++- .../qnn/builder/qnn_windows_file_mapper.cc | 43 +++++++++---------- .../qnn/builder/qnn_windows_file_mapper.h | 8 ++-- .../providers/qnn/qnn_execution_provider.cc | 22 ++++++---- .../providers/qnn/qnn_execution_provider.h | 3 +- .../test/providers/qnn/qnn_test_utils.cc | 26 ++++++++--- 6 files changed, 65 insertions(+), 42 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index f04fa8bf9429c..790b894a7b5c3 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -995,6 +995,7 @@ Status QnnBackendManager::CreateContextFromListAsyncWithCallback(const QnnContex std::vector context_params_ptr_list; context_params_list.reserve(context_bin_map.size()); + context_paramsv2_list.reserve(context_bin_map.size()); context_callbacks_list.reserve(context_bin_map.size()); context_params_ptr_list.reserve(context_bin_map.size() + 1); @@ -1030,11 +1031,13 @@ Status QnnBackendManager::CreateContextFromListAsyncWithCallback(const QnnContex QnnContext_Params_t context_params = {QnnContext_ParamsVersion_t::QNN_CONTEXT_PARAMS_VERSION_2, {}}; - context_params.v2 = &context_params_v2; + context_params_list.push_back(std::move(context_params)); context_callbacks_list.push_back(std::move(context_file_map_callbacks)); context_paramsv2_list.push_back(std::move(context_params_v2)); context_params_ptr_list.push_back(&(context_params_list.back())); + + context_params.v2 = &context_paramsv2_list.back(); } context_params_ptr_list.push_back(nullptr); auto result = qnn_interface_.contextCreateFromBinaryListAsync(backend_handle_, diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc index afd90f91f37c2..83d5abde5821f 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc @@ -23,12 +23,19 @@ WindowsFileMapper::WindowsFileMapper(const logging::Logger& logger, ORT_ENFORCE(rpcmem_lib); } -// Close all handles and registered buffers -// Use LOGS_DEFAULT here as this function will be called during destruction of QnnBackendManager -// At time of destruction. Usage of logger_ will not be available and will result in a seg fault WindowsFileMapper::~WindowsFileMapper() { } +static void UnmapFile(void* addr) noexcept { + bool successful = UnmapViewOfFile(addr); + if (!successful) { + const auto error_code = GetLastError(); + LOGS_DEFAULT(ERROR) << "Failed to unmap view of file with ptr: " << addr + << ", Error code: " << error_code << ", \"" + << std::system_category().message(error_code) << "\""; + } +} + Status WindowsFileMapper::GetContextBinMappedMemoryPtr(const std::string& bin_filepath, void** mapped_data_ptr) { LOGS(*logger_, INFO) << "Creating context bin file mapping for " @@ -36,6 +43,15 @@ Status WindowsFileMapper::GetContextBinMappedMemoryPtr(const std::string& bin_fi ORT_RETURN_IF(bin_filepath.empty(), "Context bin file path is empty"); + std::lock_guard lock(map_mutex_); + auto map_it = mapped_memory_ptrs_.find(bin_filepath); + if (map_it != mapped_memory_ptrs_.end()) { + *mapped_data_ptr = map_it->second.get(); + LOGS(*logger_, INFO) << "Found existing mapview memory pointer (" << mapped_data_ptr + << ") for context bin file: " << bin_filepath; + return Status::OK(); + } + std::wstring bin_filepath_wstr(bin_filepath.begin(), bin_filepath.end()); wil::unique_hfile file_handle{CreateFile2(bin_filepath_wstr.c_str(), GENERIC_READ, @@ -91,21 +107,11 @@ Status WindowsFileMapper::GetContextBinMappedMemoryPtr(const std::string& bin_fi }}; *mapped_data_ptr = mapped_memory_ptr.get(); - mapped_memory_ptrs.push_back(std::move(mapped_memory_ptr)); + mapped_memory_ptrs_.emplace(bin_filepath, std::move(mapped_memory_ptr)); return Status::OK(); } -void WindowsFileMapper::UnmapFile(void* addr) noexcept { - bool successful = UnmapViewOfFile(addr); - if (!successful) { - const auto error_code = GetLastError(); - LOGS_DEFAULT(ERROR) << "Failed to unmap view of file with ptr: " << addr - << ", Error code: " << error_code << ", \"" - << std::system_category().message(error_code) << "\""; - } -} - Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t request, Qnn_ContextBinaryDmaDataResponse_t* response, void* mapped_data_ptr) { @@ -139,21 +145,12 @@ Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t r << "), aligned offset(" << aligned_offset << "), delta(" << delta << "), unaligned address(" << unaligned_data_ptr << ")"; - if (!rpcmem_lib_) { - LOGS(*logger_, ERROR) << "RPCMem Libs are not initialized"; - return QNN_COMMON_ERROR_RESOURCE_UNAVAILABLE; - } - rpcmem_lib_->Api().register_buf(unaligned_data_ptr, buffer_size, NULL, rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); auto fd = rpcmem_lib_->Api().to_fd(unaligned_data_ptr); if (fd == -1) { LOGS(*logger_, ERROR) << "Failed to register DMA data mapping to RPCMEM"; - if (!UnmapViewOfFile(aligned_data_ptr)) { - LOGS(*logger_, ERROR) << "Failed to unmap DMA data with error code " << GetLastError() - << " with address : " << aligned_data_ptr; - } return QNN_COMMON_ERROR_SYSTEM; } diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h index 675c3020d13c9..dac07f7c36983 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h @@ -6,12 +6,12 @@ #include "core/providers/qnn/builder/qnn_file_mapping_callback_interface.h" #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE +#include #include #include #include -#include "core/platform/env.h" #include "core/providers/qnn/rpcmem_library.h" #include "core/providers/qnn/ort_api.h" @@ -27,8 +27,6 @@ class WindowsFileMapper : public FileMappingCallbackInterface { Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath, void** mapped_data_ptr) override; - static void UnmapFile(void* addr) noexcept; - Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, Qnn_ContextBinaryDmaDataResponse_t* response, void* mapped_data_ptr) override; @@ -43,7 +41,9 @@ class WindowsFileMapper : public FileMappingCallbackInterface { private: // A container of smart pointers of mapview memory pointers to mapped context bins - std::vector mapped_memory_ptrs; + // key: filepath to context bin, value: smart pointer of mapview memory pointers + std::mutex map_mutex_; + std::unordered_map mapped_memory_ptrs_; const logging::Logger* logger_; std::shared_ptr rpcmem_lib_; }; diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index a54f9899567f4..cc693fad58555 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -567,18 +567,24 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio } static const std::string QNN_HTP_SHARED_MEMORY_ALLOCATOR_ENABLED = "enable_htp_shared_memory_allocator"; - bool enable_htp_shared_mem_allocator = ParseBoolOption(QNN_HTP_SHARED_MEMORY_ALLOCATOR_ENABLED, false, provider_options_map); - if (enable_htp_shared_mem_allocator || enable_file_mapped_weights_) { + enable_htp_shared_mem_allocator_ = ParseBoolOption(QNN_HTP_SHARED_MEMORY_ALLOCATOR_ENABLED, false, provider_options_map); + if (enable_htp_shared_mem_allocator_) { // Initialize rpcmem_library_. - // This is necessary for HtpSharedMemoryAllocator and file mapped weights to function and - // also indicates that the allocator is available. + // This is necessary for HtpSharedMemoryAllocator to function and also indicates that the allocator is available. + rpcmem_library_ = std::make_shared(); + model_settings_.htp_shared_memory = enable_htp_shared_mem_allocator_; + } + + if (enable_file_mapped_weights_ && !rpcmem_library_) { + // Attempt to init rpcmem_library_ if needed. If this fails, then + // disable file mapped weights and proceed with normal operation try { rpcmem_library_ = std::make_shared(); - } catch (std::exception e) { - // If unable to load, return empty capability on GetCapability() call - LOGS_DEFAULT(ERROR) << "Unable to load RPCMEM Library: " << e.what(); + } catch (const std::exception& e) { + LOGS_DEFAULT(WARNING) << "Unable to load RPCMem library: " << e.what() + << " - Disabling file mapped weights."; + enable_file_mapped_weights_ = false; } - model_settings_.htp_shared_memory = enable_htp_shared_mem_allocator; } dump_json_qnn_graph_ = ParseBoolOption("dump_json_qnn_graph", false, provider_options_map); diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.h b/onnxruntime/core/providers/qnn/qnn_execution_provider.h index 05a73304806ec..f7022229f6c7b 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.h +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.h @@ -80,7 +80,7 @@ class QNNExecutionProvider : public IExecutionProvider { qnn::ProfilingLevel GetProfilingLevelFromETWLevel(unsigned char level); - bool IsHtpSharedMemoryAllocatorAvailable() const { return rpcmem_library_ != nullptr; } + bool IsHtpSharedMemoryAllocatorAvailable() const { return enable_htp_shared_mem_allocator_ && rpcmem_library_ != nullptr; } private: // Will return true if any power config options need to be updated @@ -120,6 +120,7 @@ class QNNExecutionProvider : public IExecutionProvider { bool stop_share_ep_contexts_ = false; bool enable_spill_fill_buffer_ = false; bool enable_file_mapped_weights_ = true; + bool enable_htp_shared_mem_allocator_ = false; #if defined(_WIN32) onnxruntime::logging::EtwRegistrationManager::EtwInternalCallback callback_ETWSink_provider_ = nullptr; #endif diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.cc b/onnxruntime/test/providers/qnn/qnn_test_utils.cc index 9ad34788444db..a6d43a3d3a9d9 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.cc +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.cc @@ -408,12 +408,28 @@ static BackendSupport GetHTPSupport(const onnxruntime::logging::Logger& logger) // Create QNN EP and call GetCapability(). MockKernelLookup kernel_lookup; onnxruntime::GraphViewer graph_viewer(graph); - std::unique_ptr qnn_ep = QnnExecutionProviderWithOptions( - {{"backend_type", "htp"}, {"offload_graph_io_quantization", "0"}}); - GraphOptimizerRegistry graph_optimizer_registry(nullptr, nullptr, nullptr); // as a placeholder to feed into GetCapability - qnn_ep->SetLogger(&logger); - auto result = qnn_ep->GetCapability(graph_viewer, kernel_lookup, graph_optimizer_registry, nullptr); + std::vector> result; + std::unique_ptr qnn_ep; + try { + qnn_ep = QnnExecutionProviderWithOptions( + {{"backend_type", "htp"}, {"offload_graph_io_quantization", "0"}, {"enable_htp_shared_memory_allocator", "1"}}); + GraphOptimizerRegistry graph_optimizer_registry(nullptr, nullptr, nullptr); // as a placeholder to feed into GetCapability + + qnn_ep->SetLogger(&logger); + result = qnn_ep->GetCapability(graph_viewer, kernel_lookup, graph_optimizer_registry, nullptr); + } catch (const std::exception& e) { + // handle exception that indicates that the libcdsprpc.so / dll can't be loaded + std::string_view error_message = e.what(); + std::string_view expected_error_message = "Failed to initialize RPCMEM dynamic library handle"; + + if (error_message.find(expected_error_message) != std::string_view::npos) { + return BackendSupport::UNSUPPORTED; + } + + // propagate other exceptions + throw; + } return result.empty() ? BackendSupport::UNSUPPORTED : BackendSupport::SUPPORTED; } From fbbe148b313c5f3349b7b45001df5b372b38dcff Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Thu, 22 Jan 2026 19:19:55 -0800 Subject: [PATCH 10/17] spacing for error message --- onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index 790b894a7b5c3..c9779f73923fd 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -1265,7 +1265,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t ORT_RETURN_IF_ERROR(GetFileSizeIfValid(context_bin_filepath, buffer_length)); - ORT_RETURN_IF(buffer_length == 0, "Context bin has a size of 0 bytes", context_bin_filepath); + ORT_RETURN_IF(buffer_length == 0, "Context bin has a size of 0 bytes: ", context_bin_filepath); ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappedMemoryPtr(context_bin_filepath, &bin_buffer)); } else { From 134d6fc4c1ce46cc4a23aa3d274ebf827a649159 Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Thu, 22 Jan 2026 20:55:09 -0800 Subject: [PATCH 11/17] Resolve pointer issues with CreateContextFromListAsyncWithCallback() --- .../core/providers/qnn/builder/qnn_backend_manager.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index c9779f73923fd..dab9c0bcb5d31 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -1019,6 +1019,7 @@ Status QnnBackendManager::CreateContextFromListAsyncWithCallback(const QnnContex context_file_map_callbacks.dmaBufferCallback.v1.notifyParam = reinterpret_cast(notify_param_ptr.get()); file_mapping_notify_params_.push_back(std::move(notify_param_ptr)); + context_callbacks_list.push_back(std::move(context_file_map_callbacks)); // Callbacks require QnnContext_ParamsV2_t which is new to QNN API 2.32 QnnContext_ParamsV2_t context_params_v2 = {nullptr, @@ -1027,17 +1028,16 @@ Status QnnBackendManager::CreateContextFromListAsyncWithCallback(const QnnContex nullptr, ContextCreateAsyncCallback, it.second.get(), - &context_file_map_callbacks}; + &context_callbacks_list.back()}; QnnContext_Params_t context_params = {QnnContext_ParamsVersion_t::QNN_CONTEXT_PARAMS_VERSION_2, {}}; - context_params_list.push_back(std::move(context_params)); - context_callbacks_list.push_back(std::move(context_file_map_callbacks)); context_paramsv2_list.push_back(std::move(context_params_v2)); - context_params_ptr_list.push_back(&(context_params_list.back())); context_params.v2 = &context_paramsv2_list.back(); + context_params_list.push_back(std::move(context_params)); + context_params_ptr_list.push_back(&(context_params_list.back())); } context_params_ptr_list.push_back(nullptr); auto result = qnn_interface_.contextCreateFromBinaryListAsync(backend_handle_, From 9a7b4f03d14a3f52fdacc66ee28e897d1cfc2292 Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Fri, 23 Jan 2026 13:22:18 -0800 Subject: [PATCH 12/17] Addrses minor PR comments --- .../core/providers/qnn/builder/qnn_backend_manager.cc | 2 +- .../core/providers/qnn/builder/qnn_windows_file_mapper.cc | 7 +++---- .../core/providers/qnn/builder/qnn_windows_file_mapper.h | 1 + onnxruntime/core/providers/qnn/qnn_execution_provider.cc | 5 ----- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index dab9c0bcb5d31..e310ad2019d58 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -804,7 +804,7 @@ static Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, if (pair->first == nullptr) { LOGS_DEFAULT(ERROR) << "DmaRelease: file mapper is null"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; - }; + } return pair->first->ReleaseDmaData(data_mem, pair->second); } diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc index 83d5abde5821f..96c435ce78f2b 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc @@ -4,14 +4,13 @@ #include "core/providers/qnn/builder/qnn_windows_file_mapper.h" #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE -#include +#include + +#include #include #include "core/providers/qnn/ort_api.h" -#include "core/providers/qnn/rpcmem_library.h" - -#include namespace onnxruntime { namespace qnn { diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h index dac07f7c36983..444590ad8c9d9 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h @@ -6,6 +6,7 @@ #include "core/providers/qnn/builder/qnn_file_mapping_callback_interface.h" #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE +#include #include #include #include diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index cc693fad58555..a6f1d1c1681cf 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -937,11 +937,6 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer const auto& logger = *GetLogger(); - if (enable_file_mapped_weights_ && !rpcmem_library_) { - LOGS(logger, ERROR) << "RPCMEM Library is required for file mapped weights but is not loaded."; - return result; - } - // Check BF16 compatibility early if (model_settings_.htp_bf16_enable) { // Check SoC model From 5b9a73ee1701c36282dcdd8ac88761fbec6f4b4b Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Fri, 23 Jan 2026 15:40:10 -0800 Subject: [PATCH 13/17] Decouple DMA mapping of weights and file mapping of context bin --- .../qnn/builder/qnn_backend_manager.cc | 144 ++++++++++++++---- .../qnn/builder/qnn_backend_manager.h | 15 +- .../qnn_file_mapping_callback_interface.h | 37 ----- .../qnn/builder/qnn_file_mapping_interface.h | 35 +++++ .../qnn/builder/qnn_windows_file_mapper.cc | 114 ++------------ .../qnn/builder/qnn_windows_file_mapper.h | 27 ++-- 6 files changed, 185 insertions(+), 187 deletions(-) delete mode 100644 onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h create mode 100644 onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index e310ad2019d58..5346bfaf7ccfc 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -775,39 +775,121 @@ Status SetQnnContextConfig(ContextPriority context_priority, QnnContext_Config_t } #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE -// callback required for allocating file mapping resources -static Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, - Qnn_ContextBinaryDmaDataResponse_t* response, void* notify_param) { +// Callback required for allocating file mapping resources +static Qnn_ErrorHandle_t MapDmaDataCallback(Qnn_ContextBinaryDataRequest_t request, + Qnn_ContextBinaryDmaDataResponse_t* response, void* notify_param) { if (notify_param == nullptr) { - LOGS_DEFAULT(ERROR) << "DmaProvider: notify_param is null"; + LOGS_DEFAULT(ERROR) << "MapDmaDataCallback: notify_param is null"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } - auto pair = reinterpret_cast*>(notify_param); + auto pair = reinterpret_cast*>(notify_param); if (pair->first == nullptr) { - LOGS_DEFAULT(ERROR) << "DmaProvider: file mapper is null"; + LOGS_DEFAULT(ERROR) << "MapDmaDataCallback: QnnBackendManager is null"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } return pair->first->MapDmaData(request, response, pair->second); } -// callback required for releasing file mapping resources -static Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, void* notify_param) { +Qnn_ErrorHandle_t QnnBackendManager::MapDmaData(Qnn_ContextBinaryDataRequest_t request, + Qnn_ContextBinaryDmaDataResponse_t* response, + void* mapped_base_ptr) { + if (!file_mapped_weights_enabled_) { + LOGS(*logger_, WARNING) << "Attempting to map DMA data but file mapping has been disabled, " + << "possibly due to an error in a previous request."; + return QNN_CONTEXT_ERROR_ABORTED; + } + + if (mapped_base_ptr == nullptr) { + LOGS(*logger_, ERROR) << "Attempting to map DMA data for null memory mapped base pointer"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + + LOGS(*logger_, INFO) << "Mapping DMA data for request: memory mapped base pointer(" + << mapped_base_ptr << "), offset(" << request.offset + << "), size(" << request.size << "), isBackendMappingNeeded(" + << request.isBackendMappingNeeded << ")"; + + auto size = request.size; + if (size == 0 || !request.isBackendMappingNeeded) { + LOGS(*logger_, ERROR) << "Mapping request size must be > 0 with backend mapping required"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + + auto mapped_weight_info = file_mapper_->GetMappedWeightMemoryPtr(mapped_base_ptr, request.offset); + void* unaligned_data_ptr = mapped_weight_info.unaligned_data_ptr; + + LOGS(*logger_, INFO) << "Created DMA data mapping with: address(" << mapped_weight_info.aligned_data_ptr + << "), aligned offset(" << mapped_weight_info.aligned_offset + << "), delta(" << mapped_weight_info.delta + << "), unaligned address(" << unaligned_data_ptr << ")"; + + rpcmem_library_->Api().register_buf(unaligned_data_ptr, size, NULL, + rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); + + auto fd = rpcmem_library_->Api().to_fd(unaligned_data_ptr); + if (fd == -1) { + LOGS(*logger_, ERROR) << "Failed to register DMA data mapping to RPCMEM"; + return QNN_COMMON_ERROR_SYSTEM; + } + + response->dmaBuffer.fd = fd; + response->dmaBuffer.data = reinterpret_cast(unaligned_data_ptr); + response->dataStartOffset = 0; + response->alignedSize = size; + + return QNN_SUCCESS; +} + +// Callback required for releasing file mapping resources +static Qnn_ErrorHandle_t ReleaseDmaDataCallback(Qnn_ContextBinaryDmaDataMem_t data_mem, void* notify_param) { if (notify_param == nullptr) { - LOGS_DEFAULT(ERROR) << "DmaRelease: notify_param is null"; + LOGS_DEFAULT(ERROR) << "ReleaseDmaDataCallback: notify_param is null"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } - auto pair = reinterpret_cast*>(notify_param); + auto pair = reinterpret_cast*>(notify_param); if (pair->first == nullptr) { - LOGS_DEFAULT(ERROR) << "DmaRelease: file mapper is null"; + LOGS_DEFAULT(ERROR) << "ReleaseDmaDataCallback: QnnBackendManager is null"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } return pair->first->ReleaseDmaData(data_mem, pair->second); } + +// Use LOGS_DEFAULT here as this function will be called during destruction of QnnBackendManager +// At time of destruction, usage of logger_ will not be available and will result in a seg fault +Qnn_ErrorHandle_t QnnBackendManager::ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, + void* mapped_base_ptr) { + if (mapped_base_ptr == nullptr) { + LOGS_DEFAULT(ERROR) << "Attempting to release DMA data for null memory mapped pointer"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + + LOGS_DEFAULT(INFO) << "Releasing DMA data mapping for memory mapped pointer(" + << mapped_base_ptr << "), address(" << data_mem.dmaBuffer.data + << "), size: (" << data_mem.memSize << ")"; + + if (data_mem.dmaBuffer.data == nullptr || data_mem.memSize == 0) { + LOGS_DEFAULT(ERROR) << "Mapping release request address must not be null and size must be > 0"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + + // Deregister file mapped data from NPU regardless of file_mapped_weights_enabled_ + // as there may be file mapped data registered to the NPU prior to any mapping error + void* unaligned_data_ptr = data_mem.dmaBuffer.data; + rpcmem_library_->Api().register_buf(unaligned_data_ptr, data_mem.memSize, -1, + rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); + + auto fd = rpcmem_library_->Api().to_fd(unaligned_data_ptr); + if (fd != -1) { + LOGS_DEFAULT(ERROR) << "Failed to deregister buffer from RPCMEM: " << unaligned_data_ptr; + return QNN_CONTEXT_ERROR_MEM_ALLOC; + } + return QNN_SUCCESS; +} #endif // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE // callback required to add context handles to class list @@ -1008,14 +1090,14 @@ Status QnnBackendManager::CreateContextFromListAsyncWithCallback(const QnnContex void* buffer; ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappedMemoryPtr(context_bin_filepath, &buffer)); - auto notify_param_ptr = std::make_unique>(file_mapper_.get(), - buffer); + auto notify_param_ptr = std::make_unique>(this, + buffer); Qnn_ContextBinaryCallback_t context_file_map_callbacks; context_file_map_callbacks.type = QNN_CONTEXT_CALLBACK_DMA_BUFFER; context_file_map_callbacks.dmaBufferCallback.version = QNN_CONTEXT_CALLBACK_DMA_BUFFER_VERSION_1; - context_file_map_callbacks.dmaBufferCallback.v1.dataProvide = MapDmaData; - context_file_map_callbacks.dmaBufferCallback.v1.dataRelease = ReleaseDmaData; + context_file_map_callbacks.dmaBufferCallback.v1.dataProvide = MapDmaDataCallback; + context_file_map_callbacks.dmaBufferCallback.v1.dataRelease = ReleaseDmaDataCallback; context_file_map_callbacks.dmaBufferCallback.v1.notifyParam = reinterpret_cast(notify_param_ptr.get()); file_mapping_notify_params_.push_back(std::move(notify_param_ptr)); @@ -1364,13 +1446,13 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t ORT_RETURN_IF(nullptr == qnn_interface_.contextCreateFromBinaryWithCallback, "Invalid function pointer for contextCreateFromBinaryWithCallback."); - auto notify_param_ptr = std::make_unique>(file_mapper_.get(), - bin_buffer); + auto notify_param_ptr = std::make_unique>(this, + bin_buffer); callbacks.type = QNN_CONTEXT_CALLBACK_DMA_BUFFER; callbacks.dmaBufferCallback.version = QNN_CONTEXT_CALLBACK_DMA_BUFFER_VERSION_1; - callbacks.dmaBufferCallback.v1.dataProvide = MapDmaData; - callbacks.dmaBufferCallback.v1.dataRelease = ReleaseDmaData; + callbacks.dmaBufferCallback.v1.dataProvide = MapDmaDataCallback; + callbacks.dmaBufferCallback.v1.dataRelease = ReleaseDmaDataCallback; callbacks.dmaBufferCallback.v1.notifyParam = reinterpret_cast(notify_param_ptr.get()); file_mapping_notify_params_.push_back(std::move(notify_param_ptr)); @@ -1405,8 +1487,6 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t << ". Retrying with feature disabled."; file_mapped_weights_enabled_ = false; - // Destruction of file_mapper_ to prevent resource leaks - file_mapper_.reset(); // Read context bin from file since file mapping has failed BufferInfo_t buffer_info; @@ -1507,7 +1587,6 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, return Status::OK(); } - file_mapped_weights_enabled_ = enable_file_mapped_weights; vtcm_backup_buffer_sharing_enabled_ = enable_vtcm_backup_buffer_sharing; Status status = Status::OK(); @@ -1516,6 +1595,19 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, } else { status = LoadQnnSerializerBackend(); } + +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE + // Backend is determined after LoadBackend() or LoadQnnSerializerBackend() + if (enable_file_mapped_weights && !file_mapper_ && GetQnnBackendType() == QnnBackendType::HTP) { + ORT_RETURN_IF(!rpcmem_library, "RPCMem Library is required for file mapping but is uninitialized."); + rpcmem_library_ = rpcmem_library; + file_mapped_weights_enabled_ = true; + file_mapper_ = std::make_unique(logger); + } +#else + ORT_UNUSED_PARAMETER(rpcmem_library); +#endif + if (status.IsOK()) { LOGS(logger, VERBOSE) << "LoadBackend succeed."; } @@ -1563,14 +1655,6 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, LOGS(logger, VERBOSE) << "LoadOpPackage succeed."; } -#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE - if (file_mapped_weights_enabled_ && !file_mapper_ && GetQnnBackendType() == QnnBackendType::HTP) { - file_mapper_ = std::make_unique(logger, rpcmem_library); - } -#else - ORT_UNUSED_PARAMETER(rpcmem_library); -#endif - bool enable_htp_weight_sharing = false; if (share_ep_contexts && !load_from_cached_context) { #if defined(__aarch64__) || defined(_M_ARM64) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index cd02d1f855556..6e341bc86cf1c 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -34,7 +34,7 @@ #include "core/providers/qnn/builder/qnn_node_group/qnn_node_group.h" #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE -#include "core/providers/qnn/builder/qnn_file_mapping_callback_interface.h" +#include "core/providers/qnn/builder/qnn_file_mapping_interface.h" #endif namespace onnxruntime { @@ -258,6 +258,13 @@ class QnnBackendManager : public std::enable_shared_from_this bool FileMappingIsEnabled() { return file_mapped_weights_enabled_; } +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE + Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, + Qnn_ContextBinaryDmaDataResponse_t* response, void* mapped_base_ptr); + + Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, void* mapped_base_ptr); +#endif + private: typedef struct BufferInfo { std::unique_ptr data; @@ -486,10 +493,10 @@ class QnnBackendManager : public std::enable_shared_from_this bool file_mapped_weights_enabled_ = false; #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE - std::unique_ptr file_mapper_ = nullptr; + std::unique_ptr file_mapper_ = nullptr; // Notify params for file mapping must persist throughout lifetime of // QnnBackendManager for release of DMA data callback on destruction - std::vector>> file_mapping_notify_params_; + std::vector>> file_mapping_notify_params_; #endif // NPU backend requires quantized model @@ -510,6 +517,8 @@ class QnnBackendManager : public std::enable_shared_from_this // Mapping of thread id to on-run-start/end power configs std::mutex per_thread_power_configs_mutex_; std::unordered_map per_thread_power_configs_; + + std::shared_ptr rpcmem_library_ = nullptr; }; } // namespace qnn diff --git a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h deleted file mode 100644 index ad2b267ac7f87..0000000000000 --- a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_callback_interface.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include - -#include - -#include "core/providers/qnn/ort_api.h" -#include "core/providers/qnn/builder/qnn_def.h" - -namespace onnxruntime { -namespace qnn { - -class FileMappingCallbackInterface { - public: - virtual ~FileMappingCallbackInterface() = default; - - virtual Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath, - void** mapped_data_ptr) = 0; - - virtual Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, - Qnn_ContextBinaryDmaDataResponse_t* response, - void* mapped_data_ptr) = 0; - virtual Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, - void* mapped_data_ptr) = 0; - - virtual Qnn_ErrorHandle_t MapRawData(Qnn_ContextBinaryDataRequest_t request, - Qnn_ContextBinaryRawDataResponse_t* response, - void* mapped_data_ptr) = 0; - virtual Qnn_ErrorHandle_t ReleaseRawData(Qnn_ContextBinaryRawDataMem_t data_mem, - void* mapped_data_ptr) = 0; -}; - -} // namespace qnn -} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h new file mode 100644 index 0000000000000..c6dd9e7f75460 --- /dev/null +++ b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include + +#include "core/providers/qnn/ort_api.h" +#include "core/providers/qnn/builder/qnn_def.h" + +namespace onnxruntime { +namespace qnn { + +class FileMappingInterface { + public: + typedef struct MappedWeightInfo { + size_t aligned_offset = 0; + size_t delta = 0; + void* aligned_data_ptr = nullptr; + void* unaligned_data_ptr = nullptr; + } MappedWeightInfo_t; + + virtual ~FileMappingInterface() = default; + + virtual Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath, + void** mapped_data_ptr) = 0; + + virtual MappedWeightInfo_t GetMappedWeightMemoryPtr(void* mapped_base_ptr, + const size_t offset) = 0; +}; + +} // namespace qnn +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc index 96c435ce78f2b..6800ef27fe349 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc @@ -15,11 +15,8 @@ namespace onnxruntime { namespace qnn { -WindowsFileMapper::WindowsFileMapper(const logging::Logger& logger, - std::shared_ptr rpcmem_lib) - : logger_(&logger), - rpcmem_lib_(rpcmem_lib) { - ORT_ENFORCE(rpcmem_lib); +WindowsFileMapper::WindowsFileMapper(const logging::Logger& logger) + : logger_(&logger) { } WindowsFileMapper::~WindowsFileMapper() { @@ -111,110 +108,27 @@ Status WindowsFileMapper::GetContextBinMappedMemoryPtr(const std::string& bin_fi return Status::OK(); } -Qnn_ErrorHandle_t WindowsFileMapper::MapDmaData(Qnn_ContextBinaryDataRequest_t request, - Qnn_ContextBinaryDmaDataResponse_t* response, - void* mapped_data_ptr) { - if (mapped_data_ptr == nullptr) { - LOGS(*logger_, ERROR) << "Attempting to map DMA data for null memory mapped pointer"; - return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; - } - - LOGS(*logger_, INFO) << "Mapping DMA data for request: memory mapped pointer(" - << mapped_data_ptr << "), offset(" << request.offset - << "), size(" << request.size << "), isBackendMappingNeeded(" - << request.isBackendMappingNeeded << ")"; - - auto buffer_size = request.size; - if (buffer_size == 0 || !request.isBackendMappingNeeded) { - LOGS(*logger_, ERROR) << "Mapping request size must be > 0 with backend mapping required"; - return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; - } +FileMappingInterface::MappedWeightInfo_t +WindowsFileMapper::GetMappedWeightMemoryPtr(void* mapped_base_ptr, + const size_t offset) { + MappedWeightInfo_t mapped_weight_info; // Align to nearest granularity boundary SYSTEM_INFO sys_info; GetSystemInfo(&sys_info); Qnn_ContextBinarySize_t granularity = sys_info.dwAllocationGranularity; - SIZE_T aligned_offset = request.offset & ~(granularity - 1); - SIZE_T delta = request.offset - aligned_offset; + SIZE_T aligned_offset = offset & ~(granularity - 1); + SIZE_T delta = offset - aligned_offset; - void* aligned_data_ptr = static_cast(mapped_data_ptr) + aligned_offset; + mapped_weight_info.aligned_offset = aligned_offset; + mapped_weight_info.delta = delta; - void* unaligned_data_ptr = static_cast(aligned_data_ptr) + delta; - LOGS(*logger_, INFO) << "Created DMA data mapping with: address(" << aligned_data_ptr - << "), aligned offset(" << aligned_offset << "), delta(" << delta - << "), unaligned address(" << unaligned_data_ptr << ")"; - - rpcmem_lib_->Api().register_buf(unaligned_data_ptr, buffer_size, NULL, - rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); - - auto fd = rpcmem_lib_->Api().to_fd(unaligned_data_ptr); - if (fd == -1) { - LOGS(*logger_, ERROR) << "Failed to register DMA data mapping to RPCMEM"; - return QNN_COMMON_ERROR_SYSTEM; - } + void* aligned_data_ptr = static_cast(mapped_base_ptr) + aligned_offset; + mapped_weight_info.aligned_data_ptr = aligned_data_ptr; + mapped_weight_info.unaligned_data_ptr = static_cast(aligned_data_ptr) + delta; - response->dmaBuffer.fd = fd; - response->dmaBuffer.data = reinterpret_cast(unaligned_data_ptr); - response->dataStartOffset = 0; - response->alignedSize = buffer_size; - - return QNN_SUCCESS; + return mapped_weight_info; } - -// Use LOGS_DEFAULT here as this function will be called during destruction of QnnBackendManager -// At time of destruction. Usage of logger_ will not be available and will result in a seg fault -Qnn_ErrorHandle_t WindowsFileMapper::ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, - void* mapped_data_ptr) { - if (mapped_data_ptr == nullptr) { - LOGS_DEFAULT(ERROR) << "Attempting to release DMA data for null memory mapped pointer"; - return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; - } - - LOGS_DEFAULT(INFO) << "Releasing DMA data mapping for memory mapped pointer(" - << mapped_data_ptr << "), address(" << data_mem.dmaBuffer.data - << "), size: (" << data_mem.memSize << ")"; - - if (data_mem.dmaBuffer.data == nullptr || data_mem.memSize == 0) { - LOGS_DEFAULT(ERROR) << "Mapping release request address must not be null and size must be > 0"; - return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; - } - - void* unaligned_data_ptr = data_mem.dmaBuffer.data; - rpcmem_lib_->Api().register_buf(unaligned_data_ptr, data_mem.memSize, -1, - rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); - - auto fd = rpcmem_lib_->Api().to_fd(unaligned_data_ptr); - if (fd != -1) { - LOGS_DEFAULT(ERROR) << "Failed to deregister buffer from RPCMEM: " << unaligned_data_ptr; - return QNN_CONTEXT_ERROR_MEM_ALLOC; - } - - return QNN_SUCCESS; -} - -Qnn_ErrorHandle_t WindowsFileMapper::MapRawData(Qnn_ContextBinaryDataRequest_t request, - Qnn_ContextBinaryRawDataResponse_t* response, - void* mapped_data_ptr) { - ORT_UNUSED_PARAMETER(request); - ORT_UNUSED_PARAMETER(response); - ORT_UNUSED_PARAMETER(mapped_data_ptr); - - LOGS(*logger_, ERROR) << "File mapping for raw binary data is unsupported on Windows"; - return QNN_CONTEXT_ERROR_UNSUPPORTED_FEATURE; -} - -// Use LOGS_DEFAULT for all clean up functions below as they will be called during destruction of -// QnnBackendManager at time of destruction. Usage of logger_ will not be available and will result -// in a seg fault -Qnn_ErrorHandle_t WindowsFileMapper::ReleaseRawData(Qnn_ContextBinaryRawDataMem_t data_mem, - void* mapped_data_ptr) { - ORT_UNUSED_PARAMETER(data_mem); - ORT_UNUSED_PARAMETER(mapped_data_ptr); - - LOGS_DEFAULT(ERROR) << "File mapping for raw binary data is unsupported on Windows"; - return QNN_CONTEXT_ERROR_UNSUPPORTED_FEATURE; -} - } // namespace qnn } // namespace onnxruntime diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h index 444590ad8c9d9..90dc29036eb2a 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h @@ -3,7 +3,7 @@ #pragma once -#include "core/providers/qnn/builder/qnn_file_mapping_callback_interface.h" +#include "core/providers/qnn/builder/qnn_file_mapping_interface.h" #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE #include @@ -13,32 +13,26 @@ #include -#include "core/providers/qnn/rpcmem_library.h" #include "core/providers/qnn/ort_api.h" namespace onnxruntime { namespace qnn { -class WindowsFileMapper : public FileMappingCallbackInterface { +class WindowsFileMapper : public FileMappingInterface { public: - explicit WindowsFileMapper(const logging::Logger& logger, - std::shared_ptr rpcmem_lib); + explicit WindowsFileMapper(const logging::Logger& logger); ~WindowsFileMapper() override; + // Creates a file mapping of the context binary and returns the + // mapview pointer of the file mapping Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath, void** mapped_data_ptr) override; - Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, - Qnn_ContextBinaryDmaDataResponse_t* response, - void* mapped_data_ptr) override; - Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, - void* mapped_data_ptr) override; - - Qnn_ErrorHandle_t MapRawData(Qnn_ContextBinaryDataRequest_t request, - Qnn_ContextBinaryRawDataResponse_t* response, - void* mapped_data_ptr) override; - Qnn_ErrorHandle_t ReleaseRawData(Qnn_ContextBinaryRawDataMem_t data_mem, - void* mapped_data_ptr) override; + // Given a base mapview pointer of a context binary and an offset, + // calcualte the aligned and unaligned memory pointers to a weight + // within the context binary + MappedWeightInfo_t GetMappedWeightMemoryPtr(void* mapped_base_ptr, + const size_t offset) override; private: // A container of smart pointers of mapview memory pointers to mapped context bins @@ -46,7 +40,6 @@ class WindowsFileMapper : public FileMappingCallbackInterface { std::mutex map_mutex_; std::unordered_map mapped_memory_ptrs_; const logging::Logger* logger_; - std::shared_ptr rpcmem_lib_; }; } // namespace qnn From b507234ca6742631492da940250eb0acb0c8fa26 Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Fri, 23 Jan 2026 15:50:53 -0800 Subject: [PATCH 14/17] linting --- .../qnn/builder/qnn_file_mapping_interface.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h index c6dd9e7f75460..28ddb70215295 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h @@ -15,20 +15,20 @@ namespace qnn { class FileMappingInterface { public: - typedef struct MappedWeightInfo { + typedef struct MappedWeightInfo { size_t aligned_offset = 0; size_t delta = 0; void* aligned_data_ptr = nullptr; void* unaligned_data_ptr = nullptr; - } MappedWeightInfo_t; + } MappedWeightInfo_t; - virtual ~FileMappingInterface() = default; + virtual ~FileMappingInterface() = default; - virtual Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath, - void** mapped_data_ptr) = 0; + virtual Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath, + void** mapped_data_ptr) = 0; - virtual MappedWeightInfo_t GetMappedWeightMemoryPtr(void* mapped_base_ptr, - const size_t offset) = 0; + virtual MappedWeightInfo_t GetMappedWeightMemoryPtr(void* mapped_base_ptr, + const size_t offset) = 0; }; } // namespace qnn From f9d373164fc884d52c44da71d9453e6f330e80b0 Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Fri, 23 Jan 2026 16:45:35 -0800 Subject: [PATCH 15/17] Mark enable_file_mapped_weights as unused in QnnBackendManager::SetupBackend if file mapping is not available --- onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index c0d87a6963c49..815dc2dcb341a 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -1621,6 +1621,7 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, file_mapper_ = std::make_unique(logger); } #else + ORT_UNUSED_PARAMETER(enable_file_mapped_weights); ORT_UNUSED_PARAMETER(rpcmem_library); #endif From 3abe5a94e633be78c361daba5076535647b07f89 Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Fri, 23 Jan 2026 19:58:14 -0800 Subject: [PATCH 16/17] Ensure dma mapping offset + size do not exceed file memory space Make file mapping callbacks more thread safe Do not destruct file_mapper_ until session destruction --- .../qnn/builder/qnn_backend_manager.cc | 87 ++++++++++--------- .../qnn/builder/qnn_backend_manager.h | 32 ++++++- .../qnn/builder/qnn_file_mapping_interface.h | 10 --- .../qnn/builder/qnn_windows_file_mapper.cc | 22 ----- .../qnn/builder/qnn_windows_file_mapper.h | 6 -- 5 files changed, 75 insertions(+), 82 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index 815dc2dcb341a..1e0547124a675 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -798,19 +798,23 @@ static Qnn_ErrorHandle_t MapDmaDataCallback(Qnn_ContextBinaryDataRequest_t reque LOGS_DEFAULT(ERROR) << "MapDmaDataCallback: notify_param is null"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } - auto pair = reinterpret_cast*>(notify_param); + auto callback_info = reinterpret_cast(notify_param); - if (pair->first == nullptr) { + if (callback_info->backend_manager == nullptr) { LOGS_DEFAULT(ERROR) << "MapDmaDataCallback: QnnBackendManager is null"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } - return pair->first->MapDmaData(request, response, pair->second); + return callback_info->backend_manager->MapDmaData(request, response, + callback_info->mapped_file_ptr, + callback_info->file_size); } Qnn_ErrorHandle_t QnnBackendManager::MapDmaData(Qnn_ContextBinaryDataRequest_t request, Qnn_ContextBinaryDmaDataResponse_t* response, - void* mapped_base_ptr) { + void* const mapped_base_ptr, + const size_t file_size) { + std::lock_guard lock(file_mapping_mutex_); if (!file_mapped_weights_enabled_) { LOGS(*logger_, WARNING) << "Attempting to map DMA data but file mapping has been disabled, " << "possibly due to an error in a previous request."; @@ -824,7 +828,8 @@ Qnn_ErrorHandle_t QnnBackendManager::MapDmaData(Qnn_ContextBinaryDataRequest_t r LOGS(*logger_, INFO) << "Mapping DMA data for request: memory mapped base pointer(" << mapped_base_ptr << "), offset(" << request.offset - << "), size(" << request.size << "), isBackendMappingNeeded(" + << "), size(" << request.size << "), total file size(" + << file_size << ") isBackendMappingNeeded(" << request.isBackendMappingNeeded << ")"; auto size = request.size; @@ -833,14 +838,20 @@ Qnn_ErrorHandle_t QnnBackendManager::MapDmaData(Qnn_ContextBinaryDataRequest_t r return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } - auto mapped_weight_info = file_mapper_->GetMappedWeightMemoryPtr(mapped_base_ptr, request.offset); - void* unaligned_data_ptr = mapped_weight_info.unaligned_data_ptr; + // offset & size are type uint64_t + // Should never be an issue, but if this occurs then there is something inherently wrong with QNN + if ((UINT64_MAX - request.offset) < size) { + LOGS(*logger_, ERROR) << "Critical error in QNN: mapping request offset + size will overflow 64 bits"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } - LOGS(*logger_, INFO) << "Created DMA data mapping with: address(" << mapped_weight_info.aligned_data_ptr - << "), aligned offset(" << mapped_weight_info.aligned_offset - << "), delta(" << mapped_weight_info.delta - << "), unaligned address(" << unaligned_data_ptr << ")"; + // file_size will be promoted to 64 bits on 32-bit systems + if ((request.offset + size) > file_size) { + LOGS(*logger_, ERROR) << "Requested offset and size includes memory outside of mapped file"; + return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; + } + void* unaligned_data_ptr = static_cast(mapped_base_ptr) + request.offset; rpcmem_library_->Api().register_buf(unaligned_data_ptr, size, NULL, rpcmem::RPCMEM_ATTR_IMPORT_BUFFER | rpcmem::RPCMEM_ATTR_READ_ONLY); @@ -850,8 +861,10 @@ Qnn_ErrorHandle_t QnnBackendManager::MapDmaData(Qnn_ContextBinaryDataRequest_t r return QNN_COMMON_ERROR_SYSTEM; } + LOGS(*logger_, INFO) << "Created DMA data mapping with address: " << unaligned_data_ptr; + response->dmaBuffer.fd = fd; - response->dmaBuffer.data = reinterpret_cast(unaligned_data_ptr); + response->dmaBuffer.data = unaligned_data_ptr; response->dataStartOffset = 0; response->alignedSize = size; @@ -865,14 +878,14 @@ static Qnn_ErrorHandle_t ReleaseDmaDataCallback(Qnn_ContextBinaryDmaDataMem_t da return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } - auto pair = reinterpret_cast*>(notify_param); + auto callback_info = reinterpret_cast(notify_param); - if (pair->first == nullptr) { + if (callback_info->backend_manager == nullptr) { LOGS_DEFAULT(ERROR) << "ReleaseDmaDataCallback: QnnBackendManager is null"; return QNN_CONTEXT_ERROR_INVALID_ARGUMENT; } - return pair->first->ReleaseDmaData(data_mem, pair->second); + return callback_info->backend_manager->ReleaseDmaData(data_mem, callback_info->mapped_file_ptr); } // Use LOGS_DEFAULT here as this function will be called during destruction of QnnBackendManager @@ -967,14 +980,13 @@ Status QnnBackendManager::ReadContextBinIfValid(const std::string& context_bin_f size_t buffer_size; ORT_RETURN_IF_ERROR(GetFileSizeIfValid(context_bin_filepath, buffer_size)); - std::unique_ptr buffer; + std::vector buffer; + buffer.reserve(buffer_size); + std::ifstream cache_file(context_bin_filepath.c_str(), std::ifstream::binary); ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to read context binary from: ", context_bin_filepath); - buffer = std::make_unique(buffer_size); - ORT_RETURN_IF(nullptr == buffer, "Failed to allocate memory for cache file."); - - const auto& read_result = cache_file.read(buffer.get(), buffer_size); + const auto& read_result = cache_file.read(buffer.data(), buffer_size); ORT_RETURN_IF(!read_result, "Failed to read contents from cached context file."); buffer_info.data = std::move(buffer); @@ -1020,15 +1032,14 @@ Status QnnBackendManager::CreateContextVtcmBackupBufferSharingEnabled(std::unord nullptr}; #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE - if (file_mapped_weights_enabled_ && file_mapper_) { + if (FileMappingIsEnabled() && file_mapper_) { // Retry logic -- if context creation failed with file mapped weights, then retry with feature disabled auto res = CreateContextFromListAsyncWithCallback(configs, context_bin_map); if (!res.IsOK()) { + DisableFileMapping(); LOGS(*logger_, WARNING) << res.ErrorMessage() << ". Retrying with feature disabled."; file_mapped_weights_enabled_ = false; - // Destruction of file_mapper_ to prevent resource leaks - file_mapper_.reset(); } else { return Status::OK(); } @@ -1043,7 +1054,7 @@ Status QnnBackendManager::CreateContextFromListAsync(const QnnContext_Config_t** std::vector context_params_list; std::vector context_paramsv1_list; std::vector context_params_ptr_list; - std::vector> buffer_list; + std::vector> buffer_list; context_params_list.reserve(context_bin_map.size()); context_params_ptr_list.reserve(context_bin_map.size() + 1); @@ -1054,11 +1065,11 @@ Status QnnBackendManager::CreateContextFromListAsync(const QnnContext_Config_t** BufferInfo_t buffer_info; ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info)); - std::unique_ptr buffer = std::move(buffer_info.data); + buffer_list.push_back(std::move(buffer_info.data)); size_t buffer_size = buffer_info.size; QnnContext_ParamsV1_t context_params_v1 = {nullptr, - buffer.get(), + buffer_list.back().data(), buffer_size, nullptr, ContextCreateAsyncCallback, @@ -1067,7 +1078,6 @@ Status QnnBackendManager::CreateContextFromListAsync(const QnnContext_Config_t** QnnContext_Params_t context_params = {QnnContext_ParamsVersion_t::QNN_CONTEXT_PARAMS_VERSION_1, {context_params_v1}}; - buffer_list.push_back(std::move(buffer)); context_params_list.push_back(std::move(context_params)); context_paramsv1_list.push_back(std::move(context_params_v1)); context_params_ptr_list.push_back(&context_params_list.back()); @@ -1106,8 +1116,7 @@ Status QnnBackendManager::CreateContextFromListAsyncWithCallback(const QnnContex void* buffer; ORT_RETURN_IF_ERROR(file_mapper_->GetContextBinMappedMemoryPtr(context_bin_filepath, &buffer)); - auto notify_param_ptr = std::make_unique>(this, - buffer); + auto notify_param_ptr = std::make_unique(buffer, buffer_size, this); Qnn_ContextBinaryCallback_t context_file_map_callbacks; context_file_map_callbacks.type = QNN_CONTEXT_CALLBACK_DMA_BUFFER; @@ -1358,7 +1367,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t void* bin_buffer = nullptr; #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE - if (file_mapped_weights_enabled_) { + if (FileMappingIsEnabled()) { ORT_RETURN_IF(!file_mapper_, "Attemping to use File Mapping feature but file_mapper_ is uninitialized"); ORT_RETURN_IF_ERROR(GetFileSizeIfValid(context_bin_filepath, buffer_length)); @@ -1458,12 +1467,11 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE Qnn_ContextBinaryCallback_t callbacks; - if (file_mapped_weights_enabled_ && file_mapper_) { + if (FileMappingIsEnabled() && file_mapper_) { ORT_RETURN_IF(nullptr == qnn_interface_.contextCreateFromBinaryWithCallback, "Invalid function pointer for contextCreateFromBinaryWithCallback."); - auto notify_param_ptr = std::make_unique>(this, - bin_buffer); + auto notify_param_ptr = std::make_unique(bin_buffer, buffer_length, this); callbacks.type = QNN_CONTEXT_CALLBACK_DMA_BUFFER; callbacks.dmaBufferCallback.version = QNN_CONTEXT_CALLBACK_DMA_BUFFER_VERSION_1; @@ -1485,8 +1493,8 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t #endif #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE - std::unique_ptr backup_buffer; - if (file_mapped_weights_enabled_ && file_mapper_) { + std::vector backup_buffer; + if (FileMappingIsEnabled() && file_mapper_) { rt = qnn_interface_.contextCreateFromBinaryWithCallback(backend_handle_, device_handle_, context_configs, @@ -1498,23 +1506,22 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t NULL); if (rt != QNN_SUCCESS) { + DisableFileMapping(); LOGS(*logger_, WARNING) << "Failed to create context with file mapping enabled. Error: " - << QnnErrorHandleToString(result) << ", Code : " << result + << QnnErrorHandleToString(rt) << ", Code : " << rt << ". Retrying with feature disabled."; - file_mapped_weights_enabled_ = false; - // Read context bin from file since file mapping has failed BufferInfo_t buffer_info; ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info)); backup_buffer = std::move(buffer_info.data); - bin_buffer = static_cast(backup_buffer.get()); + bin_buffer = static_cast(backup_buffer.data()); } } #endif // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE - if (!file_mapped_weights_enabled_ || rt != QNN_SUCCESS) { + if (!FileMappingIsEnabled() || rt != QNN_SUCCESS) { rt = qnn_interface_.contextCreateFromBinary(backend_handle_, device_handle_, context_configs, diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index 3b10ad6b36916..10aee6f78bab1 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -256,11 +256,16 @@ class QnnBackendManager : public std::enable_shared_from_this bool ProfilingEnabled() { return profiling_enabled_; } #endif - bool FileMappingIsEnabled() { return file_mapped_weights_enabled_; } + bool FileMappingIsEnabled() { + std::lock_guard lock(file_mapping_mutex_); + return file_mapped_weights_enabled_; + } #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request, - Qnn_ContextBinaryDmaDataResponse_t* response, void* mapped_base_ptr); + Qnn_ContextBinaryDmaDataResponse_t* response, + void* const mapped_base_ptr, + const size_t file_size); Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, void* mapped_base_ptr); #endif @@ -268,9 +273,21 @@ class QnnBackendManager : public std::enable_shared_from_this QnnLog_Level_t MapOrtSeverityToQNNLogLevel(logging::Severity ort_log_level); static logging::Severity MapQNNLogLevelToOrtSeverity(QnnLog_Level_t qnn_log_level); +#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE + typedef struct FileMappingCallbackInfo { + void* const mapped_file_ptr; + const size_t file_size; + QnnBackendManager* const backend_manager; + + FileMappingCallbackInfo(void* ptr, size_t size, QnnBackendManager* manager) + : mapped_file_ptr(ptr), file_size(size), backend_manager(manager) {} + + } FileMappingCallbackInfo_t; +#endif + private: typedef struct BufferInfo { - std::unique_ptr data; + std::vector data; size_t size; } BufferInfo_t; @@ -450,6 +467,11 @@ class QnnBackendManager : public std::enable_shared_from_this return Status::OK(); } + void DisableFileMapping() { + std::lock_guard lock(file_mapping_mutex_); + file_mapped_weights_enabled_ = false; + } + private: const std::string backend_path_; std::recursive_mutex logger_recursive_mutex_; @@ -492,13 +514,15 @@ class QnnBackendManager : public std::enable_shared_from_this bool context_created_ = false; bool backend_setup_completed_ = false; bool vtcm_backup_buffer_sharing_enabled_ = false; + + std::mutex file_mapping_mutex_; bool file_mapped_weights_enabled_ = false; #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE std::unique_ptr file_mapper_ = nullptr; // Notify params for file mapping must persist throughout lifetime of // QnnBackendManager for release of DMA data callback on destruction - std::vector>> file_mapping_notify_params_; + std::vector> file_mapping_notify_params_; #endif // NPU backend requires quantized model diff --git a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h index 28ddb70215295..f99cc7b1ee5dd 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h @@ -15,20 +15,10 @@ namespace qnn { class FileMappingInterface { public: - typedef struct MappedWeightInfo { - size_t aligned_offset = 0; - size_t delta = 0; - void* aligned_data_ptr = nullptr; - void* unaligned_data_ptr = nullptr; - } MappedWeightInfo_t; - virtual ~FileMappingInterface() = default; virtual Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath, void** mapped_data_ptr) = 0; - - virtual MappedWeightInfo_t GetMappedWeightMemoryPtr(void* mapped_base_ptr, - const size_t offset) = 0; }; } // namespace qnn diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc index 6800ef27fe349..71f562d59d847 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc @@ -107,28 +107,6 @@ Status WindowsFileMapper::GetContextBinMappedMemoryPtr(const std::string& bin_fi return Status::OK(); } - -FileMappingInterface::MappedWeightInfo_t -WindowsFileMapper::GetMappedWeightMemoryPtr(void* mapped_base_ptr, - const size_t offset) { - MappedWeightInfo_t mapped_weight_info; - - // Align to nearest granularity boundary - SYSTEM_INFO sys_info; - GetSystemInfo(&sys_info); - Qnn_ContextBinarySize_t granularity = sys_info.dwAllocationGranularity; - SIZE_T aligned_offset = offset & ~(granularity - 1); - SIZE_T delta = offset - aligned_offset; - - mapped_weight_info.aligned_offset = aligned_offset; - mapped_weight_info.delta = delta; - - void* aligned_data_ptr = static_cast(mapped_base_ptr) + aligned_offset; - mapped_weight_info.aligned_data_ptr = aligned_data_ptr; - mapped_weight_info.unaligned_data_ptr = static_cast(aligned_data_ptr) + delta; - - return mapped_weight_info; -} } // namespace qnn } // namespace onnxruntime diff --git a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h index 90dc29036eb2a..742255b26f07d 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h @@ -28,12 +28,6 @@ class WindowsFileMapper : public FileMappingInterface { Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath, void** mapped_data_ptr) override; - // Given a base mapview pointer of a context binary and an offset, - // calcualte the aligned and unaligned memory pointers to a weight - // within the context binary - MappedWeightInfo_t GetMappedWeightMemoryPtr(void* mapped_base_ptr, - const size_t offset) override; - private: // A container of smart pointers of mapview memory pointers to mapped context bins // key: filepath to context bin, value: smart pointer of mapview memory pointers From ebf245c10ebdc400bde0b61b6cee3449265aec6e Mon Sep 17 00:00:00 2001 From: quic_calvnguy Date: Mon, 26 Jan 2026 12:52:48 -0800 Subject: [PATCH 17/17] Remove state changes to QnnBackendManager::file_mapped_weights_enabled_ and unnecessary functions relating to file_mapped_weights_enabled_ --- .../qnn/builder/qnn_backend_manager.cc | 35 +++++++------------ .../qnn/builder/qnn_backend_manager.h | 15 +------- 2 files changed, 13 insertions(+), 37 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index 1e0547124a675..9fc1cd7f42939 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -814,7 +814,6 @@ Qnn_ErrorHandle_t QnnBackendManager::MapDmaData(Qnn_ContextBinaryDataRequest_t r Qnn_ContextBinaryDmaDataResponse_t* response, void* const mapped_base_ptr, const size_t file_size) { - std::lock_guard lock(file_mapping_mutex_); if (!file_mapped_weights_enabled_) { LOGS(*logger_, WARNING) << "Attempting to map DMA data but file mapping has been disabled, " << "possibly due to an error in a previous request."; @@ -976,12 +975,11 @@ Status QnnBackendManager::GetFileSizeIfValid(const std::string& filepath, } Status QnnBackendManager::ReadContextBinIfValid(const std::string& context_bin_filepath, - BufferInfo_t& buffer_info) { + std::vector& buffer) { size_t buffer_size; ORT_RETURN_IF_ERROR(GetFileSizeIfValid(context_bin_filepath, buffer_size)); - std::vector buffer; - buffer.reserve(buffer_size); + buffer.resize(buffer_size); std::ifstream cache_file(context_bin_filepath.c_str(), std::ifstream::binary); ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to read context binary from: ", context_bin_filepath); @@ -989,9 +987,6 @@ Status QnnBackendManager::ReadContextBinIfValid(const std::string& context_bin_f const auto& read_result = cache_file.read(buffer.data(), buffer_size); ORT_RETURN_IF(!read_result, "Failed to read contents from cached context file."); - buffer_info.data = std::move(buffer); - buffer_info.size = buffer_size; - return Status::OK(); } @@ -1032,14 +1027,11 @@ Status QnnBackendManager::CreateContextVtcmBackupBufferSharingEnabled(std::unord nullptr}; #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE - if (FileMappingIsEnabled() && file_mapper_) { + if (file_mapped_weights_enabled_ && file_mapper_) { // Retry logic -- if context creation failed with file mapped weights, then retry with feature disabled auto res = CreateContextFromListAsyncWithCallback(configs, context_bin_map); if (!res.IsOK()) { - DisableFileMapping(); LOGS(*logger_, WARNING) << res.ErrorMessage() << ". Retrying with feature disabled."; - - file_mapped_weights_enabled_ = false; } else { return Status::OK(); } @@ -1062,11 +1054,11 @@ Status QnnBackendManager::CreateContextFromListAsync(const QnnContext_Config_t** for (auto& it : context_bin_map) { auto context_bin_filepath = it.first; - BufferInfo_t buffer_info; - ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info)); + std::vector buffer; + ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer)); - buffer_list.push_back(std::move(buffer_info.data)); - size_t buffer_size = buffer_info.size; + size_t buffer_size = buffer.size(); + buffer_list.push_back(std::move(buffer)); QnnContext_ParamsV1_t context_params_v1 = {nullptr, buffer_list.back().data(), @@ -1367,7 +1359,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t void* bin_buffer = nullptr; #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE - if (FileMappingIsEnabled()) { + if (file_mapped_weights_enabled_) { ORT_RETURN_IF(!file_mapper_, "Attemping to use File Mapping feature but file_mapper_ is uninitialized"); ORT_RETURN_IF_ERROR(GetFileSizeIfValid(context_bin_filepath, buffer_length)); @@ -1467,7 +1459,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE Qnn_ContextBinaryCallback_t callbacks; - if (FileMappingIsEnabled() && file_mapper_) { + if (file_mapped_weights_enabled_ && file_mapper_) { ORT_RETURN_IF(nullptr == qnn_interface_.contextCreateFromBinaryWithCallback, "Invalid function pointer for contextCreateFromBinaryWithCallback."); @@ -1494,7 +1486,7 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE std::vector backup_buffer; - if (FileMappingIsEnabled() && file_mapper_) { + if (file_mapped_weights_enabled_ && file_mapper_) { rt = qnn_interface_.contextCreateFromBinaryWithCallback(backend_handle_, device_handle_, context_configs, @@ -1506,22 +1498,19 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t NULL); if (rt != QNN_SUCCESS) { - DisableFileMapping(); LOGS(*logger_, WARNING) << "Failed to create context with file mapping enabled. Error: " << QnnErrorHandleToString(rt) << ", Code : " << rt << ". Retrying with feature disabled."; // Read context bin from file since file mapping has failed - BufferInfo_t buffer_info; - ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, buffer_info)); - backup_buffer = std::move(buffer_info.data); + ORT_RETURN_IF_ERROR(ReadContextBinIfValid(context_bin_filepath, backup_buffer)); bin_buffer = static_cast(backup_buffer.data()); } } #endif // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE - if (!FileMappingIsEnabled() || rt != QNN_SUCCESS) { + if (!file_mapped_weights_enabled_ || rt != QNN_SUCCESS) { rt = qnn_interface_.contextCreateFromBinary(backend_handle_, device_handle_, context_configs, diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index 10aee6f78bab1..9b573531f7c3d 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -257,7 +257,6 @@ class QnnBackendManager : public std::enable_shared_from_this #endif bool FileMappingIsEnabled() { - std::lock_guard lock(file_mapping_mutex_); return file_mapped_weights_enabled_; } @@ -286,11 +285,6 @@ class QnnBackendManager : public std::enable_shared_from_this #endif private: - typedef struct BufferInfo { - std::vector data; - size_t size; - } BufferInfo_t; - Status LoadBackend(); Status InitializeBackend(); @@ -310,7 +304,7 @@ class QnnBackendManager : public std::enable_shared_from_this Status GetFileSizeIfValid(const std::string& filepath, size_t& file_size); Status ReadContextBinIfValid(const std::string& context_bin_filepath, - BufferInfo_t& buffer_info); + std::vector& buffer); Status CreateContextVtcmBackupBufferSharingEnabled(std::unordered_map>>& context_bin_map); @@ -467,11 +461,6 @@ class QnnBackendManager : public std::enable_shared_from_this return Status::OK(); } - void DisableFileMapping() { - std::lock_guard lock(file_mapping_mutex_); - file_mapped_weights_enabled_ = false; - } - private: const std::string backend_path_; std::recursive_mutex logger_recursive_mutex_; @@ -514,8 +503,6 @@ class QnnBackendManager : public std::enable_shared_from_this bool context_created_ = false; bool backend_setup_completed_ = false; bool vtcm_backup_buffer_sharing_enabled_ = false; - - std::mutex file_mapping_mutex_; bool file_mapped_weights_enabled_ = false; #ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE