diff --git a/.gitmodules b/.gitmodules index a8e569ef763dc..ee87ea533ebac 100644 --- a/.gitmodules +++ b/.gitmodules @@ -25,15 +25,9 @@ [submodule "cmake/external/eigen"] path = cmake/external/eigen url = https://github.com/eigenteam/eigen-git-mirror.git -[submodule "cmake/external/grpc"] - path = cmake/external/grpc - url = https://github.com/grpc/grpc [submodule "cmake/external/DNNLibrary"] path = cmake/external/DNNLibrary url = https://github.com/JDAI-CV/DNNLibrary -[submodule "cmake/external/spdlog"] - path = cmake/external/spdlog - url = https://github.com/gabime/spdlog.git [submodule "cmake/external/mimalloc"] path = cmake/external/mimalloc url = https://github.com/microsoft/mimalloc.git @@ -49,3 +43,9 @@ [submodule "cmake/external/json"] path = cmake/external/json url = https://github.com/nlohmann/json +[submodule "server/external/spdlog"] + path = server/external/spdlog + url = https://github.com/gabime/spdlog.git +[submodule "cmake/external/FeaturizersLibrary"] + path = cmake/external/FeaturizersLibrary + url = https://github.com/microsoft/FeaturizersLibrary.git diff --git a/cgmanifest.json b/cgmanifest.json index 4db0cd0b7ac04..b030f30d91d16 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -22,7 +22,7 @@ "component": { "type": "git", "git": { - "commitHash": "9bda90b7e5e08c4c37a832d0cea218aed6af6470", + "commitHash": "703bd9caab50b139428cea1aaff9974ebee5742e", "repositoryUrl": "https://github.com/google/googletest.git" } } @@ -247,7 +247,7 @@ "component": { "type": "git", "git": { - "commitHash": "48cb18e5c419ddd23d9badcfe4e9df7bde1979b2", + "commitHash": "fe1790ca0df67173702f70d5646b82f48f412b99", "repositoryUrl": "https://github.com/protocolbuffers/protobuf.git" } } @@ -450,7 +450,7 @@ { "component": { "git": { - "commitHash": "3f0f9802553944b75015aad098d856b2d17220df", + "commitHash": "ebec32ef06859b6399bf8854f18b91158c87760b", "repositoryUrl": "https://github.com/microsoft/FeaturizersLibrary.git" }, "type": "git" diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index e05ed0f43ad1d..d2db9644db0da 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -222,12 +222,7 @@ else() string(APPEND CMAKE_C_FLAGS_RELEASE " -march=native -mtune=native") string(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " -march=native -mtune=native") string(APPEND CMAKE_C_FLAGS_RELWITHDEBINFO " -march=native -mtune=native") - endif() - if(onnxruntime_BUILD_x86) - set (CMAKE_SYSTEM_PROCESSOR "x86") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse2 -mfpmath=sse -Wno-narrowing") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse2 -mfpmath=sse -Wno-narrowing") - endif() + endif() endif() if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") @@ -279,10 +274,14 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external) # use protobuf as a submodule +if (CMAKE_SYSTEM_NAME STREQUAL "Android") + set(protobuf_BUILD_PROTOC_BINARIES OFF CACHE BOOL "Build protobuf tests" FORCE) +endif() + add_subdirectory(${PROJECT_SOURCE_DIR}/external/protobuf/cmake EXCLUDE_FROM_ALL) set_target_properties(libprotobuf PROPERTIES FOLDER "External/Protobuf") set_target_properties(libprotobuf-lite PROPERTIES FOLDER "External/Protobuf") -set_target_properties(libprotoc PROPERTIES FOLDER "External/Protobuf") +set_target_properties(libprotoc PROPERTIES FOLDER "External/Protobuf") set_target_properties(protoc PROPERTIES FOLDER "External/Protobuf") if (onnxruntime_USE_FULL_PROTOBUF) add_library(protobuf::libprotobuf ALIAS libprotobuf) @@ -295,7 +294,6 @@ if(UNIX AND onnxruntime_ENABLE_LTO) #https://github.com/protocolbuffers/protobuf/issues/5923 target_link_options(protoc PRIVATE "-Wl,--no-as-needed") endif() - include(protobuf_function.cmake) if (onnxruntime_DISABLE_CONTRIB_OPS) @@ -422,6 +420,14 @@ if (onnxruntime_USE_TVM) list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES tvm nnvm_compiler) endif() +if (APPLE) + #onnx/onnx/proto_utils.h:34:16: error: 'SetTotalBytesLimit' is deprecated: Please use the single + #parameter version of SetTotalBytesLimit(). The second parameter is ignored. + # coded_stream.SetTotalBytesLimit((2048LL << 20) - 1, 512LL << 20); + #TODO: fix the warning in ONNX and re-enable this flag + string(APPEND CMAKE_CXX_FLAGS " -Wno-deprecated") + string(APPEND CMAKE_C_FLAGS " -Wno-deprecated") +endif() # ONNX add_subdirectory(onnx) @@ -498,6 +504,7 @@ else() endif() check_cxx_compiler_flag(-Wunused-but-set-variable HAS_UNUSED_BUT_SET_VARIABLE) check_cxx_compiler_flag(-Wunused-parameter HAS_UNUSED_PARAMETER) + check_cxx_compiler_flag(-Wunused-variable HAS_UNUSED_VARIABLE) check_cxx_compiler_flag(-Wcast-function-type HAS_CAST_FUNCTION_TYPE) check_cxx_compiler_flag(-Wparentheses HAS_PARENTHESES) check_cxx_compiler_flag(-Wuseless-cast HAS_USELESS_CAST) diff --git a/cmake/external/FeaturizersLibrary b/cmake/external/FeaturizersLibrary new file mode 160000 index 0000000000000..ebec32ef06859 --- /dev/null +++ b/cmake/external/FeaturizersLibrary @@ -0,0 +1 @@ +Subproject commit ebec32ef06859b6399bf8854f18b91158c87760b diff --git a/cmake/external/dml.cmake b/cmake/external/dml.cmake index 41de28fdd2e0a..35c606f6412e8 100644 --- a/cmake/external/dml.cmake +++ b/cmake/external/dml.cmake @@ -20,19 +20,17 @@ if (NOT onnxruntime_USE_CUSTOM_DIRECTML) set(NUGET_CONFIG ${PROJECT_SOURCE_DIR}/../NuGet.config) set(PACKAGES_CONFIG ${PROJECT_SOURCE_DIR}/../packages.config) set(PACKAGES_DIR ${CMAKE_CURRENT_BINARY_DIR}/packages) + set(DML_PACKAGE_DIR ${PACKAGES_DIR}/DirectML.0.0.1) # Restore nuget packages, which will pull down the DirectML redist package add_custom_command( - OUTPUT restore_packages.stamp + OUTPUT ${DML_PACKAGE_DIR}/bin/x64/DirectML.lib ${DML_PACKAGE_DIR}/bin/x86/DirectML.lib DEPENDS ${PACKAGES_CONFIG} ${NUGET_CONFIG} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/nuget/src/nuget restore ${PACKAGES_CONFIG} -PackagesDirectory ${PACKAGES_DIR} -ConfigFile ${NUGET_CONFIG} - COMMAND ${CMAKE_COMMAND} -E touch restore_packages.stamp VERBATIM) - add_custom_target(RESTORE_PACKAGES ALL DEPENDS restore_packages.stamp) + add_custom_target(RESTORE_PACKAGES ALL DEPENDS ${DML_PACKAGE_DIR}/bin/x64/DirectML.lib ${DML_PACKAGE_DIR}/bin/x86/DirectML.lib) add_dependencies(RESTORE_PACKAGES nuget) - - list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES RESTORE_PACKAGES) else() include_directories(${dml_INCLUDE_DIR}) endif() diff --git a/cmake/external/featurizers.cmake b/cmake/external/featurizers.cmake index ac7f2432db921..5c10ee15a54c0 100644 --- a/cmake/external/featurizers.cmake +++ b/cmake/external/featurizers.cmake @@ -2,50 +2,17 @@ # Licensed under the MIT License. # This source code should not depend on the onnxruntime and may be built independently -set(featurizers_URL "https://github.com/microsoft/FeaturizersLibrary.git") -set(featurizers_TAG "3f0f9802553944b75015aad098d856b2d17220df") - set(featurizers_pref FeaturizersLibrary) set(featurizers_ROOT ${PROJECT_SOURCE_DIR}/external/${featurizers_pref}) set(featurizers_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/${featurizers_pref}) -# Only due to GIT_CONFIG -# Uncoment UPDATE_COMMAND if you work locally -# on the featurizers so cmake does not undo your changes. -if (WIN32) - ExternalProject_Add(featurizers_lib - PREFIX ${featurizers_pref} - GIT_REPOSITORY ${featurizers_URL} - GIT_TAG ${featurizers_TAG} - # Need this to properly checkout crlf - GIT_CONFIG core.autocrlf=input - SOURCE_DIR ${featurizers_ROOT} - # Location of CMakeLists.txt - SOURCE_SUBDIR src/Featurizers - BINARY_DIR ${featurizers_BINARY_DIR} - CMAKE_ARGS -Dfeaturizers_MSVC_STATIC_RUNTIME=${onnxruntime_MSVC_STATIC_RUNTIME} -# UPDATE_COMMAND "" - INSTALL_COMMAND "" - ) -else() - ExternalProject_Add(featurizers_lib - PREFIX ${featurizers_pref} - GIT_REPOSITORY ${featurizers_URL} - GIT_TAG ${featurizers_TAG} - SOURCE_DIR ${featurizers_ROOT} - # Location of CMakeLists.txt - SOURCE_SUBDIR src/Featurizers - BINARY_DIR ${featurizers_BINARY_DIR} - CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON -# UPDATE_COMMAND "" - INSTALL_COMMAND "" - ) -endif() +add_subdirectory(external/FeaturizersLibrary/src/Featurizers ${featurizers_BINARY_DIR} EXCLUDE_FROM_ALL) +set_target_properties(FeaturizersCode PROPERTIES FOLDER "External/FeaturizersLibrary") add_library(onnxruntime_featurizers STATIC IMPORTED) -add_dependencies(onnxruntime_featurizers featurizers_lib) -target_include_directories(onnxruntime_featurizers INTERFACE ${featurizers_ROOT}/src) +add_dependencies(onnxruntime_featurizers FeaturizersCode) +target_include_directories(onnxruntime_featurizers INTERFACE ${featurizers_ROOT}/src) if(MSVC) set_property(TARGET onnxruntime_featurizers PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/${featurizers_pref}/${CMAKE_BUILD_TYPE}/FeaturizersCode.lib) @@ -54,6 +21,7 @@ else() ${CMAKE_CURRENT_BINARY_DIR}/external/${featurizers_pref}/libFeaturizersCode.a) endif() + if (WIN32) # Add Code Analysis properties to enable C++ Core checks. Have to do it via a props file include. set_target_properties(onnxruntime_featurizers PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props) diff --git a/cmake/external/grpc b/cmake/external/grpc deleted file mode 160000 index 02a2a458ac159..0000000000000 --- a/cmake/external/grpc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 02a2a458ac15912d7d87cc1171e811b0c5219ece diff --git a/cmake/onnx/CMakeLists.txt b/cmake/onnx/CMakeLists.txt index 3c7e6b3d7064e..1048c90bd2c5c 100644 --- a/cmake/onnx/CMakeLists.txt +++ b/cmake/onnx/CMakeLists.txt @@ -6,7 +6,11 @@ target_include_directories(onnx_proto PUBLIC $) onnxruntime_protobuf_generate(APPEND_PATH IMPORT_DIRS ${ONNXRUNTIME_ROOT}/core/protobuf TARGET onnx_proto) if (WIN32) - target_compile_options(onnx_proto PRIVATE /wd4146) # unary minus operator applied to unsigned type + target_compile_options(onnx_proto PRIVATE "/wd4146" "/wd4125" "/wd4456" "/wd4267") +else() + if(HAS_UNUSED_VARIABLE) + target_compile_options(onnx_proto PRIVATE "-Wno-unused-variable") + endif() endif() # Cpp Tests were added and they require googletest # since we have our own copy, try using that diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index 63a8283429dce..52aa2f3989d9d 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -44,6 +44,22 @@ else() endif() endif() +if(CMAKE_GENERATOR_PLATFORM) + # Multi-platform generator + set(onnxruntime_target_platform ${CMAKE_GENERATOR_PLATFORM}) +else() + set(onnxruntime_target_platform ${CMAKE_SYSTEM_PROCESSOR}) +endif() +if(onnxruntime_target_platform STREQUAL "ARM64") + set(onnxruntime_target_platform "ARM64") +elseif(onnxruntime_target_platform STREQUAL "ARM" OR CMAKE_GENERATOR MATCHES "ARM") + set(onnxruntime_target_platform "ARM") +elseif(onnxruntime_target_platform STREQUAL "x64" OR onnxruntime_target_platform STREQUAL "x86_64" OR onnxruntime_target_platform STREQUAL "AMD64" OR CMAKE_GENERATOR MATCHES "Win64") + set(onnxruntime_target_platform "x64") +elseif(onnxruntime_target_platform STREQUAL "x86" OR onnxruntime_target_platform STREQUAL "i386" OR onnxruntime_target_platform STREQUAL "i686") + set(onnxruntime_target_platform "x86") +endif() + file(GLOB onnxruntime_common_src CONFIGURE_DEPENDS ${onnxruntime_common_src_patterns} ) diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index e7b4213bc6cbb..38ec173dfc0b1 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -19,7 +19,7 @@ set(mlas_common_srcs ) if(MSVC) - if(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64") + if(onnxruntime_target_platform STREQUAL "ARM64") set(asm_filename ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm64/SgemmKernelNeon.asm) set(pre_filename ${CMAKE_CURRENT_BINARY_DIR}/SgemmKernelNeon.i) set(obj_filename ${CMAKE_CURRENT_BINARY_DIR}/SgemmKernelNeon.obj) @@ -38,11 +38,11 @@ if(MSVC) armasm64.exe ${ARMASM_FLAGS} ${pre_filename} ${obj_filename} ) set(mlas_platform_srcs ${obj_filename}) - elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM" OR CMAKE_GENERATOR MATCHES "ARM") + elseif(onnxruntime_target_platform STREQUAL "ARM") set(mlas_platform_srcs ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm/sgemmc.cpp ) - elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "x64" OR CMAKE_GENERATOR MATCHES "Win64") + elseif(onnxruntime_target_platform STREQUAL "x64") enable_language(ASM_MASM) set(mlas_platform_srcs diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 9875955e3cd89..3bf0b6859f74e 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -217,7 +217,7 @@ if (onnxruntime_USE_TENSORRT) if ( CMAKE_COMPILER_IS_GNUCC ) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-missing-field-initializers") endif() - set(CXX_VERSION_DEFINED TRUE) + set(CXX_VERSION_DEFINED TRUE) add_subdirectory(${ONNXRUNTIME_ROOT}/../cmake/external/onnx-tensorrt) set(CMAKE_CXX_FLAGS ${OLD_CMAKE_CXX_FLAGS}) if (WIN32) @@ -303,7 +303,7 @@ if (onnxruntime_USE_OPENVINO) if(WIN32) set(OPENVINO_LIB_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/lib/intel64/Release) set(OPENVINO_TBB_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/lib/intel64/Release) - set(OPENVINO_MKL_TINY_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/bin/intel64/Release) + set(OPENVINO_MKL_TINY_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/bin/intel64/Release) else() set(OPENVINO_LIB_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/lib/intel64/) set(OPENVINO_TBB_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/tbb/lib) @@ -327,9 +327,9 @@ if (onnxruntime_USE_OPENVINO) else() target_include_directories(onnxruntime_providers_openvino SYSTEM PUBLIC ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${OPENVINO_INCLUDE_DIR} ${OPENVINO_EXTENSIONS_DIR} ${OPENVINO_LIB_DIR} ${OPENVINO_TBB_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS}) endif() - - if (WIN32) - string(REPLACE "include" "libs" PYTHON_LIB ${PYTHON_INCLUDE_DIRS}) + + if (WIN32) + string(REPLACE "include" "libs" PYTHON_LIB ${PYTHON_INCLUDE_DIRS}) find_package(InferenceEngine 2.1 REQUIRED) set(PYTHON_LIBRARIES ${PYTHON_LIB}) set(OPENVINO_CPU_EXTENSION_DIR ${onnxruntime_BINARY_DIR}/ie_cpu_extension/${CMAKE_BUILD_TYPE}) @@ -430,22 +430,41 @@ if (onnxruntime_USE_DML) onnxruntime_add_include_to_target(onnxruntime_providers_dml onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf) add_dependencies(onnxruntime_providers_dml ${onnxruntime_EXTERNAL_DEPENDENCIES}) target_include_directories(onnxruntime_providers_dml PRIVATE ${ONNXRUNTIME_ROOT} ${ONNXRUNTIME_ROOT}/../cmake/external/wil/include) - - target_link_libraries(onnxruntime_providers_dml ${CMAKE_CURRENT_BINARY_DIR}/packages/DirectML.0.0.1/build/DirectML.targets) - target_link_libraries(onnxruntime_providers_dml d3d12.lib dxgi.lib) + + if (NOT onnxruntime_USE_CUSTOM_DIRECTML) + if(NOT onnxruntime_target_platform STREQUAL "x86" AND NOT onnxruntime_target_platform STREQUAL "x64") + message(FATAL_ERROR "Target platform ${onnxruntime_target_platform} is not supported by DML") + endif() + foreach(file "DirectML.dll" "DirectML.pdb" "DirectML.Debug.dll" "DirectML.Debug.pdb") + add_custom_command(TARGET onnxruntime_providers_dml + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${DML_PACKAGE_DIR}/bin/${onnxruntime_target_platform}/${file}" $) + endforeach() + endif() + + function(target_add_dml target) + if (NOT onnxruntime_USE_CUSTOM_DIRECTML) + target_link_libraries(${target} PRIVATE "${DML_PACKAGE_DIR}/bin/${onnxruntime_target_platform}/DirectML.lib") + target_include_directories(${target} PRIVATE "${DML_PACKAGE_DIR}/include") + endif() + endfunction() + + target_add_dml(onnxruntime_providers_dml) + target_link_libraries(onnxruntime_providers_dml PRIVATE d3d12.lib dxgi.lib delayimp.lib) list(APPEND ONNXRUNTIME_LINKER_FLAGS "/DELAYLOAD:DirectML.dll /DELAYLOAD:d3d12.dll /DELAYLOAD:dxgi.dll") # The DML EP requires C++17 set_target_properties(onnxruntime_providers_dml PROPERTIES CXX_STANDARD 17) set_target_properties(onnxruntime_providers_dml PROPERTIES CXX_STANDARD_REQUIRED ON) - + target_compile_definitions(onnxruntime_providers_dml PRIVATE ONNX_NAMESPACE=onnx ONNX_ML LOTUS_LOG_THRESHOLD=2 LOTUS_ENABLE_STDERR_LOGGING PLATFORM_WINDOWS) target_compile_definitions(onnxruntime_providers_dml PRIVATE UNICODE _UNICODE NOMINMAX) if (MSVC) target_compile_definitions(onnxruntime_providers_dml PRIVATE _SILENCE_CXX17_ITERATOR_BASE_CLASS_DEPRECATION_WARNING) target_compile_options(onnxruntime_providers_dml PRIVATE "/W3") endif() - + install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/dml DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_dml PROPERTIES LINKER_LANGUAGE CXX) diff --git a/cmake/onnxruntime_server.cmake b/cmake/onnxruntime_server.cmake deleted file mode 100644 index 86296105bef4e..0000000000000 --- a/cmake/onnxruntime_server.cmake +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -set(SERVER_APP_NAME "onnxruntime_server") - -set(gRPC_BUILD_TESTS OFF CACHE INTERNAL "Don't build tests") -set(gRPC_GFLAGS_PROVIDER "" CACHE INTERNAL "Don't use gflags") -set(gRPC_BENCHMARK_PROVIDER "" CACHE INTERNAL "Don't use benchmark") -set(gRPC_ZLIB_PROVIDER "package" CACHE INTERNAL "Use preinstalled zlib library") -set(gRPC_PROTOBUF_PROVIDER "" CACHE INTERNAL "Don't use grpc protobuf, set it manually.") - - -# protobuf targets have already been included as submodules - adapted from https://github.com/grpc/grpc/blob/master/cmake/protobuf.cmake -set(_gRPC_PROTOBUF_LIBRARY_NAME "libprotobuf") -set(_gRPC_PROTOBUF_LIBRARIES protobuf::${_gRPC_PROTOBUF_LIBRARY_NAME}) - -set(_gRPC_PROTOBUF_PROTOC_LIBRARIES protobuf::libprotoc) -# extract the include dir from target's properties - -set(_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR ${REPO_ROOT}/cmake/external/protobuf/src) -set(_gRPC_PROTOBUF_PROTOC protobuf::protoc) -set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $) - -set(_gRPC_PROTOBUF_INCLUDE_DIR ${PROTOBUF_INCLUDE_DIRS}) - -if(NOT WIN32) - string(REPLACE "-Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") # Disable werror for included subdirectories - c-ares<1.15 breaks with -Wall - string(REPLACE "-Werror" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") - if(HAS_UNUSED_PARAMETER) # disable warning for unused parameters because (BoringSSL specifically) have unused parameters. - string(APPEND CMAKE_CXX_FLAGS " -Wno-unused-parameter") - string(APPEND CMAKE_C_FLAGS " -Wno-unused-parameter") - endif() -endif() - -add_subdirectory(${PROJECT_SOURCE_DIR}/external/grpc EXCLUDE_FROM_ALL) -if(NOT WIN32) - if(onnxruntime_DEV_MODE) # Reenable Werror for our code subdirectories. - if(NOT onnxruntime_USE_TVM) - string(APPEND CMAKE_CXX_FLAGS " -Werror") - string(APPEND CMAKE_C_FLAGS " -Werror") - endif() - endif() - if(HAS_UNUSED_PARAMETER) # reenable warning for unused parameters for our code. - string(APPEND CMAKE_CXX_FLAGS " -Wunused-parameter") - string(APPEND CMAKE_C_FLAGS " -Wunused-parameter") - endif() -endif() - -set(_GRPC_CPP_PLUGIN_EXECUTABLE $) -set(_GRPC_PY_PLUGIN_EXECUTABLE $) - - -# Generate .h and .cc files from protobuf file -add_library(server_proto ${ONNXRUNTIME_ROOT}/server/protobuf/predict.proto ${ONNXRUNTIME_ROOT}/server/protobuf/onnx-ml.proto) -if(WIN32) - target_compile_options(server_proto PRIVATE "/wd4125" "/wd4456") -endif() -target_include_directories(server_proto PUBLIC $ "${CMAKE_CURRENT_BINARY_DIR}/.." ${CMAKE_CURRENT_BINARY_DIR}/onnx) -target_compile_definitions(server_proto PUBLIC $) -onnxruntime_protobuf_generate(APPEND_PATH IMPORT_DIRS ${REPO_ROOT}/cmake/external/protobuf/src ${ONNXRUNTIME_ROOT}/server/protobuf ${ONNXRUNTIME_ROOT}/core/protobuf TARGET server_proto) -add_dependencies(server_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) -if(NOT WIN32) - if(HAS_UNUSED_PARAMETER) - set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/predict.pb.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) - set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/onnx-ml.pb.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) - endif() -endif() - -# Setup dependencies -include(get_boost.cmake) -set(re2_src ${REPO_ROOT}/cmake/external/re2) -set(SPDLOG_BUILD_EXAMPLES OFF) -add_subdirectory(${REPO_ROOT}/cmake/external/spdlog) - -# Generate GRPC service source and headers. -get_filename_component(grpc_proto "${ONNXRUNTIME_ROOT}/server/protobuf/prediction_service.proto" ABSOLUTE) -get_filename_component(grpc_proto_path "${grpc_proto}" PATH) - -set(grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/prediction_service.grpc.pb.cc") -set(grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/prediction_service.grpc.pb.h") -add_custom_command( - OUTPUT "${grpc_srcs}" "${grpc_hdrs}" - COMMAND $ - ARGS - --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" - --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" - --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" - -I ${grpc_proto_path} - "${grpc_proto}" - DEPENDS "${grpc_proto}" ${_GRPC_CPP_PLUGIN_EXECUTABLE} - COMMENT "Running ${_GRPC_CPP_PLUGIN_EXECUTABLE} on ${grpc_proto}" - ) - -add_library(server_grpc_proto ${grpc_srcs}) -target_include_directories(server_grpc_proto PUBLIC $ "${CMAKE_CURRENT_BINARY_DIR}" ${CMAKE_CURRENT_BINARY_DIR}/onnx PRIVATE) -set(grpc_reflection -Wl,--whole-archive grpc++_reflection -Wl,--no-whole-archive) -set(grpc_static_libs grpc++ grpcpp_channelz) -target_link_libraries(server_grpc_proto ${grpc_static_libs}) -add_dependencies(server_grpc_proto server_proto) -# Include generated *.pb.h files -include_directories("${CMAKE_CURRENT_BINARY_DIR}") - -if(NOT WIN32) - if(HAS_UNUSED_PARAMETER) - set_source_files_properties(${grpc_srcs} PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) - set_source_files_properties(${onnxruntime_server_grpc_srcs} PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) - endif() -endif() - - -# Setup source code -set(onnxruntime_server_lib_srcs - "${ONNXRUNTIME_ROOT}/server/http/json_handling.cc" - "${ONNXRUNTIME_ROOT}/server/http/predict_request_handler.cc" - "${ONNXRUNTIME_ROOT}/server/http/util.cc" - "${ONNXRUNTIME_ROOT}/server/environment.cc" - "${ONNXRUNTIME_ROOT}/server/executor.cc" - "${ONNXRUNTIME_ROOT}/server/converter.cc" - "${ONNXRUNTIME_ROOT}/server/util.cc" - "${ONNXRUNTIME_ROOT}/server/core/request_id.cc" - "${ONNXRUNTIME_ROOT}/server/grpc/prediction_service_impl.cc" - "${ONNXRUNTIME_ROOT}/server/grpc/grpc_app.cc" - "${ONNXRUNTIME_ROOT}/server/serializing/tensorprotoutils.cc" - ) -if(NOT WIN32) - if(HAS_UNUSED_PARAMETER) - set_source_files_properties(${onnxruntime_server_lib_srcs} PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) - endif() -endif() - -file(GLOB_RECURSE onnxruntime_server_http_core_lib_srcs - "${ONNXRUNTIME_ROOT}/server/http/core/*.cc" - ) - -file(GLOB_RECURSE onnxruntime_server_srcs - "${ONNXRUNTIME_ROOT}/server/main.cc" -) - -# HTTP core library -add_library(onnxruntime_server_http_core_lib STATIC - ${onnxruntime_server_http_core_lib_srcs}) -target_include_directories(onnxruntime_server_http_core_lib - PUBLIC - ${ONNXRUNTIME_ROOT} - ${ONNXRUNTIME_ROOT}/server/http/core - ${ONNXRUNTIME_ROOT}/server/core - ${Boost_INCLUDE_DIR} - ${re2_src} -) -add_dependencies(onnxruntime_server_http_core_lib Boost) -target_link_libraries(onnxruntime_server_http_core_lib PRIVATE - ${Boost_LIBRARIES} -) - -# Server library -add_library(onnxruntime_server_lib ${onnxruntime_server_lib_srcs}) -onnxruntime_add_include_to_target(onnxruntime_server_lib onnx_proto server_proto) -target_include_directories(onnxruntime_server_lib PRIVATE - ${ONNXRUNTIME_INCLUDE_DIR} - ${ONNXRUNTIME_ROOT} - ${ONNXRUNTIME_ROOT}/server - ${ONNXRUNTIME_ROOT}/server/http - ${ONNXRUNTIME_ROOT}/server/logging - ${ONNXRUNTIME_ROOT}/server/core - PUBLIC - ${ONNXRUNTIME_ROOT}/server - ${Boost_INCLUDE_DIR} - ${re2_src} -) - - -target_link_libraries(onnxruntime_server_lib PRIVATE - server_proto - server_grpc_proto - ${Boost_LIBRARIES} - onnxruntime_server_http_core_lib - PUBLIC - protobuf::libprotobuf - ${onnxruntime_EXTERNAL_LIBRARIES} - spdlog::spdlog - onnxruntime -) - -if (onnxruntime_USE_SYSLOG) - target_compile_definitions(onnxruntime_server_lib PUBLIC USE_SYSLOG="1") -endif() - -# For IDE only -source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_server_srcs} ${onnxruntime_server_lib_srcs} ${onnxruntime_server_lib}) - -# Server Application -add_executable(${SERVER_APP_NAME} ${onnxruntime_server_srcs}) -add_dependencies(${SERVER_APP_NAME} onnx server_proto onnx_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) - -if(NOT WIN32) - if(HAS_UNUSED_PARAMETER) - set_source_files_properties("${ONNXRUNTIME_ROOT}/server/main.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) - endif() -endif() - -set(onnxruntime_SERVER_VERSION "local-build" CACHE STRING "Sever version") -target_compile_definitions(${SERVER_APP_NAME} PUBLIC SRV_VERSION="${onnxruntime_SERVER_VERSION}") -message(STATUS "ONNX Runtime Server version set to: ${onnxruntime_SERVER_VERSION}") - -set(onnxruntime_LATEST_COMMIT_ID "default" CACHE STRING "The latest commit id") -target_compile_definitions(${SERVER_APP_NAME} PUBLIC LATEST_COMMIT_ID="${onnxruntime_LATEST_COMMIT_ID}") -message(STATUS "ONNX Runtime Server latest commit id is: ${onnxruntime_LATEST_COMMIT_ID}") - -onnxruntime_add_include_to_target(${SERVER_APP_NAME} onnxruntime_session onnxruntime_server_lib onnx onnx_proto server_proto) - -target_include_directories(${SERVER_APP_NAME} PRIVATE - ${ONNXRUNTIME_INCLUDE_DIR} - ${ONNXRUNTIME_ROOT}/server/http -) - - -target_link_libraries(${SERVER_APP_NAME} PRIVATE - onnxruntime_server_http_core_lib - onnxruntime_server_lib - ${grpc_reflection} #Note that this will break the tests if we try to link it to the lib so just link to the executable. -) - diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 09aedbe005ae8..2bd8929c34410 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -44,7 +44,7 @@ function(AddTest) endif() if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS AND onnxruntime_ENABLE_PYTHON) target_compile_definitions(${_UT_TARGET} PRIVATE ENABLE_LANGUAGE_INTEROP_OPS) - endif() + endif() if (WIN32) if (onnxruntime_USE_CUDA) # disable a warning from the CUDA headers about unreferenced local functions @@ -162,6 +162,7 @@ endif() set(onnxruntime_test_common_libs onnxruntime_test_utils onnxruntime_common + winml_adapter ) set(onnxruntime_test_ir_libs @@ -318,7 +319,7 @@ if (onnxruntime_USE_DNNL) target_compile_definitions(onnxruntime_test_utils_for_framework PUBLIC USE_DNNL=1) endif() if (onnxruntime_USE_DML) - target_link_libraries(onnxruntime_test_utils_for_framework PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/packages/DirectML.0.0.1/build/DirectML.targets) + target_add_dml(onnxruntime_test_utils_for_framework) endif() add_dependencies(onnxruntime_test_utils_for_framework ${onnxruntime_EXTERNAL_DEPENDENCIES}) target_include_directories(onnxruntime_test_utils_for_framework PUBLIC "${TEST_SRC_DIR}/util/include" PRIVATE ${eigen_INCLUDE_DIRS} ${ONNXRUNTIME_ROOT}) @@ -336,7 +337,7 @@ if (onnxruntime_USE_DNNL) target_compile_definitions(onnxruntime_test_utils PUBLIC USE_DNNL=1) endif() if (onnxruntime_USE_DML) - target_link_libraries(onnxruntime_test_utils PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/packages/DirectML.0.0.1/build/DirectML.targets) + target_add_dml(onnxruntime_test_utils) endif() add_dependencies(onnxruntime_test_utils ${onnxruntime_EXTERNAL_DEPENDENCIES}) target_include_directories(onnxruntime_test_utils PUBLIC "${TEST_SRC_DIR}/util/include" PRIVATE ${eigen_INCLUDE_DIRS} ${ONNXRUNTIME_ROOT}) @@ -428,7 +429,7 @@ endif() # SingleUnitTestProject AddTest( TARGET onnxruntime_test_framework_session_without_environment_standalone SOURCES "${TEST_SRC_DIR}/framework/inference_session_without_environment/inference_session_without_environment_standalone_test.cc" "${TEST_SRC_DIR}/framework/test_main.cc" - LIBS onnxruntime_test_utils ${ONNXRUNTIME_TEST_LIBS} + LIBS onnxruntime_test_utils ${ONNXRUNTIME_TEST_LIBS} winml_adapter DEPENDS ${onnxruntime_EXTERNAL_DEPENDENCIES} ) @@ -486,16 +487,23 @@ if(WIN32) endif() add_library(onnx_test_data_proto ${TEST_SRC_DIR}/proto/tml.proto) -if(WIN32) - target_compile_options(onnx_test_data_proto PRIVATE "/wd4125" "/wd4456" "/wd4100") -endif() add_dependencies(onnx_test_data_proto onnx_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) -if(NOT WIN32) +if(WIN32) + target_compile_options(onnx_test_data_proto PRIVATE "/wd4125" "/wd4456" "/wd4100" "/wd4267") +else() if(HAS_UNUSED_PARAMETER) - set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/tml.pb.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-parameter") + endif() + if(HAS_UNUSED_VARIABLE) + target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-variable") + endif() + if(HAS_UNUSED_BUT_SET_VARIABLE) + target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-but-set-variable") endif() endif() +add_dependencies(onnx_test_data_proto onnx_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) + onnxruntime_add_include_to_target(onnx_test_data_proto onnx_proto) target_include_directories(onnx_test_data_proto PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx) set_target_properties(onnx_test_data_proto PROPERTIES FOLDER "ONNXRuntimeTest") @@ -554,7 +562,7 @@ if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) endif() add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc) -target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs}) +target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs} winml_adapter) target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT}) set_target_properties(onnx_test_runner PROPERTIES FOLDER "ONNXRuntimeTest") @@ -704,87 +712,6 @@ if (onnxruntime_BUILD_SHARED_LIB) ) endif() -if (onnxruntime_BUILD_SERVER) - file(GLOB onnxruntime_test_server_src - "${TEST_SRC_DIR}/server/unit_tests/*.cc" - "${TEST_SRC_DIR}/server/unit_tests/*.h" - ) - - file(GLOB onnxruntime_integration_test_server_src - "${TEST_SRC_DIR}/server/integration_tests/*.py" - ) - if(NOT WIN32) - if(HAS_UNUSED_PARAMETER) - set_source_files_properties("${TEST_SRC_DIR}/server/unit_tests/json_handling_tests.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) - set_source_files_properties("${TEST_SRC_DIR}/server/unit_tests/converter_tests.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) - set_source_files_properties("${TEST_SRC_DIR}/server/unit_tests/util_tests.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) - set_source_files_properties("${TEST_SRC_DIR}/server/unit_tests/prediction_service_impl_test.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) - set_source_files_properties("${TEST_SRC_DIR}/server/unit_tests/executor_test.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) - endif() - endif() - - add_library(onnxruntime_test_utils_for_server ${onnxruntime_test_server_src}) - onnxruntime_add_include_to_target(onnxruntime_test_utils_for_server onnxruntime_test_utils_for_framework gtest gmock onnx onnx_proto server_proto server_grpc_proto) - add_dependencies(onnxruntime_test_utils_for_server onnxruntime_server_lib onnxruntime_server_http_core_lib Boost ${onnxruntime_EXTERNAL_DEPENDENCIES}) - target_include_directories(onnxruntime_test_utils_for_server PUBLIC ${Boost_INCLUDE_DIR} ${REPO_ROOT}/cmake/external/re2 ${CMAKE_CURRENT_BINARY_DIR}/onnx ${ONNXRUNTIME_ROOT}/server ${ONNXRUNTIME_ROOT}/server/http ${ONNXRUNTIME_ROOT}/server/http/core ${ONNXRUNTIME_ROOT}/server/grpc ${ONNXRUNTIME_ROOT}/server ${ONNXRUNTIME_ROOT}/server/core PRIVATE ${ONNXRUNTIME_ROOT}) - if (onnxruntime_USE_OPENVINO) - message(${OPENVINO_INCLUDE_DIR}) - target_include_directories(onnxruntime_test_utils_for_server PUBLIC ${OPENVINO_INCLUDE_DIR} ${OPENVINO_TBB_INCLUDE_DIR}) - endif() - if(UNIX) - target_compile_options(onnxruntime_test_utils_for_server PRIVATE "$<$:SHELL:-Xcompiler -Wno-error=sign-compare>" - "$<$>:-Wno-error=sign-compare>") - endif() - target_link_libraries(onnxruntime_test_utils_for_server ${Boost_LIBRARIES} spdlog::spdlog server_grpc_proto) - - - AddTest( - TARGET onnxruntime_server_tests - SOURCES ${onnxruntime_test_server_src} - LIBS ${onnxruntime_test_server_libs} server_proto server_grpc_proto onnxruntime_server_lib ${onnxruntime_test_providers_libs} - DEPENDS ${onnxruntime_EXTERNAL_DEPENDENCIES} - ) - - onnxruntime_protobuf_generate( - APPEND_PATH IMPORT_DIRS ${REPO_ROOT}/cmake/external/protobuf/src ${ONNXRUNTIME_ROOT}/server/protobuf ${ONNXRUNTIME_ROOT}/core/protobuf - PROTOS ${ONNXRUNTIME_ROOT}/server/protobuf/predict.proto ${ONNXRUNTIME_ROOT}/server/protobuf/onnx-ml.proto - LANGUAGE python - TARGET onnxruntime_server_tests - OUT_VAR server_test_py) - - set(grpc_py "${CMAKE_CURRENT_BINARY_DIR}/prediction_service_pb2_grpc.py") - - add_custom_command( - TARGET onnxruntime_server_tests - COMMAND $ - ARGS - --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" - --plugin=protoc-gen-grpc="${_GRPC_PY_PLUGIN_EXECUTABLE}" - -I ${grpc_proto_path} - "${grpc_proto}" - DEPENDS "${grpc_proto}" - COMMENT "Running ${_GRPC_PY_PLUGIN_EXECUTABLE} on ${grpc_proto}" - ) - - add_custom_command( - TARGET onnxruntime_server_tests POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/server_test - COMMAND ${CMAKE_COMMAND} -E copy - ${onnxruntime_integration_test_server_src} - ${CMAKE_CURRENT_BINARY_DIR}/server_test/ - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/onnx_ml_pb2.py - ${CMAKE_CURRENT_BINARY_DIR}/server_test/ - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/predict_pb2.py - ${CMAKE_CURRENT_BINARY_DIR}/server_test/ - COMMAND ${CMAKE_COMMAND} -E copy - ${grpc_py} - ${CMAKE_CURRENT_BINARY_DIR}/server_test/ - ) - -endif() - #some ETW tools if(WIN32 AND onnxruntime_ENABLE_INSTRUMENT) add_executable(generate_perf_report_from_etl ${ONNXRUNTIME_ROOT}/tool/etw/main.cc ${ONNXRUNTIME_ROOT}/tool/etw/eparser.h ${ONNXRUNTIME_ROOT}/tool/etw/eparser.cc ${ONNXRUNTIME_ROOT}/tool/etw/TraceSession.h ${ONNXRUNTIME_ROOT}/tool/etw/TraceSession.cc) diff --git a/cmake/patches/ngraph/ngraph_protobuf.patch b/cmake/patches/ngraph/ngraph_protobuf.patch index 87b7b5bd39f19..57bf85a881d7a 100644 --- a/cmake/patches/ngraph/ngraph_protobuf.patch +++ b/cmake/patches/ngraph/ngraph_protobuf.patch @@ -11,7 +11,7 @@ index 32217f5d6..d33e9c631 100644 # This version of PROTOBUF is required by Microsoft ONNX Runtime. set(NGRAPH_PROTOBUF_GIT_REPO_URL "https://github.com/protocolbuffers/protobuf") -set(NGRAPH_PROTOBUF_GIT_TAG "v3.5.2") -+set(NGRAPH_PROTOBUF_GIT_TAG "v3.6.1") ++set(NGRAPH_PROTOBUF_GIT_TAG "v3.11.2") if (WIN32) ExternalProject_Add( diff --git a/cmake/precompiled_header.cmake b/cmake/precompiled_header.cmake index f41310fac524c..3bebf7341809b 100644 --- a/cmake/precompiled_header.cmake +++ b/cmake/precompiled_header.cmake @@ -2,8 +2,7 @@ # header name as input. The function will generate a .cpp file that includes the header and is used # to generate the precompiled header; this source file is added to the target's sources. function(target_precompiled_header target_name header_name) - if (MSVC) - + if (MSVC AND CMAKE_VS_PLATFORM_TOOLSET) # The input precompiled header source (i.e. the '.h' file used for the precompiled header). set(pch_header_path ${header_name}) get_filename_component(header_base_name ${header_name} NAME_WE) @@ -14,14 +13,14 @@ function(target_precompiled_header target_name header_name) set(pch_source_content "// THIS FILE IS GENERATED BY CMAKE\n#include \"${pch_header_path}\"") file(WRITE ${pch_source_path} ${pch_source_content}) set_source_files_properties(${pch_source_path} PROPERTIES COMPILE_FLAGS "/Yc${pch_header_path}") - + # The target's C++ sources use the precompiled header (/Yu). Source-level properties will - # take precedence over target-level properties, so this will not change the generated source + # take precedence over target-level properties, so this will not change the generated source # file's property to create the precompiled header (/Yc). target_compile_options(${target_name} PRIVATE $<$:/Yu${header_name}>) - + # Append generated precompiled source to target's sources. target_sources(${target_name} PRIVATE ${pch_source_path}) - - endif(MSVC) -endfunction() \ No newline at end of file + + endif() +endfunction() diff --git a/cmake/winml.cmake b/cmake/winml.cmake index 4770729305b86..0af2fbe082943 100644 --- a/cmake/winml.cmake +++ b/cmake/winml.cmake @@ -225,6 +225,7 @@ add_dependencies(winml_adapter ${onnxruntime_EXTERNAL_DEPENDENCIES}) target_precompiled_header(winml_adapter pch.h) # Includes +target_include_directories(winml_adapter PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) # windows machine learning generated component headers target_include_directories(winml_adapter PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml_api) # windows machine learning generated component headers target_include_directories(winml_adapter PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml_api/comp_generated) # windows machine learning generated component headers target_include_directories(winml_adapter PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml/sdk/cppwinrt/include) # sdk cppwinrt headers @@ -247,11 +248,11 @@ add_dependencies(winml_adapter winml_api_native_internal) # Link libraries target_link_libraries(winml_adapter PRIVATE wil) if (onnxruntime_USE_DML) - target_link_libraries(winml_adapter PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/packages/DirectML.0.0.1/build/DirectML.targets) + target_add_dml(winml_adapter) endif(onnxruntime_USE_DML) # add it to the onnxruntime shared library -set(onnxruntime_winml windowsapp.lib winml_adapter) +set(onnxruntime_winml winml_adapter) list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES winml_adapter) ########################### @@ -296,6 +297,7 @@ target_compile_definitions(winml_lib_image PRIVATE _SCL_SECURE_NO_WARNINGS) target_precompiled_header(winml_lib_image pch.h) # Includes +target_include_directories(winml_lib_image PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) # windows machine learning generated component headers target_include_directories(winml_lib_image PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml_api) # windows machine learning generated component headers target_include_directories(winml_lib_image PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml_api/comp_generated) # windows machine learning generated component headers target_include_directories(winml_lib_image PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml/sdk/cppwinrt/include) # sdk cppwinrt headers @@ -322,9 +324,9 @@ add_dependencies(winml_lib_image winml_api_native) add_dependencies(winml_lib_image winml_api_native_internal) # Link libraries -target_link_libraries(winml_lib_image PRIVATE wil) +target_link_libraries(winml_lib_image PRIVATE wil winml_lib_common) if (onnxruntime_USE_DML) - target_link_libraries(winml_lib_image PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/packages/DirectML.0.0.1/build/DirectML.targets) + target_add_dml(winml_lib_image) endif(onnxruntime_USE_DML) @@ -425,11 +427,37 @@ add_dependencies(winml_lib_api winml_api_native) add_dependencies(winml_lib_api winml_api_native_internal) # Link libraries -target_link_libraries(winml_lib_api PRIVATE wil) +target_link_libraries(winml_lib_api PRIVATE wil winml_lib_telemetry) if (onnxruntime_USE_DML) - target_link_libraries(winml_lib_api PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/packages/DirectML.0.0.1/build/DirectML.targets) + target_add_dml(winml_lib_api) endif(onnxruntime_USE_DML) +########################### +# Add winml_lib_common +########################### + +add_library(winml_lib_common STATIC + ${winml_lib_common_dir}/CommonDeviceHelpers.cpp +) + +set_target_properties(winml_lib_common PROPERTIES CXX_STANDARD 17) +set_target_properties(winml_lib_common PROPERTIES CXX_STANDARD_REQUIRED ON) +target_compile_options(winml_lib_common PRIVATE /GR- /await /bigobj /wd4238) +target_link_libraries(winml_lib_common PRIVATE wil) +target_include_directories(winml_lib_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml_api) +target_compile_definitions(winml_lib_common PRIVATE + ONNX_NAMESPACE=onnx + ONNX_ML + LOTUS_LOG_THRESHOLD=2 + LOTUS_ENABLE_STDERR_LOGGING + PLATFORM_WINDOWS + _SCL_SECURE_NO_WARNINGS) +add_dependencies(winml_lib_common winml_sdk_cppwinrt) + +target_include_directories(winml_lib_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml_api) # windows machine learning generated component headers +target_include_directories(winml_lib_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml_api/comp_generated) # windows machine learning generated component headers +target_include_directories(winml_lib_common PRIVATE ${winml_lib_api_dir}) +target_include_directories(winml_lib_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) ########################### # Add winml_dll @@ -506,6 +534,13 @@ if (onnxruntime_USE_DML) set(delayload_dml "/DELAYLOAD:directml.dll") endif(onnxruntime_USE_DML) +# The default libraries to link with in Windows are kernel32.lib;user32.lib;gdi32.lib;winspool.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;comdlg32.lib;advapi32.lib +# Remove them and use the onecore umbrella library instead +foreach(default_lib kernel32.lib user32.lib gdi32.lib winspool.lib shell32.lib ole32.lib oleaut32.lib uuid.lib comdgl32.lib advapi32.lib) + set(removed_libs "${removed_libs} /NODEFAULTLIB:${default_lib}") +endforeach() +set(CMAKE_C_STANDARD_LIBRARIES "${removed_libs} onecoreuap.lib") +set(CMAKE_CXX_STANDARD_LIBRARIES "${removed_libs} onecoreuap.lib") set_target_properties(winml_dll PROPERTIES LINK_FLAGS @@ -535,12 +570,11 @@ endif("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") target_link_libraries(winml_dll PRIVATE onnxruntime) target_link_libraries(winml_dll PRIVATE re2) target_link_libraries(winml_dll PRIVATE wil) -#target_link_libraries(winml_dll PRIVATE windowsapp.lib) target_link_libraries(winml_dll PRIVATE winml_lib_api) target_link_libraries(winml_dll PRIVATE winml_lib_image) target_link_libraries(winml_dll PRIVATE winml_lib_ort) target_link_libraries(winml_dll PRIVATE winml_lib_telemetry) -target_link_libraries(winml_dll PRIVATE onecoreuap_apiset.lib) +target_link_libraries(winml_dll PRIVATE delayimp.lib) target_link_libraries(winml_dll PRIVATE ${DBGHELP}) # 1 of 3 projects that fail in link with 'failed to do memory mapped file I/O' (Only release) @@ -552,6 +586,7 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "Release") set_target_properties(winml_dll PROPERTIES VS_GLOBAL_PreferredToolArchitecture "x64") endif("${CMAKE_BUILD_TYPE}" STREQUAL "Release") +option(onnxruntime_BUILD_WINML_TESTS "Build WinML tests" ON) if (onnxruntime_BUILD_WINML_TESTS) include(winml_unittests.cmake) endif() diff --git a/cmake/winml_cppwinrt.cmake b/cmake/winml_cppwinrt.cmake index 2d53ade270b2b..ea96a1b289764 100644 --- a/cmake/winml_cppwinrt.cmake +++ b/cmake/winml_cppwinrt.cmake @@ -102,7 +102,7 @@ function(target_cppwinrt # Get directory get_filename_component(idl_source_directory ${file} DIRECTORY) - set(target_outputs ${CMAKE_CURRENT_BINARY_DIR}/${target_name}) + set(target_outputs ${CMAKE_CURRENT_BINARY_DIR}/${target_name}) convert_forward_slashes_to_back(${target_outputs}/comp output_dir_back_slash) convert_forward_slashes_to_back(${target_outputs}/temp temp_dir_back_slash) convert_forward_slashes_to_back(${target_outputs}/comp_generated generated_dir_back_slash) @@ -126,50 +126,53 @@ function(target_cppwinrt /tlb ${tlb_filename} ${idl_file_forward_slash} COMMAND - ${cppwinrt_exe} -in \"${winmd_filename}\" -comp \"${output_dir_back_slash}\" -ref \"${sdk_metadata_directory}\" -out \"${generated_dir_back_slash}\" -verbose + ${cppwinrt_exe} -in ${winmd_filename} -comp ${output_dir_back_slash} -ref ${sdk_metadata_directory} -out ${generated_dir_back_slash} -verbose COMMAND # copy the generated component files into a temporary directory where headers exclusions will be applied - xcopy \"${output_dir_back_slash}\" \"${temp_dir_back_slash}\\\" /Y /D + xcopy ${output_dir_back_slash} ${temp_dir_back_slash}\\ /Y /D COMMAND # for each file in the temp directory, ensure it is not in the exclusions list. # if it is, then we need to delete it. - for /f %%I in ('dir /b \"${temp_dir_back_slash}\"') - do - ( - for /f %%E in (${CPPWINRT_COMPONENT_EXCLUSION_LIST}) - do - ( - if %%E == %%I - ( - del \"${temp_dir_back_slash}\\%%I\" - ) - ) - ) + cmd /C "@echo off \ + for /f %I in ('dir /b ${temp_dir_back_slash}') \ + do \ + ( \ + for /f %E in (${CPPWINRT_COMPONENT_EXCLUSION_LIST}) \ + do \ + ( \ + if %E == %I \ + ( \ + del ${temp_dir_back_slash}\\%I \ + ) \ + ) \ + )" COMMAND # for each file in the temp directory, copy the file back into the source tree # unless the file already exists - for /f %%I in ('dir /b \"${temp_dir_back_slash}\"') - do - ( - if not exist \"${out_sources_folder}\\%%I\" - ( - xcopy \"${temp_dir_back_slash}\\%%I\" \"${out_sources_folder}\\%%I\" - ) - ) + cmd /C "@echo off \ + for /f %I in ('dir /b ${temp_dir_back_slash}') \ + do \ + ( \ + if not exist ${out_sources_folder}\\%I \ + ( \ + copy ${temp_dir_back_slash}\\%I ${out_sources_folder}\\%I \ + ) \ + )" COMMAND # open the generated module.g.cpp and strip all the includes (lines) containing excluded headers # write the new file out to module.g.excl.cpp. - powershell -Command \"& { - $exclusions = get-content '${CPPWINRT_COMPONENT_EXCLUSION_LIST}'\; - (get-content '${module_g_cpp_back_slash}') - | where { - $str = $_\; - $matches = ($exclusions | where { $str -match $_ }) \; - $matches.Length -eq 0 } - | Out-File '${module_g_ecxl_cpp_back_slash}' - }\" + powershell -Command "& { \ + $exclusions = get-content '${CPPWINRT_COMPONENT_EXCLUSION_LIST}'; \ + (get-content '${module_g_cpp_back_slash}') \ + | where { \ + $str = $_; \ + $matches = ($exclusions | where { $str -match $_ }); \ + $matches.Length -eq 0 } \ + | Out-File '${module_g_ecxl_cpp_back_slash}' \ + }" BYPRODUCTS ${generated_dir_back_slash}/module.g.excl.cpp + VERBATIM ) add_custom_target( @@ -214,4 +217,4 @@ function(add_generate_cppwinrt_sdk_headers_target set_target_properties(${target_name} PROPERTIES FOLDER ${folder_name}) endif() -endfunction() \ No newline at end of file +endfunction() diff --git a/cmake/winml_sdk_helpers.cmake b/cmake/winml_sdk_helpers.cmake index 03a87653fe986..df9db93a6b467 100644 --- a/cmake/winml_sdk_helpers.cmake +++ b/cmake/winml_sdk_helpers.cmake @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0) -# utility +# utility function(convert_forward_slashes_to_back input output) string(REGEX REPLACE "/" "\\\\" backwards ${input}) set(${output} ${backwards} PARENT_SCOPE) @@ -16,7 +16,23 @@ function(get_installed_sdk set(${sdk_folder} ${win10_sdk_root} PARENT_SCOPE) # return the sdk version - set(${output_sdk_version} ${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION} PARENT_SCOPE) + if(CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION) + set(${output_sdk_version} ${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION} PARENT_SCOPE) + else() + # choose the SDK matching the system version, or fallback to the latest + file(GLOB win10_sdks RELATIVE "${win10_sdk_root}/UnionMetadata" "${win10_sdk_root}/UnionMetadata/*.*.*.*") + list(GET win10_sdks 0 latest_sdk) + foreach(sdk IN LISTS win10_sdks) + string(FIND ${sdk} ${CMAKE_SYSTEM_VERSION} is_system_version) + if(NOT ${is_system_version} EQUAL -1) + set(${output_sdk_version} ${sdk} PARENT_SCOPE) + return() + elseif(sdk VERSION_GREATER latest_sdk) + set(latest_sdk ${sdk}) + endif() + endforeach() + set(${output_sdk_version} ${latest_sdk} PARENT_SCOPE) + endif() endfunction() # current sdk binary directory @@ -95,7 +111,7 @@ function(get_sdk set(${output_sdk_version} ${winml_WINDOWS_SDK_VERSION_OVERRIDE} PARENT_SCOPE) else() message( - FATAL_ERROR + FATAL_ERROR "Options winml_WINDOWS_SDK_DIR_OVERRIDE and winml_WINDOWS_SDK_VERSION_OVERRIDE must be defined together, or not at all.") endif() -endfunction() \ No newline at end of file +endfunction() diff --git a/cmake/winml_unittests.cmake b/cmake/winml_unittests.cmake index 18e6f7cdf85ff..29ee21ce742c4 100644 --- a/cmake/winml_unittests.cmake +++ b/cmake/winml_unittests.cmake @@ -4,7 +4,6 @@ set(WINML_TEST_SRC_DIR ${REPO_ROOT}/winml/test) set(WINML_TEST_INC_DIR ${REPO_ROOT}/winml/test/common - ${REPO_ROOT}/winml/lib/Api.Image/inc ${REPO_ROOT}/winml/lib/Common/inc ${REPO_ROOT}/onnxruntime ${REPO_ROOT}/onnxruntime/core/providers/dml/DmlExecutionProvider/src/External/D3DX12 @@ -44,7 +43,7 @@ function(add_winml_test) if (_UT_DEPENDS) add_dependencies(${_UT_TARGET} ${_UT_DEPENDS}) endif() - target_link_libraries(${_UT_TARGET} PRIVATE ${_UT_LIBS} gtest windowsapp winml_lib_image ${onnxruntime_EXTERNAL_LIBRARIES} winml_lib_telemetry) + target_link_libraries(${_UT_TARGET} PRIVATE ${_UT_LIBS} gtest ${onnxruntime_EXTERNAL_LIBRARIES} winml_lib_common onnxruntime) add_test(NAME ${_UT_TARGET} COMMAND ${_UT_TARGET} @@ -60,6 +59,7 @@ add_dependencies(winml_test_common winml_api winml_dll ) +target_compile_definitions(winml_test_common PRIVATE BUILD_GOOGLE_TEST) set_winml_target_properties(winml_test_common) file(GLOB winml_test_api_src CONFIGURE_DEPENDS "${WINML_TEST_SRC_DIR}/api/*.cpp") @@ -68,6 +68,7 @@ add_winml_test( SOURCES ${winml_test_api_src} LIBS winml_test_common ) +target_compile_definitions(winml_test_api PRIVATE BUILD_GOOGLE_TEST) target_precompiled_header(winml_test_api testPch.h) if (onnxruntime_USE_DML) @@ -79,9 +80,10 @@ endif() add_winml_test( TARGET winml_test_scenario SOURCES ${winml_test_scenario_src} - LIBS winml_test_common ${winml_test_scenario_libs} + LIBS winml_test_common delayimp.lib ${winml_test_scenario_libs} ) target_precompiled_header(winml_test_scenario testPch.h) +target_compile_definitions(winml_test_scenario PRIVATE BUILD_GOOGLE_TEST) set_target_properties(winml_test_scenario PROPERTIES LINK_FLAGS "/DELAYLOAD:d2d1.dll /DELAYLOAD:d3d11.dll /DELAYLOAD:dxgi.dll" ) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj index eab50f647af01..1a254099e2a43 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj @@ -207,7 +207,7 @@ - + diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs index 70d48877da9c5..6f24f49634da0 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs @@ -51,7 +51,7 @@ public SessionOptions() #if USE_CUDA /// - /// A helper method to constuct a SessionOptions object for CUDA execution + /// A helper method to construct a SessionOptions object for CUDA execution /// /// A SessionsOptions() object configured for execution on deviceId=0 public static SessionOptions MakeSessionOptionWithCudaProvider() @@ -60,7 +60,7 @@ public static SessionOptions MakeSessionOptionWithCudaProvider() } /// - /// A helper method to constuct a SessionOptions object for CUDA execution + /// A helper method to construct a SessionOptions object for CUDA execution /// /// /// A SessionsOptions() object configured for execution on deviceId @@ -345,7 +345,7 @@ public int InterOpNumThreads private int _interOpNumThreads = 0; // set to what is set in C++ SessionOptions by default; /// - /// Sets the graph optimization level for the session. Default is set to ORT_ENABLE_BASIC. + /// Sets the graph optimization level for the session. Default is set to ORT_ENABLE_ALL. /// public GraphOptimizationLevel GraphOptimizationLevel { @@ -359,7 +359,7 @@ public GraphOptimizationLevel GraphOptimizationLevel _graphOptimizationLevel = value; } } - private GraphOptimizationLevel _graphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_BASIC; + private GraphOptimizationLevel _graphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL; /// /// Sets the execution mode for the session. Default is set to ORT_SEQUENTIAL. diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/Microsoft.ML.OnnxRuntime.EndToEndTests.RunCapi.vcxproj b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/Microsoft.ML.OnnxRuntime.EndToEndTests.RunCapi.vcxproj index eb576614c8732..3fb20d386bb2e 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/Microsoft.ML.OnnxRuntime.EndToEndTests.RunCapi.vcxproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/Microsoft.ML.OnnxRuntime.EndToEndTests.RunCapi.vcxproj @@ -1,13 +1,11 @@ - $(MSBuildThisFileDirectory)..\.. Microsoft.ML.OnnxRuntime C_Api_Sample.cpp - Debug @@ -17,7 +15,7 @@ Release x64 - + Debug x86 @@ -36,13 +34,13 @@ Application true - v141 + v142 Unicode Application false - v141 + v142 true Unicode @@ -112,7 +110,7 @@ - + Always false @@ -129,4 +127,4 @@ - + \ No newline at end of file diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/Microsoft.ML.OnnxRuntime.Gpu.EndToEndTests.RunCapi.vcxproj b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/Microsoft.ML.OnnxRuntime.Gpu.EndToEndTests.RunCapi.vcxproj index cd93ff64ff455..ea4701e9fedc1 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/Microsoft.ML.OnnxRuntime.Gpu.EndToEndTests.RunCapi.vcxproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/Microsoft.ML.OnnxRuntime.Gpu.EndToEndTests.RunCapi.vcxproj @@ -1,11 +1,9 @@ - $(MSBuildThisFileDirectory)..\.. - Debug @@ -26,13 +24,13 @@ Application true - v141 + v142 Unicode Application false - v141 + v142 true Unicode @@ -90,7 +88,7 @@ - + Always false @@ -107,4 +105,4 @@ - + \ No newline at end of file diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj index e2ffae1c4cb4e..19ba6724e8ca8 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj @@ -13,7 +13,7 @@ Microsoft.ML.OnnxRuntime - + diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 29d3181568cd3..7b221f5e33673 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -41,7 +41,7 @@ public void TestSessionOptions() Assert.Equal(LogLevel.Verbose, opt.LogVerbosityLevel); Assert.Equal(0, opt.IntraOpNumThreads); Assert.Equal(0, opt.InterOpNumThreads); - Assert.Equal(GraphOptimizationLevel.ORT_ENABLE_BASIC, opt.GraphOptimizationLevel); + Assert.Equal(GraphOptimizationLevel.ORT_ENABLE_ALL, opt.GraphOptimizationLevel); // try setting options opt.ExecutionMode = ExecutionMode.ORT_PARALLEL; @@ -328,7 +328,8 @@ private static Dictionary GetSkippedModels() { "mlperf_ssd_mobilenet_300", "Could not find file output_0.pb" }, { "tf_resnet_v1_50", "result mismatch when Conv BN Fusion is applied" }, { "tf_resnet_v1_101", "result mismatch when Conv BN Fusion is applied" }, - { "tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied" } + { "tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied" }, + { "mask_rcnn_keras", "Model should be edited to remove the extra outputs" }, }; // The following models fails on nocontribops win CI @@ -437,84 +438,86 @@ private void TestPreTrainedModels(string opset, string modelName) { testDataDirNamePattern = "seq_lens*"; // discrepency in data directory } - var testDataDir = modelDir.EnumerateDirectories(testDataDirNamePattern).First(); - var inputContainer = new List(); - var outputContainer = new List(); - foreach (var f in testDataDir.EnumerateFiles("input_*.pb")) - { - inputContainer.Add(LoadTensorFromFilePb(f.FullName, inMeta)); - } - foreach (var f in testDataDir.EnumerateFiles("output_*.pb")) + foreach (var testDataDir in modelDir.EnumerateDirectories(testDataDirNamePattern)) { - outputContainer.Add(LoadTensorFromFilePb(f.FullName, session.OutputMetadata)); - } + var inputContainer = new List(); + var outputContainer = new List(); + foreach (var f in testDataDir.EnumerateFiles("input_*.pb")) + { + inputContainer.Add(LoadTensorFromFilePb(f.FullName, inMeta)); + } + foreach (var f in testDataDir.EnumerateFiles("output_*.pb")) + { + outputContainer.Add(LoadTensorFromFilePb(f.FullName, session.OutputMetadata)); + } - using (var resultCollection = session.Run(inputContainer)) - { - foreach (var result in resultCollection) + using (var resultCollection = session.Run(inputContainer)) { - Assert.True(session.OutputMetadata.ContainsKey(result.Name)); - var outputMeta = session.OutputMetadata[result.Name]; - NamedOnnxValue outputValue = null; - foreach (var o in outputContainer) - { - if (o.Name == result.Name) - { - outputValue = o; - break; - } - } - if (outputValue == null) - { - outputValue = outputContainer.First(); // in case the output data file does not contain the name - } - if (outputMeta.IsTensor) + foreach (var result in resultCollection) { - if (outputMeta.ElementType == typeof(float)) - { - Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new floatComparer()); - } - else if (outputMeta.ElementType == typeof(int)) - { - Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); - } - else if (outputMeta.ElementType == typeof(uint)) - { - Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); - } - else if (outputMeta.ElementType == typeof(short)) + Assert.True(session.OutputMetadata.ContainsKey(result.Name)); + var outputMeta = session.OutputMetadata[result.Name]; + NamedOnnxValue outputValue = null; + foreach (var o in outputContainer) { - Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); + if (o.Name == result.Name) + { + outputValue = o; + break; + } } - else if (outputMeta.ElementType == typeof(ushort)) + if (outputValue == null) { - Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); + outputValue = outputContainer.First(); // in case the output data file does not contain the name } - else if (outputMeta.ElementType == typeof(long)) + if (outputMeta.IsTensor) { - Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); - } - else if (outputMeta.ElementType == typeof(ulong)) - { - Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); - } - else if (outputMeta.ElementType == typeof(byte)) - { - Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); - } - else if (outputMeta.ElementType == typeof(bool)) - { - Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); + if (outputMeta.ElementType == typeof(float)) + { + Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new floatComparer()); + } + else if (outputMeta.ElementType == typeof(int)) + { + Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); + } + else if (outputMeta.ElementType == typeof(uint)) + { + Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); + } + else if (outputMeta.ElementType == typeof(short)) + { + Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); + } + else if (outputMeta.ElementType == typeof(ushort)) + { + Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); + } + else if (outputMeta.ElementType == typeof(long)) + { + Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); + } + else if (outputMeta.ElementType == typeof(ulong)) + { + Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); + } + else if (outputMeta.ElementType == typeof(byte)) + { + Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); + } + else if (outputMeta.ElementType == typeof(bool)) + { + Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer()); + } + else + { + Assert.True(false, "The TestPretrainedModels does not yet support output of type " + nameof(outputMeta.ElementType)); + } } else { - Assert.True(false, "The TestPretrainedModels does not yet support output of type " + nameof(outputMeta.ElementType)); + Assert.True(false, "TestPretrainedModel cannot handle non-tensor outputs yet"); } } - else - { - Assert.True(false, "TestPretrainedModel cannot handle non-tensor outputs yet"); - } } } } diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj index 5a74dd3674fb0..04c8a1cffa639 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj @@ -47,7 +47,7 @@ - + diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs index dc04a19a2de5a..57df68e12e6d5 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs @@ -109,24 +109,24 @@ static OnnxMlReflection() { "EhAKDEVYUEVSSU1FTlRBTBAAEgoKBlNUQUJMRRABYgZwcm90bzM=")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, - new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Onnx.Version), typeof(global::Onnx.OperatorStatus), }, new pbr::GeneratedClrTypeInfo[] { - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.AttributeProto), global::Onnx.AttributeProto.Parser, new[]{ "Name", "RefAttrName", "DocString", "Type", "F", "I", "S", "T", "G", "SparseTensor", "Floats", "Ints", "Strings", "Tensors", "Graphs", "SparseTensors" }, null, new[]{ typeof(global::Onnx.AttributeProto.Types.AttributeType) }, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.ValueInfoProto), global::Onnx.ValueInfoProto.Parser, new[]{ "Name", "Type", "DocString" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.NodeProto), global::Onnx.NodeProto.Parser, new[]{ "Input", "Output", "Name", "OpType", "Domain", "Attribute", "DocString" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.ModelProto), global::Onnx.ModelProto.Parser, new[]{ "IrVersion", "OpsetImport", "ProducerName", "ProducerVersion", "Domain", "ModelVersion", "DocString", "Graph", "Functions", "MetadataProps" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.StringStringEntryProto), global::Onnx.StringStringEntryProto.Parser, new[]{ "Key", "Value" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TensorAnnotation), global::Onnx.TensorAnnotation.Parser, new[]{ "TensorName", "QuantParameterTensorNames" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.GraphProto), global::Onnx.GraphProto.Parser, new[]{ "Node", "Name", "Initializer", "SparseInitializer", "DocString", "Input", "Output", "ValueInfo", "QuantizationAnnotation" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TensorProto), global::Onnx.TensorProto.Parser, new[]{ "Dims", "DataType", "Segment", "FloatData", "Int32Data", "StringData", "Int64Data", "Name", "DocString", "RawData", "ExternalData", "DataLocation", "DoubleData", "Uint64Data" }, null, new[]{ typeof(global::Onnx.TensorProto.Types.DataType), typeof(global::Onnx.TensorProto.Types.DataLocation) }, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TensorProto.Types.Segment), global::Onnx.TensorProto.Types.Segment.Parser, new[]{ "Begin", "End" }, null, null, null)}), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.SparseTensorProto), global::Onnx.SparseTensorProto.Parser, new[]{ "Values", "Indices", "Dims" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TensorShapeProto), global::Onnx.TensorShapeProto.Parser, new[]{ "Dim" }, null, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TensorShapeProto.Types.Dimension), global::Onnx.TensorShapeProto.Types.Dimension.Parser, new[]{ "DimValue", "DimParam", "Denotation" }, new[]{ "Value" }, null, null)}), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto), global::Onnx.TypeProto.Parser, new[]{ "TensorType", "SequenceType", "MapType", "SparseTensorType", "OpaqueType", "Denotation" }, new[]{ "Value" }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto.Types.Tensor), global::Onnx.TypeProto.Types.Tensor.Parser, new[]{ "ElemType", "Shape" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto.Types.Sequence), global::Onnx.TypeProto.Types.Sequence.Parser, new[]{ "ElemType" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto.Types.Map), global::Onnx.TypeProto.Types.Map.Parser, new[]{ "KeyType", "ValueType" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto.Types.SparseTensor), global::Onnx.TypeProto.Types.SparseTensor.Parser, new[]{ "ElemType", "Shape" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto.Types.Opaque), global::Onnx.TypeProto.Types.Opaque.Parser, new[]{ "Domain", "Name" }, null, null, null)}), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.OperatorSetIdProto), global::Onnx.OperatorSetIdProto.Parser, new[]{ "Domain", "Version" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.FunctionProto), global::Onnx.FunctionProto.Parser, new[]{ "Name", "SinceVersion", "Status", "Input", "Output", "Attribute", "Node", "DocString" }, null, null, null) + new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Onnx.Version), typeof(global::Onnx.OperatorStatus), }, null, new pbr::GeneratedClrTypeInfo[] { + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.AttributeProto), global::Onnx.AttributeProto.Parser, new[]{ "Name", "RefAttrName", "DocString", "Type", "F", "I", "S", "T", "G", "SparseTensor", "Floats", "Ints", "Strings", "Tensors", "Graphs", "SparseTensors" }, null, new[]{ typeof(global::Onnx.AttributeProto.Types.AttributeType) }, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.ValueInfoProto), global::Onnx.ValueInfoProto.Parser, new[]{ "Name", "Type", "DocString" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.NodeProto), global::Onnx.NodeProto.Parser, new[]{ "Input", "Output", "Name", "OpType", "Domain", "Attribute", "DocString" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.ModelProto), global::Onnx.ModelProto.Parser, new[]{ "IrVersion", "OpsetImport", "ProducerName", "ProducerVersion", "Domain", "ModelVersion", "DocString", "Graph", "Functions", "MetadataProps" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.StringStringEntryProto), global::Onnx.StringStringEntryProto.Parser, new[]{ "Key", "Value" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TensorAnnotation), global::Onnx.TensorAnnotation.Parser, new[]{ "TensorName", "QuantParameterTensorNames" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.GraphProto), global::Onnx.GraphProto.Parser, new[]{ "Node", "Name", "Initializer", "SparseInitializer", "DocString", "Input", "Output", "ValueInfo", "QuantizationAnnotation" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TensorProto), global::Onnx.TensorProto.Parser, new[]{ "Dims", "DataType", "Segment", "FloatData", "Int32Data", "StringData", "Int64Data", "Name", "DocString", "RawData", "ExternalData", "DataLocation", "DoubleData", "Uint64Data" }, null, new[]{ typeof(global::Onnx.TensorProto.Types.DataType), typeof(global::Onnx.TensorProto.Types.DataLocation) }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TensorProto.Types.Segment), global::Onnx.TensorProto.Types.Segment.Parser, new[]{ "Begin", "End" }, null, null, null, null)}), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.SparseTensorProto), global::Onnx.SparseTensorProto.Parser, new[]{ "Values", "Indices", "Dims" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TensorShapeProto), global::Onnx.TensorShapeProto.Parser, new[]{ "Dim" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TensorShapeProto.Types.Dimension), global::Onnx.TensorShapeProto.Types.Dimension.Parser, new[]{ "DimValue", "DimParam", "Denotation" }, new[]{ "Value" }, null, null, null)}), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto), global::Onnx.TypeProto.Parser, new[]{ "TensorType", "SequenceType", "MapType", "SparseTensorType", "OpaqueType", "Denotation" }, new[]{ "Value" }, null, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto.Types.Tensor), global::Onnx.TypeProto.Types.Tensor.Parser, new[]{ "ElemType", "Shape" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto.Types.Sequence), global::Onnx.TypeProto.Types.Sequence.Parser, new[]{ "ElemType" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto.Types.Map), global::Onnx.TypeProto.Types.Map.Parser, new[]{ "KeyType", "ValueType" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto.Types.SparseTensor), global::Onnx.TypeProto.Types.SparseTensor.Parser, new[]{ "ElemType", "Shape" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.TypeProto.Types.Opaque), global::Onnx.TypeProto.Types.Opaque.Parser, new[]{ "Domain", "Name" }, null, null, null, null)}), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.OperatorSetIdProto), global::Onnx.OperatorSetIdProto.Parser, new[]{ "Domain", "Version" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Onnx.FunctionProto), global::Onnx.FunctionProto.Parser, new[]{ "Name", "SinceVersion", "Status", "Input", "Output", "Attribute", "Node", "DocString" }, null, null, null, null) })); } #endregion @@ -304,7 +304,7 @@ public string DocString { /// Field number for the "type" field. public const int TypeFieldNumber = 20; - private global::Onnx.AttributeProto.Types.AttributeType type_ = 0; + private global::Onnx.AttributeProto.Types.AttributeType type_ = global::Onnx.AttributeProto.Types.AttributeType.Undefined; /// /// The type field MUST be present for this version of the IR. /// For 0.0.1 versions of the IR, this field was not defined, and @@ -521,7 +521,7 @@ public override int GetHashCode() { if (Name.Length != 0) hash ^= Name.GetHashCode(); if (RefAttrName.Length != 0) hash ^= RefAttrName.GetHashCode(); if (DocString.Length != 0) hash ^= DocString.GetHashCode(); - if (Type != 0) hash ^= Type.GetHashCode(); + if (Type != global::Onnx.AttributeProto.Types.AttributeType.Undefined) hash ^= Type.GetHashCode(); if (F != 0F) hash ^= pbc::ProtobufEqualityComparers.BitwiseSingleEqualityComparer.GetHashCode(F); if (I != 0L) hash ^= I.GetHashCode(); if (S.Length != 0) hash ^= S.GetHashCode(); @@ -580,7 +580,7 @@ public void WriteTo(pb::CodedOutputStream output) { output.WriteRawTag(106); output.WriteString(DocString); } - if (Type != 0) { + if (Type != global::Onnx.AttributeProto.Types.AttributeType.Undefined) { output.WriteRawTag(160, 1); output.WriteEnum((int) Type); } @@ -610,7 +610,7 @@ public int CalculateSize() { if (DocString.Length != 0) { size += 1 + pb::CodedOutputStream.ComputeStringSize(DocString); } - if (Type != 0) { + if (Type != global::Onnx.AttributeProto.Types.AttributeType.Undefined) { size += 2 + pb::CodedOutputStream.ComputeEnumSize((int) Type); } if (F != 0F) { @@ -657,7 +657,7 @@ public void MergeFrom(AttributeProto other) { if (other.DocString.Length != 0) { DocString = other.DocString; } - if (other.Type != 0) { + if (other.Type != global::Onnx.AttributeProto.Types.AttributeType.Undefined) { Type = other.Type; } if (other.F != 0F) { @@ -671,19 +671,19 @@ public void MergeFrom(AttributeProto other) { } if (other.t_ != null) { if (t_ == null) { - t_ = new global::Onnx.TensorProto(); + T = new global::Onnx.TensorProto(); } T.MergeFrom(other.T); } if (other.g_ != null) { if (g_ == null) { - g_ = new global::Onnx.GraphProto(); + G = new global::Onnx.GraphProto(); } G.MergeFrom(other.G); } if (other.sparseTensor_ != null) { if (sparseTensor_ == null) { - sparseTensor_ = new global::Onnx.SparseTensorProto(); + SparseTensor = new global::Onnx.SparseTensorProto(); } SparseTensor.MergeFrom(other.SparseTensor); } @@ -722,16 +722,16 @@ public void MergeFrom(pb::CodedInputStream input) { } case 42: { if (t_ == null) { - t_ = new global::Onnx.TensorProto(); + T = new global::Onnx.TensorProto(); } - input.ReadMessage(t_); + input.ReadMessage(T); break; } case 50: { if (g_ == null) { - g_ = new global::Onnx.GraphProto(); + G = new global::Onnx.GraphProto(); } - input.ReadMessage(g_); + input.ReadMessage(G); break; } case 58: @@ -761,7 +761,7 @@ public void MergeFrom(pb::CodedInputStream input) { break; } case 160: { - type_ = (global::Onnx.AttributeProto.Types.AttributeType) input.ReadEnum(); + Type = (global::Onnx.AttributeProto.Types.AttributeType) input.ReadEnum(); break; } case 170: { @@ -770,9 +770,9 @@ public void MergeFrom(pb::CodedInputStream input) { } case 178: { if (sparseTensor_ == null) { - sparseTensor_ = new global::Onnx.SparseTensorProto(); + SparseTensor = new global::Onnx.SparseTensorProto(); } - input.ReadMessage(sparseTensor_); + input.ReadMessage(SparseTensor); break; } case 186: { @@ -978,7 +978,7 @@ public void MergeFrom(ValueInfoProto other) { } if (other.type_ != null) { if (type_ == null) { - type_ = new global::Onnx.TypeProto(); + Type = new global::Onnx.TypeProto(); } Type.MergeFrom(other.Type); } @@ -1002,9 +1002,9 @@ public void MergeFrom(pb::CodedInputStream input) { } case 18: { if (type_ == null) { - type_ = new global::Onnx.TypeProto(); + Type = new global::Onnx.TypeProto(); } - input.ReadMessage(type_); + input.ReadMessage(Type); break; } case 26: { @@ -1673,7 +1673,7 @@ public void MergeFrom(ModelProto other) { } if (other.graph_ != null) { if (graph_ == null) { - graph_ = new global::Onnx.GraphProto(); + Graph = new global::Onnx.GraphProto(); } Graph.MergeFrom(other.Graph); } @@ -1716,9 +1716,9 @@ public void MergeFrom(pb::CodedInputStream input) { } case 58: { if (graph_ == null) { - graph_ = new global::Onnx.GraphProto(); + Graph = new global::Onnx.GraphProto(); } - input.ReadMessage(graph_); + input.ReadMessage(Graph); break; } case 66: { @@ -2627,7 +2627,7 @@ public string DocString { /// Field number for the "data_location" field. public const int DataLocationFieldNumber = 14; - private global::Onnx.TensorProto.Types.DataLocation dataLocation_ = 0; + private global::Onnx.TensorProto.Types.DataLocation dataLocation_ = global::Onnx.TensorProto.Types.DataLocation.Default; /// /// If value not set, data is stored in raw_data (if set) otherwise in type-specified field. /// @@ -2717,7 +2717,7 @@ public override int GetHashCode() { if (DocString.Length != 0) hash ^= DocString.GetHashCode(); if (RawData.Length != 0) hash ^= RawData.GetHashCode(); hash ^= externalData_.GetHashCode(); - if (DataLocation != 0) hash ^= DataLocation.GetHashCode(); + if (DataLocation != global::Onnx.TensorProto.Types.DataLocation.Default) hash ^= DataLocation.GetHashCode(); hash ^= doubleData_.GetHashCode(); hash ^= uint64Data_.GetHashCode(); if (_unknownFields != null) { @@ -2761,7 +2761,7 @@ public void WriteTo(pb::CodedOutputStream output) { output.WriteString(DocString); } externalData_.WriteTo(output, _repeated_externalData_codec); - if (DataLocation != 0) { + if (DataLocation != global::Onnx.TensorProto.Types.DataLocation.Default) { output.WriteRawTag(112); output.WriteEnum((int) DataLocation); } @@ -2794,7 +2794,7 @@ public int CalculateSize() { size += 1 + pb::CodedOutputStream.ComputeBytesSize(RawData); } size += externalData_.CalculateSize(_repeated_externalData_codec); - if (DataLocation != 0) { + if (DataLocation != global::Onnx.TensorProto.Types.DataLocation.Default) { size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) DataLocation); } size += doubleData_.CalculateSize(_repeated_doubleData_codec); @@ -2816,7 +2816,7 @@ public void MergeFrom(TensorProto other) { } if (other.segment_ != null) { if (segment_ == null) { - segment_ = new global::Onnx.TensorProto.Types.Segment(); + Segment = new global::Onnx.TensorProto.Types.Segment(); } Segment.MergeFrom(other.Segment); } @@ -2834,7 +2834,7 @@ public void MergeFrom(TensorProto other) { RawData = other.RawData; } externalData_.Add(other.externalData_); - if (other.DataLocation != 0) { + if (other.DataLocation != global::Onnx.TensorProto.Types.DataLocation.Default) { DataLocation = other.DataLocation; } doubleData_.Add(other.doubleData_); @@ -2861,9 +2861,9 @@ public void MergeFrom(pb::CodedInputStream input) { } case 26: { if (segment_ == null) { - segment_ = new global::Onnx.TensorProto.Types.Segment(); + Segment = new global::Onnx.TensorProto.Types.Segment(); } - input.ReadMessage(segment_); + input.ReadMessage(Segment); break; } case 34: @@ -2912,7 +2912,7 @@ public void MergeFrom(pb::CodedInputStream input) { break; } case 112: { - dataLocation_ = (global::Onnx.TensorProto.Types.DataLocation) input.ReadEnum(); + DataLocation = (global::Onnx.TensorProto.Types.DataLocation) input.ReadEnum(); break; } } @@ -3327,13 +3327,13 @@ public void MergeFrom(SparseTensorProto other) { } if (other.values_ != null) { if (values_ == null) { - values_ = new global::Onnx.TensorProto(); + Values = new global::Onnx.TensorProto(); } Values.MergeFrom(other.Values); } if (other.indices_ != null) { if (indices_ == null) { - indices_ = new global::Onnx.TensorProto(); + Indices = new global::Onnx.TensorProto(); } Indices.MergeFrom(other.Indices); } @@ -3351,16 +3351,16 @@ public void MergeFrom(pb::CodedInputStream input) { break; case 10: { if (values_ == null) { - values_ = new global::Onnx.TensorProto(); + Values = new global::Onnx.TensorProto(); } - input.ReadMessage(values_); + input.ReadMessage(Values); break; } case 18: { if (indices_ == null) { - indices_ = new global::Onnx.TensorProto(); + Indices = new global::Onnx.TensorProto(); } - input.ReadMessage(indices_); + input.ReadMessage(Indices); break; } case 26: @@ -4240,7 +4240,7 @@ public void MergeFrom(Tensor other) { } if (other.shape_ != null) { if (shape_ == null) { - shape_ = new global::Onnx.TensorShapeProto(); + Shape = new global::Onnx.TensorShapeProto(); } Shape.MergeFrom(other.Shape); } @@ -4261,9 +4261,9 @@ public void MergeFrom(pb::CodedInputStream input) { } case 18: { if (shape_ == null) { - shape_ = new global::Onnx.TensorShapeProto(); + Shape = new global::Onnx.TensorShapeProto(); } - input.ReadMessage(shape_); + input.ReadMessage(Shape); break; } } @@ -4386,7 +4386,7 @@ public void MergeFrom(Sequence other) { } if (other.elemType_ != null) { if (elemType_ == null) { - elemType_ = new global::Onnx.TypeProto(); + ElemType = new global::Onnx.TypeProto(); } ElemType.MergeFrom(other.ElemType); } @@ -4403,9 +4403,9 @@ public void MergeFrom(pb::CodedInputStream input) { break; case 10: { if (elemType_ == null) { - elemType_ = new global::Onnx.TypeProto(); + ElemType = new global::Onnx.TypeProto(); } - input.ReadMessage(elemType_); + input.ReadMessage(ElemType); break; } } @@ -4556,7 +4556,7 @@ public void MergeFrom(Map other) { } if (other.valueType_ != null) { if (valueType_ == null) { - valueType_ = new global::Onnx.TypeProto(); + ValueType = new global::Onnx.TypeProto(); } ValueType.MergeFrom(other.ValueType); } @@ -4577,9 +4577,9 @@ public void MergeFrom(pb::CodedInputStream input) { } case 18: { if (valueType_ == null) { - valueType_ = new global::Onnx.TypeProto(); + ValueType = new global::Onnx.TypeProto(); } - input.ReadMessage(valueType_); + input.ReadMessage(ValueType); break; } } @@ -4724,7 +4724,7 @@ public void MergeFrom(SparseTensor other) { } if (other.shape_ != null) { if (shape_ == null) { - shape_ = new global::Onnx.TensorShapeProto(); + Shape = new global::Onnx.TensorShapeProto(); } Shape.MergeFrom(other.Shape); } @@ -4745,9 +4745,9 @@ public void MergeFrom(pb::CodedInputStream input) { } case 18: { if (shape_ == null) { - shape_ = new global::Onnx.TensorShapeProto(); + Shape = new global::Onnx.TensorShapeProto(); } - input.ReadMessage(shape_); + input.ReadMessage(Shape); break; } } @@ -5170,7 +5170,7 @@ public long SinceVersion { /// Field number for the "status" field. public const int StatusFieldNumber = 3; - private global::Onnx.OperatorStatus status_ = 0; + private global::Onnx.OperatorStatus status_ = global::Onnx.OperatorStatus.Experimental; /// /// This field indicates whether the syntax, semantics, or presence /// of this function is in an experimental or stable stage. Once an @@ -5281,7 +5281,7 @@ public override int GetHashCode() { int hash = 1; if (Name.Length != 0) hash ^= Name.GetHashCode(); if (SinceVersion != 0L) hash ^= SinceVersion.GetHashCode(); - if (Status != 0) hash ^= Status.GetHashCode(); + if (Status != global::Onnx.OperatorStatus.Experimental) hash ^= Status.GetHashCode(); hash ^= input_.GetHashCode(); hash ^= output_.GetHashCode(); hash ^= attribute_.GetHashCode(); @@ -5308,7 +5308,7 @@ public void WriteTo(pb::CodedOutputStream output) { output.WriteRawTag(16); output.WriteInt64(SinceVersion); } - if (Status != 0) { + if (Status != global::Onnx.OperatorStatus.Experimental) { output.WriteRawTag(24); output.WriteEnum((int) Status); } @@ -5334,7 +5334,7 @@ public int CalculateSize() { if (SinceVersion != 0L) { size += 1 + pb::CodedOutputStream.ComputeInt64Size(SinceVersion); } - if (Status != 0) { + if (Status != global::Onnx.OperatorStatus.Experimental) { size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) Status); } size += input_.CalculateSize(_repeated_input_codec); @@ -5361,7 +5361,7 @@ public void MergeFrom(FunctionProto other) { if (other.SinceVersion != 0L) { SinceVersion = other.SinceVersion; } - if (other.Status != 0) { + if (other.Status != global::Onnx.OperatorStatus.Experimental) { Status = other.Status; } input_.Add(other.input_); @@ -5391,7 +5391,7 @@ public void MergeFrom(pb::CodedInputStream input) { break; } case 24: { - status_ = (global::Onnx.OperatorStatus) input.ReadEnum(); + Status = (global::Onnx.OperatorStatus) input.ReadEnum(); break; } case 34: { diff --git a/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs index de589114e7bad..a9af0449bd15c 100644 --- a/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs +++ b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs @@ -32,8 +32,8 @@ class CommandOptions [Option('p', Required = false, HelpText = "Run with parallel exection. Default is false")] public bool ParallelExecution { get; set; } = false; - [Option('o', "optimization_level", Required = false, HelpText = "Optimization Level. Default is 1, partial optimization.")] - public GraphOptimizationLevel OptimizationLevel { get; set; } = GraphOptimizationLevel.ORT_ENABLE_BASIC; + [Option('o', "optimization_level", Required = false, HelpText = "Optimization Level. Default is 99, all optimization.")] + public GraphOptimizationLevel OptimizationLevel { get; set; } = GraphOptimizationLevel.ORT_ENABLE_ALL; } class Program diff --git a/dockerfiles/README.md b/dockerfiles/README.md index d99aecff57020..ae32c1fcac6cc 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -80,7 +80,7 @@ Use `docker pull` with any of the images and tags below to pull an image and try ``` ## TensorRT -**Ubuntu 16.04, TensorRT 5.0.2** +**Ubuntu 18.04, CUDA 10.1.243, TensorRT 6.0.1** 1. Build the docker image from the Dockerfile in this repository. ``` diff --git a/docs/ONNX_Runtime_Graph_Optimizations.md b/docs/ONNX_Runtime_Graph_Optimizations.md index 0c347f87e477d..1002b41d68c64 100644 --- a/docs/ONNX_Runtime_Graph_Optimizations.md +++ b/docs/ONNX_Runtime_Graph_Optimizations.md @@ -15,9 +15,11 @@ Graph optimizations are divided in three levels: The optimizations belonging to one level are performed after the optimizations of the previous level have been applied (e.g., extended optimizations are applied after basic optimizations have been applied). +**All optimizations are enabled by default.** + ### Basic Graph Optimizations -These are semantics-preserving graph rewrites which remove redundant nodes and redundant computation. These optimizations are enabled by default. They run before graph partitioning and thus apply to all the execution providers. Available basic graph optimizations are as follows: +These are semantics-preserving graph rewrites which remove redundant nodes and redundant computation. They run before graph partitioning and thus apply to all the execution providers. Available basic graph optimizations are as follows: * Constant Folding: Statically computes parts of the graph that rely only on constant initializers. This eliminates the need to compute them during runtime. @@ -32,15 +34,26 @@ These are semantics-preserving graph rewrites which remove redundant nodes and r * Conv Mul Fusion * Conv BatchNorm Fusion * Relu Clip Fusion + * Reshape Fusion ### Extended Graph Optimizations -These optimizations include complex node fusions. They are run after graph partitioning and are only applied to the nodes assigned to the CPU execution provider. Available extended graph optimizations are as follows: - -* GEMM Activation Fusion -* Matmul Add Fusion -* Conv Activation Fusion -* GELU Fusion +These optimizations include complex node fusions. They are run after graph partitioning and are only applied to the nodes assigned to the CPU or CUDA execution provider. Available extended graph optimizations are as follows: + +| Optimization | Execution Provider | Comment | +|---------------------------------|--------------------|-----------------------------------------------------------------------------| +| GEMM Activation Fusion | cpu | | +| Matmul Add Fusion | cpu | | +| Conv Activation Fusion | cpu | | +| GELU Fusion | cpu or cuda | | +| Layer Normalization Fusion | cpu or cuda | | +| BERT Embedding Layer Fusion | cpu or cuda | Fuse BERT embedding layer, layer normalization and attention mask length | +| Attention Fusion | cpu or cuda | Attention mask has approximation in cuda execution provider | +| Skip Layer Normalization Fusion | cpu or cuda | Fuse bias of fully connected layer, skip connection and layer normalization | +| Bias GELU Fusion | cpu or cuda | Fuse bias of fully connected layer and GELU activation | +| GELU Approximation | cuda | Erf is approximated by a formula using tanh function | + +To optimize inference performance of BERT model, approximation is used in GELU approximation and Attention fusion for cuda execution provider. There might be slight difference in result. The impact on accuracy could be neglected based on our evaluation: F1 score for a BERT model on SQuAD v1.1 is almost same (87.05 vs 87.03). ### Layout Optimizations diff --git a/docs/ONNX_Runtime_Perf_Tuning.md b/docs/ONNX_Runtime_Perf_Tuning.md index 2788ae041b85e..aaab8f8422fe5 100644 --- a/docs/ONNX_Runtime_Perf_Tuning.md +++ b/docs/ONNX_Runtime_Perf_Tuning.md @@ -86,9 +86,9 @@ sess_options.graph_optimization_level = rt.GraphOptimizationLevel.ORT_ENABLE_ALL * Sequential vs Parallel Execution * `sess_options.execution_mode = rt.ExecutionMode.ORT_SEQUENTIAL` controls whether then operators in the graph should run sequentially or in parallel. Usually when a model has many branches, setting this option to false will provide better performance. * When `sess_options.execution_mode = rt.ExecutionMode.ORT_PARALLEL`, you can set `sess_options.inter_op_num_threads` to control the - number of threads used to parallelize the execution of the graph (across nodes). -* sess_options.graph_optimization_level = rt.GraphOptimizationLevel.ORT_ENABLE_ALL. Default is ORT_ENABLE_BASIC(1). Please see [onnxruntime_c_api.h](../include/onnxruntime/core/session/onnxruntime_c_api.h#L241) (enum GraphOptimizationLevel) for the full list of all optimization levels. For details regarding available optimizations and usage please refer to the [Graph Optimizations Doc](../docs/ONNX_Runtime_Graph_Optimizations.md). + +* sess_options.graph_optimization_level = rt.GraphOptimizationLevel.ORT_ENABLE_ALL. Default is already ORT_ENABLE_ALL(99). Please see [onnxruntime_c_api.h](../include/onnxruntime/core/session/onnxruntime_c_api.h#L241) (enum GraphOptimizationLevel) for the full list of all optimization levels. For details regarding available optimizations and usage please refer to the [Graph Optimizations Doc](../docs/ONNX_Runtime_Graph_Optimizations.md). ### MKL_DNN/nGraph/MKL_ML Execution Provider MKL_DNN, MKL_ML and nGraph all depends on openmp for parallization. For those execution providers, we need to use the openmp enviroment variable to tune the performance. diff --git a/docs/execution_providers/MKL-DNN-ExecutionProvider.md b/docs/execution_providers/DNNL-ExecutionProvider.md similarity index 100% rename from docs/execution_providers/MKL-DNN-ExecutionProvider.md rename to docs/execution_providers/DNNL-ExecutionProvider.md diff --git a/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb b/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb index 2515dfed0bb45..2872f35781c9a 100644 --- a/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb +++ b/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb @@ -46,6 +46,7 @@ "outputs": [], "source": [ "import cpufeature\n", + "import hashlib\n", "import numpy as np\n", "import onnx\n", "from onnx import helper, numpy_helper\n", @@ -67,7 +68,20 @@ "\n", "def print_speedup(name, delta_baseline, delta):\n", " print(\"{} speed-up {:.2f}%\".format(name, 100*(delta_baseline/delta - 1)))\n", - " print(\" Baseline: {:.3f} s, Current: {:.3f} s\".format(delta_baseline, delta))" + " print(\" Baseline: {:.3f} s, Current: {:.3f} s\".format(delta_baseline, delta))\n", + "\n", + "def create_cache_dir(cache_dir):\n", + " # remove any stale cache files\n", + " if os.path.exists(cache_dir):\n", + " shutil.rmtree(cache_dir)\n", + " os.makedirs(cache_dir, exist_ok=True)\n", + "\n", + "def md5(file_name):\n", + " hash_md5 = hashlib.md5()\n", + " with open(file_name, \"rb\") as f:\n", + " for chunk in iter(lambda: f.read(4096), b\"\"):\n", + " hash_md5.update(chunk)\n", + " return hash_md5.hexdigest()" ] }, { @@ -86,7 +100,7 @@ "import onnxruntime\n", "from onnxruntime.nuphar.model_editor import convert_to_scan_model\n", "from onnxruntime.nuphar.model_quantizer import convert_matmul_model\n", - "from onnxruntime.nuphar.rnn_benchmark import generate_model\n", + "from onnxruntime.nuphar.rnn_benchmark import generate_model, perf_test\n", "from onnxruntime.nuphar.symbolic_shape_infer import SymbolicShapeInference" ] }, @@ -216,8 +230,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Fusion speed-up 436.78%\n", - " Baseline: 0.725 s, Current: 0.135 s\n" + "Fusion speed-up 445.44%\n", + " Baseline: 0.732 s, Current: 0.134 s\n" ] } ], @@ -226,18 +240,18 @@ "batch = 16\n", "input_data = np.random.rand(seq, batch, dim).astype(np.float32)\n", "sess = onnxruntime.InferenceSession(simple_model_name)\n", - "feed = {X:input_data}\n", - "output = sess.run([], feed)\n", + "simple_feed = {X:input_data}\n", + "simple_output = sess.run([], simple_feed)\n", "np_output = ((((input_data + input_data) * input_data) + input_data) * input_data) + input_data\n", - "assert np.allclose(output[0], np_output)\n", + "assert np.allclose(simple_output[0], np_output)\n", "\n", - "repeats = 100\n", + "simple_repeats = 100\n", "start_ort = timer()\n", - "for i in range(repeats):\n", - " output = sess.run([], feed)\n", + "for i in range(simple_repeats):\n", + " sess.run([], simple_feed)\n", "end_ort = timer()\n", "start_np = timer()\n", - "for i in range(repeats):\n", + "for i in range(simple_repeats):\n", " np_output = ((((input_data + input_data) * input_data) + input_data) * input_data) + input_data\n", "end_np = timer()\n", "print_speedup('Fusion', end_np - start_np, end_ort - start_ort)" @@ -271,7 +285,9 @@ "collapsed": true }, "source": [ - "**IMPORTANT**: Nuphar generates code before knowing shapes of input data, unlike other execution providers that do runtime shape inference. Thus, shape inference information is critical for compiler optimizations in Nuphar. To do that, we run symbolic shape inference on the model. Symbolic shape inference is based on the ONNX shape inference, and enhanced by sympy to better handle Shape/ConstantOfShape/etc. ops using symbolic computation." + "**IMPORTANT**: Nuphar generates code before knowing shapes of input data, unlike other execution providers that do runtime shape inference. Thus, shape inference information is critical for compiler optimizations in Nuphar. To do that, we run symbolic shape inference on the model. Symbolic shape inference is based on the ONNX shape inference, and enhanced by sympy to better handle Shape/ConstantOfShape/etc. ops using symbolic computation.\n", + "\n", + "**IMPORTANT**: When running multi-threaded inference, Nuphar currently uses TVM's parallel schedule with has its own thread pool that's compatible with OpenMP and MKLML. The TVM thread pool has not been integrated with ONNX runtime thread pool, so intra_op_num_threads won't control it. Please make sure the build is with OpenMP or MKLML, and use OMP_NUM_THREADS to control thread pool." ] }, { @@ -298,12 +314,11 @@ "metadata": {}, "outputs": [], "source": [ - "sess_baseline = onnxruntime.InferenceSession(lstm_model)\n", - "sess_baseline.set_providers(['CPUExecutionProvider']) # default provider in this container is Nuphar, this overrides to CPU EP\n", + "sess_baseline = onnxruntime.InferenceSession(lstm_model, providers=['CPUExecutionProvider'])\n", "seq = 128\n", "input_data = np.random.rand(seq, 1, input_dim).astype(np.float32)\n", - "feed = {sess_baseline.get_inputs()[0].name:input_data}\n", - "output = sess_baseline.run([], feed)" + "lstm_feed = {sess_baseline.get_inputs()[0].name:input_data}\n", + "lstm_output = sess_baseline.run([], lstm_feed)" ] }, { @@ -319,8 +334,8 @@ "metadata": {}, "outputs": [], "source": [ - "scan_model = 'Scan_LSTMx4.onnx'\n", - "convert_to_scan_model(lstm_model, scan_model)" + "lstm_scan_model = 'Scan_LSTMx4.onnx'\n", + "convert_to_scan_model(lstm_model, lstm_scan_model)" ] }, { @@ -339,28 +354,28 @@ "name": "stdout", "output_type": "stream", "text": [ - "Nuphar Scan speed-up 3.37%\n", - " Baseline: 3.099 s, Current: 2.998 s\n" + "Nuphar Scan speed-up 9.22%\n", + " Baseline: 3.070 s, Current: 2.811 s\n" ] } ], "source": [ - "sess_nuphar = onnxruntime.InferenceSession(scan_model)\n", - "output_nuphar = sess_nuphar.run([], feed)\n", - "assert np.allclose(output[0], output_nuphar[0])\n", + "sess_nuphar = onnxruntime.InferenceSession(lstm_scan_model)\n", + "output_nuphar = sess_nuphar.run([], lstm_feed)\n", + "assert np.allclose(lstm_output[0], output_nuphar[0])\n", "\n", - "repeats = 10\n", - "start_baseline = timer()\n", - "for i in range(repeats):\n", - " output = sess_baseline.run([], feed)\n", - "end_baseline = timer()\n", + "lstm_repeats = 10\n", + "start_lstm_baseline = timer()\n", + "for i in range(lstm_repeats):\n", + " sess_baseline.run([], lstm_feed)\n", + "end_lstm_baseline = timer()\n", "\n", "start_nuphar = timer()\n", - "for i in range(repeats):\n", - " output = sess_nuphar.run([], feed)\n", + "for i in range(lstm_repeats):\n", + " sess_nuphar.run([], lstm_feed)\n", "end_nuphar = timer()\n", "\n", - "print_speedup('Nuphar Scan', end_baseline - start_baseline, end_nuphar - start_nuphar)" + "print_speedup('Nuphar Scan', end_lstm_baseline - start_lstm_baseline, end_nuphar - start_nuphar)" ] }, { @@ -406,8 +421,8 @@ "metadata": {}, "outputs": [], "source": [ - "quantized_model = 'Scan_LSTMx4_int8.onnx'\n", - "convert_matmul_model(scan_model, quantized_model)" + "lstm_quantized_model = 'Scan_LSTMx4_int8.onnx'\n", + "convert_matmul_model(lstm_scan_model, lstm_quantized_model)" ] }, { @@ -423,9 +438,9 @@ "metadata": {}, "outputs": [], "source": [ - "sess_quantized = onnxruntime.InferenceSession(quantized_model)\n", - "output_quantized = sess_quantized.run([], feed)\n", - "assert np.allclose(output[0], output_quantized[0], rtol=1e-3, atol=1e-3)" + "sess_quantized = onnxruntime.InferenceSession(lstm_quantized_model)\n", + "output_quantized = sess_quantized.run([], lstm_feed)\n", + "assert np.allclose(lstm_output[0], output_quantized[0], rtol=1e-3, atol=1e-3)" ] }, { @@ -444,27 +459,64 @@ "name": "stdout", "output_type": "stream", "text": [ - "Quantization speed-up 299.66%\n", - " Baseline: 2.998 s, Current: 0.750 s\n" + "Quantization speed-up 722.57%\n", + " Baseline: 2.811 s, Current: 0.342 s\n" ] } ], "source": [ "start_quantized = timer()\n", - "for i in range(repeats):\n", - " output = sess_quantized.run([], feed)\n", + "for i in range(lstm_repeats):\n", + " sess_quantized.run([], lstm_feed)\n", "end_quantized = timer()\n", "\n", "print_speedup('Quantization', end_nuphar - start_nuphar, end_quantized - start_quantized)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To check RNN quantization performance, please use rnn_benchmark.perf_test." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perf_rnn (with default threads) lstm_i80_h512_bi_l6_no_batch.onnx: run for 348 iterations, top 5 avg 28.368 ms\n", + "perf_scan (with 4 threads) lstm_i80_h512_bi_l6_no_batch_scan.onnx: run for 373 iterations, top 5 avg 25.819 ms\n", + "perf_int8 (with 4 threads) lstm_i80_h512_bi_l6_no_batch_int8.onnx: run for 778 iterations, top 5 avg 12.534 ms\n", + "Nuphar Quantization speed up speed-up 126.33%\n", + " Baseline: 0.028 s, Current: 0.013 s\n" + ] + } + ], + "source": [ + "rnn_type = 'lstm' # could be 'lstm', 'gru' or 'rnn'\n", + "num_threads = cpufeature.CPUFeature['num_physical_cores'] # no hyper thread\n", + "input_dim = 80 # size of input dimension\n", + "hidden_dim = 512 # size of hidden dimension in cell\n", + "bidirectional = True # specify RNN being bidirectional\n", + "layers = 6 # number of stacked RNN layers\n", + "seq_len = 40 # length of sequence\n", + "batch_size = 1 # size of batch\n", + "original_ms, scan_ms, int8_ms = perf_test(rnn_type, num_threads, input_dim, hidden_dim, bidirectional, layers, seq_len, batch_size)\n", + "print_speedup('Nuphar Quantization speed up', original_ms / 1000, int8_ms / 1000)" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 5. Working on real models\n", "\n", - "### BERT Squad\n", + "### 5.1 BERT Squad\n", "\n", "BERT (Bidirectional Encoder Representations from Transformers) applies Transformers to language modelling. With Nuphar, we may fuse and compile the model to accelerate inference on CPU.\n", "\n", @@ -473,17 +525,17 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "# download BERT squad model\n", "cwd = os.getcwd()\n", - "model_url = 'https://onnxzoo.blob.core.windows.net/models/opset_10/bert_squad/download_sample_10.tar.gz'\n", - "model_local = os.path.join(cwd, 'download_sample_10.tar.gz')\n", - "if not os.path.exists(model_local):\n", - " urllib.request.urlretrieve(model_url, model_local)\n", - "with tarfile.open(model_local, 'r') as f:\n", + "bert_model_url = 'https://onnxzoo.blob.core.windows.net/models/opset_10/bert_squad/download_sample_10.tar.gz'\n", + "bert_model_local = os.path.join(cwd, 'download_sample_10.tar.gz')\n", + "if not os.path.exists(bert_model_local):\n", + " urllib.request.urlretrieve(bert_model_url, bert_model_local)\n", + "with tarfile.open(bert_model_local, 'r') as f:\n", " f.extractall(cwd)" ] }, @@ -497,16 +549,16 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ - "model_dir = os.path.join(cwd, 'download_sample_10')\n", - "model = os.path.join(model_dir, 'bertsquad10.onnx')\n", - "model_with_shape_inference = os.path.join(model_dir, 'bertsquad10_shaped.onnx')\n", + "bert_model_dir = os.path.join(cwd, 'download_sample_10')\n", + "bert_model = os.path.join(bert_model_dir, 'bertsquad10.onnx')\n", + "bert_model_with_shape_inference = os.path.join(bert_model_dir, 'bertsquad10_shaped.onnx')\n", "\n", "# run symbolic shape inference\n", - "SymbolicShapeInference.infer_shapes(model, model_with_shape_inference, auto_merge=True, int_max=100000)" + "SymbolicShapeInference.infer_shapes(bert_model, bert_model_with_shape_inference, auto_merge=True, int_max=100000)" ] }, { @@ -518,26 +570,25 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "sess_options = onnxruntime.SessionOptions()\n", "sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL\n", - "sess_baseline = onnxruntime.InferenceSession(model, sess_options)\n", - "sess_baseline.set_providers(['CPUExecutionProvider'])\n", + "sess_baseline = onnxruntime.InferenceSession(bert_model, sess_options=sess_options, providers=['CPUExecutionProvider'])\n", "\n", "# load test data\n", - "test_data_dir = os.path.join(model_dir, 'test_data_set_1')\n", + "test_data_dir = os.path.join(bert_model_dir, 'test_data_set_1')\n", "tps = [onnx.load_tensor(os.path.join(test_data_dir, 'input_{}.pb'.format(i))) for i in range(len(sess_baseline.get_inputs()))]\n", - "feed = {tp.name:numpy_helper.to_array(tp) for tp in tps}\n", - "output_baseline = sess_baseline.run([], feed)\n", + "bert_feed = {tp.name:numpy_helper.to_array(tp) for tp in tps}\n", + "bert_output_baseline = sess_baseline.run([], bert_feed)\n", "\n", - "repeats = 20\n", - "start_baseline = timer()\n", - "for i in range(repeats):\n", - " output = sess_baseline.run([], feed)\n", - "end_baseline = timer()" + "bert_repeats = 20\n", + "start_bert_baseline = timer()\n", + "for i in range(bert_repeats):\n", + " sess_baseline.run([], bert_feed)\n", + "end_bert_baseline = timer()" ] }, { @@ -550,13 +601,13 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ - "sess = onnxruntime.InferenceSession(model_with_shape_inference)\n", - "output = sess.run([], feed)\n", - "assert all([np.allclose(o, ob, atol=1e-4) for o, ob in zip(output, output_baseline)])" + "sess = onnxruntime.InferenceSession(bert_model_with_shape_inference)\n", + "output = sess.run([], bert_feed)\n", + "assert all([np.allclose(o, ob, atol=1e-4) for o, ob in zip(output, bert_output_baseline)])" ] }, { @@ -568,32 +619,32 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Nuphar BERT squad speed-up 60.13%\n", - " Baseline: 4.928 s, Current: 3.077 s\n" + "Nuphar BERT squad speed-up 66.70%\n", + " Baseline: 5.093 s, Current: 3.055 s\n" ] } ], "source": [ "start_nuphar = timer()\n", - "for i in range(repeats):\n", - " output = sess.run([], feed)\n", + "for i in range(bert_repeats):\n", + " sess.run([], bert_feed)\n", "end_nuphar = timer()\n", "\n", - "print_speedup('Nuphar BERT squad', end_baseline - start_baseline, end_nuphar - start_nuphar)" + "print_speedup('Nuphar BERT squad', end_bert_baseline - start_bert_baseline, end_nuphar - start_nuphar)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### GPT-2 with fixed batch size\n", + "### 5.2 GPT-2 with fixed batch size\n", "GPT-2 is a language model using Generative Pre-Trained Transformer for text generation. With Nuphar, we may fuse and compile the model to accelerate inference on CPU.\n", "\n", "#### Download model and test data" @@ -601,17 +652,17 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "# download GPT-2 model\n", "cwd = os.getcwd()\n", - "model_url = 'https://onnxzoo.blob.core.windows.net/models/opset_10/GPT2/GPT-2.tar.gz'\n", - "model_local = os.path.join(cwd, 'GPT-2.tar.gz')\n", - "if not os.path.exists(model_local):\n", - " urllib.request.urlretrieve(model_url, model_local)\n", - "with tarfile.open(model_local, 'r') as f:\n", + "gpt2_model_url = 'https://onnxzoo.blob.core.windows.net/models/opset_10/GPT2/GPT-2.tar.gz'\n", + "gpt2_model_local = os.path.join(cwd, 'GPT-2.tar.gz')\n", + "if not os.path.exists(gpt2_model_local):\n", + " urllib.request.urlretrieve(gpt2_model_url, gpt2_model_local)\n", + "with tarfile.open(gpt2_model_local, 'r') as f:\n", " f.extractall(cwd)" ] }, @@ -625,22 +676,22 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ - "model_dir = os.path.join(cwd, 'GPT2')\n", - "model = os.path.join(model_dir, 'model.onnx')\n", + "gpt2_model_dir = os.path.join(cwd, 'GPT2')\n", + "gpt2_model = os.path.join(gpt2_model_dir, 'model.onnx')\n", "\n", "# edit batch dimension from symbolic to int value for better codegen\n", - "mp = onnx.load(model)\n", + "mp = onnx.load(gpt2_model)\n", "mp.graph.input[0].type.tensor_type.shape.dim[0].dim_value = 1\n", - "onnx.save(mp, model)\n", + "onnx.save(mp, gpt2_model)\n", "\n", - "model_with_shape_inference = os.path.join(model_dir, 'model_shaped.onnx')\n", + "gpt2_model_with_shape_inference = os.path.join(gpt2_model_dir, 'model_shaped.onnx')\n", "\n", "# run symbolic shape inference\n", - "SymbolicShapeInference.infer_shapes(model, model_with_shape_inference, auto_merge=True)" + "SymbolicShapeInference.infer_shapes(gpt2_model, gpt2_model_with_shape_inference, auto_merge=True)" ] }, { @@ -652,54 +703,53 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Nuphar GPT-2 speed-up 13.48%\n", - " Baseline: 2.535 s, Current: 2.234 s\n" + "Nuphar GPT-2 speed-up 16.15%\n", + " Baseline: 2.671 s, Current: 2.299 s\n" ] } ], "source": [ "sess_options = onnxruntime.SessionOptions()\n", "sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL\n", - "sess_baseline = onnxruntime.InferenceSession(model, sess_options)\n", - "sess_baseline.set_providers(['CPUExecutionProvider'])\n", + "sess_baseline = onnxruntime.InferenceSession(gpt2_model, sess_options=sess_options, providers=['CPUExecutionProvider'])\n", "\n", "# load test data, note the tensor proto name in data does not match model, so override it in feed\n", "input_name = [i.name for i in sess_baseline.get_inputs()][0] # This model only has one input\n", - "test_data_dir = os.path.join(model_dir, 'test_data_set_0')\n", + "test_data_dir = os.path.join(gpt2_model_dir, 'test_data_set_0')\n", "tp = onnx.load_tensor(os.path.join(test_data_dir, 'input_0.pb'))\n", - "feed = {input_name:numpy_helper.to_array(tp).reshape(1,-1)} # the test data missed batch dimension\n", - "output_baseline = sess_baseline.run([], feed)\n", + "gpt2_feed = {input_name:numpy_helper.to_array(tp).reshape(1,-1)} # the test data missed batch dimension\n", + "gpt2_output_baseline = sess_baseline.run([], gpt2_feed)\n", "\n", - "repeats = 100\n", - "start_baseline = timer()\n", - "for i in range(repeats):\n", - " output = sess_baseline.run([], feed)\n", - "end_baseline = timer()\n", + "gpt2_repeats = 100\n", + "start_gpt2_baseline = timer()\n", + "for i in range(gpt2_repeats):\n", + " sess_baseline.run([], gpt2_feed)\n", + "end_gpt2_baseline = timer()\n", "\n", - "sess = onnxruntime.InferenceSession(model_with_shape_inference)\n", - "output = sess.run([], feed)\n", - "assert all([np.allclose(o, ob, atol=1e-4) for o, ob in zip(output, output_baseline)])\n", + "sess = onnxruntime.InferenceSession(gpt2_model_with_shape_inference)\n", + "output = sess.run([], gpt2_feed)\n", + "assert all([np.allclose(o, ob, atol=1e-4) for o, ob in zip(output, gpt2_output_baseline)])\n", "\n", "start_nuphar = timer()\n", - "for i in range(repeats):\n", - " output = sess.run([], feed)\n", + "for i in range(gpt2_repeats):\n", + " output = sess.run([], gpt2_feed)\n", "end_nuphar = timer()\n", "\n", - "print_speedup('Nuphar GPT-2', end_baseline - start_baseline, end_nuphar - start_nuphar)" + "print_speedup('Nuphar GPT-2', end_gpt2_baseline - start_gpt2_baseline, end_nuphar - start_nuphar)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### BiDAF with quantization\n", + "### 5.3 BiDAF with quantization\n", "\n", "BiDAF is a machine comprehension model that uses LSTMs. The inputs to this model are paragraphs of contexts and queries, and the outputs are start/end indices of words in the contexts that answers the queries.\n", "\n", @@ -708,7 +758,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -731,18 +781,18 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ - "bidaf = os.path.join(cwd, 'bidaf', 'bidaf.onnx')\n", - "sess_baseline = onnxruntime.InferenceSession(bidaf)\n", - "sess_baseline.set_providers(['CPUExecutionProvider'])\n", + "bidaf_dir = os.path.join(cwd, 'bidaf')\n", + "bidaf = os.path.join(bidaf_dir, 'bidaf.onnx')\n", + "sess_baseline = onnxruntime.InferenceSession(bidaf, providers=['CPUExecutionProvider'])\n", "# load test data\n", "test_data_dir = os.path.join(cwd, 'bidaf', 'test_data_set_3')\n", "tps = [onnx.load_tensor(os.path.join(test_data_dir, 'input_{}.pb'.format(i))) for i in range(len(sess_baseline.get_inputs()))]\n", - "feed = {tp.name:numpy_helper.to_array(tp) for tp in tps}\n", - "output_baseline = sess_baseline.run([], feed)" + "bidaf_feed = {tp.name:numpy_helper.to_array(tp) for tp in tps}\n", + "bidaf_output_baseline = sess_baseline.run([], bidaf_feed)" ] }, { @@ -754,7 +804,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -763,13 +813,13 @@ "\"with 4:51 left in regulation , carolina got the ball on their own 24 - yard line with a chance to mount a game - winning drive , and soon faced 3rd - and - 9 . on the next play , miller stripped the ball away from newton , and after several players dove for it , it took a long bounce backwards and was recovered by ward , who returned it five yards to the panthers 4 - yard line . although several players dove into the pile to attempt to recover it , newton did not and his lack of aggression later earned him heavy criticism . meanwhile , denver ' s offense was kept out of the end zone for three plays , but a holding penalty on cornerback josh norman gave the broncos a new set of downs . then anderson scored on a 2 - yard touchdown run and manning completed a pass to bennie fowler for a 2 - point conversion , giving denver a 24 – 10 lead with 3:08 left and essentially putting the game away . carolina had two more drives , but failed to get a first down on each one .\"" ] }, - "execution_count": 26, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "' '.join(list(feed['context_word'].reshape(-1)))" + "' '.join(list(bidaf_feed['context_word'].reshape(-1)))" ] }, { @@ -781,7 +831,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -790,13 +840,13 @@ "'who recovered the strip ball ?'" ] }, - "execution_count": 27, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "' '.join(list(feed['query_word'].reshape(-1)))" + "' '.join(list(bidaf_feed['query_word'].reshape(-1)))" ] }, { @@ -808,7 +858,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -817,13 +867,13 @@ "'ward'" ] }, - "execution_count": 28, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "' '.join(list(feed['context_word'][output_baseline[0][0]:output_baseline[1][0]+1].reshape(-1)))" + "' '.join(list(bidaf_feed['context_word'][bidaf_output_baseline[0][0]:bidaf_output_baseline[1][0]+1].reshape(-1)))" ] }, { @@ -835,7 +885,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -851,8 +901,8 @@ "\n", "# inference and verify accuracy\n", "sess = onnxruntime.InferenceSession(bidaf_converted)\n", - "output = sess.run([], feed)\n", - "assert all([np.allclose(o, ob) for o, ob in zip(output, output_baseline)])" + "output = sess.run([], bidaf_feed)\n", + "assert all([np.allclose(o, ob) for o, ob in zip(output, bidaf_output_baseline)])" ] }, { @@ -864,30 +914,31 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Nuphar quantized BiDAF speed-up 47.77%\n", - " Baseline: 1.564 s, Current: 1.058 s\n" + "Nuphar quantized BiDAF speed-up 43.79%\n", + " Baseline: 1.517 s, Current: 1.055 s\n" ] } ], "source": [ - "start_baseline = timer()\n", - "for i in range(repeats):\n", - " output = sess_baseline.run([], feed)\n", - "end_baseline = timer()\n", + "bidaf_repeats = 100\n", + "start_bidaf_baseline = timer()\n", + "for i in range(bidaf_repeats):\n", + " sess_baseline.run([], bidaf_feed)\n", + "end_bidaf_baseline = timer()\n", "\n", "start_nuphar = timer()\n", - "for i in range(repeats):\n", - " output = sess.run([], feed)\n", + "for i in range(bidaf_repeats):\n", + " sess.run([], bidaf_feed)\n", "end_nuphar = timer()\n", "\n", - "print_speedup('Nuphar quantized BiDAF', end_baseline - start_baseline, end_nuphar - start_nuphar)" + "print_speedup('Nuphar quantized BiDAF', end_bidaf_baseline - start_bidaf_baseline, end_nuphar - start_nuphar)" ] }, { @@ -907,16 +958,16 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'JIT took 4.721 seconds'" + "'JIT took 5.749 seconds'" ] }, - "execution_count": 31, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -930,20 +981,16 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 33, "metadata": { "scrolled": true }, "outputs": [], "source": [ - "# create a folder for JIT cache\n", - "cache_dir = os.path.join(cwd, 'bidaf_cache')\n", - "# remove any stale cache files\n", - "if os.path.exists(cache_dir):\n", - " shutil.rmtree(cache_dir)\n", - "os.makedirs(cache_dir, exist_ok=True)\n", "# use settings to enable JIT cache\n", - "settings = 'nuphar_cache_path:{}'.format(cache_dir)\n", + "bidaf_cache_dir = os.path.join(bidaf_dir, 'cache')\n", + "create_cache_dir(bidaf_cache_dir)\n", + "settings = 'nuphar_cache_path:{}'.format(bidaf_cache_dir)\n", "onnxruntime.capi._pybind_state.set_nuphar_settings(settings)\n", "sess = onnxruntime.InferenceSession(bidaf_converted)" ] @@ -957,7 +1004,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -966,17 +1013,16 @@ "['jit.so']" ] }, - "execution_count": 33, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "cache_versioned_dir = os.path.join(cache_dir, os.listdir(cache_dir)[0])\n", + "bidaf_cache_versioned_dir = os.path.join(bidaf_cache_dir, os.listdir(bidaf_cache_dir)[0])\n", "# use onnxruntime.nuphar.create_shared module to create dll\n", - "onnxruntime_dir = os.path.split(os.path.abspath(onnxruntime.__file__))[0]\n", - "subprocess.run([sys.executable, '-m', 'onnxruntime.nuphar.create_shared', '--input_dir', cache_versioned_dir], check=True)\n", - "os.listdir(cache_versioned_dir)" + "subprocess.run([sys.executable, '-m', 'onnxruntime.nuphar.create_shared', '--input_dir', bidaf_cache_versioned_dir], check=True)\n", + "os.listdir(bidaf_cache_versioned_dir)" ] }, { @@ -988,28 +1034,95 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "AOT speed-up 764.62%\n", - " Baseline: 4.721 s, Current: 0.546 s\n" + "AOT speed-up 694.06%\n", + " Baseline: 5.749 s, Current: 0.724 s\n" ] } ], "source": [ "start_aot = timer()\n", "# NOTE: Nuphar settings string is not sticky. It needs to be reset before creating InferenceSession\n", - "settings = 'nuphar_cache_path:{}'.format(cache_dir)\n", + "settings = 'nuphar_cache_path:{}'.format(bidaf_cache_dir)\n", "onnxruntime.capi._pybind_state.set_nuphar_settings(settings)\n", "sess = onnxruntime.InferenceSession(bidaf_converted)\n", "end_aot = timer()\n", "print_speedup('AOT', end_jit - start_jit, end_aot - start_aot)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Moreover, Nuphar AOT also supports:\n", + "* Generate JIT cache with AVX/AVX2/AVX-512 and build a AOT dll including support for all these CPUs, which makes deployment easier when targeting different CPUs in one package.\n", + "* Bake model checksum into AOT dll to validate model with given AOT dll." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Scan_LSTMx4_int8.onnx in avx2 speed-up 790.69%\n", + " Baseline: 3.070 s, Current: 0.345 s\n", + "Scan_LSTMx4_int8.onnx in avx speed-up 361.68%\n", + " Baseline: 3.070 s, Current: 0.665 s\n" + ] + } + ], + "source": [ + "# create object files for different CPUs\n", + "cache_dir = os.path.join(os.getcwd(), 'lstm_cache')\n", + "model_name = lstm_quantized_model\n", + "model_checksum = md5(model_name)\n", + "repeats = lstm_repeats\n", + "feed = lstm_feed\n", + "time_baseline = end_lstm_baseline - start_lstm_baseline\n", + "multi_isa_so = 'avx_avx2_avx512.so'\n", + "\n", + "create_cache_dir(cache_dir)\n", + "settings = 'nuphar_cache_path:{}'.format(cache_dir)\n", + "for isa in ['avx512', 'avx2', 'avx']:\n", + " settings_with_isa = settings + ', nuphar_codegen_target:' + isa\n", + " onnxruntime.capi._pybind_state.set_nuphar_settings(settings_with_isa)\n", + " sess = onnxruntime.InferenceSession(model_name)\n", + " cache_versioned_dir = os.path.join(cache_dir, os.listdir(cache_dir)[0])\n", + "\n", + "# link object files to AOT dll\n", + "subprocess.run([sys.executable, '-m', 'onnxruntime.nuphar.create_shared', '--input_dir', cache_versioned_dir, '--input_model', model_name, '--output_name', multi_isa_so], check=True)\n", + "\n", + "# now load the model with AOT dll\n", + "# NOTE: when nuphar_codegen_target is not set, it defaults to current CPU ISA\n", + "settings = 'nuphar_cache_path:{}, nuphar_cache_so_name:{}, nuphar_cache_model_checksum:{}, nuphar_cache_force_no_jit:on'.format(cache_dir, multi_isa_so, model_checksum)\n", + "onnxruntime.capi._pybind_state.set_nuphar_settings(settings)\n", + "sess = onnxruntime.InferenceSession(model_name)\n", + "\n", + "# force to a different ISA which is a subset of current CPU\n", + "# NOTE: if an incompatible ISA is used, exception on invalid instructions would be thrown\n", + "for valid_isa in ['avx2', 'avx']:\n", + " settings_with_isa = 'nuphar_cache_path:{}, nuphar_cache_so_name:{}, nuphar_cache_model_checksum:{}, nuphar_codegen_target:{}, nuphar_cache_force_no_jit:on'.format(cache_dir, multi_isa_so, model_checksum, valid_isa)\n", + " onnxruntime.capi._pybind_state.set_nuphar_settings(settings_with_isa)\n", + " sess = onnxruntime.InferenceSession(model_name)\n", + "\n", + " start_nuphar = timer()\n", + " for i in range(repeats):\n", + " sess.run([], feed)\n", + " end_nuphar = timer()\n", + "\n", + " print_speedup('{} in {}'.format(model_name, valid_isa), time_baseline, end_nuphar - start_nuphar)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1020,15 +1133,15 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Single thread perf w/o parallel schedule speed-up 1.05%\n", - " Baseline: 1.542 s, Current: 1.526 s\n" + "Single thread perf w/o parallel schedule speed-up 1.49%\n", + " Baseline: 1.540 s, Current: 1.518 s\n" ] } ], @@ -1039,8 +1152,8 @@ "\n", "sess = onnxruntime.InferenceSession(bidaf_converted)\n", "start_baseline = timer()\n", - "for i in range(repeats):\n", - " output_baseline = sess_baseline.run([], feed)\n", + "for i in range(bidaf_repeats):\n", + " sess_baseline.run([], bidaf_feed)\n", "end_baseline = timer()\n", "\n", "# use NUPHAR_PARALLEL_MIN_WORKLOADS=0 to turn off parallel schedule, using settings string\n", @@ -1050,10 +1163,12 @@ "sess = onnxruntime.InferenceSession(bidaf_converted)\n", "\n", "start = timer()\n", - "for i in range(repeats):\n", - " output = sess_baseline.run([], feed)\n", + "for i in range(bidaf_repeats):\n", + " sess_baseline.run([], bidaf_feed)\n", "end = timer()\n", - "print_speedup('Single thread perf w/o parallel schedule', end_baseline - start_baseline, end - start)" + "print_speedup('Single thread perf w/o parallel schedule', end_baseline - start_baseline, end - start)\n", + "\n", + "del os.environ['OMP_NUM_THREADS']" ] } ], diff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.h index f6a31494768c8..1fdcdfc02e313 100644 --- a/include/onnxruntime/core/framework/allocator.h +++ b/include/onnxruntime/core/framework/allocator.h @@ -85,26 +85,27 @@ struct OrtMemoryInfo { // use string for name, so we could have customized allocator in execution provider. const char* name; - int id; - OrtMemType mem_type; - OrtAllocatorType type; + int id = -1; + OrtMemType mem_type = OrtMemTypeDefault; + OrtAllocatorType alloc_type = Invalid; OrtDevice device; - constexpr OrtMemoryInfo(const char* name_, OrtAllocatorType type_, OrtDevice device_ = OrtDevice(), int id_ = 0, OrtMemType mem_type_ = OrtMemTypeDefault) + constexpr OrtMemoryInfo(const char* name_, OrtAllocatorType type_, OrtDevice device_ = OrtDevice(), int id_ = 0, + OrtMemType mem_type_ = OrtMemTypeDefault) #if (defined(__GNUC__) || defined(__clang__)) __attribute__((nonnull)) #endif : name(name_), id(id_), mem_type(mem_type_), - type(type_), + alloc_type(type_), device(device_) { } // To make OrtMemoryInfo become a valid key in std map - inline bool operator<(const OrtMemoryInfo& other) const { - if (type != other.type) - return type < other.type; + bool operator<(const OrtMemoryInfo& other) const { + if (alloc_type != other.alloc_type) + return alloc_type < other.alloc_type; if (mem_type != other.mem_type) return mem_type < other.mem_type; if (id != other.id) @@ -113,20 +114,22 @@ struct OrtMemoryInfo { return strcmp(name, other.name) < 0; } - inline std::string ToString() const { + std::string ToString() const { std::ostringstream ostr; ostr << "OrtMemoryInfo: [" << " name:" << name << " id:" << id << " mem_type:" << mem_type - << " type:" << type + << " alloc_type:" << alloc_type << "]"; return ostr.str(); } }; inline bool operator==(const OrtMemoryInfo& left, const OrtMemoryInfo& other) { - return left.mem_type == other.mem_type && left.type == other.type && left.id == other.id && + return left.mem_type == other.mem_type && + left.alloc_type == other.alloc_type && + left.id == other.id && strcmp(left.name, other.name) == 0; } @@ -213,9 +216,11 @@ class IAllocator { if (!std::is_void::value) { // sizeof(void) isn't valid, but the compiler isn't smart enough to ignore that this line isn't // reachable if T is void. use std::conditional to 'use' void* in the sizeof call - if (!CalcMemSizeForArray(count_or_bytes, sizeof(typename std::conditional::value, void*, T>::type), + if (!CalcMemSizeForArray(count_or_bytes, + sizeof(typename std::conditional::value, void*, T>::type), &alloc_size)) return nullptr; } + return IAllocatorUniquePtr{ static_cast(allocator->Alloc(alloc_size)), // allocate [=](T* ptr) { allocator->Free(ptr); }}; // capture IAllocator so it's always valid, and use as deleter @@ -224,22 +229,26 @@ class IAllocator { template bool IAllocator::CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t* out) noexcept { - static constexpr size_t max_allowed = (static_cast(1) << (static_cast(std::numeric_limits::digits >> 1))) - alignment; + static constexpr size_t max_allowed = (size_t(1) << (size_t(std::numeric_limits::digits >> 1))) - alignment; static constexpr size_t max_size = std::numeric_limits::max() - alignment; static constexpr size_t alignment_mask = alignment - 1; + //Indeed, we only need to check if max_size / nmemb < size //max_allowed is for avoiding unnecessary DIV. if (nmemb >= max_allowed && max_size / nmemb < size) { return false; } + if (size >= max_allowed && nmemb > 0 && max_size / nmemb < size) { return false; } + if (alignment == 0) *out = size * nmemb; else *out = (size * nmemb + alignment_mask) & ~static_cast(alignment_mask); + return true; } @@ -297,11 +306,10 @@ class MiMallocAllocator : public IDeviceAllocator { #endif - #ifdef USE_MIMALLOC - using TAllocator = MiMallocAllocator; +using TAllocator = MiMallocAllocator; #else - using TAllocator = CPUAllocator; +using TAllocator = CPUAllocator; #endif using AllocatorPtr = std::shared_ptr; diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index a9193c2fbf096..176b988ad0f3b 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -84,12 +84,12 @@ typedef enum ONNXTensorElementDataType { ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, // maps to c++ type std::string ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16, - ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 // Non-IEEE floating-point format based on IEEE754 single-precision + ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 // Non-IEEE floating-point format based on IEEE754 single-precision } ONNXTensorElementDataType; // Synced with onnx TypeProto oneof @@ -193,6 +193,7 @@ struct OrtCustomOp; typedef struct OrtCustomOp OrtCustomOp; typedef enum OrtAllocatorType { + Invalid = -1, OrtDeviceAllocator = 0, OrtArenaAllocator = 1 } OrtAllocatorType; @@ -234,14 +235,14 @@ struct OrtApi { const char*(ORT_API_CALL* GetErrorMessage)(_In_ const OrtStatus* status)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; /** - * \param out Should be freed by `OrtReleaseEnv` after use - */ + * \param out Should be freed by `OrtReleaseEnv` after use + */ OrtStatus*(ORT_API_CALL* CreateEnv)(OrtLoggingLevel default_logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; /** - * \param out Should be freed by `OrtReleaseEnv` after use - */ + * \param out Should be freed by `OrtReleaseEnv` after use + */ OrtStatus*(ORT_API_CALL* CreateEnvWithCustomLogger)(OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel default_warning_level, _In_ const char* logid, @@ -253,7 +254,7 @@ struct OrtApi { // TODO: document the path separator convention? '/' vs '\' // TODO: should specify the access characteristics of model_path. Is this read only during the - // execution of OrtCreateSession, or does the OrtSession retain a handle to the file/directory + // execution of CreateSession, or does the OrtSession retain a handle to the file/directory // and continue to access throughout the OrtSession lifetime? // What sort of access is needed to model_path : read or read/write? OrtStatus*(ORT_API_CALL* CreateSession)(_In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, @@ -268,8 +269,8 @@ struct OrtApi { _In_ const char* const* output_names, size_t output_names_len, _Outptr_ OrtValue** output)NO_EXCEPTION; /** - * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseSessionOptions after use - */ + * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseSessionOptions after use + */ OrtStatus*(ORT_API_CALL* CreateSessionOptions)(_Outptr_ OrtSessionOptions** options)NO_EXCEPTION; // Set filepath to save optimized model after graph level transformations. @@ -325,36 +326,36 @@ struct OrtApi { OrtStatus*(ORT_API_CALL* CreateCustomOpDomain)(_In_ const char* domain, _Outptr_ OrtCustomOpDomain** out)NO_EXCEPTION; /* - * Add custom ops to the OrtCustomOpDomain - * Note: The OrtCustomOp* pointer must remain valid until the OrtCustomOpDomain using it is released - */ + * Add custom ops to the OrtCustomOpDomain + * Note: The OrtCustomOp* pointer must remain valid until the OrtCustomOpDomain using it is released + */ OrtStatus*(ORT_API_CALL* CustomOpDomain_Add)(_Inout_ OrtCustomOpDomain* custom_op_domain, _In_ OrtCustomOp* op)NO_EXCEPTION; /* - * Add a custom op domain to the OrtSessionOptions - * Note: The OrtCustomOpDomain* must not be deleted until the sessions using it are released - */ + * Add a custom op domain to the OrtSessionOptions + * Note: The OrtCustomOpDomain* must not be deleted until the sessions using it are released + */ OrtStatus*(ORT_API_CALL* AddCustomOpDomain)(_Inout_ OrtSessionOptions* options, _In_ OrtCustomOpDomain* custom_op_domain)NO_EXCEPTION; /* - * Loads a DLL named 'library_path' and looks for this entry point: - * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); - * It then passes in the provided session options to this function along with the api base. - * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in - * session options are destroyed, or if an error occurs and it is non null. + * Loads a DLL named 'library_path' and looks for this entry point: + * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); + * It then passes in the provided session options to this function along with the api base. + * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in + * session options are destroyed, or if an error occurs and it is non null. */ OrtStatus*(ORT_API_CALL* RegisterCustomOpsLibrary)(_Inout_ OrtSessionOptions* options, _In_ const char* library_path, void** library_handle)NO_EXCEPTION; /** - * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these - * functions to enable them in the session: - * OrtSessionOptionsAppendExecutionProvider_CPU - * OrtSessionOptionsAppendExecutionProvider_CUDA - * OrtSessionOptionsAppendExecutionProvider_ - * The order they care called indicates the preference order as well. In other words call this method - * on your most preferred execution provider first followed by the less preferred ones. - * If none are called Ort will use its internal CPU execution provider. - */ + * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these + * functions to enable them in the session: + * OrtSessionOptionsAppendExecutionProvider_CPU + * OrtSessionOptionsAppendExecutionProvider_CUDA + * OrtSessionOptionsAppendExecutionProvider_ + * The order they care called indicates the preference order as well. In other words call this method + * on your most preferred execution provider first followed by the less preferred ones. + * If none are called Ort will use its internal CPU execution provider. + */ OrtStatus*(ORT_API_CALL* SessionGetInputCount)(_In_ const OrtSession* sess, _Out_ size_t* out)NO_EXCEPTION; OrtStatus*(ORT_API_CALL* SessionGetOutputCount)(_In_ const OrtSession* sess, _Out_ size_t* out)NO_EXCEPTION; @@ -432,45 +433,45 @@ struct OrtApi { OrtStatus*(ORT_API_CALL* GetTensorMutableData)(_Inout_ OrtValue* value, _Outptr_ void** out)NO_EXCEPTION; /** - * \param value A tensor created from OrtCreateTensor... function. - * \param s each A string array. Each string in this array must be null terminated. - * \param s_len length of s - */ + * \param value A tensor created from OrtCreateTensor... function. + * \param s each A string array. Each string in this array must be null terminated. + * \param s_len length of s + */ OrtStatus*(ORT_API_CALL* FillStringTensor)(_Inout_ OrtValue* value, _In_ const char* const* s, size_t s_len)NO_EXCEPTION; /** - * \param value A tensor created from OrtCreateTensor... function. - * \param len total data length, not including the trailing '\0' chars. - */ + * \param value A tensor created from OrtCreateTensor... function. + * \param len total data length, not including the trailing '\0' chars. + */ OrtStatus*(ORT_API_CALL* GetStringTensorDataLength)(_In_ const OrtValue* value, _Out_ size_t* len)NO_EXCEPTION; /** - * \param s string contents. Each string is NOT null-terminated. - * \param value A tensor created from OrtCreateTensor... function. - * \param s_len total data length, get it from OrtGetStringTensorDataLength - */ + * \param s string contents. Each string is NOT null-terminated. + * \param value A tensor created from OrtCreateTensor... function. + * \param s_len total data length, get it from OrtGetStringTensorDataLength + */ OrtStatus*(ORT_API_CALL* GetStringTensorContent)(_In_ const OrtValue* value, _Out_ void* s, size_t s_len, _Out_ size_t* offsets, size_t offsets_len)NO_EXCEPTION; /** - * Don't free the 'out' value - */ + * Don't free the 'out' value + */ OrtStatus*(ORT_API_CALL* CastTypeInfoToTensorInfo)(_In_ const OrtTypeInfo*, _Out_ const OrtTensorTypeAndShapeInfo** out)NO_EXCEPTION; /** - * Return OnnxType from OrtTypeInfo - */ + * Return OnnxType from OrtTypeInfo + */ OrtStatus*(ORT_API_CALL* GetOnnxTypeFromTypeInfo)(_In_ const OrtTypeInfo*, _Out_ enum ONNXType* out)NO_EXCEPTION; /** - * The 'out' value should be released by calling OrtReleaseTensorTypeAndShapeInfo - */ + * The 'out' value should be released by calling OrtReleaseTensorTypeAndShapeInfo + */ OrtStatus*(ORT_API_CALL* CreateTensorTypeAndShapeInfo)(_Outptr_ OrtTensorTypeAndShapeInfo** out)NO_EXCEPTION; OrtStatus*(ORT_API_CALL* SetTensorElementType)(_Inout_ OrtTensorTypeAndShapeInfo*, enum ONNXTensorElementDataType type)NO_EXCEPTION; /** - * \param info Created from OrtCreateTensorTypeAndShapeInfo() function + * \param info Created from CreateTensorTypeAndShapeInfo() function * \param dim_values An array with length of `dim_count`. Its elements can contain negative values. * \param dim_count length of dim_values */ @@ -592,36 +593,36 @@ struct OrtApi { _Outptr_ OrtValue** out)NO_EXCEPTION; /** - * Construct OrtValue that contains a value of non-standard type created for - * experiments or while awaiting standardization. OrtValue in this case would contain - * an internal representation of the Opaque type. Opaque types are distinguished between - * each other by two strings 1) domain and 2) type name. The combination of the two - * must be unique, so the type representation is properly identified internally. The combination - * must be properly registered from within ORT at both compile/run time or by another API. - * - * To construct the OrtValue pass domain and type names, also a pointer to a data container - * the type of which must be know to both ORT and the client program. That data container may or may - * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for - * verification purposes. - * - * \domain_name - domain name for the Opaque type, null terminated. - * \type_name - type name for the Opaque type, null terminated. - * \data_contianer - data to populate OrtValue - * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected - * data_container size internally. - */ + * Construct OrtValue that contains a value of non-standard type created for + * experiments or while awaiting standardization. OrtValue in this case would contain + * an internal representation of the Opaque type. Opaque types are distinguished between + * each other by two strings 1) domain and 2) type name. The combination of the two + * must be unique, so the type representation is properly identified internally. The combination + * must be properly registered from within ORT at both compile/run time or by another API. + * + * To construct the OrtValue pass domain and type names, also a pointer to a data container + * the type of which must be know to both ORT and the client program. That data container may or may + * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for + * verification purposes. + * + * \domain_name - domain name for the Opaque type, null terminated. + * \type_name - type name for the Opaque type, null terminated. + * \data_contianer - data to populate OrtValue + * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected + * data_container size internally. + */ OrtStatus*(ORT_API_CALL* CreateOpaqueValue)(_In_ const char* domain_name, _In_ const char* type_name, _In_ const void* data_container, size_t data_container_size, _Outptr_ OrtValue** out)NO_EXCEPTION; /** - * Fetch data from an OrtValue that contains a value of non-standard type created for - * experiments or while awaiting standardization. - * \domain_name - domain name for the Opaque type, null terminated. - * \type_name - type name for the Opaque type, null terminated. - * \data_contianer - data to populate OrtValue - * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected - * data_container size internally. - */ + * Fetch data from an OrtValue that contains a value of non-standard type created for + * experiments or while awaiting standardization. + * \domain_name - domain name for the Opaque type, null terminated. + * \type_name - type name for the Opaque type, null terminated. + * \data_contianer - data to populate OrtValue + * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected + * data_container size internally. + */ OrtStatus*(ORT_API_CALL* GetOpaqueValue)(_In_ const char* domain_name, _In_ const char* type_name, _In_ const OrtValue* in, _Out_ void* data_container, size_t data_container_size)NO_EXCEPTION; diff --git a/onnxruntime/contrib_ops/cpu/bert/attention.cc b/onnxruntime/contrib_ops/cpu/bert/attention.cc index 2027dd1da0ddc..ccef01924b3ef 100644 --- a/onnxruntime/contrib_ops/cpu/bert/attention.cc +++ b/onnxruntime/contrib_ops/cpu/bert/attention.cc @@ -238,8 +238,17 @@ Status Attention::Compute(OpKernelContext* context) const { float* x = reinterpret_cast(scratch_data) + j * D; float* y = x; - for (int i = 0; i < D; i++) - y[i] = expf(x[i]); + // e^x is represented as infinity if x is large enough, like 100.f. + // Infinity divided by Infinity is a NAN. Thus, softmax gets a NAN if one or more item are large enough. + // a math transform as below is leveraged to get a stable softmax: + // e^xi/(e^x1 + ...e^xn) = e^(xi - max) / (e^(x1 - max) + ... + e^(xn - max)) + float max = -std::numeric_limits::infinity(); + for (int i = 0; i < D; i++) { + if (max < x[i]) max = x[i]; + } + for (int i = 0; i < D; i++) { + y[i] = expf(x[i] - max); + } double sum = 0.0; diff --git a/onnxruntime/contrib_ops/cpu/nchwc_ops.cc b/onnxruntime/contrib_ops/cpu/nchwc_ops.cc index 325217a1f8e10..14e0a83384e7a 100644 --- a/onnxruntime/contrib_ops/cpu/nchwc_ops.cc +++ b/onnxruntime/contrib_ops/cpu/nchwc_ops.cc @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/framework/op_kernel_context_internal.h" #include "nchwc_ops.h" #include "core/mlas/inc/mlas.h" diff --git a/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu b/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu index 53c79598ab5b7..26aee9affb46f 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu +++ b/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu @@ -24,6 +24,7 @@ limitations under the License. #include #include #include +#include #include "core/providers/cuda/cu_inc/common.cuh" #include "core/providers/cuda/cuda_common.h" #include "attention_impl.h" @@ -54,84 +55,125 @@ size_t GetAttentionWorkspaceSize(size_t element_size, int batch_size, int num_he } template -__device__ inline void Softmax(const int ld, const int last_valid, const T* input, T* output) { +__device__ inline void Softmax(const int ld, const int num_valid, const T* input, T* output) { using BlockReduce = cub::BlockReduce; __shared__ typename BlockReduce::TempStorage tmp_storage; - __shared__ float reverse_z; + __shared__ float sum_reverse_block; + __shared__ float max_block; - float thread_data(0); + float thread_data_max(-CUDART_INF_F); + + // e^x is represented as infinity if x is large enough, like 100.f. + // Infinity divided by Infinity is a NAN. Thus, softmax gets a NAN if one or more item are large enough. + // a math transform as below is leveraged to get a stable softmax: + // e^xi/(e^x1 + ...e^xn) = e^(xi - max) / (e^(x1 - max) + ... + e^(xn - max)) const int offset = (blockIdx.y * gridDim.x + blockIdx.x) * ld; - for (int i = threadIdx.x; i < last_valid; i += TPB) { + for (int i = threadIdx.x; i < num_valid; i += TPB) { + const int index = offset + i; + if (thread_data_max < float(input[index])) { + thread_data_max = float(input[index]); + } + } + + const auto max = BlockReduce(tmp_storage).Reduce(thread_data_max, cub::Max()); + + // Store max value + if (threadIdx.x == 0) { + max_block = max; + } + __syncthreads(); + + float thread_data_sum(0.f); + for (int i = threadIdx.x; i < num_valid; i += TPB) { const int index = offset + i; const float val = input[index]; - thread_data += expf(val); + thread_data_sum += expf(val - max_block); } - cub::Sum sum; - const auto z = BlockReduce(tmp_storage).Reduce(thread_data, sum); + const auto sum = BlockReduce(tmp_storage).Reduce(thread_data_sum, cub::Sum()); if (threadIdx.x == 0) { - reverse_z = 1.f / z; + sum_reverse_block = 1.f / sum; } __syncthreads(); for (int i = threadIdx.x; i < ld; i += TPB) { const int index = offset + i; - const float val = (i < last_valid) ? expf(float(input[index])) * reverse_z : 0.f; + const float val = (i < num_valid) ? expf(float(input[index]) - max_block) * sum_reverse_block : 0.f; output[index] = T(val); } } template -__device__ inline void SoftmaxSmall(const int ld, const int last_valid, const T* input, T* output) { +__device__ inline void SoftmaxSmall(const int ld, const int num_valid, const T* input, T* output) { using BlockReduce = cub::BlockReduce; __shared__ typename BlockReduce::TempStorage tmp_storage; - __shared__ float reverse_z; + __shared__ float sum_reverse_block; + __shared__ float max_block; - float thread_data(0); const int offset = (blockIdx.y * gridDim.x + blockIdx.x) * ld; const int index = offset + threadIdx.x; - if (threadIdx.x < last_valid) { + + // e^x is represented as infinity if x is large enough, like 100.f. + // Infinity divided by Infinity is a NAN. Thus, softmax gets a NAN if one or more item are large enough. + // a math transform as below is leveraged to get a stable softmax: + // e^xi/(e^x1 + ...e^xn) = e^(xi - max) / (e^(x1 - max) + ... + e^(xn - max)) + float thread_data_max(-CUDART_INF_F); + if (threadIdx.x < num_valid) { + thread_data_max = input[index]; + } + + const auto max = BlockReduce(tmp_storage).Reduce(thread_data_max, cub::Max(), num_valid); + + // Store max value + if (threadIdx.x == 0) { + max_block = max; + } + __syncthreads(); + + float thread_data_exp(0.f); + if (threadIdx.x < num_valid) { const float val = input[index]; - thread_data = expf(val); + thread_data_exp = expf(val - max_block); } - cub::Sum sum; - const auto z = BlockReduce(tmp_storage).Reduce(thread_data, sum); + const auto sum = BlockReduce(tmp_storage).Reduce(thread_data_exp, cub::Sum(), num_valid); + + // Store max value if (threadIdx.x == 0) { - reverse_z = (1.f) / z; + sum_reverse_block = (1.f) / sum; } __syncthreads(); if (threadIdx.x < ld) { - // this will be 0 for threadIdx.x >= last_valid - output[index] = T(thread_data * reverse_z); + // this will be 0 for threadIdx.x >= num_valid + output[index] = T(thread_data_exp * sum_reverse_block); } } template __global__ void MaskedSoftmaxKernelSmall(const int sequence_length, const int* mask_index, const T* input, T* output) { - __shared__ int last_valid; + __shared__ int num_valid; if (threadIdx.x == 0) { - last_valid = min(sequence_length, mask_index[blockIdx.y]); + num_valid = min(sequence_length, mask_index[blockIdx.y]); } __syncthreads(); - SoftmaxSmall(sequence_length, last_valid, input, output); + SoftmaxSmall(sequence_length, num_valid, input, output); } template __global__ void MaskedSoftmaxKernel(const int sequence_length, const int* mask_index, const T* input, T* output) { - __shared__ int last_valid; + __shared__ int num_valid; if (threadIdx.x == 0) { - last_valid = min(sequence_length, mask_index[blockIdx.y]); + num_valid = min(sequence_length, mask_index[blockIdx.y]); } __syncthreads(); - Softmax(sequence_length, last_valid, input, output); + Softmax(sequence_length, num_valid, input, output); } template diff --git a/onnxruntime/core/codegen/mti/mti_tvm_utils.cc b/onnxruntime/core/codegen/mti/mti_tvm_utils.cc index 562834bf9580f..0afc25d627cd5 100644 --- a/onnxruntime/core/codegen/mti/mti_tvm_utils.cc +++ b/onnxruntime/core/codegen/mti/mti_tvm_utils.cc @@ -113,18 +113,17 @@ tvm::Tensor Promote(const tvm::Expr& expr, const tvm::Array& shape, c name); } -void DumpTVMModuleToFile(const std::string& filename_prefix, tvm::runtime::Module& module) { +void DumpTVMModuleToFile(const std::string& filename, tvm::runtime::Module& module) { const codegen::CodeGenSettings& settings = codegen::CodeGenSettings::Instance(); if (!settings.HasOption(codegen::CodeGenSettings::kCodeGenDumpModule)) return; - static int dump_module_cnt = 0; // ISSUE: note that all option values are converted to lower case. It doesn't cause // any issue currently, because all supported formats (i.e. file exts) are of lower case. // Just keep in mind that we might have issue if somehow we started to support dump // formats with upper case, although it's quite unlikely. std::string format = settings.GetOptionValue(codegen::CodeGenSettings::kCodeGenDumpModule); - std::string module_filename = filename_prefix + "_" + std::to_string(dump_module_cnt++) + "." + format; + std::string module_filename = filename + "." + format; module->SaveToFile(module_filename, format); } diff --git a/onnxruntime/core/codegen/mti/mti_tvm_utils.h b/onnxruntime/core/codegen/mti/mti_tvm_utils.h index 709269aacff04..487c6415a7e7d 100644 --- a/onnxruntime/core/codegen/mti/mti_tvm_utils.h +++ b/onnxruntime/core/codegen/mti/mti_tvm_utils.h @@ -51,7 +51,7 @@ tvm::Tensor Promote(const tvm::Expr& expr, tvm::Tensor MakeZeroTensor(const tvm::Array& shape, HalideIR::Type type, const std::string& name); -void DumpTVMModuleToFile(const std::string& filename_prefix, tvm::runtime::Module& module); +void DumpTVMModuleToFile(const std::string& filename, tvm::runtime::Module& module); bool BroadcastDim(const tvm::Array& shape, size_t i, size_t output_rank, tvm::Expr& dim); diff --git a/onnxruntime/core/codegen/passes/op_ir_creator/tvm_ir_builder.cc b/onnxruntime/core/codegen/passes/op_ir_creator/tvm_ir_builder.cc index a50d1ee3f3839..2fe892797886a 100644 --- a/onnxruntime/core/codegen/passes/op_ir_creator/tvm_ir_builder.cc +++ b/onnxruntime/core/codegen/passes/op_ir_creator/tvm_ir_builder.cc @@ -70,7 +70,7 @@ Status TVMIRBuilder::Evaluate( // BEGIN: Generic IR creator classes #define ADD_OP_ITEM(name) \ - op_ir_registry->Register(std::move(onnxruntime::make_unique())); + op_ir_registry->Register(onnxruntime::make_unique()); #define BINARY_OP(name) ADD_OP_ITEM(name) #define BINARY_CMP_OP(name) ADD_OP_ITEM(name) diff --git a/onnxruntime/core/codegen/passes/weight_layout/weight_layout.cc b/onnxruntime/core/codegen/passes/weight_layout/weight_layout.cc index 207b5c6efc5e1..ab3e647fd284a 100644 --- a/onnxruntime/core/codegen/passes/weight_layout/weight_layout.cc +++ b/onnxruntime/core/codegen/passes/weight_layout/weight_layout.cc @@ -31,12 +31,11 @@ const std::string WeightLayout::GetKey( ONNX_NAMESPACE::TensorProto_DataType proto_type, int input_dim, float pad_zero) { - std::string key = name; - key += "_type_" + std::to_string(static_cast(proto_type)); - key += "_dim_" + input_dim; - key += "_pad_zero_" + std::to_string(pad_zero); - key = NormalizeCppName(key); - return key; + std::ostringstream key; + key << name << "_type_" << static_cast(proto_type); + key << "_dim_" << input_dim; + key << "_pad_zero_" << pad_zero; + return NormalizeCppName(key.str()); } WeightLayout::WeightLayout( diff --git a/onnxruntime/core/framework/allocator.cc b/onnxruntime/core/framework/allocator.cc index f58020df3f426..d3a567c4a207d 100644 --- a/onnxruntime/core/framework/allocator.cc +++ b/onnxruntime/core/framework/allocator.cc @@ -71,7 +71,7 @@ ORT_API_STATUS_IMPL(OrtApis::MemoryInfoGetMemType, _In_ const OrtMemoryInfo* ptr } ORT_API_STATUS_IMPL(OrtApis::MemoryInfoGetType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtAllocatorType* out) { - *out = ptr->type; + *out = ptr->alloc_type; return nullptr; } diff --git a/onnxruntime/core/framework/data_transfer.cc b/onnxruntime/core/framework/data_transfer.cc index 465655f03d6ec..80099c2e4e6fe 100644 --- a/onnxruntime/core/framework/data_transfer.cc +++ b/onnxruntime/core/framework/data_transfer.cc @@ -9,6 +9,14 @@ common::Status IDataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const { return CopyTensor(src, dst, 0); } +common::Status IDataTransfer::CopyTensors(const Tensor* src, Tensor* dst, int size) const { + ORT_ENFORCE(nullptr != src && nullptr != dst); + for (int i = 0; i < size; ++i) { + ORT_RETURN_IF_ERROR(CopyTensor(src[i], dst[i], 0)); + } + return Status::OK(); +} + bool CPUDataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const { return src_device.Type() == OrtDevice::CPU && dst_device.Type() == OrtDevice::CPU; } diff --git a/onnxruntime/core/framework/data_transfer.h b/onnxruntime/core/framework/data_transfer.h index 1e707b196babc..798525149bff7 100644 --- a/onnxruntime/core/framework/data_transfer.h +++ b/onnxruntime/core/framework/data_transfer.h @@ -17,6 +17,7 @@ class IDataTransfer { virtual common::Status CopyTensor(const Tensor& src, Tensor& dst) const; virtual common::Status CopyTensor(const Tensor& src, Tensor& dst, int exec_queue_id) const = 0; + virtual common::Status CopyTensors(const Tensor* src, Tensor* dst, int size) const; }; class CPUDataTransfer : public IDataTransfer { diff --git a/onnxruntime/core/framework/execution_providers.h b/onnxruntime/core/framework/execution_providers.h index ec7e2fedc9e2a..b17c3a61c046d 100644 --- a/onnxruntime/core/framework/execution_providers.h +++ b/onnxruntime/core/framework/execution_providers.h @@ -107,8 +107,25 @@ class ExecutionProviders { // maps for fast lookup of an index into exec_providers_ std::unordered_map provider_idx_map_; + + // currently the allocator type is an implementation detail and we don't make any behavioral choices based on it, + // so exclude it from the key comparison for allocator_idx_map_. + // we also don't expect to have two allocators with the same name, one using an arena and one not. + struct OrtMemoryInfoLessThanIgnoreAllocType { + bool operator()(const OrtMemoryInfo& lhs, const OrtMemoryInfo& rhs) const { + /*if (lhs.alloc_type != rhs.alloc_type) + return lhs.alloc_type < rhs.alloc_type;*/ + if (lhs.mem_type != rhs.mem_type) + return lhs.mem_type < rhs.mem_type; + if (lhs.id != rhs.id) + return lhs.id < rhs.id; + + return strcmp(lhs.name, rhs.name) < 0; + } + }; + // using std::map as OrtMemoryInfo would need a custom hash function to be used with unordered_map, // and as this isn't performance critical it's not worth the maintenance overhead of adding one. - std::map allocator_idx_map_; + std::map allocator_idx_map_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/framework/ort_value_name_idx_map.h b/onnxruntime/core/framework/ort_value_name_idx_map.h index dc906642d67e9..74e41c79a3f40 100644 --- a/onnxruntime/core/framework/ort_value_name_idx_map.h +++ b/onnxruntime/core/framework/ort_value_name_idx_map.h @@ -22,8 +22,7 @@ class OrtValueNameIdxMap { int Add(const std::string& name) { auto it = map_.find(name); if (it == map_.end()) { - int idx; - idx = ort_value_max_idx_++; + int idx = next_idx_++; map_.insert(it, {name, idx}); return idx; } @@ -43,7 +42,7 @@ class OrtValueNameIdxMap { } size_t Size() const { return map_.size(); }; - int MaxIdx() const { return ort_value_max_idx_; } + int MaxIdx() const { return next_idx_ - 1; } const_iterator begin() const noexcept { return map_.cbegin(); } const_iterator end() const noexcept { return map_.cend(); } @@ -51,8 +50,7 @@ class OrtValueNameIdxMap { private: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OrtValueNameIdxMap); - int ort_value_max_idx_ = 0; + int next_idx_ = 0; std::unordered_map map_; }; -using OrtValueNameIdxMap = OrtValueNameIdxMap; } // namespace onnxruntime diff --git a/onnxruntime/core/framework/path_lib.h b/onnxruntime/core/framework/path_lib.h index 89cfaef4188e7..1339450c9a664 100644 --- a/onnxruntime/core/framework/path_lib.h +++ b/onnxruntime/core/framework/path_lib.h @@ -34,11 +34,14 @@ ptrdiff_t OrtStrToPtrDiff(const T* nptr, T** endptr); template <> inline ptrdiff_t OrtStrToPtrDiff(const wchar_t* nptr, wchar_t** endptr) { #ifdef _WIN32 - // TODO: on x86_32, it should be wcstol +#ifdef _M_AMD64 return _wcstoi64(nptr, endptr, 10); #else return wcstol(nptr, endptr, 10); #endif +#else + return wcstol(nptr, endptr, 10); +#endif } template @@ -57,11 +60,14 @@ inline size_t OrtStrftime(wchar_t* strDest, size_t maxsize, const wchar template <> inline ptrdiff_t OrtStrToPtrDiff(const char* nptr, char** endptr) { #ifdef _WIN32 - // TODO: on x86_32, it should be strtol +#ifdef _M_AMD64 return _strtoi64(nptr, endptr, 10); #else return strtol(nptr, endptr, 10); #endif +#else + return strtol(nptr, endptr, 10); +#endif } template <> diff --git a/onnxruntime/core/framework/sequential_execution_plan.h b/onnxruntime/core/framework/sequential_execution_plan.h index 3e3dca68020cb..924b62d395432 100644 --- a/onnxruntime/core/framework/sequential_execution_plan.h +++ b/onnxruntime/core/framework/sequential_execution_plan.h @@ -33,7 +33,7 @@ struct AllocPlanPerValue { bool create_fence_if_async{false}; public: - AllocPlanPerValue() : location(CPU, OrtArenaAllocator) {} + AllocPlanPerValue() : location(CPU, Invalid) {} }; // SequentialExecutionPlan: This is the data that is produced by a static diff --git a/onnxruntime/core/framework/session_options.h b/onnxruntime/core/framework/session_options.h index aaefc50cc6c9c..1c02f9f0ebbc2 100644 --- a/onnxruntime/core/framework/session_options.h +++ b/onnxruntime/core/framework/session_options.h @@ -52,7 +52,7 @@ struct SessionOptions { unsigned max_num_graph_transformation_steps = 10; // TODO choose a good default here? // set graph optimization level - TransformerLevel graph_optimization_level = TransformerLevel::Level1; + TransformerLevel graph_optimization_level = TransformerLevel::Level3; // controls the size of the thread pool used to parallelize the execution of tasks within individual nodes (ops) int intra_op_num_threads = 0; diff --git a/onnxruntime/core/framework/session_state_initializer.cc b/onnxruntime/core/framework/session_state_initializer.cc index 3dd6ad72691fa..a80d6b6d47c85 100644 --- a/onnxruntime/core/framework/session_state_initializer.cc +++ b/onnxruntime/core/framework/session_state_initializer.cc @@ -150,7 +150,7 @@ static common::Status DeserializeTensorProto(const Env& env, const std::basic_st const Tensor& p_deserialize_tensor = tmp_ort_value.Get(); p_tensor = onnxruntime::make_unique(p_deserialize_tensor.DataType(), p_deserialize_tensor.Shape(), m.GetBuffer(), - m.GetAllocInfo()); + m.GetAllocInfo()); // TODO: does this function work for string tensor? Status copy_status = data_transfer_mgr.CopyTensor(p_deserialize_tensor, *p_tensor); if (d.f) d.f(d.param); @@ -177,7 +177,7 @@ common::Status SaveInitializedTensors(const Env& env, const std::basic_string 0, "OrtValue indexes should have been populated."); + ORT_ENFORCE(ort_value_name_idx_map.MaxIdx() > -1, "OrtValue indexes should have been populated."); //1. first plan the memory const onnxruntime::InitializedTensorSet& initialized_tensor_set = graph.GetAllInitializedTensors(); diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index 422c1aad31f94..ccfe61344b190 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -162,7 +162,6 @@ void convTransposeWithDynamicPadsShapeInference(InferenceContext& ctx) { return; } } - } // namespace ONNX_NAMESPACE namespace onnxruntime { @@ -202,6 +201,126 @@ void NchwcGlobalPoolOpSchemaGenerator(OpSchema& schema) { }); } +void ValidateTypeAndShapeForScaleAndZP(ONNX_NAMESPACE::InferenceContext& ctx, int index, ::google::protobuf::int32 expectedType, bool isScalar, int expectedTensorSize = 0) { + if (ctx.getNumInputs() > static_cast(index)) { + auto data_type = ctx.getInputType(index); + if (nullptr == data_type || data_type->value_case() != ONNX_NAMESPACE::TypeProto::kTensorType || + data_type->tensor_type().elem_type() != expectedType) { + fail_type_inference( + "Input data type does not match the expected data type. Current data type is ", data_type->tensor_type().elem_type()); + } + } + + if (hasInputShape(ctx, index)) { + ONNX_NAMESPACE::TensorShapeProto shape = ctx.getInputType(index)->tensor_type().shape(); + if (isScalar) { + if (shape.dim_size() != 0) { + fail_type_inference("Scale and Zero-point must be a scalar"); + } + } else { + if (shape.dim_size() != 1) { + fail_type_inference("Scale and Zero-point must be of rank 1"); + } + + if (shape.dim((int)0).has_dim_value() && shape.dim((int)0).dim_value() != expectedTensorSize) { + fail_type_inference( + "Scale and Zero-point must be of rank 1 and the number of elements should be equal to the number of rows of the corresponding input."); + } + } + } +} + +std::function QLinearMathDocGenerator(const char* name, const char* additionalDocumentation) { + return [=](OpSchema& schema) { + std::string doc = R"DOC( +Performs element-wise binary {name} on 8 bit data types (with Numpy-style broadcasting support). + +{additionalDocumentation} +)DOC"; + ONNX_NAMESPACE::ReplaceAll(doc, "{name}", name); + ONNX_NAMESPACE::ReplaceAll(doc, "{additionalDocumentation}", additionalDocumentation); + schema.SetDoc(doc); + schema.Input(0, "A", "First operand.", "T"); + schema.Input( + 1, + "A_scale", + "Input A's scale. It's a scalar, which means a per-tensor/layer quantization.", + "tensor(float)"); + schema.Input( + 2, + "A_zero_point", + "Input A zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.", + "T", + OpSchema::Optional); + schema.Input(3, "B", "Second operand.", "T"); + schema.Input( + 4, + "B_scale", + "Input B's scale. It's a scalar, which means a per-tensor/layer quantization.", + "tensor(float)"); + schema.Input( + 5, + "B_zero_point", + "Input B zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.", + "T", + OpSchema::Optional); + schema.Input( + 6, + "C_scale", + "Output scale. It's a scalar, which means a per-tensor/layer quantization.", + "tensor(float)"); + schema.Input( + 7, + "C_zero_point", + "Output zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.", + "T", + OpSchema::Optional); + schema.Output(0, "C", "Result, has same element type as two inputs", "T"); + schema.TypeConstraint("T", {"tensor(uint8)", "tensor(int8)"}, "Constrain input and output types to 8 bit signed and unsigned tensors."); + schema.TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromInputToOutput(ctx, 0, 0); + + auto a_type = ctx.getInputType(0); + auto b_type = ctx.getInputType(3); + + if (nullptr == a_type || nullptr == b_type || + a_type->value_case() != ONNX_NAMESPACE::TypeProto::kTensorType || + b_type->value_case() != ONNX_NAMESPACE::TypeProto::kTensorType) { + fail_type_inference("inputs are expected to have tensor type."); + } + + // validate scale and zero points + ValidateTypeAndShapeForScaleAndZP(ctx, 1, ONNX_NAMESPACE::TensorProto::FLOAT, true); + ValidateTypeAndShapeForScaleAndZP(ctx, 2, a_type->tensor_type().elem_type(), true); + ValidateTypeAndShapeForScaleAndZP(ctx, 4, ONNX_NAMESPACE::TensorProto::FLOAT, true); + ValidateTypeAndShapeForScaleAndZP(ctx, 5, b_type->tensor_type().elem_type(), true); + ValidateTypeAndShapeForScaleAndZP(ctx, 6, ONNX_NAMESPACE::TensorProto::FLOAT, true); + ValidateTypeAndShapeForScaleAndZP(ctx, 7, a_type->tensor_type().elem_type(), true); + + if (hasInputShape(ctx, 0) && hasInputShape(ctx, 3)) + bidirectionalBroadcastShapeInference( + ctx.getInputType(0)->tensor_type().shape(), + ctx.getInputType(3)->tensor_type().shape(), + *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape()); + }); + }; +} + +const char* contrib_ops_pads_doc = + "Padding for the beginning and ending along each spatial axis, it can take any value greater " + "than or equal to 0. The value represent the number of pixels added to the beginning " + "and end part of the corresponding axis. `pads` format should be as follow " + "[x1_begin, x2_begin...x1_end, x2_end,...], where xi_begin the number of pixels " + "added at the beginning of axis `i` and xi_end, the number of pixels added at " + "the end of axis `i`. This attribute cannot be used simultaneously with " + "auto_pad attribute. If not present, the padding defaults to 0 along start and end of each spatial axis."; +const char* contrib_ops_auto_pad_doc = + "auto_pad must be either NOTSET, SAME_UPPER, SAME_LOWER or VALID. Where " + "default value is NOTSET, which means explicit padding is used. " + "SAME_UPPER or SAME_LOWER mean pad the input so that the output spatial size match the input." + "In case of odd number add the extra padding at the end for SAME_UPPER and at the " + "beginning for SAME_LOWER. VALID mean no padding."; + void RegisterNchwcSchemas() { ONNX_CONTRIB_OPERATOR_SCHEMA(ReorderInput) .SetDomain(kMSNchwcDomain) @@ -1541,6 +1660,307 @@ with the exception that numpy default keepdims to False instead of True.)DOC") "Keep the reduced dimension or not, default 1 mean keep reduced dimension.", AttributeProto::INT); + ONNX_CONTRIB_OPERATOR_SCHEMA(QLinearAdd) + .SetDomain(kMSDomain) + .SinceVersion(1) + .FillUsing(QLinearMathDocGenerator("addition", + "C = (A_scale * (A - A_zero_point) + B_scale * (B - B_zero_point))/C_scale + C_zero_point")); + + ONNX_CONTRIB_OPERATOR_SCHEMA(QLinearMul) + .SetDomain(kMSDomain) + .SinceVersion(1) + .FillUsing(QLinearMathDocGenerator("multiplication", + "C = ((A - A_zero_point) * (B - B_zero_point)) * (A_scale * B_scale)/C_scale + C_zero_point")); + + ONNX_CONTRIB_OPERATOR_SCHEMA(QLinearReduceMean) + .SetDomain(kMSDomain) + .SinceVersion(1) + .SetDoc(R"DOC( +Computes the mean of the low-precision input tensor's element along the provided axes. +The resulting tensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, +then the resulting tensor have the reduced dimension pruned. The above behavior is similar to numpy, +with the exception that numpy default keepdims to False instead of True. +Input and Output scales and zero points are used to requantize the output in a new range. +This helps to improve accuracy as after ReduceMean operation the range of the output is expected to decrease. + +``` +"Output = Dequantize(Input) -> ReduceMean on fp32 data -> Quantize(output)", + +``` +)DOC") + .Input(0, "data", "An input tensor.", "T") + .Input( + 1, + "data_scale", + "Input scale. It's a scalar, which means a per-tensor/layer quantization.", + "tensor(float)") + .Input( + 2, + "data_zero_point", + "Input zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.", + "T", + OpSchema::Optional) + .Input( + 3, + "reduced_scale", + "Output scale. It's a scalar, which means a per-tensor/layer quantization.", + "tensor(float)") + .Input( + 4, + "reduced_zero_point", + "Output zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.", + "T", + OpSchema::Optional) + .Output(0, "reduced", "Reduced output tensor.", "T") + .TypeConstraint("T", {"tensor(uint8)", "tensor(int8)"}, "Constrain input types to 8 bit signed and unsigned tensors.") + .Attr( + "axes", + "A list of integers, along which to reduce. The default is to reduce over all the dimensions of the input tensor.", + AttributeProto::INTS) + .Attr( + "keepdims", + "Keep the reduced dimension or not, default 1 mean keep reduced dimension.", + AttributeProto::INT) + .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromInputToOutput(ctx, 0, 0); + + if (!hasNInputShapes(ctx, 1)) { + return; + } + + auto data_type = ctx.getInputType(0); + if (nullptr == data_type || data_type->value_case() != ONNX_NAMESPACE::TypeProto::kTensorType) { + fail_type_inference("inputs are expected to have tensor type."); + } + + // validate scale and zero points + ValidateTypeAndShapeForScaleAndZP(ctx, 1, ONNX_NAMESPACE::TensorProto::FLOAT, true); + ValidateTypeAndShapeForScaleAndZP(ctx, 2, data_type->tensor_type().elem_type(), true); + ValidateTypeAndShapeForScaleAndZP(ctx, 3, ONNX_NAMESPACE::TensorProto::FLOAT, true); + ValidateTypeAndShapeForScaleAndZP(ctx, 4, data_type->tensor_type().elem_type(), true); + + int64_t keep_dims = 1; + auto attr_proto = ctx.getAttribute("keepdims"); + if (attr_proto) { + keep_dims = attr_proto->i(); + } + + auto& input_shape = ctx.getInputType(0)->tensor_type().shape(); + int64_t input_ndim = input_shape.dim_size(); + auto output_shape = + ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape(); + std::vector axes; + auto axes_proto = ctx.getAttribute("axes"); + if (axes_proto) + axes.assign(axes_proto->ints().begin(), axes_proto->ints().end()); + + for (size_t i = 0; i < axes.size(); ++i) { + if (axes[i] < -input_ndim || axes[i] >= input_ndim) { + fail_shape_inference( + "axis must be in [-rank, rank-1]. input rank was ", input_ndim); + } + if (axes[i] < 0) + axes[i] += input_ndim; + } + // do we need to handle negative axis? + for (int i = 0; i < input_ndim; ++i) { + // axes empty means reduce all dim + if (!axes.empty() && + std::find(axes.begin(), axes.end(), i) == axes.end()) { + auto dim = output_shape->add_dim(); + dim->CopyFrom(input_shape.dim(i)); + } else { + if (keep_dims == 1) { + auto dim = output_shape->add_dim(); + dim->set_dim_value(1); + } + } + } + }); + + ONNX_CONTRIB_OPERATOR_SCHEMA(MulInteger) + .SetDomain(kMSDomain) + .SinceVersion(1) + .SetDoc(R"DOC(Performs element-wise binary quantized multiplication (with Numpy-style broadcasting support). +"This operator supports **multidirectional (i.e., Numpy-style) broadcasting**" +The output of this op is the int32 accumulated result of the mul operation + +``` +C (int32) = (A - A_zero_point) * (B - B_zero_point) +``` + +)DOC") + .Input(0, "A", "First operand.", "T") + .Input( + 1, + "A_zero_point", + "Input A zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.", + "T", + OpSchema::Optional) + .Input(2, "B", "Second operand.", "T") + .Input( + 3, + "B_zero_point", + "Input B zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.", + "T", + OpSchema::Optional) + .Output(0, "C", "Constrain output to 32 bit tensor", "T1") + .TypeConstraint("T", {"tensor(uint8)", "tensor(int8)"}, "Constrain input types to 8 bit signed and unsigned tensors.") + .TypeConstraint("T1", {"tensor(int32)"}, "Constrain output types to 32 bit tensors.") + .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { + auto c_type = ctx.getOutputType(0); + c_type->mutable_tensor_type()->set_elem_type( + ONNX_NAMESPACE::TensorProto::INT32); + + auto a_type = ctx.getInputType(0); + auto b_type = ctx.getInputType(3); + if (nullptr == a_type || nullptr == b_type || + a_type->value_case() != ONNX_NAMESPACE::TypeProto::kTensorType || + b_type->value_case() != ONNX_NAMESPACE::TypeProto::kTensorType) { + fail_type_inference("inputs are expected to have tensor type."); + } + + ValidateTypeAndShapeForScaleAndZP(ctx, 1, a_type->tensor_type().elem_type(), true); + ValidateTypeAndShapeForScaleAndZP(ctx, 3, b_type->tensor_type().elem_type(), true); + + if (hasInputShape(ctx, 0) && hasInputShape(ctx, 2)) { + bidirectionalBroadcastShapeInference( + ctx.getInputType(0)->tensor_type().shape(), + ctx.getInputType(2)->tensor_type().shape(), + *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape()); + } + }); + + const char* QLinearAveragePoolDoc_ver1 = R"DOC( + QLinearAveragePool consumes an input tensor X and applies average pooling across + the tensor according to kernel sizes, stride sizes, and pad lengths. + average pooling consisting of computing the average on all values of a + subset of the input tensor according to the kernel size and downsampling the + data into the output tensor Y for further processing. The output spatial shape will be following: + ``` + output_spatial_shape[i] = floor((input_spatial_shape[i] + pad_shape[i] - kernel_spatial_shape[i]) / strides_spatial_shape[i] + 1) + ``` + or + ``` + output_spatial_shape[i] = ceil((input_spatial_shape[i] + pad_shape[i] - kernel_spatial_shape[i]) / strides_spatial_shape[i] + 1) + ``` + if ceil_mode is enabled + + ``` + * pad_shape[i] is sum of pads along axis i + ``` + + `auto_pad` is a DEPRECATED attribute. If you are using them currently, the output spatial shape will be following: + ``` + VALID: output_spatial_shape[i] = ceil((input_spatial_shape[i] - kernel_spatial_shape[i] + 1) / strides_spatial_shape[i]) + SAME_UPPER or SAME_LOWER: output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides_spatial_shape[i]) + ``` + And pad shape will be following if `SAME_UPPER` or `SAME_LOWER`: + ``` + pad_shape[i] = (output_spatial_shape[i] - 1) * strides_spatial_shape[i] + kernel_spatial_shape[i] - input_spatial_shape[i] + ``` + +The output of each pooling window is divided by the number of elements (exclude pad when attribute count_include_pad is zero). + +Input and output scales and zero points are used to convert the output to a new quantization range. +Output = Dequantize(Input) -> AveragePool on fp32 data -> Quantize(output) +)DOC"; + + ONNX_CONTRIB_OPERATOR_SCHEMA(QLinearAveragePool) + .SetDomain(kMSDomain) + .SinceVersion(1) + .SetDoc(QLinearAveragePoolDoc_ver1) + .Attr( + "count_include_pad", + "Whether include pad pixels when calculating values for the edges. Default is 0, doesn't count include pad.", + AttributeProto::INT, + static_cast(0)) + .Attr( + "kernel_shape", + "The size of the kernel along each axis.", + AttributeProto::INTS) + .Attr( + "strides", + "Stride along each spatial axis. If not present, the stride defaults to 1 along each spatial axis.", + AttributeProto::INTS, + OPTIONAL) + .Attr( + "auto_pad", + contrib_ops_auto_pad_doc, + AttributeProto::STRING, + std::string("NOTSET")) + .Attr("pads", contrib_ops_pads_doc, AttributeProto::INTS, OPTIONAL) + .Attr( + "ceil_mode", + "Whether to use ceil or floor (default) to compute the output shape.", + AttributeProto::INT, + static_cast(0)) + .Input( + 0, + "X", + "Input data tensor from the previous operator; " + "dimensions for image case are (N x C x H x W), " + "where N is the batch size, C is the number of " + "channels, and H and W are the height and the " + "width of the data. For non image case, the " + "dimensions are in the form of " + "(N x C x D1 x D2 ... Dn), where N is the batch " + "size. Optionally, if dimension denotation is " + "in effect, the operation expects the input " + "data tensor to arrive with the dimension denotation " + "of [DATA_BATCH, DATA_CHANNEL, DATA_FEATURE, DATA_FEATURE ...].", + "T") + .Input( + 1, + "x_scale", + "Input scale. It's a scalar, which means a per-tensor/layer quantization.", + "tensor(float)") + .Input( + 2, + "x_zero_point", + "Input zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.", + "T", + OpSchema::Optional) + .Input( + 3, + "y_scale", + "Output scale. It's a scalar, which means a per-tensor/layer quantization.", + "tensor(float)") + .Input( + 4, + "y_zero_point", + "Output zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.", + "T", + OpSchema::Optional) + .Output( + 0, + "Y", + "Output data tensor from average or max pooling across " + "the input tensor. Dimensions will vary based " + "on various kernel, stride, and pad sizes. Floor value of " + "the dimension is used", + "T") + .TypeConstraint( + "T", + {"tensor(uint8)", "tensor(int8)"}, + "Constrain input and output types to 8 bit tensors.") + .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { + ONNX_NAMESPACE::propagateElemTypeFromInputToOutput(ctx, 0, 0); + + auto data_type = ctx.getInputType(0); + if (nullptr == data_type || data_type->value_case() != ONNX_NAMESPACE::TypeProto::kTensorType) { + fail_type_inference("inputs are expected to have tensor type."); + } + + // validate scale and zero points + ValidateTypeAndShapeForScaleAndZP(ctx, 1, ONNX_NAMESPACE::TensorProto::FLOAT, true); + ValidateTypeAndShapeForScaleAndZP(ctx, 2, data_type->tensor_type().elem_type(), true); + ValidateTypeAndShapeForScaleAndZP(ctx, 3, ONNX_NAMESPACE::TensorProto::FLOAT, true); + ValidateTypeAndShapeForScaleAndZP(ctx, 4, data_type->tensor_type().elem_type(), true); + + ONNX_NAMESPACE::convPoolShapeInference(ctx, false, true, 0, 5); + }); + ONNX_CONTRIB_OPERATOR_SCHEMA(MurmurHash3) .SetDomain(kMSDomain) .SinceVersion(1) diff --git a/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc index d6db03b3f6c30..6497e6cd882bd 100644 --- a/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc +++ b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc @@ -34,8 +34,16 @@ using ONNX_NAMESPACE::OPTIONAL; // Forward declarations static void RegisterCatImputerFeaturizerVer1(); static void RegisterDateTimeFeaturizerVer1(); +static void RegisterHashOneHotVectorizerFeaturizerVer1(); +static void RegisterImputationMarkerFeaturizerVer1(); +static void RegisterLabelEncoderFeaturizerVer1(); static void RegisterMaxAbsScalarFeaturizerVer1(); +static void RegisterMinMaxScalarFeaturizerVer1(); +static void RegisterMissingDummiesFeaturizerVer1(); +static void RegisterOneHotEncoderFeaturizerVer1(); +static void RegisterRobustScalarFeaturizerVer1(); static void RegisterStringFeaturizerVer1(); +static void RegisterTimeSeriesImputerFeaturizerVer1(); // ---------------------------------------------------------------------- // ---------------------------------------------------------------------- @@ -43,8 +51,16 @@ static void RegisterStringFeaturizerVer1(); void RegisterMSFeaturizersSchemas() { RegisterCatImputerFeaturizerVer1(); RegisterDateTimeFeaturizerVer1(); + RegisterHashOneHotVectorizerFeaturizerVer1(); + RegisterImputationMarkerFeaturizerVer1(); + RegisterLabelEncoderFeaturizerVer1(); RegisterMaxAbsScalarFeaturizerVer1(); + RegisterMinMaxScalarFeaturizerVer1(); + RegisterMissingDummiesFeaturizerVer1(); + RegisterOneHotEncoderFeaturizerVer1(); + RegisterRobustScalarFeaturizerVer1(); RegisterStringFeaturizerVer1(); + RegisterTimeSeriesImputerFeaturizerVer1(); } // ---------------------------------------------------------------------- @@ -58,15 +74,15 @@ void RegisterCatImputerFeaturizerVer1() { within the host frameworks and programming languages. C++-style pseudo signature: - std::float_t execute(std::float_t const &value); - std::double_t execute(std::double_t const &value); + float execute(float const &value); + double execute(double const &value); template T execute(std::optional const &value); Examples (where 55.5 is the mode value): execute(1.0) -> 1.0 execute(NaN) -> 55.5 execute(2.0) -> 2.0 - )DOC"; + )DOC"; MS_FEATURIZERS_OPERATOR_SCHEMA(CatImputerTransformer) .SinceVersion(1) @@ -76,7 +92,7 @@ void RegisterCatImputerFeaturizerVer1() { 0, "State", "State generated during training that is used for prediction", - "tensor(uint8)") + "T0") .Input( 1, "Input", @@ -87,6 +103,10 @@ void RegisterCatImputerFeaturizerVer1() { "Output", "No information is available", "T") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") .TypeConstraint( "T", {"tensor(float)", "tensor(double)", "tensor(string)"}, @@ -94,10 +114,9 @@ void RegisterCatImputerFeaturizerVer1() { .TypeAndShapeInferenceFunction( [](ONNX_NAMESPACE::InferenceContext& ctx) { propagateElemTypeFromInputToOutput(ctx, 1, 0); - if (!hasNInputShapes(ctx, 1)) { - return; + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); } - propagateShapeFromInputToOutput(ctx, 1, 0); }); } @@ -106,7 +125,7 @@ void RegisterDateTimeFeaturizerVer1() { Extracts various datetime-related values from a UTC time_point. C++-style pseudo signature: - TimePoint execute(std::chron::system_clock::time_point const &value); + TimePoint execute(std::chrono::system_clock::time_point const &value); Examples: Given a time_point 'value' representing "November 17, 1976 12:27:04PM": @@ -134,7 +153,7 @@ void RegisterDateTimeFeaturizerVer1() { "holidayName": "", "isPaidTimeOff": 0 } - )DOC"; + )DOC"; MS_FEATURIZERS_OPERATOR_SCHEMA(DateTimeTransformer) .SinceVersion(1) @@ -144,7 +163,7 @@ void RegisterDateTimeFeaturizerVer1() { 0, "State", "State generated during training that is used for prediction", - "tensor(uint8)") + "T0") .Input( 1, "Input", @@ -171,6 +190,10 @@ void RegisterDateTimeFeaturizerVer1() { .Output(18, "dayOfWeekLabel", "No information available", "OutputT3") .Output(19, "holidayName", "No information available", "OutputT3") .Output(20, "isPaidTimeOff", "No information available", "OutputT1") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") .TypeConstraint( "OutputT0", {"tensor(int32)"}, @@ -189,30 +212,295 @@ void RegisterDateTimeFeaturizerVer1() { "No information is available") .TypeAndShapeInferenceFunction( [](ONNX_NAMESPACE::InferenceContext& ctx) { - ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); - ctx.getOutputType(1)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(2)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(3)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(4)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(5)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(6)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(7)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(8)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(9)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(10)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); - ctx.getOutputType(11)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); - ctx.getOutputType(12)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(13)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(14)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(15)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); - ctx.getOutputType(16)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(17)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(18)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(19)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(20)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - - for (size_t i = 0; i < ctx.getNumOutputs(); ++i) { - *ctx.getOutputType(i)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); + const bool has_shape = hasInputShape(ctx, 1); + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_INT32, 0); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 1); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 1); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 2); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 2); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 3); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 3); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 4); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 4); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 5); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 5); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 6); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 6); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 7); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 7); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 8); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 8); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 9); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 9); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT16, 10); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 10); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT16, 11); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 11); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 12); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 12); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 13); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 13); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 14); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 14); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_INT32, 15); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 15); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 16); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 16); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 17); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 17); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 18); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 18); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 19); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 19); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 20); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 20); + } + + } + ); +} + +void RegisterHashOneHotVectorizerFeaturizerVer1() { + static const char* doc = R"DOC( + Hashes the input to a categorical value, then produces a one hot encoded vector + based on that value. + + C++-style pseudo signature: + template HashOneHotVectorizerStruct execute(T const &value); + + Examples: + Assuming the hashing algorithm... + "A" -> 1 + "B" -> 2 + "C" -> 5 + + and 'numCols' set to 8: + + execute("A") -> [1, 0, 0, 0, 0, 0, 0, 0] + execute("B") -> [0, 1, 0, 0, 0, 0, 0, 0] + execute("C") -> [0, 0, 0, 0, 1, 0, 0, 0] + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(HashOneHotVectorizerTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output(0, "NumElements", "No information available", "OutputT0") + .Output(1, "Value", "No information available", "OutputT1") + .Output(2, "Index", "No information available", "OutputT0") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"}, + "No information is available") + .TypeConstraint( + "OutputT0", + {"tensor(uint64)"}, + "No information is available") + .TypeConstraint( + "OutputT1", + {"tensor(uint8)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + const bool has_shape = hasInputShape(ctx, 1); + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT64, 0); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 1); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 1); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT64, 2); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 2); + } + + } + ); +} + +void RegisterImputationMarkerFeaturizerVer1() { + static const char* doc = R"DOC( + Returns true if the input is null, false if it is not. + + C++-style pseudo signature: + bool execute(float const &value); + bool execute(double const &value); + template bool execute(std::optional const &value); + + Examples: + 3.0 -> false + NaN -> true + "foo" -> false + std::optional() -> true + std::optional("bar") -> false + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(ImputationMarkerTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output( + 0, + "Output", + "No information is available", + "tensor(bool)") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(float)", "tensor(double)", "tensor(string)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_BOOL, 0); + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } + }); +} + +void RegisterLabelEncoderFeaturizerVer1() { + static const char* doc = R"DOC( + Returns a unique id for the input based on all values encountered during training. + + C++-style pseudo signature: + template uint32 execute(T const &value); + + Examples: + Assuming the training data of ["A", "B", "C"]... + + execute("A") -> 1 + execute("B") -> 2 + execute("C") -> 3 + execute("This value was not seen during training") -> 0 + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(LabelEncoderTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output( + 0, + "Output", + "No information is available", + "tensor(uint32)") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT32, 0); + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); } }); } @@ -222,8 +510,8 @@ void RegisterMaxAbsScalarFeaturizerVer1() { Scales input based on the maximum absolute value of all data encountered during training. C++-style pseudo signature: - std::float_t execute(std::uint16_t value); - std::double_t execute(std::uint32_t value); + float execute(uint16 value); + double execute(uint32 value); Examples: Given a training set of [1.0, -2.0, 3.0, -4.0], where 4.0 is the absolute value of the @@ -232,7 +520,7 @@ void RegisterMaxAbsScalarFeaturizerVer1() { execute(1.0) -> 1.0 / 4.0 execute(-4.0) -> -4.0 / 4.0 execute(100.0) -> 100 / 4.0 - )DOC"; + )DOC"; MS_FEATURIZERS_OPERATOR_SCHEMA(MaxAbsScalarTransformer) .SinceVersion(1) @@ -242,7 +530,7 @@ void RegisterMaxAbsScalarFeaturizerVer1() { 0, "State", "State generated during training that is used for prediction", - "tensor(uint8)") + "T0") .Input( 1, "Input", @@ -253,6 +541,10 @@ void RegisterMaxAbsScalarFeaturizerVer1() { "Output", "No information is available", "OutputT") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") .TypeConstraint( "InputT", {"tensor(int8)", "tensor(int16)", "tensor(uint8)", "tensor(uint16)", "tensor(float)", "tensor(int32)", "tensor(int64)", "tensor(uint32)", "tensor(uint64)", "tensor(double)"}, @@ -269,16 +561,270 @@ void RegisterMaxAbsScalarFeaturizerVer1() { input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT8 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT16 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { - ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_FLOAT, 0); } else if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT32 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT64 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT32 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT64 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE) { - ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_DOUBLE); + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_DOUBLE, 0); + } else { + fail_type_inference("input 1 is expected to have an accepted type"); + } + + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } + + }); +} + +void RegisterMinMaxScalarFeaturizerVer1() { + static const char* doc = R"DOC( + Scales input based on the scale that results from the minimum and maximum values encountered + during training. + + C++-style pseudo signature: + template double(T const &value); + + Examples: + Given the training data [1, 2, 3, 4, 5]; + min: 1 + max: 5 + scale ( - ): 4 + + execute(2) = 2 / 4 + execute(20) = 20 / 4 + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(MinMaxScalarTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output( + 0, + "Output", + "No information is available", + "tensor(double)") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_DOUBLE, 0); + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } + }); +} + +void RegisterMissingDummiesFeaturizerVer1() { + static const char* doc = R"DOC( + Returns 1 if the input is null, 0 if it is not. + + C++-style pseudo signature: + int8 execute(float const &value); + int8 execute(double const &value); + template int8 execute(T const &value); + + Examples: + 1.0 -> 0 + NaN -> 1 + "foo" -> 0 + std::optional() -> 1 + std::optional("bar") -> 0 + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(MissingDummiesTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output( + 0, + "Output", + "No information is available", + "tensor(int8)") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(float)", "tensor(double)", "tensor(string)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_INT8, 0); + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } + }); +} + +void RegisterOneHotEncoderFeaturizerVer1() { + static const char* doc = R"DOC( + Produces a one hot vector based on categories calculated during training. + + C++-style pseudo signature: + template OneHotVector execute(T const &value); + + Examples: + Assuming the training data [10, 20, 30, 40]... + + execute(10) -> [0, 1, 0, 0, 0] + execute(20) -> [0, 0, 1, 0, 0] + execute(30) -> [0, 0, 0, 1, 0] + execute(40) -> [0, 0, 0, 0, 1] + execute(200) -> [1, 0, 0, 0, 0] + execute(-1) -> [1, 0, 0, 0, 0] + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(OneHotEncoderTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output(0, "NumElements", "No information available", "OutputT0") + .Output(1, "Value", "No information available", "OutputT1") + .Output(2, "Index", "No information available", "OutputT0") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"}, + "No information is available") + .TypeConstraint( + "OutputT0", + {"tensor(uint64)"}, + "No information is available") + .TypeConstraint( + "OutputT1", + {"tensor(uint8)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + const bool has_shape = hasInputShape(ctx, 1); + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT64, 0); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 1); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 1); + } + + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT64, 2); + if(has_shape) { + propagateShapeFromInputToOutput(ctx, 1, 2); + } + + } + ); +} + +void RegisterRobustScalarFeaturizerVer1() { + static const char* doc = R"DOC( + MinMaxScalarEstimator + centering? + + C++-style pseudo signature: + TODO + + Examples: + TODO + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(RobustScalarTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output( + 0, + "Output", + "No information is available", + "OutputT") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(uint8)", "tensor(uint16)", "tensor(float)", "tensor(int32)", "tensor(int64)", "tensor(uint32)", "tensor(uint64)", "tensor(double)"}, + "No information is available") + .TypeConstraint( + "OutputT", + {"tensor(float)", "tensor(double)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + auto input_elem_type = ctx.getInputType(1)->tensor_type().elem_type(); + if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT8 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT16 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT8 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT16 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_FLOAT, 0); + } else if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT32 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT64 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT32 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT64 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_DOUBLE, 0); + } else { + fail_type_inference("input 1 is expected to have an accepted type"); + } + + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); } - *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); }); } @@ -287,12 +833,12 @@ void RegisterStringFeaturizerVer1() { Converts the input into a string representation based on the input's type. C++-style pseudo signature: - template std::string execute(T const &value); + template string execute(T const &value); Examples: execute(1) -> "1" execute(3.14) -> "3.14" - )DOC"; + )DOC"; MS_FEATURIZERS_OPERATOR_SCHEMA(StringTransformer) .SinceVersion(1) @@ -302,7 +848,7 @@ void RegisterStringFeaturizerVer1() { 0, "State", "State generated during training that is used for prediction", - "tensor(uint8)") + "T0") .Input( 1, "Input", @@ -313,6 +859,10 @@ void RegisterStringFeaturizerVer1() { "Output", "No information is available", "tensor(string)") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") .TypeConstraint( "InputT", {"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"}, @@ -320,8 +870,178 @@ void RegisterStringFeaturizerVer1() { .TypeAndShapeInferenceFunction( [](ONNX_NAMESPACE::InferenceContext& ctx) { propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 0); + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } + }); +} + +void RegisterTimeSeriesImputerFeaturizerVer1() { + static const char* doc = R"DOC( + Imputes rows and column values such that the generated output does not contain any + time gaps per grain (based on the time gaps encountered during training) and that + all missing column values are populated according to a strategy (forward fill, + backward fill, mode, etc.). + + This Featurizer is unique in that it will produce 0:N rows per invocation, depending upon the + input data. + + C++-style pseudo signature: + template + std::vector< + std::tuple< + bool, // true if the row was added + std::chrono::system_clock::time_point, + std::tuple, + std::tuple + > + > execute( + std::chrono::system_clock::time_point const &value, + std::tuple const &grain, + std::tuple const &colData + ); + + Examples: + During training, the time period was found to be 1 day... + + Input: + +------+-------+------------------+-------------------+ + | time | grain | forward fill col | backward fill col | + +======+=======+==================+===================+ + | 1 | A | 10 | None | + +------+-------+------------------+-------------------+ + | 2 | A | None | 200 | + +------+-------+------------------+-------------------+ + | 1 | B | -10 | -100 | + +------+-------+------------------+-------------------+ + | 4 | A | 40 | 400 | + +------+-------+------------------+-------------------+ + | 6 | A | 60 | 600 | + +------+-------+------------------+-------------------+ + | 3 | B | -30 | -300 | + +------+-------+------------------+-------------------+ + + Output: + +-------+------+-------+------------------+-------------------+ + | Added | time | grain | forward fill col | backward fill col | + +=======+======+=======+==================+===================+ + | false | 1 | A | 10 | 200 (from 2) | + +-------+------+-------+------------------+-------------------+ + | false | 2 | A | 10 (from 1) | 200 | + +-------+------+-------+------------------+-------------------+ + | true | 3 | A | 10 (from 2) | 400 (from 4) | + +-------+------+-------+------------------+-------------------+ + | false | 4 | A | 40 | 400 | + +-------+------+-------+------------------+-------------------+ + | true | 5 | A | 40 (from 4) | 600 (from 6) | + +-------+------+-------+------------------+-------------------+ + | false | 6 | A | 60 | 600 | + +-------+------+-------+------------------+-------------------+ + | false | 1 | B | -10 | -100 | + +-------+------+-------+------------------+-------------------+ + | true | 2 | B | -10 (from 1) | -300 (from 3) | + +-------+------+-------+------------------+-------------------+ + | false | 3 | B | -30 | -300 | + +-------+------+-------+------------------+-------------------+ + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(TimeSeriesImputerTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Times", + "Tensor of timestamps in seconds since epoch [R] where R is a number of rows.", + "T1") + .Input( + 2, + "Keys", + "Composite keys tensor of shape [R][K]. R is the same as Input(1)", + "T2") + .Input( + 3, + "Data", + "It is a data tensor of shape [R][C] where R - rows and C - columns. R must be the same with Input(1)", + "T2") + .Output( + 0, + "Added", + "Tensor of boolean with a shape of [IR]. Contains a boolean for each row in the result where true represents added row.", + "T3") + .Output( + 1, + "ImputedTimes", + "This is a tensor of timestamps in seconds since epoch of shape [IR], where IR is the number of output rows.", + "T1") + .Output( + 2, + "ImputedKeys", + "Contains keys along with the imputed keys. Tensor of shape [IR][K].", + "T2") + .Output( + 3, + "ImputedData", + "Tensor of shape [IR][C] where IR is the number of rows in the output." + "C is the number of columns.", + "T2") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "T1", + {"tensor(int64)"}, + "Represents number of seconds since epoch") + .TypeConstraint( + "T2", + {"tensor(string)"}, + "Output data") + .TypeConstraint( + "T3", + {"tensor(bool)"}, + "Boolean Tensor") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_BOOL, 0); + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_INT64, 1); + // Number of output rows is not known + ONNX_NAMESPACE::TensorShapeProto shape_0_1; + shape_0_1.add_dim(); + ONNX_NAMESPACE::updateOutputShape(ctx, 0, shape_0_1); + ONNX_NAMESPACE::updateOutputShape(ctx, 1, shape_0_1); - *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); + // Keys + propagateElemTypeFromInputToOutput(ctx, 2, 2); + // Keys shape + if (hasInputShape(ctx, 2)) { + const auto& input2_shape = getInputShape(ctx, 2); + if (input2_shape.dim_size() != 2) { + fail_shape_inference("Expecting keys to have 2 dimensions"); + } + ONNX_NAMESPACE::TensorShapeProto shape; + shape.add_dim(); + *shape.add_dim() = input2_shape.dim(1); + ONNX_NAMESPACE::updateOutputShape(ctx, 2, shape); + } + + // Data shape + propagateElemTypeFromInputToOutput(ctx, 3, 3); + if (hasInputShape(ctx, 3)) { + const auto& input3_shape = getInputShape(ctx, 3); + if (input3_shape.dim_size() != 2) { + fail_shape_inference("Expecting data to have 2 dimensions"); + } + ONNX_NAMESPACE::TensorShapeProto shape; + shape.add_dim(); + *shape.add_dim() = input3_shape.dim(1); + ONNX_NAMESPACE::updateOutputShape(ctx, 3, shape); + } }); } diff --git a/onnxruntime/core/graph/model.cc b/onnxruntime/core/graph/model.cc index 10756253a3469..9a783c1b6c237 100644 --- a/onnxruntime/core/graph/model.cc +++ b/onnxruntime/core/graph/model.cc @@ -432,7 +432,7 @@ Status Model::Load(int fd, ONNX_NAMESPACE::ModelProto& model_proto) { CodedInputStream cis(&fs); // Allows protobuf library versions < 3.2.0 to parse messages greater than 64MB. - cis.SetTotalBytesLimit(INT_MAX, INT_MAX); + cis.SetTotalBytesLimit(INT_MAX); if (!model_proto->ParseFromCodedStream(&cis)) { return Status(ONNXRUNTIME, INVALID_PROTOBUF, "Protobuf parsing failed."); } diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm index f701fb623facb..a4d9ec8468a8f 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm @@ -170,6 +170,8 @@ ProcessNextRowM4: vpxor xmm1,xmm1,xmm1 vpxor xmm2,xmm2,xmm2 vpxor xmm3,xmm3,xmm3 + lea r13,[r8+r8*2] ; compute ldb * 3 + lea rax,[r11+r11*2] ; compute output stride * 3 mov rdx,rsi mov rcx,rdi lea rsi,[rsi+r8*4] ; advance next matrix A by 4 rows @@ -179,16 +181,14 @@ ProcessNextRowM4: jb ProcessRemainingColumnsM4 ProcessNextColumnLoopM4: - lea rax,[rdx+r8*2] ; compute matrix A plus 2 rows vmovdqu ymm4,YMMWORD PTR [rdx] vmovdqu ymm5,YMMWORD PTR [rdx+r8] - vmovdqu ymm6,YMMWORD PTR [rax] - vmovdqu ymm7,YMMWORD PTR [rax+r8] - lea rax,[rcx+r11*2] ; compute matrix D plus 2 rows + vmovdqu ymm6,YMMWORD PTR [rdx+r8*2] + vmovdqu ymm7,YMMWORD PTR [rdx+r13] vmovdqu YMMWORD PTR [rcx],ymm4 vmovdqu YMMWORD PTR [rcx+r11],ymm5 - vmovdqu YMMWORD PTR [rax],ymm6 - vmovdqu YMMWORD PTR [rax+r11],ymm7 + vmovdqu YMMWORD PTR [rcx+r11*2],ymm6 + vmovdqu YMMWORD PTR [rcx+rax],ymm7 vpmaddubsw ymm4,ymm4,ymm9 ; horizontal byte+byte=word per row vpaddw ymm0,ymm0,ymm4 ; add words to row accumulators vpmaddubsw ymm5,ymm5,ymm9 @@ -207,16 +207,14 @@ ProcessRemainingColumnsM4: jz ReduceRowSumVectorM4 test bl,16 ; (CountK & 16) != 0? jz CopyRemainingCountKLessThan16M4 - lea rax,[rdx+r8*2] ; compute matrix A plus 2 rows vmovdqu xmm4,XMMWORD PTR [rdx] vmovdqu xmm5,XMMWORD PTR [rdx+r8] - vmovdqu xmm6,XMMWORD PTR [rax] - vmovdqu xmm7,XMMWORD PTR [rax+r8] - lea rax,[rcx+r11*2] ; compute matrix D plus 2 rows + vmovdqu xmm6,XMMWORD PTR [rdx+r8*2] + vmovdqu xmm7,XMMWORD PTR [rdx+r13] vmovdqu XMMWORD PTR [rcx],xmm4 vmovdqu XMMWORD PTR [rcx+r11],xmm5 - vmovdqu XMMWORD PTR [rax],xmm6 - vmovdqu XMMWORD PTR [rax+r11],xmm7 + vmovdqu XMMWORD PTR [rcx+r11*2],xmm6 + vmovdqu XMMWORD PTR [rcx+rax],xmm7 vpmaddubsw xmm4,xmm4,xmm9 ; horizontal byte+byte=word per row vpaddw ymm0,ymm0,ymm4 ; add words to row accumulators vpmaddubsw xmm5,xmm5,xmm9 @@ -239,14 +237,13 @@ CopyRemainingCountKLessThan16M4: mov rbp,rsp ; GemmU8S8CopyPackAFrame.PaddedMatrixAData test bl,8 ; (CountK & 8) != 0? jz CopyRemainingCountKLessThan8M4 - lea r13,[rdx+r8*2] ; compute matrix A plus 2 rows mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax mov rax,QWORD PTR [rdx+r8] mov QWORD PTR [rbp+16],rax - mov rax,QWORD PTR [r13] + mov rax,QWORD PTR [rdx+r8*2] mov QWORD PTR [rbp+32],rax - mov rax,QWORD PTR [r13+r8] + mov rax,QWORD PTR [rdx+r13] mov QWORD PTR [rbp+48],rax add rdx,8 add rbp,8 ; advance padded buffer destination @@ -254,14 +251,13 @@ CopyRemainingCountKLessThan16M4: CopyRemainingCountKLessThan8M4: test bl,4 ; (CountK & 4) != 0? jz CopyRemainingCountKLessThan4M4 - lea r13,[rdx+r8*2] ; compute matrix A plus 2 rows mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax mov eax,DWORD PTR [rdx+r8] mov DWORD PTR [rbp+16],eax - mov eax,DWORD PTR [r13] + mov eax,DWORD PTR [rdx+r8*2] mov DWORD PTR [rbp+32],eax - mov eax,DWORD PTR [r13+r8] + mov eax,DWORD PTR [rdx+r13] mov DWORD PTR [rbp+48],eax add rdx,4 add rbp,4 ; advance padded buffer destination @@ -269,14 +265,13 @@ CopyRemainingCountKLessThan8M4: CopyRemainingCountKLessThan4M4: test bl,2 ; (CountK & 2) != 0? jz CopyRemainingCountKLessThan2M4 - lea r13,[rdx+r8*2] ; compute matrix A plus 2 rows movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax movzx eax,WORD PTR [rdx+r8] mov WORD PTR [rbp+16],ax - movzx eax,WORD PTR [r13] + movzx eax,WORD PTR [rdx+r8*2] mov WORD PTR [rbp+32],ax - movzx eax,WORD PTR [r13+r8] + movzx eax,WORD PTR [rdx+r13] mov WORD PTR [rbp+48],ax add rdx,2 add rbp,2 ; advance padded buffer destination @@ -284,14 +279,13 @@ CopyRemainingCountKLessThan4M4: CopyRemainingCountKLessThan2M4: test bl,1 ; (CountK & 1) != 0? jz ProcessPaddedMatrixADataM4 - lea r13,[rdx+r8*2] ; compute matrix A plus 2 rows movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al movzx eax,BYTE PTR [rdx+r8] mov BYTE PTR [rbp+16],al - movzx eax,BYTE PTR [r13] + movzx eax,BYTE PTR [rdx+r8*2] mov BYTE PTR [rbp+32],al - movzx eax,BYTE PTR [r13+r8] + movzx eax,BYTE PTR [rdx+r13] mov BYTE PTR [rbp+48],al ; @@ -507,6 +501,7 @@ ExitRoutine: END_PROLOGUE mov rsi,rdx + lea rdi,[r8+r8*2] ; compute ldb * 3 mov r10,GemmU8S8CopyPackBFrame.CountK[rsp] mov r11,GemmU8S8CopyPackBFrame.ColumnSumVector[rsp] vpbroadcastw ymm7,WORD PTR GemmU8S8CopyPackBFrame.offa[rsp] @@ -532,11 +527,10 @@ ProcessNextColumnN16: jb ProcessRemainingRowsN16 ProcessNextRowLoopN16: - lea rax,[rdx+r8*2] ; compute matrix B plus 2 rows vmovdqu xmm2,XMMWORD PTR [rdx] ; load 4 rows vmovdqu xmm3,XMMWORD PTR [rdx+r8] - vmovdqu xmm4,XMMWORD PTR [rax] - vmovdqu xmm5,XMMWORD PTR [rax+r8] + vmovdqu xmm4,XMMWORD PTR [rdx+r8*2] + vmovdqu xmm5,XMMWORD PTR [rdx+rdi] lea rdx,[rdx+r8*4] ; advance matrix B by 4 rows InterleaveRowDataN16: @@ -630,14 +624,13 @@ ProcessNextRowLoopNUnaligned: mov rbp,rsp ; GemmU8S8CopyPackBFrame.PaddedMatrixBData test r9b,8 ; (CountN & 8) != 0? jz CopyRemainingCountNLessThan8K4 - lea rdi,[rdx+r8*2] ; compute matrix B plus 2 rows mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax mov rax,QWORD PTR [rdx+r8] mov QWORD PTR [rbp+16],rax - mov rax,QWORD PTR [rdi] + mov rax,QWORD PTR [rdx+r8*2] mov QWORD PTR [rbp+32],rax - mov rax,QWORD PTR [rdi+r8] + mov rax,QWORD PTR [rdx+rdi] mov QWORD PTR [rbp+48],rax add rdx,8 ; advance matrix B add rbp,8 ; advance padded buffer destination @@ -645,14 +638,13 @@ ProcessNextRowLoopNUnaligned: CopyRemainingCountNLessThan8K4: test r9b,4 ; (CountN & 4) != 0? jz CopyRemainingCountNLessThan4K4 - lea rdi,[rdx+r8*2] ; compute matrix B plus 2 rows mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax mov eax,DWORD PTR [rdx+r8] mov DWORD PTR [rbp+16],eax - mov eax,DWORD PTR [rdi] + mov eax,DWORD PTR [rdx+r8*2] mov DWORD PTR [rbp+32],eax - mov eax,DWORD PTR [rdi+r8] + mov eax,DWORD PTR [rdx+rdi] mov DWORD PTR [rbp+48],eax add rdx,4 ; advance matrix B add rbp,4 ; advance padded buffer destination @@ -660,14 +652,13 @@ CopyRemainingCountNLessThan8K4: CopyRemainingCountNLessThan4K4: test r9b,2 ; (CountN & 2) != 0? jz CopyRemainingCountNLessThan2K4 - lea rdi,[rdx+r8*2] ; compute matrix B plus 2 rows movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax movzx eax,WORD PTR [rdx+r8] mov WORD PTR [rbp+16],ax - movzx eax,WORD PTR [rdi] + movzx eax,WORD PTR [rdx+r8*2] mov WORD PTR [rbp+32],ax - movzx eax,WORD PTR [rdi+r8] + movzx eax,WORD PTR [rdx+rdi] mov WORD PTR [rbp+48],ax add rdx,2 ; advance matrix B add rbp,2 ; advance padded buffer destination @@ -675,14 +666,13 @@ CopyRemainingCountNLessThan4K4: CopyRemainingCountNLessThan2K4: test r9b,1 ; (CountN & 1) != 0? jz ProcessPaddedMatrixBData - lea rdi,[rdx+r8*2] ; compute matrix B plus 2 rows movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al movzx eax,BYTE PTR [rdx+r8] mov BYTE PTR [rbp+16],al - movzx eax,BYTE PTR [rdi] + movzx eax,BYTE PTR [rdx+r8*2] mov BYTE PTR [rbp+32],al - movzx eax,BYTE PTR [rdi+r8] + movzx eax,BYTE PTR [rdx+rdi] mov BYTE PTR [rbp+48],al ProcessPaddedMatrixBData: diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index 4ce094eb2fee6..12ed28672e5c2 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -131,6 +131,7 @@ Module Name: #define MLAS_SGEMM_STRIDEN_THREAD_ALIGN 16 #define MLAS_DGEMM_STRIDEN_THREAD_ALIGN 8 +#define MLAS_QGEMM_STRIDEN_THREAD_ALIGN 16 // // Define the prototypes of the platform optimized routines. @@ -333,6 +334,24 @@ size_t typedef MLAS_GEMM_U8U8_KERNEL* PMLAS_GEMM_U8U8_KERNEL; +typedef +void +(MLASCALL MLAS_GEMM_X8X8_OPERATION)( + size_t M, + size_t N, + size_t K, + const uint8_t* A, + size_t lda, + int16_t offa, + const uint8_t* B, + size_t ldb, + int16_t offb, + int32_t* C, + size_t ldc + ); + +typedef MLAS_GEMM_X8X8_OPERATION* PMLAS_GEMM_X8X8_OPERATION; + typedef void (MLASCALL MLAS_CONV_FLOAT_KERNEL)( @@ -554,22 +573,10 @@ extern "C" { // Define the target number of per-thread multiplies before using another // thread to perform additional work. // -// The number is derived from performance results running SGEMM across a -// range of workloads and observing the ideal number of threads to complete -// that workload. -// -#if defined(_OPENMP) #define MLAS_SGEMM_THREAD_COMPLEXITY (64 * 1024) -#else -#if defined(MLAS_TARGET_AMD64) -#define MLAS_SGEMM_THREAD_COMPLEXITY (2 * 1024 * 1024) -#else -#define MLAS_SGEMM_THREAD_COMPLEXITY (1 * 1024 * 1024) -#endif -#endif - #define MLAS_DGEMM_THREAD_COMPLEXITY (64 * 1024) +#define MLAS_QGEMM_THREAD_COMPLEXITY (64 * 1024) // // Single-threaded single precision matrix/matrix multiply operation. @@ -673,6 +680,28 @@ MlasGetMaximumThreadCount( #endif } +inline +void +MlasPartitionWork( + int32_t ThreadId, + int32_t ThreadCount, + size_t TotalWork, + size_t* WorkIndex, + size_t* WorkRemaining + ) +{ + const size_t WorkPerThread = TotalWork / ThreadCount; + const size_t WorkPerThreadExtra = TotalWork % ThreadCount; + + if (uint32_t(ThreadId) < WorkPerThreadExtra) { + *WorkIndex = (WorkPerThread + 1) * ThreadId; + *WorkRemaining = WorkPerThread + 1; + } else { + *WorkIndex = WorkPerThread * ThreadId + WorkPerThreadExtra; + *WorkRemaining = WorkPerThread; + } +} + // // Define the missing ARM64 NEON intrinsic macros from arm64_neon.h that enable // cross-compiler support. diff --git a/onnxruntime/core/mlas/lib/qgemm.cpp b/onnxruntime/core/mlas/lib/qgemm.cpp index 776107fc5c97a..377628c3564ca 100644 --- a/onnxruntime/core/mlas/lib/qgemm.cpp +++ b/onnxruntime/core/mlas/lib/qgemm.cpp @@ -21,13 +21,33 @@ Module Name: // Define the default strides to step through slices of the input matrices. // -#define MLAS_GEMM_U8S8_STRIDEM 24 -#define MLAS_GEMM_U8S8_STRIDEN 256 -#define MLAS_GEMM_U8S8_STRIDEK 128 +#define MLAS_GEMM_X8X8_STRIDEM 24 +#define MLAS_GEMM_X8X8_STRIDEN 256 +#define MLAS_GEMM_X8X8_STRIDEK 128 -#define MLAS_GEMM_U8U8_STRIDEM 24 -#define MLAS_GEMM_U8U8_STRIDEN 256 -#define MLAS_GEMM_U8U8_STRIDEK 128 +// +// Define the parameters to execute segments of a QGEMM operation on worker +// threads. +// + +struct MLAS_GEMM_X8X8_WORK_BLOCK { + PMLAS_GEMM_X8X8_OPERATION GemmX8X8Operation; + size_t M; + size_t N; + size_t K; + const uint8_t* A; + size_t lda; + const uint8_t* B; + size_t ldb; + int32_t* C; + size_t ldc; + int32_t ThreadCountM; + int32_t ThreadCountN; + size_t StrideM; + size_t StrideN; + int16_t offa; + int16_t offb; +}; #ifdef MLAS_TARGET_AMD64_IX86 @@ -1102,45 +1122,83 @@ Return Value: void MLASCALL -MlasGemm( +MlasGemmU8S8Operation( size_t M, size_t N, size_t K, const uint8_t* A, size_t lda, - uint8_t offa, - const int8_t* B, + int16_t offa, + const uint8_t* B, size_t ldb, - int8_t offb, + int16_t offb, int32_t* C, - size_t ldc, - MLAS_THREADPOOL* ThreadPool + size_t ldc ) -{ - MLAS_DECLSPEC_ALIGN(uint8_t PanelA[MLAS_GEMM_U8S8_STRIDEM * MLAS_GEMM_U8S8_STRIDEK], 64); - MLAS_DECLSPEC_ALIGN(int8_t PanelB[MLAS_GEMM_U8S8_STRIDEN * MLAS_GEMM_U8S8_STRIDEK], 64); +/*++ + +Routine Description: + + This module implements the quantized integer matrix/matrix multiply + operation (QGEMM). + +Arguments: + + M - Supplies the number of rows of matrix A and matrix C. + + N - Supplies the number of columns of matrix B and matrix C. + + K - Supplies the number of columns of matrix A and the number of rows of + matrix B. + + A - Supplies the address of matrix A. + + lda - Supplies the first dimension of matrix A. + + offa - Supplies the zero point offset of matrix A. + + B - Supplies the address of matrix B. + + ldb - Supplies the first dimension of matrix B. + + offb - Supplies the zero point offset of matrix B. + + C - Supplies the address of matrix C. + + ldc - Supplies the first dimension of matrix C. + +Return Value: + + None. - MLAS_DECLSPEC_ALIGN(int32_t RowSumVector[MLAS_GEMM_U8S8_STRIDEM], 16); - MLAS_DECLSPEC_ALIGN(int32_t ColumnSumVector[MLAS_GEMM_U8S8_STRIDEN], 16); +--*/ +{ + MLAS_DECLSPEC_ALIGN(uint8_t PanelA[MLAS_GEMM_X8X8_STRIDEM * MLAS_GEMM_X8X8_STRIDEK], 64); + MLAS_DECLSPEC_ALIGN(int8_t PanelB[MLAS_GEMM_X8X8_STRIDEN * MLAS_GEMM_X8X8_STRIDEK], 64); - size_t StrideM = MLAS_GEMM_U8S8_STRIDEM; - size_t StrideN = MLAS_GEMM_U8S8_STRIDEN; - size_t StrideK = MLAS_GEMM_U8S8_STRIDEK; + MLAS_DECLSPEC_ALIGN(int32_t RowSumVector[MLAS_GEMM_X8X8_STRIDEM], 16); + MLAS_DECLSPEC_ALIGN(int32_t ColumnSumVector[MLAS_GEMM_X8X8_STRIDEN], 16); - MLAS_UNREFERENCED_PARAMETER(ThreadPool); + size_t StrideM = MLAS_GEMM_X8X8_STRIDEM; + size_t StrideN = MLAS_GEMM_X8X8_STRIDEN; + size_t StrideK = MLAS_GEMM_X8X8_STRIDEK; #if defined(MLAS_TARGET_AMD64) if (M == 1 && offa == 0 && offb == 0) { if (MlasPlatform.GemvU8S8Kernel != nullptr) { - MlasPlatform.GemvU8S8Kernel(A, B, C, K, N, ldb); + MlasPlatform.GemvU8S8Kernel(A, (const int8_t*)B, C, K, N, ldb); return; } } #endif + // + // Step through each slice of matrix B along the K dimension. + // + size_t CountK; for (size_t k = 0; k < K; k += CountK) { @@ -1151,6 +1209,10 @@ MlasGemm( CountK = K - k; } + // + // Step through each slice of matrix B along the N dimension. + // + size_t CountN; for (size_t n = 0; n < N; n += CountN) { @@ -1161,7 +1223,10 @@ MlasGemm( CountN = N - n; } - MlasPlatform.GemmU8S8CopyPackBRoutine(PanelB, B + n + k * ldb, ldb, CountN, CountK, ColumnSumVector, -int16_t(offa)); + const int8_t* b = (const int8_t*)B + n + k * ldb; + + MlasPlatform.GemmU8S8CopyPackBRoutine(PanelB, b, ldb, CountN, + CountK, ColumnSumVector, -int16_t(offa)); size_t CountM; @@ -1173,7 +1238,8 @@ MlasGemm( CountM = M - m; } - MlasPlatform.GemmU8S8CopyPackARoutine(PanelA, A + k + m * lda, lda, CountM, CountK, RowSumVector, -int16_t(offb)); + MlasPlatform.GemmU8S8CopyPackARoutine(PanelA, A + k + m * lda, + lda, CountM, CountK, RowSumVector, -int16_t(offb)); uint8_t* pa = PanelA; int32_t* c = C + n + m * ldc; @@ -1187,7 +1253,9 @@ MlasGemm( while (RowsRemaining > 0) { - RowsHandled = MlasPlatform.GemmU8S8Kernel(pa, PanelB, c, QuadCountK, RowsRemaining, CountN, ldc, RowSums, ColumnSumVector, int32_t(CountK) * offa * offb, k == 0); + RowsHandled = MlasPlatform.GemmU8S8Kernel(pa, PanelB, c, + QuadCountK, RowsRemaining, CountN, ldc, RowSums, + ColumnSumVector, int32_t(CountK) * offa * offb, k == 0); RowsRemaining -= RowsHandled; c += ldc * RowsHandled; @@ -1201,32 +1269,70 @@ MlasGemm( void MLASCALL -MlasGemm( +MlasGemmU8U8Operation( size_t M, size_t N, size_t K, const uint8_t* A, size_t lda, - uint8_t offa, + int16_t offa, const uint8_t* B, size_t ldb, - uint8_t offb, + int16_t offb, int32_t* C, - size_t ldc, - MLAS_THREADPOOL* ThreadPool + size_t ldc ) +/*++ + +Routine Description: + + This module implements the quantized integer matrix/matrix multiply + operation (QGEMM). + +Arguments: + + M - Supplies the number of rows of matrix A and matrix C. + + N - Supplies the number of columns of matrix B and matrix C. + + K - Supplies the number of columns of matrix A and the number of rows of + matrix B. + + A - Supplies the address of matrix A. + + lda - Supplies the first dimension of matrix A. + + offa - Supplies the zero point offset of matrix A. + + B - Supplies the address of matrix B. + + ldb - Supplies the first dimension of matrix B. + + offb - Supplies the zero point offset of matrix B. + + C - Supplies the address of matrix C. + + ldc - Supplies the first dimension of matrix C. + +Return Value: + + None. + +--*/ { - MLAS_DECLSPEC_ALIGN(int16_t PanelA[MLAS_GEMM_U8U8_STRIDEM * MLAS_GEMM_U8U8_STRIDEK], 64); - MLAS_DECLSPEC_ALIGN(uint8_t PanelB[MLAS_GEMM_U8U8_STRIDEN * MLAS_GEMM_U8U8_STRIDEK], 64); + MLAS_DECLSPEC_ALIGN(int16_t PanelA[MLAS_GEMM_X8X8_STRIDEM * MLAS_GEMM_X8X8_STRIDEK], 64); + MLAS_DECLSPEC_ALIGN(uint8_t PanelB[MLAS_GEMM_X8X8_STRIDEN * MLAS_GEMM_X8X8_STRIDEK], 64); - MLAS_DECLSPEC_ALIGN(int32_t RowSumVector[MLAS_GEMM_U8U8_STRIDEM], 16); - MLAS_DECLSPEC_ALIGN(int32_t ColumnSumVector[MLAS_GEMM_U8U8_STRIDEN], 16); + MLAS_DECLSPEC_ALIGN(int32_t RowSumVector[MLAS_GEMM_X8X8_STRIDEM], 16); + MLAS_DECLSPEC_ALIGN(int32_t ColumnSumVector[MLAS_GEMM_X8X8_STRIDEN], 16); - size_t StrideM = MLAS_GEMM_U8U8_STRIDEM; - size_t StrideN = MLAS_GEMM_U8U8_STRIDEN; - size_t StrideK = MLAS_GEMM_U8U8_STRIDEK; + size_t StrideM = MLAS_GEMM_X8X8_STRIDEM; + size_t StrideN = MLAS_GEMM_X8X8_STRIDEN; + size_t StrideK = MLAS_GEMM_X8X8_STRIDEK; - MLAS_UNREFERENCED_PARAMETER(ThreadPool); + // + // Step through each slice of matrix B along the K dimension. + // size_t CountK; @@ -1238,6 +1344,10 @@ MlasGemm( CountK = K - k; } + // + // Step through each slice of matrix B along the N dimension. + // + size_t CountN; for (size_t n = 0; n < N; n += CountN) { @@ -1248,7 +1358,10 @@ MlasGemm( CountN = N - n; } - MlasPlatform.GemmU8U8CopyPackBRoutine(PanelB, B + n + k * ldb, ldb, CountN, CountK, ColumnSumVector, -int16_t(offa)); + const uint8_t* b = (const uint8_t*)B + n + k * ldb; + + MlasPlatform.GemmU8U8CopyPackBRoutine(PanelB, b, ldb, CountN, + CountK, ColumnSumVector, -int16_t(offa)); size_t CountM; @@ -1260,7 +1373,8 @@ MlasGemm( CountM = M - m; } - MlasPlatform.GemmU8U8CopyPackARoutine(PanelA, A + k + m * lda, lda, CountM, CountK, RowSumVector, -int16_t(offb)); + MlasPlatform.GemmU8U8CopyPackARoutine(PanelA, A + k + m * lda, + lda, CountM, CountK, RowSumVector, -int16_t(offb)); int16_t* pa = PanelA; int32_t* c = C + n + m * ldc; @@ -1274,7 +1388,9 @@ MlasGemm( while (RowsRemaining > 0) { - RowsHandled = MlasPlatform.GemmU8U8Kernel(pa, PanelB, c, PairCountK, RowsRemaining, CountN, ldc, RowSums, ColumnSumVector, int32_t(CountK) * offa * offb, k == 0); + RowsHandled = MlasPlatform.GemmU8U8Kernel(pa, PanelB, c, + PairCountK, RowsRemaining, CountN, ldc, RowSums, + ColumnSumVector, int32_t(CountK) * offa * offb, k == 0); RowsRemaining -= RowsHandled; c += ldc * RowsHandled; @@ -1286,4 +1402,330 @@ MlasGemm( } } +void +MlasGemmX8X8Threaded( + void* Context, + int32_t ThreadId + ) +/*++ + +Routine Description: + + This routine is invoked from a worker thread to execute a segment of a + QGEMM operation. + +Arguments: + + Context - Supplies the pointer to the context for the threaded operation. + + ThreadId - Supplies the current index of the threaded operation. + +Return Value: + + None. + +--*/ +{ + const auto* WorkBlock = (MLAS_GEMM_X8X8_WORK_BLOCK*)Context; + + const int32_t ThreadCountM = WorkBlock->ThreadCountM; + const int32_t ThreadCountN = WorkBlock->ThreadCountN; + + const int32_t ThreadIdM = ThreadId / ThreadCountN; + const int32_t ThreadIdN = ThreadId % ThreadCountN; + + // + // Partition the operation along the M dimension. + // + + size_t M = WorkBlock->M; + size_t m; + size_t CountM; + + MlasPartitionWork(ThreadIdM, ThreadCountM, M, &m, &CountM); + + // + // Partition the operation along the N dimension. + // + + size_t N = WorkBlock->N; + size_t n; + size_t CountN; + + const size_t BlockedN = (N + MLAS_QGEMM_STRIDEN_THREAD_ALIGN - 1) / + MLAS_QGEMM_STRIDEN_THREAD_ALIGN; + + MlasPartitionWork(ThreadIdN, ThreadCountN, BlockedN, &n, &CountN); + + n *= MLAS_QGEMM_STRIDEN_THREAD_ALIGN; + CountN *= MLAS_QGEMM_STRIDEN_THREAD_ALIGN; + + if (CountN > N - n) { + CountN = N - n; + } + + // + // Dispatch the partitioned operation. + // + + const size_t lda = WorkBlock->lda; + const size_t ldb = WorkBlock->ldb; + const size_t ldc = WorkBlock->ldc; + + const uint8_t* a = WorkBlock->A + m * lda; + const uint8_t* b = WorkBlock->B + n; + int32_t* c = WorkBlock->C + n + m * ldc; + + WorkBlock->GemmX8X8Operation(CountM, CountN, WorkBlock->K, a, lda, + WorkBlock->offa, b, ldb, WorkBlock->offb, c, ldc); +} + +void +MlasGemmX8X8Schedule( + MLAS_GEMM_X8X8_WORK_BLOCK* WorkBlock, + MLAS_THREADPOOL* ThreadPool + ) +/*++ + +Routine Description: + + This module schedules the quantized integer matrix/matrix multiply + operation (QGEMM) across one or more threads. + +Arguments: + + WorkBlock - Supplies the structure containing the GEMM parameters. + + ThreadPool - Supplies the thread pool object to use, else nullptr if the + base library threading support should be used. + +Return Value: + + None. + +--*/ +{ + const size_t M = WorkBlock->M; + const size_t N = WorkBlock->N; + const size_t K = WorkBlock->K; + + // + // Compute the number of target threads given the complexity of the SGEMM + // operation. Small requests should run using the single threaded path. + // + + double Complexity = double(M) * double(N) * double(K); + + int32_t TargetThreadCount; + + if (Complexity < double(MLAS_QGEMM_THREAD_COMPLEXITY * MLAS_MAXIMUM_THREAD_COUNT)) { + TargetThreadCount = int32_t(Complexity / double(MLAS_QGEMM_THREAD_COMPLEXITY)) + 1; + } else { + TargetThreadCount = MLAS_MAXIMUM_THREAD_COUNT; + } + + int32_t MaximumThreadCount = MlasGetMaximumThreadCount(ThreadPool); + + if (TargetThreadCount >= MaximumThreadCount) { + TargetThreadCount = MaximumThreadCount; + } + + // + // Segment the operation across multiple threads. + // + // N.B. Currently, the operation is segmented as a 1D partition, which + // works okay for operations involving skinny matrices. + // + + if (N > M) { + + const size_t BlockedN = (N + MLAS_QGEMM_STRIDEN_THREAD_ALIGN - 1) / + MLAS_QGEMM_STRIDEN_THREAD_ALIGN; + + if (size_t(TargetThreadCount) > BlockedN) { + TargetThreadCount = int32_t(BlockedN); + } + + WorkBlock->ThreadCountM = 1; + WorkBlock->ThreadCountN = TargetThreadCount; + + } else { + + if (size_t(TargetThreadCount) > M) { + TargetThreadCount = int32_t(M); + } + + WorkBlock->ThreadCountM = TargetThreadCount; + WorkBlock->ThreadCountN = 1; + } + + MlasExecuteThreaded(MlasGemmX8X8Threaded, WorkBlock, TargetThreadCount, ThreadPool); +} + +void +MLASCALL +MlasGemm( + size_t M, + size_t N, + size_t K, + const uint8_t* A, + size_t lda, + uint8_t offa, + const int8_t* B, + size_t ldb, + int8_t offb, + int32_t* C, + size_t ldc, + MLAS_THREADPOOL* ThreadPool + ) +/*++ + +Routine Description: + + This module implements the quantized integer matrix/matrix multiply + operation (QGEMM). + +Arguments: + + M - Supplies the number of rows of matrix A and matrix C. + + N - Supplies the number of columns of matrix B and matrix C. + + K - Supplies the number of columns of matrix A and the number of rows of + matrix B. + + A - Supplies the address of matrix A. + + lda - Supplies the first dimension of matrix A. + + offa - Supplies the zero point offset of matrix A. + + B - Supplies the address of matrix B. + + ldb - Supplies the first dimension of matrix B. + + offb - Supplies the zero point offset of matrix B. + + C - Supplies the address of matrix C. + + ldc - Supplies the first dimension of matrix C. + + ThreadPool - Supplies the thread pool object to use, else nullptr if the + base library threading support should be used. + +Return Value: + + None. + +--*/ +{ + MLAS_GEMM_X8X8_WORK_BLOCK WorkBlock; + + // + // Capture the GEMM parameters to the work block. + // + + WorkBlock.M = M; + WorkBlock.N = N; + WorkBlock.K = K; + WorkBlock.A = A; + WorkBlock.lda = lda; + WorkBlock.B = (const uint8_t*)B; + WorkBlock.ldb = ldb; + WorkBlock.C = C; + WorkBlock.ldc = ldc; + WorkBlock.offa = int16_t(offa); + WorkBlock.offb = int16_t(offb); + WorkBlock.GemmX8X8Operation = MlasGemmU8S8Operation; + + // + // Schedule the operation across a set of worker threads. + // + + MlasGemmX8X8Schedule(&WorkBlock, ThreadPool); +} + +void +MLASCALL +MlasGemm( + size_t M, + size_t N, + size_t K, + const uint8_t* A, + size_t lda, + uint8_t offa, + const uint8_t* B, + size_t ldb, + uint8_t offb, + int32_t* C, + size_t ldc, + MLAS_THREADPOOL* ThreadPool + ) +/*++ + +Routine Description: + + This module implements the quantized integer matrix/matrix multiply + operation (QGEMM). + +Arguments: + + M - Supplies the number of rows of matrix A and matrix C. + + N - Supplies the number of columns of matrix B and matrix C. + + K - Supplies the number of columns of matrix A and the number of rows of + matrix B. + + A - Supplies the address of matrix A. + + lda - Supplies the first dimension of matrix A. + + offa - Supplies the zero point offset of matrix A. + + B - Supplies the address of matrix B. + + ldb - Supplies the first dimension of matrix B. + + offb - Supplies the zero point offset of matrix B. + + C - Supplies the address of matrix C. + + ldc - Supplies the first dimension of matrix C. + + ThreadPool - Supplies the thread pool object to use, else nullptr if the + base library threading support should be used. + +Return Value: + + None. + +--*/ +{ + MLAS_GEMM_X8X8_WORK_BLOCK WorkBlock; + + // + // Capture the GEMM parameters to the work block. + // + + WorkBlock.M = M; + WorkBlock.N = N; + WorkBlock.K = K; + WorkBlock.A = A; + WorkBlock.lda = lda; + WorkBlock.B = B; + WorkBlock.ldb = ldb; + WorkBlock.C = C; + WorkBlock.ldc = ldc; + WorkBlock.offa = int16_t(offa); + WorkBlock.offb = int16_t(offb); + WorkBlock.GemmX8X8Operation = MlasGemmU8U8Operation; + + // + // Schedule the operation across a set of worker threads. + // + + MlasGemmX8X8Schedule(&WorkBlock, ThreadPool); +} + #endif diff --git a/onnxruntime/core/mlas/lib/quantize.cpp b/onnxruntime/core/mlas/lib/quantize.cpp index 2a04b14d25083..c81897d6f9044 100644 --- a/onnxruntime/core/mlas/lib/quantize.cpp +++ b/onnxruntime/core/mlas/lib/quantize.cpp @@ -176,7 +176,7 @@ MlasQuantizeLinearKernel( MinimumValueVector, MaximumValueVector, ZeroPointVector); #if defined(MLAS_NEON64_INTRINSICS) - vst1q_lane_u8((uint8_t*)Output + n, vreinterpretq_s32_u8(IntegerVector), 0); + vst1q_lane_u8((uint8_t*)Output + n, vreinterpretq_u8_s32(IntegerVector), 0); #else *((uint8_t*)Output + n) = (uint8_t)_mm_cvtsi128_si32(IntegerVector); #endif diff --git a/onnxruntime/core/mlas/lib/snchwc.cpp b/onnxruntime/core/mlas/lib/snchwc.cpp index 843f919f44c40..7c3193c2907fb 100644 --- a/onnxruntime/core/mlas/lib/snchwc.cpp +++ b/onnxruntime/core/mlas/lib/snchwc.cpp @@ -128,7 +128,7 @@ Routine Description: Arguments: - WorkBlock - Supplies the structure that contains the commong convolution + WorkBlock - Supplies the structure that contains the common convolution and pooling parameters. Dimensions - Supplies the number of dimensions. @@ -344,28 +344,6 @@ struct MLAS_NCHWC_NN_ALGORITHM OutputCountRightPadX(WorkBlock->OutputCountRightPad[WidthShapeIndex]) { } - - static - void - PartitionWork( - int32_t Index, - const MLAS_NCHWC_WORK_BLOCK* WorkBlock, - size_t TotalWork, - size_t* WorkIndex, - size_t* WorkRemaining - ) - { - const size_t WorkPerThread = TotalWork / WorkBlock->tids; - const size_t WorkPerThreadExtra = TotalWork % WorkBlock->tids; - - if (uint32_t(Index) < WorkPerThreadExtra) { - *WorkIndex = (WorkPerThread + 1) * Index; - *WorkRemaining = WorkPerThread + 1; - } else { - *WorkIndex = WorkPerThread * Index + WorkPerThreadExtra; - *WorkRemaining = WorkPerThread; - } - } }; constexpr size_t MLAS_NCHWC_NN_ALGORITHM::HeightShapeIndex; @@ -573,7 +551,7 @@ struct MLAS_NCHWC_GROUPED_CONV_ALGORITHM : MLAS_NCHWC_CONV_ALGORITHM size_t WorkIndex; - PartitionWork(Index, WorkBlock, TotalWork, &WorkIndex, &WorkRemaining); + MlasPartitionWork(Index, WorkBlock->tids, TotalWork, &WorkIndex, &WorkRemaining); // // Extract the current batch, group, filter cluster, and output line @@ -1004,7 +982,7 @@ struct MLAS_NCHWC_CONV_DEPTHWISE_ALGORITHM : MLAS_NCHWC_CONV_ALGORITHM size_t WorkIndex; size_t WorkRemaining; - PartitionWork(Index, WorkBlock, TotalWork, &WorkIndex, &WorkRemaining); + MlasPartitionWork(Index, WorkBlock->tids, TotalWork, &WorkIndex, &WorkRemaining); // // Extract the current batch, group block, and output line from the @@ -1138,7 +1116,7 @@ struct MLAS_NCHWC_POOL_ALGORITHM : MLAS_NCHWC_NN_ALGORITHM size_t WorkIndex; size_t WorkRemaining; - PartitionWork(Index, WorkBlock, TotalWork, &WorkIndex, &WorkRemaining); + MlasPartitionWork(Index, WorkBlock->tids, TotalWork, &WorkIndex, &WorkRemaining); size_t ph = WorkIndex % OutputHeight; const size_t BatchChannel = WorkIndex / OutputHeight; diff --git a/onnxruntime/core/mlas/lib/threading.cpp b/onnxruntime/core/mlas/lib/threading.cpp index ef30de9499bb2..ab797e431231a 100644 --- a/onnxruntime/core/mlas/lib/threading.cpp +++ b/onnxruntime/core/mlas/lib/threading.cpp @@ -46,7 +46,6 @@ MlasExecuteThreaded( } #endif - // // Fallback to OpenMP or a serialized implementation. // diff --git a/onnxruntime/core/mlas/lib/x86_64/QgemmU8S8KernelAvx2.S b/onnxruntime/core/mlas/lib/x86_64/QgemmU8S8KernelAvx2.S index 3d637fb97ec59..6a6edd7e8017b 100644 --- a/onnxruntime/core/mlas/lib/x86_64/QgemmU8S8KernelAvx2.S +++ b/onnxruntime/core/mlas/lib/x86_64/QgemmU8S8KernelAvx2.S @@ -130,6 +130,8 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackAAvx2): vpxor xmm1,xmm1,xmm1 vpxor xmm2,xmm2,xmm2 vpxor xmm3,xmm3,xmm3 + lea r13,[r10+r10*2] # compute ldb * 3 + lea rax,[r12+r12*2] # compute output stride * 3 mov rdx,rsi mov rcx,rdi lea rsi,[rsi+r10*4] # advance next matrix A by 4 rows @@ -139,16 +141,14 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackAAvx2): jb .LCopyPackA.ProcessRemainingColumnsM4 .LCopyPackA.ProcessNextColumnLoopM4: - lea rax,[rdx+r10*2] # compute matrix A plus 2 rows vmovdqu ymm4,YMMWORD PTR [rdx] vmovdqu ymm5,YMMWORD PTR [rdx+r10] - vmovdqu ymm6,YMMWORD PTR [rax] - vmovdqu ymm7,YMMWORD PTR [rax+r10] - lea rax,[rcx+r12*2] # compute matrix D plus 2 rows + vmovdqu ymm6,YMMWORD PTR [rdx+r10*2] + vmovdqu ymm7,YMMWORD PTR [rdx+r13] vmovdqu YMMWORD PTR [rcx],ymm4 vmovdqu YMMWORD PTR [rcx+r12],ymm5 - vmovdqu YMMWORD PTR [rax],ymm6 - vmovdqu YMMWORD PTR [rax+r12],ymm7 + vmovdqu YMMWORD PTR [rcx+r12*2],ymm6 + vmovdqu YMMWORD PTR [rcx+rax],ymm7 vpmaddubsw ymm4,ymm4,ymm9 # horizontal byte+byte=word per row vpaddw ymm0,ymm0,ymm4 # add words to row accumulators vpmaddubsw ymm5,ymm5,ymm9 @@ -167,16 +167,14 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackAAvx2): jz .LCopyPackA.ReduceRowSumVectorM4 test bl,16 # (CountK & 16) != 0? jz .LCopyPackA.CopyRemainingCountKLessThan16M4 - lea rax,[rdx+r10*2] # compute matrix A plus 2 rows vmovdqu xmm4,XMMWORD PTR [rdx] vmovdqu xmm5,XMMWORD PTR [rdx+r10] - vmovdqu xmm6,XMMWORD PTR [rax] - vmovdqu xmm7,XMMWORD PTR [rax+r10] - lea rax,[rcx+r12*2] # compute matrix D plus 2 rows + vmovdqu xmm6,XMMWORD PTR [rdx+r10*2] + vmovdqu xmm7,XMMWORD PTR [rdx+r13] vmovdqu XMMWORD PTR [rcx],xmm4 vmovdqu XMMWORD PTR [rcx+r12],xmm5 - vmovdqu XMMWORD PTR [rax],xmm6 - vmovdqu XMMWORD PTR [rax+r12],xmm7 + vmovdqu XMMWORD PTR [rcx+r12*2],xmm6 + vmovdqu XMMWORD PTR [rcx+rax],xmm7 vpmaddubsw xmm4,xmm4,xmm9 # horizontal byte+byte=word per row vpaddw ymm0,ymm0,ymm4 # add words to row accumulators vpmaddubsw xmm5,xmm5,xmm9 @@ -198,14 +196,13 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackAAvx2): lea rbp,.LGemmU8S8CopyPackAFrame_PaddedMatrixAData[rsp] test bl,8 # (CountK & 8) != 0? jz .LCopyPackA.CopyRemainingCountKLessThan8M4 - lea r13,[rdx+r10*2] # compute matrix A plus 2 rows mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax mov rax,QWORD PTR [rdx+r10] mov QWORD PTR [rbp+16],rax - mov rax,QWORD PTR [r13] + mov rax,QWORD PTR [rdx+r10*2] mov QWORD PTR [rbp+32],rax - mov rax,QWORD PTR [r13+r10] + mov rax,QWORD PTR [rdx+r13] mov QWORD PTR [rbp+48],rax add rdx,8 add rbp,8 # advance padded buffer destination @@ -213,14 +210,13 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackAAvx2): .LCopyPackA.CopyRemainingCountKLessThan8M4: test bl,4 # (CountK & 4) != 0? jz .LCopyPackA.CopyRemainingCountKLessThan4M4 - lea r13,[rdx+r10*2] # compute matrix A plus 2 rows mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax mov eax,DWORD PTR [rdx+r10] mov DWORD PTR [rbp+16],eax - mov eax,DWORD PTR [r13] + mov eax,DWORD PTR [rdx+r10*2] mov DWORD PTR [rbp+32],eax - mov eax,DWORD PTR [r13+r10] + mov eax,DWORD PTR [rdx+r13] mov DWORD PTR [rbp+48],eax add rdx,4 add rbp,4 # advance padded buffer destination @@ -228,14 +224,13 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackAAvx2): .LCopyPackA.CopyRemainingCountKLessThan4M4: test bl,2 # (CountK & 2) != 0? jz .LCopyPackA.CopyRemainingCountKLessThan2M4 - lea r13,[rdx+r10*2] # compute matrix A plus 2 rows movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax movzx eax,WORD PTR [rdx+r10] mov WORD PTR [rbp+16],ax - movzx eax,WORD PTR [r13] + movzx eax,WORD PTR [rdx+r10*2] mov WORD PTR [rbp+32],ax - movzx eax,WORD PTR [r13+r10] + movzx eax,WORD PTR [rdx+r13] mov WORD PTR [rbp+48],ax add rdx,2 add rbp,2 # advance padded buffer destination @@ -243,14 +238,13 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackAAvx2): .LCopyPackA.CopyRemainingCountKLessThan2M4: test bl,1 # (CountK & 1) != 0? jz .LCopyPackA.ProcessPaddedMatrixADataM4 - lea r13,[rdx+r10*2] # compute matrix A plus 2 rows movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al movzx eax,BYTE PTR [rdx+r10] mov BYTE PTR [rbp+16],al - movzx eax,BYTE PTR [r13] + movzx eax,BYTE PTR [rdx+r10*2] mov BYTE PTR [rbp+32],al - movzx eax,BYTE PTR [r13+r10] + movzx eax,BYTE PTR [rdx+r13] mov BYTE PTR [rbp+48],al // @@ -446,6 +440,7 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackBAvx2): push rbx mov r10,rdx + lea r11,[r10+r10*2] # compute ldb * 3 vpbroadcastw ymm7,WORD PTR .LGemmU8S8CopyPackBFrame_offa[rsp] vpcmpeqw ymm8,ymm8,ymm8 # generate word vector [0xFFFF] vpsrlw ymm8,ymm8,15 # generate word vector [0x0001] @@ -469,11 +464,10 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackBAvx2): jb .LCopyPackB.ProcessRemainingRowsN16 .LCopyPackB.ProcessNextRowLoopN16: - lea rax,[rdx+r10*2] # compute matrix B plus 2 rows vmovdqu xmm2,XMMWORD PTR [rdx] # load 4 rows vmovdqu xmm3,XMMWORD PTR [rdx+r10] - vmovdqu xmm4,XMMWORD PTR [rax] - vmovdqu xmm5,XMMWORD PTR [rax+r10] + vmovdqu xmm4,XMMWORD PTR [rdx+r10*2] + vmovdqu xmm5,XMMWORD PTR [rdx+r11] lea rdx,[rdx+r10*4] # advance matrix B by 4 rows .LCopyPackB.InterleaveRowDataN16: @@ -558,14 +552,13 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackBAvx2): lea rbp,.LGemmU8S8CopyPackBFrame_PaddedMatrixBData[rsp] test cl,8 # (CountN & 8) != 0? jz .LCopyPackB.CopyRemainingCountNLessThan8K4 - lea r11,[rdx+r10*2] # compute matrix B plus 2 rows mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax mov rax,QWORD PTR [rdx+r10] mov QWORD PTR [rbp+16],rax - mov rax,QWORD PTR [r11] + mov rax,QWORD PTR [rdx+r10*2] mov QWORD PTR [rbp+32],rax - mov rax,QWORD PTR [r11+r10] + mov rax,QWORD PTR [rdx+r11] mov QWORD PTR [rbp+48],rax add rdx,8 # advance matrix B add rbp,8 # advance padded buffer destination @@ -573,14 +566,13 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackBAvx2): .LCopyPackB.CopyRemainingCountNLessThan8K4: test cl,4 # (CountN & 4) != 0? jz .LCopyPackB.CopyRemainingCountNLessThan4K4 - lea r11,[rdx+r10*2] # compute matrix B plus 2 rows mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax mov eax,DWORD PTR [rdx+r10] mov DWORD PTR [rbp+16],eax - mov eax,DWORD PTR [r11] + mov eax,DWORD PTR [rdx+r10*2] mov DWORD PTR [rbp+32],eax - mov eax,DWORD PTR [r11+r10] + mov eax,DWORD PTR [rdx+r11] mov DWORD PTR [rbp+48],eax add rdx,4 # advance matrix B add rbp,4 # advance padded buffer destination @@ -588,14 +580,13 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackBAvx2): .LCopyPackB.CopyRemainingCountNLessThan4K4: test cl,2 # (CountN & 2) != 0? jz .LCopyPackB.CopyRemainingCountNLessThan2K4 - lea r11,[rdx+r10*2] # compute matrix B plus 2 rows movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax movzx eax,WORD PTR [rdx+r10] mov WORD PTR [rbp+16],ax - movzx eax,WORD PTR [r11] + movzx eax,WORD PTR [rdx+r10*2] mov WORD PTR [rbp+32],ax - movzx eax,WORD PTR [r11+r10] + movzx eax,WORD PTR [rdx+r11] mov WORD PTR [rbp+48],ax add rdx,2 # advance matrix B add rbp,2 # advance padded buffer destination @@ -603,14 +594,13 @@ C_UNDERSCORE(MlasGemmU8S8CopyPackBAvx2): .LCopyPackB.CopyRemainingCountNLessThan2K4: test cl,1 # (CountN & 1) != 0? jz .LCopyPackB.ProcessPaddedMatrixBData - lea r11,[rdx+r10*2] # compute matrix B plus 2 rows movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al movzx eax,BYTE PTR [rdx+r10] mov BYTE PTR [rbp+16],al - movzx eax,BYTE PTR [r11] + movzx eax,BYTE PTR [rdx+r10*2] mov BYTE PTR [rbp+32],al - movzx eax,BYTE PTR [r11+r10] + movzx eax,BYTE PTR [rdx+r11] mov BYTE PTR [rbp+48],al .LCopyPackB.ProcessPaddedMatrixBData: diff --git a/onnxruntime/core/optimizer/gemm_activation_fusion.cc b/onnxruntime/core/optimizer/gemm_activation_fusion.cc index 05d1c864b770b..c6c5e5316c28a 100644 --- a/onnxruntime/core/optimizer/gemm_activation_fusion.cc +++ b/onnxruntime/core/optimizer/gemm_activation_fusion.cc @@ -68,7 +68,9 @@ Status GemmActivationFusion::ApplyImpl(Graph& graph, bool& modified, int graph_l if (act_node.OpType() == "LeakyRelu") { const NodeAttributes& attrs = act_node.GetAttributes(); for (const auto& attr : attrs) { - fused_gemm.AddAttribute("leaky_relu_" + attr.first, attr.second); + AttributeProto fused_gemm_attr(attr.second); + fused_gemm_attr.set_name("leaky_relu_" + attr.first); + fused_gemm.AddAttribute("leaky_relu_" + attr.first, fused_gemm_attr); } } diff --git a/onnxruntime/core/platform/posix/stacktrace.cc b/onnxruntime/core/platform/posix/stacktrace.cc index f41ef0e1d4902..73a4e2b274f62 100644 --- a/onnxruntime/core/platform/posix/stacktrace.cc +++ b/onnxruntime/core/platform/posix/stacktrace.cc @@ -3,10 +3,44 @@ #include "core/common/common.h" +#include +#include + namespace onnxruntime { std::vector GetStackTrace() { - return {""}; -} + std::vector stack; + +#ifndef NDEBUG + constexpr int kCallstackLimit = 64; // Maximum depth of callstack + + void* array[kCallstackLimit]; + char** strings = nullptr; + + size_t size = backtrace(array, kCallstackLimit); + stack.reserve(size); + strings = backtrace_symbols(array, size); + // NOTE: To get meaningful info from the output, addr2line (or atos on osx) would need to be used. + // See https://gist.github.com/jvranish/4441299 for an example. + // + // To manually translate the output, use the value in the '()' after the executable name with addr2line + // e.g. + // Stacktrace: + // /home/me/src/github/onnxruntime/build/Linux/Debug/onnxruntime_test_all(+0x3f46cc) [0x559543faf6cc] + // + // >addr2line -f -C -e /home/me/src/github/onnxruntime/build/Linux/Debug/onnxruntime_test_all +0x3f46cc + + // hide GetStackTrace so the output starts with the 'real' location + constexpr size_t start_frame = 1; + for (size_t i = start_frame; i < size; i++) { + stack.push_back(strings[i]); + } + + free(strings); + +#endif + + return stack; +} } // namespace onnxruntime diff --git a/onnxruntime/core/platform/windows/debug_alloc.cc b/onnxruntime/core/platform/windows/debug_alloc.cc index 83c38baa05248..57267f146d884 100644 --- a/onnxruntime/core/platform/windows/debug_alloc.cc +++ b/onnxruntime/core/platform/windows/debug_alloc.cc @@ -232,12 +232,8 @@ Memory_LeakCheck::~Memory_LeakCheck() { _snprintf_s(buffer, _TRUNCATE, "%d bytes of memory leaked in %d allocations", leaked_bytes, leak_count); string.append(buffer); - // If we're being actively debugged, show a message box to get the dev's attention - if (IsDebuggerPresent()) - MessageBoxA(nullptr, string.c_str(), "Warning", MB_OK | MB_ICONWARNING); - else { - // If we're on the command line (like on a build machine), output to the console and exit(-1) - std::cout << "\n----- MEMORY LEAKS: " << string.c_str() << "\n"; + std::cout << "\n----- MEMORY LEAKS: " << string.c_str() << "\n"; + if (!IsDebuggerPresent()) { exit(-1); } diff --git a/onnxruntime/core/providers/acl/math/gemm.h b/onnxruntime/core/providers/acl/math/gemm.h index d898ad17b1a32..c171837c6d108 100644 --- a/onnxruntime/core/providers/acl/math/gemm.h +++ b/onnxruntime/core/providers/acl/math/gemm.h @@ -21,6 +21,7 @@ //NEON #include "arm_compute/runtime/NEON/functions/NEGEMM.h" #include "arm_compute/runtime/NEON/functions/NETranspose.h" +#include "arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h" #undef GEMM_ACL #define CACHE_TRANSPOSED_DATA @@ -29,7 +30,7 @@ namespace onnxruntime { namespace acl { typedef struct { - std::shared_ptr layer; + std::shared_ptr layer; std::shared_ptr a, b, c, d; std::shared_ptr mm_layer; } ACLNEGEMM; @@ -51,7 +52,6 @@ class Gemm : public onnxruntime::Gemm { ORT_ENFORCE(info.GetAttr("beta", &beta_).IsOK()); } -#ifdef GEMM_ACL Status Compute(OpKernelContext* context) const override { const auto X = context->Input(0); const auto W = context->Input(1); @@ -66,6 +66,8 @@ class Gemm : public onnxruntime::Gemm { int64_t N = helper.N(); auto Y = context->Output(0, TensorShape({M, N})); + bool FC = ((alpha_ == 1 && beta_ == 1) || (alpha_ == 1 && beta_ == 0)); + int64_t K = helper.K(); LOGS_DEFAULT(VERBOSE) << "Gemm ACL:" << std::endl; if (X) LOGS_DEFAULT(VERBOSE) << "X " << X->Shape().ToString().c_str() << std::endl; @@ -93,7 +95,7 @@ class Gemm : public onnxruntime::Gemm { tGEMM.d->allocator()->init(arm_compute::TensorInfo(arm_compute::TensorShape(N, M), tGEMM.a->info()->format())); // transpose - if (trans_B_ == CblasTrans) { + if (!FC && trans_B_ == CblasTrans) { auto trans_layer = std::make_shared(); tGEMM.b->allocator()->init(arm_compute::TensorInfo(ACLTensorShape(W->Shape()), arm_compute::Format::F32)); @@ -113,13 +115,23 @@ class Gemm : public onnxruntime::Gemm { } tGEMM.mm_layer = ACLCreateMemoryManager(); - tGEMM.layer = std::make_shared(tGEMM.mm_layer); - // configure GEMM - tGEMM.layer->configure(tGEMM.a.get(), tGEMM.b.get(), tGEMM.c.get(), tGEMM.d.get(), alpha_, beta_, arm_compute::GEMMInfo()); + if(FC) { + auto layer = std::make_shared(tGEMM.mm_layer); + layer->configure(tGEMM.a.get(), tGEMM.b.get(), (B != nullptr && beta_ != 0) ? tGEMM.c.get() : nullptr, tGEMM.d.get()); + tGEMM.layer = std::move(layer); + } else { +#ifdef GEMM_ACL + auto layer = std::make_shared(tGEMM.mm_layer); + layer->configure(tGEMM.a.get(), tGEMM.b.get(), (B != nullptr && beta_ != 0) ? tGEMM.c.get() : nullptr, tGEMM.d.get(), alpha_, beta_, arm_compute::GEMMInfo()); + tGEMM.layer = std::move(layer); +#else + return onnxruntime::Gemm::Compute(context); +#endif + } // non-transpose - if (trans_B_ != CblasTrans) { + if (FC || trans_B_ != CblasTrans) { const T* b_data = W->template Data(); ACLImportMemory(tGEMM.b->allocator(), (void*)b_data, W->Shape().Size() * 4); } @@ -132,7 +144,7 @@ class Gemm : public onnxruntime::Gemm { pGEMM = &it->second; // transpose - if (trans_B_ == CblasTrans) { + if (!FC && trans_B_ == CblasTrans) { #ifndef CACHE_TRANSPOSED_DATA auto trans_layer = std::make_shared(); pGEMM->b->allocator()->init(arm_compute::TensorInfo(ACLTensorShape(W->Shape()), arm_compute::Format::F32)); @@ -193,36 +205,6 @@ class Gemm : public onnxruntime::Gemm { gemmLayers.erase(this); } -#else - - Status Compute(OpKernelContext* context) const override { - const auto X = context->Input(0); - const auto W = context->Input(1); - const auto B = context->Input(2); - - GemmHelper helper(X->Shape(), trans_A_ != CblasNoTrans, W->Shape(), trans_B_ != CblasNoTrans, B->Shape()); - - if (!helper.State().IsOK()) - return helper.State(); - - int64_t M = helper.M(); - int64_t N = helper.N(); - auto Y = context->Output(0, TensorShape({M, N})); - - int64_t K = helper.K(); - LOGS_DEFAULT(VERBOSE) << "Gemm CPU:" << std::endl; - if (X) LOGS_DEFAULT(VERBOSE) << "X " << X->Shape().ToString().c_str() << std::endl; - if (W) LOGS_DEFAULT(VERBOSE) << "W " << W->Shape().ToString().c_str() << std::endl; - if (B) LOGS_DEFAULT(VERBOSE) << "B " << B->Shape().ToString().c_str() << std::endl; - LOGS_DEFAULT(VERBOSE) << "Y " << Y->Shape().ToString().c_str() << std::endl; - LOGS_DEFAULT(VERBOSE) << "M " << (int)M << ", N " << (int)N << ", K " << (int)K << std::endl; - LOGS_DEFAULT(VERBOSE) << "Alfa " << alpha_ << ", Beta " << beta_ << std::endl; - LOGS_DEFAULT(VERBOSE) << std::endl; - - return onnxruntime::Gemm::Compute(context); - } -#endif - private: static thread_local std::map gemmLayers; diff --git a/onnxruntime/core/providers/acl/nn/conv.cc b/onnxruntime/core/providers/acl/nn/conv.cc index c191e27ed9a80..a4a42c908d24f 100644 --- a/onnxruntime/core/providers/acl/nn/conv.cc +++ b/onnxruntime/core/providers/acl/nn/conv.cc @@ -24,8 +24,14 @@ #include "arm_compute/runtime/NEON/functions/NEConvolutionLayer.h" #include "arm_compute/runtime/NEON/functions/NEDepthwiseConvolutionLayer.h" +#ifdef ACL_1902 +#include "arm_compute/core/NEON/kernels/NEDepthwiseConvolutionLayer3x3Kernel.h" +#else +#include "arm_compute/runtime/NEON/functions/assembly/NEDepthwiseConvolutionAssemblyDispatch.h" +#endif + #define CONV_ACL -#define DEPTHWISE_CPU +#undef DEPTHWISE_CPU #define PREF_DIM 4 @@ -36,7 +42,7 @@ template thread_local std::map Conv::convLayers; template -arm_compute::TensorShape Conv::ACLReshapeWeightsDepthwise(arm_compute::Tensor* kernel) { +arm_compute::TensorShape Conv::ACLReshapeWeightsDepthwise(arm_compute::Tensor* kernel) const { arm_compute::TensorShape shape = arm_compute::TensorShape(kernel->info()->tensor_shape()); shape[2] = shape[2] * shape[3]; shape[3] = 1; @@ -49,6 +55,16 @@ template Status Conv::Compute(OpKernelContext* context) const { size_t num_inputs = OpKernel::Node().InputDefs().size(); + ACLNEConv* pConv; + ConvLayersIterator it = Conv::convLayers.find((OpKernel*)this); + if (it != Conv::convLayers.end()) { + pConv = &it->second; + if(pConv->isDepthwiseCPU == true) { + Status s = onnxruntime::Conv::Compute(context); + return s; + } + } + const Tensor* X = context->Input(0); const Tensor* W = context->Input(1); const Tensor* B = num_inputs == 3 ? context->Input(2) : nullptr; @@ -109,9 +125,8 @@ Status Conv::Compute(OpKernelContext* context) const { ORT_NOT_IMPLEMENTED("Not implemented fused activation: ", conv_attrs_.activation); } - ACLNEConv* pConv; - ConvLayersIterator it = Conv::convLayers.find((OpKernel*)this); if (it == Conv::convLayers.end()) { + auto mm_layer = ACLCreateMemoryManager(); ACLNEConv tconv; @@ -133,6 +148,7 @@ Status Conv::Compute(OpKernelContext* context) const { const arm_compute::DataLayout data_layout = tconv.in->info()->data_layout(); const int idx_channel = arm_compute::get_data_layout_dimension_index(data_layout, arm_compute::DataLayoutDimension::CHANNEL); bool isDepthwise = (1 == tconv.k->info()->tensor_shape()[idx_channel]); + tconv.isDepthwiseCPU = isDepthwise; std::vector aclStrides(2); aclStrides[0] = (strides.size() == 2) ? strides[1] : 1; @@ -160,29 +176,71 @@ Status Conv::Compute(OpKernelContext* context) const { arm_compute::PadStrideInfo aclPadStride = arm_compute::PadStrideInfo(aclStrides[0], aclStrides[1], aclPads[0], aclPads[1], aclPads[2], aclPads[3], arm_compute::DimensionRoundingType::FLOOR); + unsigned int aclDilation0 = (dilations.size() == 2) ? dilations[1] : 1; if (isDepthwise) { #ifdef DEPTHWISE_CPU Status s = onnxruntime::Conv::Compute(context); + std::pair ret; + ret = Conv::convLayers.insert(std::pair((OpKernel*)this, tconv)); return s; #else - auto layer = std::make_shared(); tconv.k->info()->set_tensor_shape(ACLReshapeWeightsDepthwise(tconv.k.get())); - layer->configure(tconv.in.get(), tconv.k.get(), (B != nullptr) ? tconv.b.get() : nullptr, tconv.out.get(), - aclPadStride, 1 /* depth multiplier */, - acl_activ_enabled ? arm_compute::ActivationLayerInfo(acl_activ_func, conv_attrs_.alpha) : arm_compute::ActivationLayerInfo()); - tconv.layer = std::move(layer); + + // in the configure function for NEDepthwiseConvolutionLayer3x3, there is a separation based on the optimization +#ifdef ACL_1902 + bool optimizable = + arm_compute::NEDepthwiseConvolutionLayer3x3Kernel::is_optimized_execution_possible(tconv.in->info()->tensor_shape(), + aclPadStride, + tconv.in->info()->data_type(), + 1 /* depth multiplier */, + tconv.in->info()->data_layout()); +#else + bool optimizable = + arm_compute::NEDepthwiseConvolutionAssemblyDispatch::is_optimized_supported(tconv.in->info(), + tconv.k->info(), + aclPadStride, + 1 /* depth multiplier */, + arm_compute::Size2D(aclDilation0, dilations[0])); +#endif + if(optimizable) { + //optimized depthwise convolution + auto layer = std::make_shared(); +#ifdef ACL_1902 + layer->configure(tconv.in.get(), tconv.k.get(), (B != nullptr) ? tconv.b.get() : nullptr, tconv.out.get(), + aclPadStride, 1 /* depth multiplier */, + acl_activ_enabled ? arm_compute::ActivationLayerInfo(acl_activ_func, conv_attrs_.alpha) : arm_compute::ActivationLayerInfo()); +#else + layer->configure(tconv.in.get(), tconv.k.get(), (B != nullptr) ? tconv.b.get() : nullptr, tconv.out.get(), + aclPadStride, 1 /* depth multiplier */, + acl_activ_enabled ? arm_compute::ActivationLayerInfo(acl_activ_func, conv_attrs_.alpha) : arm_compute::ActivationLayerInfo(), + arm_compute::Size2D(aclDilation0, dilations[0])); +#endif + tconv.layer = std::move(layer); + tconv.isDepthwiseCPU = false; + } else { + // cpu depthwise convolution + Status s = onnxruntime::Conv::Compute(context); + std::pair ret; + ret = Conv::convLayers.insert(std::pair((OpKernel*)this, tconv)); + return s; + } #endif } else { - unsigned int aclDilation0 = (dilations.size() == 2) ? dilations[1] : 1; - - auto layer = std::make_shared(mm_layer); - layer->configure(tconv.in.get(), tconv.k.get(), (B != nullptr) ? tconv.b.get() : nullptr, tconv.out.get(), - aclPadStride, - arm_compute::WeightsInfo(), arm_compute::Size2D(aclDilation0, dilations[0]), - acl_activ_enabled ? arm_compute::ActivationLayerInfo(acl_activ_func, conv_attrs_.alpha) : arm_compute::ActivationLayerInfo(), - false, conv_attrs_.group); - tconv.layer = std::move(layer); + if(tconv.k->info()->tensor_shape()[0] == 1 && tconv.k->info()->tensor_shape()[1] == 1) { + //pointwise convolution + Status s = onnxruntime::Conv::Compute(context); + return s; + } else { + //convolution + auto layer = std::make_shared(mm_layer); + layer->configure(tconv.in.get(), tconv.k.get(), (B != nullptr) ? tconv.b.get() : nullptr, tconv.out.get(), + aclPadStride, + arm_compute::WeightsInfo(), arm_compute::Size2D(aclDilation0, dilations[0]), + acl_activ_enabled ? arm_compute::ActivationLayerInfo(acl_activ_func, conv_attrs_.alpha) : arm_compute::ActivationLayerInfo(), + false, conv_attrs_.group); + tconv.layer = std::move(layer); + } } tconv.out->info()->set_format(tconv.in->info()->format()); @@ -224,6 +282,7 @@ Status Conv::Compute(OpKernelContext* context) const { pConv->b->allocator()->free(); pConv->out->allocator()->free(); + return Status::OK(); } #else diff --git a/onnxruntime/core/providers/acl/nn/conv.h b/onnxruntime/core/providers/acl/nn/conv.h index 218fe9173dddd..e4e413e454349 100644 --- a/onnxruntime/core/providers/acl/nn/conv.h +++ b/onnxruntime/core/providers/acl/nn/conv.h @@ -30,7 +30,7 @@ typedef struct std::shared_ptr k; std::shared_ptr b; std::shared_ptr out; - bool isDeptwise; + bool isDepthwiseCPU; } ACLNEConv; typedef std::map::iterator ConvLayersIterator; @@ -54,7 +54,7 @@ class Conv final : public onnxruntime::Conv { ConvAttributes conv_attrs_; ACLExecutionProvider* provider_; - arm_compute::TensorShape ACLReshapeWeightsDepthwise(arm_compute::Tensor* kernel); + arm_compute::TensorShape ACLReshapeWeightsDepthwise(arm_compute::Tensor* kernel) const; }; } // namespace mkl_dnn } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index 625b6a9c40a05..654173c54ea4d 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -270,6 +270,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, float, Where); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, int32_t, Where); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, int64_t, Where); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, uint8_t, Where); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, 10, Flatten); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, BatchNormalization); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, 10, Gemm); @@ -415,7 +416,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, GatherND); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Range); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Unique); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, TopK); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, float, TopK); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, int64_t, TopK); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, int64_t_int64_t_int64_t, OneHot); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, float_int64_t_int64_t, OneHot); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, int64_t_string_int64_t, OneHot); @@ -890,6 +892,8 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { Where)>, BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo( - onnxruntime::make_unique(device_info.factory(0)))); + InsertAllocator(device_info.factory(0)); #else //Disable Arena allocator for x86_32 build because it may run into infinite loop when integer overflow happens #if defined(__amd64__) || defined(_M_AMD64) @@ -50,10 +48,7 @@ class CPUExecutionProvider : public IExecutionProvider { #else ORT_UNUSED_PARAMETER(info); #endif - InsertAllocator( - std::shared_ptr( - onnxruntime::make_unique(device_info.factory(0)))); - + InsertAllocator(device_info.factory(0)); #endif } diff --git a/onnxruntime/core/providers/cpu/math/gemm.h b/onnxruntime/core/providers/cpu/math/gemm.h index a309ad144a01e..227bf46c02d1e 100644 --- a/onnxruntime/core/providers/cpu/math/gemm.h +++ b/onnxruntime/core/providers/cpu/math/gemm.h @@ -8,7 +8,6 @@ #include "core/util/math.h" #include "core/util/math_cpuonly.h" #include "gemm_helper.h" -#include "core/framework/op_kernel_context_internal.h" namespace onnxruntime { @@ -28,7 +27,7 @@ class Gemm : public OpKernel { } Status Compute(OpKernelContext* context) const override { - concurrency::ThreadPool* tp = context->GetOperatorThreadPool(); + concurrency::ThreadPool* thread_pool = context->GetOperatorThreadPool(); const auto* X = context->Input(0); const auto* W = context->Input(1); @@ -82,7 +81,7 @@ class Gemm : public OpKernel { // but passing 0 for beta is cheaper and it will ignore any junk in the output buffer B != nullptr ? beta_ : 0, y_data, - tp); + thread_pool); FuseActivation(activation_, y_data, M * N, leaky_relu_alpha_); diff --git a/onnxruntime/core/providers/cpu/math/matmul.cc b/onnxruntime/core/providers/cpu/math/matmul.cc index cff1b1066a26f..5e4c5fb2a55ef 100644 --- a/onnxruntime/core/providers/cpu/math/matmul.cc +++ b/onnxruntime/core/providers/cpu/math/matmul.cc @@ -1,8 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/framework/op_kernel_context_internal.h" -#include "core/providers/cpu/math/matmul.h" +#include "core/providers/cpu/math/matmul.h" #include "core/util/math.h" #include "core/util/math_cpuonly.h" #include "matmul_helper.h" diff --git a/onnxruntime/core/providers/cpu/math/matmul_integer.cc b/onnxruntime/core/providers/cpu/math/matmul_integer.cc index b76470e098928..26bfaaa013970 100644 --- a/onnxruntime/core/providers/cpu/math/matmul_integer.cc +++ b/onnxruntime/core/providers/cpu/math/matmul_integer.cc @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/framework/data_types_internal.h" -#include "core/framework/op_kernel_context_internal.h" #include "core/providers/cpu/math/matmul_integer.h" #include "core/providers/cpu/math/matmul_helper.h" #include "core/util/qmath.h" diff --git a/onnxruntime/core/providers/cpu/math/top_k.cc b/onnxruntime/core/providers/cpu/math/top_k.cc index 5e8264ef939ed..4eec811d85855 100644 --- a/onnxruntime/core/providers/cpu/math/top_k.cc +++ b/onnxruntime/core/providers/cpu/math/top_k.cc @@ -276,24 +276,33 @@ Status TopK<10, float>::Compute(OpKernelContext* p_op_kernel_context) const { } // Opset ver - 11 -template <> -TopK<11, float>::TopK(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_info) { + +static void TopkOpset11ConstructorCommon(const OpKernelInfo& op_kernel_info, + int& axis, bool& largest, bool& sorted) { int64_t axis_temp; ORT_ENFORCE(op_kernel_info.GetAttr("axis", &axis_temp).IsOK()); - axis_ = gsl::narrow_cast(axis_temp); + axis = gsl::narrow_cast(axis_temp); int64_t largest_temp; ORT_ENFORCE(op_kernel_info.GetAttr("largest", &largest_temp).IsOK()); - largest_ = largest_temp == 1 ? true : false; + largest = largest_temp == 1 ? true : false; int64_t sorted_temp; ORT_ENFORCE(op_kernel_info.GetAttr("sorted", &sorted_temp).IsOK()); - sorted_ = sorted_temp == 1 ? true : false; + sorted = sorted_temp == 1 ? true : false; } -// Opset ver - 11 template <> -Status TopK<11, float>::Compute(OpKernelContext* p_op_kernel_context) const { +TopK<11, float>::TopK(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_info) { + TopkOpset11ConstructorCommon(op_kernel_info, axis_, largest_, sorted_); +} + +template <> +TopK<11, int64_t>::TopK(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_info) { + TopkOpset11ConstructorCommon(op_kernel_info, axis_, largest_, sorted_); +} + +static Status ComputeImplOpset11(OpKernelContext* p_op_kernel_context, int axis, bool is_largest, bool is_sorted) { const auto* X = p_op_kernel_context->Input(0); const auto* Y = p_op_kernel_context->Input(1); if (X == nullptr || Y == nullptr) { @@ -312,7 +321,18 @@ Status TopK<11, float>::Compute(OpKernelContext* p_op_kernel_context) const { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "value of k must not be negative"); } - return TopKImpl(p_op_kernel_context, X, axis_, gsl::narrow_cast(parsed_input_k), largest_, sorted_); + return TopKImpl(p_op_kernel_context, X, axis, gsl::narrow_cast(parsed_input_k), is_largest, is_sorted); +} + +// Opset ver - 11 +template <> +Status TopK<11, float>::Compute(OpKernelContext* p_op_kernel_context) const { + return ComputeImplOpset11(p_op_kernel_context, axis_, largest_, sorted_); +} + +template <> +Status TopK<11, int64_t>::Compute(OpKernelContext* p_op_kernel_context) const { + return ComputeImplOpset11(p_op_kernel_context, axis_, largest_, sorted_); } // Register necessary kernels @@ -329,10 +349,16 @@ ONNX_CPU_OPERATOR_VERSIONED_KERNEL(TopK, 10, 10, .TypeConstraint("I", DataTypeImpl::GetTensorType()), TopK<10, float>); -ONNX_CPU_OPERATOR_KERNEL(TopK, 11, - KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()) - .TypeConstraint("I", DataTypeImpl::GetTensorType()), - TopK<11, float>); +#define REGISTER_TOPK_TYPED_KERNEL(OPSET, TYPE) \ + ONNX_CPU_OPERATOR_TYPED_KERNEL(TopK, \ + OPSET, \ + TYPE, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("I", DataTypeImpl::GetTensorType()), \ + TopK); + +REGISTER_TOPK_TYPED_KERNEL(11, float); +REGISTER_TOPK_TYPED_KERNEL(11, int64_t); } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/conv.cc b/onnxruntime/core/providers/cpu/nn/conv.cc index f7023a98d3209..e213db54fc426 100644 --- a/onnxruntime/core/providers/cpu/nn/conv.cc +++ b/onnxruntime/core/providers/cpu/nn/conv.cc @@ -78,7 +78,7 @@ Status Conv::Compute(OpKernelContext* context) const { output_shape.GetDims().end()); const size_t kernel_rank = kernel_shape.size(); - concurrency::ThreadPool* tp = context->GetOperatorThreadPool(); + concurrency::ThreadPool* thread_pool = context->GetOperatorThreadPool(); for (int image_id = 0; image_id < N; ++image_id) { for (int group_id = 0; group_id < conv_attrs_.group; ++group_id) { @@ -125,7 +125,7 @@ Status Conv::Compute(OpKernelContext* context) const { col_buffer_data, 0, Ydata + group_id * Y_offset, - tp); + thread_pool); } if (B != nullptr) { @@ -186,7 +186,7 @@ Status Conv::Compute(OpKernelContext* context) const { auto* Ydata = Y->template MutableData(); const size_t kernel_rank = kernel_shape.size(); - concurrency::ThreadPool* tp = context->GetOperatorThreadPool(); + concurrency::ThreadPool* thread_pool = context->GetOperatorThreadPool(); if (kernel_rank == 2 || kernel_rank == 3) { MLAS_CONV_PARAMETERS Parameters; @@ -205,7 +205,7 @@ Status Conv::Compute(OpKernelContext* context) const { static_cast(M / conv_attrs_.group), &activation_, &WorkingBufferSize, - tp); + thread_pool); auto working_data = WorkingBufferSize > 0 ? alloc->Alloc(sizeof(float) * WorkingBufferSize) : nullptr; BufferUniquePtr working_buffer(working_data, BufferDeleter(alloc)); @@ -216,7 +216,7 @@ Status Conv::Compute(OpKernelContext* context) const { Bdata, static_cast(working_buffer.get()), Ydata, - tp); + thread_pool); } else { const int64_t input_image_size = input_shape.Size(); const int64_t output_image_size = output_shape.Size(); @@ -262,7 +262,7 @@ Status Conv::Compute(OpKernelContext* context) const { col_buffer_data, 0, Ydata + group_id * Y_offset, - tp); + thread_pool); } MlasActivation(&activation_, Ydata, Bdata, M, output_image_size, output_image_size); diff --git a/onnxruntime/core/providers/cpu/nn/conv_integer.cc b/onnxruntime/core/providers/cpu/nn/conv_integer.cc index 7e5951d4ec699..53ce6e1070f6a 100644 --- a/onnxruntime/core/providers/cpu/nn/conv_integer.cc +++ b/onnxruntime/core/providers/cpu/nn/conv_integer.cc @@ -90,6 +90,7 @@ Status ConvInteger::Compute(OpKernelContext* context) const { output_shape.GetDims().end()); const size_t kernel_rank = kernel_shape.size(); + concurrency::ThreadPool* thread_pool = context->GetOperatorThreadPool(); for (int image_id = 0; image_id < N; ++image_id) { for (int group_id = 0; group_id < conv_attrs_.group; ++group_id) { @@ -139,7 +140,7 @@ Status ConvInteger::Compute(OpKernelContext* context) const { input_offset, Ydata + group_id * Y_offset, static_cast(output_image_size), - nullptr); + thread_pool); } Xdata += X_offset * conv_attrs_.group; diff --git a/onnxruntime/core/providers/cpu/nn/conv_transpose.cc b/onnxruntime/core/providers/cpu/nn/conv_transpose.cc index e025c12979e36..ab763b8d9f788 100644 --- a/onnxruntime/core/providers/cpu/nn/conv_transpose.cc +++ b/onnxruntime/core/providers/cpu/nn/conv_transpose.cc @@ -16,7 +16,6 @@ /* Modifications Copyright (c) Microsoft. */ #include "core/providers/cpu/nn/conv_transpose.h" -#include "core/framework/op_kernel_context_internal.h" #include "core/util/math.h" #include "core/util/math_cpuonly.h" @@ -42,7 +41,7 @@ Status ConvTranspose::Compute(OpKernelContext* context) const { template Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_padding) const { - concurrency::ThreadPool* tp = context->GetOperatorThreadPool(); + concurrency::ThreadPool* thread_pool = context->GetOperatorThreadPool(); size_t num_inputs = OpKernel::Node().InputDefs().size(); ConvTransposeAttributes::Prepare p; @@ -87,7 +86,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ Xdata + group_id * X_offset, 0, col_buffer_data, - tp); + thread_pool); // Col2im math::Col2im( @@ -136,7 +135,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ Xdata + group_id * X_offset, 0, col_buffer_data, - tp); + thread_pool); // Col2im math::Col2imNd( @@ -155,7 +154,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ } if (p.B != nullptr) { - + auto Ymatrix = EigenMatrixMap(Ydata, output_size, p.num_output_channels); auto Bvec = ConstEigenVectorMap(p.B->template Data(), p.num_output_channels); Ymatrix.rowwise() += Bvec.transpose(); diff --git a/onnxruntime/core/providers/cpu/nn/pool.cc b/onnxruntime/core/providers/cpu/nn/pool.cc index ec7a5904c754d..8bf630eb056a9 100644 --- a/onnxruntime/core/providers/cpu/nn/pool.cc +++ b/onnxruntime/core/providers/cpu/nn/pool.cc @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/framework/op_kernel_context_internal.h" #include "core/providers/cpu/nn/pool.h" using namespace ::onnxruntime::common; diff --git a/onnxruntime/core/providers/cpu/nn/tfidfvectorizer.cc b/onnxruntime/core/providers/cpu/nn/tfidfvectorizer.cc index 176a9d5f4356d..7e9a4636f1afc 100644 --- a/onnxruntime/core/providers/cpu/nn/tfidfvectorizer.cc +++ b/onnxruntime/core/providers/cpu/nn/tfidfvectorizer.cc @@ -388,7 +388,7 @@ TfIdfVectorizer::TfIdfVectorizer(const OpKernelInfo& info) : OpKernel(info), imp } else { auto before_insert = impl_->str_set_.size(); Emplace(impl_->pool_strings_.begin() + start_idx, ngrams, ngram_size, ngram_id, impl_->str_set_); - ORT_ENFORCE((before_insert + ngrams) == impl_->str_set_.size(), "poll_strings duplicate ", std::to_string(ngram_size), "-grams detected"); + ORT_ENFORCE((before_insert + ngrams) == impl_->str_set_.size(), "pool_strings duplicate ", std::to_string(ngram_size), "-grams detected"); } } else { ngram_id += ngrams; diff --git a/onnxruntime/core/providers/cpu/tensor/where_op.cc b/onnxruntime/core/providers/cpu/tensor/where_op.cc index d518352af85a0..7585943c00453 100644 --- a/onnxruntime/core/providers/cpu/tensor/where_op.cc +++ b/onnxruntime/core/providers/cpu/tensor/where_op.cc @@ -22,7 +22,7 @@ namespace onnxruntime { WHERE_TYPED_KERNEL_WITH_TYPE_NAME(type, type) // start with a subset of types, enable more as needed... -//WHERE_TYPED_KERNEL(uint8_t) +WHERE_TYPED_KERNEL(uint8_t) //WHERE_TYPED_KERNEL(uint16_t) //WHERE_TYPED_KERNEL(uint32_t) //WHERE_TYPED_KERNEL(uint64_t) diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc index 375d714612c1f..92f7ba87e6b70 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc @@ -539,6 +539,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 9, float, Where); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 9, int32_t, Where); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 9, int64_t, Where); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 9, uint8_t, Where); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 9, bool, NonZero); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 9, uint8_t, NonZero); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 9, int32_t, NonZero); @@ -1004,6 +1005,7 @@ static void RegisterCudaKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, diff --git a/onnxruntime/core/providers/cuda/tensor/where.cc b/onnxruntime/core/providers/cuda/tensor/where.cc index 142dc9294414e..1e341f3f072cb 100644 --- a/onnxruntime/core/providers/cuda/tensor/where.cc +++ b/onnxruntime/core/providers/cuda/tensor/where.cc @@ -183,6 +183,7 @@ Status Where::ComputeInternal(OpKernelContext* context) const { #define SPECIALIZED_COMPUTE(T) \ SPECIALIZED_COMPUTE_WITH_NAME(T, T) +SPECIALIZED_COMPUTE(uint8_t) SPECIALIZED_COMPUTE(int32_t) SPECIALIZED_COMPUTE(int64_t) SPECIALIZED_COMPUTE(float) diff --git a/onnxruntime/core/providers/cuda/tensor/where_impl.cu b/onnxruntime/core/providers/cuda/tensor/where_impl.cu index 7bde79a2765da..6c7f92dcf7197 100644 --- a/onnxruntime/core/providers/cuda/tensor/where_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/where_impl.cu @@ -120,6 +120,7 @@ void WhereImpl( T* output_data, \ size_t count); +SPECIALIZED_IMPL(uint8_t) SPECIALIZED_IMPL(int32_t) SPECIALIZED_IMPL(int64_t) SPECIALIZED_IMPL(float) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/inc/IWinmlExecutionProvider.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/inc/IWinmlExecutionProvider.h index 86c5e8a22ba0c..0415976b58263 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/inc/IWinmlExecutionProvider.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/inc/IWinmlExecutionProvider.h @@ -94,14 +94,22 @@ namespace Windows::AI::MachineLearning::Adapter const void* executionHandle, DmlGraphNodeCreateInfo* graphNodeCreateInfo )>; - + struct GraphNodeFactoryRegistration { GraphNodeFactory factory; std::optional requiredInputCount; - std::vector requiredConstantCpuInputs; bool requiresFloatFormatsExceptConstInputs = false; }; - using GraphNodeFactoryMap = std::unordered_map>; -} // namespace Windows::AI::MachineLearning::Adapter \ No newline at end of file + using KernelSupportQuery = std::function; + + struct InternalRegistrationInfo + { + std::vector requiredConstantCpuInputs; + std::optional graphNodeFactoryRegistration; + KernelSupportQuery supportQuery; + }; + + using InternalRegistrationInfoMap = std::unordered_map>; +} \ No newline at end of file diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp index dae166d39761a..ff4061bc8e102 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp @@ -9,7 +9,7 @@ namespace Windows::AI::MachineLearning::Adapter AbiCustomRegistry::AbiCustomRegistry() : m_kernelRegistry(std::make_shared()), - m_graphNodeFactoryMap(std::make_shared()) + m_internalRegInfoMap(std::make_shared()) { } @@ -321,13 +321,14 @@ HRESULT STDMETHODCALLTYPE AbiCustomRegistry::RegisterOperatorKernel( IMLOperatorKernelFactory* operatorKernelFactory, _In_opt_ IMLOperatorShapeInferrer* shapeInferrer) const noexcept { - return RegisterOperatorKernel(opKernel, operatorKernelFactory, shapeInferrer, false, false, false); + return RegisterOperatorKernel(opKernel, operatorKernelFactory, shapeInferrer, nullptr, false, false, false); } HRESULT STDMETHODCALLTYPE AbiCustomRegistry::RegisterOperatorKernel( const MLOperatorKernelDescription* opKernel, IMLOperatorKernelFactory* operatorKernelFactory, _In_opt_ IMLOperatorShapeInferrer* shapeInferrer, + _In_opt_ IMLOperatorSupportQueryPrivate* supportQuery, bool isInternalOperator, bool canAliasFirstInput, bool supportsGraph, @@ -449,63 +450,91 @@ HRESULT STDMETHODCALLTYPE AbiCustomRegistry::RegisterOperatorKernel( }; onnxruntime::KernelCreateInfo create_info(builder.Build(), lotusKernelCreateFn); + onnxruntime::KernelDef* kernelDef = create_info.kernel_def.get(); - if (supportsGraph) + if (isInternalOperator) { + auto regInfo = std::make_shared(); + regInfo->requiredConstantCpuInputs = constantCpuInputCapture; + // Only internal operators support usage in DML graphs - if (!isInternalOperator) + if (supportsGraph) { - THROW_HR(E_INVALIDARG); + GraphNodeFactoryRegistration graphReg; + graphReg.factory = + [kernelFactoryCapture, + requiresInputShapesAtCreation, + requiresOutputShapesAtCreation, + shapeInferrerCapture, + defaultAttributesCapture, + constantCpuInputCapture](const onnxruntime::Node& node, MLOperatorTensorGetter& constantInputGetter, const void* executionHandle, DmlGraphNodeCreateInfo* graphNodeCreateInfo) + { + onnxruntime::ProtoHelperNodeContext nodeContext(node); + onnxruntime::OpNodeProtoHelper protoHelper(&nodeContext); + + // Use the same list of required constant inputs for the shape inferrer and the kernel. + EdgeShapes outputShapes; + InferAndVerifyOutputSizes(node, &defaultAttributesCapture, shapeInferrerCapture.Get(), constantCpuInputCapture, constantInputGetter, nullptr, outputShapes); + + // Create the kernel while allowing input shape and output shape queries according to options + ComPtr kernelInfoWrapper = wil::MakeOrThrow( + &protoHelper, + executionHandle, + true, + &outputShapes, + &defaultAttributesCapture, + graphNodeCreateInfo, + constantCpuInputCapture, + constantInputGetter); + + Microsoft::WRL::ComPtr kernel; + THROW_IF_FAILED(kernelFactoryCapture->CreateKernel(kernelInfoWrapper.Get(), kernel.GetAddressOf())); + kernelInfoWrapper->Close(); + }; + + if (requiredInputCountForGraph) + { + graphReg.requiredInputCount = *requiredInputCountForGraph; + } + + graphReg.requiresFloatFormatsExceptConstInputs = requiresFloatFormatsForGraph; + regInfo->graphNodeFactoryRegistration = graphReg; } - auto registration = std::make_shared(); + if (supportQuery) + { + ComPtr supportQueryCapture = supportQuery; - registration->factory = - [kernelFactoryCapture, - requiresInputShapesAtCreation, - requiresOutputShapesAtCreation, - shapeInferrerCapture, - defaultAttributesCapture, - constantCpuInputCapture](const onnxruntime::Node& node, MLOperatorTensorGetter& constantInputGetter, const void* executionHandle, DmlGraphNodeCreateInfo* graphNodeCreateInfo) + regInfo->supportQuery = [supportQueryCapture, defaultAttributesCapture](const onnxruntime::Node& node) { onnxruntime::ProtoHelperNodeContext nodeContext(node); onnxruntime::OpNodeProtoHelper protoHelper(&nodeContext); - - // Use the same list of required constant inputs for the shape inferrer and the kernel. - EdgeShapes outputShapes; - InferAndVerifyOutputSizes(node, &defaultAttributesCapture, shapeInferrerCapture.Get(), constantCpuInputCapture, constantInputGetter, nullptr, outputShapes); - + // Create the kernel while allowing input shape and output shape queries according to options - ComPtr kernelInfoWrapper = wil::MakeOrThrow( + ComPtr supportContext = wil::MakeOrThrow( &protoHelper, - executionHandle, - true, - &outputShapes, - &defaultAttributesCapture, - graphNodeCreateInfo, - constantCpuInputCapture, - constantInputGetter); - - Microsoft::WRL::ComPtr kernel; - THROW_IF_FAILED(kernelFactoryCapture->CreateKernel(kernelInfoWrapper.Get(), kernel.GetAddressOf())); - kernelInfoWrapper->Close(); - }; + &defaultAttributesCapture); - if (requiredInputCountForGraph) - { - registration->requiredInputCount = *requiredInputCountForGraph; + BOOL bSupported = FALSE; + THROW_IF_FAILED(supportQueryCapture->QuerySupport(supportContext.Get(), &bSupported)); + return !!bSupported; + }; } - registration->requiresFloatFormatsExceptConstInputs = requiresFloatFormatsForGraph; - registration->requiredConstantCpuInputs = constantCpuInputCapture; - - onnxruntime::KernelDef* kernelDef = create_info.kernel_def.get(); THROW_IF_NOT_OK(m_kernelRegistry->RegisterCustomKernel(create_info)); - (*m_graphNodeFactoryMap)[kernelDef] = registration; + (*m_internalRegInfoMap)[kernelDef] = regInfo; } else { - // For backward compatibility, this does not propagate errors + // Currently unsupported for external operators + if (canAliasFirstInput || supportsGraph || requiredInputCountForGraph || + requiresFloatFormatsForGraph || requiredConstantCpuInputs) + { + THROW_HR(E_INVALIDARG); + } + + // + // For backward compatibility, this does not propagate errors for external operators m_kernelRegistry->RegisterCustomKernel(create_info); } diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.h index 47e5bde4c8a60..51075019ed11f 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.h @@ -36,6 +36,7 @@ class AbiCustomRegistry : public WRL::Base GetGraphNodeFactoryMap() const + std::shared_ptr GetInternalRegInfoMap() const { - return m_graphNodeFactoryMap; + return m_internalRegInfoMap; } private: @@ -104,8 +105,9 @@ class AbiCustomRegistry : public WRL::Base, std::shared_ptr> m_customRegistryOpsetVerMap; - // Map between Lotus KernelDefs and graph node factories used for fusing nodes for graph compilation - mutable std::shared_ptr m_graphNodeFactoryMap; + // Map between Lotus KernelDefs and extended data used during partitioning + mutable std::shared_ptr m_internalRegInfoMap; + }; } // namespace Windows::AI::MachineLearning::Adapter diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.cpp index 12a98bb949a35..6b8bdd698aef9 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.cpp @@ -45,7 +45,7 @@ namespace Dml static void CreateDmlKernelRegistry( _Outptr_ std::shared_ptr* registry, - _Outptr_ std::shared_ptr* graphNodeFactoryMap) + _Outptr_ std::shared_ptr* internalRegInfoMap) { ComPtr abiRegistry = wil::MakeOrThrow(); Dml::RegisterDmlOperators(abiRegistry.Get()); @@ -54,7 +54,7 @@ namespace Dml auto customRegistry = *abiRegistry->GetRegistries().begin(); *registry = customRegistry->GetKernelRegistry(); - *graphNodeFactoryMap = abiRegistry->GetGraphNodeFactoryMap(); + *internalRegInfoMap = abiRegistry->GetInternalRegInfoMap(); } ExecutionProvider::ExecutionProvider( @@ -182,7 +182,7 @@ namespace Dml m_cpuInputAllocator = std::make_shared(OrtMemType::OrtMemTypeCPUInput); m_cpuOutputAllocator = std::make_shared(OrtMemType::OrtMemTypeCPUOutput); - CreateDmlKernelRegistry(&m_kernelRegistry, &m_graphNodeFactoryMap); + CreateDmlKernelRegistry(&m_kernelRegistry, &m_internalRegInfoMap); } HRESULT __stdcall ExecutionProviderImpl::GetD3DDevice(_COM_Outptr_ ID3D12Device** d3dDevice) const noexcept @@ -499,7 +499,14 @@ namespace Dml { std::string partitionKernelPrefix = std::to_string(m_partitionKernelPrefixVal++) + "_"; uint32_t deviceDataTypeMask = GetSuppportedDeviceDataTypeMask(); - return PartitionGraph(graph, *m_graphNodeFactoryMap, registries, deviceDataTypeMask, m_kernelRegistry.get(), partitionKernelPrefix); + + return PartitionGraph(graph, + *m_internalRegInfoMap, + registries, + deviceDataTypeMask, + m_kernelRegistry.get(), + partitionKernelPrefix + ); } Status ExecutionProviderImpl::CopyTensor(const onnxruntime::Tensor& src, onnxruntime::Tensor& dst) const diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.h index 979279ac4fc3c..4c14420441a0b 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.h @@ -170,7 +170,7 @@ namespace Dml std::shared_ptr m_cpuInputAllocator; std::shared_ptr m_cpuOutputAllocator; std::shared_ptr m_kernelRegistry; - std::shared_ptr m_graphNodeFactoryMap; + std::shared_ptr m_internalRegInfoMap; mutable uint64_t m_partitionKernelPrefixVal = 0; bool m_closed = false; diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphDescBuilder.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphDescBuilder.cpp index ff5dd3720ac65..622b7b96cb09c 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphDescBuilder.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphDescBuilder.cpp @@ -127,7 +127,7 @@ namespace Dml::GraphDescBuilder const onnxruntime::Node& node = *graph.GetNode(sortedNodeIndex); const GraphNodeProperties& graphNodeProps = graphNodePropertyMap.find(GetUniqueNodeName(node))->second; - const auto& requiredConstantCpuInputs = graphNodeProps.graphNodeFactoryRegistration->requiredConstantCpuInputs; + const auto& requiredConstantCpuInputs = graphNodeProps.internalRegInfo->requiredConstantCpuInputs; MLOperatorTensorGetter constantCpuNodeInputGetter = [&node, &constantCpuGraphInputGetter, &requiredConstantCpuInputs](uint32_t inputIndex) { @@ -144,7 +144,7 @@ namespace Dml::GraphDescBuilder }; DmlGraphNodeCreateInfo graphNodeInfo; - graphNodeProps.graphNodeFactoryRegistration->factory( + graphNodeProps.internalRegInfo->graphNodeFactoryRegistration->factory( node, constantCpuNodeInputGetter, executionHandle, diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphDescBuilder.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphDescBuilder.h index 9a28c73a10e4f..02319daab0ab9 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphDescBuilder.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphDescBuilder.h @@ -9,8 +9,8 @@ namespace Dml { struct GraphNodeProperties { - std::shared_ptr - graphNodeFactoryRegistration; + std::shared_ptr + internalRegInfo; // These are currently passed from the partitioning step since the only DML operators current // supporting graph nodes don't customize the order of edges or shapes, other than coercing diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp index 50c5121178955..cfd965eef3973 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp @@ -172,14 +172,14 @@ namespace Dml return true; } - bool NodeTensorTypesSupportedInGraph(const onnxruntime::Node& node, const GraphNodeFactoryRegistration& registration) + bool NodeTensorTypesSupportedInGraph(const onnxruntime::Node& node, const InternalRegistrationInfo& registration) { for (size_t i = 0; i < node.InputDefs().size(); ++i) { bool isConstantCpuInput = std::find(registration.requiredConstantCpuInputs.begin(), registration.requiredConstantCpuInputs.end(), i) != registration.requiredConstantCpuInputs.end(); - if (!isConstantCpuInput && !NodeArgSupportedInGraph(node.InputDefs()[i], registration.requiresFloatFormatsExceptConstInputs)) + if (!isConstantCpuInput && !NodeArgSupportedInGraph(node.InputDefs()[i], registration.graphNodeFactoryRegistration->requiresFloatFormatsExceptConstInputs)) { return false; } @@ -187,7 +187,7 @@ namespace Dml for (auto arg : node.OutputDefs()) { - if (!NodeArgSupportedInGraph(arg, registration.requiresFloatFormatsExceptConstInputs)) + if (!NodeArgSupportedInGraph(arg, registration.graphNodeFactoryRegistration->requiresFloatFormatsExceptConstInputs)) { return false; } @@ -196,8 +196,31 @@ namespace Dml return true; } + bool TryGetTensorDataType( + const onnxruntime::NodeArg& nodeArg, + _Out_ MLOperatorTensorDataType* onnxElementType + ) + { + *onnxElementType = MLOperatorTensorDataType::Undefined; + + const ::onnx::TypeProto* typeProto = nodeArg.TypeAsProto(); + if (typeProto != nullptr && typeProto->has_tensor_type()) + { + const ::onnx::TypeProto_Tensor& tensorTypeProto = typeProto->tensor_type(); + if (tensorTypeProto.has_elem_type()) + { + *onnxElementType = static_cast(tensorTypeProto.elem_type()); + return true; + } + } + + return false; + } + bool DoesNodeContainSupportedDataTypes( const onnxruntime::Node& node, + const std::unordered_map& nodeNameToPartitionMap, + _In_opt_ const InternalRegistrationInfo* regInfo, uint32_t supportedDeviceDataTypeMask // Each bit corresponds to each DML_TENSOR_DATA_TYPE. ) { @@ -209,33 +232,58 @@ namespace Dml { // Get the tensor element data type for this node, comparing against what the device actually supports. // Use the enumeration from the proto instead of nodeArg.Type() which returns a string. - - const ::onnx::TypeProto* typeProto = nodeArg.TypeAsProto(); - if (typeProto != nullptr && typeProto->has_tensor_type()) + MLOperatorTensorDataType onnxElementType; + if (TryGetTensorDataType(nodeArg, &onnxElementType)) { - const ::onnx::TypeProto_Tensor& tensorTypeProto = typeProto->tensor_type(); - if (tensorTypeProto.has_elem_type()) + DML_TENSOR_DATA_TYPE dmlElementType = GetDmlDataTypeFromMlDataTypeNoThrow(onnxElementType); + if (dmlElementType != DML_TENSOR_DATA_TYPE_UNKNOWN) { - MLOperatorTensorDataType onnxElementType = static_cast(tensorTypeProto.elem_type()); - DML_TENSOR_DATA_TYPE dmlElementType = GetDmlDataTypeFromMlDataTypeNoThrow(onnxElementType); - if (dmlElementType != DML_TENSOR_DATA_TYPE_UNKNOWN) + if (((1 << dmlElementType) & supportedDeviceDataTypeMask) == 0) { - if ((1 << dmlElementType) & supportedDeviceDataTypeMask) - { - // Leave nodeContainsSupportedDataTypes alone, since data type is supported. - return; - } + nodeContainsSupportedDataTypes = false; } } } - - // Else it's not supported (non-tensors, opaque data types, unsupported data types...). - nodeContainsSupportedDataTypes = false; }; // Check whether the node uses any data types which are unsupported by the device. node.ForEachDef(nodeCallback); + // DML kernels support int64 and uint64 are expected to not *introduce* values out of range, which allows + // the temporary trick using strides to emulate 64 bit tensors to work. If the source is a CPU operator, + // graph input or initializer, it's not safe to assume the input can be represented with 32 bits. + if (regInfo) + { + for (uint32_t i = 0; i < node.InputDefs().size(); ++i) + { + const auto* arg = node.InputDefs()[i]; + MLOperatorTensorDataType onnxElementType; + if (arg->Exists() && TryGetTensorDataType(*arg, &onnxElementType)) + { + if (((onnxElementType == MLOperatorTensorDataType::UInt64) || (onnxElementType == MLOperatorTensorDataType::Int64))) + { + // Look up the input partition. If it's a graph input or initializer it will be missing + // from the map. In this case or if the input comes from a CPU partition, it might be + // out of range. + const std::string& argName = arg->Name(); + auto partitionIter = nodeNameToPartitionMap.find(argName); + if (partitionIter == nodeNameToPartitionMap.end() || !partitionIter->second->IsDmlPartition()) + { + // Check if the operator handles the input on the CPU as a constant input + bool isConstantCpuInput = std::find(regInfo->requiredConstantCpuInputs.begin(), regInfo->requiredConstantCpuInputs.end(), i) != + regInfo->requiredConstantCpuInputs.end(); + + if (!isConstantCpuInput) + { + nodeContainsSupportedDataTypes = false; + break; + } + } + } + } + } + } + return nodeContainsSupportedDataTypes; } @@ -245,7 +293,8 @@ namespace Dml const onnxruntime::Node& node, const std::vector& dmlRegistries, uint32_t supportedDeviceDataTypeMask, // Each bit corresponds to each DML_TENSOR_DATA_TYPE. - const GraphNodeFactoryMap& graphNodeFactoryMap, + const InternalRegistrationInfoMap& internalRegInfoMap, + const std::unordered_map& nodeNameToPartitionMap, _Inout_ std::unordered_map& dmlNodePropertyMap, _Inout_ std::unordered_set& requiredInitializerMap, _Out_ bool* isDmlNode, @@ -260,11 +309,26 @@ namespace Dml for (auto registry : dmlRegistries) { const onnxruntime::KernelCreateInfo* createInfo = registry->TryFindKernel(node, onnxruntime::kDmlExecutionProvider); + if (!createInfo) + { + continue; + } + + auto regInfoIter = internalRegInfoMap.find(createInfo->kernel_def.get()); + std::shared_ptr internalRegInfo; + if (regInfoIter != internalRegInfoMap.end()) + { + internalRegInfo = regInfoIter->second; + if (internalRegInfo->supportQuery && !internalRegInfo->supportQuery(node)) + { + continue; + } + } // Check whether the node uses any data types which are unsupported by the device. - bool nodeContainsSupportedDataTypes = DoesNodeContainSupportedDataTypes(node, supportedDeviceDataTypeMask); + bool nodeContainsSupportedDataTypes = DoesNodeContainSupportedDataTypes(node, nodeNameToPartitionMap, internalRegInfo.get(), supportedDeviceDataTypeMask); - if (createInfo && nodeContainsSupportedDataTypes) + if (nodeContainsSupportedDataTypes) { *isDmlNode = true; @@ -274,12 +338,11 @@ namespace Dml // Ensure that shape information is known statically for the inputs and outputs of the node, // which is required for MLGraph compilation. - auto graphNodeFactorMapIter = graphNodeFactoryMap.find(createInfo->kernel_def.get()); - if (graphNodeFactorMapIter != graphNodeFactoryMap.end() && - NodeTensorTypesSupportedInGraph(node, *graphNodeFactorMapIter->second)) + if (internalRegInfo && internalRegInfo->graphNodeFactoryRegistration && + NodeTensorTypesSupportedInGraph(node, *internalRegInfo)) { bool requiredCpuInputsConstant = true; - for (uint32_t inputIndex : graphNodeFactorMapIter->second->requiredConstantCpuInputs) + for (uint32_t inputIndex : internalRegInfo->requiredConstantCpuInputs) { if (inputIndex >= node.InputDefs().size() || !node.InputDefs()[inputIndex]->Exists()) { @@ -298,7 +361,7 @@ namespace Dml requiredInitializerMap.insert(inputName); } - std::optional requiredInputCount = graphNodeFactorMapIter->second->requiredInputCount; + std::optional requiredInputCount = internalRegInfo->graphNodeFactoryRegistration->requiredInputCount; if (requiredCpuInputsConstant && TryGetStaticInputShapes( node, graphNodeProperty.first->second.inputShapes) && !ContainsEmptyDimensions(graphNodeProperty.first->second.inputShapes) && @@ -307,7 +370,7 @@ namespace Dml (requiredInputCount == std::nullopt || *requiredInputCount == node.InputDefs().size())) { *isDmlGraphNode = true; - graphNodeProperty.first->second.graphNodeFactoryRegistration = graphNodeFactorMapIter->second; + graphNodeProperty.first->second.internalRegInfo = internalRegInfo; } } @@ -550,7 +613,7 @@ namespace Dml std::vector> BuildPartitions( const onnxruntime::GraphViewer& graph, - const GraphNodeFactoryMap& graphNodeFactoryMap, + const InternalRegistrationInfoMap& internalRegInfoMap, const std::vector& registries, uint32_t supportedDeviceDataTypeMask, // Each bit corresponds to each DML_TENSOR_DATA_TYPE. std::unordered_map& graphNodePropertyMap, @@ -610,7 +673,8 @@ namespace Dml node, registries, supportedDeviceDataTypeMask, - graphNodeFactoryMap, + internalRegInfoMap, + nodeNameToPartitionMap, graphNodePropertyMap, requiredInitializerMap, /*out*/ &isDmlNode, @@ -726,7 +790,7 @@ namespace Dml std::vector> PartitionGraph( const onnxruntime::GraphViewer& graph, - const GraphNodeFactoryMap& graphNodeFactoryMap, + const InternalRegistrationInfoMap& internalRegInfoMap, const std::vector& registries, uint32_t supportedDeviceDataTypeMask, // Each bit corresponds to each DML_TENSOR_DATA_TYPE. onnxruntime::KernelRegistry* registryForPartitionKernels, @@ -741,7 +805,7 @@ namespace Dml std::unordered_map graphNodePropertyMap; std::vector> partitions = BuildPartitions( graph, - graphNodeFactoryMap, + internalRegInfoMap, registries, supportedDeviceDataTypeMask, graphNodePropertyMap, diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.h index 297ba5e7f916d..cba44ccb6f96e 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.h @@ -43,7 +43,7 @@ namespace Dml std::vector> BuildPartitions( const onnxruntime::GraphViewer& graph, - const Windows::AI::MachineLearning::Adapter::GraphNodeFactoryMap& graphNodeFactoryMap, + const Windows::AI::MachineLearning::Adapter::InternalRegistrationInfoMap& internalRegInfoMap, const std::vector& registries, uint32_t supportedDeviceDataTypeMask, // Each bit corresponds to each DML_TENSOR_DATA_TYPE. std::unordered_map& graphNodePropertyMap, @@ -53,7 +53,7 @@ namespace Dml std::vector> PartitionGraph( const onnxruntime::GraphViewer& graph, - const Windows::AI::MachineLearning::Adapter::GraphNodeFactoryMap& graphNodeFactoryMap, + const Windows::AI::MachineLearning::Adapter::InternalRegistrationInfoMap& internalRegInfoMap, const std::vector& registries, uint32_t supportedDeviceDataTypeMask, // Each bit corresponds to each DML_TENSOR_DATA_TYPE. onnxruntime::KernelRegistry* registryForPartitionKernels, diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp index f5ee36bf323cd..5df941e9ef887 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp @@ -1798,6 +1798,13 @@ HRESULT STDMETHODCALLTYPE MLKernelInferenceContext::SetOutputTensorShape( } CATCH_RETURN(); +MLSupportQueryContext::MLSupportQueryContext( + onnxruntime::OpNodeProtoHelper* info, + const AttributeMap* defaultAttributes) : + OpNodeInfoWrapper(info, nullptr, defaultAttributes, gsl::span(), MLOperatorTensorGetter()) +{ +} + bool TryGetStaticShapeIfTensor( const onnx::TypeProto* inputProto, std::vector& shapeDims) { diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h index d8a2eb6445418..0168da24ef4ca 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h @@ -616,6 +616,21 @@ void InferAndVerifyOutputSizes( const EdgeShapes* inputShapes, EdgeShapes& outputShapes); +class MLSupportQueryContext final : public OpNodeInfoWrapper< + onnxruntime::ProtoHelperNodeContext, + WRL::Base>, + onnxruntime::null_type> +{ + public: + MLSupportQueryContext() = delete; + + MLSupportQueryContext( + onnxruntime::OpNodeProtoHelper* info, + const AttributeMap* defaultAttributes); + + // TODO - ... +}; + onnxruntime::MLDataType ToTensorDataType(::MLOperatorTensorDataType type); std::string ToTypeString(MLOperatorEdgeDescription desc); onnx::AttributeProto_AttributeType ToProto(MLOperatorAttributeType type); diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorConvolution.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorConvolution.cpp index 990e0981c8172..6a24b9c704301 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorConvolution.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorConvolution.cpp @@ -14,14 +14,17 @@ class DmlOperatorConvolution : public DmlOperator, public ConvolutionHelperBase DmlOperatorConvolution( const MLOperatorKernelCreationContext& kernelInfo, DML_CONVOLUTION_MODE mode, - DML_CONVOLUTION_DIRECTION direction + DML_CONVOLUTION_DIRECTION direction, + bool hasDynamicPads ) : DmlOperator(kernelInfo), - ConvolutionHelperBase(kernelInfo, kernelInfo.GetTensorShapeDescription(), direction == DML_CONVOLUTION_DIRECTION_BACKWARD) + ConvolutionHelperBase(kernelInfo, kernelInfo.GetTensorShapeDescription(), direction == DML_CONVOLUTION_DIRECTION_BACKWARD, hasDynamicPads) { - ML_CHECK_VALID_ARGUMENT(kernelInfo.GetInputCount() >= 2); + uint32_t biasIndex = hasDynamicPads ? 3 : 2; + bool hasBiasInput = kernelInfo.GetInputCount() > biasIndex; + + std::vector> kernelInputIndices = { 0, 1, biasIndex }; - std::vector> kernelInputIndices = {0, 1, 2}; DmlOperator::Initialize(kernelInfo, kernelInputIndices); // Vibranium DirectML is limited to handle only 2D and 3D convolution (4D and 5D tensors). So for 1D tensors, @@ -32,7 +35,7 @@ class DmlOperatorConvolution : public DmlOperator, public ConvolutionHelperBase m_inputTensorDescs[1] = CreateTensorDescFromInput(kernelInfo, 1, TensorAxis::DoNotCoerce, TensorAxis::NoPlacementAdjustment, NonspatialDimensionCount, std::nullopt); // Bias is optional so only adjust it if it exists. - if (kernelInfo.GetInputCount() > 2) + if (hasBiasInput) { uint32_t inputDimSize = kernelInfo.GetTensorShapeDescription().GetInputTensorDimensionCount(0); ML_CHECK_VALID_ARGUMENT( @@ -43,9 +46,9 @@ class DmlOperatorConvolution : public DmlOperator, public ConvolutionHelperBase // Resize the bias to be the same dimension as the input tensor. // The 1D tensor needs to be moved to the C channel. - m_inputTensorDescs[2] = CreateTensorDescFromInput( + m_inputTensorDescs[biasIndex] = CreateTensorDescFromInput( kernelInfo, - 2, + biasIndex, TensorAxis::DoNotCoerce, TensorAxis::C, TensorAxis::LeftAligned, @@ -73,7 +76,7 @@ class DmlOperatorConvolution : public DmlOperator, public ConvolutionHelperBase DML_CONVOLUTION_OPERATOR_DESC convDesc = {}; convDesc.InputTensor = &inputDescs[0]; convDesc.FilterTensor = &inputDescs[1]; - convDesc.BiasTensor = kernelInfo.GetInputCount() > 2 ? &inputDescs[2] : nullptr; + convDesc.BiasTensor = hasBiasInput ? &inputDescs[biasIndex] : nullptr; convDesc.OutputTensor = &outputDescs[0]; convDesc.Mode = mode; convDesc.Direction = direction; @@ -92,19 +95,20 @@ class DmlOperatorConvolution : public DmlOperator, public ConvolutionHelperBase }; // A specific type of operation for registration. -template +template class DmlOperatorConvolutionTemplate : public DmlOperatorConvolution { public: DmlOperatorConvolutionTemplate(const MLOperatorKernelCreationContext& kernelInfo) - : DmlOperatorConvolution(kernelInfo, Mode, Direction) + : DmlOperatorConvolution(kernelInfo, Mode, Direction, hasDynamicPads) { } }; -DML_OP_DEFINE_CREATION_FUNCTION(Conv, DmlOperatorConvolutionTemplate); -DML_OP_DEFINE_CREATION_FUNCTION(ConvTranspose, DmlOperatorConvolutionTemplate); -DML_OP_DEFINE_CREATION_FUNCTION(FusedConv, DmlOperatorConvolutionTemplate); -DML_OP_DEFINE_CREATION_FUNCTION(FusedConvTranspose, DmlOperatorConvolutionTemplate); +DML_OP_DEFINE_CREATION_FUNCTION(Conv, DmlOperatorConvolutionTemplate); +DML_OP_DEFINE_CREATION_FUNCTION(ConvTranspose, DmlOperatorConvolutionTemplate); +DML_OP_DEFINE_CREATION_FUNCTION(FusedConv, DmlOperatorConvolutionTemplate); +DML_OP_DEFINE_CREATION_FUNCTION(FusedConvTranspose, DmlOperatorConvolutionTemplate); +DML_OP_DEFINE_CREATION_FUNCTION(ConvTransposeWithDynamicPads, DmlOperatorConvolutionTemplate); } // namespace Dml diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCopy.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCopy.cpp index 0aeb4a2fc0959..eb985f6c404c9 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCopy.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCopy.cpp @@ -24,10 +24,7 @@ class DmlOperatorCopy : public DmlOperator // element counts are the same. All this operator does is copy the resource and // rearrange the dimensions, so we tell DML that the output dimensions are the // same as the input dimensions. - m_outputTensorDescs.front() = TensorDesc( - m_outputTensorDescs.front().GetDmlDataType(), - m_inputTensorDescs.front().GetSizes() - ); + m_outputTensorDescs.front() = m_inputTensorDescs.front(); ComPtr contextPrivate; THROW_IF_FAILED(kernelInfo.GetInterface()->QueryInterface(contextPrivate.GetAddressOf())); diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorPooling.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorPooling.cpp index 9bd86a8d43b94..7f27291ede562 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorPooling.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorPooling.cpp @@ -29,6 +29,19 @@ class DmlOperatorPooling : public DmlOperator, public PoolingHelperBase assert(m_kernel.spatialDimensionCount <= ARRAYSIZE(m_kernel.windowSize)); + // The below attributes are temporarily not supported: + int ceilMode = kernelInfo.GetOptionalAttribute(AttrName::CeilMode, 0); + THROW_HR_IF(E_NOTIMPL, ceilMode != 0); + + int storageOrder = kernelInfo.GetOptionalAttribute(AttrName::StorageOrder, 0); + THROW_HR_IF(E_NOTIMPL, storageOrder != 0); + + auto dilations = kernelInfo.GetOptionalAttributeVectorInt32(AttrName::Dilations); + for (int dilation : dilations) + { + THROW_HR_IF(E_NOTIMPL, dilation != 1); + } + // DML requires that DimensionCount be equal to Input.DimCount - 2 for Pooling uint32_t expectedSpatialDimCount = m_inputTensorDescs[0].GetDimensionCount() - 2; if (m_kernel.spatialDimensionCount < expectedSpatialDimCount) @@ -121,6 +134,37 @@ class DmlOperatorPoolingTemplate : public DmlOperatorPooling } }; +void QueryMaxPool(IMLOperatorSupportQueryContextPrivate* context, bool *isSupported) +{ + *isSupported = false; + + MLOperatorAttributes attributes(context); + + // The below attributes are temporarily not supported: + int ceilMode = attributes.GetOptionalAttribute(AttrName::CeilMode, 0); + if (ceilMode != 0) + { + return; + } + + int storageOrder = attributes.GetOptionalAttribute(AttrName::StorageOrder, 0); + if (storageOrder != 0) + { + return; + } + + auto dilations = attributes.GetOptionalAttributeVectorInt32(AttrName::Dilations); + for (int dilation : dilations) + { + if (dilation != 1) + { + return; + } + } + + *isSupported = true; +} + DML_OP_DEFINE_CREATION_FUNCTION(AveragePool, DmlOperatorPoolingTemplate); DML_OP_DEFINE_CREATION_FUNCTION(GlobalAveragePool, DmlOperatorPoolingTemplate); DML_OP_DEFINE_CREATION_FUNCTION(MaxPool, DmlOperatorPoolingTemplate); diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorSlice.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorSlice.cpp index 7ae1995e74cbe..e167a89f0606e 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorSlice.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorSlice.cpp @@ -6,16 +6,22 @@ namespace Dml { -class DmlOperatorSlice : public DmlOperator, public SliceHelper +class DmlOperatorSlice : public DmlOperator, public SliceHelperBase { public: - DmlOperatorSlice(const MLOperatorKernelCreationContext& kernelInfo) + DmlOperatorSlice(const MLOperatorKernelCreationContext& kernelInfo, uint32_t opsetVersion) : DmlOperator(kernelInfo), - SliceHelper(kernelInfo, kernelInfo.GetTensorShapeDescription()) + SliceHelperBase(kernelInfo, kernelInfo.GetTensorShapeDescription(), opsetVersion) { - ML_CHECK_VALID_ARGUMENT(kernelInfo.GetInputCount() == 1); + uint32_t minInputCount = (opsetVersion < 10) ? 1 : 3; + ML_CHECK_VALID_ARGUMENT(kernelInfo.GetInputCount() >= minInputCount); ML_CHECK_VALID_ARGUMENT(kernelInfo.GetOutputCount() == 1); - DmlOperator::Initialize(kernelInfo); + + // TODO (23108599): Slice V10 introduces an optional "Steps" input which the kernel does not yet support. + THROW_HR_IF(E_NOTIMPL, kernelInfo.GetInputCount() > 4); + + std::vector> kernelInputIndices = { 0 }; + DmlOperator::Initialize(kernelInfo, kernelInputIndices); assert(m_inputTensorDescs[0].GetDimensionCount() >= gsl::narrow_cast(m_offsets.size())); assert(m_inputTensorDescs[0].GetDimensionCount() >= gsl::narrow_cast(m_sizes.size())); @@ -54,6 +60,22 @@ class DmlOperatorSlice : public DmlOperator, public SliceHelper } }; -DML_OP_DEFINE_CREATION_FUNCTION(Slice, DmlOperatorSlice); +// A specific type of operation for registration. +template +class DmlOperatorSliceTemplate : public DmlOperatorSlice +{ +public: + DmlOperatorSliceTemplate(const MLOperatorKernelCreationContext& kernelInfo) + : DmlOperatorSlice(kernelInfo, opsetVersion) + { + } +}; + +void QuerySlice(IMLOperatorSupportQueryContextPrivate* context, bool *isSupported) +{ + *isSupported = (context->GetInputCount() <= 4); +} +DML_OP_DEFINE_CREATION_FUNCTION(Slice7, DmlOperatorSliceTemplate<7>); +DML_OP_DEFINE_CREATION_FUNCTION(Slice10, DmlOperatorSliceTemplate<10>); } // namespace Dml diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp index b35c570bf8c53..c0ade42ccd5a4 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp @@ -70,12 +70,14 @@ struct OperatorRegistrationInformation // can't be represented as nodes in an optimized graph yet. std::optional requiredInputCountForDmlGraphSupport; + MLOperatorSupportQueryFunction supportQueryFunction; }; DML_OP_EXTERN_CREATION_FUNCTION(Copy); DML_OP_EXTERN_CREATION_FUNCTION(FC); DML_OP_EXTERN_CREATION_FUNCTION(Conv); DML_OP_EXTERN_CREATION_FUNCTION(ConvTranspose); +DML_OP_EXTERN_CREATION_FUNCTION(ConvTransposeWithDynamicPads); DML_OP_EXTERN_CREATION_FUNCTION(AveragePool); DML_OP_EXTERN_CREATION_FUNCTION(GlobalAveragePool); DML_OP_EXTERN_CREATION_FUNCTION(MaxPool); @@ -97,7 +99,8 @@ DML_OP_EXTERN_CREATION_FUNCTION(Split); DML_OP_EXTERN_CREATION_FUNCTION(Transpose); DML_OP_EXTERN_CREATION_FUNCTION(Tile); DML_OP_EXTERN_CREATION_FUNCTION(Concat); -DML_OP_EXTERN_CREATION_FUNCTION(Slice); +DML_OP_EXTERN_CREATION_FUNCTION(Slice7); +DML_OP_EXTERN_CREATION_FUNCTION(Slice10); DML_OP_EXTERN_CREATION_FUNCTION(Pad); DML_OP_EXTERN_CREATION_FUNCTION(SpaceToDepth); DML_OP_EXTERN_CREATION_FUNCTION(DepthToSpace); @@ -201,6 +204,9 @@ DML_OP_EXTERN_CREATION_FUNCTION(Scatter); DML_OP_EXTERN_CREATION_FUNCTION(Resize); DML_OP_EXTERN_CREATION_FUNCTION(ConstantOfShape); +DML_OP_EXTERN_QUERY_FUNCTION(MaxPool); +DML_OP_EXTERN_QUERY_FUNCTION(Slice); + const static char* const typeNameListDefault[1] = {"T"}; const static char* const typeNameListTopK[2] = { "T", "I" }; const static char* const typeNameListLogicalComparison[2] = { "T", "T1" }; @@ -220,7 +226,7 @@ const static SupportedTensorDataTypes supportedTypeListAllScalars[1] = { Support const static SupportedTensorDataTypes supportedTypeListBool[1] = {SupportedTensorDataTypes::Bool}; const static SupportedTensorDataTypes supportedTypeListTopK[2] = {SupportedTensorDataTypes::Float16to32, SupportedTensorDataTypes::Int64}; const static SupportedTensorDataTypes supportedTypeListIndices[1] = { SupportedTensorDataTypes::Int32|SupportedTensorDataTypes::Int64 }; -const static SupportedTensorDataTypes supportedTypeListCast[2] = { SupportedTensorDataTypes::Scalars8to32, SupportedTensorDataTypes::Scalars8to32 }; +const static SupportedTensorDataTypes supportedTypeListCast[2] = { SupportedTensorDataTypes::AllScalars, SupportedTensorDataTypes::Scalars8to32 }; const static SupportedTensorDataTypes supportedTypeListScatterGather[2] = { SupportedTensorDataTypes::Float16to32, SupportedTensorDataTypes::Int32 | SupportedTensorDataTypes::Int64 }; const static SupportedTensorDataTypes supportedTypeListQuantize[2] = { SupportedTensorDataTypes::Float32, SupportedTensorDataTypes::UInt8 }; const static SupportedTensorDataTypes supportedTypeListIsNan[2] = { SupportedTensorDataTypes::Float16to32, SupportedTensorDataTypes::UInt8 }; @@ -233,6 +239,10 @@ const static SupportedTensorDataTypes supportedTypeListLogicalComparison[2] = /* #define REG_INFO(version, operatorName, ...) \ #operatorName, OnnxOperatorSet##version::sc_sinceVer_##operatorName, onnxruntime::kOnnxDomain, Create##operatorName, ShapeInferenceFunction, false, false, ##__VA_ARGS__, +// Versioned operator +#define REG_INFO_VER(version, operatorName, ...) \ + #operatorName, OnnxOperatorSet##version::sc_sinceVer_##operatorName, onnxruntime::kOnnxDomain, Create##operatorName##version, ShapeInferenceFunction, false, false, ##__VA_ARGS__, + // Identity operators use Copy, alias their first input, and require floating point formats // for usage in the graph, besides constant inputs. This is because they currently use // element-wise identity operators in the graph for striding support, but issue actual copies @@ -242,12 +252,17 @@ const static SupportedTensorDataTypes supportedTypeListLogicalComparison[2] = /* // MS-domain operators #define REG_INFO_MS(version, operatorName, ...) \ + #operatorName, MsftOperatorSet##version::sc_sinceVer_##operatorName, onnxruntime::kMSDomain, Create##operatorName, ShapeInferenceFunction, false, false, ##__VA_ARGS__, + +// MS-domain operators +#define REG_INFO_MSDML(version, operatorName, ...) \ #operatorName, MsftOperatorSet##version::sc_sinceVer_##operatorName, onnxruntime::kMSDmlDomain, Create##operatorName, ShapeInferenceFunction, false, false, ##__VA_ARGS__, const static OperatorRegistrationInformation operatorRegistrationInformationTable[] = { /// Domain/Type, Ver, Name, TypeNames, Types, Graph Support, Required const CPU inputs, -/// Input count required for graph support +/// Input count required for graph support, +/// Support query function // Deep Learning Standard Layers {REG_INFO( 7, Conv, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, @@ -255,7 +270,9 @@ const static OperatorRegistrationInformation operatorRegistrationInformationTabl {REG_INFO( 7, AveragePool, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, {REG_INFO( 7, GlobalAveragePool, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, {REG_INFO( 7, MaxPool, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, - {REG_INFO( 8, MaxPool, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, + {REG_INFO( 8, MaxPool, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported, {}, std::nullopt, QueryMaxPool)}, + {REG_INFO( 10, MaxPool, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported, {}, std::nullopt, QueryMaxPool)}, + {REG_INFO( 7, GlobalMaxPool, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, {REG_INFO( 7, LpPool, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, {REG_INFO( 7, GlobalLpPool, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, @@ -269,12 +286,14 @@ const static OperatorRegistrationInformation operatorRegistrationInformationTabl {REG_INFO( 7, RNN, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::NotSupported)}, {REG_INFO( 7, GRU, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::NotSupported)}, {REG_INFO( 7, LSTM, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::NotSupported)}, + {REG_INFO_MS( 1, ConvTransposeWithDynamicPads, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported, {2})}, // Data Reorganization Layers {REG_INFO( 7, Split, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, {REG_INFO( 7, Transpose, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, {REG_INFO( 7, Concat, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, - {REG_INFO( 7, Slice, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, + {REG_INFO_VER( 7, Slice, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, + {REG_INFO_VER( 10, Slice, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported, {1, 2, 3}, std::nullopt, QuerySlice)}, {REG_INFO( 7, Pad, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, {REG_INFO( 7, SpaceToDepth, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, {REG_INFO( 7, DepthToSpace, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, @@ -400,19 +419,19 @@ const static OperatorRegistrationInformation operatorRegistrationInformationTabl {REG_INFO( 9, OneHot, typeNameListOneHot, supportedTypeListOneHot, DmGraphSupport::Supported, {1})}, // Fused operators - {REG_INFO_MS( 1, FusedConv, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, - {REG_INFO_MS( 1, FusedConvTranspose, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, - {REG_INFO_MS( 1, FusedInstanceNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, - {REG_INFO_MS( 1, FusedBatchNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, - {REG_INFO_MS( 1, FusedMeanVarianceNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, - {REG_INFO_MS( 1, FusedGemm, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, - {REG_INFO_MS( 1, FusedMatMul, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, - {REG_INFO_MS( 1, FusedAdd, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, - {REG_INFO_MS( 1, FusedSum, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported, {}, 2)}, + {REG_INFO_MSDML(1, FusedConv, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, + {REG_INFO_MSDML(1, FusedConvTranspose, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, + {REG_INFO_MSDML(1, FusedInstanceNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, + {REG_INFO_MSDML(1, FusedBatchNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, + {REG_INFO_MSDML(1, FusedMeanVarianceNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, + {REG_INFO_MSDML(1, FusedGemm, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, + {REG_INFO_MSDML(1, FusedMatMul, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, + {REG_INFO_MSDML(1, FusedAdd, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported)}, + {REG_INFO_MSDML(1, FusedSum, typeNameListDefault, supportedTypeListFloat16to32, DmGraphSupport::Supported, {}, 2)}, // TODO: DwayneR implement MaxUnpool https://dev.azure.com/microsoft/OS/_workitems/edit/21267466 }; - + template MLOperatorEdgeDescription EdgeDesc() { @@ -497,10 +516,17 @@ void RegisterDmlOperators(IMLOperatorRegistry* registry) shapeInferrer = wil::MakeOrThrow(information.shapeInferenceFunction); } + ComPtr supportQuery; + if (information.supportQueryFunction) + { + supportQuery = wil::MakeOrThrow(information.supportQueryFunction); + } + THROW_IF_FAILED(registryPrivate->RegisterOperatorKernel( &desc, factory.Get(), shapeInferrer.Get(), + supportQuery.Get(), true, // isInternalOperator information.canAliasFirstInput, // alias kernelSupportsGraph, // supportsGraph diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.h index 12ca021b60fc2..186608d78b586 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.h @@ -9,6 +9,7 @@ class MLOperatorKernelCreationContext; // Forward declares an external creation function. #define DML_OP_EXTERN_CREATION_FUNCTION(operatorName) extern void Create##operatorName(IMLOperatorKernelCreationContext* kernelInfo, IMLOperatorKernel** opKernel) +#define DML_OP_EXTERN_QUERY_FUNCTION(operatorName) extern void Query##operatorName(IMLOperatorSupportQueryContextPrivate* context, bool *isSupported); // Declares a callback creation function of the given operator class. // This does not register it, just declares it for usage by registration later. diff --git a/onnxruntime/core/providers/dml/OperatorAuthorHelper/Attributes.h b/onnxruntime/core/providers/dml/OperatorAuthorHelper/Attributes.h index 20f4d69cbdde0..dc7def086dad2 100644 --- a/onnxruntime/core/providers/dml/OperatorAuthorHelper/Attributes.h +++ b/onnxruntime/core/providers/dml/OperatorAuthorHelper/Attributes.h @@ -19,6 +19,7 @@ namespace AttrName static constexpr const char* BlockSize = "blocksize"; static constexpr const char* Border = "border"; static constexpr const char* Broadcast = "broadcast"; + static constexpr const char* CeilMode = "ceil_mode"; static constexpr const char* Clip = "clip"; static constexpr const char* CountIncludePad = "count_include_pad"; static constexpr const char* Dilations = "dilations"; @@ -60,6 +61,7 @@ namespace AttrName static constexpr const char* Split = "split"; static constexpr const char* Starts = "starts"; static constexpr const char* Steepness = "steepness"; + static constexpr const char* StorageOrder = "storage_order"; static constexpr const char* Strides = "strides"; static constexpr const char* Tiles = "tiles"; static constexpr const char* To = "to"; diff --git a/onnxruntime/core/providers/dml/OperatorAuthorHelper/MLOperatorAuthorHelper.h b/onnxruntime/core/providers/dml/OperatorAuthorHelper/MLOperatorAuthorHelper.h index 871df4070987f..b15955e0e533d 100644 --- a/onnxruntime/core/providers/dml/OperatorAuthorHelper/MLOperatorAuthorHelper.h +++ b/onnxruntime/core/providers/dml/OperatorAuthorHelper/MLOperatorAuthorHelper.h @@ -735,6 +735,7 @@ class MLOperatorKernel : public Microsoft::WRL::RuntimeClass< using MLOperatorTypeInferenceFunction = void (CALLBACK*)(IMLOperatorTypeInferenceContext*); using MLOperatorShapeInferenceFunction = void (CALLBACK*)(IMLOperatorShapeInferenceContext*); using MLOperatorKernelCreateFn = void(*)(IMLOperatorKernelCreationContext*, IMLOperatorKernel**); +using MLOperatorSupportQueryFunction = void (CALLBACK*)(IMLOperatorSupportQueryContextPrivate*, bool*); class MLOperatorShapeInferrer : public Microsoft::WRL::RuntimeClass< Microsoft::WRL::RuntimeClassFlags, IMLOperatorShapeInferrer> @@ -755,6 +756,29 @@ class MLOperatorShapeInferrer : public Microsoft::WRL::RuntimeClass< MLOperatorShapeInferenceFunction m_shapeInferenceFn = nullptr; }; +class MLOperatorSupportQuery : public Microsoft::WRL::RuntimeClass< + Microsoft::WRL::RuntimeClassFlags, IMLOperatorSupportQueryPrivate> +{ +public: + MLOperatorSupportQuery(MLOperatorSupportQueryFunction queryFn) : + m_queryFn(queryFn) + {} + + HRESULT STDMETHODCALLTYPE QuerySupport( + IMLOperatorSupportQueryContextPrivate* context, + BOOL* isSupported) noexcept override try + { + bool fIsSupported = false; + m_queryFn(context, &fIsSupported); + *isSupported = fIsSupported ? TRUE : FALSE; + return S_OK; + } + CATCH_RETURN(); + +private: + MLOperatorSupportQueryFunction m_queryFn = nullptr; +}; + class MLOperatorTypeInferrer : public Microsoft::WRL::RuntimeClass< Microsoft::WRL::RuntimeClassFlags, IMLOperatorTypeInferrer> { diff --git a/onnxruntime/core/providers/dml/OperatorAuthorHelper/MLOperatorAuthorPrivate.h b/onnxruntime/core/providers/dml/OperatorAuthorHelper/MLOperatorAuthorPrivate.h index 6b0cbab3fb1fd..216dfdcb2ba3d 100644 --- a/onnxruntime/core/providers/dml/OperatorAuthorHelper/MLOperatorAuthorPrivate.h +++ b/onnxruntime/core/providers/dml/OperatorAuthorHelper/MLOperatorAuthorPrivate.h @@ -46,6 +46,60 @@ IMLOperatorKernelCreationContextPrivate : public IMLOperatorKernelCreationContex ) const noexcept PURE; }; +//! \interface IMLOperatorAttributes1 +//! \brief Represents the values of an operator's attributes, as determined by a model using the operator. +//! This interface is called by implementations of custom operator kernels, and by implementations +//! of shape and type inferrers. +interface DECLSPEC_UUID("3a798815-dfe3-4bcd-b6a6-f70650d5f80b") DECLSPEC_NOVTABLE +IMLOperatorAttributes1 : public IMLOperatorAttributes +{ + //! Gets an interface pointer for the constant tensor. + //! Note the tensor is CPU side (IsCpuData is true). + STDMETHOD(GetTensorAttribute)( + _In_z_ const char* name, + _COM_Outptr_ IMLOperatorTensor** tensor + ) const noexcept PURE; +}; + +interface __declspec(uuid("897bb586-6cee-4106-8513-dda33151c109")) DECLSPEC_NOVTABLE +IMLOperatorSupportQueryContextPrivate : public IMLOperatorAttributes1 +{ + //! Gets the number of inputs to the operator. + STDMETHOD_(uint32_t, GetInputCount)() const noexcept PURE; + + //! Gets the number of outputs to the operator. + STDMETHOD_(uint32_t, GetOutputCount)() const noexcept PURE; + + //! Returns true if an input to the operator is valid. + //! This always returns true except for optional inputs and invalid indices. + STDMETHOD_(bool, IsInputValid)(uint32_t inputIndex) const noexcept PURE; + + //! Returns true if an output to the operator is valid. + //! This always returns true if within GetOutputCount except for optional outputs. + STDMETHOD_(bool, IsOutputValid)(uint32_t outputIndex) const noexcept PURE; + + //! Gets the description of the specified input edge of the operator. + STDMETHOD(GetInputEdgeDescription)( + uint32_t inputIndex, + _Out_ MLOperatorEdgeDescription* edgeDescription + ) const noexcept PURE; + + //! Gets the description of the specified output edge of the operator. + STDMETHOD(GetOutputEdgeDescription)( + uint32_t outputIndex, + _Out_ MLOperatorEdgeDescription* edgeDescription + ) const noexcept PURE; +}; + +interface __declspec(uuid("023954b3-aed2-4b03-b7c7-f0838053a9a1")) DECLSPEC_NOVTABLE +IMLOperatorSupportQueryPrivate : public IUnknown +{ + STDMETHOD(QuerySupport)( + IMLOperatorSupportQueryContextPrivate* context, + BOOL* isSupported + ) noexcept PURE; +}; + interface DECLSPEC_UUID("3de1dc1e-13e9-4099-ae88-7b4100083415") DECLSPEC_NOVTABLE IMLOperatorRegistryPrivate : public IUnknown { @@ -53,6 +107,7 @@ IMLOperatorRegistryPrivate : public IUnknown const MLOperatorKernelDescription* operatorKernel, IMLOperatorKernelFactory* operatorKernelFactory, _In_opt_ IMLOperatorShapeInferrer* shapeInferrer, + _In_opt_ IMLOperatorSupportQueryPrivate* supportQuery, bool isInternalOperator, bool canAliasFirstInput, bool supportsGraph, @@ -63,21 +118,6 @@ IMLOperatorRegistryPrivate : public IUnknown ) const noexcept PURE; }; -//! \interface IMLOperatorAttributes1 -//! \brief Represents the values of an operator's attributes, as determined by a model using the operator. -//! This interface is called by implementations of custom operator kernels, and by implementations -//! of shape and type inferrers. -interface DECLSPEC_UUID("3a798815-dfe3-4bcd-b6a6-f70650d5f80b") DECLSPEC_NOVTABLE -IMLOperatorAttributes1 : public IMLOperatorAttributes -{ - //! Gets an interface pointer for the constant tensor. - //! Note the tensor is CPU side (IsCpuData is true). - STDMETHOD(GetTensorAttribute)( - _In_z_ const char* name, - _COM_Outptr_ IMLOperatorTensor** tensor - ) const noexcept PURE; -}; - // Declare private enum MLOperatorAttributeType::Tensor. // // enum class MLOperatorAttributeType : uint32_t diff --git a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.cpp b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.cpp index 99425a76f49f0..9f3a0ed7e69f4 100644 --- a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.cpp +++ b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.cpp @@ -430,58 +430,7 @@ int64_t ReadAsInt64(MLOperatorTensorDataType tensorDataType, const void* p) return edgeShapes; } - void SliceHelper::Initialize( - const MLOperatorAttributes& operatorAttributes, - gsl::span inputDimensions - ) - { - const uint32_t dimCount = gsl::narrow_cast(inputDimensions.size()); - - std::vector starts = operatorAttributes.GetOptionalAttributeVectorInt32(AttrName::Starts); - std::vector ends = operatorAttributes.GetOptionalAttributeVectorInt32(AttrName::Ends); - std::vector axes = operatorAttributes.GetOptionalAttributeVectorInt32(AttrName::Axes); - HandleNegativeAxes(/*inout*/ axes, dimCount); - - ML_CHECK_VALID_ARGUMENT(starts.size() == ends.size(), "'starts' must equal 'ends' in size."); - ML_CHECK_VALID_ARGUMENT(axes.empty() || starts.size() == axes.size(), "'axes' must equal 'starts' in size, or 'axes' must be empty."); - - m_outputDimensions.assign(inputDimensions.begin(), inputDimensions.end()); - m_offsets.resize(m_outputDimensions.size()); - m_sizes.resize(m_outputDimensions.size()); - m_strides.resize(m_outputDimensions.size(), 1); // Only a stride of 1 element is supported by ONNX 1.2. - - // Set initial defaults lest 'starts' and 'ends' arrays are shorter than the dimension count. - std::copy(inputDimensions.begin(), inputDimensions.begin() + m_outputDimensions.size(), m_sizes.begin()); - - // Clamp selected dimensions to given 'starts' and 'ends'. - for (int i = 0, ci = gsl::narrow_cast(starts.size()); i < ci; ++i) - { - int dimIndex = i; - if (!axes.empty()) - { - dimIndex = axes[i]; - } - ML_CHECK_VALID_ARGUMENT(dimIndex < inputDimensions.size(), "'axes' must be valid with within actual input dimensions."); - - // Positive values are offsets from 0. - // Negative values are offsets from the dimension's size. - int dim = gsl::narrow_cast(inputDimensions[dimIndex]); - int start = (starts[i] < 0) ? (starts[i] + dim) : starts[i]; - int end = (ends[i] < 0) ? (ends[i] + dim) : ends[i]; - - // Clamp the dimensions to the slice extents. - // Clamp negative numbers to 0, per case test_slice_start_out_of_bounds. - start = std::max(start, 0); - end = std::min(end, dim); - int size = std::max(end - start, 0); - - m_outputDimensions[dimIndex] = size; - m_offsets[dimIndex] = start; - m_sizes[dimIndex] = gsl::narrow_cast(size); - } - } - - std::vector SliceHelper::GetOutputShapes(const MLShapeInferenceContext& shapeInfo) const + std::vector SliceHelperBase::GetOutputShapes(const MLShapeInferenceContext& shapeInfo) const { return { m_outputDimensions }; } diff --git a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.h b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.h index dccb9e5455b6e..c944c1b59b27e 100644 --- a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.h +++ b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.h @@ -267,29 +267,30 @@ class RandomNormalHelper : public RandomNormalHelperBase { std::vector m_tensorShape; }; -class ConvolutionHelperBase { - public: - enum FilterDims { K }; - enum InputTensor { X, - Filter, - Bias }; - enum InputDims { N, - C, - H, - W }; - - public: - // Info_t is used to obtain attributes which will be used for calculating the output shape later. - template - ConvolutionHelperBase(const Info_t& info, const Shape_t& shape, bool transpose) : m_kernel(InitializeKernel(info, shape.GetInputTensorDimensionCount(0), shape.GetInputTensorShape(1))) { - m_groupCount = info.GetOptionalAttribute(AttrName::Group, 1); - - if (!transpose) { - InitializeKernelAndShapes(shape); - } else { - InitializeKernelAndShapesTransposed(info, shape); +class ConvolutionHelperBase +{ +public: + enum FilterDims { K }; + enum InputTensor { X, Filter}; + enum InputDims { N, C, H, W }; + +public: + // Info_t is used to obtain attributes which will be used for calculating the output shape later. + template + ConvolutionHelperBase(const Info_t& info, const Shape_t& shape, bool transpose, bool hasDynamicPads) : + m_kernel(InitializeKernel(info, shape.GetInputTensorDimensionCount(0), shape.GetInputTensorShape(1))) + { + m_groupCount = info.GetOptionalAttribute(AttrName::Group, 1); + + if (!transpose) + { + InitializeKernelAndShapes(shape); + } + else + { + InitializeKernelAndShapesTransposed(info, shape, hasDynamicPads); + } } - } void ResolvingPadding(gsl::span inputDimensions); @@ -303,37 +304,68 @@ class ConvolutionHelperBase { const std::vector inputDimensions = shapeInfo.GetInputTensorShape(0); const std::vector filterDims = shapeInfo.GetInputTensorShape(1); - ML_CHECK_VALID_ARGUMENT( - inputDimensions.size() >= 3 && inputDimensions.size() <= 5, - "Input dimensions must be: 3, 4, 5."); - - ResolvingPadding(inputDimensions); - - m_outputShapes.resize(1); - m_outputShapes[0] = InitializeKernelOutputDimensions(inputDimensions, m_kernel); - m_outputShapes[0].GetShape()[C] = filterDims[K]; - } - - template - void InitializeKernelAndShapesTransposed(const Info_t& info, const Shape_t& shapeInfo) { - std::vector outputShape = info.GetOptionalAttributeVectorInt32(AttrName::OutputShape); - if (!outputShape.empty()) { - ML_CHECK_VALID_ARGUMENT( - outputShape.size() >= m_kernel.spatialDimensionCount, - "The output shape must equal the number of spatial dimensions"); + ML_CHECK_VALID_ARGUMENT( + inputDimensions.size() >= 3 && inputDimensions.size() <= 5, + "Input dimensions must be: 3, 4, 5." + ); + + ResolvingPadding(inputDimensions); + + m_outputShapes.resize(1); + m_outputShapes[0] = InitializeKernelOutputDimensions(inputDimensions, m_kernel); + m_outputShapes[0].GetShape()[C] = filterDims[K]; } + + + template + void InitializeKernelAndShapesTransposed(const Info_t& info, const Shape_t& shapeInfo, bool hasDynamicPads) + { + std::vector outputShape = info.GetOptionalAttributeVectorInt32(AttrName::OutputShape); + if (!outputShape.empty()) + { + ML_CHECK_VALID_ARGUMENT( + outputShape.size() >= m_kernel.spatialDimensionCount, + "The output shape must equal the number of spatial dimensions" + ); + } const std::vector inputDimensions = shapeInfo.GetInputTensorShape(0); const std::vector filterDims = shapeInfo.GetInputTensorShape(1); ML_CHECK_VALID_ARGUMENT(inputDimensions.size() > NonspatialDimensionCount, "Input dimensions must be >= 3"); - ResolvingPadding(inputDimensions); - m_outputShapes.resize(1); - m_outputShapes[0] = InitializeKernelOutputDimsTranspose(inputDimensions, m_kernel); - static_assert(C < NonspatialDimensionCount); - assert(m_outputShapes[0].GetShape().size() > C); - m_outputShapes[0].GetShape()[C] = filterDims[C] * m_groupCount; + if (hasDynamicPads) + { + MLOperatorTensor padsTensor = info.GetConstantInputTensor(2); + const std::vector& padsTensorDimensions = padsTensor.GetShape(); + ML_CHECK_VALID_ARGUMENT(padsTensorDimensions.size() == 1, "Pads dimensions must equal 1"); + const size_t dimCount = padsTensorDimensions[0]; + ML_CHECK_VALID_ARGUMENT(dimCount == 2 * NchwSpatialDimensionCount, "Pads count must equal 4"); + const int64_t* padsData = padsTensor.GetData(); + + for (size_t i = 0; i < dimCount; ++i) + { + ML_CHECK_VALID_ARGUMENT(padsData[i] >= 0, "Padding values must be greater than or equal to 0"); + if (i < dimCount / 2) + { + m_kernel.startPadding[i] = gsl::narrow_cast(padsData[i]); + } + else + { + m_kernel.endPadding[i - dimCount/2] = gsl::narrow_cast(padsData[i]); + } + } + } + else + { + ResolvingPadding(inputDimensions); + } + + m_outputShapes.resize(1); + m_outputShapes[0] = InitializeKernelOutputDimsTranspose(inputDimensions, m_kernel); + static_assert(C < NonspatialDimensionCount); + assert(m_outputShapes[0].GetShape().size() > C); + m_outputShapes[0].GetShape()[C] = filterDims[C] * m_groupCount; if (!outputShape.empty()) { // Start padding, end padding, and output padding are all ignored if output shape is set. @@ -376,31 +408,42 @@ class ConvolutionHelperBase { std::vector m_outputShapes; }; -class ConvHelper : public ConvolutionHelperBase { - public: - template - ConvHelper(const Info_t& info, const Shape_t& shape) : ConvolutionHelperBase(info, shape, false) {} +class ConvHelper : public ConvolutionHelperBase +{ +public: + template + ConvHelper(const Info_t& info, const Shape_t& shape) : ConvolutionHelperBase(info, shape, false, false) {} }; -class ConvTransposeHelper : public ConvolutionHelperBase { - public: - template - ConvTransposeHelper(const Info_t& info, const Shape_t& shape) : ConvolutionHelperBase(info, shape, true) {} +class ConvTransposeHelper : public ConvolutionHelperBase +{ +public: + template + ConvTransposeHelper(const Info_t& info, const Shape_t& shape) : ConvolutionHelperBase(info, shape, true, false) {} }; -class GemmHelper { - public: - // Info_t is used to obtain attributes which will be used for calculating the output shape later. - // Shape_t is used to obtain input shape which will be used for adjusting attribute value. - template - GemmHelper(const Info_t& info, const Shape_t& shape) { - ORT_UNUSED_PARAMETER(shape); - m_transA = info.GetOptionalAttribute(AttrName::TransA, 0); - m_transB = info.GetOptionalAttribute(AttrName::TransB, 0); - m_broadcast = info.GetOptionalAttribute(AttrName::Broadcast, 0); - m_alpha = info.GetOptionalAttribute(AttrName::Alpha, 1.0f); - m_beta = info.GetOptionalAttribute(AttrName::Beta, 0.0f); - } +class ConvTransposeWithDynamicPadsHelper : public ConvolutionHelperBase +{ +public: + template + ConvTransposeWithDynamicPadsHelper(const Info_t& info, const Shape_t& shape) : ConvolutionHelperBase(info, shape, true, true) {} +}; + +class GemmHelper +{ +public: + // Info_t is used to obtain attributes which will be used for calculating the output shape later. + // Shape_t is used to obtain input shape which will be used for adjusting attribute value. + template + GemmHelper(const Info_t& info, const Shape_t& shape) + { + ORT_UNUSED_PARAMETER(shape); + m_transA = info.GetOptionalAttribute(AttrName::TransA, 0); + m_transB = info.GetOptionalAttribute(AttrName::TransB, 0); + m_broadcast = info.GetOptionalAttribute(AttrName::Broadcast, 0); + m_alpha = info.GetOptionalAttribute(AttrName::Alpha, 1.0f); + m_beta = info.GetOptionalAttribute(AttrName::Beta, 0.0f); + } std::vector GetOutputShapes(const MLShapeInferenceContext& shapeInfo) const; @@ -455,18 +498,139 @@ class SplitHelper { std::vector m_split; }; -class SliceHelper { - public: - void Initialize( - const MLOperatorAttributes& operatorAttributes, - gsl::span inputDimensions); +class SliceHelperBase +{ +public: + template + void ReadIndexTensors( + const Info_t& operatorInfo, + std::vector& starts, + std::vector& ends, + std::vector& axes, + std::vector& steps + ) + { + // Get starts, ends, optional axes and optional steps from constant inputs. + MLOperatorTensor startsTensor = operatorInfo.GetConstantInputTensor(1); + const std::vector& startsTensorDimensions = startsTensor.GetShape(); + size_t dimCount = startsTensorDimensions[0]; + const Index_t* startsData = startsTensor.GetData(); + for (size_t i = 0; i < dimCount; ++i) + { + starts.push_back(gsl::narrow_cast(startsData[i])); + } + + MLOperatorTensor endsTensor = operatorInfo.GetConstantInputTensor(2); + const std::vector& endsTensorDimensions = endsTensor.GetShape(); + dimCount = endsTensorDimensions[0]; + const Index_t* endsData = endsTensor.GetData(); + for (size_t i = 0; i < dimCount; ++i) + { + ends.push_back(gsl::narrow_cast(endsData[i])); + } + uint32_t inputCount = operatorInfo.GetInputCount(); + if (inputCount > 3) + { + MLOperatorTensor axesTensor = operatorInfo.GetConstantInputTensor(3); + const std::vector& axesTensorDimensions = axesTensor.GetShape(); + dimCount = axesTensorDimensions[0]; + const Index_t* axesData = axesTensor.GetData(); + for (size_t i = 0; i < dimCount; ++i) + { + axes.push_back(gsl::narrow_cast(axesData[i])); + } + } + + if (inputCount > 4) + { + MLOperatorTensor stepsTensor = operatorInfo.GetConstantInputTensor(4); + const std::vector& stepsTensorDimensions = stepsTensor.GetShape(); + dimCount = stepsTensorDimensions[0]; + const Index_t* stepsData = stepsTensor.GetData(); + for (size_t i = 0; i < dimCount; ++i) + { + steps.push_back(gsl::narrow_cast(stepsData[i])); + } + } + } - // Info_t is used to obtain attributes which will be used for calculating the output shape later. - // Shape_t is used to obtain input shape which will be used for adjusting attribute value. - template - SliceHelper(const Info_t& info, const Shape_t& shape) { - Initialize(info, shape.GetInputTensorShape(0)); - } + template + void Initialize( + const Info_t& operatorInfo, + gsl::span inputDimensions, + uint32_t opsetVersion + ) + { + std::vector starts; + std::vector ends; + std::vector axes; + std::vector steps; + if (opsetVersion == 7) + { + // Get starts, ends and axes from attributes + starts = operatorInfo.GetOptionalAttributeVectorInt32(AttrName::Starts); + ends = operatorInfo.GetOptionalAttributeVectorInt32(AttrName::Ends); + axes = operatorInfo.GetOptionalAttributeVectorInt32(AttrName::Axes); + } + else if (opsetVersion == 10) + { + if (operatorInfo.GetConstantInputTensor(1).GetTensorDataType() == MLOperatorTensorDataType::Int32) + { + ReadIndexTensors(operatorInfo, starts, ends, axes, steps); + } + else + { + THROW_HR_IF(E_INVALIDARG, operatorInfo.GetConstantInputTensor(1).GetTensorDataType() != MLOperatorTensorDataType::Int64); + ReadIndexTensors(operatorInfo, starts, ends, axes, steps); + } + } + + ML_CHECK_VALID_ARGUMENT(starts.size() == ends.size(), "'starts' must equal 'ends' in size."); + ML_CHECK_VALID_ARGUMENT(axes.empty() || starts.size() == axes.size(), "'axes' must equal 'starts' in size, or 'axes' must be empty."); + + m_outputDimensions.assign(inputDimensions.begin(), inputDimensions.end()); + m_offsets.resize(m_outputDimensions.size()); + m_sizes.resize(m_outputDimensions.size()); + m_strides.resize(m_outputDimensions.size(), 1); // Only a stride of 1 element is supported by ONNX 1.2. + + // Set initial defaults lest 'starts' and 'ends' arrays are shorter than the dimension count. + std::copy(inputDimensions.begin(), inputDimensions.begin() + m_outputDimensions.size(), m_sizes.begin()); + + // Clamp selected dimensions to given 'starts' and 'ends'. + for (int i = 0, ci = gsl::narrow_cast(starts.size()); i < ci; ++i) + { + int dimIndex = i; + if (!axes.empty()) + { + dimIndex = axes[i]; + } + ML_CHECK_VALID_ARGUMENT(dimIndex < inputDimensions.size(), "'axes' must be valid with within actual input dimensions."); + + // Positive values are offsets from 0. + // Negative values are offsets from the dimension's size. + int dim = gsl::narrow_cast(inputDimensions[dimIndex]); + int start = (starts[i] < 0) ? (starts[i] + dim) : starts[i]; + int end = (ends[i] < 0) ? (ends[i] + dim) : ends[i]; + + // Clamp the dimensions to the slice extents. + // Clamp negative numbers to 0, per case test_slice_start_out_of_bounds. + start = std::max(start, 0); + end = std::min(end, dim); + int size = std::max(end - start, 0); + + m_outputDimensions[dimIndex] = size; + m_offsets[dimIndex] = start; + m_sizes[dimIndex] = gsl::narrow_cast(size); + } + } + + // Info_t is used to obtain attributes which will be used for calculating the output shape later. + // Shape_t is used to obtain input shape which will be used for adjusting attribute value. + template + SliceHelperBase(const Info_t& info, const Shape_t& shape, uint32_t opsetVersion) + { + Initialize(info, shape.GetInputTensorShape(0), opsetVersion); + } std::vector GetOutputShapes(const MLShapeInferenceContext& shapeInfo) const; @@ -477,9 +641,25 @@ class SliceHelper { std::vector m_strides; }; -class PaddingHelper { - public: - void Initialize(const MLOperatorAttributes& operatorAttributes); +class SliceHelper : public SliceHelperBase +{ +public: + template + SliceHelper(const Info_t& info, const Shape_t& shape) : SliceHelperBase(info, shape, 7) {} +}; + +class Slice10Helper : public SliceHelperBase +{ +public: + template + Slice10Helper(const Info_t& info, const Shape_t& shape) : SliceHelperBase(info, shape, 10) {} +}; + + +class PaddingHelper +{ +public: + void Initialize(const MLOperatorAttributes& operatorAttributes); // Info_t is used to obtain attributes which will be used for calculating the output shape later. // Shape_t is used to obtain input shape which will be used for adjusting attribute value. @@ -985,6 +1165,7 @@ class OneHotHelper { using ShapeInferenceHelper_Conv = ConvHelper; using ShapeInferenceHelper_ConvTranspose = ConvTransposeHelper; +using ShapeInferenceHelper_ConvTransposeWithDynamicPads = ConvTransposeWithDynamicPadsHelper; using ShapeInferenceHelper_AveragePool = PoolingHelper; using ShapeInferenceHelper_GlobalAveragePool = GlobalPoolingHelper; using ShapeInferenceHelper_MaxPool = PoolingHelper; @@ -1007,7 +1188,8 @@ using ShapeInferenceHelper_Flatten = FlattenHelper; using ShapeInferenceHelper_Split = SplitHelper; using ShapeInferenceHelper_Transpose = TransposeHelper; using ShapeInferenceHelper_Concat = ConcatHelper; -using ShapeInferenceHelper_Slice = SliceHelper; +using ShapeInferenceHelper_Slice7 = SliceHelper; +using ShapeInferenceHelper_Slice10 = Slice10Helper; using ShapeInferenceHelper_Pad = PaddingHelper; using ShapeInferenceHelper_SpaceToDepth = SpaceToDepthHelper; using ShapeInferenceHelper_DepthToSpace = DepthToSpaceHelper; diff --git a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorRegistration.h b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorRegistration.h index b9e429a6bf1bd..2b28bf9f6209b 100644 --- a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorRegistration.h +++ b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorRegistration.h @@ -178,6 +178,7 @@ namespace OperatorHelper static const int sc_sinceVer_Dropout = 10; static const int sc_sinceVer_ThresholdedRelu = 10; static const int sc_sinceVer_Upsample = 10; + static const int sc_sinceVer_Slice = 10; } // namespace OnnxOperatorSet10 namespace MsftOperatorSet1 @@ -193,6 +194,7 @@ namespace OperatorHelper static const int sc_sinceVer_FusedSum = 1; static const int sc_sinceVer_QuantizeLinear = 1; static const int sc_sinceVer_DequantizeLinear = 1; + static const int sc_sinceVer_ConvTransposeWithDynamicPads = 1; } // namespace MsftOperatorSet1 } // namespace OperatorHelper diff --git a/onnxruntime/core/providers/dml/OperatorAuthorHelper/SchemaInferenceOverrider.h b/onnxruntime/core/providers/dml/OperatorAuthorHelper/SchemaInferenceOverrider.h index 19fc6280246c6..62e8c30cd19ef 100644 --- a/onnxruntime/core/providers/dml/OperatorAuthorHelper/SchemaInferenceOverrider.h +++ b/onnxruntime/core/providers/dml/OperatorAuthorHelper/SchemaInferenceOverrider.h @@ -80,7 +80,7 @@ OverrideSchemaInferenceFunction #include -#define _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING // required by VS 2019 +#define _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING // required by VS 2019 #include #undef _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING #include @@ -102,117 +102,102 @@ static void ParseVersion(const char* version, int* major, int* minor, int* patch *patch = ver_num_fn(val); } -static void VerifyCacheVersion(const std::string& so_path) { - static std::atomic cache_version_checked{false}; - static std::mutex cache_version_mutex; - - // make sure we only check cache version once - if (!cache_version_checked.load(std::memory_order::memory_order_acquire)) { - std::lock_guard lock(cache_version_mutex); - if (!cache_version_checked.load(std::memory_order::memory_order_acquire)) { - cache_version_checked.store(true, std::memory_order::memory_order_release); - // ensure we have _ORTInternal_GetCacheVersion_ function - void* f = GetFuncFromLibrary(so_path, "_ORTInternal_GetCacheVersion", /*throw_if_not_found*/ true); - ORT_ENFORCE(f, "NULL library function pointer!"); - - typedef const char* (*GetVersionFunc)(); - GetVersionFunc func = reinterpret_cast(f); - const char* cache_version = func(); - ORT_ENFORCE(cache_version, "Null cache version string!"); - int cur_major, cur_minor, cur_patch; - ParseVersion(__NUPHAR_CACHE_VERSION__, &cur_major, &cur_minor, &cur_patch); - int cache_major, cache_minor, cache_patch; - ParseVersion(cache_version, &cache_major, &cache_minor, &cache_patch); - - // make version check strict until we have thorough design for compatibility - ORT_ENFORCE((cur_major == cache_major) && (cur_minor == cache_minor), - "Current nuphar runtime version (", __NUPHAR_CACHE_VERSION__, - ") doesn't match cached dll version (", cache_version, ")"); - - cache_version_checked = true; - } +static bool VerifyCacheVersion(const std::string& so_path) { + // ensure we have _ORTInternal_GetCacheVersion_ function + void* f = GetFuncFromLibrary(so_path, "_ORTInternal_GetCacheVersion", /*throw_if_not_found*/ true); + if (f == nullptr) + return false; + + typedef const char* (*GetVersionFunc)(); + GetVersionFunc func = reinterpret_cast(f); + const char* cache_version = func(); + ORT_ENFORCE(cache_version, "Null cache version string!"); + int cur_major, cur_minor, cur_patch; + ParseVersion(__NUPHAR_CACHE_VERSION__, &cur_major, &cur_minor, &cur_patch); + int cache_major, cache_minor, cache_patch; + ParseVersion(cache_version, &cache_major, &cache_minor, &cache_patch); + + // make version check strict until we have thorough design for compatibility + bool version_match = (cur_major == cache_major) && (cur_minor == cache_minor); + if (!version_match) { + LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "Current nuphar runtime version (" << __NUPHAR_CACHE_VERSION__ << ") doesn't match cached dll version (" << cache_version << ")"; } + return version_match; } -static bool disable_caching_due_to_checksum_failure = false; - static bool VerifyTVMModuleChecksum(const std::string& so_path) { - static std::string last_so_path; - static bool last_checksum_validated = false; - static std::mutex checksum_mutex; - if (last_so_path != so_path) { - std::lock_guard lock(checksum_mutex); - if (last_so_path != so_path) { - disable_caching_due_to_checksum_failure = false; // reset disabled caching for a new file - last_so_path = so_path; - void* f = GetFuncFromLibrary(so_path, "_ORTInternal_GetCheckSum", /*throw_if_not_found*/ false); - if (f) { - typedef void (*GetChecksumFunc)(const char*&, size_t&); - GetChecksumFunc func = reinterpret_cast(f); - const char* model_checksum; - size_t model_checksum_len; - func(model_checksum, - model_checksum_len); - - codegen::CodeGenSettings& setting = codegen::CodeGenSettings::Instance(); - // When checksum is expected by dll/so, user must set environment variable - // NUPHAR_CACHE_MODEL_CHECKSUM from md5 digest of running model. - // User may choose to run with base model or simplified mode and any match - // would be regarded as validated. - // Note that checksum validation here is not designed as a security measurement, - // so checksum compute is not done inside ORT. - last_checksum_validated = - setting.OptionMatches( - kNupharCacheModelChecksum, - std::string(model_checksum, model_checksum_len)); - - if (!last_checksum_validated) { - LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "Cache checksum validation failed, using JIT..."; - disable_caching_due_to_checksum_failure = true; - } - } else { - // do not validate checksum if dll didn't require it (usually during debugging) - // TODO: force checksum validation in final release - last_checksum_validated = true; + void* f = GetFuncFromLibrary(so_path, "_ORTInternal_GetCheckSum", /*throw_if_not_found*/ false); + if (f) { + typedef void (*GetChecksumFunc)(const char*&, size_t&); + GetChecksumFunc func = reinterpret_cast(f); + const char* model_checksum; + size_t model_checksum_len; + func(model_checksum, + model_checksum_len); + + codegen::CodeGenSettings& setting = codegen::CodeGenSettings::Instance(); + // When checksum is expected by dll/so, user must set environment variable + // NUPHAR_CACHE_MODEL_CHECKSUM from md5 digest of running model. + // User may choose to run with base model or simplified mode and any match + // would be regarded as validated. + // Note that checksum validation here is not designed as a security measurement, + // so checksum compute is not done inside ORT. + if (setting.OptionMatches( + kNupharCacheModelChecksum, + std::string(model_checksum, model_checksum_len))) { + return true; + } else { + static std::mutex warn_mutex; + std::lock_guard warn_lock(warn_mutex); + static std::string last_warned_so_path; + if (last_warned_so_path != so_path) { + // warning only once for each so_path + LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "Cache checksum validation failed, using JIT..."; } + return false; } + } else { + // do not validate checksum if dll didn't require it (usually during debugging) + // TODO: force checksum validation in final release + return true; } - return last_checksum_validated; } -tvm::runtime::PackedFunc LoadTVMPackedFuncFromCache(const std::string& func_name) { +CacheStatus LoadTVMPackedFuncFromCache(const std::string& func_name, tvm::runtime::PackedFunc& func) { std::string so_path; - if (!GetCacheSoFilePath(so_path)) - return nullptr; + if (!GetCacheSoFilePath(so_path)) { + if (codegen::CodeGenSettings::Instance().HasOption(kNupharCachePath)) { + return CacheStatus::Missing; + } else { + return CacheStatus::NotInUse; + } + } - VerifyCacheVersion(so_path); + if (!VerifyCacheVersion(so_path)) { + return CacheStatus::Mismatch; + } if (!VerifyTVMModuleChecksum(so_path)) - return nullptr; + return CacheStatus::Mismatch; tvm::runtime::Module module = tvm::runtime::Module::LoadFromFile(so_path); - tvm::runtime::PackedFunc func = module.GetFunction(func_name); + func = module.GetFunction(func_name); if (func == nullptr) { LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "Cannot find " << func_name << " in cache, using JIT..."; + return CacheStatus::Missing; } - return func; + return CacheStatus::Found; } void SaveTVMModuleToCache(const std::string& filename, tvm::runtime::Module& module) { fs::path path; - if (disable_caching_due_to_checksum_failure) - return; - static std::mutex save_cache_mutex; - static std::unordered_set existing_files; std::lock_guard lock(save_cache_mutex); - if (existing_files.count(filename) == 0 && - GetOrCreateTVMModuleCacheDirectory(path, /*create*/ true)) { - existing_files.insert(filename); + if (GetOrCreateTVMModuleCacheDirectory(path, /*create*/ true)) { path.append(filename + ".o"); if (fs::exists(path)) { - LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "Object file " << path << " already exists, skip saving..."; + //LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "Object file " << path << " already exists, skip saving..."; return; } module->SaveToFile(path.string(), "o"); diff --git a/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.h b/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.h index fe9162911ce52..a9f75bdf17221 100644 --- a/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.h +++ b/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.h @@ -19,8 +19,15 @@ struct NupharSubgraphUnit; //forward // Helper functions to create or load from offline cached dll // note after saving to obj file, we need to use tvm Python to create dll // using script at onnxruntime/core/codegen/mti/scripts/create_shared.py -tvm::runtime::PackedFunc -LoadTVMPackedFuncFromCache(const std::string& func_name); + +enum class CacheStatus { + NotInUse, + Mismatch, + Missing, + Found, +}; + +CacheStatus LoadTVMPackedFuncFromCache(const std::string& func_name, tvm::runtime::PackedFunc& func); void SaveTVMModuleToCache(const std::string& filename, tvm::runtime::Module& module); std::string GetPackedFuncName(const nuphar::NupharSubgraphUnit& subgraph, const CodeGenTarget& codegen_target, int64_t parallel_min_workloads); diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.cc b/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.cc index 08031691c5ee7..f96913f2f9c38 100644 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.cc +++ b/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.cc @@ -56,13 +56,16 @@ static tvm::runtime::PackedFunc LowerLayoutFunc(const tvm_codegen::WeightLayout* std::string func_name = layout->Name() + "_marshall"; - tvm::runtime::PackedFunc cached_func = nuphar::LoadTVMPackedFuncFromCache(func_name); - - if (cached_func == nullptr) { + tvm::runtime::PackedFunc cached_func; + auto cache_status = nuphar::LoadTVMPackedFuncFromCache(func_name, cached_func); + if (cache_status != nuphar::CacheStatus::Found) { + ORT_ENFORCE(cached_func == nullptr); auto lowered = tvm::lower(S, {inputs[0], outputs[0]}, func_name, {}, config); auto module = tvm::build(lowered, tvm::target::llvm(), tvm::Target(), config); tvm_codegen::DumpTVMModuleToFile(func_name, module); - nuphar::SaveTVMModuleToCache(func_name, module); + if (cache_status == nuphar::CacheStatus::Missing) { + nuphar::SaveTVMModuleToCache(func_name, module); + } cached_func = module.GetFunction(func_name); } return cached_func; diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_compiler.cc b/onnxruntime/core/providers/nuphar/compiler/nuphar_compiler.cc index 9b41ac88cf4a0..4b648e0e386a7 100644 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_compiler.cc +++ b/onnxruntime/core/providers/nuphar/compiler/nuphar_compiler.cc @@ -151,8 +151,9 @@ tvm::runtime::PackedFunc NupharCompiler::GetLoweredPackedFunc( // JIT-caching and AOT are mutual exclusive. // Change it by not always saving a compiled func unless it is in JIT-Caching model. // In AOT, there should be another member func explicitly loading - tvm::runtime::PackedFunc cached_func = nuphar::LoadTVMPackedFuncFromCache(func_name); - if (cached_func == nullptr) { + tvm::runtime::PackedFunc cached_func; + auto cache_status = nuphar::LoadTVMPackedFuncFromCache(func_name, cached_func); + if (cache_status != nuphar::CacheStatus::Found) { codegen::CodeGenSettings& settings = codegen::CodeGenSettings::Instance(); if (settings.HasOption(kNupharCacheForceNoJIT)) { @@ -180,7 +181,9 @@ tvm::runtime::PackedFunc NupharCompiler::GetLoweredPackedFunc( tvm::runtime::Module module = tvm::build(lowered, tvm_target, tvm_host_target, config); tvm_codegen::DumpTVMModuleToFile(func_name, module); - nuphar::SaveTVMModuleToCache(func_name, module); + if (cache_status == nuphar::CacheStatus::Missing) { + nuphar::SaveTVMModuleToCache(func_name, module); + } cached_func = module.GetFunction(func_name); } diff --git a/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc b/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc index 6e76bc45d0ad9..1f61783b775f5 100644 --- a/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc +++ b/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc @@ -25,7 +25,7 @@ namespace onnxruntime { thread_local int64_t NupharSubgraphUnit::counter = 0; thread_local std::unique_ptr> NupharExecutionProvider::tls_realized_dims_; -int NupharExecutionProvider::global_fused_count_ = 0; +thread_local int NupharExecutionProvider::per_model_fused_count_ = 0; static std::string GetCurrentHostTargetString() { #if USE_TVM_WITH_LLVM @@ -312,12 +312,12 @@ NupharExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie }; GraphPartitioner graph_partitioner(is_supported_func); - ORT_ENFORCE(graph_partitioner.Partition(graph_viewer, global_fused_count_, results).IsOK()); + ORT_ENFORCE(graph_partitioner.Partition(graph_viewer, per_model_fused_count_, results).IsOK()); - // reset global_fused_count_ for main graph, since there might be multiple sessions for subgraphs, + // reset per_model_fused_count_ for main graph, since there might be multiple sessions for subgraphs, // this is the time all graph cut should be finished as ORT handles main graph last if (!graph_viewer.IsSubgraph()) - global_fused_count_ = 0; + per_model_fused_count_ = 0; // for any node being fused in results, save initializer tensors // because IExecutionProvider::Compile would be called without OpKernelInfo diff --git a/onnxruntime/core/providers/nuphar/nuphar_execution_provider.h b/onnxruntime/core/providers/nuphar/nuphar_execution_provider.h index edfb5aa14ffc3..67c26194f1498 100644 --- a/onnxruntime/core/providers/nuphar/nuphar_execution_provider.h +++ b/onnxruntime/core/providers/nuphar/nuphar_execution_provider.h @@ -154,9 +154,10 @@ class NupharExecutionProvider : public IExecutionProvider { mutable std::unordered_map> constant_initializers_used_in_compiled_nodes_; mutable std::unordered_map domain_versions_; - // used to create unique fused node name, make it static because - // subsession may create multiple instances of EPs - static int global_fused_count_; + // used to create unique fused node name, make it thread_local because + // subsession of a model with subgraph may create multiple instances of EPs, + // and there might be multiple inference sessions running different models concurrently + static thread_local int per_model_fused_count_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 028a34e09d2e0..1a56b8229e2c1 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -676,7 +676,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectorbuilder; - auto trt_profile = trt_builder->createOptimizationProfile(); + nvinfer1::IOptimizationProfile* trt_profile = nullptr; for (int i = 0, end = num_binding_inputs; i < end; ++i) { // TODO: check if getInput indexing is same with binding index auto input = trt_state->network->getInput(i); @@ -714,6 +714,9 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectorcreateOptimizationProfile(); + } if (engine.isShapeBinding(i)) { std::vector shapes_min(nb_dims), shapes_opt(nb_dims), shapes_max(nb_dims); for (int j = 0, end = nb_dims; j < end; ++j) { @@ -733,7 +736,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectorsetDimensions(input->getName(), nvinfer1::OptProfileSelector::kMIN, dims_min); trt_profile->setDimensions(input->getName(), nvinfer1::OptProfileSelector::kOPT, dims_opt); trt_profile->setDimensions(input->getName(), nvinfer1::OptProfileSelector::kMAX, dims_max); @@ -746,10 +749,13 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector(trt_builder->createBuilderConfig()); trt_config->addOptimizationProfile(trt_profile); trt_state->engine = trt_builder->buildEngineWithConfig(*trt_state->network, *trt_config); - ORT_ENFORCE(trt_state->engine != nullptr); - + if (trt_state->engine == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP Failed to Build Engine."); + } trt_state->context = trt_state->engine->createExecutionContext(); - ORT_ENFORCE(trt_state->context != nullptr); + if (trt_state->context == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP Failed to Create Context."); + } trt_context = trt_state->context; } diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index 961172bce22cb..7642faee251bd 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -140,7 +140,7 @@ OrtStatus* CreateTensorImpl(MLDataType ml_type, const int64_t* shape, size_t sha size_t elem_count = 1; std::vector shapes(shape_len); for (size_t i = 0; i != shape_len; ++i) { - elem_count *= shape[i]; + elem_count *= static_cast(shape[i]); shapes[i] = shape[i]; } @@ -702,16 +702,8 @@ ORT_API_STATUS_IMPL(OrtApis::AllocatorGetInfo, _In_ const OrtAllocator* ptr, _Ou API_IMPL_END } -/////////////////////////////////////////////////////////////////////////// -// Code to handle non-tensor types -// OrtGetValueCount -// OrtGetVaue -// OrtCreateValue -/////////////////////////////////////////////////////////////////////////// const int NUM_MAP_INDICES = 2; -//////////////////// -// OrtGetValueCount template OrtStatus* OrtGetNumSequenceElements(const OrtValue* p_ml_value, size_t* out) { auto& data = p_ml_value->Get(); @@ -762,7 +754,7 @@ ORT_API_STATUS_IMPL(OrtApis::GetValueCount, const OrtValue* value, size_t* out) } /////////////////// -// OrtGetValue +// OrtGetValueImplSeqOfMap template static OrtStatus* OrtGetValueImplSeqOfMap(const OrtValue* p_ml_value, int index, OrtValue** out) { using TKey = typename T::value_type::key_type; @@ -866,28 +858,31 @@ static OrtStatus* OrtGetValueImplMapHelper(const OrtValue* p_ml_value, int index using TVal = typename T::mapped_type; auto& data = p_ml_value->Get(); int64_t num_kv_pairs = data.size(); +#if defined(_WIN32) && !defined(_M_AMD64) + ORT_ENFORCE(static_cast(num_kv_pairs) < std::numeric_limits::max()); +#endif switch (index) { case 0: { // user is requesting keys std::vector vec; - vec.reserve(num_kv_pairs); + vec.reserve(static_cast(num_kv_pairs)); for (const auto& kv : data) { vec.push_back(kv.first); } std::vector dims{num_kv_pairs}; OrtStatus* st = OrtApis::CreateTensorAsOrtValue(allocator, dims.data(), dims.size(), GetONNXTensorElementDataType(), out); - return st ? st : PopulateTensorWithData(*out, vec.data(), num_kv_pairs, sizeof(TKey)); + return st ? st : PopulateTensorWithData(*out, vec.data(), static_cast(num_kv_pairs), sizeof(TKey)); } case 1: { // user is requesting values std::vector vec; - vec.reserve(num_kv_pairs); + vec.reserve(static_cast(num_kv_pairs)); for (const auto& kv : data) { vec.push_back(kv.second); } std::vector dims{num_kv_pairs}; OrtStatus* st = OrtApis::CreateTensorAsOrtValue(allocator, dims.data(), dims.size(), GetONNXTensorElementDataType(), out); - return st ? st : PopulateTensorWithData(*out, vec.data(), num_kv_pairs, sizeof(TVal)); + return st ? st : PopulateTensorWithData(*out, vec.data(), static_cast(num_kv_pairs), sizeof(TVal)); } default: return OrtApis::CreateStatus(ORT_FAIL, "Invalid index requested for map type."); @@ -1095,7 +1090,9 @@ static OrtStatus* OrtCreateMapMLValue(const Tensor& key_tensor, const Tensor& va // iterate through the key and value tensors and populate map auto key_data = key_tensor.Data(); auto value_data = value_tensor.Data(); - size_t num_kv_pairs = key_tensor.Shape().Size(); + auto len = key_tensor.Shape().Size(); + ORT_ENFORCE(len >= 0 && static_cast(len) < std::numeric_limits::max()); + size_t num_kv_pairs = static_cast(key_tensor.Shape().Size()); for (size_t n = 0; n < num_kv_pairs; ++n, ++key_data, ++value_data) { map_ptr->insert({*key_data, *value_data}); } diff --git a/onnxruntime/core/util/protobuf_parsing_utils.cc b/onnxruntime/core/util/protobuf_parsing_utils.cc deleted file mode 100644 index 7cdf60bb783a3..0000000000000 --- a/onnxruntime/core/util/protobuf_parsing_utils.cc +++ /dev/null @@ -1,481 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: kenton@google.com (Kenton Varda) -// Based on original Protocol Buffers design by -// Sanjay Ghemawat, Jeff Dean, and others. - -// Modifications Copyright (c) Microsoft. -// Introduced this file based on suggestion by Kenton Varda as seen here -// https://groups.google.com/forum/#!topic/protobuf/J3H0GSlrfIo -// The idea is to retain the same efficiency of loading models. - -#ifndef _MSC_VER -#include -#include -#include -#include -#endif -#include -#include -#include -#include -#include -#include -#ifdef _WIN32 -#include -#endif -#include "protobuf_parsing_utils.h" -#include - -namespace google { -namespace protobuf { -namespace io { - -#ifdef _WIN32 -// Win32 lseek is broken: If invoked on a non-seekable file descriptor, its -// return value is undefined. We re-define it to always produce an error. -#define lseek(fd, offset, origin) ((off_t)-1) -// DO NOT include , instead create functions in io_win32.{h,cc} and import -// them like we do below. -using google::protobuf::internal::win32::access; -using google::protobuf::internal::win32::close; -using google::protobuf::internal::win32::open; -using google::protobuf::internal::win32::read; -using google::protobuf::internal::win32::write; -#endif - -namespace { - -// EINTR sucks. -int close_no_eintr(int fd) { - int result; - do { - result = close(fd); - } while (result < 0 && errno == EINTR); - return result; -} - -} // namespace - -// =================================================================== - -FileInputStream::FileInputStream(int file_descriptor, int block_size) - : copying_input_(file_descriptor), - impl_(©ing_input_, block_size) { -} - -bool FileInputStream::Close() { - return copying_input_.Close(); -} - -bool FileInputStream::Next(const void** data, int* size) { - return impl_.Next(data, size); -} - -void FileInputStream::BackUp(int count) { - impl_.BackUp(count); -} - -bool FileInputStream::Skip(int count) { - return impl_.Skip(count); -} - -int64 FileInputStream::ByteCount() const { - return impl_.ByteCount(); -} - -FileInputStream::CopyingFileInputStream::CopyingFileInputStream( - int file_descriptor) - : file_(file_descriptor), - close_on_delete_(false), - is_closed_(false), - errno_(0), - previous_seek_failed_(false) { -} - -FileInputStream::CopyingFileInputStream::~CopyingFileInputStream() { - if (close_on_delete_) { - if (!Close()) { -#ifdef _WIN32 - const size_t errmsglen = 100; // strerrorlen_s not available on MSVC, https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/strerror-s-strerror-s-wcserror-s-wcserror-s?view=vs-2017 - char errmsg[errmsglen]; - strerror_s(errmsg, errmsglen, errno); - GOOGLE_LOG(ERROR) << "close() failed: " << errmsg; -#else // POSIX - char* e_msg = strerror(errno); - GOOGLE_LOG(ERROR) << "close() failed: " << e_msg; -#endif - } - } -} - -bool FileInputStream::CopyingFileInputStream::Close() { - GOOGLE_CHECK(!is_closed_); - - is_closed_ = true; - if (close_no_eintr(file_) != 0) { - // The docs on close() do not specify whether a file descriptor is still - // open after close() fails with EIO. However, the glibc source code - // seems to indicate that it is not. - errno_ = errno; - return false; - } - - return true; -} - -int FileInputStream::CopyingFileInputStream::Read(void* buffer, int size) { - GOOGLE_CHECK(!is_closed_); - - int result; - do { - result = read(file_, buffer, size); - } while (result < 0 && errno == EINTR); - - if (result < 0) { - // Read error (not EOF). - errno_ = errno; - } - - return result; -} - -int FileInputStream::CopyingFileInputStream::Skip(int count) { - GOOGLE_CHECK(!is_closed_); - - if (!previous_seek_failed_ && lseek(file_, count, SEEK_CUR) != static_cast(-1)) { - // Seek succeeded. - return count; - } - // Failed to seek. - - // Note to self: Don't seek again. This file descriptor doesn't - // support it. - previous_seek_failed_ = true; - - // Use the default implementation. - return CopyingInputStream::Skip(count); -} - -// =================================================================== - -FileOutputStream::FileOutputStream(int file_descriptor, int block_size) - : copying_output_(file_descriptor), - impl_(©ing_output_, block_size) { -} - -FileOutputStream::~FileOutputStream() { - impl_.Flush(); -} - -bool FileOutputStream::Close() { - bool flush_succeeded = impl_.Flush(); - return copying_output_.Close() && flush_succeeded; -} - -bool FileOutputStream::Flush() { - return impl_.Flush(); -} - -bool FileOutputStream::Next(void** data, int* size) { - return impl_.Next(data, size); -} - -void FileOutputStream::BackUp(int count) { - impl_.BackUp(count); -} - -int64 FileOutputStream::ByteCount() const { - return impl_.ByteCount(); -} - -FileOutputStream::CopyingFileOutputStream::CopyingFileOutputStream( - int file_descriptor) - : file_(file_descriptor), - close_on_delete_(false), - is_closed_(false), - errno_(0) { -} - -FileOutputStream::CopyingFileOutputStream::~CopyingFileOutputStream() { - if (close_on_delete_) { - if (!Close()) { -#ifdef _WIN32 - // https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/strerror-s-strerror-s-wcserror-s-wcserror-s?view=vs-2017 - const size_t errmsglen = 100; //strerrorlen_s not available on MSVC - char errmsg[errmsglen]; - strerror_s(errmsg, errmsglen, errno); - GOOGLE_LOG(ERROR) << "close() failed: " << errmsg; -#else // POSIX - char* e_msg = strerror(errno); - GOOGLE_LOG(ERROR) << "close() failed: " << e_msg; -#endif - } - } -} - -bool FileOutputStream::CopyingFileOutputStream::Close() { - GOOGLE_CHECK(!is_closed_); - - is_closed_ = true; - if (close_no_eintr(file_) != 0) { - // The docs on close() do not specify whether a file descriptor is still - // open after close() fails with EIO. However, the glibc source code - // seems to indicate that it is not. - errno_ = errno; - return false; - } - - return true; -} - -bool FileOutputStream::CopyingFileOutputStream::Write( - const void* buffer, int size) { - GOOGLE_CHECK(!is_closed_); - int total_written = 0; - - const uint8* buffer_base = reinterpret_cast(buffer); - - while (total_written < size) { - int bytes; - do { - bytes = write(file_, buffer_base + total_written, size - total_written); - } while (bytes < 0 && errno == EINTR); - - if (bytes <= 0) { - // Write error. - - // FIXME(kenton): According to the man page, if write() returns zero, - // there was no error; write() simply did not write anything. It's - // unclear under what circumstances this might happen, but presumably - // errno won't be set in this case. I am confused as to how such an - // event should be handled. For now I'm treating it as an error, since - // retrying seems like it could lead to an infinite loop. I suspect - // this never actually happens anyway. - - if (bytes < 0) { - errno_ = errno; - } - return false; - } - total_written += bytes; - } - - return true; -} - -// =================================================================== - -IstreamInputStream::IstreamInputStream(std::istream* input, int block_size) - : copying_input_(input), impl_(©ing_input_, block_size) {} - -bool IstreamInputStream::Next(const void** data, int* size) { - return impl_.Next(data, size); -} - -void IstreamInputStream::BackUp(int count) { - impl_.BackUp(count); -} - -bool IstreamInputStream::Skip(int count) { - return impl_.Skip(count); -} - -int64 IstreamInputStream::ByteCount() const { - return impl_.ByteCount(); -} - -IstreamInputStream::CopyingIstreamInputStream::CopyingIstreamInputStream( - std::istream* input) - : input_(input) {} - -IstreamInputStream::CopyingIstreamInputStream::~CopyingIstreamInputStream() = default; - -int IstreamInputStream::CopyingIstreamInputStream::Read( - void* buffer, int size) { - input_->read(reinterpret_cast(buffer), size); - int result = static_cast(input_->gcount()); - if (result == 0 && input_->fail() && !input_->eof()) { - return -1; - } - return result; -} - -// =================================================================== - -OstreamOutputStream::OstreamOutputStream(std::ostream* output, int block_size) - : copying_output_(output), impl_(©ing_output_, block_size) {} - -OstreamOutputStream::~OstreamOutputStream() { - impl_.Flush(); -} - -bool OstreamOutputStream::Next(void** data, int* size) { - return impl_.Next(data, size); -} - -void OstreamOutputStream::BackUp(int count) { - impl_.BackUp(count); -} - -int64 OstreamOutputStream::ByteCount() const { - return impl_.ByteCount(); -} - -OstreamOutputStream::CopyingOstreamOutputStream::CopyingOstreamOutputStream( - std::ostream* output) - : output_(output) {} - -OstreamOutputStream::CopyingOstreamOutputStream::~CopyingOstreamOutputStream() = default; - -bool OstreamOutputStream::CopyingOstreamOutputStream::Write( - const void* buffer, int size) { - output_->write(reinterpret_cast(buffer), size); - return output_->good(); -} - -// =================================================================== - -ConcatenatingInputStream::ConcatenatingInputStream( - ZeroCopyInputStream* const streams[], int count) - : streams_(streams), stream_count_(count), bytes_retired_(0) { -} - -bool ConcatenatingInputStream::Next(const void** data, int* size) { - while (stream_count_ > 0) { - if (streams_[0]->Next(data, size)) return true; - - // That stream is done. Advance to the next one. - bytes_retired_ += streams_[0]->ByteCount(); - ++streams_; - --stream_count_; - } - - // No more streams. - return false; -} - -void ConcatenatingInputStream::BackUp(int count) { - if (stream_count_ > 0) { - streams_[0]->BackUp(count); - } else { - GOOGLE_LOG(DFATAL) << "Can't BackUp() after failed Next()."; - } -} - -bool ConcatenatingInputStream::Skip(int count) { - while (stream_count_ > 0) { - // Assume that ByteCount() can be used to find out how much we actually - // skipped when Skip() fails. - int64 target_byte_count = streams_[0]->ByteCount() + count; - if (streams_[0]->Skip(count)) return true; - - // Hit the end of the stream. Figure out how many more bytes we still have - // to skip. - int64 final_byte_count = streams_[0]->ByteCount(); - GOOGLE_DCHECK_LT(final_byte_count, target_byte_count); - count = static_cast(target_byte_count - final_byte_count); - - // That stream is done. Advance to the next one. - bytes_retired_ += final_byte_count; - ++streams_; - --stream_count_; - } - - return false; -} - -int64 ConcatenatingInputStream::ByteCount() const { - if (stream_count_ == 0) { - return bytes_retired_; - } - return bytes_retired_ + streams_[0]->ByteCount(); -} - -// =================================================================== - -LimitingInputStream::LimitingInputStream(ZeroCopyInputStream* input, - int64 limit) - : input_(input), limit_(limit) { - prior_bytes_read_ = input_->ByteCount(); -} - -LimitingInputStream::~LimitingInputStream() { - // If we overshot the limit, back up. - if (limit_ < 0) input_->BackUp(static_cast(-limit_)); -} - -bool LimitingInputStream::Next(const void** data, int* size) { - if (limit_ <= 0) return false; - if (!input_->Next(data, size)) return false; - - limit_ -= *size; - if (limit_ < 0) { - // We overshot the limit. Reduce *size to hide the rest of the buffer. - *size += static_cast(limit_); - } - return true; -} - -void LimitingInputStream::BackUp(int count) { - if (limit_ < 0) { - input_->BackUp(static_cast(count - limit_)); - limit_ = count; - } else { - input_->BackUp(count); - limit_ += count; - } -} - -bool LimitingInputStream::Skip(int count) { - if (count > limit_) { - if (limit_ < 0) return false; - input_->Skip(static_cast(limit_)); - limit_ = 0; - return false; - } - if (!input_->Skip(count)) return false; - limit_ -= count; - return true; -} - -int64 LimitingInputStream::ByteCount() const { - if (limit_ < 0) { - return input_->ByteCount() + limit_ - prior_bytes_read_; - } - return input_->ByteCount() - prior_bytes_read_; -} - -// =================================================================== - -} // namespace io -} // namespace protobuf -} // namespace google diff --git a/onnxruntime/core/util/protobuf_parsing_utils.h b/onnxruntime/core/util/protobuf_parsing_utils.h index 34c261b79f41e..4c97562653f04 100644 --- a/onnxruntime/core/util/protobuf_parsing_utils.h +++ b/onnxruntime/core/util/protobuf_parsing_utils.h @@ -1,41 +1,3 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: kenton@google.com (Kenton Varda) -// Based on original Protocol Buffers design by -// Sanjay Ghemawat, Jeff Dean, and others. -// -// This file contains common implementations of the interfaces defined in -// zero_copy_stream.h which are only included in the full (non-lite) -// protobuf library. These implementations include Unix file descriptors -// and C++ iostreams. See also: zero_copy_stream_impl_lite.h #pragma once @@ -317,30 +279,6 @@ class LIBPROTOBUF_EXPORT ConcatenatingInputStream : public ZeroCopyInputStream { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ConcatenatingInputStream); }; -// =================================================================== - -// A ZeroCopyInputStream which wraps some other stream and limits it to -// a particular byte count. -class LIBPROTOBUF_EXPORT LimitingInputStream : public ZeroCopyInputStream { - public: - LimitingInputStream(ZeroCopyInputStream* input, int64 limit); - ~LimitingInputStream() override; - - // implements ZeroCopyInputStream ---------------------------------- - bool Next(const void** data, int* size) override; - void BackUp(int count) override; - bool Skip(int count) override; - int64 ByteCount() const override; - - private: - ZeroCopyInputStream* input_; - int64 limit_; // Decreases as we go, becomes negative if we overshoot. - int64 prior_bytes_read_; // Bytes read on underlying stream at construction - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(LimitingInputStream); -}; - -// =================================================================== } // namespace io } // namespace protobuf diff --git a/onnxruntime/featurizers_ops/cpu/cat_imputer_transformer.cc b/onnxruntime/featurizers_ops/cpu/cat_imputer_transformer.cc index 7aa53d734e298..7a5ee782e3d2f 100644 --- a/onnxruntime/featurizers_ops/cpu/cat_imputer_transformer.cc +++ b/onnxruntime/featurizers_ops/cpu/cat_imputer_transformer.cc @@ -3,6 +3,7 @@ #include "core/common/common.h" #include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" #include "core/framework/op_kernel.h" #include "Featurizers/CatImputerFeaturizer.h" @@ -27,12 +28,8 @@ inline nonstd::optional PreprocessOptional(std::string value) { } template -class CatImputerTransformer final : public OpKernel { - public: - explicit CatImputerTransformer(const OpKernelInfo& info) : OpKernel(info) { - } - - Status Compute(OpKernelContext* ctx) const override { +struct CatImputerTransformerImpl { + void operator()(OpKernelContext* ctx) const { // Create the transformer Microsoft::Featurizer::Featurizers::CatImputerTransformer transformer( [ctx](void) { @@ -57,40 +54,32 @@ class CatImputerTransformer final : public OpKernel { for (int64_t i = 0; i < length; ++i) { output_data[i] = transformer.execute(PreprocessOptional(input_data[i])); } - - return Status::OK(); } }; -ONNX_OPERATOR_TYPED_KERNEL_EX( - CatImputerTransformer, - kMSFeaturizersDomain, - 1, - float, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()), - CatImputerTransformer); +class CatImputerTransformer final : public OpKernel { + public: + explicit CatImputerTransformer(const OpKernelInfo& info) : OpKernel(info) { + } -ONNX_OPERATOR_TYPED_KERNEL_EX( - CatImputerTransformer, - kMSFeaturizersDomain, - 1, - double, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()), - CatImputerTransformer); + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; -ONNX_OPERATOR_TYPED_KERNEL_EX( +ONNX_OPERATOR_KERNEL_EX( CatImputerTransformer, kMSFeaturizersDomain, 1, - string, kCpuExecutionProvider, KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()), - CatImputerTransformer); + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + CatImputerTransformer); } // namespace featurizers } // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/date_time_transformer.cc b/onnxruntime/featurizers_ops/cpu/date_time_transformer.cc index 1ac583b0e5b76..13f780414d988 100644 --- a/onnxruntime/featurizers_ops/cpu/date_time_transformer.cc +++ b/onnxruntime/featurizers_ops/cpu/date_time_transformer.cc @@ -3,6 +3,7 @@ #include "core/common/common.h" #include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" #include "core/framework/op_kernel.h" #include "Featurizers/DateTimeFeaturizer.h" @@ -29,7 +30,7 @@ class DateTimeTransformer final : public OpKernel { // Get the input const auto* input_tensor(ctx->Input(1)); - const std::int64_t* input_data(input_tensor->Data()); + const int64_t* input_data(input_tensor->Data()); // Prepare the output Tensor* year_tensor(ctx->Output(0, input_tensor->Shape())); @@ -80,7 +81,7 @@ class DateTimeTransformer final : public OpKernel { const int64_t length(input_tensor->Shape().Size()); for (int64_t i = 0; i < length; ++i) { - auto result(transformer.execute(input_data[i])); + auto result(transformer.execute(std::chrono::system_clock::from_time_t(input_data[i]))); year_data[i] = std::move(result.year); month_data[i] = std::move(result.month); @@ -115,8 +116,8 @@ ONNX_OPERATOR_KERNEL_EX( 1, kCpuExecutionProvider, KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()) - .TypeConstraint("T1", DataTypeImpl::GetTensorType()), + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("T1", DataTypeImpl::GetTensorType()), DateTimeTransformer); } // namespace featurizers diff --git a/onnxruntime/featurizers_ops/cpu/hash_one_hot_vectorizer_transformer.cc b/onnxruntime/featurizers_ops/cpu/hash_one_hot_vectorizer_transformer.cc new file mode 100644 index 0000000000000..747dcf885cb23 --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/hash_one_hot_vectorizer_transformer.cc @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/HashOneHotVectorizerFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +template +struct HashOneHotVectorizerTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::HashOneHotVectorizerTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::HashOneHotVectorizerTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* NumElements_tensor(ctx->Output(0, input_tensor->Shape())); + Tensor* Value_tensor(ctx->Output(1, input_tensor->Shape())); + Tensor* Index_tensor(ctx->Output(2, input_tensor->Shape())); + + uint64_t* NumElements_data(NumElements_tensor->MutableData()); + uint8_t* Value_data(Value_tensor->MutableData()); + uint64_t* Index_data(Index_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + auto result(transformer.execute(input_data[i])); + + NumElements_data[i] = std::move(result.NumElements); + Value_data[i] = std::move(result.Value); + Index_data[i] = std::move(result.Index); + } + } +}; + +class HashOneHotVectorizerTransformer final : public OpKernel { + public: + explicit HashOneHotVectorizerTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + HashOneHotVectorizerTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + HashOneHotVectorizerTransformer); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/imputation_marker_transformer.cc b/onnxruntime/featurizers_ops/cpu/imputation_marker_transformer.cc new file mode 100644 index 0000000000000..2612f448e7335 --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/imputation_marker_transformer.cc @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/ImputationMarkerFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +inline float const& PreprocessOptional(float const& value) { return value; } +inline double const& PreprocessOptional(double const& value) { return value; } +inline nonstd::optional PreprocessOptional(std::string value) { + return value.empty() ? nonstd::optional() : nonstd::optional(std::move(value)); +} + +template +struct ImputationMarkerTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::ImputationMarkerTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::ImputationMarkerTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + bool* output_data(output_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(PreprocessOptional(input_data[i])); + } + } +}; + +class ImputationMarkerTransformer final : public OpKernel { + public: + explicit ImputationMarkerTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + ImputationMarkerTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + ImputationMarkerTransformer); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/label_encoder_transformer.cc b/onnxruntime/featurizers_ops/cpu/label_encoder_transformer.cc new file mode 100644 index 0000000000000..6350da43fc906 --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/label_encoder_transformer.cc @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/LabelEncoderFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +template +struct LabelEncoderTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::LabelEncoderTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::LabelEncoderTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + uint32_t* output_data(output_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(input_data[i]); + } + } +}; + +class LabelEncoderTransformer final : public OpKernel { + public: + explicit LabelEncoderTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + LabelEncoderTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + LabelEncoderTransformer); +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/max_abs_scalar_transformer.cc b/onnxruntime/featurizers_ops/cpu/max_abs_scalar_transformer.cc index 98b7845997e34..0208db93af0bf 100644 --- a/onnxruntime/featurizers_ops/cpu/max_abs_scalar_transformer.cc +++ b/onnxruntime/featurizers_ops/cpu/max_abs_scalar_transformer.cc @@ -3,6 +3,7 @@ #include "core/common/common.h" #include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" #include "core/framework/op_kernel.h" #include "Featurizers/MaxAbsScalarFeaturizer.h" @@ -35,12 +36,8 @@ template <> struct OutputTypeMapper { using type = double; }; template -class MaxAbsScalarTransformer final : public OpKernel { - public: - explicit MaxAbsScalarTransformer(const OpKernelInfo& info) : OpKernel(info) { - } - - Status Compute(OpKernelContext* ctx) const override { +struct MaxAbsScalarTransformerImpl { + void operator()(OpKernelContext* ctx) const { // Create the transformer Microsoft::Featurizer::Featurizers::MaxAbsScalarTransformer::type> transformer( [ctx](void) { @@ -65,110 +62,40 @@ class MaxAbsScalarTransformer final : public OpKernel { for (int64_t i = 0; i < length; ++i) { output_data[i] = transformer.execute(input_data[i]); } - - return Status::OK(); } }; -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - int8, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - int16, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - uint8, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - uint16, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - float, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - int32, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - int64, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - uint32, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); +class MaxAbsScalarTransformer final : public OpKernel { + public: + explicit MaxAbsScalarTransformer(const OpKernelInfo& info) : OpKernel(info) { + } -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - uint64, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; -ONNX_OPERATOR_TYPED_KERNEL_EX( +ONNX_OPERATOR_KERNEL_EX( MaxAbsScalarTransformer, kMSFeaturizersDomain, 1, - double, kCpuExecutionProvider, KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + MaxAbsScalarTransformer); } // namespace featurizers } // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/min_max_scalar_transformer.cc b/onnxruntime/featurizers_ops/cpu/min_max_scalar_transformer.cc new file mode 100644 index 0000000000000..281e9cd8099d2 --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/min_max_scalar_transformer.cc @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/MinMaxScalarFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +template +struct MinMaxScalarTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::MinMaxScalarTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::MinMaxScalarTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + double* output_data(output_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(input_data[i]); + } + } +}; + +class MinMaxScalarTransformer final : public OpKernel { + public: + explicit MinMaxScalarTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + MinMaxScalarTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + MinMaxScalarTransformer); +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/missing_dummies_transformer.cc b/onnxruntime/featurizers_ops/cpu/missing_dummies_transformer.cc new file mode 100644 index 0000000000000..5d40abcf761ad --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/missing_dummies_transformer.cc @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/MissingDummiesFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +inline float const& PreprocessOptional(float const& value) { return value; } +inline double const& PreprocessOptional(double const& value) { return value; } +inline nonstd::optional PreprocessOptional(std::string value) { + return value.empty() ? nonstd::optional() : nonstd::optional(std::move(value)); +} + +template +struct MissingDummiesTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::MissingDummiesTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::MissingDummiesTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + int8_t* output_data(output_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(PreprocessOptional(input_data[i])); + } + } +}; + +class MissingDummiesTransformer final : public OpKernel { + public: + explicit MissingDummiesTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + MissingDummiesTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + MissingDummiesTransformer); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/one_hot_encoder_transformer.cc b/onnxruntime/featurizers_ops/cpu/one_hot_encoder_transformer.cc new file mode 100644 index 0000000000000..1a299456ecbed --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/one_hot_encoder_transformer.cc @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/OneHotEncoderFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +template +struct OneHotEncoderTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::OneHotEncoderTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::OneHotEncoderTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* NumElements_tensor(ctx->Output(0, input_tensor->Shape())); + Tensor* Value_tensor(ctx->Output(1, input_tensor->Shape())); + Tensor* Index_tensor(ctx->Output(2, input_tensor->Shape())); + + uint64_t* NumElements_data(NumElements_tensor->MutableData()); + uint8_t* Value_data(Value_tensor->MutableData()); + uint64_t* Index_data(Index_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + auto result(transformer.execute(input_data[i])); + + NumElements_data[i] = std::move(result.NumElements); + Value_data[i] = std::move(result.Value); + Index_data[i] = std::move(result.Index); + } + } +}; + +class OneHotEncoderTransformer final : public OpKernel { + public: + explicit OneHotEncoderTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + OneHotEncoderTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + OneHotEncoderTransformer); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/robust_scalar_transformer.cc b/onnxruntime/featurizers_ops/cpu/robust_scalar_transformer.cc new file mode 100644 index 0000000000000..728c9c710508f --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/robust_scalar_transformer.cc @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/RobustScalarFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +template +struct OutputTypeMapper {}; +template <> +struct OutputTypeMapper { using type = float; }; +template <> +struct OutputTypeMapper { using type = float; }; +template <> +struct OutputTypeMapper { using type = float; }; +template <> +struct OutputTypeMapper { using type = float; }; +template <> +struct OutputTypeMapper { using type = float; }; +template <> +struct OutputTypeMapper { using type = double; }; +template <> +struct OutputTypeMapper { using type = double; }; +template <> +struct OutputTypeMapper { using type = double; }; +template <> +struct OutputTypeMapper { using type = double; }; +template <> +struct OutputTypeMapper { using type = double; }; + +template +struct RobustScalarTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::RobustScalarTransformer::type> transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::RobustScalarTransformer::type>(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + typename OutputTypeMapper::type* output_data(output_tensor->MutableData::type>()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(input_data[i]); + } + } +}; + +class RobustScalarTransformer final : public OpKernel { + public: + explicit RobustScalarTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + RobustScalarTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + RobustScalarTransformer); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/string_transformer.cc b/onnxruntime/featurizers_ops/cpu/string_transformer.cc index 8f719552dd309..6ee65b8e2633d 100644 --- a/onnxruntime/featurizers_ops/cpu/string_transformer.cc +++ b/onnxruntime/featurizers_ops/cpu/string_transformer.cc @@ -3,6 +3,7 @@ #include "core/common/common.h" #include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" #include "core/framework/op_kernel.h" #include "Featurizers/StringFeaturizer.h" @@ -12,12 +13,8 @@ namespace onnxruntime { namespace featurizers { template -class StringTransformer final : public OpKernel { - public: - explicit StringTransformer(const OpKernelInfo& info) : OpKernel(info) { - } - - Status Compute(OpKernelContext* ctx) const override { +struct StringTransformerImpl { + void operator()(OpKernelContext* ctx) const { // Create the transformer Microsoft::Featurizer::Featurizers::StringTransformer transformer( [ctx](void) { @@ -42,130 +39,43 @@ class StringTransformer final : public OpKernel { for (int64_t i = 0; i < length; ++i) { output_data[i] = transformer.execute(input_data[i]); } - - return Status::OK(); } }; -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - int8, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - int16, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - int32, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - int64, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - uint8, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - uint16, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - uint32, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - uint64, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - float, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - double, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); +class StringTransformer final : public OpKernel { + public: + explicit StringTransformer(const OpKernelInfo& info) : OpKernel(info) { + } -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - bool, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; -ONNX_OPERATOR_TYPED_KERNEL_EX( +ONNX_OPERATOR_KERNEL_EX( StringTransformer, kMSFeaturizersDomain, 1, - string, kCpuExecutionProvider, KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + StringTransformer); } // namespace featurizers } // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/time_series_imputer_transformer.cc b/onnxruntime/featurizers_ops/cpu/time_series_imputer_transformer.cc new file mode 100644 index 0000000000000..ea8f4755eab37 --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/time_series_imputer_transformer.cc @@ -0,0 +1,244 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/op_kernel.h" + +#include +#include + +#include "Featurizers/TimeSeriesImputerFeaturizer.h" +#include "Archive.h" + +namespace ft = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace featurizers { + +namespace timeseries_imputer_details { + +inline std::chrono::system_clock::time_point ToTimePoint(int64_t secs) { + return std::chrono::system_clock::from_time_t(secs); +} + +inline int64_t ToSecs(const std::chrono::system_clock::time_point& tp) { + using namespace std::chrono; + return duration_cast(tp.time_since_epoch()).count(); +} + +template +struct ToString { + std::string operator()(T val) const { + return std::to_string(val); + } +}; + +template <> +struct ToString { + const std::string& operator()(const std::string& val) const { + return val; + } +}; + +template +struct ToStringOptional { + nonstd::optional operator()(T val) const { + nonstd::optional result; + if (std::isnan(val)) { + return result; + } + result = std::to_string(val); + return result; + } +}; + +template <> +struct ToStringOptional { + nonstd::optional operator()(std::string val) const { + return (val.empty()) ? nonstd::optional() : nonstd::optional(std::move(val)); + } +}; + +template +struct FromString; + +template <> +struct FromString { + const std::string& operator()(const std::string& val) const { + return val; + } +}; + +template <> +struct FromString { + float operator()(const std::string& val) const { + char* str_end = nullptr; + const char* str = val.c_str(); + float result = std::strtof(str, &str_end); + if (str == str_end) { + ORT_THROW("Resulting key string is not convertible to float: ", val); + } + return result; + } +}; + +template <> +struct FromString { + double operator()(const std::string& val) const { + const char* str = val.c_str(); + char* str_end = nullptr; + double result = std::strtod(str, &str_end); + if (str == str_end) { + ORT_THROW("Resulting key string is not convertible to double: ", val); + } + return result; + } +}; +template +struct FromStringOptional { + T operator()(const nonstd::optional& val) const { + if (val.has_value()) { + return FromString()(*val); + } + return std::numeric_limits::quiet_NaN(); + } +}; + +template <> +struct FromStringOptional { + std::string operator()(const nonstd::optional& val) const { + if (val.has_value()) { + return *val; + } + return std::string(); + } +}; +} // namespace timeseries_imputer_details + +template +struct TimeSeriesImputerTransformerImpl { + void operator()(OpKernelContext* ctx, int64_t rows) { + const auto& state = *ctx->Input(0); + const uint8_t* const state_data = state.template Data(); + + const auto& times = *ctx->Input(1); + const auto& keys = *ctx->Input(2); + const auto& data = *ctx->Input(3); + + const int64_t keys_per_row = keys.Shape()[1]; + const int64_t columns = data.Shape()[1]; + + using namespace timeseries_imputer_details; + + using OutputType = std::tuple, std::vector>>; + std::vector output_rows; + std::function callback_fn; + callback_fn = [&output_rows](OutputType value) -> void { + output_rows.emplace_back(std::move(value)); + }; + + Microsoft::Featurizer::Archive archive(state_data, state.Shape().Size()); + ft::Components::TimeSeriesImputerEstimator::Transformer transformer(archive); + + const int64_t* times_data = times.template Data(); + const T* const keys_data = keys.template Data(); + const T* const data_data = data.template Data(); + + // for each row get timestamp, get all keys, get all data and feed it + for (int64_t row = 0; row < rows; ++row) { + const T* const key_row_data = keys_data + (row * keys_per_row); + const T* const keys_row_end = key_row_data + keys_per_row; + std::vector str_keys; + std::transform(key_row_data, keys_row_end, std::back_inserter(str_keys), + ToString()); + + std::vector> str_data; + const T* const data_row = data_data + (row * columns); + const T* const data_row_end = data_row + columns; + std::transform(data_row, data_row_end, std::back_inserter(str_data), + ToStringOptional()); + + auto tuple_row = std::make_tuple(ToTimePoint(*times_data), std::move(str_keys), std::move(str_data)); + + transformer.execute(tuple_row, callback_fn); + ++times_data; + } + + transformer.flush(callback_fn); + + // Compute output shapes now + // Number of outputs is the number of rows, + int64_t output_rows_num = static_cast(output_rows.size()); + TensorShape rows_shape({output_rows_num}); + TensorShape keys_shape({output_rows_num, keys_per_row}); + TensorShape data_shape({output_rows_num, columns}); + + auto* added_output = ctx->Output(0, rows_shape)->template MutableData(); + auto* time_output = ctx->Output(1, rows_shape)->template MutableData(); + auto* keys_output = ctx->Output(2, keys_shape)->template MutableData(); + auto* data_output = ctx->Output(3, data_shape)->template MutableData(); + + for (const auto& out : output_rows) { + *added_output++ = std::get<0>(out); + *time_output++ = ToSecs(std::get<1>(out)); + const auto& imputed_keys = std::get<2>(out); + ORT_ENFORCE(static_cast(imputed_keys.size()) == keys_per_row, + "resulting number of keys: ", imputed_keys.size(), " expected: ", keys_per_row); + const auto& imputed_data = std::get<3>(out); + ORT_ENFORCE(static_cast(imputed_data.size()) == columns, + "resulting number of columns: ", imputed_data.size(), " expected: ", columns); + keys_output = std::transform(imputed_keys.cbegin(), imputed_keys.cend(), keys_output, + FromString()); + data_output = std::transform(imputed_data.cbegin(), imputed_data.cend(), data_output, + FromStringOptional()); + } + } +}; + +class TimeSeriesImputerTransformer final : public OpKernel { + public: + explicit TimeSeriesImputerTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + static Status CheckBatches(int64_t rows, const TensorShape& shape) { + if (shape.NumDimensions() == 2) { + ORT_RETURN_IF_NOT(rows == shape[0], "Number of rows does not match"); + } else { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Expect shape of [R][C]"); + } + return Status::OK(); + } + + Status Compute(OpKernelContext* ctx) const override { + const auto& times = *ctx->Input(1); + const auto& times_shape = times.Shape(); + ORT_RETURN_IF_NOT(times_shape.NumDimensions() == 1, "Times must have shape [B][R] or [R]"); + int64_t rows = times_shape[0]; + + const auto& keys = *ctx->Input(2); + ORT_RETURN_IF_ERROR(CheckBatches(rows, keys.Shape())); + const auto& data = *ctx->Input(3); + ORT_RETURN_IF_ERROR(CheckBatches(rows, data.Shape())); + + auto data_type = data.GetElementType(); + ORT_RETURN_IF_NOT(keys.GetElementType() == data_type, "Keys and data must have the same datatype"); + + TimeSeriesImputerTransformerImpl()(ctx, rows); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + TimeSeriesImputerTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("T1", DataTypeImpl::GetTensorType()) + .TypeConstraint("T2", DataTypeImpl::GetTensorType()), + TimeSeriesImputerTransformer); +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc index 4acd5432ec22f..96326b10de936 100644 --- a/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc +++ b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc @@ -10,61 +10,34 @@ namespace onnxruntime { namespace featurizers { // Forward declarations -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float, CatImputerTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double, CatImputerTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, CatImputerTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, CatImputerTransformer); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, DateTimeTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, bool, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, StringTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, HashOneHotVectorizerTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, ImputationMarkerTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, LabelEncoderTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MaxAbsScalarTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MinMaxScalarTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MissingDummiesTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, OneHotEncoderTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, RobustScalarTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, StringTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TimeSeriesImputerTransformer); Status RegisterCpuMSFeaturizersKernels(KernelRegistry& kernel_registry) { static const BuildKernelCreateInfoFn function_table[] = { - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo}; + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + }; for (auto& function_table_entry : function_table) { ORT_RETURN_IF_ERROR(kernel_registry.Register(function_table_entry())); diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index e48c03323cc55..ba6035ba2da70 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -587,7 +587,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") .def_property( "graph_optimization_level", [](const SessionOptions* options) -> GraphOptimizationLevel { - GraphOptimizationLevel retval = ORT_ENABLE_BASIC; + GraphOptimizationLevel retval = ORT_ENABLE_ALL; switch (options->graph_optimization_level) { case onnxruntime::TransformerLevel::Default: retval = ORT_DISABLE_ALL; @@ -602,8 +602,8 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") retval = ORT_ENABLE_ALL; break; default: - retval = ORT_ENABLE_BASIC; - LOGS_DEFAULT(WARNING) << "Got invalid graph optimization level; defaulting to ORT_ENABLE_BASIC"; + retval = ORT_ENABLE_ALL; + LOGS_DEFAULT(WARNING) << "Got invalid graph optimization level; defaulting to ORT_ENABLE_ALL"; break; } return retval; diff --git a/onnxruntime/python/onnxruntime_validation.py b/onnxruntime/python/onnxruntime_validation.py index 0cb88d3505014..73bd352c32556 100644 --- a/onnxruntime/python/onnxruntime_validation.py +++ b/onnxruntime/python/onnxruntime_validation.py @@ -13,7 +13,6 @@ def check_distro_info(): __my_distro__ = '' __my_distro_ver__ = '' __my_system__ = platform.system().lower() - __my_arch__ = platform.architecture()[0].lower() __OS_RELEASE_FILE__ = '/etc/os-release' __LSB_RELEASE_FILE__ = '/etc/lsb-release' @@ -25,8 +24,6 @@ def check_distro_info(): if __my_distro_ver__ != '10': warnings.warn('Unsupported Windows version (%s). ONNX Runtime supports Windows 10 and above, only.' % __my_distro_ver__) elif __my_system__ == 'linux': - if __my_arch__ != '64bit': - warnings.warn('Unsupported architecture (%s). ONNX Runtime supports 64bit architecture, only.' % __my_arch__) ''' Although the 'platform' python module for getting Distro information works well on standard OS images running on real hardware, it is not acurate when running on Azure VMs, Git Bash, Cygwin, etc. The returned values for release and version are unpredictable for virtualized or emulated environments. diff --git a/onnxruntime/python/tools/bert/bert_model_optimization.py b/onnxruntime/python/tools/bert/bert_model_optimization.py index 4f074a8433a7c..2508f7f64b8bc 100644 --- a/onnxruntime/python/tools/bert/bert_model_optimization.py +++ b/onnxruntime/python/tools/bert/bert_model_optimization.py @@ -6,6 +6,18 @@ # Convert Bert ONNX model exported from PyTorch to use Attention, Gelu, # SkipLayerNormalization and EmbedLayerNormalization ops to optimize # performance on NVidia GPU. + +# Note: This script is not required for Bert model optimization. +# OnnxRuntime has bert model optimization support internally. The recommended way is +# to set optimization level to ORT_ENABLE_EXTENDED during Bert model inference. +# See the following document for more information: +# https://github.com/microsoft/onnxruntime/blob/master/docs/ONNX_Runtime_Graph_Optimizations.md + +# This script is retained for experiment purpose. Useful senarios like the following: +# (1) Change model from fp32 to fp16. +# (2) Change input data type from int64 to int32. +# (3) Model cannot be handled to OnnxRuntime graph optimization, and you can modify this script to get optimized model. + import onnx import sys import argparse @@ -193,6 +205,13 @@ def get_constant_value(self, output_name): for att in node.attribute: if att.name == 'value': return numpy_helper.to_array(att.t) + + # Fall back to intializer since constant folding might have been + # applied. + initializer = self.get_initializer(output_name) + if initializer is not None: + return numpy_helper.to_array(initializer) + return None def get_constant_input(self, node): @@ -200,13 +219,14 @@ def get_constant_input(self, node): value = self.get_constant_value(input) if value is not None: return i, value + return None, None def find_constant_input(self, node, expected_value, delta=0.000001): - for i, input in enumerate(node.input): - value = self.get_constant_value(input) - if value is not None and value.size == 1 and abs(value - expected_value) < delta: - return i + i, value = self.get_constant_input(node) + if value is not None and value.size == 1 and abs(value - expected_value) < delta: + return i + return -1 def has_constant_input(self, node, expected_value, delta=0.000001): @@ -402,6 +422,9 @@ def __init__(self, model, num_heads, hidden_size, sequence_length, verbose): # A lookup table with mask input as key, and mask index output as value self.mask_indice = {} + # A lookup table with mask input as key, and cast (to int32) output as value + self.mask_casted = {} + self.bert_inputs = [] # constant node names @@ -414,21 +437,52 @@ def get_normalize_nodes(self): def normalize_children_types(self): return ['MatMul', 'MatMul', 'MatMul', 'SkipLayerNormalization'] + def cast_graph_input_to_int32(self, input_name): + graph_input = self.find_graph_input(input_name) + if graph_input is not None and graph_input.type.tensor_type.elem_type != TensorProto.INT32: + cast_output = input_name + '_int32' + cast_node = onnx.helper.make_node('Cast', inputs=[input_name], outputs=[cast_output]) + cast_node.attribute.extend([onnx.helper.make_attribute("to", int(TensorProto.INT32))]) + self.add_node(cast_node) + return True, cast_output + + return False, input_name + + def undo_cast_input_to_int32(self, input_name): + input_name_to_nodes = self.input_name_to_nodes() + nodes = input_name_to_nodes[input_name] + for node in nodes: + if node.op_type == "Cast": + is_int32 = False + for att in node.attribute: + if att.name == 'to' and att.i == int(TensorProto.INT32): + is_int32 = True + break + if is_int32: + output_name = node.output[0] + self.remove_node(node) + self.replace_input_of_all_nodes(output_name, input_name) + def process_mask(self, input): if input in self.mask_indice: return self.mask_indice[input] + # Add cast to convert int64 to int32 + casted, input_name = self.cast_graph_input_to_int32(input) + if casted: + self.mask_casted[input] = input_name + # Add a mask processing node output_name = self.create_node_name('mask_index') mask_index_node = onnx.helper.make_node('ReduceSum', - inputs=[input], + inputs=[input_name], outputs=[output_name], name=self.create_node_name('ReduceSum', 'MaskReduceSum')) mask_index_node.attribute.extend([onnx.helper.make_attribute("axes", [1]), onnx.helper.make_attribute("keepdims", 0)]) self.add_node(mask_index_node) + self.mask_indice[input] = output_name - - return self.mask_indice[input] + return output_name def create_attention_node(self, mask_index, q_matmul, k_matmul, v_matmul, q_add, k_add, v_add, input, output): q_weight = self.get_initializer(q_matmul.input[1]) @@ -437,7 +491,7 @@ def create_attention_node(self, mask_index, q_matmul, k_matmul, v_matmul, q_add, q_bias = self.get_initializer(q_add.input[1]) k_bias = self.get_initializer(k_add.input[1]) v_bias = self.get_initializer(v_add.input[1]) - + qw = numpy_helper.to_array(q_weight) assert qw.shape == (self.hidden_size, self.hidden_size) @@ -932,9 +986,9 @@ def fuse_reshape(self): | | v v +---(optional graph) SkipLayerNormalization - Optional graph is used to generate position list (0, 1, ...). It can be a constant in some model. + Optional graph is used to generate position list (0, 1, ...) per batch. It can be a constant in some model. """ - def fuse_embed_layer(self): + def fuse_embed_layer(self, input_int32): nodes = self.nodes() input_name_to_nodes = self.input_name_to_nodes() output_name_to_node = self.output_name_to_node() @@ -972,9 +1026,13 @@ def fuse_embed_layer(self): position_embedding_path = self.match_parent_path(add_node, ['Gather', 'Expand', 'Shape'], [1, 1, 1]) if position_embedding_path is None: - print("Failed to find position embedding") - return - position_embedding_gather, position_embedding_expand, position_embedding_shape = position_embedding_path + position_embedding_path2 = self.match_parent_path(add_node, ['Gather', 'Expand', 'Concat', 'Unsqueeze', 'Gather', 'Shape'], [1, 1, 1, 1, 0, 0]) + if position_embedding_path2 is None: + print("Failed to find position embedding") + return + position_embedding_gather, position_embedding_expand, _, _, _, position_embedding_shape = position_embedding_path2 + else: + position_embedding_gather, position_embedding_expand, position_embedding_shape = position_embedding_path segment_embedding_path = self.match_parent_path(normalize_node, ['Gather'], [1]) if segment_embedding_path is None: @@ -995,19 +1053,32 @@ def fuse_embed_layer(self): nodes_to_remove.extend([normalize_node, add_node, segment_embedding_gather, word_embedding_gather, position_embedding_gather, position_embedding_expand]) nodes_to_remove.extend([mask_node]) + # store inputs for further processing + self.bert_inputs = [input_ids, segment_ids, mask_input_name] + + if not input_int32: + # When mask has been casted to int32, use that casted one as input of embed layer norm. + if mask_input_name in self.mask_casted: + mask_input_name = self.mask_casted[mask_input_name] + + # Cast input_ids and segment_ids to int32. + casted, input_ids = self.cast_graph_input_to_int32(input_ids) + + casted, segment_ids = self.cast_graph_input_to_int32(segment_ids) + else: + self.undo_cast_input_to_int32(mask_input_name) + embed_node = onnx.helper.make_node('EmbedLayerNormalization', - inputs=[input_ids, segment_ids, + inputs=[input_ids, + segment_ids, word_embedding_gather.input[0], position_embedding_gather.input[0], segment_embedding_gather.input[0], normalize_node.input[2], normalize_node.input[3], # gamma and beta mask_input_name], - outputs=["embed_output", self.mask_indice[mask_input_name]], + outputs=["embed_output", mask_output_name], name="EmbedLayer") embed_node.domain = "com.microsoft" - # store inputs for further processing - self.bert_inputs = [input_ids, segment_ids, mask_input_name] - self.replace_input_of_all_nodes(normalize_node.output[0], 'embed_output') self.remove_nodes(nodes_to_remove) @@ -1015,8 +1086,12 @@ def fuse_embed_layer(self): self.update_graph() print("Fused EmbedLayerNormalization count: 1") - def get_bert_inputs(self): - return self.bert_inputs + # Change graph input data type int32 if needed. + if input_int32: + self.change_input_to_int32() + + def get_bert_inputs(self, include_mask=True): + return self.bert_inputs if include_mask else self.bert_inputs[:2] def get_batch_size_from_graph_input(self): graph = self.graph() @@ -1040,6 +1115,7 @@ def change_input_to_int32(self): batch_size = self.get_batch_size_from_graph_input() input_batch_size = batch_size if isinstance(batch_size, int) else 1 new_graph_inputs = [] + bert_inputs = self.get_bert_inputs() for input in graph.input: if input.name in bert_inputs: @@ -1063,16 +1139,6 @@ def change_input_to_int32(self): # restore opset version self.model.opset_import[0].version = original_opset_version - def cast_input_to_int32(self): - bert_inputs = self.get_bert_inputs() - for input in bert_inputs: - graph_input = self.find_graph_input(input) - if graph_input is not None and graph_input.type.tensor_type.elem_type != TensorProto.INT32: - cast_output = input + '_int32' - cast_node = onnx.helper.make_node('Cast', inputs=[input], outputs=[cast_output]) - cast_node.attribute.extend([onnx.helper.make_attribute("to", int(TensorProto.INT32))]) - self.replace_input_of_all_nodes(input, cast_output) - self.add_node(cast_node) # Update input and output using dynamic batch def update_dynamic_batch_io(self, dynamic_batch_dim='batch'): @@ -1095,7 +1161,7 @@ def update_dynamic_batch_io(self, dynamic_batch_dim='batch'): | | | v Add --> ReduceMean --> Sub --> Pow --> ReduceMean --> Add --> Sqrt --> Div --> Mul --> Add - (axis=2 or -1) | (Y=2) (axis=2 or -1) (E-6 or E-12) ^ + (axis=2 or -1) | (Y=2) (axis=2 or -1) (E-6 or E-12 or 0) ^ | | +-----------------------------------------------+ @@ -1200,6 +1266,7 @@ def fuse_layer_norm(self): normalize_node = onnx.helper.make_node('LayerNormalization', inputs=[node.input[0], weight_input, bias_input], outputs=[last_add_node.output[0]]) + normalize_node.attribute.extend([onnx.helper.make_attribute("epsilon", add_weight)]) layernorm_nodes.extend([normalize_node]) self.remove_nodes(nodes_to_remove) @@ -1250,7 +1317,7 @@ def main(): bert_model.fuse_attention() - bert_model.fuse_embed_layer() + bert_model.fuse_embed_layer(args.input_int32) # Fuse Gelu and Add Bias before it. bert_model.fuse_add_bias_gelu() @@ -1258,12 +1325,6 @@ def main(): # Fuse SkipLayerNormalization and Add Bias before it. bert_model.fuse_add_bias_skip_layer_norm() - if bert_model.get_bert_inputs(): - if args.input_int32: - bert_model.change_input_to_int32() - else: - bert_model.cast_input_to_int32() - if args.float16: bert_model.convert_model_float32_to_float16() @@ -1277,4 +1338,5 @@ def main(): with open(args.output, "wb") as out: out.write(bert_model.model.SerializeToString()) -main() +if __name__ == "__main__": + main() diff --git a/onnxruntime/python/tools/quantization/README.md b/onnxruntime/python/tools/quantization/README.md index abbd30fdd8605..c2da60a9aba8b 100644 --- a/onnxruntime/python/tools/quantization/README.md +++ b/onnxruntime/python/tools/quantization/README.md @@ -1,5 +1,45 @@ # Quantization tool Overview -This tool supports quantization of an onnx model. quantize() takes a model in ModelProto format and returns the quantized model in ModelProto format. +This tool supports 8 bit linear quantization of an onnx model. quantize() takes a model in ModelProto format and returns the quantized model in ModelProto format. +Today ORT does not guarantee support for E2E model quantization, meaning since not all ONNX ops have support for 8 bit data types therefore only the supported ops in the model are quantized. For rest of the ops inputs are reconverted to FP32. + +List of Supported Quantized Ops: +The following ops were chosen as phase 1 ops because in most of the CNN models these ops consume most amount of compute and power and therefore there is benefit in quantizing these ops to get perf benefits. + * Convolution + * Matmul + * Data type agnostic ops like transpose, identity etc ( Note: special quantization is not done for these ops. ) + + ## Quantization specifics + ONNX implements 8 bit linear quantization. During quantization the floating point real values are mapped to a 8 bit quantization space and it is of the form : + VAL_fp32 = Scale * (VAL_quantized - Zero_point) + + Scale is a positive real number used to map the floating point numbers to a quantization space. It is calculated as follows : + For unsigned 8 bit + ``` + scale = (data_rage_max - data_range_min) / (quantization_range_max - quantization_range_min) + ``` + + For signed 8 bit + ``` + scale = Abs(data_rage_max, data_range_min) * 2 / (quantization_range_max - quantization_range_min) + ``` + + Zero point represents zero in quantization space. It is important that floating point zero value be exactly representable in quantization space. This is because in lot of CNNs, zero padding is used and if after quantization it is not possible to represent 0 uniquely then it will lead to accuracy errors. + +## Quantization and model opset versions +Quantization is fairly new in ONNX and ONNXRuntime. Quantization ops were introduced in ONNX opset version 10. Therefore it is important that the model which is being quantized be opset 10 or higher. In case the model opset version is < 10 then it is recommended that the model should be reconverted to ONNX from its original framework using the latest opset. + +Quantization tool displays a warning when the model opset version is < 10 and still goes ahead and quantizes the model and at the end changes the opset version to 10. It is the responsibility of the model owner to run model checker and make sure the model is valid. If the model is not valid then use the above recommended way i.e. reconvert the model from original framework. + +## Quantization and Graph Optimization +Please note quantization and graph optimizations may not always work together. + +### Quantizing an optimized model +If a model is optimized using level 99 (i.e. all possible optimizations are run on that model) then it is possible that after these optimizations are applied the model is converted in a way that quantization cannot be applied on this model anymore and therefore after running quantization script there will be no change in the model. + +### Optimizing a quantized model +Same goes other way round. After quantizing a model some graph optimizations which otherwise might have been applicable on this model may not be applicable anymore. + +It is advised that the model owner be aware of this and run perf evaluations to understand which technique gives the best performance for their model. ## Quantize an ONNX model ```python diff --git a/onnxruntime/python/tools/quantization/quantize.py b/onnxruntime/python/tools/quantization/quantize.py index e886fd0df9923..1d5b393c10205 100644 --- a/onnxruntime/python/tools/quantization/quantize.py +++ b/onnxruntime/python/tools/quantization/quantize.py @@ -262,8 +262,12 @@ def quantize_model(self): # Create a new topologically sorted list for quantizing a model new_list = [] for node in self.model.graph.node: + # if a list of ops to be quantized is provided then only quantize those ops if self.nodes_to_quantize is not None and node.name not in self.nodes_to_quantize: new_list +=self._handle_other_ops(node, new_list) + # only onnx domain ops can be quantized today + elif node.domain != "ai.onnx" or node.domain != "": + new_list +=self._handle_other_ops(node, new_list) else: if node.op_type == 'Conv': new_list += self._quantize_convolution(node, new_list) @@ -274,7 +278,7 @@ def quantize_model(self): elif node.op_type == 'Relu' or node.op_type == 'Clip': new_list +=self._handle_activation_ops(node, new_list) else: - new_list +=self._handle_other_ops(node, new_list) + new_list +=self._handle_other_ops(node, new_list) # extend is used to append to the list for a protobuf fields # https://developers.google.com/protocol-buffers/docs/reference/python-generated?csw=1#fields @@ -284,7 +288,7 @@ def quantize_model(self): # Remove weights which are already quantized from graph. self._remove_quantized_weights() - # update opset. + # update opset. opset_info = next((opset for opset in self.model.opset_import if opset.domain == '' or opset.domain == onnx_domain), None) if opset_info is not None: self.model.opset_import.remove(opset_info) diff --git a/onnxruntime/test/featurizers_ops/categoryimputer_test.cc b/onnxruntime/test/featurizers_ops/categoryimputer_test.cc index d803b06879cce..5dc50cc245de2 100644 --- a/onnxruntime/test/featurizers_ops/categoryimputer_test.cc +++ b/onnxruntime/test/featurizers_ops/categoryimputer_test.cc @@ -11,7 +11,7 @@ namespace dft = Microsoft::Featurizer::Featurizers; namespace onnxruntime { namespace test { -TEST(CategoryImputer, Float_values) { +TEST(FeaturizersTests, CategoryImputer_float_values) { OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); @@ -20,15 +20,15 @@ TEST(CategoryImputer, Float_values) { test.AddInput("State", {8}, {1, 0, 0, 0, 0, 0, 192, 63}); // We are adding a scalar Tensor in this instance - test.AddInput("Input", {5}, {1, std::nanf("1"), std::nanf("1"), 2, std::nanf("1")}); + test.AddInput("Input", {5}, {1.f, std::nanf("1"), std::nanf("1"), 2.f, std::nanf("1")}); // Expected output. - test.AddOutput("Output", {5}, {1, 1.5, 1.5, 2, 1.5}); + test.AddOutput("Output", {5}, {1.f, 1.5f, 1.5f, 2.f, 1.5f}); test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(CategoryImputer, Double_values) { +TEST(FeaturizersTests, CategoryImputer_double_values) { OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State from when the transformer was trained. Corresponds to a @@ -36,15 +36,15 @@ TEST(CategoryImputer, Double_values) { test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 63}); // We are adding a scalar Tensor in this instance - test.AddInput("Input", {5}, {1, std::nan("1"), std::nan("1"), 2, std::nan("1")}); + test.AddInput("Input", {5}, {1., std::nan("1"), std::nan("1"), 2., std::nan("1")}); // Expected output. - test.AddOutput("Output", {5}, {1, 1.5, 1.5, 2, 1.5}); + test.AddOutput("Output", {5}, {1., 1.5, 1.5, 2., 1.5}); test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(CategoryImputer, String_values) { +TEST(FeaturizersTests, CategoryImputer_string_values) { OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State from when the transformer was trained. Corresponds to a diff --git a/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc b/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc index 863fa7f1a2872..354f201661dee 100644 --- a/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc +++ b/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc @@ -4,6 +4,7 @@ #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" +#include "Archive.h" #include "Featurizers/DateTimeFeaturizer.h" namespace dft = Microsoft::Featurizer::Featurizers; @@ -13,12 +14,21 @@ using SysClock = std::chrono::system_clock; namespace onnxruntime { namespace test { -TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_04) { +std::vector GetStream () { + dft::DateTimeTransformer dt("", ""); + Microsoft::Featurizer::Archive ar; + dt.save(ar); + return ar.commit(); +} + +TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17_12_27_04) { const time_t date = 217081624; OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input - test.AddInput("State", {8}, {1, 0, 0, 0, 0, 0, 0, 0}); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + test.AddInput("State", {dim}, stream); // We are adding a scalar Tensor in this instance test.AddInput("Date", {1}, {date}); @@ -74,19 +84,22 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_04) { test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05) { +TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17_12_27_05) { const time_t date = 217081625; + const auto date_tp = SysClock::from_time_t(date); OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input - test.AddInput("State", {8}, {1, 0, 0, 0, 0, 0, 0, 0}); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + test.AddInput("State", {dim}, stream); // We are adding a scalar Tensor in this instance test.AddInput("Date", {1}, {date}); dft::DateTimeTransformer dt("", ""); - dft::TimePoint tp(dt.execute(date)); + dft::TimePoint tp(dt.execute(date_tp)); ASSERT_EQ(tp.year, 1976); ASSERT_EQ(tp.month, dft::TimePoint::NOVEMBER); ASSERT_EQ(tp.day, 17); @@ -135,21 +148,26 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05) { test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05_and_Past_1976_Nov_17__12_27_04) { +TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17__12_27_05_and_past_1976_nov_17_12_27_04) { const time_t date1 = 217081625; + const auto date1_tp = SysClock::from_time_t(date1); const time_t date2 = 217081624; + const auto date2_tp = SysClock::from_time_t(date2); OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input - test.AddInput("State", {8}, {1, 0, 0, 0, 0, 0, 0, 0}); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + test.AddInput("State", {dim}, stream); + // We are adding a scalar Tensor in this instance test.AddInput("Date", {2}, {date1, date2}); dft::DateTimeTransformer dt("", ""); - dft::TimePoint tp1(dt.execute(date1)); - dft::TimePoint tp2(dt.execute(date2)); + dft::TimePoint tp1(dt.execute(date1_tp)); + dft::TimePoint tp2(dt.execute(date2_tp)); // Date1 ASSERT_EQ(tp1.year, 1976); @@ -223,19 +241,22 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05_and_Past_1976_Nov_17__12_27 test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(DateTimeTransformer, Future_2025_June_30) { +TEST(FeaturizersTests, DateTimeTransformer_future_2025_june_30) { const time_t date = 1751241600; + const auto date_tp = std::chrono::system_clock::from_time_t(date); OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input - test.AddInput("State", {8}, {1, 0, 0, 0, 0, 0, 0, 0}); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + test.AddInput("State", {dim}, stream); // We are adding a scalar Tensor in this instance test.AddInput("Date", {1}, {date}); dft::DateTimeTransformer dt("", ""); - dft::TimePoint tp = dt.execute(date); + dft::TimePoint tp = dt.execute(date_tp); ASSERT_EQ(tp.year, 2025); ASSERT_EQ(tp.month, dft::TimePoint::JUNE); ASSERT_EQ(tp.day, 30); diff --git a/onnxruntime/test/featurizers_ops/hash_one_hot_encoder_transformer_test.cc b/onnxruntime/test/featurizers_ops/hash_one_hot_encoder_transformer_test.cc new file mode 100644 index 0000000000000..915173110b69c --- /dev/null +++ b/onnxruntime/test/featurizers_ops/hash_one_hot_encoder_transformer_test.cc @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/HashOneHotVectorizerFeaturizer.h" +#include "Featurizers/TestHelpers.h" +#include "Archive.h" + +namespace NS = Microsoft::Featurizer; + +namespace onnxruntime { +namespace test { + +template +std::vector GetStream() { + NS::Featurizers::HashOneHotVectorizerTransformer hvtransformer(2, 100); + NS::Archive ar; + hvtransformer.save(ar); + return ar.commit(); +} + +TEST(FeaturizersTests, HashOneHotVectorizerTransformer_int8) { + using Type = int8_t; + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + OpTester test("HashOneHotVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {1}, {15}); + + test.AddOutput("NumElements", {1}, {100u}); + test.AddOutput("Value", {1}, {1u}); + test.AddOutput("Index", {1}, {29u}); + + test.Run(); +} + +TEST(FeaturizersTests, HashOneHotVectorizerTransformer_int32) { + using Type = int32_t; + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + OpTester test("HashOneHotVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {1}, {15}); + + test.AddOutput("NumElements", {1}, {100u}); + test.AddOutput("Value", {1}, {1u}); + test.AddOutput("Index", {1}, {22u}); + + test.Run(); +} + +TEST(FeaturizersTests, HashOneHotVectorizerTransformer_double) { + using Type = double; + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + OpTester test("HashOneHotVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {1}, {15.0}); + + test.AddOutput("NumElements", {1}, {100u}); + test.AddOutput("Value", {1}, {1u}); + test.AddOutput("Index", {1}, {99u}); + + test.Run(); +} + +TEST(FeaturizersTests, HashOneHotVectorizerTransformer_string) { + using Type = std::string; + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + OpTester test("HashOneHotVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {1}, {"hello"}); + + test.AddOutput("NumElements", {1}, {100u}); + test.AddOutput("Value", {1}, {1u}); + test.AddOutput("Index", {1}, {25u}); + + test.Run(); +} + +} // namespace test +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/featurizers_ops/imputationmarkertransformer_test.cc b/onnxruntime/test/featurizers_ops/imputationmarkertransformer_test.cc new file mode 100644 index 0000000000000..df83269ead6d3 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/imputationmarkertransformer_test.cc @@ -0,0 +1,146 @@ +// ---------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License +// ---------------------------------------------------------------------- + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/ImputationMarkerFeaturizer.h" + +namespace ft = Microsoft::Featurizer; + +namespace onnxruntime { +namespace test { + +template +std::vector GetStream () { + ft::Archive ar; + ft::Featurizers::ImputationMarkerTransformer inst; + inst.save(ar); + return ar.commit(); +} + +//TEST (FeaturizersTests, ImputationMarker_int8) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_uint8) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_int16) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_uint16) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_int32) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_uint32) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_int64) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_uint64) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} + +TEST(FeaturizersTests, ImputationMarker_float) { + OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {2.5f, std::numeric_limits::quiet_NaN()}); + test.AddOutput("Output", {2}, {false, true}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, ImputationMarker_double) { + OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {2.5, std::numeric_limits::quiet_NaN()}); + test.AddOutput("Output", {2}, {false, true}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, ImputationMarker_string) { + OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {"hello", ""}); + test.AddOutput("Output", {2}, {false, true}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +} +} diff --git a/onnxruntime/test/featurizers_ops/labelencodertransfomer_test.cc b/onnxruntime/test/featurizers_ops/labelencodertransfomer_test.cc new file mode 100644 index 0000000000000..32f1c380a6744 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/labelencodertransfomer_test.cc @@ -0,0 +1,110 @@ +// ---------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License +// ---------------------------------------------------------------------- + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Archive.h" +#include "Featurizers/LabelEncoderFeaturizer.h" +#include "Featurizers/TestHelpers.h" + +namespace ft = Microsoft::Featurizer; + +namespace onnxruntime { +namespace test { + +template +using IndexMap = std::unordered_map; + +template +std::vector GetStream(const IndexMap& map, bool allowMissingValues) { + ft::Archive ar; + using TransType = ft::Featurizers::LabelEncoderTransformer; + TransType inst(map, allowMissingValues); + inst.save(ar); + return ar.commit(); +} + +TEST(FeaturizersTests, LabelEncodeTransformer_uint32) { + OpTester test("LabelEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + using InputType = uint32_t; + + IndexMap index_map = { + {11, 2}, {8, 0}, {10, 1}, {15, 3}, {20, 5}}; + + auto stream = GetStream(index_map, false); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {5}, {11, 8, 10, 15, 20}); + test.AddOutput("Output", {5}, {2, 0, 1, 3, 5}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, LabelEncodeTransformer_string) { + OpTester test("LabelEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + using InputType = std::string; + + IndexMap index_map = { + {"orange", 5}, {"apple", 0}, {"grape", 3}, {"carrot", 5}, {"peach", 5}, {"banana", 1}}; + + auto stream = GetStream(index_map, false); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {3}, {"banana", "grape", "apple"}); + test.AddOutput("Output", {3}, {1, 3, 0}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, LabelEncodeTransformer_string_nothrow) { + OpTester test("LabelEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + using InputType = std::string; + + // when an inference data is not seen before, in the non-throw mode, the featurizer should generate 0 + // hello is not seen before among fruits + IndexMap index_map = { + {"banana", 1}, + {"apple", 2}, + {"grape", 3}, + {"carrot", 4}, + {"peach", 5}, + {"orange", 6}}; + + auto stream = GetStream(index_map, true); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {3}, {"banana", "grape", "hello"}); + // The transformer will add 1 to each of the output for the missing input + test.AddOutput("Output", {3}, {2, 4, 0}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, LabelEncodeTransformer_string_throw) { + OpTester test("LabelEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + using InputType = std::string; + + // when an inference data is not seen before, in the non-throw mode, the featurizer should generate 0 + // hello is not seen before among fruits + IndexMap index_map = { + {"banana", 1}, + {"apple", 2}, + {"grape", 3}, + {"carrot", 4}, + {"peach", 5}, + {"orange", 6}}; + + auto stream = GetStream(index_map, false); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {4}, {"banana", "grape", "apple", "hello"}); + test.AddOutput("Output", {4}, {1, 3, 2, 0}); + test.Run(OpTester::ExpectResult::kExpectFailure, "'input' was not found"); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc b/onnxruntime/test/featurizers_ops/maxabsscalertransformer_test.cc similarity index 79% rename from onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc rename to onnxruntime/test/featurizers_ops/maxabsscalertransformer_test.cc index a871eec3efe26..3bc607f5289fa 100644 --- a/onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc +++ b/onnxruntime/test/featurizers_ops/maxabsscalertransformer_test.cc @@ -11,7 +11,7 @@ namespace dft = Microsoft::Featurizer::Featurizers; namespace onnxruntime { namespace test { -TEST(MaxAbsScaler, Int8_values) { +TEST(FeaturizersTests, MaxAbsScaler_int8_values) { OpTester test("MaxAbsScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); @@ -23,12 +23,12 @@ TEST(MaxAbsScaler, Int8_values) { test.AddInput("X", {5}, {-4,3,0,2,-1}); // Expected output. - test.AddOutput("ScaledValues", {5}, {-1,.75,0,.5,-.25}); + test.AddOutput("ScaledValues", {5}, {-1.f,.75f,0.f,.5f,-.25f}); test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(MaxAbsScaler, Double_values) { +TEST(FeaturizersTests, MaxAbsScaler_double_values) { OpTester test("MaxAbsScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State from when the transformer was trained. Corresponds to Version 1 and a @@ -36,10 +36,10 @@ TEST(MaxAbsScaler, Double_values) { test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 64}); // We are adding a scalar Tensor in this instance - test.AddInput("X", {5}, {-4, 3, 0, 2, -1}); + test.AddInput("X", {5}, {-4, 3, 0, 2, -1}); // Expected output. - test.AddOutput("ScaledValues", {5}, {-1, .75, 0, .5, -.25}); + test.AddOutput("ScaledValues", {5}, {-1, .75, 0, .5, -.25}); test.Run(OpTester::ExpectResult::kExpectSuccess); } diff --git a/onnxruntime/test/featurizers_ops/minmaxscalartransformer_test.cc b/onnxruntime/test/featurizers_ops/minmaxscalartransformer_test.cc new file mode 100644 index 0000000000000..4551616b5d83f --- /dev/null +++ b/onnxruntime/test/featurizers_ops/minmaxscalartransformer_test.cc @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/MinMaxScalarFeaturizer.h" + +namespace onnxruntime { +namespace test { + +TEST(FeaturizersTests, MinMaxScalarTransformer_int8) { + OpTester test("MinMaxScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {6}, {1, 0, 0, 0, 1, 9}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {1}, {15}); + + // Expected output. + test.AddOutput("?2", {1}, {1.75}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, MinMaxScalarTransformer_float_t) { + OpTester test("MinMaxScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 128, 191, 0, 0, 128, 63}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {1}, {2.f}); + + // Expected output. + test.AddOutput("?2", {1}, {1.5}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, MinMaxScalarTransformer_only_one_input) { + OpTester test("MinMaxScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {6}, {1, 0, 0, 0, 255, 255}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {1}, {2}); + + // Expected output. + test.AddOutput("?2", {1}, {0}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +} +} \ No newline at end of file diff --git a/onnxruntime/test/featurizers_ops/missingdummiestransfomer_test.cc b/onnxruntime/test/featurizers_ops/missingdummiestransfomer_test.cc new file mode 100644 index 0000000000000..a457b4a5f4f43 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/missingdummiestransfomer_test.cc @@ -0,0 +1,58 @@ +// ---------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License +// ---------------------------------------------------------------------- + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Archive.h" +#include "Featurizers/MissingDummiesFeaturizer.h" + +namespace ft = Microsoft::Featurizer; + +namespace onnxruntime { +namespace test { + +template +std::vector GetStream() { + ft::Archive ar; + ft::Featurizers::MissingDummiesTransformer inst; + inst.save(ar); + return ar.commit(); +} + +TEST(FeaturizersTests, MissingDummiesTransformer_float) { + OpTester test("MissingDummiesTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {2.5f, std::numeric_limits::quiet_NaN()}); + test.AddOutput("Output", {2}, {0, 1}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, MissingDummiesTransformer_double) { + OpTester test("MissingDummiesTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {2.5, std::numeric_limits::quiet_NaN()}); + test.AddOutput("Output", {2}, {0, 1}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, MissingDummiesTransformer_string) { + OpTester test("MissingDummiesTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {"hello", ""}); + test.AddOutput("Output", {2}, {0, 1}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} +} +} diff --git a/onnxruntime/test/featurizers_ops/one_hot_encoder_test.cc b/onnxruntime/test/featurizers_ops/one_hot_encoder_test.cc new file mode 100644 index 0000000000000..f60d0b2d60ce2 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/one_hot_encoder_test.cc @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/OneHotEncoderFeaturizer.h" +#include "Featurizers/TestHelpers.h" +#include "Archive.h" + +namespace NS = Microsoft::Featurizer; + +namespace onnxruntime { +namespace test { + +template +std::vector GetStream(const std::vector>& trainingBatches, bool allowMissingValues) { + using Estimator = NS::Featurizers::OneHotEncoderEstimator; + + Estimator estimator(NS::CreateTestAnnotationMapsPtr(1), 0, allowMissingValues); + NS::TestHelpers::Train(estimator, trainingBatches); + typename Estimator::TransformerUniquePtr pTransformer(estimator.create_transformer()); + + NS::Archive ar; + pTransformer->save(ar); + return ar.commit(); +} + +TEST(FeaturizersTests, OneHotEncoder_uint32_t) { + using InputType = uint32_t; + + auto trainingBatches = NS::TestHelpers::make_vector>( + NS::TestHelpers::make_vector(10, 20, 10), + NS::TestHelpers::make_vector(30), + NS::TestHelpers::make_vector(10, 10, 11, 15), + NS::TestHelpers::make_vector(18, 8)); + + auto stream = GetStream(trainingBatches, false); + auto dim = static_cast(stream.size()); + + OpTester test("OneHotEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {5}, {11u, 8u, 10u, 15u, 20u}); + + test.AddOutput("NumElements", {5}, {7u, 7u, 7u, 7u, 7u}); + test.AddOutput("Value", {5}, {1u, 1u, 1u, 1u, 1u}); + test.AddOutput("Index", {5}, {2u, 0u, 1u, 3u, 5u}); + + test.Run(); +} + +TEST(FeaturizersTests, OneHotEncoder_string) { + using InputType = std::string; + + auto trainingBatches = NS::TestHelpers::make_vector>( + NS::TestHelpers::make_vector("orange", "apple", "orange", + "grape", "carrot", "carrot", + "peach", "banana", "orange")); + + auto stream = GetStream(trainingBatches, false); + auto dim = static_cast(stream.size()); + + OpTester test("OneHotEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {4}, {"banana", "grape", "apple", "orange"}); + + test.AddOutput("NumElements", {4}, {6u, 6u, 6u, 6u}); + test.AddOutput("Value", {4}, {1u, 1u, 1u, 1u}); + test.AddOutput("Index", {4}, {1u, 3u, 0u, 4u}); + + test.Run(); +} + +TEST(FeaturizersTests, OneHotEncoder_unseen_values) { + using InputType = std::string; + + auto trainingBatches = NS::TestHelpers::make_vector>( + NS::TestHelpers::make_vector("orange", "apple", "orange", + "grape", "carrot", "carrot", + "peach", "banana", "orange")); + + auto stream = GetStream(trainingBatches, true); + auto dim = static_cast(stream.size()); + + OpTester test("OneHotEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {5}, {"banana", "grape", "apple", "orange", "hello"}); + + test.AddOutput("NumElements", {5}, {7u, 7u, 7u, 7u, 7u}); + test.AddOutput("Value", {5}, {1u, 1u, 1u, 1u, 1u}); + test.AddOutput("Index", {5}, {2u, 4u, 1u, 5u, 0u}); + + test.Run(); +} + +TEST(FeaturizersTests, OneHotEncoder_unseen_values_throws) { + using InputType = std::string; + + auto trainingBatches = NS::TestHelpers::make_vector>( + NS::TestHelpers::make_vector("orange", "apple", "orange", + "grape", "carrot", "carrot", + "peach", "banana", "orange")); + + auto stream = GetStream(trainingBatches, false); + auto dim = static_cast(stream.size()); + + OpTester test("OneHotEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {5}, {"banana", "grape", "apple", "orange", "hello"}); + + test.AddOutput("NumElements", {5}, {7u, 7u, 7u, 7u, 7u}); + test.AddOutput("Value", {5}, {1u, 1u, 1u, 1u, 1u}); + test.AddOutput("Index", {5}, {2u, 4u, 1u, 5u, 0u}); + + test.Run(OpTester::ExpectResult::kExpectFailure); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/featurizers_ops/robustscalartransformer_test.cc b/onnxruntime/test/featurizers_ops/robustscalartransformer_test.cc new file mode 100644 index 0000000000000..60f809324e95b --- /dev/null +++ b/onnxruntime/test/featurizers_ops/robustscalartransformer_test.cc @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/RobustScalarFeaturizer.h" + +namespace onnxruntime { +namespace test { + +TEST(FeaturizersTests, RobustScalarTransformer_default_with_centering) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 160, 64, 0, 0, 128, 64}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("?2", {5}, {-1.0f,-0.5f, 0.0f, 0.5f, 1.0f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, RobustScalarTransformer_default_no_centering) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 64}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("?2", {5}, {0.25f, 0.75f, 1.25f, 1.75f, 2.25f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, RobustScalarTransformer_default_no_centering_zero_scale) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {3}, {10, 10, 10}); + + // Expected output. + test.AddOutput("?2", {3}, {10.f, 10.f, 10.f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, RobustScalarTransformer_default_with_centering_no_scaling) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 160, 64, 0, 0, 128, 63}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("?2", {5}, {-4.f, -2.f, 0.f, 2.f, 4.f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, RobustScalarTransformer_default_with_centering_custom_scaling) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 160, 64, 0, 0, 0, 65}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("?2", {5}, {-0.5f, -0.25f, 0.f, 0.25f, 0.5f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, RobustScalarTransformer_default_no_centering_custom_scaling) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("?2", {5}, {0.125f, 0.375f, 0.625f, 0.875f, 1.125f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +} +} diff --git a/onnxruntime/test/featurizers_ops/stringtransformer_test.cc b/onnxruntime/test/featurizers_ops/stringtransformer_test.cc index c2564d15c73e4..40af22313b9f7 100644 --- a/onnxruntime/test/featurizers_ops/stringtransformer_test.cc +++ b/onnxruntime/test/featurizers_ops/stringtransformer_test.cc @@ -11,7 +11,7 @@ namespace dft = Microsoft::Featurizer::Featurizers; namespace onnxruntime { namespace test { -TEST(StringTransformer, Integer_values) { +TEST(FeaturizersTests, StringTransformer_integer_values) { OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State represents version 1 @@ -26,7 +26,7 @@ TEST(StringTransformer, Integer_values) { test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(StringTransformer, Double_values) { +TEST(FeaturizersTests, StringTransformer_double_values) { OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State represents version 1 @@ -41,7 +41,7 @@ TEST(StringTransformer, Double_values) { test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(StringTransformer, Bool_values) { +TEST(FeaturizersTests, StringTransformer_bool_values) { OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State represents version 1 @@ -56,7 +56,7 @@ TEST(StringTransformer, Bool_values) { test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(StringTransformer, String_values) { +TEST(FeaturizersTests, StringTransformer_string_values) { OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State represents version 1 diff --git a/onnxruntime/test/featurizers_ops/time_series_imputer_transformer_test.cc b/onnxruntime/test/featurizers_ops/time_series_imputer_transformer_test.cc new file mode 100644 index 0000000000000..d8a351d248088 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/time_series_imputer_transformer_test.cc @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/TimeSeriesImputerFeaturizer.h" +#include "Featurizers/TestHelpers.h" +#include "Archive.h" + +namespace NS = Microsoft::Featurizer; + +namespace onnxruntime { +namespace test { + +inline std::chrono::system_clock::time_point GetTimePoint(std::chrono::system_clock::time_point tp, int unitsToAdd, std::string = "days") { + return tp + std::chrono::minutes(unitsToAdd * (60 * 24)); +} + +inline int64_t GetTimeSecs(std::chrono::system_clock::time_point tp) { + using namespace std::chrono; + return time_point_cast(tp).time_since_epoch().count(); +} + +using InputType = std::tuple< + std::chrono::system_clock::time_point, + std::vector, + std::vector>>; + +using TransformedType = std::vector< + std::tuple< + bool, + std::chrono::system_clock::time_point, + std::vector, + std::vector>>>; + +std::vector GetStream(const std::vector>& trainingBatches, + const std::vector& colsToImputeDataTypes, + bool supressError, NS::Featurizers::Components::TimeSeriesImputeStrategy tsImputeStrategy) { + using TSImputerEstimator = NS::Featurizers::TimeSeriesImputerEstimator; + + NS::AnnotationMapsPtr const pAllColumnAnnotations(NS::CreateTestAnnotationMapsPtr(1)); + TSImputerEstimator estimator(pAllColumnAnnotations, colsToImputeDataTypes, supressError, tsImputeStrategy); + + NS::TestHelpers::Train(estimator, trainingBatches); + TSImputerEstimator::TransformerUniquePtr pTransformer(estimator.create_transformer()); + + NS::Archive ar; + pTransformer->save(ar); + return ar.commit(); +} + +static void AddInputs(OpTester& test, const std::vector>& trainingBatches, + const std::vector& inferenceBatches, const std::vector& colsToImputeDataTypes, + bool supressError, NS::Featurizers::Components::TimeSeriesImputeStrategy tsImputeStrategy) { + auto stream = GetStream( + trainingBatches, + colsToImputeDataTypes, + supressError, + tsImputeStrategy); + + auto dim = static_cast(stream.size()); + test.AddInput("State", {dim}, stream); + + std::vector times; + std::vector keys; + std::vector data; + + using namespace std::chrono; + for (const auto& infb : inferenceBatches) { + times.push_back(time_point_cast(std::get<0>(infb)).time_since_epoch().count()); + keys.insert(keys.end(), std::get<1>(infb).cbegin(), std::get<1>(infb).cend()); + std::transform(std::get<2>(infb).cbegin(), std::get<2>(infb).cend(), std::back_inserter(data), + [](const nonstd::optional& opt) -> std::string { + if (opt.has_value()) return *opt; + return std::string(); + }); + } + + // Should have equal amount of keys per row + ASSERT_TRUE(keys.size() % times.size() == 0); + ASSERT_TRUE(data.size() % times.size() == 0); + test.AddInput("Times", {static_cast(times.size())}, times); + test.AddInput("Keys", {static_cast(times.size()), static_cast(keys.size() / times.size())}, keys); + test.AddInput("Data", {static_cast(times.size()), static_cast(data.size() / times.size())}, data); +} + +void AddOutputs(OpTester& test, const std::initializer_list& added, const std::initializer_list& times, + const std::vector& keys, const std::vector& data) { + ASSERT_TRUE(keys.size() % times.size() == 0); + ASSERT_TRUE(data.size() % times.size() == 0); + + std::vector times_int64; + std::transform(times.begin(), times.end(), std::back_inserter(times_int64), GetTimeSecs); + + test.AddOutput("Added", {static_cast(added.size())}, added); + test.AddOutput("ImputedTimes", {static_cast(times.size())}, times_int64); + test.AddOutput("ImputedKeys", {static_cast(times.size()), static_cast(keys.size() / times.size())}, keys); + test.AddOutput("ImputedData", {static_cast(times.size()), static_cast(data.size() / times.size())}, data); +} + +TEST(FeaturizersTests, RowImputation_1_grain_no_gaps) { + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + auto tp_0 = GetTimePoint(now, 0); + auto tp_1 = GetTimePoint(now, 1); + auto tp_2 = GetTimePoint(now, 2); + auto tuple_1 = std::make_tuple(tp_0, std::vector{"a"}, std::vector>{"14.5", "18"}); + auto tuple_2 = std::make_tuple(tp_1, std::vector{"a"}, std::vector>{nonstd::optional{}, "12"}); + auto tuple_3 = std::make_tuple(tp_2, std::vector{"a"}, std::vector>{"15.0", nonstd::optional{}}); + + std::vector inferenceBatches = {tuple_1, + tuple_2, + tuple_3}; + + OpTester test("TimeSeriesImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + AddInputs(test, {inferenceBatches}, inferenceBatches, + {NS::TypeId::Float64, NS::TypeId::Float64}, false, NS::Featurizers::Components::TimeSeriesImputeStrategy::Forward); + AddOutputs(test, {false, false, false}, {tp_0, tp_1, tp_2}, + {"a", "a", "a"}, {"14.5", "18", "14.5", "12", "15.0", "12"}); + + test.Run(); +} + +TEST(FeaturizersTests, RowImputation_1_grain_2_gaps) { + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + auto tp_0 = GetTimePoint(now, 0); + auto tp_1 = GetTimePoint(now, 1); + auto tp_2 = GetTimePoint(now, 2); + auto tp_3 = GetTimePoint(now, 3); + + auto tuple_0 = std::make_tuple(tp_0, std::vector{"a"}, std::vector>{"14.5", "18"}); + auto tuple_1 = std::make_tuple(tp_1, std::vector{"a"}, std::vector>{nonstd::optional{}, "12"}); + auto tuple_3 = std::make_tuple(tp_3, std::vector{"a"}, std::vector>{nonstd::optional{}, "15.0"}); + + OpTester test("TimeSeriesImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + AddInputs(test, {{tuple_0, tuple_1}}, {tuple_0, tuple_3}, + {NS::TypeId::Float64, NS::TypeId::Float64}, false, NS::Featurizers::Components::TimeSeriesImputeStrategy::Forward); + + AddOutputs(test, {false, true, true, false}, {tp_0, tp_1, tp_2, tp_3}, + {"a", "a", "a", "a"}, {"14.5", "18", "14.5", "18", "14.5", "18", "14.5", "15.0"}); + test.Run(); +} + +TEST(FeaturizersTests, RowImputation_2_grains_no_gaps_input_interleaved) { + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + auto tp_0 = GetTimePoint(now, 0); + auto tp_1 = GetTimePoint(now, 1); + auto tp_5 = GetTimePoint(now, 5); + auto tp_6 = GetTimePoint(now, 6); + + auto tuple_0 = std::make_tuple(tp_0, std::vector{"a"}, std::vector>{"14.5", "18"}); + auto tuple_5 = std::make_tuple(tp_5, std::vector{"b"}, std::vector>{"14.5", "18"}); + auto tuple_5_inf = std::make_tuple(GetTimePoint(now, 5), std::vector{"b"}, std::vector>{"114.5", "118"}); + auto tuple_1 = std::make_tuple(tp_1, std::vector{"a"}, std::vector>{nonstd::optional{}, "12"}); + auto tuple_6 = std::make_tuple(tp_6, std::vector{"b"}, std::vector>{nonstd::optional{}, "12"}); + auto tuple_6_inf = std::make_tuple(GetTimePoint(now, 6), std::vector{"b"}, std::vector>{nonstd::optional{}, "112"}); + + OpTester test("TimeSeriesImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + AddInputs(test, {{tuple_0, tuple_5, tuple_1, tuple_6}}, {tuple_0, tuple_5_inf, tuple_1, tuple_6_inf}, + {NS::TypeId::Float64, NS::TypeId::Float64}, false, NS::Featurizers::Components::TimeSeriesImputeStrategy::Forward); + + AddOutputs(test, {false, false, false, false}, {tp_0, tp_5, tp_1, tp_6}, + {"a", "b", "a", "b"}, {"14.5", "18", "114.5", "118", "14.5", "12", "114.5", "112"}); + test.Run(); +} + +TEST(FeaturizersTests, RowImputation_2_grains_1_gap_input_interleaved) { + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + auto tp_0 = GetTimePoint(now, 0); + auto tp_1 = GetTimePoint(now, 1); + auto tp_2 = GetTimePoint(now, 2); + auto tp_5 = GetTimePoint(now, 5); + auto tp_6 = GetTimePoint(now, 6); + auto tp_7 = GetTimePoint(now, 7); + + auto tuple_0 = std::make_tuple(tp_0, std::vector{"a"}, std::vector>{"14.5", "18"}); + auto tuple_2 = std::make_tuple(GetTimePoint(now, 2), std::vector{"a"}, std::vector>{nonstd::optional{}, "12"}); + auto tuple_5 = std::make_tuple(tp_5, std::vector{"b"}, std::vector>{"14.5", "18"}); + auto tuple_5_inf = std::make_tuple(tp_5, std::vector{"b"}, std::vector>{"114.5", "118"}); + auto tuple_1 = std::make_tuple(tp_1, std::vector{"a"}, std::vector>{nonstd::optional{}, "12"}); + auto tuple_6 = std::make_tuple(tp_6, std::vector{"b"}, std::vector>{nonstd::optional{}, "12"}); + auto tuple_7 = std::make_tuple(GetTimePoint(now, 7), std::vector{"b"}, std::vector>{nonstd::optional{}, "112"}); + + OpTester test("TimeSeriesImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + AddInputs(test, {{tuple_0, tuple_5, tuple_1, tuple_6}}, {tuple_0, tuple_5_inf, tuple_2, tuple_7}, + {NS::TypeId::Float64, NS::TypeId::Float64}, false, NS::Featurizers::Components::TimeSeriesImputeStrategy::Forward); + + AddOutputs(test, {false, false, true, false, true, false}, {tp_0, tp_5, tp_1, tp_2, tp_6, tp_7}, + {"a", "b", "a", "a", "b", "b"}, {"14.5", "18", "114.5", "118", "14.5", "18", "14.5", "12", "114.5", "118", "114.5", "112"}); + + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/allocator_test.cc b/onnxruntime/test/framework/allocator_test.cc index 6fe5faf810ae9..9a5ded2146b7b 100644 --- a/onnxruntime/test/framework/allocator_test.cc +++ b/onnxruntime/test/framework/allocator_test.cc @@ -13,7 +13,13 @@ TEST(AllocatorTest, CPUAllocatorTest) { ASSERT_STREQ(cpu_arena->Info().name, CPU); EXPECT_EQ(cpu_arena->Info().id, 0); - EXPECT_EQ(cpu_arena->Info().type, OrtAllocatorType::OrtArenaAllocator); + + // arena is disabled for CPUExecutionProvider on x86 and JEMalloc +#if (defined(__amd64__) || defined(_M_AMD64)) && !defined(USE_JEMALLOC) + EXPECT_EQ(cpu_arena->Info().alloc_type, OrtAllocatorType::OrtArenaAllocator); +#else + EXPECT_EQ(cpu_arena->Info().alloc_type, OrtAllocatorType::OrtDeviceAllocator); +#endif size_t size = 1024; auto bytes = cpu_arena->Alloc(size); diff --git a/onnxruntime/test/framework/cuda/allocator_cuda_test.cc b/onnxruntime/test/framework/cuda/allocator_cuda_test.cc index 4bcca7ee7bd66..b0388b457a156 100644 --- a/onnxruntime/test/framework/cuda/allocator_cuda_test.cc +++ b/onnxruntime/test/framework/cuda/allocator_cuda_test.cc @@ -11,8 +11,10 @@ namespace onnxruntime { namespace test { TEST(AllocatorTest, CUDAAllocatorTest) { int cuda_device_id = 0; - DeviceAllocatorRegistrationInfo default_memory_info({OrtMemTypeDefault, - [](int id) { return onnxruntime::make_unique(id, CUDA); }, std::numeric_limits::max()}); + DeviceAllocatorRegistrationInfo default_memory_info( + {OrtMemTypeDefault, + [](int id) { return onnxruntime::make_unique(id, CUDA); }, + std::numeric_limits::max()}); auto cuda_arena = CreateAllocator(default_memory_info, cuda_device_id); @@ -21,21 +23,23 @@ TEST(AllocatorTest, CUDAAllocatorTest) { EXPECT_STREQ(cuda_arena->Info().name, CUDA); EXPECT_EQ(cuda_arena->Info().id, cuda_device_id); EXPECT_EQ(cuda_arena->Info().mem_type, OrtMemTypeDefault); - EXPECT_EQ(cuda_arena->Info().type, OrtArenaAllocator); + EXPECT_EQ(cuda_arena->Info().alloc_type, OrtArenaAllocator); //test cuda allocation auto cuda_addr = cuda_arena->Alloc(size); EXPECT_TRUE(cuda_addr); - DeviceAllocatorRegistrationInfo pinned_memory_info({OrtMemTypeCPUOutput, - [](int) { return onnxruntime::make_unique(0, CUDA_PINNED); }, std::numeric_limits::max()}); + DeviceAllocatorRegistrationInfo pinned_memory_info( + {OrtMemTypeCPUOutput, + [](int) { return onnxruntime::make_unique(0, CUDA_PINNED); }, + std::numeric_limits::max()}); auto pinned_allocator = CreateAllocator(pinned_memory_info); EXPECT_STREQ(pinned_allocator->Info().name, CUDA_PINNED); EXPECT_EQ(pinned_allocator->Info().id, 0); EXPECT_EQ(pinned_allocator->Info().mem_type, OrtMemTypeCPUOutput); - EXPECT_EQ(pinned_allocator->Info().type, OrtArenaAllocator); + EXPECT_EQ(pinned_allocator->Info().alloc_type, OrtArenaAllocator); //test pinned allocation auto pinned_addr = pinned_allocator->Alloc(size); @@ -45,7 +49,7 @@ TEST(AllocatorTest, CUDAAllocatorTest) { EXPECT_STREQ(cpu_arena->Info().name, CPU); EXPECT_EQ(cpu_arena->Info().id, 0); EXPECT_EQ(cpu_arena->Info().mem_type, OrtMemTypeDefault); - EXPECT_EQ(cpu_arena->Info().type, OrtArenaAllocator); + EXPECT_EQ(cpu_arena->Info().alloc_type, OrtArenaAllocator); auto cpu_addr_a = cpu_arena->Alloc(size); EXPECT_TRUE(cpu_addr_a); diff --git a/onnxruntime/test/framework/cuda/fence_cuda_test.cc b/onnxruntime/test/framework/cuda/fence_cuda_test.cc index b2f55064deaa5..c2c057a96402c 100644 --- a/onnxruntime/test/framework/cuda/fence_cuda_test.cc +++ b/onnxruntime/test/framework/cuda/fence_cuda_test.cc @@ -127,7 +127,8 @@ TEST(CUDAFenceTests, DISABLED_PartOnCPU) { session.Run(std::unordered_map{{"X1", value}}, std::vector{"Out"}, &outputs); ASSERT_TRUE(1 == outputs.size()); const Tensor& output = outputs[0].Get(); - EXPECT_EQ(output.Shape(), shape); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + EXPECT_EQ(*reinterpret_cast*>(&output.Shape()), *reinterpret_cast*>(&shape)); EXPECT_EQ(output.DataType(), DataTypeImpl::GetType()); float expected_output[4] = {13.0f, -18.0f, -27.0f, 40.0f}; @@ -179,7 +180,8 @@ TEST(CUDAFenceTests, TileWithInitializer) { session.Run(std::unordered_map{{"X1", value}}, std::vector{"Y"}, &outputs); ASSERT_TRUE(1 == outputs.size()); const Tensor& output = outputs[0].Get(); - EXPECT_EQ(output.Shape(), TensorShape({2, 4})); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + EXPECT_EQ(*reinterpret_cast*>(&output.Shape()), (std::vector{2, 4})); EXPECT_EQ(output.DataType(), DataTypeImpl::GetType()); float expected_output[8] = {-1, 2, -1, 2, 3, -4, 3, -4}; @@ -242,7 +244,8 @@ TEST(CUDAFenceTests, TileWithComputedInput) { session.Run(std::unordered_map{{"X1", value}}, std::vector{"Out"}, &outputs); ASSERT_TRUE(1 == outputs.size()); const Tensor& output = outputs[0].Get(); - EXPECT_EQ(output.Shape(), TensorShape({4, 4})); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + EXPECT_EQ(*reinterpret_cast*>(&output.Shape()), (std::vector{4, 4})); EXPECT_EQ(output.DataType(), DataTypeImpl::GetType()); float expected_output[16] = {7, -10, 7, -10, -15, 22, -15, 22, 7, -10, 7, -10, -15, 22, -15, 22}; diff --git a/onnxruntime/test/framework/execution_frame_test.cc b/onnxruntime/test/framework/execution_frame_test.cc index ee67b5b473a97..36fcc9c1cbf8e 100644 --- a/onnxruntime/test/framework/execution_frame_test.cc +++ b/onnxruntime/test/framework/execution_frame_test.cc @@ -90,7 +90,8 @@ TEST_F(ExecutionFrameTest, TensorAllocationTest) { OrtValue* p_ml_value = frame.GetMutableNodeInputOrOutputMLValue(0); Tensor* p_tensor = p_ml_value ? p_ml_value->GetMutable() : nullptr; EXPECT_TRUE(p_tensor); - EXPECT_EQ(p_tensor->Shape(), shape); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + EXPECT_EQ(*reinterpret_cast*>(&p_tensor->Shape()), *reinterpret_cast*>(&shape)); EXPECT_EQ(p_tensor->DataType(), DataTypeImpl::GetType()); //test share memory from tensor @@ -106,7 +107,8 @@ TEST_F(ExecutionFrameTest, TensorAllocationTest) { const OrtValue* p_ml_value_const = frame.GetNodeInputOrOutputMLValue(1); auto tensor2 = p_ml_value_const ? &(p_ml_value_const->Get()) : nullptr; EXPECT_TRUE(tensor2); - EXPECT_EQ(tensor2->Shape(), shape2); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + EXPECT_EQ(*reinterpret_cast*>(&tensor2->Shape()), *reinterpret_cast*>(&shape2)); EXPECT_EQ(tensor2->template Data(), p_tensor->template Data()); } @@ -155,7 +157,8 @@ TEST_F(ExecutionFrameTest, FeedInDataTest) { OrtValue* p_ml_value = frame.GetMutableNodeInputOrOutputMLValue(0); Tensor* p_tensor_arg_0 = p_ml_value ? p_ml_value->GetMutable() : nullptr; EXPECT_TRUE(p_tensor_arg_0); - EXPECT_EQ(p_tensor_arg_0->Shape(), shape); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + EXPECT_EQ(*reinterpret_cast*>(&p_tensor_arg_0->Shape()), *reinterpret_cast*>(&shape)); EXPECT_EQ(p_tensor_arg_0->DataType(), DataTypeImpl::GetType()); EXPECT_EQ(p_tensor_arg_0->MutableData(), value.GetMutable()->MutableData()); } diff --git a/onnxruntime/test/framework/float_16_test.cc b/onnxruntime/test/framework/float_16_test.cc index c141ea817c456..bb4ccc930afd0 100644 --- a/onnxruntime/test/framework/float_16_test.cc +++ b/onnxruntime/test/framework/float_16_test.cc @@ -122,7 +122,8 @@ void RunSession(InferenceSession& session_object, ASSERT_EQ(1, fetches.size()); auto& rtensor = fetches.front().Get(); TensorShape expected_shape(dims_y); - EXPECT_EQ(expected_shape, rtensor.Shape()); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + EXPECT_EQ(*reinterpret_cast*>(&expected_shape), *reinterpret_cast*>(&rtensor.Shape())); const std::vector found(rtensor.template Data(), rtensor.template Data() + expected_shape.Size()); ASSERT_EQ(found.size(), values_y.size()); for (size_t i = 0; i < found.size(); i++) diff --git a/onnxruntime/test/framework/inference_session_test.cc b/onnxruntime/test/framework/inference_session_test.cc index ca5e2f8d7d36e..2063852cdea84 100644 --- a/onnxruntime/test/framework/inference_session_test.cc +++ b/onnxruntime/test/framework/inference_session_test.cc @@ -86,8 +86,7 @@ class FuseExecutionProvider : public IExecutionProvider { DeviceAllocatorRegistrationInfo device_info({OrtMemTypeDefault, [](int) { return onnxruntime::make_unique(); }, std::numeric_limits::max()}); - InsertAllocator(std::shared_ptr( - onnxruntime::make_unique(device_info.factory(0)))); + InsertAllocator(device_info.factory(0)); } std::vector> @@ -176,7 +175,8 @@ template void VerifyOutputs(const Tensor& tensor, const std::vector& expected_dims, const std::vector& expected_values) { TensorShape expected_shape(expected_dims); - ASSERT_EQ(expected_shape, tensor.Shape()); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + ASSERT_EQ(*reinterpret_cast*>(&expected_shape), *reinterpret_cast*>(&tensor.Shape())); const std::vector found(tensor.template Data(), tensor.template Data() + expected_values.size()); ASSERT_EQ(expected_values, found); @@ -775,8 +775,8 @@ static void TestBindHelper(const std::string& log_str, std::string s1; p_model->ToProto().SerializeToString(&s1); std::stringstream sstr(s1); - ASSERT_TRUE(session_object.Load(sstr).IsOK()); - ASSERT_TRUE(session_object.Initialize().IsOK()); + ASSERT_STATUS_OK(session_object.Load(sstr)); + ASSERT_STATUS_OK(session_object.Initialize()); RunOptions run_options; run_options.run_log_verbosity_level = so.session_log_verbosity_level; @@ -1312,7 +1312,8 @@ TEST(InferenceSessionTests, TestTruncatedSequence) { ASSERT_EQ(1, fetches.size()); auto& rtensor = fetches.front().Get(); TensorShape expected_shape(Y_dims); - ASSERT_EQ(expected_shape, rtensor.Shape()); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + ASSERT_EQ(*reinterpret_cast*>(&expected_shape), *reinterpret_cast*>(&rtensor.Shape())); for (size_t i = 0; i < Y_data.size(); ++i) EXPECT_NEAR(Y_data[i], rtensor.template Data()[i], FLT_EPSILON); @@ -1354,7 +1355,8 @@ TEST(InferenceSessionTests, TestTruncatedSequence) { std::vector truncated_output_dims = Y_dims; truncated_output_dims[0] = truncated_len; TensorShape truncated_shape(truncated_output_dims); - ASSERT_EQ(truncated_shape, truncated_rtensor.Shape()); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + ASSERT_EQ(*reinterpret_cast*>(&truncated_shape), *reinterpret_cast*>(&truncated_rtensor.Shape())); auto seq_output_stride = truncated_shape.SizeFromDimension(1); for (int i = 0; i < truncated_shape.Size(); ++i) EXPECT_NEAR(Y_data[i + seq_start * seq_output_stride], truncated_rtensor.template Data()[i], FLT_EPSILON); diff --git a/onnxruntime/test/framework/local_kernel_registry_test.cc b/onnxruntime/test/framework/local_kernel_registry_test.cc index 09352e92b0bcd..9ef4d4de9bf25 100644 --- a/onnxruntime/test/framework/local_kernel_registry_test.cc +++ b/onnxruntime/test/framework/local_kernel_registry_test.cc @@ -214,7 +214,8 @@ void RunSession(InferenceSession& session_object, ASSERT_EQ(1, fetches.size()); auto& rtensor = fetches.front().Get(); TensorShape expected_shape(dims_y); - EXPECT_EQ(expected_shape, rtensor.Shape()); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + EXPECT_EQ(*reinterpret_cast*>(&expected_shape), *reinterpret_cast*>(&rtensor.Shape())); const std::vector found(rtensor.template Data(), rtensor.template Data() + expected_shape.Size()); ASSERT_EQ(values_y, found); } diff --git a/onnxruntime/test/framework/tensor_test.cc b/onnxruntime/test/framework/tensor_test.cc index 1dea1a5e95993..6251b0259e719 100644 --- a/onnxruntime/test/framework/tensor_test.cc +++ b/onnxruntime/test/framework/tensor_test.cc @@ -21,7 +21,8 @@ void CPUTensorTest(std::vector dims, const int offset = 0) { EXPECT_TRUE(data); Tensor t(DataTypeImpl::GetType(), shape, data, alloc->Info(), offset); auto tensor_shape = t.Shape(); - EXPECT_EQ(shape, tensor_shape); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + EXPECT_EQ(*reinterpret_cast*>(&shape), *reinterpret_cast*>(&tensor_shape)); EXPECT_EQ(t.DataType(), DataTypeImpl::GetType()); auto& location = t.Location(); EXPECT_STREQ(location.name, CPU); @@ -36,7 +37,8 @@ void CPUTensorTest(std::vector dims, const int offset = 0) { Tensor new_t(DataTypeImpl::GetType(), shape, alloc, offset); tensor_shape = new_t.Shape(); - EXPECT_EQ(shape, tensor_shape); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + EXPECT_EQ(*reinterpret_cast*>(&shape), *reinterpret_cast*>(&tensor_shape)); EXPECT_EQ(new_t.DataType(), DataTypeImpl::GetType()); auto& new_location = new_t.Location(); ASSERT_STREQ(new_location.name, CPU); @@ -134,7 +136,13 @@ TEST(TensorTest, EmptyTensorTest) { auto& location = t.Location(); ASSERT_STREQ(location.name, CPU); EXPECT_EQ(location.id, 0); - EXPECT_EQ(location.type, OrtAllocatorType::OrtArenaAllocator); + + // arena is disabled for CPUExecutionProvider on x86 and JEMalloc +#if (defined(__amd64__) || defined(_M_AMD64)) && !defined(USE_JEMALLOC) + EXPECT_EQ(location.alloc_type, OrtAllocatorType::OrtArenaAllocator); +#else + EXPECT_EQ(location.alloc_type, OrtAllocatorType::OrtDeviceAllocator); +#endif } TEST(TensorTest, StringTensorTest) { @@ -150,7 +158,8 @@ TEST(TensorTest, StringTensorTest) { Tensor t(DataTypeImpl::GetType(), shape, alloc); auto& tensor_shape = t.Shape(); - EXPECT_EQ(shape, tensor_shape); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + EXPECT_EQ(*reinterpret_cast*>(&shape), *reinterpret_cast*>(&tensor_shape)); EXPECT_EQ(t.DataType(), DataTypeImpl::GetType()); auto& location = t.Location(); ASSERT_STREQ(location.name, CPU); diff --git a/onnxruntime/test/ir/onnx_model_test.cc b/onnxruntime/test/ir/onnx_model_test.cc index 4fb0f11ed4b76..0a979cf71e2e6 100644 --- a/onnxruntime/test/ir/onnx_model_test.cc +++ b/onnxruntime/test/ir/onnx_model_test.cc @@ -83,7 +83,7 @@ TEST(ONNXModelsTest1, bvlc_alexnet_1) { std::unique_ptr raw_input(new FileInputStream(fd)); std::unique_ptr coded_input(new CodedInputStream(raw_input.get())); // Allows protobuf library versions < 3.2.0 to parse messages greater than 64MB. - coded_input->SetTotalBytesLimit(INT_MAX, INT_MAX); + coded_input->SetTotalBytesLimit(INT_MAX); ModelProto model_proto; bool result = model_proto.ParseFromCodedStream(coded_input.get()); coded_input.reset(); @@ -133,7 +133,7 @@ TEST_P(ONNXModelsTest, LoadFromProtobuf) { ASSERT_TRUE(fd > 0); std::unique_ptr raw_input(new FileInputStream(fd)); std::unique_ptr coded_input(new CodedInputStream(raw_input.get())); - coded_input->SetTotalBytesLimit(INT_MAX, INT_MAX); + coded_input->SetTotalBytesLimit(INT_MAX); std::unique_ptr model_proto = onnxruntime::make_unique(); bool result = model_proto->ParseFromCodedStream(coded_input.get()); coded_input.reset(); diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index de09b17df8dcd..9b6e661c8cc50 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -203,12 +203,13 @@ class OnnxModelInfo : public TestModelInfo { if (!st.IsOK()) { ORT_THROW(st.ErrorMessage()); } - google::protobuf::io::FileInputStream f(model_fd); - f.SetCloseOnDelete(true); + ONNX_NAMESPACE::ModelProto model_pb; - if (!model_pb.ParseFromZeroCopyStream(&f)) { + if (!model_pb.ParseFromFileDescriptor(model_fd)) { + (void)Env::Default().FileClose(model_fd); ORT_THROW("Failed to load model because protobuf parsing failed."); } + (void)Env::Default().FileClose(model_fd); #ifdef __GNUG__ const RE2::Anchor re2_anchor = RE2::UNANCHORED; re2::StringPiece text(model_url); @@ -233,6 +234,7 @@ class OnnxModelInfo : public TestModelInfo { if (!init.has_name()) continue; initializer_names.insert(init.name()); } + //Ignore the inputs that are already in initializers for (const auto& p : graph.input()) { if (!p.has_name()) ORT_THROW("input without name??"); if (initializer_names.find(p.name()) == initializer_names.end()) input_value_info_.push_back(p); diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index ed444ccc67ae2..6113657cae7ee 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -41,7 +41,7 @@ void usage() { "Default: 'cpu'.\n" "\t-x: Use parallel executor, default (without -x): sequential executor.\n" "\t-d [device_id]: Specifies the device id for multi-device (e.g. GPU). The value should > 0\n" - "\t-o [optimization level]: Default is 1. Valid values are 0 (disable), 1 (basic), 2 (extended), 99 (all).\n" + "\t-o [optimization level]: Default is 99. Valid values are 0 (disable), 1 (basic), 2 (extended), 99 (all).\n" "\t\tPlease see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels. " "\n" "\t-h: help\n" @@ -354,14 +354,16 @@ int real_main(int argc, char* argv[], Ort::Env& env) { sf.SetGraphOptimizationLevel(graph_optimization_level); } - static const char* cuda_flaky_tests[] = {"fp16_inception_v1", "fp16_shufflenet", "fp16_tiny_yolov2"}; - static const char* dml_disabled_tests[] = {"mlperf_ssd_resnet34_1200", "mlperf_ssd_mobilenet_300", "mask_rcnn_keras", "mask_rcnn", "faster_rcnn"}; - static const char* dnnl_disabled_tests[] = {"test_densenet121", "test_resnet18v2", "test_resnet34v2", "test_resnet50v2", "test_resnet101v2", - "test_resnet101v2", "test_vgg19", "tf_inception_resnet_v2", "tf_inception_v1", "tf_inception_v3", "tf_inception_v4", "tf_mobilenet_v1_1.0_224", - "tf_mobilenet_v2_1.0_224", "tf_mobilenet_v2_1.4_224", "tf_nasnet_large", "tf_pnasnet_large", "tf_resnet_v1_50", "tf_resnet_v1_101", "tf_resnet_v1_101", - "tf_resnet_v2_101", "tf_resnet_v2_152"}; - - std::unordered_set all_disabled_tests; + static const ORTCHAR_T* cuda_flaky_tests[] = { + ORT_TSTR("fp16_inception_v1"), + ORT_TSTR("fp16_shufflenet"), ORT_TSTR("fp16_tiny_yolov2")}; + static const ORTCHAR_T* dml_disabled_tests[] = {ORT_TSTR("mlperf_ssd_resnet34_1200"), ORT_TSTR("mlperf_ssd_mobilenet_300"), ORT_TSTR("mask_rcnn"), ORT_TSTR("faster_rcnn")}; + static const ORTCHAR_T* dnnl_disabled_tests[] = {ORT_TSTR("test_densenet121"), ORT_TSTR("test_resnet18v2"), ORT_TSTR("test_resnet34v2"), ORT_TSTR("test_resnet50v2"), ORT_TSTR("test_resnet101v2"), + ORT_TSTR("test_resnet101v2"), ORT_TSTR("test_vgg19"), ORT_TSTR("tf_inception_resnet_v2"), ORT_TSTR("tf_inception_v1"), ORT_TSTR("tf_inception_v3"), ORT_TSTR("tf_inception_v4"), ORT_TSTR("tf_mobilenet_v1_1.0_224"), + ORT_TSTR("tf_mobilenet_v2_1.0_224"), ORT_TSTR("tf_mobilenet_v2_1.4_224"), ORT_TSTR("tf_nasnet_large"), ORT_TSTR("tf_pnasnet_large"), ORT_TSTR("tf_resnet_v1_50"), ORT_TSTR("tf_resnet_v1_101"), ORT_TSTR("tf_resnet_v1_101"), + ORT_TSTR("tf_resnet_v2_101"), ORT_TSTR("tf_resnet_v2_152")}; + + std::unordered_set > all_disabled_tests; if (enable_cuda) { all_disabled_tests.insert(std::begin(cuda_flaky_tests), std::end(cuda_flaky_tests)); } @@ -373,25 +375,15 @@ int real_main(int argc, char* argv[], Ort::Env& env) { // This will be removed after LRU implementation all_disabled_tests.insert(std::begin(dnnl_disabled_tests), std::end(dnnl_disabled_tests)); } -#if defined(__amd64__) || defined(_M_AMD64) -#else +#if !defined(__amd64__) && !defined(_M_AMD64) //out of memory - static const char* x86_disabled_tests[] = {"mlperf_ssd_resnet34_1200", "mask_rcnn_keras", "mask_rcnn", "faster_rcnn", "vgg19"}; + static const ORTCHAR_T* x86_disabled_tests[] = {ORT_TSTR("mlperf_ssd_resnet34_1200"), ORT_TSTR("mask_rcnn_keras"), ORT_TSTR("mask_rcnn"), ORT_TSTR("faster_rcnn"), ORT_TSTR("vgg19")}; all_disabled_tests.insert(std::begin(x86_disabled_tests), std::end(x86_disabled_tests)); #endif std::vector tests; - LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance, + LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance, all_disabled_tests, [&tests](ITestCase* l) { tests.push_back(l); }); - for (auto it = tests.begin(); it != tests.end();) { - auto iter = all_disabled_tests.find((*it)->GetTestCaseName()); - if (iter != all_disabled_tests.end()) { - delete *it; - it = tests.erase(it); - } else { - ++it; - } - } TestEnv args(tests, stat, env, sf); Status st = RunTests(args, p_models, concurrent_session_runs, static_cast(repeat_count), @@ -406,25 +398,22 @@ int real_main(int argc, char* argv[], Ort::Env& env) { std::string res = stat.ToString(); fwrite(res.c_str(), 1, res.size(), stdout); } - // clang-format off - struct BrokenTest - { + struct BrokenTest { std::string test_name_; std::string reason_; - std::set broken_versions_ = {}; // apply to all versions if empty + std::set broken_versions_ = {}; // apply to all versions if empty BrokenTest(std::string name, std::string reason) : test_name_(std::move(name)), reason_(std::move(reason)) {} - BrokenTest(std::string name, std::string reason, const std::initializer_list& versions) : - test_name_(std::move(name)), reason_(std::move(reason)), broken_versions_(versions) {} - bool operator < (const struct BrokenTest& test) const { - return strcmp(test_name_.c_str(), test.test_name_.c_str()) < 0; + BrokenTest(std::string name, std::string reason, const std::initializer_list& versions) : test_name_(std::move(name)), reason_(std::move(reason)), broken_versions_(versions) {} + bool operator<(const struct BrokenTest& test) const { + return strcmp(test_name_.c_str(), test.test_name_.c_str()) < 0; } }; std::set broken_tests = { {"BERT_Squad", "test data bug"}, - {"constantofshape_float_ones", "test data bug", {"onnx141","onnx150"}}, - {"constantofshape_int_zeros", "test data bug", {"onnx141","onnx150"}}, + {"constantofshape_float_ones", "test data bug", {"onnx141", "onnx150"}}, + {"constantofshape_int_zeros", "test data bug", {"onnx141", "onnx150"}}, {"convtranspose_3d", "3d convtranspose not supported yet"}, {"cast_STRING_to_FLOAT", "Linux CI has old ONNX python package with bad test data", {"onnx141"}}, // Numpy float to string has unexpected rounding for some results given numpy default precision is meant to be 8. @@ -451,102 +440,105 @@ int real_main(int argc, char* argv[], Ort::Env& env) { {"resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", "Bad onnx test output. Needs test fix."}, {"bitshift_right_uint16", "BitShift(11) uint16 support not enabled currently"}, {"bitshift_left_uint16", "BitShift(11) uint16 support not enabled currently"}, - {"maxunpool_export_with_output_shape", "Invalid output in ONNX test. See https://github.com/onnx/onnx/issues/2398" }, -}; - -#ifdef USE_NGRAPH - broken_tests.insert({"qlinearconv", "ambiguity in scalar dimensions [] vs [1]"}); - broken_tests.insert({"clip_splitbounds", "not implemented yet for opset 11"}); - broken_tests.insert({"clip_outbounds", "not implemented yet for opset 11"}); - broken_tests.insert({"clip_example", "not implemented yet for opset 11"}); - broken_tests.insert({"clip_default_min", "not implemented yet for opset 11"}); - broken_tests.insert({"clip_default_max", "not implemented yet for opset 11"}); - broken_tests.insert({"clip", "not implemented yet for opset 11"}); - broken_tests.insert({"depthtospace_crd_mode_example", "NGraph does not support CRD mode"}); - broken_tests.insert({"depthtospace_crd_mode", "NGraph does not support CRD mode"}); - broken_tests.insert({"gemm_default_no_bias", "not implemented yet for opset 11"}); - broken_tests.insert({"quantizelinear", "ambiguity in scalar dimensions [] vs [1]", {"onnx150"}}); - broken_tests.insert({"dequantizelinear", "ambiguity in scalar dimensions [] vs [1]", {"onnx150"}}); - broken_tests.insert({"mlperf_ssd_resnet34_1200", "Results mismatch"}); - broken_tests.insert({"BERT_Squad", "Invalid Feed Input Name:input4"}); - broken_tests.insert({"mask_rcnn_keras", "Results mismatch: 8 of 81000"}); - broken_tests.insert({"candy", "Results mismatch: 2 of 150528"}); - broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); -#endif + {"maxunpool_export_with_output_shape", "Invalid output in ONNX test. See https://github.com/onnx/onnx/issues/2398"}, + }; -#ifdef USE_DNNL - broken_tests.insert({"tf_mobilenet_v2_1.0_224", "result mismatch"}); - broken_tests.insert({"tf_mobilenet_v2_1.4_224", "result mismatch"}); - broken_tests.insert({"tf_mobilenet_v1_1.0_224", "result mismatch"}); - broken_tests.insert({"mobilenetv2-1.0", "result mismatch"}); - broken_tests.insert({"candy", "result mismatch"}); - broken_tests.insert({"range_float_type_positive_delta_expanded", "get unknown exception from DNNL EP"}); - broken_tests.insert({"range_int32_type_negative_delta_expanded", "get unknown exception from DNNL EP"}); - broken_tests.insert({"averagepool_2d_ceil", "maxpool ceiling not supported"}); - broken_tests.insert({"maxpool_2d_ceil", "maxpool ceiling not supported"}); - broken_tests.insert({"maxpool_2d_dilations", "maxpool dilations not supported"}); - broken_tests.insert({"mlperf_ssd_resnet34_1200", "test pass on dev box but fails on CI build"}); - broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); -#endif + if (enable_ngraph) { + broken_tests.insert({"qlinearconv", "ambiguity in scalar dimensions [] vs [1]"}); + broken_tests.insert({"clip_splitbounds", "not implemented yet for opset 11"}); + broken_tests.insert({"clip_outbounds", "not implemented yet for opset 11"}); + broken_tests.insert({"clip_example", "not implemented yet for opset 11"}); + broken_tests.insert({"clip_default_min", "not implemented yet for opset 11"}); + broken_tests.insert({"clip_default_max", "not implemented yet for opset 11"}); + broken_tests.insert({"clip", "not implemented yet for opset 11"}); + broken_tests.insert({"depthtospace_crd_mode_example", "NGraph does not support CRD mode"}); + broken_tests.insert({"depthtospace_crd_mode", "NGraph does not support CRD mode"}); + broken_tests.insert({"gemm_default_no_bias", "not implemented yet for opset 11"}); + broken_tests.insert({"quantizelinear", "ambiguity in scalar dimensions [] vs [1]", {"onnx150"}}); + broken_tests.insert({"dequantizelinear", "ambiguity in scalar dimensions [] vs [1]", {"onnx150"}}); + broken_tests.insert({"mlperf_ssd_resnet34_1200", "Results mismatch"}); + broken_tests.insert({"BERT_Squad", "Invalid Feed Input Name:input4"}); + broken_tests.insert({"candy", "Results mismatch: 2 of 150528"}); + broken_tests.insert({"tf_mobilenet_v1_1.0_224", "Results mismatch"}); + broken_tests.insert({"tf_mobilenet_v2_1.0_224", "Results mismatch"}); + broken_tests.insert({"tf_mobilenet_v2_1.4_224", "Results mismatch"}); + broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); + } + if (enable_nuphar) { + broken_tests.insert({"cgan", "TVM exception during initialization"}); + } + if (enable_dnnl) { + broken_tests.insert({"tf_mobilenet_v2_1.0_224", "result mismatch"}); + broken_tests.insert({"tf_mobilenet_v2_1.4_224", "result mismatch"}); + broken_tests.insert({"tf_mobilenet_v1_1.0_224", "result mismatch"}); + broken_tests.insert({"mobilenetv2-1.0", "result mismatch"}); + broken_tests.insert({"candy", "result mismatch"}); + broken_tests.insert({"range_float_type_positive_delta_expanded", "get unknown exception from DNNL EP"}); + broken_tests.insert({"range_int32_type_negative_delta_expanded", "get unknown exception from DNNL EP"}); + broken_tests.insert({"averagepool_2d_ceil", "maxpool ceiling not supported"}); + broken_tests.insert({"maxpool_2d_ceil", "maxpool ceiling not supported"}); + broken_tests.insert({"maxpool_2d_dilations", "maxpool dilations not supported"}); + broken_tests.insert({"mlperf_ssd_resnet34_1200", "test pass on dev box but fails on CI build"}); + broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); + } -#ifdef USE_OPENVINO - broken_tests.insert({"fp16_shufflenet", "accuracy mismatch with fp16 precision"}); - broken_tests.insert({"fp16_inception_v1", "accuracy mismatch with fp16 precision"}); - broken_tests.insert({"fp16_tiny_yolov2", "accuaracy mismatch with fp16 precision"}); - broken_tests.insert({"scan_sum", "disable temporarily"}); - broken_tests.insert({"scan9_sum", "disable temporarily"}); - broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); + if (enable_openvino) { + broken_tests.insert({"fp16_shufflenet", "accuracy mismatch with fp16 precision"}); + broken_tests.insert({"fp16_inception_v1", "accuracy mismatch with fp16 precision"}); + broken_tests.insert({"fp16_tiny_yolov2", "accuaracy mismatch with fp16 precision"}); + broken_tests.insert({"scan_sum", "disable temporarily"}); + broken_tests.insert({"scan9_sum", "disable temporarily"}); + broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); #ifdef OPENVINO_CONFIG_GPU_FP32 - broken_tests.insert({"tiny_yolov2", "accuracy mismatch"}); - broken_tests.insert({"div", "will be fixed in the next release"}); + broken_tests.insert({"tiny_yolov2", "accuracy mismatch"}); + broken_tests.insert({"div", "will be fixed in the next release"}); #ifdef OPENVINO_CONFIG_GPU_FP16 - broken_tests.insert({"div", "will be fixed in the next release"}); -#endif + broken_tests.insert({"div", "will be fixed in the next release"}); #endif #endif + } -#ifdef USE_NNAPI - broken_tests.insert({"scan9_sum", "Error with the extra graph"}); - broken_tests.insert({"scan_sum", "Error with the extra graph"}); - broken_tests.insert({"mvn_expanded", "Failed to find kernel for MemcpyFromHost(1) (node Memcpy_1)"}); - broken_tests.insert({"dynamicquantizelinear_expanded", "Temporarily disabled pending investigation"}); - broken_tests.insert({"dynamicquantizelinear_max_adjusted_expanded", "Temporarily disabled pending investigation"}); - broken_tests.insert({"dynamicquantizelinear_min_adjusted_expanded", "Temporarily disabled pending investigation"}); - broken_tests.insert({"gemm_transposeB", "Temporarily disabled pending investigation"}); - broken_tests.insert({"range_float_type_positive_delta_expanded", "Temporarily disabled pending investigation"}); - broken_tests.insert({"range_int32_type_negative_delta_expanded", "Temporarily disabled pending investigation"}); - broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); -#endif + if (enable_nnapi) { + broken_tests.insert({"scan9_sum", "Error with the extra graph"}); + broken_tests.insert({"scan_sum", "Error with the extra graph"}); + broken_tests.insert({"mvn_expanded", "Failed to find kernel for MemcpyFromHost(1) (node Memcpy_1)"}); + broken_tests.insert({"dynamicquantizelinear_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"dynamicquantizelinear_max_adjusted_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"dynamicquantizelinear_min_adjusted_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"gemm_transposeB", "Temporarily disabled pending investigation"}); + broken_tests.insert({"range_float_type_positive_delta_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"range_int32_type_negative_delta_expanded", "Temporarily disabled pending investigation"}); + broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); + } -#ifdef USE_TENSORRT - broken_tests.insert({"fp16_shufflenet", "TRT EP bug"}); - broken_tests.insert({"fp16_inception_v1", "TRT EP bug"}); - broken_tests.insert({"fp16_tiny_yolov2", "TRT EP bug"}); - broken_tests.insert({"tf_inception_v3", "TRT Engine couldn't be created"}); - broken_tests.insert({"tf_mobilenet_v1_1.0_224", "TRT Engine couldn't be created"}); - broken_tests.insert({"tf_mobilenet_v2_1.0_224", "TRT Engine couldn't be created"}); - broken_tests.insert({"tf_mobilenet_v2_1.4_224", "TRT Engine couldn't be created"}); - broken_tests.insert({"tf_resnet_v1_101", "TRT Engine couldn't be created"}); - broken_tests.insert({"tf_resnet_v1_152", "TRT Engine couldn't be created"}); - broken_tests.insert({"tf_resnet_v1_50", "TRT Engine couldn't be created"}); - broken_tests.insert({"tf_resnet_v2_101", "TRT Engine couldn't be created"}); - broken_tests.insert({"tf_resnet_v2_152", "TRT Engine couldn't be created"}); - broken_tests.insert({"tf_resnet_v2_50", "TRT Engine couldn't be created"}); - broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); -#endif + if (enable_tensorrt) { + broken_tests.insert({"fp16_shufflenet", "TRT EP bug"}); + broken_tests.insert({"fp16_inception_v1", "TRT EP bug"}); + broken_tests.insert({"fp16_tiny_yolov2", "TRT EP bug"}); + broken_tests.insert({"tf_inception_v3", "TRT Engine couldn't be created"}); + broken_tests.insert({"tf_mobilenet_v1_1.0_224", "TRT Engine couldn't be created"}); + broken_tests.insert({"tf_mobilenet_v2_1.0_224", "TRT Engine couldn't be created"}); + broken_tests.insert({"tf_mobilenet_v2_1.4_224", "TRT Engine couldn't be created"}); + broken_tests.insert({"tf_resnet_v1_101", "TRT Engine couldn't be created"}); + broken_tests.insert({"tf_resnet_v1_152", "TRT Engine couldn't be created"}); + broken_tests.insert({"tf_resnet_v1_50", "TRT Engine couldn't be created"}); + broken_tests.insert({"tf_resnet_v2_101", "TRT Engine couldn't be created"}); + broken_tests.insert({"tf_resnet_v2_152", "TRT Engine couldn't be created"}); + broken_tests.insert({"tf_resnet_v2_50", "TRT Engine couldn't be created"}); + broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); + } -#ifdef USE_CUDA - broken_tests.insert({"candy", "result mismatch"}); - broken_tests.insert({"mask_rcnn_keras", "result mismatch"}); - broken_tests.insert({"mlperf_ssd_mobilenet_300", "unknown error"}); - broken_tests.insert({"mlperf_ssd_resnet34_1200", "unknown error"}); - broken_tests.insert({"tf_inception_v1", "flaky test"}); //TODO: Investigate cause for flakiness - broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); -#endif + if (enable_cuda) { + broken_tests.insert({"candy", "result mismatch"}); + broken_tests.insert({"tinyyolov3", "The parameter is incorrect"}); + broken_tests.insert({"mlperf_ssd_mobilenet_300", "unknown error"}); + broken_tests.insert({"mlperf_ssd_resnet34_1200", "unknown error"}); + broken_tests.insert({"tf_inception_v1", "flaky test"}); //TODO: Investigate cause for flakiness + broken_tests.insert({"convtranspose_1d", "1d convtranspose not supported yet"}); + } -#ifdef USE_DML - if (enable_dml) - { + if (enable_dml) { + broken_tests.insert({"tinyyolov3", "The parameter is incorrect"}); broken_tests.insert({"PixelShuffle", "Test requires 6D Reshape, which isn't supported by DirectML"}); broken_tests.insert({"operator_permute2", "Test requires 6D Transpose, which isn't supported by DirectML"}); broken_tests.insert({"resize_downsample_linear", "ORT 0.4 uses asymmetric but will conform to half_pixel in the next ONNX version."}); @@ -563,7 +555,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) { broken_tests.insert({"dynamicquantizelinear_expanded", "Temporarily disabled pending investigation"}); broken_tests.insert({"dynamicquantizelinear_max_adjusted_expanded", "Temporarily disabled pending investigation"}); broken_tests.insert({"dynamicquantizelinear_min_adjusted_expanded", "Temporarily disabled pending investigation"}); - broken_tests.insert({"maxpool_with_argmax_2d_precomputed_pads", "Temporarily disabled pending investigation"}); broken_tests.insert({"mxnet_arcface", "Temporarily disabled pending investigation"}); broken_tests.insert({"yolov3", "Temporarily disabled pending investigation"}); broken_tests.insert({"tf_inception_v2", "Temporarily disabled pending investigation"}); @@ -571,8 +562,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) { broken_tests.insert({"candy", "Temporarily disabled pending investigation"}); broken_tests.insert({"BERT_Squad", "Temporarily disabled pending investigation"}); } -#endif - // clang-format on #if defined(_WIN32) && !defined(_WIN64) broken_tests.insert({"vgg19", "failed: bad allocation"}); diff --git a/onnxruntime/test/onnx/runner.cc b/onnxruntime/test/onnx/runner.cc index d8942fc222138..3356f3b1b90fd 100644 --- a/onnxruntime/test/onnx/runner.cc +++ b/onnxruntime/test/onnx/runner.cc @@ -284,6 +284,7 @@ Status RunTests(TestEnv& env, int p_models, int concurrent_runs, size_t repeat_c void LoadTests(const std::vector>& input_paths, const std::vector>& whitelisted_test_cases, double default_per_sample_tolerance, double default_relative_per_sample_tolerance, + const std::unordered_set>& disabled_tests, const std::function& process_function) { std::vector> paths(input_paths); while (!paths.empty()) { @@ -302,10 +303,13 @@ void LoadTests(const std::vector>& input_paths std::basic_string test_case_name = my_dir_name; if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) test_case_name = test_case_name.substr(5); + if (!whitelisted_test_cases.empty() && std::find(whitelisted_test_cases.begin(), whitelisted_test_cases.end(), test_case_name) == whitelisted_test_cases.end()) { return true; } + if (disabled_tests.find(test_case_name) != disabled_tests.end()) return true; + std::basic_string p = ConcatPathComponent(node_data_root_path, filename_str); ITestCase* l = CreateOnnxTestCase(ToMBString(test_case_name), TestModelInfo::LoadOnnxModel(p.c_str()), diff --git a/onnxruntime/test/onnx/runner.h b/onnxruntime/test/onnx/runner.h index 9a26496b454f1..5f07b6bd78849 100644 --- a/onnxruntime/test/onnx/runner.h +++ b/onnxruntime/test/onnx/runner.h @@ -130,6 +130,7 @@ struct DataTask { void LoadTests(const std::vector>& input_paths, const std::vector>& whitelisted_test_cases, double default_per_sample_tolerance, double default_relative_per_sample_tolerance, + const std::unordered_set>& disabled_tests, const std::function& process_function); //Do not run this function in the thread pool passed in diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc index bbfc1fcc0dd68..6bce5a239c0e1 100644 --- a/onnxruntime/test/optimizer/graph_transform_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_test.cc @@ -609,6 +609,23 @@ TEST(GraphTransformationTests, Gemm_Relu_three_input) { std::map op_to_count = CountOpsInGraph(graph); ASSERT_TRUE(op_to_count["Relu"] == 0); } + +TEST(GraphTransformationTests, Gemm_LeakyRelu_Fusion) { + auto model_uri = MODEL_FOLDER "gemm_activation_fusion/gemm_activation_fusion.onnx"; + + std::shared_ptr p_model; + ASSERT_TRUE(Model::Load(model_uri, p_model, nullptr, DefaultLoggingManager().DefaultLogger()).IsOK()); + Graph& graph = p_model->MainGraph(); + std::map op_to_count1 = CountOpsInGraph(graph); + onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); + ASSERT_TRUE(graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, DefaultLoggingManager().DefaultLogger()).IsOK()); + + std::map op_to_count = CountOpsInGraph(graph); + ASSERT_TRUE(op_to_count["LeakyRelu"] == 0); + ASSERT_TRUE(op_to_count["Gemm"] == 0); + ASSERT_TRUE(op_to_count["FusedGemm"] == 1); +} #endif TEST(GraphTransformationTests, FuseConvBnAddMulFloat16) { @@ -661,7 +678,8 @@ TEST(GraphTransformationTests, FuseConvBnAddMulFloat16) { ASSERT_EQ(1, fetches.size()); auto& rtensor = fetches.front().Get(); TensorShape expected_shape(expected_dims_prod); - ASSERT_EQ(expected_shape, rtensor.Shape()); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + ASSERT_EQ(*reinterpret_cast*>(&expected_shape), *reinterpret_cast*>(&rtensor.Shape())); const std::vector found(rtensor.template Data(), rtensor.template Data() + expected_dims_prod.size()); ASSERT_EQ(expected_values_prod, found); diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index 7820639946fad..6e539162bf88a 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -69,15 +69,17 @@ Status PerformanceRunner::Run() { performance_result_.average_CPU_usage = p_ICPUUsage->GetUsage(); performance_result_.peak_workingset_size = utils::GetPeakWorkingSetSize(); + std::chrono::duration session_create_duration = session_create_end_ - session_create_start_; // TODO: end profiling // if (!performance_test_config_.run_config.profile_file.empty()) session_object->EndProfiling(); - std::chrono::duration duration = performance_result_.end_ - performance_result_.start_; + std::chrono::duration inference_duration = performance_result_.end_ - performance_result_.start_; - std::cout << "Total time cost:" << performance_result_.total_time_cost << std::endl // sum of time taken by each request - << "Total iterations:" << performance_result_.time_costs.size() << std::endl - << "Average time cost:" << performance_result_.total_time_cost / performance_result_.time_costs.size() * 1000 << " ms" << std::endl + std::cout << "Session creation time cost:" << session_create_duration.count() << " s" << std::endl + << "Total inference time cost:" << performance_result_.total_time_cost << " s" << std::endl // sum of time taken by each request + << "Total inference requests:" << performance_result_.time_costs.size() << std::endl + << "Average inference time cost:" << performance_result_.total_time_cost / performance_result_.time_costs.size() * 1000 << " ms" << std::endl // Time between start and end of run. Less than Total time cost when running requests in parallel. - << "Total run time:" << duration.count() << " s" << std::endl; + << "Total inference run time:" << inference_duration.count() << " s" << std::endl; return Status::OK(); } @@ -191,8 +193,11 @@ static TestSession* CreateSession(Ort::Env& env, std::random_device& rd, } PerformanceRunner::PerformanceRunner(Ort::Env& env, const PerformanceTestConfig& test_config, std::random_device& rd) : performance_test_config_(test_config), - test_model_info_(CreateModelInfo(test_config)), - session_(CreateSession(env, rd, test_config, test_model_info_)) {} + test_model_info_(CreateModelInfo(test_config)) { + session_create_start_ = std::chrono::high_resolution_clock::now(); + session_.reset(CreateSession(env, rd, test_config, test_model_info_)); + session_create_end_ = std::chrono::high_resolution_clock::now(); +} PerformanceRunner::~PerformanceRunner() = default; diff --git a/onnxruntime/test/perftest/performance_runner.h b/onnxruntime/test/perftest/performance_runner.h index bd3d03e80475f..902422bb4af66 100644 --- a/onnxruntime/test/perftest/performance_runner.h +++ b/onnxruntime/test/perftest/performance_runner.h @@ -132,6 +132,8 @@ class PerformanceRunner { } private: + std::chrono::time_point session_create_start_; + std::chrono::time_point session_create_end_; PerformanceResult performance_result_; PerformanceTestConfig performance_test_config_; TestModelInfo* test_model_info_; diff --git a/onnxruntime/test/perftest/test_configuration.h b/onnxruntime/test/perftest/test_configuration.h index 9d0bf46774057..36186e129f0f8 100644 --- a/onnxruntime/test/perftest/test_configuration.h +++ b/onnxruntime/test/perftest/test_configuration.h @@ -47,7 +47,7 @@ struct RunConfig { ExecutionMode execution_mode{ExecutionMode::ORT_SEQUENTIAL}; int intra_op_num_threads{0}; int inter_op_num_threads{0}; - GraphOptimizationLevel optimization_level{ORT_ENABLE_EXTENDED}; + GraphOptimizationLevel optimization_level{ORT_ENABLE_ALL}; std::basic_string optimized_model_path; }; diff --git a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc index e6813e2a07875..b6e1660364c23 100644 --- a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc @@ -620,7 +620,8 @@ TEST(Loop, SubgraphInputShadowsOuterScopeValue) { auto& b_out = fetches[0].Get(); TensorShape expected_shape(scalar); - ASSERT_EQ(expected_shape, b_out.Shape()); + //Use reinterpret_cast to bypass a gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51213 + ASSERT_EQ(*reinterpret_cast*>(&expected_shape), *reinterpret_cast*>(&b_out.Shape())); ASSERT_EQ(b_out.DataAsSpan()[0], expected_value_b); auto user_defined_vals_out = fetches[1].Get().DataAsSpan(); diff --git a/onnxruntime/test/python/onnxruntime_test_python.py b/onnxruntime/test/python/onnxruntime_test_python.py index 0a0060767ebea..21b0fc8a8c457 100644 --- a/onnxruntime/test/python/onnxruntime_test_python.py +++ b/onnxruntime/test/python/onnxruntime_test_python.py @@ -521,10 +521,10 @@ def test_run_model_mlnet(self): def testGraphOptimizationLevel(self): opt = onnxrt.SessionOptions() - self.assertEqual(opt.graph_optimization_level, onnxrt.GraphOptimizationLevel.ORT_ENABLE_BASIC) - # default should be basic optimization - opt.graph_optimization_level = onnxrt.GraphOptimizationLevel.ORT_ENABLE_ALL + # default should be all optimizations optimization self.assertEqual(opt.graph_optimization_level, onnxrt.GraphOptimizationLevel.ORT_ENABLE_ALL) + opt.graph_optimization_level = onnxrt.GraphOptimizationLevel.ORT_ENABLE_EXTENDED + self.assertEqual(opt.graph_optimization_level, onnxrt.GraphOptimizationLevel.ORT_ENABLE_EXTENDED) sess = onnxrt.InferenceSession(self.get_name("logicaland.onnx"), sess_options=opt) a = np.array([[True, True], [False, False]], dtype=np.bool) b = np.array([[True, False], [True, False]], dtype=np.bool) @@ -622,7 +622,7 @@ def testLoadingSessionOptionsFromModel(self): finally: # Make sure the usage of the feature is disabled after this test - os.environ['ORT_LOAD_CONFIG_FROM_MODEL'] = str(0) + os.environ['ORT_LOAD_CONFIG_FROM_MODEL'] = str(0) if __name__ == '__main__': unittest.main() diff --git a/onnxruntime/test/python/onnxruntime_test_python_nuphar.py b/onnxruntime/test/python/onnxruntime_test_python_nuphar.py index b496a2b93e6a6..f9387ffb73c17 100644 --- a/onnxruntime/test/python/onnxruntime_test_python_nuphar.py +++ b/onnxruntime/test/python/onnxruntime_test_python_nuphar.py @@ -41,11 +41,13 @@ def test_bidaf(self): # run onnx_test_runner to verify results # use -M to disable memory pattern - # use -j 1 -c 1 to run one model/session at a time when running multiple models onnx_test_runner = os.path.join(cwd, 'onnx_test_runner') - subprocess.run([onnx_test_runner, '-e', 'nuphar', '-M', '-c', '1', '-j', '1', '-n', 'bidaf', cwd], check=True, cwd=cwd) + subprocess.run([onnx_test_runner, '-e', 'nuphar', '-M', '-n', 'bidaf', cwd], check=True, cwd=cwd) # test AOT on the quantized model + if os.name not in ['nt', 'posix']: + return # don't run the rest of test if AOT is not supported + cache_dir = os.path.join(cwd, 'nuphar_cache') if os.path.exists(cache_dir): shutil.rmtree(cache_dir) @@ -57,33 +59,28 @@ def test_bidaf(self): tp = onnx.load_tensor(os.path.join(bidaf_dir, 'test_data_set_0', 'input_{}.pb'.format(i))) feed[tp.name] = numpy_helper.to_array(tp) - # force codegen_target to be avx - nuphar_settings = 'nuphar_codegen_target:avx' - onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings) - sess = onnxrt.InferenceSession(bidaf_int8_scan_only_model) - assert 'NupharExecutionProvider' in sess.get_providers() - output = sess.run([], feed) - - nuphar_settings = 'nuphar_cache_path:{}'.format(cache_dir) - onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings) - sess = onnxrt.InferenceSession(bidaf_int8_scan_only_model) # JIT cache happens when initializing session - assert 'NupharExecutionProvider' in sess.get_providers() - output = sess.run([], feed) - - cache_dir_content = os.listdir(cache_dir) - assert len(cache_dir_content) == 1 - cache_versioned_dir = os.path.join(cache_dir, cache_dir_content[0]) - so_name = 'bidaf.so' - if os.name in ['nt', 'posix'] : # Windows or Linux + for model in [bidaf_opt_scan_model, bidaf_int8_scan_only_model]: + nuphar_settings = 'nuphar_cache_path:{}'.format(cache_dir) + for isa in ['avx', 'avx2', 'avx512']: + onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings + ', nuphar_codegen_target:' + isa) + sess = onnxrt.InferenceSession(model) # JIT cache happens when initializing session + + cache_dir_content = os.listdir(cache_dir) + assert len(cache_dir_content) == 1 + cache_versioned_dir = os.path.join(cache_dir, cache_dir_content[0]) + so_name = os.path.basename(model) + '.so' subprocess.run([sys.executable, '-m', 'onnxruntime.nuphar.create_shared', '--input_dir', cache_versioned_dir, '--output_name', so_name], check=True) - else: - return # don't run the rest of test if AOT is not supported - nuphar_settings = 'nuphar_cache_path:{}, nuphar_cache_so_name:{}, nuphar_cache_force_no_jit:{}'.format(cache_dir, so_name, 'on') - onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings) - sess = onnxrt.InferenceSession(bidaf_int8_scan_only_model) # JIT cache happens when initializing session - assert 'NupharExecutionProvider' in sess.get_providers() - sess.run([], feed) + nuphar_settings = 'nuphar_cache_path:{}, nuphar_cache_so_name:{}, nuphar_cache_force_no_jit:{}'.format(cache_dir, so_name, 'on') + onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings) + sess = onnxrt.InferenceSession(model) + sess.run([], feed) + + # test avx + nuphar_settings = 'nuphar_cache_path:{}, nuphar_cache_so_name:{}, nuphar_cache_force_no_jit:{}, nuphar_codegen_target:{}'.format(cache_dir, so_name, 'on', 'avx') + onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings) + sess = onnxrt.InferenceSession(model) + sess.run([], feed) def test_bert_squad(self): diff --git a/onnxruntime/test/server/unit_tests/converter_tests.cc b/onnxruntime/test/server/unit_tests/converter_tests.cc deleted file mode 100644 index 6387391bbf98f..0000000000000 --- a/onnxruntime/test/server/unit_tests/converter_tests.cc +++ /dev/null @@ -1,1113 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "gtest/gtest.h" -#include "gmock/gmock.h" - -#include "core/framework/tensor.h" -#include "core/graph/basic_types.h" -#include "core/framework/allocatormgr.h" -#include "test/framework/test_utils.h" -#include "test/test_environment.h" -#include "server/converter.h" -#include "server/serializing/tensorprotoutils.h" -#include -#include -#include -#include "onnx-ml.pb.h" - -namespace onnxruntime { -namespace server { -namespace test { - -void CreateMLValueBool(AllocatorPtr alloc, const std::vector& dims, const bool* value, Ort::Value& p_mlvalue); - -template -void CreateMLValue(AllocatorPtr alloc, const std::vector& dims, const std::vector& value, - Ort::Value& p_mlvalue) { - OrtValue* ml_value = new OrtValue{}; - onnxruntime::test::CreateMLValue(alloc, dims, value, ml_value); - p_mlvalue = Ort::Value{ml_value}; -} - -IExecutionProvider* TestCPUExecutionProvider() { - static CPUExecutionProviderInfo info; - static CPUExecutionProvider cpu_provider(info); - return &cpu_provider; -} - -TEST(MLDataTypeToTensorProtoDataTypeTests, MLDataTypeToTensorProtoDataTypeTests) { - onnx::TensorProto_DataType result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); - EXPECT_EQ(result, onnx::TensorProto_DataType_FLOAT); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16); - EXPECT_EQ(result, onnx::TensorProto_DataType_FLOAT16); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16); - EXPECT_EQ(result, onnx::TensorProto_DataType_BFLOAT16); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE); - EXPECT_EQ(result, onnx::TensorProto_DataType_DOUBLE); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8); - EXPECT_EQ(result, onnx::TensorProto_DataType_UINT8); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8); - EXPECT_EQ(result, onnx::TensorProto_DataType_INT8); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16); - EXPECT_EQ(result, onnx::TensorProto_DataType_UINT16); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16); - EXPECT_EQ(result, onnx::TensorProto_DataType_INT16); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32); - EXPECT_EQ(result, onnx::TensorProto_DataType_UINT32); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32); - EXPECT_EQ(result, onnx::TensorProto_DataType_INT32); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64); - EXPECT_EQ(result, onnx::TensorProto_DataType_UINT64); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64); - EXPECT_EQ(result, onnx::TensorProto_DataType_INT64); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING); - EXPECT_EQ(result, onnx::TensorProto_DataType_STRING); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL); - EXPECT_EQ(result, onnx::TensorProto_DataType_BOOL); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); - EXPECT_EQ(result, onnx::TensorProto_DataType_UNDEFINED); - - result = onnxruntime::server::MLDataTypeToTensorProtoDataType(static_cast(17)); - EXPECT_EQ(result, onnx::TensorProto_DataType_UNDEFINED); -} - -TEST(MLValueToTensorProtoTests, FloatToRaw) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_FLOAT); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(float); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_FLOAT_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, FloatToFloatData) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_FLOAT); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - EXPECT_EQ(tp.float_data().size(), 6); - auto data = tp.float_data().data(); - for (int j = 0; j < tp.float_data().size(); ++j) { - EXPECT_FLOAT_EQ(data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, Int32ToRaw) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_INT32); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(int32_t); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, Int32ToInt32Data) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_INT32); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - EXPECT_EQ(tp.int32_data().size(), 6); - auto data = tp.int32_data().data(); - for (int j = 0; j < tp.int32_data().size(); ++j) { - EXPECT_EQ(data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, UInt8ToRaw) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_UINT8); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(uint8_t); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, UInt8ToInt32Data) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_UINT8); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - auto count = tp.int32_data().size(); - EXPECT_EQ(count, 6); - for (int x = 0; x < 6; ++x) { - EXPECT_EQ(tp.int32_data()[x], values_mul_x[x]); - } -} - -TEST(MLValueToTensorProtoTests, UInt8ProtoRoundTrip) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - - onnx::TensorProto tp; - for (auto const& val : values_mul_x) { - tp.add_int32_data(val); - } - for (auto const& dim : dims_mul_x) { - tp.add_dims(dim); - } - tp.set_data_type(onnx::TensorProto_DataType_UINT8); - Ort::Value ml_value{nullptr}; - char buf[1000]; - Ort::AllocatorWithDefaultOptions allocator; - auto info = allocator.GetInfo(); - MemBuffer buffer((void*)&buf, tp.ByteSizeLong(), *info); - onnxruntime::server::TensorProtoToMLValue(tp, buffer, ml_value); - - onnx::TensorProto tp_out; - - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp_out); - - // Verify data type - EXPECT_TRUE(tp_out.has_data_type()); - EXPECT_EQ(tp_out.data_type(), onnx::TensorProto_DataType_UINT8); - - // Verify data location - EXPECT_FALSE(tp_out.has_data_location()); - - // Verify dimensions - const auto& dims = tp_out.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp_out.has_raw_data()); - - EXPECT_EQ(tp_out.int32_data_size(), tp.int32_data_size()); - auto in_data = tp.int32_data(); - auto out_data = tp_out.int32_data(); - for (auto x = 0; x < tp_out.int32_data_size(); ++x) { - EXPECT_EQ(values_mul_x[x], in_data[x]); - EXPECT_EQ(static_cast(out_data[x]), in_data[x]); - } -} - -TEST(MLValueToTensorProtoTests, Int8ToRaw) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_INT8); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(uint8_t); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, Int8ToInt32Data) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_INT8); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - auto count = tp.int32_data().size(); - EXPECT_EQ(count, 6); - auto data = tp.int32_data(); - for (int x = 0; x < 6; ++x) { - EXPECT_EQ(tp.int32_data()[x], values_mul_x[x]); - } -} - -TEST(MLValueToTensorProtoTests, UInt16ToRaw) { - std::vector dims_mul_x = {3, 3}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6, 7, 8, 9}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_UINT16); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(uint16_t); - EXPECT_EQ(count, 9); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, UInt16ToInt32Data) { - std::vector dims_mul_x = {3, 3}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6, 7, 8, 9}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_UINT16); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - auto count = tp.int32_data().size(); - EXPECT_EQ(count, 9); - auto data = tp.int32_data(); - for (int x = 0; x < 9; ++x) { - EXPECT_EQ(tp.int32_data()[x], values_mul_x[x]); - } -} - -TEST(MLValueToTensorProtoTests, Int16ToRaw) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_INT16); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(uint16_t); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, Int16ToInt32Data) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_INT16); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - auto count = tp.int32_data().size(); - EXPECT_EQ(count, 6); - auto data = tp.int32_data(); - for (int x = 0; x < 6; ++x) { - EXPECT_EQ(tp.int32_data()[x], values_mul_x[x]); - } -} - -TEST(MLValueToTensorProtoTests, BoolToRaw) { - std::vector dims_mul_x = {3, 2}; - bool values_mul_x[] = {true, false, false, true, true, false}; - Ort::Value ml_value{nullptr}; - CreateMLValueBool(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_BOOL); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(bool); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, BoolToInt32Data) { - std::vector dims_mul_x = {3, 2}; - bool values_mul_x[] = {true, false, false, true, true, false}; - Ort::Value ml_value{nullptr}; - CreateMLValueBool(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_BOOL); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - auto count = tp.int32_data().size(); - EXPECT_EQ(count, 6); - auto data = tp.int32_data(); - for (int x = 0; x < 6; ++x) { - EXPECT_EQ(tp.int32_data()[x], values_mul_x[x]); - } -} - -TEST(MLValueToTensorProtoTests, Float16ToRaw) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x{ - onnxruntime::MLFloat16(1), - onnxruntime::MLFloat16(2), - onnxruntime::MLFloat16(3), - onnxruntime::MLFloat16(4), - onnxruntime::MLFloat16(5), - onnxruntime::MLFloat16(6)}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_FLOAT16); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(onnxruntime::MLFloat16); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, FloatToInt32Data) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x{ - onnxruntime::MLFloat16(1), - onnxruntime::MLFloat16(2), - onnxruntime::MLFloat16(3), - onnxruntime::MLFloat16(4), - onnxruntime::MLFloat16(5), - onnxruntime::MLFloat16(6)}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_FLOAT16); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - auto count = tp.int32_data().size(); - EXPECT_EQ(count, 6); - auto data = tp.int32_data().data(); - for (int x = 0; x < 6; ++x) { - const u_int16_t data16 = data[x]; - const auto data_float_16 = static_cast(data16); - EXPECT_EQ(data_float_16, values_mul_x[x]); - } -} - -TEST(MLValueToTensorProtoTests, BFloat16ToRaw) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x{ - onnxruntime::BFloat16(1.0f), - onnxruntime::BFloat16(2.0f), - onnxruntime::BFloat16(3.0f), - onnxruntime::BFloat16(4.0f), - onnxruntime::BFloat16(5.0f), - onnxruntime::BFloat16(6.0f)}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_BFLOAT16); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(uint16_t); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_EQ(tensor_data[j], values_mul_x[j].val); - } -} - -TEST(MLValueToTensorProtoTests, BFloatToInt32Data) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x{ - onnxruntime::BFloat16(1.0f), - onnxruntime::BFloat16(2.0f), - onnxruntime::BFloat16(3.0f), - onnxruntime::BFloat16(4.0f), - onnxruntime::BFloat16(5.0f), - onnxruntime::BFloat16(6.0f)}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_BFLOAT16); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - auto count = tp.int32_data().size(); - EXPECT_EQ(count, 6); - auto data = tp.int32_data().data(); - for (int x = 0; x < 6; ++x) { - const u_int16_t data16 = data[x]; - const auto data_float_16 = static_cast(data16); - EXPECT_EQ(data_float_16, values_mul_x[x]); - } -} - -TEST(MLValueToTensorProtoTests, StringToStringData) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x{"A", "BC", "DEF", "123", "45", "6"}; - OrtValue* p_mlValue = new OrtValue{}; - onnxruntime::test::AllocateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, p_mlValue); - - Tensor* mutable_tensor = p_mlValue->GetMutable(); - std::string* mutable_data = mutable_tensor->MutableData(); - for (size_t i = 0; i < values_mul_x.size(); ++i) { - mutable_data[i] = values_mul_x[i]; - } - Ort::Value value{p_mlValue}; - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_STRING); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - auto count = tp.string_data().size(); - EXPECT_EQ(count, 6); - const auto* data = tp.string_data().data(); - for (int x = 0; x < 6; ++x) { - EXPECT_STREQ(data[x]->c_str(), values_mul_x[x].c_str()); - } -} - -TEST(MLValueToTensorProtoTests, Int64ToRaw) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_INT64); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(int64_t); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, Int64ToInt64Data) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_INT64); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - EXPECT_EQ(tp.int64_data().size(), 6); - auto data = tp.int64_data().data(); - for (int j = 0; j < tp.int64_data().size(); ++j) { - EXPECT_EQ(data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, UInt32ToRaw) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_UINT32); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(uint32_t); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - auto* tensor_data = (uint32_t*)raw; - for (size_t j = 0; j < count; ++j) { - EXPECT_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, UInt32ToUint64Data) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_UINT32); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - auto count = tp.uint64_data().size(); - EXPECT_EQ(count, 6); - - auto data = tp.uint64_data().data(); - for (size_t x = 0; x < count; ++x) { - EXPECT_EQ(data[x], values_mul_x[x]); - } -} - -TEST(MLValueToTensorProtoTests, UInt64ToRaw) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_UINT64); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(uint64_t); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, UInt64ToInt64Data) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_UINT64); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - EXPECT_EQ(tp.uint64_data().size(), 6); - auto data = tp.uint64_data().data(); - for (int j = 0; j < tp.uint64_data().size(); ++j) { - EXPECT_EQ(data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, DoubleToRaw) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ true, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_DOUBLE); - - // Verify data location - EXPECT_TRUE(tp.has_data_location()); - EXPECT_EQ(tp.data_location(), onnx::TensorProto_DataLocation_DEFAULT); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_TRUE(tp.has_raw_data()); - auto count = tp.raw_data().size() / sizeof(uint64_t); - EXPECT_EQ(count, 6); - - auto raw = tp.raw_data().data(); - const auto* tensor_data = reinterpret_cast(raw); - for (size_t j = 0; j < count; ++j) { - EXPECT_DOUBLE_EQ(tensor_data[j], values_mul_x[j]); - } -} - -TEST(MLValueToTensorProtoTests, DoubleToInt64Data) { - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; - Ort::Value ml_value{nullptr}; - onnxruntime::server::test::CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, ml_value); - - onnx::TensorProto tp; - onnxruntime::server::MLValueToTensorProto(ml_value, /* using_raw_data */ false, spdlog::default_logger(), tp); - - // Verify data type - EXPECT_TRUE(tp.has_data_type()); - EXPECT_EQ(tp.data_type(), onnx::TensorProto_DataType_DOUBLE); - - // Verify data location - EXPECT_FALSE(tp.has_data_location()); - - // Verify dimensions - const auto& dims = tp.dims(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - EXPECT_EQ(dims[i], dims_mul_x[i]); - } - - // Verify data - EXPECT_FALSE(tp.has_raw_data()); - EXPECT_EQ(tp.double_data().size(), 6); - auto data = tp.double_data().data(); - for (int j = 0; j < tp.double_data().size(); ++j) { - EXPECT_DOUBLE_EQ(data[j], values_mul_x[j]); - } -} - -void CreateMLValueBool(AllocatorPtr alloc, const std::vector& dims, const bool* value, Ort::Value& p_value) { - TensorShape shape(dims); - OrtValue* p_mlvalue = new OrtValue{}; - auto element_type = DataTypeImpl::GetType(); - std::unique_ptr p_tensor = onnxruntime::make_unique(element_type, - shape, - alloc); - memcpy(p_tensor->MutableData(), &value[0], element_type->Size() * shape.Size()); - p_mlvalue->Init(p_tensor.release(), - DataTypeImpl::GetType(), - DataTypeImpl::GetType()->GetDeleteFunc()); - p_value = Ort::Value{p_mlvalue}; -} - -} // namespace test -} // namespace server -} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/testdata/transform/gemm_activation_fusion/gemm_activation_fusion.onnx b/onnxruntime/test/testdata/transform/gemm_activation_fusion/gemm_activation_fusion.onnx new file mode 100644 index 0000000000000..87dd469343005 --- /dev/null +++ b/onnxruntime/test/testdata/transform/gemm_activation_fusion/gemm_activation_fusion.onnx @@ -0,0 +1,24 @@ + onnx-helper: + +a +b +cy"Gemm + +yz" LeakyRelugemm_activation_fusionZ +a +  + +Z +b +  + +Z +c +  + +b +z +  + +B +ai.onnx \ No newline at end of file diff --git a/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/input_0.pb b/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/input_0.pb new file mode 100644 index 0000000000000..ee690bab01f44 --- /dev/null +++ b/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/input_0.pb @@ -0,0 +1 @@ +BaJ<  ?7?N?w} ?H>QY%?n >~J?e?^k?l?Z{= \ No newline at end of file diff --git a/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/input_1.pb b/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/input_1.pb new file mode 100644 index 0000000000000..6945e22c76d0a --- /dev/null +++ b/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/input_1.pb @@ -0,0 +1 @@ +BbJPp= <&U?H5G?^?z?L?G>G?9=#?4>q?ڗ?N>s>.4F?>?< \ No newline at end of file diff --git a/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/input_2.pb b/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/input_2.pb new file mode 100644 index 0000000000000..7077e4ffe336e Binary files /dev/null and b/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/input_2.pb differ diff --git a/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/output_0.pb b/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/output_0.pb new file mode 100644 index 0000000000000..f53ed3aaf25e2 --- /dev/null +++ b/onnxruntime/test/testdata/transform/gemm_activation_fusion/test_data_set_0/output_0.pb @@ -0,0 +1 @@ +BzJ0c?C?%?~?@D?\@?p?ߎ? N? ? \ No newline at end of file diff --git a/onnxruntime/test/tvm/tvm_basic_test.cc b/onnxruntime/test/tvm/tvm_basic_test.cc index 0b9d338d0d6e7..70399f2f03679 100644 --- a/onnxruntime/test/tvm/tvm_basic_test.cc +++ b/onnxruntime/test/tvm/tvm_basic_test.cc @@ -150,7 +150,7 @@ class FuseExecutionProviderX : public CPUExecutionProvider { onnxruntime::make_unique(std::move(sub_graph))); } } - return std::move(result); + return result; } common::Status Compile(const std::vector& fused_nodes, diff --git a/onnxruntime/test/tvm/tvm_demo/demo_compiler.cc b/onnxruntime/test/tvm/tvm_demo/demo_compiler.cc index 886ecc9e1af31..15ca50f66664a 100644 --- a/onnxruntime/test/tvm/tvm_demo/demo_compiler.cc +++ b/onnxruntime/test/tvm/tvm_demo/demo_compiler.cc @@ -110,7 +110,7 @@ bool TVM_SCHEDULER_CLASS(AlwaysInline, DemoTVM)::Evaluate( // Register the always inline Scheduler to sched_registry static void RegisterAlwaysInlineScheduler(tvm_codegen::TVMScheduleRegistry* sched_registry) { sched_registry->Register( - std::move(onnxruntime::make_unique())); + onnxruntime::make_unique()); } // Declare a schedule dispatcher that always dispatches the always inline Scheduler diff --git a/onnxruntime/test/util/compare_ortvalue.cc b/onnxruntime/test/util/compare_ortvalue.cc index cfeaa11363185..958597997733c 100644 --- a/onnxruntime/test/util/compare_ortvalue.cc +++ b/onnxruntime/test/util/compare_ortvalue.cc @@ -5,11 +5,39 @@ #include #include +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-qualifiers" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#else +#pragma warning(push) +#pragma warning(disable : 4018) /*'expression' : signed/unsigned mismatch */ +#pragma warning(disable : 4065) /*switch statement contains 'default' but no 'case' labels*/ +#pragma warning(disable : 4100) +#pragma warning(disable : 4146) /*unary minus operator applied to unsigned type, result still unsigned*/ +#pragma warning(disable : 4244) /*'conversion' conversion from 'type1' to 'type2', possible loss of data*/ +#pragma warning(disable : 4251) /*'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2'*/ +#pragma warning(disable : 4267) /*'var' : conversion from 'size_t' to 'type', possible loss of data*/ +#pragma warning(disable : 4305) /*'identifier' : truncation from 'type1' to 'type2'*/ +#pragma warning(disable : 4307) /*'operator' : integral constant overflow*/ +#pragma warning(disable : 4309) /*'conversion' : truncation of constant value*/ +#pragma warning(disable : 4334) /*'operator' : result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?)*/ +#pragma warning(disable : 4355) /*'this' : used in base member initializer list*/ +#pragma warning(disable : 4506) /*no definition for inline function 'function'*/ +#pragma warning(disable : 4800) /*'type' : forcing value to bool 'true' or 'false' (performance warning)*/ +#pragma warning(disable : 4996) /*The compiler encountered a deprecated declaration.*/ +#endif #ifdef USE_FULL_PROTOBUF #include #else #include #endif +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#else +#pragma warning(pop) +#endif + #include "core/graph/onnx_protobuf.h" #include "core/framework/tensorprotoutils.h" diff --git a/samples/README.md b/samples/README.md index 7b399953ff83e..50cada3d13a56 100644 --- a/samples/README.md +++ b/samples/README.md @@ -18,8 +18,8 @@ For a list of available dockerfiles and published images to help with getting st * [Model Inferencing using NUPHAR Execution Provider](../docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb) **Inference with model conversion** -* [SKL Pipeline: Train, Convert, and Inference](https://microsoft.github.io/onnxruntime/tutorial.html) -* [Keras: Convert and Inference](https://microsoft.github.io/onnxruntime/auto_examples/plot_dl_keras.html#sphx-glr-auto-examples-plot-dl-keras-py) +* [SKL Pipeline: Train, Convert, and Inference](https://microsoft.github.io/onnxruntime/python/tutorial.html) +* [Keras: Convert and Inference](https://microsoft.github.io/onnxruntime/python/auto_examples/plot_dl_keras.html#sphx-glr-auto-examples-plot-dl-keras-py) **Inference and deploy through AzureML** * Inferencing on CPU using [ONNX Model Zoo](https://github.com/onnx/models) models: @@ -43,7 +43,7 @@ For a list of available dockerfiles and published images to help with getting st **Other** * [Running ONNX model tests](./docs/Model_Test.md) -* [Common Errors with explanations](https://microsoft.github.io/onnxruntime/auto_examples/plot_common_errors.html#sphx-glr-auto-examples-plot-common-errors-py) +* [Common Errors with explanations](https://microsoft.github.io/onnxruntime/python/auto_examples/plot_common_errors.html#sphx-glr-auto-examples-plot-common-errors-py) ## C# * [Inferencing Tutorial](../docs/CSharp_API.md#getting-started) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt new file mode 100755 index 0000000000000..efefb84aec691 --- /dev/null +++ b/server/CMakeLists.txt @@ -0,0 +1,258 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Minimum CMake required +cmake_minimum_required(VERSION 3.5) +# Project +project(onnxruntime C CXX) +set(CMAKE_CXX_STANDARD 14) + +find_package(Threads) +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) +include(protobuf_function.cmake) +find_package(gRPC REQUIRED) + +set(SERVER_APP_NAME "onnxruntime_server") +set(onnxruntime_USE_FULL_PROTOBUF ON) + +set(ONNXRUNTIME_SERVER_ROOT ${PROJECT_SOURCE_DIR}) + +# Generate .h and .cc files from protobuf file +add_library(server_proto ${ONNXRUNTIME_SERVER_ROOT}/protobuf/predict.proto ${ONNXRUNTIME_SERVER_ROOT}/protobuf/onnx-ml.proto) +if(WIN32) + target_compile_options(server_proto PRIVATE "/wd4125" "/wd4456") +endif() +target_include_directories(server_proto PUBLIC $ "${CMAKE_CURRENT_BINARY_DIR}/.." ${CMAKE_CURRENT_BINARY_DIR}/onnx) +target_compile_definitions(server_proto PUBLIC $) +onnxruntime_protobuf_generate(APPEND_PATH ${ONNXRUNTIME_SERVER_ROOT}/protobuf ${ONNXRUNTIME_ROOT}/core/protobuf TARGET server_proto) +if(NOT WIN32) + if(HAS_UNUSED_PARAMETER) + set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/predict.pb.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/onnx-ml.pb.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + endif() +endif() + +# Setup dependencies +include(get_boost.cmake) +set(SPDLOG_BUILD_EXAMPLES OFF) +add_subdirectory(external/spdlog) + +# Generate GRPC service source and headers. +get_filename_component(grpc_proto "${ONNXRUNTIME_SERVER_ROOT}/protobuf/prediction_service.proto" ABSOLUTE) +get_filename_component(grpc_proto_path "${grpc_proto}" PATH) + +set(grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/prediction_service.grpc.pb.cc") +set(grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/prediction_service.grpc.pb.h") + +if(NOT _gRPC_PYTHON_PLUGIN_EXECUTABLE) + find_program(_gRPC_PYTHON_PLUGIN_EXECUTABLE grpc_python_plugin DOC "The gRPC Python plugin for protoc") +endif() + +add_custom_command( + OUTPUT "${grpc_srcs}" "${grpc_hdrs}" + COMMAND protoc + ARGS + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" + --plugin=protoc-gen-grpc="${_gRPC_CPP_PLUGIN_EXECUTABLE}" + -I ${grpc_proto_path} + "${grpc_proto}" + DEPENDS "${grpc_proto}" + COMMENT "Running ${_gRPC_CPP_PLUGIN_EXECUTABLE} on ${grpc_proto}" + ) + +add_library(server_grpc_proto ${grpc_srcs}) +target_include_directories(server_grpc_proto PUBLIC $ "${CMAKE_CURRENT_BINARY_DIR}" ${CMAKE_CURRENT_BINARY_DIR}/onnx PRIVATE) +set(grpc_reflection -Wl,--whole-archive grpc++_reflection -Wl,--no-whole-archive) +set(grpc_static_libs gRPC::grpc++ grpcpp_channelz) + +add_dependencies(server_grpc_proto server_proto Boost) +# Include generated *.pb.h files +include_directories("${CMAKE_CURRENT_BINARY_DIR}") + +if(NOT WIN32) + if(HAS_UNUSED_PARAMETER) + set_source_files_properties(${grpc_srcs} PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + set_source_files_properties(${onnxruntime_server_grpc_srcs} PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + endif() +endif() + + +# Setup source code +set(onnxruntime_server_lib_srcs + "${ONNXRUNTIME_SERVER_ROOT}/http/json_handling.cc" + "${ONNXRUNTIME_SERVER_ROOT}/http/predict_request_handler.cc" + "${ONNXRUNTIME_SERVER_ROOT}/http/util.cc" + "${ONNXRUNTIME_SERVER_ROOT}/environment.cc" + "${ONNXRUNTIME_SERVER_ROOT}/executor.cc" + "${ONNXRUNTIME_SERVER_ROOT}/converter.cc" + "${ONNXRUNTIME_SERVER_ROOT}/util.cc" + "${ONNXRUNTIME_SERVER_ROOT}/core/request_id.cc" + "${ONNXRUNTIME_SERVER_ROOT}/grpc/prediction_service_impl.cc" + "${ONNXRUNTIME_SERVER_ROOT}/grpc/grpc_app.cc" + "${ONNXRUNTIME_SERVER_ROOT}/serializing/tensorprotoutils.cc" + ) +if(NOT WIN32) + if(HAS_UNUSED_PARAMETER) + set_source_files_properties(${onnxruntime_server_lib_srcs} PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + endif() +endif() + +file(GLOB_RECURSE onnxruntime_server_http_core_lib_srcs + "${ONNXRUNTIME_SERVER_ROOT}/http/core/*.cc" + ) + +file(GLOB_RECURSE onnxruntime_server_srcs + "${ONNXRUNTIME_SERVER_ROOT}/main.cc" +) + + + + +# HTTP core library +add_library(onnxruntime_server_http_core_lib STATIC + ${onnxruntime_server_http_core_lib_srcs}) +target_include_directories(onnxruntime_server_http_core_lib + PUBLIC + ${ONNXRUNTIME_ROOT} + ${ONNXRUNTIME_SERVER_ROOT}/http/core + ${ONNXRUNTIME_SERVER_ROOT}/core + ${Boost_INCLUDE_DIR} + +) +add_dependencies(onnxruntime_server_http_core_lib Boost) + +# Server library +add_library(onnxruntime_server_lib ${onnxruntime_server_lib_srcs}) +target_include_directories(onnxruntime_server_lib PRIVATE + ${ONNXRUNTIME_INCLUDE_DIR} + ${ONNXRUNTIME_ROOT} + ${ONNXRUNTIME_SERVER_ROOT} + ${ONNXRUNTIME_SERVER_ROOT}/http + ${ONNXRUNTIME_SERVER_ROOT}/logging + ${ONNXRUNTIME_SERVER_ROOT}/core + PUBLIC + ${ONNXRUNTIME_SERVER_ROOT}/external/spdlog/include + ${ONNXRUNTIME_SERVER_ROOT}/http/core + ${ONNXRUNTIME_SERVER_ROOT} + ${Boost_INCLUDE_DIR} +) + + +if (onnxruntime_USE_SYSLOG) + target_compile_definitions(onnxruntime_server_lib PUBLIC USE_SYSLOG="1") +endif() +add_dependencies(onnxruntime_server_lib server_proto Boost) + +# Server Application +add_executable(${SERVER_APP_NAME} ${onnxruntime_server_srcs}) +add_dependencies(${SERVER_APP_NAME} server_proto Boost) + +if(NOT WIN32) + if(HAS_UNUSED_PARAMETER) + set_source_files_properties("${ONNXRUNTIME_SERVER_ROOT}/main.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + endif() +endif() + +set(onnxruntime_SERVER_VERSION "local-build" CACHE STRING "Sever version") +target_compile_definitions(${SERVER_APP_NAME} PUBLIC SRV_VERSION="${onnxruntime_SERVER_VERSION}") +message(STATUS "ONNX Runtime Server version set to: ${onnxruntime_SERVER_VERSION}") + +set(onnxruntime_LATEST_COMMIT_ID "default" CACHE STRING "The latest commit id") +target_compile_definitions(${SERVER_APP_NAME} PUBLIC LATEST_COMMIT_ID="${onnxruntime_LATEST_COMMIT_ID}") +message(STATUS "ONNX Runtime Server latest commit id is: ${onnxruntime_LATEST_COMMIT_ID}") + +target_include_directories(${SERVER_APP_NAME} PRIVATE + ${ONNXRUNTIME_INCLUDE_DIR} + ${ONNXRUNTIME_SERVER_ROOT}/http +) + + +target_link_libraries(${SERVER_APP_NAME} PRIVATE + onnxruntime_server_http_core_lib + onnxruntime_server_lib + ${grpc_reflection} #Note that this will break the tests if we try to link it to the lib so just link to the executable. + server_grpc_proto server_proto + ${Boost_LIBRARIES} + onnxruntime_server_http_core_lib + ${grpc_static_libs} protobuf::libprotobuf + spdlog::spdlog + onnxruntime gtest re2 ${CMAKE_DL_LIBS} Threads::Threads +) + + + + +file(GLOB onnxruntime_test_server_src + "test/unit_tests/*.cc" + "test/unit_tests/*.h" + ) + + file(GLOB onnxruntime_integration_test_server_src + "test/integration_tests/*.py" + ) + if(NOT WIN32) + if(HAS_UNUSED_PARAMETER) + set_source_files_properties("test/unit_tests/json_handling_tests.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + set_source_files_properties("test/unit_tests/converter_tests.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + set_source_files_properties("test/unit_tests/util_tests.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + set_source_files_properties("test/unit_tests/prediction_service_impl_test.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + set_source_files_properties("test/unit_tests/executor_test.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + endif() + endif() + + add_executable(onnxruntime_server_tests ${onnxruntime_test_server_src}) + add_dependencies(onnxruntime_server_tests server_proto Boost) + target_link_libraries(onnxruntime_server_tests PRIVATE onnxruntime_server_http_core_lib + onnxruntime_server_lib + ${grpc_reflection} #Note that this will break the tests if we try to link it to the lib so just link to the executable. + server_grpc_proto server_proto + ${Boost_LIBRARIES} + onnxruntime_server_http_core_lib + ${grpc_static_libs} protobuf::libprotobuf + spdlog::spdlog + onnxruntime gtest re2 gtest ${CMAKE_DL_LIBS} Threads::Threads) + target_include_directories(onnxruntime_server_tests PRIVATE ${ONNXRUNTIME_SERVER_ROOT}/external/spdlog/include + ${ONNXRUNTIME_SERVER_ROOT}/http/core + ${ONNXRUNTIME_SERVER_ROOT} ${ONNXRUNTIME_SERVER_ROOT}/core) + onnxruntime_protobuf_generate( + APPEND_PATH IMPORT_DIRS ${ONNXRUNTIME_SERVER_ROOT}/protobuf + PROTOS ${ONNXRUNTIME_SERVER_ROOT}/protobuf/predict.proto protobuf/onnx-ml.proto + LANGUAGE python + TARGET onnxruntime_server_tests + OUT_VAR server_test_py) + + set(grpc_py "${CMAKE_CURRENT_BINARY_DIR}/prediction_service_pb2_grpc.py") + + add_custom_command( + TARGET onnxruntime_server_tests + COMMAND protoc + ARGS + --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" + --plugin=protoc-gen-grpc=${_gRPC_PYTHON_PLUGIN_EXECUTABLE} + -I ${grpc_proto_path} + "${grpc_proto}" + DEPENDS "${grpc_proto}" + COMMENT "Running ${_gRPC_PYTHON_PLUGIN_EXECUTABLE} on ${grpc_proto}" + ) + + add_custom_command( + TARGET onnxruntime_server_tests POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/server_test + COMMAND ${CMAKE_COMMAND} -E copy + ${onnxruntime_integration_test_server_src} + ${CMAKE_CURRENT_BINARY_DIR}/server_test/ + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/onnx_ml_pb2.py + ${CMAKE_CURRENT_BINARY_DIR}/server_test/ + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/predict_pb2.py + ${CMAKE_CURRENT_BINARY_DIR}/server_test/ + COMMAND ${CMAKE_COMMAND} -E copy + ${grpc_py} + ${CMAKE_CURRENT_BINARY_DIR}/server_test/) + + add_test(NAME onnxruntime_server_tests + COMMAND onnxruntime_server_tests + WORKING_DIRECTORY ${ONNXRUNTIME_SERVER_ROOT}/test/testdata> + ) diff --git a/server/ci/run.sh b/server/ci/run.sh new file mode 100755 index 0000000000000..62ef99b8e4a01 --- /dev/null +++ b/server/ci/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +cd /build +cmake -DCMAKE_BUILD_TYPE=Debug /onnxruntime_src +make -j$(getconf _NPROCESSORS_ONLN) +cd /onnxruntime_src/test +/build/onnxruntime_server_tests +cd /build +pip3 install grpcio requests protobuf +python3 server_test/test_main.py /build/onnxruntime_server /build/models/opset8/test_mnist /onnxruntime_src/test/testdata/server /build /build/server_test \ No newline at end of file diff --git a/server/cmake/FindProtobufWithTargets.cmake b/server/cmake/FindProtobufWithTargets.cmake new file mode 100644 index 0000000000000..6ae90023a04b6 --- /dev/null +++ b/server/cmake/FindProtobufWithTargets.cmake @@ -0,0 +1,203 @@ +# ~~~ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ~~~ + +#[=======================================================================[.rst: +FindProtobufWithTargets +------------------- + +A module to use ``Protobuf`` with less complications. + +Using ``find_package(Protobuf)`` should be simple, but it is not. + +CMake provides a ``FindProtobuf`` module. Unfortunately it does not generate +``protobuf::*`` targets until CMake-3.9, and ``protobuf::protoc`` does not +appear until CMake-3.10. + +The CMake-config files generated by ``protobuf`` always create these targets, +but on some Linux distributions (e.g. Fedora>=29, and openSUSE-Tumbleweed) there +are system packages for protobuf, but these packages are installed without the +CMake-config files. One must either use the ``FindProtobuf`` module, find the +libraries via ``pkg-config``, or find the libraries manually. + +When the CMake-config files are installed they produce the same targets as +recent versions of ``FindProtobuf``. However, they do not produce the +``Protobuf_LIBRARY``, ``Protobuf_INCLUDE_DIR``, etc. that are generated by the +module. Furthermore, the ``protobuf::protoc`` library is not usable when loaded +from the CMake-config files: its ``IMPORTED_LOCATION`` variable is not defined. + +This module is designed to provide a single, uniform, ``find_package()`` +module that always produces the same outputs: + +- It always generates the ``protobuf::*`` targets. +- It always defines ``ProtobufWithTargets_FOUND`` and + ``ProtobufWithTargets_VERSION``. +- It *prefers* using the CMake config files if they are available. +- It fallsback on the ``FindProtobuf`` module if the config files are not found. +- It populates any missing targets and their properties. + +The following ``IMPORTED`` targets are defined: + +``protobuf::libprotobuf`` + The protobuf library. +``protobuf::libprotobuf-lite`` + The protobuf lite library. +``protobuf::libprotoc`` + The protoc library. +``protobuf::protoc`` + The protoc compiler. + +Example: + +.. code-block:: cmake + + find_package(ProtobufWithTargets REQUIRED) + add_executable(bar bar.cc) + target_link_libraries(bar PRIVATE protobuf::libprotobuf) + +#]=======================================================================] + +if (protobuf_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "protobuf_USE_STATIC_LIBS = ${protobuf_USE_STATIC_LIBS}" + " ProtobufWithTargets = ${ProtobufWithTargets_FOUND}") +endif () + +# Always load thread support, even on Windows. +find_package(Threads REQUIRED) + +# First try to use the ``protobufConfig.cmake`` or ``protobuf-config.cmake`` +# file if it was installed. This is common on systems (or package managers) +# where protobuf was compiled and installed with `CMake`. Note that on Linux +# this *must* be all lowercase ``protobuf``, while on Windows it does not +# matter. +find_package(protobuf CONFIG QUIET) + +if (protobuf_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "protobuf_FOUND = ${protobuf_FOUND}" + " protobuf_VERSION = ${protobuf_VERSION}") +endif () + +if (NOT protobuf_FOUND) + find_package(Protobuf QUIET) +endif () + +if (protobuf_FOUND) + set(ProtobufWithTargets_FOUND 1) + set(ProtobufWithTargets_VERSION ${protobuf_VERSION}) + + if (NOT TARGET protobuf::libprotobuf) + add_library(protobuf::libprotobuf INTERFACE IMPORTED) + set_property( + TARGET protobuf::libprotobuf PROPERTY INTERFACE_INCLUDE_DIRECTORIES + ${Protobuf_INCLUDE_DIR}) + set_property( + TARGET protobuf::libprotobuf + APPEND + PROPERTY INTERFACE_LINK_LIBRARIES ${Protobuf_LIBRARY} + Threads::Threads) + endif () + + if (NOT TARGET protobuf::libprotobuf-lite) + add_library(protobuf::libprotobuf-lite INTERFACE IMPORTED) + set_property( + TARGET protobuf::libprotobuf-lite + PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${Protobuf_INCLUDE_DIR}) + set_property( + TARGET protobuf::libprotobuf-lite + APPEND + PROPERTY INTERFACE_LINK_LIBRARIES ${Protobuf_LITE_LIBRARY} + Threads::Threads) + endif () + + if (NOT TARGET protobuf::libprotoc) + add_library(protobuf::libprotoc INTERFACE IMPORTED) + set_property( + TARGET protobuf::libprotoc PROPERTY INTERFACE_INCLUDE_DIRECTORIES + ${Protobuf_INCLUDE_DIR}) + set_property( + TARGET protobuf::libprotoc + APPEND + PROPERTY INTERFACE_LINK_LIBRARIES ${Protobuf_PROTOC_LIBRARY} + Threads::Threads) + endif () + + if (NOT TARGET protobuf::protoc) + add_executable(protobuf::protoc IMPORTED) + + # Discover the protoc compiler location. + find_program( + _protobuf_PROTOC_EXECUTABLE + NAMES protoc + DOC "The Google Protocol Buffers Compiler") + if (protobuf_DEBUG) + message( + STATUS + "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "ProtobufWithTargets_FOUND = ${ProtobufWithTargets_FOUND}" + " ProtobufWithTargets_VERSION = ${ProtobufWithTargets_VERSION}" + " EXE = ${_protobuf_PROTOC_EXECUTABLE}") + endif () + set_property(TARGET protobuf::protoc + PROPERTY IMPORTED_LOCATION ${_protobuf_PROTOC_EXECUTABLE}) + set_property( + TARGET protobuf::protoc PROPERTY IMPORTED_LOCATION_DEBUG + ${_protobuf_PROTOC_EXECUTABLE}) + set_property( + TARGET protobuf::protoc PROPERTY IMPORTED_LOCATION_RELEASE + ${_protobuf_PROTOC_EXECUTABLE}) + unset(_protobuf_PROTOC_EXECUTABLE) + + if (protobuf_DEBUG) + get_target_property(_protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION) + message( + STATUS + "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "LOCATION=${_protobuf_PROTOC_EXECUTABLE}") + endif () + endif () +endif () + +if (protobuf_DEBUG) + message( + STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "ProtobufWithTargets_FOUND = ${ProtobufWithTargets_FOUND}" + " ProtobufWithTargets_VERSION = ${ProtobufWithTargets_VERSION}") +endif () + +if (protobuf_DEBUG) + message( + STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "ProtobufWithTargets_FOUND = ${ProtobufWithTargets_FOUND}" + " ProtobufWithTargets_VERSION = ${ProtobufWithTargets_VERSION}") + if (ProtobufWithTargets_FOUND) + foreach (_target protobuf::libprotobuf protobuf::libprotobuf-lite + protobuf::libprotoc) + if (NOT TARGET ${_target}) + message( + STATUS + "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "target=${_target} is NOT a target") + endif () + endforeach () + unset(_target) + endif () +endif () + +find_package_handle_standard_args( + ProtobufWithTargets REQUIRED_VARS ProtobufWithTargets_FOUND + ProtobufWithTargets_VERSION) diff --git a/server/cmake/FindgRPC.cmake b/server/cmake/FindgRPC.cmake new file mode 100644 index 0000000000000..e6686596a74ca --- /dev/null +++ b/server/cmake/FindgRPC.cmake @@ -0,0 +1,353 @@ +# ~~~ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ~~~ + +#[=======================================================================[.rst: +FindgRPC +-------- + +Locate and configure the ``gRPC`` library. + +The following variables can be set and are optional: + +``gRPC_DEBUG`` + Show debug messages. +``gRPC_USE_STATIC_LIBS`` + Set to ON to force the use of the static libraries. + Default is OFF. + +Defines the following variables: + +``gRPC_FOUND`` + Found the gRPC library +``gRPC_VERSION`` + Version of package found. + +The following ``IMPORTED`` targets are also defined: + +``gRPC::grpc++`` + The gRPC C++ library. +``gRPC::grpc`` + The gRPC C core library. +``gRPC::cpp_plugin`` + The C++ plugin for the Protobuf protoc compiler. + +The following cache variables are also available to set or use: + +Example: + +.. code-block:: cmake + + find_package(gRPC REQUIRED) + add_executable(bar bar.cc) + target_link_libraries(bar PRIVATE gRPC::grpc++) + +#]=======================================================================] + +if (gRPC_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "gRPC_USE_STATIC_LIBS = ${gRPC_USE_STATIC_LIBS}" + " gRPC_FOUND = ${gRPC_FOUND}") +endif () + +# gRPC always requires Thread support. +find_package(Threads REQUIRED) + +# Load the module to find protobuf with proper targets. Do not use +# `find_package()` because we (have to) install this module in non-standard +# locations. +include(${CMAKE_CURRENT_LIST_DIR}/FindProtobufWithTargets.cmake) + +# The gRPC::grpc_cpp_plugin target is sometimes defined, but without a +# IMPORTED_LOCATION +function (_grpc_fix_grpc_cpp_plugin_target) + # The target may already exist, do not create it again if it does. + if (NOT TARGET gRPC::grpc_cpp_plugin) + add_executable(gRPC::grpc_cpp_plugin IMPORTED) + endif () + get_target_property(_gRPC_CPP_PLUGIN_EXECUTABLE gRPC::grpc_cpp_plugin + IMPORTED_LOCATION) + if (gRPC_DEBUG) + message( + STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "LOCATION=${_gRPC_CPP_PLUGIN_EXECUTABLE}") + endif () + # Even if the target exists, gRPC CMake support files do not define the + # executable for the imported target (at least they do not in v1.19.1), so + # we need to define it ourselves. + if (NOT _gRPC_CPP_PLUGIN_EXECUTABLE) + find_program(_gRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin + DOC "The gRPC C++ plugin for protoc") + mark_as_advanced(_gRPC_CPP_PLUGIN_EXECUTABLE) + if (_gRPC_CPP_PLUGIN_EXECUTABLE) + set_property( + TARGET gRPC::grpc_cpp_plugin + PROPERTY IMPORTED_LOCATION ${_gRPC_CPP_PLUGIN_EXECUTABLE}) + else () + set(gRPC_FOUND "grpc_cpp_plugin-NOTFOUND") + endif () + endif () +endfunction () + +# The gRPC::* targets sometimes lack the right definitions to compile cleanly on +# WIN32 +function (_grpc_fix_grpc_target_definitions) + # Including gRPC headers without this definition results in a build error. + if (WIN32) + set_property( + TARGET gRPC::grpc + APPEND + PROPERTY INTERFACE_COMPILE_DEFINITIONS _WIN32_WINNT=0x600) + set_property( + TARGET gRPC::grpc++ + APPEND + PROPERTY INTERFACE_COMPILE_DEFINITIONS _WIN32_WINNT=0x600) + endif () +endfunction () + +# First try to use the `gRPCConfig.cmake` or `grpc-config.cmake` file if it was +# installed. This is common on systems (or package managers) where gRPC was +# compiled and installed with `CMake`. +find_package(gRPC NO_MODULE QUIET) + +if (gRPC_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "NO_MODULE result gRPC_FOUND = ${gRPC_FOUND}") +endif () + +if (gRPC_FOUND) + _grpc_fix_grpc_cpp_plugin_target() + _grpc_fix_grpc_target_definitions() + return() +endif () + +include(SelectLibraryConfigurations) + +# Internal function: search for normal library as well as a debug one if the +# debug one is specified also include debug/optimized keywords in *_LIBRARIES +# variable +function (_gRPC_find_library name filename) + if (${name}_LIBRARY) + # Use result recorded by a previous call. + return() + else () + find_library(${name}_LIBRARY_RELEASE NAMES ${filename}) + mark_as_advanced(${name}_LIBRARY_RELEASE) + + find_library(${name}_LIBRARY_DEBUG NAMES ${filename}d ${filename}) + mark_as_advanced(${name}_LIBRARY_DEBUG) + + select_library_configurations(${name}) + + if (gRPC_DEBUG) + message( + STATUS + "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "${name} ${filename} RELEASE=${${name}_LIBRARY}" + " DEBUG=${${name}_LIBRARY_DEBUG} DEFAULT=${${name}_LIBRARY}" + ) + endif () + + set(${name}_LIBRARY + "${${name}_LIBRARY}" + PARENT_SCOPE) + endif () +endfunction () + +# +# Main +# + +# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES +if (_gRPC_USE_STATIC_LIBS) + set(_gRPC_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) + if (WIN32) + set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES}) + else () + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + endif () +endif () + +_grpc_find_library(_gRPC_grpc grpc) +_grpc_find_library(_gRPC_grpc++ grpc++) + +if (NOT _gRPC_INCLUDE_DIR) + find_path(_gRPC_INCLUDE_DIR grpcpp/grpcpp.h) + mark_as_advanced(_gRPC_INCLUDE_DIR) +endif () + +if (gRPC_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " _gRPC_grpc_LIBRARY = ${_gRPC_grpc_LIBRARY}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " _gRPC_grpc++_LIBRARY = ${_gRPC_grpc++_LIBRARY}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " _gRPC_INCLUDE_DIR = ${_gRPC_INCLUDE_DIR}") +endif () + +if (_gRPC_grpc_LIBRARY) + if (NOT TARGET gRPC::grpc) + add_library(gRPC::grpc UNKNOWN IMPORTED) + set_target_properties( + gRPC::grpc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${_gRPC_INCLUDE_DIR}") + if (EXISTS "${_gRPC_grpc_LIBRARY}") + set_target_properties(gRPC::grpc PROPERTIES IMPORTED_LOCATION + "${_gRPC_grpc_LIBRARY}") + endif () + if (EXISTS "${_gRPC_grpc_LIBRARY_RELEASE}") + set_property( + TARGET gRPC::grpc + APPEND + PROPERTY IMPORTED_CONFIGURATIONS RELEASE) + set_target_properties( + gRPC::grpc PROPERTIES IMPORTED_LOCATION_RELEASE + "${_gRPC_grpc_LIBRARY_RELEASE}") + endif () + if (EXISTS "${_gRPC_grpc_LIBRARY_DEBUG}") + set_property( + TARGET gRPC::grpc + APPEND + PROPERTY IMPORTED_CONFIGURATIONS DEBUG) + set_target_properties( + gRPC::grpc PROPERTIES IMPORTED_LOCATION_DEBUG + "${_gRPC_grpc_LIBRARY_DEBUG}") + endif () + set_property( + TARGET gRPC::grpc + APPEND + PROPERTY INTERFACE_LINK_LIBRARIES protobuf::libprotobuf + Threads::Threads) + endif () +endif () + +if (_gRPC_grpc++_LIBRARY) + if (NOT TARGET gRPC::grpc++) + add_library(gRPC::grpc++ UNKNOWN IMPORTED) + set_target_properties( + gRPC::grpc++ PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${_gRPC++_INCLUDE_DIR}") + if (EXISTS "${_gRPC_grpc++_LIBRARY}") + set_target_properties( + gRPC::grpc++ PROPERTIES IMPORTED_LOCATION + "${_gRPC_grpc++_LIBRARY}") + endif () + if (EXISTS "${_gRPC_grpc++_LIBRARY_RELEASE}") + set_property( + TARGET gRPC::grpc++ + APPEND + PROPERTY IMPORTED_CONFIGURATIONS RELEASE) + set_target_properties( + gRPC::grpc++ PROPERTIES IMPORTED_LOCATION_RELEASE + "${_gRPC_grpc++_LIBRARY_RELEASE}") + endif () + if (EXISTS "${_gRPC_grpc++_LIBRARY_DEBUG}") + set_property( + TARGET gRPC::grpc++ + APPEND + PROPERTY IMPORTED_CONFIGURATIONS DEBUG) + set_target_properties( + gRPC::grpc++ PROPERTIES IMPORTED_LOCATION_DEBUG + "${_gRPC_grpc++_LIBRARY_DEBUG}") + endif () + set_property( + TARGET gRPC::grpc++ + APPEND + PROPERTY INTERFACE_LINK_LIBRARIES gRPC::grpc protobuf::libprotobuf + Threads::Threads) + if (CMAKE_VERSION VERSION_GREATER 3.8) + # gRPC++ requires C++11, but only CMake-3.8 introduced a target + # compiler feature to meet that requirement. + set_property( + TARGET gRPC::grpc++ + APPEND + PROPERTY INTERFACE_COMPILE_FEATURES cxx_std_11) + elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + # CMake 3.5 is still alive and kicking in some older distros, use + # the compiler-specific versions in these cases. + set_property( + TARGET gRPC::grpc++ + APPEND + PROPERTY INTERFACE_COMPILE_OPTIONS "-std=c++11") + elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set_property( + TARGET gRPC::grpc++ + APPEND + PROPERTY INTERFACE_COMPILE_OPTIONS "-std=c++11") + else () + message( + WARNING + "gRPC::grpc++ requires C++11, but this module" + " (${CMAKE_CURRENT_LIST_FILE})" + " cannot enable it for the library target in your CMake and" + " compiler versions. You need to enable C++11 in the" + " CMakeLists.txt for your project. Consider filing a bug" + " so we can fix this problem.") + endif () + endif () +endif () + +# Restore original find library prefixes +if (_gRPC_USE_STATIC_LIBS) + set(CMAKE_FIND_LIBRARY_PREFIXES "${_gRPC_ORIG_FIND_LIBRARY_PREFIXES}") +endif () + +file( + WRITE "${CMAKE_BINARY_DIR}/get_gRPC_version.cc" + [====[ +#include +#include +int main() { + std::cout << grpc::Version(); // no newline to simplify CMake module + return 0; +} + ]====]) + +try_run( + _gRPC_GET_VERSION_STATUS + _gRPC_GET_VERSION_COMPILE_STATUS + "${CMAKE_BINARY_DIR}" + "${CMAKE_BINARY_DIR}/get_gRPC_version.cc" + LINK_LIBRARIES + gRPC::grpc++ + gRPC::grpc + COMPILE_OUTPUT_VARIABLE _gRPC_GET_VERSION_COMPILE_OUTPUT + RUN_OUTPUT_VARIABLE gRPC_VERSION) + +file(REMOVE "${CMAKE_BINARY_DIR}/get_gRPC_version.cc") + +_grpc_fix_grpc_cpp_plugin_target() + +if (gRPC_DEBUG) + foreach ( + _var + _gRPC_CPP_PLUGIN_EXECUTABLE + _gRPC_VERSION_RAW + _gRPC_GET_VERSION_STATUS + _gRPC_GET_VERSION_COMPILE_STATUS + _gRPC_GET_VERSION_COMPILE_OUTPUT + _gRPC_grpc_LIBRARY + _gRPC_grpc++_LIBRARY + _gRPC_INCLUDE_DIR) + message( + STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "${_var} = ${${_var}}") + endforeach () + unset(_var) +endif () + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(gRPC REQUIRED_VARS _gRPC_grpc_LIBRARY + _gRPC_INCLUDE_DIR VERSION_VAR gRPC_VERSION) diff --git a/onnxruntime/server/converter.cc b/server/converter.cc similarity index 88% rename from onnxruntime/server/converter.cc rename to server/converter.cc index 523c022de7545..03ee5d2e42ce1 100644 --- a/onnxruntime/server/converter.cc +++ b/server/converter.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" #include "onnx-ml.pb.h" #include "predict.pb.h" @@ -159,35 +159,6 @@ void MLValueToTensorProto(Ort::Value& ml_value, bool using_raw_data, } break; } - case onnx::TensorProto_DataType_FLOAT16: { // Target: raw_data or int32_data - const auto* data = ml_value.GetTensorMutableData(); - if (using_raw_data) { - tensor_proto.set_raw_data(data, sizeof(onnxruntime::MLFloat16) * elem_count); - } else { - for (size_t i = 0, count = elem_count; i < count; ++i) { - tensor_proto.add_int32_data(reinterpret_cast(data)[i]); - } - } - break; - } - case onnx::TensorProto_DataType_BFLOAT16: { // Target: raw_data or int32_data - const auto* data = ml_value.GetTensorMutableData(); - - std::vector raw_data; - raw_data.reserve(elem_count); - for (size_t i = 0; i < elem_count; ++i) { - raw_data.push_back(data[i].val); - } - - if (using_raw_data) { - tensor_proto.set_raw_data(raw_data.data(), raw_data.size() * sizeof(uint16_t)); - } else { - for (size_t i = 0, count = elem_count; i < count; ++i) { - tensor_proto.add_int32_data(raw_data[i]); - } - } - break; - } case onnx::TensorProto_DataType_STRING: { // Target: string_data // string could not be written into "raw_data" auto length = ml_value.GetStringTensorDataLength(); @@ -260,4 +231,4 @@ void MLValueToTensorProto(Ort::Value& ml_value, bool using_raw_data, return; } } // namespace server -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/server/converter.h b/server/converter.h similarity index 88% rename from onnxruntime/server/converter.h rename to server/converter.h index 3e705f5766204..fc8ef84446526 100644 --- a/onnxruntime/server/converter.h +++ b/server/converter.h @@ -2,11 +2,7 @@ // Licensed under the MIT License. #pragma once -#include "core/session/onnxruntime_cxx_api.h" - -#include - -#include "core/framework/data_types.h" +#include "onnxruntime_cxx_api.h" #include "environment.h" #include "predict.pb.h" diff --git a/onnxruntime/server/core/request_id.cc b/server/core/request_id.cc similarity index 87% rename from onnxruntime/server/core/request_id.cc rename to server/core/request_id.cc index 8e0c5495d6879..4e08dc9e1fecb 100644 --- a/onnxruntime/server/core/request_id.cc +++ b/server/core/request_id.cc @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #include "request_id.h" // boost random is using a deprecated header in 1.69 // See: https://github.com/boostorg/random/issues/49 diff --git a/onnxruntime/server/core/request_id.h b/server/core/request_id.h similarity index 75% rename from onnxruntime/server/core/request_id.h rename to server/core/request_id.h index b45ca7113ba1a..03e0c9beecf0a 100644 --- a/onnxruntime/server/core/request_id.h +++ b/server/core/request_id.h @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #pragma once #include diff --git a/onnxruntime/server/environment.cc b/server/environment.cc similarity index 99% rename from onnxruntime/server/environment.cc rename to server/environment.cc index f928216a380c1..be12314722d6b 100644 --- a/onnxruntime/server/environment.cc +++ b/server/environment.cc @@ -3,7 +3,7 @@ #include #include "environment.h" -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" #ifdef USE_DNNL diff --git a/onnxruntime/server/environment.h b/server/environment.h similarity index 97% rename from onnxruntime/server/environment.h rename to server/environment.h index 86af21029031c..062e4025cd437 100644 --- a/onnxruntime/server/environment.h +++ b/server/environment.h @@ -6,7 +6,7 @@ #include #include -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" #include #include #include diff --git a/onnxruntime/server/executor.cc b/server/executor.cc similarity index 96% rename from onnxruntime/server/executor.cc rename to server/executor.cc index 52836c844e765..25849d73d313f 100644 --- a/onnxruntime/server/executor.cc +++ b/server/executor.cc @@ -2,13 +2,7 @@ // Licensed under the MIT License. #include -#include "core/common/logging/logging.h" -#include "core/framework/data_types.h" -#include "core/session/environment.h" -#include "core/framework/framework_common.h" #include "serializing/mem_buffer.h" -#include "core/framework/ml_value.h" -#include "core/framework/tensor.h" #include "serializing/tensorprotoutils.h" #include "onnx-ml.pb.h" diff --git a/onnxruntime/server/executor.h b/server/executor.h similarity index 97% rename from onnxruntime/server/executor.h rename to server/executor.h index fbfcd25b20854..7cd7cc3e3dc01 100644 --- a/onnxruntime/server/executor.h +++ b/server/executor.h @@ -8,7 +8,7 @@ #include "environment.h" #include "predict.pb.h" #include "util.h" -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" namespace onnxruntime { namespace server { diff --git a/cmake/external/spdlog b/server/external/spdlog similarity index 100% rename from cmake/external/spdlog rename to server/external/spdlog diff --git a/cmake/get_boost.cmake b/server/get_boost.cmake similarity index 100% rename from cmake/get_boost.cmake rename to server/get_boost.cmake diff --git a/onnxruntime/server/grpc/grpc_app.cc b/server/grpc/grpc_app.cc similarity index 91% rename from onnxruntime/server/grpc/grpc_app.cc rename to server/grpc/grpc_app.cc index fe73b91abdc4c..1d571b43fe012 100644 --- a/onnxruntime/server/grpc/grpc_app.cc +++ b/server/grpc/grpc_app.cc @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #include "grpc_app.h" #include #include diff --git a/onnxruntime/server/grpc/grpc_app.h b/server/grpc/grpc_app.h similarity index 87% rename from onnxruntime/server/grpc/grpc_app.h rename to server/grpc/grpc_app.h index 27cea08d15155..a9345c34441ff 100644 --- a/onnxruntime/server/grpc/grpc_app.h +++ b/server/grpc/grpc_app.h @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #pragma once #include #include "prediction_service_impl.h" diff --git a/onnxruntime/server/grpc/prediction_service_impl.cc b/server/grpc/prediction_service_impl.cc similarity index 94% rename from onnxruntime/server/grpc/prediction_service_impl.cc rename to server/grpc/prediction_service_impl.cc index cd2817abab300..834f5c894939a 100644 --- a/onnxruntime/server/grpc/prediction_service_impl.cc +++ b/server/grpc/prediction_service_impl.cc @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #include "prediction_service_impl.h" #include "request_id.h" diff --git a/onnxruntime/server/grpc/prediction_service_impl.h b/server/grpc/prediction_service_impl.h similarity index 89% rename from onnxruntime/server/grpc/prediction_service_impl.h rename to server/grpc/prediction_service_impl.h index b1024d31c28c2..be8d7ecb694f3 100644 --- a/onnxruntime/server/grpc/prediction_service_impl.h +++ b/server/grpc/prediction_service_impl.h @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #pragma once #include "prediction_service.grpc.pb.h" #include "environment.h" diff --git a/onnxruntime/server/http/core/context.h b/server/http/core/context.h similarity index 100% rename from onnxruntime/server/http/core/context.h rename to server/http/core/context.h diff --git a/onnxruntime/server/http/core/http_server.cc b/server/http/core/http_server.cc similarity index 100% rename from onnxruntime/server/http/core/http_server.cc rename to server/http/core/http_server.cc diff --git a/onnxruntime/server/http/core/http_server.h b/server/http/core/http_server.h similarity index 100% rename from onnxruntime/server/http/core/http_server.h rename to server/http/core/http_server.h diff --git a/onnxruntime/server/http/core/listener.cc b/server/http/core/listener.cc similarity index 100% rename from onnxruntime/server/http/core/listener.cc rename to server/http/core/listener.cc diff --git a/onnxruntime/server/http/core/listener.h b/server/http/core/listener.h similarity index 100% rename from onnxruntime/server/http/core/listener.h rename to server/http/core/listener.h diff --git a/onnxruntime/server/http/core/routes.cc b/server/http/core/routes.cc similarity index 100% rename from onnxruntime/server/http/core/routes.cc rename to server/http/core/routes.cc diff --git a/onnxruntime/server/http/core/routes.h b/server/http/core/routes.h similarity index 100% rename from onnxruntime/server/http/core/routes.h rename to server/http/core/routes.h diff --git a/onnxruntime/server/http/core/session.cc b/server/http/core/session.cc similarity index 100% rename from onnxruntime/server/http/core/session.cc rename to server/http/core/session.cc diff --git a/onnxruntime/server/http/core/session.h b/server/http/core/session.h similarity index 100% rename from onnxruntime/server/http/core/session.h rename to server/http/core/session.h diff --git a/onnxruntime/server/http/core/util.cc b/server/http/core/util.cc similarity index 100% rename from onnxruntime/server/http/core/util.cc rename to server/http/core/util.cc diff --git a/onnxruntime/server/http/core/util.h b/server/http/core/util.h similarity index 100% rename from onnxruntime/server/http/core/util.h rename to server/http/core/util.h diff --git a/onnxruntime/server/http/json_handling.cc b/server/http/json_handling.cc similarity index 100% rename from onnxruntime/server/http/json_handling.cc rename to server/http/json_handling.cc diff --git a/onnxruntime/server/http/json_handling.h b/server/http/json_handling.h similarity index 100% rename from onnxruntime/server/http/json_handling.h rename to server/http/json_handling.h diff --git a/onnxruntime/server/http/predict_request_handler.cc b/server/http/predict_request_handler.cc similarity index 100% rename from onnxruntime/server/http/predict_request_handler.cc rename to server/http/predict_request_handler.cc diff --git a/onnxruntime/server/http/predict_request_handler.h b/server/http/predict_request_handler.h similarity index 100% rename from onnxruntime/server/http/predict_request_handler.h rename to server/http/predict_request_handler.h diff --git a/onnxruntime/server/http/util.cc b/server/http/util.cc similarity index 100% rename from onnxruntime/server/http/util.cc rename to server/http/util.cc diff --git a/onnxruntime/server/http/util.h b/server/http/util.h similarity index 100% rename from onnxruntime/server/http/util.h rename to server/http/util.h diff --git a/onnxruntime/server/main.cc b/server/main.cc similarity index 94% rename from onnxruntime/server/main.cc rename to server/main.cc index 9c0e98ec44719..bf039d8a4ae90 100644 --- a/onnxruntime/server/main.cc +++ b/server/main.cc @@ -60,10 +60,12 @@ int main(int argc, char* argv[]) { const auto env = std::make_shared(config.logging_level, spdlog::sinks_init_list{std::make_shared(), std::make_shared()}); auto logger = env->GetAppLogger(); - logger->info("Model path: {}", config.model_path); + logger->info("Model path: {}, ", config.model_path); + logger->info("Model name: {}", config.model_name); + logger->info("Model version: {}", config.model_version); try { - env->InitializeModel(config.model_path, "default", "1"); + env->InitializeModel(config.model_path, config.model_name, config.model_version); logger->debug("Initialize Model Successfully!"); } catch (const Ort::Exception& ex) { logger->critical("Initialize Model Failed: {} ---- Error: [{}]", ex.GetOrtErrorCode(), ex.what()); diff --git a/onnxruntime/server/protobuf/onnx-ml.proto b/server/protobuf/onnx-ml.proto similarity index 100% rename from onnxruntime/server/protobuf/onnx-ml.proto rename to server/protobuf/onnx-ml.proto diff --git a/onnxruntime/server/protobuf/predict.proto b/server/protobuf/predict.proto similarity index 100% rename from onnxruntime/server/protobuf/predict.proto rename to server/protobuf/predict.proto diff --git a/onnxruntime/server/protobuf/prediction_service.proto b/server/protobuf/prediction_service.proto similarity index 100% rename from onnxruntime/server/protobuf/prediction_service.proto rename to server/protobuf/prediction_service.proto diff --git a/server/protobuf_function.cmake b/server/protobuf_function.cmake new file mode 100644 index 0000000000000..d3c0b6e688c66 --- /dev/null +++ b/server/protobuf_function.cmake @@ -0,0 +1,158 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#Changelog: +#copied from https://github.com/protocolbuffers/protobuf/blob/master/cmake/protobuf-config.cmake.in +#sed -i 's/protobuf_generate/onnxruntime_protobuf_generate/g' protobuf-config.cmake.orig +#replace 'protobuf::protoc' with ${PROTOC_EXECUTABLE} and ${PROTOC_DEPS} +#remove OUTDIR + +function(onnxruntime_protobuf_generate) + include(CMakeParseArguments) + if(EXISTS "${ONNX_CUSTOM_PROTOC_EXECUTABLE}") + set(PROTOC_EXECUTABLE ${ONNX_CUSTOM_PROTOC_EXECUTABLE}) + else() + set(PROTOC_EXECUTABLE $) + set(PROTOC_DEPS protobuf::protoc) + endif() + set(_options APPEND_PATH) + set(_singleargs LANGUAGE OUT_VAR EXPORT_MACRO) + if(COMMAND target_sources) + list(APPEND _singleargs TARGET) + endif() + set(_multiargs PROTOS IMPORT_DIRS GENERATE_EXTENSIONS) + + cmake_parse_arguments(onnxruntime_protobuf_generate "${_options}" "${_singleargs}" "${_multiargs}" "${ARGN}") + + if(NOT onnxruntime_protobuf_generate_PROTOS AND NOT onnxruntime_protobuf_generate_TARGET) + message(SEND_ERROR "Error: onnxruntime_protobuf_generate called without any targets or source files") + return() + endif() + + if(NOT onnxruntime_protobuf_generate_OUT_VAR AND NOT onnxruntime_protobuf_generate_TARGET) + message(SEND_ERROR "Error: onnxruntime_protobuf_generate called without a target or output variable") + return() + endif() + + if(NOT onnxruntime_protobuf_generate_LANGUAGE) + set(onnxruntime_protobuf_generate_LANGUAGE cpp) + endif() + string(TOLOWER ${onnxruntime_protobuf_generate_LANGUAGE} onnxruntime_protobuf_generate_LANGUAGE) + + if(onnxruntime_protobuf_generate_EXPORT_MACRO AND onnxruntime_protobuf_generate_LANGUAGE STREQUAL cpp) + set(_dll_export_decl "dllexport_decl=${onnxruntime_protobuf_generate_EXPORT_MACRO}:") + endif() + + if(NOT onnxruntime_protobuf_generate_EXTENSIONS) + if(onnxruntime_protobuf_generate_LANGUAGE STREQUAL cpp) + set(onnxruntime_protobuf_generate_EXTENSIONS .pb.h .pb.cc) + elseif(onnxruntime_protobuf_generate_LANGUAGE STREQUAL python) + set(onnxruntime_protobuf_generate_EXTENSIONS _pb2.py) + else() + message(SEND_ERROR "Error: onnxruntime_protobuf_generate given unknown Language ${LANGUAGE}, please provide a value for GENERATE_EXTENSIONS") + return() + endif() + endif() + + if(onnxruntime_protobuf_generate_TARGET) + get_target_property(_source_list ${onnxruntime_protobuf_generate_TARGET} SOURCES) + foreach(_file ${_source_list}) + if(_file MATCHES "proto$") + list(APPEND onnxruntime_protobuf_generate_PROTOS ${_file}) + endif() + endforeach() + endif() + + if(NOT onnxruntime_protobuf_generate_PROTOS) + message(SEND_ERROR "Error: onnxruntime_protobuf_generate could not find any .proto files") + return() + endif() + + if(onnxruntime_protobuf_generate_APPEND_PATH) + # Create an include path for each file specified + foreach(_file ${onnxruntime_protobuf_generate_PROTOS}) + get_filename_component(_abs_file ${_file} ABSOLUTE) + get_filename_component(_abs_path ${_abs_file} PATH) + list(FIND _protobuf_include_path ${_abs_path} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${_abs_path}) + endif() + endforeach() + else() + set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + foreach(DIR ${onnxruntime_protobuf_generate_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + + set(_generated_srcs_all) + foreach(_proto ${onnxruntime_protobuf_generate_PROTOS}) + get_filename_component(_abs_file ${_proto} ABSOLUTE) + get_filename_component(_basename ${_proto} NAME_WE) + + set(_generated_srcs) + foreach(_ext ${onnxruntime_protobuf_generate_EXTENSIONS}) + list(APPEND _generated_srcs "${CMAKE_CURRENT_BINARY_DIR}/${_basename}${_ext}") + endforeach() + list(APPEND _generated_srcs_all ${_generated_srcs}) + + if (onnxruntime_USE_FULL_PROTOBUF) + add_custom_command( + OUTPUT ${_generated_srcs} + COMMAND ${PROTOC_EXECUTABLE} + ARGS --${onnxruntime_protobuf_generate_LANGUAGE}_out ${_dll_export_decl}${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${_abs_file} + DEPENDS ${_abs_file} ${PROTOC_DEPS} + COMMENT "Running ${onnxruntime_protobuf_generate_LANGUAGE} protocol buffer (full) compiler on ${_proto}" + VERBATIM ) + else() + add_custom_command( + OUTPUT ${_generated_srcs} + COMMAND ${PROTOC_EXECUTABLE} + ARGS --${onnxruntime_protobuf_generate_LANGUAGE}_out lite:${_dll_export_decl}${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${_abs_file} + DEPENDS ${_abs_file} ${PROTOC_DEPS} + COMMENT "Running ${onnxruntime_protobuf_generate_LANGUAGE} protocol buffer compiler (lite) on ${_proto}" + VERBATIM ) + endif() + endforeach() + + set_source_files_properties(${_generated_srcs_all} PROPERTIES GENERATED TRUE) + if(onnxruntime_protobuf_generate_OUT_VAR) + set(${onnxruntime_protobuf_generate_OUT_VAR} ${_generated_srcs_all} PARENT_SCOPE) + endif() + if(onnxruntime_protobuf_generate_TARGET) + target_sources(${onnxruntime_protobuf_generate_TARGET} PRIVATE ${_generated_srcs_all}) + endif() + +endfunction() diff --git a/onnxruntime/server/serializing/mem_buffer.h b/server/serializing/mem_buffer.h similarity index 72% rename from onnxruntime/server/serializing/mem_buffer.h rename to server/serializing/mem_buffer.h index bcda8ce0e189f..fa2f28b73a028 100644 --- a/onnxruntime/server/serializing/mem_buffer.h +++ b/server/serializing/mem_buffer.h @@ -1,5 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. #pragma once -#include "core/common/common.h" + +#include "onnxruntime_c_api.h" +#include "onnxruntime_cxx_api.h" namespace onnxruntime { namespace server { @@ -18,4 +22,4 @@ class MemBuffer { const OrtMemoryInfo& alloc_info_; }; } // namespace server -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/server/serializing/tensorprotoutils.cc b/server/serializing/tensorprotoutils.cc similarity index 82% rename from onnxruntime/server/serializing/tensorprotoutils.cc rename to server/serializing/tensorprotoutils.cc index 45e062374e790..ee6d6abf7ffae 100644 --- a/onnxruntime/server/serializing/tensorprotoutils.cc +++ b/server/serializing/tensorprotoutils.cc @@ -6,24 +6,12 @@ #include #include #include -#include -#include "core/framework/data_types.h" -#include "core/framework/allocator.h" +#include #include "onnx-ml.pb.h" -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" namespace onnxruntime { -namespace server { -#ifdef __GNUC__ -constexpr inline bool IsLittleEndianOrder() noexcept { return __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; } -#else -// On Windows and Mac, this function should always return true -GSL_SUPPRESS(type .1) // allow use of reinterpret_cast for this special case -inline bool IsLittleEndianOrder() noexcept { - static int n = 1; - return (*reinterpret_cast(&n) == 1); -} -#endif + //From core common inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept { @@ -56,6 +44,19 @@ inline std::string MakeString(const char* p_str) { return p_str; } + + +namespace server { +#ifdef __GNUC__ +constexpr inline bool IsLittleEndianOrder() noexcept { return __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; } +#else +// On Windows and Mac, this function should always return true +GSL_SUPPRESS(type .1) // allow use of reinterpret_cast for this special case +inline bool IsLittleEndianOrder() noexcept { + static int n = 1; + return (*reinterpret_cast(&n) == 1); +} +#endif std::vector GetTensorShapeFromTensorProto(const onnx::TensorProto& tensor_proto) { const auto& dims = tensor_proto.dims(); std::vector tensor_shape_vec(static_cast(dims.size())); @@ -66,17 +67,40 @@ std::vector GetTensorShapeFromTensorProto(const onnx::TensorProto& tens return tensor_shape_vec; } + +template +static bool CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t* out) noexcept { + static constexpr size_t max_allowed = (static_cast(1) << (static_cast(std::numeric_limits::digits >> 1))) - alignment; + static constexpr size_t max_size = std::numeric_limits::max() - alignment; + static constexpr size_t alignment_mask = alignment - 1; + //Indeed, we only need to check if max_size / nmemb < size + //max_allowed is for avoiding unnecessary DIV. + if (nmemb >= max_allowed && max_size / nmemb < size) { + return false; + } + if (size >= max_allowed && + nmemb > 0 && max_size / nmemb < size) { + return false; + } + if (alignment == 0) + *out = size * nmemb; + else + *out = (size * nmemb + alignment_mask) & ~static_cast(alignment_mask); + return true; +} + +static bool CalcMemSizeForArray(size_t nmemb, size_t size, size_t* out) noexcept { + return CalcMemSizeForArrayWithAlignment<0>(nmemb, size, out); +} + // This function doesn't support string tensors template static void UnpackTensorWithRawData(const void* raw_data, size_t raw_data_length, size_t expected_size, /*out*/ T* p_data) { // allow this low level routine to be somewhat unsafe. assuming it's thoroughly tested and valid - GSL_SUPPRESS(type) // type.1 reinterpret-cast; type.4 C-style casts; type.5 'T result;' is uninitialized; - GSL_SUPPRESS(bounds .1) // pointer arithmetic - GSL_SUPPRESS(f .23) // buff and temp_bytes never tested for nullness and could be gsl::not_null { size_t expected_size_in_bytes; - if (!onnxruntime::IAllocator::CalcMemSizeForArray(expected_size, sizeof(T), &expected_size_in_bytes)) { + if (!CalcMemSizeForArray(expected_size, sizeof(T), &expected_size_in_bytes)) { throw Ort::Exception("size overflow", OrtErrorCode::ORT_FAIL); } if (raw_data_length != expected_size_in_bytes) @@ -187,77 +211,11 @@ void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t return; } -template <> -void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len, - /*out*/ MLFloat16* p_data, int64_t expected_size) { - if (nullptr == p_data) { - const size_t size = raw_data != nullptr ? raw_data_len : tensor.int32_data_size(); - if (size == 0) return; - throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); - } - if (onnx::TensorProto_DataType_FLOAT16 != tensor.data_type()) { - throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); - } - - if (raw_data != nullptr) { - return UnpackTensorWithRawData(raw_data, raw_data_len, expected_size, p_data); - } - - if (tensor.int32_data_size() != expected_size) - throw Ort::Exception( - "UnpackTensor: the pre-allocate size does not match the size in proto", OrtErrorCode::ORT_FAIL); - constexpr int max_value = std::numeric_limits::max(); - for (int i = 0; i < static_cast(expected_size); i++) { - int v = tensor.int32_data()[i]; - if (v < 0 || v > max_value) { - throw Ort::Exception( - "data overflow", OrtErrorCode::ORT_FAIL); - } - p_data[i] = MLFloat16(static_cast(v)); - } - - return; -} - -template <> -void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len, - /*out*/ BFloat16* p_data, int64_t expected_size) { - if (nullptr == p_data) { - const size_t size = raw_data != nullptr ? raw_data_len : tensor.int32_data_size(); - if (size == 0) - return; - - throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); - } - if (onnx::TensorProto_DataType_BFLOAT16 != tensor.data_type()) { - throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); - } - - if (raw_data != nullptr) { - return UnpackTensorWithRawData(raw_data, raw_data_len, expected_size, p_data); - } - - if (tensor.int32_data_size() != expected_size) - throw Ort::Exception( - "UnpackTensor: the pre-allocate size does not match the size in proto", OrtErrorCode::ORT_FAIL); - - constexpr int max_value = std::numeric_limits::max(); - for (int i = 0; i < static_cast(expected_size); i++) { - int v = tensor.int32_data()[i]; - if (v < 0 || v > max_value) { - throw Ort::Exception( - "data overflow", OrtErrorCode::ORT_FAIL); - } - p_data[i] = BFloat16(static_cast(v)); - } - - return; -} #define CASE_PROTO_TRACE(X, Y) \ case onnx::TensorProto_DataType::TensorProto_DataType_##X: \ - if (!IAllocator::CalcMemSizeForArrayWithAlignment(size, sizeof(Y), out)) { \ + if (!CalcMemSizeForArrayWithAlignment(size, sizeof(Y), out)) { \ throw Ort::Exception("Invalid TensorProto", OrtErrorCode::ORT_FAIL); \ } \ break; @@ -270,7 +228,7 @@ void GetSizeInBytesFromTensorProto(const onnx::TensorProto& tensor_proto, size_t if (dim < 0 || static_cast(dim) >= std::numeric_limits::max()) { throw Ort::Exception("Invalid TensorProto", OrtErrorCode::ORT_FAIL); } - if (!IAllocator::CalcMemSizeForArray(size, static_cast(dim), &size)) { + if (!CalcMemSizeForArray(size, static_cast(dim), &size)) { throw Ort::Exception("Invalid TensorProto", OrtErrorCode::ORT_FAIL); } } @@ -286,8 +244,6 @@ void GetSizeInBytesFromTensorProto(const onnx::TensorProto& tensor_proto, size_t CASE_PROTO_TRACE(UINT16, uint16_t); CASE_PROTO_TRACE(UINT32, uint32_t); CASE_PROTO_TRACE(UINT64, uint64_t); - CASE_PROTO_TRACE(FLOAT16, MLFloat16); - CASE_PROTO_TRACE(BFLOAT16, BFloat16); CASE_PROTO_TRACE(STRING, std::string); default: throw Ort::Exception("", OrtErrorCode::ORT_NOT_IMPLEMENTED); @@ -401,8 +357,6 @@ void TensorProtoToMLValue(const onnx::TensorProto& tensor_proto, const MemBuffer CASE_PROTO(UINT16, uint16_t); CASE_PROTO(UINT32, uint32_t); CASE_PROTO(UINT64, uint64_t); - CASE_PROTO(FLOAT16, MLFloat16); - CASE_PROTO(BFLOAT16, BFloat16); case onnx::TensorProto_DataType::TensorProto_DataType_STRING: if (preallocated != nullptr) { OrtInitializeBufferForTensor(preallocated, preallocated_size, ele_type); @@ -428,4 +382,4 @@ template void GetSizeInBytesFromTensorProto<256>(const onnx::TensorProto& tensor size_t* out); template void GetSizeInBytesFromTensorProto<0>(const onnx::TensorProto& tensor_proto, size_t* out); } // namespace server -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/server/serializing/tensorprotoutils.h b/server/serializing/tensorprotoutils.h similarity index 91% rename from onnxruntime/server/serializing/tensorprotoutils.h rename to server/serializing/tensorprotoutils.h index 243b122ef64b5..4b08a29448efa 100644 --- a/onnxruntime/server/serializing/tensorprotoutils.h +++ b/server/serializing/tensorprotoutils.h @@ -5,8 +5,8 @@ #include #include -#include "core/session/onnxruntime_c_api.h" -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_c_api.h" +#include "onnxruntime_cxx_api.h" #include "mem_buffer.h" @@ -34,4 +34,4 @@ void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t ONNXTensorElementDataType CApiElementTypeFromProtoType(int type); ONNXTensorElementDataType GetTensorElementType(const onnx::TensorProto& tensor_proto); } // namespace server -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/server/server_configuration.h b/server/server_configuration.h similarity index 93% rename from onnxruntime/server/server_configuration.h rename to server/server_configuration.h index 93d6983d33e51..eeb883f8999a7 100644 --- a/onnxruntime/server/server_configuration.h +++ b/server/server_configuration.h @@ -8,7 +8,7 @@ #include #include "boost/program_options.hpp" -#include "core/session/onnxruntime_cxx_api.h" +#include "onnxruntime_cxx_api.h" namespace onnxruntime { namespace server { @@ -39,6 +39,8 @@ class ServerConfiguration { public: const std::string full_desc = "ONNX Server: host an ONNX model with ONNX Runtime"; std::string model_path; + std::string model_name = "default"; + std::string model_version = "1"; std::string address = "0.0.0.0"; unsigned short http_port = 8001; unsigned short grpc_port = 50051; @@ -49,6 +51,8 @@ class ServerConfiguration { desc.add_options()("help,h", "Shows a help message and exits"); desc.add_options()("log_level", po::value(&log_level_str)->default_value(log_level_str), "Logging level. Allowed options (case sensitive): verbose, info, warning, error, fatal"); desc.add_options()("model_path", po::value(&model_path)->required(), "Path to ONNX model"); + desc.add_options()("model_name", po::value(&model_name)->default_value(model_name), "ONNX model name"); + desc.add_options()("model_version", po::value(&model_version)->default_value(model_version), "ONNX model version"); desc.add_options()("address", po::value(&address)->default_value(address), "The base HTTP address"); desc.add_options()("http_port", po::value(&http_port)->default_value(http_port), "HTTP port to listen to requests"); desc.add_options()("num_http_threads", po::value(&num_http_threads)->default_value(num_http_threads), "Number of http threads"); diff --git a/onnxruntime/test/server/integration_tests/README.MD b/server/test/integration_tests/README.MD similarity index 100% rename from onnxruntime/test/server/integration_tests/README.MD rename to server/test/integration_tests/README.MD diff --git a/onnxruntime/test/server/integration_tests/function_tests.py b/server/test/integration_tests/function_tests.py similarity index 100% rename from onnxruntime/test/server/integration_tests/function_tests.py rename to server/test/integration_tests/function_tests.py diff --git a/onnxruntime/test/server/integration_tests/model_zoo_data_prep.py b/server/test/integration_tests/model_zoo_data_prep.py similarity index 100% rename from onnxruntime/test/server/integration_tests/model_zoo_data_prep.py rename to server/test/integration_tests/model_zoo_data_prep.py diff --git a/onnxruntime/test/server/integration_tests/model_zoo_tests.py b/server/test/integration_tests/model_zoo_tests.py similarity index 100% rename from onnxruntime/test/server/integration_tests/model_zoo_tests.py rename to server/test/integration_tests/model_zoo_tests.py diff --git a/onnxruntime/test/server/integration_tests/test_main.py b/server/test/integration_tests/test_main.py similarity index 100% rename from onnxruntime/test/server/integration_tests/test_main.py rename to server/test/integration_tests/test_main.py diff --git a/onnxruntime/test/server/integration_tests/test_util.py b/server/test/integration_tests/test_util.py similarity index 100% rename from onnxruntime/test/server/integration_tests/test_util.py rename to server/test/integration_tests/test_util.py diff --git a/server/test/testdata/mul_1.onnx b/server/test/testdata/mul_1.onnx new file mode 100644 index 0000000000000..0b6dc51026132 Binary files /dev/null and b/server/test/testdata/mul_1.onnx differ diff --git a/onnxruntime/test/testdata/server/mnist_test_data_set_0_input.json b/server/test/testdata/server/mnist_test_data_set_0_input.json similarity index 100% rename from onnxruntime/test/testdata/server/mnist_test_data_set_0_input.json rename to server/test/testdata/server/mnist_test_data_set_0_input.json diff --git a/onnxruntime/test/testdata/server/mnist_test_data_set_0_input.pb b/server/test/testdata/server/mnist_test_data_set_0_input.pb similarity index 100% rename from onnxruntime/test/testdata/server/mnist_test_data_set_0_input.pb rename to server/test/testdata/server/mnist_test_data_set_0_input.pb diff --git a/onnxruntime/test/testdata/server/mnist_test_data_set_0_output.json b/server/test/testdata/server/mnist_test_data_set_0_output.json similarity index 100% rename from onnxruntime/test/testdata/server/mnist_test_data_set_0_output.json rename to server/test/testdata/server/mnist_test_data_set_0_output.json diff --git a/onnxruntime/test/testdata/server/mnist_test_data_set_0_output.pb b/server/test/testdata/server/mnist_test_data_set_0_output.pb similarity index 100% rename from onnxruntime/test/testdata/server/mnist_test_data_set_0_output.pb rename to server/test/testdata/server/mnist_test_data_set_0_output.pb diff --git a/onnxruntime/test/testdata/server/request_0.pb b/server/test/testdata/server/request_0.pb similarity index 100% rename from onnxruntime/test/testdata/server/request_0.pb rename to server/test/testdata/server/request_0.pb diff --git a/onnxruntime/test/testdata/server/response_0.pb b/server/test/testdata/server/response_0.pb similarity index 100% rename from onnxruntime/test/testdata/server/response_0.pb rename to server/test/testdata/server/response_0.pb diff --git a/onnxruntime/test/server/unit_tests/executor_test.cc b/server/test/unit_tests/executor_test.cc similarity index 96% rename from onnxruntime/test/server/unit_tests/executor_test.cc rename to server/test/unit_tests/executor_test.cc index 40561198d8a1d..0ff196f63a127 100644 --- a/onnxruntime/test/server/unit_tests/executor_test.cc +++ b/server/test/unit_tests/executor_test.cc @@ -5,8 +5,8 @@ #include "gtest/gtest.h" -#include "server/executor.h" -#include "server/http/json_handling.h" +#include "executor.h" +#include "http/json_handling.h" #include #include #include diff --git a/onnxruntime/test/server/unit_tests/external/server_context_test_spouse.h b/server/test/unit_tests/external/server_context_test_spouse.h similarity index 100% rename from onnxruntime/test/server/unit_tests/external/server_context_test_spouse.h rename to server/test/unit_tests/external/server_context_test_spouse.h diff --git a/onnxruntime/test/server/unit_tests/http_routes_tests.cc b/server/test/unit_tests/http_routes_tests.cc similarity index 99% rename from onnxruntime/test/server/unit_tests/http_routes_tests.cc rename to server/test/unit_tests/http_routes_tests.cc index 842eabb071134..e084a8befd907 100644 --- a/onnxruntime/test/server/unit_tests/http_routes_tests.cc +++ b/server/test/unit_tests/http_routes_tests.cc @@ -4,7 +4,7 @@ #include #include "gtest/gtest.h" -#include "server/http/core/routes.h" +#include "http/core/routes.h" namespace onnxruntime { namespace server { diff --git a/onnxruntime/test/server/unit_tests/json_handling_tests.cc b/server/test/unit_tests/json_handling_tests.cc similarity index 98% rename from onnxruntime/test/server/unit_tests/json_handling_tests.cc rename to server/test/unit_tests/json_handling_tests.cc index 0bd2d16e45795..5e6f8dfc0a416 100644 --- a/onnxruntime/test/server/unit_tests/json_handling_tests.cc +++ b/server/test/unit_tests/json_handling_tests.cc @@ -7,7 +7,7 @@ #include "gtest/gtest.h" #include "predict.pb.h" -#include "server/http/json_handling.h" +#include "http/json_handling.h" namespace onnxruntime { namespace server { @@ -36,7 +36,7 @@ TEST(JsonDeserializationTests, InvalidData) { protobufutil::Status status = onnxruntime::server::GetRequestFromJson(input_json, request); EXPECT_EQ(protobufutil::error::INVALID_ARGUMENT, status.error_code()); - EXPECT_EQ("inputs[0].value.raw_data: invalid value \"hello\" for type TYPE_BYTES", status.error_message()); + EXPECT_EQ("(inputs[0].value.raw_data): invalid value \"hello\" for type TYPE_BYTES", status.error_message()) << status.error_message(); } TEST(JsonDeserializationTests, InvalidJson) { @@ -125,4 +125,4 @@ TEST(JsonErrorMessageTests, MessageWithManyCarriageCharacters) { } // namespace test } // namespace server -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/test/server/unit_tests/prediction_service_impl_test.cc b/server/test/unit_tests/prediction_service_impl_test.cc similarity index 96% rename from onnxruntime/test/server/unit_tests/prediction_service_impl_test.cc rename to server/test/unit_tests/prediction_service_impl_test.cc index 39f32b62c1ad8..968a42c1512d5 100644 --- a/onnxruntime/test/server/unit_tests/prediction_service_impl_test.cc +++ b/server/test/unit_tests/prediction_service_impl_test.cc @@ -1,10 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. #include #include "gtest/gtest.h" -#include "server/executor.h" -#include "server/grpc/prediction_service_impl.h" -#include "test/test_environment.h" +#include "executor.h" +#include "grpc/prediction_service_impl.h" #include "test_server_environment.h" #include "external/server_context_test_spouse.h" #include diff --git a/onnxruntime/test/server/unit_tests/server_configuration_test.cc b/server/test/unit_tests/server_configuration_test.cc similarity index 87% rename from onnxruntime/test/server/unit_tests/server_configuration_test.cc rename to server/test/unit_tests/server_configuration_test.cc index 00e772d902274..578a550f4fa83 100644 --- a/onnxruntime/test/server/unit_tests/server_configuration_test.cc +++ b/server/test/unit_tests/server_configuration_test.cc @@ -4,7 +4,7 @@ #include "gtest/gtest.h" #include "gmock/gmock.h" -#include "server/server_configuration.h" +#include "server_configuration.h" namespace onnxruntime { namespace server { @@ -14,15 +14,19 @@ TEST(ConfigParsingTests, AllArgs) { char* test_argv[] = { const_cast("/path/to/binary"), const_cast("--model_path"), const_cast("testdata/mul_1.onnx"), + const_cast("--model_name"), const_cast("mul_1"), + const_cast("--model_version"), const_cast("1"), const_cast("--address"), const_cast("4.4.4.4"), const_cast("--http_port"), const_cast("80"), const_cast("--num_http_threads"), const_cast("1"), const_cast("--log_level"), const_cast("info")}; onnxruntime::server::ServerConfiguration config{}; - Result res = config.ParseInput(11, test_argv); + Result res = config.ParseInput(15, test_argv); EXPECT_EQ(res, Result::ContinueSuccess); EXPECT_EQ(config.model_path, "testdata/mul_1.onnx"); + EXPECT_EQ(config.model_name, "mul_1"); + EXPECT_EQ(config.model_version, "1"); EXPECT_EQ(config.address, "4.4.4.4"); EXPECT_EQ(config.http_port, 80); EXPECT_EQ(config.num_http_threads, 1); @@ -32,13 +36,15 @@ TEST(ConfigParsingTests, AllArgs) { TEST(ConfigParsingTests, Defaults) { char* test_argv[] = { const_cast("/path/to/binary"), - const_cast("--model"), const_cast("testdata/mul_1.onnx"), + const_cast("--model_path"), const_cast("testdata/mul_1.onnx"), const_cast("--num_http_threads"), const_cast("3")}; onnxruntime::server::ServerConfiguration config{}; Result res = config.ParseInput(5, test_argv); EXPECT_EQ(res, Result::ContinueSuccess); EXPECT_EQ(config.model_path, "testdata/mul_1.onnx"); + EXPECT_EQ(config.model_name, "default"); + EXPECT_EQ(config.model_version, "1"); EXPECT_EQ(config.address, "0.0.0.0"); EXPECT_EQ(config.http_port, 8001); EXPECT_EQ(config.num_http_threads, 3); diff --git a/onnxruntime/test/server/unit_tests/test_main.cc b/server/test/unit_tests/test_main.cc similarity index 81% rename from onnxruntime/test/server/unit_tests/test_main.cc rename to server/test/unit_tests/test_main.cc index 09a5304d5f8d7..ee106cc3286b6 100644 --- a/onnxruntime/test/server/unit_tests/test_main.cc +++ b/server/test/unit_tests/test_main.cc @@ -2,15 +2,14 @@ // Licensed under the MIT License. #include "gtest/gtest.h" -#include "server/environment.h" +#include "environment.h" #include "test_server_environment.h" GTEST_API_ int main(int argc, char** argv) { int status = 0; - try { + testing::InitGoogleTest(&argc, argv); onnxruntime::server::test::TestServerEnvironment server_env{}; - onnxruntime::test::TestEnvironment env{argc, argv, false}; status = RUN_ALL_TESTS(); } catch (const std::exception& ex) { std::cerr << ex.what(); diff --git a/onnxruntime/test/server/unit_tests/test_server_environment.cc b/server/test/unit_tests/test_server_environment.cc similarity index 71% rename from onnxruntime/test/server/unit_tests/test_server_environment.cc rename to server/test/unit_tests/test_server_environment.cc index bd42e2ba31b81..8ce32769d70a8 100644 --- a/onnxruntime/test/server/unit_tests/test_server_environment.cc +++ b/server/test/unit_tests/test_server_environment.cc @@ -1,7 +1,10 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. #include "test_server_environment.h" #include #include #include +#include namespace onnxruntime { namespace server { @@ -9,9 +12,7 @@ namespace test { static std::unique_ptr s_env; ServerEnvironment* ServerEnv() { - ORT_ENFORCE(s_env != nullptr, - "Need a TestServerEnvironment instance to provide the server environment."); - + assert(s_env != nullptr); return s_env.get(); } @@ -19,7 +20,7 @@ TestServerEnvironment::TestServerEnvironment() { auto console = spdlog::stdout_logger_mt("console"); spdlog::set_default_logger(console); spdlog::sink_ptr ptr = std::make_shared(); - s_env = onnxruntime::make_unique(ORT_LOGGING_LEVEL_WARNING, spdlog::sinks_init_list{ptr}); + s_env = std::make_unique(ORT_LOGGING_LEVEL_WARNING, spdlog::sinks_init_list{ptr}); } TestServerEnvironment::~TestServerEnvironment() { //destruct env to make sure the default logger is destoryed before the logger mutex. @@ -28,4 +29,4 @@ TestServerEnvironment::~TestServerEnvironment() { } // namespace test } // namespace server -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/test/server/unit_tests/test_server_environment.h b/server/test/unit_tests/test_server_environment.h similarity index 71% rename from onnxruntime/test/server/unit_tests/test_server_environment.h rename to server/test/unit_tests/test_server_environment.h index bfd8072f13d4b..e725d15d7f23c 100644 --- a/onnxruntime/test/server/unit_tests/test_server_environment.h +++ b/server/test/unit_tests/test_server_environment.h @@ -1,7 +1,8 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. #pragma once -#include "server/environment.h" -#include "test/test_environment.h" +#include "environment.h" namespace onnxruntime { namespace server { @@ -17,4 +18,4 @@ class TestServerEnvironment { }; } // namespace test } // namespace server -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/test/server/unit_tests/util_tests.cc b/server/test/unit_tests/util_tests.cc similarity index 98% rename from onnxruntime/test/server/unit_tests/util_tests.cc rename to server/test/unit_tests/util_tests.cc index c16f87c4a712b..48b17dd769d82 100644 --- a/onnxruntime/test/server/unit_tests/util_tests.cc +++ b/server/test/unit_tests/util_tests.cc @@ -3,8 +3,8 @@ #include #include "gtest/gtest.h" -#include "server/http/core/context.h" -#include "server/http/util.h" +#include "http/core/context.h" +#include "http/util.h" namespace onnxruntime { namespace server { diff --git a/onnxruntime/server/util.cc b/server/util.cc similarity index 60% rename from onnxruntime/server/util.cc rename to server/util.cc index a4dbc3ed291ae..2124e1f7bf8f5 100644 --- a/onnxruntime/server/util.cc +++ b/server/util.cc @@ -4,7 +4,6 @@ #include #include -#include "core/common/status.h" #include "util.h" namespace onnxruntime { @@ -15,23 +14,23 @@ namespace protobufutil = google::protobuf::util; protobufutil::Status GenerateProtobufStatus(const int& onnx_status, const std::string& message) { protobufutil::error::Code code = protobufutil::error::Code::UNKNOWN; switch (onnx_status) { - case onnxruntime::common::StatusCode::OK: - case onnxruntime::common::StatusCode::MODEL_LOADED: + case ORT_OK: + case ORT_MODEL_LOADED: code = protobufutil::error::Code::OK; break; - case onnxruntime::common::StatusCode::FAIL: - case onnxruntime::common::StatusCode::INVALID_ARGUMENT: - case onnxruntime::common::StatusCode::INVALID_PROTOBUF: - case onnxruntime::common::StatusCode::INVALID_GRAPH: - case onnxruntime::common::StatusCode::NO_SUCHFILE: - case onnxruntime::common::StatusCode::NO_MODEL: + case ORT_FAIL: + case ORT_INVALID_ARGUMENT: + case ORT_INVALID_PROTOBUF: + case ORT_INVALID_GRAPH: + case ORT_NO_SUCHFILE: + case ORT_NO_MODEL: code = protobufutil::error::Code::INVALID_ARGUMENT; break; - case onnxruntime::common::StatusCode::NOT_IMPLEMENTED: + case ORT_NOT_IMPLEMENTED: code = protobufutil::error::Code::UNIMPLEMENTED; break; - case onnxruntime::common::StatusCode::RUNTIME_EXCEPTION: - case onnxruntime::common::StatusCode::EP_FAIL: + case ORT_RUNTIME_EXCEPTION: + case ORT_EP_FAIL: code = protobufutil::error::Code::INTERNAL; break; default: diff --git a/onnxruntime/server/util.h b/server/util.h similarity index 79% rename from onnxruntime/server/util.h rename to server/util.h index f136324e195ca..8ca204ad83bc8 100644 --- a/onnxruntime/server/util.h +++ b/server/util.h @@ -4,8 +4,9 @@ #pragma once #include +#include "onnxruntime_c_api.h" +#include "onnxruntime_cxx_api.h" -#include "core/common/status.h" namespace onnxruntime { namespace server { @@ -39,8 +40,7 @@ class MemBufferArray { }; google::protobuf::util::Status GenerateProtobufStatus(const int& onnx_status, const std::string& message); -// Generate protobuf status from ONNX Runtime status -google::protobuf::util::Status GenerateProtobufStatus(const onnxruntime::common::Status& onnx_status, const std::string& message); + } // namespace server } // namespace onnxruntime diff --git a/setup.py b/setup.py index b53a6519acdd5..4af75ccc873da 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,10 @@ elif '--use_ngraph' in sys.argv: package_name = 'onnxruntime-ngraph' sys.argv.remove('--use_ngraph') + +elif '--use_dnnl' in sys.argv: + package_name = 'onnxruntime-dnnl' + sys.argv.remove('--use_dnnl') elif '--use_openvino' in sys.argv: package_name = 'onnxruntime-openvino' @@ -119,7 +123,7 @@ def run(self): if platform.system() == 'Linux': libs = ['onnxruntime_pybind11_state.so', 'libdnnl.so.1', 'libmklml_intel.so', 'libiomp5.so', 'mimalloc.so'] # nGraph Libs - libs.extend(['libngraph.so', 'libcodegen.so', 'libcpu_backend.so', 'libdnnl.so', 'libtbb_debug.so', 'libtbb_debug.so.2', 'libtbb.so', 'libtbb.so.2']) + libs.extend(['libngraph.so', 'libcodegen.so', 'libcpu_backend.so', 'libmkldnn.so', 'libtbb_debug.so', 'libtbb_debug.so.2', 'libtbb.so', 'libtbb.so.2']) # Nuphar Libs libs.extend(['libtvm.so.0.5.1']) # Openvino Libs @@ -183,7 +187,13 @@ def run(self): with open('VERSION_NUMBER') as f: version_number = f.readline().strip() if nightly_build: - date_suffix = str(datetime.datetime.now().date().strftime("%m%d")) + #https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables + date_suffix = environ.get('BUILD_BUILDNUMBER') + if date_suffix is None: + #The following line is only for local testing + date_suffix = str(datetime.datetime.now().date().strftime("%Y%m%d")) + else: + date_suffix = date_suffix.replace('.','') version_number = version_number + ".dev" + date_suffix cmd_classes = {} @@ -212,19 +222,20 @@ def run(self): 'onnxruntime': data + examples + extra, }, py_modules=python_modules_list, - extras_require={ - 'backend': ['onnx>=1.2.3'], - 'numpy': ['numpy>=1.15.0'] - }, + install_requires=[ + 'onnx>=1.2.3', + 'numpy>=1.18.0' + ], entry_points= { 'console_scripts': [ 'onnxruntime_test = onnxruntime.tools.onnxruntime_test:main', ] }, classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 399dd4d7c8591..65e2ba8020185 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -95,12 +95,7 @@ def parse_arguments(): # Build a shared lib parser.add_argument("--build_shared_lib", action='store_true', help="Build a shared library for the ONNXRuntime.") - - # Build ONNX Runtime server - parser.add_argument("--build_server", action='store_true', help="Build server application for the ONNXRuntime.") - parser.add_argument("--enable_server_tests", action='store_true', help="Run server application tests.") - parser.add_argument("--enable_server_model_tests", action='store_true', help="Run server model tests.") - + # Build options parser.add_argument("--cmake_extra_defines", nargs="+", help="Extra definitions to pass to CMake during build system generation. " + @@ -141,7 +136,6 @@ def parse_arguments(): parser.add_argument("--use_tvm", action="store_true", help="Build with tvm") parser.add_argument("--use_openmp", action='store_true', help="Build with OpenMP.") parser.add_argument("--use_llvm", action="store_true", help="Build tvm with llvm") - parser.add_argument("--use_eigenthreadpool", action="store_true", help="Build with eigenthreadpool") parser.add_argument("--enable_msinternal", action="store_true", help="Enable for Microsoft internal builds only.") parser.add_argument("--llvm_path", help="Path to llvm dir") parser.add_argument("--use_brainslice", action="store_true", help="Build with brain slice") @@ -175,8 +169,17 @@ def resolve_executable_path(command_or_path): def is_windows(): return sys.platform.startswith("win") +def get_linux_distro(): + try: + with open('/etc/os-release', 'r') as f: + dist_info = dict(line.strip().split('=', 1) for line in f.readlines()) + return dist_info.get('NAME', '').strip('"'), dist_info.get('VERSION', '').strip('"') + except: + return '', '' + def is_ubuntu_1604(): - return platform.linux_distribution()[0] == 'Ubuntu' and platform.linux_distribution()[1] == '16.04' + dist, ver = get_linux_distro() + return dist == 'Ubuntu' and ver.startswith('16.04') def get_config_build_dir(build_dir, config): # build directory per configuration @@ -243,7 +246,7 @@ def install_ubuntu_deps(args): def install_python_deps(numpy_version=""): dep_packages = ['setuptools', 'wheel', 'pytest'] - dep_packages.append('numpy=={}'.format(numpy_version) if numpy_version else 'numpy>=1.15.0') + dep_packages.append('numpy=={}'.format(numpy_version) if numpy_version else 'numpy>=1.18.0') dep_packages.append('sympy>=1.1') dep_packages.append('packaging') run_subprocess([sys.executable, '-m', 'pip', 'install', '--trusted-host', 'files.pythonhosted.org'] + dep_packages) @@ -300,7 +303,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_ENABLE_PYTHON=" + ("ON" if args.enable_pybind else "OFF"), "-Donnxruntime_BUILD_CSHARP=" + ("ON" if args.build_csharp else "OFF"), "-Donnxruntime_BUILD_JAVA=" + ("ON" if args.build_java else "OFF"), - "-Donnxruntime_BUILD_SHARED_LIB=" + ("ON" if args.build_shared_lib or args.build_server else "OFF"), + "-Donnxruntime_BUILD_SHARED_LIB=" + ("ON" if args.build_shared_lib else "OFF"), "-Donnxruntime_USE_EIGEN_FOR_BLAS=" + ("OFF" if args.use_openblas else "ON"), "-Donnxruntime_USE_OPENBLAS=" + ("ON" if args.use_openblas else "OFF"), "-Donnxruntime_USE_DNNL=" + ("ON" if args.use_dnnl else "OFF"), @@ -321,15 +324,12 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_ENABLE_MICROSOFT_INTERNAL=" + ("ON" if args.enable_msinternal else "OFF"), "-Donnxruntime_USE_BRAINSLICE=" + ("ON" if args.use_brainslice else "OFF"), "-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"), - "-Donnxruntime_USE_EIGEN_THREADPOOL=" + ("ON" if args.use_eigenthreadpool else "OFF"), "-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"), "-Donnxruntime_TENSORRT_HOME=" + (tensorrt_home if args.use_tensorrt else ""), # By default - we currently support only cross compiling for ARM/ARM64 (no native compilation supported through this script) - "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 or args.arm else "OFF"), - "-Donnxruntime_BUILD_SERVER=" + ("ON" if args.build_server else "OFF"), - "-Donnxruntime_BUILD_x86=" + ("ON" if args.x86 else "OFF"), + "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 or args.arm else "OFF"), # nGraph and TensorRT providers currently only supports full_protobuf option. - "-Donnxruntime_USE_FULL_PROTOBUF=" + ("ON" if args.use_full_protobuf or args.use_ngraph or args.use_tensorrt or args.build_server or args.gen_doc else "OFF"), + "-Donnxruntime_USE_FULL_PROTOBUF=" + ("ON" if args.use_full_protobuf or args.use_ngraph or args.use_tensorrt or args.gen_doc else "OFF"), "-Donnxruntime_DISABLE_CONTRIB_OPS=" + ("ON" if args.disable_contrib_ops else "OFF"), "-Donnxruntime_MSVC_STATIC_RUNTIME=" + ("ON" if args.enable_msvc_static_runtime else "OFF"), # enable pyop if it is nightly build @@ -709,70 +709,7 @@ def nuphar_run_python_tests(build_dir, configs): run_subprocess([sys.executable, 'onnxruntime_test_python_nuphar.py'], cwd=cwd, dll_path=dll_path) -def split_server_binary_and_symbol(build_dir, configs): - if is_windows(): - # TODO: Windows support - pass - else: - for config in configs: - if config == 'RelWithDebInfo': - config_build_dir = get_config_build_dir(build_dir, config) - run_subprocess(['objcopy', '--only-keep-debug', 'onnxruntime_server', 'onnxruntime_server.symbol'], cwd=config_build_dir) - run_subprocess(['strip', '--strip-debug', '--strip-unneeded', 'onnxruntime_server'], cwd=config_build_dir) - run_subprocess(['objcopy', '--add-gnu-debuglink=onnxruntime_server.symbol', 'onnxruntime_server'], cwd=config_build_dir) - libonnx = glob.glob(os.path.join(config_build_dir, "libonnxruntime.so.*")) - if len(libonnx) != 1 : - raise ValueError("Too many libonxruntime.so.*") - libonnx = libonnx[0] - run_subprocess(['objcopy', '--only-keep-debug', libonnx, libonnx+'.symbol'], cwd=config_build_dir) - run_subprocess(['strip', '--strip-debug', libonnx], cwd=config_build_dir) - run_subprocess(['objcopy', '--add-gnu-debuglink={}.symbol'.format(libonnx), libonnx], cwd=config_build_dir) - - -def run_server_tests(build_dir, configs): - pip_freeze_result = run_subprocess([sys.executable, '-m', 'pip', 'freeze'], capture=True).stdout - installed_packages = [r.decode().split('==')[0] for r in pip_freeze_result.split()] - if not (('requests' in installed_packages) and ('protobuf' in installed_packages) and ('numpy' in installed_packages) and ('grpcio' in installed_packages)): - if hasattr(sys, 'real_prefix'): - # In virtualenv - run_subprocess([sys.executable, '-m', 'pip', 'install', '--trusted-host', 'files.pythonhosted.org', 'requests', 'protobuf', 'numpy', 'grpcio']) - else: - # Outside virtualenv - run_subprocess([sys.executable, '-m', 'pip', 'install', '--user', '--trusted-host', 'files.pythonhosted.org', 'requests', 'protobuf', 'numpy', 'grpcio']) - for config in configs: - config_build_dir = get_config_build_dir(build_dir, config) - if is_windows(): - server_app_path = os.path.join(config_build_dir, config, 'onnxruntime_server.exe') - python_package_path = os.path.join(config_build_dir, config) - else: - server_app_path = os.path.join(config_build_dir, 'onnxruntime_server') - python_package_path = config_build_dir - server_test_folder = os.path.join(config_build_dir, 'server_test') - server_test_model_folder = os.path.join(build_dir, 'models', 'opset8', 'test_mnist') - server_test_data_folder = os.path.join(config_build_dir, 'testdata', 'server') - run_subprocess([sys.executable, 'test_main.py', server_app_path, server_test_model_folder, server_test_data_folder, python_package_path, server_test_folder], cwd=server_test_folder, dll_path=None) - - -def run_server_model_tests(build_dir, configs): - for config in configs: - config_build_dir = get_config_build_dir(build_dir, config) - server_test_folder = os.path.join(config_build_dir, 'server_test') - server_test_data_folder = os.path.join(config_build_dir, 'server_test_data') - - if is_windows(): - server_app_path = os.path.join(config_build_dir, config, 'onnxruntime_server.exe') - test_raw_data_folder = os.path.join(config_build_dir, 'models') - python_package_path = os.path.join(config_build_dir, config) - else: - server_app_path = os.path.join(config_build_dir, 'onnxruntime_server') - test_raw_data_folder = os.path.join(build_dir, 'models') - python_package_path = config_build_dir - - run_subprocess([sys.executable, 'model_zoo_data_prep.py', test_raw_data_folder, server_test_data_folder, python_package_path, server_test_folder], cwd=server_test_folder, dll_path=None) - run_subprocess([sys.executable, 'model_zoo_tests.py', server_app_path, test_raw_data_folder, server_test_data_folder, python_package_path, server_test_folder], cwd=server_test_folder, dll_path=None) - - -def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_ngraph, use_tensorrt, use_openvino, use_nuphar, nightly_build = False): +def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_ngraph, use_dnnl, use_tensorrt, use_openvino, use_nuphar, nightly_build = False): for config in configs: cwd = get_config_build_dir(build_dir, config) if is_windows(): @@ -786,6 +723,8 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_ngraph, use args.append('--use_cuda') elif use_ngraph: args.append('--use_ngraph') + elif use_dnnl: + args.append('--use_dnnl') elif use_openvino: args.append('--use_openvino') elif use_nuphar: @@ -891,7 +830,7 @@ def main(): if args.use_tensorrt: args.use_cuda = True - if args.build_wheel or args.enable_server_model_tests: + if args.build_wheel: args.enable_pybind = True if args.build_csharp or args.build_java: @@ -1006,7 +945,7 @@ def main(): else: tensorrt_run_onnx_tests(build_dir, configs, "") - if args.use_cuda: + if args.use_cuda and not args.use_tensorrt: run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda', args.enable_multi_device_test, False, 2) if args.use_ngraph: @@ -1033,17 +972,10 @@ def main(): if args.enable_pybind and not args.skip_onnx_tests and args.use_nuphar: nuphar_run_python_tests(build_dir, configs) - if args.build_server: - split_server_binary_and_symbol(build_dir, configs) - if args.enable_server_tests: - run_server_tests(build_dir, configs) - if args.enable_server_model_tests: - run_server_model_tests(build_dir, configs) - if args.build: if args.build_wheel: nightly_build = bool(os.getenv('NIGHTLY_BUILD') == '1') - build_python_wheel(source_dir, build_dir, configs, args.use_cuda, args.use_ngraph, args.use_tensorrt, args.use_openvino, args.use_nuphar, nightly_build) + build_python_wheel(source_dir, build_dir, configs, args.use_cuda, args.use_ngraph, args.use_dnnl, args.use_tensorrt, args.use_openvino, args.use_nuphar, nightly_build) if args.gen_doc and (args.build or args.test): generate_documentation(source_dir, build_dir, configs) diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml index 0b925ed087878..529306c6c5ec6 100644 --- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml +++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml @@ -8,12 +8,19 @@ jobs: Python35: python.version: '3.5' python.dir: '/opt/python/cp35-cp35m' + python.include.dir: '/opt/python/cp35-cp35m/include/python3.5m' Python36: python.version: '3.6' python.dir: '/opt/python/cp36-cp36m' + python.include.dir: '/opt/python/cp36-cp36m/include/python3.6m' Python37: python.version: '3.7' python.dir: '/opt/python/cp37-cp37m' + python.include.dir: '/opt/python/cp37-cp37m/include/python3.7m' + Python38: + python.version: '3.8' + python.dir: '/opt/python/cp38-cp38' + python.include.dir: '/opt/python/cp38-cp38/include/python3.8' steps: - template: templates/set-test-data-variables-step.yml @@ -40,7 +47,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD -e BUILD_BUILDNUMBER onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so workingDirectory: $(Build.SourcesDirectory) - task: CopyFiles@2 @@ -69,12 +76,19 @@ jobs: Python35: python.version: '3.5' python.dir: '/opt/python/cp35-cp35m' + python.include.dir: '/opt/python/cp35-cp35m/include/python3.5m' Python36: python.version: '3.6' python.dir: '/opt/python/cp36-cp36m' + python.include.dir: '/opt/python/cp36-cp36m/include/python3.6m' Python37: python.version: '3.7' python.dir: '/opt/python/cp37-cp37m' + python.include.dir: '/opt/python/cp37-cp37m/include/python3.7m' + Python38: + python.version: '3.8' + python.dir: '/opt/python/cp38-cp38' + python.include.dir: '/opt/python/cp38-cp38/include/python3.8' steps: - template: templates/set-test-data-variables-step.yml @@ -101,7 +115,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD -e BUILD_BUILDNUMBER onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 workingDirectory: $(Build.SourcesDirectory) - task: CopyFiles@2 @@ -131,6 +145,8 @@ jobs: python.version: '3.6' Python37: python.version: '3.7' + Python38: + python.version: '3.8' variables: OrtPackageId: 'Microsoft.ML.OnnxRuntime' MsbuildArguments: '-maxcpucount' @@ -158,7 +174,7 @@ jobs: workingFolder: '$(Build.BinariesDirectory)' - script: | - python -m pip install -q pyopenssl setuptools wheel numpy + python -m pip install -q pyopenssl setuptools wheel numpy==1.18 workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' @@ -174,13 +190,13 @@ jobs: displayName: 'BUILD' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --enable_onnx_tests --parallel' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests --parallel' workingDirectory: '$(Build.BinariesDirectory)' - task: CopyFiles@2 displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: - SourceFolder: '$(Build.BinariesDirectory)\$(BuildConfig)' + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' Contents: '**\dist\*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' @@ -198,11 +214,11 @@ jobs: - job: Windows_py_GPU_Wheels workspace: clean: all - pool: Win-GPU-CUDA10 - timeoutInMinutes: 120 + pool: 'Win-GPU-2019' + timeoutInMinutes: 60 variables: - buildDirectory: '$(Build.SourcesDirectory)\build' CUDA_VERSION: '10.0' + EnvSetupScript: setup_env.bat strategy: matrix: Python35: @@ -217,30 +233,39 @@ jobs: versionSpec: $(python.version) addToPath: true architecture: 'x64' + - task: BatchScript@1 - displayName: 'Setup VS2017 env vars' + displayName: 'setup env' inputs: - filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat' - arguments: 'amd64' + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' modifyEnvironment: true - - task: PowerShell@1 - displayName: 'Set CUDA path' + workingFolder: '$(Build.BinariesDirectory)' + + + - script: | + python -m pip install -q pyopenssl setuptools wheel numpy==1.18 + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + + - template: templates/set-test-data-variables-step.yml + - task: PythonScript@0 + displayName: 'Download test data' inputs: - scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1' - arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-10.0.130-win10 -CudaVersion 10.0' + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) + workingDirectory: $(Build.BinariesDirectory) - - task: BatchScript@1 - displayName: 'Run build script' + - task: PythonScript@0 + displayName: 'build' inputs: - filename: 'build.bat' - arguments: ' --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" - --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --build_dir $(buildDirectory) --config Release --build_wheel' - workingFolder: "$(Build.SourcesDirectory)" + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: --config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_generator "Visual Studio 16 2019" --msvc_toolset 14.16 --build_wheel --enable_onnx_tests --parallel --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows10-x64-v7.3.1.20\cuda" + workingDirectory: '$(Build.BinariesDirectory)' - task: CopyFiles@2 displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: - SourceFolder: '$(buildDirectory)' + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' Contents: '**\dist\*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' @@ -249,11 +274,6 @@ jobs: inputs: ArtifactName: onnxruntime_gpu - - task: PowerShell@1 - displayName: 'Clean up CUDA props files' - inputs: - scriptName: 'tools/ci_build/github/windows/clean_up_cuda_prop_files.ps1' - arguments: '-CudaVersion 10.0' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' @@ -272,6 +292,8 @@ jobs: python.version: '3.6' Python37: python.version: '3.7' + Python38: + python.version: '3.8' steps: - task: UsePythonVersion@0 displayName: 'Use Python' diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml index 9f18e3d0d3dbd..b04222df0a45e 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml @@ -93,9 +93,6 @@ jobs: msbuildArch: amd64 msbuildPlatform: x64 buildparameter: - variables: - buildDirectory: '$(Build.BinariesDirectory)' - buildArch: 'x64' steps: - task: UsePythonVersion@0 @@ -161,49 +158,78 @@ jobs: commitId: $(OnnxRuntimeGitCommitHash) - template: templates/clean-agent-build-directory-step.yml -- job: Windows_Packaging_GPU_x64 + +- job: Windows_Packaging_GPU workspace: clean: all - timeoutInMinutes: 120 - pool: 'Win-GPU-CUDA10' + pool: 'Win-GPU-2019' + timeoutInMinutes: 120 variables: - buildDirectory: '$(Build.BinariesDirectory)' - buildConfig: 'RelWithDebInfo' - buildArch: 'x64' - + EnvSetupScript: setup_env.bat + buildArch: x64 + msbuildArch: amd64 + msbuildPlatform: x64 + CUDA_VERSION: '10.0' + buildparameter: --msvc_toolset 14.16 --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows10-x64-v7.3.1.20\cuda" steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: $(buildArch) + + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + + - script: | + python -m pip install -q pyopenssl setuptools wheel numpy + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + - template: templates/set-test-data-variables-step.yml - template: templates/set-version-number-variables-step.yml + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) + workingDirectory: $(Build.BinariesDirectory) - - template: templates/windows-build-tools-setup-steps.yml - parameters: - EnvSetupScript: 'setup_env.bat' - buildArch: 'amd64' # amd64 is needed for vcvars target arch - setVcvars: true - - task: PowerShell@1 - displayName: 'Set CUDA path' + - task: PythonScript@0 + displayName: 'Generate cmake config' inputs: - scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1' - arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-10.0.130-win10 -CudaVersion 10.0' + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' + workingDirectory: '$(Build.BinariesDirectory)' - - task: CmdLine@2 - displayName: 'Build and Test OnnxRuntime' + + - task: VSBuild@1 + displayName: 'Build' inputs: - script: | - $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + platform: $(msbuildPlatform) + configuration: RelWithDebInfo + msbuildArchitecture: $(buildArch) + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true + + - task: PythonScript@0 + displayName: 'test' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' workingDirectory: '$(Build.BinariesDirectory)' - template: templates/c-api-artifacts-package-and-publish-steps-windows.yml parameters: - buildConfig: $(buildConfig) + buildConfig: RelWithDebInfo artifactName: 'onnxruntime-win-$(buildArch)-gpu-$(OnnxRuntimeVersion)' commitId: $(OnnxRuntimeGitCommitHash) - - - task: PowerShell@1 - displayName: 'Clean up CUDA props files' - inputs: - scriptName: 'tools/ci_build/github/windows/clean_up_cuda_prop_files.ps1' - arguments: '-CudaVersion 10.0' - - template: templates/clean-agent-build-directory-step.yml \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml index 2786df3c4e905..6d9b79a274890 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml @@ -14,6 +14,12 @@ jobs: continueOnError: true condition: always() + - task: CmdLine@2 + displayName: 'Build docker image' + inputs: + script: docker build --pull -t onnxruntime-server-ubuntu16.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=3.5 -f Dockerfile.ubuntu_server . + workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker + - task: CmdLine@2 displayName: 'Download azcopy' inputs: @@ -30,8 +36,11 @@ jobs: pythonInterpreter: '/usr/bin/python3' workingDirectory: $(Build.BinariesDirectory) - - script: 'tools/ci_build/github/linux/server_run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--config Debug Release --build_server --use_openmp --use_full_protobuf --enable_server_tests"' - displayName: 'Run build script' + - task: CmdLine@2 + displayName: 'Run docker image' + inputs: + script: docker run --rm --volume $(Build.SourcesDirectory)/server:/onnxruntime_src --volume $(Build.BinariesDirectory):/build onnxruntime-server-ubuntu16.04 /bin/bash /onnxruntime_src/ci/run.sh + workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' diff --git a/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml index 39aca7ec08596..5455b55a1639e 100644 --- a/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-nocontribops-ci-pipeline.yml @@ -1,35 +1,6 @@ -variables: - DisableContribOps: ON - TestDataUrlNoContribOps : https://onnxruntimetestdata.blob.core.windows.net/models/20181210.zip - jobs: -- job: MacOS_CI_Dev - pool: - vmImage: 'macOS-10.13' - steps: - - template: templates/set-test-data-variables-step.yml - - task: CmdLine@2 - displayName: 'Download azcopy' - inputs: - script: | - curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-mac' - tar -zxvf azcopy.tar.gz --strip 1 - workingDirectory: $(Build.BinariesDirectory) - - task: PythonScript@0 - displayName: 'Download test data' - inputs: - scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrlNoContribOps) --azure_region centralus --build_dir $(Build.BinariesDirectory) - pythonInterpreter: '/usr/local/bin/python3' - workingDirectory: $(Build.BinariesDirectory) - - script: | - sudo python3 -m pip install numpy==1.15.0 - sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --enable_onnx_tests --skip_submodule_sync --parallel --build_shared_lib --disable_contrib_ops --config Debug Release - displayName: 'Build and Test OnnxRuntime lib for MacOS' - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) - - - template: templates/clean-agent-build-directory-step.yml +- template: templates/mac-ci.yml + parameters: + AgentPool : 'Hosted macOS High Sierra' + DoNugetPack: 'false' + BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --build_wheel --skip_submodule_sync --parallel --build_shared_lib --disable_contrib_ops --enable_onnx_tests --config Debug RelWithDebInfo' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml index 062ff28790fba..64f82c148364c 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml @@ -1,6 +1,5 @@ # Defined as pipeline variables # variables: -# AgentPoolWin : 'Win-CPU' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-esrp-pipeline.yml index aac065f365019..e7d942c4ab57a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-esrp-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-esrp-pipeline.yml @@ -1,6 +1,5 @@ # Defined as pipeline variables # variables: -# AgentPoolWin : 'Win-CPU' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' @@ -18,5 +17,5 @@ variables: jobs: - template: templates/cpu-mklml.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' DoEsrp: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-pipeline.yml index 135469669c7ed..29581701c71e2 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-pipeline.yml @@ -1,6 +1,5 @@ # Defined as pipeline variables # variables: -# AgentPoolWin : 'Win-CPU' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' @@ -10,5 +9,5 @@ variables: jobs: - template: templates/cpu-mklml.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-esrp-pipeline.yml index 56c9a86167b74..773295e3d23c3 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-esrp-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-esrp-pipeline.yml @@ -1,6 +1,5 @@ # Defined as pipeline variables # variables: -# AgentPoolWin : 'Win-CPU' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' @@ -18,5 +17,5 @@ variables: jobs: - template: templates/cpu-nocontribops-arm64.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' DoEsrp: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-pipeline.yml index b09b21ef1ca44..ba0a94e3bd105 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-arm64-pipeline.yml @@ -1,6 +1,5 @@ # Defined as pipeline variables # variables: -# AgentPoolWin : 'Win-CPU' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' @@ -10,5 +9,5 @@ variables: jobs: - template: templates/cpu-nocontribops-arm64.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml index 82b795c05fcc4..917c83a6393b1 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml @@ -19,5 +19,5 @@ variables: jobs: - template: templates/gpu.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-GPU-2019' DoEsrp: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml index ef040845eefe1..5057490121b9a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml @@ -1,6 +1,5 @@ # Defined as pipeline variables # variables: -# AgentPoolWin : 'Win-CPU-CUDA10' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' @@ -11,5 +10,5 @@ variables: jobs: - template: templates/gpu.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-GPU-2019' DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml index 5d25a5d29f3e8..9d1ff3e8673a7 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml @@ -1,6 +1,5 @@ # Defined as pipeline variables # variables: -# AgentPoolWin : 'Win-CPU' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' @@ -13,6 +12,10 @@ jobs: AgentPool : 'Win-CPU-2019' JobName: 'Windows_CI_Dev' BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_mklml --build_shared_lib --enable_onnx_tests --cmake_generator "Visual Studio 16 2019"' + BuildArch: 'x64' + msbuildArchitecture: 'amd64' + EnvSetupScript: 'setup_env.bat' + sln_platform: 'x64' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' @@ -163,6 +166,12 @@ jobs: FolderPath: '$(Build.ArtifactStagingDirectory)' DoEsrp: ${{ parameters.DoEsrp }} + - template: ../../templates/validate-nuget.yml + parameters: + NugetPath: '$(Build.ArtifactStagingDirectory)' + PlatformsSupported: 'win-x64,linux-x64,osx-x64' + VerifyNugetSigning: ${{ parameters.DoEsrp }} + - task: PublishPipelineArtifact@0 displayName: 'Publish Pipeline NuGet Artifact' inputs: diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml index 6aa762839af33..7e2bdaf14b3f4 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml @@ -1,21 +1,25 @@ # Defined as pipeline variables # variables: -# AgentPoolWin : 'Win-CPU' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' parameters: DoEsrp: 'false' -jobs: -- template: ../../templates/win-ci.yml +jobs: +- template: ../../templates/win-ci-2019.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' + ArtifactName: 'drop-nuget' JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests' + BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --use_telemetry --cmake_generator "Visual Studio 16 2019" --disable_contrib_ops --enable_msvc_static_runtime' + BuildArch: 'x64' + msbuildArchitecture: 'amd64' + EnvSetupScript: 'setup_env.bat' + sln_platform: 'x64' DoDebugBuild: 'false' DoNugetPack : 'true' - DoCompliance: 'false' + DoCompliance: ${{ parameters.DoCompliance }} DoEsrp: ${{ parameters.DoEsrp }} NuPackScript: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage @@ -23,11 +27,16 @@ jobs: mkdir $(Build.ArtifactStagingDirectory)\testdata copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata -- template: ../../templates/win-x86-ci.yml +- template: ../../templates/win-ci-2019.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' + ArtifactName: 'drop-win-x86-zip' JobName: 'Windows_CI_Dev_x86' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --x86' + BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --x86 --use_telemetry --cmake_generator "Visual Studio 16 2019" --disable_contrib_ops --enable_msvc_static_runtime' + BuildArch: 'x86' + msbuildArchitecture: 'x86' + EnvSetupScript: 'setup_env_x86.bat' + sln_platform: 'Win32' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' @@ -42,7 +51,7 @@ jobs: - template: ../../templates/win-ci-arm.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' JobName: 'Windows_Arm64_Dev' BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --arm64' DoDebugBuild: 'false' @@ -76,7 +85,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --enable_onnx_tests --disable_contrib_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --enable_onnx_tests --disable_contrib_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x @@ -102,7 +111,7 @@ jobs: parameters: AgentPool : $(AgentPoolMacOS) JobName: 'MacOS_CI_Dev' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --disable_contrib_ops --use_openmp --enable_onnx_tests --config RelWithDebInfo' + BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --disable_contrib_ops --enable_onnx_tests --config RelWithDebInfo' DoNugetPack : 'true' NuPackScript: | set -e -x @@ -123,7 +132,7 @@ jobs: - job: NuGet_Packaging workspace: clean: all - pool: $(AgentPoolWin) + pool: 'Win-CPU-2019' dependsOn: - Windows_CI_Dev - Windows_CI_Dev_x86 @@ -214,6 +223,12 @@ jobs: FolderPath: '$(Build.ArtifactStagingDirectory)' DoEsrp: ${{ parameters.DoEsrp }} + - template: ../../templates/validate-nuget.yml + parameters: + NugetPath: '$(Build.ArtifactStagingDirectory)' + PlatformsSupported: 'win-x64,win-x86,win10-arm,linux-x64,osx-x64' + VerifyNugetSigning: ${{ parameters.DoEsrp }} + - task: PublishPipelineArtifact@0 displayName: 'Publish Pipeline NuGet Artifact' inputs: diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index f55d35ca57da4..b1239966d645c 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -5,14 +5,20 @@ parameters: DoEsrp: 'false' + PackageName: 'Microsoft.ML.OnnxRuntime' DoCompliance: 'false' jobs: - template: ../../templates/win-ci-2019.yml parameters: AgentPool : 'Win-CPU-2019' + ArtifactName: 'drop-nuget' JobName: 'Windows_CI_Dev' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' + BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --use_telemetry --cmake_generator "Visual Studio 16 2019"' + BuildArch: 'x64' + msbuildArchitecture: 'amd64' + EnvSetupScript: 'setup_env.bat' + sln_platform: 'x64' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: ${{ parameters.DoCompliance }} @@ -23,11 +29,16 @@ jobs: mkdir $(Build.ArtifactStagingDirectory)\testdata copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata -- template: ../../templates/win-x86-ci-2019.yml +- template: ../../templates/win-ci-2019.yml parameters: AgentPool : 'Win-CPU-2019' + ArtifactName: 'drop-win-x86-zip' JobName: 'Windows_CI_Dev_x86' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --x86 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' + BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --x86 --use_telemetry --cmake_generator "Visual Studio 16 2019"' + BuildArch: 'x86' + msbuildArchitecture: 'x86' + EnvSetupScript: 'setup_env_x86.bat' + sln_platform: 'Win32' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' @@ -55,7 +66,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_featurizers --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x @@ -81,7 +92,7 @@ jobs: parameters: AgentPool : $(AgentPoolMacOS) JobName: 'MacOS_CI_Dev' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' + BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --use_featurizers --use_openmp --enable_onnx_tests --config RelWithDebInfo' DoNugetPack : 'true' NuPackScript: | set -e -x @@ -116,7 +127,6 @@ jobs: inputs: artifactName: 'drop-nuget' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - task: DownloadPipelineArtifact@0 @@ -124,21 +134,18 @@ jobs: inputs: artifactName: 'drop-win-x86-zip' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - Linux' inputs: artifactName: 'drop-linux' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - MacOS' inputs: artifactName: 'drop-osx' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - template: bundle_dlls.yml @@ -148,6 +155,12 @@ jobs: FolderPath: '$(Build.ArtifactStagingDirectory)' DoEsrp: ${{ parameters.DoEsrp }} + - template: ../../templates/validate-nuget.yml + parameters: + NugetPath: '$(Build.ArtifactStagingDirectory)' + PlatformsSupported: 'win-x64,win-x86,linux-x64,osx-x64' + VerifyNugetSigning: ${{ parameters.DoEsrp }} + - task: PublishPipelineArtifact@0 displayName: 'Publish Pipeline NuGet Artifact' inputs: diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index 7a89dc24c1728..fd7e38df2a600 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -1,6 +1,5 @@ # Defined as pipeline variables # variables: -# AgentPoolWin : 'Win-CPU-CUDA10' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' @@ -9,21 +8,21 @@ parameters: PackageName: 'Microsoft.ML.OnnxRuntime.Gpu' jobs: -- template: ../../templates/win-ci.yml +- template: ../../templates/win-ci-2019.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-GPU-2019' + ArtifactName: 'drop-nuget' JobName: 'Windows_CI_GPU_CUDA_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --cuda_version 10.0' + BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --use_featurizers --enable_onnx_tests --use_telemetry --cmake_generator "Visual Studio 16 2019" --msvc_toolset 14.16 --use_cuda --cuda_version=10.0 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" + BuildArch: 'x64' + msbuildArchitecture: 'amd64' + EnvSetupScript: 'setup_env.bat' + sln_platform: 'x64' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' - DoEsrp: ${{ parameters.DoEsrp }} - BuildArch: 'amd64' - SetVcvars: 'true' - MsbuildArguments: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\' - EnvSetupScript: 'setup_env_cuda.bat' + DoEsrp: ${{ parameters.DoEsrp }} CudaVersion: '10.0' - ArtifactName: drop-nuget-cuda OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu' NuPackScript: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu @@ -62,7 +61,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_featurizers --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" - script: | set -e -x mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 @@ -84,7 +83,7 @@ jobs: - template: ../../templates/clean-agent-build-directory-step.yml - job: NuGet_Packaging - pool: $(AgentPoolWin) + pool: 'Win-GPU-2019' dependsOn: - Windows_CI_GPU_CUDA_Dev - Windows_CI_GPU_DML_Dev @@ -96,7 +95,6 @@ jobs: inputs: artifactName: 'drop-nuget' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - NuGet' @@ -110,7 +108,6 @@ jobs: inputs: artifactName: 'drop-linux' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - continueOnError: true - script: | pushd $(Build.BinariesDirectory)\nuget-artifact @@ -141,6 +138,12 @@ jobs: FolderPath: '$(Build.ArtifactStagingDirectory)' DoEsrp: ${{ parameters.DoEsrp }} + - template: ../../templates/validate-nuget.yml + parameters: + NugetPath: '$(Build.ArtifactStagingDirectory)' + PlatformsSupported: 'win-x64,linux-x64' + VerifyNugetSigning: ${{ parameters.DoEsrp }} + - task: PublishPipelineArtifact@0 displayName: 'Publish Pipeline NuGet Artifact' inputs: @@ -149,7 +152,7 @@ jobs: - template: test_win.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-GPU-2019' - template: test_linux.yml parameters: AgentPool : $(AgentPoolLinux) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml index 8a60f4a2675ef..1eac74b5fa532 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_all_os.yml @@ -2,7 +2,7 @@ jobs: - template: test_win.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' - template: test_linux.yml parameters: AgentPool : $(AgentPoolLinux) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml index e4fc8b2e2d018..c2b426b609528 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml @@ -25,9 +25,9 @@ jobs: - template: win-download-test-data.yml - task: BatchScript@1 - displayName: 'Setup VS2017 env vars' + displayName: 'Setup VS2019 env vars' inputs: - filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat' + filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat' arguments: 'amd64' modifyEnvironment: true diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml index 82cb2edab499e..cf3b4506c1f18 100644 --- a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml @@ -1,35 +1,72 @@ # Jobs to run on master after every successful merge is committed jobs: - job: Windows_Debug_CPU_x64 - pool: 'Win-CPU' + pool: 'Win-CPU-2019' timeoutInMinutes: 120 variables: - group: dashboard_mysql_secret - name: buildDirectory value: '$(Build.BinariesDirectory)' - - name: buildConfig + - name: BuildConfig value: 'Debug' - name: buildArch value: 'x64' + - name: EnvSetupScript + value: 'setup_env.bat' steps: - template: templates/set-test-data-variables-step.yml - template: templates/set-version-number-variables-step.yml - - template: templates/windows-build-tools-setup-steps.yml - parameters: - EnvSetupScript: 'setup_env.bat' - buildArch: 'amd64' # amd64 is needed for vcvars target arch - setVcvars: false + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: $(buildArch) - - template: templates/windows-build-and-test-steps.yml - parameters: - buildAdditionalParams: ' --use_openmp ' - # --use_openmp is added, similar to the build config of the C-api package distribution. - # MLAS(default) with OpenMP is known to produce better performance (latency) than without OpenMP. - buildArch: $(buildArch) - msbuildPlatform: $(buildArch) - buildConfig: $(buildConfig) + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) + workingDirectory: $(Build.BinariesDirectory) + + + - powershell: 'Start-Process -FilePath "$(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe" -ArgumentList ''/SP-'', ''/VERYSILENT'', ''/SUPPRESSMSGBOXES'', ''/NORESTART'', ''/DIR="$(Build.BinariesDirectory)\OpenCppCoverage'' -NoNewWindow -Wait' + displayName: 'Run OpenCPPCoverage installer' + + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests' + workingDirectory: '$(Build.BinariesDirectory)' + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln' + platform: 'x64' + configuration: $(BuildConfig) + msbuildArchitecture: $(buildArch) + maximumCpuCount: true + logProjectEvents: false + workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)' + createLogFile: true + + - task: PythonScript@0 + displayName: 'test' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests' + workingDirectory: '$(Build.BinariesDirectory)' - template: templates/windows-code-coverage-steps.yml parameters: @@ -38,5 +75,3 @@ jobs: PublishReport: true PostToDashboard: true - - template: templates/clean-agent-build-directory-step.yml - diff --git a/tools/ci_build/github/azure-pipelines/templates/compliance.yml b/tools/ci_build/github/azure-pipelines/templates/compliance.yml index f96857856bc4a..87e3c21bfa6e7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/compliance.yml +++ b/tools/ci_build/github/azure-pipelines/templates/compliance.yml @@ -11,6 +11,15 @@ steps: arguments: 'analyze $(Build.BinariesDirectory)\RelWithDebInfo\*.dll --recurse --verbose' continueOnError: true +- task: DeleteFiles@1 + displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + Contents: | + **/*.obj + **/*.pdb + **/*.dll + - task: securedevelopmentteam.vss-secure-development-tools.build-task-prefast.SDLNativeRules@2 displayName: 'Run the PREfast SDL Native Rules for MSBuild' continueOnError: true diff --git a/tools/ci_build/github/azure-pipelines/templates/set-test-data-variables-step.yml b/tools/ci_build/github/azure-pipelines/templates/set-test-data-variables-step.yml index 658e3515bd44c..abec88edb183f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/set-test-data-variables-step.yml +++ b/tools/ci_build/github/azure-pipelines/templates/set-test-data-variables-step.yml @@ -1,18 +1,11 @@ -# sets variables $(TestDataUrl) and $(TestDataChecksum) +# sets variables $(TestDataUrl) parameters: - TestDataUrl: https://onnxruntimetestdata.blob.core.windows.net/models/20190925.zip - TestDataChecksum: ae5fb5e3dd5e4937c8343d13e3be680c + TestDataUrl: https://onnxruntimetestdata.blob.core.windows.net/models/20191107.zip steps: - task: CmdLine@1 displayName: 'Set TestDataUrl variable' inputs: filename: echo - arguments: '##vso[task.setvariable variable=TestDataUrl;]${{parameters.TestDataUrl}}' - -- task: CmdLine@1 - displayName: 'Set TestDataChecksum variable' - inputs: - filename: echo - arguments: '##vso[task.setvariable variable=TestDataChecksum;]${{parameters.TestDataChecksum}}' + arguments: '##vso[task.setvariable variable=TestDataUrl;]${{parameters.TestDataUrl}}' \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/validate-nuget.yml b/tools/ci_build/github/azure-pipelines/templates/validate-nuget.yml new file mode 100644 index 0000000000000..a162219f5bd9c --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/validate-nuget.yml @@ -0,0 +1,17 @@ +parameters: + NugetPath: '' + PlatformsSupported: '' + VerifyNugetSigning: '' + +steps: + - task: UsePythonVersion@0 + displayName: 'Use Python' + inputs: + versionSpec: 3.7 + + - task: PythonScript@0 + displayName: 'Validate Nuget' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\nuget\validate_nuget.py' + arguments: '--nuget_path ${{parameters.NugetPath}} --platforms_supported ${{parameters.PlatformsSupported}} --verify_nuget_signing ${{parameters.VerifyNugetSigning}}' + workingDirectory: "$(Build.BinariesDirectory)" \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml index c018d6f02089e..b32e6958cf8d7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml @@ -8,12 +8,12 @@ parameters: ArtifactName: 'drop-nuget' DoEsrp: 'false' DoTestCoverage: 'false' - BuildArch: 'x64' - SetVcvars: 'false' - msbuildArchitecture: 'amd64' + BuildArch: 'x64' # Optional. Options: x86, x64 + msbuildArchitecture: 'amd64' # Options: x86, amd64 + sln_platform: 'x64' # Options: Win32, x64 EnvSetupScript: 'setup_env.bat' CudaVersion: '' - AgentPool: 'Win-CPU' + AgentPool: 'Win-CPU-2019' AgentDemands: [] OrtPackageId: Microsoft.ML.OnnxRuntime jobs: @@ -30,6 +30,9 @@ jobs: OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' DotNetExe: 'dotnet.exe' CUDA_VERSION: ${{ parameters.CudaVersion }} + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + PlatformTarget: ${{ parameters.BuildArch }} + RuntimeIdentifier: win-${{ parameters.BuildArch }} steps: - powershell: | @@ -38,9 +41,12 @@ jobs: $length = $env:TELEMETRYGUID.length $fileContent = "#define ENABLE_TELEMETRY`n#define TraceLoggingOptionMicrosoftTelemetry() \ TraceLoggingOptionGroup("+$env:TELEMETRYGUID.substring(1, $length-2)+")" - New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force + New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force + Write-Output "Enabling TELEMETRY" } displayName: 'Create TraceLoggingConfigPrivate.h For WinML Telemetry' + env: + TELEMETRYGUID: $(TELEMETRYGUID) - task: UsePythonVersion@0 inputs: versionSpec: '3.7' @@ -71,14 +77,13 @@ jobs: displayName: 'Build' inputs: solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' - platform: 'x64' + platform: ${{ parameters.sln_platform }} configuration: RelWithDebInfo msbuildArchitecture: ${{ parameters.BuildArch }} maximumCpuCount: true logProjectEvents: true workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' createLogFile: true - clean: true # Build RelWithDebInfo -- this variable required to build C# - script: | diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 92ebe320a26de..033923dc34ebb 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -14,7 +14,7 @@ parameters: MsbuildArguments: '/m' EnvSetupScript: 'setup_env.bat' CudaVersion: '' - AgentPool: 'Win-CPU' + AgentPool: 'Win-CPU-2019' AgentDemands: [] OrtPackageId: Microsoft.ML.OnnxRuntime jobs: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-x86-ci-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-x86-ci-2019.yml deleted file mode 100644 index 167d1c0b2f5de..0000000000000 --- a/tools/ci_build/github/azure-pipelines/templates/win-x86-ci-2019.yml +++ /dev/null @@ -1,190 +0,0 @@ -parameters: - DoDebugBuild: 'true' - DoCompliance: 'false' - BuildCommand: '' - JobName: 'Windows_CI_Dev_x86' - DoNugetPack: 'false' - NuPackScript : '' - ArtifactName: 'drop-win-x86-zip' - DoEsrp: 'false' - DoTestCoverage: 'false' - BuildArch: 'x86' - SetVcvars: 'false' - msbuildArchitecture: 'x86' - EnvSetupScript: 'setup_env_x86.bat' - CudaVersion: '' - AgentPool: 'Win-CPU' - AgentDemands: [] - OrtPackageId: Microsoft.ML.OnnxRuntime -jobs: -- job: ${{ parameters.JobName }} - timeoutInMinutes: 120 - workspace: - clean: all - pool: - name: ${{ parameters.AgentPool }} - demands: ${{ parameters.AgentDemands }} - variables: - buildDirectory: '$(Build.BinariesDirectory)' - BuildCommand: ${{ parameters.BuildCommand }} - OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' - DotNetExe: 'dotnet.exe' - CUDA_VERSION: ${{ parameters.CudaVersion }} - PlatformTarget: x86 - RuntimeIdentifier: win-x86 - - steps: - - powershell: | - if($env:TELEMETRYGUID) - { - $length = $env:TELEMETRYGUID.length - $fileContent = "#define ENABLE_TELEMETRY`n#define TraceLoggingOptionMicrosoftTelemetry() \ - TraceLoggingOptionGroup("+$env:TELEMETRYGUID.substring(1, $length-2)+")" - New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force - } - displayName: 'Create TraceLoggingConfigPrivate.h For WinML Telemetry' - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: ${{ parameters.BuildArch }} - - - task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' - - - script: | - python -m pip install -q pyopenssl setuptools wheel numpy - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' - - - - task: PythonScript@0 - displayName: 'Generate cmake config' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '$(BuildCommand) --update --config RelWithDebInfo' - workingDirectory: '$(Build.BinariesDirectory)' - - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' - platform: 'Win32' - configuration: RelWithDebInfo - msbuildArchitecture: ${{ parameters.BuildArch }} - maximumCpuCount: true - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - createLogFile: true - clean: true - - # Build RelWithDebInfo -- this variable required to build C# - - script: | - @echo ##vso[task.setvariable variable=Configuration]RelWithDebInfo - - - template: set-test-data-variables-step.yml - - - task: NuGetToolInstaller@0 - displayName: Use Nuget 4.9 - inputs: - versionSpec: 4.9.4 - - - task: PythonScript@0 - displayName: 'Download test data' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' - arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) - workingDirectory: $(Build.BinariesDirectory) - - - task: DotNetCoreCLI@2 - displayName: 'Restore nuget packages' - inputs: - command: restore - projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - configuration: 'RelWithDebInfo' - arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=${{ parameters.OrtPackageId }}' - workingDirectory: '$(Build.SourcesDirectory)\csharp' - - - task: DotNetCoreCLI@2 - displayName: 'Build C#' - inputs: - command: build - projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - configuration: 'RelWithDebInfo' - arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}' - workingDirectory: '$(Build.SourcesDirectory)\csharp' - - - task: DotNetCoreCLI@2 - displayName: 'Test C#' - inputs: - command: test - projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj' - configuration: 'RelWithDebInfo' - arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}' - workingDirectory: '$(Build.SourcesDirectory)\csharp' - - - script: | - mklink /D /J $(Build.BinariesDirectory)\RelWithDebInfo\models $(Build.BinariesDirectory)\models - DIR dist\ /S /B > wheel_filename_file - set /p WHEEL_FILENAME= wheel_filename_file + set /p WHEEL_FILENAME=GetAllocator(0, ::OrtMemType::OrtMemTypeDefault); const auto& info = allocator->Info(); - *memory_info = new (std::nothrow) OrtMemoryInfo(info.name, info.type, info.device, info.id, info.mem_type); + *memory_info = new (std::nothrow) OrtMemoryInfo(info.name, info.alloc_type, info.device, info.id, info.mem_type); if (*memory_info == nullptr) { return OrtApis::CreateStatus(ORT_FAIL, "Out of memory"); } @@ -81,7 +81,7 @@ ORT_API_STATUS_IMPL(winmla::GetValueMemoryInfo, const OrtValue* value, OrtMemory API_IMPL_BEGIN const auto& tensor = value->Get(); auto info = tensor.Location(); - *memory_info = new OrtMemoryInfo(info.name, info.type, info.device, info.id, info.mem_type); + *memory_info = new OrtMemoryInfo(info.name, info.alloc_type, info.device, info.id, info.mem_type); if (*memory_info == nullptr) { return OrtApis::CreateStatus(ORT_FAIL, "Out of memory"); } diff --git a/winml/lib/Api.Image/D3DDeviceCache.cpp b/winml/lib/Api.Image/D3DDeviceCache.cpp index a551bb96562c5..1b6314cd8a0bb 100644 --- a/winml/lib/Api.Image/D3DDeviceCache.cpp +++ b/winml/lib/Api.Image/D3DDeviceCache.cpp @@ -6,6 +6,7 @@ #include #include #include "inc/DeviceHelpers.h" +#include "CommonDeviceHelpers.h" namespace float32 { #include "shaders\SurfaceToTensor-SurfaceToTensorBGR8.h" @@ -50,8 +51,8 @@ D3DDeviceCache::D3DDeviceCache(Windows::AI::MachineLearning::LearningModelDevice DXGI_GPU_PREFERENCE preference; WINML_THROW_IF_FAILED(DeviceHelpers::GetGPUPreference(deviceKind, &preference)); - DeviceHelpers::AdapterEnumerationSupport support; - WINML_THROW_IF_FAILED(DeviceHelpers::GetAdapterEnumerationSupport(&support)); + CommonDeviceHelpers::AdapterEnumerationSupport support; + WINML_THROW_IF_FAILED(CommonDeviceHelpers::GetAdapterEnumerationSupport(&support)); const char errStr[] = "No hardware adapters available"; if (support.has_dxgi) { @@ -130,7 +131,7 @@ D3DDeviceCache::~D3DDeviceCache() { bool D3DDeviceCache::IsFloat16Supported() { if (device_ != nullptr) { - return DeviceHelpers::IsFloat16Supported(device_.get()); + return CommonDeviceHelpers::IsFloat16Supported(device_.get()); } return true; @@ -671,4 +672,4 @@ void D3DDeviceCache::SyncD3D11DeviceToConverter(_In_ ID3D11Fence* pD3D11Fence) { bool D3DDeviceCache::SharedHandleInitialized() { return d3d11_fence_ != nullptr; } -} // namespace winrt::Windows::AI::MachineLearning::implementation \ No newline at end of file +} // namespace winrt::Windows::AI::MachineLearning::implementation diff --git a/winml/lib/Api.Image/DeviceHelpers.cpp b/winml/lib/Api.Image/DeviceHelpers.cpp index 67824624a7b0a..6e0b5a7ffd946 100644 --- a/winml/lib/Api.Image/DeviceHelpers.cpp +++ b/winml/lib/Api.Image/DeviceHelpers.cpp @@ -9,210 +9,10 @@ #include #include #include "inc/DeviceHelpers.h" +#include "CommonDeviceHelpers.h" +#include "LearningModelDevice.h" namespace DeviceHelpers { -constexpr uint32_t c_intelVendorId = 0x8086; -constexpr uint32_t c_nvidiaVendorId = 0x10DE; -constexpr uint32_t c_amdVendorId = 0x1002; -static std::optional s_adapterEnumerationSupport; - -static bool CheckAdapterFP16Blocked(bool isMcdmAdapter, uint32_t vendorId, uint32_t majorVersion, uint32_t minorVersion) { - switch (vendorId) { - case c_intelVendorId: { - if (isMcdmAdapter) { - return false; - } - - // Check Intel GPU driver version - return (majorVersion < 25) || (majorVersion == 25 && minorVersion < 6574) || (majorVersion == 26 && minorVersion < 6572); - } - } - return false; -} - -static void ParseDriverVersion(LARGE_INTEGER& version, uint32_t& majorVersion, uint32_t& minorVersion) { - majorVersion = HIWORD(version.HighPart); - minorVersion = LOWORD(version.LowPart); -} - -static HRESULT GetDXGIAdapterMetadata(ID3D12Device& device, uint32_t& vendorId, uint32_t& majorVersion, uint32_t& minorVersion) { - winrt::com_ptr spFactory; - RETURN_IF_FAILED(CreateDXGIFactory1(IID_PPV_ARGS(spFactory.put()))); - - winrt::com_ptr spAdapter; - RETURN_IF_FAILED(spFactory->EnumAdapterByLuid(device.GetAdapterLuid(), IID_PPV_ARGS(spAdapter.put()))); - - DXGI_ADAPTER_DESC adapterDesc = {}; - RETURN_IF_FAILED(spAdapter->GetDesc(&adapterDesc)); - - LARGE_INTEGER driverVersion; - RETURN_IF_FAILED(spAdapter->CheckInterfaceSupport(__uuidof(IDXGIDevice), &driverVersion)); - - vendorId = adapterDesc.VendorId; - ParseDriverVersion(driverVersion, majorVersion, minorVersion); - return S_OK; -} - -#ifdef ENABLE_DXCORE -static HRESULT GetDXCoreAdapterMetadata(ID3D12Device& device, bool& isMcdmAdapter, uint32_t& vendorId, uint32_t& majorVersion, uint32_t& minorVersion) { - winrt::com_ptr spFactory; - RETURN_IF_FAILED(DXCoreCreateAdapterFactory(IID_PPV_ARGS(spFactory.put()))); - - winrt::com_ptr spAdapter; - RETURN_IF_FAILED(spFactory->GetAdapterByLuid(device.GetAdapterLuid(), IID_PPV_ARGS(spAdapter.put()))); - - if (spAdapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D12_CORE_COMPUTE) && - (!(spAdapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS) || - spAdapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D11_GRAPHICS)))) { - isMcdmAdapter = true; - } else { - isMcdmAdapter = false; - } - - DXCoreHardwareID hardwareId; - RETURN_IF_FAILED(spAdapter->GetProperty(DXCoreAdapterProperty::HardwareID, &hardwareId)); - vendorId = hardwareId.vendorID; - - uint64_t rawDriverVersion; - RETURN_IF_FAILED(spAdapter->GetProperty(DXCoreAdapterProperty::DriverVersion, &rawDriverVersion)); - - LARGE_INTEGER driverVersion; - driverVersion.QuadPart = static_cast(rawDriverVersion); - ParseDriverVersion(driverVersion, majorVersion, minorVersion); - return S_OK; -} -#endif - -static HRESULT GetD3D12Device(const winrt::Windows::AI::MachineLearning::LearningModelDevice& device, ID3D12Device** outDevice) { - _LUID id; - id.LowPart = device.AdapterId().LowPart; - id.HighPart = device.AdapterId().HighPart; - AdapterEnumerationSupport support; - RETURN_IF_FAILED(GetAdapterEnumerationSupport(&support)); - - if (support.has_dxgi) { - winrt::com_ptr spFactory; - RETURN_IF_FAILED(CreateDXGIFactory1(IID_PPV_ARGS(spFactory.put()))); - - winrt::com_ptr spAdapter; - RETURN_IF_FAILED(spFactory->EnumAdapterByLuid(id, IID_PPV_ARGS(spAdapter.put()))); - RETURN_IF_FAILED(D3D12CreateDevice(spAdapter.get(), D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(outDevice))); - } -#ifdef ENABLE_DXCORE - if (support.has_dxgi == false) { - winrt::com_ptr spFactory; - RETURN_IF_FAILED(DXCoreCreateAdapterFactory(IID_PPV_ARGS(spFactory.put()))); - - winrt::com_ptr spAdapter; - RETURN_IF_FAILED(spFactory->GetAdapterByLuid(id, IID_PPV_ARGS(spAdapter.put()))); - RETURN_IF_FAILED(D3D12CreateDevice(spAdapter.get(), D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(outDevice))); - } -#endif - return S_OK; -} - -static HRESULT IsFloat16Blocked(ID3D12Device& device, bool* isBlocked) { - uint32_t vendorId; - uint32_t majorVersion; - uint32_t minorVersion; - bool isMcdmAdapter; - *isBlocked = true; - AdapterEnumerationSupport support; - RETURN_IF_FAILED(GetAdapterEnumerationSupport(&support)); -#ifdef ENABLE_DXCORE - if (support.has_dxcore) { - RETURN_IF_FAILED(GetDXCoreAdapterMetadata(device, isMcdmAdapter, vendorId, majorVersion, minorVersion)); - *isBlocked = CheckAdapterFP16Blocked(isMcdmAdapter, vendorId, majorVersion, minorVersion); - return S_OK; - } -#endif - RETURN_IF_FAILED(GetDXGIAdapterMetadata(device, vendorId, majorVersion, minorVersion)); - isMcdmAdapter = false; - *isBlocked = CheckAdapterFP16Blocked(isMcdmAdapter, vendorId, majorVersion, minorVersion); - return S_OK; -} - -bool IsFloat16Supported(const winrt::Windows::AI::MachineLearning::LearningModelDevice& device) { - winrt::com_ptr d3d12Device; - if (FAILED(GetD3D12Device(device, d3d12Device.put()))) { - return false; - } - return IsFloat16Supported(d3d12Device.get()); -} - -bool IsFloat16Supported(ID3D12Device* device) { -#ifndef USE_DML - WINML_THROW_HR_IF_TRUE_MSG(ERROR_NOT_SUPPORTED, true, "IsFloat16Supported is not implemented for WinML only build."); - return false; -#else - bool isBlocked; - if (FAILED(IsFloat16Blocked(*device, &isBlocked)) || isBlocked) { - return false; - } - winrt::com_ptr dmlDevice; - winrt::check_hresult(DMLCreateDevice( - device, - DML_CREATE_DEVICE_FLAG_NONE, - IID_PPV_ARGS(dmlDevice.put()))); - - DML_FEATURE_QUERY_TENSOR_DATA_TYPE_SUPPORT float16Query = {DML_TENSOR_DATA_TYPE_FLOAT16}; - DML_FEATURE_DATA_TENSOR_DATA_TYPE_SUPPORT float16Data = {}; - - winrt::check_hresult(dmlDevice->CheckFeatureSupport( - DML_FEATURE_TENSOR_DATA_TYPE_SUPPORT, - sizeof(float16Query), - &float16Query, - sizeof(float16Data), - &float16Data)); - return float16Data.IsSupported; -#endif USE_DML -} - -// uses Structured Exception Handling (SEH) to detect for delay load failures of target API. -// You cannot mix and match SEH with C++ exception and object unwinding -// In this case we will catch it, and report up to the caller via HRESULT so our callers can use -// C++ exceptions -template -static HRESULT RunDelayLoadedApi(TFunc& tfunc, TArgs&&... args) { - __try { - return tfunc(std::forward(args)...); - } __except (GetExceptionCode() == VcppException(ERROR_SEVERITY_ERROR, ERROR_MOD_NOT_FOUND) ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) { - // this could be ok, just let people know that it failed to load - return HRESULT_FROM_WIN32(ERROR_MOD_NOT_FOUND); - } -} - -HRESULT GetAdapterEnumerationSupport(AdapterEnumerationSupport* support) { - if (!s_adapterEnumerationSupport.has_value()) { - // check for support, starting with DXGI - winrt::com_ptr dxgiFactory; -#ifdef ENABLE_DXCORE - winrt::com_ptr dxcoreFactory; - // necessary because DXCoreCreateAdapterFactory is overloaded - HRESULT(WINAPI * pDxCoreTestFunc) - (REFIID, void**) = DXCoreCreateAdapterFactory; -#endif - AdapterEnumerationSupport adapterEnumerationSupport = {}; - - if (SUCCEEDED(RunDelayLoadedApi(CreateDXGIFactory1, IID_PPV_ARGS(dxgiFactory.put())))) { - adapterEnumerationSupport.has_dxgi = true; - } -#ifdef ENABLE_DXCORE - if (SUCCEEDED(RunDelayLoadedApi(pDxCoreTestFunc, IID_PPV_ARGS(dxcoreFactory.put())))) { - adapterEnumerationSupport.has_dxcore = true; - } -#endif - - s_adapterEnumerationSupport = adapterEnumerationSupport; - - if (!(adapterEnumerationSupport.has_dxgi || adapterEnumerationSupport.has_dxcore)) { - return TYPE_E_CANTLOADLIBRARY; - } - } - *support = s_adapterEnumerationSupport.value(); - return S_OK; -} - HRESULT GetDXGIHardwareAdapterWithPreference(DXGI_GPU_PREFERENCE preference, IDXGIAdapter1** ppAdapter) { winrt::com_ptr spFactory; RETURN_IF_FAILED(CreateDXGIFactory1(IID_PPV_ARGS(spFactory.put()))); @@ -299,7 +99,7 @@ HRESULT GetDXCoreHardwareAdapterWithPreference(DXGI_GPU_PREFERENCE preference, I #endif HRESULT CreateD3D11On12Device(ID3D12Device* device12, ID3D11Device** device11) { - return DeviceHelpers::RunDelayLoadedApi( + return CommonDeviceHelpers::RunDelayLoadedApi( D3D11On12CreateDevice, device12, // pointer to d3d12 device D3D11_CREATE_DEVICE_BGRA_SUPPORT, // required in order to interop with Direct2D diff --git a/winml/lib/Api.Image/inc/DeviceHelpers.h b/winml/lib/Api.Image/inc/DeviceHelpers.h index 2a86c7d40cd96..d5c409c02b680 100644 --- a/winml/lib/Api.Image/inc/DeviceHelpers.h +++ b/winml/lib/Api.Image/inc/DeviceHelpers.h @@ -14,28 +14,11 @@ #include #endif -// -// Exception information -// -#ifndef FACILITY_VISUALCPP -#define FACILITY_VISUALCPP ((LONG)0x6d) -#endif - -#define VcppException(sev, err) ((sev) | (FACILITY_VISUALCPP << 16) | err) - namespace DeviceHelpers { -struct AdapterEnumerationSupport { - bool has_dxgi; - bool has_dxcore; -}; - -HRESULT GetAdapterEnumerationSupport(AdapterEnumerationSupport* support); -bool IsFloat16Supported(ID3D12Device* device); -bool IsFloat16Supported(const winrt::Windows::AI::MachineLearning::LearningModelDevice& device); HRESULT CreateD3D11On12Device(ID3D12Device* device12, ID3D11Device** device11); #ifdef ENABLE_DXCORE HRESULT GetDXCoreHardwareAdapterWithPreference(DXGI_GPU_PREFERENCE preference, _COM_Outptr_ IDXCoreAdapter** ppAdapter); #endif HRESULT GetDXGIHardwareAdapterWithPreference(DXGI_GPU_PREFERENCE preference, _COM_Outptr_ IDXGIAdapter1** adapter); HRESULT GetGPUPreference(winrt::Windows::AI::MachineLearning::LearningModelDeviceKind deviceKind, DXGI_GPU_PREFERENCE* preference) noexcept; -} // namespace DeviceHelpers \ No newline at end of file +} // namespace DeviceHelpers diff --git a/winml/lib/Common/CommonDeviceHelpers.cpp b/winml/lib/Common/CommonDeviceHelpers.cpp new file mode 100644 index 0000000000000..292d665ffdf1b --- /dev/null +++ b/winml/lib/Common/CommonDeviceHelpers.cpp @@ -0,0 +1,215 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// #include "dx.h" +// #include + +#if USE_DML +#include +#endif USE_DML +#include "inc/CommonDeviceHelpers.h" +#include +#include +#include "LearningModelDevice.h" + +namespace { +constexpr uint32_t c_intelVendorId = 0x8086; +constexpr uint32_t c_nvidiaVendorId = 0x10DE; +constexpr uint32_t c_amdVendorId = 0x1002; + +bool CheckAdapterFP16Blocked(bool isMcdmAdapter, uint32_t vendorId, uint32_t majorVersion, uint32_t minorVersion) { + switch (vendorId) { + case c_intelVendorId: { + if (isMcdmAdapter) { + return false; + } + + // Check Intel GPU driver version + return (majorVersion < 25) || (majorVersion == 25 && minorVersion < 6574) || (majorVersion == 26 && minorVersion < 6572); + } + } + return false; +} + +void ParseDriverVersion(LARGE_INTEGER& version, uint32_t& majorVersion, uint32_t& minorVersion) { + majorVersion = HIWORD(version.HighPart); + minorVersion = LOWORD(version.LowPart); +} + +HRESULT GetDXGIAdapterMetadata(ID3D12Device& device, uint32_t& vendorId, uint32_t& majorVersion, uint32_t& minorVersion) { + winrt::com_ptr spFactory; + RETURN_IF_FAILED(CreateDXGIFactory1(IID_PPV_ARGS(spFactory.put()))); + + winrt::com_ptr spAdapter; + RETURN_IF_FAILED(spFactory->EnumAdapterByLuid(device.GetAdapterLuid(), IID_PPV_ARGS(spAdapter.put()))); + + DXGI_ADAPTER_DESC adapterDesc = {}; + RETURN_IF_FAILED(spAdapter->GetDesc(&adapterDesc)); + + LARGE_INTEGER driverVersion; + RETURN_IF_FAILED(spAdapter->CheckInterfaceSupport(__uuidof(IDXGIDevice), &driverVersion)); + + vendorId = adapterDesc.VendorId; + ParseDriverVersion(driverVersion, majorVersion, minorVersion); + return S_OK; +} + +#ifdef ENABLE_DXCORE +HRESULT GetDXCoreAdapterMetadata(ID3D12Device& device, bool& isMcdmAdapter, uint32_t& vendorId, uint32_t& majorVersion, uint32_t& minorVersion) { + winrt::com_ptr spFactory; + RETURN_IF_FAILED(DXCoreCreateAdapterFactory(IID_PPV_ARGS(spFactory.put()))); + + winrt::com_ptr spAdapter; + RETURN_IF_FAILED(spFactory->GetAdapterByLuid(device.GetAdapterLuid(), IID_PPV_ARGS(spAdapter.put()))); + + if (spAdapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D12_CORE_COMPUTE) && + (!(spAdapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS) || + spAdapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D11_GRAPHICS)))) { + isMcdmAdapter = true; + } else { + isMcdmAdapter = false; + } + + DXCoreHardwareID hardwareId; + RETURN_IF_FAILED(spAdapter->GetProperty(DXCoreAdapterProperty::HardwareID, &hardwareId)); + vendorId = hardwareId.vendorID; + + uint64_t rawDriverVersion; + RETURN_IF_FAILED(spAdapter->GetProperty(DXCoreAdapterProperty::DriverVersion, &rawDriverVersion)); + + LARGE_INTEGER driverVersion; + driverVersion.QuadPart = static_cast(rawDriverVersion); + ParseDriverVersion(driverVersion, majorVersion, minorVersion); + return S_OK; +} +#endif + +HRESULT GetD3D12Device(const winrt::Windows::AI::MachineLearning::LearningModelDevice& device, ID3D12Device** outDevice) { + _LUID id; + id.LowPart = device.AdapterId().LowPart; + id.HighPart = device.AdapterId().HighPart; + CommonDeviceHelpers::AdapterEnumerationSupport support; + RETURN_IF_FAILED(GetAdapterEnumerationSupport(&support)); + + if (support.has_dxgi) { + winrt::com_ptr spFactory; + RETURN_IF_FAILED(CreateDXGIFactory1(IID_PPV_ARGS(spFactory.put()))); + + winrt::com_ptr spAdapter; + RETURN_IF_FAILED(spFactory->EnumAdapterByLuid(id, IID_PPV_ARGS(spAdapter.put()))); + RETURN_IF_FAILED(D3D12CreateDevice(spAdapter.get(), D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(outDevice))); + } +#ifdef ENABLE_DXCORE + if (support.has_dxgi == false) { + winrt::com_ptr spFactory; + RETURN_IF_FAILED(DXCoreCreateAdapterFactory(IID_PPV_ARGS(spFactory.put()))); + + winrt::com_ptr spAdapter; + RETURN_IF_FAILED(spFactory->GetAdapterByLuid(id, IID_PPV_ARGS(spAdapter.put()))); + RETURN_IF_FAILED(D3D12CreateDevice(spAdapter.get(), D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(outDevice))); + } +#endif + return S_OK; +} + +HRESULT IsFloat16Blocked(ID3D12Device& device, bool* isBlocked) { + uint32_t vendorId; + uint32_t majorVersion; + uint32_t minorVersion; + bool isMcdmAdapter; + *isBlocked = true; + CommonDeviceHelpers::AdapterEnumerationSupport support; + RETURN_IF_FAILED(CommonDeviceHelpers::GetAdapterEnumerationSupport(&support)); +#ifdef ENABLE_DXCORE + if (support.has_dxcore) { + RETURN_IF_FAILED(GetDXCoreAdapterMetadata(device, isMcdmAdapter, vendorId, majorVersion, minorVersion)); + *isBlocked = CheckAdapterFP16Blocked(isMcdmAdapter, vendorId, majorVersion, minorVersion); + return S_OK; + } +#endif + RETURN_IF_FAILED(GetDXGIAdapterMetadata(device, vendorId, majorVersion, minorVersion)); + isMcdmAdapter = false; + *isBlocked = CheckAdapterFP16Blocked(isMcdmAdapter, vendorId, majorVersion, minorVersion); + return S_OK; +} +} + +namespace CommonDeviceHelpers { +constexpr uint32_t c_intelVendorId = 0x8086; +constexpr uint32_t c_nvidiaVendorId = 0x10DE; +constexpr uint32_t c_amdVendorId = 0x1002; + +bool IsFloat16Supported(const winrt::Windows::AI::MachineLearning::LearningModelDevice& device) { + auto adapterId = device.AdapterId(); + if (!adapterId.HighPart && !adapterId.LowPart) { + // CPU device + return true; + } + winrt::com_ptr d3d12Device; + if (FAILED(GetD3D12Device(device, d3d12Device.put()))) { + return false; + } + return IsFloat16Supported(d3d12Device.get()); +} + +bool IsFloat16Supported(ID3D12Device* device) { +#ifndef USE_DML + WINML_THROW_HR_IF_TRUE_MSG(ERROR_NOT_SUPPORTED, true, "IsFloat16Supported is not implemented for WinML only build."); + return false; +#else + bool isBlocked; + if (FAILED(IsFloat16Blocked(*device, &isBlocked)) || isBlocked) { + return false; + } + winrt::com_ptr dmlDevice; + winrt::check_hresult(DMLCreateDevice( + device, + DML_CREATE_DEVICE_FLAG_NONE, + IID_PPV_ARGS(dmlDevice.put()))); + + DML_FEATURE_QUERY_TENSOR_DATA_TYPE_SUPPORT float16Query = {DML_TENSOR_DATA_TYPE_FLOAT16}; + DML_FEATURE_DATA_TENSOR_DATA_TYPE_SUPPORT float16Data = {}; + + winrt::check_hresult(dmlDevice->CheckFeatureSupport( + DML_FEATURE_TENSOR_DATA_TYPE_SUPPORT, + sizeof(float16Query), + &float16Query, + sizeof(float16Data), + &float16Data)); + return float16Data.IsSupported; +#endif +} + +HRESULT GetAdapterEnumerationSupport(AdapterEnumerationSupport* support) { + static std::optional s_adapterEnumerationSupport; + if (!s_adapterEnumerationSupport.has_value()) { + // check for support, starting with DXGI + winrt::com_ptr dxgiFactory; +#ifdef ENABLE_DXCORE + winrt::com_ptr dxcoreFactory; + // necessary because DXCoreCreateAdapterFactory is overloaded + HRESULT(WINAPI * pDxCoreTestFunc) + (REFIID, void**) = DXCoreCreateAdapterFactory; +#endif + AdapterEnumerationSupport adapterEnumerationSupport = {}; + + if (SUCCEEDED(RunDelayLoadedApi(CreateDXGIFactory1, IID_PPV_ARGS(dxgiFactory.put())))) { + adapterEnumerationSupport.has_dxgi = true; + } +#ifdef ENABLE_DXCORE + if (SUCCEEDED(RunDelayLoadedApi(pDxCoreTestFunc, IID_PPV_ARGS(dxcoreFactory.put())))) { + adapterEnumerationSupport.has_dxcore = true; + } +#endif + + s_adapterEnumerationSupport = adapterEnumerationSupport; + + if (!(adapterEnumerationSupport.has_dxgi || adapterEnumerationSupport.has_dxcore)) { + return TYPE_E_CANTLOADLIBRARY; + } + } + *support = s_adapterEnumerationSupport.value(); + return S_OK; +} + +} // namespace CommonDeviceHelpers diff --git a/winml/lib/Common/inc/CommonDeviceHelpers.h b/winml/lib/Common/inc/CommonDeviceHelpers.h new file mode 100644 index 0000000000000..38f1aa1e8a13c --- /dev/null +++ b/winml/lib/Common/inc/CommonDeviceHelpers.h @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "winrt_headers.h" +#include +#include +#include + +#if __has_include("dxcore.h") +#define ENABLE_DXCORE 1 +#endif +#ifdef ENABLE_DXCORE +#include +#endif + +// +// Exception information +// +#ifndef FACILITY_VISUALCPP +#define FACILITY_VISUALCPP ((LONG)0x6d) +#endif + +#define VcppException(sev, err) ((sev) | (FACILITY_VISUALCPP << 16) | err) + +namespace CommonDeviceHelpers { +struct AdapterEnumerationSupport { + bool has_dxgi; + bool has_dxcore; +}; + +// uses Structured Exception Handling (SEH) to detect for delay load failures of target API. +// You cannot mix and match SEH with C++ exception and object unwinding +// In this case we will catch it, and report up to the caller via HRESULT so our callers can use +// C++ exceptions +template +HRESULT RunDelayLoadedApi(TFunc& tfunc, TArgs&&... args) { + __try { + return tfunc(std::forward(args)...); + } __except (GetExceptionCode() == VcppException(ERROR_SEVERITY_ERROR, ERROR_MOD_NOT_FOUND) ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) { + // this could be ok, just let people know that it failed to load + return HRESULT_FROM_WIN32(ERROR_MOD_NOT_FOUND); + } +} + +HRESULT GetAdapterEnumerationSupport(AdapterEnumerationSupport* support); +bool IsFloat16Supported(ID3D12Device* device); +bool IsFloat16Supported(const winrt::Windows::AI::MachineLearning::LearningModelDevice& device); +} // namespace CommonDeviceHelpers diff --git a/winml/test/api/APITest.h b/winml/test/api/APITest.h index 68e68f724ddf5..fd40a19b787f4 100644 --- a/winml/test/api/APITest.h +++ b/winml/test/api/APITest.h @@ -5,37 +5,25 @@ //----------------------------------------------------------------------------- #pragma once +#include "fileHelpers.h" +namespace APITest { +static void LoadModel(const std::wstring& modelPath, + winrt::Windows::AI::MachineLearning::LearningModel& learningModel) { + std::wstring fullPath = FileHelpers::GetModulePath() + modelPath; + learningModel = winrt::Windows::AI::MachineLearning::LearningModel::LoadFromFilePath(fullPath); +}; -#include - -class APITest : public ::testing::Test -{ -protected: - void LoadModel(const std::wstring& modelPath) - { - std::wstring fullPath = FileHelpers::GetModulePath() + modelPath; - m_model = winrt::Windows::AI::MachineLearning::LearningModel::LoadFromFilePath(fullPath); - } - - winrt::Windows::AI::MachineLearning::LearningModel m_model = nullptr; - winrt::Windows::AI::MachineLearning::LearningModelDevice m_device = nullptr; - winrt::Windows::AI::MachineLearning::LearningModelSession m_session = nullptr; - - uint64_t GetAdapterIdQuadPart() - { - LARGE_INTEGER id; - id.LowPart = m_device.AdapterId().LowPart; - id.HighPart = m_device.AdapterId().HighPart; - return id.QuadPart; - }; - - _LUID GetAdapterIdAsLUID() - { - _LUID id; - id.LowPart = m_device.AdapterId().LowPart; - id.HighPart = m_device.AdapterId().HighPart; - return id; - } - - bool m_runGPUTests = true; +static uint64_t GetAdapterIdQuadPart(winrt::Windows::AI::MachineLearning::LearningModelDevice& device) { + LARGE_INTEGER id; + id.LowPart = device.AdapterId().LowPart; + id.HighPart = device.AdapterId().HighPart; + return id.QuadPart; }; + +static _LUID GetAdapterIdAsLUID(winrt::Windows::AI::MachineLearning::LearningModelDevice& device) { + _LUID id; + id.LowPart = device.AdapterId().LowPart; + id.HighPart = device.AdapterId().HighPart; + return id; +} +}; // namespace APITest diff --git a/winml/test/api/LearningModelAPITest.cpp b/winml/test/api/LearningModelAPITest.cpp index ad592a9a86301..639940551d580 100644 --- a/winml/test/api/LearningModelAPITest.cpp +++ b/winml/test/api/LearningModelAPITest.cpp @@ -1,7 +1,6 @@ #include "testPch.h" - +#include "LearningModelAPITest.h" #include "APITest.h" - #include #include #include @@ -15,107 +14,96 @@ using namespace winrt::Windows::Media; using namespace winrt::Windows::Storage; using namespace winrt::Windows::Storage::Streams; -class LearningModelAPITest : public APITest -{ -protected: - LearningModelAPITest() { - init_apartment(); - m_model = nullptr; - m_device = nullptr; - m_session = nullptr; - } -}; - -class LearningModelAPITestGpu : public LearningModelAPITest -{ -protected: - void SetUp() override - { - GPUTEST - } -}; +static void LearningModelAPITestSetup() { + init_apartment(); +} -TEST_F(LearningModelAPITest, CreateModelFromFilePath) -{ - EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); +static void LearningModelAPITestGpuSetup() { + GPUTEST; + init_apartment(); } -TEST_F(LearningModelAPITest, CreateModelFromIStorage) -{ - std::wstring path = FileHelpers::GetModulePath() + L"squeezenet_modifiedforruntimestests.onnx"; - auto storageFile = winrt::Windows::Storage::StorageFile::GetFileFromPathAsync(path).get(); - EXPECT_NO_THROW(m_model = LearningModel::LoadFromStorageFileAsync(storageFile).get()); - EXPECT_TRUE(m_model != nullptr); - - // check the author so we know the model was populated correctly. - std::wstring author(m_model.Author()); - EXPECT_EQ(L"onnx-caffe2", author); +static void CreateModelFromFilePath() { + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"squeezenet_modifiedforruntimestests.onnx", learningModel)); } -TEST_F(LearningModelAPITest, CreateModelFromIStorageOutsideCwd) -{ - std::wstring path = FileHelpers::GetModulePath() + L"ModelSubdirectory\\ModelInSubdirectory.onnx"; - auto storageFile = winrt::Windows::Storage::StorageFile::GetFileFromPathAsync(path).get(); - EXPECT_NO_THROW(m_model = LearningModel::LoadFromStorageFileAsync(storageFile).get()); - EXPECT_TRUE(m_model != nullptr); - - // check the author so we know the model was populated correctly. - std::wstring author(m_model.Author()); - EXPECT_EQ(L"onnx-caffe2", author); +static void CreateModelFromIStorage() { + std::wstring path = FileHelpers::GetModulePath() + L"squeezenet_modifiedforruntimestests.onnx"; + auto storageFile = winrt::Windows::Storage::StorageFile::GetFileFromPathAsync(path).get(); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromStorageFileAsync(storageFile).get()); + WINML_EXPECT_TRUE(learningModel != nullptr); + + // check the author so we know the model was populated correctly. + std::wstring author(learningModel.Author()); + WINML_EXPECT_EQUAL(L"onnx-caffe2", author); } -TEST_F(LearningModelAPITest, CreateModelFromIStream) -{ - std::wstring path = FileHelpers::GetModulePath() + L"squeezenet_modifiedforruntimestests.onnx"; - auto storageFile = winrt::Windows::Storage::StorageFile::GetFileFromPathAsync(path).get(); - winrt::Windows::Storage::Streams::IRandomAccessStreamReference streamref; - storageFile.as(streamref); +static void CreateModelFromIStorageOutsideCwd() { + std::wstring path = FileHelpers::GetModulePath() + L"ModelSubdirectory\\ModelInSubdirectory.onnx"; + auto storageFile = winrt::Windows::Storage::StorageFile::GetFileFromPathAsync(path).get(); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromStorageFileAsync(storageFile).get()); + WINML_EXPECT_TRUE(learningModel != nullptr); - EXPECT_NO_THROW(m_model = LearningModel::LoadFromStreamAsync(streamref).get()); - EXPECT_TRUE(m_model != nullptr); + // check the author so we know the model was populated correctly. + std::wstring author(learningModel.Author()); + WINML_EXPECT_EQUAL(L"onnx-caffe2", author); +} - // check the author so we know the model was populated correctly. - std::wstring author(m_model.Author()); - EXPECT_EQ(L"onnx-caffe2", author); +static void CreateModelFromIStream() { + std::wstring path = FileHelpers::GetModulePath() + L"squeezenet_modifiedforruntimestests.onnx"; + auto storageFile = winrt::Windows::Storage::StorageFile::GetFileFromPathAsync(path).get(); + winrt::Windows::Storage::Streams::IRandomAccessStreamReference streamref; + storageFile.as(streamref); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromStreamAsync(streamref).get()); + WINML_EXPECT_TRUE(learningModel != nullptr); + + // check the author so we know the model was populated correctly. + std::wstring author(learningModel.Author()); + WINML_EXPECT_EQUAL(L"onnx-caffe2", author); } -TEST_F(LearningModelAPITest, GetAuthor) -{ - EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); - std::wstring author(m_model.Author()); - EXPECT_EQ(L"onnx-caffe2", author); +static void ModelGetAuthor() { + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"squeezenet_modifiedforruntimestests.onnx", learningModel)); + std::wstring author(learningModel.Author()); + WINML_EXPECT_EQUAL(L"onnx-caffe2", author); } -TEST_F(LearningModelAPITest, GetName) -{ - EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); - std::wstring name(m_model.Name()); - EXPECT_EQ(L"squeezenet_old", name); +static void ModelGetName() { + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"squeezenet_modifiedforruntimestests.onnx", learningModel)); + std::wstring name(learningModel.Name()); + WINML_EXPECT_EQUAL(L"squeezenet_old", name); } -TEST_F(LearningModelAPITest, GetDomain) -{ - EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); - std::wstring domain(m_model.Domain()); - EXPECT_EQ(L"test-domain", domain); +static void ModelGetDomain() { + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"squeezenet_modifiedforruntimestests.onnx", learningModel)); + std::wstring domain(learningModel.Domain()); + WINML_EXPECT_EQUAL(L"test-domain", domain); } -TEST_F(LearningModelAPITest, GetDescription) -{ - EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); - std::wstring description(m_model.Description()); - EXPECT_EQ(L"test-doc_string", description); +static void ModelGetDescription() { + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"squeezenet_modifiedforruntimestests.onnx", learningModel)); + std::wstring description(learningModel.Description()); + WINML_EXPECT_EQUAL(L"test-doc_string", description); } -TEST_F(LearningModelAPITest, GetVersion) -{ - EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); - int64_t version(m_model.Version()); - (void)(version); +static void ModelGetVersion() { + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"squeezenet_modifiedforruntimestests.onnx", learningModel)); + int64_t version(learningModel.Version()); + (void)(version); } typedef std::vector> Metadata; +/* class MetadataTest : public LearningModelAPITest, public testing::WithParamInterface> {}; @@ -125,16 +113,16 @@ TEST_P(MetadataTest, GetMetaData) std::vector> keyValuePairs; tie(fileName, keyValuePairs) = GetParam(); - EXPECT_NO_THROW(LoadModel(fileName.c_str())); - EXPECT_TRUE(m_model.Metadata() != nullptr); - EXPECT_EQ(keyValuePairs.size(), m_model.Metadata().Size()); + WINML_EXPECT_NO_THROW(LoadModel(fileName.c_str())); + WINML_EXPECT_TRUE(m_model.Metadata() != nullptr); + WINML_EXPECT_EQUAL(keyValuePairs.size(), m_model.Metadata().Size()); auto iter = m_model.Metadata().First(); for (auto& keyValue : keyValuePairs) { - EXPECT_TRUE(iter.HasCurrent()); - EXPECT_EQ(keyValue.first, std::wstring(iter.Current().Key())); - EXPECT_EQ(keyValue.second, std::wstring(iter.Current().Value())); + WINML_EXPECT_TRUE(iter.HasCurrent()); + WINML_EXPECT_EQUAL(keyValue.first, std::wstring(iter.Current().Key())); + WINML_EXPECT_EQUAL(keyValue.second, std::wstring(iter.Current().Value())); iter.MoveNext(); } } @@ -147,122 +135,141 @@ INSTANTIATE_TEST_SUITE_P( std::pair(L"modelWithMetaData.onnx", Metadata{{L"thisisalongkey", L"thisisalongvalue"}}), std::pair(L"modelWith2MetaData.onnx", Metadata{{L"thisisalongkey", L"thisisalongvalue"}, {L"key2", L"val2"}}) )); +*/ -TEST_F(LearningModelAPITest, EnumerateInputs) -{ - EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); +static void EnumerateInputs() { + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"squeezenet_modifiedforruntimestests.onnx", learningModel)); - // purposely don't cache "InputFeatures" in order to exercise calling it multiple times - EXPECT_TRUE(m_model.InputFeatures().First().HasCurrent()); + // purposely don't cache "InputFeatures" in order to exercise calling it multiple times + WINML_EXPECT_TRUE(learningModel.InputFeatures().First().HasCurrent()); - std::wstring name(m_model.InputFeatures().First().Current().Name()); - EXPECT_EQ(L"data_0", name); + std::wstring name(learningModel.InputFeatures().First().Current().Name()); + WINML_EXPECT_EQUAL(L"data_0", name); - // make sure it's either tensor or image - TensorFeatureDescriptor tensorDescriptor = nullptr; - m_model.InputFeatures().First().Current().try_as(tensorDescriptor); - if (tensorDescriptor == nullptr) - { - ImageFeatureDescriptor imageDescriptor = nullptr; - EXPECT_NO_THROW(m_model.InputFeatures().First().Current().as(imageDescriptor)); - } + // make sure it's either tensor or image + TensorFeatureDescriptor tensorDescriptor = nullptr; + learningModel.InputFeatures().First().Current().try_as(tensorDescriptor); + if (tensorDescriptor == nullptr) { + ImageFeatureDescriptor imageDescriptor = nullptr; + WINML_EXPECT_NO_THROW(learningModel.InputFeatures().First().Current().as(imageDescriptor)); + } - auto modelDataKind = tensorDescriptor.TensorKind(); - EXPECT_EQ(TensorKind::Float, modelDataKind); + auto modelDataKind = tensorDescriptor.TensorKind(); + WINML_EXPECT_EQUAL(TensorKind::Float, modelDataKind); - EXPECT_TRUE(tensorDescriptor.IsRequired()); + WINML_EXPECT_TRUE(tensorDescriptor.IsRequired()); - std::vector expectedShapes = { 1,3,224,224 }; - EXPECT_EQ(expectedShapes.size(), tensorDescriptor.Shape().Size()); - for (uint32_t j = 0; j < tensorDescriptor.Shape().Size(); j++) - { - EXPECT_EQ(expectedShapes.at(j), tensorDescriptor.Shape().GetAt(j)); - } + std::vector expectedShapes = {1, 3, 224, 224}; + WINML_EXPECT_EQUAL(expectedShapes.size(), tensorDescriptor.Shape().Size()); + for (uint32_t j = 0; j < tensorDescriptor.Shape().Size(); j++) { + WINML_EXPECT_EQUAL(expectedShapes.at(j), tensorDescriptor.Shape().GetAt(j)); + } - auto first = m_model.InputFeatures().First(); - first.MoveNext(); - EXPECT_FALSE(first.HasCurrent()); + auto first = learningModel.InputFeatures().First(); + first.MoveNext(); + WINML_EXPECT_FALSE(first.HasCurrent()); } -TEST_F(LearningModelAPITest, EnumerateOutputs) -{ - EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); +static void EnumerateOutputs() { + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"squeezenet_modifiedforruntimestests.onnx", learningModel)); - // purposely don't cache "OutputFeatures" in order to exercise calling it multiple times - std::wstring name(m_model.OutputFeatures().First().Current().Name()); - EXPECT_EQ(L"softmaxout_1", name); + // purposely don't cache "OutputFeatures" in order to exercise calling it multiple times + std::wstring name(learningModel.OutputFeatures().First().Current().Name()); + WINML_EXPECT_EQUAL(L"softmaxout_1", name); - TensorFeatureDescriptor tensorDescriptor = nullptr; - EXPECT_NO_THROW(m_model.OutputFeatures().First().Current().as(tensorDescriptor)); - EXPECT_TRUE(tensorDescriptor != nullptr); + TensorFeatureDescriptor tensorDescriptor = nullptr; + WINML_EXPECT_NO_THROW(learningModel.OutputFeatures().First().Current().as(tensorDescriptor)); + WINML_EXPECT_TRUE(tensorDescriptor != nullptr); - auto tensorName = tensorDescriptor.Name(); - EXPECT_EQ(L"softmaxout_1", tensorName); + auto tensorName = tensorDescriptor.Name(); + WINML_EXPECT_EQUAL(L"softmaxout_1", tensorName); - auto modelDataKind = tensorDescriptor.TensorKind(); - EXPECT_EQ(TensorKind::Float, modelDataKind); + auto modelDataKind = tensorDescriptor.TensorKind(); + WINML_EXPECT_EQUAL(TensorKind::Float, modelDataKind); - EXPECT_TRUE(tensorDescriptor.IsRequired()); + WINML_EXPECT_TRUE(tensorDescriptor.IsRequired()); - std::vector expectedShapes = { 1, 1000, 1, 1 }; - EXPECT_EQ(expectedShapes.size(), tensorDescriptor.Shape().Size()); - for (uint32_t j = 0; j < tensorDescriptor.Shape().Size(); j++) - { - EXPECT_EQ(expectedShapes.at(j), tensorDescriptor.Shape().GetAt(j)); - } + std::vector expectedShapes = {1, 1000, 1, 1}; + WINML_EXPECT_EQUAL(expectedShapes.size(), tensorDescriptor.Shape().Size()); + for (uint32_t j = 0; j < tensorDescriptor.Shape().Size(); j++) { + WINML_EXPECT_EQUAL(expectedShapes.at(j), tensorDescriptor.Shape().GetAt(j)); + } - auto first = m_model.OutputFeatures().First(); - first.MoveNext(); - EXPECT_FALSE(first.HasCurrent()); + auto first = learningModel.OutputFeatures().First(); + first.MoveNext(); + WINML_EXPECT_FALSE(first.HasCurrent()); } -TEST_F(LearningModelAPITest, CloseModelCheckMetadata) -{ - EXPECT_NO_THROW(LoadModel(L"squeezenet_modifiedforruntimestests.onnx")); - EXPECT_NO_THROW(m_model.Close()); - std::wstring author(m_model.Author()); - EXPECT_EQ(L"onnx-caffe2", author); - std::wstring name(m_model.Name()); - EXPECT_EQ(L"squeezenet_old", name); - std::wstring domain(m_model.Domain()); - EXPECT_EQ(L"test-domain", domain); - std::wstring description(m_model.Description()); - EXPECT_EQ(L"test-doc_string", description); - int64_t version(m_model.Version()); - EXPECT_EQ(123456, version); +static void CloseModelCheckMetadata() { + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"squeezenet_modifiedforruntimestests.onnx", learningModel)); + WINML_EXPECT_NO_THROW(learningModel.Close()); + std::wstring author(learningModel.Author()); + WINML_EXPECT_EQUAL(L"onnx-caffe2", author); + std::wstring name(learningModel.Name()); + WINML_EXPECT_EQUAL(L"squeezenet_old", name); + std::wstring domain(learningModel.Domain()); + WINML_EXPECT_EQUAL(L"test-domain", domain); + std::wstring description(learningModel.Description()); + WINML_EXPECT_EQUAL(L"test-doc_string", description); + int64_t version(learningModel.Version()); + WINML_EXPECT_EQUAL(123456, version); } -TEST_F(LearningModelAPITestGpu, CloseModelCheckEval) -{ - EXPECT_NO_THROW(LoadModel(L"model.onnx")); - LearningModelSession session = nullptr; - EXPECT_NO_THROW(session = LearningModelSession(m_model)); - EXPECT_NO_THROW(m_model.Close()); - - std::wstring fullImagePath = FileHelpers::GetModulePath() + L"kitten_224.png"; - StorageFile imagefile = StorageFile::GetFileFromPathAsync(fullImagePath).get(); - IRandomAccessStream stream = imagefile.OpenAsync(FileAccessMode::Read).get(); - SoftwareBitmap softwareBitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); - VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); - - LearningModelBinding binding = nullptr; - EXPECT_NO_THROW(binding = LearningModelBinding(session)); - EXPECT_NO_THROW(binding.Bind(m_model.InputFeatures().First().Current().Name(), frame)); - - EXPECT_NO_THROW(session.Evaluate(binding, L"")); +static void CloseModelCheckEval() { + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"model.onnx", learningModel)); + LearningModelSession session = nullptr; + WINML_EXPECT_NO_THROW(session = LearningModelSession(learningModel)); + WINML_EXPECT_NO_THROW(learningModel.Close()); + + std::wstring fullImagePath = FileHelpers::GetModulePath() + L"kitten_224.png"; + StorageFile imagefile = StorageFile::GetFileFromPathAsync(fullImagePath).get(); + IRandomAccessStream stream = imagefile.OpenAsync(FileAccessMode::Read).get(); + SoftwareBitmap softwareBitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); + + LearningModelBinding binding = nullptr; + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); + WINML_EXPECT_NO_THROW(binding.Bind(learningModel.InputFeatures().First().Current().Name(), frame)); + + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); } -TEST_F(LearningModelAPITest, CloseModelNoNewSessions) -{ - EXPECT_NO_THROW(LoadModel(L"model.onnx")); - EXPECT_NO_THROW(m_model.Close()); - LearningModelSession session = nullptr; - EXPECT_THROW( - try { - session = LearningModelSession(m_model); - } catch (const winrt::hresult_error& e) { - EXPECT_EQ(E_INVALIDARG, e.code()); - throw; - } - , winrt::hresult_error); +static void CloseModelNoNewSessions() { + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"model.onnx", learningModel)); + WINML_EXPECT_NO_THROW(learningModel.Close()); + LearningModelSession session = nullptr; + WINML_EXPECT_THROW_SPECIFIC( + session = LearningModelSession(learningModel);, + winrt::hresult_error, + [](const winrt::hresult_error& e) -> bool { + return e.code() == E_INVALIDARG; + }); } + +const LearningModelApiTestApi& getapi() { + static constexpr LearningModelApiTestApi api = + { + LearningModelAPITestSetup, + LearningModelAPITestGpuSetup, + CreateModelFromFilePath, + CreateModelFromIStorage, + CreateModelFromIStorageOutsideCwd, + CreateModelFromIStream, + ModelGetAuthor, + ModelGetName, + ModelGetDomain, + ModelGetDescription, + ModelGetVersion, + EnumerateInputs, + EnumerateOutputs, + CloseModelCheckMetadata, + CloseModelCheckEval, + CloseModelNoNewSessions + }; + return api; +} \ No newline at end of file diff --git a/winml/test/api/LearningModelAPITest.h b/winml/test/api/LearningModelAPITest.h new file mode 100644 index 0000000000000..4a723ab2d9d29 --- /dev/null +++ b/winml/test/api/LearningModelAPITest.h @@ -0,0 +1,41 @@ +#include "test.h" +struct LearningModelApiTestApi +{ + SetupTest LearningModelAPITestSetup; + SetupTest LearningModelAPITestGpuSetup; + VoidTest CreateModelFromFilePath; + VoidTest CreateModelFromIStorage; + VoidTest CreateModelFromIStorageOutsideCwd; + VoidTest CreateModelFromIStream; + VoidTest ModelGetAuthor; + VoidTest ModelGetName; + VoidTest ModelGetDomain; + VoidTest ModelGetDescription; + VoidTest ModelGetVersion; + VoidTest EnumerateInputs; + VoidTest EnumerateOutputs; + VoidTest CloseModelCheckMetadata; + VoidTest CloseModelCheckEval; + VoidTest CloseModelNoNewSessions; +}; +const LearningModelApiTestApi& getapi(); + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelAPITest, LearningModelAPITestSetup) +WINML_TEST(LearningModelAPITest, CreateModelFromFilePath) +WINML_TEST(LearningModelAPITest, CreateModelFromIStorage) +WINML_TEST(LearningModelAPITest, CreateModelFromIStorageOutsideCwd) +WINML_TEST(LearningModelAPITest, CreateModelFromIStream) +WINML_TEST(LearningModelAPITest, ModelGetAuthor) +WINML_TEST(LearningModelAPITest, ModelGetName) +WINML_TEST(LearningModelAPITest, ModelGetDomain) +WINML_TEST(LearningModelAPITest, ModelGetDescription) +WINML_TEST(LearningModelAPITest, ModelGetVersion) +WINML_TEST(LearningModelAPITest, EnumerateInputs) +WINML_TEST(LearningModelAPITest, EnumerateOutputs) +WINML_TEST(LearningModelAPITest, CloseModelCheckMetadata) +WINML_TEST(LearningModelAPITest, CloseModelNoNewSessions) +WINML_TEST_CLASS_END() + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelAPITestGpu, LearningModelAPITestGpuSetup) +WINML_TEST(LearningModelAPITestGpu, CloseModelCheckEval) +WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/api/LearningModelBindingAPITest.cpp b/winml/test/api/LearningModelBindingAPITest.cpp index f7cd11430a288..2c771500df76b 100644 --- a/winml/test/api/LearningModelBindingAPITest.cpp +++ b/winml/test/api/LearningModelBindingAPITest.cpp @@ -1,13 +1,13 @@ #include "testPch.h" #include "APITest.h" +#include "LearningModelBindingAPITest.h" #include "SqueezeNetValidator.h" #include #include #include "winrt/Windows.Storage.h" -#include "DeviceHelpers.h" - +#include using namespace winrt; using namespace winrt::Windows::AI::MachineLearning; using namespace winrt::Windows::Foundation::Collections; @@ -15,25 +15,22 @@ using namespace winrt::Windows::Graphics::Imaging; using namespace winrt::Windows::Media; using namespace winrt::Windows::Storage; -class LearningModelBindingAPITest : public APITest -{}; +static void LearningModelBindingAPITestSetup() { + init_apartment(); +} -class LearningModelBindingAPITestGpu : public LearningModelBindingAPITest -{ -protected: - void SetUp() override - { - GPUTEST - } -}; +static void LearningModelBindingAPITestGpuSetup() { + GPUTEST; + init_apartment(); +} -TEST_F(LearningModelBindingAPITest, CpuSqueezeNet) +static void CpuSqueezeNet() { std::string cpuInstance("CPU"); WinML::Engine::Test::ModelValidator::SqueezeNet(cpuInstance, LearningModelDeviceKind::Cpu, /*dataTolerance*/ 0.00001f, false); } -TEST_F(LearningModelBindingAPITest, CpuSqueezeNetEmptyOutputs) +static void CpuSqueezeNetEmptyOutputs() { std::string cpuInstance("CPU"); WinML::Engine::Test::ModelValidator::SqueezeNet( @@ -44,7 +41,7 @@ TEST_F(LearningModelBindingAPITest, CpuSqueezeNetEmptyOutputs) OutputBindingStrategy::Empty); } -TEST_F(LearningModelBindingAPITest, CpuSqueezeNetUnboundOutputs) +static void CpuSqueezeNetUnboundOutputs() { std::string cpuInstance("CPU"); WinML::Engine::Test::ModelValidator::SqueezeNet( @@ -55,7 +52,7 @@ TEST_F(LearningModelBindingAPITest, CpuSqueezeNetUnboundOutputs) OutputBindingStrategy::Unbound); } -TEST_F(LearningModelBindingAPITest, CpuSqueezeNetBindInputTensorAsInspectable) +static void CpuSqueezeNetBindInputTensorAsInspectable() { std::string cpuInstance("CPU"); WinML::Engine::Test::ModelValidator::SqueezeNet( @@ -67,27 +64,28 @@ TEST_F(LearningModelBindingAPITest, CpuSqueezeNetBindInputTensorAsInspectable) true /* bind inputs as inspectables */); } -TEST_F(LearningModelBindingAPITest, CastMapInt64) +static void CastMapInt64() { - EXPECT_NO_THROW(LoadModel(L"castmap-int64.onnx")); + WINML_EXPECT_NO_THROW(LearningModel::LoadFromFilePath(FileHelpers::GetModulePath() + L"castmap-int64.onnx")); // TODO: Check Descriptor } -TEST_F(LearningModelBindingAPITest, DictionaryVectorizerMapInt64) +static void DictionaryVectorizerMapInt64() { - EXPECT_NO_THROW(LoadModel(L"dictvectorizer-int64.onnx")); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"dictvectorizer-int64.onnx", learningModel)); - auto inputDescriptor = m_model.InputFeatures().First().Current(); - EXPECT_TRUE(inputDescriptor.Kind() == LearningModelFeatureKind::Map); + auto inputDescriptor = learningModel.InputFeatures().First().Current(); + WINML_EXPECT_TRUE(inputDescriptor.Kind() == LearningModelFeatureKind::Map); auto mapDescriptor = inputDescriptor.as(); - EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::Int64); - EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); + WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::Int64); + WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); auto tensorDescriptor = mapDescriptor.ValueDescriptor().as(); // empty size means tensor of scalar value - EXPECT_TRUE(tensorDescriptor.Shape().Size() == 0); - EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); + WINML_EXPECT_TRUE(tensorDescriptor.Shape().Size() == 0); + WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); - LearningModelSession modelSession(m_model); + LearningModelSession modelSession(learningModel); LearningModelBinding binding(modelSession); std::unordered_map map; map[1] = 1.f; @@ -102,38 +100,39 @@ TEST_F(LearningModelBindingAPITest, DictionaryVectorizerMapInt64) binding.Bind(mapInputName, abiMap); auto mapInputInspectable = abiMap.as(); auto first = binding.First(); - EXPECT_TRUE(first.Current().Key() == mapInputName); - EXPECT_TRUE(first.Current().Value() == mapInputInspectable); - EXPECT_TRUE(binding.Lookup(mapInputName) == mapInputInspectable); + WINML_EXPECT_TRUE(first.Current().Key() == mapInputName); + WINML_EXPECT_TRUE(first.Current().Value() == mapInputInspectable); + WINML_EXPECT_TRUE(binding.Lookup(mapInputName) == mapInputInspectable); // Bind as IMapView auto mapView = abiMap.GetView(); binding.Bind(mapInputName, mapView); mapInputInspectable = mapView.as(); first = binding.First(); - EXPECT_TRUE(first.Current().Key() == mapInputName); - EXPECT_TRUE(first.Current().Value() == mapView); - EXPECT_TRUE(binding.Lookup(mapInputName) == mapView); + WINML_EXPECT_TRUE(first.Current().Key() == mapInputName); + WINML_EXPECT_TRUE(first.Current().Value() == mapView); + WINML_EXPECT_TRUE(binding.Lookup(mapInputName) == mapView); } -TEST_F(LearningModelBindingAPITest, DictionaryVectorizerMapString) +static void DictionaryVectorizerMapString() { - EXPECT_NO_THROW(LoadModel(L"dictvectorizer-string.onnx")); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"dictvectorizer-string.onnx", learningModel)); - auto inputDescriptor = m_model.InputFeatures().First().Current(); - EXPECT_TRUE(inputDescriptor.Kind() == LearningModelFeatureKind::Map); + auto inputDescriptor = learningModel.InputFeatures().First().Current(); + WINML_EXPECT_TRUE(inputDescriptor.Kind() == LearningModelFeatureKind::Map); auto mapDescriptor = inputDescriptor.as(); - EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::String); - EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); + WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::String); + WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); auto tensorDescriptor = mapDescriptor.ValueDescriptor().as(); // empty size means tensor of scalar value - EXPECT_TRUE(tensorDescriptor.Shape().Size() == 0); - EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); + WINML_EXPECT_TRUE(tensorDescriptor.Shape().Size() == 0); + WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); - LearningModelSession modelSession(m_model); + LearningModelSession modelSession(learningModel); LearningModelBinding binding(modelSession); std::unordered_map map; map[L"1"] = 1.f; @@ -146,9 +145,9 @@ TEST_F(LearningModelBindingAPITest, DictionaryVectorizerMapString) auto mapInputInspectable = abiMap.as(); auto first = binding.First(); - EXPECT_TRUE(first.Current().Key() == mapInputName); - EXPECT_TRUE(first.Current().Value() == mapInputInspectable); - EXPECT_TRUE(binding.Lookup(mapInputName) == mapInputInspectable); + WINML_EXPECT_TRUE(first.Current().Key() == mapInputName); + WINML_EXPECT_TRUE(first.Current().Value() == mapInputInspectable); + WINML_EXPECT_TRUE(binding.Lookup(mapInputName) == mapInputInspectable); modelSession.Evaluate(binding, L""); } @@ -159,15 +158,15 @@ static void RunZipMapInt64( { auto outputFeatures = model.OutputFeatures(); auto outputDescriptor = outputFeatures.First().Current(); - EXPECT_TRUE(outputDescriptor.Kind() == LearningModelFeatureKind::Sequence); + WINML_EXPECT_TRUE(outputDescriptor.Kind() == LearningModelFeatureKind::Sequence); auto seqDescriptor = outputDescriptor.as(); auto mapDescriptor = seqDescriptor.ElementDescriptor().as(); - EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::Int64); + WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::Int64); - EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); + WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); auto tensorDescriptor = mapDescriptor.ValueDescriptor().as(); - EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); + WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); LearningModelSession session(model); LearningModelBinding binding(session); @@ -201,63 +200,66 @@ static void RunZipMapInt64( // from output binding const auto &out1 = abiOutput.GetAt(0); const auto &out2 = result.Lookup(L"Y").as>().GetAt(0); - SCOPED_TRACE((std::ostringstream() << "size: " << out1.Size()).str()); + WINML_LOG_COMMENT((std::ostringstream() << "size: " << out1.Size()).str()); // check outputs auto iter1 = out1.First(); auto iter2 = out2.First(); for (uint32_t i = 0, size = (uint32_t)inputs.size(); i < size; ++i) { - EXPECT_TRUE(iter1.HasCurrent()); - EXPECT_TRUE(iter2.HasCurrent()); + WINML_EXPECT_TRUE(iter1.HasCurrent()); + WINML_EXPECT_TRUE(iter2.HasCurrent()); const auto &pair1 = iter1.Current(); const auto &pair2 = iter2.Current(); - SCOPED_TRACE((std::ostringstream() << "key: " << pair1.Key() << ", value: " << pair2.Value()).str()); - EXPECT_TRUE(pair1.Key() == i && pair2.Key() == i); - EXPECT_TRUE(pair1.Value() == inputs[i] && pair2.Value() == inputs[i]); + WINML_LOG_COMMENT((std::ostringstream() << "key: " << pair1.Key() << ", value: " << pair2.Value()).str()); + WINML_EXPECT_TRUE(pair1.Key() == i && pair2.Key() == i); + WINML_EXPECT_TRUE(pair1.Value() == inputs[i] && pair2.Value() == inputs[i]); iter1.MoveNext(); iter2.MoveNext(); } - EXPECT_TRUE(!iter1.HasCurrent()); - EXPECT_TRUE(!iter2.HasCurrent()); + WINML_EXPECT_TRUE(!iter1.HasCurrent()); + WINML_EXPECT_TRUE(!iter2.HasCurrent()); } else { abiOutput = result.Lookup(L"Y").as(); - EXPECT_TRUE(abiOutput.Size() == 1); + WINML_EXPECT_TRUE(abiOutput.Size() == 1); ABIMap map = abiOutput.GetAt(0); - EXPECT_TRUE(map.Size() == 3); - EXPECT_TRUE(map.Lookup(0) == 0.5); - EXPECT_TRUE(map.Lookup(1) == .25); - EXPECT_TRUE(map.Lookup(2) == .125); + WINML_EXPECT_TRUE(map.Size() == 3); + WINML_EXPECT_TRUE(map.Lookup(0) == 0.5); + WINML_EXPECT_TRUE(map.Lookup(1) == .25); + WINML_EXPECT_TRUE(map.Lookup(2) == .125); } } -TEST_F(LearningModelBindingAPITest, ZipMapInt64) +static void ZipMapInt64() { - EXPECT_NO_THROW(LoadModel(L"zipmap-int64.onnx")); - RunZipMapInt64(m_model, OutputBindingStrategy::Bound); + LearningModel learningModel= nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-int64.onnx", learningModel)); + RunZipMapInt64(learningModel, OutputBindingStrategy::Bound); } -TEST_F(LearningModelBindingAPITest, ZipMapInt64Unbound) +static void ZipMapInt64Unbound() { - EXPECT_NO_THROW(LoadModel(L"zipmap-int64.onnx")); - RunZipMapInt64(m_model, OutputBindingStrategy::Unbound); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-int64.onnx", learningModel)); + RunZipMapInt64(learningModel, OutputBindingStrategy::Unbound); } -TEST_F(LearningModelBindingAPITest, ZipMapString) +static void ZipMapString() { // output constraint: "seq(map(string, float))" or "seq(map(int64, float))" - EXPECT_NO_THROW(LoadModel(L"zipmap-string.onnx")); - auto outputs = m_model.OutputFeatures(); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-string.onnx", learningModel)); + auto outputs = learningModel.OutputFeatures(); auto outputDescriptor = outputs.First().Current(); - EXPECT_TRUE(outputDescriptor.Kind() == LearningModelFeatureKind::Sequence); + WINML_EXPECT_TRUE(outputDescriptor.Kind() == LearningModelFeatureKind::Sequence); auto mapDescriptor = outputDescriptor.as().ElementDescriptor().as(); - EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::String); - EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); + WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::String); + WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor); auto tensorDescriptor = mapDescriptor.ValueDescriptor().as(); - EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); + WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float); - LearningModelSession session(m_model); + LearningModelSession session(learningModel); LearningModelBinding binding(session); std::vector inputs = { 0.5f, 0.25f, 0.125f }; @@ -276,27 +278,27 @@ TEST_F(LearningModelBindingAPITest, ZipMapString) // from output binding const auto &out1 = ABIOutput.GetAt(0); const auto &out2 = result.Lookup(L"Y").as>().GetAt(0); - SCOPED_TRACE((std::ostringstream() << "size: " << out1.Size()).str()); + WINML_LOG_COMMENT((std::ostringstream() << "size: " << out1.Size()).str()); // single key,value pair for each map auto iter1 = out1.First(); auto iter2 = out2.First(); for (uint32_t i = 0, size = (uint32_t)inputs.size(); i < size; ++i) { - EXPECT_TRUE(iter2.HasCurrent()); + WINML_EXPECT_TRUE(iter2.HasCurrent()); const auto &pair1 = iter1.Current(); const auto &pair2 = iter2.Current(); - SCOPED_TRACE((std::ostringstream() << "key: " << pair1.Key().c_str() << ", value " << pair2.Value()).str()); - EXPECT_TRUE(std::wstring(pair1.Key().c_str()).compare(labels[i]) == 0); - EXPECT_TRUE(std::wstring(pair2.Key().c_str()).compare(labels[i]) == 0); - EXPECT_TRUE(pair1.Value() == inputs[i] && pair2.Value() == inputs[i]); + WINML_LOG_COMMENT((std::ostringstream() << "key: " << pair1.Key().c_str() << ", value " << pair2.Value()).str()); + WINML_EXPECT_TRUE(std::wstring(pair1.Key().c_str()).compare(labels[i]) == 0); + WINML_EXPECT_TRUE(std::wstring(pair2.Key().c_str()).compare(labels[i]) == 0); + WINML_EXPECT_TRUE(pair1.Value() == inputs[i] && pair2.Value() == inputs[i]); iter1.MoveNext(); iter2.MoveNext(); } - EXPECT_TRUE(!iter1.HasCurrent()); - EXPECT_TRUE(!iter2.HasCurrent()); + WINML_EXPECT_TRUE(!iter1.HasCurrent()); + WINML_EXPECT_TRUE(!iter2.HasCurrent()); } -TEST_F(LearningModelBindingAPITestGpu, GpuSqueezeNet) +static void GpuSqueezeNet() { std::string gpuInstance("GPU"); WinML::Engine::Test::ModelValidator::SqueezeNet( @@ -305,7 +307,7 @@ TEST_F(LearningModelBindingAPITestGpu, GpuSqueezeNet) /*dataTolerance*/ 0.00001f); } -TEST_F(LearningModelBindingAPITestGpu, GpuSqueezeNetEmptyOutputs) +static void GpuSqueezeNetEmptyOutputs() { std::string gpuInstance("GPU"); WinML::Engine::Test::ModelValidator::SqueezeNet( @@ -316,7 +318,7 @@ TEST_F(LearningModelBindingAPITestGpu, GpuSqueezeNetEmptyOutputs) OutputBindingStrategy::Empty); } -TEST_F(LearningModelBindingAPITestGpu, GpuSqueezeNetUnboundOutputs) +static void GpuSqueezeNetUnboundOutputs() { std::string gpuInstance("GPU"); WinML::Engine::Test::ModelValidator::SqueezeNet( @@ -328,44 +330,48 @@ TEST_F(LearningModelBindingAPITestGpu, GpuSqueezeNetUnboundOutputs) } // Validates that when the input image is the same as the model expects, the binding step is executed correctly. -TEST_F(LearningModelBindingAPITestGpu, ImageBindingDimensions) +static void ImageBindingDimensions() { - LearningModelBinding m_binding = nullptr; + LearningModelBinding learningModelBinding = nullptr; + LearningModel learningModel = nullptr; + LearningModelSession learningModelSession = nullptr; + LearningModelDevice leraningModelDevice = nullptr; std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; // load a model with expected input size: 224 x 224 - EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Default)); - EXPECT_NO_THROW(m_model = LearningModel::LoadFromFilePath(filePath)); - EXPECT_TRUE(m_model != nullptr); - EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); - EXPECT_NO_THROW(m_binding = LearningModelBinding(m_session)); + WINML_EXPECT_NO_THROW(leraningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default)); + WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(filePath)); + WINML_EXPECT_TRUE(learningModel != nullptr); + WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, leraningModelDevice)); + WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession)); // Create input images and execute bind // Test Case 1: both width and height are larger than model expects VideoFrame inputImage1(BitmapPixelFormat::Rgba8, 1000, 1000); ImageFeatureValue inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage1); - EXPECT_NO_THROW(m_binding.Bind(L"data_0", inputTensor)); + WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor)); // Test Case 2: only height is larger, while width is smaller VideoFrame inputImage2(BitmapPixelFormat::Rgba8, 20, 1000); inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage2); - EXPECT_NO_THROW(m_binding.Bind(L"data_0", inputTensor)); + WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor)); // Test Case 3: only width is larger, while height is smaller VideoFrame inputImage3(BitmapPixelFormat::Rgba8, 1000, 20); inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage3); - EXPECT_NO_THROW(m_binding.Bind(L"data_0", inputTensor)); + WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor)); // Test Case 4: both width and height are smaller than model expects VideoFrame inputImage4(BitmapPixelFormat::Rgba8, 20, 20); inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage4); - EXPECT_NO_THROW(m_binding.Bind(L"data_0", inputTensor)); + WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor)); } -TEST_F(LearningModelBindingAPITestGpu, VerifyInvalidBindExceptions) +static void VerifyInvalidBindExceptions() { - EXPECT_NO_THROW(LoadModel(L"zipmap-int64.onnx")); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-int64.onnx", learningModel)); - LearningModelSession session(m_model); + LearningModelSession session(learningModel); LearningModelBinding binding(session); std::vector inputs = { 0.5f, 0.25f, 0.125f }; @@ -386,47 +392,47 @@ TEST_F(LearningModelBindingAPITestGpu, VerifyInvalidBindExceptions) // Bind invalid image as tensorfloat input auto image = FileHelpers::LoadImageFeatureValue(L"227x227.png"); - EXPECT_THROW_SPECIFIC(binding.Bind(L"X", image), winrt::hresult_error, ensureWinmlSizeMismatch); + WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", image), winrt::hresult_error, ensureWinmlSizeMismatch); // Bind invalid map as tensorfloat input std::unordered_map map; auto abiMap = winrt::single_threaded_map(std::move(map)); - EXPECT_THROW_SPECIFIC(binding.Bind(L"X", abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); // Bind invalid sequence as tensorfloat input std::vector sequence; auto abiSequence = winrt::single_threaded_vector(std::move(sequence)); - EXPECT_THROW_SPECIFIC(binding.Bind(L"X", abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); // Bind invalid tensor size as tensorfloat input auto tensorBoolean = TensorBoolean::Create(); - EXPECT_THROW_SPECIFIC(binding.Bind(L"X", tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); // Bind invalid tensor shape as tensorfloat input auto tensorInvalidShape = TensorFloat::Create(std::vector { 2, 3, 4 }); - EXPECT_THROW_SPECIFIC(binding.Bind(L"X", tensorInvalidShape), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", tensorInvalidShape), winrt::hresult_error, ensureWinmlInvalidBinding); /* Verify sequence bindings throw correct bind exceptions */ // Bind invalid image as sequence output - EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", image), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", image), winrt::hresult_error, ensureWinmlInvalidBinding); // Bind invalid map as sequence output - EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); // Bind invalid sequence as sequence output - EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); // Bind invalid tensor as sequence output - EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); /* Verify image bindings throw correct bind exceptions */ - // EXPECT_NO_THROW(LoadModel(L"fns-candy.onnx")); + // WINML_EXPECT_NO_THROW(LoadModel(L"fns-candy.onnx")); // LearningModelSession imageSession(m_model); // LearningModelBinding imageBinding(imageSession); @@ -434,74 +440,77 @@ TEST_F(LearningModelBindingAPITestGpu, VerifyInvalidBindExceptions) // auto inputName = m_model.InputFeatures().First().Current().Name(); // // Bind invalid map as image input - // EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); + // WINML_EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); // // Bind invalid sequence as image input - // EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); + // WINML_EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); // // Bind invalid tensor type as image input - // EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); + // WINML_EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); // // Bind invalid tensor size as image input // auto tensorFloat = TensorFloat::Create(std::vector { 1, 1, 100, 100 }); - // EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, tensorFloat), winrt::hresult_error, ensureWinmlInvalidBinding); + // WINML_EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, tensorFloat), winrt::hresult_error, ensureWinmlInvalidBinding); // // Bind invalid tensor shape as image input - // EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, tensorInvalidShape), winrt::hresult_error, ensureWinmlInvalidBinding); + // WINML_EXPECT_THROW_SPECIFIC(imageBinding.Bind(inputName, tensorInvalidShape), winrt::hresult_error, ensureWinmlInvalidBinding); /* Verify map bindings throw correct bind exceptions */ - EXPECT_NO_THROW(LoadModel(L"dictvectorizer-int64.onnx")); + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"dictvectorizer-int64.onnx", learningModel)); - LearningModelSession mapSession(m_model); + LearningModelSession mapSession(learningModel); LearningModelBinding mapBinding(mapSession); - auto inputName = m_model.InputFeatures().First().Current().Name(); + auto inputName = learningModel.InputFeatures().First().Current().Name(); // Bind invalid image as image input auto smallImage = FileHelpers::LoadImageFeatureValue(L"100x100.png"); - EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, smallImage), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, smallImage), winrt::hresult_error, ensureWinmlInvalidBinding); // Bind invalid map as image input - EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, abiMap), winrt::hresult_error, ensureWinmlInvalidBinding); // Bind invalid sequence as image input - EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding); // Bind invalid tensor type as image input - EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); + WINML_EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding); } // Verify that it throws an error when binding an invalid name. -TEST_F(LearningModelBindingAPITestGpu, BindInvalidInputName) +static void BindInvalidInputName() { - LearningModelBinding m_binding = nullptr; + LearningModel learningModel = nullptr; + LearningModelBinding learningModelBinding = nullptr; + LearningModelDevice learningModelDevice = nullptr; + LearningModelSession learningModelSession = nullptr; std::wstring modelPath = FileHelpers::GetModulePath() + L"Add_ImageNet1920.onnx"; - EXPECT_NO_THROW(m_model = LearningModel::LoadFromFilePath(modelPath)); - EXPECT_TRUE(m_model != nullptr); - EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Default)); - EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); - EXPECT_NO_THROW(m_binding = LearningModelBinding(m_session)); + WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(modelPath)); + WINML_EXPECT_TRUE(learningModel != nullptr); + WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default)); + WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, learningModelDevice)); + WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession)); VideoFrame iuputImage(BitmapPixelFormat::Rgba8, 1920, 1080); ImageFeatureValue inputTensor = ImageFeatureValue::CreateFromVideoFrame(iuputImage); - auto first = m_model.InputFeatures().First(); + auto first = learningModel.InputFeatures().First(); std::wstring testInvalidName = L"0"; // Verify that testInvalidName is not in model's InputFeatures while (first.HasCurrent()) { - EXPECT_NE(testInvalidName, first.Current().Name()); + WINML_EXPECT_NOT_EQUAL(testInvalidName, first.Current().Name()); first.MoveNext(); } // Bind inputTensor to a valid input name - EXPECT_NO_THROW(m_binding.Bind(L"input_39:0", inputTensor)); + WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"input_39:0", inputTensor)); // Bind inputTensor to an invalid input name - EXPECT_THROW_SPECIFIC(m_binding.Bind(testInvalidName, inputTensor), + WINML_EXPECT_THROW_SPECIFIC(learningModelBinding.Bind(testInvalidName, inputTensor), winrt::hresult_error, [](const winrt::hresult_error& e) -> bool { @@ -509,15 +518,18 @@ TEST_F(LearningModelBindingAPITestGpu, BindInvalidInputName) }); } -TEST_F(LearningModelBindingAPITest, VerifyOutputAfterEvaluateAsyncCalledTwice) +static void VerifyOutputAfterEvaluateAsyncCalledTwice() { - LearningModelBinding m_binding = nullptr; + LearningModel learningModel = nullptr; + LearningModelBinding learningModelBinding = nullptr; + LearningModelDevice learningModelDevice = nullptr; + LearningModelSession learningModelSession = nullptr; std::wstring filePath = FileHelpers::GetModulePath() + L"relu.onnx"; - EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Default)); - EXPECT_NO_THROW(m_model = LearningModel::LoadFromFilePath(filePath)); - EXPECT_TRUE(m_model != nullptr); - EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); - EXPECT_NO_THROW(m_binding = LearningModelBinding(m_session)); + WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default)); + WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(filePath)); + WINML_EXPECT_TRUE(learningModel != nullptr); + WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, learningModelDevice)); + WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession)); auto inputShape = std::vector{ 5 }; auto inputData1 = std::vector{ -50.f, -25.f, 0.f, 25.f, 50.f }; @@ -532,22 +544,22 @@ TEST_F(LearningModelBindingAPITest, VerifyOutputAfterEvaluateAsyncCalledTwice) inputShape, single_threaded_vector(std::move(inputData2)).GetView()); - EXPECT_NO_THROW(m_binding.Bind(L"X", inputValue1)); + WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"X", inputValue1)); auto outputValue = TensorFloat::Create(); - EXPECT_NO_THROW(m_binding.Bind(L"Y", outputValue)); + WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"Y", outputValue)); - EXPECT_NO_THROW(m_session.Evaluate(m_binding, L"")); + WINML_EXPECT_NO_THROW(learningModelSession.Evaluate(learningModelBinding, L"")); auto buffer1 = outputValue.GetAsVectorView(); - EXPECT_TRUE(buffer1 != nullptr); + WINML_EXPECT_TRUE(buffer1 != nullptr); // The second evaluation // If we don't bind output again, the output value will not change - EXPECT_NO_THROW(m_binding.Bind(L"X", inputValue2)); - EXPECT_NO_THROW(m_session.Evaluate(m_binding, L"")); + WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"X", inputValue2)); + WINML_EXPECT_NO_THROW(learningModelSession.Evaluate(learningModelBinding, L"")); auto buffer2 = outputValue.GetAsVectorView(); - EXPECT_EQ(buffer1.Size(), buffer2.Size()); + WINML_EXPECT_EQUAL(buffer1.Size(), buffer2.Size()); bool isSame = true; for (uint32_t i = 0; i < buffer1.Size(); ++i) { @@ -557,7 +569,7 @@ TEST_F(LearningModelBindingAPITest, VerifyOutputAfterEvaluateAsyncCalledTwice) break; } } - EXPECT_FALSE(isSame); + WINML_EXPECT_FALSE(isSame); } static VideoFrame CreateVideoFrame(const wchar_t* path) @@ -569,7 +581,7 @@ static VideoFrame CreateVideoFrame(const wchar_t* path) return VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); } -TEST_F(LearningModelBindingAPITest, VerifyOutputAfterImageBindCalledTwice) +static void VerifyOutputAfterImageBindCalledTwice() { std::wstring fullModelPath = FileHelpers::GetModulePath() + L"model.onnx"; std::wstring fullImagePath1 = FileHelpers::GetModulePath() + L"kitten_224.png"; @@ -577,9 +589,9 @@ TEST_F(LearningModelBindingAPITest, VerifyOutputAfterImageBindCalledTwice) // winml model creation LearningModel model = nullptr; - EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath)); + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath)); LearningModelSession modelSession = nullptr; - EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(LearningModelDeviceKind::Default))); + WINML_EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(LearningModelDeviceKind::Default))); LearningModelBinding modelBinding(modelSession); // create the tensor for the actual output @@ -589,8 +601,8 @@ TEST_F(LearningModelBindingAPITest, VerifyOutputAfterImageBindCalledTwice) // Bind image 1 and evaluate auto frame = CreateVideoFrame(fullImagePath1.c_str()); auto imageTensor = ImageFeatureValue::CreateFromVideoFrame(frame); - EXPECT_NO_THROW(modelBinding.Bind(L"data_0", imageTensor)); - EXPECT_NO_THROW(modelSession.Evaluate(modelBinding, L"")); + WINML_EXPECT_NO_THROW(modelBinding.Bind(L"data_0", imageTensor)); + WINML_EXPECT_NO_THROW(modelSession.Evaluate(modelBinding, L"")); // Store 1st result auto outputVectorView1 = output.GetAsVectorView(); @@ -600,13 +612,13 @@ TEST_F(LearningModelBindingAPITest, VerifyOutputAfterImageBindCalledTwice) // The expected result is that the videoframe will be re-tensorized at bind auto frame2 = CreateVideoFrame(fullImagePath2.c_str()); frame2.CopyToAsync(frame).get(); - EXPECT_NO_THROW(modelBinding.Bind(L"data_0", imageTensor)); - EXPECT_NO_THROW(modelSession.Evaluate(modelBinding, L"")); + WINML_EXPECT_NO_THROW(modelBinding.Bind(L"data_0", imageTensor)); + WINML_EXPECT_NO_THROW(modelSession.Evaluate(modelBinding, L"")); // Store 2nd result auto outputVectorView2 = output.GetAsVectorView(); - EXPECT_EQ(outputVectorView1.Size(), outputVectorView2.Size()); + WINML_EXPECT_EQUAL(outputVectorView1.Size(), outputVectorView2.Size()); bool isSame = true; for (uint32_t i = 0; i < outputVectorView1.Size(); ++i) { @@ -616,5 +628,32 @@ TEST_F(LearningModelBindingAPITest, VerifyOutputAfterImageBindCalledTwice) break; } } - EXPECT_FALSE(isSame); + WINML_EXPECT_FALSE(isSame); +} + +const LearningModelBindingAPITestApi& getapi() { + static constexpr LearningModelBindingAPITestApi api = + { + LearningModelBindingAPITestSetup, + LearningModelBindingAPITestGpuSetup, + CpuSqueezeNet, + CpuSqueezeNetEmptyOutputs, + CpuSqueezeNetUnboundOutputs, + CpuSqueezeNetBindInputTensorAsInspectable, + CastMapInt64, + DictionaryVectorizerMapInt64, + DictionaryVectorizerMapString, + ZipMapInt64, + ZipMapInt64Unbound, + ZipMapString, + GpuSqueezeNet, + GpuSqueezeNetEmptyOutputs, + GpuSqueezeNetUnboundOutputs, + ImageBindingDimensions, + VerifyInvalidBindExceptions, + BindInvalidInputName, + VerifyOutputAfterEvaluateAsyncCalledTwice, + VerifyOutputAfterImageBindCalledTwice + }; + return api; } diff --git a/winml/test/api/LearningModelBindingAPITest.h b/winml/test/api/LearningModelBindingAPITest.h new file mode 100644 index 0000000000000..c93be87c3cb60 --- /dev/null +++ b/winml/test/api/LearningModelBindingAPITest.h @@ -0,0 +1,49 @@ +#include "test.h" + +struct LearningModelBindingAPITestApi { + SetupTest LearningModelBindingAPITestSetup; + SetupTest LearningModelBindingAPITestGpuSetup; + VoidTest CpuSqueezeNet; + VoidTest CpuSqueezeNetEmptyOutputs; + VoidTest CpuSqueezeNetUnboundOutputs; + VoidTest CpuSqueezeNetBindInputTensorAsInspectable; + VoidTest CastMapInt64; + VoidTest DictionaryVectorizerMapInt64; + VoidTest DictionaryVectorizerMapString; + VoidTest ZipMapInt64; + VoidTest ZipMapInt64Unbound; + VoidTest ZipMapString; + VoidTest GpuSqueezeNet; + VoidTest GpuSqueezeNetEmptyOutputs; + VoidTest GpuSqueezeNetUnboundOutputs; + VoidTest ImageBindingDimensions; + VoidTest VerifyInvalidBindExceptions; + VoidTest BindInvalidInputName; + VoidTest VerifyOutputAfterEvaluateAsyncCalledTwice; + VoidTest VerifyOutputAfterImageBindCalledTwice; +}; +const LearningModelBindingAPITestApi& getapi(); + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelBindingAPITest, LearningModelBindingAPITestSetup) +WINML_TEST(LearningModelBindingAPITest, CpuSqueezeNet) +WINML_TEST(LearningModelBindingAPITest, CpuSqueezeNetEmptyOutputs) +WINML_TEST(LearningModelBindingAPITest, CpuSqueezeNetUnboundOutputs) +WINML_TEST(LearningModelBindingAPITest, CpuSqueezeNetBindInputTensorAsInspectable) +WINML_TEST(LearningModelBindingAPITest, CastMapInt64) +WINML_TEST(LearningModelBindingAPITest, DictionaryVectorizerMapInt64) +WINML_TEST(LearningModelBindingAPITest, DictionaryVectorizerMapString) +WINML_TEST(LearningModelBindingAPITest, ZipMapInt64) +WINML_TEST(LearningModelBindingAPITest, ZipMapInt64Unbound) +WINML_TEST(LearningModelBindingAPITest, ZipMapString) +WINML_TEST(LearningModelBindingAPITest, VerifyOutputAfterEvaluateAsyncCalledTwice) +WINML_TEST(LearningModelBindingAPITest, VerifyOutputAfterImageBindCalledTwice) +WINML_TEST_CLASS_END() + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelBindingAPITestGpu, LearningModelBindingAPITestGpuSetup) +WINML_TEST(LearningModelBindingAPITestGpu, GpuSqueezeNet) +WINML_TEST(LearningModelBindingAPITestGpu, GpuSqueezeNetEmptyOutputs) +WINML_TEST(LearningModelBindingAPITestGpu, GpuSqueezeNetUnboundOutputs) +WINML_TEST(LearningModelBindingAPITestGpu, ImageBindingDimensions) +WINML_TEST(LearningModelBindingAPITestGpu, VerifyInvalidBindExceptions) +WINML_TEST(LearningModelBindingAPITestGpu, BindInvalidInputName) +WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/api/LearningModelSessionAPITest.cpp b/winml/test/api/LearningModelSessionAPITest.cpp index bc6494a3a71ac..45af7cbd831b1 100644 --- a/winml/test/api/LearningModelSessionAPITest.cpp +++ b/winml/test/api/LearningModelSessionAPITest.cpp @@ -1,10 +1,10 @@ #include "testPch.h" -#include "APITest.h" - -#include "winrt/Windows.Storage.h" -#include "DeviceHelpers.h" +#include "APITest.h" +#include "CommonDeviceHelpers.h" +#include "LearningModelSessionAPITest.h" #include "protobufHelpers.h" +#include "winrt/Windows.Storage.h" #include #include @@ -16,144 +16,151 @@ using namespace winrt::Windows::Foundation::Collections; using winrt::Windows::Foundation::IPropertyValue; -class LearningModelSessionAPITests : public APITest -{}; +static void LearningModelSessionAPITestSetup() { + init_apartment(); +} -class LearningModelSessionAPITestsGpu : public APITest -{ -protected: - void SetUp() override - { - GPUTEST - } -}; +static void LearningModelSessionAPITestGpuSetup() { + GPUTEST; + init_apartment(); +} -class LearningModelSessionAPITestsSkipEdgeCore : public LearningModelSessionAPITestsGpu -{ -protected: - void SetUp() override - { - LearningModelSessionAPITestsGpu::SetUp(); - SKIP_EDGECORE - } -}; +static void LearningModelSessionAPITestsSkipEdgeCoreSetup() { + LearningModelSessionAPITestGpuSetup(); + SKIP_EDGECORE +} -TEST_F(LearningModelSessionAPITests, CreateSessionDeviceDefault) +static void CreateSessionDeviceDefault() { - EXPECT_NO_THROW(LoadModel(L"model.onnx")); + LearningModel learningModel = nullptr; + LearningModelDevice learningModelDevice = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"model.onnx", learningModel)); - EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Default)); - EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default)); + WINML_EXPECT_NO_THROW(LearningModelSession(learningModel, learningModelDevice)); } -TEST_F(LearningModelSessionAPITests, CreateSessionDeviceCpu) +static void CreateSessionDeviceCpu() { - EXPECT_NO_THROW(LoadModel(L"model.onnx")); + LearningModel learningModel = nullptr; + LearningModelDevice learningModelDevice = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"model.onnx", learningModel)); - EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Cpu)); - EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Cpu)); + WINML_EXPECT_NO_THROW(LearningModelSession(learningModel, learningModelDevice)); // for the CPU device, make sure that we get back NULL and 0 for any device properties - EXPECT_FALSE(m_device.Direct3D11Device()); + WINML_EXPECT_EQUAL(learningModelDevice.Direct3D11Device(), nullptr); LARGE_INTEGER id; - id.QuadPart = GetAdapterIdQuadPart(); - EXPECT_EQ(id.LowPart, static_cast(0)); - EXPECT_EQ(id.HighPart, 0); + id.QuadPart = APITest::GetAdapterIdQuadPart(learningModelDevice); + WINML_EXPECT_EQUAL(id.LowPart, static_cast(0)); + WINML_EXPECT_EQUAL(id.HighPart, 0); } -TEST_F(LearningModelSessionAPITests, CreateSessionWithModelLoadedFromStream) +static void CreateSessionWithModelLoadedFromStream() { + LearningModel learningModel = nullptr; + LearningModelDevice learningModelDevice = nullptr; std::wstring path = FileHelpers::GetModulePath() + L"model.onnx"; auto storageFile = winrt::Windows::Storage::StorageFile::GetFileFromPathAsync(path).get(); - EXPECT_NO_THROW(m_model = LearningModel::LoadFromStream(storageFile)); + WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromStream(storageFile)); - EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::Default)); - EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default)); + WINML_EXPECT_NO_THROW(LearningModelSession(learningModel, learningModelDevice)); } -TEST_F(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectX) +static void CreateSessionDeviceDirectX() { - EXPECT_NO_THROW(LoadModel(L"model.onnx")); + LearningModel learningModel = nullptr; + LearningModelDevice learningModelDevice = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"model.onnx", learningModel)); - EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::DirectX)); - EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::DirectX)); + WINML_EXPECT_NO_THROW(LearningModelSession(learningModel, learningModelDevice)); } -TEST_F(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectXHighPerformance) +static void CreateSessionDeviceDirectXHighPerformance() { - EXPECT_NO_THROW(LoadModel(L"model.onnx")); + LearningModel learningModel = nullptr; + LearningModelDevice learningModelDevice = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"model.onnx", learningModel)); - EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::DirectXHighPerformance)); - EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::DirectXHighPerformance)); + WINML_EXPECT_NO_THROW(LearningModelSession(learningModel, learningModelDevice)); } -TEST_F(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectXMinimumPower) +static void CreateSessionDeviceDirectXMinimumPower() { - EXPECT_NO_THROW(LoadModel(L"model.onnx")); + LearningModel learningModel = nullptr; + LearningModelDevice learningModelDevice = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"model.onnx", learningModel)); - EXPECT_NO_THROW(m_device = LearningModelDevice(LearningModelDeviceKind::DirectXMinPower)); - EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); + WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::DirectXMinPower)); + WINML_EXPECT_NO_THROW(LearningModelSession(learningModel, learningModelDevice)); } -TEST_F(LearningModelSessionAPITestsSkipEdgeCore, AdapterIdAndDevice) -{ - EXPECT_NO_THROW(LoadModel(L"model.onnx")); +static void AdapterIdAndDevice() { + LearningModel learningModel = nullptr; + LearningModelDevice learningModelDevice = nullptr; + LearningModelSession learningModelSession = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"model.onnx", learningModel)); com_ptr factory; - EXPECT_HRESULT_SUCCEEDED(CreateDXGIFactory1(__uuidof(IDXGIFactory6), factory.put_void())); + WINML_EXPECT_HRESULT_SUCCEEDED(CreateDXGIFactory1(__uuidof(IDXGIFactory6), factory.put_void())); com_ptr adapter; - m_device = LearningModelDevice(LearningModelDeviceKind::DirectX); - EXPECT_HRESULT_SUCCEEDED(factory->EnumAdapters(0, adapter.put())); + learningModelDevice = LearningModelDevice(LearningModelDeviceKind::DirectX); + WINML_EXPECT_HRESULT_SUCCEEDED(factory->EnumAdapters(0, adapter.put())); DXGI_ADAPTER_DESC desc; - EXPECT_HRESULT_SUCCEEDED(adapter->GetDesc(&desc)); + WINML_EXPECT_HRESULT_SUCCEEDED(adapter->GetDesc(&desc)); LARGE_INTEGER id; - id.QuadPart = GetAdapterIdQuadPart(); - EXPECT_EQ(desc.AdapterLuid.LowPart, id.LowPart); - EXPECT_EQ(desc.AdapterLuid.HighPart, id.HighPart); - EXPECT_TRUE(m_device.Direct3D11Device() != nullptr); + id.QuadPart = APITest::GetAdapterIdQuadPart(learningModelDevice); + WINML_EXPECT_EQUAL(desc.AdapterLuid.LowPart, id.LowPart); + WINML_EXPECT_EQUAL(desc.AdapterLuid.HighPart, id.HighPart); + WINML_EXPECT_TRUE(learningModelDevice.Direct3D11Device() != nullptr); - m_device = LearningModelDevice(LearningModelDeviceKind::DirectXHighPerformance); + learningModelDevice = LearningModelDevice(LearningModelDeviceKind::DirectXHighPerformance); adapter = nullptr; - EXPECT_HRESULT_SUCCEEDED(factory->EnumAdapterByGpuPreference(0, DXGI_GPU_PREFERENCE_HIGH_PERFORMANCE, __uuidof(IDXGIAdapter), adapter.put_void())); - EXPECT_HRESULT_SUCCEEDED(adapter->GetDesc(&desc)); - id.QuadPart = GetAdapterIdQuadPart(); - EXPECT_EQ(desc.AdapterLuid.LowPart, id.LowPart); - EXPECT_EQ(desc.AdapterLuid.HighPart, id.HighPart); - EXPECT_TRUE(m_device.Direct3D11Device() != nullptr); + WINML_EXPECT_HRESULT_SUCCEEDED(factory->EnumAdapterByGpuPreference(0, DXGI_GPU_PREFERENCE_HIGH_PERFORMANCE, __uuidof(IDXGIAdapter), adapter.put_void())); + WINML_EXPECT_HRESULT_SUCCEEDED(adapter->GetDesc(&desc)); + id.QuadPart = APITest::GetAdapterIdQuadPart(learningModelDevice); + WINML_EXPECT_EQUAL(desc.AdapterLuid.LowPart, id.LowPart); + WINML_EXPECT_EQUAL(desc.AdapterLuid.HighPart, id.HighPart); + WINML_EXPECT_TRUE(learningModelDevice.Direct3D11Device() != nullptr); adapter = nullptr; - m_device = LearningModelDevice(LearningModelDeviceKind::DirectXMinPower); - EXPECT_HRESULT_SUCCEEDED(factory->EnumAdapterByGpuPreference(0, DXGI_GPU_PREFERENCE_MINIMUM_POWER, __uuidof(IDXGIAdapter), adapter.put_void())); - EXPECT_HRESULT_SUCCEEDED(adapter->GetDesc(&desc)); - id.QuadPart = GetAdapterIdQuadPart(); - EXPECT_EQ(desc.AdapterLuid.LowPart, id.LowPart); - EXPECT_EQ(desc.AdapterLuid.HighPart, id.HighPart); - EXPECT_TRUE(m_device.Direct3D11Device() != nullptr); - - EXPECT_NO_THROW(m_session = LearningModelSession(m_model, m_device)); - EXPECT_EQ(m_session.Device().AdapterId(), m_device.AdapterId()); + learningModelDevice = LearningModelDevice(LearningModelDeviceKind::DirectXMinPower); + WINML_EXPECT_HRESULT_SUCCEEDED(factory->EnumAdapterByGpuPreference(0, DXGI_GPU_PREFERENCE_MINIMUM_POWER, __uuidof(IDXGIAdapter), adapter.put_void())); + WINML_EXPECT_HRESULT_SUCCEEDED(adapter->GetDesc(&desc)); + id.QuadPart = APITest::GetAdapterIdQuadPart(learningModelDevice); + WINML_EXPECT_EQUAL(desc.AdapterLuid.LowPart, id.LowPart); + WINML_EXPECT_EQUAL(desc.AdapterLuid.HighPart, id.HighPart); + WINML_EXPECT_TRUE(learningModelDevice.Direct3D11Device() != nullptr); + + WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, learningModelDevice)); + WINML_EXPECT_EQUAL(learningModelSession.Device().AdapterId(), learningModelDevice.AdapterId()); } -TEST_F(LearningModelSessionAPITests, EvaluateFeatures) +static void EvaluateFeatures() { std::vector shape = { 4 }; std::vector data = { L"one", L"two", L"three", L"four" }; // create from buffer auto tensor = TensorString::CreateFromArray(shape, data); - EXPECT_EQ(tensor.GetAsVectorView().Size(), data.size()); - EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); + WINML_EXPECT_EQUAL(tensor.GetAsVectorView().Size(), data.size()); + WINML_EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); // create from vector view auto dataCopy = data; tensor = TensorString::CreateFromIterable( shape, winrt::single_threaded_vector(std::move(dataCopy)).GetView()); - EXPECT_EQ(tensor.GetAsVectorView().Size(), data.size()); - EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); + WINML_EXPECT_EQUAL(tensor.GetAsVectorView().Size(), data.size()); + WINML_EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); - EXPECT_NO_THROW(LoadModel(L"id-tensor-string.onnx")); - LearningModelSession session(m_model); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"id-tensor-string.onnx", learningModel)); + LearningModelSession session(learningModel); auto outputTensor = TensorString::Create(); @@ -164,29 +171,30 @@ TEST_F(LearningModelSessionAPITests, EvaluateFeatures) session.EvaluateFeatures(featureswinrtmap, L"0"); // verify identity model round-trip works - EXPECT_EQ(outputTensor.GetAsVectorView().Size(), data.size()); - EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(outputTensor.GetAsVectorView()))); + WINML_EXPECT_EQUAL(outputTensor.GetAsVectorView().Size(), data.size()); + WINML_EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(outputTensor.GetAsVectorView()))); } -TEST_F(LearningModelSessionAPITests, EvaluateFeaturesAsync) +static void EvaluateFeaturesAsync() { std::vector shape = { 4 }; std::vector data = { L"one", L"two", L"three", L"four" }; // create from buffer auto tensor = TensorString::CreateFromArray(shape, data); - EXPECT_EQ(tensor.GetAsVectorView().Size(), data.size()); - EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); + WINML_EXPECT_EQUAL(tensor.GetAsVectorView().Size(), data.size()); + WINML_EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); // create from vector view auto dataCopy = data; tensor = TensorString::CreateFromIterable( shape, winrt::single_threaded_vector(std::move(dataCopy)).GetView()); - EXPECT_EQ(tensor.GetAsVectorView().Size(), data.size()); - EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); + WINML_EXPECT_EQUAL(tensor.GetAsVectorView().Size(), data.size()); + WINML_EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(tensor.GetAsVectorView()))); - EXPECT_NO_THROW(LoadModel(L"id-tensor-string.onnx")); - LearningModelSession session(m_model); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"id-tensor-string.onnx", learningModel)); + LearningModelSession session(learningModel); auto outputTensor = TensorString::Create(shape); @@ -197,37 +205,39 @@ TEST_F(LearningModelSessionAPITests, EvaluateFeaturesAsync) session.EvaluateFeaturesAsync(featureswinrtmap, L"0").get(); // verify identity model round-trip works - EXPECT_EQ(outputTensor.GetAsVectorView().Size(), data.size()); - EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(outputTensor.GetAsVectorView()))); + WINML_EXPECT_EQUAL(outputTensor.GetAsVectorView().Size(), data.size()); + WINML_EXPECT_TRUE(std::equal(data.cbegin(), data.cend(), begin(outputTensor.GetAsVectorView()))); } -TEST_F(LearningModelSessionAPITests, EvaluationProperties) +static void EvaluationProperties() { // load a model - EXPECT_NO_THROW(LoadModel(L"model.onnx")); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"model.onnx", learningModel)); // create a session - m_session = LearningModelSession(m_model); + LearningModelSession learningModelSession = nullptr; + learningModelSession = LearningModelSession(learningModel); // set a property auto value = winrt::Windows::Foundation::PropertyValue::CreateBoolean(true); - m_session.EvaluationProperties().Insert(L"propName1", value); + learningModelSession.EvaluationProperties().Insert(L"propName1", value); // get the property and make sure it's there with the right value - auto value2 = m_session.EvaluationProperties().Lookup(L"propName1"); - EXPECT_EQ(value2.as().GetBoolean(), true); + auto value2 = learningModelSession.EvaluationProperties().Lookup(L"propName1"); + WINML_EXPECT_EQUAL(value2.as().GetBoolean(), true); } static LearningModelSession CreateSession(LearningModel model) { LearningModelDevice device(nullptr); - EXPECT_NO_THROW(device = LearningModelDevice(LearningModelDeviceKind::DirectX)); + WINML_EXPECT_NO_THROW(device = LearningModelDevice(LearningModelDeviceKind::DirectX)); LearningModelSession session(nullptr); - if (DeviceHelpers::IsFloat16Supported(device)) + if (CommonDeviceHelpers::IsFloat16Supported(device)) { - EXPECT_NO_THROW(session = LearningModelSession(model, device)); + WINML_EXPECT_NO_THROW(session = LearningModelSession(model, device)); } else { - EXPECT_THROW_SPECIFIC( + WINML_EXPECT_THROW_SPECIFIC( session = LearningModelSession(model, device), winrt::hresult_error, [](const winrt::hresult_error& e) -> bool @@ -239,26 +249,28 @@ static LearningModelSession CreateSession(LearningModel model) return session; } -TEST_F(LearningModelSessionAPITestsGpu, CreateSessionWithCastToFloat16InModel) +static void CreateSessionWithCastToFloat16InModel() { // load a model - EXPECT_NO_THROW(LoadModel(L"fp16-truncate-with-cast.onnx")); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"fp16-truncate-with-cast.onnx", learningModel)); - CreateSession(m_model); + CreateSession(learningModel); } -TEST_F(LearningModelSessionAPITestsGpu, DISABLED_CreateSessionWithFloat16InitializersInModel) +static void DISABLED_CreateSessionWithFloat16InitializersInModel() { // Disabled due to https://microsoft.visualstudio.com/DefaultCollection/OS/_workitems/edit/21624720: // Model fails to resolve due to ORT using incorrect IR version within partition // load a model - EXPECT_NO_THROW(LoadModel(L"fp16-initializer.onnx")); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"fp16-initializer.onnx", learningModel)); - CreateSession(m_model); + CreateSession(learningModel); } -static void EvaluateSessionAndCloseModel( +static void EvaluateSessionAndCloseModelHelper( LearningModelDeviceKind kind, bool close_model_on_session_creation) { @@ -275,7 +287,7 @@ static void EvaluateSessionAndCloseModel( // ensure you can create a session from the model LearningModelSession session(nullptr); - EXPECT_NO_THROW(session = LearningModelSession(model, device, options)); + WINML_EXPECT_NO_THROW(session = LearningModelSession(model, device, options)); std::vector input(1000); std::iota(std::begin(input), std::end(input), 0.0f); @@ -284,12 +296,12 @@ static void EvaluateSessionAndCloseModel( binding.Bind(L"input", tensor_input); LearningModelEvaluationResult result(nullptr); - EXPECT_NO_THROW(result = session.Evaluate(binding, L"")); + WINML_EXPECT_NO_THROW(result = session.Evaluate(binding, L"")); if (close_model_on_session_creation) { // ensure that the model has been closed - EXPECT_THROW_SPECIFIC( + WINML_EXPECT_THROW_SPECIFIC( LearningModelSession(model, device, options), winrt::hresult_error, [](const winrt::hresult_error& e) -> bool @@ -299,19 +311,20 @@ static void EvaluateSessionAndCloseModel( } else { - EXPECT_NO_THROW(LearningModelSession(model, device, options)); + WINML_EXPECT_NO_THROW(LearningModelSession(model, device, options)); } } -TEST_F(LearningModelSessionAPITests, EvaluateSessionAndCloseModel) +static void EvaluateSessionAndCloseModel() { - EXPECT_NO_THROW(::EvaluateSessionAndCloseModel(LearningModelDeviceKind::Cpu, true)); - EXPECT_NO_THROW(::EvaluateSessionAndCloseModel(LearningModelDeviceKind::Cpu, false)); + WINML_EXPECT_NO_THROW(::EvaluateSessionAndCloseModelHelper(LearningModelDeviceKind::Cpu, true)); + WINML_EXPECT_NO_THROW(::EvaluateSessionAndCloseModelHelper(LearningModelDeviceKind::Cpu, false)); } -TEST_F(LearningModelSessionAPITests, CloseSession) +static void CloseSession() { - EXPECT_NO_THROW(LoadModel(L"model.onnx")); + LearningModel learningModel = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"model.onnx", learningModel)); LearningModelSession session = nullptr; /* @@ -329,7 +342,7 @@ TEST_F(LearningModelSessionAPITests, CloseSession) SIZE_T afterSessionCloseWorkingSetSize = 0; bool getProcessMemoryInfoSuccess = false; */ - EXPECT_NO_THROW(session = LearningModelSession(m_model)); + WINML_EXPECT_NO_THROW(session = LearningModelSession(learningModel)); /* // Get the current process memory info after session creation. @@ -341,7 +354,7 @@ TEST_F(LearningModelSessionAPITests, CloseSession) beforeSessionCloseWorkingSetSize = pmc.WorkingSetSize; pmc = { 0 }; */ - EXPECT_NO_THROW(session.Close()); + WINML_EXPECT_NO_THROW(session.Close()); /* Bug 23659026: Working set difference tolerance is too tight for LearningModelSessionAPITests::CloseSession @@ -367,17 +380,41 @@ TEST_F(LearningModelSessionAPITests, CloseSession) */ // verify that model still has metadata info after session close - std::wstring author(m_model.Author()); - EXPECT_EQ(author, L"onnx-caffe2"); + std::wstring author(learningModel.Author()); + WINML_EXPECT_EQUAL(author, L"onnx-caffe2"); // verify that session throws RO_E_CLOSED error std::vector input(1 * 3 * 224 * 224, 0); std::vector shape = { 1, 3, 224, 224 }; auto tensor_input = TensorFloat::CreateFromShapeArrayAndDataArray(shape, input); - EXPECT_THROW_SPECIFIC(LearningModelBinding binding(session), + WINML_EXPECT_THROW_SPECIFIC(LearningModelBinding binding(session), winrt::hresult_error, [](const winrt::hresult_error &e) -> bool { return e.code() == RO_E_CLOSED; }); } + +const LearningModelSesssionAPITestApi& getapi() { + static constexpr LearningModelSesssionAPITestApi api = + { + LearningModelSessionAPITestSetup, + LearningModelSessionAPITestGpuSetup, + LearningModelSessionAPITestsSkipEdgeCoreSetup, + CreateSessionDeviceDefault, + CreateSessionDeviceCpu, + CreateSessionWithModelLoadedFromStream, + CreateSessionDeviceDirectX, + CreateSessionDeviceDirectXHighPerformance, + CreateSessionDeviceDirectXMinimumPower, + AdapterIdAndDevice, + EvaluateFeatures, + EvaluateFeaturesAsync, + EvaluationProperties, + CreateSessionWithCastToFloat16InModel, + DISABLED_CreateSessionWithFloat16InitializersInModel, + EvaluateSessionAndCloseModel, + CloseSession, + }; + return api; +} diff --git a/winml/test/api/LearningModelSessionAPITest.h b/winml/test/api/LearningModelSessionAPITest.h new file mode 100644 index 0000000000000..b98cb56f9fbd7 --- /dev/null +++ b/winml/test/api/LearningModelSessionAPITest.h @@ -0,0 +1,44 @@ +#include "test.h" + +struct LearningModelSesssionAPITestApi { + SetupTest LearningModelSessionAPITestSetup; + SetupTest LearningModelSessionAPITestGpuSetup; + SetupTest LearningModelSessionAPITestsSkipEdgeCoreSetup; + VoidTest CreateSessionDeviceDefault; + VoidTest CreateSessionDeviceCpu; + VoidTest CreateSessionWithModelLoadedFromStream; + VoidTest CreateSessionDeviceDirectX; + VoidTest CreateSessionDeviceDirectXHighPerformance; + VoidTest CreateSessionDeviceDirectXMinimumPower; + VoidTest AdapterIdAndDevice; + VoidTest EvaluateFeatures; + VoidTest EvaluateFeaturesAsync; + VoidTest EvaluationProperties; + VoidTest CreateSessionWithCastToFloat16InModel; + VoidTest DISABLED_CreateSessionWithFloat16InitializersInModel; + VoidTest EvaluateSessionAndCloseModel; + VoidTest CloseSession; +}; +const LearningModelSesssionAPITestApi& getapi(); + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelSessionAPITest, LearningModelSessionAPITestSetup) +WINML_TEST(LearningModelSessionAPITest, CreateSessionDeviceDefault) +WINML_TEST(LearningModelSessionAPITest,CreateSessionDeviceCpu) +WINML_TEST(LearningModelSessionAPITest,CreateSessionWithModelLoadedFromStream) +WINML_TEST(LearningModelSessionAPITest,EvaluateFeatures) +WINML_TEST(LearningModelSessionAPITest,EvaluateFeaturesAsync) +WINML_TEST(LearningModelSessionAPITest,EvaluationProperties) +WINML_TEST(LearningModelSessionAPITest,EvaluateSessionAndCloseModel) +WINML_TEST_CLASS_END() + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelSessionAPITestGpu, LearningModelSessionAPITestGpuSetup) +WINML_TEST(LearningModelSessionAPITestGpu, CreateSessionDeviceDirectX) +WINML_TEST(LearningModelSessionAPITestGpu, CreateSessionDeviceDirectXHighPerformance) +WINML_TEST(LearningModelSessionAPITestGpu, CreateSessionDeviceDirectXMinimumPower) +WINML_TEST(LearningModelSessionAPITestGpu, CreateSessionWithCastToFloat16InModel) +WINML_TEST(LearningModelSessionAPITestGpu, DISABLED_CreateSessionWithFloat16InitializersInModel) +WINML_TEST_CLASS_END() + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(LearningModelSessionAPITestsSkipEdgeCore, LearningModelSessionAPITestsSkipEdgeCoreSetup) +WINML_TEST(LearningModelSessionAPITestsSkipEdgeCore, AdapterIdAndDevice) +WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/common/SqueezeNetValidator.cpp b/winml/test/common/SqueezeNetValidator.cpp index e3fc0978ad086..2eacc456cd017 100644 --- a/winml/test/common/SqueezeNetValidator.cpp +++ b/winml/test/common/SqueezeNetValidator.cpp @@ -2,12 +2,11 @@ #include "protobufHelpers.h" #include "fileHelpers.h" #include "core/common/common.h" -#include #include #include #include #include - +#include // using namespace winrt::Windows::Foundation; using namespace winrt::Windows::AI::MachineLearning; using namespace winrt::Windows::Foundation::Collections; @@ -35,12 +34,12 @@ static void BindImage( if (bindAsInspectable) { - EXPECT_NO_THROW(binding.Bind(name, frame)); + WINML_EXPECT_NO_THROW(binding.Bind(name, frame)); } else { auto imagetensor = ImageFeatureValue::CreateFromVideoFrame(frame); - EXPECT_NO_THROW(binding.Bind(name, imagetensor)); + WINML_EXPECT_NO_THROW(binding.Bind(name, imagetensor)); } } @@ -50,15 +49,15 @@ static void BindTensor( ITensor inputTensor, bool bindAsInspectable = false) { - EXPECT_TRUE(inputTensor != nullptr); + WINML_EXPECT_TRUE(inputTensor != nullptr); if (bindAsInspectable) { - EXPECT_NO_THROW(binding.Bind(name, inputTensor.as().GetAsVectorView())); + WINML_EXPECT_NO_THROW(binding.Bind(name, inputTensor.as().GetAsVectorView())); } else { - EXPECT_NO_THROW(binding.Bind(name, inputTensor)); + WINML_EXPECT_NO_THROW(binding.Bind(name, inputTensor)); } } @@ -75,11 +74,11 @@ ITensor BindOutput( { case OutputBindingStrategy::Bound: outputTensor = T::Create(shape); - EXPECT_NO_THROW(binding.Bind(name, outputTensor)); + WINML_EXPECT_NO_THROW(binding.Bind(name, outputTensor)); break; case OutputBindingStrategy::Empty: outputTensor = T::Create(); - EXPECT_NO_THROW(binding.Bind(name, outputTensor)); + WINML_EXPECT_NO_THROW(binding.Bind(name, outputTensor)); break; case OutputBindingStrategy::Unbound: __fallthrough; @@ -104,7 +103,7 @@ ImageFeatureValue BindImageOutput( SoftwareBitmap bitmap(BitmapPixelFormat::Bgra8, 720, 720); VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(bitmap); outputTensor = ImageFeatureValue::CreateFromVideoFrame(frame); - EXPECT_NO_THROW(binding.Bind(name, outputTensor)); + WINML_EXPECT_NO_THROW(binding.Bind(name, outputTensor)); break; } case OutputBindingStrategy::Unbound: @@ -136,10 +135,10 @@ void ModelValidator::FnsCandy16( // WinML model creation LearningModel model = nullptr; - EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath)); + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath)); LearningModelSession modelSession = nullptr; - EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(deviceKind))); + WINML_EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(deviceKind))); LearningModelBinding modelBinding(modelSession); auto fullImagePath = modulePath + inputDataImageFileName; @@ -147,7 +146,7 @@ void ModelValidator::FnsCandy16( // create the tensor for the actual output auto output = model.OutputFeatures().First().Current(); - EXPECT_TRUE(output.Kind() == LearningModelFeatureKind::Tensor); + WINML_EXPECT_TRUE(output.Kind() == LearningModelFeatureKind::Tensor); auto shape = winrt::single_threaded_vector(std::vector {1, 1}); auto outputTensor = BindImageOutput(outputBindingStrategy, modelBinding, outputDataBindingName); @@ -155,7 +154,7 @@ void ModelValidator::FnsCandy16( // Evaluate the model std::cout << "Calling EvaluateSync on instance" << instance << "\n"; LearningModelEvaluationResult result = nullptr; - EXPECT_NO_THROW(result = modelSession.Evaluate(modelBinding, {})); + WINML_EXPECT_NO_THROW(result = modelSession.Evaluate(modelBinding, {})); // Get results if (outputBindingStrategy == OutputBindingStrategy::Unbound) @@ -167,7 +166,7 @@ void ModelValidator::FnsCandy16( } else { - EXPECT_EQ(result.Outputs().Lookup(outputDataBindingName), outputTensor); + WINML_EXPECT_EQUAL(result.Outputs().Lookup(outputDataBindingName), outputTensor); auto softwareBitmap = outputTensor.VideoFrame().SoftwareBitmap(); @@ -203,10 +202,10 @@ void ModelValidator::SqueezeNet( // WinML model creation LearningModel model = nullptr; - EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath)); + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath)); LearningModelSession modelSession = nullptr; - EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(deviceKind))); + WINML_EXPECT_NO_THROW(modelSession = LearningModelSession(model, LearningModelDevice(deviceKind))); LearningModelBinding modelBinding(modelSession); @@ -224,11 +223,11 @@ void ModelValidator::SqueezeNet( // load up the expected output auto expectedResultsTensor = ProtobufHelpers::LoadTensorFromProtobufFile(outputFileName, false); - EXPECT_TRUE(expectedResultsTensor != nullptr); + WINML_EXPECT_TRUE(expectedResultsTensor != nullptr); // create the tensor for the actual output auto output = model.OutputFeatures().First().Current(); - EXPECT_TRUE(output.Kind() == LearningModelFeatureKind::Tensor); + WINML_EXPECT_TRUE(output.Kind() == LearningModelFeatureKind::Tensor); auto outputTensor = BindOutput( outputBindingStrategy, modelBinding, outputDataBindingName, expectedResultsTensor.Shape()); @@ -236,7 +235,7 @@ void ModelValidator::SqueezeNet( // Evaluate the model std::cout << "Calling EvaluateSync on instance" << instance << "\n"; LearningModelEvaluationResult result = nullptr; - EXPECT_NO_THROW(result = modelSession.Evaluate(modelBinding, {})); + WINML_EXPECT_NO_THROW(result = modelSession.Evaluate(modelBinding, {})); // Get results if (outputBindingStrategy == OutputBindingStrategy::Unbound) @@ -247,21 +246,22 @@ void ModelValidator::SqueezeNet( } else { - EXPECT_EQ(result.Outputs().Lookup(outputDataBindingName), outputTensor); + WINML_EXPECT_EQUAL(result.Outputs().Lookup(outputDataBindingName), outputTensor); } auto outDataExpected = expectedResultsTensor.as().GetAsVectorView(); auto outDataActual = outputTensor.as().GetAsVectorView(); - EXPECT_TRUE(outDataActual.Size() == outDataExpected.Size()); + WINML_EXPECT_TRUE(outDataActual.Size() == outDataExpected.Size()); for (uint32_t i = 0; i < outDataActual.Size(); i++) { float delta = std::abs(outDataActual.GetAt(i) - outDataExpected.GetAt(i)); if (delta > dataTolerance) { - ADD_FAILURE() << "EXPECTED: " << outDataExpected.GetAt(i) << " , ACTUAL: " << outDataActual.GetAt(i) + std::stringstream ss; + ss << "EXPECTED: " << outDataExpected.GetAt(i) << " , ACTUAL: " << outDataActual.GetAt(i) << "instance " << instance << ", element " << i; - + WINML_LOG_ERROR(ss.str().c_str()); } } } diff --git a/winml/test/common/SqueezeNetValidator.h b/winml/test/common/SqueezeNetValidator.h index 68b44ddb2dd3b..613df41853fbe 100644 --- a/winml/test/common/SqueezeNetValidator.h +++ b/winml/test/common/SqueezeNetValidator.h @@ -6,7 +6,7 @@ #pragma once -#include +#include "std.h" enum OutputBindingStrategy { Bound, Unbound, Empty }; diff --git a/winml/test/common/googleTestMacros.h b/winml/test/common/googleTestMacros.h new file mode 100644 index 0000000000000..b5199df0bb668 --- /dev/null +++ b/winml/test/common/googleTestMacros.h @@ -0,0 +1,82 @@ +#include +#include "runtimeParameters.h" + +#define TEST_GROUP_BEGIN(group_name) +#define TEST_GROUP_END() + +#define WINML_TEST(group_name, test_name) \ + TEST_F(group_name, test_name) { \ + getapi().test_name(); \ + } + +#define WINML_TEST_CLASS_BEGIN_NO_SETUP(test_class_name) \ + namespace { \ + class test_class_name : public ::testing::Test { \ + }; + +#define WINML_TEST_CLASS_BEGIN_WITH_SETUP(test_class_name, setup_method) \ + namespace { \ + class test_class_name : public ::testing::Test { \ + protected: \ + void SetUp() override { \ + getapi().setup_method(); \ + } \ + }; + +#define WINML_TEST_CLASS_END() } + +// For old versions of gtest without GTEST_SKIP, stream the message and return success instead +#ifndef GTEST_SKIP +#define GTEST_SKIP_(message) \ + return GTEST_MESSAGE_(message, ::testing::TestPartResult::kSuccess) +#define GTEST_SKIP GTEST_SKIP_("") +#endif + +#define EXPECT_THROW_SPECIFIC(statement, exception, condition) \ + EXPECT_THROW( \ + try { \ + statement; \ + } catch (const exception& e) { \ + EXPECT_TRUE(condition(e)); \ + throw; \ + } \ + , exception); + +#ifndef INSTANTIATE_TEST_SUITE_P +// Use the old name, removed in newer versions of googletest +#define INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_CASE_P +#endif + +#define WINML_SKIP_TEST(message) \ + GTEST_SKIP() << message; + +#define WINML_EXPECT_NO_THROW(statement) EXPECT_NO_THROW(statement) +#define WINML_EXPECT_TRUE(statement) EXPECT_TRUE(statement) +#define WINML_EXPECT_FALSE(statement) EXPECT_FALSE(statement) +#define WINML_EXPECT_EQUAL(val1, val2) EXPECT_EQ(val1, val2) +#define WINML_EXPECT_NOT_EQUAL(val1, val2) EXPECT_NE(val1, val2) + +#define WINML_LOG_ERROR(message) \ + ADD_FAILURE() << message +#define WINML_LOG_COMMENT(message)\ + SCOPED_TRACE(message) +#define WINML_EXPECT_HRESULT_SUCCEEDED(hresult_expression) EXPECT_HRESULT_SUCCEEDED(hresult_expression) +#define WINML_EXPECT_HRESULT_FAILED(hresult_expression) EXPECT_HRESULT_FAILED(hresult_expression) +#define WINML_EXPECT_THROW_SPECIFIC(statement, exception, condition) EXPECT_THROW_SPECIFIC(statement, exception, condition) + +#ifndef USE_DML +#define GPUTEST \ + WINML_SUPRESS_UNREACHABLE_BELOW(WINML_SKIP_TEST("GPU tests disabled because this is a WinML only build (no DML)")) +#else +#define GPUTEST \ + if (auto noGpuTests = RuntimeParameters::Parameters.find("noGPUtests"); \ + noGpuTests != RuntimeParameters::Parameters.end() && noGpuTests->second != "0") { \ + WINML_SKIP_TEST("GPU tests disabled"); \ + } +#endif + +#define SKIP_EDGECORE \ + if (auto isEdgeCore = RuntimeParameters::Parameters.find("EdgeCore"); \ + isEdgeCore != RuntimeParameters::Parameters.end() && isEdgeCore->second != "0") { \ + WINML_SKIP_TEST("Test can't be run in EdgeCore"); \ + } diff --git a/winml/test/common/protobufHelpers.cpp b/winml/test/common/protobufHelpers.cpp index a5bc12a70e4a2..dbd440b9ab71b 100644 --- a/winml/test/common/protobufHelpers.cpp +++ b/winml/test/common/protobufHelpers.cpp @@ -1,10 +1,11 @@ -#define _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS +#ifndef _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS +#define _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS +#endif // LotusRT #include "core/framework/allocatormgr.h" #include "core/common/logging/logging.h" #include "core/common/logging/sinks/clog_sink.h" - #include "protobufHelpers.h" #pragma warning(push) @@ -12,7 +13,6 @@ #include "onnx/onnx-ml.pb.h" #pragma warning(pop) -#include #include #include "winrt/Windows.Storage.Streams.h" @@ -66,33 +66,35 @@ bool LoadTensorFromPb(onnx::TensorProto& tensor, std::wstring filePath) { template std::vector GetTypeSpecificDataFromTensorProto( - onnx::TensorProto /*tensorProto*/){ + onnx::TensorProto /*tensorProto*/) { static_assert(false, "UNDEFINED! TensorProto methods aren't templated, so add a new template specialization."); } template <> std::vector GetTypeSpecificDataFromTensorProto( - onnx::TensorProto tensorProto){ + onnx::TensorProto tensorProto) { return std::vector(std::begin(tensorProto.float_data()), std::end(tensorProto.float_data())); } template <> std::vector GetTypeSpecificDataFromTensorProto( - onnx::TensorProto tensorProto){ + onnx::TensorProto tensorProto) { return std::vector(std::begin(tensorProto.int32_data()), std::end(tensorProto.int32_data())); } template <> std::vector GetTypeSpecificDataFromTensorProto( - onnx::TensorProto tensorProto){ + onnx::TensorProto tensorProto) { return std::vector(std::begin(tensorProto.int64_data()), std::end(tensorProto.int64_data())); } template std::vector GetTensorDataFromTensorProto( - onnx::TensorProto tensorProto, - uint64_t elementCount) { + onnx::TensorProto tensorProto, + uint64_t elementCount) { if (tensorProto.has_raw_data()) { std::vector tensorData; auto& values = tensorProto.raw_data(); - EXPECT_EQ(elementCount, values.size() / sizeof(DataType)) << L"TensorProto elementcount should match raw data buffer size in elements."; + if (elementCount != values.size() / sizeof(DataType)) { + WINML_LOG_ERROR("TensorProto element count should match raw data buffer size in elements."); + } tensorData = std::vector(elementCount); memcpy(tensorData.data(), values.data(), values.size()); @@ -105,7 +107,7 @@ std::vector GetTensorDataFromTensorProto( static std::vector GetTensorStringDataFromTensorProto( onnx::TensorProto tensorProto, uint64_t elementCount) { - EXPECT_EQ(tensorProto.string_data_size(), elementCount); + WINML_EXPECT_EQUAL(tensorProto.string_data_size(), elementCount); auto& values = tensorProto.string_data(); auto returnVector = std::vector(elementCount); std::transform(std::begin(values), std::end(values), std::begin(returnVector), @@ -131,15 +133,15 @@ ITensor ProtobufHelpers::LoadTensorFromProtobufFile( } switch (tensorProto.data_type()) { case (onnx::TensorProto::DataType::TensorProto_DataType_FLOAT): - return TensorFloat::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto(tensorProto, elementCount)); + return TensorFloat::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto(tensorProto, elementCount)); case (onnx::TensorProto::DataType::TensorProto_DataType_INT32): - return TensorInt32Bit::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto(tensorProto, elementCount)); + return TensorInt32Bit::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto(tensorProto, elementCount)); case (onnx::TensorProto::DataType::TensorProto_DataType_INT64): - return TensorInt64Bit::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto(tensorProto, elementCount)); + return TensorInt64Bit::CreateFromIterable(tensorShape, GetTensorDataFromTensorProto(tensorProto, elementCount)); case (onnx::TensorProto::DataType::TensorProto_DataType_STRING): return TensorString::CreateFromIterable(tensorShape, GetTensorStringDataFromTensorProto(tensorProto, elementCount)); default: - ADD_FAILURE() << L"Tensor type for creating tensor from protobuf file not supported."; + WINML_LOG_ERROR("Tensor type for creating tensor from protobuf file not supported."); break; } } @@ -152,7 +154,7 @@ TensorFloat16Bit ProtobufHelpers::LoadTensorFloat16FromProtobufFile( onnx::TensorProto tensorProto; if (LoadTensorFromPb(tensorProto, filePath)) { if (tensorProto.has_data_type()) { - EXPECT_EQ(onnx::TensorProto::DataType::TensorProto_DataType_FLOAT16, tensorProto.data_type()); + WINML_EXPECT_EQUAL(onnx::TensorProto::DataType::TensorProto_DataType_FLOAT16, tensorProto.data_type()); } else { std::cerr << "Loading unknown TensorProto datatype as TensorFloat16Bit.\n"; } @@ -166,7 +168,10 @@ TensorFloat16Bit ProtobufHelpers::LoadTensorFloat16FromProtobufFile( uint32_t sizeInBytes; spTensorValueNative->GetBuffer(reinterpret_cast(&data), &sizeInBytes); - EXPECT_TRUE(tensorProto.has_raw_data()) << L"Float16 tensor proto buffers are expected to contain raw data."; + if (!tensorProto.has_raw_data()) + { + WINML_LOG_ERROR("Float16 tensor proto buffers are expected to contain raw data."); + } auto& raw_data = tensorProto.raw_data(); auto buff = raw_data.c_str(); @@ -318,9 +323,9 @@ winrt::Windows::AI::MachineLearning::LearningModel ProtobufHelpers::CreateModel( DataWriter m_dataWriter; }; - auto size = model.ByteSize(); + auto size = model.ByteSizeLong(); auto raw_array = std::unique_ptr(new char[size]); - model.SerializeToArray(raw_array.get(), size); + model.SerializeToArray(raw_array.get(), static_cast(size)); BufferStreamAdapter buffer; std::ostream os(&buffer); diff --git a/winml/test/common/runtimeParameters.h b/winml/test/common/runtimeParameters.h index 6b0edc7a9cc8b..a98e6c5e283a0 100644 --- a/winml/test/common/runtimeParameters.h +++ b/winml/test/common/runtimeParameters.h @@ -6,4 +6,4 @@ namespace RuntimeParameters { // Runtime parameters passed through CLI arguments extern std::unordered_map Parameters; -} +} \ No newline at end of file diff --git a/winml/test/common/std.h b/winml/test/common/std.h index a2ea803ab70a9..99a3af8e5943f 100644 --- a/winml/test/common/std.h +++ b/winml/test/common/std.h @@ -19,59 +19,15 @@ #include #include -#include +#include "test.h" // IUnknown must be declared before winrt/base.h is included to light up support for native COM // interfaces with C++/WinRT types (e.g. winrt::com_ptr). #include +#include #include "winrt/base.h" #include "winrt/Windows.Foundation.Collections.h" #include "comp_generated/winrt/windows.ai.machinelearning.h" // WinML #include "Windows.AI.MachineLearning.Native.h" - -#include "runtimeParameters.h" - -#define EXPECT_THROW_SPECIFIC(statement, exception, condition) \ - EXPECT_THROW( \ - try { \ - statement; \ - } catch (const exception& e) { \ - EXPECT_TRUE(condition(e)); \ - throw; \ - } \ - , exception); - -// For old versions of gtest without GTEST_SKIP, stream the message and return success instead -#ifndef GTEST_SKIP -#define GTEST_SKIP_(message) \ - return GTEST_MESSAGE_(message, ::testing::TestPartResult::kSuccess) -#define GTEST_SKIP GTEST_SKIP_("") -#endif - -#ifndef INSTANTIATE_TEST_SUITE_P -// Use the old name, removed in newer versions of googletest -#define INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_CASE_P -#endif - - -#ifndef USE_DML -#define GPUTEST \ - GTEST_SKIP() << "GPU tests disabled because this is a WinML only build (no DML)"; -#else -#define GPUTEST \ - if (auto noGpuTests = RuntimeParameters::Parameters.find("noGPUtests"); \ - noGpuTests != RuntimeParameters::Parameters.end() && noGpuTests->second != "0") \ - { \ - GTEST_SKIP() << "GPU tests disabled"; \ - } -#endif - - -#define SKIP_EDGECORE \ - if (auto isEdgeCore = RuntimeParameters::Parameters.find("EdgeCore"); \ - isEdgeCore != RuntimeParameters::Parameters.end() && isEdgeCore->second != "0") \ - { \ - GTEST_SKIP() << "Test can't be run in EdgeCore"; \ - } diff --git a/winml/test/common/taefTestMacros.h b/winml/test/common/taefTestMacros.h new file mode 100644 index 0000000000000..083baa531a99e --- /dev/null +++ b/winml/test/common/taefTestMacros.h @@ -0,0 +1,63 @@ +#include "WexTestClass.h" + +using namespace WEX::Logging; +using namespace WEX::Common; +using namespace WEX::TestExecution; + +#define WINML_EXPECT_NO_THROW(statement) VERIFY_NO_THROW(statement) + +#define WINML_TEST_CLASS_BEGIN_WITH_SETUP(test_class_name, setup_method) \ + class test_class_name { \ + TEST_CLASS(test_class_name); \ + TEST_CLASS_SETUP(TestClassSetup) { \ + getapi().setup_method(); \ + return true; \ + } + +#define WINML_TEST_CLASS_END() \ + } \ + ; + +#define WINML_TEST(group_name, test_name) \ + TEST_METHOD(test_name) { \ + getapi().test_name(); \ + } + +#define WINML_SKIP_TEST(message) \ + do { \ + Log::Result(TestResults::Skipped, \ + std::wstring_convert>().from_bytes(message).c_str()); \ + return; \ + } while (0) + +#define WINML_EXPECT_NO_THROW(statement) VERIFY_NO_THROW(statement) +#define WINML_EXPECT_TRUE(statement) VERIFY_IS_TRUE(statement) +#define WINML_EXPECT_FALSE(statement) VERIFY_IS_FALSE(statement) +#define WINML_EXPECT_EQUAL(val1, val2) VERIFY_ARE_EQUAL(val1, val2) +#define WINML_EXPECT_NOT_EQUAL(val1, val2) VERIFY_ARE_NOT_EQUAL(val1, val2) +#define WINML_LOG_ERROR(message) \ + VERIFY_FAIL(std::wstring_convert>().from_bytes(message).c_str()) +#define WINML_LOG_COMMENT(message)\ + WEX::Logging::Log::Comment(std::wstring_convert>().from_bytes(message).c_str()) +#define WINML_EXPECT_HRESULT_SUCCEEDED(hresult_expression) VERIFY_SUCCEEDED(hresult_expression) +#define WINML_EXPECT_THROW_SPECIFIC(statement, exception, condition) VERIFY_THROWS_SPECIFIC(statement, exception, condition) +#define WINML_EXPECT_HRESULT_FAILED(hresult_expression) VERIFY_FAILED(hresult_expression) + +#ifndef USE_DML +#define GPUTEST \ + WINML_SUPRESS_UNREACHABLE_BELOW(WINML_SKIP_TEST("GPU tests disabled because this is a WinML only build (no DML)")) +#else +#define GPUTEST \ + bool noGPUTests; \ + if (SUCCEEDED(RuntimeParameters::TryGetValue(L"noGPUtests", noGPUTests)) && noGPUTests) { \ + WINML_SKIP_TEST("This test is disabled by the noGPUTests runtime parameter."); \ + return; \ + } +#endif + +#define SKIP_EDGECORE \ + bool edgeCoreRun; \ + if (SUCCEEDED(RuntimeParameters::TryGetValue(L"EdgeCore", edgeCoreRun)) && edgeCoreRun) { \ + WINML_SKIP_TEST("This test is disabled by the EdgeCore runtime parameter."); \ + return; \ + } diff --git a/winml/test/common/test.h b/winml/test/common/test.h new file mode 100644 index 0000000000000..037f59e932484 --- /dev/null +++ b/winml/test/common/test.h @@ -0,0 +1,18 @@ +#pragma once + +using VoidTest = void (*)(); +using SetupTest = VoidTest; + +constexpr bool alwaysTrue() { + return true; +} +#define WINML_SUPRESS_UNREACHABLE_BELOW(statement) \ + if (alwaysTrue()) { statement; } + +#ifdef BUILD_GOOGLE_TEST +#include "googleTestMacros.h" +#else +#ifdef BUILD_TAEF_TEST +#include "taefTestMacros.h" +#endif +#endif diff --git a/winml/test/common/testPch.h b/winml/test/common/testPch.h index 952f0bbccf6a1..4bc156fa0c1c2 100644 --- a/winml/test/common/testPch.h +++ b/winml/test/common/testPch.h @@ -3,7 +3,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // //----------------------------------------------------------------------------- +#ifndef _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS #define _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS +#endif #include "std.h" #include diff --git a/winml/test/scenario/cppwinrt/CustomNullOp.h b/winml/test/scenario/cppwinrt/CustomNullOp.h index d7dfdf75ac488..7d2e5efa899f4 100644 --- a/winml/test/scenario/cppwinrt/CustomNullOp.h +++ b/winml/test/scenario/cppwinrt/CustomNullOp.h @@ -4,14 +4,14 @@ #pragma once -#include +#include "test.h" template struct NullShapeInferrer : winrt::implements, IMLOperatorShapeInferrer> { STDMETHOD(InferOutputShapes)(IMLOperatorShapeInferenceContext* context) noexcept { - EXPECT_NO_THROW(OperatorHelper::ShapeInferenceFunction(context)); + WINML_EXPECT_NO_THROW(OperatorHelper::ShapeInferenceFunction(context)); return S_OK; } }; @@ -23,7 +23,7 @@ struct NullOperator : winrt::implements STDMETHOD(Compute)(IMLOperatorKernelContext* context) { winrt::com_ptr outputTensor; - EXPECT_HRESULT_SUCCEEDED(context->GetOutputTensor(0, outputTensor.put())); + WINML_EXPECT_HRESULT_SUCCEEDED(context->GetOutputTensor(0, outputTensor.put())); ++(*m_callCount); return S_OK; @@ -92,7 +92,7 @@ struct NullOperatorFactory : winrt::implements(callCount); - EXPECT_HRESULT_SUCCEEDED(registry->RegisterOperatorKernel( + WINML_EXPECT_HRESULT_SUCCEEDED(registry->RegisterOperatorKernel( &kernelDescription, factory.get(), shapeInferrer.get() diff --git a/winml/test/scenario/cppwinrt/CustomOperatorProvider.h b/winml/test/scenario/cppwinrt/CustomOperatorProvider.h index 215cd4c4f29b8..698eeeb396095 100644 --- a/winml/test/scenario/cppwinrt/CustomOperatorProvider.h +++ b/winml/test/scenario/cppwinrt/CustomOperatorProvider.h @@ -54,4 +54,4 @@ struct CustomOperatorProvider : m_registry.copy_to(ppOperatorRegistry); return S_OK; } -}; +}; \ No newline at end of file diff --git a/winml/test/scenario/cppwinrt/CustomOps.cpp b/winml/test/scenario/cppwinrt/CustomOps.cpp index a9ab916fbc1bd..eb47843b95de6 100644 --- a/winml/test/scenario/cppwinrt/CustomOps.cpp +++ b/winml/test/scenario/cppwinrt/CustomOps.cpp @@ -12,7 +12,7 @@ #include #include #include "CustomOperatorProvider.h" -#include "runtimeParameters.h" +#include "CustomOps.h" // For custom operator and shape inferencing support #include "core/providers/dml/DmlExecutionProvider/inc/MLOperatorAuthor.h" @@ -31,26 +31,19 @@ using namespace winrt::Windows::Graphics::Imaging; using namespace winrt::Windows::Storage; using namespace winrt::Windows::Storage::Streams; -class CustomOpsScenarioTest : public ::testing::Test +static void CustomOpsScenarioTestSetup() { -protected: - CustomOpsScenarioTest() { - init_apartment(); - } -}; + init_apartment(); +} -class CustomOpsScenarioGpuTest : public CustomOpsScenarioTest +static void CustomOpsScenarioGpuTestSetup() { -protected: - void SetUp() override - { - GPUTEST - } -}; + init_apartment(); + GPUTEST; +} // Tests that the execution provider correctly fuses operators together when custom ops are involved. -TEST_F(CustomOpsScenarioGpuTest, CustomOperatorFusion) -{ +static void CustomOperatorFusion() { constexpr const wchar_t* c_modelFilename = L"squeezenet_tensor_input.onnx"; // This particular model has 25 Conv ops and 25 Relu ops, all of which are eligible for fusion so we expect them @@ -96,7 +89,7 @@ TEST_F(CustomOpsScenarioGpuTest, CustomOperatorFusion) { using namespace OperatorHelper; - EXPECT_HRESULT_SUCCEEDED(MLCreateOperatorRegistry(m_registry.put())); + WINML_EXPECT_HRESULT_SUCCEEDED(MLCreateOperatorRegistry(m_registry.put())); #pragma push_macro("REGISTER_KERNEL") #define REGISTER_KERNEL(_name, _domain, _opSet, _shapeInferrer, _callCount) \ @@ -143,14 +136,14 @@ TEST_F(CustomOpsScenarioGpuTest, CustomOperatorFusion) auto provider = customOperatorProvider.as(); LearningModelDevice device = nullptr; - EXPECT_NO_THROW(device = LearningModelDevice(LearningModelDeviceKind::DirectX)); + WINML_EXPECT_NO_THROW(device = LearningModelDevice(LearningModelDeviceKind::DirectX)); std::wstring fullPath = FileHelpers::GetModulePath() + c_modelFilename; auto model = LearningModel::LoadFromFilePath(fullPath, provider); auto featureValue = FileHelpers::LoadImageFeatureValue(L"227x227.png"); LearningModelSession session = nullptr; - EXPECT_NO_THROW(session = LearningModelSession(model, device)); + WINML_EXPECT_NO_THROW(session = LearningModelSession(model, device)); LearningModelBinding modelBinding(session); modelBinding.Bind(L"data", featureValue); @@ -159,15 +152,15 @@ TEST_F(CustomOpsScenarioGpuTest, CustomOperatorFusion) const auto& callCounts = customOperatorProvider.as()->GetCallCounts(); // Verify that the correct number of each operator was seen (i.e. that none were dropped / incorrectly fused) - EXPECT_EQ(c_expectedConvOps, callCounts.conv); - EXPECT_EQ(c_expectedReluOps, callCounts.relu); - EXPECT_EQ(c_expectedFusedConvOps, callCounts.fusedConv); - EXPECT_EQ(c_expectedGemmOps, callCounts.gemm); - EXPECT_EQ(c_expectedSigmoidOps, callCounts.sigmoid); - EXPECT_EQ(c_expectedFusedGemmOps, callCounts.fusedGemm); - EXPECT_EQ(c_expectedBatchNormOps, callCounts.batchNorm); - EXPECT_EQ(c_expectedMaxPoolOps, callCounts.maxPool); - EXPECT_EQ(c_expectedConcatOps, callCounts.concat); + WINML_EXPECT_EQUAL(c_expectedConvOps, callCounts.conv); + WINML_EXPECT_EQUAL(c_expectedReluOps, callCounts.relu); + WINML_EXPECT_EQUAL(c_expectedFusedConvOps, callCounts.fusedConv); + WINML_EXPECT_EQUAL(c_expectedGemmOps, callCounts.gemm); + WINML_EXPECT_EQUAL(c_expectedSigmoidOps, callCounts.sigmoid); + WINML_EXPECT_EQUAL(c_expectedFusedGemmOps, callCounts.fusedGemm); + WINML_EXPECT_EQUAL(c_expectedBatchNormOps, callCounts.batchNorm); + WINML_EXPECT_EQUAL(c_expectedMaxPoolOps, callCounts.maxPool); + WINML_EXPECT_EQUAL(c_expectedConcatOps, callCounts.concat); } struct LocalCustomOperatorProvider : @@ -178,7 +171,7 @@ struct LocalCustomOperatorProvider : { LocalCustomOperatorProvider() { - EXPECT_HRESULT_SUCCEEDED(MLCreateOperatorRegistry(m_registry.put())); + WINML_EXPECT_HRESULT_SUCCEEDED(MLCreateOperatorRegistry(m_registry.put())); } STDMETHOD(GetRegistry)(IMLOperatorRegistry** ppOperatorRegistry) @@ -205,20 +198,20 @@ struct LocalCustomOperatorProvider : void VerifyTestAttributes(const MLOperatorAttributes& attrs) { std::string strAttr = attrs.GetAttribute("DefaultedNonRequiredString"); - EXPECT_EQ(strAttr, "1"); + WINML_EXPECT_EQUAL(strAttr, "1"); std::vector strArrayAttr = attrs.GetAttributeVector("DefaultedNonRequiredStringArray"); std::vector expected = std::vector({ "1", "2" }); for (size_t i = 0; i < expected.size(); ++i) { - EXPECT_EQ(strArrayAttr[i], expected[i]); + WINML_EXPECT_EQUAL(strArrayAttr[i], expected[i]); } - EXPECT_EQ(1, attrs.GetAttribute("DefaultedNonRequiredInt")); - EXPECT_EQ(1.0f, attrs.GetAttribute("DefaultedNonRequiredFloat")); + WINML_EXPECT_EQUAL(1, attrs.GetAttribute("DefaultedNonRequiredInt")); + WINML_EXPECT_EQUAL(1.0f, attrs.GetAttribute("DefaultedNonRequiredFloat")); - EXPECT_EQ(std::vector({ 1, 2 }), attrs.GetAttributeVector("DefaultedNonRequiredIntArray")); - EXPECT_EQ(std::vector({ 1.0f, 2.0f }), attrs.GetAttributeVector("DefaultedNonRequiredFloatArray")); + WINML_EXPECT_EQUAL(std::vector({ 1, 2 }), attrs.GetAttributeVector("DefaultedNonRequiredIntArray")); + WINML_EXPECT_EQUAL(std::vector({ 1.0f, 2.0f }), attrs.GetAttributeVector("DefaultedNonRequiredFloatArray")); } // Foo kernel which is doing Add and optionally truncates its output @@ -241,14 +234,14 @@ class FooKernel if (!Truncate) { com_ptr shapeInfo; - EXPECT_EQ(info.GetInterface()->HasTensorShapeDescription(), false); - EXPECT_HRESULT_FAILED(info.GetInterface()->GetTensorShapeDescription(shapeInfo.put())); + WINML_EXPECT_EQUAL(info.GetInterface()->HasTensorShapeDescription(), false); + WINML_EXPECT_HRESULT_FAILED(info.GetInterface()->GetTensorShapeDescription(shapeInfo.put())); } else { com_ptr shapeInfo; - EXPECT_EQ(info.GetInterface()->HasTensorShapeDescription(), true); - EXPECT_EQ(info.GetInterface()->GetTensorShapeDescription(shapeInfo.put()), S_OK); + WINML_EXPECT_EQUAL(info.GetInterface()->HasTensorShapeDescription(), true); + WINML_EXPECT_EQUAL(info.GetInterface()->GetTensorShapeDescription(shapeInfo.put()), S_OK); } } @@ -271,7 +264,7 @@ class FooKernel if (!Truncate) { com_ptr tensor; - EXPECT_HRESULT_FAILED(context.GetInterface()->GetOutputTensor(0, tensor.put())); + WINML_EXPECT_HRESULT_FAILED(context.GetInterface()->GetOutputTensor(0, tensor.put())); } else { @@ -308,7 +301,7 @@ void CreateTruncatedABIFooKernel(IMLOperatorKernelCreationContext* kernelInfo, I } // Test using a foo kernel which is doing Add, but register it as "Mul". -TEST_F(CustomOpsScenarioTest, CustomKernelWithBuiltInSchema) +static void CustomKernelWithBuiltInSchema() { // Create the registry auto operatorProvider = winrt::make(); @@ -337,7 +330,7 @@ TEST_F(CustomOpsScenarioTest, CustomKernelWithBuiltInSchema) }; Microsoft::WRL::ComPtr factory = wil::MakeOrThrow(CreateABIFooKernel); - EXPECT_HRESULT_SUCCEEDED(registry->RegisterOperatorKernel(&kernelDesc, factory.Get(), nullptr)); + WINML_EXPECT_HRESULT_SUCCEEDED(registry->RegisterOperatorKernel(&kernelDesc, factory.Get(), nullptr)); // Prepare inputs std::vector dimsX = { 3, 2 }; @@ -361,23 +354,23 @@ TEST_F(CustomOpsScenarioTest, CustomKernelWithBuiltInSchema) bindings.Bind(winrt::hstring(L"X"), inputTensor); auto outputValue = TensorFloat::Create(); - EXPECT_NO_THROW(bindings.Bind(L"Y", outputValue)); + WINML_EXPECT_NO_THROW(bindings.Bind(L"Y", outputValue)); // Evaluate the model hstring correlationId; - EXPECT_NO_THROW(session.Evaluate(bindings, correlationId)); + WINML_EXPECT_NO_THROW(session.Evaluate(bindings, correlationId)); // Check the result shape - EXPECT_EQ(expectedDimsY.size(), outputValue.Shape().Size()); + WINML_EXPECT_EQUAL(expectedDimsY.size(), outputValue.Shape().Size()); for (uint32_t j = 0; j < outputValue.Shape().Size(); j++) { - EXPECT_EQ(expectedDimsY.at(j), outputValue.Shape().GetAt(j)); + WINML_EXPECT_EQUAL(expectedDimsY.at(j), outputValue.Shape().GetAt(j)); } // Check the results auto buffer = outputValue.GetAsVectorView(); - EXPECT_TRUE(buffer != nullptr); - EXPECT_TRUE(std::equal(expectedValuesY.cbegin(), expectedValuesY.cend(), begin(buffer))); + WINML_EXPECT_TRUE(buffer != nullptr); + WINML_EXPECT_TRUE(std::equal(expectedValuesY.cbegin(), expectedValuesY.cend(), begin(buffer))); // Release the model before operatorProvider goes out of scope model = nullptr; @@ -404,7 +397,7 @@ class MLOperatorShapeInferrerFromFunc : public Microsoft::WRL::RuntimeClass< }; // Test using a custom kernel and schema, while verifying attribute defaults, type mapping, and inference methods -TEST_F(CustomOpsScenarioTest, CustomKernelWithCustomSchema) +static void CustomKernelWithCustomSchema() { // Test cases struct @@ -591,7 +584,7 @@ TEST_F(CustomOpsScenarioTest, CustomKernelWithCustomSchema) // Register the schema MLOperatorSetId opsetId = { "", 7 }; MLOperatorSchemaDescription* opSchemaDescs = &schemaDesc; - EXPECT_EQ(S_OK, registry->RegisterOperatorSetSchema( + WINML_EXPECT_EQUAL(S_OK, registry->RegisterOperatorSetSchema( &opsetId, 1, &opSchemaDescs, @@ -608,7 +601,7 @@ TEST_F(CustomOpsScenarioTest, CustomKernelWithCustomSchema) MLOperatorSetId id = { "", 9 }; MLOperatorSchemaDescription* schemaDescs = &futureSchemaDesc; - EXPECT_EQ(S_OK, registry->RegisterOperatorSetSchema( + WINML_EXPECT_EQUAL(S_OK, registry->RegisterOperatorSetSchema( &id, 7, &schemaDescs, @@ -624,7 +617,7 @@ TEST_F(CustomOpsScenarioTest, CustomKernelWithCustomSchema) MLOperatorSetId id = { "otherDomain", 7 }; MLOperatorSchemaDescription* schemaDescs = &otherSchemaDesc; - EXPECT_EQ(S_OK, registry->RegisterOperatorSetSchema( + WINML_EXPECT_EQUAL(S_OK, registry->RegisterOperatorSetSchema( &id, 1, &schemaDescs, @@ -661,12 +654,12 @@ TEST_F(CustomOpsScenarioTest, CustomKernelWithCustomSchema) kernelDesc.options = MLOperatorKernelOptions::AllowDynamicInputShapes; Microsoft::WRL::ComPtr factory = wil::MakeOrThrow(CreateABIFooKernel); - EXPECT_EQ(S_OK, registry->RegisterOperatorKernel(&kernelDesc, factory.Get(), nullptr)); + WINML_EXPECT_EQUAL(S_OK, registry->RegisterOperatorKernel(&kernelDesc, factory.Get(), nullptr)); } else { Microsoft::WRL::ComPtr factory = wil::MakeOrThrow(CreateTruncatedABIFooKernel); - EXPECT_EQ(S_OK, registry->RegisterOperatorKernel( + WINML_EXPECT_EQUAL(S_OK, registry->RegisterOperatorKernel( &kernelDesc, factory.Get(), testCases[caseIndex].useShapeInferenceInKernel ? shapeInferrer.Get() : nullptr @@ -699,23 +692,23 @@ TEST_F(CustomOpsScenarioTest, CustomKernelWithCustomSchema) bindings.Bind(winrt::hstring(L"X"), inputTensor); auto outputValue = TensorFloat::Create(); - EXPECT_NO_THROW(bindings.Bind(L"Y", outputValue)); + WINML_EXPECT_NO_THROW(bindings.Bind(L"Y", outputValue)); // Evaluate the model hstring correlationId; - EXPECT_NO_THROW(session.Evaluate(bindings, correlationId)); + WINML_EXPECT_NO_THROW(session.Evaluate(bindings, correlationId)); // Verify the result shape - EXPECT_EQ(expectedDimsY.size(), outputValue.Shape().Size()); + WINML_EXPECT_EQUAL(expectedDimsY.size(), outputValue.Shape().Size()); for (uint32_t j = 0; j < outputValue.Shape().Size(); j++) { - EXPECT_EQ(expectedDimsY.at(j), outputValue.Shape().GetAt(j)); + WINML_EXPECT_EQUAL(expectedDimsY.at(j), outputValue.Shape().GetAt(j)); } // Verify the result values auto buffer = outputValue.GetAsVectorView(); - EXPECT_TRUE(buffer != nullptr); - EXPECT_TRUE(std::equal(expectedValuesY.cbegin(), expectedValuesY.cend(), begin(buffer))); + WINML_EXPECT_TRUE(buffer != nullptr); + WINML_EXPECT_TRUE(std::equal(expectedValuesY.cbegin(), expectedValuesY.cend(), begin(buffer))); // Release the model before operatorProvider goes out of scope model = nullptr; @@ -724,7 +717,19 @@ TEST_F(CustomOpsScenarioTest, CustomKernelWithCustomSchema) { // Check that the shape inference context is closed and safely fails MLOperatorEdgeDescription edgeDesc; - EXPECT_EQ(E_INVALIDARG, shapeInferenceContext->GetInputEdgeDescription(0, &edgeDesc)); + WINML_EXPECT_EQUAL(E_INVALIDARG, shapeInferenceContext->GetInputEdgeDescription(0, &edgeDesc)); } } } + +const CustomOpsTestApi& getapi() { + static constexpr CustomOpsTestApi api = + { + CustomOpsScenarioTestSetup, + CustomOpsScenarioGpuTestSetup, + CustomOperatorFusion, + CustomKernelWithBuiltInSchema, + CustomKernelWithCustomSchema + }; + return api; +} \ No newline at end of file diff --git a/winml/test/scenario/cppwinrt/CustomOps.h b/winml/test/scenario/cppwinrt/CustomOps.h new file mode 100644 index 0000000000000..df48bd51af990 --- /dev/null +++ b/winml/test/scenario/cppwinrt/CustomOps.h @@ -0,0 +1,19 @@ +#include "test.h" +struct CustomOpsTestApi +{ + SetupTest CustomOpsScenarioTestSetup; + SetupTest CustomOpsScenarioGpuTestSetup; + VoidTest CustomOperatorFusion; + VoidTest CustomKernelWithBuiltInSchema; + VoidTest CustomKernelWithCustomSchema; +}; +const CustomOpsTestApi& getapi(); + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(CustomOpsScenarioTest, CustomOpsScenarioTestSetup) +WINML_TEST(CustomOpsScenarioTest, CustomKernelWithBuiltInSchema) +WINML_TEST(CustomOpsScenarioTest, CustomKernelWithCustomSchema) +WINML_TEST_CLASS_END() + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(CustomOpsScenarioGpuTest, CustomOpsScenarioGpuTestSetup) +WINML_TEST(CustomOpsScenarioGpuTest, CustomOperatorFusion) +WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp index 5100dd08b71d1..e38f5c29b3079 100644 --- a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp +++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp @@ -1,7 +1,6 @@ #include "testPch.h" #include -#include #include "winrt/Windows.Devices.Enumeration.Pnp.h" #include "winrt/Windows.Graphics.DirectX.Direct3D11.h" @@ -16,21 +15,22 @@ #ifdef GetCurrentTime #undef GetCurrentTime #endif +#include "CommonDeviceHelpers.h" #include "CustomOperatorProvider.h" -#include "DeviceHelpers.h" #include "filehelpers.h" #include "robuffer.h" -#include "runtimeParameters.h" +#include "scenariotestscppwinrt.h" #include "Windows.AI.MachineLearning.Native.h" #include "Windows.Graphics.DirectX.Direct3D11.interop.h" #include "windows.ui.xaml.media.dxinterop.h" #include "winrt/Windows.UI.Xaml.Controls.h" #include "winrt/Windows.UI.Xaml.Media.Imaging.h" + #include #include #include #include - +#include #if __has_include("dxcore.h") #define ENABLE_DXCORE 1 #endif @@ -49,1408 +49,1285 @@ using namespace winrt::Windows::Storage; using namespace winrt::Windows::Storage::Streams; using namespace winrt::Windows::UI::Xaml::Media::Imaging; -class ScenarioCppWinrtTest : public ::testing::Test -{ -protected: - ScenarioCppWinrtTest() - { - init_apartment(); - } -}; +static void ScenarioCppWinrtTestSetup() { + init_apartment(); +} -class ScenarioCppWinrtGpuTest : public ScenarioCppWinrtTest -{ -protected: - void SetUp() override - { - GPUTEST - } +static void ScenarioCppWinrtGpuTestSetup() { + ScenarioCppWinrtTestSetup(); + GPUTEST }; -using ScenarioCppWinrtGpuTestDeathTest = ScenarioCppWinrtGpuTest; - -class ScenarioCppWinrtGpuSkipEdgeCoreTest : public ScenarioCppWinrtGpuTest -{ -protected: - void SetUp() override - { - ScenarioCppWinrtGpuTest::SetUp(); - SKIP_EDGECORE - } + +static void ScenarioCppWinrtGpuSkipEdgeCoreTestSetup() { + ScenarioCppWinrtGpuTestSetup(); + SKIP_EDGECORE }; -TEST_F(ScenarioCppWinrtTest, Sample1) -{ - LearningModel model = nullptr; - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(filePath)); +static void Sample1() { + LearningModel model = nullptr; + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(filePath)); } -ILearningModelFeatureValue MakeTensor(const ITensorFeatureDescriptor& descriptor) -{ - auto dataType = descriptor.TensorKind(); - std::vector shape; - int64_t size = 1; - for (auto&& dim : descriptor.Shape()) - { - if (dim == -1) dim = 1; - shape.push_back(dim); - size *= dim; - } +ILearningModelFeatureValue MakeTensor(const ITensorFeatureDescriptor& descriptor) { + auto dataType = descriptor.TensorKind(); + std::vector shape; + int64_t size = 1; + for (auto&& dim : descriptor.Shape()) { + if (dim == -1) dim = 1; + shape.push_back(dim); + size *= dim; + } - switch (dataType) - { - case TensorKind::Float: - { - std::vector buffer; - buffer.resize(size); - auto ftv = TensorFloat::CreateFromIterable(shape, winrt::single_threaded_vector(std::move(buffer))); - return ftv; + switch (dataType) { + case TensorKind::Float: { + std::vector buffer; + buffer.resize(size); + auto ftv = TensorFloat::CreateFromIterable(shape, winrt::single_threaded_vector(std::move(buffer))); + return ftv; } default: - throw_hresult(E_NOTIMPL); - break; - } + throw_hresult(E_NOTIMPL); + break; + } } -ILearningModelFeatureValue MakeImage(const IImageFeatureDescriptor& /*descriptor*/, winrt::Windows::Foundation::IInspectable data) -{ - VideoFrame videoFrame = nullptr; - if (data != nullptr) - { - SoftwareBitmap sb = nullptr; - data.as(sb); - videoFrame = VideoFrame::CreateWithSoftwareBitmap(sb); - } - else - { - SoftwareBitmap sb = SoftwareBitmap(BitmapPixelFormat::Bgra8, 28, 28); - videoFrame = VideoFrame::CreateWithSoftwareBitmap(sb); - } - auto imageValue = ImageFeatureValue::CreateFromVideoFrame(videoFrame); - return imageValue; +ILearningModelFeatureValue MakeImage(const IImageFeatureDescriptor& /*descriptor*/, winrt::Windows::Foundation::IInspectable data) { + VideoFrame videoFrame = nullptr; + if (data != nullptr) { + SoftwareBitmap sb = nullptr; + data.as(sb); + videoFrame = VideoFrame::CreateWithSoftwareBitmap(sb); + } else { + SoftwareBitmap sb = SoftwareBitmap(BitmapPixelFormat::Bgra8, 28, 28); + videoFrame = VideoFrame::CreateWithSoftwareBitmap(sb); + } + auto imageValue = ImageFeatureValue::CreateFromVideoFrame(videoFrame); + return imageValue; } -ILearningModelFeatureValue FeatureValueFromFeatureValueDescriptor(ILearningModelFeatureDescriptor descriptor, winrt::Windows::Foundation::IInspectable data = nullptr) -{ - auto kind = descriptor.Kind(); - switch (kind) - { - case LearningModelFeatureKind::Image: - { - ImageFeatureDescriptor imageDescriptor = nullptr; - descriptor.as(imageDescriptor); - return MakeImage(imageDescriptor, data); +ILearningModelFeatureValue FeatureValueFromFeatureValueDescriptor(ILearningModelFeatureDescriptor descriptor, winrt::Windows::Foundation::IInspectable data = nullptr) { + auto kind = descriptor.Kind(); + switch (kind) { + case LearningModelFeatureKind::Image: { + ImageFeatureDescriptor imageDescriptor = nullptr; + descriptor.as(imageDescriptor); + return MakeImage(imageDescriptor, data); } case LearningModelFeatureKind::Map: - throw_hresult(E_NOTIMPL); - break; + throw_hresult(E_NOTIMPL); + break; case LearningModelFeatureKind::Sequence: - throw_hresult(E_NOTIMPL); - break; - case LearningModelFeatureKind::Tensor: - { - TensorFeatureDescriptor tensorDescriptor = nullptr; - descriptor.as(tensorDescriptor); - return MakeTensor(tensorDescriptor); + throw_hresult(E_NOTIMPL); + break; + case LearningModelFeatureKind::Tensor: { + TensorFeatureDescriptor tensorDescriptor = nullptr; + descriptor.as(tensorDescriptor); + return MakeTensor(tensorDescriptor); } default: - throw_hresult(E_INVALIDARG); - break; - } + throw_hresult(E_INVALIDARG); + break; + } } // helper method that populates a binding object with default data -static void BindFeatures(LearningModelBinding binding, IVectorView features) -{ - for (auto&& feature : features) - { - auto featureValue = FeatureValueFromFeatureValueDescriptor(feature); - // set an actual buffer here. we're using uninitialized data for simplicity. - binding.Bind(feature.Name(), featureValue); - } +static void BindFeatures(LearningModelBinding binding, IVectorView features) { + for (auto&& feature : features) { + auto featureValue = FeatureValueFromFeatureValueDescriptor(feature); + // set an actual buffer here. we're using uninitialized data for simplicity. + binding.Bind(feature.Name(), featureValue); + } } //! Scenario1 : Load , bind, eval a model using all the system defaults (easy path) -TEST_F(ScenarioCppWinrtTest, Scenario1LoadBindEvalDefault) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - // create a session on the default device - LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); - // create a binding set - LearningModelBinding binding(session); - // bind the input and the output buffers by name - auto inputs = model.InputFeatures(); - for (auto&& input : inputs) - { - auto featureValue = FeatureValueFromFeatureValueDescriptor(input); - // set an actual buffer here. we're using uninitialized data for simplicity. - binding.Bind(input.Name(), featureValue); - } - // run eval - EXPECT_NO_THROW(session.Evaluate(binding, L"")); +static void Scenario1LoadBindEvalDefault() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + // create a session on the default device + LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); + // create a binding set + LearningModelBinding binding(session); + // bind the input and the output buffers by name + auto inputs = model.InputFeatures(); + for (auto&& input : inputs) { + auto featureValue = FeatureValueFromFeatureValueDescriptor(input); + // set an actual buffer here. we're using uninitialized data for simplicity. + binding.Bind(input.Name(), featureValue); + } + // run eval + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); } //! Scenario2: Load a model from stream // - winRT, and win32 -TEST_F(ScenarioCppWinrtTest, Scenario2LoadModelFromStream) -{ - // get a stream - std::wstring path = FileHelpers::GetModulePath() + L"model.onnx"; - auto storageFile = StorageFile::GetFileFromPathAsync(path).get(); - - // load the stream - Streams::IRandomAccessStreamReference streamref; - storageFile.as(streamref); - - // load a model - LearningModel model = nullptr; - EXPECT_NO_THROW(model = LearningModel::LoadFromStreamAsync(streamref).get()); - EXPECT_TRUE(model != nullptr); +static void Scenario2LoadModelFromStream() { + // get a stream + std::wstring path = FileHelpers::GetModulePath() + L"model.onnx"; + auto storageFile = StorageFile::GetFileFromPathAsync(path).get(); + + // load the stream + Streams::IRandomAccessStreamReference streamref; + storageFile.as(streamref); + + // load a model + LearningModel model = nullptr; + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromStreamAsync(streamref).get()); + WINML_EXPECT_TRUE(model != nullptr); } //! Scenario3: pass a SoftwareBitmap into a model -TEST_F(ScenarioCppWinrtGpuTest, Scenario3SoftwareBitmapInputBinding) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - // create a session on the default device - LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); - // create a binding set - LearningModelBinding binding(session); - // bind the input and the output buffers by name - auto inputs = model.InputFeatures(); - for (auto&& input : inputs) - { - // load the SoftwareBitmap - SoftwareBitmap sb = FileHelpers::GetSoftwareBitmapFromFile(FileHelpers::GetModulePath() + L"fish.png"); - auto videoFrame = VideoFrame::CreateWithSoftwareBitmap(sb); - auto imageValue = ImageFeatureValue::CreateFromVideoFrame(videoFrame); - - EXPECT_NO_THROW(binding.Bind(input.Name(), imageValue)); - } - // run eval - EXPECT_NO_THROW(session.Evaluate(binding, L"")); +static void Scenario3SoftwareBitmapInputBinding() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + // create a session on the default device + LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); + // create a binding set + LearningModelBinding binding(session); + // bind the input and the output buffers by name + auto inputs = model.InputFeatures(); + for (auto&& input : inputs) { + // load the SoftwareBitmap + SoftwareBitmap sb = FileHelpers::GetSoftwareBitmapFromFile(FileHelpers::GetModulePath() + L"fish.png"); + auto videoFrame = VideoFrame::CreateWithSoftwareBitmap(sb); + auto imageValue = ImageFeatureValue::CreateFromVideoFrame(videoFrame); + + WINML_EXPECT_NO_THROW(binding.Bind(input.Name(), imageValue)); + } + // run eval + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); } //! Scenario5: run an async eval -winrt::Windows::Foundation::IAsyncOperation DoEvalAsync() -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - // create a session on the default device - LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); - // create a binding set - LearningModelBinding binding(session); - // bind the input and the output buffers by name - auto inputs = model.InputFeatures(); - for (auto&& input : inputs) - { - auto featureValue = FeatureValueFromFeatureValueDescriptor(input); - // set an actual buffer here. we're using uninitialized data for simplicity. - binding.Bind(input.Name(), featureValue); - } - // run eval async - return session.EvaluateAsync(binding, L""); +winrt::Windows::Foundation::IAsyncOperation DoEvalAsync() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + // create a session on the default device + LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); + // create a binding set + LearningModelBinding binding(session); + // bind the input and the output buffers by name + auto inputs = model.InputFeatures(); + for (auto&& input : inputs) { + auto featureValue = FeatureValueFromFeatureValueDescriptor(input); + // set an actual buffer here. we're using uninitialized data for simplicity. + binding.Bind(input.Name(), featureValue); + } + // run eval async + return session.EvaluateAsync(binding, L""); } -TEST_F(ScenarioCppWinrtTest, Scenario5AsyncEval) -{ - auto task = DoEvalAsync(); +static void Scenario5AsyncEval() { + auto task = DoEvalAsync(); - while (task.Status() == winrt::Windows::Foundation::AsyncStatus::Started) - { - std::cout << "Waiting...\n"; - Sleep(30); - } - std::cout << "Done\n"; - EXPECT_NO_THROW(task.get()); + while (task.Status() == winrt::Windows::Foundation::AsyncStatus::Started) { + std::cout << "Waiting...\n"; + Sleep(30); + } + std::cout << "Done\n"; + WINML_EXPECT_NO_THROW(task.get()); } //! Scenario6: use BindInputWithProperties - BitmapBounds, BitmapPixelFormat // apparently this scenario is cut for rs5. - not cut, just rewprked. move props // to the image value when that is checked in. -TEST_F(ScenarioCppWinrtGpuTest, Scenario6BindWithProperties) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - // create a session on the default device - LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); - // create a binding set - LearningModelBinding binding(session); - // bind the input and the output buffers by name - auto inputs = model.InputFeatures(); - for (auto&& input : inputs) - { - SoftwareBitmap sb = SoftwareBitmap(BitmapPixelFormat::Bgra8, 224, 224); - auto videoFrame = VideoFrame::CreateWithSoftwareBitmap(sb); - auto imageValue = ImageFeatureValue::CreateFromVideoFrame(videoFrame); - - PropertySet propertySet; - - // make a BitmapBounds - BitmapBounds bounds; - bounds.X = 0; - bounds.Y = 0; - bounds.Height = 100; - bounds.Width = 100; - - auto bitmapsBoundsProperty = winrt::Windows::Foundation::PropertyValue::CreateUInt32Array({ bounds.X, bounds.Y, bounds.Width, bounds.Height }); - // insert it in the property set - propertySet.Insert(L"BitmapBounds", bitmapsBoundsProperty); - - // make a BitmapPixelFormat - BitmapPixelFormat bitmapPixelFormat = BitmapPixelFormat::Bgra8; - // translate it to an int so it can be used as a PropertyValue; - int intFromBitmapPixelFormat = static_cast(bitmapPixelFormat); - auto bitmapPixelFormatProperty = winrt::Windows::Foundation::PropertyValue::CreateInt32(intFromBitmapPixelFormat); - // insert it in the property set - propertySet.Insert(L"BitmapPixelFormat", bitmapPixelFormatProperty); - - // bind with properties - EXPECT_NO_THROW(binding.Bind(input.Name(), imageValue, propertySet)); - } - // run eval - EXPECT_NO_THROW(session.Evaluate(binding, L"")); +static void Scenario6BindWithProperties() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + // create a session on the default device + LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); + // create a binding set + LearningModelBinding binding(session); + // bind the input and the output buffers by name + auto inputs = model.InputFeatures(); + for (auto&& input : inputs) { + SoftwareBitmap sb = SoftwareBitmap(BitmapPixelFormat::Bgra8, 224, 224); + auto videoFrame = VideoFrame::CreateWithSoftwareBitmap(sb); + auto imageValue = ImageFeatureValue::CreateFromVideoFrame(videoFrame); + + PropertySet propertySet; + + // make a BitmapBounds + BitmapBounds bounds; + bounds.X = 0; + bounds.Y = 0; + bounds.Height = 100; + bounds.Width = 100; + + auto bitmapsBoundsProperty = winrt::Windows::Foundation::PropertyValue::CreateUInt32Array({bounds.X, bounds.Y, bounds.Width, bounds.Height}); + // insert it in the property set + propertySet.Insert(L"BitmapBounds", bitmapsBoundsProperty); + + // make a BitmapPixelFormat + BitmapPixelFormat bitmapPixelFormat = BitmapPixelFormat::Bgra8; + // translate it to an int so it can be used as a PropertyValue; + int intFromBitmapPixelFormat = static_cast(bitmapPixelFormat); + auto bitmapPixelFormatProperty = winrt::Windows::Foundation::PropertyValue::CreateInt32(intFromBitmapPixelFormat); + // insert it in the property set + propertySet.Insert(L"BitmapPixelFormat", bitmapPixelFormatProperty); + + // bind with properties + WINML_EXPECT_NO_THROW(binding.Bind(input.Name(), imageValue, propertySet)); + } + // run eval + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); } //! Scenario7: run eval without creating a binding object -TEST_F(ScenarioCppWinrtTest, Scenario7EvalWithNoBind) -{ - auto map = winrt::single_threaded_map(); - - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - // create a session on the default device - LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); - // enumerate feature descriptors and create features (but don't bind them) - auto inputs = model.InputFeatures(); - for (auto&& input : inputs) - { - auto featureValue = FeatureValueFromFeatureValueDescriptor(input); - map.Insert(input.Name(), featureValue); - } - // run eval - EXPECT_NO_THROW(session.EvaluateFeaturesAsync(map, L"").get()); +static void Scenario7EvalWithNoBind() { + auto map = winrt::single_threaded_map(); + + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + // create a session on the default device + LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); + // enumerate feature descriptors and create features (but don't bind them) + auto inputs = model.InputFeatures(); + for (auto&& input : inputs) { + auto featureValue = FeatureValueFromFeatureValueDescriptor(input); + map.Insert(input.Name(), featureValue); + } + // run eval + WINML_EXPECT_NO_THROW(session.EvaluateFeaturesAsync(map, L"").get()); } //! Scenario8: choose which device to run the model on - PreferredDeviceType, PreferredDevicePerformance, SetDeviceFromSurface, SetDevice // create a session on the default device -TEST_F(ScenarioCppWinrtTest, Scenario8SetDeviceSampleDefault) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - - LearningModelDevice anyDevice(LearningModelDeviceKind::Default); - LearningModelSession anySession(model, anyDevice); +static void Scenario8SetDeviceSampleDefault() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + + LearningModelDevice anyDevice(LearningModelDeviceKind::Default); + LearningModelSession anySession(model, anyDevice); } // create a session on the CPU device -TEST_F(ScenarioCppWinrtTest, Scenario8SetDeviceSampleCPU) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - - LearningModelDevice cpuDevice(LearningModelDeviceKind::Cpu); - LearningModelSession cpuSession(model, cpuDevice); +static void Scenario8SetDeviceSampleCPU() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + + LearningModelDevice cpuDevice(LearningModelDeviceKind::Cpu); + LearningModelSession cpuSession(model, cpuDevice); } // create a session on the default DML device -TEST_F(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleDefaultDirectX) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - - LearningModelDevice dmlDeviceDefault(LearningModelDeviceKind::DirectX); - LearningModelSession dmlSessionDefault(model, dmlDeviceDefault); +static void Scenario8SetDeviceSampleDefaultDirectX() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + + LearningModelDevice dmlDeviceDefault(LearningModelDeviceKind::DirectX); + LearningModelSession dmlSessionDefault(model, dmlDeviceDefault); } // create a session on the DML device that provides best power -TEST_F(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleMinPower) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - - LearningModelDevice dmlDeviceMinPower(LearningModelDeviceKind::DirectXMinPower); - LearningModelSession dmlSessionMinPower(model, dmlDeviceMinPower); +static void Scenario8SetDeviceSampleMinPower() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + + LearningModelDevice dmlDeviceMinPower(LearningModelDeviceKind::DirectXMinPower); + LearningModelSession dmlSessionMinPower(model, dmlDeviceMinPower); } // create a session on the DML device that provides best perf -TEST_F(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleMaxPerf) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - - LearningModelDevice dmlDeviceMaxPerf(LearningModelDeviceKind::DirectXHighPerformance); - LearningModelSession dmlSessionMaxPerf(model, dmlDeviceMaxPerf); +static void Scenario8SetDeviceSampleMaxPerf() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + + LearningModelDevice dmlDeviceMaxPerf(LearningModelDeviceKind::DirectXHighPerformance); + LearningModelSession dmlSessionMaxPerf(model, dmlDeviceMaxPerf); } // create a session on the same device my camera is on -TEST_F(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleMyCameraDevice) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - - auto devices = winrt::Windows::Devices::Enumeration::DeviceInformation::FindAllAsync(winrt::Windows::Devices::Enumeration::DeviceClass::VideoCapture).get(); - hstring deviceId; - if (devices.Size() > 0) - { - auto device = devices.GetAt(0); - deviceId = device.Id(); - auto deviceName = device.Name(); - auto enabled = device.IsEnabled(); - std::cout << "Found device " << deviceName.c_str() << ", enabled = " << enabled << "\n"; - winrt::Windows::Media::Capture::MediaCapture captureManager; - winrt::Windows::Media::Capture::MediaCaptureInitializationSettings settings; - settings.VideoDeviceId(deviceId); - captureManager.InitializeAsync(settings).get(); - auto mediaCaptureSettings = captureManager.MediaCaptureSettings(); - auto direct3D11Device = mediaCaptureSettings.Direct3D11Device(); - LearningModelDevice dmlDeviceCamera = LearningModelDevice::CreateFromDirect3D11Device(direct3D11Device); - LearningModelSession dmlSessionCamera(model, dmlDeviceCamera); - } - else - { - GTEST_SKIP() << "Test skipped because video capture device is missing"; - } +static void Scenario8SetDeviceSampleMyCameraDevice() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + + auto devices = winrt::Windows::Devices::Enumeration::DeviceInformation::FindAllAsync(winrt::Windows::Devices::Enumeration::DeviceClass::VideoCapture).get(); + hstring deviceId; + if (devices.Size() > 0) { + auto device = devices.GetAt(0); + deviceId = device.Id(); + auto deviceName = device.Name(); + auto enabled = device.IsEnabled(); + std::cout << "Found device " << deviceName.c_str() << ", enabled = " << enabled << "\n"; + winrt::Windows::Media::Capture::MediaCapture captureManager; + winrt::Windows::Media::Capture::MediaCaptureInitializationSettings settings; + settings.VideoDeviceId(deviceId); + captureManager.InitializeAsync(settings).get(); + auto mediaCaptureSettings = captureManager.MediaCaptureSettings(); + auto direct3D11Device = mediaCaptureSettings.Direct3D11Device(); + LearningModelDevice dmlDeviceCamera = LearningModelDevice::CreateFromDirect3D11Device(direct3D11Device); + LearningModelSession dmlSessionCamera(model, dmlDeviceCamera); + } else { + WINML_SKIP_TEST("Test skipped because video capture device is missing"); + } } // create a device from D3D11 Device -TEST_F(ScenarioCppWinrtGpuSkipEdgeCoreTest, Scenario8SetDeviceSampleD3D11Device) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - - com_ptr pD3D11Device = nullptr; - com_ptr pContext = nullptr; - D3D_FEATURE_LEVEL fl; - HRESULT result = D3D11CreateDevice( - nullptr, D3D_DRIVER_TYPE::D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, nullptr, 0, - D3D11_SDK_VERSION, pD3D11Device.put(), &fl, pContext.put()); - if (FAILED(result)) - { - GTEST_SKIP() << "Test skipped because d3d11 device is missing"; - } +static void Scenario8SetDeviceSampleD3D11Device() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); - // get dxgiDevice from d3ddevice - com_ptr pDxgiDevice; - pD3D11Device.get()->QueryInterface(pDxgiDevice.put()); + com_ptr pD3D11Device = nullptr; + com_ptr pContext = nullptr; + D3D_FEATURE_LEVEL fl; + HRESULT result = D3D11CreateDevice( + nullptr, D3D_DRIVER_TYPE::D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, nullptr, 0, + D3D11_SDK_VERSION, pD3D11Device.put(), &fl, pContext.put()); + if (FAILED(result)) { + WINML_SKIP_TEST("Test skipped because d3d11 device is missing"); + } + + // get dxgiDevice from d3ddevice + com_ptr pDxgiDevice; + pD3D11Device.get()->QueryInterface(pDxgiDevice.put()); - com_ptr<::IInspectable> pInspectable; - CreateDirect3D11DeviceFromDXGIDevice(pDxgiDevice.get(), pInspectable.put()); + com_ptr<::IInspectable> pInspectable; + CreateDirect3D11DeviceFromDXGIDevice(pDxgiDevice.get(), pInspectable.put()); - LearningModelDevice device = LearningModelDevice::CreateFromDirect3D11Device( - pInspectable.as()); - LearningModelSession session(model, device); + LearningModelDevice device = LearningModelDevice::CreateFromDirect3D11Device( + pInspectable.as()); + LearningModelSession session(model, device); } // create a session on the a specific dx device that I chose some other way , note we have to use native interop here and pass a cmd queue -TEST_F(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleCustomCommandQueue) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - - com_ptr pD3D12Device = nullptr; - DeviceHelpers::AdapterEnumerationSupport support; - if (FAILED(DeviceHelpers::GetAdapterEnumerationSupport(&support))) - { - FAIL() << "Unable to load DXGI or DXCore"; - return; - } - HRESULT result = S_OK; - if (support.has_dxgi) - { - EXPECT_NO_THROW(result = D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_12_0, __uuidof(ID3D12Device), reinterpret_cast(pD3D12Device.put()))); - } +static void Scenario8SetDeviceSampleCustomCommandQueue() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + + com_ptr pD3D12Device = nullptr; + CommonDeviceHelpers::AdapterEnumerationSupport support; + if (FAILED(CommonDeviceHelpers::GetAdapterEnumerationSupport(&support))) { + WINML_LOG_ERROR("Unable to load DXGI or DXCore"); + return; + } + HRESULT result = S_OK; + if (support.has_dxgi) { + WINML_EXPECT_NO_THROW(result = D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_12_0, __uuidof(ID3D12Device), reinterpret_cast(pD3D12Device.put()))); + } #ifdef ENABLE_DXCORE - if (support.has_dxgi == false) - { - com_ptr spFactory; - DXCoreCreateAdapterFactory(IID_PPV_ARGS(spFactory.put())); - const GUID gpuFilter[] = { DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS }; - com_ptr spAdapterList; - spFactory->CreateAdapterList(1, gpuFilter, IID_PPV_ARGS(spAdapterList.put())); - com_ptr spAdapter; - EXPECT_NO_THROW(spAdapterList->GetAdapter(0, IID_PPV_ARGS(spAdapter.put()))); - ::IUnknown* pAdapter = spAdapter.get(); - EXPECT_NO_THROW(result = D3D12CreateDevice(pAdapter, D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_12_0, __uuidof(ID3D12Device), reinterpret_cast(pD3D12Device.put()))); - } + if (support.has_dxgi == false) { + com_ptr spFactory; + DXCoreCreateAdapterFactory(IID_PPV_ARGS(spFactory.put())); + const GUID gpuFilter[] = {DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS}; + com_ptr spAdapterList; + spFactory->CreateAdapterList(1, gpuFilter, IID_PPV_ARGS(spAdapterList.put())); + com_ptr spAdapter; + WINML_EXPECT_NO_THROW(spAdapterList->GetAdapter(0, IID_PPV_ARGS(spAdapter.put()))); + ::IUnknown* pAdapter = spAdapter.get(); + WINML_EXPECT_NO_THROW(result = D3D12CreateDevice(pAdapter, D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_12_0, __uuidof(ID3D12Device), reinterpret_cast(pD3D12Device.put()))); + } #endif - if (FAILED(result)) - { - GTEST_SKIP() << "Test skipped because d3d12 device is missing"; - return; - } - com_ptr dxQueue = nullptr; - D3D12_COMMAND_QUEUE_DESC commandQueueDesc = {}; - commandQueueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; - pD3D12Device->CreateCommandQueue(&commandQueueDesc, __uuidof(ID3D12CommandQueue), reinterpret_cast(&dxQueue)); - auto factory = get_activation_factory(); + if (FAILED(result)) { + WINML_SKIP_TEST("Test skipped because d3d12 device is missing"); + return; + } + com_ptr dxQueue = nullptr; + D3D12_COMMAND_QUEUE_DESC commandQueueDesc = {}; + commandQueueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; + pD3D12Device->CreateCommandQueue(&commandQueueDesc, __uuidof(ID3D12CommandQueue), reinterpret_cast(&dxQueue)); + auto factory = get_activation_factory(); - com_ptr<::IUnknown> spUnk; - factory->CreateFromD3D12CommandQueue(dxQueue.get(), spUnk.put()); + com_ptr<::IUnknown> spUnk; + factory->CreateFromD3D12CommandQueue(dxQueue.get(), spUnk.put()); - auto dmlDeviceCustom = spUnk.as(); - LearningModelSession dmlSessionCustom(model, dmlDeviceCustom); + auto dmlDeviceCustom = spUnk.as(); + LearningModelSession dmlSessionCustom(model, dmlDeviceCustom); } - //pass a Tensor in as an input GPU -TEST_F(ScenarioCppWinrtGpuTest, DISABLED_Scenario9LoadBindEvalInputTensorGPU) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"fns-candy.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - - com_ptr pD3D12Device; - EXPECT_NO_THROW(D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device), pD3D12Device.put_void())); - com_ptr dxQueue; - D3D12_COMMAND_QUEUE_DESC commandQueueDesc = {}; - commandQueueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; - pD3D12Device->CreateCommandQueue(&commandQueueDesc, __uuidof(ID3D12CommandQueue), dxQueue.put_void()); - auto devicefactory = get_activation_factory(); - auto tensorfactory = get_activation_factory(); - - - com_ptr<::IUnknown> spUnk; - EXPECT_NO_THROW(devicefactory->CreateFromD3D12CommandQueue(dxQueue.get(), spUnk.put())); - - LearningModelDevice dmlDeviceCustom = nullptr; - EXPECT_NO_THROW(spUnk.as(dmlDeviceCustom)); - LearningModelSession dmlSessionCustom = nullptr; - EXPECT_NO_THROW(dmlSessionCustom = LearningModelSession(model, dmlDeviceCustom)); - - LearningModelBinding modelBinding(dmlSessionCustom); - - UINT64 bufferbytesize = 720 * 720 * 3 * sizeof(float); - D3D12_HEAP_PROPERTIES heapProperties = { - D3D12_HEAP_TYPE_DEFAULT, - D3D12_CPU_PAGE_PROPERTY_UNKNOWN, - D3D12_MEMORY_POOL_UNKNOWN, - 0, - 0 - }; - D3D12_RESOURCE_DESC resourceDesc = { - D3D12_RESOURCE_DIMENSION_BUFFER, - 0, - bufferbytesize, - 1, - 1, - 1, - DXGI_FORMAT_UNKNOWN, - { 1, 0 }, - D3D12_TEXTURE_LAYOUT_ROW_MAJOR, - D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS - }; - - com_ptr pGPUResource = nullptr; - pD3D12Device->CreateCommittedResource( - &heapProperties, - D3D12_HEAP_FLAG_NONE, - &resourceDesc, - D3D12_RESOURCE_STATE_COMMON, - nullptr, - __uuidof(ID3D12Resource), - pGPUResource.put_void() - ); - com_ptr<::IUnknown> spUnkTensor; - TensorFloat input1imagetensor(nullptr); - __int64 shape[4] = { 1,3, 720, 720 }; - tensorfactory->CreateFromD3D12Resource(pGPUResource.get(), shape, 4, spUnkTensor.put()); - spUnkTensor.try_as(input1imagetensor); - - auto feature = model.InputFeatures().First(); - EXPECT_NO_THROW(modelBinding.Bind(feature.Current().Name(), input1imagetensor)); - - auto outputtensordescriptor = model.OutputFeatures().First().Current().as(); - auto outputtensorshape = outputtensordescriptor.Shape(); - VideoFrame outputimage( - BitmapPixelFormat::Rgba8, - static_cast(outputtensorshape.GetAt(3)), - static_cast(outputtensorshape.GetAt(2))); - ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); - - EXPECT_NO_THROW(modelBinding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); - - // Testing GetAsD3D12Resource - com_ptr pReturnedResource; - input1imagetensor.as()->GetD3D12Resource(pReturnedResource.put()); - EXPECT_EQ(pReturnedResource.get(), pGPUResource.get()); - - // Evaluate the model - winrt::hstring correlationId; - dmlSessionCustom.EvaluateAsync(modelBinding, correlationId).get(); +static void Scenario9LoadBindEvalInputTensorGPU() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"fns-candy.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + com_ptr pD3D12Device; + WINML_EXPECT_NO_THROW(D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device), pD3D12Device.put_void())); + com_ptr dxQueue; + D3D12_COMMAND_QUEUE_DESC commandQueueDesc = {}; + commandQueueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; + pD3D12Device->CreateCommandQueue(&commandQueueDesc, __uuidof(ID3D12CommandQueue), dxQueue.put_void()); + auto devicefactory = get_activation_factory(); + auto tensorfactory = get_activation_factory(); + + com_ptr<::IUnknown> spUnk; + WINML_EXPECT_NO_THROW(devicefactory->CreateFromD3D12CommandQueue(dxQueue.get(), spUnk.put())); + + LearningModelDevice dmlDeviceCustom = nullptr; + WINML_EXPECT_NO_THROW(spUnk.as(dmlDeviceCustom)); + LearningModelSession dmlSessionCustom = nullptr; + WINML_EXPECT_NO_THROW(dmlSessionCustom = LearningModelSession(model, dmlDeviceCustom)); + + LearningModelBinding modelBinding(dmlSessionCustom); + + UINT64 bufferbytesize = 720 * 720 * 3 * sizeof(float); + D3D12_HEAP_PROPERTIES heapProperties = { + D3D12_HEAP_TYPE_DEFAULT, + D3D12_CPU_PAGE_PROPERTY_UNKNOWN, + D3D12_MEMORY_POOL_UNKNOWN, + 0, + 0}; + D3D12_RESOURCE_DESC resourceDesc = { + D3D12_RESOURCE_DIMENSION_BUFFER, + 0, + bufferbytesize, + 1, + 1, + 1, + DXGI_FORMAT_UNKNOWN, + {1, 0}, + D3D12_TEXTURE_LAYOUT_ROW_MAJOR, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS}; + + com_ptr pGPUResource = nullptr; + pD3D12Device->CreateCommittedResource( + &heapProperties, + D3D12_HEAP_FLAG_NONE, + &resourceDesc, + D3D12_RESOURCE_STATE_COMMON, + nullptr, + __uuidof(ID3D12Resource), + pGPUResource.put_void()); + com_ptr<::IUnknown> spUnkTensor; + TensorFloat input1imagetensor(nullptr); + __int64 shape[4] = {1, 3, 720, 720}; + tensorfactory->CreateFromD3D12Resource(pGPUResource.get(), shape, 4, spUnkTensor.put()); + spUnkTensor.try_as(input1imagetensor); + + auto feature = model.InputFeatures().First(); + WINML_EXPECT_NO_THROW(modelBinding.Bind(feature.Current().Name(), input1imagetensor)); + + auto outputtensordescriptor = model.OutputFeatures().First().Current().as(); + auto outputtensorshape = outputtensordescriptor.Shape(); + VideoFrame outputimage( + BitmapPixelFormat::Rgba8, + static_cast(outputtensorshape.GetAt(3)), + static_cast(outputtensorshape.GetAt(2))); + ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); + + WINML_EXPECT_NO_THROW(modelBinding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); + + // Testing GetAsD3D12Resource + com_ptr pReturnedResource; + input1imagetensor.as()->GetD3D12Resource(pReturnedResource.put()); + WINML_EXPECT_EQUAL(pReturnedResource.get(), pGPUResource.get()); + + // Evaluate the model + winrt::hstring correlationId; + dmlSessionCustom.EvaluateAsync(modelBinding, correlationId).get(); } -TEST_F(ScenarioCppWinrtGpuTest, Scenario13SingleModelOnCPUandGPU) -{ - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - LearningModelSession cpuSession(model, LearningModelDevice(LearningModelDeviceKind::Cpu)); - LearningModelSession gpuSession(model, LearningModelDevice(LearningModelDeviceKind::DirectX)); - - LearningModelBinding cpuBinding(cpuSession); - LearningModelBinding gpuBinding(gpuSession); - auto inputs = model.InputFeatures(); - for (auto&& input : inputs) - { - auto cpuFeatureValue = FeatureValueFromFeatureValueDescriptor(input); - cpuBinding.Bind(input.Name(), cpuFeatureValue); - - auto gpuFeatureValue = FeatureValueFromFeatureValueDescriptor(input); - gpuBinding.Bind(input.Name(), gpuFeatureValue); - } +static void Scenario13SingleModelOnCPUandGPU() { + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + LearningModelSession cpuSession(model, LearningModelDevice(LearningModelDeviceKind::Cpu)); + LearningModelSession gpuSession(model, LearningModelDevice(LearningModelDeviceKind::DirectX)); + + LearningModelBinding cpuBinding(cpuSession); + LearningModelBinding gpuBinding(gpuSession); + auto inputs = model.InputFeatures(); + for (auto&& input : inputs) { + auto cpuFeatureValue = FeatureValueFromFeatureValueDescriptor(input); + cpuBinding.Bind(input.Name(), cpuFeatureValue); + + auto gpuFeatureValue = FeatureValueFromFeatureValueDescriptor(input); + gpuBinding.Bind(input.Name(), gpuFeatureValue); + } - auto cpuTask = cpuSession.EvaluateAsync(cpuBinding, L"cpu"); - auto gpuTask = gpuSession.EvaluateAsync(gpuBinding, L"gpu"); + auto cpuTask = cpuSession.EvaluateAsync(cpuBinding, L"cpu"); + auto gpuTask = gpuSession.EvaluateAsync(gpuBinding, L"gpu"); - EXPECT_NO_THROW(cpuTask.get()); - EXPECT_NO_THROW(gpuTask.get()); + WINML_EXPECT_NO_THROW(cpuTask.get()); + WINML_EXPECT_NO_THROW(gpuTask.get()); } // Validates when binding input image with free dimensions, the binding step is executed correctly. -TEST_F(ScenarioCppWinrtGpuTest, Scenario11FreeDimensionsTensor) -{ - std::wstring filePath = FileHelpers::GetModulePath() + L"free_dimensional_image_input.onnx"; - // load a model with expected input size: -1 x -1 - auto model = LearningModel::LoadFromFilePath(filePath); - auto session = LearningModelSession(model); - auto binding = LearningModelBinding(session); +static void Scenario11FreeDimensionsTensor() { + std::wstring filePath = FileHelpers::GetModulePath() + L"free_dimensional_image_input.onnx"; + // load a model with expected input size: -1 x -1 + auto model = LearningModel::LoadFromFilePath(filePath); + auto session = LearningModelSession(model); + auto binding = LearningModelBinding(session); - VideoFrame inputImage(BitmapPixelFormat::Rgba8, 1000, 1000); - ImageFeatureValue inputimagetensor = ImageFeatureValue::CreateFromVideoFrame(inputImage); + VideoFrame inputImage(BitmapPixelFormat::Rgba8, 1000, 1000); + ImageFeatureValue inputimagetensor = ImageFeatureValue::CreateFromVideoFrame(inputImage); - auto feature = model.InputFeatures().First(); - binding.Bind(feature.Current().Name(), inputimagetensor); - feature.MoveNext(); - binding.Bind(feature.Current().Name(), inputimagetensor); + auto feature = model.InputFeatures().First(); + binding.Bind(feature.Current().Name(), inputimagetensor); + feature.MoveNext(); + binding.Bind(feature.Current().Name(), inputimagetensor); - session.Evaluate(binding, L""); + session.Evaluate(binding, L""); } -TEST_F(ScenarioCppWinrtGpuTest, Scenario11FreeDimensionsImage) -{ - std::wstring filePath = FileHelpers::GetModulePath() + L"free_dimensional_imageDes.onnx"; - // load a model with expected input size: -1 x -1 - auto model = LearningModel::LoadFromFilePath(filePath); - auto session = LearningModelSession(model); - auto binding = LearningModelBinding(session); +static void Scenario11FreeDimensionsImage() { + std::wstring filePath = FileHelpers::GetModulePath() + L"free_dimensional_imageDes.onnx"; + // load a model with expected input size: -1 x -1 + auto model = LearningModel::LoadFromFilePath(filePath); + auto session = LearningModelSession(model); + auto binding = LearningModelBinding(session); - VideoFrame inputImage(BitmapPixelFormat::Bgra8, 1000, 1000); - ImageFeatureValue inputimagetensor = ImageFeatureValue::CreateFromVideoFrame(inputImage); + VideoFrame inputImage(BitmapPixelFormat::Bgra8, 1000, 1000); + ImageFeatureValue inputimagetensor = ImageFeatureValue::CreateFromVideoFrame(inputImage); - auto feature = model.InputFeatures().First(); - ImageFeatureDescriptor imageDescriptor = nullptr; - feature.Current().as(imageDescriptor); - binding.Bind(feature.Current().Name(), inputimagetensor); + auto feature = model.InputFeatures().First(); + ImageFeatureDescriptor imageDescriptor = nullptr; + feature.Current().as(imageDescriptor); + binding.Bind(feature.Current().Name(), inputimagetensor); - feature.MoveNext(); - feature.Current().as(imageDescriptor); - binding.Bind(feature.Current().Name(), inputimagetensor); + feature.MoveNext(); + feature.Current().as(imageDescriptor); + binding.Bind(feature.Current().Name(), inputimagetensor); - session.Evaluate(binding, L""); + session.Evaluate(binding, L""); } -struct SwapChainEntry -{ - LearningModelSession session; - LearningModelBinding binding; - winrt::Windows::Foundation::IAsyncOperation activetask; - SwapChainEntry() :session(nullptr), binding(nullptr), activetask(nullptr) - {} +struct SwapChainEntry { + LearningModelSession session; + LearningModelBinding binding; + winrt::Windows::Foundation::IAsyncOperation activetask; + SwapChainEntry() : session(nullptr), binding(nullptr), activetask(nullptr) {} }; -void SubmitEval(LearningModel model, SwapChainEntry *sessionBindings, int swapchaindex) -{ - if (sessionBindings[swapchaindex].activetask != nullptr) - { - //make sure the previously submitted work for this swapchain index is complete before reusing resources - sessionBindings[swapchaindex].activetask.get(); - } - // bind the input and the output buffers by name - auto inputs = model.InputFeatures(); - for (auto&& input : inputs) - { - auto featureValue = FeatureValueFromFeatureValueDescriptor(input); - // set an actual buffer here. we're using uninitialized data for simplicity. - sessionBindings[swapchaindex].binding.Bind(input.Name(), featureValue); - } - // submit an eval and wait for it to finish submitting work - sessionBindings[swapchaindex].activetask = sessionBindings[swapchaindex].session.EvaluateAsync(sessionBindings[swapchaindex].binding, L"0"); - // return without waiting for the submit to finish, setup the completion handler +void SubmitEval(LearningModel model, SwapChainEntry* sessionBindings, int swapchaindex) { + if (sessionBindings[swapchaindex].activetask != nullptr) { + //make sure the previously submitted work for this swapchain index is complete before reusing resources + sessionBindings[swapchaindex].activetask.get(); + } + // bind the input and the output buffers by name + auto inputs = model.InputFeatures(); + for (auto&& input : inputs) { + auto featureValue = FeatureValueFromFeatureValueDescriptor(input); + // set an actual buffer here. we're using uninitialized data for simplicity. + sessionBindings[swapchaindex].binding.Bind(input.Name(), featureValue); + } + // submit an eval and wait for it to finish submitting work + sessionBindings[swapchaindex].activetask = sessionBindings[swapchaindex].session.EvaluateAsync(sessionBindings[swapchaindex].binding, L"0"); + // return without waiting for the submit to finish, setup the completion handler } //Scenario14:Load single model, run it mutliple times on a single gpu device using a fast swapchain pattern -TEST_F(ScenarioCppWinrtGpuTest, Scenario14RunModelSwapchain) -{ - const int swapchainentrycount = 3; - SwapChainEntry sessionBindings[swapchainentrycount]; - - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - // create a session on gpu1 - LearningModelDevice dmlDevice = LearningModelDevice(LearningModelDeviceKind::DirectX); - // create the swapchain style bindings to cycle through - for (int i = 0; i < swapchainentrycount; i++) - { - sessionBindings[i].session = LearningModelSession(model, dmlDevice); - sessionBindings[i].binding = LearningModelBinding(sessionBindings[i].session); - } +static void Scenario14RunModelSwapchain() { + const int swapchainentrycount = 3; + SwapChainEntry sessionBindings[swapchainentrycount]; - //submit 10 evaluations to 3 swapchain entries - int swapchaindex = 0; - for (int i = 0; i < 10; i++) - { - swapchaindex = swapchaindex % swapchainentrycount; - SubmitEval(model, sessionBindings, (swapchaindex)++); - } + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + // create a session on gpu1 + LearningModelDevice dmlDevice = LearningModelDevice(LearningModelDeviceKind::DirectX); + // create the swapchain style bindings to cycle through + for (int i = 0; i < swapchainentrycount; i++) { + sessionBindings[i].session = LearningModelSession(model, dmlDevice); + sessionBindings[i].binding = LearningModelBinding(sessionBindings[i].session); + } + + //submit 10 evaluations to 3 swapchain entries + int swapchaindex = 0; + for (int i = 0; i < 10; i++) { + swapchaindex = swapchaindex % swapchainentrycount; + SubmitEval(model, sessionBindings, (swapchaindex)++); + } - //wait for all work to be completed - for (int i = 0; i < swapchainentrycount; i++) - { - if (sessionBindings[i].activetask != nullptr) - { - //make sure the previously submitted work for this swapchain index is compolete before resuing resources - sessionBindings[i].activetask.get(); - } + //wait for all work to be completed + for (int i = 0; i < swapchainentrycount; i++) { + if (sessionBindings[i].activetask != nullptr) { + //make sure the previously submitted work for this swapchain index is compolete before resuing resources + sessionBindings[i].activetask.get(); } + } } -static void LoadBindEval_CustomOperator_CPU(const wchar_t* fileName) -{ - auto customOperatorProvider = winrt::make(); - auto provider = customOperatorProvider.as(); - - LearningModel model = LearningModel::LoadFromFilePath(fileName, provider); - LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); - LearningModelBinding bindings(session); - - auto inputShape = std::vector{ 5 }; - auto inputData = std::vector{ -50.f, -25.f, 0.f, 25.f, 50.f }; - auto inputValue = - TensorFloat::CreateFromIterable( - inputShape, - single_threaded_vector(std::move(inputData)).GetView()); - EXPECT_NO_THROW(bindings.Bind(L"X", inputValue)); - - auto outputValue = TensorFloat::Create(); - EXPECT_NO_THROW(bindings.Bind(L"Y", outputValue)); - - hstring correlationId; - EXPECT_NO_THROW(session.Evaluate(bindings, correlationId)); - - auto buffer = outputValue.GetAsVectorView(); - EXPECT_TRUE(buffer != nullptr); +static void LoadBindEval_CustomOperator_CPU(const wchar_t* fileName) { + auto customOperatorProvider = winrt::make(); + auto provider = customOperatorProvider.as(); + + LearningModel model = LearningModel::LoadFromFilePath(fileName, provider); + LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); + LearningModelBinding bindings(session); + + auto inputShape = std::vector{5}; + auto inputData = std::vector{-50.f, -25.f, 0.f, 25.f, 50.f}; + auto inputValue = + TensorFloat::CreateFromIterable( + inputShape, + single_threaded_vector(std::move(inputData)).GetView()); + WINML_EXPECT_NO_THROW(bindings.Bind(L"X", inputValue)); + + auto outputValue = TensorFloat::Create(); + WINML_EXPECT_NO_THROW(bindings.Bind(L"Y", outputValue)); + + hstring correlationId; + WINML_EXPECT_NO_THROW(session.Evaluate(bindings, correlationId)); + + auto buffer = outputValue.GetAsVectorView(); + WINML_EXPECT_TRUE(buffer != nullptr); } //! Scenario17 : Control the dev diagnostics features of WinML Tracing -TEST_F(ScenarioCppWinrtTest, Scenario17DevDiagnostics) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - // create a session on the default device - LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); - // create a binding set - LearningModelBinding binding(session); - // bind the input and the output buffers by name - auto inputs = model.InputFeatures(); - for (auto&& input : inputs) - { - auto featureValue = FeatureValueFromFeatureValueDescriptor(input); - // set an actual buffer here. we're using uninitialized data for simplicity. - binding.Bind(input.Name(), featureValue); - } - session.EvaluationProperties().Insert(L"EnableDebugOutput", nullptr); - // run eval - EXPECT_NO_THROW(session.Evaluate(binding, L"")); +static void Scenario17DevDiagnostics() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + // create a session on the default device + LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); + // create a binding set + LearningModelBinding binding(session); + // bind the input and the output buffers by name + auto inputs = model.InputFeatures(); + for (auto&& input : inputs) { + auto featureValue = FeatureValueFromFeatureValueDescriptor(input); + // set an actual buffer here. we're using uninitialized data for simplicity. + binding.Bind(input.Name(), featureValue); + } + session.EvaluationProperties().Insert(L"EnableDebugOutput", nullptr); + // run eval + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); } -/** +/** * Custom Operator Tests are labeled as GPU tests because the DML code is interlaced with the custom op code * even though CPU custom ops shouldn't be dependent on GPU functionality. - * These should be reclassed to ScenarioCppWinrtTest once the DML code is decoupled from the custom op code. -**/ - + * These should be reclassed to ScenarioCppWinrt once the DML code is decoupled from the custom op code. +**/ // create a session that loads a model with a branch new operator, register the custom operator, and load/bind/eval -TEST_F(ScenarioCppWinrtGpuTest, Scenario20aLoadBindEvalCustomOperatorCPU) -{ - std::wstring filePath = FileHelpers::GetModulePath() + L"noisy_relu.onnx"; - LoadBindEval_CustomOperator_CPU(filePath.c_str()); +static void Scenario20aLoadBindEvalCustomOperatorCPU() { + std::wstring filePath = FileHelpers::GetModulePath() + L"noisy_relu.onnx"; + LoadBindEval_CustomOperator_CPU(filePath.c_str()); } // create a session that loads a model with an overridden operator, register the replacement custom operator, and load/bind/eval -TEST_F(ScenarioCppWinrtGpuTest, Scenario20bLoadBindEvalReplacementCustomOperatorCPU) -{ - std::wstring filePath = FileHelpers::GetModulePath() + L"relu.onnx"; - LoadBindEval_CustomOperator_CPU(filePath.c_str()); +static void Scenario20bLoadBindEvalReplacementCustomOperatorCPU() { + std::wstring filePath = FileHelpers::GetModulePath() + L"relu.onnx"; + LoadBindEval_CustomOperator_CPU(filePath.c_str()); } //! Scenario21: Load two models, set them up to run chained after one another on the same gpu hardware device -TEST_F(ScenarioCppWinrtGpuTest, DISABLED_Scenario21RunModel2ChainZ) -{ - // load a model, TODO: get a model that has an image descriptor - std::wstring filePath = FileHelpers::GetModulePath() + L"fns-candy.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - // create both session on the default gpu - LearningModelSession session1(model, LearningModelDevice(LearningModelDeviceKind::DirectX)); - LearningModelSession session2(model, LearningModelDevice(LearningModelDeviceKind::DirectX)); - // create both binding sets - LearningModelBinding binding1(session1); - LearningModelBinding binding2(session2); - // get the input descriptor - auto input = model.InputFeatures().GetAt(0); - // load a SoftwareBitmap - auto sb = FileHelpers::GetSoftwareBitmapFromFile(FileHelpers::GetModulePath() + L"fish_720.png"); - auto videoFrame = VideoFrame::CreateWithSoftwareBitmap(sb); - // bind it - binding1.Bind(input.Name(), videoFrame); - // get the output descriptor - auto output = model.OutputFeatures().GetAt(0); - // create an empty output tensor since we don't want the first model to detensorize into an image. - - std::vector shape = { 1, 3, 720, 720 }; - auto outputValue = TensorFloat::Create(shape); // FeatureValueFromFeatureValueDescriptor(input, nullptr); - // now bind the(empty) output so we have a marker to chain with - binding1.Bind(output.Name(), outputValue); - // and leave the output unbound on the second model, we will fetch it later - // run both models async - EXPECT_NO_THROW(session1.EvaluateAsync(binding1, L"")); - - // now bind that output to the next models input - binding2.Bind(input.Name(), outputValue); - - //eval the second model - auto session2AsyncOp = session2.EvaluateAsync(binding2, L""); - - // now get the output don't wait, queue up the next model - auto finalOutput = session2AsyncOp.get().Outputs().First().Current().Value(); +static void Scenario21RunModel2ChainZ() { + // load a model, TODO: get a model that has an image descriptor + std::wstring filePath = FileHelpers::GetModulePath() + L"fns-candy.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + // create both session on the default gpu + LearningModelSession session1(model, LearningModelDevice(LearningModelDeviceKind::DirectX)); + LearningModelSession session2(model, LearningModelDevice(LearningModelDeviceKind::DirectX)); + // create both binding sets + LearningModelBinding binding1(session1); + LearningModelBinding binding2(session2); + // get the input descriptor + auto input = model.InputFeatures().GetAt(0); + // load a SoftwareBitmap + auto sb = FileHelpers::GetSoftwareBitmapFromFile(FileHelpers::GetModulePath() + L"fish_720.png"); + auto videoFrame = VideoFrame::CreateWithSoftwareBitmap(sb); + // bind it + binding1.Bind(input.Name(), videoFrame); + // get the output descriptor + auto output = model.OutputFeatures().GetAt(0); + // create an empty output tensor since we don't want the first model to detensorize into an image. + + std::vector shape = {1, 3, 720, 720}; + auto outputValue = TensorFloat::Create(shape); // FeatureValueFromFeatureValueDescriptor(input, nullptr); + // now bind the(empty) output so we have a marker to chain with + binding1.Bind(output.Name(), outputValue); + // and leave the output unbound on the second model, we will fetch it later + // run both models async + WINML_EXPECT_NO_THROW(session1.EvaluateAsync(binding1, L"")); + + // now bind that output to the next models input + binding2.Bind(input.Name(), outputValue); + + //eval the second model + auto session2AsyncOp = session2.EvaluateAsync(binding2, L""); + + // now get the output don't wait, queue up the next model + auto finalOutput = session2AsyncOp.get().Outputs().First().Current().Value(); } -bool VerifyHelper(ImageFeatureValue actual, ImageFeatureValue expected) -{ - auto softwareBitmapActual = actual.VideoFrame().SoftwareBitmap(); - auto softwareBitmapExpected = expected.VideoFrame().SoftwareBitmap(); - EXPECT_EQ(softwareBitmapActual.PixelHeight(), softwareBitmapExpected.PixelHeight()); - EXPECT_EQ(softwareBitmapActual.PixelWidth(), softwareBitmapExpected.PixelWidth()); - EXPECT_EQ(softwareBitmapActual.BitmapPixelFormat(), softwareBitmapExpected.BitmapPixelFormat()); - - // 4 means 4 channels - uint32_t size = 4 * softwareBitmapActual.PixelHeight() * softwareBitmapActual.PixelWidth(); - - winrt::Windows::Storage::Streams::Buffer actualOutputBuffer(size); - winrt::Windows::Storage::Streams::Buffer expectedOutputBuffer(size); - - softwareBitmapActual.CopyToBuffer(actualOutputBuffer); - softwareBitmapExpected.CopyToBuffer(expectedOutputBuffer); - - byte* actualBytes; - actualOutputBuffer.try_as<::Windows::Storage::Streams::IBufferByteAccess>()->Buffer(&actualBytes); - byte* expectedBytes; - expectedOutputBuffer.try_as<::Windows::Storage::Streams::IBufferByteAccess>()->Buffer(&expectedBytes); - - byte* pActualByte = actualBytes; - byte* pExpectedByte = expectedBytes; - - // hard code, might need to be modified later. - const float cMaxErrorRate = 0.06f; - byte epsilon = 20; - - UINT errors = 0; - for (uint32_t i = 0; i < size; i++, pActualByte++, pExpectedByte++) - { - auto diff = (*pActualByte - *pExpectedByte); - if (diff > epsilon) - { - errors++; - } - } - std::cout << "total errors is " << errors << "/" << size << ", errors rate is " << (float)errors / size << "\n"; +bool VerifyHelper(ImageFeatureValue actual, ImageFeatureValue expected) { + auto softwareBitmapActual = actual.VideoFrame().SoftwareBitmap(); + auto softwareBitmapExpected = expected.VideoFrame().SoftwareBitmap(); + WINML_EXPECT_EQUAL(softwareBitmapActual.PixelHeight(), softwareBitmapExpected.PixelHeight()); + WINML_EXPECT_EQUAL(softwareBitmapActual.PixelWidth(), softwareBitmapExpected.PixelWidth()); + WINML_EXPECT_EQUAL(softwareBitmapActual.BitmapPixelFormat(), softwareBitmapExpected.BitmapPixelFormat()); - return ((float)errors / size < cMaxErrorRate); -} + // 4 means 4 channels + uint32_t size = 4 * softwareBitmapActual.PixelHeight() * softwareBitmapActual.PixelWidth(); -TEST_F(ScenarioCppWinrtTest, DISABLED_Scenario22ImageBindingAsCPUTensor) -{ - std::wstring modulePath = FileHelpers::GetModulePath(); - std::wstring inputImagePath = modulePath + L"fish_720.png"; - std::wstring bmImagePath = modulePath + L"bm_fish_720.jpg"; - std::wstring modelPath = modulePath + L"fns-candy.onnx"; + winrt::Windows::Storage::Streams::Buffer actualOutputBuffer(size); + winrt::Windows::Storage::Streams::Buffer expectedOutputBuffer(size); - auto device = LearningModelDevice(LearningModelDeviceKind::Default); - auto model = LearningModel::LoadFromFilePath(modelPath); - auto session = LearningModelSession(model, device); - auto binding = LearningModelBinding(session); + softwareBitmapActual.CopyToBuffer(actualOutputBuffer); + softwareBitmapExpected.CopyToBuffer(expectedOutputBuffer); - SoftwareBitmap softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(inputImagePath); - softwareBitmap = SoftwareBitmap::Convert(softwareBitmap, BitmapPixelFormat::Bgra8); - - // Put softwareBitmap into buffer - BYTE* pData = nullptr; - UINT32 size = 0; - winrt::Windows::Graphics::Imaging::BitmapBuffer spBitmapBuffer(softwareBitmap.LockBuffer(winrt::Windows::Graphics::Imaging::BitmapBufferAccessMode::Read)); - winrt::Windows::Foundation::IMemoryBufferReference reference = spBitmapBuffer.CreateReference(); - auto spByteAccess = reference.as<::Windows::Foundation::IMemoryBufferByteAccess>(); - spByteAccess->GetBuffer(&pData, &size); - - std::vector shape = { 1, 3, softwareBitmap.PixelHeight() , softwareBitmap.PixelWidth() }; - float* pCPUTensor; - uint32_t uCapacity; - TensorFloat tf = TensorFloat::Create(shape); - com_ptr itn = tf.as(); - itn->GetBuffer(reinterpret_cast(&pCPUTensor), &uCapacity); - - uint32_t height = softwareBitmap.PixelHeight(); - uint32_t width = softwareBitmap.PixelWidth(); - for (UINT32 i = 0; i < size; i += 4) - { - UINT32 pixelInd = i / 4; - pCPUTensor[pixelInd] = (float)pData[i]; - pCPUTensor[(height * width) + pixelInd] = (float)pData[i + 1]; - pCPUTensor[(height * width * 2) + pixelInd] = (float)pData[i + 2]; - } + byte* actualBytes; + actualOutputBuffer.try_as<::Windows::Storage::Streams::IBufferByteAccess>()->Buffer(&actualBytes); + byte* expectedBytes; + expectedOutputBuffer.try_as<::Windows::Storage::Streams::IBufferByteAccess>()->Buffer(&expectedBytes); - // Bind input - binding.Bind(model.InputFeatures().First().Current().Name(), tf); - - // Bind output - auto outputtensordescriptor = model.OutputFeatures().First().Current().as(); - auto outputtensorshape = outputtensordescriptor.Shape(); - VideoFrame outputimage( - BitmapPixelFormat::Bgra8, - static_cast(outputtensorshape.GetAt(3)), - static_cast(outputtensorshape.GetAt(2))); - ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); - EXPECT_NO_THROW(binding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); - - // Evaluate the model - winrt::hstring correlationId; - EXPECT_NO_THROW(session.EvaluateAsync(binding, correlationId).get()); - - // Verify the output by comparing with the benchmark image - SoftwareBitmap bm_softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(bmImagePath); - bm_softwareBitmap = SoftwareBitmap::Convert(bm_softwareBitmap, BitmapPixelFormat::Bgra8); - VideoFrame bm_videoFrame = VideoFrame::CreateWithSoftwareBitmap(bm_softwareBitmap); - ImageFeatureValue bm_imagevalue = ImageFeatureValue::CreateFromVideoFrame(bm_videoFrame); - EXPECT_TRUE(VerifyHelper(bm_imagevalue, outputTensor)); - - // check the output video frame object by saving output image to disk - std::wstring outputDataImageFileName = L"out_cpu_tensor_fish_720.jpg"; - StorageFolder currentfolder = StorageFolder::GetFolderFromPathAsync(modulePath).get(); - StorageFile outimagefile = currentfolder.CreateFileAsync(outputDataImageFileName, CreationCollisionOption::ReplaceExisting).get(); - IRandomAccessStream writestream = outimagefile.OpenAsync(FileAccessMode::ReadWrite).get(); - BitmapEncoder encoder = BitmapEncoder::CreateAsync(BitmapEncoder::JpegEncoderId(), writestream).get(); - // Set the software bitmap - encoder.SetSoftwareBitmap(outputimage.SoftwareBitmap()); - encoder.FlushAsync().get(); -} + byte* pActualByte = actualBytes; + byte* pExpectedByte = expectedBytes; -TEST_F(ScenarioCppWinrtGpuTest, DISABLED_Scenario22ImageBindingAsGPUTensor) -{ - std::wstring modulePath = FileHelpers::GetModulePath(); - std::wstring inputImagePath = modulePath + L"fish_720.png"; - std::wstring bmImagePath = modulePath + L"bm_fish_720.jpg"; - std::wstring modelPath = modulePath + L"fns-candy.onnx"; - std::wstring outputDataImageFileName = L"out_gpu_tensor_fish_720.jpg"; - - SoftwareBitmap softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(inputImagePath); - softwareBitmap = SoftwareBitmap::Convert(softwareBitmap, BitmapPixelFormat::Bgra8); - - // Put softwareBitmap into cpu buffer - BYTE* pData = nullptr; - UINT32 size = 0; - winrt::Windows::Graphics::Imaging::BitmapBuffer spBitmapBuffer(softwareBitmap.LockBuffer(winrt::Windows::Graphics::Imaging::BitmapBufferAccessMode::Read)); - winrt::Windows::Foundation::IMemoryBufferReference reference = spBitmapBuffer.CreateReference(); - auto spByteAccess = reference.as<::Windows::Foundation::IMemoryBufferByteAccess>(); - spByteAccess->GetBuffer(&pData, &size); - - std::vector shape = { 1, 3, softwareBitmap.PixelHeight() , softwareBitmap.PixelWidth() }; - FLOAT* pCPUTensor; - uint32_t uCapacity; - - // CPU tensorization - TensorFloat tf = TensorFloat::Create(shape); - com_ptr itn = tf.as(); - itn->GetBuffer(reinterpret_cast(&pCPUTensor), &uCapacity); - - uint32_t height = softwareBitmap.PixelHeight(); - uint32_t width = softwareBitmap.PixelWidth(); - for (UINT32 i = 0; i < size; i += 4) - { - UINT32 pixelInd = i / 4; - pCPUTensor[pixelInd] = (FLOAT)pData[i]; - pCPUTensor[(height * width) + pixelInd] = (FLOAT)pData[i + 1]; - pCPUTensor[(height * width * 2) + pixelInd] = (FLOAT)pData[i + 2]; + // hard code, might need to be modified later. + const float cMaxErrorRate = 0.06f; + byte epsilon = 20; + + UINT errors = 0; + for (uint32_t i = 0; i < size; i++, pActualByte++, pExpectedByte++) { + auto diff = (*pActualByte - *pExpectedByte); + if (diff > epsilon) { + errors++; } + } + std::cout << "total errors is " << errors << "/" << size << ", errors rate is " << (float)errors / size << "\n"; - // create the d3d device. - com_ptr pD3D12Device = nullptr; - EXPECT_NO_THROW(D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device), reinterpret_cast(&pD3D12Device))); - - // create the command queue. - com_ptr dxQueue = nullptr; - D3D12_COMMAND_QUEUE_DESC commandQueueDesc = {}; - commandQueueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; - pD3D12Device->CreateCommandQueue(&commandQueueDesc, __uuidof(ID3D12CommandQueue), reinterpret_cast(&dxQueue)); - auto devicefactory = get_activation_factory(); - auto tensorfactory = get_activation_factory(); - com_ptr<::IUnknown> spUnk; - devicefactory->CreateFromD3D12CommandQueue(dxQueue.get(), spUnk.put()); - - LearningModel model(nullptr); - EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(modelPath)); - LearningModelDevice dmlDeviceCustom = nullptr; - EXPECT_NO_THROW(spUnk.as(dmlDeviceCustom)); - LearningModelSession dmlSessionCustom = nullptr; - EXPECT_NO_THROW(dmlSessionCustom = LearningModelSession(model, dmlDeviceCustom)); - LearningModelBinding modelBinding = nullptr; - EXPECT_NO_THROW(modelBinding = LearningModelBinding(dmlSessionCustom)); - - // Create ID3D12GraphicsCommandList and Allocator - D3D12_COMMAND_LIST_TYPE queuetype = dxQueue->GetDesc().Type; - com_ptr alloctor; - com_ptr cmdList; - - pD3D12Device->CreateCommandAllocator( - queuetype, - winrt::guid_of(), - alloctor.put_void()); - - pD3D12Device->CreateCommandList( - 0, - queuetype, - alloctor.get(), - nullptr, - winrt::guid_of(), - cmdList.put_void()); - - // Create Committed Resource - // 3 is number of channels we use. R G B without alpha. - UINT64 bufferbytesize = 3 * sizeof(float) * softwareBitmap.PixelWidth()*softwareBitmap.PixelHeight(); - D3D12_HEAP_PROPERTIES heapProperties = { - D3D12_HEAP_TYPE_DEFAULT, - D3D12_CPU_PAGE_PROPERTY_UNKNOWN, - D3D12_MEMORY_POOL_UNKNOWN, - 0, - 0 - }; - D3D12_RESOURCE_DESC resourceDesc = { - D3D12_RESOURCE_DIMENSION_BUFFER, - 0, - bufferbytesize, - 1, - 1, - 1, - DXGI_FORMAT_UNKNOWN, - { 1, 0 }, - D3D12_TEXTURE_LAYOUT_ROW_MAJOR, - D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS - }; - - com_ptr pGPUResource = nullptr; - com_ptr imageUploadHeap; - pD3D12Device->CreateCommittedResource( - &heapProperties, - D3D12_HEAP_FLAG_NONE, - &resourceDesc, - D3D12_RESOURCE_STATE_COMMON, - nullptr, - __uuidof(ID3D12Resource), - pGPUResource.put_void() - ); - - // Create the GPU upload buffer. - CD3DX12_HEAP_PROPERTIES props(D3D12_HEAP_TYPE_UPLOAD); - auto buffer = CD3DX12_RESOURCE_DESC::Buffer(bufferbytesize); - EXPECT_NO_THROW(pD3D12Device->CreateCommittedResource( - &props, - D3D12_HEAP_FLAG_NONE, - &buffer, - D3D12_RESOURCE_STATE_GENERIC_READ, - nullptr, - __uuidof(ID3D12Resource), - imageUploadHeap.put_void())); - - // Copy from Cpu to GPU - D3D12_SUBRESOURCE_DATA CPUData = {}; - CPUData.pData = reinterpret_cast(pCPUTensor); - CPUData.RowPitch = bufferbytesize; - CPUData.SlicePitch = bufferbytesize; - UpdateSubresources(cmdList.get(), pGPUResource.get(), imageUploadHeap.get(), 0, 0, 1, &CPUData); - - // Close the command list and execute it to begin the initial GPU setup. - EXPECT_NO_THROW(cmdList->Close()); - ID3D12CommandList* ppCommandLists[] = { cmdList.get() }; - dxQueue->ExecuteCommandLists(_countof(ppCommandLists), ppCommandLists); - - // GPU tensorize - com_ptr<::IUnknown> spUnkTensor; - TensorFloat input1imagetensor(nullptr); - __int64 shapes[4] = { 1,3, softwareBitmap.PixelWidth(), softwareBitmap.PixelHeight() }; - tensorfactory->CreateFromD3D12Resource(pGPUResource.get(), shapes, 4, spUnkTensor.put()); - spUnkTensor.try_as(input1imagetensor); - - auto feature = model.InputFeatures().First(); - EXPECT_NO_THROW(modelBinding.Bind(feature.Current().Name(), input1imagetensor)); - - auto outputtensordescriptor = model.OutputFeatures().First().Current().as(); - auto outputtensorshape = outputtensordescriptor.Shape(); - VideoFrame outputimage( - BitmapPixelFormat::Rgba8, - static_cast(outputtensorshape.GetAt(3)), - static_cast(outputtensorshape.GetAt(2))); - ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); - - EXPECT_NO_THROW(modelBinding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); - - // Evaluate the model - winrt::hstring correlationId; - dmlSessionCustom.EvaluateAsync(modelBinding, correlationId).get(); - - // Verify the output by comparing with the benchmark image - SoftwareBitmap bm_softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(bmImagePath); - bm_softwareBitmap = SoftwareBitmap::Convert(bm_softwareBitmap, BitmapPixelFormat::Rgba8); - VideoFrame bm_videoFrame = VideoFrame::CreateWithSoftwareBitmap(bm_softwareBitmap); - ImageFeatureValue bm_imagevalue = ImageFeatureValue::CreateFromVideoFrame(bm_videoFrame); - EXPECT_TRUE(VerifyHelper(bm_imagevalue, outputTensor)); - - - //check the output video frame object - StorageFolder currentfolder = StorageFolder::GetFolderFromPathAsync(modulePath).get(); - StorageFile outimagefile = currentfolder.CreateFileAsync(outputDataImageFileName, CreationCollisionOption::ReplaceExisting).get(); - IRandomAccessStream writestream = outimagefile.OpenAsync(FileAccessMode::ReadWrite).get(); - BitmapEncoder encoder = BitmapEncoder::CreateAsync(BitmapEncoder::JpegEncoderId(), writestream).get(); - // Set the software bitmap - encoder.SetSoftwareBitmap(outputimage.SoftwareBitmap()); - encoder.FlushAsync().get(); + return ((float)errors / size < cMaxErrorRate); } -TEST_F(ScenarioCppWinrtTest, QuantizedModels) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"onnxzoo_lotus_inception_v1-dq.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - // create a session on the default device - LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); - // create a binding set - LearningModelBinding binding(session); - // bind the input and the output buffers by name - auto inputs = model.InputFeatures(); - for (auto&& input : inputs) - { - auto featureValue = FeatureValueFromFeatureValueDescriptor(input); - // set an actual buffer here. we're using uninitialized data for simplicity. - binding.Bind(input.Name(), featureValue); - } - // run eval - EXPECT_NO_THROW(session.Evaluate(binding, filePath)); +static void Scenario22ImageBindingAsCPUTensor() { + std::wstring modulePath = FileHelpers::GetModulePath(); + std::wstring inputImagePath = modulePath + L"fish_720.png"; + std::wstring bmImagePath = modulePath + L"bm_fish_720.jpg"; + std::wstring modelPath = modulePath + L"fns-candy.onnx"; + + auto device = LearningModelDevice(LearningModelDeviceKind::Default); + auto model = LearningModel::LoadFromFilePath(modelPath); + auto session = LearningModelSession(model, device); + auto binding = LearningModelBinding(session); + + SoftwareBitmap softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(inputImagePath); + softwareBitmap = SoftwareBitmap::Convert(softwareBitmap, BitmapPixelFormat::Bgra8); + + // Put softwareBitmap into buffer + BYTE* pData = nullptr; + UINT32 size = 0; + winrt::Windows::Graphics::Imaging::BitmapBuffer spBitmapBuffer(softwareBitmap.LockBuffer(winrt::Windows::Graphics::Imaging::BitmapBufferAccessMode::Read)); + winrt::Windows::Foundation::IMemoryBufferReference reference = spBitmapBuffer.CreateReference(); + auto spByteAccess = reference.as<::Windows::Foundation::IMemoryBufferByteAccess>(); + spByteAccess->GetBuffer(&pData, &size); + + std::vector shape = {1, 3, softwareBitmap.PixelHeight(), softwareBitmap.PixelWidth()}; + float* pCPUTensor; + uint32_t uCapacity; + TensorFloat tf = TensorFloat::Create(shape); + com_ptr itn = tf.as(); + itn->GetBuffer(reinterpret_cast(&pCPUTensor), &uCapacity); + + uint32_t height = softwareBitmap.PixelHeight(); + uint32_t width = softwareBitmap.PixelWidth(); + for (UINT32 i = 0; i < size; i += 4) { + UINT32 pixelInd = i / 4; + pCPUTensor[pixelInd] = (float)pData[i]; + pCPUTensor[(height * width) + pixelInd] = (float)pData[i + 1]; + pCPUTensor[(height * width * 2) + pixelInd] = (float)pData[i + 2]; + } + + // Bind input + binding.Bind(model.InputFeatures().First().Current().Name(), tf); + + // Bind output + auto outputtensordescriptor = model.OutputFeatures().First().Current().as(); + auto outputtensorshape = outputtensordescriptor.Shape(); + VideoFrame outputimage( + BitmapPixelFormat::Bgra8, + static_cast(outputtensorshape.GetAt(3)), + static_cast(outputtensorshape.GetAt(2))); + ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); + WINML_EXPECT_NO_THROW(binding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); + + // Evaluate the model + winrt::hstring correlationId; + WINML_EXPECT_NO_THROW(session.EvaluateAsync(binding, correlationId).get()); + + // Verify the output by comparing with the benchmark image + SoftwareBitmap bm_softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(bmImagePath); + bm_softwareBitmap = SoftwareBitmap::Convert(bm_softwareBitmap, BitmapPixelFormat::Bgra8); + VideoFrame bm_videoFrame = VideoFrame::CreateWithSoftwareBitmap(bm_softwareBitmap); + ImageFeatureValue bm_imagevalue = ImageFeatureValue::CreateFromVideoFrame(bm_videoFrame); + WINML_EXPECT_TRUE(VerifyHelper(bm_imagevalue, outputTensor)); + + // check the output video frame object by saving output image to disk + std::wstring outputDataImageFileName = L"out_cpu_tensor_fish_720.jpg"; + StorageFolder currentfolder = StorageFolder::GetFolderFromPathAsync(modulePath).get(); + StorageFile outimagefile = currentfolder.CreateFileAsync(outputDataImageFileName, CreationCollisionOption::ReplaceExisting).get(); + IRandomAccessStream writestream = outimagefile.OpenAsync(FileAccessMode::ReadWrite).get(); + BitmapEncoder encoder = BitmapEncoder::CreateAsync(BitmapEncoder::JpegEncoderId(), writestream).get(); + // Set the software bitmap + encoder.SetSoftwareBitmap(outputimage.SoftwareBitmap()); + encoder.FlushAsync().get(); } -TEST_F(ScenarioCppWinrtGpuTest, MsftQuantizedModels) -{ - // load a model - std::wstring filePath = FileHelpers::GetModulePath() + L"coreml_Resnet50_ImageNet-dq.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::DirectX)); - // create a binding set - LearningModelBinding binding(session); - // bind the input and the output buffers by name - - std::wstring fullImagePath = FileHelpers::GetModulePath() + L"kitten_224.png"; - StorageFile imagefile = StorageFile::GetFileFromPathAsync(fullImagePath).get(); - IRandomAccessStream stream = imagefile.OpenAsync(FileAccessMode::Read).get(); - SoftwareBitmap softwareBitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); - - auto inputs = model.InputFeatures(); - for (auto&& input : inputs) - { - auto featureValue = FeatureValueFromFeatureValueDescriptor(input, softwareBitmap); - // set an actual buffer here. we're using uninitialized data for simplicity. - binding.Bind(input.Name(), featureValue); - } - // run eval - EXPECT_NO_THROW(session.Evaluate(binding, filePath)); +static void Scenario22ImageBindingAsGPUTensor() { + std::wstring modulePath = FileHelpers::GetModulePath(); + std::wstring inputImagePath = modulePath + L"fish_720.png"; + std::wstring bmImagePath = modulePath + L"bm_fish_720.jpg"; + std::wstring modelPath = modulePath + L"fns-candy.onnx"; + std::wstring outputDataImageFileName = L"out_gpu_tensor_fish_720.jpg"; + + SoftwareBitmap softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(inputImagePath); + softwareBitmap = SoftwareBitmap::Convert(softwareBitmap, BitmapPixelFormat::Bgra8); + + // Put softwareBitmap into cpu buffer + BYTE* pData = nullptr; + UINT32 size = 0; + winrt::Windows::Graphics::Imaging::BitmapBuffer spBitmapBuffer(softwareBitmap.LockBuffer(winrt::Windows::Graphics::Imaging::BitmapBufferAccessMode::Read)); + winrt::Windows::Foundation::IMemoryBufferReference reference = spBitmapBuffer.CreateReference(); + auto spByteAccess = reference.as<::Windows::Foundation::IMemoryBufferByteAccess>(); + spByteAccess->GetBuffer(&pData, &size); + + std::vector shape = {1, 3, softwareBitmap.PixelHeight(), softwareBitmap.PixelWidth()}; + FLOAT* pCPUTensor; + uint32_t uCapacity; + + // CPU tensorization + TensorFloat tf = TensorFloat::Create(shape); + com_ptr itn = tf.as(); + itn->GetBuffer(reinterpret_cast(&pCPUTensor), &uCapacity); + + uint32_t height = softwareBitmap.PixelHeight(); + uint32_t width = softwareBitmap.PixelWidth(); + for (UINT32 i = 0; i < size; i += 4) { + UINT32 pixelInd = i / 4; + pCPUTensor[pixelInd] = (FLOAT)pData[i]; + pCPUTensor[(height * width) + pixelInd] = (FLOAT)pData[i + 1]; + pCPUTensor[(height * width * 2) + pixelInd] = (FLOAT)pData[i + 2]; + } + + // create the d3d device. + com_ptr pD3D12Device = nullptr; + WINML_EXPECT_NO_THROW(D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device), reinterpret_cast(&pD3D12Device))); + + // create the command queue. + com_ptr dxQueue = nullptr; + D3D12_COMMAND_QUEUE_DESC commandQueueDesc = {}; + commandQueueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; + pD3D12Device->CreateCommandQueue(&commandQueueDesc, __uuidof(ID3D12CommandQueue), reinterpret_cast(&dxQueue)); + auto devicefactory = get_activation_factory(); + auto tensorfactory = get_activation_factory(); + com_ptr<::IUnknown> spUnk; + devicefactory->CreateFromD3D12CommandQueue(dxQueue.get(), spUnk.put()); + + LearningModel model(nullptr); + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(modelPath)); + LearningModelDevice dmlDeviceCustom = nullptr; + WINML_EXPECT_NO_THROW(spUnk.as(dmlDeviceCustom)); + LearningModelSession dmlSessionCustom = nullptr; + WINML_EXPECT_NO_THROW(dmlSessionCustom = LearningModelSession(model, dmlDeviceCustom)); + LearningModelBinding modelBinding = nullptr; + WINML_EXPECT_NO_THROW(modelBinding = LearningModelBinding(dmlSessionCustom)); + + // Create ID3D12GraphicsCommandList and Allocator + D3D12_COMMAND_LIST_TYPE queuetype = dxQueue->GetDesc().Type; + com_ptr alloctor; + com_ptr cmdList; + + pD3D12Device->CreateCommandAllocator( + queuetype, + winrt::guid_of(), + alloctor.put_void()); + + pD3D12Device->CreateCommandList( + 0, + queuetype, + alloctor.get(), + nullptr, + winrt::guid_of(), + cmdList.put_void()); + + // Create Committed Resource + // 3 is number of channels we use. R G B without alpha. + UINT64 bufferbytesize = 3 * sizeof(float) * softwareBitmap.PixelWidth() * softwareBitmap.PixelHeight(); + D3D12_HEAP_PROPERTIES heapProperties = { + D3D12_HEAP_TYPE_DEFAULT, + D3D12_CPU_PAGE_PROPERTY_UNKNOWN, + D3D12_MEMORY_POOL_UNKNOWN, + 0, + 0}; + D3D12_RESOURCE_DESC resourceDesc = { + D3D12_RESOURCE_DIMENSION_BUFFER, + 0, + bufferbytesize, + 1, + 1, + 1, + DXGI_FORMAT_UNKNOWN, + {1, 0}, + D3D12_TEXTURE_LAYOUT_ROW_MAJOR, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS}; + + com_ptr pGPUResource = nullptr; + com_ptr imageUploadHeap; + pD3D12Device->CreateCommittedResource( + &heapProperties, + D3D12_HEAP_FLAG_NONE, + &resourceDesc, + D3D12_RESOURCE_STATE_COMMON, + nullptr, + __uuidof(ID3D12Resource), + pGPUResource.put_void()); + + // Create the GPU upload buffer. + CD3DX12_HEAP_PROPERTIES props(D3D12_HEAP_TYPE_UPLOAD); + auto buffer = CD3DX12_RESOURCE_DESC::Buffer(bufferbytesize); + WINML_EXPECT_NO_THROW(pD3D12Device->CreateCommittedResource( + &props, + D3D12_HEAP_FLAG_NONE, + &buffer, + D3D12_RESOURCE_STATE_GENERIC_READ, + nullptr, + __uuidof(ID3D12Resource), + imageUploadHeap.put_void())); + + // Copy from Cpu to GPU + D3D12_SUBRESOURCE_DATA CPUData = {}; + CPUData.pData = reinterpret_cast(pCPUTensor); + CPUData.RowPitch = bufferbytesize; + CPUData.SlicePitch = bufferbytesize; + UpdateSubresources(cmdList.get(), pGPUResource.get(), imageUploadHeap.get(), 0, 0, 1, &CPUData); + + // Close the command list and execute it to begin the initial GPU setup. + WINML_EXPECT_NO_THROW(cmdList->Close()); + ID3D12CommandList* ppCommandLists[] = {cmdList.get()}; + dxQueue->ExecuteCommandLists(_countof(ppCommandLists), ppCommandLists); + + // GPU tensorize + com_ptr<::IUnknown> spUnkTensor; + TensorFloat input1imagetensor(nullptr); + __int64 shapes[4] = {1, 3, softwareBitmap.PixelWidth(), softwareBitmap.PixelHeight()}; + tensorfactory->CreateFromD3D12Resource(pGPUResource.get(), shapes, 4, spUnkTensor.put()); + spUnkTensor.try_as(input1imagetensor); + + auto feature = model.InputFeatures().First(); + WINML_EXPECT_NO_THROW(modelBinding.Bind(feature.Current().Name(), input1imagetensor)); + + auto outputtensordescriptor = model.OutputFeatures().First().Current().as(); + auto outputtensorshape = outputtensordescriptor.Shape(); + VideoFrame outputimage( + BitmapPixelFormat::Rgba8, + static_cast(outputtensorshape.GetAt(3)), + static_cast(outputtensorshape.GetAt(2))); + ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); + + WINML_EXPECT_NO_THROW(modelBinding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); + + // Evaluate the model + winrt::hstring correlationId; + dmlSessionCustom.EvaluateAsync(modelBinding, correlationId).get(); + + // Verify the output by comparing with the benchmark image + SoftwareBitmap bm_softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(bmImagePath); + bm_softwareBitmap = SoftwareBitmap::Convert(bm_softwareBitmap, BitmapPixelFormat::Rgba8); + VideoFrame bm_videoFrame = VideoFrame::CreateWithSoftwareBitmap(bm_softwareBitmap); + ImageFeatureValue bm_imagevalue = ImageFeatureValue::CreateFromVideoFrame(bm_videoFrame); + WINML_EXPECT_TRUE(VerifyHelper(bm_imagevalue, outputTensor)); + + //check the output video frame object + StorageFolder currentfolder = StorageFolder::GetFolderFromPathAsync(modulePath).get(); + StorageFile outimagefile = currentfolder.CreateFileAsync(outputDataImageFileName, CreationCollisionOption::ReplaceExisting).get(); + IRandomAccessStream writestream = outimagefile.OpenAsync(FileAccessMode::ReadWrite).get(); + BitmapEncoder encoder = BitmapEncoder::CreateAsync(BitmapEncoder::JpegEncoderId(), writestream).get(); + // Set the software bitmap + encoder.SetSoftwareBitmap(outputimage.SoftwareBitmap()); + encoder.FlushAsync().get(); } -TEST_F(ScenarioCppWinrtGpuTest, DISABLED_SyncVsAsync) -{ - // create model, device and session - LearningModel model = nullptr; - EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(FileHelpers::GetModulePath() + L"fns-candy.onnx")); - - LearningModelSession session = nullptr; - EXPECT_NO_THROW(session = LearningModelSession(model, LearningModelDevice(LearningModelDeviceKind::DirectX))); - - // create the binding - LearningModelBinding modelBinding(session); - - // bind the input - std::wstring fullImagePath = FileHelpers::GetModulePath() + L"fish_720.png"; - StorageFile imagefile = StorageFile::GetFileFromPathAsync(fullImagePath).get(); - IRandomAccessStream stream = imagefile.OpenAsync(FileAccessMode::Read).get(); - SoftwareBitmap softwareBitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); - VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); - - auto imagetensor = ImageFeatureValue::CreateFromVideoFrame(frame); - auto inputFeatureDescriptor = model.InputFeatures().First(); - EXPECT_NO_THROW(modelBinding.Bind(inputFeatureDescriptor.Current().Name(), imagetensor)); - - UINT N = 20; - - auto outputtensordescriptor = model.OutputFeatures().First().Current().as(); - auto outputtensorshape = outputtensordescriptor.Shape(); - VideoFrame outputimage( - BitmapPixelFormat::Rgba8, - static_cast(outputtensorshape.GetAt(3)), - static_cast(outputtensorshape.GetAt(2))); - ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); - EXPECT_NO_THROW(modelBinding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); - - // evaluate N times synchronously and time it - auto startSync = std::chrono::high_resolution_clock::now(); - for (UINT i = 0; i < N; i++) - { - session.Evaluate(modelBinding, L""); - } - auto syncTime = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - startSync); - std::cout << "Synchronous time for " << N << " evaluations: " << syncTime.count() << " milliseconds\n"; - - // evaluate N times Asynchronously and time it - std::vector> tasks; - std::vector bindings(N, nullptr); - - for (size_t i = 0; i < bindings.size(); i++) - { - bindings[i] = LearningModelBinding(session); - bindings[i].Bind(inputFeatureDescriptor.Current().Name(), imagetensor); - bindings[i].Bind( - model.OutputFeatures().First().Current().Name(), - VideoFrame(BitmapPixelFormat::Rgba8, - static_cast(outputtensorshape.GetAt(3)), - static_cast(outputtensorshape.GetAt(2)))); - } +static void QuantizedModels() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"onnxzoo_lotus_inception_v1-dq.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + // create a session on the default device + LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::Default)); + // create a binding set + LearningModelBinding binding(session); + // bind the input and the output buffers by name + auto inputs = model.InputFeatures(); + for (auto&& input : inputs) { + auto featureValue = FeatureValueFromFeatureValueDescriptor(input); + // set an actual buffer here. we're using uninitialized data for simplicity. + binding.Bind(input.Name(), featureValue); + } + // run eval + WINML_EXPECT_NO_THROW(session.Evaluate(binding, filePath)); +} - auto startAsync = std::chrono::high_resolution_clock::now(); - for (UINT i = 0; i < N; i++) - { - tasks.emplace_back(session.EvaluateAsync(bindings[i], L"")); - } - // wait for them all to complete - for (auto&& task : tasks) - { - task.get(); - } - auto asyncTime = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - startAsync); - std::cout << "Asynchronous time for " << N << " evaluations: " << asyncTime.count() << " milliseconds\n"; +static void MsftQuantizedModels() { + // load a model + std::wstring filePath = FileHelpers::GetModulePath() + L"coreml_Resnet50_ImageNet-dq.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + LearningModelSession session(model, LearningModelDevice(LearningModelDeviceKind::DirectX)); + // create a binding set + LearningModelBinding binding(session); + // bind the input and the output buffers by name + + std::wstring fullImagePath = FileHelpers::GetModulePath() + L"kitten_224.png"; + StorageFile imagefile = StorageFile::GetFileFromPathAsync(fullImagePath).get(); + IRandomAccessStream stream = imagefile.OpenAsync(FileAccessMode::Read).get(); + SoftwareBitmap softwareBitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); + + auto inputs = model.InputFeatures(); + for (auto&& input : inputs) { + auto featureValue = FeatureValueFromFeatureValueDescriptor(input, softwareBitmap); + // set an actual buffer here. we're using uninitialized data for simplicity. + binding.Bind(input.Name(), featureValue); + } + // run eval + WINML_EXPECT_NO_THROW(session.Evaluate(binding, filePath)); } +static void SyncVsAsync() { + // create model, device and session + LearningModel model = nullptr; + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(FileHelpers::GetModulePath() + L"fns-candy.onnx")); + + LearningModelSession session = nullptr; + WINML_EXPECT_NO_THROW(session = LearningModelSession(model, LearningModelDevice(LearningModelDeviceKind::DirectX))); + + // create the binding + LearningModelBinding modelBinding(session); + + // bind the input + std::wstring fullImagePath = FileHelpers::GetModulePath() + L"fish_720.png"; + StorageFile imagefile = StorageFile::GetFileFromPathAsync(fullImagePath).get(); + IRandomAccessStream stream = imagefile.OpenAsync(FileAccessMode::Read).get(); + SoftwareBitmap softwareBitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); + + auto imagetensor = ImageFeatureValue::CreateFromVideoFrame(frame); + auto inputFeatureDescriptor = model.InputFeatures().First(); + WINML_EXPECT_NO_THROW(modelBinding.Bind(inputFeatureDescriptor.Current().Name(), imagetensor)); + + UINT N = 20; + + auto outputtensordescriptor = model.OutputFeatures().First().Current().as(); + auto outputtensorshape = outputtensordescriptor.Shape(); + VideoFrame outputimage( + BitmapPixelFormat::Rgba8, + static_cast(outputtensorshape.GetAt(3)), + static_cast(outputtensorshape.GetAt(2))); + ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); + WINML_EXPECT_NO_THROW(modelBinding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); + + // evaluate N times synchronously and time it + auto startSync = std::chrono::high_resolution_clock::now(); + for (UINT i = 0; i < N; i++) { + session.Evaluate(modelBinding, L""); + } + auto syncTime = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - startSync); + std::cout << "Synchronous time for " << N << " evaluations: " << syncTime.count() << " milliseconds\n"; + + // evaluate N times Asynchronously and time it + std::vector> tasks; + std::vector bindings(N, nullptr); + + for (size_t i = 0; i < bindings.size(); i++) { + bindings[i] = LearningModelBinding(session); + bindings[i].Bind(inputFeatureDescriptor.Current().Name(), imagetensor); + bindings[i].Bind( + model.OutputFeatures().First().Current().Name(), + VideoFrame(BitmapPixelFormat::Rgba8, + static_cast(outputtensorshape.GetAt(3)), + static_cast(outputtensorshape.GetAt(2)))); + } -TEST_F(ScenarioCppWinrtGpuTest, DISABLED_CustomCommandQueueWithFence) -{ - static const wchar_t* const modelFileName = L"fns-candy.onnx"; - static const wchar_t* const inputDataImageFileName = L"fish_720.png"; + auto startAsync = std::chrono::high_resolution_clock::now(); + for (UINT i = 0; i < N; i++) { + tasks.emplace_back(session.EvaluateAsync(bindings[i], L"")); + } + // wait for them all to complete + for (auto&& task : tasks) { + task.get(); + } + auto asyncTime = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - startAsync); + std::cout << "Asynchronous time for " << N << " evaluations: " << asyncTime.count() << " milliseconds\n"; +} - com_ptr d3d12Device; - EXPECT_HRESULT_SUCCEEDED(D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device), d3d12Device.put_void())); +static void CustomCommandQueueWithFence() { + static const wchar_t* const modelFileName = L"fns-candy.onnx"; + static const wchar_t* const inputDataImageFileName = L"fish_720.png"; - D3D12_COMMAND_QUEUE_DESC queueDesc = {}; - queueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; + com_ptr d3d12Device; + WINML_EXPECT_HRESULT_SUCCEEDED(D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device), d3d12Device.put_void())); - com_ptr queue; - EXPECT_HRESULT_SUCCEEDED(d3d12Device->CreateCommandQueue(&queueDesc, __uuidof(ID3D12CommandQueue), queue.put_void())); + D3D12_COMMAND_QUEUE_DESC queueDesc = {}; + queueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; - com_ptr fence; - EXPECT_HRESULT_SUCCEEDED(d3d12Device->CreateFence(0, D3D12_FENCE_FLAG_NONE, __uuidof(ID3D12Fence), fence.put_void())); + com_ptr queue; + WINML_EXPECT_HRESULT_SUCCEEDED(d3d12Device->CreateCommandQueue(&queueDesc, __uuidof(ID3D12CommandQueue), queue.put_void())); - auto devicefactory = get_activation_factory(); + com_ptr fence; + WINML_EXPECT_HRESULT_SUCCEEDED(d3d12Device->CreateFence(0, D3D12_FENCE_FLAG_NONE, __uuidof(ID3D12Fence), fence.put_void())); - com_ptr<::IUnknown> learningModelDeviceUnknown; - EXPECT_HRESULT_SUCCEEDED(devicefactory->CreateFromD3D12CommandQueue(queue.get(), learningModelDeviceUnknown.put())); + auto devicefactory = get_activation_factory(); - LearningModelDevice device = nullptr; - EXPECT_NO_THROW(learningModelDeviceUnknown.as(device)); + com_ptr<::IUnknown> learningModelDeviceUnknown; + WINML_EXPECT_HRESULT_SUCCEEDED(devicefactory->CreateFromD3D12CommandQueue(queue.get(), learningModelDeviceUnknown.put())); - std::wstring modulePath = FileHelpers::GetModulePath(); + LearningModelDevice device = nullptr; + WINML_EXPECT_NO_THROW(learningModelDeviceUnknown.as(device)); - // WinML model creation - std::wstring fullModelPath = modulePath + modelFileName; - LearningModel model(nullptr); - EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath)); - LearningModelSession modelSession = nullptr; - EXPECT_NO_THROW(modelSession = LearningModelSession(model, device)); - LearningModelBinding modelBinding = nullptr; - EXPECT_NO_THROW(modelBinding = LearningModelBinding(modelSession)); + std::wstring modulePath = FileHelpers::GetModulePath(); - std::wstring fullImagePath = modulePath + inputDataImageFileName; + // WinML model creation + std::wstring fullModelPath = modulePath + modelFileName; + LearningModel model(nullptr); + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath)); + LearningModelSession modelSession = nullptr; + WINML_EXPECT_NO_THROW(modelSession = LearningModelSession(model, device)); + LearningModelBinding modelBinding = nullptr; + WINML_EXPECT_NO_THROW(modelBinding = LearningModelBinding(modelSession)); - StorageFile imagefile = StorageFile::GetFileFromPathAsync(fullImagePath).get(); - IRandomAccessStream stream = imagefile.OpenAsync(FileAccessMode::Read).get(); - SoftwareBitmap softwareBitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); - VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); - ImageFeatureValue input1imagetensor = ImageFeatureValue::CreateFromVideoFrame(frame); + std::wstring fullImagePath = modulePath + inputDataImageFileName; - auto feature = model.InputFeatures().First(); - EXPECT_NO_THROW(modelBinding.Bind(feature.Current().Name(), input1imagetensor)); + StorageFile imagefile = StorageFile::GetFileFromPathAsync(fullImagePath).get(); + IRandomAccessStream stream = imagefile.OpenAsync(FileAccessMode::Read).get(); + SoftwareBitmap softwareBitmap = (BitmapDecoder::CreateAsync(stream).get()).GetSoftwareBitmapAsync().get(); + VideoFrame frame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); + ImageFeatureValue input1imagetensor = ImageFeatureValue::CreateFromVideoFrame(frame); - auto outputtensordescriptor = model.OutputFeatures().First().Current().as(); - auto outputtensorshape = outputtensordescriptor.Shape(); - VideoFrame outputimage( - BitmapPixelFormat::Rgba8, - static_cast(outputtensorshape.GetAt(3)), - static_cast(outputtensorshape.GetAt(2))); - ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); + auto feature = model.InputFeatures().First(); + WINML_EXPECT_NO_THROW(modelBinding.Bind(feature.Current().Name(), input1imagetensor)); - EXPECT_NO_THROW(modelBinding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); + auto outputtensordescriptor = model.OutputFeatures().First().Current().as(); + auto outputtensorshape = outputtensordescriptor.Shape(); + VideoFrame outputimage( + BitmapPixelFormat::Rgba8, + static_cast(outputtensorshape.GetAt(3)), + static_cast(outputtensorshape.GetAt(2))); + ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); - // Block the queue on the fence, evaluate the model, then queue a signal. The model evaluation should not complete - // until after the wait is unblocked, and the signal should not complete until model evaluation does. This can - // only be true if WinML executes the workload on the supplied queue (instead of using its own). + WINML_EXPECT_NO_THROW(modelBinding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); - EXPECT_HRESULT_SUCCEEDED(queue->Wait(fence.get(), 1)); + // Block the queue on the fence, evaluate the model, then queue a signal. The model evaluation should not complete + // until after the wait is unblocked, and the signal should not complete until model evaluation does. This can + // only be true if WinML executes the workload on the supplied queue (instead of using its own). - EXPECT_HRESULT_SUCCEEDED(queue->Signal(fence.get(), 2)); + WINML_EXPECT_HRESULT_SUCCEEDED(queue->Wait(fence.get(), 1)); - winrt::hstring correlationId; - winrt::Windows::Foundation::IAsyncOperation asyncOp; - EXPECT_NO_THROW(asyncOp = modelSession.EvaluateAsync(modelBinding, correlationId)); + WINML_EXPECT_HRESULT_SUCCEEDED(queue->Signal(fence.get(), 2)); - Sleep(1000); // Give the model a chance to run (which it shouldn't if everything is working correctly) + winrt::hstring correlationId; + winrt::Windows::Foundation::IAsyncOperation asyncOp; + WINML_EXPECT_NO_THROW(asyncOp = modelSession.EvaluateAsync(modelBinding, correlationId)); - // Because we haven't unblocked the wait yet, model evaluation must not have completed (nor the fence signal) - EXPECT_NE(asyncOp.Status(), winrt::Windows::Foundation::AsyncStatus::Completed); - EXPECT_EQ(fence->GetCompletedValue(), 0); + Sleep(1000); // Give the model a chance to run (which it shouldn't if everything is working correctly) - // Unblock the queue - EXPECT_HRESULT_SUCCEEDED(fence->Signal(1)); + // Because we haven't unblocked the wait yet, model evaluation must not have completed (nor the fence signal) + WINML_EXPECT_NOT_EQUAL(asyncOp.Status(), winrt::Windows::Foundation::AsyncStatus::Completed); + WINML_EXPECT_EQUAL(fence->GetCompletedValue(), 0); - // Wait for model evaluation to complete - asyncOp.get(); + // Unblock the queue + WINML_EXPECT_HRESULT_SUCCEEDED(fence->Signal(1)); - // The fence must be signaled by now (because model evaluation has completed) - EXPECT_EQ(fence->GetCompletedValue(), 2); -} + // Wait for model evaluation to complete + asyncOp.get(); -TEST_F(ScenarioCppWinrtGpuTest, DISABLED_ReuseVideoFrame) -{ - std::wstring modulePath = FileHelpers::GetModulePath(); - std::wstring inputImagePath = modulePath + L"fish_720.png"; - std::wstring bmImagePath = modulePath + L"bm_fish_720.jpg"; - std::wstring modelPath = modulePath + L"fns-candy.onnx"; - - std::vector deviceKinds = { LearningModelDeviceKind::Cpu, LearningModelDeviceKind::DirectX }; - std::vector videoFrameSources; - DeviceHelpers::AdapterEnumerationSupport support; - DeviceHelpers::GetAdapterEnumerationSupport(&support); - if (support.has_dxgi) - { - videoFrameSources = { "SoftwareBitmap", "Direct3DSurface" }; - } - else { - videoFrameSources = { "SoftwareBitmap" }; + // The fence must be signaled by now (because model evaluation has completed) + WINML_EXPECT_EQUAL(fence->GetCompletedValue(), 2); +} - } +static void ReuseVideoFrame() { + std::wstring modulePath = FileHelpers::GetModulePath(); + std::wstring inputImagePath = modulePath + L"fish_720.png"; + std::wstring bmImagePath = modulePath + L"bm_fish_720.jpg"; + std::wstring modelPath = modulePath + L"fns-candy.onnx"; + + std::vector deviceKinds = {LearningModelDeviceKind::Cpu, LearningModelDeviceKind::DirectX}; + std::vector videoFrameSources; + CommonDeviceHelpers::AdapterEnumerationSupport support; + CommonDeviceHelpers::GetAdapterEnumerationSupport(&support); + if (support.has_dxgi) { + videoFrameSources = {"SoftwareBitmap", "Direct3DSurface"}; + } else { + videoFrameSources = {"SoftwareBitmap"}; + } - for (auto deviceKind : deviceKinds) - { - auto device = LearningModelDevice(deviceKind); - auto model = LearningModel::LoadFromFilePath(modelPath); - auto session = LearningModelSession(model, device); - auto binding = LearningModelBinding(session); - for (auto videoFrameSource : videoFrameSources) - { - VideoFrame reuseVideoFrame = nullptr; - if (videoFrameSource == "SoftwareBitmap") - { - reuseVideoFrame = VideoFrame::CreateWithSoftwareBitmap(SoftwareBitmap(BitmapPixelFormat::Bgra8, 720, 720)); - } - else - { - reuseVideoFrame = VideoFrame::CreateAsDirect3D11SurfaceBacked(DirectXPixelFormat::B8G8R8X8UIntNormalized, 720, 720); - } - for (uint32_t i = 0; i < 3; ++i) - { - SoftwareBitmap softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(inputImagePath); - VideoFrame videoFrame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); - // reuse video frame - videoFrame.CopyToAsync(reuseVideoFrame).get(); - - // bind input - binding.Bind(model.InputFeatures().First().Current().Name(), reuseVideoFrame); - - // bind output - VideoFrame outputimage(BitmapPixelFormat::Bgra8, 720, 720); - ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); - EXPECT_NO_THROW(binding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); - - // evaluate - winrt::hstring correlationId; - EXPECT_NO_THROW(session.EvaluateAsync(binding, correlationId).get()); - - // verify result - SoftwareBitmap bm_softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(bmImagePath); - bm_softwareBitmap = SoftwareBitmap::Convert(bm_softwareBitmap, BitmapPixelFormat::Bgra8); - VideoFrame bm_videoFrame = VideoFrame::CreateWithSoftwareBitmap(bm_softwareBitmap); - ImageFeatureValue bm_imagevalue = ImageFeatureValue::CreateFromVideoFrame(bm_videoFrame); - EXPECT_TRUE(VerifyHelper(bm_imagevalue, outputTensor)); - } - } + for (auto deviceKind : deviceKinds) { + auto device = LearningModelDevice(deviceKind); + auto model = LearningModel::LoadFromFilePath(modelPath); + auto session = LearningModelSession(model, device); + auto binding = LearningModelBinding(session); + for (auto videoFrameSource : videoFrameSources) { + VideoFrame reuseVideoFrame = nullptr; + if (videoFrameSource == "SoftwareBitmap") { + reuseVideoFrame = VideoFrame::CreateWithSoftwareBitmap(SoftwareBitmap(BitmapPixelFormat::Bgra8, 720, 720)); + } else { + reuseVideoFrame = VideoFrame::CreateAsDirect3D11SurfaceBacked(DirectXPixelFormat::B8G8R8X8UIntNormalized, 720, 720); + } + for (uint32_t i = 0; i < 3; ++i) { + SoftwareBitmap softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(inputImagePath); + VideoFrame videoFrame = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap); + // reuse video frame + videoFrame.CopyToAsync(reuseVideoFrame).get(); + + // bind input + binding.Bind(model.InputFeatures().First().Current().Name(), reuseVideoFrame); + + // bind output + VideoFrame outputimage(BitmapPixelFormat::Bgra8, 720, 720); + ImageFeatureValue outputTensor = ImageFeatureValue::CreateFromVideoFrame(outputimage); + WINML_EXPECT_NO_THROW(binding.Bind(model.OutputFeatures().First().Current().Name(), outputTensor)); + + // evaluate + winrt::hstring correlationId; + WINML_EXPECT_NO_THROW(session.EvaluateAsync(binding, correlationId).get()); + + // verify result + SoftwareBitmap bm_softwareBitmap = FileHelpers::GetSoftwareBitmapFromFile(bmImagePath); + bm_softwareBitmap = SoftwareBitmap::Convert(bm_softwareBitmap, BitmapPixelFormat::Bgra8); + VideoFrame bm_videoFrame = VideoFrame::CreateWithSoftwareBitmap(bm_softwareBitmap); + ImageFeatureValue bm_imagevalue = ImageFeatureValue::CreateFromVideoFrame(bm_videoFrame); + WINML_EXPECT_TRUE(VerifyHelper(bm_imagevalue, outputTensor)); + } } + } } - -TEST_F(ScenarioCppWinrtTest, EncryptedStream) -{ - // get a stream - std::wstring path = FileHelpers::GetModulePath() + L"model.onnx"; - auto storageFile = StorageFile::GetFileFromPathAsync(path).get(); - auto fileBuffer = winrt::Windows::Storage::FileIO::ReadBufferAsync(storageFile).get(); - - // encrypt - auto algorithmName = winrt::Windows::Security::Cryptography::Core::SymmetricAlgorithmNames::AesCbcPkcs7(); - auto algorithm = winrt::Windows::Security::Cryptography::Core::SymmetricKeyAlgorithmProvider::OpenAlgorithm(algorithmName); - uint32_t keyLength = 32; - auto keyBuffer = winrt::Windows::Security::Cryptography::CryptographicBuffer::GenerateRandom(keyLength); - auto key = algorithm.CreateSymmetricKey(keyBuffer); - auto iv = winrt::Windows::Security::Cryptography::CryptographicBuffer::GenerateRandom(algorithm.BlockLength()); - auto encryptedBuffer = winrt::Windows::Security::Cryptography::Core::CryptographicEngine::Encrypt(key, fileBuffer, iv); - - // verify loading the encrypted stream fails appropriately. - auto encryptedStream = InMemoryRandomAccessStream(); - encryptedStream.WriteAsync(encryptedBuffer).get(); - EXPECT_THROW_SPECIFIC(LearningModel::LoadFromStream(RandomAccessStreamReference::CreateFromStream(encryptedStream)), - winrt::hresult_error, - [](const winrt::hresult_error& e) -> bool - { - return e.code() == E_INVALIDARG; - }); - - // now decrypt - auto decryptedBuffer = winrt::Windows::Security::Cryptography::Core::CryptographicEngine::Decrypt(key, encryptedBuffer, iv); - auto decryptedStream = InMemoryRandomAccessStream(); - decryptedStream.WriteAsync(decryptedBuffer).get(); - - // load! - LearningModel model = nullptr; - EXPECT_NO_THROW(model = LearningModel::LoadFromStream(RandomAccessStreamReference::CreateFromStream(decryptedStream))); - LearningModelSession session = nullptr; - EXPECT_NO_THROW(session = LearningModelSession(model)); +static void EncryptedStream() { + // get a stream + std::wstring path = FileHelpers::GetModulePath() + L"model.onnx"; + auto storageFile = StorageFile::GetFileFromPathAsync(path).get(); + auto fileBuffer = winrt::Windows::Storage::FileIO::ReadBufferAsync(storageFile).get(); + + // encrypt + auto algorithmName = winrt::Windows::Security::Cryptography::Core::SymmetricAlgorithmNames::AesCbcPkcs7(); + auto algorithm = winrt::Windows::Security::Cryptography::Core::SymmetricKeyAlgorithmProvider::OpenAlgorithm(algorithmName); + uint32_t keyLength = 32; + auto keyBuffer = winrt::Windows::Security::Cryptography::CryptographicBuffer::GenerateRandom(keyLength); + auto key = algorithm.CreateSymmetricKey(keyBuffer); + auto iv = winrt::Windows::Security::Cryptography::CryptographicBuffer::GenerateRandom(algorithm.BlockLength()); + auto encryptedBuffer = winrt::Windows::Security::Cryptography::Core::CryptographicEngine::Encrypt(key, fileBuffer, iv); + + // verify loading the encrypted stream fails appropriately. + auto encryptedStream = InMemoryRandomAccessStream(); + encryptedStream.WriteAsync(encryptedBuffer).get(); + WINML_EXPECT_THROW_SPECIFIC(LearningModel::LoadFromStream(RandomAccessStreamReference::CreateFromStream(encryptedStream)), + winrt::hresult_error, + [](const winrt::hresult_error& e) -> bool { + return e.code() == E_INVALIDARG; + }); + + // now decrypt + auto decryptedBuffer = winrt::Windows::Security::Cryptography::Core::CryptographicEngine::Decrypt(key, encryptedBuffer, iv); + auto decryptedStream = InMemoryRandomAccessStream(); + decryptedStream.WriteAsync(decryptedBuffer).get(); + + // load! + LearningModel model = nullptr; + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromStream(RandomAccessStreamReference::CreateFromStream(decryptedStream))); + LearningModelSession session = nullptr; + WINML_EXPECT_NO_THROW(session = LearningModelSession(model)); } -void DeviceLostRecoveryHelper() -{ +static void DeviceLostRecovery() { // load a model std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; LearningModel model = LearningModel::LoadFromFilePath(filePath); @@ -1469,13 +1346,10 @@ void DeviceLostRecoveryHelper() } // evaluate should fail - try - { + try { session.Evaluate(binding, L""); - FAIL() << "Evaluate should fail after removing the device"; - } - catch (...) - { + WINML_LOG_ERROR("Evaluate should fail after removing the device"); + } catch (...) { } // remove all references to the device by reseting the session and binding. @@ -1483,67 +1357,97 @@ void DeviceLostRecoveryHelper() binding = nullptr; // create new session and binding and try again! - session = LearningModelSession(model, LearningModelDevice(LearningModelDeviceKind::DirectX)); - binding = LearningModelBinding(session); + WINML_EXPECT_NO_THROW(session = LearningModelSession(model, LearningModelDevice(LearningModelDeviceKind::DirectX))); + WINML_EXPECT_NO_THROW(binding = LearningModelBinding(session)); BindFeatures(binding, model.InputFeatures()); - session.Evaluate(binding, L""); - exit(0); + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); } -TEST_F(ScenarioCppWinrtGpuTestDeathTest, DeviceLostRecovery) { - - ::testing::FLAGS_gtest_death_test_style = "threadsafe"; - EXPECT_EXIT( - DeviceLostRecoveryHelper(), - ::testing::ExitedWithCode(0), - "" - ); +static void D2DInterop() { + // load a model (model.onnx == squeezenet[1,3,224,224]) + std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; + LearningModel model = LearningModel::LoadFromFilePath(filePath); + // create a dx12 device + com_ptr device = nullptr; + WINML_EXPECT_HRESULT_SUCCEEDED(D3D12CreateDevice(NULL, D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device1), device.put_void())); + // now create a command queue from it + com_ptr commandQueue = nullptr; + D3D12_COMMAND_QUEUE_DESC queueDesc = {}; + queueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; + WINML_EXPECT_HRESULT_SUCCEEDED(device->CreateCommandQueue(&queueDesc, winrt::guid_of(), commandQueue.put_void())); + // create a winml learning device based on that dx12 queue + auto factory = get_activation_factory(); + com_ptr<::IUnknown> spUnk; + WINML_EXPECT_HRESULT_SUCCEEDED(factory->CreateFromD3D12CommandQueue(commandQueue.get(), spUnk.put())); + auto learningDevice = spUnk.as(); + // create a winml session from that dx device + LearningModelSession session(model, learningDevice); + // now lets try and do some XAML/d2d on that same device, first prealloc a VideoFrame + VideoFrame frame = VideoFrame::CreateAsDirect3D11SurfaceBacked( + DirectXPixelFormat::B8G8R8A8UIntNormalized, + 224, + 224, + session.Device().Direct3D11Device()); + // create a D2D factory + D2D1_FACTORY_OPTIONS options = {}; + com_ptr d2dFactory; + WINML_EXPECT_HRESULT_SUCCEEDED(D2D1CreateFactory(D2D1_FACTORY_TYPE_SINGLE_THREADED, __uuidof(ID2D1Factory), &options, d2dFactory.put_void())); + // grab the dxgi surface back from our video frame + com_ptr dxgiSurface; + com_ptr dxgiInterfaceAccess = frame.Direct3DSurface().as(); + WINML_EXPECT_HRESULT_SUCCEEDED(dxgiInterfaceAccess->GetInterface(__uuidof(IDXGISurface), dxgiSurface.put_void())); + // and try and use our surface to create a render targer + com_ptr renderTarget; + D2D1_RENDER_TARGET_PROPERTIES props = D2D1::RenderTargetProperties(); + props.pixelFormat = D2D1::PixelFormat( + DXGI_FORMAT_B8G8R8A8_UNORM, + D2D1_ALPHA_MODE_IGNORE); + WINML_EXPECT_HRESULT_SUCCEEDED(d2dFactory->CreateDxgiSurfaceRenderTarget( + dxgiSurface.get(), + props, + renderTarget.put())); } -TEST_F(ScenarioCppWinrtGpuSkipEdgeCoreTest, D2DInterop) -{ - // load a model (model.onnx == squeezenet[1,3,224,224]) - std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; - LearningModel model = LearningModel::LoadFromFilePath(filePath); - // create a dx12 device - com_ptr device = nullptr; - EXPECT_HRESULT_SUCCEEDED(D3D12CreateDevice(NULL, D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device1), device.put_void())); - // now create a command queue from it - com_ptr commandQueue = nullptr; - D3D12_COMMAND_QUEUE_DESC queueDesc = {}; - queueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; - EXPECT_HRESULT_SUCCEEDED(device->CreateCommandQueue(&queueDesc, winrt::guid_of(), commandQueue.put_void())); - // create a winml learning device based on that dx12 queue - auto factory = get_activation_factory(); - com_ptr<::IUnknown> spUnk; - EXPECT_HRESULT_SUCCEEDED(factory->CreateFromD3D12CommandQueue(commandQueue.get(), spUnk.put())); - auto learningDevice = spUnk.as(); - // create a winml session from that dx device - LearningModelSession session(model, learningDevice); - // now lets try and do some XAML/d2d on that same device, first prealloc a VideoFrame - VideoFrame frame = VideoFrame::CreateAsDirect3D11SurfaceBacked( - DirectXPixelFormat::B8G8R8A8UIntNormalized, - 224, - 224, - session.Device().Direct3D11Device() - ); - // create a D2D factory - D2D1_FACTORY_OPTIONS options = {}; - com_ptr d2dFactory; - EXPECT_HRESULT_SUCCEEDED(D2D1CreateFactory(D2D1_FACTORY_TYPE_SINGLE_THREADED, __uuidof(ID2D1Factory), &options, d2dFactory.put_void())); - // grab the dxgi surface back from our video frame - com_ptr dxgiSurface; - com_ptr dxgiInterfaceAccess = frame.Direct3DSurface().as(); - EXPECT_HRESULT_SUCCEEDED(dxgiInterfaceAccess->GetInterface(__uuidof(IDXGISurface), dxgiSurface.put_void())); - // and try and use our surface to create a render targer - com_ptr renderTarget; - D2D1_RENDER_TARGET_PROPERTIES props = D2D1::RenderTargetProperties(); - props.pixelFormat = D2D1::PixelFormat( - DXGI_FORMAT_B8G8R8A8_UNORM, - D2D1_ALPHA_MODE_IGNORE - ); - EXPECT_HRESULT_SUCCEEDED(d2dFactory->CreateDxgiSurfaceRenderTarget( - dxgiSurface.get(), - props, - renderTarget.put())); +const ScenarioTestApi& getapi() { + static constexpr ScenarioTestApi api = + { + ScenarioCppWinrtTestSetup, + ScenarioCppWinrtGpuTestSetup, + ScenarioCppWinrtGpuSkipEdgeCoreTestSetup, + Sample1, + Scenario1LoadBindEvalDefault, + Scenario2LoadModelFromStream, + Scenario5AsyncEval, + Scenario7EvalWithNoBind, + Scenario8SetDeviceSampleDefault, + Scenario8SetDeviceSampleCPU, + Scenario17DevDiagnostics, + Scenario22ImageBindingAsCPUTensor, + QuantizedModels, + EncryptedStream, + Scenario3SoftwareBitmapInputBinding, + Scenario6BindWithProperties, + Scenario8SetDeviceSampleDefaultDirectX, + Scenario8SetDeviceSampleMinPower, + Scenario8SetDeviceSampleMaxPerf, + Scenario8SetDeviceSampleMyCameraDevice, + Scenario8SetDeviceSampleCustomCommandQueue, + Scenario9LoadBindEvalInputTensorGPU, + Scenario13SingleModelOnCPUandGPU, + Scenario11FreeDimensionsTensor, + Scenario11FreeDimensionsImage, + Scenario14RunModelSwapchain, + Scenario20aLoadBindEvalCustomOperatorCPU, + Scenario20bLoadBindEvalReplacementCustomOperatorCPU, + Scenario21RunModel2ChainZ, + Scenario22ImageBindingAsGPUTensor, + MsftQuantizedModels, + SyncVsAsync, + CustomCommandQueueWithFence, + ReuseVideoFrame, + DeviceLostRecovery, + Scenario8SetDeviceSampleD3D11Device, + D2DInterop, + }; + return api; } diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.h b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.h new file mode 100644 index 0000000000000..f3d335eb3ba45 --- /dev/null +++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.h @@ -0,0 +1,85 @@ +#include "test.h" +struct ScenarioTestApi +{ + SetupTest ScenarioCppWinrtTestSetup; + SetupTest ScenarioCppWinrtGpuTestSetup; + SetupTest ScenarioCppWinrtGpuSkipEdgeCoreTestSetup; + VoidTest Sample1; + VoidTest Scenario1LoadBindEvalDefault; + VoidTest Scenario2LoadModelFromStream; + VoidTest Scenario5AsyncEval; + VoidTest Scenario7EvalWithNoBind; + VoidTest Scenario8SetDeviceSampleDefault; + VoidTest Scenario8SetDeviceSampleCPU; + VoidTest Scenario17DevDiagnostics; + VoidTest DISABLED_Scenario22ImageBindingAsCPUTensor; + VoidTest QuantizedModels; + VoidTest EncryptedStream; + VoidTest Scenario3SoftwareBitmapInputBinding; + VoidTest Scenario6BindWithProperties; + VoidTest Scenario8SetDeviceSampleDefaultDirectX; + VoidTest Scenario8SetDeviceSampleMinPower; + VoidTest Scenario8SetDeviceSampleMaxPerf; + VoidTest Scenario8SetDeviceSampleMyCameraDevice; + VoidTest Scenario8SetDeviceSampleCustomCommandQueue; + VoidTest DISABLED_Scenario9LoadBindEvalInputTensorGPU; + VoidTest Scenario13SingleModelOnCPUandGPU; + VoidTest Scenario11FreeDimensionsTensor; + VoidTest Scenario11FreeDimensionsImage; + VoidTest Scenario14RunModelSwapchain; + VoidTest Scenario20aLoadBindEvalCustomOperatorCPU; + VoidTest Scenario20bLoadBindEvalReplacementCustomOperatorCPU; + VoidTest DISABLED_Scenario21RunModel2ChainZ; + VoidTest DISABLED_Scenario22ImageBindingAsGPUTensor; + VoidTest MsftQuantizedModels; + VoidTest DISABLED_SyncVsAsync; + VoidTest DISABLED_CustomCommandQueueWithFence; + VoidTest DISABLED_ReuseVideoFrame; + VoidTest DeviceLostRecovery; + VoidTest Scenario8SetDeviceSampleD3D11Device; + VoidTest D2DInterop; +}; +const ScenarioTestApi& getapi(); + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(ScenarioCppWinrtTest, ScenarioCppWinrtTestSetup) +WINML_TEST(ScenarioCppWinrtTest, Sample1) +WINML_TEST(ScenarioCppWinrtTest, Scenario1LoadBindEvalDefault) +WINML_TEST(ScenarioCppWinrtTest, Scenario2LoadModelFromStream) +WINML_TEST(ScenarioCppWinrtTest, Scenario5AsyncEval) +WINML_TEST(ScenarioCppWinrtTest, Scenario7EvalWithNoBind) +WINML_TEST(ScenarioCppWinrtTest, Scenario8SetDeviceSampleDefault) +WINML_TEST(ScenarioCppWinrtTest, Scenario8SetDeviceSampleCPU) +WINML_TEST(ScenarioCppWinrtTest, Scenario17DevDiagnostics) +WINML_TEST(ScenarioCppWinrtTest, DISABLED_Scenario22ImageBindingAsCPUTensor) +WINML_TEST(ScenarioCppWinrtTest, QuantizedModels) +WINML_TEST(ScenarioCppWinrtTest, EncryptedStream) +WINML_TEST_CLASS_END() + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(ScenarioCppWinrtGpuTest, ScenarioCppWinrtGpuTestSetup) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario3SoftwareBitmapInputBinding) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario6BindWithProperties) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleDefaultDirectX) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleMinPower) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleMaxPerf) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario8SetDeviceSampleCustomCommandQueue) +WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_Scenario9LoadBindEvalInputTensorGPU) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario13SingleModelOnCPUandGPU) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario11FreeDimensionsTensor) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario11FreeDimensionsImage) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario14RunModelSwapchain) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario20aLoadBindEvalCustomOperatorCPU) +WINML_TEST(ScenarioCppWinrtGpuTest, Scenario20bLoadBindEvalReplacementCustomOperatorCPU) +WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_Scenario21RunModel2ChainZ) +WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_Scenario22ImageBindingAsGPUTensor) +WINML_TEST(ScenarioCppWinrtGpuTest, MsftQuantizedModels) +WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_SyncVsAsync) +WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_CustomCommandQueueWithFence) +WINML_TEST(ScenarioCppWinrtGpuTest, DISABLED_ReuseVideoFrame) +WINML_TEST(ScenarioCppWinrtGpuTest, DeviceLostRecovery) +WINML_TEST_CLASS_END() + +WINML_TEST_CLASS_BEGIN_WITH_SETUP(ScenarioCppWinrtGpuSkipEdgeCoreTest, ScenarioCppWinrtGpuSkipEdgeCoreTestSetup) +WINML_TEST(ScenarioCppWinrtGpuSkipEdgeCoreTest, Scenario8SetDeviceSampleMyCameraDevice) +WINML_TEST(ScenarioCppWinrtGpuSkipEdgeCoreTest, Scenario8SetDeviceSampleD3D11Device ) +WINML_TEST(ScenarioCppWinrtGpuSkipEdgeCoreTest, D2DInterop) +WINML_TEST_CLASS_END() \ No newline at end of file