diff --git a/CMake/itkExternal_Eigen3.cmake b/CMake/itkExternal_Eigen3.cmake index 2ab425e5d5c..563430069d4 100644 --- a/CMake/itkExternal_Eigen3.cmake +++ b/CMake/itkExternal_Eigen3.cmake @@ -3,6 +3,24 @@ include(ITK_CheckCCompilerFlag) set(_additional_external_project_args) +if(APPLE) + if(DEFINED CMAKE_APPLE_SILICON_PROCESSOR) + list(APPEND _additional_external_project_args + -DCMAKE_APPLE_SILICON_PROCESSOR:STRING=${CMAKE_APPLE_SILICON_PROCESSOR}) + endif() +endif() + +if(NOT CMAKE_CONFIGURATION_TYPES) + list(APPEND _additional_external_project_args + -DCMAKE_BUILD_TYPE:STRING=Release) +endif() + +if(CMAKE_SH AND CMAKE_VERSION VERSION_LESS "3.17.0") + # Setting CMAKE_SH is required when using "MinGW Makefiles" generator with CMake < 3.17 + # See https://github.com/InsightSoftwareConsortium/ITK/issues/66#issuecomment-424374973 + list(APPEND _additional_external_project_args + -DCMAKE_SH:PATH=${CMAKE_SH}) +endif() # Because the header-only nature of Eigen3, EIGEN_MPL2_ONLY definition could be leaked outside ITK. # This would wrongly enforce EIGEN_MPL2_ONLY to other libraries using Eigen. @@ -10,9 +28,6 @@ set(_additional_external_project_args) # to avoid introducing GPL code from Eigen3 internally in ITK. option(ITK_USE_EIGEN_MPL2_ONLY "Set compile definition EIGEN_MPL2_ONLY for ITKInternalEigen3." OFF) mark_as_advanced(ITK_USE_EIGEN_MPL2_ONLY) -if(ITK_USE_EIGEN_MPL2_ONLY) - list(APPEND _additional_external_project_args "-DITK_USE_EIGEN_MPL2_ONLY:BOOL=ON") -endif() if(ITK_USE_SYSTEM_EIGEN) find_package(Eigen3) @@ -36,33 +51,40 @@ else() # However, generators and c,cxx compilers have to be explicitly passed # for CMake configuration to work. # Configure Eigen - file(MAKE_DIRECTORY ${_eigen3_build_dir}) execute_process( COMMAND - ${CMAKE_COMMAND} ${_eigen3_source_dir} "-DCMAKE_APPLE_SILICON_PROCESSOR=${CMAKE_APPLE_SILICON_PROCESSOR}" - "-DCMAKE_INSTALL_PREFIX=${_eigen3_cmake_install_prefix}" - "-DCMAKE_INSTALL_INCLUDEDIR=${_eigen3_cmake_install_includedir}" - "-DCMAKE_INSTALL_DATADIR=${_eigen3_cmake_install_datadir}" "-DCMAKE_GENERATOR=${CMAKE_GENERATOR}" - "-DCMAKE_SH=${CMAKE_SH}" "-DCMAKE_GENERATOR_TOOLSET=${CMAKE_GENERATOR_TOOLSET}" "-DCMAKE_SH=${CMAKE_SH}" - "-DCMAKE_GENERATOR_PLATFORM=${CMAKE_GENERATOR_PLATFORM}" "-DCMAKE_GENERATOR_INSTANCE=${CMAKE_GENERATOR_INSTANCE}" - "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}" - "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}" - "-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}" "-DCMAKE_SYSTEM_VERSION=${CMAKE_SYSTEM_VERSION}" - ${_additional_external_project_args} - WORKING_DIRECTORY ${_eigen3_build_dir} + ${CMAKE_COMMAND} + -DCMAKE_SYSTEM_VERSION:STRING=${CMAKE_SYSTEM_VERSION} + -DCMAKE_GENERATOR:STRING=${CMAKE_GENERATOR} + -DCMAKE_GENERATOR_TOOLSET:STRING=${CMAKE_GENERATOR_TOOLSET} + -DCMAKE_GENERATOR_PLATFORM:STRING=${CMAKE_GENERATOR_PLATFORM} + -DCMAKE_GENERATOR_INSTANCE:STRING=${CMAKE_GENERATOR_INSTANCE} + -DCMAKE_C_COMPILER:PATH=${CMAKE_C_COMPILER} + -DCMAKE_C_FLAGS:STRING=${CMAKE_C_FLAGS} + -DCMAKE_CXX_COMPILER:PATH=${CMAKE_CXX_COMPILER} + -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS} + -DCMAKE_OSX_ARCHITECTURES:STRING=${CMAKE_OSX_ARCHITECTURES} + -DCMAKE_INSTALL_PREFIX:PATH=${_eigen3_cmake_install_prefix} + -DCMAKE_INSTALL_INCLUDEDIR:PATH=${_eigen3_cmake_install_includedir} + -DCMAKE_INSTALL_DATADIR:PATH=${_eigen3_cmake_install_datadir} + -DITK_USE_EIGEN_MPL2_ONLY:BOOL=${ITK_USE_EIGEN_MPL2_ONLY} + ${_additional_external_project_args} + -S ${_eigen3_source_dir} + -B ${_eigen3_build_dir} OUTPUT_VARIABLE ITKEigen3Config_STDOUT - ERROR_VARIABLE ITKEigen3Config_STDERR) + ERROR_VARIABLE ITKEigen3Config_STDERR + ) set(ITKInternalEigen3_DIR ${_eigen3_build_dir}) find_package(ITKInternalEigen3) if(NOT ITKInternalEigen3_FOUND) message( FATAL_ERROR - "ITKInternalEigen3 configuration faileed\nREPORT:\n${ITKEigen3Config_STDOUT}\n${ITKEigen3Config_STDERR}") + "ITKInternalEigen3 configuration failed\nREPORT:\n${ITKEigen3Config_STDOUT}\n${ITKEigen3Config_STDERR}") endif() install( CODE "execute_process( COMMAND - ${CMAKE_COMMAND} --build . --target install + ${CMAKE_COMMAND} --build . --config Release --target install WORKING_DIRECTORY ${_eigen3_build_dir} )" COMPONENT Development) diff --git a/Modules/ThirdParty/Eigen3/UpdateFromUpstream.sh b/Modules/ThirdParty/Eigen3/UpdateFromUpstream.sh index e6c37c161bc..938f543b617 100755 --- a/Modules/ThirdParty/Eigen3/UpdateFromUpstream.sh +++ b/Modules/ThirdParty/Eigen3/UpdateFromUpstream.sh @@ -7,10 +7,8 @@ shopt -s dotglob readonly name="Eigen3" readonly ownership="Eigen Upstream " readonly subtree="Modules/ThirdParty/Eigen3/src/itkeigen" -# readonly repo="https://gitlab.com/libeigen/eigen" -# readonly repo="https://gitlab.kitware.com/third-party/eigen.git" -readonly repo="https://gitlab.kitware.com/phcerdan/eigen.git" -readonly tag="for/itk-3.4" +readonly repo="https://github.com/InsightSoftwareConsortium/eigen" +readonly tag="for/itk-20231108-master-4d54c43d" readonly paths=" Eigen/Cholesky Eigen/CholmodSupport diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/CMakeLists.txt b/Modules/ThirdParty/Eigen3/src/itkeigen/CMakeLists.txt index cba78ca454f..bd610f07e1d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/CMakeLists.txt +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/CMakeLists.txt @@ -1,11 +1,40 @@ -# cmake_minimum_require must be the first command of the file # Same version of ITK cmake_minimum_required -cmake_minimum_required(VERSION 3.16.3) +cmake_minimum_required(VERSION 3.10.2) project(Eigen3) # ITK doesn't compile anything here, just generates targets for the INTERFACE library. if(FALSE) + +# NOTE Remove setting the policy once the minimum required CMake version is +# increased to at least 3.15. Retain enabling the export to package registry. +if (POLICY CMP0090) + # The export command does not populate package registry by default + cmake_policy (SET CMP0090 NEW) + + # Unless otherwise specified, always export to package registry to ensure + # backwards compatibility. + if (NOT DEFINED CMAKE_EXPORT_PACKAGE_REGISTRY) + set (CMAKE_EXPORT_PACKAGE_REGISTRY ON) + endif (NOT DEFINED CMAKE_EXPORT_PACKAGE_REGISTRY) +endif (POLICY CMP0090) + +# Remove this block after bumping CMake to v3.21.0 +# PROJECT_IS_TOP_LEVEL is defined then by default +if(CMAKE_VERSION VERSION_LESS 3.21.0) + if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(PROJECT_IS_TOP_LEVEL TRUE) + else() + set(PROJECT_IS_TOP_LEVEL FALSE) + endif() +endif() + +if(PROJECT_IS_TOP_LEVEL) + set(CMAKE_CXX_STANDARD 14 CACHE STRING "Default C++ standard") + set(CMAKE_CXX_STANDARD_REQUIRED ON CACHE BOOL "Require C++ standard") + set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "Allow C++ extensions") +endif() + # guard against in-source builds if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) @@ -26,7 +55,7 @@ endif() ############################################################################# -# retrieve version information # +# retrieve version information # ############################################################################# # automatically parse the version number @@ -64,10 +93,6 @@ include(CMakeDependentOption) set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) - -option(EIGEN_TEST_CXX11 "Enable testing with C++11 and C++11 features (e.g. Tensor module)." OFF) - - macro(ei_add_cxx_compiler_flag FLAG) string(REGEX REPLACE "-" "" SFLAG1 ${FLAG}) string(REGEX REPLACE "\\+" "p" SFLAG ${SFLAG1}) @@ -77,19 +102,19 @@ macro(ei_add_cxx_compiler_flag FLAG) endif() endmacro() -check_cxx_compiler_flag("-std=c++11" EIGEN_COMPILER_SUPPORT_CPP11) - -if(EIGEN_TEST_CXX11) - set(CMAKE_CXX_STANDARD 11) - set(CMAKE_CXX_EXTENSIONS OFF) - if(EIGEN_COMPILER_SUPPORT_CPP11) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +function(ei_maybe_separate_arguments variable mode args) + # Use separate_arguments if the input is a single string containing a space. + # Otherwise, if it is already a list or doesn't have a space, just propagate + # the original value. This is to better support multi-argument lists. + list(LENGTH args list_length) + if (${list_length} EQUAL 1) + string(FIND "${args}" " " has_space) + if (${has_space} GREATER -1) + separate_arguments(args ${mode} "${args}") + endif() endif() -else() - #set(CMAKE_CXX_STANDARD 03) - #set(CMAKE_CXX_EXTENSIONS OFF) - ei_add_cxx_compiler_flag("-std=c++03") -endif() + set(${variable} ${args} PARENT_SCOPE) +endfunction(ei_maybe_separate_arguments) # Determine if we should build shared libraries on this platform. get_cmake_property(EIGEN_BUILD_SHARED_LIBS TARGET_SUPPORTS_SHARED_LIBS) @@ -104,6 +129,11 @@ find_package(StandardMathLibrary) set(EIGEN_TEST_CUSTOM_LINKER_FLAGS "" CACHE STRING "Additional linker flags when linking unit tests.") set(EIGEN_TEST_CUSTOM_CXX_FLAGS "" CACHE STRING "Additional compiler flags when compiling unit tests.") +# Convert space-separated arguments into CMake lists for downstream consumption. +ei_maybe_separate_arguments(EIGEN_TEST_CUSTOM_LINKER_FLAGS NATIVE_COMMAND "${EIGEN_TEST_CUSTOM_LINKER_FLAGS}") +ei_maybe_separate_arguments(EIGEN_TEST_CUSTOM_CXX_FLAGS NATIVE_COMMAND "${EIGEN_TEST_CUSTOM_CXX_FLAGS}") + + set(EIGEN_STANDARD_LIBRARIES_TO_LINK_TO "") if(NOT STANDARD_MATH_LIBRARY_FOUND) @@ -112,13 +142,11 @@ if(NOT STANDARD_MATH_LIBRARY_FOUND) "Can't link to the standard math library. Please report to the Eigen developers, telling them about your platform.") else() - if(EIGEN_STANDARD_LIBRARIES_TO_LINK_TO) set(EIGEN_STANDARD_LIBRARIES_TO_LINK_TO "${EIGEN_STANDARD_LIBRARIES_TO_LINK_TO} ${STANDARD_MATH_LIBRARY}") else() set(EIGEN_STANDARD_LIBRARIES_TO_LINK_TO "${STANDARD_MATH_LIBRARY}") endif() - endif() if(EIGEN_STANDARD_LIBRARIES_TO_LINK_TO) @@ -128,6 +156,7 @@ else() endif() option(EIGEN_BUILD_BTL "Build benchmark suite" OFF) +option(EIGEN_BUILD_SPBENCH "Build sparse benchmark suite" OFF) # Disable pkgconfig only for native Windows builds if(NOT WIN32 OR NOT CMAKE_HOST_SYSTEM_NAME MATCHES Windows) @@ -186,18 +215,6 @@ if(NOT MSVC) ei_add_cxx_compiler_flag("-wd981") # disable ICC's "operands are evaluated in unspecified order" remark ei_add_cxx_compiler_flag("-wd2304") # disable ICC's "warning #2304: non-explicit constructor with single argument may cause implicit type conversion" produced by -Wnon-virtual-dtor - - # The -ansi flag must be added last, otherwise it is also used as a linker flag by check_cxx_compiler_flag making it fails - # Moreover we should not set both -strict-ansi and -ansi - check_cxx_compiler_flag("-strict-ansi" COMPILER_SUPPORT_STRICTANSI) - ei_add_cxx_compiler_flag("-Qunused-arguments") # disable clang warning: argument unused during compilation: '-ansi' - - if(COMPILER_SUPPORT_STRICTANSI) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -strict-ansi") - else() - ei_add_cxx_compiler_flag("-ansi") - endif() - if(ANDROID_NDK) ei_add_cxx_compiler_flag("-pie") ei_add_cxx_compiler_flag("-fPIE") @@ -265,6 +282,12 @@ if(NOT MSVC) message(STATUS "Enabling AVX512DQ in tests/examples") endif() + option(EIGEN_TEST_AVX512FP16 "Enable/Disable AVX512-FP16 in tests/examples" OFF) + if(EIGEN_TEST_AVX512FP16) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mfma -mavx512vl -mavx512fp16") + message(STATUS "Enabling AVX512-FP16 in tests/examples") + endif() + option(EIGEN_TEST_F16C "Enable/Disable F16C in tests/examples" OFF) if(EIGEN_TEST_F16C) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mf16c") @@ -366,11 +389,19 @@ else() endif() option(EIGEN_TEST_FMA "Enable/Disable FMA/AVX2 in tests/examples" OFF) - if(EIGEN_TEST_FMA AND NOT EIGEN_TEST_NEON) + option(EIGEN_TEST_AVX2 "Enable/Disable FMA/AVX2 in tests/examples" OFF) + if((EIGEN_TEST_FMA AND NOT EIGEN_TEST_NEON) OR EIGEN_TEST_AVX2) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2") message(STATUS "Enabling FMA/AVX2 in tests/examples") endif() + option(EIGEN_TEST_AVX512 "Enable/Disable AVX512 in tests/examples" OFF) + option(EIGEN_TEST_AVX512DQ "Enable/Disable AVX512DQ in tests/examples" OFF) + if(EIGEN_TEST_AVX512 OR EIGEN_TEST_AVX512DQ) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX512") + message(STATUS "Enabling AVX512 in tests/examples") + endif() + endif() option(EIGEN_TEST_NO_EXPLICIT_VECTORIZATION "Disable explicit vectorization in tests/examples" OFF) @@ -413,7 +444,8 @@ if(EIGEN_TEST_NO_EXCEPTIONS) message(STATUS "Disabling exceptions in tests/examples") endif() -set(EIGEN_CUDA_COMPUTE_ARCH 30 CACHE STRING "The CUDA compute architecture level to target when compiling CUDA code") +set(EIGEN_CUDA_CXX_FLAGS "" CACHE STRING "Additional flags to pass to the cuda compiler.") +set(EIGEN_CUDA_COMPUTE_ARCH 30 CACHE STRING "The CUDA compute architecture(s) to target when compiling CUDA code") include_directories(${CMAKE_CURRENT_SOURCE_DIR}) @@ -447,17 +479,6 @@ foreach(var INCLUDE_INSTALL_DIR CMAKEPACKAGE_INSTALL_DIR PKGCONFIG_INSTALL_DIR) endif() endforeach() -# similar to set_target_properties but append the property instead of overwriting it -macro(ei_add_target_property target prop value) - - get_target_property(previous ${target} ${prop}) - # if the property wasn't previously set, ${previous} is now "previous-NOTFOUND" which cmake allows catching with plain if() - if(NOT previous) - set(previous "") - endif() - set_target_properties(${target} PROPERTIES ${prop} "${previous} ${value}") -endmacro() - install(FILES signature_of_eigen3_matrix_library DESTINATION ${INCLUDE_INSTALL_DIR} COMPONENT Devel @@ -473,43 +494,49 @@ endif() install(DIRECTORY Eigen DESTINATION ${INCLUDE_INSTALL_DIR} COMPONENT Devel) -option(EIGEN_BUILD_DOC "Enable creation of Eigen documentation" ON) +# Avoid building docs if included from another project. +# Building documentation requires creating and running executables on the host +# platform. We shouldn't do this if cross-compiling. +if (${PROJECT_IS_TOP_LEVEL} AND NOT ${CMAKE_CROSSCOMPILING}) + set(EIGEN_BUILD_DOC_DEFAULT ON) +else() + set(EIGEN_BUILD_DOC_DEFAULT OFF) +endif() +option(EIGEN_BUILD_DOC "Enable creation of Eigen documentation" ${EIGEN_BUILD_DOC_DEFAULT}) if(EIGEN_BUILD_DOC) add_subdirectory(doc EXCLUDE_FROM_ALL) endif() - -option(BUILD_TESTING "Enable creation of Eigen tests." ON) -if(BUILD_TESTING) - include(EigenConfigureTesting) - - if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) - add_subdirectory(test) # can't do EXCLUDE_FROM_ALL here, breaks CTest - else() - add_subdirectory(test EXCLUDE_FROM_ALL) - endif() - - add_subdirectory(failtest) -endif() - -if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) - add_subdirectory(blas) - add_subdirectory(lapack) -else() - add_subdirectory(blas EXCLUDE_FROM_ALL) - add_subdirectory(lapack EXCLUDE_FROM_ALL) -endif() - # add SYCL option(EIGEN_TEST_SYCL "Add Sycl support." OFF) -option(EIGEN_SYCL_TRISYCL "Use the triSYCL Sycl implementation (ComputeCPP by default)." OFF) if(EIGEN_TEST_SYCL) + option(EIGEN_SYCL_DPCPP "Use the DPCPP Sycl implementation (DPCPP is default SYCL-Compiler)." ON) + option(EIGEN_SYCL_TRISYCL "Use the triSYCL Sycl implementation." OFF) + option(EIGEN_SYCL_ComputeCpp "Use the ComputeCPP Sycl implementation." OFF) + + # Building options + # https://developer.codeplay.com/products/computecpp/ce/2.11.0/guides/eigen-overview/options-for-building-eigen-sycl + option(EIGEN_SYCL_USE_DEFAULT_SELECTOR "Use sycl default selector to select the preferred device." OFF) + option(EIGEN_SYCL_NO_LOCAL_MEM "Build for devices without dedicated shared memory." OFF) + option(EIGEN_SYCL_LOCAL_MEM "Allow the use of local memory (enabled by default)." ON) + option(EIGEN_SYCL_LOCAL_THREAD_DIM0 "Set work group size for dimension 0." 16) + option(EIGEN_SYCL_LOCAL_THREAD_DIM1 "Set work group size for dimension 1." 16) + option(EIGEN_SYCL_ASYNC_EXECUTION "Allow asynchronous execution (enabled by default)." ON) + option(EIGEN_SYCL_DISABLE_SKINNY "Disable optimization for tall/skinny matrices." OFF) + option(EIGEN_SYCL_DISABLE_DOUBLE_BUFFER "Disable double buffer." OFF) + option(EIGEN_SYCL_DISABLE_SCALAR "Disable scalar contraction." OFF) + option(EIGEN_SYCL_DISABLE_GEMV "Disable GEMV and create a single kernel to calculate contraction instead." OFF) + + set(EIGEN_SYCL ON) + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations -Wno-shorten-64-to-32 -Wno-cast-align") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-copy-with-user-provided-copy -Wno-unused-variable") set (CMAKE_MODULE_PATH "${CMAKE_ROOT}/Modules" "cmake/Modules/" "${CMAKE_MODULE_PATH}") find_package(Threads REQUIRED) if(EIGEN_SYCL_TRISYCL) message(STATUS "Using triSYCL") include(FindTriSYCL) - else() + elseif(EIGEN_SYCL_ComputeCpp) message(STATUS "Using ComputeCPP SYCL") include(FindComputeCpp) set(COMPUTECPP_DRIVER_DEFAULT_VALUE OFF) @@ -517,11 +544,15 @@ if(EIGEN_TEST_SYCL) set(COMPUTECPP_DRIVER_DEFAULT_VALUE ON) endif() option(COMPUTECPP_USE_COMPILER_DRIVER - "Use ComputeCpp driver instead of a 2 steps compilation" - ${COMPUTECPP_DRIVER_DEFAULT_VALUE} - ) + "Use ComputeCpp driver instead of a 2 steps compilation" + ${COMPUTECPP_DRIVER_DEFAULT_VALUE} + ) + else() #Default SYCL compiler is DPCPP (EIGEN_SYCL_DPCPP) + set(DPCPP_SYCL_TARGET "spir64" CACHE STRING "Default target for Intel CPU/GPU") + message(STATUS "Using DPCPP") + find_package(DPCPP) + add_definitions(-DSYCL_COMPILER_IS_DPCPP) endif(EIGEN_SYCL_TRISYCL) - option(EIGEN_DONT_VECTORIZE_SYCL "Don't use vectorisation in the SYCL tests." OFF) if(EIGEN_DONT_VECTORIZE_SYCL) message(STATUS "Disabling SYCL vectorization in tests/examples") # When disabling SYCL vectorization, also disable Eigen default vectorization @@ -530,6 +561,31 @@ if(EIGEN_TEST_SYCL) endif() endif() +cmake_dependent_option(BUILD_TESTING "Enable creation of tests." ON "PROJECT_IS_TOP_LEVEL" OFF) +option(EIGEN_BUILD_TESTING "Enable creation of Eigen tests." ${BUILD_TESTING}) +if(EIGEN_BUILD_TESTING) + include(EigenConfigureTesting) + + if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) + add_subdirectory(test) # can't do EXCLUDE_FROM_ALL here, breaks CTest + else() + add_subdirectory(test EXCLUDE_FROM_ALL) + endif() + + add_subdirectory(failtest) +endif() + +include(CMakeDetermineFortranCompiler) +option(EIGEN_BUILD_BLAS "Toggles the building of the Eigen Blas library" ${CMAKE_Fortran_COMPILER}) +option(EIGEN_BUILD_LAPACK "Toggles the building of the included Eigen LAPACK library" ${CMAKE_Fortran_COMPILER}) +if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) + add_subdirectory(blas) + add_subdirectory(lapack) +else() + add_subdirectory(blas EXCLUDE_FROM_ALL) + add_subdirectory(lapack EXCLUDE_FROM_ALL) +endif() + add_subdirectory(unsupported) add_subdirectory(demos EXCLUDE_FROM_ALL) @@ -542,13 +598,30 @@ if(EIGEN_BUILD_BTL) add_subdirectory(bench/btl EXCLUDE_FROM_ALL) endif() -if(NOT WIN32) +find_package(CLANG_FORMAT 9 EXACT) +if(CLANG_FORMAT_FOUND) +set(FORMAT_SOURCES) +list(APPEND FORMAT_SUBDIRS blas bench demos "doc" Eigen include lapack scripts share unsupported test failtest) +foreach(DIR ${FORMAT_SUBDIRS}) + set(ABS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${DIR}) + file(GLOB_RECURSE ${DIR}_SOURCES ${ABS_DIR}/*.cc ${ABS_DIR}/*.h ${ABS_DIR}/*.cpp ${ABS_DIR}/*.hpp ${ABS_DIR}/*.c) + list(APPEND FORMAT_SOURCES ${${DIR}_SOURCES}) + endforeach() + file(GLOB FORMAT_SOURCES_WITHOUTENDING LIST_DIRECTORIES false ${CMAKE_CURRENT_SOURCE_DIR}/Eigen/* ${CMAKE_CURRENT_SOURCE_DIR}/Eigen/CXX11/* ${CMAKE_CURRENT_SOURCE_DIR}/unsupported/Eigen/* ${CMAKE_CURRENT_SOURCE_DIR}/unsupported/Eigen/CXX11/*) + list(FILTER FORMAT_SOURCES_WITHOUTENDING EXCLUDE REGEX ".*.txt$") + list (APPEND FORMAT_SOURCES ${FORMAT_SOURCES_WITHOUTENDING}) + add_custom_target(format + COMMAND ${CLANG_FORMAT_EXECUTABLE} -i -style=file ${FORMAT_SOURCES} + DEPENDS ${FORMAT_SOURCES}) +endif() + +if(NOT WIN32 AND EIGEN_BUILD_SPBENCH) add_subdirectory(bench/spbench EXCLUDE_FROM_ALL) endif() configure_file(scripts/cdashtesting.cmake.in cdashtesting.cmake @ONLY) -if(BUILD_TESTING) +if(EIGEN_BUILD_TESTING) ei_testing_print_summary() endif() @@ -556,49 +629,49 @@ message(STATUS "") message(STATUS "Configured Eigen ${EIGEN_VERSION_NUMBER}") message(STATUS "") -string(TOLOWER "${CMAKE_GENERATOR}" cmake_generator_tolower) -if(cmake_generator_tolower MATCHES "makefile") - message(STATUS "Available targets (use: make TARGET):") -else() - message(STATUS "Available targets (use: cmake --build . --target TARGET):") -endif() -message(STATUS "---------+--------------------------------------------------------------") -message(STATUS "Target | Description") -message(STATUS "---------+--------------------------------------------------------------") -message(STATUS "install | Install Eigen. Headers will be installed to:") -message(STATUS " | /") -message(STATUS " | Using the following values:") -message(STATUS " | CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}") -message(STATUS " | INCLUDE_INSTALL_DIR: ${INCLUDE_INSTALL_DIR}") -message(STATUS " | Change the install location of Eigen headers using:") -message(STATUS " | cmake . -DCMAKE_INSTALL_PREFIX=yourprefix") -message(STATUS " | Or:") -message(STATUS " | cmake . -DINCLUDE_INSTALL_DIR=yourdir") -message(STATUS "doc | Generate the API documentation, requires Doxygen & LaTeX") -if(BUILD_TESTING) - message(STATUS "check | Build and run the unit-tests. Read this page:") - message(STATUS " | http://eigen.tuxfamily.org/index.php?title=Tests") -endif() -message(STATUS "blas | Build BLAS library (not the same thing as Eigen)") -message(STATUS "uninstall| Remove files installed by the install target") -message(STATUS "---------+--------------------------------------------------------------") -message(STATUS "") - +if(PROJECT_IS_TOP_LEVEL) + string(TOLOWER "${CMAKE_GENERATOR}" cmake_generator_tolower) + if(cmake_generator_tolower MATCHES "makefile") + message(STATUS "Available targets (use: make TARGET):") + else() + message(STATUS "Available targets (use: cmake --build . --target TARGET):") + endif() + message(STATUS "---------+--------------------------------------------------------------") + message(STATUS "Target | Description") + message(STATUS "---------+--------------------------------------------------------------") + message(STATUS "install | Install Eigen. Headers will be installed to:") + message(STATUS " | /") + message(STATUS " | Using the following values:") + message(STATUS " | CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}") + message(STATUS " | INCLUDE_INSTALL_DIR: ${INCLUDE_INSTALL_DIR}") + message(STATUS " | Change the install location of Eigen headers using:") + message(STATUS " | cmake . -DCMAKE_INSTALL_PREFIX=yourprefix") + message(STATUS " | Or:") + message(STATUS " | cmake . -DINCLUDE_INSTALL_DIR=yourdir") + message(STATUS "doc | Generate the API documentation, requires Doxygen & LaTeX") + if(EIGEN_BUILD_TESTING) + message(STATUS "check | Build and run the unit-tests. Read this page:") + message(STATUS " | http://eigen.tuxfamily.org/index.php?title=Tests") + endif() + if(CLANG_FORMAT_FOUND) + message(STATUS "format | Formats the source code according to .clang-format file") + endif() + message(STATUS "blas | Build BLAS library (not the same thing as Eigen)") + message(STATUS "uninstall| Remove files installed by the install target") + message(STATUS "---------+--------------------------------------------------------------") + message(STATUS "") +endif() set ( EIGEN_VERSION_STRING ${EIGEN_VERSION_NUMBER} ) set ( EIGEN_VERSION_MAJOR ${EIGEN_WORLD_VERSION} ) set ( EIGEN_VERSION_MINOR ${EIGEN_MAJOR_VERSION} ) set ( EIGEN_VERSION_PATCH ${EIGEN_MINOR_VERSION} ) -set ( EIGEN_DEFINITIONS "") -set ( EIGEN_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/${INCLUDE_INSTALL_DIR}" ) -set ( EIGEN_ROOT_DIR ${CMAKE_INSTALL_PREFIX} ) include (CMakePackageConfigHelpers) # Imported target support add_library (eigen INTERFACE) add_library (Eigen3::Eigen ALIAS eigen) -target_compile_definitions (eigen INTERFACE ${EIGEN_DEFINITIONS}) target_include_directories (eigen INTERFACE $ $ @@ -609,23 +682,35 @@ set_target_properties (eigen PROPERTIES EXPORT_NAME Eigen) install (TARGETS eigen EXPORT Eigen3Targets) +option(EIGEN_BUILD_CMAKE_PACKAGE "Enables the creation of EigenConfig.cmake and related files" ON) +if(EIGEN_BUILD_CMAKE_PACKAGE) configure_package_config_file ( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Eigen3Config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake - PATH_VARS EIGEN_INCLUDE_DIR EIGEN_ROOT_DIR INSTALL_DESTINATION ${CMAKEPACKAGE_INSTALL_DIR} + NO_SET_AND_CHECK_MACRO # Eigen does not provide legacy style defines NO_CHECK_REQUIRED_COMPONENTS_MACRO # Eigen does not provide components ) -# Remove CMAKE_SIZEOF_VOID_P from Eigen3ConfigVersion.cmake since Eigen does -# not depend on architecture specific settings or libraries. More -# specifically, an Eigen3Config.cmake generated from a 64 bit target can be -# used for 32 bit targets as well (and vice versa). -set (_Eigen3_CMAKE_SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P}) -unset (CMAKE_SIZEOF_VOID_P) -write_basic_package_version_file (Eigen3ConfigVersion.cmake - VERSION ${EIGEN_VERSION_NUMBER} - COMPATIBILITY SameMajorVersion) -set (CMAKE_SIZEOF_VOID_P ${_Eigen3_CMAKE_SIZEOF_VOID_P}) + +# NOTE Remove the first code path once the minimum required CMake version is +# bumped to 3.14 or above. +if (CMAKE_VERSION VERSION_LESS 3.14) + # Remove CMAKE_SIZEOF_VOID_P from Eigen3ConfigVersion.cmake since Eigen does + # not depend on architecture specific settings or libraries. More + # specifically, an Eigen3Config.cmake generated from a 64 bit target can be + # used for 32 bit targets as well (and vice versa). + set (_Eigen3_CMAKE_SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P}) + unset (CMAKE_SIZEOF_VOID_P) + write_basic_package_version_file (Eigen3ConfigVersion.cmake + VERSION ${EIGEN_VERSION_NUMBER} + COMPATIBILITY SameMajorVersion) + set (CMAKE_SIZEOF_VOID_P ${_Eigen3_CMAKE_SIZEOF_VOID_P}) +else (CMAKE_VERSION VERSION_LESS 3.14) + write_basic_package_version_file (Eigen3ConfigVersion.cmake + VERSION ${EIGEN_VERSION_NUMBER} + COMPATIBILITY SameMajorVersion + ARCH_INDEPENDENT) +endif (CMAKE_VERSION VERSION_LESS 3.14) # The Eigen target will be located in the Eigen3 namespace. Other CMake # targets can refer to it using Eigen3::Eigen. @@ -636,14 +721,16 @@ export (PACKAGE Eigen3) install (EXPORT Eigen3Targets NAMESPACE Eigen3:: DESTINATION ${CMAKEPACKAGE_INSTALL_DIR}) -install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/UseEigen3.cmake - ${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/Eigen3ConfigVersion.cmake - DESTINATION ${CMAKEPACKAGE_INSTALL_DIR} ) +install (FILES ${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/Eigen3ConfigVersion.cmake + DESTINATION ${CMAKEPACKAGE_INSTALL_DIR}) # Add uninstall target -add_custom_target ( uninstall - COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/EigenUninstall.cmake) +if(NOT TARGET uninstall) + add_custom_target ( uninstall + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/EigenUninstall.cmake) +endif() +endif() if (EIGEN_SPLIT_TESTSUITE) ei_split_testsuite("${EIGEN_SPLIT_TESTSUITE}") @@ -758,12 +845,6 @@ if(ITK_USE_EIGEN_MPL2_ONLY) target_compile_definitions (eigen_internal INTERFACE "EIGEN_MPL2_ONLY") endif() -# Hack to disable warning. To remove when/if fixed upstream. -# See issue: https://gitlab.com/libeigen/eigen/-/issues/2416 -if(CMAKE_CXX_STANDARD GREATER_EQUAL 17 AND CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(eigen_internal INTERFACE "-Wno-used-but-marked-unused") -endif() - # BUILD: go one directory before to localize the headers: # #include # INSTALL: headers require pre-prend itkeigen/Eigen/X. diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/COPYING.MPL2 b/Modules/ThirdParty/Eigen3/src/itkeigen/COPYING.MPL2 index 14e2f777f6c..ee6256cdb62 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/COPYING.MPL2 +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/COPYING.MPL2 @@ -357,7 +357,7 @@ Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. + file, You can obtain one at https://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/COPYING.README b/Modules/ThirdParty/Eigen3/src/itkeigen/COPYING.README index de5b6321582..11af93ca790 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/COPYING.README +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/COPYING.README @@ -2,17 +2,5 @@ Eigen is primarily MPL2 licensed. See COPYING.MPL2 and these links: http://www.mozilla.org/MPL/2.0/ http://www.mozilla.org/MPL/2.0/FAQ.html -Some files contain third-party code under BSD or LGPL licenses, whence the other -COPYING.* files here. - -All the LGPL code is either LGPL 2.1-only, or LGPL 2.1-or-later. -For this reason, the COPYING.LGPL file contains the LGPL 2.1 text. - -If you want to guarantee that the Eigen code that you are #including is licensed -under the MPL2 and possibly more permissive licenses (like BSD), #define this -preprocessor symbol: - EIGEN_MPL2_ONLY -For example, with most compilers, you could add this to your project CXXFLAGS: - -DEIGEN_MPL2_ONLY -This will cause a compilation error to be generated if you #include any code that is -LGPL licensed. +Some files contain third-party code under BSD or other MPL2-compatible licenses, +whence the other COPYING.* files here. \ No newline at end of file diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Cholesky b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Cholesky index a318ceb797d..4211151d458 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Cholesky +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Cholesky @@ -29,16 +29,14 @@ * \endcode */ +// IWYU pragma: begin_exports #include "src/Cholesky/LLT.h" #include "src/Cholesky/LDLT.h" #ifdef EIGEN_USE_LAPACKE -#ifdef EIGEN_USE_MKL -#include "mkl_lapacke.h" -#else -#include "src/misc/lapacke.h" -#endif +#include "src/misc/lapacke_helpers.h" #include "src/Cholesky/LLT_LAPACKE.h" #endif +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/CholmodSupport b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/CholmodSupport index bed8924d31e..2ccb21143a4 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/CholmodSupport +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/CholmodSupport @@ -22,7 +22,7 @@ extern "C" { * This module provides an interface to the Cholmod library which is part of the suitesparse package. * It provides the two following main factorization classes: * - class CholmodSupernodalLLT: a supernodal LLT Cholesky factorization. - * - class CholmodDecomposiiton: a general L(D)LT Cholesky factorization with automatic or explicit runtime selection of the underlying factorization method (supernodal or simplicial). + * - class CholmodDecomposition: a general L(D)LT Cholesky factorization with automatic or explicit runtime selection of the underlying factorization method (supernodal or simplicial). * * For the sake of completeness, this module also propose the two following classes: * - class CholmodSimplicialLLT @@ -40,7 +40,9 @@ extern "C" { * */ +// IWYU pragma: begin_exports #include "src/CholmodSupport/CholmodSupport.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Core b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Core index 3c03519fe92..93d47890a38 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Core +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Core @@ -8,8 +8,8 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#ifndef EIGEN_CORE_H -#define EIGEN_CORE_H +#ifndef EIGEN_CORE_MODULE_H +#define EIGEN_CORE_MODULE_H // first thing Eigen does: stop the compiler from reporting useless warnings. #include "src/Core/util/DisableStupidWarnings.h" @@ -34,16 +34,16 @@ #include #endif -// Disable the ipa-cp-clone optimization flag with MinGW 6.x or newer (enabled by default with -O3) +// Disable the ipa-cp-clone optimization flag with MinGW 6.x or older (enabled by default with -O3) // See http://eigen.tuxfamily.org/bz/show_bug.cgi?id=556 for details. -#if EIGEN_COMP_MINGW && EIGEN_GNUC_AT_LEAST(4,6) && EIGEN_GNUC_AT_MOST(5,5) +#if EIGEN_COMP_MINGW && EIGEN_GNUC_STRICT_LESS_THAN(6,0,0) #pragma GCC optimize ("-fno-ipa-cp-clone") #endif // Prevent ICC from specializing std::complex operators that silently fail // on device. This allows us to use our own device-compatible specializations // instead. -#if defined(EIGEN_COMP_ICC) && defined(EIGEN_GPU_COMPILE_PHASE) \ +#if EIGEN_COMP_ICC && defined(EIGEN_GPU_COMPILE_PHASE) \ && !defined(_OVERRIDE_COMPLEX_SPECIALIZATION_) #define _OVERRIDE_COMPLEX_SPECIALIZATION_ 1 #endif @@ -67,6 +67,7 @@ #endif #ifdef EIGEN_HAS_OPENMP +#include #include #endif @@ -81,10 +82,9 @@ #include #include #include -#include #include -#include #ifndef EIGEN_NO_IO + #include #include #endif #include @@ -94,14 +94,11 @@ // for min/max: #include -#if EIGEN_HAS_CXX11 #include -#endif +#include // for std::is_nothrow_move_assignable -#ifdef EIGEN_INCLUDE_TYPE_TRAITS #include -#endif // for outputting debug info #ifdef EIGEN_DEBUG_ASSIGN @@ -159,14 +156,27 @@ using std::ptrdiff_t; * \endcode */ +#ifdef EIGEN_USE_LAPACKE + #ifdef EIGEN_USE_MKL + #include "mkl_lapacke.h" + #else + #include "src/misc/lapacke.h" + #endif +#endif + +// IWYU pragma: begin_exports #include "src/Core/util/Constants.h" #include "src/Core/util/Meta.h" +#include "src/Core/util/Assert.h" #include "src/Core/util/ForwardDeclarations.h" #include "src/Core/util/StaticAssert.h" #include "src/Core/util/XprHelper.h" #include "src/Core/util/Memory.h" #include "src/Core/util/IntegralConstant.h" +#include "src/Core/util/Serializer.h" #include "src/Core/util/SymbolicIndex.h" +#include "src/Core/util/EmulateArray.h" +#include "src/Core/util/MoreMeta.h" #include "src/Core/NumTraits.h" #include "src/Core/MathFunctions.h" @@ -176,10 +186,12 @@ using std::ptrdiff_t; // Generic half float support #include "src/Core/arch/Default/Half.h" #include "src/Core/arch/Default/BFloat16.h" -#include "src/Core/arch/Default/TypeCasting.h" #include "src/Core/arch/Default/GenericPacketMathFunctionsFwd.h" #if defined EIGEN_VECTORIZE_AVX512 + #if defined EIGEN_VECTORIZE_AVX512FP16 + #include "src/Core/arch/AVX512/PacketMathFP16.h" + #endif #include "src/Core/arch/SSE/PacketMath.h" #include "src/Core/arch/SSE/TypeCasting.h" #include "src/Core/arch/SSE/Complex.h" @@ -192,6 +204,7 @@ using std::ptrdiff_t; #include "src/Core/arch/SSE/MathFunctions.h" #include "src/Core/arch/AVX/MathFunctions.h" #include "src/Core/arch/AVX512/MathFunctions.h" + #include "src/Core/arch/AVX512/TrsmKernel.h" #elif defined EIGEN_VECTORIZE_AVX // Use AVX for floats and doubles, SSE for integers #include "src/Core/arch/SSE/PacketMath.h" @@ -209,6 +222,7 @@ using std::ptrdiff_t; #include "src/Core/arch/SSE/Complex.h" #elif defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) #include "src/Core/arch/AltiVec/PacketMath.h" + #include "src/Core/arch/AltiVec/TypeCasting.h" #include "src/Core/arch/AltiVec/MathFunctions.h" #include "src/Core/arch/AltiVec/Complex.h" #elif defined EIGEN_VECTORIZE_NEON @@ -228,6 +242,8 @@ using std::ptrdiff_t; #include "src/Core/arch/MSA/PacketMath.h" #include "src/Core/arch/MSA/MathFunctions.h" #include "src/Core/arch/MSA/Complex.h" +#elif defined EIGEN_VECTORIZE_HVX + #include "src/Core/arch/HVX/PacketMath.h" #endif #if defined EIGEN_VECTORIZE_GPU @@ -237,7 +253,6 @@ using std::ptrdiff_t; #endif #if defined(EIGEN_USE_SYCL) - #include "src/Core/arch/SYCL/SyclMemoryModel.h" #include "src/Core/arch/SYCL/InteropHeaders.h" #if !defined(EIGEN_DONT_VECTORIZE_SYCL) #include "src/Core/arch/SYCL/PacketMath.h" @@ -257,10 +272,14 @@ using std::ptrdiff_t; #include "src/Core/functors/StlFunctors.h" #include "src/Core/functors/AssignmentFunctors.h" -// Specialized functors to enable the processing of complex numbers -// on CUDA devices -#ifdef EIGEN_CUDACC -#include "src/Core/arch/CUDA/Complex.h" +// Specialized functors for GPU. +#ifdef EIGEN_GPUCC +#include "src/Core/arch/GPU/Complex.h" +#endif + +// Specializations of vectorized activation functions for NEON. +#ifdef EIGEN_VECTORIZE_NEON +#include "src/Core/arch/NEON/UnaryFunctors.h" #endif #include "src/Core/util/IndexedViewHelper.h" @@ -315,6 +334,7 @@ using std::ptrdiff_t; #include "src/Core/DiagonalMatrix.h" #include "src/Core/Diagonal.h" #include "src/Core/DiagonalProduct.h" +#include "src/Core/SkewSymmetricMatrix3.h" #include "src/Core/Redux.h" #include "src/Core/Visitor.h" #include "src/Core/Fuzzy.h" @@ -347,13 +367,20 @@ using std::ptrdiff_t; #include "src/Core/CoreIterators.h" #include "src/Core/ConditionEstimator.h" -#if defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) +#if defined(EIGEN_VECTORIZE_VSX) #include "src/Core/arch/AltiVec/MatrixProduct.h" #elif defined EIGEN_VECTORIZE_NEON #include "src/Core/arch/NEON/GeneralBlockPanelKernel.h" #endif -#include "src/Core/BooleanRedux.h" +#if defined(EIGEN_VECTORIZE_AVX512) + #include "src/Core/arch/AVX512/GemmKernel.h" +#endif + +#if defined(EIGEN_VECTORIZE_HVX) + #include "src/Core/arch/HVX/GeneralBlockPanelKernel.h" +#endif + #include "src/Core/Select.h" #include "src/Core/VectorwiseOp.h" #include "src/Core/PartialReduxEvaluator.h" @@ -379,7 +406,8 @@ using std::ptrdiff_t; #endif #include "src/Core/GlobalFunctions.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" -#endif // EIGEN_CORE_H +#endif // EIGEN_CORE_MODULE_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Eigenvalues b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Eigenvalues index 5467a2e7b3e..97c526d91b3 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Eigenvalues +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Eigenvalues @@ -33,6 +33,8 @@ */ #include "src/misc/RealSvd2x2.h" + +// IWYU pragma: begin_exports #include "src/Eigenvalues/Tridiagonalization.h" #include "src/Eigenvalues/RealSchur.h" #include "src/Eigenvalues/EigenSolver.h" @@ -54,6 +56,7 @@ #include "src/Eigenvalues/ComplexSchur_LAPACKE.h" #include "src/Eigenvalues/SelfAdjointEigenSolver_LAPACKE.h" #endif +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Geometry b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Geometry index bc78110a846..0a733c5ecbc 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Geometry +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Geometry @@ -33,9 +33,9 @@ * \endcode */ +// IWYU pragma: begin_exports #include "src/Geometry/OrthoMethods.h" #include "src/Geometry/EulerAngles.h" - #include "src/Geometry/Homogeneous.h" #include "src/Geometry/RotationBase.h" #include "src/Geometry/Rotation2D.h" @@ -53,6 +53,7 @@ #if (defined EIGEN_VECTORIZE_SSE) || (defined EIGEN_VECTORIZE_NEON) #include "src/Geometry/arch/Geometry_SIMD.h" #endif +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Householder b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Householder index f2fa79969c9..0f7f9b3d33e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Householder +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Householder @@ -20,9 +20,11 @@ * \endcode */ +// IWYU pragma: begin_exports #include "src/Householder/Householder.h" #include "src/Householder/HouseholderSequence.h" #include "src/Householder/BlockHouseholder.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/IterativeLinearSolvers b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/IterativeLinearSolvers index 957d5750b2c..833189b132d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/IterativeLinearSolvers +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/IterativeLinearSolvers @@ -27,13 +27,14 @@ * - DiagonalPreconditioner - also called Jacobi preconditioner, work very well on diagonal dominant matrices. * - IncompleteLUT - incomplete LU factorization with dual thresholding * - * Such problems can also be solved using the direct sparse decomposition modules: SparseCholesky, CholmodSupport, UmfPackSupport, SuperLUSupport. + * Such problems can also be solved using the direct sparse decomposition modules: SparseCholesky, CholmodSupport, UmfPackSupport, SuperLUSupport, AccelerateSupport. * \code #include \endcode */ +// IWYU pragma: begin_exports #include "src/IterativeLinearSolvers/SolveWithGuess.h" #include "src/IterativeLinearSolvers/IterativeSolverBase.h" #include "src/IterativeLinearSolvers/BasicPreconditioners.h" @@ -42,6 +43,7 @@ #include "src/IterativeLinearSolvers/BiCGSTAB.h" #include "src/IterativeLinearSolvers/IncompleteLUT.h" #include "src/IterativeLinearSolvers/IncompleteCholesky.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Jacobi b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Jacobi index 43edc7a1946..490a6a8c57e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Jacobi +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/Jacobi @@ -24,7 +24,9 @@ * - MatrixBase::applyOnTheRight(). */ +// IWYU pragma: begin_exports #include "src/Jacobi/Jacobi.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/LU b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/LU index 1236ceb0467..49d7b91fe05 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/LU +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/LU @@ -25,14 +25,12 @@ #include "src/misc/Kernel.h" #include "src/misc/Image.h" + +// IWYU pragma: begin_exports #include "src/LU/FullPivLU.h" #include "src/LU/PartialPivLU.h" #ifdef EIGEN_USE_LAPACKE -#ifdef EIGEN_USE_MKL -#include "mkl_lapacke.h" -#else -#include "src/misc/lapacke.h" -#endif +#include "src/misc/lapacke_helpers.h" #include "src/LU/PartialPivLU_LAPACKE.h" #endif #include "src/LU/Determinant.h" @@ -41,6 +39,7 @@ #if defined EIGEN_VECTORIZE_SSE || defined EIGEN_VECTORIZE_NEON #include "src/LU/arch/InverseSize4.h" #endif +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/MetisSupport b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/MetisSupport index 85c41bf3400..e74c3bb6ad7 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/MetisSupport +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/MetisSupport @@ -27,8 +27,9 @@ extern "C" { * It can be used just as any other built-in method as explained in \link OrderingMethods_Module here. \endlink */ - +// IWYU pragma: begin_exports #include "src/MetisSupport/MetisSupport.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/OrderingMethods b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/OrderingMethods index 29691a62b44..5031de7a520 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/OrderingMethods +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/OrderingMethods @@ -63,8 +63,11 @@ * \endcode */ +// IWYU pragma: begin_exports #include "src/OrderingMethods/Amd.h" #include "src/OrderingMethods/Ordering.h" +// IWYU pragma: end_exports + #include "src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_ORDERINGMETHODS_MODULE_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/PaStiXSupport b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/PaStiXSupport index 234619accee..5d3cb38fdf3 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/PaStiXSupport +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/PaStiXSupport @@ -42,7 +42,9 @@ extern "C" { * */ +// IWYU pragma: begin_exports #include "src/PaStiXSupport/PaStiXSupport.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/PardisoSupport b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/PardisoSupport index 340edf51fe2..c022ce1fb3b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/PardisoSupport +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/PardisoSupport @@ -28,7 +28,9 @@ * */ +// IWYU pragma: begin_exports #include "src/PardisoSupport/PardisoSupport.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/QR b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/QR index 8465b62ceee..2f57a602bc9 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/QR +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/QR @@ -31,19 +31,17 @@ * \endcode */ +// IWYU pragma: begin_exports #include "src/QR/HouseholderQR.h" #include "src/QR/FullPivHouseholderQR.h" #include "src/QR/ColPivHouseholderQR.h" #include "src/QR/CompleteOrthogonalDecomposition.h" #ifdef EIGEN_USE_LAPACKE -#ifdef EIGEN_USE_MKL -#include "mkl_lapacke.h" -#else -#include "src/misc/lapacke.h" -#endif +#include "src/misc/lapacke_helpers.h" #include "src/QR/HouseholderQR_LAPACKE.h" #include "src/QR/ColPivHouseholderQR_LAPACKE.h" #endif +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SPQRSupport b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SPQRSupport index f70390c1766..a25f5ef9f01 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SPQRSupport +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SPQRSupport @@ -28,7 +28,12 @@ * */ -#include "src/CholmodSupport/CholmodSupport.h" +#include "CholmodSupport" + +// IWYU pragma: begin_exports #include "src/SPQRSupport/SuiteSparseQRSupport.h" +// IWYU pragma: end_exports + +#include "src/Core/util/ReenableStupidWarnings.h" #endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SVD b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SVD index 34517949632..b13ec2fb1ca 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SVD +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SVD @@ -31,19 +31,24 @@ * \endcode */ +// IWYU pragma: begin_exports #include "src/misc/RealSvd2x2.h" #include "src/SVD/UpperBidiagonalization.h" #include "src/SVD/SVDBase.h" #include "src/SVD/JacobiSVD.h" #include "src/SVD/BDCSVD.h" -#if defined(EIGEN_USE_LAPACKE) && !defined(EIGEN_USE_LAPACKE_STRICT) +#ifdef EIGEN_USE_LAPACKE #ifdef EIGEN_USE_MKL #include "mkl_lapacke.h" #else #include "src/misc/lapacke.h" #endif +#ifndef EIGEN_USE_LAPACKE_STRICT #include "src/SVD/JacobiSVD_LAPACKE.h" #endif +#include "src/SVD/BDCSVD_LAPACKE.h" +#endif +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseCholesky b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseCholesky index d2b1f1276da..967196f049e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseCholesky +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseCholesky @@ -30,8 +30,11 @@ * \endcode */ +// IWYU pragma: begin_exports #include "src/SparseCholesky/SimplicialCholesky.h" #include "src/SparseCholesky/SimplicialCholesky_impl.h" +// IWYU pragma: end_exports + #include "src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_SPARSECHOLESKY_MODULE_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseCore b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseCore index 76966c4c4cb..292b0891029 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseCore +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseCore @@ -17,6 +17,7 @@ #include #include #include +#include /** * \defgroup SparseCore_Module SparseCore module @@ -33,6 +34,7 @@ * This module depends on: Core. */ +// IWYU pragma: begin_exports #include "src/SparseCore/SparseUtil.h" #include "src/SparseCore/SparseMatrixBase.h" #include "src/SparseCore/SparseAssign.h" @@ -41,7 +43,6 @@ #include "src/SparseCore/SparseCompressedBase.h" #include "src/SparseCore/SparseMatrix.h" #include "src/SparseCore/SparseMap.h" -#include "src/SparseCore/MappedSparseMatrix.h" #include "src/SparseCore/SparseVector.h" #include "src/SparseCore/SparseRef.h" #include "src/SparseCore/SparseCwiseUnaryOp.h" @@ -62,6 +63,7 @@ #include "src/SparseCore/SparsePermutation.h" #include "src/SparseCore/SparseFuzzy.h" #include "src/SparseCore/SparseSolverBase.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseLU b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseLU index 37c4a5c5a8b..9ec6ac73571 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseLU +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseLU @@ -25,8 +25,7 @@ #include "src/Core/util/DisableStupidWarnings.h" -#include "src/SparseLU/SparseLU_gemm_kernel.h" - +// IWYU pragma: begin_exports #include "src/SparseLU/SparseLU_Structs.h" #include "src/SparseLU/SparseLU_SupernodalMatrix.h" #include "src/SparseLU/SparseLUImpl.h" @@ -44,6 +43,7 @@ #include "src/SparseLU/SparseLU_pruneL.h" #include "src/SparseLU/SparseLU_Utils.h" #include "src/SparseLU/SparseLU.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseQR b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseQR index f5fc5fa7fee..4bc144ccdae 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseQR +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SparseQR @@ -28,8 +28,10 @@ * */ +// IWYU pragma: begin_exports #include "src/SparseCore/SparseColEtree.h" #include "src/SparseQR/SparseQR.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdDeque b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdDeque index bc68397be25..29550e2a72c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdDeque +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdDeque @@ -20,7 +20,9 @@ #else +// IWYU pragma: begin_exports #include "src/StlSupport/StdDeque.h" +// IWYU pragma: end_exports #endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdList b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdList index 4c6262c08cc..8a22fc8f106 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdList +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdList @@ -19,7 +19,9 @@ #else +// IWYU pragma: begin_exports #include "src/StlSupport/StdList.h" +// IWYU pragma: end_exports #endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdVector b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdVector index 0c4697ad5be..e68f6b5b8ac 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdVector +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/StdVector @@ -20,7 +20,9 @@ #else +// IWYU pragma: begin_exports #include "src/StlSupport/StdVector.h" +// IWYU pragma: end_exports #endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SuperLUSupport b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SuperLUSupport index 59312a82db0..33997fa5036 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SuperLUSupport +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/SuperLUSupport @@ -57,7 +57,9 @@ namespace Eigen { struct SluMatrix; } * */ +// IWYU pragma: begin_exports #include "src/SuperLUSupport/SuperLUSupport.h" +// IWYU pragma: end_exports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/UmfPackSupport b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/UmfPackSupport index 00eec80875f..eea150eedcb 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/UmfPackSupport +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/UmfPackSupport @@ -33,7 +33,9 @@ extern "C" { * */ +// IWYU pragma: begin_exports #include "src/UmfPackSupport/UmfPackSupport.h" +// IWYU pragma: endexports #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/AccelerateSupport/AccelerateSupport.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/AccelerateSupport/AccelerateSupport.h new file mode 100644 index 00000000000..0417688b0c0 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/AccelerateSupport/AccelerateSupport.h @@ -0,0 +1,421 @@ +#ifndef EIGEN_ACCELERATESUPPORT_H +#define EIGEN_ACCELERATESUPPORT_H + +#include + +#include + +namespace Eigen { + +template +class AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \class AccelerateLLT + * \brief A direct Cholesky (LLT) factorization and solver based on Accelerate + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ additional information about the matrix structure. Default is Lower. + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateLLT + */ +template +using AccelerateLLT = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \class AccelerateLDLT + * \brief The default Cholesky (LDLT) factorization and solver based on Accelerate + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ additional information about the matrix structure. Default is Lower. + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateLDLT + */ +template +using AccelerateLDLT = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \class AccelerateLDLTUnpivoted + * \brief A direct Cholesky-like LDL^T factorization and solver based on Accelerate with only 1x1 pivots and no pivoting + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ additional information about the matrix structure. Default is Lower. + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateLDLTUnpivoted + */ +template +using AccelerateLDLTUnpivoted = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \class AccelerateLDLTSBK + * \brief A direct Cholesky (LDLT) factorization and solver based on Accelerate with Supernode Bunch-Kaufman and static pivoting + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ additional information about the matrix structure. Default is Lower. + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateLDLTSBK + */ +template +using AccelerateLDLTSBK = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \class AccelerateLDLTTPP + * \brief A direct Cholesky (LDLT) factorization and solver based on Accelerate with full threshold partial pivoting + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ additional information about the matrix structure. Default is Lower. + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateLDLTTPP + */ +template +using AccelerateLDLTTPP = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \class AccelerateQR + * \brief A QR factorization and solver based on Accelerate + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateQR + */ +template +using AccelerateQR = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \class AccelerateCholeskyAtA + * \brief A QR factorization and solver based on Accelerate without storing Q (equivalent to A^TA = R^T R) + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateCholeskyAtA + */ +template +using AccelerateCholeskyAtA = AccelerateImpl; + +namespace internal { +template +struct AccelFactorizationDeleter { + void operator()(T* sym) { + if (sym) { + SparseCleanup(*sym); + delete sym; + sym = nullptr; + } + } +}; + +template +struct SparseTypesTraitBase { + typedef DenseVecT AccelDenseVector; + typedef DenseMatT AccelDenseMatrix; + typedef SparseMatT AccelSparseMatrix; + + typedef SparseOpaqueSymbolicFactorization SymbolicFactorization; + typedef NumFactT NumericFactorization; + + typedef AccelFactorizationDeleter SymbolicFactorizationDeleter; + typedef AccelFactorizationDeleter NumericFactorizationDeleter; +}; + +template +struct SparseTypesTrait {}; + +template <> +struct SparseTypesTrait : SparseTypesTraitBase {}; + +template <> +struct SparseTypesTrait + : SparseTypesTraitBase { +}; + +} // end namespace internal + +template +class AccelerateImpl : public SparseSolverBase > { + protected: + using Base = SparseSolverBase; + using Base::derived; + using Base::m_isInitialized; + + public: + using Base::_solve_impl; + + typedef MatrixType_ MatrixType; + typedef typename MatrixType::Scalar Scalar; + typedef typename MatrixType::StorageIndex StorageIndex; + enum { ColsAtCompileTime = Dynamic, MaxColsAtCompileTime = Dynamic }; + enum { UpLo = UpLo_ }; + + using AccelDenseVector = typename internal::SparseTypesTrait::AccelDenseVector; + using AccelDenseMatrix = typename internal::SparseTypesTrait::AccelDenseMatrix; + using AccelSparseMatrix = typename internal::SparseTypesTrait::AccelSparseMatrix; + using SymbolicFactorization = typename internal::SparseTypesTrait::SymbolicFactorization; + using NumericFactorization = typename internal::SparseTypesTrait::NumericFactorization; + using SymbolicFactorizationDeleter = typename internal::SparseTypesTrait::SymbolicFactorizationDeleter; + using NumericFactorizationDeleter = typename internal::SparseTypesTrait::NumericFactorizationDeleter; + + AccelerateImpl() { + m_isInitialized = false; + + auto check_flag_set = [](int value, int flag) { return ((value & flag) == flag); }; + + if (check_flag_set(UpLo_, Symmetric)) { + m_sparseKind = SparseSymmetric; + m_triType = (UpLo_ & Lower) ? SparseLowerTriangle : SparseUpperTriangle; + } else if (check_flag_set(UpLo_, UnitLower)) { + m_sparseKind = SparseUnitTriangular; + m_triType = SparseLowerTriangle; + } else if (check_flag_set(UpLo_, UnitUpper)) { + m_sparseKind = SparseUnitTriangular; + m_triType = SparseUpperTriangle; + } else if (check_flag_set(UpLo_, StrictlyLower)) { + m_sparseKind = SparseTriangular; + m_triType = SparseLowerTriangle; + } else if (check_flag_set(UpLo_, StrictlyUpper)) { + m_sparseKind = SparseTriangular; + m_triType = SparseUpperTriangle; + } else if (check_flag_set(UpLo_, Lower)) { + m_sparseKind = SparseTriangular; + m_triType = SparseLowerTriangle; + } else if (check_flag_set(UpLo_, Upper)) { + m_sparseKind = SparseTriangular; + m_triType = SparseUpperTriangle; + } else { + m_sparseKind = SparseOrdinary; + m_triType = (UpLo_ & Lower) ? SparseLowerTriangle : SparseUpperTriangle; + } + + m_order = SparseOrderDefault; + } + + explicit AccelerateImpl(const MatrixType& matrix) : AccelerateImpl() { compute(matrix); } + + ~AccelerateImpl() {} + + inline Index cols() const { return m_nCols; } + inline Index rows() const { return m_nRows; } + + ComputationInfo info() const { + eigen_assert(m_isInitialized && "Decomposition is not initialized."); + return m_info; + } + + void analyzePattern(const MatrixType& matrix); + + void factorize(const MatrixType& matrix); + + void compute(const MatrixType& matrix); + + template + void _solve_impl(const MatrixBase& b, MatrixBase& dest) const; + + /** Sets the ordering algorithm to use. */ + void setOrder(SparseOrder_t order) { m_order = order; } + + private: + template + void buildAccelSparseMatrix(const SparseMatrix& a, AccelSparseMatrix& A, std::vector& columnStarts) { + const Index nColumnsStarts = a.cols() + 1; + + columnStarts.resize(nColumnsStarts); + + for (Index i = 0; i < nColumnsStarts; i++) columnStarts[i] = a.outerIndexPtr()[i]; + + SparseAttributes_t attributes{}; + attributes.transpose = false; + attributes.triangle = m_triType; + attributes.kind = m_sparseKind; + + SparseMatrixStructure structure{}; + structure.attributes = attributes; + structure.rowCount = static_cast(a.rows()); + structure.columnCount = static_cast(a.cols()); + structure.blockSize = 1; + structure.columnStarts = columnStarts.data(); + structure.rowIndices = const_cast(a.innerIndexPtr()); + + A.structure = structure; + A.data = const_cast(a.valuePtr()); + } + + void doAnalysis(AccelSparseMatrix& A) { + m_numericFactorization.reset(nullptr); + + SparseSymbolicFactorOptions opts{}; + opts.control = SparseDefaultControl; + opts.orderMethod = m_order; + opts.order = nullptr; + opts.ignoreRowsAndColumns = nullptr; + opts.malloc = malloc; + opts.free = free; + opts.reportError = nullptr; + + m_symbolicFactorization.reset(new SymbolicFactorization(SparseFactor(Solver_, A.structure, opts))); + + SparseStatus_t status = m_symbolicFactorization->status; + + updateInfoStatus(status); + + if (status != SparseStatusOK) m_symbolicFactorization.reset(nullptr); + } + + void doFactorization(AccelSparseMatrix& A) { + SparseStatus_t status = SparseStatusReleased; + + if (m_symbolicFactorization) { + m_numericFactorization.reset(new NumericFactorization(SparseFactor(*m_symbolicFactorization, A))); + + status = m_numericFactorization->status; + + if (status != SparseStatusOK) m_numericFactorization.reset(nullptr); + } + + updateInfoStatus(status); + } + + protected: + void updateInfoStatus(SparseStatus_t status) const { + switch (status) { + case SparseStatusOK: + m_info = Success; + break; + case SparseFactorizationFailed: + case SparseMatrixIsSingular: + m_info = NumericalIssue; + break; + case SparseInternalError: + case SparseParameterError: + case SparseStatusReleased: + default: + m_info = InvalidInput; + break; + } + } + + mutable ComputationInfo m_info; + Index m_nRows, m_nCols; + std::unique_ptr m_symbolicFactorization; + std::unique_ptr m_numericFactorization; + SparseKind_t m_sparseKind; + SparseTriangle_t m_triType; + SparseOrder_t m_order; +}; + +/** Computes the symbolic and numeric decomposition of matrix \a a */ +template +void AccelerateImpl::compute(const MatrixType& a) { + if (EnforceSquare_) eigen_assert(a.rows() == a.cols()); + + m_nRows = a.rows(); + m_nCols = a.cols(); + + AccelSparseMatrix A{}; + std::vector columnStarts; + + buildAccelSparseMatrix(a, A, columnStarts); + + doAnalysis(A); + + if (m_symbolicFactorization) doFactorization(A); + + m_isInitialized = true; +} + +/** Performs a symbolic decomposition on the sparsity pattern of matrix \a a. + * + * This function is particularly useful when solving for several problems having the same structure. + * + * \sa factorize() + */ +template +void AccelerateImpl::analyzePattern(const MatrixType& a) { + if (EnforceSquare_) eigen_assert(a.rows() == a.cols()); + + m_nRows = a.rows(); + m_nCols = a.cols(); + + AccelSparseMatrix A{}; + std::vector columnStarts; + + buildAccelSparseMatrix(a, A, columnStarts); + + doAnalysis(A); + + m_isInitialized = true; +} + +/** Performs a numeric decomposition of matrix \a a. + * + * The given matrix must have the same sparsity pattern as the matrix on which the symbolic decomposition has been performed. + * + * \sa analyzePattern() + */ +template +void AccelerateImpl::factorize(const MatrixType& a) { + eigen_assert(m_symbolicFactorization && "You must first call analyzePattern()"); + eigen_assert(m_nRows == a.rows() && m_nCols == a.cols()); + + if (EnforceSquare_) eigen_assert(a.rows() == a.cols()); + + AccelSparseMatrix A{}; + std::vector columnStarts; + + buildAccelSparseMatrix(a, A, columnStarts); + + doFactorization(A); +} + +template +template +void AccelerateImpl::_solve_impl(const MatrixBase& b, + MatrixBase& x) const { + if (!m_numericFactorization) { + m_info = InvalidInput; + return; + } + + eigen_assert(m_nRows == b.rows()); + eigen_assert(((b.cols() == 1) || b.outerStride() == b.rows())); + + SparseStatus_t status = SparseStatusOK; + + Scalar* b_ptr = const_cast(b.derived().data()); + Scalar* x_ptr = const_cast(x.derived().data()); + + AccelDenseMatrix xmat{}; + xmat.attributes = SparseAttributes_t(); + xmat.columnCount = static_cast(x.cols()); + xmat.rowCount = static_cast(x.rows()); + xmat.columnStride = xmat.rowCount; + xmat.data = x_ptr; + + AccelDenseMatrix bmat{}; + bmat.attributes = SparseAttributes_t(); + bmat.columnCount = static_cast(b.cols()); + bmat.rowCount = static_cast(b.rows()); + bmat.columnStride = bmat.rowCount; + bmat.data = b_ptr; + + SparseSolve(*m_numericFactorization, bmat, xmat); + + updateInfoStatus(status); +} + +} // end namespace Eigen + +#endif // EIGEN_ACCELERATESUPPORT_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/AccelerateSupport/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/AccelerateSupport/InternalHeaderCheck.h new file mode 100644 index 00000000000..69bcff50a65 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/AccelerateSupport/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_ACCELERATESUPPORT_MODULE_H +#error "Please include Eigen/AccelerateSupport instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/InternalHeaderCheck.h new file mode 100644 index 00000000000..5de2b219b3f --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_CHOLESKY_MODULE_H +#error "Please include Eigen/Cholesky instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LDLT.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LDLT.h index 1013ca045df..f0664e98706 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LDLT.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LDLT.h @@ -13,11 +13,14 @@ #ifndef EIGEN_LDLT_H #define EIGEN_LDLT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { - template struct traits > - : traits<_MatrixType> + template struct traits > + : traits { typedef MatrixXpr XprKind; typedef SolverStorage StorageKind; @@ -37,8 +40,8 @@ namespace internal { * * \brief Robust Cholesky decomposition of a matrix with pivoting * - * \tparam _MatrixType the type of the matrix of which to compute the LDL^T Cholesky decomposition - * \tparam _UpLo the triangular part that will be used for the decompositon: Lower (default) or Upper. + * \tparam MatrixType_ the type of the matrix of which to compute the LDL^T Cholesky decomposition + * \tparam UpLo_ the triangular part that will be used for the decomposition: Lower (default) or Upper. * The other triangular part won't be read. * * Perform a robust Cholesky decomposition of a positive semidefinite or negative semidefinite @@ -56,11 +59,11 @@ namespace internal { * * \sa MatrixBase::ldlt(), SelfAdjointView::ldlt(), class LLT */ -template class LDLT - : public SolverBase > +template class LDLT + : public SolverBase > { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef SolverBase Base; friend class SolverBase; @@ -68,7 +71,7 @@ template class LDLT enum { MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, - UpLo = _UpLo + UpLo = UpLo_ }; typedef Matrix TmpMatrixType; @@ -244,7 +247,7 @@ template class LDLT * This method is provided for compatibility with other matrix decompositions, thus enabling generic code such as: * \code x = decomposition.adjoint().solve(b) \endcode */ - const LDLT& adjoint() const { return *this; }; + const LDLT& adjoint() const { return *this; } EIGEN_DEVICE_FUNC inline EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_matrix.rows(); } EIGEN_DEVICE_FUNC inline EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_matrix.cols(); } @@ -270,10 +273,7 @@ template class LDLT protected: - static void check_template_parameters() - { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - } + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) /** \internal * Used to compute and store the Cholesky decomposition A = L D L^* = U^* D U. @@ -441,7 +441,7 @@ template<> struct ldlt_inplace // Update the terms of L Index rs = size-j-1; w.tail(rs) -= wj * mat.col(j).tail(rs); - if(gamma != 0) + if(!numext::is_exactly_zero(gamma)) mat.col(j).tail(rs) += (sigma*numext::conj(wj)/gamma)*w.tail(rs); } return true; @@ -494,12 +494,10 @@ template struct LDLT_Traits /** Compute / recompute the LDLT decomposition A = L D L^* = U^* D U of \a matrix */ -template +template template -LDLT& LDLT::compute(const EigenBase& a) +LDLT& LDLT::compute(const EigenBase& a) { - check_template_parameters(); - eigen_assert(a.rows()==a.cols()); const Index size = a.rows(); @@ -510,7 +508,7 @@ LDLT& LDLT::compute(const EigenBase() + m_matrix.row(col).head(col).template lpNorm<1>(); else abs_col_sum = m_matrix.col(col).head(col).template lpNorm<1>() + m_matrix.row(col).tail(size - col).template lpNorm<1>(); @@ -534,9 +532,9 @@ LDLT& LDLT::compute(const EigenBase +template template -LDLT& LDLT::rankUpdate(const MatrixBase& w, const typename LDLT::RealScalar& sigma) +LDLT& LDLT::rankUpdate(const MatrixBase& w, const typename LDLT::RealScalar& sigma) { typedef typename TranspositionType::StorageIndex IndexType; const Index size = w.rows(); @@ -562,16 +560,16 @@ LDLT& LDLT::rankUpdate(const MatrixBase +template template -void LDLT<_MatrixType,_UpLo>::_solve_impl(const RhsType &rhs, DstType &dst) const +void LDLT::_solve_impl(const RhsType &rhs, DstType &dst) const { _solve_impl_transposed(rhs, dst); } -template +template template -void LDLT<_MatrixType,_UpLo>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +void LDLT::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const { // dst = P b dst = m_transpositions * rhs; @@ -624,9 +622,9 @@ void LDLT<_MatrixType,_UpLo>::_solve_impl_transposed(const RhsType &rhs, DstType * * \sa LDLT::solve(), MatrixBase::ldlt() */ -template +template template -bool LDLT::solveInPlace(MatrixBase &bAndX) const +bool LDLT::solveInPlace(MatrixBase &bAndX) const { eigen_assert(m_isInitialized && "LDLT is not initialized."); eigen_assert(m_matrix.rows() == bAndX.rows()); @@ -639,8 +637,8 @@ bool LDLT::solveInPlace(MatrixBase &bAndX) const /** \returns the matrix represented by the decomposition, * i.e., it returns the product: P^T L D L^* P. * This function is provided for debug purpose. */ -template -MatrixType LDLT::reconstructedMatrix() const +template +MatrixType LDLT::reconstructedMatrix() const { eigen_assert(m_isInitialized && "LDLT is not initialized."); const Index size = m_matrix.rows(); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LLT.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LLT.h index 8c9b2b39879..55b39f60607 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LLT.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LLT.h @@ -10,12 +10,15 @@ #ifndef EIGEN_LLT_H #define EIGEN_LLT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal{ -template struct traits > - : traits<_MatrixType> +template struct traits > + : traits { typedef MatrixXpr XprKind; typedef SolverStorage StorageKind; @@ -32,8 +35,8 @@ template struct LLT_Traits; * * \brief Standard Cholesky decomposition (LL^T) of a matrix and associated features * - * \tparam _MatrixType the type of the matrix of which we are computing the LL^T Cholesky decomposition - * \tparam _UpLo the triangular part that will be used for the decompositon: Lower (default) or Upper. + * \tparam MatrixType_ the type of the matrix of which we are computing the LL^T Cholesky decomposition + * \tparam UpLo_ the triangular part that will be used for the decomposition: Lower (default) or Upper. * The other triangular part won't be read. * * This class performs a LL^T Cholesky decomposition of a symmetric, positive definite @@ -58,16 +61,16 @@ template struct LLT_Traits; * * This class supports the \link InplaceDecomposition inplace decomposition \endlink mechanism. * - * Note that during the decomposition, only the lower (or upper, as defined by _UpLo) triangular part of A is considered. + * Note that during the decomposition, only the lower (or upper, as defined by UpLo_) triangular part of A is considered. * Therefore, the strict lower part does not have to store correct values. * * \sa MatrixBase::llt(), SelfAdjointView::llt(), class LDLT */ -template class LLT - : public SolverBase > +template class LLT + : public SolverBase > { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef SolverBase Base; friend class SolverBase; @@ -79,7 +82,7 @@ template class LLT enum { PacketSize = internal::packet_traits::size, AlignmentMask = int(PacketSize)-1, - UpLo = _UpLo + UpLo = UpLo_ }; typedef internal::LLT_Traits Traits; @@ -199,7 +202,7 @@ template class LLT * This method is provided for compatibility with other matrix decompositions, thus enabling generic code such as: * \code x = decomposition.adjoint().solve(b) \endcode */ - const LLT& adjoint() const EIGEN_NOEXCEPT { return *this; }; + const LLT& adjoint() const EIGEN_NOEXCEPT { return *this; } inline EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_matrix.rows(); } inline EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_matrix.cols(); } @@ -217,10 +220,7 @@ template class LLT protected: - static void check_template_parameters() - { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - } + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) /** \internal * Used to compute and store L @@ -243,7 +243,7 @@ static Index llt_rank_update_lower(MatrixType& mat, const VectorType& vec, const typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef typename MatrixType::ColXpr ColXpr; - typedef typename internal::remove_all::type ColXprCleaned; + typedef internal::remove_all_t ColXprCleaned; typedef typename ColXprCleaned::SegmentReturnType ColXprSegment; typedef Matrix TempVectorType; typedef typename TempVectorType::SegmentReturnType TempVecSegment; @@ -298,7 +298,7 @@ static Index llt_rank_update_lower(MatrixType& mat, const VectorType& vec, const if(rs) { temp.tail(rs) -= (wj/Ljj) * mat.col(j).tail(rs); - if(gamma != 0) + if(!numext::is_exactly_zero(gamma)) mat.col(j).tail(rs) = (nLjj/Ljj) * mat.col(j).tail(rs) + (nLjj * sigma*numext::conj(wj)/gamma)*temp.tail(rs); } } @@ -427,12 +427,10 @@ template struct LLT_Traits * Example: \include TutorialLinAlgComputeTwice.cpp * Output: \verbinclude TutorialLinAlgComputeTwice.out */ -template +template template -LLT& LLT::compute(const EigenBase& a) +LLT& LLT::compute(const EigenBase& a) { - check_template_parameters(); - eigen_assert(a.rows()==a.cols()); const Index size = a.rows(); m_matrix.resize(size, size); @@ -444,7 +442,7 @@ LLT& LLT::compute(const EigenBase // TODO move this code to SelfAdjointView for (Index col = 0; col < size; ++col) { RealScalar abs_col_sum; - if (_UpLo == Lower) + if (UpLo_ == Lower) abs_col_sum = m_matrix.col(col).tail(size - col).template lpNorm<1>() + m_matrix.row(col).head(col).template lpNorm<1>(); else abs_col_sum = m_matrix.col(col).head(col).template lpNorm<1>() + m_matrix.row(col).tail(size - col).template lpNorm<1>(); @@ -464,9 +462,9 @@ LLT& LLT::compute(const EigenBase * then after it we have LL^* = A + sigma * v v^* where \a v must be a vector * of same dimension. */ -template +template template -LLT<_MatrixType,_UpLo> & LLT<_MatrixType,_UpLo>::rankUpdate(const VectorType& v, const RealScalar& sigma) +LLT & LLT::rankUpdate(const VectorType& v, const RealScalar& sigma) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(VectorType); eigen_assert(v.size()==m_matrix.cols()); @@ -480,16 +478,16 @@ LLT<_MatrixType,_UpLo> & LLT<_MatrixType,_UpLo>::rankUpdate(const VectorType& v, } #ifndef EIGEN_PARSED_BY_DOXYGEN -template +template template -void LLT<_MatrixType,_UpLo>::_solve_impl(const RhsType &rhs, DstType &dst) const +void LLT::_solve_impl(const RhsType &rhs, DstType &dst) const { _solve_impl_transposed(rhs, dst); } -template +template template -void LLT<_MatrixType,_UpLo>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +void LLT::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const { dst = rhs; @@ -511,9 +509,9 @@ void LLT<_MatrixType,_UpLo>::_solve_impl_transposed(const RhsType &rhs, DstType * * \sa LLT::solve(), MatrixBase::llt() */ -template +template template -void LLT::solveInPlace(const MatrixBase &bAndX) const +void LLT::solveInPlace(const MatrixBase &bAndX) const { eigen_assert(m_isInitialized && "LLT is not initialized."); eigen_assert(m_matrix.rows()==bAndX.rows()); @@ -524,8 +522,8 @@ void LLT::solveInPlace(const MatrixBase &bAndX) const /** \returns the matrix represented by the decomposition, * i.e., it returns the product: L L^*. * This function is provided for debug purpose. */ -template -MatrixType LLT::reconstructedMatrix() const +template +MatrixType LLT::reconstructedMatrix() const { eigen_assert(m_isInitialized && "LLT is not initialized."); return matrixL() * matrixL().adjoint().toDenseMatrix(); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LLT_LAPACKE.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LLT_LAPACKE.h index bc6489e69a9..15eb2b8da47 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LLT_LAPACKE.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Cholesky/LLT_LAPACKE.h @@ -33,64 +33,89 @@ #ifndef EIGEN_LLT_LAPACKE_H #define EIGEN_LLT_LAPACKE_H -namespace Eigen { +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { namespace internal { -template struct lapacke_llt; - -#define EIGEN_LAPACKE_LLT(EIGTYPE, BLASTYPE, LAPACKE_PREFIX) \ -template<> struct lapacke_llt \ -{ \ - template \ - static inline Index potrf(MatrixType& m, char uplo) \ - { \ - lapack_int matrix_order; \ - lapack_int size, lda, info, StorageOrder; \ - EIGTYPE* a; \ - eigen_assert(m.rows()==m.cols()); \ - /* Set up parameters for ?potrf */ \ - size = convert_index(m.rows()); \ - StorageOrder = MatrixType::Flags&RowMajorBit?RowMajor:ColMajor; \ - matrix_order = StorageOrder==RowMajor ? LAPACK_ROW_MAJOR : LAPACK_COL_MAJOR; \ - a = &(m.coeffRef(0,0)); \ - lda = convert_index(m.outerStride()); \ -\ - info = LAPACKE_##LAPACKE_PREFIX##potrf( matrix_order, uplo, size, (BLASTYPE*)a, lda ); \ - info = (info==0) ? -1 : info>0 ? info-1 : size; \ - return info; \ - } \ -}; \ -template<> struct llt_inplace \ -{ \ - template \ - static Index blocked(MatrixType& m) \ - { \ - return lapacke_llt::potrf(m, 'L'); \ - } \ - template \ - static Index rankUpdate(MatrixType& mat, const VectorType& vec, const typename MatrixType::RealScalar& sigma) \ - { return Eigen::internal::llt_rank_update_lower(mat, vec, sigma); } \ -}; \ -template<> struct llt_inplace \ -{ \ - template \ - static Index blocked(MatrixType& m) \ - { \ - return lapacke_llt::potrf(m, 'U'); \ - } \ - template \ - static Index rankUpdate(MatrixType& mat, const VectorType& vec, const typename MatrixType::RealScalar& sigma) \ - { \ - Transpose matt(mat); \ - return llt_inplace::rankUpdate(matt, vec.conjugate(), sigma); \ - } \ -}; - -EIGEN_LAPACKE_LLT(double, double, d) -EIGEN_LAPACKE_LLT(float, float, s) -EIGEN_LAPACKE_LLT(dcomplex, lapack_complex_double, z) -EIGEN_LAPACKE_LLT(scomplex, lapack_complex_float, c) +namespace lapacke_helpers { + // ------------------------------------------------------------------------------------------------------------------- + // Dispatch for rank update handling upper and lower parts + // ------------------------------------------------------------------------------------------------------------------- + + template + struct rank_update {}; + + template<> + struct rank_update { + template + static Index run(MatrixType &mat, const VectorType &vec, const typename MatrixType::RealScalar &sigma) { + return Eigen::internal::llt_rank_update_lower(mat, vec, sigma); + } + }; + + template<> + struct rank_update { + template + static Index run(MatrixType &mat, const VectorType &vec, const typename MatrixType::RealScalar &sigma) { + Transpose matt(mat); + return Eigen::internal::llt_rank_update_lower(matt, vec.conjugate(), sigma); + } + }; + + // ------------------------------------------------------------------------------------------------------------------- + // Generic lapacke llt implementation that hands of to the dispatches + // ------------------------------------------------------------------------------------------------------------------- + + template + struct lapacke_llt { + EIGEN_STATIC_ASSERT(((Mode == Lower) || (Mode == Upper)),MODE_MUST_BE_UPPER_OR_LOWER) + template + static Index blocked(MatrixType& m) + { + eigen_assert(m.rows() == m.cols()); + if(m.rows() == 0) { + return -1; + } + /* Set up parameters for ?potrf */ + lapack_int size = to_lapack(m.rows()); + lapack_int matrix_order = lapack_storage_of(m); + constexpr char uplo = Mode == Upper ? 'U' : 'L'; + Scalar* a = &(m.coeffRef(0,0)); + lapack_int lda = to_lapack(m.outerStride()); + + lapack_int info = potrf(matrix_order, uplo, size, to_lapack(a), lda ); + info = (info==0) ? -1 : info>0 ? info-1 : size; + return info; + } + + template + static Index rankUpdate(MatrixType& mat, const VectorType& vec, const typename MatrixType::RealScalar& sigma) + { + return rank_update::run(mat, vec, sigma); + } + }; +} +// end namespace lapacke_helpers + +/* + * Here, we just put the generic implementation from lapacke_llt into a full specialization of the llt_inplace + * type. By being a full specialization, the versions defined here thus get precedence over the generic implementation + * in LLT.h for double, float and complex double, complex float types. + */ + +#define EIGEN_LAPACKE_LLT(EIGTYPE) \ +template<> struct llt_inplace : public lapacke_helpers::lapacke_llt {}; \ +template<> struct llt_inplace : public lapacke_helpers::lapacke_llt {}; + +EIGEN_LAPACKE_LLT(double) +EIGEN_LAPACKE_LLT(float) +EIGEN_LAPACKE_LLT(std::complex) +EIGEN_LAPACKE_LLT(std::complex) + +#undef EIGEN_LAPACKE_LLT } // end namespace internal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/CholmodSupport/CholmodSupport.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/CholmodSupport/CholmodSupport.h index adaf52858e4..e4e58f3e76b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/CholmodSupport/CholmodSupport.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/CholmodSupport/CholmodSupport.h @@ -10,6 +10,9 @@ #ifndef EIGEN_CHOLMODSUPPORT_H #define EIGEN_CHOLMODSUPPORT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -54,8 +57,8 @@ template<> struct cholmod_configure_matrix > { /** Wraps the Eigen sparse matrix \a mat into a Cholmod sparse matrix object. * Note that the data are shared. */ -template -cholmod_sparse viewAsCholmod(Ref > mat) +template +cholmod_sparse viewAsCholmod(Ref > mat) { cholmod_sparse res; res.nzmax = mat.nonZeros(); @@ -80,11 +83,11 @@ cholmod_sparse viewAsCholmod(Ref > res.dtype = 0; res.stype = -1; - if (internal::is_same<_StorageIndex,int>::value) + if (internal::is_same::value) { res.itype = CHOLMOD_INT; } - else if (internal::is_same<_StorageIndex,SuiteSparse_long>::value) + else if (internal::is_same::value) { res.itype = CHOLMOD_LONG; } @@ -94,39 +97,39 @@ cholmod_sparse viewAsCholmod(Ref > } // setup res.xtype - internal::cholmod_configure_matrix<_Scalar>::run(res); + internal::cholmod_configure_matrix::run(res); res.stype = 0; return res; } -template -const cholmod_sparse viewAsCholmod(const SparseMatrix<_Scalar,_Options,_Index>& mat) +template +const cholmod_sparse viewAsCholmod(const SparseMatrix& mat) { - cholmod_sparse res = viewAsCholmod(Ref >(mat.const_cast_derived())); + cholmod_sparse res = viewAsCholmod(Ref >(mat.const_cast_derived())); return res; } -template -const cholmod_sparse viewAsCholmod(const SparseVector<_Scalar,_Options,_Index>& mat) +template +const cholmod_sparse viewAsCholmod(const SparseVector& mat) { - cholmod_sparse res = viewAsCholmod(Ref >(mat.const_cast_derived())); + cholmod_sparse res = viewAsCholmod(Ref >(mat.const_cast_derived())); return res; } /** Returns a view of the Eigen sparse matrix \a mat as Cholmod sparse matrix. * The data are not copied but shared. */ -template -cholmod_sparse viewAsCholmod(const SparseSelfAdjointView, UpLo>& mat) +template +cholmod_sparse viewAsCholmod(const SparseSelfAdjointView, UpLo>& mat) { - cholmod_sparse res = viewAsCholmod(Ref >(mat.matrix().const_cast_derived())); + cholmod_sparse res = viewAsCholmod(Ref >(mat.matrix().const_cast_derived())); if(UpLo==Upper) res.stype = 1; if(UpLo==Lower) res.stype = -1; // swap stype for rowmajor matrices (only works for real matrices) - EIGEN_STATIC_ASSERT((_Options & RowMajorBit) == 0 || NumTraits<_Scalar>::IsComplex == 0, THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES); - if(_Options & RowMajorBit) res.stype *=-1; + EIGEN_STATIC_ASSERT((Options_ & RowMajorBit) == 0 || NumTraits::IsComplex == 0, THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES); + if(Options_ & RowMajorBit) res.stype *=-1; return res; } @@ -155,9 +158,9 @@ cholmod_dense viewAsCholmod(MatrixBase& mat) /** Returns a view of the Cholmod sparse matrix \a cm as an Eigen sparse matrix. * The data are not copied but shared. */ template -MappedSparseMatrix viewAsEigen(cholmod_sparse& cm) +Map > viewAsEigen(cholmod_sparse& cm) { - return MappedSparseMatrix + return Map > (cm.nrow, cm.ncol, static_cast(cm.p)[cm.ncol], static_cast(cm.p), static_cast(cm.i),static_cast(cm.x) ); } @@ -167,11 +170,11 @@ namespace internal { // template specializations for int and long that call the correct cholmod method #define EIGEN_CHOLMOD_SPECIALIZE0(ret, name) \ - template inline ret cm_ ## name (cholmod_common &Common) { return cholmod_ ## name (&Common); } \ + template inline ret cm_ ## name (cholmod_common &Common) { return cholmod_ ## name (&Common); } \ template<> inline ret cm_ ## name (cholmod_common &Common) { return cholmod_l_ ## name (&Common); } #define EIGEN_CHOLMOD_SPECIALIZE1(ret, name, t1, a1) \ - template inline ret cm_ ## name (t1& a1, cholmod_common &Common) { return cholmod_ ## name (&a1, &Common); } \ + template inline ret cm_ ## name (t1& a1, cholmod_common &Common) { return cholmod_ ## name (&a1, &Common); } \ template<> inline ret cm_ ## name (t1& a1, cholmod_common &Common) { return cholmod_l_ ## name (&a1, &Common); } EIGEN_CHOLMOD_SPECIALIZE0(int, start) @@ -183,14 +186,14 @@ EIGEN_CHOLMOD_SPECIALIZE1(int, free_sparse, cholmod_sparse*, A) EIGEN_CHOLMOD_SPECIALIZE1(cholmod_factor*, analyze, cholmod_sparse, A) -template inline cholmod_dense* cm_solve (int sys, cholmod_factor& L, cholmod_dense& B, cholmod_common &Common) { return cholmod_solve (sys, &L, &B, &Common); } +template inline cholmod_dense* cm_solve (int sys, cholmod_factor& L, cholmod_dense& B, cholmod_common &Common) { return cholmod_solve (sys, &L, &B, &Common); } template<> inline cholmod_dense* cm_solve (int sys, cholmod_factor& L, cholmod_dense& B, cholmod_common &Common) { return cholmod_l_solve (sys, &L, &B, &Common); } -template inline cholmod_sparse* cm_spsolve (int sys, cholmod_factor& L, cholmod_sparse& B, cholmod_common &Common) { return cholmod_spsolve (sys, &L, &B, &Common); } +template inline cholmod_sparse* cm_spsolve (int sys, cholmod_factor& L, cholmod_sparse& B, cholmod_common &Common) { return cholmod_spsolve (sys, &L, &B, &Common); } template<> inline cholmod_sparse* cm_spsolve (int sys, cholmod_factor& L, cholmod_sparse& B, cholmod_common &Common) { return cholmod_l_spsolve (sys, &L, &B, &Common); } -template -inline int cm_factorize_p (cholmod_sparse* A, double beta[2], _StorageIndex* fset, std::size_t fsize, cholmod_factor* L, cholmod_common &Common) { return cholmod_factorize_p (A, beta, fset, fsize, L, &Common); } +template +inline int cm_factorize_p (cholmod_sparse* A, double beta[2], StorageIndex_* fset, std::size_t fsize, cholmod_factor* L, cholmod_common &Common) { return cholmod_factorize_p (A, beta, fset, fsize, L, &Common); } template<> inline int cm_factorize_p (cholmod_sparse* A, double beta[2], SuiteSparse_long* fset, std::size_t fsize, cholmod_factor* L, cholmod_common &Common) { return cholmod_l_factorize_p (A, beta, fset, fsize, L, &Common); } @@ -210,7 +213,7 @@ enum CholmodMode { * \brief The base class for the direct Cholesky factorization of Cholmod * \sa class CholmodSupernodalLLT, class CholmodSimplicialLDLT, class CholmodSimplicialLLT */ -template +template class CholmodBase : public SparseSolverBase { protected: @@ -218,8 +221,8 @@ class CholmodBase : public SparseSolverBase using Base::derived; using Base::m_isInitialized; public: - typedef _MatrixType MatrixType; - enum { UpLo = _UpLo }; + typedef MatrixType_ MatrixType; + enum { UpLo = UpLo_ }; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef MatrixType CholMatrixType; @@ -436,7 +439,7 @@ class CholmodBase : public SparseSolverBase if (m_cholmodFactor->is_ll) logDet *= 2.0; return logDet; - }; + } template void dumpMemory(Stream& /*s*/) @@ -461,8 +464,8 @@ class CholmodBase : public SparseSolverBase * The sparse matrix A must be selfadjoint and positive definite. The vectors or matrices * X and B can be either dense or sparse. * - * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> - * \tparam _UpLo the triangular part that will be used for the computations. It can be Lower + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ the triangular part that will be used for the computations. It can be Lower * or Upper. Default is Lower. * * \implsparsesolverconcept @@ -473,15 +476,15 @@ class CholmodBase : public SparseSolverBase * * \sa \ref TutorialSparseSolverConcept, class CholmodSupernodalLLT, class SimplicialLLT */ -template -class CholmodSimplicialLLT : public CholmodBase<_MatrixType, _UpLo, CholmodSimplicialLLT<_MatrixType, _UpLo> > +template +class CholmodSimplicialLLT : public CholmodBase > { - typedef CholmodBase<_MatrixType, _UpLo, CholmodSimplicialLLT> Base; + typedef CholmodBase Base; using Base::m_cholmod; public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; CholmodSimplicialLLT() : Base() { init(); } @@ -512,8 +515,8 @@ class CholmodSimplicialLLT : public CholmodBase<_MatrixType, _UpLo, CholmodSimpl * The sparse matrix A must be selfadjoint and positive definite. The vectors or matrices * X and B can be either dense or sparse. * - * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> - * \tparam _UpLo the triangular part that will be used for the computations. It can be Lower + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ the triangular part that will be used for the computations. It can be Lower * or Upper. Default is Lower. * * \implsparsesolverconcept @@ -524,15 +527,15 @@ class CholmodSimplicialLLT : public CholmodBase<_MatrixType, _UpLo, CholmodSimpl * * \sa \ref TutorialSparseSolverConcept, class CholmodSupernodalLLT, class SimplicialLDLT */ -template -class CholmodSimplicialLDLT : public CholmodBase<_MatrixType, _UpLo, CholmodSimplicialLDLT<_MatrixType, _UpLo> > +template +class CholmodSimplicialLDLT : public CholmodBase > { - typedef CholmodBase<_MatrixType, _UpLo, CholmodSimplicialLDLT> Base; + typedef CholmodBase Base; using Base::m_cholmod; public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; CholmodSimplicialLDLT() : Base() { init(); } @@ -561,8 +564,8 @@ class CholmodSimplicialLDLT : public CholmodBase<_MatrixType, _UpLo, CholmodSimp * The sparse matrix A must be selfadjoint and positive definite. The vectors or matrices * X and B can be either dense or sparse. * - * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> - * \tparam _UpLo the triangular part that will be used for the computations. It can be Lower + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ the triangular part that will be used for the computations. It can be Lower * or Upper. Default is Lower. * * \implsparsesolverconcept @@ -573,15 +576,15 @@ class CholmodSimplicialLDLT : public CholmodBase<_MatrixType, _UpLo, CholmodSimp * * \sa \ref TutorialSparseSolverConcept */ -template -class CholmodSupernodalLLT : public CholmodBase<_MatrixType, _UpLo, CholmodSupernodalLLT<_MatrixType, _UpLo> > +template +class CholmodSupernodalLLT : public CholmodBase > { - typedef CholmodBase<_MatrixType, _UpLo, CholmodSupernodalLLT> Base; + typedef CholmodBase Base; using Base::m_cholmod; public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; CholmodSupernodalLLT() : Base() { init(); } @@ -612,8 +615,8 @@ class CholmodSupernodalLLT : public CholmodBase<_MatrixType, _UpLo, CholmodSuper * On the other hand, it does not provide access to the result of the factorization. * The default is to let Cholmod automatically choose between a simplicial and supernodal factorization. * - * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> - * \tparam _UpLo the triangular part that will be used for the computations. It can be Lower + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ the triangular part that will be used for the computations. It can be Lower * or Upper. Default is Lower. * * \implsparsesolverconcept @@ -624,15 +627,15 @@ class CholmodSupernodalLLT : public CholmodBase<_MatrixType, _UpLo, CholmodSuper * * \sa \ref TutorialSparseSolverConcept */ -template -class CholmodDecomposition : public CholmodBase<_MatrixType, _UpLo, CholmodDecomposition<_MatrixType, _UpLo> > +template +class CholmodDecomposition : public CholmodBase > { - typedef CholmodBase<_MatrixType, _UpLo, CholmodDecomposition> Base; + typedef CholmodBase Base; using Base::m_cholmod; public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; CholmodDecomposition() : Base() { init(); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/CholmodSupport/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/CholmodSupport/InternalHeaderCheck.h new file mode 100644 index 00000000000..0fb3abc7040 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/CholmodSupport/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_CHOLMODSUPPORT_MODULE_H +#error "Please include Eigen/CholmodSupport instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArithmeticSequence.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArithmeticSequence.h index b6200fac1b0..34689c66dbf 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArithmeticSequence.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArithmeticSequence.h @@ -10,69 +10,19 @@ #ifndef EIGEN_ARITHMETIC_SEQUENCE_H #define EIGEN_ARITHMETIC_SEQUENCE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -#if (!EIGEN_HAS_CXX11) || !((!EIGEN_COMP_GNUC) || EIGEN_COMP_GNUC>=48) -template struct aseq_negate {}; - -template<> struct aseq_negate { - typedef Index type; -}; - -template struct aseq_negate > { - typedef FixedInt<-N> type; -}; - -// Compilation error in the following case: -template<> struct aseq_negate > {}; - -template::value, - bool SizeIsSymbolic =symbolic::is_symbolic::value> -struct aseq_reverse_first_type { - typedef Index type; -}; - -template -struct aseq_reverse_first_type { - typedef symbolic::AddExpr > >, - symbolic::ValueExpr > - > type; -}; - -template -struct aseq_reverse_first_type_aux { - typedef Index type; -}; - -template -struct aseq_reverse_first_type_aux::type> { - typedef FixedInt<(SizeType::value-1)*IncrType::value> type; -}; - -template -struct aseq_reverse_first_type { - typedef typename aseq_reverse_first_type_aux::type Aux; - typedef symbolic::AddExpr > type; -}; - -template -struct aseq_reverse_first_type { - typedef symbolic::AddExpr > >, - symbolic::ValueExpr >, - symbolic::ValueExpr<> > type; -}; -#endif - // Helper to cleanup the type of the increment: template struct cleanup_seq_incr { typedef typename cleanup_index_type::type type; }; -} +} // namespace internal //-------------------------------------------------------------------------------- // seq(first,last,incr) and seqN(first,size,incr) @@ -137,21 +87,9 @@ class ArithmeticSequence IncrType m_incr; public: - -#if EIGEN_HAS_CXX11 && ((!EIGEN_COMP_GNUC) || EIGEN_COMP_GNUC>=48) auto reverse() const -> decltype(Eigen::seqN(m_first+(m_size+fix<-1>())*m_incr,m_size,-m_incr)) { return seqN(m_first+(m_size+fix<-1>())*m_incr,m_size,-m_incr); } -#else -protected: - typedef typename internal::aseq_negate::type ReverseIncrType; - typedef typename internal::aseq_reverse_first_type::type ReverseFirstType; -public: - ArithmeticSequence - reverse() const { - return seqN(m_first+(m_size+fix<-1>())*m_incr,m_size,-m_incr); - } -#endif }; /** \returns an ArithmeticSequence starting at \a first, of length \a size, and increment \a incr @@ -200,7 +138,6 @@ auto seq(FirstType f, LastType l); #else // EIGEN_PARSED_BY_DOXYGEN -#if EIGEN_HAS_CXX11 template auto seq(FirstType f, LastType l) -> decltype(seqN(typename internal::cleanup_index_type::type(f), ( typename internal::cleanup_index_type::type(l) @@ -226,101 +163,11 @@ auto seq(FirstType f, LastType l, IncrType incr) CleanedIncrType(incr)); } -#else // EIGEN_HAS_CXX11 - -template -typename internal::enable_if::value || symbolic::is_symbolic::value), - ArithmeticSequence::type,Index> >::type -seq(FirstType f, LastType l) -{ - return seqN(typename internal::cleanup_index_type::type(f), - Index((typename internal::cleanup_index_type::type(l)-typename internal::cleanup_index_type::type(f)+fix<1>()))); -} - -template -typename internal::enable_if::value, - ArithmeticSequence,symbolic::ValueExpr<> >, - symbolic::ValueExpr > > > >::type -seq(const symbolic::BaseExpr &f, LastType l) -{ - return seqN(f.derived(),(typename internal::cleanup_index_type::type(l)-f.derived()+fix<1>())); -} - -template -typename internal::enable_if::value, - ArithmeticSequence::type, - symbolic::AddExpr >, - symbolic::ValueExpr > > > >::type -seq(FirstType f, const symbolic::BaseExpr &l) -{ - return seqN(typename internal::cleanup_index_type::type(f),(l.derived()-typename internal::cleanup_index_type::type(f)+fix<1>())); -} - -template -ArithmeticSequence >,symbolic::ValueExpr > > > -seq(const symbolic::BaseExpr &f, const symbolic::BaseExpr &l) -{ - return seqN(f.derived(),(l.derived()-f.derived()+fix<1>())); -} - - -template -typename internal::enable_if::value || symbolic::is_symbolic::value), - ArithmeticSequence::type,Index,typename internal::cleanup_seq_incr::type> >::type -seq(FirstType f, LastType l, IncrType incr) -{ - typedef typename internal::cleanup_seq_incr::type CleanedIncrType; - return seqN(typename internal::cleanup_index_type::type(f), - Index((typename internal::cleanup_index_type::type(l)-typename internal::cleanup_index_type::type(f)+CleanedIncrType(incr))/CleanedIncrType(incr)), incr); -} - -template -typename internal::enable_if::value, - ArithmeticSequence, - symbolic::ValueExpr<> >, - symbolic::ValueExpr::type> >, - symbolic::ValueExpr::type> >, - typename internal::cleanup_seq_incr::type> >::type -seq(const symbolic::BaseExpr &f, LastType l, IncrType incr) -{ - typedef typename internal::cleanup_seq_incr::type CleanedIncrType; - return seqN(f.derived(),(typename internal::cleanup_index_type::type(l)-f.derived()+CleanedIncrType(incr))/CleanedIncrType(incr), incr); -} - -template -typename internal::enable_if::value, - ArithmeticSequence::type, - symbolic::QuotientExpr >, - symbolic::ValueExpr::type> >, - symbolic::ValueExpr::type> >, - typename internal::cleanup_seq_incr::type> >::type -seq(FirstType f, const symbolic::BaseExpr &l, IncrType incr) -{ - typedef typename internal::cleanup_seq_incr::type CleanedIncrType; - return seqN(typename internal::cleanup_index_type::type(f), - (l.derived()-typename internal::cleanup_index_type::type(f)+CleanedIncrType(incr))/CleanedIncrType(incr), incr); -} - -template -ArithmeticSequence >, - symbolic::ValueExpr::type> >, - symbolic::ValueExpr::type> >, - typename internal::cleanup_seq_incr::type> -seq(const symbolic::BaseExpr &f, const symbolic::BaseExpr &l, IncrType incr) -{ - typedef typename internal::cleanup_seq_incr::type CleanedIncrType; - return seqN(f.derived(),(l.derived()-f.derived()+CleanedIncrType(incr))/CleanedIncrType(incr), incr); -} -#endif // EIGEN_HAS_CXX11 #endif // EIGEN_PARSED_BY_DOXYGEN +namespace placeholders { -#if EIGEN_HAS_CXX11 || defined(EIGEN_PARSED_BY_DOXYGEN) /** \cpp11 * \returns a symbolic ArithmeticSequence representing the last \a size elements with increment \a incr. * @@ -329,9 +176,9 @@ seq(const symbolic::BaseExpr &f, const symbolic::BaseExpr auto lastN(SizeType size, IncrType incr) --> decltype(seqN(Eigen::last-(size-fix<1>())*incr, size, incr)) +-> decltype(seqN(Eigen::placeholders::last-(size-fix<1>())*incr, size, incr)) { - return seqN(Eigen::last-(size-fix<1>())*incr, size, incr); + return seqN(Eigen::placeholders::last-(size-fix<1>())*incr, size, incr); } /** \cpp11 @@ -342,18 +189,19 @@ auto lastN(SizeType size, IncrType incr) * \sa lastN(SizeType,IncrType, seqN(FirstType,SizeType), seq(FirstType,LastType) */ template auto lastN(SizeType size) --> decltype(seqN(Eigen::last+fix<1>()-size, size)) +-> decltype(seqN(Eigen::placeholders::last+fix<1>()-size, size)) { - return seqN(Eigen::last+fix<1>()-size, size); + return seqN(Eigen::placeholders::last+fix<1>()-size, size); } -#endif + +} // namespace placeholders namespace internal { // Convert a symbolic span into a usable one (i.e., remove last/end "keywords") template struct make_size_type { - typedef typename internal::conditional::value, Index, T>::type type; + typedef std::conditional_t::value, Index, T> type; }; template @@ -387,25 +235,23 @@ struct get_compile_time_incr > { * \code using namespace Eigen::indexing; \endcode * is equivalent to: * \code - using Eigen::all; + using Eigen::fix; using Eigen::seq; using Eigen::seqN; - using Eigen::lastN; // c++11 only - using Eigen::last; - using Eigen::lastp1; - using Eigen::fix; + using Eigen::placeholders::all; + using Eigen::placeholders::last; + using Eigen::placeholders::lastN; // c++11 only + using Eigen::placeholders::lastp1; \endcode */ namespace indexing { - using Eigen::all; + using Eigen::fix; using Eigen::seq; using Eigen::seqN; - #if EIGEN_HAS_CXX11 - using Eigen::lastN; - #endif - using Eigen::last; - using Eigen::lastp1; - using Eigen::fix; + using Eigen::placeholders::all; + using Eigen::placeholders::last; + using Eigen::placeholders::lastN; + using Eigen::placeholders::lastp1; } } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Array.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Array.h index 20c789b10c7..005583836fe 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Array.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Array.h @@ -10,14 +10,17 @@ #ifndef EIGEN_ARRAY_H #define EIGEN_ARRAY_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template -struct traits > : traits > +template +struct traits > : traits > { typedef ArrayXpr XprKind; - typedef ArrayBase > XprBase; + typedef ArrayBase > XprBase; }; } @@ -41,16 +44,16 @@ struct traits > : tra * * \sa \blank \ref TutorialArrayClass, \ref TopicClassHierarchy */ -template +template class Array - : public PlainObjectBase > + : public PlainObjectBase > { public: typedef PlainObjectBase Base; EIGEN_DENSE_PUBLIC_INTERFACE(Array) - enum { Options = _Options }; + enum { Options = Options_ }; typedef typename Base::PlainObject PlainObject; protected: @@ -131,7 +134,6 @@ class Array EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array() : Base() { - Base::_check_template_params(); EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } @@ -142,17 +144,14 @@ class Array Array(internal::constructor_without_unaligned_array_assert) : Base(internal::constructor_without_unaligned_array_assert()) { - Base::_check_template_params(); EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } #endif -#if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC Array(Array&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_constructible::value) : Base(std::move(other)) { - Base::_check_template_params(); } EIGEN_DEVICE_FUNC Array& operator=(Array&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_assignable::value) @@ -160,9 +159,7 @@ class Array Base::operator=(std::move(other)); return *this; } -#endif - #if EIGEN_HAS_CXX11 /** \copydoc PlainObjectBase(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) * * Example: \include Array_variadic_ctor_cxx11.cpp @@ -197,16 +194,15 @@ class Array * * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) */ - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Array(const std::initializer_list>& list) : Base(list) {} - #endif // end EIGEN_HAS_CXX11 + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Array( + const std::initializer_list>& list) + : Base(list) {} #ifndef EIGEN_PARSED_BY_DOXYGEN template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Array(const T& x) { - Base::_check_template_params(); Base::template _init1(x); } @@ -214,7 +210,6 @@ class Array EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const T0& val0, const T1& val1) { - Base::_check_template_params(); this->template _init2(val0, val1); } @@ -249,7 +244,6 @@ class Array EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const Scalar& val0, const Scalar& val1, const Scalar& val2) { - Base::_check_template_params(); EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Array, 3) m_storage.data()[0] = val0; m_storage.data()[1] = val1; @@ -261,7 +255,6 @@ class Array EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const Scalar& val0, const Scalar& val1, const Scalar& val2, const Scalar& val3) { - Base::_check_template_params(); EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Array, 4) m_storage.data()[0] = val0; m_storage.data()[1] = val1; @@ -283,8 +276,8 @@ class Array template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const EigenBase &other, - typename internal::enable_if::value, - PrivateType>::type = PrivateType()) + std::enable_if_t::value, + PrivateType> = PrivateType()) : Base(other.derived()) { } @@ -359,8 +352,6 @@ EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(std::complex, cd) #undef EIGEN_MAKE_ARRAY_TYPEDEFS #undef EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS -#if EIGEN_HAS_CXX11 - #define EIGEN_MAKE_ARRAY_TYPEDEFS(Size, SizeSuffix) \ /** \ingroup arraytypedefs */ \ /** \brief \cpp11 */ \ @@ -392,8 +383,6 @@ EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(4) #undef EIGEN_MAKE_ARRAY_TYPEDEFS #undef EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS -#endif // EIGEN_HAS_CXX11 - #define EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE_AND_SIZE(TypeSuffix, SizeSuffix) \ using Eigen::Matrix##SizeSuffix##TypeSuffix; \ using Eigen::Vector##SizeSuffix##TypeSuffix; \ diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArrayBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArrayBase.h index ea3dd1c3b38..83001b27787 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArrayBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArrayBase.h @@ -10,6 +10,9 @@ #ifndef EIGEN_ARRAYBASE_H #define EIGEN_ARRAYBASE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template class MatrixWrapper; @@ -21,7 +24,7 @@ template class MatrixWrapper; * * An array is similar to a dense vector or matrix. While matrices are mathematical * objects with well defined linear algebra operators, an array is just a collection - * of scalar values arranged in a one or two dimensionnal fashion. As the main consequence, + * of scalar values arranged in a one or two dimensional fashion. As the main consequence, * all operations applied to an array are performed coefficient wise. Furthermore, * arrays support scalar math functions of the c++ standard library (e.g., std::sin(x)), and convenient * constructors allowing to easily write generic code working for both scalar values @@ -89,11 +92,11 @@ template class ArrayBase #define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::ArrayBase #define EIGEN_DOC_UNARY_ADDONS(X,Y) -# include "../plugins/MatrixCwiseUnaryOps.h" -# include "../plugins/ArrayCwiseUnaryOps.h" -# include "../plugins/CommonCwiseBinaryOps.h" -# include "../plugins/MatrixCwiseBinaryOps.h" -# include "../plugins/ArrayCwiseBinaryOps.h" +# include "../plugins/MatrixCwiseUnaryOps.inc" +# include "../plugins/ArrayCwiseUnaryOps.inc" +# include "../plugins/CommonCwiseBinaryOps.inc" +# include "../plugins/MatrixCwiseBinaryOps.inc" +# include "../plugins/ArrayCwiseBinaryOps.inc" # ifdef EIGEN_ARRAYBASE_PLUGIN # include EIGEN_ARRAYBASE_PLUGIN # endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArrayWrapper.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArrayWrapper.h index 2e9555b5374..986fad8b674 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArrayWrapper.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ArrayWrapper.h @@ -10,6 +10,9 @@ #ifndef EIGEN_ARRAYWRAPPER_H #define EIGEN_ARRAYWRAPPER_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class ArrayWrapper @@ -26,12 +29,12 @@ namespace Eigen { namespace internal { template struct traits > - : public traits::type > + : public traits > { typedef ArrayXpr XprKind; // Let's remove NestByRefBit enum { - Flags0 = traits::type >::Flags, + Flags0 = traits >::Flags, LvalueBitFlag = is_lvalue::value ? LvalueBit : 0, Flags = (Flags0 & ~(NestByRefBit | LvalueBit)) | LvalueBitFlag }; @@ -45,13 +48,13 @@ class ArrayWrapper : public ArrayBase > typedef ArrayBase Base; EIGEN_DENSE_PUBLIC_INTERFACE(ArrayWrapper) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ArrayWrapper) - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; - typedef typename internal::conditional< + typedef std::conditional_t< internal::is_lvalue::value, Scalar, const Scalar - >::type ScalarWithConstIfNotLvalue; + > ScalarWithConstIfNotLvalue; typedef typename internal::ref_selector::non_const_type NestedExpressionType; @@ -91,7 +94,7 @@ class ArrayWrapper : public ArrayBase > inline void evalTo(Dest& dst) const { dst = m_expression; } EIGEN_DEVICE_FUNC - const typename internal::remove_all::type& + const internal::remove_all_t& nestedExpression() const { return m_expression; @@ -124,12 +127,12 @@ class ArrayWrapper : public ArrayBase > namespace internal { template struct traits > - : public traits::type > + : public traits > { typedef MatrixXpr XprKind; // Let's remove NestByRefBit enum { - Flags0 = traits::type >::Flags, + Flags0 = traits >::Flags, LvalueBitFlag = is_lvalue::value ? LvalueBit : 0, Flags = (Flags0 & ~(NestByRefBit | LvalueBit)) | LvalueBitFlag }; @@ -143,13 +146,13 @@ class MatrixWrapper : public MatrixBase > typedef MatrixBase > Base; EIGEN_DENSE_PUBLIC_INTERFACE(MatrixWrapper) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(MatrixWrapper) - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; - typedef typename internal::conditional< - internal::is_lvalue::value, - Scalar, - const Scalar - >::type ScalarWithConstIfNotLvalue; + typedef std::conditional_t< + internal::is_lvalue::value, + Scalar, + const Scalar + > ScalarWithConstIfNotLvalue; typedef typename internal::ref_selector::non_const_type NestedExpressionType; @@ -185,7 +188,7 @@ class MatrixWrapper : public MatrixBase > } EIGEN_DEVICE_FUNC - const typename internal::remove_all::type& + const internal::remove_all_t& nestedExpression() const { return m_expression; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Assign.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Assign.h index 655412efd7f..374558cbc0e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Assign.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Assign.h @@ -12,6 +12,9 @@ #ifndef EIGEN_ASSIGN_H #define EIGEN_ASSIGN_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/AssignEvaluator.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/AssignEvaluator.h index 7d76f0c256f..8d8dc2fed41 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/AssignEvaluator.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/AssignEvaluator.h @@ -12,6 +12,9 @@ #ifndef EIGEN_ASSIGN_EVALUATOR_H #define EIGEN_ASSIGN_EVALUATOR_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { // This implementation is based on Assign.h @@ -40,7 +43,7 @@ struct copy_using_evaluator_traits DstAlignment = DstEvaluator::Alignment, SrcAlignment = SrcEvaluator::Alignment, DstHasDirectAccess = (DstFlags & DirectAccessBit) == DirectAccessBit, - JointAlignment = EIGEN_PLAIN_ENUM_MIN(DstAlignment,SrcAlignment) + JointAlignment = plain_enum_min(DstAlignment, SrcAlignment) }; private: @@ -51,8 +54,8 @@ struct copy_using_evaluator_traits InnerMaxSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::MaxSizeAtCompileTime) : int(DstFlags)&RowMajorBit ? int(Dst::MaxColsAtCompileTime) : int(Dst::MaxRowsAtCompileTime), - RestrictedInnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(InnerSize,MaxPacketSize), - RestrictedLinearSize = EIGEN_SIZE_MIN_PREFER_FIXED(Dst::SizeAtCompileTime,MaxPacketSize), + RestrictedInnerSize = min_size_prefer_fixed(InnerSize, MaxPacketSize), + RestrictedLinearSize = min_size_prefer_fixed(Dst::SizeAtCompileTime, MaxPacketSize), OuterStride = int(outer_stride_at_compile_time::ret), MaxSizeAtCompileTime = Dst::SizeAtCompileTime }; @@ -111,7 +114,7 @@ struct copy_using_evaluator_traits || int(Traversal) == SliceVectorizedTraversal }; - typedef typename conditional::type PacketType; + typedef std::conditional_t PacketType; private: enum { @@ -216,7 +219,7 @@ struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling template struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&) { } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel&) { } }; template @@ -285,7 +288,7 @@ struct copy_using_evaluator_innervec_CompleteUnrolling template struct copy_using_evaluator_innervec_CompleteUnrolling { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&) { } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel&) { } }; template @@ -325,10 +328,9 @@ struct dense_assignment_loop; template struct dense_assignment_loop { - EIGEN_DEVICE_FUNC static void EIGEN_STRONG_INLINE run(Kernel& /*kernel*/) + EIGEN_DEVICE_FUNC static void EIGEN_STRONG_INLINE EIGEN_CONSTEXPR run(Kernel& /*kernel*/) { - typedef typename Kernel::DstEvaluatorType::XprType DstXprType; - EIGEN_STATIC_ASSERT(int(DstXprType::SizeAtCompileTime) == 0, + EIGEN_STATIC_ASSERT(int(Kernel::DstEvaluatorType::XprType::SizeAtCompileTime) == 0, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT) } }; @@ -386,7 +388,7 @@ struct unaligned_dense_assignment_loop { // if IsAligned = true, then do nothing template - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&, Index, Index) {} + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel&, Index, Index) {} }; template <> @@ -402,7 +404,7 @@ struct unaligned_dense_assignment_loop Index end) #else template - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel, + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel &kernel, Index start, Index end) #endif @@ -412,10 +414,34 @@ struct unaligned_dense_assignment_loop } }; +template +struct copy_using_evaluator_linearvec_CompleteUnrolling { + // FIXME: this is not very clean, perhaps this information should be provided by the kernel? + typedef typename Kernel::DstEvaluatorType DstEvaluatorType; + typedef typename DstEvaluatorType::XprType DstXprType; + typedef typename Kernel::PacketType PacketType; + + enum { + SrcAlignment = Kernel::AssignmentTraits::SrcAlignment, + DstAlignment = Kernel::AssignmentTraits::DstAlignment + }; + + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) { + kernel.template assignPacket(Index); + enum { NextIndex = Index + unpacket_traits::size }; + copy_using_evaluator_linearvec_CompleteUnrolling::run(kernel); + } +}; + +template +struct copy_using_evaluator_linearvec_CompleteUnrolling { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel&) {} +}; + template struct dense_assignment_loop { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel &kernel) { const Index size = kernel.size(); typedef typename Kernel::Scalar Scalar; @@ -443,7 +469,7 @@ struct dense_assignment_loop template struct dense_assignment_loop { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel &kernel) { typedef typename Kernel::DstEvaluatorType::XprType DstXprType; typedef typename Kernel::PacketType PacketType; @@ -452,8 +478,8 @@ struct dense_assignment_loop::size, alignedSize = (int(size)/packetSize)*packetSize }; - copy_using_evaluator_innervec_CompleteUnrolling::run(kernel); - copy_using_evaluator_DefaultTraversal_CompleteUnrolling::run(kernel); + copy_using_evaluator_linearvec_CompleteUnrolling::run(kernel); + copy_using_evaluator_LinearTraversal_CompleteUnrolling::run(kernel); } }; @@ -469,7 +495,7 @@ struct dense_assignment_loop SrcAlignment = Kernel::AssignmentTraits::SrcAlignment, DstAlignment = Kernel::AssignmentTraits::DstAlignment }; - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel &kernel) { const Index innerSize = kernel.innerSize(); const Index outerSize = kernel.outerSize(); @@ -511,7 +537,7 @@ struct dense_assignment_loop template struct dense_assignment_loop { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel &kernel) { const Index size = kernel.size(); for(Index i = 0; i < size; ++i) @@ -522,7 +548,7 @@ struct dense_assignment_loop template struct dense_assignment_loop { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel &kernel) { typedef typename Kernel::DstEvaluatorType::XprType DstXprType; copy_using_evaluator_LinearTraversal_CompleteUnrolling::run(kernel); @@ -536,7 +562,7 @@ struct dense_assignment_loop template struct dense_assignment_loop { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel &kernel) { typedef typename Kernel::Scalar Scalar; typedef typename Kernel::PacketType PacketType; @@ -549,7 +575,7 @@ struct dense_assignment_loop : int(Kernel::AssignmentTraits::DstAlignment) }; const Scalar *dst_ptr = kernel.dstDataPtr(); - if((!bool(dstIsAligned)) && (UIntPtr(dst_ptr) % sizeof(Scalar))>0) + if((!bool(dstIsAligned)) && (std::uintptr_t(dst_ptr) % sizeof(Scalar))>0) { // the pointer is not aligned-on scalar, so alignment is not possible return dense_assignment_loop::run(kernel); @@ -584,7 +610,7 @@ struct dense_assignment_loop template struct dense_assignment_loop { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel &kernel) { typedef typename Kernel::DstEvaluatorType::XprType DstXprType; typedef typename Kernel::PacketType PacketType; @@ -669,24 +695,24 @@ class generic_dense_assignment_kernel } - template + template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index row, Index col) { - m_functor.template assignPacket(&m_dst.coeffRef(row,col), m_src.template packet(row,col)); + m_functor.template assignPacket(&m_dst.coeffRef(row,col), m_src.template packet(row,col)); } - template + template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index index) { - m_functor.template assignPacket(&m_dst.coeffRef(index), m_src.template packet(index)); + m_functor.template assignPacket(&m_dst.coeffRef(index), m_src.template packet(index)); } - template + template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner) { Index row = rowIndexByOuterInner(outer, inner); Index col = colIndexByOuterInner(outer, inner); - assignPacket(row, col); + assignPacket(row, col); } EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index rowIndexByOuterInner(Index outer, Index inner) @@ -766,7 +792,7 @@ void resize_if_allowed(DstXprType &dst, const SrcXprType& src, const internal::a } template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(DstXprType& dst, const SrcXprType& src, const Functor &func) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_dense_assignment_loop(DstXprType& dst, const SrcXprType& src, const Functor &func) { typedef evaluator DstEvaluatorType; typedef evaluator SrcEvaluatorType; @@ -844,8 +870,8 @@ void call_assignment(const Dst& dst, const Src& src) // Deal with "assume-aliasing" template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if< evaluator_assume_aliasing::value, void*>::type = 0) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR +void call_assignment(Dst& dst, const Src& src, const Func& func, std::enable_if_t< evaluator_assume_aliasing::value, void*> = 0) { typename plain_matrix_type::type tmp(src); call_assignment_no_alias(dst, tmp, func); @@ -853,7 +879,7 @@ void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if::value, void*>::type = 0) +void call_assignment(Dst& dst, const Src& src, const Func& func, std::enable_if_t::value, void*> = 0) { call_assignment_no_alias(dst, src, func); } @@ -861,7 +887,7 @@ void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable // by-pass "assume-aliasing" // When there is no aliasing, we require that 'dst' has been properly resized template class StorageBase, typename Src, typename Func> -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment(NoAlias& dst, const Src& src, const Func& func) { call_assignment_no_alias(dst.expression(), src, func); @@ -869,7 +895,7 @@ void call_assignment(NoAlias& dst, const Src& src, const Func& template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment_no_alias(Dst& dst, const Src& src, const Func& func) { enum { @@ -878,8 +904,8 @@ void call_assignment_no_alias(Dst& dst, const Src& src, const Func& func) ) && int(Dst::SizeAtCompileTime) != 1 }; - typedef typename internal::conditional, Dst>::type ActualDstTypeCleaned; - typedef typename internal::conditional, Dst&>::type ActualDstType; + typedef std::conditional_t, Dst> ActualDstTypeCleaned; + typedef std::conditional_t, Dst&> ActualDstType; ActualDstType actualDst(dst); // TODO check whether this is the right place to perform these checks: @@ -911,14 +937,14 @@ void call_restricted_packet_assignment_no_alias(Dst& dst, const Src& src, const } template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment_no_alias(Dst& dst, const Src& src) { call_assignment_no_alias(dst, src, internal::assign_op()); } template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src, const Func& func) { // TODO check whether this is the right place to perform these checks: @@ -929,14 +955,14 @@ void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src, const Func& Assignment::run(dst, src, func); } template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src) { call_assignment_no_alias_no_transpose(dst, src, internal::assign_op()); } // forward declaration -template void check_for_aliasing(const Dst &dst, const Src &src); +template EIGEN_DEVICE_FUNC void check_for_aliasing(const Dst &dst, const Src &src); // Generic Dense to Dense assignment // Note that the last template argument "Weak" is needed to make it possible to perform diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Assign_MKL.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Assign_MKL.h index c6140d185ba..448dae2b6e5 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Assign_MKL.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Assign_MKL.h @@ -34,6 +34,9 @@ #ifndef EIGEN_ASSIGN_VML_H #define EIGEN_ASSIGN_VML_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -82,7 +85,7 @@ class vml_assign_traits #define EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, VMLOP, EIGENTYPE, VMLTYPE, VMLMODE) \ template< typename DstXprType, typename SrcXprNested> \ struct Assignment, SrcXprNested>, assign_op, \ - Dense2Dense, typename enable_if::EnableVml>::type> { \ + Dense2Dense, std::enable_if_t::EnableVml>> { \ typedef CwiseUnaryOp, SrcXprNested> SrcXprType; \ static void run(DstXprType &dst, const SrcXprType &src, const assign_op &func) { \ resize_if_allowed(dst, src, func); \ @@ -137,12 +140,13 @@ EIGEN_MKL_VML_DECLARE_UNARY_CALLS_CPLX(arg, Arg, _) EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(round, Round, _) EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(floor, Floor, _) EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(ceil, Ceil, _) +EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(cbrt, Cbrt, _) #define EIGEN_MKL_VML_DECLARE_POW_CALL(EIGENOP, VMLOP, EIGENTYPE, VMLTYPE, VMLMODE) \ template< typename DstXprType, typename SrcXprNested, typename Plain> \ struct Assignment, SrcXprNested, \ const CwiseNullaryOp,Plain> >, assign_op, \ - Dense2Dense, typename enable_if::EnableVml>::type> { \ + Dense2Dense, std::enable_if_t::EnableVml>> { \ typedef CwiseBinaryOp, SrcXprNested, \ const CwiseNullaryOp,Plain> > SrcXprType; \ static void run(DstXprType &dst, const SrcXprType &src, const assign_op &func) { \ diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/BandMatrix.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/BandMatrix.h index 878c0240ac1..8955bcd331b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/BandMatrix.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/BandMatrix.h @@ -10,6 +10,9 @@ #ifndef EIGEN_BANDMATRIX_H #define EIGEN_BANDMATRIX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -41,7 +44,7 @@ class BandMatrixBase : public EigenBase DataRowsAtCompileTime = ((Supers!=Dynamic) && (Subs!=Dynamic)) ? 1 + Supers + Subs : Dynamic, - SizeAtCompileTime = EIGEN_SIZE_MIN_PREFER_DYNAMIC(RowsAtCompileTime,ColsAtCompileTime) + SizeAtCompileTime = min_size_prefer_dynamic(RowsAtCompileTime,ColsAtCompileTime) }; public: @@ -96,13 +99,13 @@ class BandMatrixBase : public EigenBase DiagonalSize = (RowsAtCompileTime==Dynamic || ColsAtCompileTime==Dynamic) ? Dynamic : (ActualIndex<0 - ? EIGEN_SIZE_MIN_PREFER_DYNAMIC(ColsAtCompileTime, RowsAtCompileTime + ActualIndex) - : EIGEN_SIZE_MIN_PREFER_DYNAMIC(RowsAtCompileTime, ColsAtCompileTime - ActualIndex)) + ? min_size_prefer_dynamic(ColsAtCompileTime, RowsAtCompileTime + ActualIndex) + : min_size_prefer_dynamic(RowsAtCompileTime, ColsAtCompileTime - ActualIndex)) }; typedef Block BuildType; - typedef typename internal::conditional,BuildType >, - BuildType>::type Type; + BuildType> Type; }; /** \returns a vector expression of the \a N -th sub or super diagonal */ @@ -161,12 +164,12 @@ class BandMatrixBase : public EigenBase * * \brief Represents a rectangular matrix with a banded storage * - * \tparam _Scalar Numeric type, i.e. float, double, int - * \tparam _Rows Number of rows, or \b Dynamic - * \tparam _Cols Number of columns, or \b Dynamic - * \tparam _Supers Number of super diagonal - * \tparam _Subs Number of sub diagonal - * \tparam _Options A combination of either \b #RowMajor or \b #ColMajor, and of \b #SelfAdjoint + * \tparam Scalar_ Numeric type, i.e. float, double, int + * \tparam Rows_ Number of rows, or \b Dynamic + * \tparam Cols_ Number of columns, or \b Dynamic + * \tparam Supers_ Number of super diagonal + * \tparam Subs_ Number of sub diagonal + * \tparam Options_ A combination of either \b #RowMajor or \b #ColMajor, and of \b #SelfAdjoint * The former controls \ref TopicStorageOrders "storage order", and defaults to * column-major. The latter controls whether the matrix represents a selfadjoint * matrix in which case either Supers of Subs have to be null. @@ -174,29 +177,29 @@ class BandMatrixBase : public EigenBase * \sa class TridiagonalMatrix */ -template -struct traits > +template +struct traits > { - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef Dense StorageKind; typedef Eigen::Index StorageIndex; enum { CoeffReadCost = NumTraits::ReadCost, - RowsAtCompileTime = _Rows, - ColsAtCompileTime = _Cols, - MaxRowsAtCompileTime = _Rows, - MaxColsAtCompileTime = _Cols, + RowsAtCompileTime = Rows_, + ColsAtCompileTime = Cols_, + MaxRowsAtCompileTime = Rows_, + MaxColsAtCompileTime = Cols_, Flags = LvalueBit, - Supers = _Supers, - Subs = _Subs, - Options = _Options, + Supers = Supers_, + Subs = Subs_, + Options = Options_, DataRowsAtCompileTime = ((Supers!=Dynamic) && (Subs!=Dynamic)) ? 1 + Supers + Subs : Dynamic }; typedef Matrix CoefficientsType; }; -template -class BandMatrix : public BandMatrixBase > +template +class BandMatrix : public BandMatrixBase > { public: @@ -233,32 +236,32 @@ class BandMatrix : public BandMatrixBase m_subs; }; -template +template class BandMatrixWrapper; -template -struct traits > +template +struct traits > { - typedef typename _CoefficientsType::Scalar Scalar; - typedef typename _CoefficientsType::StorageKind StorageKind; - typedef typename _CoefficientsType::StorageIndex StorageIndex; + typedef typename CoefficientsType_::Scalar Scalar; + typedef typename CoefficientsType_::StorageKind StorageKind; + typedef typename CoefficientsType_::StorageIndex StorageIndex; enum { - CoeffReadCost = internal::traits<_CoefficientsType>::CoeffReadCost, - RowsAtCompileTime = _Rows, - ColsAtCompileTime = _Cols, - MaxRowsAtCompileTime = _Rows, - MaxColsAtCompileTime = _Cols, + CoeffReadCost = internal::traits::CoeffReadCost, + RowsAtCompileTime = Rows_, + ColsAtCompileTime = Cols_, + MaxRowsAtCompileTime = Rows_, + MaxColsAtCompileTime = Cols_, Flags = LvalueBit, - Supers = _Supers, - Subs = _Subs, - Options = _Options, + Supers = Supers_, + Subs = Subs_, + Options = Options_, DataRowsAtCompileTime = ((Supers!=Dynamic) && (Subs!=Dynamic)) ? 1 + Supers + Subs : Dynamic }; - typedef _CoefficientsType CoefficientsType; + typedef CoefficientsType_ CoefficientsType; }; -template -class BandMatrixWrapper : public BandMatrixBase > +template +class BandMatrixWrapper : public BandMatrixBase > { public: @@ -266,12 +269,12 @@ class BandMatrixWrapper : public BandMatrixBase::CoefficientsType CoefficientsType; typedef typename internal::traits::StorageIndex StorageIndex; - explicit inline BandMatrixWrapper(const CoefficientsType& coeffs, Index rows=_Rows, Index cols=_Cols, Index supers=_Supers, Index subs=_Subs) + explicit inline BandMatrixWrapper(const CoefficientsType& coeffs, Index rows=Rows_, Index cols=Cols_, Index supers=Supers_, Index subs=Subs_) : m_coeffs(coeffs), m_rows(rows), m_supers(supers), m_subs(subs) { EIGEN_UNUSED_VARIABLE(cols); - //internal::assert(coeffs.cols()==cols() && (supers()+subs()+1)==coeffs.rows()); + // eigen_assert(coeffs.cols()==cols() && (supers()+subs()+1)==coeffs.rows()); } /** \returns the number of columns */ @@ -291,9 +294,9 @@ class BandMatrixWrapper : public BandMatrixBase m_rows; - internal::variable_if_dynamic m_supers; - internal::variable_if_dynamic m_subs; + internal::variable_if_dynamic m_rows; + internal::variable_if_dynamic m_supers; + internal::variable_if_dynamic m_subs; }; /** @@ -330,16 +333,16 @@ class TridiagonalMatrix : public BandMatrix -struct evaluator_traits > - : public evaluator_traits_base > +template +struct evaluator_traits > + : public evaluator_traits_base > { typedef BandShape Shape; }; -template -struct evaluator_traits > - : public evaluator_traits_base > +template +struct evaluator_traits > + : public evaluator_traits_base > { typedef BandShape Shape; }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Block.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Block.h index d0b95d50b47..248d297ae4c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Block.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Block.h @@ -11,51 +11,55 @@ #ifndef EIGEN_BLOCK_H #define EIGEN_BLOCK_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template -struct traits > : traits +template +struct traits > : traits { - typedef typename traits::Scalar Scalar; - typedef typename traits::StorageKind StorageKind; - typedef typename traits::XprKind XprKind; - typedef typename ref_selector::type XprTypeNested; - typedef typename remove_reference::type _XprTypeNested; + typedef typename traits::Scalar Scalar; + typedef typename traits::StorageKind StorageKind; + typedef typename traits::XprKind XprKind; + typedef typename ref_selector::type XprTypeNested; + typedef std::remove_reference_t XprTypeNested_; enum{ - MatrixRows = traits::RowsAtCompileTime, - MatrixCols = traits::ColsAtCompileTime, + MatrixRows = traits::RowsAtCompileTime, + MatrixCols = traits::ColsAtCompileTime, RowsAtCompileTime = MatrixRows == 0 ? 0 : BlockRows, ColsAtCompileTime = MatrixCols == 0 ? 0 : BlockCols, MaxRowsAtCompileTime = BlockRows==0 ? 0 : RowsAtCompileTime != Dynamic ? int(RowsAtCompileTime) - : int(traits::MaxRowsAtCompileTime), + : int(traits::MaxRowsAtCompileTime), MaxColsAtCompileTime = BlockCols==0 ? 0 : ColsAtCompileTime != Dynamic ? int(ColsAtCompileTime) - : int(traits::MaxColsAtCompileTime), + : int(traits::MaxColsAtCompileTime), - XprTypeIsRowMajor = (int(traits::Flags)&RowMajorBit) != 0, + XprTypeIsRowMajor = (int(traits::Flags)&RowMajorBit) != 0, IsRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1 : (MaxColsAtCompileTime==1&&MaxRowsAtCompileTime!=1) ? 0 : XprTypeIsRowMajor, HasSameStorageOrderAsXprType = (IsRowMajor == XprTypeIsRowMajor), InnerSize = IsRowMajor ? int(ColsAtCompileTime) : int(RowsAtCompileTime), InnerStrideAtCompileTime = HasSameStorageOrderAsXprType - ? int(inner_stride_at_compile_time::ret) - : int(outer_stride_at_compile_time::ret), + ? int(inner_stride_at_compile_time::ret) + : int(outer_stride_at_compile_time::ret), OuterStrideAtCompileTime = HasSameStorageOrderAsXprType - ? int(outer_stride_at_compile_time::ret) - : int(inner_stride_at_compile_time::ret), + ? int(outer_stride_at_compile_time::ret) + : int(inner_stride_at_compile_time::ret), // FIXME, this traits is rather specialized for dense object and it needs to be cleaned further - FlagsLvalueBit = is_lvalue::value ? LvalueBit : 0, + FlagsLvalueBit = is_lvalue::value ? LvalueBit : 0, FlagsRowMajorBit = IsRowMajor ? RowMajorBit : 0, - Flags = (traits::Flags & (DirectAccessBit | (InnerPanel?CompressedAccessBit:0))) | FlagsLvalueBit | FlagsRowMajorBit, + Flags = (traits::Flags & (DirectAccessBit | (InnerPanel_?CompressedAccessBit:0))) | FlagsLvalueBit | FlagsRowMajorBit, // FIXME DirectAccessBit should not be handled by expressions // // Alignment is needed by MapBase's assertions // We can sefely set it to false here. Internal alignment errors will be detected by an eigen_internal_assert in the respective evaluator - Alignment = 0 + Alignment = 0, + InnerPanel = InnerPanel_ ? 1 : 0 }; }; @@ -82,7 +86,7 @@ template(Index,Index) and * most of the time this is the only way it is used. * - * However, if you want to directly maniputate block expressions, + * However, if you want to directly manipulate block expressions, * for instance if you want to write a function returning such an expression, you * will need to use this class. * @@ -104,13 +108,14 @@ template class : public BlockImpl::StorageKind> { typedef BlockImpl::StorageKind> Impl; + using BlockHelper = internal::block_xpr_helper; public: //typedef typename Impl::Base Base; typedef Impl Base; EIGEN_GENERIC_PUBLIC_INTERFACE(Block) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Block) - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; /** Column or Row constructor */ @@ -146,9 +151,25 @@ template class eigen_assert(startRow >= 0 && blockRows >= 0 && startRow <= xpr.rows() - blockRows && startCol >= 0 && blockCols >= 0 && startCol <= xpr.cols() - blockCols); } + + // convert nested blocks (e.g. Block>) to a simple block expression (Block) + + using ConstUnwindReturnType = Block; + using UnwindReturnType = Block; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ConstUnwindReturnType unwind() const { + return ConstUnwindReturnType(BlockHelper::base(*this), BlockHelper::row(*this, 0), BlockHelper::col(*this, 0), + this->rows(), this->cols()); + } + + template ::value>> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE UnwindReturnType unwind() { + return UnwindReturnType(BlockHelper::base(*this), BlockHelper::row(*this, 0), BlockHelper::col(*this, 0), + this->rows(), this->cols()); + } }; -// The generic default implementation for dense block simplu forward to the internal::BlockImpl_dense +// The generic default implementation for dense block simply forward to the internal::BlockImpl_dense // that must be specialized for direct and non-direct access... template class BlockImpl @@ -295,7 +316,7 @@ template::type& nestedExpression() const + const internal::remove_all_t& nestedExpression() const { return m_xpr; } @@ -334,6 +355,17 @@ class BlockImpl_dense enum { XprTypeIsRowMajor = (int(traits::Flags)&RowMajorBit) != 0 }; + + /** \internal Returns base+offset (unless base is null, in which case returns null). + * Adding an offset to nullptr is undefined behavior, so we must avoid it. + */ + template + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR EIGEN_ALWAYS_INLINE + static Scalar* add_to_nullable_pointer(Scalar* base, Index offset) + { + return base != nullptr ? base+offset : nullptr; + } + public: typedef MapBase Base; @@ -344,8 +376,9 @@ class BlockImpl_dense */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockImpl_dense(XprType& xpr, Index i) - : Base(xpr.data() + i * ( ((BlockRows==1) && (BlockCols==XprType::ColsAtCompileTime) && (!XprTypeIsRowMajor)) - || ((BlockRows==XprType::RowsAtCompileTime) && (BlockCols==1) && ( XprTypeIsRowMajor)) ? xpr.innerStride() : xpr.outerStride()), + : Base((BlockRows == 0 || BlockCols == 0) ? nullptr : add_to_nullable_pointer(xpr.data(), + i * ( ((BlockRows==1) && (BlockCols==XprType::ColsAtCompileTime) && (!XprTypeIsRowMajor)) + || ((BlockRows==XprType::RowsAtCompileTime) && (BlockCols==1) && ( XprTypeIsRowMajor)) ? xpr.innerStride() : xpr.outerStride())), BlockRows==1 ? 1 : xpr.rows(), BlockCols==1 ? 1 : xpr.cols()), m_xpr(xpr), @@ -359,7 +392,8 @@ class BlockImpl_dense */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockImpl_dense(XprType& xpr, Index startRow, Index startCol) - : Base(xpr.data()+xpr.innerStride()*(XprTypeIsRowMajor?startCol:startRow) + xpr.outerStride()*(XprTypeIsRowMajor?startRow:startCol)), + : Base((BlockRows == 0 || BlockCols == 0) ? nullptr : add_to_nullable_pointer(xpr.data(), + xpr.innerStride()*(XprTypeIsRowMajor?startCol:startRow) + xpr.outerStride()*(XprTypeIsRowMajor?startRow:startCol))), m_xpr(xpr), m_startRow(startRow), m_startCol(startCol) { init(); @@ -371,14 +405,16 @@ class BlockImpl_dense BlockImpl_dense(XprType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) - : Base(xpr.data()+xpr.innerStride()*(XprTypeIsRowMajor?startCol:startRow) + xpr.outerStride()*(XprTypeIsRowMajor?startRow:startCol), blockRows, blockCols), + : Base((blockRows == 0 || blockCols == 0) ? nullptr : add_to_nullable_pointer(xpr.data(), + xpr.innerStride()*(XprTypeIsRowMajor?startCol:startRow) + xpr.outerStride()*(XprTypeIsRowMajor?startRow:startCol)), + blockRows, blockCols), m_xpr(xpr), m_startRow(startRow), m_startCol(startCol) { init(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const typename internal::remove_all::type& nestedExpression() const EIGEN_NOEXCEPT + const internal::remove_all_t& nestedExpression() const EIGEN_NOEXCEPT { return m_xpr; } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/BooleanRedux.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/BooleanRedux.h deleted file mode 100644 index 852de8b90a5..00000000000 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/BooleanRedux.h +++ /dev/null @@ -1,162 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2008 Gael Guennebaud -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_ALLANDANY_H -#define EIGEN_ALLANDANY_H - -namespace Eigen { - -namespace internal { - -template -struct all_unroller -{ - enum { - col = (UnrollCount-1) / Rows, - row = (UnrollCount-1) % Rows - }; - - EIGEN_DEVICE_FUNC static inline bool run(const Derived &mat) - { - return all_unroller::run(mat) && mat.coeff(row, col); - } -}; - -template -struct all_unroller -{ - EIGEN_DEVICE_FUNC static inline bool run(const Derived &/*mat*/) { return true; } -}; - -template -struct all_unroller -{ - EIGEN_DEVICE_FUNC static inline bool run(const Derived &) { return false; } -}; - -template -struct any_unroller -{ - enum { - col = (UnrollCount-1) / Rows, - row = (UnrollCount-1) % Rows - }; - - EIGEN_DEVICE_FUNC static inline bool run(const Derived &mat) - { - return any_unroller::run(mat) || mat.coeff(row, col); - } -}; - -template -struct any_unroller -{ - EIGEN_DEVICE_FUNC static inline bool run(const Derived & /*mat*/) { return false; } -}; - -template -struct any_unroller -{ - EIGEN_DEVICE_FUNC static inline bool run(const Derived &) { return false; } -}; - -} // end namespace internal - -/** \returns true if all coefficients are true - * - * Example: \include MatrixBase_all.cpp - * Output: \verbinclude MatrixBase_all.out - * - * \sa any(), Cwise::operator<() - */ -template -EIGEN_DEVICE_FUNC inline bool DenseBase::all() const -{ - typedef internal::evaluator Evaluator; - enum { - unroll = SizeAtCompileTime != Dynamic - && SizeAtCompileTime * (int(Evaluator::CoeffReadCost) + int(NumTraits::AddCost)) <= EIGEN_UNROLLING_LIMIT - }; - Evaluator evaluator(derived()); - if(unroll) - return internal::all_unroller::RowsAtCompileTime>::run(evaluator); - else - { - for(Index j = 0; j < cols(); ++j) - for(Index i = 0; i < rows(); ++i) - if (!evaluator.coeff(i, j)) return false; - return true; - } -} - -/** \returns true if at least one coefficient is true - * - * \sa all() - */ -template -EIGEN_DEVICE_FUNC inline bool DenseBase::any() const -{ - typedef internal::evaluator Evaluator; - enum { - unroll = SizeAtCompileTime != Dynamic - && SizeAtCompileTime * (int(Evaluator::CoeffReadCost) + int(NumTraits::AddCost)) <= EIGEN_UNROLLING_LIMIT - }; - Evaluator evaluator(derived()); - if(unroll) - return internal::any_unroller::RowsAtCompileTime>::run(evaluator); - else - { - for(Index j = 0; j < cols(); ++j) - for(Index i = 0; i < rows(); ++i) - if (evaluator.coeff(i, j)) return true; - return false; - } -} - -/** \returns the number of coefficients which evaluate to true - * - * \sa all(), any() - */ -template -EIGEN_DEVICE_FUNC inline Eigen::Index DenseBase::count() const -{ - return derived().template cast().template cast().sum(); -} - -/** \returns true is \c *this contains at least one Not A Number (NaN). - * - * \sa allFinite() - */ -template -inline bool DenseBase::hasNaN() const -{ -#if EIGEN_COMP_MSVC || (defined __FAST_MATH__) - return derived().array().isNaN().any(); -#else - return !((derived().array()==derived().array()).all()); -#endif -} - -/** \returns true if \c *this contains only finite numbers, i.e., no NaN and no +/-INF values. - * - * \sa hasNaN() - */ -template -inline bool DenseBase::allFinite() const -{ -#if EIGEN_COMP_MSVC || (defined __FAST_MATH__) - return derived().array().isFinite().all(); -#else - return !((derived()-derived()).hasNaN()); -#endif -} - -} // end namespace Eigen - -#endif // EIGEN_ALLANDANY_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CommaInitializer.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CommaInitializer.h index c0e29c75c22..51bf87619f6 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CommaInitializer.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CommaInitializer.h @@ -11,6 +11,9 @@ #ifndef EIGEN_COMMAINITIALIZER_H #define EIGEN_COMMAINITIALIZER_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class CommaInitializer @@ -45,7 +48,7 @@ struct CommaInitializer { eigen_assert(m_xpr.rows() >= other.rows() && m_xpr.cols() >= other.cols() && "Cannot comma-initialize a 0x0 matrix (operator<<)"); - m_xpr.block(0, 0, other.rows(), other.cols()) = other; + m_xpr.template block(0, 0, other.rows(), other.cols()) = other; } /* Copy/Move constructor which transfers ownership. This is crucial in diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ConditionEstimator.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ConditionEstimator.h index 51a2e5f1b6f..7d77f43d329 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ConditionEstimator.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ConditionEstimator.h @@ -10,6 +10,9 @@ #ifndef EIGEN_CONDITIONESTIMATOR_H #define EIGEN_CONDITIONESTIMATOR_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -160,12 +163,12 @@ rcond_estimate_helper(typename Decomposition::RealScalar matrix_norm, const Deco { typedef typename Decomposition::RealScalar RealScalar; eigen_assert(dec.rows() == dec.cols()); - if (dec.rows() == 0) return NumTraits::infinity(); - if (matrix_norm == RealScalar(0)) return RealScalar(0); - if (dec.rows() == 1) return RealScalar(1); + if (dec.rows() == 0) return NumTraits::infinity(); + if (numext::is_exactly_zero(matrix_norm)) return RealScalar(0); + if (dec.rows() == 1) return RealScalar(1); const RealScalar inverse_matrix_norm = rcond_invmatrix_L1_norm_estimate(dec); - return (inverse_matrix_norm == RealScalar(0) ? RealScalar(0) - : (RealScalar(1) / inverse_matrix_norm) / matrix_norm); + return (numext::is_exactly_zero(inverse_matrix_norm) ? RealScalar(0) + : (RealScalar(1) / inverse_matrix_norm) / matrix_norm); } } // namespace internal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CoreEvaluators.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CoreEvaluators.h index 0ff8c8deb82..7249f7ebac3 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CoreEvaluators.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CoreEvaluators.h @@ -13,6 +13,9 @@ #ifndef EIGEN_COREEVALUATORS_H #define EIGEN_COREEVALUATORS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -498,7 +501,7 @@ struct evaluator > : evaluator_base > { typedef CwiseNullaryOp XprType; - typedef typename internal::remove_all::type PlainObjectTypeCleaned; + typedef internal::remove_all_t PlainObjectTypeCleaned; enum { CoeffReadCost = internal::functor_traits::Cost, @@ -619,6 +622,207 @@ struct unary_evaluator, IndexBased > Data m_d; }; +// ----------------------- Casting --------------------- + +template +struct unary_evaluator, ArgType>, IndexBased> { + using CastOp = core_cast_op; + using XprType = CwiseUnaryOp; + + // Use the largest packet type by default + using SrcPacketType = typename packet_traits::type; + static constexpr int SrcPacketSize = unpacket_traits::size; + static constexpr int SrcPacketBytes = SrcPacketSize * sizeof(SrcType); + + enum { + CoeffReadCost = int(evaluator::CoeffReadCost) + int(functor_traits::Cost), + PacketAccess = functor_traits::PacketAccess, + ActualPacketAccessBit = PacketAccess ? PacketAccessBit : 0, + Flags = evaluator::Flags & (HereditaryBits | LinearAccessBit | ActualPacketAccessBit), + IsRowMajor = (evaluator::Flags & RowMajorBit), + Alignment = evaluator::Alignment + }; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& xpr) + : m_argImpl(xpr.nestedExpression()), m_rows(xpr.rows()), m_cols(xpr.cols()) { + EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits::Cost); + EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); + } + + template + using AltSrcScalarOp = std::enable_if_t<(unpacket_traits::size < SrcPacketSize && !find_packet_by_size::size>::value), bool>; + template + using SrcPacketArgs1 = std::enable_if_t<(find_packet_by_size::size>::value), bool>; + template + using SrcPacketArgs2 = std::enable_if_t<(unpacket_traits::size) == (2 * SrcPacketSize), bool>; + template + using SrcPacketArgs4 = std::enable_if_t<(unpacket_traits::size) == (4 * SrcPacketSize), bool>; + template + using SrcPacketArgs8 = std::enable_if_t<(unpacket_traits::size) == (8 * SrcPacketSize), bool>; + + template = true> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool check_array_bounds(Index, Index col, Index packetSize) const { + return col + packetSize <= cols(); + } + template = true> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool check_array_bounds(Index row, Index, Index packetSize) const { + return row + packetSize <= rows(); + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool check_array_bounds(Index index, Index packetSize) const { + return index + packetSize <= size(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE SrcType srcCoeff(Index row, Index col, Index offset) const { + Index actualRow = IsRowMajor ? row : row + offset; + Index actualCol = IsRowMajor ? col + offset : col; + return m_argImpl.coeff(actualRow, actualCol); + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE SrcType srcCoeff(Index index, Index offset) const { + Index actualIndex = index + offset; + return m_argImpl.coeff(actualIndex); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstType coeff(Index row, Index col) const { + return cast(srcCoeff(row, col, 0)); + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstType coeff(Index index) const { return cast(srcCoeff(index, 0)); } + + template + EIGEN_STRONG_INLINE PacketType srcPacket(Index row, Index col, Index offset) const { + constexpr int PacketSize = unpacket_traits::size; + Index actualRow = IsRowMajor ? row : row + (offset * PacketSize); + Index actualCol = IsRowMajor ? col + (offset * PacketSize) : col; + eigen_assert(check_array_bounds(actualRow, actualCol, PacketSize) && "Array index out of bounds"); + return m_argImpl.template packet(actualRow, actualCol); + } + template + EIGEN_STRONG_INLINE PacketType srcPacket(Index index, Index offset) const { + constexpr int PacketSize = unpacket_traits::size; + Index actualIndex = index + (offset * PacketSize); + eigen_assert(check_array_bounds(actualIndex, PacketSize) && "Array index out of bounds"); + return m_argImpl.template packet(actualIndex); + } + + // There is no source packet type with equal or fewer elements than DstPacketType. + // This is problematic as the evaluation loop may attempt to access data outside the bounds of the array. + // For example, consider the cast utilizing pcast with an array of size 4: {0.0f,1.0f,2.0f,3.0f}. + // The first iteration of the evaulation loop will load 16 bytes: {0.0f,1.0f,2.0f,3.0f} and cast to {0.0,1.0}, which is acceptable. + // The second iteration will load 16 bytes: {2.0f,3.0f,?,?}, which is outside the bounds of the array. + + // Instead, perform runtime check to determine if the load would access data outside the bounds of the array. + // If not, perform full load. Otherwise, revert to a scalar loop to perform a partial load. + // In either case, perform a vectorized cast of the source packet. + template = true> + EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + constexpr int DstPacketSize = unpacket_traits::size; + constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); + constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode); + SrcPacketType src; + if (EIGEN_PREDICT_TRUE(check_array_bounds(row, col, SrcPacketSize))) { + src = srcPacket(row, col, 0); + } else { + Array srcArray; + for (size_t k = 0; k < DstPacketSize; k++) srcArray[k] = srcCoeff(row, col, k); + for (size_t k = DstPacketSize; k < SrcPacketSize; k++) srcArray[k] = SrcType(0); + src = pload(srcArray.data()); + } + return pcast(src); + } + // Use the source packet type with the same size as DstPacketType, if it exists + template = true> + EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + constexpr int DstPacketSize = unpacket_traits::size; + using SizedSrcPacketType = typename find_packet_by_size::type; + constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); + constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode); + return pcast( + srcPacket(row, col, 0)); + } + // unpacket_traits::size == 2 * SrcPacketSize + template = true> + EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); + return pcast( + srcPacket(row, col, 0), srcPacket(row, col, 1)); + } + // unpacket_traits::size == 4 * SrcPacketSize + template = true> + EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); + return pcast( + srcPacket(row, col, 0), srcPacket(row, col, 1), + srcPacket(row, col, 2), srcPacket(row, col, 3)); + } + // unpacket_traits::size == 8 * SrcPacketSize + template = true> + EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); + return pcast( + srcPacket(row, col, 0), srcPacket(row, col, 1), + srcPacket(row, col, 2), srcPacket(row, col, 3), + srcPacket(row, col, 4), srcPacket(row, col, 5), + srcPacket(row, col, 6), srcPacket(row, col, 7)); + } + + // Analagous routines for linear access. + template = true> + EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + constexpr int DstPacketSize = unpacket_traits::size; + constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); + constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode); + SrcPacketType src; + if (EIGEN_PREDICT_TRUE(check_array_bounds(index, SrcPacketSize))) { + src = srcPacket(index, 0); + } else { + Array srcArray; + for (size_t k = 0; k < DstPacketSize; k++) srcArray[k] = srcCoeff(index, k); + for (size_t k = DstPacketSize; k < SrcPacketSize; k++) srcArray[k] = SrcType(0); + src = pload(srcArray.data()); + } + return pcast(src); + } + template = true> + EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + constexpr int DstPacketSize = unpacket_traits::size; + using SizedSrcPacketType = typename find_packet_by_size::type; + constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); + constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode); + return pcast( + srcPacket(index, 0)); + } + template = true> + EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); + return pcast( + srcPacket(index, 0), srcPacket(index, 1)); + } + template = true> + EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); + return pcast( + srcPacket(index, 0), srcPacket(index, 1), + srcPacket(index, 2), srcPacket(index, 3)); + } + template = true> + EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); + return pcast( + srcPacket(index, 0), srcPacket(index, 1), + srcPacket(index, 2), srcPacket(index, 3), + srcPacket(index, 4), srcPacket(index, 5), + srcPacket(index, 6), srcPacket(index, 7)); + } + + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rows() const { return m_rows; } + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index cols() const { return m_cols; } + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index size() const { return m_rows * m_cols; } + + protected: + const evaluator m_argImpl; + const variable_if_dynamic m_rows; + const variable_if_dynamic m_cols; +}; + // -------------------- CwiseTernaryOp -------------------- // this is a ternary expression @@ -655,9 +859,9 @@ struct ternary_evaluator, IndexBased ) ), Flags = (Flags0 & ~RowMajorBit) | (Arg1Flags & RowMajorBit), - Alignment = EIGEN_PLAIN_ENUM_MIN( - EIGEN_PLAIN_ENUM_MIN(evaluator::Alignment, evaluator::Alignment), - evaluator::Alignment) + Alignment = plain_enum_min( + plain_enum_min(evaluator::Alignment, evaluator::Alignment), + evaluator::Alignment) }; EIGEN_DEVICE_FUNC explicit ternary_evaluator(const XprType& xpr) : m_d(xpr) @@ -751,7 +955,7 @@ struct binary_evaluator, IndexBased, IndexBase ) ), Flags = (Flags0 & ~RowMajorBit) | (LhsFlags & RowMajorBit), - Alignment = EIGEN_PLAIN_ENUM_MIN(evaluator::Alignment,evaluator::Alignment) + Alignment = plain_enum_min(evaluator::Alignment, evaluator::Alignment) }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -810,11 +1014,11 @@ struct binary_evaluator, IndexBased, IndexBase // -------------------- CwiseUnaryView -------------------- -template -struct unary_evaluator, IndexBased> - : evaluator_base > +template +struct unary_evaluator, IndexBased> + : evaluator_base > { - typedef CwiseUnaryView XprType; + typedef CwiseUnaryView XprType; enum { CoeffReadCost = int(evaluator::CoeffReadCost) + int(functor_traits::Cost), @@ -900,7 +1104,8 @@ struct mapbase_evaluator : evaluator_base m_innerStride(map.innerStride()), m_outerStride(map.outerStride()) { - EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(evaluator::Flags&PacketAccessBit, internal::inner_stride_at_compile_time::ret==1), + EIGEN_STATIC_ASSERT(check_implication((evaluator::Flags & PacketAccessBit) != 0, + internal::inner_stride_at_compile_time::ret == 1), PACKET_ACCESS_REQUIRES_TO_HAVE_INNER_STRIDE_FIXED_TO_1); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } @@ -1072,7 +1277,7 @@ struct evaluator > Alignment0 = (InnerPanel && (OuterStrideAtCompileTime!=Dynamic) && (OuterStrideAtCompileTime!=0) && (((OuterStrideAtCompileTime * int(sizeof(Scalar))) % int(PacketAlignment)) == 0)) ? int(PacketAlignment) : 0, - Alignment = EIGEN_PLAIN_ENUM_MIN(evaluator::Alignment, Alignment0) + Alignment = plain_enum_min(evaluator::Alignment, Alignment0) }; typedef block_evaluator block_evaluator_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -1222,8 +1427,8 @@ struct block_evaluator(block) { - // TODO: for the 3.3 release, this should be turned to an internal assertion, but let's keep it as is for the beta lifetime - eigen_assert(((internal::UIntPtr(block.data()) % EIGEN_PLAIN_ENUM_MAX(1,evaluator::Alignment)) == 0) && "data is not aligned"); + eigen_internal_assert((internal::is_constant_evaluated() || (std::uintptr_t(block.data()) % plain_enum_max(1,evaluator::Alignment)) == 0) \ + && "data is not aligned"); } }; @@ -1239,12 +1444,12 @@ struct evaluator > typedef Select XprType; enum { CoeffReadCost = evaluator::CoeffReadCost - + EIGEN_PLAIN_ENUM_MAX(evaluator::CoeffReadCost, - evaluator::CoeffReadCost), + + plain_enum_max(evaluator::CoeffReadCost, + evaluator::CoeffReadCost), Flags = (unsigned int)evaluator::Flags & evaluator::Flags & HereditaryBits, - Alignment = EIGEN_PLAIN_ENUM_MIN(evaluator::Alignment, evaluator::Alignment) + Alignment = plain_enum_min(evaluator::Alignment, evaluator::Alignment) }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -1295,7 +1500,7 @@ struct unary_evaluator > Factor = (RowFactor==Dynamic || ColFactor==Dynamic) ? Dynamic : RowFactor*ColFactor }; typedef typename internal::nested_eval::type ArgTypeNested; - typedef typename internal::remove_all::type ArgTypeNestedCleaned; + typedef internal::remove_all_t ArgTypeNestedCleaned; enum { CoeffReadCost = evaluator::CoeffReadCost, @@ -1379,7 +1584,7 @@ template struct evaluator_wrapper_base : evaluator_base { - typedef typename remove_all::type ArgType; + typedef remove_all_t ArgType; enum { CoeffReadCost = evaluator::CoeffReadCost, Flags = evaluator::Flags, @@ -1720,14 +1925,14 @@ struct evaluator > EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : m_result(xpr.arg()) { - ::new (static_cast(this)) Base(m_result); + internal::construct_at(this, m_result); } // This constructor is used when nesting an EvalTo evaluator in another evaluator EIGEN_DEVICE_FUNC evaluator(const ArgType& arg) : m_result(arg) { - ::new (static_cast(this)) Base(m_result); + internal::construct_at(this, m_result); } protected: diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CoreIterators.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CoreIterators.h index b967196813b..d768cbcd614 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CoreIterators.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CoreIterators.h @@ -10,6 +10,9 @@ #ifndef EIGEN_COREITERATORS_H #define EIGEN_COREITERATORS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /* This file contains the respective InnerIterator definition of the expressions defined in Eigen/Core diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseBinaryOp.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseBinaryOp.h index 2202b1cc6b7..d5cf5d5de96 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseBinaryOp.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseBinaryOp.h @@ -11,6 +11,9 @@ #ifndef EIGEN_CWISE_BINARY_OP_H #define EIGEN_CWISE_BINARY_OP_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -19,7 +22,7 @@ struct traits > { // we must not inherit from traits since it has // the potential to cause problems with MSVC - typedef typename remove_all::type Ancestor; + typedef remove_all_t Ancestor; typedef typename traits::XprKind XprKind; enum { RowsAtCompileTime = traits::RowsAtCompileTime, @@ -43,10 +46,10 @@ struct traits > typename traits::StorageIndex>::type StorageIndex; typedef typename Lhs::Nested LhsNested; typedef typename Rhs::Nested RhsNested; - typedef typename remove_reference::type _LhsNested; - typedef typename remove_reference::type _RhsNested; + typedef std::remove_reference_t LhsNested_; + typedef std::remove_reference_t RhsNested_; enum { - Flags = cwise_promote_storage_order::StorageKind,typename traits::StorageKind,_LhsNested::Flags & RowMajorBit,_RhsNested::Flags & RowMajorBit>::value + Flags = cwise_promote_storage_order::StorageKind,typename traits::StorageKind,LhsNested_::Flags & RowMajorBit,RhsNested_::Flags & RowMajorBit>::value }; }; } // end namespace internal @@ -84,9 +87,9 @@ class CwiseBinaryOp : { public: - typedef typename internal::remove_all::type Functor; - typedef typename internal::remove_all::type Lhs; - typedef typename internal::remove_all::type Rhs; + typedef internal::remove_all_t Functor; + typedef internal::remove_all_t Lhs; + typedef internal::remove_all_t Rhs; typedef typename CwiseBinaryOpImpl< BinaryOp, LhsType, RhsType, @@ -95,12 +98,15 @@ class CwiseBinaryOp : BinaryOp>::ret>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseBinaryOp) + EIGEN_CHECK_BINARY_COMPATIBILIY(BinaryOp,typename Lhs::Scalar,typename Rhs::Scalar) + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Lhs, Rhs) + typedef typename internal::ref_selector::type LhsNested; typedef typename internal::ref_selector::type RhsNested; - typedef typename internal::remove_reference::type _LhsNested; - typedef typename internal::remove_reference::type _RhsNested; + typedef std::remove_reference_t LhsNested_; + typedef std::remove_reference_t RhsNested_; -#if EIGEN_COMP_MSVC && EIGEN_HAS_CXX11 +#if EIGEN_COMP_MSVC //Required for Visual Studio or the Copy constructor will probably not get inlined! EIGEN_STRONG_INLINE CwiseBinaryOp(const CwiseBinaryOp&) = default; @@ -110,29 +116,26 @@ class CwiseBinaryOp : CwiseBinaryOp(const Lhs& aLhs, const Rhs& aRhs, const BinaryOp& func = BinaryOp()) : m_lhs(aLhs), m_rhs(aRhs), m_functor(func) { - EIGEN_CHECK_BINARY_COMPATIBILIY(BinaryOp,typename Lhs::Scalar,typename Rhs::Scalar); - // require the sizes to match - EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Lhs, Rhs) eigen_assert(aLhs.rows() == aRhs.rows() && aLhs.cols() == aRhs.cols()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { // return the fixed size type if available to enable compile time optimizations - return internal::traits::type>::RowsAtCompileTime==Dynamic ? m_rhs.rows() : m_lhs.rows(); + return internal::traits>::RowsAtCompileTime==Dynamic ? m_rhs.rows() : m_lhs.rows(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { // return the fixed size type if available to enable compile time optimizations - return internal::traits::type>::ColsAtCompileTime==Dynamic ? m_rhs.cols() : m_lhs.cols(); + return internal::traits>::ColsAtCompileTime==Dynamic ? m_rhs.cols() : m_lhs.cols(); } /** \returns the left hand side nested expression */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const _LhsNested& lhs() const { return m_lhs; } + const LhsNested_& lhs() const { return m_lhs; } /** \returns the right hand side nested expression */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const _RhsNested& rhs() const { return m_rhs; } + const RhsNested_& rhs() const { return m_rhs; } /** \returns the functor representing the binary operation */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const BinaryOp& functor() const { return m_functor; } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseNullaryOp.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseNullaryOp.h index 289ec510a8a..b246cca8617 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseNullaryOp.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseNullaryOp.h @@ -10,6 +10,9 @@ #ifndef EIGEN_CWISE_NULLARY_OP_H #define EIGEN_CWISE_NULLARY_OP_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -304,6 +307,20 @@ DenseBase::LinSpaced(const Scalar& low, const Scalar& high) return DenseBase::NullaryExpr(Derived::SizeAtCompileTime, internal::linspaced_op(low,high,Derived::SizeAtCompileTime)); } +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::RandomAccessEqualSpacedReturnType +DenseBase::EqualSpaced(Index size, const Scalar& low, const Scalar& step) { + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return DenseBase::NullaryExpr(size, internal::equalspaced_op(low, step)); +} + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::RandomAccessEqualSpacedReturnType +DenseBase::EqualSpaced(const Scalar& low, const Scalar& step) { + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return DenseBase::NullaryExpr(Derived::SizeAtCompileTime, internal::equalspaced_op(low, step)); +} + /** \returns true if all coefficients in this matrix are approximately equal to \a val, to within precision \a prec */ template EIGEN_DEVICE_FUNC bool DenseBase::isApproxToConstant @@ -453,6 +470,19 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::setLinSpaced( return setLinSpaced(size(), low, high); } +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::setEqualSpaced(Index newSize, const Scalar& low, + const Scalar& step) { + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return derived() = Derived::NullaryExpr(newSize, internal::equalspaced_op(low, step)); +} +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::setEqualSpaced(const Scalar& low, + const Scalar& step) { + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return setEqualSpaced(size(), low, step); +} + // zero: /** \returns an expression of a zero matrix. diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseTernaryOp.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseTernaryOp.h index 9f3576fecef..d8d912fd28a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseTernaryOp.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseTernaryOp.h @@ -12,6 +12,9 @@ #ifndef EIGEN_CWISE_TERNARY_OP_H #define EIGEN_CWISE_TERNARY_OP_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -19,7 +22,7 @@ template struct traits > { // we must not inherit from traits since it has // the potential to cause problems with MSVC - typedef typename remove_all::type Ancestor; + typedef remove_all_t Ancestor; typedef typename traits::XprKind XprKind; enum { RowsAtCompileTime = traits::RowsAtCompileTime, @@ -41,10 +44,10 @@ struct traits > { typedef typename Arg1::Nested Arg1Nested; typedef typename Arg2::Nested Arg2Nested; typedef typename Arg3::Nested Arg3Nested; - typedef typename remove_reference::type _Arg1Nested; - typedef typename remove_reference::type _Arg2Nested; - typedef typename remove_reference::type _Arg3Nested; - enum { Flags = _Arg1Nested::Flags & RowMajorBit }; + typedef std::remove_reference_t Arg1Nested_; + typedef std::remove_reference_t Arg2Nested_; + typedef std::remove_reference_t Arg3Nested_; + enum { Flags = Arg1Nested_::Flags & RowMajorBit }; }; } // end namespace internal @@ -87,9 +90,23 @@ class CwiseTernaryOp : public CwiseTernaryOpImpl< internal::no_assignment_operator { public: - typedef typename internal::remove_all::type Arg1; - typedef typename internal::remove_all::type Arg2; - typedef typename internal::remove_all::type Arg3; + typedef internal::remove_all_t Arg1; + typedef internal::remove_all_t Arg2; + typedef internal::remove_all_t Arg3; + + // require the sizes to match + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Arg1, Arg2) + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Arg1, Arg3) + + // The index types should match + EIGEN_STATIC_ASSERT((internal::is_same< + typename internal::traits::StorageKind, + typename internal::traits::StorageKind>::value), + STORAGE_KIND_MUST_MATCH) + EIGEN_STATIC_ASSERT((internal::is_same< + typename internal::traits::StorageKind, + typename internal::traits::StorageKind>::value), + STORAGE_KIND_MUST_MATCH) typedef typename CwiseTernaryOpImpl< TernaryOp, Arg1Type, Arg2Type, Arg3Type, @@ -99,29 +116,15 @@ class CwiseTernaryOp : public CwiseTernaryOpImpl< typedef typename internal::ref_selector::type Arg1Nested; typedef typename internal::ref_selector::type Arg2Nested; typedef typename internal::ref_selector::type Arg3Nested; - typedef typename internal::remove_reference::type _Arg1Nested; - typedef typename internal::remove_reference::type _Arg2Nested; - typedef typename internal::remove_reference::type _Arg3Nested; + typedef std::remove_reference_t Arg1Nested_; + typedef std::remove_reference_t Arg2Nested_; + typedef std::remove_reference_t Arg3Nested_; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CwiseTernaryOp(const Arg1& a1, const Arg2& a2, const Arg3& a3, const TernaryOp& func = TernaryOp()) : m_arg1(a1), m_arg2(a2), m_arg3(a3), m_functor(func) { - // require the sizes to match - EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Arg1, Arg2) - EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Arg1, Arg3) - - // The index types should match - EIGEN_STATIC_ASSERT((internal::is_same< - typename internal::traits::StorageKind, - typename internal::traits::StorageKind>::value), - STORAGE_KIND_MUST_MATCH) - EIGEN_STATIC_ASSERT((internal::is_same< - typename internal::traits::StorageKind, - typename internal::traits::StorageKind>::value), - STORAGE_KIND_MUST_MATCH) - eigen_assert(a1.rows() == a2.rows() && a1.cols() == a2.cols() && a1.rows() == a3.rows() && a1.cols() == a3.cols()); } @@ -130,14 +133,14 @@ class CwiseTernaryOp : public CwiseTernaryOpImpl< EIGEN_STRONG_INLINE Index rows() const { // return the fixed size type if available to enable compile time // optimizations - if (internal::traits::type>:: + if (internal::traits>:: RowsAtCompileTime == Dynamic && - internal::traits::type>:: + internal::traits>:: RowsAtCompileTime == Dynamic) return m_arg3.rows(); - else if (internal::traits::type>:: + else if (internal::traits>:: RowsAtCompileTime == Dynamic && - internal::traits::type>:: + internal::traits>:: RowsAtCompileTime == Dynamic) return m_arg2.rows(); else @@ -147,14 +150,14 @@ class CwiseTernaryOp : public CwiseTernaryOpImpl< EIGEN_STRONG_INLINE Index cols() const { // return the fixed size type if available to enable compile time // optimizations - if (internal::traits::type>:: + if (internal::traits>:: ColsAtCompileTime == Dynamic && - internal::traits::type>:: + internal::traits>:: ColsAtCompileTime == Dynamic) return m_arg3.cols(); - else if (internal::traits::type>:: + else if (internal::traits>:: ColsAtCompileTime == Dynamic && - internal::traits::type>:: + internal::traits>:: ColsAtCompileTime == Dynamic) return m_arg2.cols(); else @@ -163,13 +166,13 @@ class CwiseTernaryOp : public CwiseTernaryOpImpl< /** \returns the first argument nested expression */ EIGEN_DEVICE_FUNC - const _Arg1Nested& arg1() const { return m_arg1; } + const Arg1Nested_& arg1() const { return m_arg1; } /** \returns the first argument nested expression */ EIGEN_DEVICE_FUNC - const _Arg2Nested& arg2() const { return m_arg2; } + const Arg2Nested_& arg2() const { return m_arg2; } /** \returns the third argument nested expression */ EIGEN_DEVICE_FUNC - const _Arg3Nested& arg3() const { return m_arg3; } + const Arg3Nested_& arg3() const { return m_arg3; } /** \returns the functor representing the ternary operation */ EIGEN_DEVICE_FUNC const TernaryOp& functor() const { return m_functor; } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseUnaryOp.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseUnaryOp.h index e68c4f74808..ec3b2e43f99 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseUnaryOp.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseUnaryOp.h @@ -11,6 +11,9 @@ #ifndef EIGEN_CWISE_UNARY_OP_H #define EIGEN_CWISE_UNARY_OP_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -22,9 +25,9 @@ struct traits > UnaryOp(const typename XprType::Scalar&) >::type Scalar; typedef typename XprType::Nested XprTypeNested; - typedef typename remove_reference::type _XprTypeNested; + typedef std::remove_reference_t XprTypeNested_; enum { - Flags = _XprTypeNested::Flags & RowMajorBit + Flags = XprTypeNested_::Flags & RowMajorBit }; }; } @@ -59,7 +62,7 @@ class CwiseUnaryOp : public CwiseUnaryOpImpl::StorageKind>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseUnaryOp) typedef typename internal::ref_selector::type XprTypeNested; - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit CwiseUnaryOp(const XprType& xpr, const UnaryOp& func = UnaryOp()) @@ -76,12 +79,12 @@ class CwiseUnaryOp : public CwiseUnaryOpImpl::type& + const internal::remove_all_t& nestedExpression() const { return m_xpr; } /** \returns the nested expression */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - typename internal::remove_all::type& + internal::remove_all_t& nestedExpression() { return m_xpr; } protected: diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseUnaryView.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseUnaryView.h index a06d7621ec1..ef1c208cb75 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseUnaryView.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/CwiseUnaryView.h @@ -10,35 +10,43 @@ #ifndef EIGEN_CWISE_UNARY_VIEW_H #define EIGEN_CWISE_UNARY_VIEW_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template -struct traits > +template +struct traits > : traits { typedef typename result_of< ViewOp(const typename traits::Scalar&) >::type Scalar; typedef typename MatrixType::Nested MatrixTypeNested; - typedef typename remove_all::type _MatrixTypeNested; + typedef remove_all_t MatrixTypeNested_; enum { FlagsLvalueBit = is_lvalue::value ? LvalueBit : 0, - Flags = traits<_MatrixTypeNested>::Flags & (RowMajorBit | FlagsLvalueBit | DirectAccessBit), // FIXME DirectAccessBit should not be handled by expressions + Flags = traits::Flags & (RowMajorBit | FlagsLvalueBit | DirectAccessBit), // FIXME DirectAccessBit should not be handled by expressions MatrixTypeInnerStride = inner_stride_at_compile_time::ret, // need to cast the sizeof's from size_t to int explicitly, otherwise: // "error: no integral type can represent all of the enumerator values - InnerStrideAtCompileTime = MatrixTypeInnerStride == Dynamic - ? int(Dynamic) - : int(MatrixTypeInnerStride) * int(sizeof(typename traits::Scalar) / sizeof(Scalar)), - OuterStrideAtCompileTime = outer_stride_at_compile_time::ret == Dynamic - ? int(Dynamic) - : outer_stride_at_compile_time::ret * int(sizeof(typename traits::Scalar) / sizeof(Scalar)) + InnerStrideAtCompileTime = StrideType::InnerStrideAtCompileTime == 0 + ? (MatrixTypeInnerStride == Dynamic + ? int(Dynamic) + : int(MatrixTypeInnerStride) * int(sizeof(typename traits::Scalar) / sizeof(Scalar))) + : int(StrideType::InnerStrideAtCompileTime), + + OuterStrideAtCompileTime = StrideType::OuterStrideAtCompileTime == 0 + ? (outer_stride_at_compile_time::ret == Dynamic + ? int(Dynamic) + : outer_stride_at_compile_time::ret * int(sizeof(typename traits::Scalar) / sizeof(Scalar))) + : int(StrideType::OuterStrideAtCompileTime) }; }; } -template +template class CwiseUnaryViewImpl; /** \class CwiseUnaryView @@ -54,15 +62,15 @@ class CwiseUnaryViewImpl; * * \sa MatrixBase::unaryViewExpr(const CustomUnaryOp &) const, class CwiseUnaryOp */ -template -class CwiseUnaryView : public CwiseUnaryViewImpl::StorageKind> +template +class CwiseUnaryView : public CwiseUnaryViewImpl::StorageKind> { public: - typedef typename CwiseUnaryViewImpl::StorageKind>::Base Base; + typedef typename CwiseUnaryViewImpl::StorageKind>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseUnaryView) typedef typename internal::ref_selector::non_const_type MatrixTypeNested; - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; explicit EIGEN_DEVICE_FUNC inline CwiseUnaryView(MatrixType& mat, const ViewOp& func = ViewOp()) : m_matrix(mat), m_functor(func) {} @@ -78,11 +86,11 @@ class CwiseUnaryView : public CwiseUnaryViewImpl::type& + EIGEN_DEVICE_FUNC const internal::remove_all_t& nestedExpression() const { return m_matrix; } /** \returns the nested expression */ - EIGEN_DEVICE_FUNC typename internal::remove_reference::type& + EIGEN_DEVICE_FUNC std::remove_reference_t& nestedExpression() { return m_matrix; } protected: @@ -91,22 +99,22 @@ class CwiseUnaryView : public CwiseUnaryViewImpl +template class CwiseUnaryViewImpl - : public internal::generic_xpr_base >::type + : public internal::generic_xpr_base >::type { public: - typedef typename internal::generic_xpr_base >::type Base; + typedef typename internal::generic_xpr_base >::type Base; }; -template -class CwiseUnaryViewImpl - : public internal::dense_xpr_base< CwiseUnaryView >::type +template +class CwiseUnaryViewImpl + : public internal::dense_xpr_base< CwiseUnaryView >::type { public: - typedef CwiseUnaryView Derived; - typedef typename internal::dense_xpr_base< CwiseUnaryView >::type Base; + typedef CwiseUnaryView Derived; + typedef typename internal::dense_xpr_base< CwiseUnaryView >::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Derived) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(CwiseUnaryViewImpl) @@ -116,12 +124,16 @@ class CwiseUnaryViewImpl EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const { - return derived().nestedExpression().innerStride() * sizeof(typename internal::traits::Scalar) / sizeof(Scalar); + return StrideType::InnerStrideAtCompileTime != 0 + ? int(StrideType::InnerStrideAtCompileTime) + : derived().nestedExpression().innerStride() * sizeof(typename internal::traits::Scalar) / sizeof(Scalar); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const { - return derived().nestedExpression().outerStride() * sizeof(typename internal::traits::Scalar) / sizeof(Scalar); + return StrideType::OuterStrideAtCompileTime != 0 + ? int(StrideType::OuterStrideAtCompileTime) + : derived().nestedExpression().outerStride() * sizeof(typename internal::traits::Scalar) / sizeof(Scalar); } protected: EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(CwiseUnaryViewImpl) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseBase.h index 9b16db68d48..3b687a4fe2a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseBase.h @@ -11,17 +11,13 @@ #ifndef EIGEN_DENSEBASE_H #define EIGEN_DENSEBASE_H -namespace Eigen { +// IWYU pragma: private +#include "./InternalHeaderCheck.h" -namespace internal { +namespace Eigen { // The index type defined by EIGEN_DEFAULT_DENSE_INDEX_TYPE must be a signed type. -// This dummy function simply aims at checking that at compile time. -static inline void check_DenseIndex_is_signed() { - EIGEN_STATIC_ASSERT(NumTraits::IsSigned,THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE) -} - -} // end namespace internal +EIGEN_STATIC_ASSERT(NumTraits::IsSigned,THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE) /** \class DenseBase * \ingroup Core_Module @@ -110,8 +106,7 @@ template class DenseBase * \sa MatrixBase::rows(), MatrixBase::cols(), RowsAtCompileTime, SizeAtCompileTime */ - SizeAtCompileTime = (internal::size_at_compile_time::RowsAtCompileTime, - internal::traits::ColsAtCompileTime>::ret), + SizeAtCompileTime = (internal::size_of_xpr_at_compile_time::ret), /**< This is equal to the number of coefficients, i.e. the number of * rows times the number of columns, or to \a Dynamic if this is not * known at compile-time. \sa RowsAtCompileTime, ColsAtCompileTime */ @@ -138,8 +133,8 @@ template class DenseBase * \sa ColsAtCompileTime, MaxRowsAtCompileTime, MaxSizeAtCompileTime */ - MaxSizeAtCompileTime = (internal::size_at_compile_time::MaxRowsAtCompileTime, - internal::traits::MaxColsAtCompileTime>::ret), + MaxSizeAtCompileTime = internal::size_at_compile_time(internal::traits::MaxRowsAtCompileTime, + internal::traits::MaxColsAtCompileTime), /**< This value is equal to the maximum possible number of coefficients that this expression * might have. If this expression might have an arbitrarily high number of coefficients, * this value is set to \a Dynamic. @@ -206,13 +201,8 @@ template class DenseBase * the return type of eval() is a const reference to a matrix, not a matrix! It is however guaranteed * that the return type of eval() is either PlainObject or const PlainObject&. */ - typedef typename internal::conditional::XprKind,MatrixXpr >::value, - PlainMatrix, PlainArray>::type PlainObject; - - /** \returns the number of nonzero coefficients which is in practice the number - * of stored coefficients. */ - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - inline Index nonZeros() const { return size(); } + typedef std::conditional_t::XprKind,MatrixXpr >::value, + PlainMatrix, PlainArray> PlainObject; /** \returns the outer size. * @@ -269,6 +259,8 @@ template class DenseBase EIGEN_DEPRECATED typedef CwiseNullaryOp,PlainObject> SequentialLinSpacedReturnType; /** \internal Represents a vector with linearly spaced coefficients that allows random access. */ typedef CwiseNullaryOp,PlainObject> RandomAccessLinSpacedReturnType; + /** \internal Represents a vector with equally spaced coefficients that allows random access. */ + typedef CwiseNullaryOp, PlainObject> RandomAccessEqualSpacedReturnType; /** \internal the return type of MatrixBase::eigenvalues() */ typedef Matrix::Scalar>::Real, internal::traits::ColsAtCompileTime, 1> EigenvaluesReturnType; @@ -324,9 +316,9 @@ template class DenseBase typedef Transpose TransposeReturnType; EIGEN_DEVICE_FUNC TransposeReturnType transpose(); - typedef typename internal::add_const >::type ConstTransposeReturnType; + typedef Transpose ConstTransposeReturnType; EIGEN_DEVICE_FUNC - ConstTransposeReturnType transpose() const; + const ConstTransposeReturnType transpose() const; EIGEN_DEVICE_FUNC void transposeInPlace(); @@ -347,6 +339,11 @@ template class DenseBase EIGEN_DEVICE_FUNC static const RandomAccessLinSpacedReturnType LinSpaced(const Scalar& low, const Scalar& high); + EIGEN_DEVICE_FUNC static const RandomAccessEqualSpacedReturnType + EqualSpaced(Index size, const Scalar& low, const Scalar& step); + EIGEN_DEVICE_FUNC static const RandomAccessEqualSpacedReturnType + EqualSpaced(const Scalar& low, const Scalar& step); + template EIGEN_DEVICE_FUNC static const CwiseNullaryOp NullaryExpr(Index rows, Index cols, const CustomNullaryOp& func); @@ -368,6 +365,8 @@ template class DenseBase EIGEN_DEVICE_FUNC Derived& setConstant(const Scalar& value); EIGEN_DEVICE_FUNC Derived& setLinSpaced(Index size, const Scalar& low, const Scalar& high); EIGEN_DEVICE_FUNC Derived& setLinSpaced(const Scalar& low, const Scalar& high); + EIGEN_DEVICE_FUNC Derived& setEqualSpaced(Index size, const Scalar& low, const Scalar& step); + EIGEN_DEVICE_FUNC Derived& setEqualSpaced(const Scalar& low, const Scalar& step); EIGEN_DEVICE_FUNC Derived& setZero(); EIGEN_DEVICE_FUNC Derived& setOnes(); EIGEN_DEVICE_FUNC Derived& setRandom(); @@ -387,15 +386,15 @@ template class DenseBase EIGEN_DEVICE_FUNC bool isZero(const RealScalar& prec = NumTraits::dummy_precision()) const; EIGEN_DEVICE_FUNC bool isOnes(const RealScalar& prec = NumTraits::dummy_precision()) const; - inline bool hasNaN() const; - inline bool allFinite() const; + EIGEN_DEVICE_FUNC inline bool hasNaN() const; + EIGEN_DEVICE_FUNC inline bool allFinite() const; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator*=(const Scalar& other); EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator/=(const Scalar& other); - typedef typename internal::add_const_on_value_type::type>::type EvalReturnType; + typedef internal::add_const_on_value_type_t::type> EvalReturnType; /** \returns the matrix or vector obtained by evaluating this expression. * * Notice that in the case of a plain matrix or vector (not an expression) this function just returns @@ -439,9 +438,9 @@ template class DenseBase EIGEN_DEVICE_FUNC inline const ForceAlignedAccess forceAlignedAccess() const; EIGEN_DEVICE_FUNC inline ForceAlignedAccess forceAlignedAccess(); template EIGEN_DEVICE_FUNC - inline const typename internal::conditional,Derived&>::type forceAlignedAccessIf() const; + inline const std::conditional_t,Derived&> forceAlignedAccessIf() const; template EIGEN_DEVICE_FUNC - inline typename internal::conditional,Derived&>::type forceAlignedAccessIf(); + inline std::conditional_t,Derived&> forceAlignedAccessIf(); EIGEN_DEVICE_FUNC Scalar sum() const; EIGEN_DEVICE_FUNC Scalar mean() const; @@ -569,18 +568,28 @@ template class DenseBase static const RandomReturnType Random(Index size); static const RandomReturnType Random(); - template - inline EIGEN_DEVICE_FUNC const Select - select(const DenseBase& thenMatrix, - const DenseBase& elseMatrix) const; - - template - inline EIGEN_DEVICE_FUNC const Select - select(const DenseBase& thenMatrix, const typename ThenDerived::Scalar& elseScalar) const; - - template - inline EIGEN_DEVICE_FUNC const Select - select(const typename ElseDerived::Scalar& thenScalar, const DenseBase& elseMatrix) const; + template + inline EIGEN_DEVICE_FUNC + CwiseTernaryOp::Scalar, + typename DenseBase::Scalar, Scalar>, + ThenDerived, ElseDerived, Derived> + select(const DenseBase& thenMatrix, const DenseBase& elseMatrix) const; + + template + inline EIGEN_DEVICE_FUNC + CwiseTernaryOp::Scalar, + typename DenseBase::Scalar, Scalar>, + ThenDerived, typename DenseBase::ConstantReturnType, Derived> + select(const DenseBase& thenMatrix, + const typename DenseBase::Scalar& elseScalar) const; + + template + inline EIGEN_DEVICE_FUNC + CwiseTernaryOp::Scalar, + typename DenseBase::Scalar, Scalar>, + typename DenseBase::ConstantReturnType, ElseDerived, Derived> + select(const typename DenseBase::Scalar& thenScalar, + const DenseBase& elseMatrix) const; template RealScalar lpNorm() const; @@ -621,27 +630,21 @@ template class DenseBase /** This is the const version of iterator (aka read-only) */ typedef random_access_iterator_type const_iterator; #else - typedef typename internal::conditional< (Flags&DirectAccessBit)==DirectAccessBit, - internal::pointer_based_stl_iterator, - internal::generic_randaccess_stl_iterator - >::type iterator_type; + typedef std::conditional_t< (Flags&DirectAccessBit)==DirectAccessBit, + internal::pointer_based_stl_iterator, + internal::generic_randaccess_stl_iterator + > iterator_type; - typedef typename internal::conditional< (Flags&DirectAccessBit)==DirectAccessBit, - internal::pointer_based_stl_iterator, - internal::generic_randaccess_stl_iterator - >::type const_iterator_type; + typedef std::conditional_t< (Flags&DirectAccessBit)==DirectAccessBit, + internal::pointer_based_stl_iterator, + internal::generic_randaccess_stl_iterator + > const_iterator_type; // Stl-style iterators are supported only for vectors. - typedef typename internal::conditional< IsVectorAtCompileTime, - iterator_type, - void - >::type iterator; + typedef std::conditional_t iterator; - typedef typename internal::conditional< IsVectorAtCompileTime, - const_iterator_type, - void - >::type const_iterator; + typedef std::conditional_t const_iterator; #endif inline iterator begin(); @@ -655,10 +658,10 @@ template class DenseBase #define EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL #define EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(COND) #define EIGEN_DOC_UNARY_ADDONS(X,Y) -# include "../plugins/CommonCwiseUnaryOps.h" -# include "../plugins/BlockMethods.h" -# include "../plugins/IndexedViewMethods.h" -# include "../plugins/ReshapedMethods.h" +# include "../plugins/CommonCwiseUnaryOps.inc" +# include "../plugins/BlockMethods.inc" +# include "../plugins/IndexedViewMethods.inc" +# include "../plugins/ReshapedMethods.inc" # ifdef EIGEN_DENSEBASE_PLUGIN # include EIGEN_DENSEBASE_PLUGIN # endif @@ -678,14 +681,13 @@ template class DenseBase protected: EIGEN_DEFAULT_COPY_CONSTRUCTOR(DenseBase) /** Default constructor. Do nothing. */ - EIGEN_DEVICE_FUNC DenseBase() - { + EIGEN_DEVICE_FUNC constexpr DenseBase() { /* Just checks for self-consistency of the flags. * Only do it when debugging Eigen, as this borders on paranoia and could slow compilation down */ #ifdef EIGEN_INTERNAL_DEBUGGING - EIGEN_STATIC_ASSERT((EIGEN_IMPLIES(MaxRowsAtCompileTime==1 && MaxColsAtCompileTime!=1, int(IsRowMajor)) - && EIGEN_IMPLIES(MaxColsAtCompileTime==1 && MaxRowsAtCompileTime!=1, int(!IsRowMajor))), + EIGEN_STATIC_ASSERT((internal::check_implication(MaxRowsAtCompileTime==1 && MaxColsAtCompileTime!=1, int(IsRowMajor)) + && internal::check_implication(MaxColsAtCompileTime==1 && MaxRowsAtCompileTime!=1, int(!IsRowMajor))), INVALID_STORAGE_ORDER_FOR_THIS_VECTOR_EXPRESSION) #endif } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseCoeffsBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseCoeffsBase.h index 37fcdb59119..93a2965e1f3 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseCoeffsBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseCoeffsBase.h @@ -10,12 +10,15 @@ #ifndef EIGEN_DENSECOEFFSBASE_H #define EIGEN_DENSECOEFFSBASE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { template struct add_const_on_value_type_if_arithmetic { - typedef typename conditional::value, T, typename add_const_on_value_type::type>::type type; + typedef std::conditional_t::value, T, add_const_on_value_type_t> type; }; } @@ -43,13 +46,13 @@ class DenseCoeffsBase : public EigenBase // - This is the return type of the coeff() method. // - The LvalueBit means exactly that we can offer a coeffRef() method, which means exactly that we can get references // to coeffs, which means exactly that we can have coeff() return a const reference (as opposed to returning a value). - // - The is_artihmetic check is required since "const int", "const double", etc. will cause warnings on some systems + // - The is_arithmetic check is required since "const int", "const double", etc. will cause warnings on some systems // while the declaration of "const T", where T is a non arithmetic type does not. Always returning "const Scalar&" is // not possible, since the underlying expressions might not offer a valid address the reference could be referring to. - typedef typename internal::conditional::Flags&LvalueBit), - const Scalar&, - typename internal::conditional::value, Scalar, const Scalar>::type - >::type CoeffReturnType; + typedef std::conditional_t::Flags&LvalueBit), + const Scalar&, + std::conditional_t::value, Scalar, const Scalar> + > CoeffReturnType; typedef typename internal::add_const_on_value_type_if_arithmetic< typename internal::packet_traits::type diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseStorage.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseStorage.h index 08ef6c53061..b0303785899 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseStorage.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DenseStorage.h @@ -18,20 +18,21 @@ #define EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(X) #endif +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { struct constructor_without_unaligned_array_assert {}; -template -EIGEN_DEVICE_FUNC -void check_static_allocation_size() -{ - // if EIGEN_STACK_ALLOCATION_LIMIT is defined to 0, then no limit - #if EIGEN_STACK_ALLOCATION_LIMIT +template +EIGEN_DEVICE_FUNC constexpr void check_static_allocation_size() { +// if EIGEN_STACK_ALLOCATION_LIMIT is defined to 0, then no limit +#if EIGEN_STACK_ALLOCATION_LIMIT EIGEN_STATIC_ASSERT(Size * sizeof(T) <= EIGEN_STACK_ALLOCATION_LIMIT, OBJECT_ALLOCATED_ON_STACK_IS_TOO_BIG); - #endif +#endif } /** \internal @@ -45,35 +46,18 @@ struct plain_array { T array[Size]; - EIGEN_DEVICE_FUNC - plain_array() - { - check_static_allocation_size(); - } + EIGEN_DEVICE_FUNC constexpr plain_array() { check_static_allocation_size(); } - EIGEN_DEVICE_FUNC - plain_array(constructor_without_unaligned_array_assert) - { - check_static_allocation_size(); + EIGEN_DEVICE_FUNC constexpr plain_array(constructor_without_unaligned_array_assert) { + check_static_allocation_size(); } }; #if defined(EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT) #define EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(sizemask) -#elif EIGEN_GNUC_AT_LEAST(4,7) - // GCC 4.7 is too aggressive in its optimizations and remove the alignment test based on the fact the array is declared to be aligned. - // See this bug report: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=53900 - // Hiding the origin of the array pointer behind a function argument seems to do the trick even if the function is inlined: - template - EIGEN_ALWAYS_INLINE PtrType eigen_unaligned_array_assert_workaround_gcc47(PtrType array) { return array; } - #define EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(sizemask) \ - eigen_assert((internal::UIntPtr(eigen_unaligned_array_assert_workaround_gcc47(array)) & (sizemask)) == 0 \ - && "this assertion is explained here: " \ - "http://eigen.tuxfamily.org/dox-devel/group__TopicUnalignedArrayAssert.html" \ - " **** READ THIS WEB PAGE !!! ****"); #else #define EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(sizemask) \ - eigen_assert((internal::UIntPtr(array) & (sizemask)) == 0 \ + eigen_assert((internal::is_constant_evaluated() || (std::uintptr_t(array) & (sizemask)) == 0) \ && "this assertion is explained here: " \ "http://eigen.tuxfamily.org/dox-devel/group__TopicUnalignedArrayAssert.html" \ " **** READ THIS WEB PAGE !!! ****"); @@ -84,17 +68,13 @@ struct plain_array { EIGEN_ALIGN_TO_BOUNDARY(8) T array[Size]; - EIGEN_DEVICE_FUNC - plain_array() - { + EIGEN_DEVICE_FUNC constexpr plain_array() { EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(7); check_static_allocation_size(); } - EIGEN_DEVICE_FUNC - plain_array(constructor_without_unaligned_array_assert) - { - check_static_allocation_size(); + EIGEN_DEVICE_FUNC constexpr plain_array(constructor_without_unaligned_array_assert) { + check_static_allocation_size(); } }; @@ -103,17 +83,13 @@ struct plain_array { EIGEN_ALIGN_TO_BOUNDARY(16) T array[Size]; - EIGEN_DEVICE_FUNC - plain_array() - { + EIGEN_DEVICE_FUNC constexpr plain_array() { EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(15); check_static_allocation_size(); } - EIGEN_DEVICE_FUNC - plain_array(constructor_without_unaligned_array_assert) - { - check_static_allocation_size(); + EIGEN_DEVICE_FUNC constexpr plain_array(constructor_without_unaligned_array_assert) { + check_static_allocation_size(); } }; @@ -122,17 +98,13 @@ struct plain_array { EIGEN_ALIGN_TO_BOUNDARY(32) T array[Size]; - EIGEN_DEVICE_FUNC - plain_array() - { + EIGEN_DEVICE_FUNC constexpr plain_array() { EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(31); check_static_allocation_size(); } - EIGEN_DEVICE_FUNC - plain_array(constructor_without_unaligned_array_assert) - { - check_static_allocation_size(); + EIGEN_DEVICE_FUNC constexpr plain_array(constructor_without_unaligned_array_assert) { + check_static_allocation_size(); } }; @@ -141,17 +113,13 @@ struct plain_array { EIGEN_ALIGN_TO_BOUNDARY(64) T array[Size]; - EIGEN_DEVICE_FUNC - plain_array() - { + EIGEN_DEVICE_FUNC constexpr plain_array() { EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(63); check_static_allocation_size(); } - EIGEN_DEVICE_FUNC - plain_array(constructor_without_unaligned_array_assert) - { - check_static_allocation_size(); + EIGEN_DEVICE_FUNC constexpr plain_array(constructor_without_unaligned_array_assert) { + check_static_allocation_size(); } }; @@ -159,8 +127,8 @@ template struct plain_array { T array[1]; - EIGEN_DEVICE_FUNC plain_array() {} - EIGEN_DEVICE_FUNC plain_array(constructor_without_unaligned_array_assert) {} + EIGEN_DEVICE_FUNC constexpr plain_array() {} + EIGEN_DEVICE_FUNC constexpr plain_array(constructor_without_unaligned_array_assert) {} }; struct plain_array_helper { @@ -201,57 +169,32 @@ struct plain_array_helper { * * \sa Matrix */ -template class DenseStorage; +template class DenseStorage; // purely fixed-size matrix -template class DenseStorage +template class DenseStorage { - internal::plain_array m_data; + internal::plain_array m_data; public: - EIGEN_DEVICE_FUNC DenseStorage() { + constexpr EIGEN_DEVICE_FUNC DenseStorage() { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = Size) } - EIGEN_DEVICE_FUNC - explicit DenseStorage(internal::constructor_without_unaligned_array_assert) + EIGEN_DEVICE_FUNC explicit constexpr DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(internal::constructor_without_unaligned_array_assert()) {} -#if !EIGEN_HAS_CXX11 || defined(EIGEN_DENSE_STORAGE_CTOR_PLUGIN) - EIGEN_DEVICE_FUNC +#if defined(EIGEN_DENSE_STORAGE_CTOR_PLUGIN) + EIGEN_DEVICE_FUNC constexpr DenseStorage(const DenseStorage& other) : m_data(other.m_data) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = Size) } #else - EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage&) = default; -#endif -#if !EIGEN_HAS_CXX11 - EIGEN_DEVICE_FUNC - DenseStorage& operator=(const DenseStorage& other) - { - if (this != &other) m_data = other.m_data; - return *this; - } -#else - EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage&) = default; -#endif -#if EIGEN_HAS_RVALUE_REFERENCES -#if !EIGEN_HAS_CXX11 - EIGEN_DEVICE_FUNC DenseStorage(DenseStorage&& other) EIGEN_NOEXCEPT - : m_data(std::move(other.m_data)) - { - } - EIGEN_DEVICE_FUNC DenseStorage& operator=(DenseStorage&& other) EIGEN_NOEXCEPT - { - if (this != &other) - m_data = std::move(other.m_data); - return *this; - } -#else - EIGEN_DEVICE_FUNC DenseStorage(DenseStorage&&) = default; - EIGEN_DEVICE_FUNC DenseStorage& operator=(DenseStorage&&) = default; + EIGEN_DEVICE_FUNC constexpr DenseStorage(const DenseStorage&) = default; #endif -#endif - EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) { + EIGEN_DEVICE_FUNC constexpr DenseStorage& operator=(const DenseStorage&) = default; + EIGEN_DEVICE_FUNC constexpr DenseStorage(DenseStorage&&) = default; + EIGEN_DEVICE_FUNC constexpr DenseStorage& operator=(DenseStorage&&) = default; + EIGEN_DEVICE_FUNC constexpr DenseStorage(Index size, Index rows, Index cols) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) - eigen_internal_assert(size==rows*cols && rows==_Rows && cols==_Cols); + eigen_internal_assert(size == rows * cols && rows == Rows_ && cols == Cols_); EIGEN_UNUSED_VARIABLE(size); EIGEN_UNUSED_VARIABLE(rows); EIGEN_UNUSED_VARIABLE(cols); @@ -259,57 +202,148 @@ template class DenseSt EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { numext::swap(m_data, other.m_data); } - EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index rows(void) EIGEN_NOEXCEPT {return _Rows;} - EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index cols(void) EIGEN_NOEXCEPT {return _Cols;} - EIGEN_DEVICE_FUNC void conservativeResize(Index,Index,Index) {} - EIGEN_DEVICE_FUNC void resize(Index,Index,Index) {} - EIGEN_DEVICE_FUNC const T *data() const { return m_data.array; } - EIGEN_DEVICE_FUNC T *data() { return m_data.array; } + EIGEN_DEVICE_FUNC static constexpr Index rows(void) EIGEN_NOEXCEPT { return Rows_; } + EIGEN_DEVICE_FUNC static constexpr Index cols(void) EIGEN_NOEXCEPT { return Cols_; } + EIGEN_DEVICE_FUNC constexpr void conservativeResize(Index, Index, Index) {} + EIGEN_DEVICE_FUNC constexpr void resize(Index, Index, Index) {} + EIGEN_DEVICE_FUNC constexpr const T* data() const { return m_data.array; } + EIGEN_DEVICE_FUNC constexpr T* data() { return m_data.array; } }; // null matrix -template class DenseStorage +template +class DenseStorage { public: - EIGEN_DEVICE_FUNC DenseStorage() {} - EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) {} - EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage&) {} - EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage&) { return *this; } - EIGEN_DEVICE_FUNC DenseStorage(Index,Index,Index) {} - EIGEN_DEVICE_FUNC void swap(DenseStorage& ) {} - EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index rows(void) EIGEN_NOEXCEPT {return _Rows;} - EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index cols(void) EIGEN_NOEXCEPT {return _Cols;} - EIGEN_DEVICE_FUNC void conservativeResize(Index,Index,Index) {} - EIGEN_DEVICE_FUNC void resize(Index,Index,Index) {} - EIGEN_DEVICE_FUNC const T *data() const { return 0; } - EIGEN_DEVICE_FUNC T *data() { return 0; } + static_assert(Rows_ * Cols_ == 0, "The fixed number of rows times columns must equal the storage size."); + EIGEN_DEVICE_FUNC constexpr DenseStorage() {} + EIGEN_DEVICE_FUNC explicit constexpr DenseStorage(internal::constructor_without_unaligned_array_assert) {} + EIGEN_DEVICE_FUNC constexpr DenseStorage(const DenseStorage&) {} + EIGEN_DEVICE_FUNC constexpr DenseStorage& operator=(const DenseStorage&) { return *this; } + EIGEN_DEVICE_FUNC constexpr DenseStorage(Index,Index,Index) {} + EIGEN_DEVICE_FUNC constexpr void swap(DenseStorage& ) {} + EIGEN_DEVICE_FUNC static constexpr Index rows(void) EIGEN_NOEXCEPT {return Rows_;} + EIGEN_DEVICE_FUNC static constexpr Index cols(void) EIGEN_NOEXCEPT {return Cols_;} + EIGEN_DEVICE_FUNC constexpr void conservativeResize(Index,Index,Index) {} + EIGEN_DEVICE_FUNC constexpr void resize(Index,Index,Index) {} + EIGEN_DEVICE_FUNC constexpr const T *data() const { return 0; } + EIGEN_DEVICE_FUNC constexpr T *data() { return 0; } }; // more specializations for null matrices; these are necessary to resolve ambiguities -template class DenseStorage -: public DenseStorage { }; +template +class DenseStorage { + Index m_rows; + Index m_cols; + public: + EIGEN_DEVICE_FUNC DenseStorage() : m_rows(0), m_cols(0) {} + EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : DenseStorage() {} + EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) : m_rows(other.m_rows), m_cols(other.m_cols) {} + EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { + m_rows = other.m_rows; + m_cols = other.m_cols; + return *this; + } + EIGEN_DEVICE_FUNC DenseStorage(Index, Index rows, Index cols) : m_rows(rows), m_cols(cols) { + eigen_assert(m_rows * m_cols == 0 && "The number of rows times columns must equal the storage size."); + } + EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { + numext::swap(m_rows,other.m_rows); + numext::swap(m_cols,other.m_cols); + } + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT {return m_rows;} + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT {return m_cols;} + EIGEN_DEVICE_FUNC void conservativeResize(Index, Index rows, Index cols) { + m_rows = rows; + m_cols = cols; + eigen_assert(m_rows * m_cols == 0 && "The number of rows times columns must equal the storage size."); + } + EIGEN_DEVICE_FUNC void resize(Index, Index rows, Index cols) { + m_rows = rows; + m_cols = cols; + eigen_assert(m_rows * m_cols == 0 && "The number of rows times columns must equal the storage size."); + } + EIGEN_DEVICE_FUNC const T *data() const { return nullptr; } + EIGEN_DEVICE_FUNC T *data() { return nullptr; } +}; -template class DenseStorage -: public DenseStorage { }; +template +class DenseStorage { + Index m_cols; + public: + EIGEN_DEVICE_FUNC DenseStorage() : m_cols(0) {} + EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : DenseStorage() {} + EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) : m_cols(other.m_cols) {} + EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { + m_cols = other.m_cols; + return *this; + } + EIGEN_DEVICE_FUNC DenseStorage(Index, Index, Index cols) : m_cols(cols) { + eigen_assert(Rows_ * m_cols == 0 && "The number of rows times columns must equal the storage size."); + } + EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { + numext::swap(m_cols, other.m_cols); + } + EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index rows(void) EIGEN_NOEXCEPT {return Rows_;} + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols(void) const EIGEN_NOEXCEPT {return m_cols;} + EIGEN_DEVICE_FUNC void conservativeResize(Index, Index, Index cols) { + m_cols = cols; + eigen_assert(Rows_ * m_cols == 0 && "The number of rows times columns must equal the storage size."); + } + EIGEN_DEVICE_FUNC void resize(Index, Index, Index cols) { + m_cols = cols; + eigen_assert(Rows_ * m_cols == 0 && "The number of rows times columns must equal the storage size."); + } + EIGEN_DEVICE_FUNC const T *data() const { return nullptr; } + EIGEN_DEVICE_FUNC T *data() { return nullptr; } +}; -template class DenseStorage -: public DenseStorage { }; +template +class DenseStorage { + Index m_rows; + public: + EIGEN_DEVICE_FUNC DenseStorage() : m_rows(0) {} + EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : DenseStorage() {} + EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) : m_rows(other.m_rows) {} + EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { + m_rows = other.m_rows; + return *this; + } + EIGEN_DEVICE_FUNC DenseStorage(Index, Index rows, Index) : m_rows(rows) { + eigen_assert(m_rows * Cols_ == 0 && "The number of rows times columns must equal the storage size."); + } + EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { + numext::swap(m_rows, other.m_rows); + } + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows(void) const EIGEN_NOEXCEPT {return m_rows;} + EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index cols(void) EIGEN_NOEXCEPT {return Cols_;} + EIGEN_DEVICE_FUNC void conservativeResize(Index, Index rows, Index) { + m_rows = rows; + eigen_assert(m_rows * Cols_ == 0 && "The number of rows times columns must equal the storage size."); + } + EIGEN_DEVICE_FUNC void resize(Index, Index rows, Index) { + m_rows = rows; + eigen_assert(m_rows * Cols_ == 0 && "The number of rows times columns must equal the storage size."); + } + EIGEN_DEVICE_FUNC const T *data() const { return nullptr; } + EIGEN_DEVICE_FUNC T *data() { return nullptr; } +}; // dynamic-size matrix with fixed-size storage -template class DenseStorage +template +class DenseStorage { - internal::plain_array m_data; + internal::plain_array m_data; Index m_rows; Index m_cols; public: - EIGEN_DEVICE_FUNC DenseStorage() : m_rows(0), m_cols(0) {} - EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) - : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(0), m_cols(0) {} - EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) - : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(other.m_rows), m_cols(other.m_cols) - { - internal::plain_array_helper::copy(other.m_data, m_rows * m_cols, m_data); - } + EIGEN_DEVICE_FUNC constexpr DenseStorage() : m_data(), m_rows(0), m_cols(0) {} + EIGEN_DEVICE_FUNC explicit constexpr DenseStorage(internal::constructor_without_unaligned_array_assert) + : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(0), m_cols(0) {} + EIGEN_DEVICE_FUNC constexpr DenseStorage(const DenseStorage& other) + : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(other.m_rows), m_cols(other.m_cols) { + internal::plain_array_helper::copy(other.m_data, m_rows * m_cols, m_data); + } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { if (this != &other) @@ -320,113 +354,121 @@ template class DenseStorage class DenseStorage +template +class DenseStorage { - internal::plain_array m_data; + internal::plain_array m_data; Index m_rows; public: - EIGEN_DEVICE_FUNC DenseStorage() : m_rows(0) {} - EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) - : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(0) {} - EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) - : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(other.m_rows) - { - internal::plain_array_helper::copy(other.m_data, m_rows * _Cols, m_data); - } - + EIGEN_DEVICE_FUNC constexpr DenseStorage() : m_rows(0) {} + EIGEN_DEVICE_FUNC explicit constexpr DenseStorage(internal::constructor_without_unaligned_array_assert) + : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(0) {} + EIGEN_DEVICE_FUNC constexpr DenseStorage(const DenseStorage& other) + : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(other.m_rows) { + internal::plain_array_helper::copy(other.m_data, m_rows * Cols_, m_data); + } + EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { if (this != &other) { m_rows = other.m_rows; - internal::plain_array_helper::copy(other.m_data, m_rows * _Cols, m_data); + internal::plain_array_helper::copy(other.m_data, m_rows * Cols_, m_data); } return *this; } - EIGEN_DEVICE_FUNC DenseStorage(Index, Index rows, Index) : m_rows(rows) {} + EIGEN_DEVICE_FUNC constexpr DenseStorage(Index, Index rows, Index) : m_rows(rows) {} EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { - internal::plain_array_helper::swap(m_data, m_rows * _Cols, other.m_data, other.m_rows * _Cols); + internal::plain_array_helper::swap(m_data, m_rows * Cols_, other.m_data, other.m_rows * Cols_); numext::swap(m_rows, other.m_rows); } - EIGEN_DEVICE_FUNC Index rows(void) const EIGEN_NOEXCEPT {return m_rows;} - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols(void) const EIGEN_NOEXCEPT {return _Cols;} - EIGEN_DEVICE_FUNC void conservativeResize(Index, Index rows, Index) { m_rows = rows; } - EIGEN_DEVICE_FUNC void resize(Index, Index rows, Index) { m_rows = rows; } - EIGEN_DEVICE_FUNC const T *data() const { return m_data.array; } - EIGEN_DEVICE_FUNC T *data() { return m_data.array; } + EIGEN_DEVICE_FUNC constexpr Index rows(void) const EIGEN_NOEXCEPT { return m_rows; } + EIGEN_DEVICE_FUNC constexpr Index cols(void) const EIGEN_NOEXCEPT { return Cols_; } + EIGEN_DEVICE_FUNC constexpr void conservativeResize(Index, Index rows, Index) { m_rows = rows; } + EIGEN_DEVICE_FUNC constexpr void resize(Index, Index rows, Index) { m_rows = rows; } + EIGEN_DEVICE_FUNC constexpr const T* data() const { return m_data.array; } + EIGEN_DEVICE_FUNC constexpr T* data() { return m_data.array; } }; // dynamic-size matrix with fixed-size storage and fixed height -template class DenseStorage +template +class DenseStorage { - internal::plain_array m_data; + internal::plain_array m_data; Index m_cols; public: - EIGEN_DEVICE_FUNC DenseStorage() : m_cols(0) {} - EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) - : m_data(internal::constructor_without_unaligned_array_assert()), m_cols(0) {} - EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) - : m_data(internal::constructor_without_unaligned_array_assert()), m_cols(other.m_cols) - { - internal::plain_array_helper::copy(other.m_data, _Rows * m_cols, m_data); - } + EIGEN_DEVICE_FUNC constexpr DenseStorage() : m_cols(0) {} + EIGEN_DEVICE_FUNC explicit constexpr DenseStorage(internal::constructor_without_unaligned_array_assert) + : m_data(internal::constructor_without_unaligned_array_assert()), m_cols(0) {} + EIGEN_DEVICE_FUNC constexpr DenseStorage(const DenseStorage& other) + : m_data(internal::constructor_without_unaligned_array_assert()), m_cols(other.m_cols) { + internal::plain_array_helper::copy(other.m_data, Rows_ * m_cols, m_data); + } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { if (this != &other) { m_cols = other.m_cols; - internal::plain_array_helper::copy(other.m_data, _Rows * m_cols, m_data); + internal::plain_array_helper::copy(other.m_data, Rows_ * m_cols, m_data); } return *this; } EIGEN_DEVICE_FUNC DenseStorage(Index, Index, Index cols) : m_cols(cols) {} EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { - internal::plain_array_helper::swap(m_data, _Rows * m_cols, other.m_data, _Rows * other.m_cols); + internal::plain_array_helper::swap(m_data, Rows_ * m_cols, other.m_data, Rows_ * other.m_cols); numext::swap(m_cols, other.m_cols); } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows(void) const EIGEN_NOEXCEPT {return _Rows;} - EIGEN_DEVICE_FUNC Index cols(void) const EIGEN_NOEXCEPT {return m_cols;} - EIGEN_DEVICE_FUNC void conservativeResize(Index, Index, Index cols) { m_cols = cols; } - EIGEN_DEVICE_FUNC void resize(Index, Index, Index cols) { m_cols = cols; } - EIGEN_DEVICE_FUNC const T *data() const { return m_data.array; } - EIGEN_DEVICE_FUNC T *data() { return m_data.array; } + EIGEN_DEVICE_FUNC constexpr Index rows(void) const EIGEN_NOEXCEPT { return Rows_; } + EIGEN_DEVICE_FUNC constexpr Index cols(void) const EIGEN_NOEXCEPT { return m_cols; } + EIGEN_DEVICE_FUNC constexpr void conservativeResize(Index, Index, Index cols) { m_cols = cols; } + EIGEN_DEVICE_FUNC constexpr void resize(Index, Index, Index cols) { m_cols = cols; } + EIGEN_DEVICE_FUNC constexpr const T* data() const { return m_data.array; } + EIGEN_DEVICE_FUNC constexpr T* data() { return m_data.array; } }; // purely dynamic matrix. -template class DenseStorage +template +class DenseStorage { T *m_data; Index m_rows; Index m_cols; public: - EIGEN_DEVICE_FUNC DenseStorage() : m_data(0), m_rows(0), m_cols(0) {} - EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) + EIGEN_DEVICE_FUNC constexpr DenseStorage() : m_data(0), m_rows(0), m_cols(0) {} + EIGEN_DEVICE_FUNC explicit constexpr DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(0), m_rows(0), m_cols(0) {} - EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) - : m_data(internal::conditional_aligned_new_auto(size)), m_rows(rows), m_cols(cols) - { + EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) + : m_data(internal::conditional_aligned_new_auto(size)), + m_rows(rows), + m_cols(cols) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) eigen_internal_assert(size==rows*cols && rows>=0 && cols >=0); - } + } EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) - : m_data(internal::conditional_aligned_new_auto(other.m_rows*other.m_cols)) + : m_data(internal::conditional_aligned_new_auto(other.m_rows*other.m_cols)) , m_rows(other.m_rows) , m_cols(other.m_cols) { @@ -442,7 +484,6 @@ template class DenseStorage class DenseStorage(m_data, m_rows*m_cols); } + EIGEN_DEVICE_FUNC ~DenseStorage() { internal::conditional_aligned_delete_auto(m_data, m_rows*m_cols); } EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { numext::swap(m_data,other.m_data); @@ -473,7 +513,7 @@ template class DenseStorage(m_data, size, m_rows*m_cols); + m_data = internal::conditional_aligned_realloc_new_auto(m_data, size, m_rows*m_cols); m_rows = rows; m_cols = cols; } @@ -481,9 +521,9 @@ template class DenseStorage(m_data, m_rows*m_cols); + internal::conditional_aligned_delete_auto(m_data, m_rows*m_cols); if (size>0) // >0 and not simply !=0 to let the compiler knows that size cannot be negative - m_data = internal::conditional_aligned_new_auto(size); + m_data = internal::conditional_aligned_new_auto(size); else m_data = 0; EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) @@ -496,25 +536,25 @@ template class DenseStorage class DenseStorage -{ +template +class DenseStorage { T *m_data; Index m_cols; public: - EIGEN_DEVICE_FUNC DenseStorage() : m_data(0), m_cols(0) {} - explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(0), m_cols(0) {} - EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) : m_data(internal::conditional_aligned_new_auto(size)), m_cols(cols) - { + EIGEN_DEVICE_FUNC constexpr DenseStorage() : m_data(0), m_cols(0) {} + explicit constexpr DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(0), m_cols(0) {} + EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) + : m_data(internal::conditional_aligned_new_auto(size)), m_cols(cols) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) - eigen_internal_assert(size==rows*cols && rows==_Rows && cols >=0); + eigen_internal_assert(size==rows*cols && rows==Rows_ && cols >=0); EIGEN_UNUSED_VARIABLE(rows); - } + } EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) - : m_data(internal::conditional_aligned_new_auto(_Rows*other.m_cols)) + : m_data(internal::conditional_aligned_new_auto(Rows_*other.m_cols)) , m_cols(other.m_cols) { - EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = m_cols*_Rows) - internal::smart_copy(other.m_data, other.m_data+_Rows*m_cols, m_data); + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = m_cols*Rows_) + internal::smart_copy(other.m_data, other.m_data+Rows_*m_cols, m_data); } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { @@ -525,7 +565,6 @@ template class DenseStorage class DenseStorage(m_data, _Rows*m_cols); } + EIGEN_DEVICE_FUNC ~DenseStorage() { internal::conditional_aligned_delete_auto(m_data, Rows_*m_cols); } EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { numext::swap(m_data,other.m_data); numext::swap(m_cols,other.m_cols); } - EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index rows(void) EIGEN_NOEXCEPT {return _Rows;} + EIGEN_DEVICE_FUNC static constexpr Index rows(void) EIGEN_NOEXCEPT { return Rows_; } EIGEN_DEVICE_FUNC Index cols(void) const EIGEN_NOEXCEPT {return m_cols;} EIGEN_DEVICE_FUNC void conservativeResize(Index size, Index, Index cols) { - m_data = internal::conditional_aligned_realloc_new_auto(m_data, size, _Rows*m_cols); + m_data = internal::conditional_aligned_realloc_new_auto(m_data, size, Rows_*m_cols); m_cols = cols; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void resize(Index size, Index, Index cols) { - if(size != _Rows*m_cols) + if(size != Rows_*m_cols) { - internal::conditional_aligned_delete_auto(m_data, _Rows*m_cols); + internal::conditional_aligned_delete_auto(m_data, Rows_*m_cols); if (size>0) // >0 and not simply !=0 to let the compiler knows that size cannot be negative - m_data = internal::conditional_aligned_new_auto(size); + m_data = internal::conditional_aligned_new_auto(size); else m_data = 0; EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) @@ -572,25 +610,26 @@ template class DenseStorage class DenseStorage +template +class DenseStorage { T *m_data; Index m_rows; public: - EIGEN_DEVICE_FUNC DenseStorage() : m_data(0), m_rows(0) {} - explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(0), m_rows(0) {} - EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) : m_data(internal::conditional_aligned_new_auto(size)), m_rows(rows) - { + EIGEN_DEVICE_FUNC constexpr DenseStorage() : m_data(0), m_rows(0) {} + explicit constexpr DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(0), m_rows(0) {} + EIGEN_DEVICE_FUNC constexpr DenseStorage(Index size, Index rows, Index cols) + : m_data(internal::conditional_aligned_new_auto(size)), m_rows(rows) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) - eigen_internal_assert(size==rows*cols && rows>=0 && cols == _Cols); + eigen_internal_assert(size==rows*cols && rows>=0 && cols == Cols_); EIGEN_UNUSED_VARIABLE(cols); - } + } EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) - : m_data(internal::conditional_aligned_new_auto(other.m_rows*_Cols)) + : m_data(internal::conditional_aligned_new_auto(other.m_rows*Cols_)) , m_rows(other.m_rows) { - EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = m_rows*_Cols) - internal::smart_copy(other.m_data, other.m_data+other.m_rows*_Cols, m_data); + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = m_rows*Cols_) + internal::smart_copy(other.m_data, other.m_data+other.m_rows*Cols_, m_data); } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { @@ -601,7 +640,6 @@ template class DenseStorage class DenseStorage(m_data, _Cols*m_rows); } + EIGEN_DEVICE_FUNC ~DenseStorage() { internal::conditional_aligned_delete_auto(m_data, Cols_*m_rows); } EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { numext::swap(m_data,other.m_data); numext::swap(m_rows,other.m_rows); } EIGEN_DEVICE_FUNC Index rows(void) const EIGEN_NOEXCEPT {return m_rows;} - EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index cols(void) {return _Cols;} + EIGEN_DEVICE_FUNC static constexpr Index cols(void) { return Cols_; } void conservativeResize(Index size, Index rows, Index) { - m_data = internal::conditional_aligned_realloc_new_auto(m_data, size, m_rows*_Cols); + m_data = internal::conditional_aligned_realloc_new_auto(m_data, size, m_rows*Cols_); m_rows = rows; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void resize(Index size, Index rows, Index) { - if(size != m_rows*_Cols) + if(size != m_rows*Cols_) { - internal::conditional_aligned_delete_auto(m_data, _Cols*m_rows); + internal::conditional_aligned_delete_auto(m_data, Cols_*m_rows); if (size>0) // >0 and not simply !=0 to let the compiler knows that size cannot be negative - m_data = internal::conditional_aligned_new_auto(size); + m_data = internal::conditional_aligned_new_auto(size); else m_data = 0; EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Diagonal.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Diagonal.h index 3112d2c16a9..d3694867f1d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Diagonal.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Diagonal.h @@ -11,6 +11,9 @@ #ifndef EIGEN_DIAGONAL_H #define EIGEN_DIAGONAL_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class Diagonal @@ -18,8 +21,8 @@ namespace Eigen { * * \brief Expression of a diagonal/subdiagonal/superdiagonal in a matrix * - * \param MatrixType the type of the object in which we are taking a sub/main/super diagonal - * \param DiagIndex the index of the sub/super diagonal. The default is 0 and it means the main diagonal. + * \tparam MatrixType the type of the object in which we are taking a sub/main/super diagonal + * \tparam DiagIndex the index of the sub/super diagonal. The default is 0 and it means the main diagonal. * A positive value means a superdiagonal, a negative value means a subdiagonal. * You can also use DynamicIndex so the index can be set at runtime. * @@ -38,21 +41,21 @@ struct traits > : traits { typedef typename ref_selector::type MatrixTypeNested; - typedef typename remove_reference::type _MatrixTypeNested; + typedef std::remove_reference_t MatrixTypeNested_; typedef typename MatrixType::StorageKind StorageKind; enum { RowsAtCompileTime = (int(DiagIndex) == DynamicIndex || int(MatrixType::SizeAtCompileTime) == Dynamic) ? Dynamic - : (EIGEN_PLAIN_ENUM_MIN(MatrixType::RowsAtCompileTime - EIGEN_PLAIN_ENUM_MAX(-DiagIndex, 0), - MatrixType::ColsAtCompileTime - EIGEN_PLAIN_ENUM_MAX( DiagIndex, 0))), + : (plain_enum_min(MatrixType::RowsAtCompileTime - plain_enum_max(-DiagIndex, 0), + MatrixType::ColsAtCompileTime - plain_enum_max( DiagIndex, 0))), ColsAtCompileTime = 1, MaxRowsAtCompileTime = int(MatrixType::MaxSizeAtCompileTime) == Dynamic ? Dynamic - : DiagIndex == DynamicIndex ? EIGEN_SIZE_MIN_PREFER_FIXED(MatrixType::MaxRowsAtCompileTime, - MatrixType::MaxColsAtCompileTime) - : (EIGEN_PLAIN_ENUM_MIN(MatrixType::MaxRowsAtCompileTime - EIGEN_PLAIN_ENUM_MAX(-DiagIndex, 0), - MatrixType::MaxColsAtCompileTime - EIGEN_PLAIN_ENUM_MAX( DiagIndex, 0))), + : DiagIndex == DynamicIndex ? min_size_prefer_fixed(MatrixType::MaxRowsAtCompileTime, + MatrixType::MaxColsAtCompileTime) + : (plain_enum_min(MatrixType::MaxRowsAtCompileTime - plain_enum_max(-DiagIndex, 0), + MatrixType::MaxColsAtCompileTime - plain_enum_max( DiagIndex, 0))), MaxColsAtCompileTime = 1, MaskLvalueBit = is_lvalue::value ? LvalueBit : 0, - Flags = (unsigned int)_MatrixTypeNested::Flags & (RowMajorBit | MaskLvalueBit | DirectAccessBit) & ~RowMajorBit, // FIXME DirectAccessBit should not be handled by expressions + Flags = (unsigned int)MatrixTypeNested_::Flags & (RowMajorBit | MaskLvalueBit | DirectAccessBit) & ~RowMajorBit, // FIXME DirectAccessBit should not be handled by expressions MatrixTypeOuterStride = outer_stride_at_compile_time::ret, InnerStrideAtCompileTime = MatrixTypeOuterStride == Dynamic ? Dynamic : MatrixTypeOuterStride+1, OuterStrideAtCompileTime = 0 @@ -60,12 +63,12 @@ struct traits > }; } -template class Diagonal - : public internal::dense_xpr_base< Diagonal >::type +template class Diagonal + : public internal::dense_xpr_base< Diagonal >::type { public: - enum { DiagIndex = _DiagIndex }; + enum { DiagIndex = DiagIndex_ }; typedef typename internal::dense_xpr_base::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Diagonal) @@ -95,11 +98,11 @@ template class Diagonal EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return 0; } - typedef typename internal::conditional< - internal::is_lvalue::value, - Scalar, - const Scalar - >::type ScalarWithConstIfNotLvalue; + typedef std::conditional_t< + internal::is_lvalue::value, + Scalar, + const Scalar + > ScalarWithConstIfNotLvalue; EIGEN_DEVICE_FUNC inline ScalarWithConstIfNotLvalue* data() { return &(m_matrix.coeffRef(rowOffset(), colOffset())); } @@ -145,7 +148,7 @@ template class Diagonal } EIGEN_DEVICE_FUNC - inline const typename internal::remove_all::type& + inline const internal::remove_all_t& nestedExpression() const { return m_matrix; @@ -191,7 +194,8 @@ MatrixBase::diagonal() /** This is the const version of diagonal(). */ template -EIGEN_DEVICE_FUNC inline typename MatrixBase::ConstDiagonalReturnType +EIGEN_DEVICE_FUNC inline +const typename MatrixBase::ConstDiagonalReturnType MatrixBase::diagonal() const { return ConstDiagonalReturnType(derived()); @@ -209,18 +213,18 @@ MatrixBase::diagonal() const * * \sa MatrixBase::diagonal(), class Diagonal */ template -EIGEN_DEVICE_FUNC inline typename MatrixBase::DiagonalDynamicIndexReturnType +EIGEN_DEVICE_FUNC inline Diagonal MatrixBase::diagonal(Index index) { - return DiagonalDynamicIndexReturnType(derived(), index); + return Diagonal(derived(), index); } /** This is the const version of diagonal(Index). */ template -EIGEN_DEVICE_FUNC inline typename MatrixBase::ConstDiagonalDynamicIndexReturnType +EIGEN_DEVICE_FUNC inline const Diagonal MatrixBase::diagonal(Index index) const { - return ConstDiagonalDynamicIndexReturnType(derived(), index); + return Diagonal(derived(), index); } /** \returns an expression of the \a DiagIndex-th sub or super diagonal of the matrix \c *this @@ -237,20 +241,20 @@ MatrixBase::diagonal(Index index) const template template EIGEN_DEVICE_FUNC -inline typename MatrixBase::template DiagonalIndexReturnType::Type +inline Diagonal MatrixBase::diagonal() { - return typename DiagonalIndexReturnType::Type(derived()); + return Diagonal(derived()); } /** This is the const version of diagonal(). */ template template EIGEN_DEVICE_FUNC -inline typename MatrixBase::template ConstDiagonalIndexReturnType::Type +inline const Diagonal MatrixBase::diagonal() const { - return typename ConstDiagonalIndexReturnType::Type(derived()); + return Diagonal(derived()); } } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DiagonalMatrix.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DiagonalMatrix.h index 542685c6594..2b745dcf3df 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DiagonalMatrix.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DiagonalMatrix.h @@ -11,9 +11,24 @@ #ifndef EIGEN_DIAGONALMATRIX_H #define EIGEN_DIAGONALMATRIX_H -namespace Eigen { - -#ifndef EIGEN_PARSED_BY_DOXYGEN +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +/** \class DiagonalBase + * \ingroup Core_Module + * + * \brief Base class for diagonal matrices and expressions + * + * This is the base class that is inherited by diagonal matrix and related expression + * types, which internally use a vector for storing the diagonal entries. Diagonal + * types always represent square matrices. + * + * \tparam Derived is the derived type, a DiagonalMatrix or DiagonalWrapper. + * + * \sa class DiagonalMatrix, class DiagonalWrapper + */ template class DiagonalBase : public EigenBase { @@ -37,24 +52,42 @@ class DiagonalBase : public EigenBase typedef DenseMatrixType DenseType; typedef DiagonalMatrix PlainObject; + /** \returns a reference to the derived object. */ EIGEN_DEVICE_FUNC inline const Derived& derived() const { return *static_cast(this); } + /** \returns a const reference to the derived object. */ EIGEN_DEVICE_FUNC inline Derived& derived() { return *static_cast(this); } + /** + * Constructs a dense matrix from \c *this. Note, this directly returns a dense matrix type, + * not an expression. + * \returns A dense matrix, with its diagonal entries set from the the derived object. */ EIGEN_DEVICE_FUNC DenseMatrixType toDenseMatrix() const { return derived(); } + /** \returns a reference to the derived object's vector of diagonal coefficients. */ EIGEN_DEVICE_FUNC inline const DiagonalVectorType& diagonal() const { return derived().diagonal(); } + /** \returns a const reference to the derived object's vector of diagonal coefficients. */ EIGEN_DEVICE_FUNC inline DiagonalVectorType& diagonal() { return derived().diagonal(); } + /** \returns the value of the coefficient as if \c *this was a dense matrix. */ EIGEN_DEVICE_FUNC + inline Scalar coeff(Index row, Index col) const { + eigen_assert(row >= 0 && col >= 0 && row < rows() && col <= cols()); + return row == col ? diagonal().coeff(row) : Scalar(0); + } + + /** \returns the number of rows. */ + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const { return diagonal().size(); } - EIGEN_DEVICE_FUNC + /** \returns the number of columns. */ + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const { return diagonal().size(); } + /** \returns the diagonal matrix product of \c *this by the dense matrix, \a matrix */ template EIGEN_DEVICE_FUNC const Product @@ -63,88 +96,99 @@ class DiagonalBase : public EigenBase return Product(derived(),matrix.derived()); } - typedef DiagonalWrapper, const DiagonalVectorType> > InverseReturnType; - EIGEN_DEVICE_FUNC - inline const InverseReturnType - inverse() const - { - return InverseReturnType(diagonal().cwiseInverse()); + template + using DiagonalProductReturnType = DiagonalWrapper; + + /** \returns the diagonal matrix product of \c *this by the diagonal matrix \a other */ + template + EIGEN_DEVICE_FUNC const DiagonalProductReturnType operator*( + const DiagonalBase& other) const { + return diagonal().cwiseProduct(other.diagonal()).asDiagonal(); } - + + using DiagonalInverseReturnType = + DiagonalWrapper, const DiagonalVectorType>>; + + /** \returns the inverse \c *this. Computed as the coefficient-wise inverse of the diagonal. */ EIGEN_DEVICE_FUNC - inline const DiagonalWrapper - operator*(const Scalar& scalar) const - { - return DiagonalWrapper(diagonal() * scalar); - } + inline const DiagonalInverseReturnType inverse() const { return diagonal().cwiseInverse().asDiagonal(); } + + using DiagonalScaleReturnType = + DiagonalWrapper; + + /** \returns the product of \c *this by the scalar \a scalar */ EIGEN_DEVICE_FUNC - friend inline const DiagonalWrapper - operator*(const Scalar& scalar, const DiagonalBase& other) - { - return DiagonalWrapper(scalar * other.diagonal()); + inline const DiagonalScaleReturnType operator*(const Scalar& scalar) const { + return (diagonal() * scalar).asDiagonal(); } - template + using ScaleDiagonalReturnType = + DiagonalWrapper; + + /** \returns the product of a scalar and the diagonal matrix \a other */ EIGEN_DEVICE_FUNC - #ifdef EIGEN_PARSED_BY_DOXYGEN - inline unspecified_expression_type - #else - inline const DiagonalWrapper - #endif - operator+(const DiagonalBase& other) const - { + friend inline const ScaleDiagonalReturnType operator*(const Scalar& scalar, const DiagonalBase& other) { + return (scalar * other.diagonal()).asDiagonal(); + } + + template + using DiagonalSumReturnType = DiagonalWrapper; + + /** \returns the sum of \c *this and the diagonal matrix \a other */ + template + EIGEN_DEVICE_FUNC inline const DiagonalSumReturnType operator+( + const DiagonalBase& other) const { return (diagonal() + other.diagonal()).asDiagonal(); } - template - EIGEN_DEVICE_FUNC - #ifdef EIGEN_PARSED_BY_DOXYGEN - inline unspecified_expression_type - #else - inline const DiagonalWrapper - #endif - operator-(const DiagonalBase& other) const - { + template + using DiagonalDifferenceReturnType = DiagonalWrapper; + + /** \returns the difference of \c *this and the diagonal matrix \a other */ + template + EIGEN_DEVICE_FUNC inline const DiagonalDifferenceReturnType operator-( + const DiagonalBase& other) const { return (diagonal() - other.diagonal()).asDiagonal(); } }; -#endif - /** \class DiagonalMatrix - * \ingroup Core_Module - * - * \brief Represents a diagonal matrix with its storage - * - * \param _Scalar the type of coefficients - * \param SizeAtCompileTime the dimension of the matrix, or Dynamic - * \param MaxSizeAtCompileTime the dimension of the matrix, or Dynamic. This parameter is optional and defaults - * to SizeAtCompileTime. Most of the time, you do not need to specify it. - * - * \sa class DiagonalWrapper - */ + * \ingroup Core_Module + * + * \brief Represents a diagonal matrix with its storage + * + * \tparam Scalar_ the type of coefficients + * \tparam SizeAtCompileTime the dimension of the matrix, or Dynamic + * \tparam MaxSizeAtCompileTime the dimension of the matrix, or Dynamic. This parameter is optional and defaults + * to SizeAtCompileTime. Most of the time, you do not need to specify it. + * + * \sa class DiagonalBase, class DiagonalWrapper + */ namespace internal { -template -struct traits > - : traits > +template +struct traits > + : traits > { - typedef Matrix<_Scalar,SizeAtCompileTime,1,0,MaxSizeAtCompileTime,1> DiagonalVectorType; + typedef Matrix DiagonalVectorType; typedef DiagonalShape StorageKind; enum { - Flags = LvalueBit | NoPreferredStorageOrderBit + Flags = LvalueBit | NoPreferredStorageOrderBit | NestByRefBit }; }; } -template +template class DiagonalMatrix - : public DiagonalBase > + : public DiagonalBase > { public: #ifndef EIGEN_PARSED_BY_DOXYGEN typedef typename internal::traits::DiagonalVectorType DiagonalVectorType; typedef const DiagonalMatrix& Nested; - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef typename internal::traits::StorageKind StorageKind; typedef typename internal::traits::StorageIndex StorageIndex; #endif @@ -178,10 +222,7 @@ class DiagonalMatrix EIGEN_DEVICE_FUNC inline DiagonalMatrix(const Scalar& x, const Scalar& y, const Scalar& z) : m_diagonal(x,y,z) {} - #if EIGEN_HAS_CXX11 - /** \brief Construct a diagonal matrix with fixed size from an arbitrary number of coefficients. \cpp11 - * - * There exists C++98 anologue constructors for fixed-size diagonal matrices having 2 or 3 coefficients. + /** \brief Construct a diagonal matrix with fixed size from an arbitrary number of coefficients. * * \warning To construct a diagonal matrix of fixed size, the number of values passed to this * constructor must match the fixed dimension of \c *this. @@ -200,7 +241,10 @@ class DiagonalMatrix EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE DiagonalMatrix(const std::initializer_list>& list) : m_diagonal(list) {} - #endif // EIGEN_HAS_CXX11 + + /** \brief Constructs a DiagonalMatrix from an r-value diagonal vector type */ + EIGEN_DEVICE_FUNC + explicit inline DiagonalMatrix(DiagonalVectorType&& diag) : m_diagonal(std::move(diag)) {} /** Copy constructor. */ template @@ -239,6 +283,22 @@ class DiagonalMatrix } #endif + typedef DiagonalWrapper, DiagonalVectorType>> + InitializeReturnType; + + /** Initializes a diagonal matrix of size SizeAtCompileTime with coefficients set to zero */ + EIGEN_DEVICE_FUNC + static const InitializeReturnType Zero() { return DiagonalVectorType::Zero().asDiagonal(); } + /** Initializes a diagonal matrix of size dim with coefficients set to zero */ + EIGEN_DEVICE_FUNC + static const InitializeReturnType Zero(Index size) { return DiagonalVectorType::Zero(size).asDiagonal(); } + /** Initializes a identity matrix of size SizeAtCompileTime */ + EIGEN_DEVICE_FUNC + static const InitializeReturnType Identity() { return DiagonalVectorType::Ones().asDiagonal(); } + /** Initializes a identity matrix of size dim */ + EIGEN_DEVICE_FUNC + static const InitializeReturnType Identity(Index size) { return DiagonalVectorType::Ones(size).asDiagonal(); } + /** Resizes to given size. */ EIGEN_DEVICE_FUNC inline void resize(Index size) { m_diagonal.resize(size); } @@ -261,7 +321,7 @@ class DiagonalMatrix * * \brief Expression of a diagonal matrix * - * \param _DiagonalVectorType the type of the vector of diagonal coefficients + * \tparam DiagonalVectorType_ the type of the vector of diagonal coefficients * * This class is an expression of a diagonal matrix, but not storing its own vector of diagonal coefficients, * instead wrapping an existing vector expression. It is the return type of MatrixBase::asDiagonal() @@ -271,10 +331,10 @@ class DiagonalMatrix */ namespace internal { -template -struct traits > +template +struct traits > { - typedef _DiagonalVectorType DiagonalVectorType; + typedef DiagonalVectorType_ DiagonalVectorType; typedef typename DiagonalVectorType::Scalar Scalar; typedef typename DiagonalVectorType::StorageIndex StorageIndex; typedef DiagonalShape StorageKind; @@ -289,13 +349,13 @@ struct traits > }; } -template +template class DiagonalWrapper - : public DiagonalBase >, internal::no_assignment_operator + : public DiagonalBase >, internal::no_assignment_operator { public: #ifndef EIGEN_PARSED_BY_DOXYGEN - typedef _DiagonalVectorType DiagonalVectorType; + typedef DiagonalVectorType_ DiagonalVectorType; typedef DiagonalWrapper Nested; #endif @@ -386,6 +446,6 @@ struct Assignment } // namespace internal -} // end namespace Eigen +} // end namespace Eigen #endif // EIGEN_DIAGONALMATRIX_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DiagonalProduct.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DiagonalProduct.h index 7911d1cd174..aad474d95a8 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DiagonalProduct.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/DiagonalProduct.h @@ -11,6 +11,9 @@ #ifndef EIGEN_DIAGONALPRODUCT_H #define EIGEN_DIAGONALPRODUCT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \returns the diagonal matrix product of \c *this by the diagonal matrix \a diagonal. diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Dot.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Dot.h index 5c3441b9266..a8ce736fc06 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Dot.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Dot.h @@ -10,6 +10,9 @@ #ifndef EIGEN_DOT_H #define EIGEN_DOT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -18,14 +21,9 @@ namespace internal { // with mismatched types, the compiler emits errors about failing to instantiate cwiseProduct BEFORE // looking at the static assertions. Thus this is a trick to get better compile errors. template + bool NeedToTranspose = T::IsVectorAtCompileTime && U::IsVectorAtCompileTime && + ((int(T::RowsAtCompileTime) == 1 && int(U::ColsAtCompileTime) == 1) || + (int(T::ColsAtCompileTime) == 1 && int(U::RowsAtCompileTime) == 1))> struct dot_nocheck { typedef scalar_conj_product_op::Scalar,typename traits::Scalar> conj_prod; @@ -75,8 +73,9 @@ MatrixBase::dot(const MatrixBase& other) const EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_SAME_VECTOR_SIZE(Derived,OtherDerived) #if !(defined(EIGEN_NO_STATIC_ASSERT) && defined(EIGEN_NO_DEBUG)) - typedef internal::scalar_conj_product_op func; - EIGEN_CHECK_BINARY_COMPATIBILIY(func,Scalar,typename OtherDerived::Scalar); + EIGEN_CHECK_BINARY_COMPATIBILIY( + Eigen::internal::scalar_conj_product_op, + Scalar, typename OtherDerived::Scalar); #endif eigen_assert(size() == other.size()); @@ -123,8 +122,8 @@ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::PlainObject MatrixBase::normalized() const { - typedef typename internal::nested_eval::type _Nested; - _Nested n(derived()); + typedef typename internal::nested_eval::type Nested_; + Nested_ n(derived()); RealScalar z = n.squaredNorm(); // NOTE: after extensive benchmarking, this conditional does not impact performance, at least on recent x86 CPU if(z>RealScalar(0)) @@ -166,8 +165,8 @@ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::PlainObject MatrixBase::stableNormalized() const { - typedef typename internal::nested_eval::type _Nested; - _Nested n(derived()); + typedef typename internal::nested_eval::type Nested_; + Nested_ n(derived()); RealScalar w = n.cwiseAbs().maxCoeff(); RealScalar z = (n/w).squaredNorm(); if(z>RealScalar(0)) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/EigenBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/EigenBase.h index 6b3c7d3745e..251125ba5e0 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/EigenBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/EigenBase.h @@ -11,6 +11,9 @@ #ifndef EIGEN_EIGENBASE_H #define EIGEN_EIGENBASE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class EigenBase diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ForceAlignedAccess.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ForceAlignedAccess.h index 817a43afced..643ff9b7549 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ForceAlignedAccess.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ForceAlignedAccess.h @@ -10,6 +10,9 @@ #ifndef EIGEN_FORCEALIGNEDACCESS_H #define EIGEN_FORCEALIGNEDACCESS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class ForceAlignedAccess @@ -128,7 +131,7 @@ MatrixBase::forceAlignedAccess() */ template template -inline typename internal::add_const_on_value_type,Derived&>::type>::type +inline add_const_on_value_type_t,Derived&>> MatrixBase::forceAlignedAccessIf() const { return derived(); // FIXME This should not work but apparently is never used @@ -139,7 +142,7 @@ MatrixBase::forceAlignedAccessIf() const */ template template -inline typename internal::conditional,Derived&>::type +inline std::conditional_t,Derived&> MatrixBase::forceAlignedAccessIf() { return derived(); // FIXME This should not work but apparently is never used diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Fuzzy.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Fuzzy.h index 43aa49b2bc2..26b25c200a5 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Fuzzy.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Fuzzy.h @@ -11,6 +11,9 @@ #ifndef EIGEN_FUZZY_H #define EIGEN_FUZZY_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal @@ -24,7 +27,7 @@ struct isApprox_selector { typename internal::nested_eval::type nested(x); typename internal::nested_eval::type otherNested(y); - return (nested - otherNested).cwiseAbs2().sum() <= prec * prec * numext::mini(nested.cwiseAbs2().sum(), otherNested.cwiseAbs2().sum()); + return (nested.matrix() - otherNested.matrix()).cwiseAbs2().sum() <= prec * prec * numext::mini(nested.cwiseAbs2().sum(), otherNested.cwiseAbs2().sum()); } }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GeneralProduct.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GeneralProduct.h index 6906aa75d10..24a8a79bcd5 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GeneralProduct.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GeneralProduct.h @@ -11,6 +11,9 @@ #ifndef EIGEN_GENERAL_PRODUCT_H #define EIGEN_GENERAL_PRODUCT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { enum { @@ -50,17 +53,17 @@ template struct product_size_category template struct product_type { - typedef typename remove_all::type _Lhs; - typedef typename remove_all::type _Rhs; + typedef remove_all_t Lhs_; + typedef remove_all_t Rhs_; enum { - MaxRows = traits<_Lhs>::MaxRowsAtCompileTime, - Rows = traits<_Lhs>::RowsAtCompileTime, - MaxCols = traits<_Rhs>::MaxColsAtCompileTime, - Cols = traits<_Rhs>::ColsAtCompileTime, - MaxDepth = EIGEN_SIZE_MIN_PREFER_FIXED(traits<_Lhs>::MaxColsAtCompileTime, - traits<_Rhs>::MaxRowsAtCompileTime), - Depth = EIGEN_SIZE_MIN_PREFER_FIXED(traits<_Lhs>::ColsAtCompileTime, - traits<_Rhs>::RowsAtCompileTime) + MaxRows = traits::MaxRowsAtCompileTime, + Rows = traits::RowsAtCompileTime, + MaxCols = traits::MaxColsAtCompileTime, + Cols = traits::ColsAtCompileTime, + MaxDepth = min_size_prefer_fixed(traits::MaxColsAtCompileTime, + traits::MaxRowsAtCompileTime), + Depth = min_size_prefer_fixed(traits::ColsAtCompileTime, + traits::RowsAtCompileTime) }; // the splitting into different lines of code here, introducing the _select enums and the typedef below, @@ -180,15 +183,16 @@ struct gemv_static_vector_if PacketSize = internal::packet_traits::size }; #if EIGEN_MAX_STATIC_ALIGN_BYTES!=0 - internal::plain_array m_data; + internal::plain_array m_data; EIGEN_STRONG_INLINE Scalar* data() { return m_data.array; } #else // Some architectures cannot align on the stack, // => let's manually enforce alignment by allocating more data and return the address of the first aligned element. - internal::plain_array m_data; + internal::plain_array m_data; EIGEN_STRONG_INLINE Scalar* data() { return ForceAlignment - ? reinterpret_cast((internal::UIntPtr(m_data.array) & ~(std::size_t(EIGEN_MAX_ALIGN_BYTES-1))) + EIGEN_MAX_ALIGN_BYTES) + ? reinterpret_cast((std::uintptr_t(m_data.array) & ~(std::size_t(EIGEN_MAX_ALIGN_BYTES-1))) + EIGEN_MAX_ALIGN_BYTES) : m_data.array; } #endif @@ -216,14 +220,13 @@ template<> struct gemv_dense_selector typedef typename Lhs::Scalar LhsScalar; typedef typename Rhs::Scalar RhsScalar; typedef typename Dest::Scalar ResScalar; - typedef typename Dest::RealScalar RealScalar; typedef internal::blas_traits LhsBlasTraits; typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; typedef internal::blas_traits RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; - typedef Map, EIGEN_PLAIN_ENUM_MIN(AlignedMax,internal::packet_traits::size)> MappedDest; + typedef Map, plain_enum_min(AlignedMax, internal::packet_traits::size)> MappedDest; ActualLhsType actualLhs = LhsBlasTraits::extract(lhs); ActualRhsType actualRhs = RhsBlasTraits::extract(rhs); @@ -231,7 +234,7 @@ template<> struct gemv_dense_selector ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs); // make sure Dest is a compile-time vector type (bug 1166) - typedef typename conditional::type ActualDest; + typedef std::conditional_t ActualDest; enum { // FIXME find a way to allow an inner stride on the result if packet_traits::size==1 @@ -261,7 +264,7 @@ template<> struct gemv_dense_selector { gemv_static_vector_if static_dest; - const bool alphaIsCompatible = (!ComplexByReal) || (numext::imag(actualAlpha)==RealScalar(0)); + const bool alphaIsCompatible = (!ComplexByReal) || (numext::is_exactly_zero(numext::imag(actualAlpha))); const bool evalToDest = EvalToDestAtCompileTime && alphaIsCompatible; ei_declare_aligned_stack_constructed_variable(ResScalar,actualDestPtr,dest.size(), @@ -314,10 +317,10 @@ template<> struct gemv_dense_selector typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; typedef internal::blas_traits RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; - typedef typename internal::remove_all::type ActualRhsTypeCleaned; + typedef internal::remove_all_t ActualRhsTypeCleaned; - typename add_const::type actualLhs = LhsBlasTraits::extract(lhs); - typename add_const::type actualRhs = RhsBlasTraits::extract(rhs); + std::add_const_t actualLhs = LhsBlasTraits::extract(lhs); + std::add_const_t actualRhs = RhsBlasTraits::extract(rhs); ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GenericPacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GenericPacketMath.h index cf677a1905f..d04f7138ae5 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GenericPacketMath.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GenericPacketMath.h @@ -11,6 +11,9 @@ #ifndef EIGEN_GENERIC_PACKET_MATH_H #define EIGEN_GENERIC_PACKET_MATH_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -42,8 +45,6 @@ namespace internal { struct default_packet_traits { enum { - HasHalfPacket = 0, - HasAdd = 1, HasSub = 1, HasShift = 1, @@ -57,12 +58,14 @@ struct default_packet_traits HasMax = 1, HasConj = 1, HasSetLinear = 1, + HasSign = 1, HasBlend = 0, // This flag is used to indicate whether packet comparison is supported. // pcmp_eq, pcmp_lt and pcmp_le should be defined for it to be true. HasCmp = 0, HasDiv = 0, + HasReciprocal = 0, HasSqrt = 0, HasRsqrt = 0, HasExp = 0, @@ -78,6 +81,7 @@ struct default_packet_traits HasASin = 0, HasACos = 0, HasATan = 0, + HasATanh = 0, HasSinh = 0, HasCosh = 0, HasTanh = 0, @@ -98,8 +102,7 @@ struct default_packet_traits HasRound = 0, HasRint = 0, HasFloor = 0, - HasCeil = 0, - HasSign = 0 + HasCeil = 0 }; }; @@ -111,7 +114,6 @@ template struct packet_traits : default_packet_traits Vectorizable = 0, size = 1, AlignedOnScalar = 0, - HasHalfPacket = 0 }; enum { HasAdd = 0, @@ -145,14 +147,78 @@ template struct unpacket_traits template struct unpacket_traits : unpacket_traits { }; -template struct type_casting_traits { +/** \internal A convenience utility for determining if the type is a scalar. + * This is used to enable some generic packet implementations. + */ +template +struct is_scalar { + using Scalar = typename unpacket_traits::type; + enum { value = internal::is_same::value }; +}; + +// automatically and succinctly define combinations of pcast when +// 1) the packets are the same type, or +// 2) the packets differ only in sign. +// In both of these cases, preinterpret (bit_cast) is equivalent to pcast (static_cast) +template ::value && is_scalar::value> +struct is_degenerate_helper : is_same {}; +template <> +struct is_degenerate_helper : std::true_type {}; +template <> +struct is_degenerate_helper : std::true_type {}; +template <> +struct is_degenerate_helper : std::true_type {}; +template <> +struct is_degenerate_helper : std::true_type {}; + +template +struct is_degenerate_helper { + using SrcScalar = typename unpacket_traits::type; + static constexpr int SrcSize = unpacket_traits::size; + using TgtScalar = typename unpacket_traits::type; + static constexpr int TgtSize = unpacket_traits::size; + static constexpr bool value = is_degenerate_helper::value && (SrcSize == TgtSize); +}; + +// is_degenerate::value == is_degenerate::value +template +struct is_degenerate { + static constexpr bool value = + is_degenerate_helper::value || is_degenerate_helper::value; +}; + +template +struct is_half { + using Scalar = typename unpacket_traits::type; + static constexpr int Size = unpacket_traits::size; + using DefaultPacket = typename packet_traits::type; + static constexpr int DefaultSize = unpacket_traits::size; + static constexpr bool value = Size < DefaultSize; +}; + +template +struct type_casting_traits { enum { - VectorizedCast = 0, + VectorizedCast = + is_degenerate::value && packet_traits::Vectorizable && packet_traits::Vectorizable, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; +// provides a succint template to define vectorized casting traits with respect to the largest accessible packet types +template +struct vectorized_type_casting_traits { + enum : int { + DefaultSrcPacketSize = packet_traits::size, + DefaultTgtPacketSize = packet_traits::size, + VectorizedCast = 1, + SrcCoeffRatio = plain_enum_max(DefaultTgtPacketSize / DefaultSrcPacketSize, 1), + TgtCoeffRatio = plain_enum_max(DefaultSrcPacketSize / DefaultTgtPacketSize, 1) + }; +}; + /** \internal Wrapper to ensure that multiple packet types can map to the same same underlying vector type. */ template @@ -160,7 +226,7 @@ struct eigen_packet_wrapper { EIGEN_ALWAYS_INLINE operator T&() { return m_val; } EIGEN_ALWAYS_INLINE operator const T&() const { return m_val; } - EIGEN_ALWAYS_INLINE eigen_packet_wrapper() {} + EIGEN_ALWAYS_INLINE eigen_packet_wrapper() = default; EIGEN_ALWAYS_INLINE eigen_packet_wrapper(const T &v) : m_val(v) {} EIGEN_ALWAYS_INLINE eigen_packet_wrapper& operator=(const T &v) { m_val = v; @@ -170,45 +236,84 @@ struct eigen_packet_wrapper T m_val; }; +template ::value> +struct preinterpret_generic; -/** \internal A convenience utility for determining if the type is a scalar. - * This is used to enable some generic packet implementations. - */ -template -struct is_scalar { - typedef typename unpacket_traits::type Scalar; - enum { - value = internal::is_same::value - }; +template +struct preinterpret_generic { + // the packets are not the same, attempt scalar bit_cast + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Target run(const Packet& a) { + return numext::bit_cast(a); + } +}; + +template +struct preinterpret_generic { + // the packets are the same type: do nothing + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& a) { return a; } +}; + +/** \internal \returns reinterpret_cast(a) */ +template +EIGEN_DEVICE_FUNC inline Target preinterpret(const Packet& a) { + return preinterpret_generic::run(a); +} + +template ::value, bool TgtIsHalf = is_half::value> +struct pcast_generic; + +template +struct pcast_generic { + // the packets are not degenerate: attempt scalar static_cast + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TgtPacket run(const SrcPacket& a) { + return cast_impl::run(a); + } +}; + +template +struct pcast_generic { + // the packets are the same: do nothing + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& a) { return a; } }; +template +struct pcast_generic { + // the packets are degenerate: preinterpret is equivalent to pcast + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TgtPacket run(const SrcPacket& a) { return preinterpret(a); } +}; + + + /** \internal \returns static_cast(a) (coeff-wise) */ template -EIGEN_DEVICE_FUNC inline TgtPacket -pcast(const SrcPacket& a) { - return static_cast(a); +EIGEN_DEVICE_FUNC inline TgtPacket pcast(const SrcPacket& a) { + return pcast_generic::run(a); } template -EIGEN_DEVICE_FUNC inline TgtPacket -pcast(const SrcPacket& a, const SrcPacket& /*b*/) { - return static_cast(a); +EIGEN_DEVICE_FUNC inline TgtPacket pcast(const SrcPacket& a, const SrcPacket& b) { + return pcast_generic::run(a, b); } template -EIGEN_DEVICE_FUNC inline TgtPacket -pcast(const SrcPacket& a, const SrcPacket& /*b*/, const SrcPacket& /*c*/, const SrcPacket& /*d*/) { - return static_cast(a); +EIGEN_DEVICE_FUNC inline TgtPacket pcast(const SrcPacket& a, const SrcPacket& b, const SrcPacket& c, + const SrcPacket& d) { + return pcast_generic::run(a, b, c, d); } template -EIGEN_DEVICE_FUNC inline TgtPacket -pcast(const SrcPacket& a, const SrcPacket& /*b*/, const SrcPacket& /*c*/, const SrcPacket& /*d*/, - const SrcPacket& /*e*/, const SrcPacket& /*f*/, const SrcPacket& /*g*/, const SrcPacket& /*h*/) { - return static_cast(a); +EIGEN_DEVICE_FUNC inline TgtPacket pcast(const SrcPacket& a, const SrcPacket& b, const SrcPacket& c, const SrcPacket& d, + const SrcPacket& e, const SrcPacket& f, const SrcPacket& g, + const SrcPacket& h) { + return pcast_generic::run(a, b, c, d, e, f, g, h); } -/** \internal \returns reinterpret_cast(a) */ -template -EIGEN_DEVICE_FUNC inline Target -preinterpret(const Packet& a); /* { return reinterpret_cast(a); } */ +template +struct pcast_generic { + // TgtPacket is a half packet of some other type + // perform cast and truncate result + using DefaultTgtPacket = typename is_half::DefaultPacket; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TgtPacket run(const SrcPacket& a) { + return preinterpret(pcast(a)); + } +}; /** \internal \returns a + b (coeff-wise) */ template EIGEN_DEVICE_FUNC inline Packet @@ -217,6 +322,15 @@ padd(const Packet& a, const Packet& b) { return a+b; } template<> EIGEN_DEVICE_FUNC inline bool padd(const bool& a, const bool& b) { return a || b; } +/** \internal \returns a packet version of \a *from, (un-aligned masked add) + * There is no generic implementation. We only have implementations for specialized + * cases. Generic case should not be called. + */ +template EIGEN_DEVICE_FUNC inline +std::enable_if_t::masked_fpops_available, Packet> +padd(const Packet& a, const Packet& b, typename unpacket_traits::mask_t umask); + + /** \internal \returns a - b (coeff-wise) */ template EIGEN_DEVICE_FUNC inline Packet psub(const Packet& a, const Packet& b) { return a-b; } @@ -259,7 +373,7 @@ struct ptrue_impl { // have another option, since the scalar type requires initialization. template struct ptrue_impl::value && NumTraits::RequireInitialization>::type > { + std::enable_if_t::value && NumTraits::RequireInitialization> > { static EIGEN_DEVICE_FUNC inline T run(const T& /*a*/){ return T(1); } @@ -285,7 +399,7 @@ struct pzero_impl { // for zero may not consist of all-zero bits. template struct pzero_impl::value>::type> { + std::enable_if_t::value>> { static EIGEN_DEVICE_FUNC inline T run(const T& /*a*/) { return T(0); } @@ -356,16 +470,16 @@ struct bytewise_bitwise_helper { EIGEN_DEVICE_FUNC static inline T bitwise_and(const T& a, const T& b) { return binary(a, b, bit_and()); } - EIGEN_DEVICE_FUNC static inline T bitwise_or(const T& a, const T& b) { + EIGEN_DEVICE_FUNC static inline T bitwise_or(const T& a, const T& b) { return binary(a, b, bit_or()); } EIGEN_DEVICE_FUNC static inline T bitwise_xor(const T& a, const T& b) { return binary(a, b, bit_xor()); } - EIGEN_DEVICE_FUNC static inline T bitwise_not(const T& a) { + EIGEN_DEVICE_FUNC static inline T bitwise_not(const T& a) { return unary(a,bit_not()); } - + private: template EIGEN_DEVICE_FUNC static inline T unary(const T& a, Op op) { @@ -398,8 +512,8 @@ struct bitwise_helper : public bytewise_bitwise_helper {}; // For integers or non-trivial scalars, use binary operators. template struct bitwise_helper::value && (NumTraits::IsInteger || NumTraits::RequireInitialization)>::type + typename std::enable_if_t< + is_scalar::value && (NumTraits::IsInteger || NumTraits::RequireInitialization)> > : public operator_bitwise_helper {}; /** \internal \returns the bitwise and of \a a and \a b */ @@ -430,6 +544,12 @@ pnot(const Packet& a) { template EIGEN_DEVICE_FUNC inline Packet pandnot(const Packet& a, const Packet& b) { return pand(a, pnot(b)); } +/** \internal \returns isnan(a) */ +template EIGEN_DEVICE_FUNC inline Packet +pisnan(const Packet& a) { + return pandnot(ptrue(a), pcmp_eq(a, a)); +} + // In the general case, use bitwise select. template struct pselect_impl { @@ -441,7 +561,7 @@ struct pselect_impl { // For scalars, use ternary select. template struct pselect_impl::value>::type > { + std::enable_if_t::value> > { static EIGEN_DEVICE_FUNC inline Packet run(const Packet& mask, const Packet& a, const Packet& b) { return numext::equal_strict(mask, Packet(0)) ? b : a; } @@ -551,13 +671,13 @@ template EIGEN_DEVICE_FUNC inline Packet parg(const Packet& a) { using numext::arg; return arg(a); } -/** \internal \returns \a a logically shifted by N bits to the right */ +/** \internal \returns \a a arithmetically shifted by N bits to the right */ template EIGEN_DEVICE_FUNC inline int parithmetic_shift_right(const int& a) { return a >> N; } template EIGEN_DEVICE_FUNC inline long int parithmetic_shift_right(const long int& a) { return a >> N; } -/** \internal \returns \a a arithmetically shifted by N bits to the right */ +/** \internal \returns \a a logically shifted by N bits to the right */ template EIGEN_DEVICE_FUNC inline int plogical_shift_right(const int& a) { return static_cast(static_cast(a) >> N); } template EIGEN_DEVICE_FUNC inline long int @@ -594,20 +714,52 @@ pldexp(const Packet &a, const Packet &exponent) { template EIGEN_DEVICE_FUNC inline Packet pabsdiff(const Packet& a, const Packet& b) { return pselect(pcmp_lt(a, b), psub(b, a), psub(a, b)); } -/** \internal \returns a packet version of \a *from, from must be 16 bytes aligned */ +/** \internal \returns a packet version of \a *from, from must be properly aligned */ template EIGEN_DEVICE_FUNC inline Packet pload(const typename unpacket_traits::type* from) { return *from; } +/** \internal \returns n elements of a packet version of \a *from, from must be properly aligned + * offset indicates the starting element in which to load and + * offset + n <= unpacket_traits::size + * All elements before offset and after the last element loaded will initialized with zero */ +template EIGEN_DEVICE_FUNC inline Packet +pload_partial(const typename unpacket_traits::type* from, const Index n, const Index offset = 0) +{ + const Index packet_size = unpacket_traits::size; + eigen_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet"); + typedef typename unpacket_traits::type Scalar; + EIGEN_ALIGN_MAX Scalar elements[packet_size] = { Scalar(0) }; + for (Index i = offset; i < numext::mini(n+offset,packet_size); i++) { + elements[i] = from[i-offset]; + } + return pload(elements); +} + /** \internal \returns a packet version of \a *from, (un-aligned load) */ template EIGEN_DEVICE_FUNC inline Packet ploadu(const typename unpacket_traits::type* from) { return *from; } +/** \internal \returns n elements of a packet version of \a *from, (un-aligned load) + * All elements after the last element loaded will initialized with zero */ +template EIGEN_DEVICE_FUNC inline Packet +ploadu_partial(const typename unpacket_traits::type* from, const Index n, const Index offset = 0) +{ + const Index packet_size = unpacket_traits::size; + eigen_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet"); + typedef typename unpacket_traits::type Scalar; + EIGEN_ALIGN_MAX Scalar elements[packet_size] = { Scalar(0) }; + for (Index i = offset; i < numext::mini(n+offset,packet_size); i++) { + elements[i] = from[i-offset]; + } + return pload(elements); +} + /** \internal \returns a packet version of \a *from, (un-aligned masked load) * There is no generic implementation. We only have implementations for specialized * cases. Generic case should not be called. */ template EIGEN_DEVICE_FUNC inline -typename enable_if::masked_load_available, Packet>::type +std::enable_if_t::masked_load_available, Packet> ploadu(const typename unpacket_traits::type* from, typename unpacket_traits::mask_t umask); /** \internal \returns a packet with constant coefficients \a a, e.g.: (a,a,a,a) */ @@ -692,28 +844,74 @@ peven_mask(const Packet& /*a*/) { } -/** \internal copy the packet \a from to \a *to, \a to must be 16 bytes aligned */ +/** \internal copy the packet \a from to \a *to, \a to must be properly aligned */ template EIGEN_DEVICE_FUNC inline void pstore(Scalar* to, const Packet& from) { (*to) = from; } +/** \internal copy n elements of the packet \a from to \a *to, \a to must be properly aligned + * offset indicates the starting element in which to store and + * offset + n <= unpacket_traits::size */ +template EIGEN_DEVICE_FUNC inline void pstore_partial(Scalar* to, const Packet& from, const Index n, const Index offset = 0) +{ + const Index packet_size = unpacket_traits::size; + eigen_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet"); + EIGEN_ALIGN_MAX Scalar elements[packet_size]; + pstore(elements, from); + for (Index i = 0; i < numext::mini(n,packet_size-offset); i++) { + to[i] = elements[i + offset]; + } +} + /** \internal copy the packet \a from to \a *to, (un-aligned store) */ template EIGEN_DEVICE_FUNC inline void pstoreu(Scalar* to, const Packet& from) { (*to) = from; } +/** \internal copy n elements of the packet \a from to \a *to, (un-aligned store) */ +template EIGEN_DEVICE_FUNC inline void pstoreu_partial(Scalar* to, const Packet& from, const Index n, const Index offset = 0) +{ + const Index packet_size = unpacket_traits::size; + eigen_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet"); + EIGEN_ALIGN_MAX Scalar elements[packet_size]; + pstore(elements, from); + for (Index i = 0; i < numext::mini(n,packet_size-offset); i++) { + to[i] = elements[i + offset]; + } +} + /** \internal copy the packet \a from to \a *to, (un-aligned store with a mask) * There is no generic implementation. We only have implementations for specialized * cases. Generic case should not be called. */ template EIGEN_DEVICE_FUNC inline -typename enable_if::masked_store_available, void>::type +std::enable_if_t::masked_store_available, void> pstoreu(Scalar* to, const Packet& from, typename unpacket_traits::mask_t umask); - template EIGEN_DEVICE_FUNC inline Packet pgather(const Scalar* from, Index /*stride*/) - { return ploadu(from); } +template EIGEN_DEVICE_FUNC inline Packet pgather(const Scalar* from, Index /*stride*/) +{ return ploadu(from); } + +template EIGEN_DEVICE_FUNC inline Packet pgather_partial(const Scalar* from, Index stride, const Index n) +{ + const Index packet_size = unpacket_traits::size; + EIGEN_ALIGN_MAX Scalar elements[packet_size] = { Scalar(0) }; + for (Index i = 0; i < numext::mini(n,packet_size); i++) { + elements[i] = from[i*stride]; + } + return pload(elements); +} + +template EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from, Index /*stride*/) +{ pstore(to, from); } - template EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from, Index /*stride*/) - { pstore(to, from); } +template EIGEN_DEVICE_FUNC inline void pscatter_partial(Scalar* to, const Packet& from, Index stride, const Index n) +{ + const Index packet_size = unpacket_traits::size; + EIGEN_ALIGN_MAX Scalar elements[packet_size]; + pstore(elements, from); + for (Index i = 0; i < numext::mini(n,packet_size); i++) { + to[i*stride] = elements[i]; + } +} /** \internal tries to do cache prefetching of \a addr */ template EIGEN_DEVICE_FUNC inline void prefetch(const Scalar* addr) @@ -767,9 +965,6 @@ Packet pasin(const Packet& a) { EIGEN_USING_STD(asin); return asin(a); } template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pacos(const Packet& a) { EIGEN_USING_STD(acos); return acos(a); } -/** \internal \returns the arc tangent of \a a (coeff-wise) */ -template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -Packet patan(const Packet& a) { EIGEN_USING_STD(atan); return atan(a); } /** \internal \returns the hyperbolic sine of \a a (coeff-wise) */ template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS @@ -779,10 +974,18 @@ Packet psinh(const Packet& a) { EIGEN_USING_STD(sinh); return sinh(a); } template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcosh(const Packet& a) { EIGEN_USING_STD(cosh); return cosh(a); } +/** \internal \returns the arc tangent of \a a (coeff-wise) */ +template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet patan(const Packet& a) { EIGEN_USING_STD(atan); return atan(a); } + /** \internal \returns the hyperbolic tan of \a a (coeff-wise) */ template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet ptanh(const Packet& a) { EIGEN_USING_STD(tanh); return tanh(a); } +/** \internal \returns the arc tangent of \a a (coeff-wise) */ +template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet patanh(const Packet& a) { EIGEN_USING_STD(atanh); return atanh(a); } + /** \internal \returns the exp of \a a (coeff-wise) */ template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp(const Packet& a) { EIGEN_USING_STD(exp); return exp(a); } @@ -807,19 +1010,16 @@ Packet plog10(const Packet& a) { EIGEN_USING_STD(log10); return log10(a); } template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog2(const Packet& a) { typedef typename internal::unpacket_traits::type Scalar; - return pmul(pset1(Scalar(EIGEN_LOG2E)), plog(a)); + return pmul(pset1(Scalar(EIGEN_LOG2E)), plog(a)); } /** \internal \returns the square-root of \a a (coeff-wise) */ template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psqrt(const Packet& a) { return numext::sqrt(a); } -/** \internal \returns the reciprocal square-root of \a a (coeff-wise) */ +/** \internal \returns the cube-root of \a a (coeff-wise) */ template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -Packet prsqrt(const Packet& a) { - typedef typename internal::unpacket_traits::type Scalar; - return pdiv(pset1(Scalar(1)), psqrt(a)); -} +Packet pcbrt(const Packet& a) { return numext::cbrt(a); } /** \internal \returns the rounded value of \a a (coeff-wise) */ template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS @@ -838,6 +1038,24 @@ Packet print(const Packet& a) { using numext::rint; return rint(a); } template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pceil(const Packet& a) { using numext::ceil; return ceil(a); } +template +struct psign_impl { + static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { + return numext::sign(a); + } +}; + +/** \internal \returns the sign of \a a (coeff-wise) */ +template EIGEN_DEVICE_FUNC inline Packet +psign(const Packet& a) { + return psign_impl::run(a); +} + +template<> EIGEN_DEVICE_FUNC inline bool +psign(const bool& a) { + return a; +} + /** \internal \returns the first element of a packet */ template EIGEN_DEVICE_FUNC inline typename unpacket_traits::type @@ -849,7 +1067,7 @@ pfirst(const Packet& a) * For packet-size smaller or equal to 4, this boils down to a noop. */ template -EIGEN_DEVICE_FUNC inline typename conditional<(unpacket_traits::size%8)==0,typename unpacket_traits::half,Packet>::type +EIGEN_DEVICE_FUNC inline std::conditional_t<(unpacket_traits::size%8)==0,typename unpacket_traits::half,Packet> predux_half_dowto4(const Packet& a) { return a; } @@ -881,7 +1099,7 @@ predux(const Packet& a) template EIGEN_DEVICE_FUNC inline typename unpacket_traits::type predux_mul( const Packet& a) { - typedef typename unpacket_traits::type Scalar; + typedef typename unpacket_traits::type Scalar; return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmul))); } @@ -889,14 +1107,14 @@ EIGEN_DEVICE_FUNC inline typename unpacket_traits::type predux_mul( template EIGEN_DEVICE_FUNC inline typename unpacket_traits::type predux_min( const Packet &a) { - typedef typename unpacket_traits::type Scalar; + typedef typename unpacket_traits::type Scalar; return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmin))); } template EIGEN_DEVICE_FUNC inline typename unpacket_traits::type predux_min( const Packet& a) { - typedef typename unpacket_traits::type Scalar; + typedef typename unpacket_traits::type Scalar; return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmin))); } @@ -904,14 +1122,14 @@ EIGEN_DEVICE_FUNC inline typename unpacket_traits::type predux_min( template EIGEN_DEVICE_FUNC inline typename unpacket_traits::type predux_max( const Packet &a) { - typedef typename unpacket_traits::type Scalar; + typedef typename unpacket_traits::type Scalar; return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmax))); } template EIGEN_DEVICE_FUNC inline typename unpacket_traits::type predux_max( const Packet& a) { - typedef typename unpacket_traits::type Scalar; + typedef typename unpacket_traits::type Scalar; return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmax))); } @@ -943,6 +1161,35 @@ template EIGEN_DEVICE_FUNC inline bool predux_any(const Packet& * The following functions might not have to be overwritten for vectorized types ***************************************************************************/ +// FMA instructions. +/** \internal \returns a * b + c (coeff-wise) */ +template +EIGEN_DEVICE_FUNC inline Packet pmadd(const Packet& a, const Packet& b, + const Packet& c) { + return padd(pmul(a, b), c); +} + +/** \internal \returns a * b - c (coeff-wise) */ +template +EIGEN_DEVICE_FUNC inline Packet pmsub(const Packet& a, const Packet& b, + const Packet& c) { + return psub(pmul(a, b), c); +} + +/** \internal \returns -(a * b) + c (coeff-wise) */ +template +EIGEN_DEVICE_FUNC inline Packet pnmadd(const Packet& a, const Packet& b, + const Packet& c) { + return padd(pnegate(pmul(a, b)), c); +} + +/** \internal \returns -(a * b) - c (coeff-wise) */ +template +EIGEN_DEVICE_FUNC inline Packet pnmsub(const Packet& a, const Packet& b, + const Packet& c) { + return psub(pnegate(pmul(a, b)), c); +} + /** \internal copy a packet with constant coefficient \a a (e.g., [a,a,a,a]) to \a *to. \a to must be 16 bytes aligned */ // NOTE: this function must really be templated on the packet type (think about different packet types for the same scalar type) template @@ -951,13 +1198,6 @@ inline void pstore1(typename unpacket_traits::type* to, const typename u pstore(to, pset1(a)); } -/** \internal \returns a * b + c (coeff-wise) */ -template EIGEN_DEVICE_FUNC inline Packet -pmadd(const Packet& a, - const Packet& b, - const Packet& c) -{ return padd(pmul(a, b),c); } - /** \internal \returns a packet version of \a *from. * The pointer \a from must be aligned on a \a Alignment bytes boundary. */ template @@ -969,6 +1209,17 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_trai return ploadu(from); } +/** \internal \returns n elements of a packet version of \a *from. + * The pointer \a from must be aligned on a \a Alignment bytes boundary. */ +template +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_partial(const typename unpacket_traits::type* from, const Index n, const Index offset = 0) +{ + if(Alignment >= unpacket_traits::alignment) + return pload_partial(from, n, offset); + else + return ploadu_partial(from, n, offset); +} + /** \internal copy the packet \a from to \a *to. * The pointer \a from must be aligned on a \a Alignment bytes boundary. */ template @@ -980,6 +1231,17 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& fro pstoreu(to, from); } +/** \internal copy n elements of the packet \a from to \a *to. + * The pointer \a from must be aligned on a \a Alignment bytes boundary. */ +template +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret_partial(Scalar* to, const Packet& from, const Index n, const Index offset = 0) +{ + if(Alignment >= unpacket_traits::alignment) + pstore_partial(to, from, n, offset); + else + pstoreu_partial(to, from, n, offset); +} + /** \internal \returns a packet version of \a *from. * Unlike ploadt, ploadt_ro takes advantage of the read-only memory path on the * hardware if available to speedup the loading of data that won't be modified @@ -1033,6 +1295,101 @@ pblend(const Selector::size>& ifPacket, const Packet& th return ifPacket.select[0] ? thenPacket : elsePacket; } +/** \internal \returns 1 / a (coeff-wise) */ +template +EIGEN_DEVICE_FUNC inline Packet preciprocal(const Packet& a) { + using Scalar = typename unpacket_traits::type; + return pdiv(pset1(Scalar(1)), a); +} + +/** \internal \returns the reciprocal square-root of \a a (coeff-wise) */ +template EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet prsqrt(const Packet& a) { + return preciprocal(psqrt(a)); +} + +template ::value, + bool IsInteger = NumTraits::type>::IsInteger> +struct psignbit_impl; +template +struct psignbit_impl { + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static constexpr Packet run(const Packet& a) { return numext::signbit(a); } +}; +template +struct psignbit_impl { + // generic implementation if not specialized in PacketMath.h + // slower than arithmetic shift + typedef typename unpacket_traits::type Scalar; + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static Packet run(const Packet& a) { + const Packet cst_pos_one = pset1(Scalar(1)); + const Packet cst_neg_one = pset1(Scalar(-1)); + return pcmp_eq(por(pand(a, cst_neg_one), cst_pos_one), cst_neg_one); + } +}; +template +struct psignbit_impl { + // generic implementation for integer packets + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static constexpr Packet run(const Packet& a) { return pcmp_lt(a, pzero(a)); } +}; +/** \internal \returns the sign bit of \a a as a bitmask*/ +template +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE constexpr Packet +psignbit(const Packet& a) { return psignbit_impl::run(a); } + +/** \internal \returns the 2-argument arc tangent of \a y and \a x (coeff-wise) */ +template ::value, int> = 0> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet patan2(const Packet& y, const Packet& x) { + return numext::atan2(y, x); +} + +/** \internal \returns the 2-argument arc tangent of \a y and \a x (coeff-wise) */ +template ::value, int> = 0> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet patan2(const Packet& y, const Packet& x) { + typedef typename internal::unpacket_traits::type Scalar; + + // See https://en.cppreference.com/w/cpp/numeric/math/atan2 + // for how corner cases are supposed to be handled according to the + // IEEE floating-point standard (IEC 60559). + const Packet kSignMask = pset1(-Scalar(0)); + const Packet kZero = pzero(x); + const Packet kOne = pset1(Scalar(1)); + const Packet kPi = pset1(Scalar(EIGEN_PI)); + + const Packet x_has_signbit = psignbit(x); + const Packet y_signmask = pand(y, kSignMask); + const Packet x_signmask = pand(x, kSignMask); + const Packet result_signmask = pxor(y_signmask, x_signmask); + const Packet shift = por(pand(x_has_signbit, kPi), y_signmask); + + const Packet x_and_y_are_same = pcmp_eq(pabs(x), pabs(y)); + const Packet x_and_y_are_zero = pcmp_eq(por(x, y), kZero); + + Packet arg = pdiv(y, x); + arg = pselect(x_and_y_are_same, por(kOne, result_signmask), arg); + arg = pselect(x_and_y_are_zero, result_signmask, arg); + + Packet result = patan(arg); + result = padd(result, shift); + return result; +} + +/** \internal \returns the argument of \a a as a complex number */ +template ::value, int> = 0> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pcarg(const Packet& a) { + return Packet(numext::arg(a)); +} + +/** \internal \returns the argument of \a a as a complex number */ +template ::value, int> = 0> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pcarg(const Packet& a) { + EIGEN_STATIC_ASSERT(NumTraits::type>::IsComplex, THIS METHOD IS FOR COMPLEX TYPES ONLY) + using RealPacket = typename unpacket_traits::as_real; + // a // r i r i ... + RealPacket aflip = pcplxflip(a).v; // i r i r ... + RealPacket result = patan2(aflip, a.v); // atan2 crap atan2 crap ... + return (Packet)pand(result, peven_mask(result)); // atan2 0 atan2 0 ... +} + } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GlobalFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GlobalFunctions.h index 629af94b994..2d8bb806114 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GlobalFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/GlobalFunctions.h @@ -51,6 +51,9 @@ } \ }; +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(real,scalar_real_op,real part,\sa ArrayBase::real) @@ -66,11 +69,9 @@ namespace Eigen EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sinh,scalar_sinh_op,hyperbolic sine,\sa ArrayBase::sinh) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cosh,scalar_cosh_op,hyperbolic cosine,\sa ArrayBase::cosh) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(tanh,scalar_tanh_op,hyperbolic tangent,\sa ArrayBase::tanh) -#if EIGEN_HAS_CXX11_MATH EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(asinh,scalar_asinh_op,inverse hyperbolic sine,\sa ArrayBase::asinh) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(acosh,scalar_acosh_op,inverse hyperbolic cosine,\sa ArrayBase::acosh) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(atanh,scalar_atanh_op,inverse hyperbolic tangent,\sa ArrayBase::atanh) -#endif EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(logistic,scalar_logistic_op,logistic function,\sa ArrayBase::logistic) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(lgamma,scalar_lgamma_op,natural logarithm of the gamma function,\sa ArrayBase::lgamma) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(digamma,scalar_digamma_op,derivative of lgamma,\sa ArrayBase::digamma) @@ -86,7 +87,9 @@ namespace Eigen EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(abs,scalar_abs_op,absolute value,\sa ArrayBase::abs DOXCOMMA MatrixBase::cwiseAbs) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(abs2,scalar_abs2_op,squared absolute value,\sa ArrayBase::abs2 DOXCOMMA MatrixBase::cwiseAbs2) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(arg,scalar_arg_op,complex argument,\sa ArrayBase::arg DOXCOMMA MatrixBase::cwiseArg) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(carg, scalar_carg_op, complex argument, \sa ArrayBase::carg DOXCOMMA MatrixBase::cwiseCArg) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sqrt,scalar_sqrt_op,square root,\sa ArrayBase::sqrt DOXCOMMA MatrixBase::cwiseSqrt) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cbrt,scalar_cbrt_op,cube root,\sa ArrayBase::cbrt DOXCOMMA MatrixBase::cwiseCbrt) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(rsqrt,scalar_rsqrt_op,reciprocal square root,\sa ArrayBase::rsqrt) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(square,scalar_square_op,square (power 2),\sa Eigen::abs2 DOXCOMMA Eigen::pow DOXCOMMA ArrayBase::square) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cube,scalar_cube_op,cube (power 3),\sa Eigen::pow DOXCOMMA ArrayBase::cube) @@ -99,31 +102,31 @@ namespace Eigen EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(isfinite,scalar_isfinite_op,finite value test,\sa Eigen::isinf DOXCOMMA Eigen::isnan DOXCOMMA ArrayBase::isfinite) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sign,scalar_sign_op,sign (or 0),\sa ArrayBase::sign) + template + using GlobalUnaryPowReturnType = std::enable_if_t< + !internal::is_arithmetic::Real>::value && + internal::is_arithmetic::Real>::value, + CwiseUnaryOp, const Derived> >; + /** \returns an expression of the coefficient-wise power of \a x to the given constant \a exponent. - * - * \tparam ScalarExponent is the scalar type of \a exponent. It must be compatible with the scalar type of the given expression (\c Derived::Scalar). - * - * \sa ArrayBase::pow() - * - * \relates ArrayBase - */ + * + * \tparam ScalarExponent is the scalar type of \a exponent. It must be compatible with the scalar type of the given + * expression (\c Derived::Scalar). + * + * \sa ArrayBase::pow() + * + * \relates ArrayBase + */ #ifdef EIGEN_PARSED_BY_DOXYGEN - template - inline const CwiseBinaryOp,Derived,Constant > - pow(const Eigen::ArrayBase& x, const ScalarExponent& exponent); + template + EIGEN_DEVICE_FUNC inline const GlobalUnaryPowReturnType pow( + const Eigen::ArrayBase& x, const ScalarExponent& exponent); #else - template - EIGEN_DEVICE_FUNC inline - EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE( - const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,typename internal::promote_scalar_arg::type,pow)) - pow(const Eigen::ArrayBase& x, const ScalarExponent& exponent) - { - typedef typename internal::promote_scalar_arg::type PromotedExponent; - return EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,PromotedExponent,pow)(x.derived(), - typename internal::plain_constant_type::type(x.derived().rows(), x.derived().cols(), internal::scalar_constant_op(exponent))); + template + EIGEN_DEVICE_FUNC inline const GlobalUnaryPowReturnType pow( + const Eigen::ArrayBase& x, const ScalarExponent& exponent) { + return GlobalUnaryPowReturnType( + x.derived(), internal::scalar_unary_pow_op(exponent)); } #endif @@ -168,10 +171,9 @@ namespace Eigen #else template EIGEN_DEVICE_FUNC inline - EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE( const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(typename internal::promote_scalar_arg::type,Derived,pow)) + EIGEN_SCALAR_BINARY_SUPPORTED(pow,Scalar,typename Derived::Scalar)>::type,Derived,pow) pow(const Scalar& x, const Eigen::ArrayBase& exponents) { typedef typename internal::promote_scalar_arg::type PromotedScalar; @@ -180,6 +182,25 @@ namespace Eigen } #endif + /** \returns an expression of the coefficient-wise atan2(\a x, \a y). \a x and \a y must be of the same type. + * + * This function computes the coefficient-wise atan2(). + * + * \sa ArrayBase::atan2() + * + * \relates ArrayBase + */ + template + inline const std::enable_if_t< + std::is_same::value, + Eigen::CwiseBinaryOp, const LhsDerived, const RhsDerived> + > + atan2(const Eigen::ArrayBase& x, const Eigen::ArrayBase& exponents) { + return Eigen::CwiseBinaryOp, const LhsDerived, const RhsDerived>( + x.derived(), + exponents.derived() + ); + } namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/IO.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/IO.h index e81c3152168..454cbc35895 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/IO.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/IO.h @@ -11,6 +11,9 @@ #ifndef EIGEN_IO_H #define EIGEN_IO_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { enum { DontAlignCols = 1 }; @@ -115,13 +118,13 @@ namespace internal { // NOTE: This helper is kept for backward compatibility with previous code specializing // this internal::significant_decimals_impl structure. In the future we should directly -// call digits10() which has been introduced in July 2016 in 3.3. +// call max_digits10(). template struct significant_decimals_impl { static inline int run() { - return NumTraits::digits10(); + return NumTraits::max_digits10(); } }; @@ -131,7 +134,6 @@ template std::ostream & print_matrix(std::ostream & s, const Derived& _m, const IOFormat& fmt) { using internal::is_same; - using internal::conditional; if(_m.size() == 0) { @@ -141,22 +143,21 @@ std::ostream & print_matrix(std::ostream & s, const Derived& _m, const IOFormat& typename Derived::Nested m = _m; typedef typename Derived::Scalar Scalar; - typedef typename - conditional< + typedef std::conditional_t< is_same::value || is_same::value || is_same::value || is_same::value, int, - typename conditional< + std::conditional_t< is_same >::value || is_same >::value || is_same >::value || is_same >::value, std::complex, const Scalar& - >::type - >::type PrintType; + > + > PrintType; Index width = 0; @@ -253,6 +254,11 @@ std::ostream & operator << return internal::print_matrix(s, m.eval(), EIGEN_DEFAULT_IO_FORMAT); } +template +std::ostream& operator<<(std::ostream& s, const DiagonalBase& m) { + return internal::print_matrix(s, m.derived(), EIGEN_DEFAULT_IO_FORMAT); +} + } // end namespace Eigen #endif // EIGEN_IO_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/IndexedView.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/IndexedView.h index 08476251d32..4898e5190fe 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/IndexedView.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/IndexedView.h @@ -10,6 +10,9 @@ #ifndef EIGEN_INDEXED_VIEW_H #define EIGEN_INDEXED_VIEW_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -21,8 +24,8 @@ struct traits > enum { RowsAtCompileTime = int(array_size::value), ColsAtCompileTime = int(array_size::value), - MaxRowsAtCompileTime = RowsAtCompileTime != Dynamic ? int(RowsAtCompileTime) : Dynamic, - MaxColsAtCompileTime = ColsAtCompileTime != Dynamic ? int(ColsAtCompileTime) : Dynamic, + MaxRowsAtCompileTime = RowsAtCompileTime, + MaxColsAtCompileTime = ColsAtCompileTime, XprTypeIsRowMajor = (int(traits::Flags)&RowMajorBit) != 0, IsRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1 @@ -40,10 +43,10 @@ struct traits > InnerSize = XprTypeIsRowMajor ? ColsAtCompileTime : RowsAtCompileTime, IsBlockAlike = InnerIncr==1 && OuterIncr==1, - IsInnerPannel = HasSameStorageOrderAsXprType && is_same,typename conditional::type>::value, + IsInnerPannel = HasSameStorageOrderAsXprType && is_same,std::conditional_t>::value, - InnerStrideAtCompileTime = InnerIncr<0 || InnerIncr==DynamicIndex || XprInnerStride==Dynamic ? Dynamic : XprInnerStride * InnerIncr, - OuterStrideAtCompileTime = OuterIncr<0 || OuterIncr==DynamicIndex || XprOuterstride==Dynamic ? Dynamic : XprOuterstride * OuterIncr, + InnerStrideAtCompileTime = InnerIncr<0 || InnerIncr==DynamicIndex || XprInnerStride==Dynamic || InnerIncr==UndefinedIncr ? Dynamic : XprInnerStride * InnerIncr, + OuterStrideAtCompileTime = OuterIncr<0 || OuterIncr==DynamicIndex || XprOuterstride==Dynamic || OuterIncr==UndefinedIncr ? Dynamic : XprOuterstride * OuterIncr, ReturnAsScalar = is_same::value && is_same::value, ReturnAsBlock = (!ReturnAsScalar) && IsBlockAlike, @@ -91,12 +94,11 @@ class IndexedViewImpl; * - std::vector * - std::valarray * - std::array - * - Plain C arrays: int[N] * - Eigen::ArrayXi * - decltype(ArrayXi::LinSpaced(...)) * - Any view/expressions of the previous types * - Eigen::ArithmeticSequence - * - Eigen::internal::AllRange (helper for Eigen::all) + * - Eigen::internal::AllRange (helper for Eigen::placeholders::all) * - Eigen::internal::SingleRange (helper for single index) * - etc. * @@ -114,7 +116,7 @@ class IndexedView : public IndexedViewImpl::non_const_type MatrixTypeNested; - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; template IndexedView(XprType& xpr, const T0& rowIndices, const T1& colIndices) @@ -122,17 +124,17 @@ class IndexedView : public IndexedViewImpl::type& + const internal::remove_all_t& nestedExpression() const { return m_xpr; } /** \returns the nested expression */ - typename internal::remove_reference::type& + std::remove_reference_t& nestedExpression() { return m_xpr; } /** \returns a const reference to the object storing/generating the row indices */ @@ -189,12 +191,16 @@ struct unary_evaluator, IndexBased> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { + eigen_assert(m_xpr.rowIndices()[row] >= 0 && m_xpr.rowIndices()[row] < m_xpr.nestedExpression().rows() + && m_xpr.colIndices()[col] >= 0 && m_xpr.colIndices()[col] < m_xpr.nestedExpression().cols()); return m_argImpl.coeff(m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { + eigen_assert(m_xpr.rowIndices()[row] >= 0 && m_xpr.rowIndices()[row] < m_xpr.nestedExpression().rows() + && m_xpr.colIndices()[col] >= 0 && m_xpr.colIndices()[col] < m_xpr.nestedExpression().cols()); return m_argImpl.coeffRef(m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); } @@ -204,6 +210,8 @@ struct unary_evaluator, IndexBased> EIGEN_STATIC_ASSERT_LVALUE(XprType) Index row = XprType::RowsAtCompileTime == 1 ? 0 : index; Index col = XprType::RowsAtCompileTime == 1 ? index : 0; + eigen_assert(m_xpr.rowIndices()[row] >= 0 && m_xpr.rowIndices()[row] < m_xpr.nestedExpression().rows() + && m_xpr.colIndices()[col] >= 0 && m_xpr.colIndices()[col] < m_xpr.nestedExpression().cols()); return m_argImpl.coeffRef( m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); } @@ -212,6 +220,8 @@ struct unary_evaluator, IndexBased> { Index row = XprType::RowsAtCompileTime == 1 ? 0 : index; Index col = XprType::RowsAtCompileTime == 1 ? index : 0; + eigen_assert(m_xpr.rowIndices()[row] >= 0 && m_xpr.rowIndices()[row] < m_xpr.nestedExpression().rows() + && m_xpr.colIndices()[col] >= 0 && m_xpr.colIndices()[col] < m_xpr.nestedExpression().cols()); return m_argImpl.coeffRef( m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); } @@ -220,6 +230,8 @@ struct unary_evaluator, IndexBased> { Index row = XprType::RowsAtCompileTime == 1 ? 0 : index; Index col = XprType::RowsAtCompileTime == 1 ? index : 0; + eigen_assert(m_xpr.rowIndices()[row] >= 0 && m_xpr.rowIndices()[row] < m_xpr.nestedExpression().rows() + && m_xpr.colIndices()[col] >= 0 && m_xpr.colIndices()[col] < m_xpr.nestedExpression().cols()); return m_argImpl.coeff( m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/InternalHeaderCheck.h new file mode 100644 index 00000000000..1cea572dcf9 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_CORE_MODULE_H +#error "Please include Eigen/Core instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Inverse.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Inverse.h index c514438c45e..54f257e63c7 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Inverse.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Inverse.h @@ -10,6 +10,9 @@ #ifndef EIGEN_INVERSE_H #define EIGEN_INVERSE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template class InverseImpl; @@ -46,9 +49,9 @@ class Inverse : public InverseImpl::S typedef typename XprType::StorageIndex StorageIndex; typedef typename XprType::Scalar Scalar; typedef typename internal::ref_selector::type XprTypeNested; - typedef typename internal::remove_all::type XprTypeNestedCleaned; + typedef internal::remove_all_t XprTypeNestedCleaned; typedef typename internal::ref_selector::type Nested; - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; explicit EIGEN_DEVICE_FUNC Inverse(const XprType &xpr) : m_xpr(xpr) @@ -102,7 +105,7 @@ struct unary_evaluator > unary_evaluator(const InverseType& inv_xpr) : m_result(inv_xpr.rows(), inv_xpr.cols()) { - ::new (static_cast(this)) Base(m_result); + internal::construct_at(this, m_result); internal::call_assignment_no_alias(m_result, inv_xpr); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Map.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Map.h index 218cc157f38..5544b2d0429 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Map.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Map.h @@ -11,6 +11,9 @@ #ifndef EIGEN_MAP_H #define EIGEN_MAP_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -129,7 +132,6 @@ template class Ma explicit inline Map(PointerArgType dataPtr, const StrideType& stride = StrideType()) : Base(cast_to_pointer_type(dataPtr)), m_stride(stride) { - PlainObjectType::Base::_check_template_params(); } /** Constructor in the dynamic-size vector case. @@ -142,7 +144,6 @@ template class Ma inline Map(PointerArgType dataPtr, Index size, const StrideType& stride = StrideType()) : Base(cast_to_pointer_type(dataPtr), size), m_stride(stride) { - PlainObjectType::Base::_check_template_params(); } /** Constructor in the dynamic-size matrix case. @@ -156,7 +157,6 @@ template class Ma inline Map(PointerArgType dataPtr, Index rows, Index cols, const StrideType& stride = StrideType()) : Base(cast_to_pointer_type(dataPtr), rows, cols), m_stride(stride) { - PlainObjectType::Base::_check_template_params(); } EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Map) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MapBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MapBase.h index d856447f03e..622a780b23f 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MapBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MapBase.h @@ -15,6 +15,9 @@ EIGEN_STATIC_ASSERT((int(internal::evaluator::Flags) & LinearAccessBit) || Derived::IsVectorAtCompileTime, \ YOU_ARE_TRYING_TO_USE_AN_INDEX_BASED_ACCESSOR_ON_AN_EXPRESSION_THAT_DOES_NOT_SUPPORT_THAT) +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \ingroup Core_Module @@ -51,11 +54,11 @@ template class MapBase typedef typename internal::traits::Scalar Scalar; typedef typename internal::packet_traits::type PacketScalar; typedef typename NumTraits::Real RealScalar; - typedef typename internal::conditional< - bool(internal::is_lvalue::value), - Scalar *, - const Scalar *>::type - PointerType; + typedef std::conditional_t< + bool(internal::is_lvalue::value), + Scalar *, + const Scalar *> + PointerType; using Base::derived; // using Base::RowsAtCompileTime; @@ -189,21 +192,32 @@ template class MapBase template EIGEN_DEVICE_FUNC - void checkSanity(typename internal::enable_if<(internal::traits::Alignment>0),void*>::type = 0) const + void checkSanity(std::enable_if_t<(internal::traits::Alignment>0),void*> = 0) const { +// Temporary macro to allow scalars to not be properly aligned. This is while we sort out failures +// in TensorFlow Lite that are currently relying on this UB. +#ifndef EIGEN_ALLOW_UNALIGNED_SCALARS + // Pointer must be aligned to the Scalar type, otherwise we get UB. + eigen_assert((std::uintptr_t(m_data) % alignof(Scalar) == 0) && "data is not scalar-aligned"); +#endif #if EIGEN_MAX_ALIGN_BYTES>0 // innerStride() is not set yet when this function is called, so we optimistically assume the lowest plausible value: const Index minInnerStride = InnerStrideAtCompileTime == Dynamic ? 1 : Index(InnerStrideAtCompileTime); EIGEN_ONLY_USED_FOR_DEBUG(minInnerStride); - eigen_assert(( ((internal::UIntPtr(m_data) % internal::traits::Alignment) == 0) + eigen_assert(( ((std::uintptr_t(m_data) % internal::traits::Alignment) == 0) || (cols() * rows() * minInnerStride * sizeof(Scalar)) < internal::traits::Alignment ) && "data is not aligned"); #endif } template EIGEN_DEVICE_FUNC - void checkSanity(typename internal::enable_if::Alignment==0,void*>::type = 0) const - {} + void checkSanity(std::enable_if_t::Alignment==0,void*> = 0) const + { +#ifndef EIGEN_ALLOW_UNALIGNED_SCALARS + // Pointer must be aligned to the Scalar type, otherwise we get UB. + eigen_assert((std::uintptr_t(m_data) % alignof(Scalar) == 0) && "data is not scalar-aligned"); +#endif + } PointerType m_data; const internal::variable_if_dynamic m_rows; @@ -245,11 +259,11 @@ template class MapBase using Base::rowStride; using Base::colStride; - typedef typename internal::conditional< + typedef std::conditional_t< internal::is_lvalue::value, Scalar, const Scalar - >::type ScalarWithConstIfNotLvalue; + > ScalarWithConstIfNotLvalue; EIGEN_DEVICE_FUNC inline const Scalar* data() const { return this->m_data; } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MathFunctions.h index 61b78f4f20a..6f2fd6d0590 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MathFunctions.h @@ -17,16 +17,10 @@ #define EIGEN_LOG2E 1.442695040888963407359924681001892137426645954152985934135449406931109219L #define EIGEN_LN2 0.693147180559945309417232121458176568075500134360255254120680009493393621L -namespace Eigen { +// IWYU pragma: private +#include "./InternalHeaderCheck.h" -// On WINCE, std::abs is defined for int only, so let's defined our own overloads: -// This issue has been confirmed with MSVC 2008 only, but the issue might exist for more recent versions too. -#if EIGEN_OS_WINCE && EIGEN_COMP_MSVC && EIGEN_COMP_MSVC<=1500 -long abs(long x) { return (labs(x)); } -double abs(double x) { return (fabs(x)); } -float abs(float x) { return (fabsf(x)); } -long double abs(long double x) { return (fabsl(x)); } -#endif +namespace Eigen { namespace internal { @@ -437,13 +431,20 @@ struct cast_impl } }; +template +struct cast_impl { + EIGEN_DEVICE_FUNC + static inline bool run(const OldType& x) { return x != OldType(0); } +}; + + // Casting from S -> Complex leads to an implicit conversion from S to T, // generating warnings on clang. Here we explicitly cast the real component. template struct cast_impl::IsComplex && NumTraits::IsComplex - >::type> + >> { EIGEN_DEVICE_FUNC static inline NewType run(const OldType& x) @@ -462,119 +463,13 @@ inline NewType cast(const OldType& x) return cast_impl::run(x); } -/**************************************************************************** -* Implementation of round * -****************************************************************************/ - -template -struct round_impl -{ - EIGEN_DEVICE_FUNC - static inline Scalar run(const Scalar& x) - { - EIGEN_STATIC_ASSERT((!NumTraits::IsComplex), NUMERIC_TYPE_MUST_BE_REAL) -#if EIGEN_HAS_CXX11_MATH - EIGEN_USING_STD(round); -#endif - return Scalar(round(x)); - } -}; - -#if !EIGEN_HAS_CXX11_MATH -#if EIGEN_HAS_C99_MATH -// Use ::roundf for float. -template<> -struct round_impl { - EIGEN_DEVICE_FUNC - static inline float run(const float& x) - { - return ::roundf(x); - } -}; -#else -template -struct round_using_floor_ceil_impl -{ - EIGEN_DEVICE_FUNC - static inline Scalar run(const Scalar& x) - { - EIGEN_STATIC_ASSERT((!NumTraits::IsComplex), NUMERIC_TYPE_MUST_BE_REAL) - // Without C99 round/roundf, resort to floor/ceil. - EIGEN_USING_STD(floor); - EIGEN_USING_STD(ceil); - // If not enough precision to resolve a decimal at all, return the input. - // Otherwise, adding 0.5 can trigger an increment by 1. - const Scalar limit = Scalar(1ull << (NumTraits::digits() - 1)); - if (x >= limit || x <= -limit) { - return x; - } - return (x > Scalar(0)) ? Scalar(floor(x + Scalar(0.5))) : Scalar(ceil(x - Scalar(0.5))); - } -}; - -template<> -struct round_impl : round_using_floor_ceil_impl {}; - -template<> -struct round_impl : round_using_floor_ceil_impl {}; -#endif // EIGEN_HAS_C99_MATH -#endif // !EIGEN_HAS_CXX11_MATH - -template -struct round_retval -{ - typedef Scalar type; -}; - -/**************************************************************************** -* Implementation of rint * -****************************************************************************/ - -template -struct rint_impl { - EIGEN_DEVICE_FUNC - static inline Scalar run(const Scalar& x) - { - EIGEN_STATIC_ASSERT((!NumTraits::IsComplex), NUMERIC_TYPE_MUST_BE_REAL) -#if EIGEN_HAS_CXX11_MATH - EIGEN_USING_STD(rint); -#endif - return rint(x); - } -}; - -#if !EIGEN_HAS_CXX11_MATH -template<> -struct rint_impl { - EIGEN_DEVICE_FUNC - static inline double run(const double& x) - { - return ::rint(x); - } -}; -template<> -struct rint_impl { - EIGEN_DEVICE_FUNC - static inline float run(const float& x) - { - return ::rintf(x); - } -}; -#endif - -template -struct rint_retval -{ - typedef Scalar type; -}; - /**************************************************************************** * Implementation of arg * ****************************************************************************/ // Visual Studio 2017 has a bug where arg(float) returns 0 for negative inputs. // This seems to be fixed in VS 2019. -#if EIGEN_HAS_CXX11_MATH && (!EIGEN_COMP_MSVC || EIGEN_COMP_MSVC >= 1920) +#if (!EIGEN_COMP_MSVC || EIGEN_COMP_MSVC >= 1920) // std::arg is only defined for types of std::complex, or integer types or float/double/long double template::IsComplex || is_integral::value @@ -588,12 +483,8 @@ struct arg_default_impl { EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) { - #if defined(EIGEN_HIP_DEVICE_COMPILE) - // HIP does not seem to have a native device side implementation for the math routine "arg" + // There is no official ::arg on device in CUDA/HIP, so we always need to use std::arg. using std::arg; - #else - EIGEN_USING_STD(arg); - #endif return static_cast(arg(x)); } }; @@ -675,11 +566,7 @@ struct expm1_impl { EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) { EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) - #if EIGEN_HAS_CXX11_MATH - using std::expm1; - #else - using std_fallback::expm1; - #endif + EIGEN_USING_STD(expm1); return expm1(x); } }; @@ -736,14 +623,11 @@ namespace std_fallback { template struct log1p_impl { + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) + EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) - #if EIGEN_HAS_CXX11_MATH - using std::log1p; - #else - using std_fallback::log1p; - #endif + EIGEN_USING_STD(log1p); return log1p(x); } }; @@ -751,9 +635,10 @@ struct log1p_impl { // Specialization for complex types that are not supported by std::log1p. template struct log1p_impl > { + EIGEN_STATIC_ASSERT_NON_INTEGER(RealScalar) + EIGEN_DEVICE_FUNC static inline std::complex run( const std::complex& x) { - EIGEN_STATIC_ASSERT_NON_INTEGER(RealScalar) return std_fallback::log1p(x); } }; @@ -893,7 +778,7 @@ struct random_default_impl // ScalarX is the widest of ScalarU and unsigned int. // We'll deal only with ScalarX and unsigned int below thus avoiding signed // types and arithmetic and signed overflows (which are undefined behavior). - typedef typename conditional<(ScalarU(-1) > unsigned(-1)), ScalarU, unsigned>::type ScalarX; + typedef std::conditional_t<(ScalarU(-1) > unsigned(-1)), ScalarU, unsigned> ScalarX; // The following difference doesn't overflow, provided our integer types are two's // complement and have the same number of padding bits in signed and unsigned variants. // This is the case in most modern implementations of C++. @@ -918,8 +803,8 @@ struct random_default_impl #else enum { rand_bits = meta_floor_log2<(unsigned int)(RAND_MAX)+1>::value, scalar_bits = sizeof(Scalar) * CHAR_BIT, - shift = EIGEN_PLAIN_ENUM_MAX(0, int(rand_bits) - int(scalar_bits)), - offset = NumTraits::IsSigned ? (1 << (EIGEN_PLAIN_ENUM_MIN(rand_bits,scalar_bits)-1)) : 0 + shift = plain_enum_max(0, int(rand_bits) - int(scalar_bits)), + offset = NumTraits::IsSigned ? (1 << (plain_enum_min(rand_bits, scalar_bits)-1)) : 0 }; return Scalar((std::rand() >> shift) - offset); #endif @@ -955,120 +840,130 @@ inline EIGEN_MATHFUNC_RETVAL(random, Scalar) random() // Implementation of is* functions -// std::is* do not work with fast-math and gcc, std::is* are available on MSVC 2013 and newer, as well as in clang. -#if (EIGEN_HAS_CXX11_MATH && !(EIGEN_COMP_GNUC_STRICT && __FINITE_MATH_ONLY__)) || (EIGEN_COMP_MSVC>=1800) || (EIGEN_COMP_CLANG) -#define EIGEN_USE_STD_FPCLASSIFY 1 -#else -#define EIGEN_USE_STD_FPCLASSIFY 0 -#endif - -template -EIGEN_DEVICE_FUNC -typename internal::enable_if::value,bool>::type -isnan_impl(const T&) { return false; } - -template -EIGEN_DEVICE_FUNC -typename internal::enable_if::value,bool>::type -isinf_impl(const T&) { return false; } - -template -EIGEN_DEVICE_FUNC -typename internal::enable_if::value,bool>::type -isfinite_impl(const T&) { return true; } - -template -EIGEN_DEVICE_FUNC -typename internal::enable_if<(!internal::is_integral::value)&&(!NumTraits::IsComplex),bool>::type -isfinite_impl(const T& x) -{ - #if defined(EIGEN_GPU_COMPILE_PHASE) - return (::isfinite)(x); - #elif EIGEN_USE_STD_FPCLASSIFY - using std::isfinite; - return isfinite EIGEN_NOT_A_MACRO (x); - #else - return x<=NumTraits::highest() && x>=NumTraits::lowest(); - #endif +template +EIGEN_DEVICE_FUNC std::enable_if_t::has_infinity || + std::numeric_limits::has_quiet_NaN || + std::numeric_limits::has_signaling_NaN), + bool> +isfinite_impl(const T&) { + return true; } -template -EIGEN_DEVICE_FUNC -typename internal::enable_if<(!internal::is_integral::value)&&(!NumTraits::IsComplex),bool>::type -isinf_impl(const T& x) -{ - #if defined(EIGEN_GPU_COMPILE_PHASE) - return (::isinf)(x); - #elif EIGEN_USE_STD_FPCLASSIFY - using std::isinf; - return isinf EIGEN_NOT_A_MACRO (x); - #else - return x>NumTraits::highest() || x::lowest(); - #endif +template +EIGEN_DEVICE_FUNC std::enable_if_t<(std::numeric_limits::has_infinity || std::numeric_limits::has_quiet_NaN || + std::numeric_limits::has_signaling_NaN) && + (!NumTraits::IsComplex), + bool> +isfinite_impl(const T& x) { + EIGEN_USING_STD(isfinite); + return isfinite EIGEN_NOT_A_MACRO (x); } -template -EIGEN_DEVICE_FUNC -typename internal::enable_if<(!internal::is_integral::value)&&(!NumTraits::IsComplex),bool>::type -isnan_impl(const T& x) -{ - #if defined(EIGEN_GPU_COMPILE_PHASE) - return (::isnan)(x); - #elif EIGEN_USE_STD_FPCLASSIFY - using std::isnan; - return isnan EIGEN_NOT_A_MACRO (x); - #else - return x != x; - #endif +template +EIGEN_DEVICE_FUNC std::enable_if_t::has_infinity, bool> +isinf_impl(const T&) { + return false; } -#if (!EIGEN_USE_STD_FPCLASSIFY) - -#if EIGEN_COMP_MSVC +template +EIGEN_DEVICE_FUNC std::enable_if_t< + (std::numeric_limits::has_infinity && !NumTraits::IsComplex), bool> +isinf_impl(const T& x) { + EIGEN_USING_STD(isinf); + return isinf EIGEN_NOT_A_MACRO (x); +} -template EIGEN_DEVICE_FUNC bool isinf_msvc_helper(T x) -{ - return _fpclass(x)==_FPCLASS_NINF || _fpclass(x)==_FPCLASS_PINF; +template +EIGEN_DEVICE_FUNC std::enable_if_t::has_quiet_NaN || + std::numeric_limits::has_signaling_NaN), + bool> +isnan_impl(const T&) { + return false; } -//MSVC defines a _isnan builtin function, but for double only -EIGEN_DEVICE_FUNC inline bool isnan_impl(const long double& x) { return _isnan(x)!=0; } -EIGEN_DEVICE_FUNC inline bool isnan_impl(const double& x) { return _isnan(x)!=0; } -EIGEN_DEVICE_FUNC inline bool isnan_impl(const float& x) { return _isnan(x)!=0; } +template +EIGEN_DEVICE_FUNC + std::enable_if_t<(std::numeric_limits::has_quiet_NaN || + std::numeric_limits::has_signaling_NaN) && + (!NumTraits::IsComplex), + bool> + isnan_impl(const T& x) { + EIGEN_USING_STD(isnan); + return isnan EIGEN_NOT_A_MACRO (x); +} -EIGEN_DEVICE_FUNC inline bool isinf_impl(const long double& x) { return isinf_msvc_helper(x); } -EIGEN_DEVICE_FUNC inline bool isinf_impl(const double& x) { return isinf_msvc_helper(x); } -EIGEN_DEVICE_FUNC inline bool isinf_impl(const float& x) { return isinf_msvc_helper(x); } +// The following overload are defined at the end of this file +template +EIGEN_DEVICE_FUNC bool isfinite_impl(const std::complex& x); +template +EIGEN_DEVICE_FUNC bool isnan_impl(const std::complex& x); +template +EIGEN_DEVICE_FUNC bool isinf_impl(const std::complex& x); +template +T generic_fast_tanh_float(const T& a_x); -#elif (defined __FINITE_MATH_ONLY__ && __FINITE_MATH_ONLY__ && EIGEN_COMP_GNUC) +/**************************************************************************** + * Implementation of sign * + ****************************************************************************/ +template ::IsComplex != 0), + bool IsInteger = (NumTraits::IsInteger != 0)> +struct sign_impl { + EIGEN_DEVICE_FUNC + static inline Scalar run(const Scalar& a) { + return Scalar((a > Scalar(0)) - (a < Scalar(0))); + } +}; -#if EIGEN_GNUC_AT_LEAST(5,0) - #define EIGEN_TMP_NOOPT_ATTRIB EIGEN_DEVICE_FUNC inline __attribute__((optimize("no-finite-math-only"))) -#else - // NOTE the inline qualifier and noinline attribute are both needed: the former is to avoid linking issue (duplicate symbol), - // while the second prevent too aggressive optimizations in fast-math mode: - #define EIGEN_TMP_NOOPT_ATTRIB EIGEN_DEVICE_FUNC inline __attribute__((noinline,optimize("no-finite-math-only"))) -#endif +template +struct sign_impl { + EIGEN_DEVICE_FUNC + static inline Scalar run(const Scalar& a) { + return (isnan_impl)(a) ? a + : Scalar((a > Scalar(0)) - (a < Scalar(0))); + } +}; -template<> EIGEN_TMP_NOOPT_ATTRIB bool isnan_impl(const long double& x) { return __builtin_isnan(x); } -template<> EIGEN_TMP_NOOPT_ATTRIB bool isnan_impl(const double& x) { return __builtin_isnan(x); } -template<> EIGEN_TMP_NOOPT_ATTRIB bool isnan_impl(const float& x) { return __builtin_isnan(x); } -template<> EIGEN_TMP_NOOPT_ATTRIB bool isinf_impl(const double& x) { return __builtin_isinf(x); } -template<> EIGEN_TMP_NOOPT_ATTRIB bool isinf_impl(const float& x) { return __builtin_isinf(x); } -template<> EIGEN_TMP_NOOPT_ATTRIB bool isinf_impl(const long double& x) { return __builtin_isinf(x); } +template +struct sign_impl { + EIGEN_DEVICE_FUNC + static inline Scalar run(const Scalar& a) { + using real_type = typename NumTraits::Real; + EIGEN_USING_STD(abs); + real_type aa = abs(a); + if (aa == real_type(0)) return Scalar(0); + aa = real_type(1) / aa; + return Scalar(a.real() * aa, a.imag() * aa); + } +}; -#undef EIGEN_TMP_NOOPT_ATTRIB +// The sign function for bool is the identity. +template <> +struct sign_impl { + EIGEN_DEVICE_FUNC + static inline bool run(const bool& a) { return a; } +}; -#endif +template +struct sign_retval { + typedef Scalar type; +}; -#endif -// The following overload are defined at the end of this file -template EIGEN_DEVICE_FUNC bool isfinite_impl(const std::complex& x); -template EIGEN_DEVICE_FUNC bool isnan_impl(const std::complex& x); -template EIGEN_DEVICE_FUNC bool isinf_impl(const std::complex& x); +template ::type>::IsInteger> +struct nearest_integer_impl { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_floor(const Scalar& x) { EIGEN_USING_STD(floor) return floor(x); } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_ceil(const Scalar& x) { EIGEN_USING_STD(ceil) return ceil(x); } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_rint(const Scalar& x) { EIGEN_USING_STD(rint) return rint(x); } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_round(const Scalar& x) { EIGEN_USING_STD(round) return round(x); } +}; +template +struct nearest_integer_impl { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_floor(const Scalar& x) { return x; } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_ceil(const Scalar& x) { return x; } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_rint(const Scalar& x) { return x; } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_round(const Scalar& x) { return x; } +}; -template T generic_fast_tanh_float(const T& a_x); } // end namespace internal /**************************************************************************** @@ -1112,6 +1007,8 @@ EIGEN_ALWAYS_INLINE double mini(const double& x, const double& y) { return fmin(x, y); } + +#ifndef EIGEN_GPU_COMPILE_PHASE template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE long double mini(const long double& x, const long double& y) @@ -1123,6 +1020,7 @@ EIGEN_ALWAYS_INLINE long double mini(const long double& x, const long double& y) return fminl(x, y); #endif } +#endif template EIGEN_DEVICE_FUNC @@ -1142,6 +1040,7 @@ EIGEN_ALWAYS_INLINE double maxi(const double& x, const double& y) { return fmax(x, y); } +#ifndef EIGEN_GPU_COMPILE_PHASE template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE long double maxi(const long double& x, const long double& y) @@ -1154,6 +1053,7 @@ EIGEN_ALWAYS_INLINE long double maxi(const long double& x, const long double& y) #endif } #endif +#endif #if defined(SYCL_DEVICE_ONLY) @@ -1234,7 +1134,7 @@ inline EIGEN_MATHFUNC_RETVAL(real, Scalar) real(const Scalar& x) template EIGEN_DEVICE_FUNC -inline typename internal::add_const_on_value_type< EIGEN_MATHFUNC_RETVAL(real_ref, Scalar) >::type real_ref(const Scalar& x) +inline internal::add_const_on_value_type_t< EIGEN_MATHFUNC_RETVAL(real_ref, Scalar) > real_ref(const Scalar& x) { return internal::real_ref_impl::run(x); } @@ -1262,7 +1162,7 @@ inline EIGEN_MATHFUNC_RETVAL(arg, Scalar) arg(const Scalar& x) template EIGEN_DEVICE_FUNC -inline typename internal::add_const_on_value_type< EIGEN_MATHFUNC_RETVAL(imag_ref, Scalar) >::type imag_ref(const Scalar& x) +inline internal::add_const_on_value_type_t< EIGEN_MATHFUNC_RETVAL(imag_ref, Scalar) > imag_ref(const Scalar& x) { return internal::imag_ref_impl::run(x); } @@ -1281,6 +1181,13 @@ inline EIGEN_MATHFUNC_RETVAL(conj, Scalar) conj(const Scalar& x) return EIGEN_MATHFUNC_IMPL(conj, Scalar)::run(x); } +template +EIGEN_DEVICE_FUNC +inline EIGEN_MATHFUNC_RETVAL(sign, Scalar) sign(const Scalar& x) +{ + return EIGEN_MATHFUNC_IMPL(sign, Scalar)::run(x); +} + template EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(abs2, Scalar) abs2(const Scalar& x) @@ -1310,8 +1217,8 @@ EIGEN_ALWAYS_INLINE double absdiff(const double& x, const double& y) return fabs(x - y); } -#if !defined(EIGEN_GPUCC) // HIP and CUDA do not support long double. +#ifndef EIGEN_GPU_COMPILE_PHASE template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE long double absdiff(const long double& x, const long double& y) { @@ -1378,29 +1285,28 @@ SYCL_SPECIALIZE_FLOATING_TYPES_UNARY_FUNC_RET_TYPE(isfinite, isfinite, bool) #endif template -EIGEN_DEVICE_FUNC -inline EIGEN_MATHFUNC_RETVAL(rint, Scalar) rint(const Scalar& x) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +Scalar rint(const Scalar& x) { - return EIGEN_MATHFUNC_IMPL(rint, Scalar)::run(x); + return internal::nearest_integer_impl::run_rint(x); } template -EIGEN_DEVICE_FUNC -inline EIGEN_MATHFUNC_RETVAL(round, Scalar) round(const Scalar& x) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +Scalar round(const Scalar& x) { - return EIGEN_MATHFUNC_IMPL(round, Scalar)::run(x); + return internal::nearest_integer_impl::run_round(x); } #if defined(SYCL_DEVICE_ONLY) SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(round, round) #endif -template -EIGEN_DEVICE_FUNC -T (floor)(const T& x) +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +Scalar (floor)(const Scalar& x) { - EIGEN_USING_STD(floor) - return floor(x); + return internal::nearest_integer_impl::run_floor(x); } #if defined(SYCL_DEVICE_ONLY) @@ -1415,12 +1321,11 @@ template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double floor(const double &x) { return ::floor(x); } #endif -template -EIGEN_DEVICE_FUNC -T (ceil)(const T& x) +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +Scalar (ceil)(const Scalar& x) { - EIGEN_USING_STD(ceil); - return ceil(x); + return internal::nearest_integer_impl::run_ceil(x); } #if defined(SYCL_DEVICE_ONLY) @@ -1436,6 +1341,19 @@ double ceil(const double &x) { return ::ceil(x); } #endif +// Integer division with rounding up. +// T is assumed to be an integer type with a>=0, and b>0 +template +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE EIGEN_CONSTEXPR +T div_ceil(T a, T b) +{ + EIGEN_STATIC_ASSERT((NumTraits::IsInteger), THIS FUNCTION IS FOR INTEGER TYPES) + eigen_assert(a >= 0); + eigen_assert(b > 0); + // Note: This form is used because it cannot overflow. + return a == 0 ? 0 : (a - 1) / b + 1; +} + /** Log base 2 for 32 bits positive integers. * Conveniently returns 0 for x==0. */ inline int log2(int x) @@ -1476,6 +1394,14 @@ bool sqrt(const bool &x) { return x; } SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(sqrt, sqrt) #endif +/** \returns the cube root of \a x. **/ +template +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +T cbrt(const T &x) { + EIGEN_USING_STD(cbrt); + return static_cast(cbrt(x)); +} + /** \returns the reciprocal square root of \a x. **/ template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE @@ -1505,7 +1431,7 @@ double log(const double &x) { return ::log(x); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE -typename internal::enable_if::IsSigned || NumTraits::IsComplex,typename NumTraits::Real>::type +std::enable_if_t::IsSigned || NumTraits::IsComplex,typename NumTraits::Real> abs(const T &x) { EIGEN_USING_STD(abs); return abs(x); @@ -1513,7 +1439,7 @@ abs(const T &x) { template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE -typename internal::enable_if::IsSigned || NumTraits::IsComplex),typename NumTraits::Real>::type +std::enable_if_t::IsSigned || NumTraits::IsComplex),typename NumTraits::Real> abs(const T &x) { return x; } @@ -1541,6 +1467,37 @@ double abs(const std::complex& x) { } #endif +template ::IsInteger, bool IsSigned = NumTraits::IsSigned> +struct signbit_impl; +template +struct signbit_impl { + static constexpr size_t Size = sizeof(Scalar); + static constexpr size_t Shift = (CHAR_BIT * Size) - 1; + using intSize_t = typename get_integer_by_size::signed_type; + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static Scalar run(const Scalar& x) { + intSize_t a = bit_cast(x); + a = a >> Shift; + Scalar result = bit_cast(a); + return result; + } +}; +template +struct signbit_impl { + static constexpr size_t Size = sizeof(Scalar); + static constexpr size_t Shift = (CHAR_BIT * Size) - 1; + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static constexpr Scalar run(const Scalar& x) { return x >> Shift; } +}; +template +struct signbit_impl { + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static constexpr Scalar run(const Scalar& ) { + return Scalar(0); + } +}; +template +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static constexpr Scalar signbit(const Scalar& x) { + return signbit_impl::run(x); +} + template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T exp(const T &x) { @@ -1659,14 +1616,12 @@ T acos(const T &x) { return acos(x); } -#if EIGEN_HAS_CXX11_MATH template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T acosh(const T &x) { EIGEN_USING_STD(acosh); return static_cast(acosh(x)); } -#endif #if defined(SYCL_DEVICE_ONLY) SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(acos, acos) @@ -1688,14 +1643,12 @@ T asin(const T &x) { return asin(x); } -#if EIGEN_HAS_CXX11_MATH template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T asinh(const T &x) { EIGEN_USING_STD(asinh); return static_cast(asinh(x)); } -#endif #if defined(SYCL_DEVICE_ONLY) SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(asin, asin) @@ -1717,14 +1670,18 @@ T atan(const T &x) { return static_cast(atan(x)); } -#if EIGEN_HAS_CXX11_MATH +template ::IsComplex, int> = 0> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T atan2(const T& y, const T& x) { + EIGEN_USING_STD(atan2); + return static_cast(atan2(y, x)); +} + template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T atanh(const T &x) { EIGEN_USING_STD(atanh); return static_cast(atanh(x)); } -#endif #if defined(SYCL_DEVICE_ONLY) SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(atan, atan) @@ -2006,9 +1963,10 @@ namespace internal { // Specialization for complex types that are not supported by std::expm1. template struct expm1_impl > { + EIGEN_STATIC_ASSERT_NON_INTEGER(RealScalar) + EIGEN_DEVICE_FUNC static inline std::complex run( const std::complex& x) { - EIGEN_STATIC_ASSERT_NON_INTEGER(RealScalar) RealScalar xr = x.real(); RealScalar xi = x.imag(); // expm1(z) = exp(z) - 1 diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MathFunctionsImpl.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MathFunctionsImpl.h index 4eaaaa78449..5b29fee7593 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MathFunctionsImpl.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MathFunctionsImpl.h @@ -11,17 +11,155 @@ #ifndef EIGEN_MATHFUNCTIONSIMPL_H #define EIGEN_MATHFUNCTIONSIMPL_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { +/** \internal Fast reciprocal using Newton-Raphson's method. + + Preconditions: + 1. The starting guess provided in approx_a_recip must have at least half + the leading mantissa bits in the correct result, such that a single + Newton-Raphson step is sufficient to get within 1-2 ulps of the currect + result. + 2. If a is zero, approx_a_recip must be infinite with the same sign as a. + 3. If a is infinite, approx_a_recip must be zero with the same sign as a. + + If the preconditions are satisfied, which they are for for the _*_rcp_ps + instructions on x86, the result has a maximum relative error of 2 ulps, + and correctly handles reciprocals of zero, infinity, and NaN. +*/ +template +struct generic_reciprocal_newton_step { + static_assert(Steps > 0, "Steps must be at least 1."); + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet + run(const Packet& a, const Packet& approx_a_recip) { + using Scalar = typename unpacket_traits::type; + const Packet two = pset1(Scalar(2)); + // Refine the approximation using one Newton-Raphson step: + // x_{i} = x_{i-1} * (2 - a * x_{i-1}) + const Packet x = + generic_reciprocal_newton_step::run(a, approx_a_recip); + const Packet tmp = pnmadd(a, x, two); + // If tmp is NaN, it means that a is either +/-0 or +/-Inf. + // In this case return the approximation directly. + const Packet is_not_nan = pcmp_eq(tmp, tmp); + return pselect(is_not_nan, pmul(x, tmp), x); + } +}; + +template +struct generic_reciprocal_newton_step { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet + run(const Packet& /*unused*/, const Packet& approx_rsqrt) { + return approx_rsqrt; + } +}; + + +/** \internal Fast reciprocal sqrt using Newton-Raphson's method. + + Preconditions: + 1. The starting guess provided in approx_a_recip must have at least half + the leading mantissa bits in the correct result, such that a single + Newton-Raphson step is sufficient to get within 1-2 ulps of the currect + result. + 2. If a is zero, approx_a_recip must be infinite with the same sign as a. + 3. If a is infinite, approx_a_recip must be zero with the same sign as a. + + If the preconditions are satisfied, which they are for for the _*_rcp_ps + instructions on x86, the result has a maximum relative error of 2 ulps, + and correctly handles zero, infinity, and NaN. Positive denormals are + treated as zero. +*/ +template +struct generic_rsqrt_newton_step { + static_assert(Steps > 0, "Steps must be at least 1."); + using Scalar = typename unpacket_traits::type; + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet + run(const Packet& a, const Packet& approx_rsqrt) { + constexpr Scalar kMinusHalf = Scalar(-1)/Scalar(2); + const Packet cst_minus_half = pset1(kMinusHalf); + const Packet cst_minus_one = pset1(Scalar(-1)); + + Packet inv_sqrt = approx_rsqrt; + for (int step = 0; step < Steps; ++step) { + // Refine the approximation using one Newton-Raphson step: + // h_n = (x * inv_sqrt) * inv_sqrt - 1 (so that h_n is nearly 0). + // inv_sqrt = inv_sqrt - 0.5 * inv_sqrt * h_n + Packet r2 = pmul(a, inv_sqrt); + Packet half_r = pmul(inv_sqrt, cst_minus_half); + Packet h_n = pmadd(r2, inv_sqrt, cst_minus_one); + inv_sqrt = pmadd(half_r, h_n, inv_sqrt); + } + + // If x is NaN, then either: + // 1) the input is NaN + // 2) zero and infinity were multiplied + // In either of these cases, return approx_rsqrt + return pselect(pisnan(inv_sqrt), approx_rsqrt, inv_sqrt); + } +}; + +template +struct generic_rsqrt_newton_step { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet + run(const Packet& /*unused*/, const Packet& approx_rsqrt) { + return approx_rsqrt; + } +}; + +/** \internal Fast sqrt using Newton-Raphson's method. + + Preconditions: + 1. The starting guess for the reciprocal sqrt provided in approx_rsqrt must + have at least half the leading mantissa bits in the correct result, such + that a single Newton-Raphson step is sufficient to get within 1-2 ulps of + the currect result. + 2. If a is zero, approx_rsqrt must be infinite. + 3. If a is infinite, approx_rsqrt must be zero. + + If the preconditions are satisfied, which they are for for the _*_rsqrt_ps + instructions on x86, the result has a maximum relative error of 2 ulps, + and correctly handles zero and infinity, and NaN. Positive denormal inputs + are treated as zero. +*/ +template +struct generic_sqrt_newton_step { + static_assert(Steps > 0, "Steps must be at least 1."); + + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet + run(const Packet& a, const Packet& approx_rsqrt) { + using Scalar = typename unpacket_traits::type; + const Packet one_point_five = pset1(Scalar(1.5)); + const Packet minus_half = pset1(Scalar(-0.5)); + // If a is inf or zero, return a directly. + const Packet inf_mask = pcmp_eq(a, pset1(NumTraits::infinity())); + const Packet return_a = por(pcmp_eq(a, pzero(a)), inf_mask); + // Do a single step of Newton's iteration for reciprocal square root: + // x_{n+1} = x_n * (1.5 + (-0.5 * x_n) * (a * x_n))). + // The Newton's step is computed this way to avoid over/under-flows. + Packet rsqrt = pmul(approx_rsqrt, pmadd(pmul(minus_half, approx_rsqrt), pmul(a, approx_rsqrt), one_point_five)); + for (int step = 1; step < Steps; ++step) { + rsqrt = pmul(rsqrt, pmadd(pmul(minus_half, rsqrt), pmul(a, rsqrt), one_point_five)); + } + + // Return sqrt(x) = x * rsqrt(x) for non-zero finite positive arguments. + // Return a itself for 0 or +inf, NaN for negative arguments. + return pselect(return_a, a, pmul(a, rsqrt)); + } +}; + /** \internal \returns the hyperbolic tan of \a a (coeff-wise) Doesn't do anything fancy, just a 13/6-degree rational interpolant which is accurate up to a couple of ulps in the (approximate) range [-8, 8], outside of which tanh(x) = +/-1 in single precision. The input is clamped to the range [-c, c]. The value c is chosen as the smallest value where the approximation evaluates to exactly 1. In the reange [-0.0004, 0.0004] - the approxmation tanh(x) ~= x is used for better accuracy as x tends to zero. + the approximation tanh(x) ~= x is used for better accuracy as x tends to zero. This implementation works on both scalars and packets. */ @@ -88,7 +226,7 @@ RealScalar positive_real_hypot(const RealScalar& x, const RealScalar& y) EIGEN_USING_STD(sqrt); RealScalar p, qp; p = numext::maxi(x,y); - if(p==RealScalar(0)) return RealScalar(0); + if(numext::is_exactly_zero(p)) return RealScalar(0); qp = numext::mini(y,x) / p; return p * sqrt(RealScalar(1) + qp*qp); } @@ -138,8 +276,8 @@ EIGEN_DEVICE_FUNC std::complex complex_sqrt(const std::complex& z) { return (numext::isinf)(y) ? std::complex(NumTraits::infinity(), y) - : x == zero ? std::complex(w, y < zero ? -w : w) - : x > zero ? std::complex(w, y / (2 * w)) + : numext::is_exactly_zero(x) ? std::complex(w, y < zero ? -w : w) + : x > zero ? std::complex(w, y / (2 * w)) : std::complex(numext::abs(y) / (2 * w), y < zero ? -w : w ); } @@ -177,10 +315,10 @@ EIGEN_DEVICE_FUNC std::complex complex_rsqrt(const std::complex& z) { const T woz = w / abs_z; // Corner cases consistent with 1/sqrt(z) on gcc/clang. return - abs_z == zero ? std::complex(NumTraits::infinity(), NumTraits::quiet_NaN()) - : ((numext::isinf)(x) || (numext::isinf)(y)) ? std::complex(zero, zero) - : x == zero ? std::complex(woz, y < zero ? woz : -woz) - : x > zero ? std::complex(woz, -y / (2 * w * abs_z)) + numext::is_exactly_zero(abs_z) ? std::complex(NumTraits::infinity(), NumTraits::quiet_NaN()) + : ((numext::isinf)(x) || (numext::isinf)(y)) ? std::complex(zero, zero) + : numext::is_exactly_zero(x) ? std::complex(woz, y < zero ? woz : -woz) + : x > zero ? std::complex(woz, -y / (2 * w * abs_z)) : std::complex(numext::abs(y) / (2 * w * abs_z), y < zero ? woz : -woz ); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Matrix.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Matrix.h index f0e59a911d2..91008f8e840 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Matrix.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Matrix.h @@ -11,37 +11,40 @@ #ifndef EIGEN_MATRIX_H #define EIGEN_MATRIX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template -struct traits > +template +struct traits > { private: - enum { size = internal::size_at_compile_time<_Rows,_Cols>::ret }; - typedef typename find_best_packet<_Scalar,size>::type PacketScalar; + constexpr static int size = internal::size_at_compile_time(Rows_,Cols_); + typedef typename find_best_packet::type PacketScalar; enum { - row_major_bit = _Options&RowMajor ? RowMajorBit : 0, - is_dynamic_size_storage = _MaxRows==Dynamic || _MaxCols==Dynamic, - max_size = is_dynamic_size_storage ? Dynamic : _MaxRows*_MaxCols, - default_alignment = compute_default_alignment<_Scalar,max_size>::value, - actual_alignment = ((_Options&DontAlign)==0) ? default_alignment : 0, + row_major_bit = Options_&RowMajor ? RowMajorBit : 0, + is_dynamic_size_storage = MaxRows_==Dynamic || MaxCols_==Dynamic, + max_size = is_dynamic_size_storage ? Dynamic : MaxRows_*MaxCols_, + default_alignment = compute_default_alignment::value, + actual_alignment = ((Options_&DontAlign)==0) ? default_alignment : 0, required_alignment = unpacket_traits::alignment, - packet_access_bit = (packet_traits<_Scalar>::Vectorizable && (EIGEN_UNALIGNED_VECTORIZE || (actual_alignment>=required_alignment))) ? PacketAccessBit : 0 + packet_access_bit = (packet_traits::Vectorizable && (EIGEN_UNALIGNED_VECTORIZE || (actual_alignment>=required_alignment))) ? PacketAccessBit : 0 }; public: - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef Dense StorageKind; typedef Eigen::Index StorageIndex; typedef MatrixXpr XprKind; enum { - RowsAtCompileTime = _Rows, - ColsAtCompileTime = _Cols, - MaxRowsAtCompileTime = _MaxRows, - MaxColsAtCompileTime = _MaxCols, - Flags = compute_matrix_flags<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>::ret, - Options = _Options, + RowsAtCompileTime = Rows_, + ColsAtCompileTime = Cols_, + MaxRowsAtCompileTime = MaxRows_, + MaxColsAtCompileTime = MaxCols_, + Flags = compute_matrix_flags(Options_), + Options = Options_, InnerStrideAtCompileTime = 1, OuterStrideAtCompileTime = (Options&RowMajor) ? ColsAtCompileTime : RowsAtCompileTime, @@ -63,18 +66,18 @@ struct traits > * The %Matrix class encompasses \em both fixed-size and dynamic-size objects (\ref fixedsize "note"). * * The first three template parameters are required: - * \tparam _Scalar Numeric type, e.g. float, double, int or std::complex. + * \tparam Scalar_ Numeric type, e.g. float, double, int or std::complex. * User defined scalar types are supported as well (see \ref user_defined_scalars "here"). - * \tparam _Rows Number of rows, or \b Dynamic - * \tparam _Cols Number of columns, or \b Dynamic + * \tparam Rows_ Number of rows, or \b Dynamic + * \tparam Cols_ Number of columns, or \b Dynamic * * The remaining template parameters are optional -- in most cases you don't have to worry about them. - * \tparam _Options A combination of either \b #RowMajor or \b #ColMajor, and of either + * \tparam Options_ A combination of either \b #RowMajor or \b #ColMajor, and of either * \b #AutoAlign or \b #DontAlign. * The former controls \ref TopicStorageOrders "storage order", and defaults to column-major. The latter controls alignment, which is required * for vectorization. It defaults to aligning matrices except for fixed sizes that aren't a multiple of the packet size. - * \tparam _MaxRows Maximum number of rows. Defaults to \a _Rows (\ref maxrows "note"). - * \tparam _MaxCols Maximum number of columns. Defaults to \a _Cols (\ref maxrows "note"). + * \tparam MaxRows_ Maximum number of rows. Defaults to \a Rows_ (\ref maxrows "note"). + * \tparam MaxCols_ Maximum number of columns. Defaults to \a Cols_ (\ref maxrows "note"). * * Eigen provides a number of typedefs covering the usual cases. Here are some examples: * @@ -128,12 +131,12 @@ struct traits > * Note that \em dense matrices, be they Fixed-size or Dynamic-size, do not expand dynamically in the sense of a std::map. * If you want this behavior, see the Sparse module. * - *
\anchor maxrows _MaxRows and _MaxCols:
+ *
\anchor maxrows MaxRows_ and MaxCols_:
*
In most cases, one just leaves these parameters to the default values. * These parameters mean the maximum size of rows and columns that the matrix may have. They are useful in cases * when the exact numbers of rows and columns are not known are compile-time, but it is known at compile-time that they cannot - * exceed a certain value. This happens when taking dynamic-size blocks inside fixed-size matrices: in this case _MaxRows and _MaxCols - * are the dimensions of the original matrix, while _Rows and _Cols are Dynamic.
+ * exceed a certain value. This happens when taking dynamic-size blocks inside fixed-size matrices: in this case MaxRows_ and MaxCols_ + * are the dimensions of the original matrix, while Rows_ and Cols_ are Dynamic. * * * ABI and storage layout @@ -174,9 +177,9 @@ struct traits > * \ref TopicStorageOrders */ -template +template class Matrix - : public PlainObjectBase > + : public PlainObjectBase > { public: @@ -185,7 +188,7 @@ class Matrix */ typedef PlainObjectBase Base; - enum { Options = _Options }; + enum { Options = Options_ }; EIGEN_DENSE_PUBLIC_INTERFACE(Matrix) @@ -258,7 +261,6 @@ class Matrix EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix() : Base() { - Base::_check_template_params(); EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } @@ -266,24 +268,18 @@ class Matrix EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Matrix(internal::constructor_without_unaligned_array_assert) : Base(internal::constructor_without_unaligned_array_assert()) - { Base::_check_template_params(); EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } + { EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } -#if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(Matrix&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_constructible::value) - : Base(std::move(other)) - { - Base::_check_template_params(); - } + : Base(std::move(other)) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix& operator=(Matrix&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_assignable::value) { Base::operator=(std::move(other)); return *this; } -#endif -#if EIGEN_HAS_CXX11 /** \copydoc PlainObjectBase(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&... args) * * Example: \include Matrix_variadic_ctor_cxx11.cpp @@ -317,9 +313,9 @@ class Matrix * * \sa Matrix(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) */ - EIGEN_DEVICE_FUNC - explicit EIGEN_STRONG_INLINE Matrix(const std::initializer_list>& list) : Base(list) {} -#endif // end EIGEN_HAS_CXX11 + EIGEN_DEVICE_FUNC explicit constexpr EIGEN_STRONG_INLINE Matrix( + const std::initializer_list>& list) + : Base(list) {} #ifndef EIGEN_PARSED_BY_DOXYGEN @@ -328,7 +324,6 @@ class Matrix EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Matrix(const T& x) { - Base::_check_template_params(); Base::template _init1(x); } @@ -336,7 +331,6 @@ class Matrix EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const T0& x, const T1& y) { - Base::_check_template_params(); Base::template _init2(x, y); } @@ -388,7 +382,6 @@ class Matrix EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const Scalar& x, const Scalar& y, const Scalar& z) { - Base::_check_template_params(); EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Matrix, 3) m_storage.data()[0] = x; m_storage.data()[1] = y; @@ -400,7 +393,6 @@ class Matrix EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const Scalar& x, const Scalar& y, const Scalar& z, const Scalar& w) { - Base::_check_template_params(); EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Matrix, 4) m_storage.data()[0] = x; m_storage.data()[1] = y; @@ -480,16 +472,21 @@ class Matrix #define EIGEN_MAKE_TYPEDEFS(Type, TypeSuffix, Size, SizeSuffix) \ /** \ingroup matrixtypedefs */ \ +/** \brief `Size`×`Size` matrix of type `Type`. */ \ typedef Matrix Matrix##SizeSuffix##TypeSuffix; \ /** \ingroup matrixtypedefs */ \ +/** \brief `Size`×`1` vector of type `Type`. */ \ typedef Matrix Vector##SizeSuffix##TypeSuffix; \ /** \ingroup matrixtypedefs */ \ +/** \brief `1`×`Size` vector of type `Type`. */ \ typedef Matrix RowVector##SizeSuffix##TypeSuffix; #define EIGEN_MAKE_FIXED_TYPEDEFS(Type, TypeSuffix, Size) \ /** \ingroup matrixtypedefs */ \ +/** \brief `Size`×`Dynamic` matrix of type `Type`. */ \ typedef Matrix Matrix##Size##X##TypeSuffix; \ /** \ingroup matrixtypedefs */ \ +/** \brief `Dynamic`×`Size` matrix of type `Type`. */ \ typedef Matrix Matrix##X##Size##TypeSuffix; #define EIGEN_MAKE_TYPEDEFS_ALL_SIZES(Type, TypeSuffix) \ @@ -511,30 +508,28 @@ EIGEN_MAKE_TYPEDEFS_ALL_SIZES(std::complex, cd) #undef EIGEN_MAKE_TYPEDEFS #undef EIGEN_MAKE_FIXED_TYPEDEFS -#if EIGEN_HAS_CXX11 - -#define EIGEN_MAKE_TYPEDEFS(Size, SizeSuffix) \ -/** \ingroup matrixtypedefs */ \ -/** \brief \cpp11 */ \ -template \ -using Matrix##SizeSuffix = Matrix; \ -/** \ingroup matrixtypedefs */ \ -/** \brief \cpp11 */ \ -template \ -using Vector##SizeSuffix = Matrix; \ -/** \ingroup matrixtypedefs */ \ -/** \brief \cpp11 */ \ -template \ +#define EIGEN_MAKE_TYPEDEFS(Size, SizeSuffix) \ +/** \ingroup matrixtypedefs */ \ +/** \brief \cpp11 `Size`×`Size` matrix of type `Type`.*/ \ +template \ +using Matrix##SizeSuffix = Matrix; \ +/** \ingroup matrixtypedefs */ \ +/** \brief \cpp11 `Size`×`1` vector of type `Type`.*/ \ +template \ +using Vector##SizeSuffix = Matrix; \ +/** \ingroup matrixtypedefs */ \ +/** \brief \cpp11 `1`×`Size` vector of type `Type`.*/ \ +template \ using RowVector##SizeSuffix = Matrix; -#define EIGEN_MAKE_FIXED_TYPEDEFS(Size) \ -/** \ingroup matrixtypedefs */ \ -/** \brief \cpp11 */ \ -template \ -using Matrix##Size##X = Matrix; \ -/** \ingroup matrixtypedefs */ \ -/** \brief \cpp11 */ \ -template \ +#define EIGEN_MAKE_FIXED_TYPEDEFS(Size) \ +/** \ingroup matrixtypedefs */ \ +/** \brief \cpp11 `Size`×`Dynamic` matrix of type `Type` */ \ +template \ +using Matrix##Size##X = Matrix; \ +/** \ingroup matrixtypedefs */ \ +/** \brief \cpp11 `Dynamic`×`Size` matrix of type `Type`. */ \ +template \ using Matrix##X##Size = Matrix; EIGEN_MAKE_TYPEDEFS(2, 2) @@ -546,20 +541,18 @@ EIGEN_MAKE_FIXED_TYPEDEFS(3) EIGEN_MAKE_FIXED_TYPEDEFS(4) /** \ingroup matrixtypedefs - * \brief \cpp11 */ + * \brief \cpp11 `Size`×`1` vector of type `Type`. */ template using Vector = Matrix; /** \ingroup matrixtypedefs - * \brief \cpp11 */ + * \brief \cpp11 `1`×`Size` vector of type `Type`. */ template using RowVector = Matrix; #undef EIGEN_MAKE_TYPEDEFS #undef EIGEN_MAKE_FIXED_TYPEDEFS -#endif // EIGEN_HAS_CXX11 - } // end namespace Eigen #endif // EIGEN_MATRIX_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MatrixBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MatrixBase.h index 45c3a596ecb..7a0942f226a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MatrixBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/MatrixBase.h @@ -11,6 +11,9 @@ #ifndef EIGEN_MATRIXBASE_H #define EIGEN_MATRIXBASE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class MatrixBase @@ -92,8 +95,8 @@ template class MatrixBase #ifndef EIGEN_PARSED_BY_DOXYGEN /** type of the equivalent square matrix */ - typedef Matrix SquareMatrixType; + typedef Matrix SquareMatrixType; #endif // not EIGEN_PARSED_BY_DOXYGEN /** \returns the size of the main diagonal, which is min(rows(),cols()). @@ -107,10 +110,10 @@ template class MatrixBase /** \internal Represents a matrix with all coefficients equal to one another*/ typedef CwiseNullaryOp,PlainObject> ConstantReturnType; /** \internal the return type of MatrixBase::adjoint() */ - typedef typename internal::conditional::IsComplex, - CwiseUnaryOp, ConstTransposeReturnType>, - ConstTransposeReturnType - >::type AdjointReturnType; + typedef std::conditional_t::IsComplex, + CwiseUnaryOp, ConstTransposeReturnType>, + ConstTransposeReturnType + > AdjointReturnType; /** \internal Return type of eigenvalues() */ typedef Matrix, internal::traits::ColsAtCompileTime, 1, ColMajor> EigenvaluesReturnType; /** \internal the return type of identity */ @@ -123,9 +126,9 @@ template class MatrixBase #define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::MatrixBase #define EIGEN_DOC_UNARY_ADDONS(X,Y) -# include "../plugins/CommonCwiseBinaryOps.h" -# include "../plugins/MatrixCwiseUnaryOps.h" -# include "../plugins/MatrixCwiseBinaryOps.h" +# include "../plugins/CommonCwiseBinaryOps.inc" +# include "../plugins/MatrixCwiseUnaryOps.inc" +# include "../plugins/MatrixCwiseBinaryOps.inc" # ifdef EIGEN_MATRIXBASE_PLUGIN # include EIGEN_MATRIXBASE_PLUGIN # endif @@ -184,6 +187,11 @@ template class MatrixBase const Product operator*(const DiagonalBase &diagonal) const; + template + EIGEN_DEVICE_FUNC + const Product + operator*(const SkewSymmetricBase &skew) const; + template EIGEN_DEVICE_FUNC typename ScalarBinaryOpTraits::Scalar,typename internal::traits::Scalar>::ReturnType @@ -206,28 +214,22 @@ template class MatrixBase EIGEN_DEVICE_FUNC DiagonalReturnType diagonal(); - typedef typename internal::add_const >::type ConstDiagonalReturnType; + typedef Diagonal ConstDiagonalReturnType; EIGEN_DEVICE_FUNC - ConstDiagonalReturnType diagonal() const; - - template struct DiagonalIndexReturnType { typedef Diagonal Type; }; - template struct ConstDiagonalIndexReturnType { typedef const Diagonal Type; }; + const ConstDiagonalReturnType diagonal() const; template EIGEN_DEVICE_FUNC - typename DiagonalIndexReturnType::Type diagonal(); + Diagonal diagonal(); template EIGEN_DEVICE_FUNC - typename ConstDiagonalIndexReturnType::Type diagonal() const; - - typedef Diagonal DiagonalDynamicIndexReturnType; - typedef typename internal::add_const >::type ConstDiagonalDynamicIndexReturnType; + const Diagonal diagonal() const; EIGEN_DEVICE_FUNC - DiagonalDynamicIndexReturnType diagonal(Index index); + Diagonal diagonal(Index index); EIGEN_DEVICE_FUNC - ConstDiagonalDynamicIndexReturnType diagonal(Index index) const; + const Diagonal diagonal(Index index) const; template struct TriangularViewReturnType { typedef TriangularView Type; }; template struct ConstTriangularViewReturnType { typedef const TriangularView Type; }; @@ -263,6 +265,8 @@ template class MatrixBase EIGEN_DEVICE_FUNC const DiagonalWrapper asDiagonal() const; const PermutationWrapper asPermutation() const; + EIGEN_DEVICE_FUNC + const SkewSymmetricWrapper asSkewSymmetric() const; EIGEN_DEVICE_FUNC Derived& setIdentity(); @@ -277,6 +281,8 @@ template class MatrixBase bool isUpperTriangular(const RealScalar& prec = NumTraits::dummy_precision()) const; bool isLowerTriangular(const RealScalar& prec = NumTraits::dummy_precision()) const; + bool isSkewSymmetric(const RealScalar& prec = NumTraits::dummy_precision()) const; + template bool isOrthogonal(const MatrixBase& other, const RealScalar& prec = NumTraits::dummy_precision()) const; @@ -323,10 +329,10 @@ template class MatrixBase /////////// LU module /////////// - inline const FullPivLU fullPivLu() const; - inline const PartialPivLU partialPivLu() const; + template inline const FullPivLU fullPivLu() const; + template inline const PartialPivLU partialPivLu() const; - inline const PartialPivLU lu() const; + template inline const PartialPivLU lu() const; EIGEN_DEVICE_FUNC inline const Inverse inverse() const; @@ -357,9 +363,9 @@ template class MatrixBase /////////// QR module /////////// inline const HouseholderQR householderQr() const; - inline const ColPivHouseholderQR colPivHouseholderQr() const; - inline const FullPivHouseholderQR fullPivHouseholderQr() const; - inline const CompleteOrthogonalDecomposition completeOrthogonalDecomposition() const; + template inline const ColPivHouseholderQR colPivHouseholderQr() const; + template inline const FullPivHouseholderQR fullPivHouseholderQr() const; + template inline const CompleteOrthogonalDecomposition completeOrthogonalDecomposition() const; /////////// Eigenvalues module /////////// @@ -368,25 +374,23 @@ template class MatrixBase /////////// SVD module /////////// - inline JacobiSVD jacobiSvd(unsigned int computationOptions = 0) const; - inline BDCSVD bdcSvd(unsigned int computationOptions = 0) const; + template + inline JacobiSVD jacobiSvd() const; + template + EIGEN_DEPRECATED + inline JacobiSVD jacobiSvd(unsigned int computationOptions) const; + + template + inline BDCSVD bdcSvd() const; + template + EIGEN_DEPRECATED + inline BDCSVD bdcSvd(unsigned int computationOptions) const; /////////// Geometry module /////////// - #ifndef EIGEN_PARSED_BY_DOXYGEN - /// \internal helper struct to form the return type of the cross product - template struct cross_product_return_type { - typedef typename ScalarBinaryOpTraits::Scalar,typename internal::traits::Scalar>::ReturnType Scalar; - typedef Matrix type; - }; - #endif // EIGEN_PARSED_BY_DOXYGEN template EIGEN_DEVICE_FUNC -#ifndef EIGEN_PARSED_BY_DOXYGEN - inline typename cross_product_return_type::type -#else - inline PlainObject -#endif + inline typename internal::cross_impl::return_type cross(const MatrixBase& other) const; template @@ -396,9 +400,12 @@ template class MatrixBase EIGEN_DEVICE_FUNC inline PlainObject unitOrthogonal(void) const; - EIGEN_DEVICE_FUNC + EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline Matrix eulerAngles(Index a0, Index a1, Index a2) const; + EIGEN_DEVICE_FUNC + inline Matrix canonicalEulerAngles(Index a0, Index a1, Index a2) const; + // put this as separate enum value to work around possible GCC 4.3 bug (?) enum { HomogeneousReturnTypeDirection = ColsAtCompileTime==1&&RowsAtCompileTime==1 ? ((internal::traits::Flags&RowMajorBit)==RowMajorBit ? Horizontal : Vertical) : ColsAtCompileTime==1 ? Vertical : Horizontal }; @@ -468,11 +475,9 @@ template class MatrixBase const MatrixFunctionReturnValue matrixFunction(StemFunction f) const; EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, cosh, hyperbolic cosine) EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, sinh, hyperbolic sine) -#if EIGEN_HAS_CXX11_MATH EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, atanh, inverse hyperbolic cosine) EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, acosh, inverse hyperbolic cosine) EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, asinh, inverse hyperbolic sine) -#endif EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, cos, cosine) EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, sin, sine) EIGEN_MATRIX_FUNCTION(MatrixSquareRootReturnValue, sqrt, square root) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NestByValue.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NestByValue.h index b4275768a01..7bcd97e2d2a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NestByValue.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NestByValue.h @@ -11,6 +11,9 @@ #ifndef EIGEN_NESTBYVALUE_H #define EIGEN_NESTBYVALUE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -41,6 +44,8 @@ template class NestByValue public: typedef typename internal::dense_xpr_base::type Base; + static constexpr bool HasDirectAccess = internal::has_direct_access::ret; + EIGEN_DENSE_PUBLIC_INTERFACE(NestByValue) EIGEN_DEVICE_FUNC explicit inline NestByValue(const ExpressionType& matrix) : m_expression(matrix) {} @@ -52,6 +57,18 @@ template class NestByValue EIGEN_DEVICE_FUNC const ExpressionType& nestedExpression() const { return m_expression; } + EIGEN_DEVICE_FUNC typename std::enable_if::type data() const { + return m_expression.data(); + } + + EIGEN_DEVICE_FUNC typename std::enable_if::type innerStride() const { + return m_expression.innerStride(); + } + + EIGEN_DEVICE_FUNC typename std::enable_if::type outerStride() const { + return m_expression.outerStride(); + } + protected: const ExpressionType m_expression; }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NoAlias.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NoAlias.h index 570283d90f1..8b032ffe5f4 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NoAlias.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NoAlias.h @@ -10,6 +10,9 @@ #ifndef EIGEN_NOALIAS_H #define EIGEN_NOALIAS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class NoAlias diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NumTraits.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NumTraits.h index 72eac5a93c0..a417b4c228f 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NumTraits.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/NumTraits.h @@ -10,12 +10,45 @@ #ifndef EIGEN_NUMTRAITS_H #define EIGEN_NUMTRAITS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { +// default implementation of digits(), based on numeric_limits if specialized, +// 0 for integer types, and log2(epsilon()) otherwise. +template< typename T, + bool use_numeric_limits = std::numeric_limits::is_specialized, + bool is_integer = NumTraits::IsInteger> +struct default_digits_impl +{ + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR + static int run() { return std::numeric_limits::digits; } +}; + +template +struct default_digits_impl // Floating point +{ + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR + static int run() { + using std::log2; + using std::ceil; + typedef typename NumTraits::Real Real; + return int(ceil(-log2(NumTraits::epsilon()))); + } +}; + +template +struct default_digits_impl // Integer +{ + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR + static int run() { return 0; } +}; + // default implementation of digits10(), based on numeric_limits if specialized, -// 0 for integer types, and log10(epsilon()) otherwise. +// 0 for integer types, and floor((digits()-1)*log10(2)) otherwise. template< typename T, bool use_numeric_limits = std::numeric_limits::is_specialized, bool is_integer = NumTraits::IsInteger> @@ -31,9 +64,9 @@ struct default_digits10_impl // Floating point EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { using std::log10; - using std::ceil; + using std::floor; typedef typename NumTraits::Real Real; - return int(ceil(-log10(NumTraits::epsilon()))); + return int(floor((internal::default_digits_impl::run()-1)*log10(2))); } }; @@ -44,32 +77,31 @@ struct default_digits10_impl // Integer static int run() { return 0; } }; - -// default implementation of digits(), based on numeric_limits if specialized, -// 0 for integer types, and log2(epsilon()) otherwise. +// default implementation of max_digits10(), based on numeric_limits if specialized, +// 0 for integer types, and log10(2) * digits() + 1 otherwise. template< typename T, bool use_numeric_limits = std::numeric_limits::is_specialized, bool is_integer = NumTraits::IsInteger> -struct default_digits_impl +struct default_max_digits10_impl { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static int run() { return std::numeric_limits::digits; } + static int run() { return std::numeric_limits::max_digits10; } }; template -struct default_digits_impl // Floating point +struct default_max_digits10_impl // Floating point { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { - using std::log; + using std::log10; using std::ceil; typedef typename NumTraits::Real Real; - return int(ceil(-log(NumTraits::epsilon())/log(static_cast(2)))); + return int(ceil(internal::default_digits_impl::run()*log10(2)+1)); } }; template -struct default_digits_impl // Integer +struct default_max_digits10_impl // Integer { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { return 0; } @@ -83,17 +115,17 @@ namespace numext { // TODO: Replace by std::bit_cast (available in C++20) template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Tgt bit_cast(const Src& src) { -#if EIGEN_HAS_TYPE_TRAITS // The behaviour of memcpy is not specified for non-trivially copyable types EIGEN_STATIC_ASSERT(std::is_trivially_copyable::value, THIS_TYPE_IS_NOT_SUPPORTED); EIGEN_STATIC_ASSERT(std::is_trivially_copyable::value && std::is_default_constructible::value, THIS_TYPE_IS_NOT_SUPPORTED); -#endif - EIGEN_STATIC_ASSERT(sizeof(Src) == sizeof(Tgt), THIS_TYPE_IS_NOT_SUPPORTED); + Tgt tgt; + // Load src into registers first. This allows the memcpy to be elided by CUDA. + const Src staged = src; EIGEN_USING_STD(memcpy) - memcpy(&tgt, &src, sizeof(Tgt)); + memcpy(static_cast(&tgt),static_cast(&staged), sizeof(Tgt)); return tgt; } } // namespace numext @@ -141,6 +173,9 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Tgt bit_cast(const Src& src) { * \li digits10() function returning the number of decimal digits that can be represented without change. This is * the analogue of std::numeric_limits::digits10 * which is used as the default implementation if specialized. + * \li max_digits10() function returning the number of decimal digits required to uniquely represent all distinct values of the type. This is + * the analogue of std::numeric_limits::max_digits10 + * which is used as the default implementation if specialized. * \li min_exponent() and max_exponent() functions returning the highest and lowest possible values, respectively, * such that the radix raised to the power exponent-1 is a normalized floating-point number. These are equivalent to * std::numeric_limits::min_exponent/ @@ -162,11 +197,7 @@ template struct GenericNumTraits }; typedef T Real; - typedef typename internal::conditional< - IsInteger, - typename internal::conditional::type, - T - >::type NonInteger; + typedef std::conditional_t, T> NonInteger; typedef T Nested; typedef T Literal; @@ -182,6 +213,12 @@ template struct GenericNumTraits return internal::default_digits10_impl::run(); } + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR + static inline int max_digits10() + { + return internal::default_max_digits10_impl::run(); + } + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int digits() { @@ -245,22 +282,35 @@ template<> struct NumTraits : GenericNumTraits static inline double dummy_precision() { return 1e-12; } }; +// GPU devices treat `long double` as `double`. +#ifndef EIGEN_GPU_COMPILE_PHASE template<> struct NumTraits : GenericNumTraits { - EIGEN_CONSTEXPR - static inline long double dummy_precision() { return 1e-15l; } + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR + static inline long double dummy_precision() { return static_cast(1e-15l); } + +#if defined(EIGEN_ARCH_PPC) && (__LDBL_MANT_DIG__ == 106) + // PowerPC double double causes issues with some values + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR + static inline long double epsilon() + { + // 2^(-(__LDBL_MANT_DIG__)+1) + return static_cast(2.4651903288156618919116517665087e-32l); + } +#endif }; +#endif -template struct NumTraits > - : GenericNumTraits > +template struct NumTraits > + : GenericNumTraits > { - typedef _Real Real; - typedef typename NumTraits<_Real>::Literal Literal; + typedef Real_ Real; + typedef typename NumTraits::Literal Literal; enum { IsComplex = 1, - RequireInitialization = NumTraits<_Real>::RequireInitialization, - ReadCost = 2 * NumTraits<_Real>::ReadCost, + RequireInitialization = NumTraits::RequireInitialization, + ReadCost = 2 * NumTraits::ReadCost, AddCost = 2 * NumTraits::AddCost, MulCost = 4 * NumTraits::MulCost + 2 * NumTraits::AddCost }; @@ -271,6 +321,8 @@ template struct NumTraits > static inline Real dummy_precision() { return NumTraits::dummy_precision(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int digits10() { return NumTraits::digits10(); } + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR + static inline int max_digits10() { return NumTraits::max_digits10(); } }; template @@ -301,6 +353,8 @@ struct NumTraits > EIGEN_CONSTEXPR static inline int digits10() { return NumTraits::digits10(); } + EIGEN_CONSTEXPR + static inline int max_digits10() { return NumTraits::max_digits10(); } }; template<> struct NumTraits @@ -315,8 +369,10 @@ template<> struct NumTraits EIGEN_CONSTEXPR static inline int digits10() { return 0; } + EIGEN_CONSTEXPR + static inline int max_digits10() { return 0; } -private: + private: static inline std::string epsilon(); static inline std::string dummy_precision(); static inline std::string lowest(); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PartialReduxEvaluator.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PartialReduxEvaluator.h index 17c06f0783d..f7653eb871a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PartialReduxEvaluator.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PartialReduxEvaluator.h @@ -10,6 +10,9 @@ #ifndef EIGEN_PARTIALREDUX_H #define EIGEN_PARTIALREDUX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -29,7 +32,7 @@ namespace internal { * some (optional) processing of the outcome, e.g., division by n for mean. * * For the vectorized path let's observe that the packet-size and outer-unrolling -* are both decided by the assignement logic. So all we have to do is to decide +* are both decided by the assignment logic. So all we have to do is to decide * on the inner unrolling. * * For the unrolling, we can reuse "internal::redux_vec_unroller" from Redux.h, @@ -91,7 +94,7 @@ struct packetwise_redux_impl * This specialization is not required for general reductions, which is * why it is defined here. */ -template +template struct redux_vec_unroller { template @@ -139,8 +142,8 @@ struct evaluator > { typedef PartialReduxExpr XprType; typedef typename internal::nested_eval::type ArgTypeNested; - typedef typename internal::add_const_on_value_type::type ConstArgTypeNested; - typedef typename internal::remove_all::type ArgTypeNestedCleaned; + typedef add_const_on_value_type_t ConstArgTypeNested; + typedef internal::remove_all_t ArgTypeNestedCleaned; typedef typename ArgType::Scalar InputScalar; typedef typename XprType::Scalar Scalar; enum { @@ -152,16 +155,16 @@ struct evaluator > : TraversalSize==0 ? 1 : int(TraversalSize) * int(evaluator::CoeffReadCost) + int(CostOpType::value), - _ArgFlags = evaluator::Flags, + ArgFlags_ = evaluator::Flags, - _Vectorizable = bool(int(_ArgFlags)&PacketAccessBit) + Vectorizable_ = bool(int(ArgFlags_)&PacketAccessBit) && bool(MemberOp::Vectorizable) - && (Direction==int(Vertical) ? bool(_ArgFlags&RowMajorBit) : (_ArgFlags&RowMajorBit)==0) + && (Direction==int(Vertical) ? bool(ArgFlags_&RowMajorBit) : (ArgFlags_&RowMajorBit)==0) && (TraversalSize!=0), Flags = (traits::Flags&RowMajorBit) | (evaluator::Flags&(HereditaryBits&(~RowMajorBit))) - | (_Vectorizable ? PacketAccessBit : 0) + | (Vectorizable_ ? PacketAccessBit : 0) | LinearAccessBit, Alignment = 0 // FIXME this will need to be improved once PartialReduxExpr is vectorized diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PermutationMatrix.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PermutationMatrix.h index 69401bf41e5..92554655130 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PermutationMatrix.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PermutationMatrix.h @@ -11,6 +11,9 @@ #ifndef EIGEN_PERMUTATIONMATRIX_H #define EIGEN_PERMUTATIONMATRIX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -269,13 +272,13 @@ class PermutationBase : public EigenBase }; namespace internal { -template -struct traits > - : traits > +template +struct traits > + : traits > { typedef PermutationStorage StorageKind; - typedef Matrix<_StorageIndex, SizeAtCompileTime, 1, 0, MaxSizeAtCompileTime, 1> IndicesType; - typedef _StorageIndex StorageIndex; + typedef Matrix IndicesType; + typedef StorageIndex_ StorageIndex; typedef void Scalar; }; } @@ -287,14 +290,14 @@ struct traits -class PermutationMatrix : public PermutationBase > +template +class PermutationMatrix : public PermutationBase > { typedef PermutationBase Base; typedef internal::traits Traits; @@ -389,20 +392,20 @@ class PermutationMatrix : public PermutationBase -struct traits,_PacketAccess> > - : traits > +template +struct traits,PacketAccess_> > + : traits > { typedef PermutationStorage StorageKind; - typedef Map, _PacketAccess> IndicesType; - typedef _StorageIndex StorageIndex; + typedef Map, PacketAccess_> IndicesType; + typedef StorageIndex_ StorageIndex; typedef void Scalar; }; } -template -class Map,_PacketAccess> - : public PermutationBase,_PacketAccess> > +template +class Map,PacketAccess_> + : public PermutationBase,PacketAccess_> > { typedef PermutationBase Base; typedef internal::traits Traits; @@ -452,18 +455,18 @@ class Map class TranspositionsWrapper; +template class TranspositionsWrapper; namespace internal { -template -struct traits > +template +struct traits > { typedef PermutationStorage StorageKind; typedef void Scalar; - typedef typename _IndicesType::Scalar StorageIndex; - typedef _IndicesType IndicesType; + typedef typename IndicesType_::Scalar StorageIndex; + typedef IndicesType_ IndicesType; enum { - RowsAtCompileTime = _IndicesType::SizeAtCompileTime, - ColsAtCompileTime = _IndicesType::SizeAtCompileTime, + RowsAtCompileTime = IndicesType_::SizeAtCompileTime, + ColsAtCompileTime = IndicesType_::SizeAtCompileTime, MaxRowsAtCompileTime = IndicesType::MaxSizeAtCompileTime, MaxColsAtCompileTime = IndicesType::MaxSizeAtCompileTime, Flags = 0 @@ -476,14 +479,14 @@ struct traits > * * \brief Class to view a vector of integers as a permutation matrix * - * \tparam _IndicesType the type of the vector of integer (can be any compatible expression) + * \tparam IndicesType_ the type of the vector of integer (can be any compatible expression) * * This class allows to view any vector expression of integers as a permutation matrix. * * \sa class PermutationBase, class PermutationMatrix */ -template -class PermutationWrapper : public PermutationBase > +template +class PermutationWrapper : public PermutationBase > { typedef PermutationBase Base; typedef internal::traits Traits; @@ -498,7 +501,7 @@ class PermutationWrapper : public PermutationBase::type& + const internal::remove_all_t& indices() const { return m_indices; } protected: diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PlainObjectBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PlainObjectBase.h index e2ddbd1d522..2c3c5851e18 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PlainObjectBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/PlainObjectBase.h @@ -22,30 +22,47 @@ # define EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED #endif +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template struct check_rows_cols_for_overflow { - template - EIGEN_DEVICE_FUNC - static EIGEN_ALWAYS_INLINE void run(Index, Index) - { +template +struct check_rows_cols_for_overflow { + EIGEN_STATIC_ASSERT(MaxRowsAtCompileTime * MaxColsAtCompileTime == MaxSizeAtCompileTime,YOU MADE A PROGRAMMING MISTAKE) + template + EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE constexpr void run(Index, Index) {} +}; + +template +struct check_rows_cols_for_overflow { + template + EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE constexpr void run(Index, Index cols) { + constexpr Index MaxIndex = NumTraits::highest(); + bool error = cols > MaxIndex / MaxRowsAtCompileTime; + if (error) throw_std_bad_alloc(); } }; -template<> struct check_rows_cols_for_overflow { - template - EIGEN_DEVICE_FUNC - static EIGEN_ALWAYS_INLINE void run(Index rows, Index cols) - { - // http://hg.mozilla.org/mozilla-central/file/6c8a909977d3/xpcom/ds/CheckedInt.h#l242 - // we assume Index is signed - Index max_index = (std::size_t(1) << (8 * sizeof(Index) - 1)) - 1; // assume Index is signed - bool error = (rows == 0 || cols == 0) ? false - : (rows > max_index / cols); - if (error) - throw_std_bad_alloc(); +template +struct check_rows_cols_for_overflow { + template + EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE constexpr void run(Index rows, Index) { + constexpr Index MaxIndex = NumTraits::highest(); + bool error = rows > MaxIndex / MaxColsAtCompileTime; + if (error) throw_std_bad_alloc(); + } +}; + +template <> +struct check_rows_cols_for_overflow { + template + EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE constexpr void run(Index rows, Index cols) { + constexpr Index MaxIndex = NumTraits::highest(); + bool error = cols == 0 ? false : (rows > MaxIndex / cols); + if (error) throw_std_bad_alloc(); } }; @@ -64,18 +81,18 @@ namespace doxygen { // This is a workaround to doxygen not being able to understand the inheritance logic // when it is hidden by the dense_xpr_base helper struct. // Moreover, doxygen fails to include members that are not documented in the declaration body of -// MatrixBase if we inherits MatrixBase >, +// MatrixBase if we inherits MatrixBase >, // this is why we simply inherits MatrixBase, though this does not make sense. /** This class is just a workaround for Doxygen and it does not not actually exist. */ template struct dense_xpr_base_dispatcher; /** This class is just a workaround for Doxygen and it does not not actually exist. */ -template -struct dense_xpr_base_dispatcher > +template +struct dense_xpr_base_dispatcher > : public MatrixBase {}; /** This class is just a workaround for Doxygen and it does not not actually exist. */ -template -struct dense_xpr_base_dispatcher > +template +struct dense_xpr_base_dispatcher > : public ArrayBase {}; } // namespace doxygen @@ -134,6 +151,16 @@ class PlainObjectBase : public internal::dense_xpr_base::type enum { NeedsToAlign = (SizeAtCompileTime != Dynamic) && (internal::traits::Alignment>0) }; EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) + EIGEN_STATIC_ASSERT(internal::check_implication(MaxRowsAtCompileTime==1 && MaxColsAtCompileTime!=1, (int(Options)&RowMajor)==RowMajor), INVALID_MATRIX_TEMPLATE_PARAMETERS) + EIGEN_STATIC_ASSERT(internal::check_implication(MaxColsAtCompileTime==1 && MaxRowsAtCompileTime!=1, (int(Options)&RowMajor)==0), INVALID_MATRIX_TEMPLATE_PARAMETERS) + EIGEN_STATIC_ASSERT((RowsAtCompileTime == Dynamic) || (RowsAtCompileTime >= 0), INVALID_MATRIX_TEMPLATE_PARAMETERS) + EIGEN_STATIC_ASSERT((ColsAtCompileTime == Dynamic) || (ColsAtCompileTime >= 0), INVALID_MATRIX_TEMPLATE_PARAMETERS) + EIGEN_STATIC_ASSERT((MaxRowsAtCompileTime == Dynamic) || (MaxRowsAtCompileTime >= 0), INVALID_MATRIX_TEMPLATE_PARAMETERS) + EIGEN_STATIC_ASSERT((MaxColsAtCompileTime == Dynamic) || (MaxColsAtCompileTime >= 0), INVALID_MATRIX_TEMPLATE_PARAMETERS) + EIGEN_STATIC_ASSERT((MaxRowsAtCompileTime == RowsAtCompileTime || RowsAtCompileTime==Dynamic), INVALID_MATRIX_TEMPLATE_PARAMETERS) + EIGEN_STATIC_ASSERT((MaxColsAtCompileTime == ColsAtCompileTime || ColsAtCompileTime==Dynamic), INVALID_MATRIX_TEMPLATE_PARAMETERS) + EIGEN_STATIC_ASSERT(((Options & (DontAlign|RowMajor)) == Options), INVALID_MATRIX_TEMPLATE_PARAMETERS) + EIGEN_DEVICE_FUNC Base& base() { return *static_cast(this); } EIGEN_DEVICE_FUNC @@ -148,12 +175,10 @@ class PlainObjectBase : public internal::dense_xpr_base::type * provided to by-pass the creation of an evaluator of the expression, thus saving compilation efforts. * * See DenseCoeffsBase::coeff(Index) const for details. */ - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const Scalar& coeff(Index rowId, Index colId) const - { - if(Flags & RowMajorBit) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const Scalar& coeff(Index rowId, Index colId) const { + if (Flags & RowMajorBit) return m_storage.data()[colId + rowId * m_storage.cols()]; - else // column-major + else // column-major return m_storage.data()[rowId + colId * m_storage.rows()]; } @@ -171,12 +196,10 @@ class PlainObjectBase : public internal::dense_xpr_base::type * provided to by-pass the creation of an evaluator of the expression, thus saving compilation efforts. * * See DenseCoeffsBase::coeffRef(Index,Index) const for details. */ - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Scalar& coeffRef(Index rowId, Index colId) - { - if(Flags & RowMajorBit) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Scalar& coeffRef(Index rowId, Index colId) { + if (Flags & RowMajorBit) return m_storage.data()[colId + rowId * m_storage.cols()]; - else // column-major + else // column-major return m_storage.data()[rowId + colId * m_storage.rows()]; } @@ -184,28 +207,20 @@ class PlainObjectBase : public internal::dense_xpr_base::type * provided to by-pass the creation of an evaluator of the expression, thus saving compilation efforts. * * See DenseCoeffsBase::coeffRef(Index) const for details. */ - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) - { - return m_storage.data()[index]; - } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Scalar& coeffRef(Index index) { return m_storage.data()[index]; } /** This is the const version of coeffRef(Index,Index) which is thus synonym of coeff(Index,Index). * It is provided for convenience. */ - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const Scalar& coeffRef(Index rowId, Index colId) const - { - if(Flags & RowMajorBit) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const Scalar& coeffRef(Index rowId, Index colId) const { + if (Flags & RowMajorBit) return m_storage.data()[colId + rowId * m_storage.cols()]; - else // column-major + else // column-major return m_storage.data()[rowId + colId * m_storage.rows()]; } /** This is the const version of coeffRef(Index) which is thus synonym of coeff(Index). * It is provided for convenience. */ - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const Scalar& coeffRef(Index index) const - { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const Scalar& coeffRef(Index index) const { return m_storage.data()[index]; } @@ -267,15 +282,13 @@ class PlainObjectBase : public internal::dense_xpr_base::type * * \sa resize(Index) for vectors, resize(NoChange_t, Index), resize(Index, NoChange_t) */ - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE void resize(Index rows, Index cols) - { - eigen_assert( EIGEN_IMPLIES(RowsAtCompileTime!=Dynamic,rows==RowsAtCompileTime) - && EIGEN_IMPLIES(ColsAtCompileTime!=Dynamic,cols==ColsAtCompileTime) - && EIGEN_IMPLIES(RowsAtCompileTime==Dynamic && MaxRowsAtCompileTime!=Dynamic,rows<=MaxRowsAtCompileTime) - && EIGEN_IMPLIES(ColsAtCompileTime==Dynamic && MaxColsAtCompileTime!=Dynamic,cols<=MaxColsAtCompileTime) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index rows, Index cols) { + eigen_assert(internal::check_implication(RowsAtCompileTime!=Dynamic, rows==RowsAtCompileTime) + && internal::check_implication(ColsAtCompileTime!=Dynamic, cols==ColsAtCompileTime) + && internal::check_implication(RowsAtCompileTime==Dynamic && MaxRowsAtCompileTime!=Dynamic, rows<=MaxRowsAtCompileTime) + && internal::check_implication(ColsAtCompileTime==Dynamic && MaxColsAtCompileTime!=Dynamic, cols<=MaxColsAtCompileTime) && rows>=0 && cols>=0 && "Invalid sizes when resizing a matrix or array."); - internal::check_rows_cols_for_overflow::run(rows, cols); + internal::check_rows_cols_for_overflow::run(rows, cols); #ifdef EIGEN_INITIALIZE_COEFFS Index size = rows*cols; bool size_changed = size != this->size(); @@ -297,12 +310,13 @@ class PlainObjectBase : public internal::dense_xpr_base::type * * \sa resize(Index,Index), resize(NoChange_t, Index), resize(Index, NoChange_t) */ - EIGEN_DEVICE_FUNC - inline void resize(Index size) - { - EIGEN_STATIC_ASSERT_VECTOR_ONLY(PlainObjectBase) - eigen_assert(((SizeAtCompileTime == Dynamic && (MaxSizeAtCompileTime==Dynamic || size<=MaxSizeAtCompileTime)) || SizeAtCompileTime == size) && size>=0); - #ifdef EIGEN_INITIALIZE_COEFFS + EIGEN_DEVICE_FUNC inline constexpr void resize(Index size) { + EIGEN_STATIC_ASSERT_VECTOR_ONLY(PlainObjectBase) + eigen_assert( + ((SizeAtCompileTime == Dynamic && (MaxSizeAtCompileTime == Dynamic || size <= MaxSizeAtCompileTime)) || + SizeAtCompileTime == size) && + size >= 0); +#ifdef EIGEN_INITIALIZE_COEFFS bool size_changed = size != this->size(); #endif if(RowsAtCompileTime == 1) @@ -322,11 +336,7 @@ class PlainObjectBase : public internal::dense_xpr_base::type * * \sa resize(Index,Index) */ - EIGEN_DEVICE_FUNC - inline void resize(NoChange_t, Index cols) - { - resize(rows(), cols); - } + EIGEN_DEVICE_FUNC inline constexpr void resize(NoChange_t, Index cols) { resize(rows(), cols); } /** Resizes the matrix, changing only the number of rows. For the parameter of type NoChange_t, just pass the special value \c NoChange * as in the example below. @@ -336,11 +346,7 @@ class PlainObjectBase : public internal::dense_xpr_base::type * * \sa resize(Index,Index) */ - EIGEN_DEVICE_FUNC - inline void resize(Index rows, NoChange_t) - { - resize(rows, cols()); - } + EIGEN_DEVICE_FUNC inline constexpr void resize(Index rows, NoChange_t) { resize(rows, cols()); } /** Resizes \c *this to have the same dimensions as \a other. * Takes care of doing all the checking that's needed. @@ -354,7 +360,7 @@ class PlainObjectBase : public internal::dense_xpr_base::type EIGEN_STRONG_INLINE void resizeLike(const EigenBase& _other) { const OtherDerived& other = _other.derived(); - internal::check_rows_cols_for_overflow::run(other.rows(), other.cols()); + internal::check_rows_cols_for_overflow::run(other.rows(), other.cols()); const Index othersize = other.rows()*other.cols(); if(RowsAtCompileTime == 1) { @@ -475,7 +481,6 @@ class PlainObjectBase : public internal::dense_xpr_base::type EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PlainObjectBase() : m_storage() { -// _check_template_params(); // EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } @@ -486,11 +491,10 @@ class PlainObjectBase : public internal::dense_xpr_base::type explicit PlainObjectBase(internal::constructor_without_unaligned_array_assert) : m_storage(internal::constructor_without_unaligned_array_assert()) { -// _check_template_params(); EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED + // EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } #endif -#if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC PlainObjectBase(PlainObjectBase&& other) EIGEN_NOEXCEPT : m_storage( std::move(other.m_storage) ) @@ -500,11 +504,9 @@ class PlainObjectBase : public internal::dense_xpr_base::type EIGEN_DEVICE_FUNC PlainObjectBase& operator=(PlainObjectBase&& other) EIGEN_NOEXCEPT { - _check_template_params(); m_storage = std::move(other.m_storage); return *this; } -#endif /** Copy constructor */ EIGEN_DEVICE_FUNC @@ -514,17 +516,14 @@ class PlainObjectBase : public internal::dense_xpr_base::type EIGEN_STRONG_INLINE PlainObjectBase(Index size, Index rows, Index cols) : m_storage(size, rows, cols) { -// _check_template_params(); // EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } - #if EIGEN_HAS_CXX11 - /** \brief Construct a row of column vector with fixed size from an arbitrary number of coefficients. \cpp11 + /** \brief Construct a row of column vector with fixed size from an arbitrary number of coefficients. * * \only_for_vectors * * This constructor is for 1D array or vectors with more than 4 coefficients. - * There exists C++98 analogue constructors for fixed-size array/vector having 1, 2, 3, or 4 coefficients. * * \warning To construct a column (resp. row) vector of fixed length, the number of values passed to this * constructor must match the the fixed number of rows (resp. columns) of \c *this. @@ -534,7 +533,6 @@ class PlainObjectBase : public internal::dense_xpr_base::type PlainObjectBase(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) : m_storage() { - _check_template_params(); EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(PlainObjectBase, sizeof...(args) + 4); m_storage.data()[0] = a0; m_storage.data()[1] = a1; @@ -546,14 +544,11 @@ class PlainObjectBase : public internal::dense_xpr_base::type } /** \brief Constructs a Matrix or Array and initializes it by elements given by an initializer list of initializer - * lists \cpp11 + * lists */ - EIGEN_DEVICE_FUNC - explicit EIGEN_STRONG_INLINE PlainObjectBase(const std::initializer_list>& list) - : m_storage() - { - _check_template_params(); - + EIGEN_DEVICE_FUNC explicit constexpr EIGEN_STRONG_INLINE PlainObjectBase( + const std::initializer_list>& list) + : m_storage() { size_t list_size = 0; if (list.begin() != list.end()) { list_size = list.begin()->size(); @@ -563,7 +558,9 @@ class PlainObjectBase : public internal::dense_xpr_base::type if (ColsAtCompileTime == 1 && list.size() == 1) { eigen_assert(list_size == static_cast(RowsAtCompileTime) || RowsAtCompileTime == Dynamic); resize(list_size, ColsAtCompileTime); - std::copy(list.begin()->begin(), list.begin()->end(), m_storage.data()); + if (list.begin()->begin() != nullptr) { + std::copy(list.begin()->begin(), list.begin()->end(), m_storage.data()); + } } else { eigen_assert(list.size() == static_cast(RowsAtCompileTime) || RowsAtCompileTime == Dynamic); eigen_assert(list_size == static_cast(ColsAtCompileTime) || ColsAtCompileTime == Dynamic); @@ -581,7 +578,6 @@ class PlainObjectBase : public internal::dense_xpr_base::type } } } - #endif // end EIGEN_HAS_CXX11 /** \sa PlainObjectBase::operator=(const EigenBase&) */ template @@ -589,7 +585,6 @@ class PlainObjectBase : public internal::dense_xpr_base::type EIGEN_STRONG_INLINE PlainObjectBase(const DenseBase &other) : m_storage() { - _check_template_params(); resizeLike(other); _set_noalias(other); } @@ -600,7 +595,6 @@ class PlainObjectBase : public internal::dense_xpr_base::type EIGEN_STRONG_INLINE PlainObjectBase(const EigenBase &other) : m_storage() { - _check_template_params(); resizeLike(other); *this = other.derived(); } @@ -609,7 +603,6 @@ class PlainObjectBase : public internal::dense_xpr_base::type EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PlainObjectBase(const ReturnByValue& other) { - _check_template_params(); // FIXME this does not automatically transpose vectors if necessary resize(other.rows(), other.cols()); other.evalTo(this->derived()); @@ -640,7 +633,7 @@ class PlainObjectBase : public internal::dense_xpr_base::type * * \see class Map */ - //@{ + ///@{ static inline ConstMapType Map(const Scalar* data) { return ConstMapType(data); } static inline MapType Map(Scalar* data) @@ -704,7 +697,7 @@ class PlainObjectBase : public internal::dense_xpr_base::type template static inline typename StridedAlignedMapType >::type MapAligned(Scalar* data, Index rows, Index cols, const Stride& stride) { return typename StridedAlignedMapType >::type(data, rows, cols, stride); } - //@} + ///@} using Base::setConstant; EIGEN_DEVICE_FUNC Derived& setConstant(Index size, const Scalar& val); @@ -800,19 +793,17 @@ class PlainObjectBase : public internal::dense_xpr_base::type template EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE void _init2(Index rows, Index cols, typename internal::enable_if::type* = 0) + EIGEN_STRONG_INLINE void _init2(Index rows, Index cols, std::enable_if_t* = 0) { - const bool t0_is_integer_alike = internal::is_valid_index_type::value; - const bool t1_is_integer_alike = internal::is_valid_index_type::value; - EIGEN_STATIC_ASSERT(t0_is_integer_alike && - t1_is_integer_alike, - FLOATING_POINT_ARGUMENT_PASSED__INTEGER_WAS_EXPECTED) + EIGEN_STATIC_ASSERT(internal::is_valid_index_type::value && + internal::is_valid_index_type::value, + T0 AND T1 MUST BE INTEGER TYPES) resize(rows,cols); } template EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE void _init2(const T0& val0, const T1& val1, typename internal::enable_if::type* = 0) + EIGEN_STRONG_INLINE void _init2(const T0& val0, const T1& val1, std::enable_if_t* = 0) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(PlainObjectBase, 2) m_storage.data()[0] = Scalar(val0); @@ -822,10 +813,10 @@ class PlainObjectBase : public internal::dense_xpr_base::type template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _init2(const Index& val0, const Index& val1, - typename internal::enable_if< (!internal::is_same::value) - && (internal::is_same::value) - && (internal::is_same::value) - && Base::SizeAtCompileTime==2,T1>::type* = 0) + std::enable_if_t< (!internal::is_same::value) + && (internal::is_same::value) + && (internal::is_same::value) + && Base::SizeAtCompileTime==2,T1>* = 0) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(PlainObjectBase, 2) m_storage.data()[0] = Scalar(val0); @@ -836,8 +827,8 @@ class PlainObjectBase : public internal::dense_xpr_base::type // then the argument is meant to be the size of the object. template EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE void _init1(Index size, typename internal::enable_if< (Base::SizeAtCompileTime!=1 || !internal::is_convertible::value) - && ((!internal::is_same::XprKind,ArrayXpr>::value || Base::SizeAtCompileTime==Dynamic)),T>::type* = 0) + EIGEN_STRONG_INLINE void _init1(Index size, std::enable_if_t< (Base::SizeAtCompileTime!=1 || !internal::is_convertible::value) + && ((!internal::is_same::XprKind,ArrayXpr>::value || Base::SizeAtCompileTime==Dynamic)),T>* = 0) { // NOTE MSVC 2008 complains if we directly put bool(NumTraits::IsInteger) as the EIGEN_STATIC_ASSERT argument. const bool is_integer_alike = internal::is_valid_index_type::value; @@ -850,7 +841,7 @@ class PlainObjectBase : public internal::dense_xpr_base::type // We have a 1x1 matrix/array => the argument is interpreted as the value of the unique coefficient (case where scalar type can be implicitly converted) template EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE void _init1(const Scalar& val0, typename internal::enable_if::value,T>::type* = 0) + EIGEN_STRONG_INLINE void _init1(const Scalar& val0, std::enable_if_t::value,T>* = 0) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(PlainObjectBase, 1) m_storage.data()[0] = val0; @@ -860,10 +851,10 @@ class PlainObjectBase : public internal::dense_xpr_base::type template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _init1(const Index& val0, - typename internal::enable_if< (!internal::is_same::value) - && (internal::is_same::value) - && Base::SizeAtCompileTime==1 - && internal::is_convertible::value,T*>::type* = 0) + std::enable_if_t< (!internal::is_same::value) + && (internal::is_same::value) + && Base::SizeAtCompileTime==1 + && internal::is_convertible::value,T*>* = 0) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(PlainObjectBase, 1) m_storage.data()[0] = Scalar(val0); @@ -916,10 +907,10 @@ class PlainObjectBase : public internal::dense_xpr_base::type template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _init1(const Scalar& val0, - typename internal::enable_if< Base::SizeAtCompileTime!=Dynamic - && Base::SizeAtCompileTime!=1 - && internal::is_convertible::value - && internal::is_same::XprKind,ArrayXpr>::value,T>::type* = 0) + std::enable_if_t< Base::SizeAtCompileTime!=Dynamic + && Base::SizeAtCompileTime!=1 + && internal::is_convertible::value + && internal::is_same::XprKind,ArrayXpr>::value,T>* = 0) { Base::setConstant(val0); } @@ -928,12 +919,12 @@ class PlainObjectBase : public internal::dense_xpr_base::type template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _init1(const Index& val0, - typename internal::enable_if< (!internal::is_same::value) - && (internal::is_same::value) - && Base::SizeAtCompileTime!=Dynamic - && Base::SizeAtCompileTime!=1 - && internal::is_convertible::value - && internal::is_same::XprKind,ArrayXpr>::value,T*>::type* = 0) + std::enable_if_t< (!internal::is_same::value) + && (internal::is_same::value) + && Base::SizeAtCompileTime!=Dynamic + && Base::SizeAtCompileTime!=1 + && internal::is_convertible::value + && internal::is_same::XprKind,ArrayXpr>::value,T*>* = 0) { Base::setConstant(val0); } @@ -964,21 +955,6 @@ class PlainObjectBase : public internal::dense_xpr_base::type void swap(DenseBase const & other) { Base::swap(other.derived()); } - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE void _check_template_params() - { - EIGEN_STATIC_ASSERT((EIGEN_IMPLIES(MaxRowsAtCompileTime==1 && MaxColsAtCompileTime!=1, (int(Options)&RowMajor)==RowMajor) - && EIGEN_IMPLIES(MaxColsAtCompileTime==1 && MaxRowsAtCompileTime!=1, (int(Options)&RowMajor)==0) - && ((RowsAtCompileTime == Dynamic) || (RowsAtCompileTime >= 0)) - && ((ColsAtCompileTime == Dynamic) || (ColsAtCompileTime >= 0)) - && ((MaxRowsAtCompileTime == Dynamic) || (MaxRowsAtCompileTime >= 0)) - && ((MaxColsAtCompileTime == Dynamic) || (MaxColsAtCompileTime >= 0)) - && (MaxRowsAtCompileTime == RowsAtCompileTime || RowsAtCompileTime==Dynamic) - && (MaxColsAtCompileTime == ColsAtCompileTime || ColsAtCompileTime==Dynamic) - && (Options & (DontAlign|RowMajor)) == Options), - INVALID_MATRIX_TEMPLATE_PARAMETERS) - } - enum { IsPlainObjectBase = 1 }; #endif public: @@ -999,11 +975,7 @@ namespace internal { template struct conservative_resize_like_impl { - #if EIGEN_HAS_TYPE_TRAITS - static const bool IsRelocatable = std::is_trivially_copyable::value; - #else - static const bool IsRelocatable = !NumTraits::RequireInitialization; - #endif + static constexpr bool IsRelocatable = std::is_trivially_copyable::value; static void run(DenseBase& _this, Index rows, Index cols) { if (_this.rows() == rows && _this.cols() == cols) return; @@ -1013,7 +985,7 @@ struct conservative_resize_like_impl && (( Derived::IsRowMajor && _this.cols() == cols) || // row-major and we change only the number of rows (!Derived::IsRowMajor && _this.rows() == rows) )) // column-major and we change only the number of columns { - internal::check_rows_cols_for_overflow::run(rows, cols); + internal::check_rows_cols_for_overflow::run(rows, cols); _this.derived().m_storage.conservativeResize(rows*cols,rows,cols); } else diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Product.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Product.h index 70a6c106396..107b47e5119 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Product.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Product.h @@ -10,6 +10,9 @@ #ifndef EIGEN_PRODUCT_H #define EIGEN_PRODUCT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template class ProductImpl; @@ -19,8 +22,8 @@ namespace internal { template struct traits > { - typedef typename remove_all::type LhsCleaned; - typedef typename remove_all::type RhsCleaned; + typedef remove_all_t LhsCleaned; + typedef remove_all_t RhsCleaned; typedef traits LhsTraits; typedef traits RhsTraits; @@ -40,7 +43,7 @@ struct traits > MaxColsAtCompileTime = RhsTraits::MaxColsAtCompileTime, // FIXME: only needed by GeneralMatrixMatrixTriangular - InnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(LhsTraits::ColsAtCompileTime, RhsTraits::RowsAtCompileTime), + InnerSize = min_size_prefer_fixed(LhsTraits::ColsAtCompileTime, RhsTraits::RowsAtCompileTime), // The storage order is somewhat arbitrary here. The correct one will be determined through the evaluator. Flags = (MaxRowsAtCompileTime==1 && MaxColsAtCompileTime!=1) ? RowMajorBit @@ -58,8 +61,8 @@ struct traits > * * \brief Expression of the product of two arbitrary matrices or vectors * - * \tparam _Lhs the type of the left-hand side expression - * \tparam _Rhs the type of the right-hand side expression + * \tparam Lhs_ the type of the left-hand side expression + * \tparam Rhs_ the type of the right-hand side expression * * This class represents an expression of the product of two arbitrary matrices. * @@ -67,16 +70,16 @@ struct traits > * \tparam Option can be DefaultProduct, AliasFreeProduct, or LazyProduct * */ -template -class Product : public ProductImpl<_Lhs,_Rhs,Option, - typename internal::product_promote_storage_type::StorageKind, - typename internal::traits<_Rhs>::StorageKind, - internal::product_type<_Lhs,_Rhs>::ret>::ret> +template +class Product : public ProductImpl::StorageKind, + typename internal::traits::StorageKind, + internal::product_type::ret>::ret> { public: - typedef _Lhs Lhs; - typedef _Rhs Rhs; + typedef Lhs_ Lhs; + typedef Rhs_ Rhs; typedef typename ProductImpl< Lhs, Rhs, Option, @@ -87,8 +90,8 @@ class Product : public ProductImpl<_Lhs,_Rhs,Option, typedef typename internal::ref_selector::type LhsNested; typedef typename internal::ref_selector::type RhsNested; - typedef typename internal::remove_all::type LhsNestedCleaned; - typedef typename internal::remove_all::type RhsNestedCleaned; + typedef internal::remove_all_t LhsNestedCleaned; + typedef internal::remove_all_t RhsNestedCleaned; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Product(const Lhs& lhs, const Rhs& rhs) : m_lhs(lhs), m_rhs(rhs) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ProductEvaluators.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ProductEvaluators.h index 8cf294b287b..67b0434dada 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ProductEvaluators.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ProductEvaluators.h @@ -13,6 +13,9 @@ #ifndef EIGEN_PRODUCTEVALUATORS_H #define EIGEN_PRODUCTEVALUATORS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -107,14 +110,14 @@ struct product_evaluator, ProductTag, LhsShape, RhsSh explicit product_evaluator(const XprType& xpr) : m_result(xpr.rows(), xpr.cols()) { - ::new (static_cast(this)) Base(m_result); + internal::construct_at(this, m_result); // FIXME shall we handle nested_eval here?, // if so, then we must take care at removing the call to nested_eval in the specializations (e.g., in permutation_matrix_product, transposition_matrix_product, etc.) // typedef typename internal::nested_eval::type LhsNested; // typedef typename internal::nested_eval::type RhsNested; -// typedef typename internal::remove_all::type LhsNestedCleaned; -// typedef typename internal::remove_all::type RhsNestedCleaned; +// typedef internal::remove_all_t LhsNestedCleaned; +// typedef internal::remove_all_t RhsNestedCleaned; // // const LhsNested lhs(xpr.lhs()); // const RhsNested rhs(xpr.rhs()); @@ -134,7 +137,7 @@ struct product_evaluator, ProductTag, LhsShape, RhsSh // Dense = Product template< typename DstXprType, typename Lhs, typename Rhs, int Options, typename Scalar> struct Assignment, internal::assign_op, Dense2Dense, - typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct)>::type> + std::enable_if_t<(Options==DefaultProduct || Options==AliasFreeProduct)>> { typedef Product SrcXprType; static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -152,7 +155,7 @@ struct Assignment, internal::assign_op struct Assignment, internal::add_assign_op, Dense2Dense, - typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct)>::type> + std::enable_if_t<(Options==DefaultProduct || Options==AliasFreeProduct)>> { typedef Product SrcXprType; static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -167,7 +170,7 @@ struct Assignment, internal::add_assign_op< // Dense -= Product template< typename DstXprType, typename Lhs, typename Rhs, int Options, typename Scalar> struct Assignment, internal::sub_assign_op, Dense2Dense, - typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct)>::type> + std::enable_if_t<(Options==DefaultProduct || Options==AliasFreeProduct)>> { typedef Product SrcXprType; static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -296,7 +299,7 @@ void EIGEN_DEVICE_FUNC outer_product_selector_run(Dst& dst, const Lhs &lhs, cons template struct generic_product_impl { - template struct is_row_major : internal::conditional<(int(T::Flags)&RowMajorBit), internal::true_type, internal::false_type>::type {}; + template struct is_row_major : std::conditional_t<(int(T::Flags)&RowMajorBit), internal::true_type, internal::false_type> {}; typedef typename Product::Scalar Scalar; // TODO it would be nice to be able to exploit our *_assign_op functors for that purpose @@ -370,7 +373,7 @@ struct generic_product_impl typedef typename nested_eval::type RhsNested; typedef typename Product::Scalar Scalar; enum { Side = Lhs::IsVectorAtCompileTime ? OnTheLeft : OnTheRight }; - typedef typename internal::remove_all::type>::type MatrixType; + typedef internal::remove_all_t> MatrixType; template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) @@ -427,8 +430,8 @@ struct generic_product_impl // 3 - it makes this fallback consistent with the heavy GEMM routine. // 4 - it fully by-passes huge stack allocation attempts when multiplying huge fixed-size matrices. // (see https://stackoverflow.com/questions/54738495) - // For small fixed sizes matrices, howver, the gains are less obvious, it is sometimes x2 faster, but sometimes x3 slower, - // and the behavior depends also a lot on the compiler... This is why this re-writting strategy is currently + // For small fixed sizes matrices, however, the gains are less obvious, it is sometimes x2 faster, but sometimes x3 slower, + // and the behavior depends also a lot on the compiler... This is why this re-writing strategy is currently // enabled only when falling back from the main GEMM. template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -448,7 +451,7 @@ struct generic_product_impl blas_traits::extract(rhs).template conjugateIf(), func, actualAlpha, - typename conditional::type()); + std::conditional_t()); } protected: @@ -458,7 +461,7 @@ struct generic_product_impl void eval_dynamic_impl(Dst& dst, const LhsT& lhs, const RhsT& rhs, const Func &func, const Scalar& s /* == 1 */, false_type) { EIGEN_UNUSED_VARIABLE(s); - eigen_internal_assert(s==Scalar(1)); + eigen_internal_assert(numext::is_exactly_one(s)); call_restricted_packet_assignment_no_alias(dst, lhs.lazyProduct(rhs), func); } @@ -526,8 +529,8 @@ struct product_evaluator, ProductTag, DenseShape, typedef typename internal::nested_eval::type LhsNested; typedef typename internal::nested_eval::type RhsNested; - typedef typename internal::remove_all::type LhsNestedCleaned; - typedef typename internal::remove_all::type RhsNestedCleaned; + typedef internal::remove_all_t LhsNestedCleaned; + typedef internal::remove_all_t RhsNestedCleaned; typedef evaluator LhsEtorType; typedef evaluator RhsEtorType; @@ -535,7 +538,7 @@ struct product_evaluator, ProductTag, DenseShape, enum { RowsAtCompileTime = LhsNestedCleaned::RowsAtCompileTime, ColsAtCompileTime = RhsNestedCleaned::ColsAtCompileTime, - InnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(LhsNestedCleaned::ColsAtCompileTime, RhsNestedCleaned::RowsAtCompileTime), + InnerSize = min_size_prefer_fixed(LhsNestedCleaned::ColsAtCompileTime, RhsNestedCleaned::RowsAtCompileTime), MaxRowsAtCompileTime = LhsNestedCleaned::MaxRowsAtCompileTime, MaxColsAtCompileTime = RhsNestedCleaned::MaxColsAtCompileTime }; @@ -564,8 +567,8 @@ struct product_evaluator, ProductTag, DenseShape, RhsVecPacketSize = unpacket_traits::size, // Here, we don't care about alignment larger than the usable packet size. - LhsAlignment = EIGEN_PLAIN_ENUM_MIN(LhsEtorType::Alignment,LhsVecPacketSize*int(sizeof(typename LhsNestedCleaned::Scalar))), - RhsAlignment = EIGEN_PLAIN_ENUM_MIN(RhsEtorType::Alignment,RhsVecPacketSize*int(sizeof(typename RhsNestedCleaned::Scalar))), + LhsAlignment = plain_enum_min(LhsEtorType::Alignment, LhsVecPacketSize*int(sizeof(typename LhsNestedCleaned::Scalar))), + RhsAlignment = plain_enum_min(RhsEtorType::Alignment, RhsVecPacketSize*int(sizeof(typename RhsNestedCleaned::Scalar))), SameType = is_same::value, @@ -585,8 +588,8 @@ struct product_evaluator, ProductTag, DenseShape, LhsOuterStrideBytes = int(LhsNestedCleaned::OuterStrideAtCompileTime) * int(sizeof(typename LhsNestedCleaned::Scalar)), RhsOuterStrideBytes = int(RhsNestedCleaned::OuterStrideAtCompileTime) * int(sizeof(typename RhsNestedCleaned::Scalar)), - Alignment = bool(CanVectorizeLhs) ? (LhsOuterStrideBytes<=0 || (int(LhsOuterStrideBytes) % EIGEN_PLAIN_ENUM_MAX(1,LhsAlignment))!=0 ? 0 : LhsAlignment) - : bool(CanVectorizeRhs) ? (RhsOuterStrideBytes<=0 || (int(RhsOuterStrideBytes) % EIGEN_PLAIN_ENUM_MAX(1,RhsAlignment))!=0 ? 0 : RhsAlignment) + Alignment = bool(CanVectorizeLhs) ? (LhsOuterStrideBytes<=0 || (int(LhsOuterStrideBytes) % plain_enum_max(1, LhsAlignment))!=0 ? 0 : LhsAlignment) + : bool(CanVectorizeRhs) ? (RhsOuterStrideBytes<=0 || (int(RhsOuterStrideBytes) % plain_enum_max(1, RhsAlignment))!=0 ? 0 : RhsAlignment) : 0, /* CanVectorizeInner deserves special explanation. It does not affect the product flags. It is not used outside @@ -640,8 +643,8 @@ struct product_evaluator, ProductTag, DenseShape, } protected: - typename internal::add_const_on_value_type::type m_lhs; - typename internal::add_const_on_value_type::type m_rhs; + add_const_on_value_type_t m_lhs; + add_const_on_value_type_t m_rhs; LhsEtorType m_lhsImpl; RhsEtorType m_rhsImpl; @@ -836,22 +839,22 @@ struct diagonal_product_evaluator_base MatrixFlags = evaluator::Flags, DiagFlags = evaluator::Flags, - _StorageOrder = (Derived::MaxRowsAtCompileTime==1 && Derived::MaxColsAtCompileTime!=1) ? RowMajor + StorageOrder_ = (Derived::MaxRowsAtCompileTime==1 && Derived::MaxColsAtCompileTime!=1) ? RowMajor : (Derived::MaxColsAtCompileTime==1 && Derived::MaxRowsAtCompileTime!=1) ? ColMajor : MatrixFlags & RowMajorBit ? RowMajor : ColMajor, - _SameStorageOrder = _StorageOrder == (MatrixFlags & RowMajorBit ? RowMajor : ColMajor), + SameStorageOrder_ = StorageOrder_ == (MatrixFlags & RowMajorBit ? RowMajor : ColMajor), - _ScalarAccessOnDiag = !((int(_StorageOrder) == ColMajor && int(ProductOrder) == OnTheLeft) - ||(int(_StorageOrder) == RowMajor && int(ProductOrder) == OnTheRight)), - _SameTypes = is_same::value, + ScalarAccessOnDiag_ = !((int(StorageOrder_) == ColMajor && int(ProductOrder) == OnTheLeft) + ||(int(StorageOrder_) == RowMajor && int(ProductOrder) == OnTheRight)), + SameTypes_ = is_same::value, // FIXME currently we need same types, but in the future the next rule should be the one - //_Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) && ((!_PacketOnDiag) || (_SameTypes && bool(int(DiagFlags)&PacketAccessBit))), - _Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) - && _SameTypes - && (_SameStorageOrder || (MatrixFlags&LinearAccessBit)==LinearAccessBit) - && (_ScalarAccessOnDiag || (bool(int(DiagFlags)&PacketAccessBit))), - _LinearAccessMask = (MatrixType::RowsAtCompileTime==1 || MatrixType::ColsAtCompileTime==1) ? LinearAccessBit : 0, - Flags = ((HereditaryBits|_LinearAccessMask) & (unsigned int)(MatrixFlags)) | (_Vectorizable ? PacketAccessBit : 0), + //Vectorizable_ = bool(int(MatrixFlags)&PacketAccessBit) && ((!_PacketOnDiag) || (SameTypes_ && bool(int(DiagFlags)&PacketAccessBit))), + Vectorizable_ = bool(int(MatrixFlags)&PacketAccessBit) + && SameTypes_ + && (SameStorageOrder_ || (MatrixFlags&LinearAccessBit)==LinearAccessBit) + && (ScalarAccessOnDiag_ || (bool(int(DiagFlags)&PacketAccessBit))), + LinearAccessMask_ = (MatrixType::RowsAtCompileTime==1 || MatrixType::ColsAtCompileTime==1) ? LinearAccessBit : 0, + Flags = ((HereditaryBits|LinearAccessMask_) & (unsigned int)(MatrixFlags)) | (Vectorizable_ ? PacketAccessBit : 0), Alignment = evaluator::Alignment, AsScalarProduct = (DiagonalType::SizeAtCompileTime==1) @@ -887,7 +890,7 @@ struct diagonal_product_evaluator_base { enum { InnerSize = (MatrixType::Flags & RowMajorBit) ? MatrixType::ColsAtCompileTime : MatrixType::RowsAtCompileTime, - DiagonalPacketLoadMode = EIGEN_PLAIN_ENUM_MIN(LoadMode,((InnerSize%16) == 0) ? int(Aligned16) : int(evaluator::Alignment)) // FIXME hardcoded 16!! + DiagonalPacketLoadMode = plain_enum_min(LoadMode,((InnerSize%16) == 0) ? int(Aligned16) : int(evaluator::Alignment)) // FIXME hardcoded 16!! }; return internal::pmul(m_matImpl.template packet(row, col), m_diagImpl.template packet(id)); @@ -913,7 +916,7 @@ struct product_evaluator, ProductTag, DiagonalSha typedef typename Lhs::DiagonalVectorType DiagonalType; - enum { StorageOrder = Base::_StorageOrder }; + enum { StorageOrder = Base::StorageOrder_ }; EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr) : Base(xpr.rhs(), xpr.lhs().diagonal()) @@ -932,7 +935,7 @@ struct product_evaluator, ProductTag, DiagonalSha // FIXME: NVCC used to complain about the template keyword, but we have to check whether this is still the case. // See also similar calls below. return this->template packet_impl(row,col, row, - typename internal::conditional::type()); + std::conditional_t()); } template @@ -957,7 +960,7 @@ struct product_evaluator, ProductTag, DenseShape, typedef Product XprType; typedef typename XprType::PlainObject PlainObject; - enum { StorageOrder = Base::_StorageOrder }; + enum { StorageOrder = Base::StorageOrder_ }; EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr) : Base(xpr.lhs(), xpr.rhs().diagonal()) @@ -974,7 +977,7 @@ struct product_evaluator, ProductTag, DenseShape, EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return this->template packet_impl(row,col, col, - typename internal::conditional::type()); + std::conditional_t()); } template @@ -1001,7 +1004,7 @@ template struct permutation_matrix_product { typedef typename nested_eval::type MatrixType; - typedef typename remove_all::type MatrixTypeCleaned; + typedef remove_all_t MatrixTypeCleaned; template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Dest& dst, const PermutationType& perm, const ExpressionType& xpr) @@ -1109,7 +1112,7 @@ template::type MatrixType; - typedef typename remove_all::type MatrixTypeCleaned; + typedef remove_all_t MatrixTypeCleaned; template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Dest& dst, const TranspositionType& tr, const ExpressionType& xpr) @@ -1172,6 +1175,40 @@ struct generic_product_impl, MatrixShape, TranspositionsShap } }; +/*************************************************************************** +* skew symmetric products +* for now we just call the generic implementation +***************************************************************************/ +template +struct generic_product_impl +{ + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) + { + generic_product_impl::evalTo(dst, lhs, rhs); + } +}; + +template +struct generic_product_impl +{ + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) + { + generic_product_impl::evalTo(dst, lhs, rhs); + } +}; + +template +struct generic_product_impl +{ + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) + { + generic_product_impl::evalTo(dst, lhs, rhs); + } +}; + } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Random.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Random.h index dab2ac8e9e8..2e9784f7e07 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Random.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Random.h @@ -10,12 +10,14 @@ #ifndef EIGEN_RANDOM_H #define EIGEN_RANDOM_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { template struct scalar_random_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_random_op) inline const Scalar operator() () const { return random(); } }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Redux.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Redux.h index b6790d11050..9ccbf69f87a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Redux.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Redux.h @@ -11,6 +11,9 @@ #ifndef EIGEN_REDUX_H #define EIGEN_REDUX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -42,9 +45,10 @@ struct redux_traits }; enum { + MayLinearize = (int(Evaluator::Flags) & LinearAccessBit), MightVectorize = (int(Evaluator::Flags)&ActualPacketAccessBit) && (functor_traits::PacketAccess), - MayLinearVectorize = bool(MightVectorize) && (int(Evaluator::Flags)&LinearAccessBit), + MayLinearVectorize = bool(MightVectorize) && bool(MayLinearize), MaySliceVectorize = bool(MightVectorize) && (int(SliceVectorizedWork)==Dynamic || int(SliceVectorizedWork)>=3) }; @@ -52,6 +56,7 @@ struct redux_traits enum { Traversal = int(MayLinearVectorize) ? int(LinearVectorizedTraversal) : int(MaySliceVectorize) ? int(SliceVectorizedTraversal) + : int(MayLinearize) ? int(LinearTraversal) : int(DefaultTraversal) }; @@ -95,12 +100,10 @@ struct redux_traits /*** no vectorization ***/ -template +template struct redux_novec_unroller { - enum { - HalfLength = Length/2 - }; + static constexpr Index HalfLength = Length/2; typedef typename Evaluator::Scalar Scalar; @@ -112,13 +115,11 @@ struct redux_novec_unroller } }; -template +template struct redux_novec_unroller { - enum { - outer = Start / Evaluator::InnerSizeAtCompileTime, - inner = Start % Evaluator::InnerSizeAtCompileTime - }; + static constexpr Index outer = Start / Evaluator::InnerSizeAtCompileTime; + static constexpr Index inner = Start % Evaluator::InnerSizeAtCompileTime; typedef typename Evaluator::Scalar Scalar; @@ -132,7 +133,7 @@ struct redux_novec_unroller // This is actually dead code and will never be called. It is required // to prevent false warnings regarding failed inlining though // for 0 length run() will never be called at all. -template +template struct redux_novec_unroller { typedef typename Evaluator::Scalar Scalar; @@ -140,19 +141,54 @@ struct redux_novec_unroller static EIGEN_STRONG_INLINE Scalar run(const Evaluator&, const Func&) { return Scalar(); } }; +template +struct redux_novec_linear_unroller +{ + static constexpr Index HalfLength = Length/2; + + typedef typename Evaluator::Scalar Scalar; + + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Evaluator &eval, const Func& func) + { + return func(redux_novec_linear_unroller::run(eval,func), + redux_novec_linear_unroller::run(eval,func)); + } +}; + +template +struct redux_novec_linear_unroller +{ + typedef typename Evaluator::Scalar Scalar; + + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Evaluator &eval, const Func&) + { + return eval.coeff(Start); + } +}; + +// This is actually dead code and will never be called. It is required +// to prevent false warnings regarding failed inlining though +// for 0 length run() will never be called at all. +template +struct redux_novec_linear_unroller +{ + typedef typename Evaluator::Scalar Scalar; + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Evaluator&, const Func&) { return Scalar(); } +}; + /*** vectorization ***/ -template +template struct redux_vec_unroller { template EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator &eval, const Func& func) { - enum { - PacketSize = unpacket_traits::size, - HalfLength = Length/2 - }; + constexpr Index HalfLength = Length/2; return func.packetOp( redux_vec_unroller::template run(eval,func), @@ -160,24 +196,52 @@ struct redux_vec_unroller } }; -template +template struct redux_vec_unroller { template EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator &eval, const Func&) { - enum { - PacketSize = unpacket_traits::size, - index = Start * PacketSize, - outer = index / int(Evaluator::InnerSizeAtCompileTime), - inner = index % int(Evaluator::InnerSizeAtCompileTime), - alignment = Evaluator::Alignment - }; + constexpr Index PacketSize = unpacket_traits::size; + constexpr Index index = Start * PacketSize; + constexpr Index outer = index / int(Evaluator::InnerSizeAtCompileTime); + constexpr Index inner = index % int(Evaluator::InnerSizeAtCompileTime); + constexpr int alignment = Evaluator::Alignment; + return eval.template packetByOuterInner(outer, inner); } }; +template +struct redux_vec_linear_unroller +{ + template + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE PacketType run(const Evaluator &eval, const Func& func) + { + constexpr Index HalfLength = Length/2; + + return func.packetOp( + redux_vec_linear_unroller::template run(eval,func), + redux_vec_linear_unroller::template run(eval,func) ); + } +}; + +template +struct redux_vec_linear_unroller +{ + template + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE PacketType run(const Evaluator &eval, const Func&) + { + constexpr Index PacketSize = unpacket_traits::size; + constexpr Index index = (Start * PacketSize); + constexpr int alignment = Evaluator::Alignment; + return eval.template packet(index); + } +}; + /*************************************************************************** * Part 3 : implementation of all cases ***************************************************************************/ @@ -198,8 +262,7 @@ struct redux_impl Scalar run(const Evaluator &eval, const Func& func, const XprType& xpr) { eigen_assert(xpr.rows()>0 && xpr.cols()>0 && "you are using an empty matrix"); - Scalar res; - res = eval.coeffByOuterInner(0, 0); + Scalar res = eval.coeffByOuterInner(0, 0); for(Index i = 1; i < xpr.innerSize(); ++i) res = func(res, eval.coeffByOuterInner(0, i)); for(Index i = 1; i < xpr.outerSize(); ++i) @@ -209,6 +272,23 @@ struct redux_impl } }; +template +struct redux_impl +{ + typedef typename Evaluator::Scalar Scalar; + + template + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE + Scalar run(const Evaluator &eval, const Func& func, const XprType& xpr) + { + eigen_assert(xpr.size()>0 && "you are using an empty matrix"); + Scalar res = eval.coeff(0); + for(Index k = 1; k < xpr.size(); ++k) + res = func(res, eval.coeff(k)); + return res; + } +}; + template struct redux_impl : redux_novec_unroller @@ -223,6 +303,20 @@ struct redux_impl } }; +template +struct redux_impl + : redux_novec_linear_unroller +{ + typedef redux_novec_linear_unroller Base; + typedef typename Evaluator::Scalar Scalar; + template + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE + Scalar run(const Evaluator &eval, const Func& func, const XprType& /*xpr*/) + { + return Base::run(eval,func); + } +}; + template struct redux_impl { @@ -234,12 +328,10 @@ struct redux_impl { const Index size = xpr.size(); - const Index packetSize = redux_traits::PacketSize; - const int packetAlignment = unpacket_traits::alignment; - enum { - alignment0 = (bool(Evaluator::Flags & DirectAccessBit) && bool(packet_traits::AlignedOnScalar)) ? int(packetAlignment) : int(Unaligned), - alignment = EIGEN_PLAIN_ENUM_MAX(alignment0, Evaluator::Alignment) - }; + constexpr Index packetSize = redux_traits::PacketSize; + constexpr int packetAlignment = unpacket_traits::alignment; + constexpr int alignment0 = (bool(Evaluator::Flags & DirectAccessBit) && bool(packet_traits::AlignedOnScalar)) ? int(packetAlignment) : int(Unaligned); + constexpr int alignment = plain_enum_max(alignment0, Evaluator::Alignment); const Index alignedStart = internal::first_default_aligned(xpr); const Index alignedSize2 = ((size-alignedStart)/(2*packetSize))*(2*packetSize); const Index alignedSize = ((size-alignedStart)/(packetSize))*(packetSize); @@ -293,11 +385,9 @@ struct redux_impl EIGEN_DEVICE_FUNC static Scalar run(const Evaluator &eval, const Func& func, const XprType& xpr) { eigen_assert(xpr.rows()>0 && xpr.cols()>0 && "you are using an empty matrix"); + constexpr Index packetSize = redux_traits::PacketSize; const Index innerSize = xpr.innerSize(); const Index outerSize = xpr.outerSize(); - enum { - packetSize = redux_traits::PacketSize - }; const Index packetedInnerSize = ((innerSize)/packetSize)*packetSize; Scalar res; if(packetedInnerSize) @@ -328,11 +418,9 @@ struct redux_impl typedef typename Evaluator::Scalar Scalar; typedef typename redux_traits::PacketType PacketType; - enum { - PacketSize = redux_traits::PacketSize, - Size = Evaluator::SizeAtCompileTime, - VectorizedSize = (int(Size) / int(PacketSize)) * int(PacketSize) - }; + static constexpr Index PacketSize = redux_traits::PacketSize; + static constexpr Index Size = Evaluator::SizeAtCompileTime; + static constexpr Index VectorizedSize = (int(Size) / int(PacketSize)) * int(PacketSize); template EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE @@ -341,24 +429,24 @@ struct redux_impl EIGEN_ONLY_USED_FOR_DEBUG(xpr) eigen_assert(xpr.rows()>0 && xpr.cols()>0 && "you are using an empty matrix"); if (VectorizedSize > 0) { - Scalar res = func.predux(redux_vec_unroller::template run(eval,func)); + Scalar res = func.predux(redux_vec_linear_unroller::template run(eval,func)); if (VectorizedSize != Size) - res = func(res,redux_novec_unroller::run(eval,func)); + res = func(res,redux_novec_linear_unroller::run(eval,func)); return res; } else { - return redux_novec_unroller::run(eval,func); + return redux_novec_linear_unroller::run(eval,func); } } }; // evaluator adaptor -template -class redux_evaluator : public internal::evaluator<_XprType> +template +class redux_evaluator : public internal::evaluator { - typedef internal::evaluator<_XprType> Base; + typedef internal::evaluator Base; public: - typedef _XprType XprType; + typedef XprType_ XprType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit redux_evaluator(const XprType &xpr) : Base(xpr) {} diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Ref.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Ref.h index c2a37eadbb6..df43c05e433 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Ref.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Ref.h @@ -10,20 +10,23 @@ #ifndef EIGEN_REF_H #define EIGEN_REF_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template -struct traits > - : public traits > +template +struct traits > + : public traits > { - typedef _PlainObjectType PlainObjectType; - typedef _StrideType StrideType; + typedef PlainObjectType_ PlainObjectType; + typedef StrideType_ StrideType; enum { - Options = _Options, - Flags = traits >::Flags | NestByRefBit, - Alignment = traits >::Alignment + Options = Options_, + Flags = traits >::Flags | NestByRefBit, + Alignment = traits >::Alignment }; template struct match { @@ -46,7 +49,7 @@ struct traits > ScalarTypeMatch = internal::is_same::value, MatchAtCompileTime = HasDirectAccess && StorageOrderMatch && InnerStrideMatch && OuterStrideMatch && AlignmentMatch && ScalarTypeMatch }; - typedef typename internal::conditional::type type; + typedef std::conditional_t type; }; }; @@ -197,8 +200,8 @@ template class RefBase return false; } - ::new (static_cast(this)) Base(expr.data(), rows, cols); - ::new (&m_stride) StrideBase( + internal::construct_at(this, expr.data(), rows, cols); + internal::construct_at(&m_stride, (StrideType::OuterStrideAtCompileTime == 0) ? 0 : outer_stride, (StrideType::InnerStrideAtCompileTime == 0) ? 0 : inner_stride ); return true; @@ -285,7 +288,7 @@ template class Ref typedef internal::traits Traits; template EIGEN_DEVICE_FUNC inline Ref(const PlainObjectBase& expr, - typename internal::enable_if::MatchAtCompileTime),Derived>::type* = 0); + std::enable_if_t::MatchAtCompileTime),Derived>* = 0); public: typedef RefBase Base; @@ -295,17 +298,17 @@ template class Ref #ifndef EIGEN_PARSED_BY_DOXYGEN template EIGEN_DEVICE_FUNC inline Ref(PlainObjectBase& expr, - typename internal::enable_if::MatchAtCompileTime),Derived>::type* = 0) + std::enable_if_t::MatchAtCompileTime),Derived>* = 0) { EIGEN_STATIC_ASSERT(bool(Traits::template match::MatchAtCompileTime), STORAGE_LAYOUT_DOES_NOT_MATCH); - // Construction must pass since we will not create temprary storage in the non-const case. + // Construction must pass since we will not create temporary storage in the non-const case. const bool success = Base::construct(expr.derived()); EIGEN_UNUSED_VARIABLE(success) eigen_assert(success); } template EIGEN_DEVICE_FUNC inline Ref(const DenseBase& expr, - typename internal::enable_if::MatchAtCompileTime),Derived>::type* = 0) + std::enable_if_t::MatchAtCompileTime),Derived>* = 0) #else /** Implicit constructor from any dense expression */ template @@ -330,6 +333,16 @@ template class Ref< : public RefBase > { typedef internal::traits Traits; + + static constexpr bool may_map_m_object_successfully = + (static_cast(StrideType::InnerStrideAtCompileTime) == 0 || + static_cast(StrideType::InnerStrideAtCompileTime) == 1 || + static_cast(StrideType::InnerStrideAtCompileTime) == Dynamic) && + (TPlainObjectType::IsVectorAtCompileTime || + static_cast(StrideType::OuterStrideAtCompileTime) == 0 || + static_cast(StrideType::OuterStrideAtCompileTime) == Dynamic || + static_cast(StrideType::OuterStrideAtCompileTime) == static_cast(TPlainObjectType::InnerSizeAtCompileTime) || + static_cast(TPlainObjectType::InnerSizeAtCompileTime) == Dynamic); public: typedef RefBase Base; @@ -337,11 +350,13 @@ template class Ref< template EIGEN_DEVICE_FUNC inline Ref(const DenseBase& expr, - typename internal::enable_if::ScalarTypeMatch),Derived>::type* = 0) + std::enable_if_t::ScalarTypeMatch),Derived>* = 0) { // std::cout << match_helper::HasDirectAccess << "," << match_helper::OuterStrideMatch << "," << match_helper::InnerStrideMatch << "\n"; // std::cout << int(StrideType::OuterStrideAtCompileTime) << " - " << int(Derived::OuterStrideAtCompileTime) << "\n"; // std::cout << int(StrideType::InnerStrideAtCompileTime) << " - " << int(Derived::InnerStrideAtCompileTime) << "\n"; + EIGEN_STATIC_ASSERT(Traits::template match::type::value || may_map_m_object_successfully, + STORAGE_LAYOUT_DOES_NOT_MATCH); construct(expr.derived(), typename Traits::template match::type()); } @@ -349,8 +364,19 @@ template class Ref< // copy constructor shall not copy the m_object, to avoid unnecessary malloc and copy } + EIGEN_DEVICE_FUNC inline Ref(Ref&& other) { + if (other.data() == other.m_object.data()) { + m_object = std::move(other.m_object); + Base::construct(m_object); + } + else + Base::construct(other); + } + template EIGEN_DEVICE_FUNC inline Ref(const RefBase& other) { + EIGEN_STATIC_ASSERT(Traits::template match::type::value || may_map_m_object_successfully, + STORAGE_LAYOUT_DOES_NOT_MATCH); construct(other.derived(), typename Traits::template match::type()); } @@ -369,7 +395,9 @@ template class Ref< EIGEN_DEVICE_FUNC void construct(const Expression& expr, internal::false_type) { internal::call_assignment_no_alias(m_object,expr,internal::assign_op()); - Base::construct(m_object); + const bool success = Base::construct(m_object); + EIGEN_ONLY_USED_FOR_DEBUG(success) + eigen_assert(success); } protected: diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Replicate.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Replicate.h index ab5be7e64bc..e06eaf2d929 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Replicate.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Replicate.h @@ -10,6 +10,9 @@ #ifndef EIGEN_REPLICATE_H #define EIGEN_REPLICATE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -21,7 +24,7 @@ struct traits > typedef typename traits::StorageKind StorageKind; typedef typename traits::XprKind XprKind; typedef typename ref_selector::type MatrixTypeNested; - typedef typename remove_reference::type _MatrixTypeNested; + typedef std::remove_reference_t MatrixTypeNested_; enum { RowsAtCompileTime = RowFactor==Dynamic || int(MatrixType::RowsAtCompileTime)==Dynamic ? Dynamic @@ -62,19 +65,19 @@ template class Replicate : public internal::dense_xpr_base< Replicate >::type { typedef typename internal::traits::MatrixTypeNested MatrixTypeNested; - typedef typename internal::traits::_MatrixTypeNested _MatrixTypeNested; + typedef typename internal::traits::MatrixTypeNested_ MatrixTypeNested_; public: typedef typename internal::dense_xpr_base::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Replicate) - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; template EIGEN_DEVICE_FUNC inline explicit Replicate(const OriginalMatrixType& matrix) : m_matrix(matrix), m_rowFactor(RowFactor), m_colFactor(ColFactor) { - EIGEN_STATIC_ASSERT((internal::is_same::type,OriginalMatrixType>::value), + EIGEN_STATIC_ASSERT((internal::is_same,OriginalMatrixType>::value), THE_MATRIX_OR_EXPRESSION_THAT_YOU_PASSED_DOES_NOT_HAVE_THE_EXPECTED_TYPE) eigen_assert(RowFactor!=Dynamic && ColFactor!=Dynamic); } @@ -84,7 +87,7 @@ template class Replicate inline Replicate(const OriginalMatrixType& matrix, Index rowFactor, Index colFactor) : m_matrix(matrix), m_rowFactor(rowFactor), m_colFactor(colFactor) { - EIGEN_STATIC_ASSERT((internal::is_same::type,OriginalMatrixType>::value), + EIGEN_STATIC_ASSERT((internal::is_same,OriginalMatrixType>::value), THE_MATRIX_OR_EXPRESSION_THAT_YOU_PASSED_DOES_NOT_HAVE_THE_EXPECTED_TYPE) } @@ -94,7 +97,7 @@ template class Replicate inline Index cols() const { return m_matrix.cols() * m_colFactor.value(); } EIGEN_DEVICE_FUNC - const _MatrixTypeNested& nestedExpression() const + const MatrixTypeNested_& nestedExpression() const { return m_matrix; } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Reshaped.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Reshaped.h index 52de73b6fc3..c118469dd01 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Reshaped.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Reshaped.h @@ -11,6 +11,9 @@ #ifndef EIGEN_RESHAPED_H #define EIGEN_RESHAPED_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class Reshaped @@ -27,10 +30,9 @@ namespace Eigen { * It is the return type of DenseBase::reshaped(NRowsType,NColsType) and * most of the time this is the only way it is used. * - * However, in C++98, if you want to directly maniputate reshaped expressions, - * for instance if you want to write a function returning such an expression, you - * will need to use this class. In C++11, it is advised to use the \em auto - * keyword for such use cases. + * If you want to directly manipulate reshaped expressions, + * for instance if you want to write a function returning such an expression, + * it is advised to use the \em auto keyword for such use cases. * * Here is an example illustrating the dynamic case: * \include class_Reshaped.cpp @@ -156,7 +158,7 @@ class ReshapedImpl_dense EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ReshapedImpl_dense) typedef typename internal::ref_selector::non_const_type MatrixTypeNested; - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; class InnerIterator; @@ -186,12 +188,12 @@ class ReshapedImpl_dense /** \returns the nested expression */ EIGEN_DEVICE_FUNC - const typename internal::remove_all::type& + const internal::remove_all_t& nestedExpression() const { return m_xpr; } /** \returns the nested expression */ EIGEN_DEVICE_FUNC - typename internal::remove_reference::type& + std::remove_reference_t& nestedExpression() { return m_xpr; } protected: @@ -231,7 +233,7 @@ class ReshapedImpl_dense {} EIGEN_DEVICE_FUNC - const typename internal::remove_all::type& nestedExpression() const + const internal::remove_all_t& nestedExpression() const { return m_xpr; } @@ -250,7 +252,7 @@ class ReshapedImpl_dense EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const { - return ((Flags&RowMajorBit)==RowMajorBit) ? this->cols() : this->rows(); + return (((Flags&RowMajorBit)==RowMajorBit) ? this->cols() : this->rows()) * m_xpr.innerStride(); } protected: @@ -324,7 +326,7 @@ struct reshaped_evaluator RowCol; - inline RowCol index_remap(Index rowId, Index colId) const + EIGEN_DEVICE_FUNC inline RowCol index_remap(Index rowId, Index colId) const { if(Order==ColMajor) { @@ -443,7 +445,7 @@ struct reshaped_evaluator(xpr) { // TODO: for the 3.4 release, this should be turned to an internal assertion, but let's keep it as is for the beta lifetime - eigen_assert(((internal::UIntPtr(xpr.data()) % EIGEN_PLAIN_ENUM_MAX(1,evaluator::Alignment)) == 0) && "data is not aligned"); + eigen_assert(((std::uintptr_t(xpr.data()) % plain_enum_max(1, evaluator::Alignment)) == 0) && "data is not aligned"); } }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ReturnByValue.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ReturnByValue.h index 4dad13ea118..c71e5f5d714 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ReturnByValue.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/ReturnByValue.h @@ -11,6 +11,9 @@ #ifndef EIGEN_RETURNBYVALUE_H #define EIGEN_RETURNBYVALUE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -104,7 +107,7 @@ struct evaluator > EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : m_result(xpr.rows(), xpr.cols()) { - ::new (static_cast(this)) Base(m_result); + internal::construct_at(this, m_result); xpr.evalTo(m_result); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Reverse.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Reverse.h index 28cdd76acaa..78ac4b1c2e0 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Reverse.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Reverse.h @@ -12,6 +12,9 @@ #ifndef EIGEN_REVERSE_H #define EIGEN_REVERSE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -24,13 +27,13 @@ struct traits > typedef typename traits::StorageKind StorageKind; typedef typename traits::XprKind XprKind; typedef typename ref_selector::type MatrixTypeNested; - typedef typename remove_reference::type _MatrixTypeNested; + typedef std::remove_reference_t MatrixTypeNested_; enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, - Flags = _MatrixTypeNested::Flags & (RowMajorBit | LvalueBit) + Flags = MatrixTypeNested_::Flags & (RowMajorBit | LvalueBit) }; }; @@ -67,7 +70,7 @@ template class Reverse typedef typename internal::dense_xpr_base::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Reverse) - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; using Base::IsRowMajor; protected: @@ -99,7 +102,7 @@ template class Reverse return -m_matrix.innerStride(); } - EIGEN_DEVICE_FUNC const typename internal::remove_all::type& + EIGEN_DEVICE_FUNC const internal::remove_all_t& nestedExpression() const { return m_matrix; @@ -173,10 +176,10 @@ struct vectorwise_reverse_inplace_impl template static void run(ExpressionType &xpr) { - const int HalfAtCompileTime = ExpressionType::RowsAtCompileTime==Dynamic?Dynamic:ExpressionType::RowsAtCompileTime/2; + constexpr Index HalfAtCompileTime = ExpressionType::RowsAtCompileTime==Dynamic?Dynamic:ExpressionType::RowsAtCompileTime/2; Index half = xpr.rows()/2; - xpr.topRows(fix(half)) - .swap(xpr.bottomRows(fix(half)).colwise().reverse()); + xpr.template topRows(half) + .swap(xpr.template bottomRows(half).colwise().reverse()); } }; @@ -186,10 +189,10 @@ struct vectorwise_reverse_inplace_impl template static void run(ExpressionType &xpr) { - const int HalfAtCompileTime = ExpressionType::ColsAtCompileTime==Dynamic?Dynamic:ExpressionType::ColsAtCompileTime/2; + constexpr Index HalfAtCompileTime = ExpressionType::ColsAtCompileTime==Dynamic?Dynamic:ExpressionType::ColsAtCompileTime/2; Index half = xpr.cols()/2; - xpr.leftCols(fix(half)) - .swap(xpr.rightCols(fix(half)).rowwise().reverse()); + xpr.template leftCols(half) + .swap(xpr.template rightCols(half).rowwise().reverse()); } }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Select.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Select.h index 7c86bf87c17..9180a5cd0f0 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Select.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Select.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SELECT_H #define EIGEN_SELECT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class Select @@ -17,9 +20,9 @@ namespace Eigen { * * \brief Expression of a coefficient wise version of the C++ ternary operator ?: * - * \param ConditionMatrixType the type of the \em condition expression which must be a boolean matrix - * \param ThenMatrixType the type of the \em then expression - * \param ElseMatrixType the type of the \em else expression + * \tparam ConditionMatrixType the type of the \em condition expression which must be a boolean matrix + * \tparam ThenMatrixType the type of the \em then expression + * \tparam ElseMatrixType the type of the \em else expression * * This class represents an expression of a coefficient wise version of the C++ ternary operator ?:. * It is the return type of DenseBase::select() and most of the time this is the only way it is used. @@ -111,52 +114,77 @@ class Select : public internal::dense_xpr_base< Select -template -inline EIGEN_DEVICE_FUNC const Select + * if \c *this(i,j) != Scalar(0), and \a elseMatrix(i,j) otherwise. + * + * Example: \include MatrixBase_select.cpp + * Output: \verbinclude MatrixBase_select.out + * + * \sa DenseBase::bitwiseSelect(const DenseBase&, const DenseBase&) + */ +template +template +inline EIGEN_DEVICE_FUNC CwiseTernaryOp< + internal::scalar_boolean_select_op::Scalar, + typename DenseBase::Scalar, + typename DenseBase::Scalar>, + ThenDerived, ElseDerived, Derived> DenseBase::select(const DenseBase& thenMatrix, - const DenseBase& elseMatrix) const -{ - return Select(derived(), thenMatrix.derived(), elseMatrix.derived()); + const DenseBase& elseMatrix) const { + using Op = internal::scalar_boolean_select_op< + typename DenseBase::Scalar, + typename DenseBase::Scalar, Scalar>; + return CwiseTernaryOp( + thenMatrix.derived(), elseMatrix.derived(), derived(), Op()); } - /** Version of DenseBase::select(const DenseBase&, const DenseBase&) with - * the \em else expression being a scalar value. - * - * \sa DenseBase::select(const DenseBase&, const DenseBase&) const, class Select - */ -template -template -inline EIGEN_DEVICE_FUNC const Select -DenseBase::select(const DenseBase& thenMatrix, - const typename ThenDerived::Scalar& elseScalar) const -{ - return Select( - derived(), thenMatrix.derived(), ThenDerived::Constant(rows(),cols(),elseScalar)); + * the \em else expression being a scalar value. + * + * \sa DenseBase::booleanSelect(const DenseBase&, const DenseBase&) const, class Select + */ +template +template +inline EIGEN_DEVICE_FUNC CwiseTernaryOp< + internal::scalar_boolean_select_op::Scalar, + typename DenseBase::Scalar, + typename DenseBase::Scalar>, + ThenDerived, typename DenseBase::ConstantReturnType, Derived> +DenseBase::select( + const DenseBase& thenMatrix, + const typename DenseBase::Scalar& elseScalar) const { + using ElseConstantType = + typename DenseBase::ConstantReturnType; + using Op = internal::scalar_boolean_select_op< + typename DenseBase::Scalar, + typename DenseBase::Scalar, Scalar>; + return CwiseTernaryOp( + thenMatrix.derived(), ElseConstantType(rows(), cols(), elseScalar), + derived(), Op()); } - /** Version of DenseBase::select(const DenseBase&, const DenseBase&) with - * the \em then expression being a scalar value. - * - * \sa DenseBase::select(const DenseBase&, const DenseBase&) const, class Select - */ -template -template -inline EIGEN_DEVICE_FUNC const Select -DenseBase::select(const typename ElseDerived::Scalar& thenScalar, - const DenseBase& elseMatrix) const -{ - return Select( - derived(), ElseDerived::Constant(rows(),cols(),thenScalar), elseMatrix.derived()); + * the \em then expression being a scalar value. + * + * \sa DenseBase::booleanSelect(const DenseBase&, const DenseBase&) const, class Select + */ +template +template +inline EIGEN_DEVICE_FUNC CwiseTernaryOp< + internal::scalar_boolean_select_op::Scalar, + typename DenseBase::Scalar, + typename DenseBase::Scalar>, + typename DenseBase::ConstantReturnType, ElseDerived, + Derived> +DenseBase::select( + const typename DenseBase::Scalar& thenScalar, + const DenseBase& elseMatrix) const { + using ThenConstantType = + typename DenseBase::ConstantReturnType; + using Op = internal::scalar_boolean_select_op< + typename DenseBase::Scalar, + typename DenseBase::Scalar, Scalar>; + return CwiseTernaryOp( + ThenConstantType(rows(), cols(), thenScalar), elseMatrix.derived(), + derived(), Op()); } } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SelfAdjointView.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SelfAdjointView.h index 8ce3b372a0f..0c8a3334f96 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SelfAdjointView.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SelfAdjointView.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SELFADJOINTMATRIX_H #define EIGEN_SELFADJOINTMATRIX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class SelfAdjointView @@ -18,8 +21,8 @@ namespace Eigen { * * \brief Expression of a selfadjoint matrix from a triangular part of a dense matrix * - * \param MatrixType the type of the dense matrix storing the coefficients - * \param TriangularPart can be either \c #Lower or \c #Upper + * \tparam MatrixType the type of the dense matrix storing the coefficients + * \tparam TriangularPart can be either \c #Lower or \c #Upper * * This class is an expression of a sefladjoint matrix from a triangular part of a matrix * with given dense storage of the coefficients. It is the return type of MatrixBase::selfadjointView() @@ -33,7 +36,7 @@ template struct traits > : traits { typedef typename ref_selector::non_const_type MatrixTypeNested; - typedef typename remove_all::type MatrixTypeNestedCleaned; + typedef remove_all_t MatrixTypeNestedCleaned; typedef MatrixType ExpressionType; typedef typename MatrixType::PlainObject FullMatrixType; enum { @@ -46,12 +49,13 @@ struct traits > : traits } -template class SelfAdjointView - : public TriangularBase > +template class SelfAdjointView + : public TriangularBase > { public: + EIGEN_STATIC_ASSERT(UpLo==Lower || UpLo==Upper,SELFADJOINTVIEW_ACCEPTS_UPPER_AND_LOWER_MODE_ONLY) - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef TriangularBase Base; typedef typename internal::traits::MatrixTypeNested MatrixTypeNested; typedef typename internal::traits::MatrixTypeNestedCleaned MatrixTypeNestedCleaned; @@ -60,8 +64,8 @@ template class SelfAdjointView /** \brief The type of coefficients in this matrix */ typedef typename internal::traits::Scalar Scalar; typedef typename MatrixType::StorageIndex StorageIndex; - typedef typename internal::remove_all::type MatrixConjugateReturnType; - typedef SelfAdjointView::type, UpLo> ConstSelfAdjointView; + typedef internal::remove_all_t MatrixConjugateReturnType; + typedef SelfAdjointView, UpLo> ConstSelfAdjointView; enum { Mode = internal::traits::Mode, @@ -71,10 +75,7 @@ template class SelfAdjointView typedef typename MatrixType::PlainObject PlainObject; EIGEN_DEVICE_FUNC - explicit inline SelfAdjointView(MatrixType& matrix) : m_matrix(matrix) - { - EIGEN_STATIC_ASSERT(UpLo==Lower || UpLo==Upper,SELFADJOINTVIEW_ACCEPTS_UPPER_AND_LOWER_MODE_ONLY); - } + explicit inline SelfAdjointView(MatrixType& matrix) : m_matrix(matrix) { } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_matrix.rows(); } @@ -180,16 +181,16 @@ template class SelfAdjointView */ template EIGEN_DEVICE_FUNC - typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), - TriangularView, - TriangularView >::type + std::conditional_t<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), + TriangularView, + TriangularView > triangularView() const { - typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), MatrixType&, typename MatrixType::ConstTransposeReturnType>::type tmp1(m_matrix); - typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), MatrixType&, typename MatrixType::AdjointReturnType>::type tmp2(tmp1); - return typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), - TriangularView, - TriangularView >::type(tmp2); + std::conditional_t<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), MatrixType&, typename MatrixType::ConstTransposeReturnType> tmp1(m_matrix); + std::conditional_t<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), MatrixType&, typename MatrixType::AdjointReturnType> tmp2(tmp1); + return std::conditional_t<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), + TriangularView, + TriangularView >(tmp2); } typedef SelfAdjointView ConjugateReturnType; @@ -203,10 +204,10 @@ template class SelfAdjointView */ template EIGEN_DEVICE_FUNC - inline typename internal::conditional::type + inline std::conditional_t conjugateIf() const { - typedef typename internal::conditional::type ReturnType; + typedef std::conditional_t ReturnType; return ReturnType(m_matrix.template conjugateIf()); } @@ -218,10 +219,10 @@ template class SelfAdjointView typedef SelfAdjointView TransposeReturnType; /** \sa MatrixBase::transpose() */ + template EIGEN_DEVICE_FUNC - inline TransposeReturnType transpose() + inline TransposeReturnType transpose(std::enable_if_t::value, Dummy*> = nullptr) { - EIGEN_STATIC_ASSERT_LVALUE(MatrixType) typename MatrixType::TransposeReturnType tmp(m_matrix); return TransposeReturnType(tmp); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SelfCwiseBinaryOp.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SelfCwiseBinaryOp.h index 7c89c2e23c6..5ed85c7fb0e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SelfCwiseBinaryOp.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SelfCwiseBinaryOp.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SELFCWISEBINARYOP_H #define EIGEN_SELFCWISEBINARYOP_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { // TODO generalize the scalar type of 'other' diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SkewSymmetricMatrix3.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SkewSymmetricMatrix3.h new file mode 100644 index 00000000000..6812236084a --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SkewSymmetricMatrix3.h @@ -0,0 +1,415 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2009 Gael Guennebaud +// Copyright (C) 2007-2009 Benoit Jacob +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SKEWSYMMETRICMATRIX3_H +#define EIGEN_SKEWSYMMETRICMATRIX3_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +/** \class SkewSymmetricBase + * \ingroup Core_Module + * + * \brief Base class for skew symmetric matrices and expressions + * + * This is the base class that is inherited by SkewSymmetricMatrix3 and related expression + * types, which internally use a three vector for storing the entries. SkewSymmetric + * types always represent square three times three matrices. + * + * This implementations follows class DiagonalMatrix + * + * \tparam Derived is the derived type, a SkewSymmetricMatrix3 or SkewSymmetricWrapper. + * + * \sa class SkewSymmetricMatrix3, class SkewSymmetricWrapper + */ +template +class SkewSymmetricBase : public EigenBase +{ + public: + typedef typename internal::traits::SkewSymmetricVectorType SkewSymmetricVectorType; + typedef typename SkewSymmetricVectorType::Scalar Scalar; + typedef typename SkewSymmetricVectorType::RealScalar RealScalar; + typedef typename internal::traits::StorageKind StorageKind; + typedef typename internal::traits::StorageIndex StorageIndex; + + enum { + RowsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime, + ColsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime, + MaxRowsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime, + MaxColsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime, + IsVectorAtCompileTime = 0, + Flags = NoPreferredStorageOrderBit + }; + + typedef Matrix DenseMatrixType; + typedef DenseMatrixType DenseType; + typedef SkewSymmetricMatrix3 PlainObject; + + /** \returns a reference to the derived object. */ + EIGEN_DEVICE_FUNC + inline const Derived& derived() const { return *static_cast(this); } + /** \returns a const reference to the derived object. */ + EIGEN_DEVICE_FUNC + inline Derived& derived() { return *static_cast(this); } + + /** + * Constructs a dense matrix from \c *this. Note, this directly returns a dense matrix type, + * not an expression. + * \returns A dense matrix, with its entries set from the the derived object. */ + EIGEN_DEVICE_FUNC + DenseMatrixType toDenseMatrix() const { return derived(); } + + /** Determinant vanishes */ + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR + inline Scalar determinant() const { return 0; } + + /** A.transpose() = -A */ + EIGEN_DEVICE_FUNC + PlainObject transpose() const { return (-vector()).asSkewSymmetric(); } + + /** \returns the exponential of this matrix using Rodrigues’ formula */ + EIGEN_DEVICE_FUNC + DenseMatrixType exponential() const { + DenseMatrixType retVal = DenseMatrixType::Identity(); + const SkewSymmetricVectorType& v = vector(); + if (v.isZero()) { + return retVal; + } + const Scalar norm2 = v.squaredNorm(); + const Scalar norm = numext::sqrt(norm2); + retVal += ((((1 - numext::cos(norm))/norm2)*derived())*derived()) + (numext::sin(norm)/norm)*derived().toDenseMatrix(); + return retVal; + } + + /** \returns a reference to the derived object's vector of coefficients. */ + EIGEN_DEVICE_FUNC + inline const SkewSymmetricVectorType& vector() const { return derived().vector(); } + /** \returns a const reference to the derived object's vector of coefficients. */ + EIGEN_DEVICE_FUNC + inline SkewSymmetricVectorType& vector() { return derived().vector(); } + + /** \returns the number of rows. */ + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR + inline Index rows() const { return 3; } + /** \returns the number of columns. */ + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR + inline Index cols() const { return 3; } + + /** \returns the matrix product of \c *this by the dense matrix, \a matrix */ + template + EIGEN_DEVICE_FUNC + Product + operator*(const MatrixBase &matrix) const + { + return Product(derived(), matrix.derived()); + } + + /** \returns the matrix product of \c *this by the skew symmetric matrix, \a matrix */ + template + EIGEN_DEVICE_FUNC + Product + operator*(const SkewSymmetricBase &matrix) const + { + return Product(derived(), matrix.derived()); + } + + template + using SkewSymmetricProductReturnType = SkewSymmetricWrapper; + + /** \returns the wedge product of \c *this by the skew symmetric matrix \a other + * A wedge B = AB - BA */ + template + EIGEN_DEVICE_FUNC SkewSymmetricProductReturnType wedge( + const SkewSymmetricBase& other) const { + return vector().cross(other.vector()).asSkewSymmetric(); + } + + using SkewSymmetricScaleReturnType = + SkewSymmetricWrapper; + + /** \returns the product of \c *this by the scalar \a scalar */ + EIGEN_DEVICE_FUNC + inline SkewSymmetricScaleReturnType operator*(const Scalar& scalar) const { + return (vector() * scalar).asSkewSymmetric(); + } + + using ScaleSkewSymmetricReturnType = + SkewSymmetricWrapper; + + /** \returns the product of a scalar and the skew symmetric matrix \a other */ + EIGEN_DEVICE_FUNC + friend inline ScaleSkewSymmetricReturnType operator*(const Scalar& scalar, const SkewSymmetricBase& other) { + return (scalar * other.vector()).asSkewSymmetric(); + } + + template + using SkewSymmetricSumReturnType = SkewSymmetricWrapper; + + /** \returns the sum of \c *this and the skew symmetric matrix \a other */ + template + EIGEN_DEVICE_FUNC inline SkewSymmetricSumReturnType operator+( + const SkewSymmetricBase& other) const { + return (vector() + other.vector()).asSkewSymmetric(); + } + + template + using SkewSymmetricDifferenceReturnType = SkewSymmetricWrapper; + + /** \returns the difference of \c *this and the skew symmetric matrix \a other */ + template + EIGEN_DEVICE_FUNC inline SkewSymmetricDifferenceReturnType operator-( + const SkewSymmetricBase& other) const { + return (vector() - other.vector()).asSkewSymmetric(); + } +}; + +/** \class SkewSymmetricMatrix3 + * \ingroup Core_Module + * + * \brief Represents a 3x3 skew symmetric matrix with its storage + * + * \tparam Scalar_ the type of coefficients + * + * \sa class SkewSymmetricBase, class SkewSymmetricWrapper + */ + +namespace internal { +template +struct traits > + : traits > +{ + typedef Matrix SkewSymmetricVectorType; + typedef SkewSymmetricShape StorageKind; + enum { + Flags = LvalueBit | NoPreferredStorageOrderBit | NestByRefBit + }; +}; +} +template +class SkewSymmetricMatrix3 + : public SkewSymmetricBase > +{ + public: + #ifndef EIGEN_PARSED_BY_DOXYGEN + typedef typename internal::traits::SkewSymmetricVectorType SkewSymmetricVectorType; + typedef const SkewSymmetricMatrix3& Nested; + typedef Scalar_ Scalar; + typedef typename internal::traits::StorageKind StorageKind; + typedef typename internal::traits::StorageIndex StorageIndex; + #endif + + protected: + + SkewSymmetricVectorType m_vector; + + public: + + /** const version of vector(). */ + EIGEN_DEVICE_FUNC + inline const SkewSymmetricVectorType& vector() const { return m_vector; } + /** \returns a reference to the stored vector of coefficients. */ + EIGEN_DEVICE_FUNC + inline SkewSymmetricVectorType& vector() { return m_vector; } + + /** Default constructor without initialization */ + EIGEN_DEVICE_FUNC + inline SkewSymmetricMatrix3() {} + + /** Constructor from three scalars */ + EIGEN_DEVICE_FUNC + inline SkewSymmetricMatrix3(const Scalar& x, const Scalar& y, const Scalar& z) : m_vector(x,y,z) {} + + /** \brief Constructs a SkewSymmetricMatrix3 from an r-value vector type */ + EIGEN_DEVICE_FUNC + explicit inline SkewSymmetricMatrix3(SkewSymmetricVectorType&& vec) : m_vector(std::move(vec)) {} + + /** generic constructor from expression of the coefficients */ + template + EIGEN_DEVICE_FUNC + explicit inline SkewSymmetricMatrix3(const MatrixBase& other) : m_vector(other) + {} + + /** Copy constructor. */ + template + EIGEN_DEVICE_FUNC + inline SkewSymmetricMatrix3(const SkewSymmetricBase& other) : m_vector(other.vector()) {} + + #ifndef EIGEN_PARSED_BY_DOXYGEN + /** copy constructor. prevent a default copy constructor from hiding the other templated constructor */ + inline SkewSymmetricMatrix3(const SkewSymmetricMatrix3& other) : m_vector(other.vector()) {} + #endif + + /** Copy operator. */ + template + EIGEN_DEVICE_FUNC + SkewSymmetricMatrix3& operator=(const SkewSymmetricBase& other) + { + m_vector = other.vector(); + return *this; + } + + #ifndef EIGEN_PARSED_BY_DOXYGEN + /** This is a special case of the templated operator=. Its purpose is to + * prevent a default operator= from hiding the templated operator=. + */ + EIGEN_DEVICE_FUNC + SkewSymmetricMatrix3& operator=(const SkewSymmetricMatrix3& other) + { + m_vector = other.vector(); + return *this; + } + #endif + + typedef SkewSymmetricWrapper, SkewSymmetricVectorType>> + InitializeReturnType; + + /** Initializes a skew symmetric matrix with coefficients set to zero */ + EIGEN_DEVICE_FUNC + static InitializeReturnType Zero() { return SkewSymmetricVectorType::Zero().asSkewSymmetric(); } + + /** Sets all coefficients to zero. */ + EIGEN_DEVICE_FUNC + inline void setZero() { m_vector.setZero(); } +}; + +/** \class SkewSymmetricWrapper + * \ingroup Core_Module + * + * \brief Expression of a skew symmetric matrix + * + * \tparam SkewSymmetricVectorType_ the type of the vector of coefficients + * + * This class is an expression of a skew symmetric matrix, but not storing its own vector of coefficients, + * instead wrapping an existing vector expression. It is the return type of MatrixBase::asSkewSymmetric() + * and most of the time this is the only way that it is used. + * + * \sa class SkewSymmetricMatrix3, class SkewSymmetricBase, MatrixBase::asSkewSymmetric() + */ + +namespace internal { +template +struct traits > +{ + typedef SkewSymmetricVectorType_ SkewSymmetricVectorType; + typedef typename SkewSymmetricVectorType::Scalar Scalar; + typedef typename SkewSymmetricVectorType::StorageIndex StorageIndex; + typedef SkewSymmetricShape StorageKind; + typedef typename traits::XprKind XprKind; + enum { + RowsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime, + ColsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime, + MaxRowsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime, + MaxColsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime, + Flags = (traits::Flags & LvalueBit) | NoPreferredStorageOrderBit + }; +}; +} + +template +class SkewSymmetricWrapper + : public SkewSymmetricBase >, internal::no_assignment_operator +{ + public: + #ifndef EIGEN_PARSED_BY_DOXYGEN + typedef SkewSymmetricVectorType_ SkewSymmetricVectorType; + typedef SkewSymmetricWrapper Nested; + #endif + + /** Constructor from expression of coefficients to wrap. */ + EIGEN_DEVICE_FUNC + explicit inline SkewSymmetricWrapper(SkewSymmetricVectorType& a_vector) : m_vector(a_vector) {} + + /** \returns a const reference to the wrapped expression of coefficients. */ + EIGEN_DEVICE_FUNC + const SkewSymmetricVectorType& vector() const { return m_vector; } + + protected: + typename SkewSymmetricVectorType::Nested m_vector; +}; + +/** \returns a pseudo-expression of a skew symmetric matrix with *this as vector of coefficients + * + * \only_for_vectors + * + * \sa class SkewSymmetricWrapper, class SkewSymmetricMatrix3, vector(), isSkewSymmetric() + **/ +template +EIGEN_DEVICE_FUNC inline const SkewSymmetricWrapper +MatrixBase::asSkewSymmetric() const +{ + return SkewSymmetricWrapper(derived()); +} + +/** \returns true if *this is approximately equal to a skew symmetric matrix, + * within the precision given by \a prec. + */ +template +bool MatrixBase::isSkewSymmetric(const RealScalar& prec) const +{ + if(cols() != rows()) return false; + return (this->transpose() + *this).isZero(prec); +} + +/** \returns the matrix product of \c *this by the skew symmetric matrix \skew. + */ +template +template +EIGEN_DEVICE_FUNC inline const Product +MatrixBase::operator*(const SkewSymmetricBase &skew) const +{ + return Product(derived(), skew.derived()); +} + +namespace internal { + +template<> struct storage_kind_to_shape { typedef SkewSymmetricShape Shape; }; + +struct SkewSymmetric2Dense {}; + +template<> struct AssignmentKind { typedef SkewSymmetric2Dense Kind; }; + +// SkewSymmetric matrix to Dense assignment +template< typename DstXprType, typename SrcXprType, typename Functor> +struct Assignment +{ + EIGEN_DEVICE_FUNC + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op &/*func*/) + { + if((dst.rows()!=3) || (dst.cols()!=3)) { + dst.resize(3, 3); + } + dst.diagonal().setZero(); + const typename SrcXprType::SkewSymmetricVectorType v = src.vector(); + dst(0, 1) = -v(2); + dst(1, 0) = v(2); + dst(0, 2) = v(1); + dst(2, 0) = -v(1); + dst(1, 2) = -v(0); + dst(2, 1) = v(0); + } + EIGEN_DEVICE_FUNC + static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op &/*func*/) + { dst.vector() += src.vector(); } + + EIGEN_DEVICE_FUNC + static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op &/*func*/) + { dst.vector() -= src.vector(); } +}; + +} // namespace internal + +} // end namespace Eigen + +#endif // EIGEN_SKEWSYMMETRICMATRIX3_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Solve.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Solve.h index 23d5cb70728..616dd168bbc 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Solve.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Solve.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SOLVE_H #define EIGEN_SOLVE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template class SolveImpl; @@ -77,7 +80,7 @@ class Solve : public SolveImpl::type m_rhs; }; @@ -123,7 +126,7 @@ struct evaluator > EIGEN_DEVICE_FUNC explicit evaluator(const SolveType& solve) : m_result(solve.rows(), solve.cols()) { - ::new (static_cast(this)) Base(m_result); + internal::construct_at(this, m_result); solve.dec()._solve_impl(solve.rhs(), m_result); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SolveTriangular.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SolveTriangular.h index dfbf99523a9..b29a7da3695 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SolveTriangular.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SolveTriangular.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SOLVETRIANGULAR_H #define EIGEN_SOLVETRIANGULAR_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -87,7 +90,7 @@ struct triangular_solver_selector static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, Rhs& rhs) { - typename internal::add_const_on_value_type::type actualLhs = LhsProductTraits::extract(lhs); + add_const_on_value_type_t actualLhs = LhsProductTraits::extract(lhs); const Index size = lhs.rows(); const Index othersize = Side==OnTheLeft? rhs.cols() : rhs.rows(); @@ -95,6 +98,11 @@ struct triangular_solver_selector typedef internal::gemm_blocking_space<(Rhs::Flags&RowMajorBit) ? RowMajor : ColMajor,Scalar,Scalar, Rhs::MaxRowsAtCompileTime, Rhs::MaxColsAtCompileTime, Lhs::MaxRowsAtCompileTime,4> BlockingType; + // Nothing to solve. + if (actualLhs.size() == 0 || rhs.size() == 0) { + return; + } + BlockingType blocking(rhs.rows(), rhs.cols(), size, 1, false); triangular_solve_matrix::solveInPlace(c return; enum { copy = (internal::traits::Flags & RowMajorBit) && OtherDerived::IsVectorAtCompileTime && OtherDerived::SizeAtCompileTime!=1}; - typedef typename internal::conditional::type, OtherDerived&>::type OtherCopy; + typedef std::conditional_t::type, OtherDerived&> OtherCopy; OtherCopy otherCopy(other); - internal::triangular_solver_selector::type, + internal::triangular_solver_selector, Side, Mode>::run(derived().nestedExpression(), otherCopy); if (copy) @@ -206,7 +214,7 @@ struct traits > template struct triangular_solve_retval : public ReturnByValue > { - typedef typename remove_all::type RhsNestedCleaned; + typedef remove_all_t RhsNestedCleaned; typedef ReturnByValue Base; triangular_solve_retval(const TriangularType& tri, const Rhs& rhs) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SolverBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SolverBase.h index 5014610420f..f45b5192d33 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SolverBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/SolverBase.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SOLVERBASE_H #define EIGEN_SOLVERBASE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -28,7 +31,7 @@ struct solve_assertion > template static void run(const type& transpose, const Rhs& b) { - internal::solve_assertion::type>::template run(transpose.nestedExpression(), b); + internal::solve_assertion>::template run(transpose.nestedExpression(), b); } }; @@ -40,7 +43,7 @@ struct solve_assertion template static void run(const type& adjoint, const Rhs& b) { - internal::solve_assertion >::type>::template run(adjoint.nestedExpression(), b); + internal::solve_assertion >>::template run(adjoint.nestedExpression(), b); } }; } // end namespace internal @@ -79,12 +82,11 @@ class SolverBase : public EigenBase enum { RowsAtCompileTime = internal::traits::RowsAtCompileTime, ColsAtCompileTime = internal::traits::ColsAtCompileTime, - SizeAtCompileTime = (internal::size_at_compile_time::RowsAtCompileTime, - internal::traits::ColsAtCompileTime>::ret), + SizeAtCompileTime = (internal::size_of_xpr_at_compile_time::ret), MaxRowsAtCompileTime = internal::traits::MaxRowsAtCompileTime, MaxColsAtCompileTime = internal::traits::MaxColsAtCompileTime, - MaxSizeAtCompileTime = (internal::size_at_compile_time::MaxRowsAtCompileTime, - internal::traits::MaxColsAtCompileTime>::ret), + MaxSizeAtCompileTime = internal::size_at_compile_time(internal::traits::MaxRowsAtCompileTime, + internal::traits::MaxColsAtCompileTime), IsVectorAtCompileTime = internal::traits::MaxRowsAtCompileTime == 1 || internal::traits::MaxColsAtCompileTime == 1, NumDimensions = int(MaxSizeAtCompileTime) == 1 ? 0 : bool(IsVectorAtCompileTime) ? 1 : 2 @@ -105,12 +107,12 @@ class SolverBase : public EigenBase inline const Solve solve(const MatrixBase& b) const { - internal::solve_assertion::type>::template run(derived(), b); + internal::solve_assertion>::template run(derived(), b); return Solve(derived(), b.derived()); } /** \internal the return type of transpose() */ - typedef typename internal::add_const >::type ConstTransposeReturnType; + typedef Transpose ConstTransposeReturnType; /** \returns an expression of the transposed of the factored matrix. * * A typical usage is to solve for the transposed problem A^T x = b: @@ -118,16 +120,16 @@ class SolverBase : public EigenBase * * \sa adjoint(), solve() */ - inline ConstTransposeReturnType transpose() const + inline const ConstTransposeReturnType transpose() const { return ConstTransposeReturnType(derived()); } /** \internal the return type of adjoint() */ - typedef typename internal::conditional::IsComplex, - CwiseUnaryOp, ConstTransposeReturnType>, - ConstTransposeReturnType - >::type AdjointReturnType; + typedef std::conditional_t::IsComplex, + CwiseUnaryOp, const ConstTransposeReturnType>, + const ConstTransposeReturnType + > AdjointReturnType; /** \returns an expression of the adjoint of the factored matrix * * A typical usage is to solve for the adjoint problem A' x = b: @@ -137,7 +139,7 @@ class SolverBase : public EigenBase * * \sa transpose(), solve() */ - inline AdjointReturnType adjoint() const + inline const AdjointReturnType adjoint() const { return AdjointReturnType(derived().transpose()); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/StableNorm.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/StableNorm.h index 4a3f0cca8c4..dd36f2c0474 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/StableNorm.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/StableNorm.h @@ -10,6 +10,9 @@ #ifndef EIGEN_STABLENORM_H #define EIGEN_STABLENORM_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -57,7 +60,7 @@ void stable_norm_impl_inner_step(const VectorType &vec, RealScalar& ssq, RealSca const Index blockSize = 4096; typedef typename internal::nested_eval::type VectorTypeCopy; - typedef typename internal::remove_all::type VectorTypeCopyClean; + typedef internal::remove_all_t VectorTypeCopyClean; const VectorTypeCopy copy(vec); enum { @@ -66,8 +69,8 @@ void stable_norm_impl_inner_step(const VectorType &vec, RealScalar& ssq, RealSca ) && (blockSize*sizeof(Scalar)*20) // if we cannot allocate on the stack, then let's not bother about this optimization }; - typedef typename internal::conditional, internal::evaluator::Alignment>, - typename VectorTypeCopyClean::ConstSegmentReturnType>::type SegmentWrapper; + typedef std::conditional_t, internal::evaluator::Alignment>, + typename VectorTypeCopyClean::ConstSegmentReturnType> SegmentWrapper; Index n = vec.size(); Index bi = internal::first_default_aligned(copy); @@ -79,7 +82,7 @@ void stable_norm_impl_inner_step(const VectorType &vec, RealScalar& ssq, RealSca template typename VectorType::RealScalar -stable_norm_impl(const VectorType &vec, typename enable_if::type* = 0 ) +stable_norm_impl(const VectorType &vec, std::enable_if_t* = 0 ) { using std::sqrt; using std::abs; @@ -101,7 +104,7 @@ stable_norm_impl(const VectorType &vec, typename enable_if typename MatrixType::RealScalar -stable_norm_impl(const MatrixType &mat, typename enable_if::type* = 0 ) +stable_norm_impl(const MatrixType &mat, std::enable_if_t* = 0 ) { using std::sqrt; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/StlIterators.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/StlIterators.h index 09041db1d55..445199c0693 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/StlIterators.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/StlIterators.h @@ -10,6 +10,9 @@ #ifndef EIGEN_STLITERATORS_H #define EIGEN_STLITERATORS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -25,7 +28,7 @@ class indexed_based_stl_iterator_base typedef typename traits::XprType XprType; typedef indexed_based_stl_iterator_base non_const_iterator; typedef indexed_based_stl_iterator_base const_iterator; - typedef typename internal::conditional::value,non_const_iterator,const_iterator>::type other_iterator; + typedef std::conditional_t::value,non_const_iterator,const_iterator> other_iterator; // NOTE: in C++03 we cannot declare friend classes through typedefs because we need to write friend class: friend class indexed_based_stl_iterator_base; friend class indexed_based_stl_iterator_base; @@ -104,7 +107,7 @@ class indexed_based_stl_reverse_iterator_base typedef typename traits::XprType XprType; typedef indexed_based_stl_reverse_iterator_base non_const_iterator; typedef indexed_based_stl_reverse_iterator_base const_iterator; - typedef typename internal::conditional::value,non_const_iterator,const_iterator>::type other_iterator; + typedef std::conditional_t::value,non_const_iterator,const_iterator> other_iterator; // NOTE: in C++03 we cannot declare friend classes through typedefs because we need to write friend class: friend class indexed_based_stl_reverse_iterator_base; friend class indexed_based_stl_reverse_iterator_base; @@ -179,18 +182,18 @@ template class pointer_based_stl_iterator { enum { is_lvalue = internal::is_lvalue::value }; - typedef pointer_based_stl_iterator::type> non_const_iterator; - typedef pointer_based_stl_iterator::type> const_iterator; - typedef typename internal::conditional::value,non_const_iterator,const_iterator>::type other_iterator; + typedef pointer_based_stl_iterator> non_const_iterator; + typedef pointer_based_stl_iterator> const_iterator; + typedef std::conditional_t::value,non_const_iterator,const_iterator> other_iterator; // NOTE: in C++03 we cannot declare friend classes through typedefs because we need to write friend class: - friend class pointer_based_stl_iterator::type>; - friend class pointer_based_stl_iterator::type>; + friend class pointer_based_stl_iterator>; + friend class pointer_based_stl_iterator>; public: typedef Index difference_type; typedef typename XprType::Scalar value_type; typedef std::random_access_iterator_tag iterator_category; - typedef typename internal::conditional::type pointer; - typedef typename internal::conditional::type reference; + typedef std::conditional_t pointer; + typedef std::conditional_t reference; pointer_based_stl_iterator() EIGEN_NO_THROW : m_ptr(0) {} @@ -256,12 +259,12 @@ class pointer_based_stl_iterator internal::variable_if_dynamic m_incr; }; -template -struct indexed_based_stl_iterator_traits > +template +struct indexed_based_stl_iterator_traits > { - typedef _XprType XprType; - typedef generic_randaccess_stl_iterator::type> non_const_iterator; - typedef generic_randaccess_stl_iterator::type> const_iterator; + typedef XprType_ XprType; + typedef generic_randaccess_stl_iterator> non_const_iterator; + typedef generic_randaccess_stl_iterator> const_iterator; }; template @@ -283,13 +286,13 @@ class generic_randaccess_stl_iterator : public indexed_based_stl_iterator_base::type read_only_ref_t; + //typedef std::conditional_t read_only_ref_t; typedef const value_type read_only_ref_t; public: - typedef typename internal::conditional::type pointer; - typedef typename internal::conditional::type reference; + typedef std::conditional_t pointer; + typedef std::conditional_t reference; generic_randaccess_stl_iterator() : Base() {} generic_randaccess_stl_iterator(XprType& xpr, Index index) : Base(xpr,index) {} @@ -301,12 +304,12 @@ class generic_randaccess_stl_iterator : public indexed_based_stl_iterator_base() const { return &((*mp_xpr)(m_index)); } }; -template -struct indexed_based_stl_iterator_traits > +template +struct indexed_based_stl_iterator_traits > { - typedef _XprType XprType; - typedef subvector_stl_iterator::type, Direction> non_const_iterator; - typedef subvector_stl_iterator::type, Direction> const_iterator; + typedef XprType_ XprType; + typedef subvector_stl_iterator, Direction> non_const_iterator; + typedef subvector_stl_iterator, Direction> const_iterator; }; template @@ -320,12 +323,12 @@ class subvector_stl_iterator : public indexed_based_stl_iterator_base::type SubVectorType; - typedef typename internal::conditional::type ConstSubVectorType; + typedef std::conditional_t SubVectorType; + typedef std::conditional_t ConstSubVectorType; public: - typedef typename internal::conditional::type reference; + typedef std::conditional_t reference; typedef typename reference::PlainObject value_type; private: @@ -349,12 +352,12 @@ class subvector_stl_iterator : public indexed_based_stl_iterator_base() const { return (*mp_xpr).template subVector(m_index); } }; -template -struct indexed_based_stl_iterator_traits > +template +struct indexed_based_stl_iterator_traits > { - typedef _XprType XprType; - typedef subvector_stl_reverse_iterator::type, Direction> non_const_iterator; - typedef subvector_stl_reverse_iterator::type, Direction> const_iterator; + typedef XprType_ XprType; + typedef subvector_stl_reverse_iterator, Direction> non_const_iterator; + typedef subvector_stl_reverse_iterator, Direction> const_iterator; }; template @@ -368,12 +371,12 @@ class subvector_stl_reverse_iterator : public indexed_based_stl_reverse_iterator using Base::m_index; using Base::mp_xpr; - typedef typename internal::conditional::type SubVectorType; - typedef typename internal::conditional::type ConstSubVectorType; + typedef std::conditional_t SubVectorType; + typedef std::conditional_t ConstSubVectorType; public: - typedef typename internal::conditional::type reference; + typedef std::conditional_t reference; typedef typename reference::PlainObject value_type; private: diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Stride.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Stride.h index d164e539969..31c70024b46 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Stride.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Stride.h @@ -10,6 +10,9 @@ #ifndef EIGEN_STRIDE_H #define EIGEN_STRIDE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class Stride @@ -31,8 +34,8 @@ namespace Eigen { * arguments to the constructor. * * Indeed, this class takes two template parameters: - * \tparam _OuterStrideAtCompileTime the outer stride, or Dynamic if you want to specify it at runtime. - * \tparam _InnerStrideAtCompileTime the inner stride, or Dynamic if you want to specify it at runtime. + * \tparam OuterStrideAtCompileTime_ the outer stride, or Dynamic if you want to specify it at runtime. + * \tparam InnerStrideAtCompileTime_ the inner stride, or Dynamic if you want to specify it at runtime. * * Here is an example: * \include Map_general_stride.cpp @@ -48,14 +51,14 @@ namespace Eigen { * * \sa class InnerStride, class OuterStride, \ref TopicStorageOrders */ -template +template class Stride { public: typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 enum { - InnerStrideAtCompileTime = _InnerStrideAtCompileTime, - OuterStrideAtCompileTime = _OuterStrideAtCompileTime + InnerStrideAtCompileTime = InnerStrideAtCompileTime_, + OuterStrideAtCompileTime = OuterStrideAtCompileTime_ }; /** Default constructor, for use when strides are fixed at compile time */ diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Swap.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Swap.h index 180a4e5adff..fd679631f89 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Swap.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Swap.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SWAP_H #define EIGEN_SWAP_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Transpose.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Transpose.h index 2bc658f40b8..89b011be83f 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Transpose.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Transpose.h @@ -11,6 +11,9 @@ #ifndef EIGEN_TRANSPOSE_H #define EIGEN_TRANSPOSE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -18,7 +21,7 @@ template struct traits > : public traits { typedef typename ref_selector::type MatrixTypeNested; - typedef typename remove_reference::type MatrixTypeNestedPlain; + typedef std::remove_reference_t MatrixTypeNestedPlain; enum { RowsAtCompileTime = MatrixType::ColsAtCompileTime, ColsAtCompileTime = MatrixType::RowsAtCompileTime, @@ -58,7 +61,7 @@ template class Transpose typedef typename TransposeImpl::StorageKind>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(Transpose) - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE Transpose(MatrixType& matrix) : m_matrix(matrix) {} @@ -72,12 +75,12 @@ template class Transpose /** \returns the nested expression */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const typename internal::remove_all::type& + const internal::remove_all_t& nestedExpression() const { return m_matrix; } /** \returns the nested expression */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - typename internal::remove_reference::type& + std::remove_reference_t& nestedExpression() { return m_matrix; } /** \internal */ @@ -130,11 +133,11 @@ template class TransposeImpl EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index outerStride() const { return derived().nestedExpression().outerStride(); } - typedef typename internal::conditional< - internal::is_lvalue::value, - Scalar, - const Scalar - >::type ScalarWithConstIfNotLvalue; + typedef std::conditional_t< + internal::is_lvalue::value, + Scalar, + const Scalar + > ScalarWithConstIfNotLvalue; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ScalarWithConstIfNotLvalue* data() { return derived().nestedExpression().data(); } @@ -178,7 +181,7 @@ template class TransposeImpl * \sa transposeInPlace(), adjoint() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -Transpose +typename DenseBase::TransposeReturnType DenseBase::transpose() { return TransposeReturnType(derived()); @@ -191,7 +194,7 @@ DenseBase::transpose() * \sa transposeInPlace(), adjoint() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -typename DenseBase::ConstTransposeReturnType +const typename DenseBase::ConstTransposeReturnType DenseBase::transpose() const { return ConstTransposeReturnType(derived()); @@ -400,7 +403,7 @@ struct check_transpose_aliasing_compile_time_selector struct check_transpose_aliasing_run_time_selector { - static bool run(const Scalar* dest, const OtherDerived& src) + EIGEN_DEVICE_FUNC static bool run(const Scalar* dest, const OtherDerived& src) { return (bool(blas_traits::IsTransposed) != DestIsTransposed) && (dest!=0 && dest==(const Scalar*)extract_data(src)); } @@ -409,7 +412,7 @@ struct check_transpose_aliasing_run_time_selector template struct check_transpose_aliasing_run_time_selector > { - static bool run(const Scalar* dest, const CwiseBinaryOp& src) + EIGEN_DEVICE_FUNC static bool run(const Scalar* dest, const CwiseBinaryOp& src) { return ((blas_traits::IsTransposed != DestIsTransposed) && (dest!=0 && dest==(const Scalar*)extract_data(src.lhs()))) || ((blas_traits::IsTransposed != DestIsTransposed) && (dest!=0 && dest==(const Scalar*)extract_data(src.rhs()))); @@ -429,7 +432,7 @@ template struct checkTransposeAliasing_impl { - static void run(const Derived& dst, const OtherDerived& other) + EIGEN_DEVICE_FUNC static void run(const Derived& dst, const OtherDerived& other) { eigen_assert((!check_transpose_aliasing_run_time_selector ::IsTransposed,OtherDerived> @@ -443,13 +446,13 @@ struct checkTransposeAliasing_impl template struct checkTransposeAliasing_impl { - static void run(const Derived&, const OtherDerived&) + EIGEN_DEVICE_FUNC static void run(const Derived&, const OtherDerived&) { } }; template -void check_for_aliasing(const Dst &dst, const Src &src) +EIGEN_DEVICE_FUNC inline void check_for_aliasing(const Dst &dst, const Src &src) { if((!Dst::IsVectorAtCompileTime) && dst.rows()>1 && dst.cols()>1) internal::checkTransposeAliasing_impl::run(dst, src); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Transpositions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Transpositions.h index 38a7b01cb51..f10ca3349b2 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Transpositions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Transpositions.h @@ -10,6 +10,9 @@ #ifndef EIGEN_TRANSPOSITIONS_H #define EIGEN_TRANSPOSITIONS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template @@ -113,11 +116,11 @@ class TranspositionsBase }; namespace internal { -template -struct traits > - : traits > +template +struct traits > + : traits > { - typedef Matrix<_StorageIndex, SizeAtCompileTime, 1, 0, MaxSizeAtCompileTime, 1> IndicesType; + typedef Matrix IndicesType; typedef TranspositionsStorage StorageKind; }; } @@ -151,8 +154,8 @@ struct traits -class Transpositions : public TranspositionsBase > +template +class Transpositions : public TranspositionsBase > { typedef internal::traits Traits; public: @@ -199,19 +202,19 @@ class Transpositions : public TranspositionsBase -struct traits,_PacketAccess> > - : traits > +template +struct traits,PacketAccess_> > + : traits > { - typedef Map, _PacketAccess> IndicesType; - typedef _StorageIndex StorageIndex; + typedef Map, PacketAccess_> IndicesType; + typedef StorageIndex_ StorageIndex; typedef TranspositionsStorage StorageKind; }; } -template -class Map,PacketAccess> - : public TranspositionsBase,PacketAccess> > +template +class Map,PacketAccess> + : public TranspositionsBase,PacketAccess> > { typedef internal::traits Traits; public: @@ -260,17 +263,17 @@ class Map,P }; namespace internal { -template -struct traits > - : traits > +template +struct traits > + : traits > { typedef TranspositionsStorage StorageKind; }; } -template +template class TranspositionsWrapper - : public TranspositionsBase > + : public TranspositionsBase > { typedef internal::traits Traits; public: diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/TriangularMatrix.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/TriangularMatrix.h index fdb8bc15a5b..44af65dc4d7 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/TriangularMatrix.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/TriangularMatrix.h @@ -11,6 +11,9 @@ #ifndef EIGEN_TRIANGULARMATRIX_H #define EIGEN_TRIANGULARMATRIX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -35,14 +38,13 @@ template class TriangularBase : public EigenBase MaxRowsAtCompileTime = internal::traits::MaxRowsAtCompileTime, MaxColsAtCompileTime = internal::traits::MaxColsAtCompileTime, - SizeAtCompileTime = (internal::size_at_compile_time::RowsAtCompileTime, - internal::traits::ColsAtCompileTime>::ret), + SizeAtCompileTime = (internal::size_of_xpr_at_compile_time::ret), /**< This is equal to the number of coefficients, i.e. the number of * rows times the number of columns, or to \a Dynamic if this is not * known at compile-time. \sa RowsAtCompileTime, ColsAtCompileTime */ - MaxSizeAtCompileTime = (internal::size_at_compile_time::MaxRowsAtCompileTime, - internal::traits::MaxColsAtCompileTime>::ret) + MaxSizeAtCompileTime = internal::size_at_compile_time(internal::traits::MaxRowsAtCompileTime, + internal::traits::MaxColsAtCompileTime) }; typedef typename internal::traits::Scalar Scalar; @@ -153,8 +155,8 @@ template class TriangularBase : public EigenBase * * \brief Expression of a triangular part in a matrix * - * \param MatrixType the type of the object in which we are taking the triangular part - * \param Mode the kind of triangular matrix expression to construct. Can be #Upper, + * \tparam MatrixType the type of the object in which we are taking the triangular part + * \tparam Mode the kind of triangular matrix expression to construct. Can be #Upper, * #Lower, #UnitUpper, #UnitLower, #StrictlyUpper, or #StrictlyLower. * This is in fact a bit field; it must have either #Upper or #Lower, * and additionally it may have #UnitDiag or #ZeroDiag or neither. @@ -166,39 +168,39 @@ template class TriangularBase : public EigenBase * \sa MatrixBase::triangularView() */ namespace internal { -template -struct traits > : traits +template +struct traits > : traits { typedef typename ref_selector::non_const_type MatrixTypeNested; - typedef typename remove_reference::type MatrixTypeNestedNonRef; - typedef typename remove_all::type MatrixTypeNestedCleaned; + typedef std::remove_reference_t MatrixTypeNestedNonRef; + typedef remove_all_t MatrixTypeNestedCleaned; typedef typename MatrixType::PlainObject FullMatrixType; typedef MatrixType ExpressionType; enum { - Mode = _Mode, + Mode = Mode_, FlagsLvalueBit = is_lvalue::value ? LvalueBit : 0, Flags = (MatrixTypeNestedCleaned::Flags & (HereditaryBits | FlagsLvalueBit) & (~(PacketAccessBit | DirectAccessBit | LinearAccessBit))) }; }; } -template class TriangularViewImpl; +template class TriangularViewImpl; -template class TriangularView - : public TriangularViewImpl<_MatrixType, _Mode, typename internal::traits<_MatrixType>::StorageKind > +template class TriangularView + : public TriangularViewImpl::StorageKind > { public: - typedef TriangularViewImpl<_MatrixType, _Mode, typename internal::traits<_MatrixType>::StorageKind > Base; + typedef TriangularViewImpl::StorageKind > Base; typedef typename internal::traits::Scalar Scalar; - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; protected: typedef typename internal::traits::MatrixTypeNested MatrixTypeNested; typedef typename internal::traits::MatrixTypeNestedNonRef MatrixTypeNestedNonRef; - typedef typename internal::remove_all::type MatrixConjugateReturnType; - typedef TriangularView::type, _Mode> ConstTriangularView; + typedef internal::remove_all_t MatrixConjugateReturnType; + typedef TriangularView, Mode_> ConstTriangularView; public: @@ -206,7 +208,7 @@ template class TriangularView typedef typename internal::traits::MatrixTypeNestedCleaned NestedExpression; enum { - Mode = _Mode, + Mode = Mode_, Flags = internal::traits::Flags, TransposeMode = (Mode & Upper ? Lower : 0) | (Mode & Lower ? Upper : 0) @@ -247,10 +249,10 @@ template class TriangularView */ template EIGEN_DEVICE_FUNC - inline typename internal::conditional::type + inline std::conditional_t conjugateIf() const { - typedef typename internal::conditional::type ReturnType; + typedef std::conditional_t ReturnType; return ReturnType(m_matrix.template conjugateIf()); } @@ -262,10 +264,10 @@ template class TriangularView typedef TriangularView TransposeReturnType; /** \sa MatrixBase::transpose() */ + template EIGEN_DEVICE_FUNC - inline TransposeReturnType transpose() + inline TransposeReturnType transpose(std::enable_if_t::value, Dummy*> = nullptr) { - EIGEN_STATIC_ASSERT_LVALUE(MatrixType) typename MatrixType::TransposeReturnType tmp(m_matrix); return TransposeReturnType(tmp); } @@ -342,16 +344,17 @@ template class TriangularView * * \sa class TriangularView, MatrixBase::triangularView() */ -template class TriangularViewImpl<_MatrixType,_Mode,Dense> - : public TriangularBase > +template class TriangularViewImpl + : public TriangularBase > { public: - typedef TriangularView<_MatrixType, _Mode> TriangularViewType; + typedef TriangularView TriangularViewType; + typedef TriangularBase Base; typedef typename internal::traits::Scalar Scalar; - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef typename MatrixType::PlainObject DenseMatrixType; typedef DenseMatrixType PlainObject; @@ -362,7 +365,7 @@ template class TriangularViewImpl<_Mat typedef typename internal::traits::StorageKind StorageKind; enum { - Mode = _Mode, + Mode = Mode_, Flags = internal::traits::Flags }; @@ -728,10 +731,10 @@ struct evaluator_traits > template struct unary_evaluator, IndexBased> - : evaluator::type> + : evaluator> { typedef TriangularView XprType; - typedef evaluator::type> Base; + typedef evaluator> Base; EIGEN_DEVICE_FUNC unary_evaluator(const XprType &xpr) : Base(xpr.nestedExpression()) {} }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/VectorBlock.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/VectorBlock.h index 71c5b95eec1..64798c3da80 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/VectorBlock.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/VectorBlock.h @@ -11,6 +11,9 @@ #ifndef EIGEN_VECTORBLOCK_H #define EIGEN_VECTORBLOCK_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -66,8 +69,8 @@ template class VectorBlock }; public: EIGEN_DENSE_PUBLIC_INTERFACE(VectorBlock) - - using Base::operator=; + EIGEN_STATIC_ASSERT_VECTOR_ONLY(VectorBlock) + EIGEN_INHERIT_ASSIGNMENT_OPERATORS(VectorBlock) /** Dynamic-size constructor */ @@ -76,18 +79,14 @@ template class VectorBlock : Base(vector, IsColVector ? start : 0, IsColVector ? 0 : start, IsColVector ? size : 1, IsColVector ? 1 : size) - { - EIGEN_STATIC_ASSERT_VECTOR_ONLY(VectorBlock); - } + { } /** Fixed-size constructor */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE VectorBlock(VectorType& vector, Index start) : Base(vector, IsColVector ? start : 0, IsColVector ? 0 : start) - { - EIGEN_STATIC_ASSERT_VECTOR_ONLY(VectorBlock); - } + { } }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/VectorwiseOp.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/VectorwiseOp.h index 870f4f1e40b..caaaef6df91 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/VectorwiseOp.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/VectorwiseOp.h @@ -11,6 +11,9 @@ #ifndef EIGEN_PARTIAL_REDUX_H #define EIGEN_PARTIAL_REDUX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \class PartialReduxExpr @@ -86,7 +89,6 @@ template struct partial_redux_dummy_func; #define EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(MEMBER,COST,VECTORIZABLE,BINARYOP) \ template \ struct member_##MEMBER { \ - EIGEN_EMPTY_STRUCT_CTOR(member_##MEMBER) \ typedef ResultType result_type; \ typedef BINARYOP BinaryOp; \ template struct Cost { enum { value = COST }; }; \ @@ -191,7 +193,7 @@ template class VectorwiseOp typedef typename ExpressionType::RealScalar RealScalar; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 typedef typename internal::ref_selector::non_const_type ExpressionTypeNested; - typedef typename internal::remove_all::type ExpressionTypeNestedCleaned; + typedef internal::remove_all_t ExpressionTypeNestedCleaned; template class Functor, typename ReturnScalar=Scalar> struct ReturnType @@ -230,9 +232,9 @@ template class VectorwiseOp typename ExtendedType::Type extendedTo(const DenseBase& other) const { - EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(isVertical, OtherDerived::MaxColsAtCompileTime==1), + EIGEN_STATIC_ASSERT(internal::check_implication(isVertical, OtherDerived::MaxColsAtCompileTime==1), YOU_PASSED_A_ROW_VECTOR_BUT_A_COLUMN_VECTOR_WAS_EXPECTED) - EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(isHorizontal, OtherDerived::MaxRowsAtCompileTime==1), + EIGEN_STATIC_ASSERT(internal::check_implication(isHorizontal, OtherDerived::MaxRowsAtCompileTime==1), YOU_PASSED_A_COLUMN_VECTOR_BUT_A_ROW_VECTOR_WAS_EXPECTED) return typename ExtendedType::Type (other.derived(), @@ -253,9 +255,9 @@ template class VectorwiseOp typename OppositeExtendedType::Type extendedToOpposite(const DenseBase& other) const { - EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(isHorizontal, OtherDerived::MaxColsAtCompileTime==1), + EIGEN_STATIC_ASSERT(internal::check_implication(isHorizontal, OtherDerived::MaxColsAtCompileTime==1), YOU_PASSED_A_ROW_VECTOR_BUT_A_COLUMN_VECTOR_WAS_EXPECTED) - EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(isVertical, OtherDerived::MaxRowsAtCompileTime==1), + EIGEN_STATIC_ASSERT(internal::check_implication(isVertical, OtherDerived::MaxRowsAtCompileTime==1), YOU_PASSED_A_COLUMN_VECTOR_BUT_A_ROW_VECTOR_WAS_EXPECTED) return typename OppositeExtendedType::Type (other.derived(), @@ -349,8 +351,8 @@ template class VectorwiseOp typedef typename ReturnType::Type HypotNormReturnType; typedef typename ReturnType::Type SumReturnType; typedef EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(SumReturnType,Scalar,quotient) MeanReturnType; - typedef typename ReturnType::Type AllReturnType; - typedef typename ReturnType::Type AnyReturnType; + typedef typename ReturnType::Type AllReturnType; + typedef typename ReturnType::Type AnyReturnType; typedef PartialReduxExpr, Direction> CountReturnType; typedef typename ReturnType::Type ProdReturnType; typedef Reverse ConstReverseReturnType; @@ -594,7 +596,7 @@ template class VectorwiseOp return m_matrix += extendedTo(other.derived()); } - /** Substracts the vector \a other to each subvector of \c *this */ + /** Subtracts the vector \a other to each subvector of \c *this */ template EIGEN_DEVICE_FUNC ExpressionType& operator-=(const DenseBase& other) @@ -604,7 +606,7 @@ template class VectorwiseOp return m_matrix -= extendedTo(other.derived()); } - /** Multiples each subvector of \c *this by the vector \a other */ + /** Multiplies each subvector of \c *this by the vector \a other */ template EIGEN_DEVICE_FUNC ExpressionType& operator*=(const DenseBase& other) @@ -744,7 +746,7 @@ template class VectorwiseOp # endif protected: - Index redux_length() const + EIGEN_DEVICE_FUNC Index redux_length() const { return Direction==Vertical ? m_matrix.rows() : m_matrix.cols(); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Visitor.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Visitor.h index 00bcca87768..6fb00e02589 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Visitor.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/Visitor.h @@ -10,56 +10,304 @@ #ifndef EIGEN_VISITOR_H #define EIGEN_VISITOR_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template -struct visitor_impl -{ - enum { - col = (UnrollCount-1) / Derived::RowsAtCompileTime, - row = (UnrollCount-1) % Derived::RowsAtCompileTime - }; +template ::PacketAccess), bool LinearAccess = false, + bool ShortCircuitEvaluation = false> +struct visitor_impl; - EIGEN_DEVICE_FUNC - static inline void run(const Derived &mat, Visitor& visitor) - { - visitor_impl::run(mat, visitor); - visitor(mat.coeff(row, col), row, col); +template +struct short_circuit_eval_impl { + // if short circuit evaluation is not used, do nothing + static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(const Visitor&) { return false; } +}; +template +struct short_circuit_eval_impl { + // if short circuit evaluation is used, check the visitor + static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(const Visitor& visitor) { + return visitor.done(); } }; -template -struct visitor_impl -{ - EIGEN_DEVICE_FUNC - static inline void run(const Derived &mat, Visitor& visitor) +// unrolled inner-outer traversal +template +struct visitor_impl { + // don't use short circuit evaulation for unrolled version + using Scalar = typename Derived::Scalar; + using Packet = typename packet_traits::type; + static constexpr bool RowMajor = Derived::IsRowMajor; + static constexpr int RowsAtCompileTime = Derived::RowsAtCompileTime; + static constexpr int ColsAtCompileTime = Derived::ColsAtCompileTime; + static constexpr int PacketSize = packet_traits::size; + + static constexpr bool CanVectorize(int K) { + constexpr int InnerSizeAtCompileTime = RowMajor ? ColsAtCompileTime : RowsAtCompileTime; + if(InnerSizeAtCompileTime < PacketSize) return false; + return Vectorize && (InnerSizeAtCompileTime - (K % InnerSizeAtCompileTime) >= PacketSize); + } + + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived&, Visitor&) {} + + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) { - return visitor.init(mat.coeff(0, 0), 0, 0); + visitor.init(mat.coeff(0, 0), 0, 0); + run<1>(mat, visitor); + } + + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) + { + static constexpr int R = RowMajor ? (K / ColsAtCompileTime) : (K % RowsAtCompileTime); + static constexpr int C = RowMajor ? (K % ColsAtCompileTime) : (K / RowsAtCompileTime); + visitor(mat.coeff(R, C), R, C); + run(mat, visitor); + } + + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) + { + Packet P = mat.template packet(0, 0); + visitor.initpacket(P, 0, 0); + run(mat, visitor); + } + + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) + { + static constexpr int R = RowMajor ? (K / ColsAtCompileTime) : (K % RowsAtCompileTime); + static constexpr int C = RowMajor ? (K % ColsAtCompileTime) : (K / RowsAtCompileTime); + Packet P = mat.template packet(R, C); + visitor.packet(P, R, C); + run(mat, visitor); } }; -// This specialization enables visitors on empty matrices at compile-time -template -struct visitor_impl { - EIGEN_DEVICE_FUNC - static inline void run(const Derived &/*mat*/, Visitor& /*visitor*/) - {} +// unrolled linear traversal +template +struct visitor_impl { + // don't use short circuit evaulation for unrolled version + using Scalar = typename Derived::Scalar; + using Packet = typename packet_traits::type; + static constexpr int PacketSize = packet_traits::size; + + static constexpr bool CanVectorize(int K) { + return Vectorize && ((UnrollCount - K) >= PacketSize); + } + + // empty + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived&, Visitor&) {} + + // scalar initialization + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) { + visitor.init(mat.coeff(0), 0); + run<1>(mat, visitor); + } + + // scalar iteration + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) { + visitor(mat.coeff(K), K); + run(mat, visitor); + } + + // vector initialization + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) { + Packet P = mat.template packet(0); + visitor.initpacket(P, 0); + run(mat, visitor); + } + + // vector iteration + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) { + Packet P = mat.template packet(K); + visitor.packet(P, K); + run(mat, visitor); + } }; -template -struct visitor_impl -{ - EIGEN_DEVICE_FUNC - static inline void run(const Derived& mat, Visitor& visitor) - { - visitor.init(mat.coeff(0,0), 0, 0); - for(Index i = 1; i < mat.rows(); ++i) - visitor(mat.coeff(i, 0), i, 0); - for(Index j = 1; j < mat.cols(); ++j) - for(Index i = 0; i < mat.rows(); ++i) - visitor(mat.coeff(i, j), i, j); +// dynamic scalar outer-inner traversal +template +struct visitor_impl { + using short_circuit = short_circuit_eval_impl; + static constexpr bool RowMajor = Derived::IsRowMajor; + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) { + const Index innerSize = RowMajor ? mat.cols() : mat.rows(); + const Index outerSize = RowMajor ? mat.rows() : mat.cols(); + if (innerSize == 0 || outerSize == 0) return; + { + visitor.init(mat.coeff(0, 0), 0, 0); + if (short_circuit::run(visitor)) return; + for (Index i = 1; i < innerSize; ++i) { + Index r = RowMajor ? 0 : i; + Index c = RowMajor ? i : 0; + visitor(mat.coeff(r, c), r, c); + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + } + } + for (Index j = 1; j < outerSize; j++) { + for (Index i = 0; i < innerSize; ++i) { + Index r = RowMajor ? j : i; + Index c = RowMajor ? i : j; + visitor(mat.coeff(r, c), r, c); + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + } + } + } +}; + +// dynamic vectorized outer-inner traversal +template +struct visitor_impl { + using Scalar = typename Derived::Scalar; + using Packet = typename packet_traits::type; + static constexpr int PacketSize = packet_traits::size; + using short_circuit = short_circuit_eval_impl; + static constexpr bool RowMajor = Derived::IsRowMajor; + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) { + const Index innerSize = RowMajor ? mat.cols() : mat.rows(); + const Index outerSize = RowMajor ? mat.rows() : mat.cols(); + if (innerSize == 0 || outerSize == 0) return; + { + Index i = 0; + if (innerSize < PacketSize) { + visitor.init(mat.coeff(0, 0), 0, 0); + i = 1; + } else { + Packet p = mat.template packet(0, 0); + visitor.initpacket(p, 0, 0); + i = PacketSize; + } + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + for (; i + PacketSize - 1 < innerSize; i += PacketSize) { + Index r = RowMajor ? 0 : i; + Index c = RowMajor ? i : 0; + Packet p = mat.template packet(r, c); + visitor.packet(p, r, c); + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + } + for (; i < innerSize; ++i) { + Index r = RowMajor ? 0 : i; + Index c = RowMajor ? i : 0; + visitor(mat.coeff(r, c), r, c); + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + } + } + for (Index j = 1; j < outerSize; j++) { + Index i = 0; + for (; i + PacketSize - 1 < innerSize; i += PacketSize) { + Index r = RowMajor ? j : i; + Index c = RowMajor ? i : j; + Packet p = mat.template packet(r, c); + visitor.packet(p, r, c); + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + } + for (; i < innerSize; ++i) { + Index r = RowMajor ? j : i; + Index c = RowMajor ? i : j; + visitor(mat.coeff(r, c), r, c); + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + } + } + } +}; + +// dynamic scalar linear traversal +template +struct visitor_impl { + using short_circuit = short_circuit_eval_impl; + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) { + const Index size = mat.size(); + if (size == 0) return; + visitor.init(mat.coeff(0), 0); + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + for (Index k = 1; k < size; k++) { + visitor(mat.coeff(k), k); + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + } + } +}; + +// dynamic vectorized linear traversal +template +struct visitor_impl { + using Scalar = typename Derived::Scalar; + using Packet = typename packet_traits::type; + static constexpr int PacketSize = packet_traits::size; + using short_circuit = short_circuit_eval_impl; + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) { + const Index size = mat.size(); + if (size == 0) return; + Index k = 0; + if (size < PacketSize) { + visitor.init(mat.coeff(0), 0); + k = 1; + } else { + Packet p = mat.template packet(k); + visitor.initpacket(p, k); + k = PacketSize; + } + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + for (; k + PacketSize - 1 < size; k += PacketSize) { + Packet p = mat.template packet(k); + visitor.packet(p, k); + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + } + for (; k < size; k++) { + visitor(mat.coeff(k), k); + if EIGEN_PREDICT_FALSE(short_circuit::run(visitor)) return; + } } }; @@ -68,28 +316,77 @@ template class visitor_evaluator { public: - EIGEN_DEVICE_FUNC - explicit visitor_evaluator(const XprType &xpr) : m_evaluator(xpr), m_xpr(xpr) {} - + typedef evaluator Evaluator; typedef typename XprType::Scalar Scalar; - typedef typename XprType::CoeffReturnType CoeffReturnType; + using Packet = typename packet_traits::type; + typedef std::remove_const_t CoeffReturnType; - enum { - RowsAtCompileTime = XprType::RowsAtCompileTime, - CoeffReadCost = internal::evaluator::CoeffReadCost - }; + static constexpr bool PacketAccess = static_cast(Evaluator::Flags & PacketAccessBit); + static constexpr bool LinearAccess = static_cast(Evaluator::Flags & LinearAccessBit); + static constexpr bool IsRowMajor = static_cast(XprType::IsRowMajor); + static constexpr int RowsAtCompileTime = XprType::RowsAtCompileTime; + static constexpr int ColsAtCompileTime = XprType::ColsAtCompileTime; + static constexpr int XprAlignment = Evaluator::Alignment; + static constexpr int CoeffReadCost = Evaluator::CoeffReadCost; + + EIGEN_DEVICE_FUNC + explicit visitor_evaluator(const XprType &xpr) : m_evaluator(xpr), m_xpr(xpr) { } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_xpr.rows(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_xpr.cols(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index size() const EIGEN_NOEXCEPT { return m_xpr.size(); } - - EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const - { return m_evaluator.coeff(row, col); } + // outer-inner access + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { return m_evaluator.coeff(row, col); } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(Index row, Index col) const { + return m_evaluator.template packet(row, col); + } + // linear access + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_evaluator.coeff(index); } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(Index index) const { + return m_evaluator.template packet(index); + } protected: - internal::evaluator m_evaluator; + Evaluator m_evaluator; const XprType &m_xpr; }; + +template +struct visit_impl { + using Evaluator = visitor_evaluator; + using Scalar = typename DenseBase::Scalar; + + static constexpr bool IsRowMajor = DenseBase::IsRowMajor; + static constexpr int SizeAtCompileTime = DenseBase::SizeAtCompileTime; + static constexpr int RowsAtCompileTime = DenseBase::RowsAtCompileTime; + static constexpr int ColsAtCompileTime = DenseBase::ColsAtCompileTime; + static constexpr int InnerSizeAtCompileTime = IsRowMajor ? ColsAtCompileTime : RowsAtCompileTime; + static constexpr int OuterSizeAtCompileTime = IsRowMajor ? RowsAtCompileTime : ColsAtCompileTime; + + static constexpr bool LinearAccess = Evaluator::LinearAccess && static_cast(functor_traits::LinearAccess); + static constexpr bool Vectorize = Evaluator::PacketAccess && static_cast(functor_traits::PacketAccess); + + static constexpr int PacketSize = packet_traits::size; + static constexpr int VectorOps = Vectorize ? (LinearAccess ? (SizeAtCompileTime / PacketSize) : (OuterSizeAtCompileTime * (InnerSizeAtCompileTime / PacketSize))) : 0; + static constexpr int ScalarOps = SizeAtCompileTime - (VectorOps * PacketSize); + // treat vector op and scalar op as same cost for unroll logic + static constexpr int TotalOps = VectorOps + ScalarOps; + + static constexpr int UnrollCost = int(Evaluator::CoeffReadCost) + int(functor_traits::Cost); + static constexpr bool Unroll = (SizeAtCompileTime != Dynamic) && ((TotalOps * UnrollCost) <= EIGEN_UNROLLING_LIMIT); + static constexpr int UnrollCount = Unroll ? int(SizeAtCompileTime) : Dynamic; + + + using impl = visitor_impl; + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const DenseBase& mat, Visitor& visitor) { + Evaluator evaluator(mat.derived()); + impl::run(evaluator, visitor); + } +}; + } // end namespace internal /** Applies the visitor \a visitor to the whole coefficients of the matrix or vector. @@ -116,17 +413,8 @@ template EIGEN_DEVICE_FUNC void DenseBase::visit(Visitor& visitor) const { - if(size()==0) - return; - - typedef typename internal::visitor_evaluator ThisEvaluator; - ThisEvaluator thisEval(derived()); - - enum { - unroll = SizeAtCompileTime != Dynamic - && SizeAtCompileTime * int(ThisEvaluator::CoeffReadCost) + (SizeAtCompileTime-1) * int(internal::functor_traits::Cost) <= EIGEN_UNROLLING_LIMIT - }; - return internal::visitor_impl::run(thisEval, visitor); + using impl = internal::visit_impl; + impl::run(derived(), visitor); } namespace internal { @@ -152,123 +440,237 @@ struct coeff_visitor } }; -/** \internal - * \brief Visitor computing the min coefficient with its value and coordinates - * - * \sa DenseBase::minCoeff(Index*, Index*) - */ -template -struct min_coeff_visitor : coeff_visitor -{ - typedef typename Derived::Scalar Scalar; - EIGEN_DEVICE_FUNC - void operator() (const Scalar& value, Index i, Index j) - { - if(value < this->res) - { + +template +struct minmax_compare { + typedef typename packet_traits::type Packet; + static EIGEN_DEVICE_FUNC inline bool compare(Scalar a, Scalar b) { return a < b; } + static EIGEN_DEVICE_FUNC inline Scalar predux(const Packet& p) { return predux_min(p); } +}; + +template +struct minmax_compare { + typedef typename packet_traits::type Packet; + static EIGEN_DEVICE_FUNC inline bool compare(Scalar a, Scalar b) { return a > b; } + static EIGEN_DEVICE_FUNC inline Scalar predux(const Packet& p) { return predux_max(p); } +}; + +// Default implementation used by non-floating types, where we do not +// need special logic for NaN handling. +template ::IsInteger> +struct minmax_coeff_visitor : coeff_visitor { + using Scalar = typename Derived::Scalar; + using Packet = typename packet_traits::type; + using Comparator = minmax_compare; + static constexpr Index PacketSize = packet_traits::size; + + EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index i, Index j) { + if (Comparator::compare(value, this->res)) { this->res = value; this->row = i; this->col = j; } } + EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index i, Index j) { + Scalar value = Comparator::predux(p); + if (Comparator::compare(value, this->res)) { + const Packet range = preverse(plset(Scalar(1))); + Packet mask = pcmp_eq(pset1(value), p); + Index max_idx = PacketSize - static_cast(predux_max(pand(range, mask))); + this->res = value; + this->row = Derived::IsRowMajor ? i : i + max_idx; + this->col = Derived::IsRowMajor ? j + max_idx : j; + } + } + EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index i, Index j) { + Scalar value = Comparator::predux(p); + const Packet range = preverse(plset(Scalar(1))); + Packet mask = pcmp_eq(pset1(value), p); + Index max_idx = PacketSize - static_cast(predux_max(pand(range, mask))); + this->res = value; + this->row = Derived::IsRowMajor ? i : i + max_idx; + this->col = Derived::IsRowMajor ? j + max_idx : j; + } }; -template -struct min_coeff_visitor : coeff_visitor -{ +// Suppress NaN. The only case in which we return NaN is if the matrix is all NaN, +// in which case, row=0, col=0 is returned for the location. +template +struct minmax_coeff_visitor : coeff_visitor { typedef typename Derived::Scalar Scalar; - EIGEN_DEVICE_FUNC - void operator() (const Scalar& value, Index i, Index j) - { - if((numext::isnan)(this->res) || (!(numext::isnan)(value) && value < this->res)) - { + using Packet = typename packet_traits::type; + using Comparator = minmax_compare; + + EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index i, Index j) { + if ((!(numext::isnan)(value) && (numext::isnan)(this->res)) || Comparator::compare(value, this->res)) { this->res = value; this->row = i; this->col = j; } } + EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index i, Index j) { + const Index PacketSize = packet_traits::size; + Scalar value = Comparator::predux(p); + if ((!(numext::isnan)(value) && (numext::isnan)(this->res)) || Comparator::compare(value, this->res)) { + const Packet range = preverse(plset(Scalar(1))); + /* mask will be zero for NaNs, so they will be ignored. */ + Packet mask = pcmp_eq(pset1(value), p); + Index max_idx = PacketSize - static_cast(predux_max(pand(range, mask))); + this->res = value; + this->row = Derived::IsRowMajor ? i : i + max_idx; + this->col = Derived::IsRowMajor ? j + max_idx : j; + } + } + EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index i, Index j) { + const Index PacketSize = packet_traits::size; + Scalar value = Comparator::predux(p); + if ((numext::isnan)(value)) { + this->res = value; + this->row = 0; + this->col = 0; + return; + } + const Packet range = preverse(plset(Scalar(1))); + /* mask will be zero for NaNs, so they will be ignored. */ + Packet mask = pcmp_eq(pset1(value), p); + Index max_idx = PacketSize - static_cast(predux_max(pand(range, mask))); + this->res = value; + this->row = Derived::IsRowMajor ? i : i + max_idx; + this->col = Derived::IsRowMajor ? j + max_idx : j; + } }; -template -struct min_coeff_visitor : coeff_visitor -{ +// Propagate NaNs. If the matrix contains NaN, the location of the first NaN +// will be returned in row and col. +template + struct minmax_coeff_visitor : coeff_visitor { typedef typename Derived::Scalar Scalar; - EIGEN_DEVICE_FUNC - void operator() (const Scalar& value, Index i, Index j) - { - if((numext::isnan)(value) || value < this->res) - { + using Packet = typename packet_traits::type; + using Comparator = minmax_compare; + + EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index i, Index j) { + const bool value_is_nan = (numext::isnan)(value); + if ((value_is_nan && !(numext::isnan)(this->res)) || Comparator::compare(value, this->res)) { this->res = value; this->row = i; this->col = j; } } + EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index i, Index j) { + const Index PacketSize = packet_traits::size; + Scalar value = Comparator::predux(p); + const bool value_is_nan = (numext::isnan)(value); + if ((value_is_nan && !(numext::isnan)(this->res)) || Comparator::compare(value, this->res)) { + const Packet range = preverse(plset(Scalar(1))); + // If the value is NaN, pick the first position of a NaN, otherwise pick the first extremal value. + Packet mask = value_is_nan ? pnot(pcmp_eq(p, p)) : pcmp_eq(pset1(value), p); + Index max_idx = PacketSize - static_cast(predux_max(pand(range, mask))); + this->res = value; + this->row = Derived::IsRowMajor ? i : i + max_idx; + this->col = Derived::IsRowMajor ? j + max_idx : j; + } + } + EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index i, Index j) { + const Index PacketSize = packet_traits::size; + Scalar value = Comparator::predux(p); + const bool value_is_nan = (numext::isnan)(value); + const Packet range = preverse(plset(Scalar(1))); + // If the value is NaN, pick the first position of a NaN, otherwise pick the first extremal value. + Packet mask = value_is_nan ? pnot(pcmp_eq(p, p)) : pcmp_eq(pset1(value), p); + Index max_idx = PacketSize - static_cast(predux_max(pand(range, mask))); + this->res = value; + this->row = Derived::IsRowMajor ? i : i + max_idx; + this->col = Derived::IsRowMajor ? j + max_idx : j; + } }; -template - struct functor_traits > { +template +struct functor_traits > { + using Scalar = typename Derived::Scalar; enum { - Cost = NumTraits::AddCost + Cost = NumTraits::AddCost, + LinearAccess = false, + PacketAccess = packet_traits::HasCmp }; }; -/** \internal - * \brief Visitor computing the max coefficient with its value and coordinates - * - * \sa DenseBase::maxCoeff(Index*, Index*) - */ -template -struct max_coeff_visitor : coeff_visitor -{ - typedef typename Derived::Scalar Scalar; - EIGEN_DEVICE_FUNC - void operator() (const Scalar& value, Index i, Index j) - { - if(value > this->res) - { - this->res = value; - this->row = i; - this->col = j; - } - } +template +struct all_visitor { + using result_type = bool; + using Packet = typename packet_traits::type; + EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = (value != Scalar(0)); } + EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = (value != Scalar(0)); } + EIGEN_DEVICE_FUNC inline bool all_predux(const Packet& p) const { return !predux_any(pcmp_eq(p, pzero(p))); } + EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = all_predux(p); } + EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = all_predux(p); } + EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) { res = res && (value != Scalar(0)); } + EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index) { res = res && (value != Scalar(0)); } + EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index, Index) { res = res && all_predux(p); } + EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index) { res = res && all_predux(p); } + EIGEN_DEVICE_FUNC inline bool done() const { return !res; } + bool res = true; +}; +template +struct functor_traits> { + enum { Cost = NumTraits::ReadCost, LinearAccess = true, PacketAccess = packet_traits::HasCmp }; }; -template -struct max_coeff_visitor : coeff_visitor -{ - typedef typename Derived::Scalar Scalar; - EIGEN_DEVICE_FUNC - void operator() (const Scalar& value, Index i, Index j) - { - if((numext::isnan)(this->res) || (!(numext::isnan)(value) && value > this->res)) - { - this->res = value; - this->row = i; - this->col = j; - } +template +struct any_visitor { + using result_type = bool; + using Packet = typename packet_traits::type; + EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = (value != Scalar(0)); } + EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = (value != Scalar(0)); } + EIGEN_DEVICE_FUNC inline bool any_predux(const Packet& p) const { + return predux_any(pandnot(ptrue(p), pcmp_eq(p, pzero(p)))); } + EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = any_predux(p); } + EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = any_predux(p); } + EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) { res = res || (value != Scalar(0)); } + EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index) { res = res || (value != Scalar(0)); } + EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index, Index) { res = res || any_predux(p); } + EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index) { res = res || any_predux(p); } + EIGEN_DEVICE_FUNC inline bool done() const { return res; } + bool res = false; +}; +template +struct functor_traits> { + enum { Cost = NumTraits::ReadCost, LinearAccess = true, PacketAccess = packet_traits::HasCmp }; }; -template -struct max_coeff_visitor : coeff_visitor -{ - typedef typename Derived::Scalar Scalar; - EIGEN_DEVICE_FUNC - void operator() (const Scalar& value, Index i, Index j) - { - if((numext::isnan)(value) || value > this->res) - { - this->res = value; - this->row = i; - this->col = j; - } +template +struct count_visitor { + using result_type = Index; + using Packet = typename packet_traits::type; + EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = value != Scalar(0) ? 1 : 0; } + EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = value != Scalar(0) ? 1 : 0; } + EIGEN_DEVICE_FUNC inline Index count_redux(const Packet& p) const { + const Packet cst_one = pset1(Scalar(1)); + Packet true_vals = pandnot(cst_one, pcmp_eq(p, pzero(p))); + Scalar num_true = predux(true_vals); + return static_cast(num_true); + } + EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = count_redux(p); } + EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = count_redux(p); } + EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) { + if (value != Scalar(0)) res++; } + EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index) { + if (value != Scalar(0)) res++; + } + EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index, Index) { res += count_redux(p); } + EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index) { res += count_redux(p); } + Index res = 0; }; -template -struct functor_traits > { +template +struct functor_traits> { enum { - Cost = NumTraits::AddCost + Cost = NumTraits::AddCost, + LinearAccess = true, + // predux is problematic for bool + PacketAccess = packet_traits::HasCmp && packet_traits::HasAdd && !is_same::value }; }; @@ -293,7 +695,7 @@ DenseBase::minCoeff(IndexType* rowId, IndexType* colId) const { eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); - internal::min_coeff_visitor minVisitor; + internal::minmax_coeff_visitor minVisitor; this->visit(minVisitor); *rowId = minVisitor.row; if (colId) *colId = minVisitor.col; @@ -316,10 +718,10 @@ EIGEN_DEVICE_FUNC typename internal::traits::Scalar DenseBase::minCoeff(IndexType* index) const { - eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); - + eigen_assert(this->rows() > 0 && this->cols() > 0 && "you are using an empty matrix"); EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - internal::min_coeff_visitor minVisitor; + + internal::minmax_coeff_visitor minVisitor; this->visit(minVisitor); *index = IndexType((RowsAtCompileTime==1) ? minVisitor.col : minVisitor.row); return minVisitor.res; @@ -344,7 +746,7 @@ DenseBase::maxCoeff(IndexType* rowPtr, IndexType* colPtr) const { eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); - internal::max_coeff_visitor maxVisitor; + internal::minmax_coeff_visitor maxVisitor; this->visit(maxVisitor); *rowPtr = maxVisitor.row; if (colPtr) *colPtr = maxVisitor.col; @@ -370,12 +772,71 @@ DenseBase::maxCoeff(IndexType* index) const eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - internal::max_coeff_visitor maxVisitor; + internal::minmax_coeff_visitor maxVisitor; this->visit(maxVisitor); *index = (RowsAtCompileTime==1) ? maxVisitor.col : maxVisitor.row; return maxVisitor.res; } +/** \returns true if all coefficients are true + * + * Example: \include MatrixBase_all.cpp + * Output: \verbinclude MatrixBase_all.out + * + * \sa any(), Cwise::operator<() + */ +template +EIGEN_DEVICE_FUNC inline bool DenseBase::all() const { + using Visitor = internal::all_visitor; + using impl = internal::visit_impl; + Visitor visitor; + impl::run(derived(), visitor); + return visitor.res; +} + +/** \returns true if at least one coefficient is true + * + * \sa all() + */ +template +EIGEN_DEVICE_FUNC inline bool DenseBase::any() const { + using Visitor = internal::any_visitor; + using impl = internal::visit_impl; + Visitor visitor; + impl::run(derived(), visitor); + return visitor.res; +} + +/** \returns the number of coefficients which evaluate to true + * + * \sa all(), any() + */ +template +EIGEN_DEVICE_FUNC +Index DenseBase::count() const +{ + using Visitor = internal::count_visitor; + using impl = internal::visit_impl; + Visitor visitor; + impl::run(derived(), visitor); + return visitor.res; + +} + +template +EIGEN_DEVICE_FUNC inline bool DenseBase::hasNaN() const { + return derived().cwiseTypedNotEqual(derived()).any(); +} + +/** \returns true if \c *this contains only finite numbers, i.e., no NaN and no +/-INF values. + * + * \sa hasNaN() + */ +template +EIGEN_DEVICE_FUNC inline bool DenseBase::allFinite() const { + return derived().array().isFinite().all(); +} + } // end namespace Eigen #endif // EIGEN_VISITOR_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/Complex.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/Complex.h index e9096c0a1ac..238edc86c0b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/Complex.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/Complex.h @@ -10,6 +10,9 @@ #ifndef EIGEN_COMPLEX_AVX_H #define EIGEN_COMPLEX_AVX_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -31,7 +34,6 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 1, HasAdd = 1, HasSub = 1, @@ -169,15 +171,12 @@ template<> EIGEN_STRONG_INLINE std::complex predux_mul(const P Packet2cf(_mm256_extractf128_ps(a.v, 1)))); } + EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet4cf,Packet8f) template<> EIGEN_STRONG_INLINE Packet4cf pdiv(const Packet4cf& a, const Packet4cf& b) { - Packet4cf num = pmul(a, pconj(b)); - __m256 tmp = _mm256_mul_ps(b.v, b.v); - __m256 tmp2 = _mm256_shuffle_ps(tmp,tmp,0xB1); - __m256 denom = _mm256_add_ps(tmp, tmp2); - return Packet4cf(_mm256_div_ps(num.v, denom)); + return pdiv_complex(a, b); } template<> EIGEN_STRONG_INLINE Packet4cf pcplxflip(const Packet4cf& x) @@ -202,7 +201,6 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 0, size = 2, - HasHalfPacket = 1, HasAdd = 1, HasSub = 1, @@ -323,10 +321,7 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cd,Packet4d) template<> EIGEN_STRONG_INLINE Packet2cd pdiv(const Packet2cd& a, const Packet2cd& b) { - Packet2cd num = pmul(a, pconj(b)); - __m256d tmp = _mm256_mul_pd(b.v, b.v); - __m256d denom = _mm256_hadd_pd(tmp, tmp); - return Packet2cd(_mm256_div_pd(num.v, denom)); + return pdiv_complex(a, b); } template<> EIGEN_STRONG_INLINE Packet2cd pcplxflip(const Packet2cd& x) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/MathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/MathFunctions.h index 67041c812c3..6e83cfc4936 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/MathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/MathFunctions.h @@ -14,175 +14,48 @@ * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ */ +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f -psin(const Packet8f& _x) { - return psin_float(_x); -} - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f -pcos(const Packet8f& _x) { - return pcos_float(_x); -} - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f -plog(const Packet8f& _x) { - return plog_float(_x); -} - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4d -plog(const Packet4d& _x) { - return plog_double(_x); -} - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f -plog2(const Packet8f& _x) { - return plog2_float(_x); -} - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4d -plog2(const Packet4d& _x) { - return plog2_double(_x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet8f plog1p(const Packet8f& _x) { - return generic_plog1p(_x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet8f pexpm1(const Packet8f& _x) { - return generic_expm1(_x); -} - -// Exponential function. Works by writing "x = m*log(2) + r" where -// "m = floor(x/log(2)+1/2)" and "r" is the remainder. The result is then -// "exp(x) = 2^m*exp(r)" where exp(r) is in the range [-1,1). -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f -pexp(const Packet8f& _x) { - return pexp_float(_x); -} - -// Hyperbolic Tangent function. -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f -ptanh(const Packet8f& _x) { - return internal::generic_fast_tanh_float(_x); -} - -// Exponential function for doubles. -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4d -pexp(const Packet4d& _x) { - return pexp_double(_x); -} - -// Functions for sqrt. -// The EIGEN_FAST_MATH version uses the _mm_rsqrt_ps approximation and one step -// of Newton's method, at a cost of 1-2 bits of precision as opposed to the -// exact solution. It does not handle +inf, or denormalized numbers correctly. -// The main advantage of this approach is not just speed, but also the fact that -// it can be inlined and pipelined with other computations, further reducing its -// effective latency. This is similar to Quake3's fast inverse square root. -// For detail see here: http://www.beyond3d.com/content/articles/8/ -#if EIGEN_FAST_MATH -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet8f psqrt(const Packet8f& _x) { - Packet8f minus_half_x = pmul(_x, pset1(-0.5f)); - Packet8f denormal_mask = pandnot( - pcmp_lt(_x, pset1((std::numeric_limits::min)())), - pcmp_lt(_x, pzero(_x))); - - // Compute approximate reciprocal sqrt. - Packet8f x = _mm256_rsqrt_ps(_x); - // Do a single step of Newton's iteration. - x = pmul(x, pmadd(minus_half_x, pmul(x,x), pset1(1.5f))); - // Flush results for denormals to zero. - return pandnot(pmul(_x,x), denormal_mask); -} - -#else +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet8f) +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE(Packet4d) -template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +// Notice that for newer processors, it is counterproductive to use Newton +// iteration for square root. In particular, Skylake and Zen2 processors +// have approximately doubled throughput of the _mm_sqrt_ps instruction +// compared to their predecessors. +template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8f psqrt(const Packet8f& _x) { return _mm256_sqrt_ps(_x); } - -#endif - -template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4d psqrt(const Packet4d& _x) { return _mm256_sqrt_pd(_x); } -#if EIGEN_FAST_MATH -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet8f prsqrt(const Packet8f& _x) { - _EIGEN_DECLARE_CONST_Packet8f_FROM_INT(inf, 0x7f800000); - _EIGEN_DECLARE_CONST_Packet8f(one_point_five, 1.5f); - _EIGEN_DECLARE_CONST_Packet8f(minus_half, -0.5f); - _EIGEN_DECLARE_CONST_Packet8f_FROM_INT(flt_min, 0x00800000); - - Packet8f neg_half = pmul(_x, p8f_minus_half); - - // select only the inverse sqrt of positive normal inputs (denormals are - // flushed to zero and cause infs as well). - Packet8f lt_min_mask = _mm256_cmp_ps(_x, p8f_flt_min, _CMP_LT_OQ); - Packet8f inf_mask = _mm256_cmp_ps(_x, p8f_inf, _CMP_EQ_OQ); - Packet8f not_normal_finite_mask = _mm256_or_ps(lt_min_mask, inf_mask); - - // Compute an approximate result using the rsqrt intrinsic. - Packet8f y_approx = _mm256_rsqrt_ps(_x); - - // Do a single step of Newton-Raphson iteration to improve the approximation. - // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n). - // It is essential to evaluate the inner term like this because forming - // y_n^2 may over- or underflow. - Packet8f y_newton = pmul(y_approx, pmadd(y_approx, pmul(neg_half, y_approx), p8f_one_point_five)); - - // Select the result of the Newton-Raphson step for positive normal arguments. - // For other arguments, choose the output of the intrinsic. This will - // return rsqrt(+inf) = 0, rsqrt(x) = NaN if x < 0, and rsqrt(x) = +inf if - // x is zero or a positive denormalized float (equivalent to flushing positive - // denormalized inputs to zero). - return pselect(not_normal_finite_mask, y_approx, y_newton); -} -#else -template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet8f prsqrt(const Packet8f& _x) { - _EIGEN_DECLARE_CONST_Packet8f(one, 1.0f); - return _mm256_div_ps(p8f_one, _mm256_sqrt_ps(_x)); +// Even on Skylake, using Newton iteration is a win for reciprocal square root. +#if EIGEN_FAST_MATH +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet8f prsqrt(const Packet8f& a) { + // _mm256_rsqrt_ps returns -inf for negative denormals. + // _mm512_rsqrt**_ps returns -NaN for negative denormals. We may want + // consistency here. + // const Packet8f rsqrt = pselect(pcmp_lt(a, pzero(a)), + // pset1(-NumTraits::quiet_NaN()), + // _mm256_rsqrt_ps(a)); + return generic_rsqrt_newton_step::run(a, _mm256_rsqrt_ps(a)); } -#endif -template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4d prsqrt(const Packet4d& _x) { - _EIGEN_DECLARE_CONST_Packet4d(one, 1.0); - return _mm256_div_pd(p4d_one, _mm256_sqrt_pd(_x)); +template<> EIGEN_STRONG_INLINE Packet8f preciprocal(const Packet8f& a) { + return generic_reciprocal_newton_step::run(a, _mm256_rcp_ps(a)); } -F16_PACKET_FUNCTION(Packet8f, Packet8h, psin) -F16_PACKET_FUNCTION(Packet8f, Packet8h, pcos) -F16_PACKET_FUNCTION(Packet8f, Packet8h, plog) -F16_PACKET_FUNCTION(Packet8f, Packet8h, plog2) -F16_PACKET_FUNCTION(Packet8f, Packet8h, plog1p) -F16_PACKET_FUNCTION(Packet8f, Packet8h, pexpm1) -F16_PACKET_FUNCTION(Packet8f, Packet8h, pexp) -F16_PACKET_FUNCTION(Packet8f, Packet8h, ptanh) -F16_PACKET_FUNCTION(Packet8f, Packet8h, psqrt) -F16_PACKET_FUNCTION(Packet8f, Packet8h, prsqrt) +#endif template <> EIGEN_STRONG_INLINE Packet8h pfrexp(const Packet8h& a, Packet8h& exponent) { @@ -197,17 +70,6 @@ EIGEN_STRONG_INLINE Packet8h pldexp(const Packet8h& a, const Packet8h& exponent) return float2half(pldexp(half2float(a), half2float(exponent))); } -BF16_PACKET_FUNCTION(Packet8f, Packet8bf, psin) -BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pcos) -BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog) -BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog2) -BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog1p) -BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pexpm1) -BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pexp) -BF16_PACKET_FUNCTION(Packet8f, Packet8bf, ptanh) -BF16_PACKET_FUNCTION(Packet8f, Packet8bf, psqrt) -BF16_PACKET_FUNCTION(Packet8f, Packet8bf, prsqrt) - template <> EIGEN_STRONG_INLINE Packet8bf pfrexp(const Packet8bf& a, Packet8bf& exponent) { Packet8f fexponent; @@ -221,6 +83,30 @@ EIGEN_STRONG_INLINE Packet8bf pldexp(const Packet8bf& a, const Packet8bf& expone return F32ToBf16(pldexp(Bf16ToF32(a), Bf16ToF32(exponent))); } +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pcos) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pexp) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pexpm1) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog1p) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog2) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, preciprocal) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, prsqrt) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, psin) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, psqrt) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, ptanh) +F16_PACKET_FUNCTION(Packet8f, Packet8h, pcos) +F16_PACKET_FUNCTION(Packet8f, Packet8h, pexp) +F16_PACKET_FUNCTION(Packet8f, Packet8h, pexpm1) +F16_PACKET_FUNCTION(Packet8f, Packet8h, plog) +F16_PACKET_FUNCTION(Packet8f, Packet8h, plog1p) +F16_PACKET_FUNCTION(Packet8f, Packet8h, plog2) +F16_PACKET_FUNCTION(Packet8f, Packet8h, preciprocal) +F16_PACKET_FUNCTION(Packet8f, Packet8h, prsqrt) +F16_PACKET_FUNCTION(Packet8f, Packet8h, psin) +F16_PACKET_FUNCTION(Packet8f, Packet8h, psqrt) +F16_PACKET_FUNCTION(Packet8f, Packet8h, ptanh) + + } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/PacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/PacketMath.h index 7fc32fd7192..6f37ba0a0e1 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/PacketMath.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/PacketMath.h @@ -10,6 +10,9 @@ #ifndef EIGEN_PACKET_MATH_AVX_H #define EIGEN_PACKET_MATH_AVX_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -29,28 +32,39 @@ namespace internal { #endif typedef __m256 Packet8f; -typedef __m256i Packet8i; +typedef eigen_packet_wrapper<__m256i, 0> Packet8i; typedef __m256d Packet4d; +#ifndef EIGEN_VECTORIZE_AVX512FP16 typedef eigen_packet_wrapper<__m128i, 2> Packet8h; +#endif typedef eigen_packet_wrapper<__m128i, 3> Packet8bf; +typedef eigen_packet_wrapper<__m256i, 4> Packet8ui; + +#ifdef EIGEN_VECTORIZE_AVX2 +// Start from 3 to be compatible with AVX512 +typedef eigen_packet_wrapper<__m256i, 3> Packet4l; +typedef eigen_packet_wrapper<__m256i, 5> Packet4ul; +#endif template<> struct is_arithmetic<__m256> { enum { value = true }; }; template<> struct is_arithmetic<__m256i> { enum { value = true }; }; template<> struct is_arithmetic<__m256d> { enum { value = true }; }; +template<> struct is_arithmetic { enum { value = true }; }; +// Note that `Packet8ui` uses the underlying type `__m256i`, which is +// interpreted as a vector of _signed_ `int32`s, which breaks some arithmetic +// operations used in `GenericPacketMath.h`. +template<> struct is_arithmetic { enum { value = false }; }; +#ifndef EIGEN_VECTORIZE_AVX512FP16 template<> struct is_arithmetic { enum { value = true }; }; +#endif template<> struct is_arithmetic { enum { value = true }; }; - -#define _EIGEN_DECLARE_CONST_Packet8f(NAME,X) \ - const Packet8f p8f_##NAME = pset1(X) - -#define _EIGEN_DECLARE_CONST_Packet4d(NAME,X) \ - const Packet4d p4d_##NAME = pset1(X) - -#define _EIGEN_DECLARE_CONST_Packet8f_FROM_INT(NAME,X) \ - const Packet8f p8f_##NAME = _mm256_castsi256_ps(pset1(X)) - -#define _EIGEN_DECLARE_CONST_Packet8i(NAME,X) \ - const Packet8i p8i_##NAME = pset1(X) +#ifdef EIGEN_VECTORIZE_AVX2 +template<> struct is_arithmetic { enum { value = true }; }; +// Note that `Packet4ul` uses the underlying type `__m256i`, which is +// interpreted as a vector of _signed_ `int32`s, which breaks some arithmetic +// operations used in `GenericPacketMath.h`. +template<> struct is_arithmetic { enum { value = false }; }; +#endif // Use the packet_traits defined in AVX512/PacketMath.h instead if we're going // to leverage AVX512 instructions. @@ -63,12 +77,16 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasHalfPacket = 1, HasCmp = 1, HasDiv = 1, + HasReciprocal = EIGEN_FAST_MATH, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, + HasACos = 1, + HasASin = 1, + HasATan = 1, + HasATanh = 1, HasLog = 1, HasLog1p = 1, HasExpm1 = 1, @@ -94,7 +112,6 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size=4, - HasHalfPacket = 1, HasCmp = 1, HasDiv = 1, @@ -102,6 +119,7 @@ template<> struct packet_traits : default_packet_traits HasExp = 1, HasSqrt = 1, HasRsqrt = 1, + HasATan = 1, HasBlend = 1, HasRound = 1, HasFloor = 1, @@ -119,7 +137,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, @@ -163,7 +180,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, @@ -196,38 +212,125 @@ struct packet_traits : default_packet_traits { HasNdtri = 1 }; }; -#endif -template<> struct scalar_div_cost { enum { value = 14 }; }; -template<> struct scalar_div_cost { enum { value = 16 }; }; - -/* Proper support for integers is only provided by AVX2. In the meantime, we'll - use SSE instructions and packets to deal with integers. -template<> struct packet_traits : default_packet_traits +template<> struct packet_traits : default_packet_traits { typedef Packet8i type; + typedef Packet4i half; enum { Vectorizable = 1, AlignedOnScalar = 1, + HasCmp = 1, + HasDiv = 1, size=8 }; }; -*/ +template<> struct packet_traits : default_packet_traits +{ + typedef Packet8ui type; + typedef Packet4ui half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 8, + + HasDiv = 0, + HasNegate = 0, + HasSqrt = 0, + + HasCmp = 1, + HasMin = 1, + HasMax = 1, + HasShift = 1 + }; +}; + +#ifdef EIGEN_VECTORIZE_AVX2 +template<> struct packet_traits : default_packet_traits +{ + typedef Packet4l type; + // There is no half-size packet for current Packet4l. + // TODO: support as SSE path. + typedef Packet4l half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + HasCmp = 1, + size=4 + }; +}; +template<> struct packet_traits : default_packet_traits +{ + typedef Packet4ul type; + // There is no half-size packet for current Packet4ul. + // TODO: support as SSE path. + typedef Packet4ul half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 4, + + // HasMin = 0, + // HasMax = 0, + HasDiv = 0, + HasBlend = 0, + HasTranspose = 0, + HasNegate = 0, + HasSqrt = 0, + HasCmp = 1, + HasShift = 1 + }; +}; +#endif + +#endif + +template<> struct scalar_div_cost { enum { value = 14 }; }; +template<> struct scalar_div_cost { enum { value = 16 }; }; template<> struct unpacket_traits { typedef float type; typedef Packet4f half; typedef Packet8i integer_packet; typedef uint8_t mask_t; - enum {size=8, alignment=Aligned32, vectorizable=true, masked_load_available=true, masked_store_available=true}; + enum {size=8, alignment=Aligned32, vectorizable=true, masked_load_available=true, masked_store_available=true +#ifdef EIGEN_VECTORIZE_AVX512 + , masked_fpops_available=true +#endif + }; }; template<> struct unpacket_traits { typedef double type; typedef Packet2d half; enum {size=4, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; -template<> struct unpacket_traits { typedef int type; typedef Packet4i half; enum {size=8, alignment=Aligned32, vectorizable=false, masked_load_available=false, masked_store_available=false}; }; -template<> struct unpacket_traits { typedef bfloat16 type; typedef Packet8bf half; enum {size=8, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; +template<> struct unpacket_traits { + typedef int type; + typedef Packet4i half; + enum {size=8, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false}; +}; +template<> struct unpacket_traits { + typedef uint32_t type; + typedef Packet4ui half; + enum {size = 8, alignment = Aligned32, vectorizable = true, masked_load_available = false, masked_store_available = false}; +}; +#ifdef EIGEN_VECTORIZE_AVX2 +template<> struct unpacket_traits { + typedef int64_t type; + typedef Packet4l half; + enum {size=4, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false}; +}; +template<> struct unpacket_traits { + typedef uint64_t type; + typedef Packet4ul half; + enum {size = 4, alignment = Aligned32, vectorizable = true, masked_load_available = false, masked_store_available = false}; +}; +#endif +template<> struct unpacket_traits { + typedef bfloat16 type; + typedef Packet8bf half; + enum {size=8, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; +}; // Helper function for bit packing snippet of low precision comparison. // It packs the flags from 16x16 to 8x16. @@ -236,10 +339,338 @@ EIGEN_STRONG_INLINE __m128i Pack16To8(Packet8f rf) { _mm256_extractf128_si256(_mm256_castps_si256(rf), 1)); } +#ifdef EIGEN_VECTORIZE_AVX2 +template <> +EIGEN_STRONG_INLINE Packet4l pset1(const int64_t& from) { + return _mm256_set1_epi64x(from); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pset1(const uint64_t& from) { + return _mm256_set1_epi64x(numext::bit_cast(from)); +} +template <> +EIGEN_STRONG_INLINE Packet4l pzero(const Packet4l& /*a*/) { + return _mm256_setzero_si256(); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pzero(const Packet4ul& /*a*/) { + return _mm256_setzero_si256(); +} +template <> +EIGEN_STRONG_INLINE Packet4l peven_mask(const Packet4l& /*a*/) { + return _mm256_set_epi64x(0ll, -1ll, 0ll, -1ll); +} +template <> +EIGEN_STRONG_INLINE Packet4ul peven_mask(const Packet4ul& /*a*/) { + return _mm256_set_epi64x(0ll, -1ll, 0ll, -1ll); +} +template <> +EIGEN_STRONG_INLINE Packet4l pload1(const int64_t* from) { + return _mm256_set1_epi64x(*from); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pload1(const uint64_t* from) { + return _mm256_set1_epi64x(*from); +} +template <> +EIGEN_STRONG_INLINE Packet4l padd(const Packet4l& a, const Packet4l& b) { + return _mm256_add_epi64(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ul padd(const Packet4ul& a, const Packet4ul& b) { + return _mm256_add_epi64(a, b); +} +template<> +EIGEN_STRONG_INLINE Packet4l plset(const int64_t& a) { + return padd(pset1(a), Packet4l(_mm256_set_epi64x(3ll, 2ll, 1ll, 0ll))); +} +template <> +EIGEN_STRONG_INLINE Packet4ul plset(const uint64_t& a) { + return padd(pset1(a), Packet4ul(_mm256_set_epi64x(3ll, 2ll, 1ll, 0ll))); +} +template <> +EIGEN_STRONG_INLINE Packet4l psub(const Packet4l& a, const Packet4l& b) { + return _mm256_sub_epi64(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ul psub(const Packet4ul& a, const Packet4ul& b) { + return _mm256_sub_epi64(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4l pnegate(const Packet4l& a) { + return psub(pzero(a), a); +} +template <> +EIGEN_STRONG_INLINE Packet4l pconj(const Packet4l& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet4l pcmp_le(const Packet4l& a, const Packet4l& b) { + return _mm256_xor_si256(_mm256_cmpgt_epi64(a, b), _mm256_set1_epi32(-1)); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pcmp_le(const Packet4ul& a, const Packet4ul& b) { + return (Packet4ul)pcmp_le((Packet4l)psub(a, pset1(0x8000000000000000UL)), + (Packet4l)psub(b, pset1(0x8000000000000000UL))); +} +template <> +EIGEN_STRONG_INLINE Packet4l pcmp_lt(const Packet4l& a, const Packet4l& b) { + return _mm256_cmpgt_epi64(b, a); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pcmp_lt(const Packet4ul& a, const Packet4ul& b) { + return (Packet4ul)pcmp_lt((Packet4l)psub(a, pset1(0x8000000000000000UL)), + (Packet4l)psub(b, pset1(0x8000000000000000UL))); +} +template <> +EIGEN_STRONG_INLINE Packet4l pcmp_eq(const Packet4l& a, const Packet4l& b) { + return _mm256_cmpeq_epi64(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pcmp_eq(const Packet4ul& a, const Packet4ul& b) { + return _mm256_cmpeq_epi64(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4l ptrue(const Packet4l& a) { + return _mm256_cmpeq_epi64(a, a); +} +template <> +EIGEN_STRONG_INLINE Packet4ul ptrue(const Packet4ul& a) { + return _mm256_cmpeq_epi64(a, a); +} +template <> +EIGEN_STRONG_INLINE Packet4l pand(const Packet4l& a, const Packet4l& b) { + return _mm256_and_si256(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4l por(const Packet4l& a, const Packet4l& b) { + return _mm256_or_si256(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4l pxor(const Packet4l& a, const Packet4l& b) { + return _mm256_xor_si256(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pxor(const Packet4ul& a, const Packet4ul& b) { + return _mm256_xor_si256(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4l pandnot(const Packet4l& a, const Packet4l& b) { + return _mm256_andnot_si256(b, a); +} +template +EIGEN_STRONG_INLINE Packet4l plogical_shift_right(Packet4l a) { + return _mm256_srli_epi64(a, N); +} +template +EIGEN_STRONG_INLINE Packet4l plogical_shift_left(Packet4l a) { + return _mm256_slli_epi64(a, N); +} +#ifdef EIGEN_VECTORIZE_AVX512FP16 +template +EIGEN_STRONG_INLINE Packet4l parithmetic_shift_right(Packet4l a) { return _mm256_srai_epi64(a, N); } +#else +template +EIGEN_STRONG_INLINE std::enable_if_t< (N == 0), Packet4l> parithmetic_shift_right(Packet4l a) { + return a; +} +template +EIGEN_STRONG_INLINE std::enable_if_t< (N > 0) && (N < 32), Packet4l> parithmetic_shift_right(Packet4l a) { + __m256i hi_word = _mm256_srai_epi32(a, N); + __m256i lo_word = _mm256_srli_epi64(a, N); + return _mm256_blend_epi32(hi_word, lo_word, 0b01010101); +} +template +EIGEN_STRONG_INLINE std::enable_if_t< (N >= 32) && (N < 63), Packet4l> parithmetic_shift_right(Packet4l a) { + __m256i hi_word = _mm256_srai_epi32(a, 31); + __m256i lo_word = _mm256_shuffle_epi32(_mm256_srai_epi32(a, N - 32), (shuffle_mask<1, 1, 3, 3>::mask)); + return _mm256_blend_epi32(hi_word, lo_word, 0b01010101); +} +template +EIGEN_STRONG_INLINE std::enable_if_t< (N == 63), Packet4l> parithmetic_shift_right(Packet4l a) { + return _mm256_shuffle_epi32(_mm256_srai_epi32(a, 31), (shuffle_mask<1, 1, 3, 3>::mask)); +} +template +EIGEN_STRONG_INLINE std::enable_if_t< (N < 0) || (N > 63), Packet4l> parithmetic_shift_right(Packet4l a) { + return parithmetic_shift_right(a); +} +#endif +template <> +EIGEN_STRONG_INLINE Packet4l pload(const int64_t* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(reinterpret_cast(from)); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pload(const uint64_t* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(reinterpret_cast(from)); +} +template <> +EIGEN_STRONG_INLINE Packet4l ploadu(const int64_t* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(reinterpret_cast(from)); +} +template <> +EIGEN_STRONG_INLINE Packet4ul ploadu(const uint64_t* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(reinterpret_cast(from)); +} +// Loads 2 int64_ts from memory a returns the packet {a0, a0, a1, a1} +template <> +EIGEN_STRONG_INLINE Packet4l ploaddup(const int64_t* from) { + const Packet4l a = _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast(from))); + return _mm256_permutevar8x32_epi32(a, _mm256_setr_epi32(0, 1, 0, 1, 2, 3, 2, 3)); +} +// Loads 2 uint64_ts from memory a returns the packet {a0, a0, a1, a1} +template <> +EIGEN_STRONG_INLINE Packet4ul ploaddup(const uint64_t* from) { + const Packet4ul a = _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast(from))); + return _mm256_permutevar8x32_epi32(a, _mm256_setr_epi32(0, 1, 0, 1, 2, 3, 2, 3)); +} +template<> +EIGEN_STRONG_INLINE void pstore(int64_t* to, const Packet4l& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to), from); +} +template <> +EIGEN_STRONG_INLINE void pstore(uint64_t* to, const Packet4ul& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to), from); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(int64_t* to, const Packet4l& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(uint64_t* to, const Packet4ul& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from); +} +template <> +EIGEN_DEVICE_FUNC inline Packet4l pgather(const int64_t* from, Index stride) { + return _mm256_set_epi64x(from[3 * stride], from[2 * stride], from[1 * stride], from[0 * stride]); +} +template <> +EIGEN_DEVICE_FUNC inline Packet4ul pgather(const uint64_t* from, Index stride) { + return _mm256_set_epi64x(from[3 * stride], from[2 * stride], from[1 * stride], from[0 * stride]); +} +template <> +EIGEN_DEVICE_FUNC inline void pscatter(int64_t* to, const Packet4l& from, Index stride) { + __m128i low = _mm256_extractf128_si256(from, 0); + to[stride * 0] = _mm_extract_epi64(low, 0); + to[stride * 1] = _mm_extract_epi64(low, 1); + + __m128i high = _mm256_extractf128_si256(from, 1); + to[stride * 2] = _mm_extract_epi64(high, 0); + to[stride * 3] = _mm_extract_epi64(high, 1); +} +template <> +EIGEN_DEVICE_FUNC inline void pscatter(uint64_t* to, const Packet4ul& from, Index stride) { + __m128i low = _mm256_extractf128_si256(from, 0); + to[stride * 0] = _mm_extract_epi64(low, 0); + to[stride * 1] = _mm_extract_epi64(low, 1); + + __m128i high = _mm256_extractf128_si256(from, 1); + to[stride * 2] = _mm_extract_epi64(high, 0); + to[stride * 3] = _mm_extract_epi64(high, 1); +} +template <> +EIGEN_STRONG_INLINE void pstore1(int64_t* to, const int64_t& a) { + Packet4l pa = pset1(a); + pstore(to, pa); +} +template <> +EIGEN_STRONG_INLINE void pstore1(uint64_t* to, const uint64_t& a) { + Packet4ul pa = pset1(a); + pstore(to, pa); +} +template<> +EIGEN_STRONG_INLINE int64_t pfirst(const Packet4l& a) { + return _mm_cvtsi128_si64(_mm256_castsi256_si128(a)); +} +template <> +EIGEN_STRONG_INLINE uint64_t pfirst(const Packet4ul& a) { + return _mm_cvtsi128_si64(_mm256_castsi256_si128(a)); +} +template <> +EIGEN_STRONG_INLINE int64_t predux(const Packet4l& a) { + __m128i r = _mm_add_epi64(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1)); + return _mm_extract_epi64(r, 0) + _mm_extract_epi64(r, 1); +} +template <> +EIGEN_STRONG_INLINE uint64_t predux(const Packet4ul& a) { + __m128i r = _mm_add_epi64(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1)); + return numext::bit_cast(_mm_extract_epi64(r, 0) + _mm_extract_epi64(r, 1)); +} +#define MM256_SHUFFLE_EPI64(A, B, M) _mm256_shuffle_pd(_mm256_castsi256_pd(A), _mm256_castsi256_pd(B), M) +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { + __m256d T0 = MM256_SHUFFLE_EPI64(kernel.packet[0], kernel.packet[1], 15); + __m256d T1 = MM256_SHUFFLE_EPI64(kernel.packet[0], kernel.packet[1], 0); + __m256d T2 = MM256_SHUFFLE_EPI64(kernel.packet[2], kernel.packet[3], 15); + __m256d T3 = MM256_SHUFFLE_EPI64(kernel.packet[2], kernel.packet[3], 0); + + kernel.packet[1] = _mm256_castpd_si256(_mm256_permute2f128_pd(T0, T2, 32)); + kernel.packet[3] = _mm256_castpd_si256(_mm256_permute2f128_pd(T0, T2, 49)); + kernel.packet[0] = _mm256_castpd_si256(_mm256_permute2f128_pd(T1, T3, 32)); + kernel.packet[2] = _mm256_castpd_si256(_mm256_permute2f128_pd(T1, T3, 49)); +} +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { + ptranspose((PacketBlock&)kernel); +} +template <> +EIGEN_STRONG_INLINE Packet4l pmin(const Packet4l& a, const Packet4l& b) { + __m256i cmp = _mm256_cmpgt_epi64(a, b); + __m256i a_min = _mm256_andnot_si256(cmp, a); + __m256i b_min = _mm256_and_si256(cmp, b); + return Packet4l(_mm256_or_si256(a_min, b_min)); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pmin(const Packet4ul& a, const Packet4ul& b) { + return padd((Packet4ul)pmin((Packet4l)psub(a, pset1(0x8000000000000000UL)), + (Packet4l)psub(b, pset1(0x8000000000000000UL))), + pset1(0x8000000000000000UL)); +} +template <> +EIGEN_STRONG_INLINE Packet4l pmax(const Packet4l& a, const Packet4l& b) { + __m256i cmp = _mm256_cmpgt_epi64(a, b); + __m256i a_min = _mm256_and_si256(cmp, a); + __m256i b_min = _mm256_andnot_si256(cmp, b); + return Packet4l(_mm256_or_si256(a_min, b_min)); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pmax(const Packet4ul& a, const Packet4ul& b) { + return padd((Packet4ul)pmax((Packet4l)psub(a, pset1(0x8000000000000000UL)), + (Packet4l)psub(b, pset1(0x8000000000000000UL))), + pset1(0x8000000000000000UL)); +} +template <> +EIGEN_STRONG_INLINE Packet4l pabs(const Packet4l& a) { + Packet4l pz = pzero(a); + Packet4l cmp = _mm256_cmpgt_epi64(a, pz); + return psub(cmp, pxor(a, cmp)); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pabs(const Packet4ul& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet4l pmul(const Packet4l& a, const Packet4l& b) { + // 64-bit mul requires avx512, so do this with 32-bit multiplication + __m256i upper32_a = _mm256_srli_epi64(a, 32); + __m256i upper32_b = _mm256_srli_epi64(b, 32); + + // upper * lower + __m256i mul1 = _mm256_mul_epu32(upper32_a, b); + __m256i mul2 = _mm256_mul_epu32(upper32_b, a); + // Gives us both upper*upper and lower*lower + __m256i mul3 = _mm256_mul_epu32(a, b); + + __m256i high = _mm256_slli_epi64(_mm256_add_epi64(mul1, mul2), 32); + return _mm256_add_epi64(high, mul3); +} +template <> +EIGEN_STRONG_INLINE Packet4ul pmul(const Packet4ul& a, const Packet4ul& b) { + return (Packet4ul)pmul((Packet4l)a, (Packet4l)b); +} +#endif template<> EIGEN_STRONG_INLINE Packet8f pset1(const float& from) { return _mm256_set1_ps(from); } template<> EIGEN_STRONG_INLINE Packet4d pset1(const double& from) { return _mm256_set1_pd(from); } template<> EIGEN_STRONG_INLINE Packet8i pset1(const int& from) { return _mm256_set1_epi32(from); } +template<> EIGEN_STRONG_INLINE Packet8ui pset1(const uint32_t& from) { return _mm256_set1_epi32(from); } template<> EIGEN_STRONG_INLINE Packet8f pset1frombits(unsigned int from) { return _mm256_castsi256_ps(pset1(from)); } template<> EIGEN_STRONG_INLINE Packet4d pset1frombits(uint64_t from) { return _mm256_castsi256_pd(_mm256_set1_epi64x(from)); } @@ -247,19 +678,28 @@ template<> EIGEN_STRONG_INLINE Packet4d pset1frombits(uint64_t from) { template<> EIGEN_STRONG_INLINE Packet8f pzero(const Packet8f& /*a*/) { return _mm256_setzero_ps(); } template<> EIGEN_STRONG_INLINE Packet4d pzero(const Packet4d& /*a*/) { return _mm256_setzero_pd(); } template<> EIGEN_STRONG_INLINE Packet8i pzero(const Packet8i& /*a*/) { return _mm256_setzero_si256(); } +template<> EIGEN_STRONG_INLINE Packet8ui pzero(const Packet8ui& /*a*/) { return _mm256_setzero_si256(); } template<> EIGEN_STRONG_INLINE Packet8f peven_mask(const Packet8f& /*a*/) { return _mm256_castsi256_ps(_mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1)); } template<> EIGEN_STRONG_INLINE Packet8i peven_mask(const Packet8i& /*a*/) { return _mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1); } +template<> EIGEN_STRONG_INLINE Packet8ui peven_mask(const Packet8ui& /*a*/) { return _mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1); } template<> EIGEN_STRONG_INLINE Packet4d peven_mask(const Packet4d& /*a*/) { return _mm256_castsi256_pd(_mm256_set_epi32(0, 0, -1, -1, 0, 0, -1, -1)); } template<> EIGEN_STRONG_INLINE Packet8f pload1(const float* from) { return _mm256_broadcast_ss(from); } template<> EIGEN_STRONG_INLINE Packet4d pload1(const double* from) { return _mm256_broadcast_sd(from); } -template<> EIGEN_STRONG_INLINE Packet8f plset(const float& a) { return _mm256_add_ps(_mm256_set1_ps(a), _mm256_set_ps(7.0,6.0,5.0,4.0,3.0,2.0,1.0,0.0)); } -template<> EIGEN_STRONG_INLINE Packet4d plset(const double& a) { return _mm256_add_pd(_mm256_set1_pd(a), _mm256_set_pd(3.0,2.0,1.0,0.0)); } - template<> EIGEN_STRONG_INLINE Packet8f padd(const Packet8f& a, const Packet8f& b) { return _mm256_add_ps(a,b); } +#ifdef EIGEN_VECTORIZE_AVX512 +template <> +EIGEN_STRONG_INLINE Packet8f padd(const Packet8f& a, const Packet8f& b, uint8_t umask) { + __mmask16 mask = static_cast<__mmask16>(umask & 0x00FF); + return _mm512_castps512_ps256(_mm512_maskz_add_ps( + mask, + _mm512_castps256_ps512(a), + _mm512_castps256_ps512(b))); +} +#endif template<> EIGEN_STRONG_INLINE Packet4d padd(const Packet4d& a, const Packet4d& b) { return _mm256_add_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet8i padd(const Packet8i& a, const Packet8i& b) { #ifdef EIGEN_VECTORIZE_AVX2 @@ -270,6 +710,21 @@ template<> EIGEN_STRONG_INLINE Packet8i padd(const Packet8i& a, const return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); #endif } +template<> EIGEN_STRONG_INLINE Packet8ui padd(const Packet8ui& a, const Packet8ui& b) +{ +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_add_epi32(a, b); +#else + __m128i lo = _mm_add_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + __m128i hi = _mm_add_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} + +template<> EIGEN_STRONG_INLINE Packet8f plset(const float& a) { return padd(pset1(a), _mm256_set_ps(7.0,6.0,5.0,4.0,3.0,2.0,1.0,0.0)); } +template<> EIGEN_STRONG_INLINE Packet4d plset(const double& a) { return padd(pset1(a), _mm256_set_pd(3.0,2.0,1.0,0.0)); } +template<> EIGEN_STRONG_INLINE Packet8i plset(const int& a) { return padd(pset1(a), (Packet8i)_mm256_set_epi32(7,6,5,4,3,2,1,0)); } +template<> EIGEN_STRONG_INLINE Packet8ui plset(const uint32_t& a) { return padd(pset1(a), (Packet8ui)_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)); } template<> EIGEN_STRONG_INLINE Packet8f psub(const Packet8f& a, const Packet8f& b) { return _mm256_sub_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d psub(const Packet4d& a, const Packet4d& b) { return _mm256_sub_pd(a,b); } @@ -282,14 +737,30 @@ template<> EIGEN_STRONG_INLINE Packet8i psub(const Packet8i& a, const return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); #endif } +template<> EIGEN_STRONG_INLINE Packet8ui psub(const Packet8ui& a, const Packet8ui& b) +{ +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_sub_epi32(a, b); +#else + __m128i lo = _mm_sub_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + __m128i hi = _mm_sub_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f pnegate(const Packet8f& a) { - return _mm256_sub_ps(_mm256_set1_ps(0.0),a); + const Packet8f mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x80000000)); + return _mm256_xor_ps(a, mask); } template<> EIGEN_STRONG_INLINE Packet4d pnegate(const Packet4d& a) { - return _mm256_sub_pd(_mm256_set1_pd(0.0),a); + const Packet4d mask = _mm256_castsi256_pd(_mm256_set1_epi64x(0x8000000000000000ULL)); + return _mm256_xor_pd(a, mask); +} +template<> EIGEN_STRONG_INLINE Packet8i pnegate(const Packet8i& a) +{ + return psub(pzero(a), a); } template<> EIGEN_STRONG_INLINE Packet8f pconj(const Packet8f& a) { return a; } @@ -307,52 +778,104 @@ template<> EIGEN_STRONG_INLINE Packet8i pmul(const Packet8i& a, const return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); #endif } +template<> EIGEN_STRONG_INLINE Packet8ui pmul(const Packet8ui& a, const Packet8ui& b) +{ +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_mullo_epi32(a, b); +#else + const __m128i lo = _mm_mullo_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + const __m128i hi = _mm_mullo_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f pdiv(const Packet8f& a, const Packet8f& b) { return _mm256_div_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d pdiv(const Packet4d& a, const Packet4d& b) { return _mm256_div_pd(a,b); } -template<> EIGEN_STRONG_INLINE Packet8i pdiv(const Packet8i& /*a*/, const Packet8i& /*b*/) -{ eigen_assert(false && "packet integer division are not supported by AVX"); - return pset1(0); -} -#ifdef EIGEN_VECTORIZE_FMA -template<> EIGEN_STRONG_INLINE Packet8f pmadd(const Packet8f& a, const Packet8f& b, const Packet8f& c) { -#if ( (EIGEN_COMP_GNUC_STRICT && EIGEN_COMP_GNUC<80) || (EIGEN_COMP_CLANG) ) - // Clang stupidly generates a vfmadd213ps instruction plus some vmovaps on registers, - // and even register spilling with clang>=6.0 (bug 1637). - // Gcc stupidly generates a vfmadd132ps instruction. - // So let's enforce it to generate a vfmadd231ps instruction since the most common use - // case is to accumulate the result of the product. - Packet8f res = c; - __asm__("vfmadd231ps %[a], %[b], %[c]" : [c] "+x" (res) : [a] "x" (a), [b] "x" (b)); - return res; +template<> EIGEN_STRONG_INLINE Packet8i pdiv(const Packet8i& a, const Packet8i& b) +{ +#ifdef EIGEN_VECTORIZE_AVX512 + return _mm512_cvttpd_epi32(_mm512_div_pd(_mm512_cvtepi32_pd(a), _mm512_cvtepi32_pd(b))); #else - return _mm256_fmadd_ps(a,b,c); + Packet4i lo = pdiv(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + Packet4i hi = pdiv(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1); #endif } -template<> EIGEN_STRONG_INLINE Packet4d pmadd(const Packet4d& a, const Packet4d& b, const Packet4d& c) { -#if ( (EIGEN_COMP_GNUC_STRICT && EIGEN_COMP_GNUC<80) || (EIGEN_COMP_CLANG) ) - // see above - Packet4d res = c; - __asm__("vfmadd231pd %[a], %[b], %[c]" : [c] "+x" (res) : [a] "x" (a), [b] "x" (b)); - return res; -#else - return _mm256_fmadd_pd(a,b,c); -#endif + +#ifdef EIGEN_VECTORIZE_FMA +template <> +EIGEN_STRONG_INLINE Packet8f pmadd(const Packet8f& a, const Packet8f& b, const Packet8f& c) { + return _mm256_fmadd_ps(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet4d pmadd(const Packet4d& a, const Packet4d& b, const Packet4d& c) { + return _mm256_fmadd_pd(a, b, c); } + +template <> +EIGEN_STRONG_INLINE Packet8f pmsub(const Packet8f& a, const Packet8f& b, const Packet8f& c) { + return _mm256_fmsub_ps(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet4d pmsub(const Packet4d& a, const Packet4d& b, const Packet4d& c) { + return _mm256_fmsub_pd(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet8f pnmadd(const Packet8f& a, const Packet8f& b, const Packet8f& c) { + return _mm256_fnmadd_ps(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet4d pnmadd(const Packet4d& a, const Packet4d& b, const Packet4d& c) { + return _mm256_fnmadd_pd(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet8f pnmsub(const Packet8f& a, const Packet8f& b, const Packet8f& c) { + return _mm256_fnmsub_ps(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet4d pnmsub(const Packet4d& a, const Packet4d& b, const Packet4d& c) { + return _mm256_fnmsub_pd(a, b, c); +} + #endif template<> EIGEN_STRONG_INLINE Packet8f pcmp_le(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a,b,_CMP_LE_OQ); } template<> EIGEN_STRONG_INLINE Packet8f pcmp_lt(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a,b,_CMP_LT_OQ); } template<> EIGEN_STRONG_INLINE Packet8f pcmp_lt_or_nan(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a, b, _CMP_NGE_UQ); } template<> EIGEN_STRONG_INLINE Packet8f pcmp_eq(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a,b,_CMP_EQ_OQ); } +template<> EIGEN_STRONG_INLINE Packet8f pisnan(const Packet8f& a) { return _mm256_cmp_ps(a,a,_CMP_UNORD_Q); } template<> EIGEN_STRONG_INLINE Packet4d pcmp_le(const Packet4d& a, const Packet4d& b) { return _mm256_cmp_pd(a,b,_CMP_LE_OQ); } template<> EIGEN_STRONG_INLINE Packet4d pcmp_lt(const Packet4d& a, const Packet4d& b) { return _mm256_cmp_pd(a,b,_CMP_LT_OQ); } template<> EIGEN_STRONG_INLINE Packet4d pcmp_lt_or_nan(const Packet4d& a, const Packet4d& b) { return _mm256_cmp_pd(a, b, _CMP_NGE_UQ); } template<> EIGEN_STRONG_INLINE Packet4d pcmp_eq(const Packet4d& a, const Packet4d& b) { return _mm256_cmp_pd(a,b,_CMP_EQ_OQ); } - +template<> EIGEN_STRONG_INLINE Packet8i pcmp_le(const Packet8i& a, const Packet8i& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_xor_si256(_mm256_cmpgt_epi32(a,b), _mm256_set1_epi32(-1)); +#else + __m128i lo = _mm_cmpgt_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + lo = _mm_xor_si128(lo, _mm_set1_epi32(-1)); + __m128i hi = _mm_cmpgt_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + hi = _mm_xor_si128(hi, _mm_set1_epi32(-1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} +template<> EIGEN_STRONG_INLINE Packet8i pcmp_lt(const Packet8i& a, const Packet8i& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_cmpgt_epi32(b,a); +#else + __m128i lo = _mm_cmpgt_epi32(_mm256_extractf128_si256(b, 0), _mm256_extractf128_si256(a, 0)); + __m128i hi = _mm_cmpgt_epi32(_mm256_extractf128_si256(b, 1), _mm256_extractf128_si256(a, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} template<> EIGEN_STRONG_INLINE Packet8i pcmp_eq(const Packet8i& a, const Packet8i& b) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_cmpeq_epi32(a,b); @@ -362,9 +885,18 @@ template<> EIGEN_STRONG_INLINE Packet8i pcmp_eq(const Packet8i& a, const Packet8 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); #endif } +template<> EIGEN_STRONG_INLINE Packet8ui pcmp_eq(const Packet8ui& a, const Packet8ui& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_cmpeq_epi32(a, b); +#else + __m128i lo = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + __m128i hi = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f pmin(const Packet8f& a, const Packet8f& b) { -#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 +#if EIGEN_GNUC_STRICT_LESS_THAN(6,3,0) // There appears to be a bug in GCC, by which the optimizer may flip // the argument order in calls to _mm_min_ps/_mm_max_ps, so we have to // resort to inline ASM here. This is supposed to be fixed in gcc6.3, @@ -378,7 +910,7 @@ template<> EIGEN_STRONG_INLINE Packet8f pmin(const Packet8f& a, const #endif } template<> EIGEN_STRONG_INLINE Packet4d pmin(const Packet4d& a, const Packet4d& b) { -#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 +#if EIGEN_GNUC_STRICT_LESS_THAN(6,3,0) // See pmin above Packet4d res; asm("vminpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); @@ -388,9 +920,27 @@ template<> EIGEN_STRONG_INLINE Packet4d pmin(const Packet4d& a, const return _mm256_min_pd(b,a); #endif } +template<> EIGEN_STRONG_INLINE Packet8i pmin(const Packet8i& a, const Packet8i& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_min_epi32(a, b); +#else + __m128i lo = _mm_min_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + __m128i hi = _mm_min_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} +template<> EIGEN_STRONG_INLINE Packet8ui pmin(const Packet8ui& a, const Packet8ui& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_min_epu32(a, b); +#else + __m128i lo = _mm_min_epu32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + __m128i hi = _mm_min_epu32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f pmax(const Packet8f& a, const Packet8f& b) { -#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 +#if EIGEN_GNUC_STRICT_LESS_THAN(6,3,0) // See pmin above Packet8f res; asm("vmaxps %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); @@ -401,7 +951,7 @@ template<> EIGEN_STRONG_INLINE Packet8f pmax(const Packet8f& a, const #endif } template<> EIGEN_STRONG_INLINE Packet4d pmax(const Packet4d& a, const Packet4d& b) { -#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 +#if EIGEN_GNUC_STRICT_LESS_THAN(6,3,0) // See pmin above Packet4d res; asm("vmaxpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); @@ -411,6 +961,30 @@ template<> EIGEN_STRONG_INLINE Packet4d pmax(const Packet4d& a, const return _mm256_max_pd(b,a); #endif } +template<> EIGEN_STRONG_INLINE Packet8i pmax(const Packet8i& a, const Packet8i& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_max_epi32(a, b); +#else + __m128i lo = _mm_max_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + __m128i hi = _mm_max_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} +template<> EIGEN_STRONG_INLINE Packet8ui pmax(const Packet8ui& a, const Packet8ui& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_max_epu32(a, b); +#else + __m128i lo = _mm_max_epu32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + __m128i hi = _mm_max_epu32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} + +#ifdef EIGEN_VECTORIZE_AVX2 +template<> EIGEN_STRONG_INLINE Packet8i psign(const Packet8i& a) { + return _mm256_sign_epi32(_mm256_set1_epi32(1), a); +} +#endif // Add specializations for min/max with prescribed NaN progation. template<> @@ -495,6 +1069,13 @@ template<> EIGEN_STRONG_INLINE Packet8i pand(const Packet8i& a, const return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a),_mm256_castsi256_ps(b))); #endif } +template<> EIGEN_STRONG_INLINE Packet8ui pand(const Packet8ui& a, const Packet8ui& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_and_si256(a,b); +#else + return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a),_mm256_castsi256_ps(b))); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f por(const Packet8f& a, const Packet8f& b) { return _mm256_or_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d por(const Packet4d& a, const Packet4d& b) { return _mm256_or_pd(a,b); } @@ -505,6 +1086,13 @@ template<> EIGEN_STRONG_INLINE Packet8i por(const Packet8i& a, const P return _mm256_castps_si256(_mm256_or_ps(_mm256_castsi256_ps(a),_mm256_castsi256_ps(b))); #endif } +template<> EIGEN_STRONG_INLINE Packet8ui por(const Packet8ui& a, const Packet8ui& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_or_si256(a,b); +#else + return _mm256_castps_si256(_mm256_or_ps(_mm256_castsi256_ps(a),_mm256_castsi256_ps(b))); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f pxor(const Packet8f& a, const Packet8f& b) { return _mm256_xor_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d pxor(const Packet4d& a, const Packet4d& b) { return _mm256_xor_pd(a,b); } @@ -515,6 +1103,13 @@ template<> EIGEN_STRONG_INLINE Packet8i pxor(const Packet8i& a, const return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a),_mm256_castsi256_ps(b))); #endif } +template<> EIGEN_STRONG_INLINE Packet8ui pxor(const Packet8ui& a, const Packet8ui& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_xor_si256(a, b); +#else + return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f pandnot(const Packet8f& a, const Packet8f& b) { return _mm256_andnot_ps(b,a); } template<> EIGEN_STRONG_INLINE Packet4d pandnot(const Packet4d& a, const Packet4d& b) { return _mm256_andnot_pd(b,a); } @@ -525,6 +1120,20 @@ template<> EIGEN_STRONG_INLINE Packet8i pandnot(const Packet8i& a, con return _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(b),_mm256_castsi256_ps(a))); #endif } +template<> EIGEN_STRONG_INLINE Packet8ui pandnot(const Packet8ui& a, const Packet8ui& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_andnot_si256(b,a); +#else + return _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(b),_mm256_castsi256_ps(a))); +#endif +} + +template<> EIGEN_STRONG_INLINE Packet8ui pcmp_lt(const Packet8ui& a, const Packet8ui& b) { + return pxor(pcmp_eq(a, pmax(a, b)), ptrue(a)); +} +template<> EIGEN_STRONG_INLINE Packet8ui pcmp_le(const Packet8ui& a, const Packet8ui& b) { + return pcmp_eq(a, pmin(a, b)); +} template<> EIGEN_STRONG_INLINE Packet8f pround(const Packet8f& a) { @@ -541,6 +1150,11 @@ template<> EIGEN_STRONG_INLINE Packet4d pround(const Packet4d& a) template<> EIGEN_STRONG_INLINE Packet8f pselect(const Packet8f& mask, const Packet8f& a, const Packet8f& b) { return _mm256_blendv_ps(b,a,mask); } +template<> EIGEN_STRONG_INLINE Packet8i pselect(const Packet8i& mask, const Packet8i& a, const Packet8i& b) +{ return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(a), _mm256_castsi256_ps(mask))); } +template<> EIGEN_STRONG_INLINE Packet8ui pselect(const Packet8ui& mask, const Packet8ui& a, const Packet8ui& b) +{ return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(a), _mm256_castsi256_ps(mask))); } + template<> EIGEN_STRONG_INLINE Packet4d pselect(const Packet4d& mask, const Packet4d& a, const Packet4d& b) { return _mm256_blendv_pd(b,a,mask); } @@ -574,27 +1188,44 @@ template EIGEN_STRONG_INLINE Packet8i plogical_shift_left(Packet8i a) { #endif } +template EIGEN_STRONG_INLINE Packet8ui parithmetic_shift_right(Packet8ui a) { + return (Packet8ui)plogical_shift_right((Packet8i)a); +} +template EIGEN_STRONG_INLINE Packet8ui plogical_shift_right(Packet8ui a) { + return (Packet8ui)plogical_shift_right((Packet8i)a); +} +template EIGEN_STRONG_INLINE Packet8ui plogical_shift_left(Packet8ui a) { + return (Packet8ui)plogical_shift_left((Packet8i)a); +} + template<> EIGEN_STRONG_INLINE Packet8f pload(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_ps(from); } template<> EIGEN_STRONG_INLINE Packet4d pload(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_pd(from); } template<> EIGEN_STRONG_INLINE Packet8i pload(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(reinterpret_cast(from)); } +template<> EIGEN_STRONG_INLINE Packet8ui pload(const uint32_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet8f ploadu(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_ps(from); } template<> EIGEN_STRONG_INLINE Packet4d ploadu(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_pd(from); } template<> EIGEN_STRONG_INLINE Packet8i ploadu(const int* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(reinterpret_cast(from)); } +template<> EIGEN_STRONG_INLINE Packet8ui ploadu(const uint32_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet8f ploadu(const float* from, uint8_t umask) { +#ifdef EIGEN_VECTORIZE_AVX512 + __mmask16 mask = static_cast<__mmask16>(umask & 0x00FF); + EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_castps512_ps256(_mm512_maskz_loadu_ps(mask, from)); +#else Packet8i mask = _mm256_set1_epi8(static_cast(umask)); const Packet8i bit_mask = _mm256_set_epi32(0xffffff7f, 0xffffffbf, 0xffffffdf, 0xffffffef, 0xfffffff7, 0xfffffffb, 0xfffffffd, 0xfffffffe); mask = por(mask, bit_mask); mask = pcmp_eq(mask, _mm256_set1_epi32(0xffffffff)); EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_maskload_ps(from, mask); +#endif } // Loads 4 floats from memory a returns the packet {a0, a0 a1, a1, a2, a2, a3, a3} template<> EIGEN_STRONG_INLINE Packet8f ploaddup(const float* from) { // TODO try to find a way to avoid the need of a temporary register -// Packet8f tmp = _mm256_castps128_ps256(_mm_loadu_ps(from)); + // Packet8f tmp = _mm256_castps128_ps256(_mm_loadu_ps(from)); // tmp = _mm256_insertf128_ps(tmp, _mm_movehl_ps(_mm256_castps256_ps128(tmp),_mm256_castps256_ps128(tmp)), 1); // return _mm256_unpacklo_ps(tmp,tmp); @@ -605,12 +1236,40 @@ template<> EIGEN_STRONG_INLINE Packet8f ploaddup(const float* from) // then we can perform a consistent permutation on the global register to get everything in shape: return _mm256_permute_ps(tmp, _MM_SHUFFLE(3,3,2,2)); } -// Loads 2 doubles from memory a returns the packet {a0, a0 a1, a1} +// Loads 2 doubles from memory a returns the packet {a0, a0, a1, a1} template<> EIGEN_STRONG_INLINE Packet4d ploaddup(const double* from) { Packet4d tmp = _mm256_broadcast_pd((const __m128d*)(const void*)from); return _mm256_permute_pd(tmp, 3<<2); } +// Loads 4 integers from memory a returns the packet {a0, a0, a1, a1, a2, a2, a3, a3} +template<> EIGEN_STRONG_INLINE Packet8i ploaddup(const int* from) +{ +#ifdef EIGEN_VECTORIZE_AVX2 + const Packet8i a = _mm256_castsi128_si256(ploadu(from)); + return _mm256_permutevar8x32_epi32(a, _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3)); +#else + __m256 tmp = _mm256_broadcast_ps((const __m128*)(const void*)from); + // mimic an "inplace" permutation of the lower 128bits using a blend + tmp = _mm256_blend_ps(tmp,_mm256_castps128_ps256(_mm_permute_ps( _mm256_castps256_ps128(tmp), _MM_SHUFFLE(1,0,1,0))), 15); + // then we can perform a consistent permutation on the global register to get everything in shape: + return _mm256_castps_si256(_mm256_permute_ps(tmp, _MM_SHUFFLE(3,3,2,2))); +#endif +} +template<> EIGEN_STRONG_INLINE Packet8ui ploaddup(const uint32_t* from) { +#ifdef EIGEN_VECTORIZE_AVX2 + const Packet8ui a = _mm256_castsi128_si256(ploadu(from)); + return _mm256_permutevar8x32_epi32(a, _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3)); +#else + __m256 tmp = _mm256_broadcast_ps((const __m128*)(const void*)from); + // mimic an "inplace" permutation of the lower 128bits using a blend + tmp = _mm256_blend_ps( + tmp, _mm256_castps128_ps256(_mm_permute_ps(_mm256_castps256_ps128(tmp), _MM_SHUFFLE(1, 0, 1, 0))), 15); + // then we can perform a consistent permutation on the global register to get + // everything in shape: + return _mm256_castps_si256(_mm256_permute_ps(tmp, _MM_SHUFFLE(3, 3, 2, 2))); +#endif +} // Loads 2 floats from memory a returns the packet {a0, a0 a0, a0, a1, a1, a1, a1} template<> EIGEN_STRONG_INLINE Packet8f ploadquad(const float* from) @@ -618,21 +1277,42 @@ template<> EIGEN_STRONG_INLINE Packet8f ploadquad(const float* from) Packet8f tmp = _mm256_castps128_ps256(_mm_broadcast_ss(from)); return _mm256_insertf128_ps(tmp, _mm_broadcast_ss(from+1), 1); } +template<> EIGEN_STRONG_INLINE Packet8i ploadquad(const int* from) +{ + return _mm256_insertf128_si256(_mm256_set1_epi32(*from), _mm_set1_epi32(*(from+1)), 1); +} +template<> EIGEN_STRONG_INLINE Packet8ui ploadquad(const uint32_t* from) { + return _mm256_insertf128_si256(_mm256_set1_epi32(*from), _mm_set1_epi32(*(from + 1)), 1); +} template<> EIGEN_STRONG_INLINE void pstore(float* to, const Packet8f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm256_store_ps(to, from); } template<> EIGEN_STRONG_INLINE void pstore(double* to, const Packet4d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm256_store_pd(to, from); } -template<> EIGEN_STRONG_INLINE void pstore(int* to, const Packet8i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from); } +template<> EIGEN_STRONG_INLINE void pstore(int* to, const Packet8i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to), from); } +template<> EIGEN_STRONG_INLINE void pstore(uint32_t* to, const Packet8ui& from) { EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet8f& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_ps(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet4d& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_pd(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(int* to, const Packet8i& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from); } +template<> EIGEN_STRONG_INLINE void pstoreu(uint32_t* to, const Packet8ui& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet8f& from, uint8_t umask) { +#ifdef EIGEN_VECTORIZE_AVX512 + __mmask16 mask = static_cast<__mmask16>(umask & 0x00FF); + EIGEN_DEBUG_UNALIGNED_STORE _mm512_mask_storeu_ps(to, mask, _mm512_castps256_ps512(from)); +#else Packet8i mask = _mm256_set1_epi8(static_cast(umask)); - const Packet8i bit_mask = _mm256_set_epi32(0xffffff7f, 0xffffffbf, 0xffffffdf, 0xffffffef, 0xfffffff7, 0xfffffffb, 0xfffffffd, 0xfffffffe); + const Packet8i bit_mask = _mm256_set_epi32(0x7f7f7f7f, 0xbfbfbfbf, 0xdfdfdfdf, 0xefefefef, 0xf7f7f7f7, 0xfbfbfbfb, 0xfdfdfdfd, 0xfefefefe); mask = por(mask, bit_mask); mask = pcmp_eq(mask, _mm256_set1_epi32(0xffffffff)); - EIGEN_DEBUG_UNALIGNED_STORE return _mm256_maskstore_ps(to, mask, from); +#if EIGEN_COMP_MSVC + // MSVC sometimes seems to use a bogus mask with maskstore. + const __m256i ifrom = _mm256_castps_si256(from); + EIGEN_DEBUG_UNALIGNED_STORE _mm_maskmoveu_si128(_mm256_extractf128_si256(ifrom, 0), _mm256_extractf128_si256(mask, 0), reinterpret_cast(to)); + EIGEN_DEBUG_UNALIGNED_STORE _mm_maskmoveu_si128(_mm256_extractf128_si256(ifrom, 1), _mm256_extractf128_si256(mask, 1), reinterpret_cast(to + 4)); +#else + EIGEN_DEBUG_UNALIGNED_STORE _mm256_maskstore_ps(to, mask, from); +#endif +#endif } // NOTE: leverage _mm256_i32gather_ps and _mm256_i32gather_pd if AVX2 instructions are available @@ -646,6 +1326,14 @@ template<> EIGEN_DEVICE_FUNC inline Packet4d pgather(const dou { return _mm256_set_pd(from[3*stride], from[2*stride], from[1*stride], from[0*stride]); } +template<> EIGEN_DEVICE_FUNC inline Packet8i pgather(const int* from, Index stride) +{ + return _mm256_set_epi32(from[7*stride], from[6*stride], from[5*stride], from[4*stride], + from[3*stride], from[2*stride], from[1*stride], from[0*stride]); +} +template<> EIGEN_DEVICE_FUNC inline Packet8ui pgather(const uint32_t* from, Index stride) { + return (Packet8ui)pgather((int*)from, stride); +} template<> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const Packet8f& from, Index stride) { @@ -670,6 +1358,23 @@ template<> EIGEN_DEVICE_FUNC inline void pscatter(double* to, to[stride*2] = _mm_cvtsd_f64(high); to[stride*3] = _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1)); } +template<> EIGEN_DEVICE_FUNC inline void pscatter(int* to, const Packet8i& from, Index stride) +{ + __m128i low = _mm256_extractf128_si256(from, 0); + to[stride*0] = _mm_extract_epi32(low, 0); + to[stride*1] = _mm_extract_epi32(low, 1); + to[stride*2] = _mm_extract_epi32(low, 2); + to[stride*3] = _mm_extract_epi32(low, 3); + + __m128i high = _mm256_extractf128_si256(from, 1); + to[stride*4] = _mm_extract_epi32(high, 0); + to[stride*5] = _mm_extract_epi32(high, 1); + to[stride*6] = _mm_extract_epi32(high, 2); + to[stride*7] = _mm_extract_epi32(high, 3); +} +template<> EIGEN_DEVICE_FUNC inline void pscatter(uint32_t* to, const Packet8ui& from, Index stride) { + pscatter((int*)to, (Packet8i)from, stride); +} template<> EIGEN_STRONG_INLINE void pstore1(float* to, const float& a) { @@ -691,6 +1396,7 @@ template<> EIGEN_STRONG_INLINE void pstore1(int* to, const int& a) template<> EIGEN_STRONG_INLINE void prefetch(const float* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE void prefetch(const double* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE void prefetch(const int* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } +template<> EIGEN_STRONG_INLINE void prefetch(const uint32_t* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } #endif template<> EIGEN_STRONG_INLINE float pfirst(const Packet8f& a) { @@ -702,6 +1408,9 @@ template<> EIGEN_STRONG_INLINE double pfirst(const Packet4d& a) { template<> EIGEN_STRONG_INLINE int pfirst(const Packet8i& a) { return _mm_cvtsi128_si32(_mm256_castsi256_si128(a)); } +template<> EIGEN_STRONG_INLINE uint32_t pfirst(const Packet8ui& a) { + return numext::bit_cast(_mm_cvtsi128_si32(_mm256_castsi256_si128(a))); +} template<> EIGEN_STRONG_INLINE Packet8f preverse(const Packet8f& a) @@ -713,13 +1422,30 @@ template<> EIGEN_STRONG_INLINE Packet4d preverse(const Packet4d& a) { __m256d tmp = _mm256_shuffle_pd(a,a,5); return _mm256_permute2f128_pd(tmp, tmp, 1); - #if 0 +#if 0 // This version is unlikely to be faster as _mm256_shuffle_ps and _mm256_permute_pd // exhibit the same latency/throughput, but it is here for future reference/benchmarking... __m256d swap_halves = _mm256_permute2f128_pd(a,a,1); return _mm256_permute_pd(swap_halves,5); - #endif +#endif +} +template<> EIGEN_STRONG_INLINE Packet8i preverse(const Packet8i& a) +{ + return _mm256_castps_si256(preverse(_mm256_castsi256_ps(a))); } +template<> EIGEN_STRONG_INLINE Packet8ui preverse(const Packet8ui& a) { + return _mm256_castps_si256(preverse(_mm256_castsi256_ps(a))); +} + +#ifdef EIGEN_VECTORIZE_AVX2 +template<> EIGEN_STRONG_INLINE Packet4l preverse(const Packet4l& a) + { + return _mm256_castpd_si256(preverse(_mm256_castsi256_pd(a))); +} +template<> EIGEN_STRONG_INLINE Packet4ul preverse(const Packet4ul& a) { + return _mm256_castpd_si256(preverse(_mm256_castsi256_pd(a))); +} +#endif // pabs should be ok template<> EIGEN_STRONG_INLINE Packet8f pabs(const Packet8f& a) @@ -732,6 +1458,26 @@ template<> EIGEN_STRONG_INLINE Packet4d pabs(const Packet4d& a) const Packet4d mask = _mm256_castsi256_pd(_mm256_setr_epi32(0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF)); return _mm256_and_pd(a,mask); } +template<> EIGEN_STRONG_INLINE Packet8i pabs(const Packet8i& a) +{ +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_abs_epi32(a); +#else + __m128i lo = _mm_abs_epi32(_mm256_extractf128_si256(a, 0)); + __m128i hi = _mm_abs_epi32(_mm256_extractf128_si256(a, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} +template<> EIGEN_STRONG_INLINE Packet8ui pabs(const Packet8ui& a) { return a; } + +template<> EIGEN_STRONG_INLINE Packet8h psignbit(const Packet8h& a) { return _mm_srai_epi16(a, 15); } +template<> EIGEN_STRONG_INLINE Packet8bf psignbit(const Packet8bf& a) { return _mm_srai_epi16(a, 15); } +template<> EIGEN_STRONG_INLINE Packet8f psignbit(const Packet8f& a) { return _mm256_castsi256_ps(parithmetic_shift_right<31>((Packet8i)_mm256_castps_si256(a))); } +template<> EIGEN_STRONG_INLINE Packet8ui psignbit(const Packet8ui& a) { return pzero(a); } +#ifdef EIGEN_VECTORIZE_AVX2 +template<> EIGEN_STRONG_INLINE Packet4d psignbit(const Packet4d& a) { return _mm256_castsi256_pd(parithmetic_shift_right<63>((Packet4l)_mm256_castpd_si256(a))); } +template<> EIGEN_STRONG_INLINE Packet4ul psignbit(const Packet4ul& a) { return pzero(a); } +#endif template<> EIGEN_STRONG_INLINE Packet8f pfrexp(const Packet8f& a, Packet8f& exponent) { return pfrexp_generic(a,exponent); @@ -773,18 +1519,18 @@ template<> EIGEN_STRONG_INLINE Packet4d pldexp(const Packet4d& a, cons // Clamp exponent to [-2099, 2099] const Packet4d max_exponent = pset1(2099.0); const Packet4i e = _mm256_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); - + // Split 2^e into four factors and multiply. const Packet4i bias = pset1(1023); Packet4i b = parithmetic_shift_right<2>(e); // floor(e/4) - + // 2^b Packet4i hi = vec4i_swizzle1(padd(b, bias), 0, 2, 1, 3); Packet4i lo = _mm_slli_epi64(hi, 52); hi = _mm_slli_epi64(_mm_srli_epi64(hi, 32), 52); Packet4d c = _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1)); Packet4d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b) - + // 2^(e - 3b) b = psub(psub(psub(e, b), b), b); // e - 3b hi = vec4i_swizzle1(padd(b, bias), 0, 2, 1, 3); @@ -803,11 +1549,25 @@ template<> EIGEN_STRONG_INLINE double predux(const Packet4d& a) { return predux(Packet2d(_mm_add_pd(_mm256_castpd256_pd128(a),_mm256_extractf128_pd(a,1)))); } +template<> EIGEN_STRONG_INLINE int predux(const Packet8i& a) +{ + return predux(Packet4i(_mm_add_epi32(_mm256_castsi256_si128(a),_mm256_extractf128_si256(a,1)))); +} +template<> EIGEN_STRONG_INLINE uint32_t predux(const Packet8ui& a) { + return predux(Packet4ui(_mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1)))); +} template<> EIGEN_STRONG_INLINE Packet4f predux_half_dowto4(const Packet8f& a) { return _mm_add_ps(_mm256_castps256_ps128(a),_mm256_extractf128_ps(a,1)); } +template<> EIGEN_STRONG_INLINE Packet4i predux_half_dowto4(const Packet8i& a) +{ + return _mm_add_epi32(_mm256_castsi256_si128(a),_mm256_extractf128_si256(a,1)); +} +template<> EIGEN_STRONG_INLINE Packet4ui predux_half_dowto4(const Packet8ui& a) { + return _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1)); +} template<> EIGEN_STRONG_INLINE float predux_mul(const Packet8f& a) { @@ -856,7 +1616,16 @@ template<> EIGEN_STRONG_INLINE double predux_max(const Packet4d& a) template<> EIGEN_STRONG_INLINE bool predux_any(const Packet8f& x) { - return _mm256_movemask_ps(x)!=0; + return _mm256_movemask_ps(x) != 0; +} + +template<> EIGEN_STRONG_INLINE bool predux_any(const Packet8i& x) +{ + return _mm256_movemask_ps(_mm256_castsi256_ps(x)) != 0; +} +template<> EIGEN_STRONG_INLINE bool predux_any(const Packet8ui& x) +{ + return _mm256_movemask_ps(_mm256_castsi256_ps(x)) != 0; } EIGEN_DEVICE_FUNC inline void @@ -905,6 +1674,72 @@ ptranspose(PacketBlock& kernel) { kernel.packet[3] = _mm256_permute2f128_ps(S2, S3, 0x31); } +#define MM256_SHUFFLE_EPI32(A, B, M) \ + _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(A), _mm256_castsi256_ps(B), M)) + +#ifndef EIGEN_VECTORIZE_AVX2 +#define MM256_UNPACKLO_EPI32(A, B) \ + _mm256_castps_si256(_mm256_unpacklo_ps(_mm256_castsi256_ps(A), _mm256_castsi256_ps(B))) +#define MM256_UNPACKHI_EPI32(A, B) \ + _mm256_castps_si256(_mm256_unpackhi_ps(_mm256_castsi256_ps(A), _mm256_castsi256_ps(B))) +#else +#define MM256_UNPACKLO_EPI32(A, B) _mm256_unpacklo_epi32(A, B) +#define MM256_UNPACKHI_EPI32(A, B) _mm256_unpackhi_epi32(A, B) +#endif + + +EIGEN_DEVICE_FUNC inline void +ptranspose(PacketBlock& kernel) { + __m256i T0 = MM256_UNPACKLO_EPI32(kernel.packet[0], kernel.packet[1]); + __m256i T1 = MM256_UNPACKHI_EPI32(kernel.packet[0], kernel.packet[1]); + __m256i T2 = MM256_UNPACKLO_EPI32(kernel.packet[2], kernel.packet[3]); + __m256i T3 = MM256_UNPACKHI_EPI32(kernel.packet[2], kernel.packet[3]); + __m256i T4 = MM256_UNPACKLO_EPI32(kernel.packet[4], kernel.packet[5]); + __m256i T5 = MM256_UNPACKHI_EPI32(kernel.packet[4], kernel.packet[5]); + __m256i T6 = MM256_UNPACKLO_EPI32(kernel.packet[6], kernel.packet[7]); + __m256i T7 = MM256_UNPACKHI_EPI32(kernel.packet[6], kernel.packet[7]); + __m256i S0 = MM256_SHUFFLE_EPI32(T0,T2,_MM_SHUFFLE(1,0,1,0)); + __m256i S1 = MM256_SHUFFLE_EPI32(T0,T2,_MM_SHUFFLE(3,2,3,2)); + __m256i S2 = MM256_SHUFFLE_EPI32(T1,T3,_MM_SHUFFLE(1,0,1,0)); + __m256i S3 = MM256_SHUFFLE_EPI32(T1,T3,_MM_SHUFFLE(3,2,3,2)); + __m256i S4 = MM256_SHUFFLE_EPI32(T4,T6,_MM_SHUFFLE(1,0,1,0)); + __m256i S5 = MM256_SHUFFLE_EPI32(T4,T6,_MM_SHUFFLE(3,2,3,2)); + __m256i S6 = MM256_SHUFFLE_EPI32(T5,T7,_MM_SHUFFLE(1,0,1,0)); + __m256i S7 = MM256_SHUFFLE_EPI32(T5,T7,_MM_SHUFFLE(3,2,3,2)); + kernel.packet[0] = _mm256_permute2f128_si256(S0, S4, 0x20); + kernel.packet[1] = _mm256_permute2f128_si256(S1, S5, 0x20); + kernel.packet[2] = _mm256_permute2f128_si256(S2, S6, 0x20); + kernel.packet[3] = _mm256_permute2f128_si256(S3, S7, 0x20); + kernel.packet[4] = _mm256_permute2f128_si256(S0, S4, 0x31); + kernel.packet[5] = _mm256_permute2f128_si256(S1, S5, 0x31); + kernel.packet[6] = _mm256_permute2f128_si256(S2, S6, 0x31); + kernel.packet[7] = _mm256_permute2f128_si256(S3, S7, 0x31); +} +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { + ptranspose((PacketBlock&)kernel); +} + +EIGEN_DEVICE_FUNC inline void +ptranspose(PacketBlock& kernel) { + __m256i T0 = MM256_UNPACKLO_EPI32(kernel.packet[0], kernel.packet[1]); + __m256i T1 = MM256_UNPACKHI_EPI32(kernel.packet[0], kernel.packet[1]); + __m256i T2 = MM256_UNPACKLO_EPI32(kernel.packet[2], kernel.packet[3]); + __m256i T3 = MM256_UNPACKHI_EPI32(kernel.packet[2], kernel.packet[3]); + + __m256i S0 = MM256_SHUFFLE_EPI32(T0,T2,_MM_SHUFFLE(1,0,1,0)); + __m256i S1 = MM256_SHUFFLE_EPI32(T0,T2,_MM_SHUFFLE(3,2,3,2)); + __m256i S2 = MM256_SHUFFLE_EPI32(T1,T3,_MM_SHUFFLE(1,0,1,0)); + __m256i S3 = MM256_SHUFFLE_EPI32(T1,T3,_MM_SHUFFLE(3,2,3,2)); + + kernel.packet[0] = _mm256_permute2f128_si256(S0, S1, 0x20); + kernel.packet[1] = _mm256_permute2f128_si256(S2, S3, 0x20); + kernel.packet[2] = _mm256_permute2f128_si256(S0, S1, 0x31); + kernel.packet[3] = _mm256_permute2f128_si256(S2, S3, 0x31); +} +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { + ptranspose((PacketBlock&)kernel); +} + EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m256d T0 = _mm256_shuffle_pd(kernel.packet[0], kernel.packet[1], 15); @@ -919,21 +1754,37 @@ ptranspose(PacketBlock& kernel) { } template<> EIGEN_STRONG_INLINE Packet8f pblend(const Selector<8>& ifPacket, const Packet8f& thenPacket, const Packet8f& elsePacket) { +#ifdef EIGEN_VECTORIZE_AVX2 + const __m256i zero = _mm256_setzero_si256(); + const __m256i select = _mm256_set_epi32(ifPacket.select[7], ifPacket.select[6], ifPacket.select[5], ifPacket.select[4], ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]); + __m256i false_mask = _mm256_cmpeq_epi32(zero, select); + return _mm256_blendv_ps(thenPacket, elsePacket, _mm256_castsi256_ps(false_mask)); +#else const __m256 zero = _mm256_setzero_ps(); const __m256 select = _mm256_set_ps(ifPacket.select[7], ifPacket.select[6], ifPacket.select[5], ifPacket.select[4], ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]); __m256 false_mask = _mm256_cmp_ps(select, zero, _CMP_EQ_UQ); return _mm256_blendv_ps(thenPacket, elsePacket, false_mask); +#endif } + template<> EIGEN_STRONG_INLINE Packet4d pblend(const Selector<4>& ifPacket, const Packet4d& thenPacket, const Packet4d& elsePacket) { +#ifdef EIGEN_VECTORIZE_AVX2 + const __m256i zero = _mm256_setzero_si256(); + const __m256i select = _mm256_set_epi64x(ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]); + __m256i false_mask = _mm256_cmpeq_epi64(select, zero); + return _mm256_blendv_pd(thenPacket, elsePacket, _mm256_castsi256_pd(false_mask)); +#else const __m256d zero = _mm256_setzero_pd(); const __m256d select = _mm256_set_pd(ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]); __m256d false_mask = _mm256_cmp_pd(select, zero, _CMP_EQ_UQ); return _mm256_blendv_pd(thenPacket, elsePacket, false_mask); +#endif } // Packet math for Eigen::half - +#ifndef EIGEN_VECTORIZE_AVX512FP16 template<> struct unpacket_traits { typedef Eigen::half type; enum {size=8, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet8h half; }; +#endif template<> EIGEN_STRONG_INLINE Packet8h pset1(const Eigen::half& from) { return _mm_set1_epi16(numext::bit_cast(from)); @@ -976,7 +1827,7 @@ ploadquad(const Eigen::half* from) { } template<> EIGEN_STRONG_INLINE Packet8h ptrue(const Packet8h& a) { - return _mm_cmpeq_epi32(a, a); + return _mm_cmpeq_epi32(a, a); } template <> @@ -989,36 +1840,19 @@ EIGEN_STRONG_INLINE Packet8f half2float(const Packet8h& a) { #ifdef EIGEN_HAS_FP16_C return _mm256_cvtph_ps(a); #else - EIGEN_ALIGN32 Eigen::half aux[8]; - pstore(aux, a); - float f0(aux[0]); - float f1(aux[1]); - float f2(aux[2]); - float f3(aux[3]); - float f4(aux[4]); - float f5(aux[5]); - float f6(aux[6]); - float f7(aux[7]); - - return _mm256_set_ps(f7, f6, f5, f4, f3, f2, f1, f0); + Eigen::internal::Packet8f pp = _mm256_castsi256_ps(_mm256_insertf128_si256( + _mm256_castsi128_si256(half2floatsse(a)), half2floatsse(_mm_srli_si128(a, 8)), 1)); + return pp; #endif } EIGEN_STRONG_INLINE Packet8h float2half(const Packet8f& a) { #ifdef EIGEN_HAS_FP16_C - return _mm256_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC); + return _mm256_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT); #else - EIGEN_ALIGN32 float aux[8]; - pstore(aux, a); - const numext::uint16_t s0 = numext::bit_cast(Eigen::half(aux[0])); - const numext::uint16_t s1 = numext::bit_cast(Eigen::half(aux[1])); - const numext::uint16_t s2 = numext::bit_cast(Eigen::half(aux[2])); - const numext::uint16_t s3 = numext::bit_cast(Eigen::half(aux[3])); - const numext::uint16_t s4 = numext::bit_cast(Eigen::half(aux[4])); - const numext::uint16_t s5 = numext::bit_cast(Eigen::half(aux[5])); - const numext::uint16_t s6 = numext::bit_cast(Eigen::half(aux[6])); - const numext::uint16_t s7 = numext::bit_cast(Eigen::half(aux[7])); - return _mm_set_epi16(s7, s6, s5, s4, s3, s2, s1, s0); + __m128i lo = float2half(_mm256_extractf128_ps(a, 0)); + __m128i hi = float2half(_mm256_extractf128_ps(a, 1)); + return _mm_packus_epi32(lo, hi); #endif } @@ -1097,6 +1931,7 @@ template<> EIGEN_STRONG_INLINE Packet8h pnegate(const Packet8h& a) { return _mm_xor_si128(a, sign_mask); } +#ifndef EIGEN_VECTORIZE_AVX512FP16 template<> EIGEN_STRONG_INLINE Packet8h padd(const Packet8h& a, const Packet8h& b) { Packet8f af = half2float(a); Packet8f bf = half2float(b); @@ -1124,6 +1959,7 @@ template<> EIGEN_STRONG_INLINE Packet8h pdiv(const Packet8h& a, const Packet8f rf = pdiv(af, bf); return float2half(rf); } +#endif template<> EIGEN_STRONG_INLINE Packet8h pgather(const Eigen::half* from, Index stride) { @@ -1152,11 +1988,14 @@ template<> EIGEN_STRONG_INLINE void pscatter(Eigen::half* to[stride*7] = aux[7]; } + +#ifndef EIGEN_VECTORIZE_AVX512FP16 template<> EIGEN_STRONG_INLINE Eigen::half predux(const Packet8h& a) { Packet8f af = half2float(a); float reduced = predux(af); return Eigen::half(reduced); } +#endif template<> EIGEN_STRONG_INLINE Eigen::half predux_max(const Packet8h& a) { Packet8f af = half2float(a); @@ -1272,7 +2111,6 @@ EIGEN_STRONG_INLINE Packet8f Bf16ToF32(const Packet8bf& a) { // Convert float to bfloat16 according to round-to-nearest-even/denormals algorithm. EIGEN_STRONG_INLINE Packet8bf F32ToBf16(const Packet8f& a) { - Packet8bf r; __m256i input = _mm256_castps_si256(a); @@ -1361,7 +2199,7 @@ ploadquad(const bfloat16* from) { } template<> EIGEN_STRONG_INLINE Packet8bf ptrue(const Packet8bf& a) { - return _mm_cmpeq_epi32(a, a); + return _mm_cmpeq_epi32(a, a); } template <> diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/TypeCasting.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/TypeCasting.h index d507fb67b21..49927b83950 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/TypeCasting.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX/TypeCasting.h @@ -10,80 +10,120 @@ #ifndef EIGEN_TYPE_CASTING_AVX_H #define EIGEN_TYPE_CASTING_AVX_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { -// For now we use SSE to handle integers, so we can't use AVX instructions to cast -// from int to float -template <> -struct type_casting_traits { - enum { - VectorizedCast = 0, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; - -template <> -struct type_casting_traits { - enum { - VectorizedCast = 0, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; +#ifndef EIGEN_VECTORIZE_AVX512 +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; -#ifndef EIGEN_VECTORIZE_AVX512 +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +#endif template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; +EIGEN_STRONG_INLINE Packet16b pcast(const Packet8f& a, + const Packet8f& b) { + __m256 nonzero_a = _mm256_cmp_ps(a, pzero(a), _CMP_NEQ_UQ); + __m256 nonzero_b = _mm256_cmp_ps(b, pzero(b), _CMP_NEQ_UQ); + constexpr char kFF = '\255'; +#ifndef EIGEN_VECTORIZE_AVX2 + __m128i shuffle_mask128_a_lo = _mm_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0); + __m128i shuffle_mask128_a_hi = _mm_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF); + __m128i shuffle_mask128_b_lo = _mm_set_epi8(kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF); + __m128i shuffle_mask128_b_hi = _mm_set_epi8(12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF); + __m128i a_hi = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_a), 1), shuffle_mask128_a_hi); + __m128i a_lo = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_a), 0), shuffle_mask128_a_lo); + __m128i b_hi = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_b), 1), shuffle_mask128_b_hi); + __m128i b_lo = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_b), 0), shuffle_mask128_b_lo); + __m128i merged = _mm_or_si128(_mm_or_si128(b_lo, b_hi), _mm_or_si128(a_lo, a_hi)); + return _mm_and_si128(merged, _mm_set1_epi8(1)); + #else + __m256i a_shuffle_mask = _mm256_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, + kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0); + __m256i b_shuffle_mask = _mm256_set_epi8( 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, + kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF); + __m256i a_shuff = _mm256_shuffle_epi8(_mm256_castps_si256(nonzero_a), a_shuffle_mask); + __m256i b_shuff = _mm256_shuffle_epi8(_mm256_castps_si256(nonzero_b), b_shuffle_mask); + __m256i a_or_b = _mm256_or_si256(a_shuff, b_shuff); + __m256i merged = _mm256_or_si256(a_or_b, _mm256_castsi128_si256(_mm256_extractf128_si256(a_or_b, 1))); + return _mm256_castsi256_si128(_mm256_and_si256(merged, _mm256_set1_epi8(1))); +#endif +} template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; - -#endif // EIGEN_VECTORIZE_AVX512 +EIGEN_STRONG_INLINE Packet8f pcast(const Packet16b& a) { + const __m256 cst_one = _mm256_set1_ps(1.0f); + #ifdef EIGEN_VECTORIZE_AVX2 + __m256i a_extended = _mm256_cvtepi8_epi32(a); + __m256i abcd_efgh = _mm256_cmpeq_epi32(a_extended, _mm256_setzero_si256()); + #else + __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128()); + __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop); + __m128i aaaa_bbbb_cccc_dddd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh); + __m128i eeee_ffff_gggg_hhhh = _mm_unpackhi_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh); + __m256i abcd_efgh = _mm256_setr_m128i(aaaa_bbbb_cccc_dddd, eeee_ffff_gggg_hhhh); + #endif + __m256 result = _mm256_andnot_ps(_mm256_castsi256_ps(abcd_efgh), cst_one); + return result; +} template<> EIGEN_STRONG_INLINE Packet8i pcast(const Packet8f& a) { return _mm256_cvttps_epi32(a); } +template<> EIGEN_STRONG_INLINE Packet8i pcast(const Packet4d& a, const Packet4d& b) { + return _mm256_set_m128i(_mm256_cvttpd_epi32(b), _mm256_cvttpd_epi32(a)); +} + +template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet4d& a) { + return _mm256_cvttpd_epi32(a); +} + template<> EIGEN_STRONG_INLINE Packet8f pcast(const Packet8i& a) { return _mm256_cvtepi32_ps(a); } +template<> EIGEN_STRONG_INLINE Packet8f pcast(const Packet4d& a, const Packet4d& b) { + return _mm256_set_m128(_mm256_cvtpd_ps(b), _mm256_cvtpd_ps(a)); +} + +template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4d& a) { + return _mm256_cvtpd_ps(a); +} + +template <> EIGEN_STRONG_INLINE Packet4d pcast(const Packet8i& a) { + return _mm256_cvtepi32_pd(_mm256_castsi256_si128(a)); +} + +template <> EIGEN_STRONG_INLINE Packet4d pcast(const Packet4i& a) { + return _mm256_cvtepi32_pd(a); +} + +template <> EIGEN_STRONG_INLINE Packet4d pcast(const Packet8f& a) { + return _mm256_cvtps_pd(_mm256_castps256_ps128(a)); +} + +template <> EIGEN_STRONG_INLINE Packet4d pcast(const Packet4f& a) { + return _mm256_cvtps_pd(a); +} + template<> EIGEN_STRONG_INLINE Packet8i preinterpret(const Packet8f& a) { return _mm256_castps_si256(a); } @@ -92,6 +132,44 @@ template<> EIGEN_STRONG_INLINE Packet8f preinterpret(const Pa return _mm256_castsi256_ps(a); } +template<> EIGEN_STRONG_INLINE Packet8ui preinterpret(const Packet8i& a) { + return Packet8ui(a); +} + +template<> EIGEN_STRONG_INLINE Packet8i preinterpret(const Packet8ui& a) { + return Packet8i(a); +} + +// truncation operations + +template<> EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet8f& a) { + return _mm256_castps256_ps128(a); +} + +template<> EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet4d& a) { + return _mm256_castpd256_pd128(a); +} + +template<> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet8i& a) { + return _mm256_castsi256_si128(a); +} + +template<> EIGEN_STRONG_INLINE Packet4ui preinterpret(const Packet8ui& a) { + return _mm256_castsi256_si128(a); +} + + +#ifdef EIGEN_VECTORIZE_AVX2 +template<> EIGEN_STRONG_INLINE Packet4ul preinterpret(const Packet4l& a) { + return Packet4ul(a); +} + +template<> EIGEN_STRONG_INLINE Packet4l preinterpret(const Packet4ul& a) { + return Packet4l(a); +} + +#endif + template<> EIGEN_STRONG_INLINE Packet8f pcast(const Packet8h& a) { return half2float(a); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/Complex.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/Complex.h index 07425385934..c484517e79a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/Complex.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/Complex.h @@ -10,6 +10,9 @@ #ifndef EIGEN_COMPLEX_AVX512_H #define EIGEN_COMPLEX_AVX512_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -30,7 +33,6 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasHalfPacket = 1, HasAdd = 1, HasSub = 1, @@ -159,11 +161,7 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet8cf,Packet16f) template<> EIGEN_STRONG_INLINE Packet8cf pdiv(const Packet8cf& a, const Packet8cf& b) { - Packet8cf num = pmul(a, pconj(b)); - __m512 tmp = _mm512_mul_ps(b.v, b.v); - __m512 tmp2 = _mm512_shuffle_ps(tmp,tmp,0xB1); - __m512 denom = _mm512_add_ps(tmp, tmp2); - return Packet8cf(_mm512_div_ps(num.v, denom)); + return pdiv_complex(a, b); } template<> EIGEN_STRONG_INLINE Packet8cf pcplxflip(const Packet8cf& x) @@ -187,7 +185,6 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 0, size = 4, - HasHalfPacket = 1, HasAdd = 1, HasSub = 1, @@ -255,11 +252,7 @@ template<> EIGEN_STRONG_INLINE Packet4cd ploadu(const std::complex EIGEN_STRONG_INLINE Packet4cd pset1(const std::complex& from) { - #ifdef EIGEN_VECTORIZE_AVX512DQ - return Packet4cd(_mm512_broadcast_f64x2(pset1(from).v)); - #else return Packet4cd(_mm512_castps_pd(_mm512_broadcast_f32x4( _mm_castpd_ps(pset1(from).v)))); - #endif } template<> EIGEN_STRONG_INLINE Packet4cd ploaddup(const std::complex* from) { @@ -311,47 +304,11 @@ template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet2cd(_mm512_extractf64x4_pd(a.v,1)))); } -template<> struct conj_helper -{ - EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const - { return padd(pmul(x,y),c); } - - EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const - { - return internal::pmul(a, pconj(b)); - } -}; - -template<> struct conj_helper -{ - EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const - { return padd(pmul(x,y),c); } - - EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const - { - return internal::pmul(pconj(a), b); - } -}; - -template<> struct conj_helper -{ - EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const - { return padd(pmul(x,y),c); } - - EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const - { - return pconj(internal::pmul(a, b)); - } -}; - EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet4cd,Packet8d) template<> EIGEN_STRONG_INLINE Packet4cd pdiv(const Packet4cd& a, const Packet4cd& b) { - Packet4cd num = pmul(a, pconj(b)); - __m512d tmp = _mm512_mul_pd(b.v, b.v); - __m512d denom = padd(_mm512_permute_pd(tmp,0x55), tmp); - return Packet4cd(_mm512_div_pd(num.v, denom)); + return pdiv_complex(a, b); } template<> EIGEN_STRONG_INLINE Packet4cd pcplxflip(const Packet4cd& x) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/GemmKernel.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/GemmKernel.h new file mode 100644 index 00000000000..2df17040aed --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/GemmKernel.h @@ -0,0 +1,1243 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2022 Intel Corporation +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CORE_ARCH_AVX512_GEMM_KERNEL_H +#define EIGEN_CORE_ARCH_AVX512_GEMM_KERNEL_H + +#if EIGEN_COMP_MSVC +#include +#else +#include +#endif +#include +#include + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +#if !defined(EIGEN_USE_AVX512_GEMM_KERNELS) +#define EIGEN_USE_AVX512_GEMM_KERNELS 1 +#endif + +#define SECOND_FETCH (32) +#if (EIGEN_COMP_GNUC_STRICT != 0) && !defined(EIGEN_ARCH_AVX512_GEMM_KERNEL_USE_LESS_A_REGS) +// Use less registers to load A elements to workaround compiler spills. Loose a +// bit of performance (less than ~2%). +#define EIGEN_ARCH_AVX512_GEMM_KERNEL_USE_LESS_A_REGS +#endif + +namespace Eigen { +namespace internal { + +template +class gemm_class { + using vec = typename packet_traits::type; + using vec_ymm = typename unpacket_traits::half; + using vec_xmm = typename unpacket_traits::half; + using umask_t = typename unpacket_traits::mask_t; + + static constexpr bool is_f32 = sizeof(Scalar) == sizeof(float); + static constexpr bool is_f64 = sizeof(Scalar) == sizeof(double); + +#ifndef EIGEN_ARCH_AVX512_GEMM_KERNEL_USE_LESS_A_REGS + static constexpr bool use_less_a_regs = !is_unit_inc; +#else + static constexpr bool use_less_a_regs = true; +#endif +#ifndef EIGEN_ARCH_AVX512_GEMM_KERNEL_USE_LESS_B_REGS + static constexpr bool use_less_b_regs = !is_unit_inc; +#else + static constexpr bool use_less_b_regs = true; +#endif + + static constexpr int a_regs[] = {0, 1, 2, use_less_a_regs ? 0 : 3, use_less_a_regs ? 1 : 4, use_less_a_regs ? 2 : 5}; + static constexpr int b_regs[] = {6, use_less_b_regs ? 6 : 7}; + static constexpr int c_regs[] = { + 8, 16, 24, 9, 17, 25, 10, 18, 26, 11, 19, 27, 12, 20, 28, 13, 21, 29, 14, 22, 30, 15, 23, 31, + }; + + static constexpr int alpha_load_reg = 0; + static constexpr int c_load_regs[] = {1, 2, 6}; + + static constexpr int a_shift = 128; + static constexpr int b_shift = 128; + + static constexpr int nelems_in_cache_line = is_f32 ? 16 : 8; + static constexpr int a_prefetch_size = nelems_in_cache_line * 2; + static constexpr int b_prefetch_size = nelems_in_cache_line * 8; + + vec zmm[32]; + umask_t mask; + + // gemm arguments. + Index m; + const Index n, k, ldc; + const Index inc; + const Scalar *alpha; + + const Scalar *a, *b; + Scalar *c; + + const bool is_alpha1; + const bool is_beta0; + + const Index a_stride, b_stride; + const Index a_off, b_off; + + EIGEN_ALWAYS_INLINE void prefetch_a(const Scalar *a_addr) { + _mm_prefetch((char *)(a_prefetch_size + a_addr - a_shift), _MM_HINT_T0); + } + + EIGEN_ALWAYS_INLINE void prefetch_b(const Scalar *b_addr) { + _mm_prefetch((char *)(b_prefetch_size + b_addr - b_shift), _MM_HINT_T0); + } + + EIGEN_ALWAYS_INLINE void prefetch_x(const Scalar *x_addr) { _mm_prefetch((char *)(x_addr - a_shift), _MM_HINT_T2); } + + EIGEN_ALWAYS_INLINE void prefetch_c(const Scalar *c_addr) { +#if defined(__PRFCHW__) && __PRFCHW__ == 1 + _m_prefetchw((void *)c_addr); +#else + _mm_prefetch((char *)c_addr, _MM_HINT_T0); +#endif + } + + template + EIGEN_ALWAYS_INLINE void a_load(vec &a_reg, const Scalar *a_addr) { + switch (nelems * sizeof(*a_addr) * 8) { + default: + case 512 * 3: + a_reg = ploadu(a_addr); + break; + case 512 * 2: + a_reg = ploadu(a_addr); + break; + case 512 * 1: + a_reg = ploadu(a_addr); + break; + case 256 * 1: + a_reg = preinterpret(_mm512_broadcast_f64x4(ploadu(reinterpret_cast(a_addr)))); + break; + case 128 * 1: + a_reg = preinterpret(_mm512_broadcast_f32x4(ploadu(reinterpret_cast(a_addr)))); + break; + case 64 * 1: + a_reg = preinterpret(pload1(reinterpret_cast(a_addr))); + break; + case 32 * 1: + a_reg = pload1(a_addr); + break; + } + } + + EIGEN_ALWAYS_INLINE void b_load(vec &b_reg, const Scalar *b_addr) { b_reg = pload1(b_addr); } + + template + EIGEN_ALWAYS_INLINE void c_store(Scalar *mem, vec &src) { + if (is_unit_inc) { + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + pstoreu(mem, src); + break; + case 512 * 2: + pstoreu(mem, src); + break; + case 512 * 1: + pstoreu(mem, src); + break; + case 256 * 1: + pstoreu(mem, preinterpret(src)); + break; + case 128 * 1: + pstoreu(mem, preinterpret(src)); + break; + case 64 * 1: + pstorel(mem, preinterpret(src)); + break; + case 32 * 1: + pstores(mem, preinterpret(src)); + break; + } + } else { + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + pscatter(mem, src, inc); + break; + case 512 * 2: + pscatter(mem, src, inc); + break; + case 512 * 1: + pscatter(mem, src, inc); + break; + case 256 * 1: + pscatter(mem, src, inc, mask); + break; + case 128 * 1: + pscatter(mem, src, inc, mask); + break; + case 64 * 1: + pscatter(mem, src, inc, mask); + break; + case 32 * 1: + pscatter(mem, src, inc, mask); + break; + } + } + } + + template + EIGEN_ALWAYS_INLINE void vaddm(vec &dst, const Scalar *mem, vec &src, vec ®) { + if (is_unit_inc) { + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + dst = padd(src, ploadu(mem)); + break; + case 512 * 2: + dst = padd(src, ploadu(mem)); + break; + case 512 * 1: + dst = padd(src, ploadu(mem)); + break; + case 256 * 1: + dst = preinterpret(padd(preinterpret(src), ploadu(mem))); + break; + case 128 * 1: + dst = preinterpret(padd(preinterpret(src), ploadu(mem))); + break; + case 64 * 1: + dst = preinterpret(padd(preinterpret(src), ploadl(mem))); + break; + case 32 * 1: + dst = preinterpret(padds(preinterpret(src), ploads(mem))); + break; + } + } else { + // Zero out scratch register + reg = pzero(reg); + + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + reg = pgather(mem, inc); + dst = padd(src, reg); + break; + case 512 * 2: + reg = pgather(mem, inc); + dst = padd(src, reg); + break; + case 512 * 1: + reg = pgather(mem, inc); + dst = padd(src, reg); + break; + case 256 * 1: + reg = preinterpret(pgather(mem, inc)); + dst = preinterpret(padd(preinterpret(src), preinterpret(reg))); + break; + case 128 * 1: + reg = preinterpret(pgather(mem, inc)); + dst = preinterpret(padd(preinterpret(src), preinterpret(reg))); + break; + case 64 * 1: + if (is_f32) { + reg = pgather(reg, mem, inc, mask); + dst = preinterpret(padd(preinterpret(src), preinterpret(reg))); + } else { + dst = preinterpret(padd(preinterpret(src), ploadl(mem))); + } + break; + case 32 * 1: + dst = preinterpret(padds(preinterpret(src), ploads(mem))); + break; + } + } + } + + EIGEN_STRONG_INLINE void vfmadd(vec &dst, const vec &src1, const vec &src2) { + dst = pmadd(src1, src2, dst); + +#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0) + // Workaround register spills for gcc and clang + __asm__("#" : [dst] "+v"(dst) : [src1] "%v"(src1), [src2] "v"(src2)); +#endif + } + + template + EIGEN_ALWAYS_INLINE void vfmaddm(vec &dst, const Scalar *mem, vec &src, vec &scale, vec ®) { + if (is_unit_inc) { + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + dst = pmadd(scale, src, ploadu(mem)); + break; + case 512 * 2: + dst = pmadd(scale, src, ploadu(mem)); + break; + case 512 * 1: + dst = pmadd(scale, src, ploadu(mem)); + break; + case 256 * 1: + dst = + preinterpret(pmadd(preinterpret(scale), preinterpret(src), ploadu(mem))); + break; + case 128 * 1: + dst = + preinterpret(pmadd(preinterpret(scale), preinterpret(src), ploadu(mem))); + break; + case 64 * 1: + dst = + preinterpret(pmadd(preinterpret(scale), preinterpret(src), ploadl(mem))); + break; + case 32 * 1: + dst = + preinterpret(pmadds(preinterpret(scale), preinterpret(src), ploads(mem))); + break; + } + } else { + // Zero out scratch register + reg = pzero(reg); + + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + reg = pgather(mem, inc); + dst = pmadd(scale, src, reg); + break; + case 512 * 2: + reg = pgather(mem, inc); + dst = pmadd(scale, src, reg); + break; + case 512 * 1: + reg = pgather(mem, inc); + dst = pmadd(scale, src, reg); + break; + case 256 * 1: + reg = preinterpret(pgather(mem, inc)); + dst = preinterpret( + pmadd(preinterpret(scale), preinterpret(src), preinterpret(reg))); + break; + case 128 * 1: + reg = preinterpret(pgather(mem, inc)); + dst = preinterpret( + pmadd(preinterpret(scale), preinterpret(src), preinterpret(reg))); + break; + case 64 * 1: + if (is_f32) { + reg = pgather(reg, mem, inc, mask); + dst = preinterpret( + pmadd(preinterpret(scale), preinterpret(src), preinterpret(reg))); + } else { + dst = preinterpret( + pmadd(preinterpret(scale), preinterpret(src), ploadl(mem))); + } + break; + case 32 * 1: + dst = + preinterpret(pmadds(preinterpret(scale), preinterpret(src), ploads(mem))); + break; + } + } + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(j > endX) || (i > endY)> a_loads(const Scalar *ao) { + EIGEN_UNUSED_VARIABLE(ao); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(j <= endX) && (i <= endY)> a_loads(const Scalar *ao) { + if (j < endX) { + if (i < endY) { + auto &a_reg = zmm[a_regs[i + (j % 2) * 3]]; + const Scalar *a_addr = ao + nelems * j + nelems_in_cache_line * i - a_shift; + a_load(a_reg, a_addr); + + a_loads(ao); + } else { + a_loads(ao); + } + } + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(un > max_b_unroll) || (i > um_vecs)> prefetch_cs(const Scalar *co1, + const Scalar *co2) { + EIGEN_UNUSED_VARIABLE(co1); + EIGEN_UNUSED_VARIABLE(co2); + } + + /* C prefetch loop structure. + * for (int un = 0; un < 8; un++) { + * if (b_unroll >= un + 1) { + * if (un == 4) co2 = co1 + 4 * ldc; + * + * for (int i = 0; i < um_vecs; i++) { + * Scalar *co = (un + 1 <= 4) ? co1 : co2; + * auto co_off = (un % 4) * ldc + a_unroll - 1 + i * nelems_in_cache_line * sizeof *co; + * prefetch_c(co + co_off); + * } + * } + * } + */ + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(un <= max_b_unroll) && (i <= um_vecs)> prefetch_cs(Scalar *&co1, Scalar *&co2) { + if (un < max_b_unroll) { + if (b_unroll >= un + 1) { + if (un == 4 && i == 0) co2 = co1 + 4 * ldc; + + if (i < um_vecs) { + Scalar *co = (un + 1 <= 4) ? co1 : co2; + auto co_off = (un % 4) * ldc + a_unroll - 1 + i * nelems_in_cache_line * sizeof *co; + prefetch_c(co + co_off); + + prefetch_cs(co1, co2); + } else { + prefetch_cs(co1, co2); + } + + } else { + prefetch_cs(co1, co2); + } + } + } + + // load_c + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> scale_load_c(const Scalar *cox, vec &alpha_reg) { + EIGEN_UNUSED_VARIABLE(cox); + EIGEN_UNUSED_VARIABLE(alpha_reg); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> scale_load_c(const Scalar *cox, vec &alpha_reg) { + if (i < um_vecs) { + auto &c_reg = zmm[c_regs[i + idx * 3]]; + auto &c_load_reg = zmm[c_load_regs[i % 3]]; + auto c_mem = cox; + if (is_unit_inc) + c_mem += i * nelems_in_cache_line; + else + c_mem += i * nelems_in_cache_line * inc; + + if (!is_beta0 && is_alpha1) + vaddm(c_reg, c_mem, c_reg, c_load_reg); + else if (!is_beta0 && !is_alpha1) + vfmaddm(c_reg, c_mem, c_reg, alpha_reg, c_load_reg); + else if (is_beta0 && !is_alpha1) + c_reg = pmul(alpha_reg, c_reg); + + scale_load_c(cox, alpha_reg); + } + } + + // store_c + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> write_c(Scalar *cox) { + EIGEN_UNUSED_VARIABLE(cox); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> write_c(Scalar *cox) { + if (i < um_vecs) { + auto &c_reg = zmm[c_regs[i + idx * 3]]; + auto c_mem = cox; + if (is_unit_inc) + c_mem += i * nelems_in_cache_line; + else + c_mem += i * nelems_in_cache_line * inc; + + c_store(c_mem, c_reg); + c_reg = pzero(c_reg); + + write_c(cox); + } + } + + /* C update loop structure. + * co2 = co1 + ldc; + * + * auto &alpha_reg = zmm[alpha_load_reg]; + * if (!is_alpha1) alpha_reg = pload1(alpha); + * + * int idx = 0; + * for (pow = 1; pow <= 8; pow <<= 1) { + * + * if (b_unroll >= pow) { + * for (count = 1; count < (pow + 1) / 2 + 1; count++) { + * if (pow >= 4) co2 += ldc; + * + * const Scalar *cox = (idx == 0) ? co1 : co2; + * + * const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); + * scale_load_c<0, um_vecs, idx, a_unroll>(cox, alpha_reg); + * write_c<0, um_vecs, idx, a_unroll>(cox); + * + * idx++; + * } + * } + * } + * + * if (b_unroll == 1) + * co1 += ldc; + * else + * co1 = co2 + ldc; + */ + + template + EIGEN_ALWAYS_INLINE void c_update_1count(Scalar *&cox) { + if (pow >= 4) cox += ldc; + + const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); + auto &alpha_reg = zmm[alpha_load_reg]; + + scale_load_c<0, um_vecs, idx, a_unroll>(cox, alpha_reg); + write_c<0, um_vecs, idx, a_unroll>(cox); + } + + template + EIGEN_ALWAYS_INLINE void c_update_1pow(Scalar *&co1, Scalar *&co2) { + constexpr int idx = pow / 2; + Scalar *&cox = idx == 0 ? co1 : co2; + + constexpr int max_count = (pow + 1) / 2; + static_assert(max_count <= 4, "Unsupported max_count."); + + if (1 <= max_count) c_update_1count(cox); + if (2 <= max_count) c_update_1count(cox); + if (3 <= max_count) c_update_1count(cox); + if (4 <= max_count) c_update_1count(cox); + } + + template + EIGEN_ALWAYS_INLINE void c_update(Scalar *&co1, Scalar *&co2) { + auto &alpha_reg = zmm[alpha_load_reg]; + + co2 = co1 + ldc; + if (!is_alpha1) alpha_reg = pload1(alpha); + if (!is_unit_inc && a_unroll < nelems_in_cache_line) mask = static_cast((1ull << a_unroll) - 1); + + static_assert(max_b_unroll <= 8, "Unsupported max_b_unroll"); + + if (1 <= max_b_unroll && 1 <= b_unroll) c_update_1pow<1, a_unroll>(co1, co2); + if (2 <= max_b_unroll && 2 <= b_unroll) c_update_1pow<2, a_unroll>(co1, co2); + if (4 <= max_b_unroll && 4 <= b_unroll) c_update_1pow<4, a_unroll>(co1, co2); + if (8 <= max_b_unroll && 8 <= b_unroll) c_update_1pow<8, a_unroll>(co1, co2); + + if (b_unroll == 1) + co1 += ldc; + else + co1 = co2 + ldc; + } + + // compute + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> compute(const Scalar *ao, const Scalar *bo, int &fetchA_idx, + int &fetchB_idx, vec &b_reg) { + EIGEN_UNUSED_VARIABLE(ao); + EIGEN_UNUSED_VARIABLE(bo); + EIGEN_UNUSED_VARIABLE(fetchA_idx); + EIGEN_UNUSED_VARIABLE(fetchB_idx); + EIGEN_UNUSED_VARIABLE(b_reg); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> compute(const Scalar *ao, const Scalar *bo, int &fetchA_idx, + int &fetchB_idx, vec &b_reg) { + if (um < um_vecs) { + auto &c_reg = zmm[c_regs[um + idx * 3]]; + auto &a_reg = zmm[a_regs[um + (uk % 2) * 3]]; + + vfmadd(c_reg, a_reg, b_reg); + + if (!fetch_x && um == 0 && + (((idx == 0 || idx == 6) && (uk % 2 == 0 || is_f64 || ktail)) || + (idx == 3 && (uk % 2 == 1 || is_f64 || ktail)))) { + prefetch_a(ao + nelems_in_cache_line * fetchA_idx); + fetchA_idx++; + } + + if (um == 0 && idx == 1 && (uk % 2 == 0 || is_f64 || ktail)) { + prefetch_b(bo + nelems_in_cache_line * fetchB_idx); + fetchB_idx++; + } + + compute(ao, bo, fetchA_idx, fetchB_idx, b_reg); + } + } + + // load_a + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> load_a(const Scalar *ao) { + EIGEN_UNUSED_VARIABLE(ao); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> load_a(const Scalar *ao) { + if (um < um_vecs) { + auto &a_reg = zmm[a_regs[um + (uk % 2) * 3]]; + const Scalar *a_addr = ao + nelems * (1 + !ktail * !use_less_a_regs + uk) + nelems_in_cache_line * um - a_shift; + a_load(a_reg, a_addr); + + load_a(ao); + } + } + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(count > (pow + 1) / 2)> innerkernel_1pow(const Scalar *&aa, + const Scalar *const &ao, + const Scalar *const &bo, Scalar *&co2, + int &fetchA_idx, int &fetchB_idx) { + EIGEN_UNUSED_VARIABLE(aa); + EIGEN_UNUSED_VARIABLE(ao); + EIGEN_UNUSED_VARIABLE(bo); + EIGEN_UNUSED_VARIABLE(co2); + EIGEN_UNUSED_VARIABLE(fetchA_idx); + EIGEN_UNUSED_VARIABLE(fetchB_idx); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(count <= (pow + 1) / 2)> innerkernel_1pow(const Scalar *&aa, + const Scalar *const &ao, + const Scalar *const &bo, Scalar *&co2, + int &fetchA_idx, int &fetchB_idx) { + const int idx = (pow / 2) + count; + + if (count < (pow + 1) / 2) { + auto &b_reg = zmm[b_regs[idx % 2]]; + + if (fetch_x && uk == 3 && idx == 0) prefetch_x(aa); + if (fetch_x && uk == 3 && idx == 4) aa += 8; + + if (b_unroll >= pow) { + compute<0, um_vecs, idx, uk, fetch_x, ktail>(ao, bo, fetchA_idx, fetchB_idx, b_reg); + + const Scalar *b_addr = bo + b_unroll * uk + idx + 1 + (b_unroll > 1) * !use_less_b_regs - b_shift; + b_load(b_reg, b_addr); + } + + // Go to the next count. + innerkernel_1pow(aa, ao, bo, co2, fetchA_idx, + fetchB_idx); + + } else { + // Maybe prefetch C data after count-loop. + if (pow == 2 && c_fetch) { + if (uk % 3 == 0 && uk > 0) { + co2 += ldc; + } else { + prefetch_c(co2 + (uk % 3) * nelems_in_cache_line); + } + } + } + } + + template + EIGEN_ALWAYS_INLINE void innerkernel_1uk(const Scalar *&aa, const Scalar *const &ao, const Scalar *const &bo, + Scalar *&co2, int &fetchA_idx, int &fetchB_idx) { + const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); + + if (max_b_unroll >= 1) + innerkernel_1pow(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + if (max_b_unroll >= 2) + innerkernel_1pow(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + if (max_b_unroll >= 4) + innerkernel_1pow(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + if (max_b_unroll >= 8) + innerkernel_1pow(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + + // Load A after pow-loop. Skip this at the end to prevent running over the buffer + if (!no_a_preload) load_a<0, um_vecs, uk, a_unroll, ktail>(ao); + } + + /* Inner kernel loop structure. + * for (int uk = 0; uk < kfactor; uk++) { + * int idx = 0; + * + * for (pow = 1; pow < max_b_unroll << 1; pow <<= 1) { + * for (int count = 0; count < (pow + 1) / 2; count++) { + * auto &b_reg = zmm[b_regs[idx % 2]]; + * + * if (fetch_x && uk == 3 && idx == 0) prefetch_x(aa); + * if (fetch_x && uk == 3 && idx == 4) aa += 8; + * + * if (b_unroll >= pow) { + * compute<0, um_vecs, idx, uk, fetchx, ktail>(ao, bo, fetchA_idx, fetchB_idx, b_reg); + * + * const Scalar *b_addr = bo + b_unroll * uk + idx + 1 + (b_unroll > 1) - b_shift ; + * b_load(b_reg, b_addr); + * } + * idx++; + * } + * + * Maybe prefetch C data. + * if (pow == 2 && c_fetch) { + * if (uk % 3 == 0 && uk > 0) { + * co2 += ldc; + * } else { + * prefetch_c(co2 + (uk % 3) * nelems_in_cache_line); + * } + * } + * } + * + * Load A. + * load_a<0, um_vecs, uk, ktail, a_unroll>(ao); + * } + * + * Advance A/B pointers after uk-loop. + * ao += a_unroll * kfactor; + * bo += b_unroll * kfactor; + */ + + template + EIGEN_ALWAYS_INLINE void innerkernel(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co2) { + int fetchA_idx = 0; + int fetchB_idx = 0; + + const bool fetch_x = k_factor == max_k_factor; + const bool ktail = k_factor == 1; + + static_assert(k_factor <= 4 && k_factor > 0, "innerkernel maximum k_factor supported is 4"); + static_assert(no_a_preload == false || (no_a_preload == true && k_factor == 1), "skipping a preload only allowed when k unroll is 1"); + + if (k_factor > 0) + innerkernel_1uk<0, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>(aa, ao, bo, co2, fetchA_idx, + fetchB_idx); + if (k_factor > 1) + innerkernel_1uk<1, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>(aa, ao, bo, co2, fetchA_idx, + fetchB_idx); + if (k_factor > 2) + innerkernel_1uk<2, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>(aa, ao, bo, co2, fetchA_idx, + fetchB_idx); + if (k_factor > 3) + innerkernel_1uk<3, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>(aa, ao, bo, co2, fetchA_idx, + fetchB_idx); + + // Advance A/B pointers after uk-loop. + ao += a_unroll * k_factor; + bo += b_unroll * k_factor; + } + + template + EIGEN_ALWAYS_INLINE void kloop(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { + const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); + if (!use_less_a_regs && k > 1) + a_loads<0, 2, 0, um_vecs, a_unroll>(ao); + else + a_loads<0, 1, 0, um_vecs, a_unroll>(ao); + + b_load(zmm[b_regs[0]], bo - b_shift + 0); + if (!use_less_b_regs) b_load(zmm[b_regs[1]], bo - b_shift + 1); + +#ifndef SECOND_FETCH + prefetch_cs<0, max_b_unroll, 0, um_vecs, a_unroll, b_unroll>(co1, co2); +#endif // SECOND_FETCH + + // Unrolling k-loop by a factor of 4. + const int max_k_factor = 4; + Index kRem = k % max_k_factor; + Index k_ = k - kRem; + if (k_ >= max_k_factor) { + k_ -= max_k_factor; + kRem += max_k_factor; + } + Index loop_count = k_ / max_k_factor; + + if (loop_count > 0) { +#ifdef SECOND_FETCH + loop_count -= SECOND_FETCH; +#endif + while (loop_count > 0) { + innerkernel(aa, ao, bo, co2); + loop_count--; + } +#ifdef SECOND_FETCH + co2 = co1 + nelems_in_cache_line - 1; + + loop_count += b_unroll; + while (loop_count > 0) { + innerkernel(aa, ao, bo, co2); + loop_count--; + } + + loop_count += SECOND_FETCH - b_unroll; + while (loop_count > 0) { + innerkernel(aa, ao, bo, co2); + loop_count--; + } +#endif + } + + // k-loop remainder handling. + loop_count = kRem; + while (loop_count > 1) { + innerkernel(aa, ao, bo, co2); + loop_count--; + } + if (loop_count > 0) { + innerkernel(aa, ao, bo, co2); + } + + // Update C matrix. + c_update(co1, co2); + } + + template + EIGEN_ALWAYS_INLINE void nloop(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { + // Set A matrix pointer. + ao = a + a_off * a_unroll; + + // Set B matrix pointer if needed. + bo += b_unroll * b_off; + + kloop(aa, ao, bo, co1, co2); + + // Advance B matrix pointer if needed. + bo += b_unroll * (b_stride - k - b_off); + + // Advance prefetch A pointer. + aa += 16; + } + + template + EIGEN_ALWAYS_INLINE void mloop(const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { + // Set prefetch A pointers. + const Scalar *aa = a + a_unroll * a_stride; + + // Set C matrix pointers. + co1 = c; + if (a_unroll >= max_a_unroll) co2 = c + 2 * ldc; + if (is_unit_inc) + c += a_unroll; + else + c += a_unroll * inc; + + // Set B matrix pointer. + bo = b; + + // Main n-loop. + for (Index i = n / max_b_unroll; i > 0; i--) nloop(aa, ao, bo, co1, co2); + + // n-remainders. + if (n & 4 && max_b_unroll > 4) nloop(aa, ao, bo, co1, co2); +#if 0 + if (n & 2 && max_b_unroll > 2) nloop(aa, ao, bo, co1, co2); + if (n & 1 && max_b_unroll > 1) nloop(aa, ao, bo, co1, co2); +#else + // Copy kernels don't support tails of n = 2 for single/double precision. + // Loop over ones. + int n_rem = 2 * ((n & 2) != 0) + 1 * ((n & 1) != 0); + while (n_rem > 0) { + nloop(aa, ao, bo, co1, co2); + n_rem--; + } +#endif + + // Advance A matrix pointer. + a = ao + a_unroll * (a_stride - k - a_off); + } + + public: + // Compute kernel unrolling C matrix by max_a_unroll x max_b_unroll. + template + EIGEN_ALWAYS_INLINE void compute_kern() { + a -= -a_shift; + b -= -b_shift; + + const Scalar *ao = nullptr; + const Scalar *bo = nullptr; + Scalar *co1 = nullptr; + Scalar *co2 = nullptr; + + // Main m-loop. + for (; m >= max_a_unroll; m -= max_a_unroll) mloop(ao, bo, co1, co2); + + // m-remainders. + if (m & 32 && max_a_unroll > 32) mloop<32, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + if (m & 16 && max_a_unroll > 16) mloop<16, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + if (m & 8 && max_a_unroll > 8) mloop<8, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + if (m & 4 && max_a_unroll > 4) mloop<4, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + if (m & 2 && max_a_unroll > 2 && is_f64) mloop<2, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + if (m & 1 && max_a_unroll > 1 && is_f64) mloop<1, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + + // Copy kernels don't support tails of m = 2 for single precision. + // Loop over ones. + if (is_f32) { + int m_rem = 2 * ((m & 2) != 0) + 1 * ((m & 1) != 0); + while (m_rem > 0) { + mloop<1, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + m_rem--; + } + } + } + + gemm_class(Index m_, Index n_, Index k_, Index ldc_, Index inc_, const Scalar *alpha_, const Scalar *a_, + const Scalar *b_, Scalar *c_, bool is_alpha1_, bool is_beta0_, Index a_stride_, Index b_stride_, + Index a_off_, Index b_off_) + : m(m_), + n(n_), + k(k_), + ldc(ldc_), + inc(inc_), + alpha(alpha_), + a(a_), + b(b_), + c(c_), + is_alpha1(is_alpha1_), + is_beta0(is_beta0_), + a_stride(a_stride_), + b_stride(b_stride_), + a_off(a_off_), + b_off(b_off_) { + // Zero out all accumulation registers. + zmm[8] = pzero(zmm[8]); + zmm[9] = pzero(zmm[9]); + zmm[10] = pzero(zmm[10]); + zmm[11] = pzero(zmm[11]); + zmm[12] = pzero(zmm[12]); + zmm[13] = pzero(zmm[13]); + zmm[14] = pzero(zmm[14]); + zmm[15] = pzero(zmm[15]); + zmm[16] = pzero(zmm[16]); + zmm[17] = pzero(zmm[17]); + zmm[18] = pzero(zmm[18]); + zmm[19] = pzero(zmm[19]); + zmm[20] = pzero(zmm[20]); + zmm[21] = pzero(zmm[21]); + zmm[22] = pzero(zmm[22]); + zmm[23] = pzero(zmm[23]); + zmm[24] = pzero(zmm[24]); + zmm[25] = pzero(zmm[25]); + zmm[26] = pzero(zmm[26]); + zmm[27] = pzero(zmm[27]); + zmm[28] = pzero(zmm[28]); + zmm[29] = pzero(zmm[29]); + zmm[30] = pzero(zmm[30]); + zmm[31] = pzero(zmm[31]); + } +}; + +// Compute kernel with max unroll support of: +// Single precision: +// max_a_unroll: 48, 32, 16, 8, 4, 2, 1 +// max_b_unroll: 8, 4, 2, 1 +// Double precision: +// max_a_unroll: 24, 16, 8, 4, 2, 1 +// max_b_unroll: 8, 4, 2, 1 +template +EIGEN_DONT_INLINE void gemm_kern_avx512(Index m, Index n, Index k, Scalar *alpha, const Scalar *a, const Scalar *b, + Scalar *c, Index ldc, Index inc = 1, Index a_stride = -1, Index b_stride = -1, + Index a_off = 0, Index b_off = 0) { + if (a_stride == -1) a_stride = k; + if (b_stride == -1) b_stride = k; + + gemm_class g(m, n, k, ldc, inc, alpha, a, b, c, is_alpha1, is_beta0, a_stride, b_stride, a_off, + b_off); + g.template compute_kern(); +} + +// Template specializations of GEBP kernels with nr = 8. +#if EIGEN_USE_AVX512_GEMM_KERNELS +template +class gebp_traits + : public gebp_traits { + using Base = gebp_traits; + + public: + enum { nr = Base::Vectorizable ? 8 : 4 }; +}; + +template +class gebp_traits + : public gebp_traits { + using Base = gebp_traits; + + public: + enum { nr = Base::Vectorizable ? 8 : 4 }; +}; + +template +struct gemm_pack_rhs { + typedef typename packet_traits::type Packet; + typedef typename DataMapper::LinearMapper LinearMapper; + enum { PacketSize = packet_traits::size }; + EIGEN_DONT_INLINE void operator()(Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride = 0, + Index offset = 0); +}; + +template +EIGEN_DONT_INLINE void gemm_pack_rhs::operator()( + Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride, Index offset) { + constexpr int nr = 8; + EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK RHS COLMAJOR"); + EIGEN_UNUSED_VARIABLE(stride); + EIGEN_UNUSED_VARIABLE(offset); + eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride)); + conj_if::IsComplex && Conjugate> cj; + Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0; + Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0; + Index count = 0; + const Index peeled_k = (depth / PacketSize) * PacketSize; + if (nr >= 8) { + for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { + // skip what we have before + if (PanelMode) count += 8 * offset; + const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); + const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); + const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); + const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3); + const LinearMapper dm4 = rhs.getLinearMapper(0, j2 + 4); + const LinearMapper dm5 = rhs.getLinearMapper(0, j2 + 5); + const LinearMapper dm6 = rhs.getLinearMapper(0, j2 + 6); + const LinearMapper dm7 = rhs.getLinearMapper(0, j2 + 7); + Index k = 0; + if ((PacketSize % 8) == 0) // TODO enable vectorized transposition for PacketSize==4 + { + for (; k < peeled_k; k += PacketSize) { + PacketBlock kernel; + + kernel.packet[0] = dm0.template loadPacket(k); + kernel.packet[1] = dm1.template loadPacket(k); + kernel.packet[2] = dm2.template loadPacket(k); + kernel.packet[3] = dm3.template loadPacket(k); + kernel.packet[4] = dm4.template loadPacket(k); + kernel.packet[5] = dm5.template loadPacket(k); + kernel.packet[6] = dm6.template loadPacket(k); + kernel.packet[7] = dm7.template loadPacket(k); + + ptranspose(kernel); + + pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel.packet[0])); + pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel.packet[1 % PacketSize])); + pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel.packet[2 % PacketSize])); + pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel.packet[3 % PacketSize])); + pstoreu(blockB + count + 4 * PacketSize, cj.pconj(kernel.packet[4 % PacketSize])); + pstoreu(blockB + count + 5 * PacketSize, cj.pconj(kernel.packet[5 % PacketSize])); + pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel.packet[6 % PacketSize])); + pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel.packet[7 % PacketSize])); + count += 8 * PacketSize; + } + } + for (; k < depth; k++) { + blockB[count + 0] = cj(dm0(k)); + blockB[count + 1] = cj(dm1(k)); + blockB[count + 2] = cj(dm2(k)); + blockB[count + 3] = cj(dm3(k)); + blockB[count + 4] = cj(dm4(k)); + blockB[count + 5] = cj(dm5(k)); + blockB[count + 6] = cj(dm6(k)); + blockB[count + 7] = cj(dm7(k)); + count += 8; + } + // skip what we have after + if (PanelMode) count += 8 * (stride - offset - depth); + } + } + + if (nr >= 4) { + for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { + // skip what we have before + if (PanelMode) count += 4 * offset; + const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); + const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); + const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); + const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3); + + Index k = 0; + if ((PacketSize % 4) == 0) // TODO enable vectorized transposition for PacketSize==2 ?? + { + for (; k < peeled_k; k += PacketSize) { + PacketBlock kernel; + kernel.packet[0] = dm0.template loadPacket(k); + kernel.packet[1 % PacketSize] = dm1.template loadPacket(k); + kernel.packet[2 % PacketSize] = dm2.template loadPacket(k); + kernel.packet[3 % PacketSize] = dm3.template loadPacket(k); + ptranspose(kernel); + pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel.packet[0])); + pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel.packet[1 % PacketSize])); + pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel.packet[2 % PacketSize])); + pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel.packet[3 % PacketSize])); + count += 4 * PacketSize; + } + } + for (; k < depth; k++) { + blockB[count + 0] = cj(dm0(k)); + blockB[count + 1] = cj(dm1(k)); + blockB[count + 2] = cj(dm2(k)); + blockB[count + 3] = cj(dm3(k)); + count += 4; + } + // skip what we have after + if (PanelMode) count += 4 * (stride - offset - depth); + } + } + + // copy the remaining columns one at a time (nr==1) + for (Index j2 = packet_cols4; j2 < cols; ++j2) { + if (PanelMode) count += offset; + const LinearMapper dm0 = rhs.getLinearMapper(0, j2); + for (Index k = 0; k < depth; k++) { + blockB[count] = cj(dm0(k)); + count += 1; + } + if (PanelMode) count += (stride - offset - depth); + } +} + +template +struct gemm_pack_rhs { + typedef typename packet_traits::type Packet; + typedef typename unpacket_traits::half HalfPacket; + typedef typename unpacket_traits::half>::half QuarterPacket; + typedef typename DataMapper::LinearMapper LinearMapper; + enum { + PacketSize = packet_traits::size, + HalfPacketSize = unpacket_traits::size, + QuarterPacketSize = unpacket_traits::size + }; + EIGEN_DONT_INLINE void operator()(Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride = 0, + Index offset = 0) { + constexpr int nr = 8; + EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK RHS ROWMAJOR"); + EIGEN_UNUSED_VARIABLE(stride); + EIGEN_UNUSED_VARIABLE(offset); + eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride)); + const bool HasHalf = (int)HalfPacketSize < (int)PacketSize; + const bool HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize; + conj_if::IsComplex && Conjugate> cj; + Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0; + Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0; + Index count = 0; + + if (nr >= 8) { + for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { + // skip what we have before + if (PanelMode) count += 8 * offset; + for (Index k = 0; k < depth; k++) { + if (PacketSize == 8) { + // Packet A = ploadu(&rhs.data()[k*rhs.stride() + j2]); + Packet A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + } else if (HasHalf && HalfPacketSize == 8) { + HalfPacket A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + } else if (HasQuarter && QuarterPacketSize == 8) { + QuarterPacket A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + } else if (PacketSize == 4) { + // Packet A = ploadu(&rhs.data()[k*rhs.stride() + j2]); + // Packet B = ploadu(&rhs.data()[k*rhs.stride() + j2 + PacketSize]); + Packet A = rhs.template loadPacket(k, j2); + Packet B = rhs.template loadPacket(k, j2 + PacketSize); + pstoreu(blockB + count, cj.pconj(A)); + pstoreu(blockB + count + PacketSize, cj.pconj(B)); + } else { + // const Scalar* b0 = &rhs.data()[k*rhs.stride() + j2]; + const LinearMapper dm0 = rhs.getLinearMapper(k, j2); + blockB[count + 0] = cj(dm0(0)); + blockB[count + 1] = cj(dm0(1)); + blockB[count + 2] = cj(dm0(2)); + blockB[count + 3] = cj(dm0(3)); + blockB[count + 4] = cj(dm0(4)); + blockB[count + 5] = cj(dm0(5)); + blockB[count + 6] = cj(dm0(6)); + blockB[count + 7] = cj(dm0(7)); + } + count += 8; + } + // skip what we have after + if (PanelMode) count += 8 * (stride - offset - depth); + } + } + + if (nr >= 4) { + for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { + // skip what we have before + if (PanelMode) count += 4 * offset; + for (Index k = 0; k < depth; k++) { + if (PacketSize == 4) { + Packet A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + count += PacketSize; + } else if (HasHalf && HalfPacketSize == 4) { + HalfPacket A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + count += HalfPacketSize; + } else if (HasQuarter && QuarterPacketSize == 4) { + QuarterPacket A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + count += QuarterPacketSize; + } else { + const LinearMapper dm0 = rhs.getLinearMapper(k, j2); + blockB[count + 0] = cj(dm0(0)); + blockB[count + 1] = cj(dm0(1)); + blockB[count + 2] = cj(dm0(2)); + blockB[count + 3] = cj(dm0(3)); + count += 4; + } + } + // skip what we have after + if (PanelMode) count += 4 * (stride - offset - depth); + } + } + // copy the remaining columns one at a time (nr==1) + for (Index j2 = packet_cols4; j2 < cols; ++j2) { + if (PanelMode) count += offset; + for (Index k = 0; k < depth; k++) { + blockB[count] = cj(rhs(k, j2)); + count += 1; + } + if (PanelMode) count += stride - offset - depth; + } + } +}; + +template +struct gebp_kernel { + EIGEN_ALWAYS_INLINE + void operator()(const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, Index depth, + Index cols, Scalar alpha, Index strideA = -1, Index strideB = -1, Index offsetA = 0, + Index offsetB = 0); +}; + +template +EIGEN_ALWAYS_INLINE void gebp_kernel::operator()( + const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, Index depth, Index cols, + Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { + if (res.incr() == 1) { + if (alpha == 1) { + gemm_kern_avx512(rows, cols, depth, &alpha, blockA, blockB, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, + strideB, offsetA, offsetB); + } else { + gemm_kern_avx512(rows, cols, depth, &alpha, blockA, blockB, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, + strideB, offsetA, offsetB); + } + } else { + if (alpha == 1) { + gemm_kern_avx512(rows, cols, depth, &alpha, blockA, blockB, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, + strideB, offsetA, offsetB); + } else { + gemm_kern_avx512(rows, cols, depth, &alpha, blockA, blockB, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, + strideB, offsetA, offsetB); + } + } +} +#endif // EIGEN_USE_AVX512_GEMM_KERNELS + +} // namespace internal +} // namespace Eigen + +#undef SECOND_FETCH + +#endif // EIGEN_CORE_ARCH_AVX512_GEMM_KERNEL_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/MathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/MathFunctions.h index 6fd726d29cf..08e5fe8de95 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/MathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/MathFunctions.h @@ -10,123 +10,14 @@ #ifndef THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_ #define THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_ +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { - -// Disable the code for older versions of gcc that don't support many of the required avx512 instrinsics. -#if EIGEN_GNUC_AT_LEAST(5, 3) || EIGEN_COMP_CLANG || EIGEN_COMP_MSVC >= 1923 - -#define _EIGEN_DECLARE_CONST_Packet16f(NAME, X) \ - const Packet16f p16f_##NAME = pset1(X) - -#define _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(NAME, X) \ - const Packet16f p16f_##NAME = preinterpret(pset1(X)) - -#define _EIGEN_DECLARE_CONST_Packet8d(NAME, X) \ - const Packet8d p8d_##NAME = pset1(X) - -#define _EIGEN_DECLARE_CONST_Packet8d_FROM_INT64(NAME, X) \ - const Packet8d p8d_##NAME = _mm512_castsi512_pd(_mm512_set1_epi64(X)) - -#define _EIGEN_DECLARE_CONST_Packet16bf(NAME, X) \ - const Packet16bf p16bf_##NAME = pset1(X) - -#define _EIGEN_DECLARE_CONST_Packet16bf_FROM_INT(NAME, X) \ - const Packet16bf p16bf_##NAME = preinterpret(pset1(X)) - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f -plog(const Packet16f& _x) { - return plog_float(_x); -} - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d -plog(const Packet8d& _x) { - return plog_double(_x); -} - -F16_PACKET_FUNCTION(Packet16f, Packet16h, plog) -BF16_PACKET_FUNCTION(Packet16f, Packet16bf, plog) - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f -plog2(const Packet16f& _x) { - return plog2_float(_x); -} - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d -plog2(const Packet8d& _x) { - return plog2_double(_x); -} - -F16_PACKET_FUNCTION(Packet16f, Packet16h, plog2) -BF16_PACKET_FUNCTION(Packet16f, Packet16bf, plog2) - -// Exponential function. Works by writing "x = m*log(2) + r" where -// "m = floor(x/log(2)+1/2)" and "r" is the remainder. The result is then -// "exp(x) = 2^m*exp(r)" where exp(r) is in the range [-1,1). -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f -pexp(const Packet16f& _x) { - _EIGEN_DECLARE_CONST_Packet16f(1, 1.0f); - _EIGEN_DECLARE_CONST_Packet16f(half, 0.5f); - _EIGEN_DECLARE_CONST_Packet16f(127, 127.0f); - - _EIGEN_DECLARE_CONST_Packet16f(exp_hi, 88.3762626647950f); - _EIGEN_DECLARE_CONST_Packet16f(exp_lo, -88.3762626647949f); - - _EIGEN_DECLARE_CONST_Packet16f(cephes_LOG2EF, 1.44269504088896341f); - - _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p0, 1.9875691500E-4f); - _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p1, 1.3981999507E-3f); - _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p2, 8.3334519073E-3f); - _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p3, 4.1665795894E-2f); - _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p4, 1.6666665459E-1f); - _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p5, 5.0000001201E-1f); - - // Clamp x. - Packet16f x = pmax(pmin(_x, p16f_exp_hi), p16f_exp_lo); - - // Express exp(x) as exp(m*ln(2) + r), start by extracting - // m = floor(x/ln(2) + 0.5). - Packet16f m = _mm512_floor_ps(pmadd(x, p16f_cephes_LOG2EF, p16f_half)); - - // Get r = x - m*ln(2). Note that we can do this without losing more than one - // ulp precision due to the FMA instruction. - _EIGEN_DECLARE_CONST_Packet16f(nln2, -0.6931471805599453f); - Packet16f r = _mm512_fmadd_ps(m, p16f_nln2, x); - Packet16f r2 = pmul(r, r); - Packet16f r3 = pmul(r2, r); - - // Evaluate the polynomial approximant,improved by instruction-level parallelism. - Packet16f y, y1, y2; - y = pmadd(p16f_cephes_exp_p0, r, p16f_cephes_exp_p1); - y1 = pmadd(p16f_cephes_exp_p3, r, p16f_cephes_exp_p4); - y2 = padd(r, p16f_1); - y = pmadd(y, r, p16f_cephes_exp_p2); - y1 = pmadd(y1, r, p16f_cephes_exp_p5); - y = pmadd(y, r3, y1); - y = pmadd(y, r2, y2); - - // Build emm0 = 2^m. - Packet16i emm0 = _mm512_cvttps_epi32(padd(m, p16f_127)); - emm0 = _mm512_slli_epi32(emm0, 23); - - // Return 2^m * exp(r). - return pmax(pmul(y, _mm512_castsi512_ps(emm0)), _x); -} - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d -pexp(const Packet8d& _x) { - return pexp_double(_x); -} - -F16_PACKET_FUNCTION(Packet16f, Packet16h, pexp) -BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pexp) +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet16f) +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE(Packet8d) template <> EIGEN_STRONG_INLINE Packet16h pfrexp(const Packet16h& a, Packet16h& exponent) { @@ -154,49 +45,21 @@ EIGEN_STRONG_INLINE Packet16bf pldexp(const Packet16bf& a, const Packet16bf& exp return F32ToBf16(pldexp(Bf16ToF32(a), Bf16ToF32(exponent))); } -// Functions for sqrt. -// The EIGEN_FAST_MATH version uses the _mm_rsqrt_ps approximation and one step -// of Newton's method, at a cost of 1-2 bits of precision as opposed to the -// exact solution. The main advantage of this approach is not just speed, but -// also the fact that it can be inlined and pipelined with other computations, -// further reducing its effective latency. #if EIGEN_FAST_MATH template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16f psqrt(const Packet16f& _x) { - Packet16f neg_half = pmul(_x, pset1(-.5f)); - __mmask16 denormal_mask = _mm512_kand( - _mm512_cmp_ps_mask(_x, pset1((std::numeric_limits::min)()), - _CMP_LT_OQ), - _mm512_cmp_ps_mask(_x, _mm512_setzero_ps(), _CMP_GE_OQ)); - - Packet16f x = _mm512_rsqrt14_ps(_x); - - // Do a single step of Newton's iteration. - x = pmul(x, pmadd(neg_half, pmul(x, x), pset1(1.5f))); - - // Flush results for denormals to zero. - return _mm512_mask_blend_ps(denormal_mask, pmul(_x,x), _mm512_setzero_ps()); + return generic_sqrt_newton_step::run(_x, _mm512_rsqrt14_ps(_x)); } template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8d psqrt(const Packet8d& _x) { - Packet8d neg_half = pmul(_x, pset1(-.5)); - __mmask16 denormal_mask = _mm512_kand( - _mm512_cmp_pd_mask(_x, pset1((std::numeric_limits::min)()), - _CMP_LT_OQ), - _mm512_cmp_pd_mask(_x, _mm512_setzero_pd(), _CMP_GE_OQ)); - - Packet8d x = _mm512_rsqrt14_pd(_x); - - // Do a single step of Newton's iteration. - x = pmul(x, pmadd(neg_half, pmul(x, x), pset1(1.5))); - - // Do a second step of Newton's iteration. - x = pmul(x, pmadd(neg_half, pmul(x, x), pset1(1.5))); - - return _mm512_mask_blend_pd(denormal_mask, pmul(_x,x), _mm512_setzero_pd()); +#ifdef EIGEN_VECTORIZE_AVX512ER + return generic_sqrt_newton_step::run(_x, _mm512_rsqrt28_pd(_x)); +#else + return generic_sqrt_newton_step::run(_x, _mm512_rsqrt14_pd(_x)); +#endif } #else template <> @@ -210,12 +73,8 @@ EIGEN_STRONG_INLINE Packet8d psqrt(const Packet8d& x) { } #endif -F16_PACKET_FUNCTION(Packet16f, Packet16h, psqrt) -BF16_PACKET_FUNCTION(Packet16f, Packet16bf, psqrt) - // prsqrt for float. #if defined(EIGEN_VECTORIZE_AVX512ER) - template <> EIGEN_STRONG_INLINE Packet16f prsqrt(const Packet16f& x) { return _mm512_rsqrt28_ps(x); @@ -223,138 +82,57 @@ EIGEN_STRONG_INLINE Packet16f prsqrt(const Packet16f& x) { #elif EIGEN_FAST_MATH template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16f prsqrt(const Packet16f& _x) { - _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(inf, 0x7f800000); - _EIGEN_DECLARE_CONST_Packet16f(one_point_five, 1.5f); - _EIGEN_DECLARE_CONST_Packet16f(minus_half, -0.5f); - - Packet16f neg_half = pmul(_x, p16f_minus_half); - - // Identity infinite, negative and denormal arguments. - __mmask16 inf_mask = _mm512_cmp_ps_mask(_x, p16f_inf, _CMP_EQ_OQ); - __mmask16 not_pos_mask = _mm512_cmp_ps_mask(_x, _mm512_setzero_ps(), _CMP_LE_OQ); - __mmask16 not_finite_pos_mask = not_pos_mask | inf_mask; - - // Compute an approximate result using the rsqrt intrinsic, forcing +inf - // for denormals for consistency with AVX and SSE implementations. - Packet16f y_approx = _mm512_rsqrt14_ps(_x); - - // Do a single step of Newton-Raphson iteration to improve the approximation. - // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n). - // It is essential to evaluate the inner term like this because forming - // y_n^2 may over- or underflow. - Packet16f y_newton = pmul(y_approx, pmadd(y_approx, pmul(neg_half, y_approx), p16f_one_point_five)); - - // Select the result of the Newton-Raphson step for positive finite arguments. - // For other arguments, choose the output of the intrinsic. This will - // return rsqrt(+inf) = 0, rsqrt(x) = NaN if x < 0, and rsqrt(0) = +inf. - return _mm512_mask_blend_ps(not_finite_pos_mask, y_newton, y_approx); -} -#else - -template <> -EIGEN_STRONG_INLINE Packet16f prsqrt(const Packet16f& x) { - _EIGEN_DECLARE_CONST_Packet16f(one, 1.0f); - return _mm512_div_ps(p16f_one, _mm512_sqrt_ps(x)); + return generic_rsqrt_newton_step::run(_x, _mm512_rsqrt14_ps(_x)); } #endif -F16_PACKET_FUNCTION(Packet16f, Packet16h, prsqrt) -BF16_PACKET_FUNCTION(Packet16f, Packet16bf, prsqrt) // prsqrt for double. #if EIGEN_FAST_MATH template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8d prsqrt(const Packet8d& _x) { - _EIGEN_DECLARE_CONST_Packet8d(one_point_five, 1.5); - _EIGEN_DECLARE_CONST_Packet8d(minus_half, -0.5); - _EIGEN_DECLARE_CONST_Packet8d_FROM_INT64(inf, 0x7ff0000000000000LL); - - Packet8d neg_half = pmul(_x, p8d_minus_half); - - // Identity infinite, negative and denormal arguments. - __mmask8 inf_mask = _mm512_cmp_pd_mask(_x, p8d_inf, _CMP_EQ_OQ); - __mmask8 not_pos_mask = _mm512_cmp_pd_mask(_x, _mm512_setzero_pd(), _CMP_LE_OQ); - __mmask8 not_finite_pos_mask = not_pos_mask | inf_mask; + #ifdef EIGEN_VECTORIZE_AVX512ER + return generic_rsqrt_newton_step::run(_x, _mm512_rsqrt28_pd(_x)); + #else + return generic_rsqrt_newton_step::run(_x, _mm512_rsqrt14_pd(_x)); + #endif +} - // Compute an approximate result using the rsqrt intrinsic, forcing +inf - // for denormals for consistency with AVX and SSE implementations. -#if defined(EIGEN_VECTORIZE_AVX512ER) - Packet8d y_approx = _mm512_rsqrt28_pd(_x); +template<> EIGEN_STRONG_INLINE Packet16f preciprocal(const Packet16f& a) { +#ifdef EIGEN_VECTORIZE_AVX512ER + return _mm512_rcp28_ps(a); #else - Packet8d y_approx = _mm512_rsqrt14_pd(_x); -#endif - // Do one or two steps of Newton-Raphson's to improve the approximation, depending on the - // starting accuracy (either 2^-14 or 2^-28, depending on whether AVX512ER is available). - // The Newton-Raphson algorithm has quadratic convergence and roughly doubles the number - // of correct digits for each step. - // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n). - // It is essential to evaluate the inner term like this because forming - // y_n^2 may over- or underflow. - Packet8d y_newton = pmul(y_approx, pmadd(neg_half, pmul(y_approx, y_approx), p8d_one_point_five)); -#if !defined(EIGEN_VECTORIZE_AVX512ER) - y_newton = pmul(y_newton, pmadd(y_newton, pmul(neg_half, y_newton), p8d_one_point_five)); + return generic_reciprocal_newton_step::run(a, _mm512_rcp14_ps(a)); #endif - // Select the result of the Newton-Raphson step for positive finite arguments. - // For other arguments, choose the output of the intrinsic. This will - // return rsqrt(+inf) = 0, rsqrt(x) = NaN if x < 0, and rsqrt(0) = +inf. - return _mm512_mask_blend_pd(not_finite_pos_mask, y_newton, y_approx); -} -#else -template <> -EIGEN_STRONG_INLINE Packet8d prsqrt(const Packet8d& x) { - _EIGEN_DECLARE_CONST_Packet8d(one, 1.0f); - return _mm512_div_pd(p8d_one, _mm512_sqrt_pd(x)); } #endif -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet16f plog1p(const Packet16f& _x) { - return generic_plog1p(_x); -} - -F16_PACKET_FUNCTION(Packet16f, Packet16h, plog1p) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pcos) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pexp) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pexpm1) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, plog) BF16_PACKET_FUNCTION(Packet16f, Packet16bf, plog1p) - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet16f pexpm1(const Packet16f& _x) { - return generic_expm1(_x); -} - +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, plog2) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, preciprocal) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, prsqrt) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, psin) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, psqrt) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, ptanh) +F16_PACKET_FUNCTION(Packet16f, Packet16h, pcos) +F16_PACKET_FUNCTION(Packet16f, Packet16h, pexp) F16_PACKET_FUNCTION(Packet16f, Packet16h, pexpm1) -BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pexpm1) - -#endif - - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f -psin(const Packet16f& _x) { - return psin_float(_x); -} - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f -pcos(const Packet16f& _x) { - return pcos_float(_x); -} - -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f -ptanh(const Packet16f& _x) { - return internal::generic_fast_tanh_float(_x); -} - +F16_PACKET_FUNCTION(Packet16f, Packet16h, plog) +F16_PACKET_FUNCTION(Packet16f, Packet16h, plog1p) +F16_PACKET_FUNCTION(Packet16f, Packet16h, plog2) +F16_PACKET_FUNCTION(Packet16f, Packet16h, preciprocal) +F16_PACKET_FUNCTION(Packet16f, Packet16h, prsqrt) F16_PACKET_FUNCTION(Packet16f, Packet16h, psin) -F16_PACKET_FUNCTION(Packet16f, Packet16h, pcos) +F16_PACKET_FUNCTION(Packet16f, Packet16h, psqrt) F16_PACKET_FUNCTION(Packet16f, Packet16h, ptanh) -BF16_PACKET_FUNCTION(Packet16f, Packet16bf, psin) -BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pcos) -BF16_PACKET_FUNCTION(Packet16f, Packet16bf, ptanh) - } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/PacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/PacketMath.h index 34d49ab660c..c6566a4cfee 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/PacketMath.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/PacketMath.h @@ -10,6 +10,9 @@ #ifndef EIGEN_PACKET_MATH_AVX512_H #define EIGEN_PACKET_MATH_AVX512_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -31,7 +34,9 @@ namespace internal { typedef __m512 Packet16f; typedef __m512i Packet16i; typedef __m512d Packet8d; +#ifndef EIGEN_VECTORIZE_AVX512FP16 typedef eigen_packet_wrapper<__m256i, 1> Packet16h; +#endif typedef eigen_packet_wrapper<__m256i, 2> Packet16bf; template <> @@ -47,6 +52,7 @@ struct is_arithmetic<__m512d> { enum { value = true }; }; +#ifndef EIGEN_VECTORIZE_AVX512FP16 template<> struct is_arithmetic { enum { value = true }; }; template <> @@ -58,7 +64,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 16, - HasHalfPacket = 1, HasCmp = 1, HasAdd = 1, @@ -72,25 +77,26 @@ struct packet_traits : default_packet_traits { HasMax = 1, HasConj = 1, HasSetLinear = 0, + HasSqrt = 1, + HasRsqrt = 1, HasLog = 1, HasLog1p = 1, - HasExpm1 = 1, HasExp = 1, - HasSqrt = 1, - HasRsqrt = 1, + HasExpm1 = 1, + HasBessel = 1, + HasNdtri = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, - HasBlend = 0, + HasBlend = 0, HasRound = 1, HasFloor = 1, HasCeil = 1, - HasRint = 1, - HasBessel = 1, - HasNdtri = 1 + HasRint = 1 }; }; +#endif template<> struct packet_traits : default_packet_traits { @@ -100,27 +106,29 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 16, - HasHalfPacket = 1, HasAbs = 1, - HasMin = 1, - HasMax = 1, - HasConj = 1, - HasBlend = 0, + HasMin = 1, + HasMax = 1, + HasConj = 1, + HasBlend = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, -#if EIGEN_GNUC_AT_LEAST(5, 3) || (!EIGEN_COMP_GNUC_STRICT) + HasACos = 1, + HasASin = 1, + HasATan = 1, + HasATanh = 1, + HasSqrt = 1, + HasRsqrt = 1, HasLog = 1, HasLog1p = 1, HasExpm1 = 1, HasNdtri = 1, HasBessel = 1, HasExp = 1, - HasSqrt = EIGEN_FAST_MATH, - HasRsqrt = EIGEN_FAST_MATH, + HasReciprocal = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, -#endif HasCmp = 1, HasDiv = 1, HasRound = 1, @@ -137,13 +145,12 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasHalfPacket = 1, -#if EIGEN_GNUC_AT_LEAST(5, 3) || (!EIGEN_COMP_GNUC_STRICT) + HasBlend = 1, + HasSqrt = 1, + HasRsqrt = 1, HasLog = 1, HasExp = 1, - HasSqrt = EIGEN_FAST_MATH, - HasRsqrt = EIGEN_FAST_MATH, -#endif + HasATan = 1, HasCmp = 1, HasDiv = 1, HasRound = 1, @@ -153,17 +160,19 @@ template<> struct packet_traits : default_packet_traits }; }; -/* TODO Implement AVX512 for integers -template<> struct packet_traits : default_packet_traits +template<> struct packet_traits : default_packet_traits { typedef Packet16i type; + typedef Packet8i half; enum { Vectorizable = 1, AlignedOnScalar = 1, - size=8 + HasBlend = 0, + HasCmp = 1, + HasDiv = 1, + size=16 }; }; -*/ template <> struct unpacket_traits { @@ -171,27 +180,30 @@ struct unpacket_traits { typedef Packet8f half; typedef Packet16i integer_packet; typedef uint16_t mask_t; - enum { size = 16, alignment=Aligned64, vectorizable=true, masked_load_available=true, masked_store_available=true }; + enum { size = 16, alignment=Aligned64, vectorizable=true, masked_load_available=true, masked_store_available=true, masked_fpops_available=true }; }; template <> struct unpacket_traits { typedef double type; typedef Packet4d half; - enum { size = 8, alignment=Aligned64, vectorizable=true, masked_load_available=false, masked_store_available=false }; + typedef uint8_t mask_t; + enum { size = 8, alignment=Aligned64, vectorizable=true, masked_load_available=true, masked_store_available=true, masked_fpops_available=true }; }; template <> struct unpacket_traits { typedef int type; typedef Packet8i half; - enum { size = 16, alignment=Aligned64, vectorizable=false, masked_load_available=false, masked_store_available=false }; + enum { size = 16, alignment=Aligned64, vectorizable=true, masked_load_available=false, masked_store_available=false }; }; +#ifndef EIGEN_VECTORIZE_AVX512FP16 template<> struct unpacket_traits { typedef Eigen::half type; typedef Packet8h half; enum {size=16, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; +#endif template <> EIGEN_STRONG_INLINE Packet16f pset1(const float& from) { @@ -235,11 +247,25 @@ template<> EIGEN_STRONG_INLINE Packet8d peven_mask(const Packet8d& /*a*/) { template <> EIGEN_STRONG_INLINE Packet16f pload1(const float* from) { +#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0) + // Inline asm here helps reduce some register spilling in TRSM kernels. + // See note in unrolls::gemm::microKernel in TrsmKernel.h + Packet16f ret; + __asm__ ("vbroadcastss %[mem], %[dst]" : [dst] "=v" (ret) : [mem] "m" (*from)); + return ret; +#else return _mm512_broadcastss_ps(_mm_load_ps1(from)); +#endif } template <> EIGEN_STRONG_INLINE Packet8d pload1(const double* from) { +#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0) + Packet8d ret; + __asm__ ("vbroadcastsd %[mem], %[dst]" : [dst] "=v" (ret) : [mem] "m" (*from)); + return ret; +#else return _mm512_set1_pd(*from); +#endif } template <> @@ -254,6 +280,12 @@ EIGEN_STRONG_INLINE Packet8d plset(const double& a) { return _mm512_add_pd(_mm512_set1_pd(a), _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0)); } +template <> +EIGEN_STRONG_INLINE Packet16i plset(const int& a) { + return _mm512_add_epi32( + _mm512_set1_epi32(a), + _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); +} template <> EIGEN_STRONG_INLINE Packet16f padd(const Packet16f& a, @@ -271,6 +303,21 @@ EIGEN_STRONG_INLINE Packet16i padd(const Packet16i& a, return _mm512_add_epi32(a, b); } +template <> +EIGEN_STRONG_INLINE Packet16f padd(const Packet16f& a, + const Packet16f& b, + uint16_t umask) { + __mmask16 mask = static_cast<__mmask16>(umask); + return _mm512_maskz_add_ps(mask, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8d padd(const Packet8d& a, + const Packet8d& b, + uint8_t umask) { + __mmask8 mask = static_cast<__mmask8>(umask); + return _mm512_maskz_add_pd(mask, a, b); +} + template <> EIGEN_STRONG_INLINE Packet16f psub(const Packet16f& a, const Packet16f& b) { @@ -289,11 +336,24 @@ EIGEN_STRONG_INLINE Packet16i psub(const Packet16i& a, template <> EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) { - return _mm512_sub_ps(_mm512_set1_ps(0.0), a); + // NOTE: MSVC seems to struggle with _mm512_set1_epi32, leading to random results. + // The intel docs give it a relatively high latency as well, so we're probably + // better off with using _mm512_set_epi32 directly anyways. + const __m512i mask = _mm512_set_epi32(0x80000000,0x80000000,0x80000000,0x80000000, + 0x80000000,0x80000000,0x80000000,0x80000000, + 0x80000000,0x80000000,0x80000000,0x80000000, + 0x80000000,0x80000000,0x80000000,0x80000000); + return _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(a), mask)); } template <> EIGEN_STRONG_INLINE Packet8d pnegate(const Packet8d& a) { - return _mm512_sub_pd(_mm512_set1_pd(0.0), a); + const __m512i mask = _mm512_set_epi64(0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL); + return _mm512_castsi512_pd(_mm512_xor_epi64(_mm512_castpd_si512(a), mask)); +} +template <> +EIGEN_STRONG_INLINE Packet16i pnegate(const Packet16i& a) { + return _mm512_sub_epi32(_mm512_setzero_si512(), a); } template <> @@ -330,12 +390,21 @@ EIGEN_STRONG_INLINE Packet16f pdiv(const Packet16f& a, const Packet16f& b) { return _mm512_div_ps(a, b); } + template <> EIGEN_STRONG_INLINE Packet8d pdiv(const Packet8d& a, const Packet8d& b) { return _mm512_div_pd(a, b); } +template <> +EIGEN_STRONG_INLINE Packet16i pdiv(const Packet16i& a, + const Packet16i& b) { + Packet8i q_lo = pdiv(_mm512_extracti64x4_epi64(a, 0), _mm512_extracti64x4_epi64(b,0)); + Packet8i q_hi = pdiv(_mm512_extracti64x4_epi64(a, 1), _mm512_extracti64x4_epi64(b, 1)); + return _mm512_inserti64x4(_mm512_castsi256_si512(q_lo), q_hi, 1); +} + #ifdef EIGEN_VECTORIZE_FMA template <> EIGEN_STRONG_INLINE Packet16f pmadd(const Packet16f& a, const Packet16f& b, @@ -347,17 +416,57 @@ EIGEN_STRONG_INLINE Packet8d pmadd(const Packet8d& a, const Packet8d& b, const Packet8d& c) { return _mm512_fmadd_pd(a, b, c); } + +template <> +EIGEN_STRONG_INLINE Packet16f pmsub(const Packet16f& a, const Packet16f& b, + const Packet16f& c) { + return _mm512_fmsub_ps(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet8d pmsub(const Packet8d& a, const Packet8d& b, + const Packet8d& c) { + return _mm512_fmsub_pd(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pnmadd(const Packet16f& a, const Packet16f& b, + const Packet16f& c) { + return _mm512_fnmadd_ps(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet8d pnmadd(const Packet8d& a, const Packet8d& b, + const Packet8d& c) { + return _mm512_fnmadd_pd(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pnmsub(const Packet16f& a, const Packet16f& b, + const Packet16f& c) { + return _mm512_fnmsub_ps(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet8d pnmsub(const Packet8d& a, const Packet8d& b, + const Packet8d& c) { + return _mm512_fnmsub_pd(a, b, c); +} #endif template <> EIGEN_DEVICE_FUNC inline Packet16f pselect(const Packet16f& mask, const Packet16f& a, const Packet16f& b) { - __mmask16 mask16 = _mm512_cmp_epi32_mask( - _mm512_castps_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ); + __mmask16 mask16 = _mm512_cmpeq_epi32_mask(_mm512_castps_si512(mask), _mm512_setzero_epi32()); return _mm512_mask_blend_ps(mask16, a, b); } +template <> +EIGEN_DEVICE_FUNC inline Packet16i pselect(const Packet16i& mask, + const Packet16i& a, + const Packet16i& b) { + __mmask16 mask16 = _mm512_cmpeq_epi32_mask(mask, _mm512_setzero_epi32()); + return _mm512_mask_blend_epi32(mask16, a, b); +} + template <> EIGEN_DEVICE_FUNC inline Packet8d pselect(const Packet8d& mask, const Packet8d& a, @@ -379,6 +488,11 @@ EIGEN_STRONG_INLINE Packet8d pmin(const Packet8d& a, // Arguments are reversed to match NaN propagation behavior of std::min. return _mm512_min_pd(b, a); } +template <> +EIGEN_STRONG_INLINE Packet16i pmin(const Packet16i& a, + const Packet16i& b) { + return _mm512_min_epi32(b, a); +} template <> EIGEN_STRONG_INLINE Packet16f pmax(const Packet16f& a, @@ -392,6 +506,11 @@ EIGEN_STRONG_INLINE Packet8d pmax(const Packet8d& a, // Arguments are reversed to match NaN propagation behavior of std::max. return _mm512_max_pd(b, a); } +template <> +EIGEN_STRONG_INLINE Packet16i pmax(const Packet16i& a, + const Packet16i& b) { + return _mm512_max_epi32(b, a); +} // Add specializations for min/max with prescribed NaN progation. template<> @@ -432,6 +551,7 @@ EIGEN_STRONG_INLINE Packet8d pmax(const Packet8d& a, con template EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) { return _mm512_extractf32x8_ps(x,I_); } template EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) { return _mm512_extractf64x2_pd(x,I_); } EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) { return _mm512_insertf32x8(_mm512_castps256_ps512(a),b,1); } +EIGEN_STRONG_INLINE Packet16i cat256i(Packet8i a, Packet8i b) { return _mm512_inserti32x8(_mm512_castsi256_si512(a), b, 1); } #else // AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512 template EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) { @@ -447,6 +567,9 @@ EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) { return _mm512_castsi512_ps(_mm512_inserti64x4(_mm512_castsi256_si512(_mm256_castps_si256(a)), _mm256_castps_si256(b),1)); } +EIGEN_STRONG_INLINE Packet16i cat256i(Packet8i a, Packet8i b) { + return _mm512_inserti64x4(_mm512_castsi256_si512(a), b, 1); +} #endif // Helper function for bit packing snippet of low precision comparison. @@ -468,59 +591,72 @@ EIGEN_STRONG_INLINE __m256i Pack32To16(Packet16f rf) { return _mm256_insertf128_si256(_mm256_castsi128_si256(result_lo), result_hi, 1); } +template <> +EIGEN_STRONG_INLINE Packet16f pisnan(const Packet16f& a) { + __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_UNORD_Q); + return _mm512_castsi512_ps(_mm512_maskz_set1_epi32(mask, 0xffffffffu)); +} + template <> EIGEN_STRONG_INLINE Packet16f pcmp_eq(const Packet16f& a, const Packet16f& b) { __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ); return _mm512_castsi512_ps( - _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); + _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu)); } template<> EIGEN_STRONG_INLINE Packet16f pcmp_le(const Packet16f& a, const Packet16f& b) { __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LE_OQ); return _mm512_castsi512_ps( - _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); + _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu)); } template<> EIGEN_STRONG_INLINE Packet16f pcmp_lt(const Packet16f& a, const Packet16f& b) { __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LT_OQ); return _mm512_castsi512_ps( - _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); + _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu)); } template<> EIGEN_STRONG_INLINE Packet16f pcmp_lt_or_nan(const Packet16f& a, const Packet16f& b) { __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_NGE_UQ); return _mm512_castsi512_ps( - _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); + _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu)); } template<> EIGEN_STRONG_INLINE Packet16i pcmp_eq(const Packet16i& a, const Packet16i& b) { - __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _CMP_EQ_OQ); - return _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu); + __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_EQ); + return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu); +} +template<> EIGEN_STRONG_INLINE Packet16i pcmp_le(const Packet16i& a, const Packet16i& b) { + __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LE); + return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu); +} +template<> EIGEN_STRONG_INLINE Packet16i pcmp_lt(const Packet16i& a, const Packet16i& b) { + __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LT); + return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu); } - template <> EIGEN_STRONG_INLINE Packet8d pcmp_eq(const Packet8d& a, const Packet8d& b) { __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ); return _mm512_castsi512_pd( - _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu)); + _mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu)); } template <> EIGEN_STRONG_INLINE Packet8d pcmp_le(const Packet8d& a, const Packet8d& b) { __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LE_OQ); return _mm512_castsi512_pd( - _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu)); + _mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu)); } template <> EIGEN_STRONG_INLINE Packet8d pcmp_lt(const Packet8d& a, const Packet8d& b) { __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LT_OQ); return _mm512_castsi512_pd( - _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu)); + _mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu)); } template <> EIGEN_STRONG_INLINE Packet8d pcmp_lt_or_nan(const Packet8d& a, const Packet8d& b) { __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_NGE_UQ); return _mm512_castsi512_pd( - _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu)); + _mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu)); } template<> EIGEN_STRONG_INLINE Packet16f print(const Packet16f& a) { return _mm512_roundscale_ps(a, _MM_FROUND_CUR_DIRECTION); } @@ -686,7 +822,7 @@ EIGEN_STRONG_INLINE Packet8d pload(const double* from) { template <> EIGEN_STRONG_INLINE Packet16i pload(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512( - reinterpret_cast(from)); + reinterpret_cast(from)); } template <> @@ -708,6 +844,11 @@ EIGEN_STRONG_INLINE Packet16f ploadu(const float* from, uint16_t umas __mmask16 mask = static_cast<__mmask16>(umask); EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_ps(mask, from); } +template <> +EIGEN_STRONG_INLINE Packet8d ploadu(const double* from, uint8_t umask) { + __mmask8 mask = static_cast<__mmask8>(umask); + EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_pd(mask, from); +} // Loads 8 floats from memory a returns the packet // {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7} @@ -746,6 +887,16 @@ EIGEN_STRONG_INLINE Packet8d ploaddup(const double* from) { } #endif +// Loads 8 integers from memory and returns the packet +// {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7} +template <> +EIGEN_STRONG_INLINE Packet16i ploaddup(const int* from) { + __m256i low_half = _mm256_loadu_si256(reinterpret_cast(from)); + __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half)); + __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0)); + return _mm512_castps_si512(pairs); +} + // Loads 4 floats from memory a returns the packet // {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3} template <> @@ -766,6 +917,15 @@ EIGEN_STRONG_INLINE Packet8d ploadquad(const double* from) { return _mm512_insertf64x4(tmp, lane1, 1); } +// Loads 4 integers from memory and returns the packet +// {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3} +template <> +EIGEN_STRONG_INLINE Packet16i ploadquad(const int* from) { + Packet16i tmp = _mm512_castsi128_si512(ploadu(from)); + const Packet16i scatter_mask = _mm512_set_epi32(3,3,3,3, 2,2,2,2, 1,1,1,1, 0,0,0,0); + return _mm512_permutexvar_epi32(scatter_mask, tmp); +} + template <> EIGEN_STRONG_INLINE void pstore(float* to, const Packet16f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from); @@ -798,6 +958,40 @@ EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet16f& from, uint16 __mmask16 mask = static_cast<__mmask16>(umask); EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_ps(to, mask, from); } +template <> +EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet8d& from, uint8_t umask) { + __mmask8 mask = static_cast<__mmask8>(umask); + EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_pd(to, mask, from); +} + +template +EIGEN_DEVICE_FUNC inline Packet pgather(const Packet& src, const Scalar* from, + Index stride, typename unpacket_traits::mask_t umask); +template <> +EIGEN_DEVICE_FUNC inline Packet16f pgather(const Packet16f& src, + const float* from, + Index stride, + uint16_t umask) { + Packet16i stride_vector = _mm512_set1_epi32(convert_index(stride)); + Packet16i stride_multiplier = + _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); + __mmask16 mask = static_cast<__mmask16>(umask); + + return _mm512_mask_i32gather_ps(src, mask, indices, from, 4); +} +template <> +EIGEN_DEVICE_FUNC inline Packet8d pgather(const Packet8d& src, + const double* from, + Index stride, + uint8_t umask) { + Packet8i stride_vector = _mm256_set1_epi32(convert_index(stride)); + Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); + __mmask8 mask = static_cast<__mmask8>(umask); + + return _mm512_mask_i32gather_pd(src, mask, indices, from, 8); +} template <> EIGEN_DEVICE_FUNC inline Packet16f pgather(const float* from, @@ -818,6 +1012,42 @@ EIGEN_DEVICE_FUNC inline Packet8d pgather(const double* from, return _mm512_i32gather_pd(indices, from, 8); } +template <> +EIGEN_DEVICE_FUNC inline Packet16i pgather(const int* from, + Index stride) { + Packet16i stride_vector = _mm512_set1_epi32(convert_index(stride)); + Packet16i stride_multiplier = + _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); + return _mm512_i32gather_epi32(indices, from, 4); +} + +template +EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from, + Index stride, typename unpacket_traits::mask_t umask); +template <> +EIGEN_DEVICE_FUNC inline void pscatter(float* to, + const Packet16f& from, + Index stride, + uint16_t umask) { + Packet16i stride_vector = _mm512_set1_epi32(convert_index(stride)); + Packet16i stride_multiplier = + _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); + __mmask16 mask = static_cast<__mmask16>(umask); + _mm512_mask_i32scatter_ps(to, mask, indices, from, 4); +} +template <> +EIGEN_DEVICE_FUNC inline void pscatter(double* to, + const Packet8d& from, + Index stride, + uint8_t umask) { + Packet8i stride_vector = _mm256_set1_epi32(convert_index(stride)); + Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); + __mmask8 mask = static_cast<__mmask8>(umask); + _mm512_mask_i32scatter_pd(to, mask, indices, from, 8); +} template <> EIGEN_DEVICE_FUNC inline void pscatter(float* to, @@ -838,6 +1068,16 @@ EIGEN_DEVICE_FUNC inline void pscatter(double* to, Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); _mm512_i32scatter_pd(to, indices, from, 8); } +template <> +EIGEN_DEVICE_FUNC inline void pscatter(int* to, + const Packet16i& from, + Index stride) { + Packet16i stride_vector = _mm512_set1_epi32(convert_index(stride)); + Packet16i stride_multiplier = + _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); + _mm512_i32scatter_epi32(to, indices, from, 4); +} template <> EIGEN_STRONG_INLINE void pstore1(float* to, const float& a) { @@ -882,6 +1122,11 @@ template<> EIGEN_STRONG_INLINE Packet8d preverse(const Packet8d& a) return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a); } +template<> EIGEN_STRONG_INLINE Packet16i preverse(const Packet16i& a) +{ + return _mm512_permutexvar_epi32(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a); +} + template<> EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a) { // _mm512_abs_ps intrinsic not found, so hack around it @@ -893,6 +1138,15 @@ EIGEN_STRONG_INLINE Packet8d pabs(const Packet8d& a) { return _mm512_castsi512_pd(_mm512_and_si512(_mm512_castpd_si512(a), _mm512_set1_epi64(0x7fffffffffffffff))); } +template<> EIGEN_STRONG_INLINE Packet16i pabs(const Packet16i& a) +{ + return _mm512_abs_epi32(a); +} + +template<> EIGEN_STRONG_INLINE Packet16h psignbit(const Packet16h& a) { return _mm256_srai_epi16(a, 15); } +template<> EIGEN_STRONG_INLINE Packet16bf psignbit(const Packet16bf& a) { return _mm256_srai_epi16(a, 15); } +template<> EIGEN_STRONG_INLINE Packet16f psignbit(const Packet16f& a) { return _mm512_castsi512_ps(_mm512_srai_epi32(_mm512_castps_si512(a), 31)); } +template<> EIGEN_STRONG_INLINE Packet8d psignbit(const Packet8d& a) { return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), 63)); } template<> EIGEN_STRONG_INLINE Packet16f pfrexp(const Packet16f& a, Packet16f& exponent){ @@ -901,7 +1155,7 @@ EIGEN_STRONG_INLINE Packet16f pfrexp(const Packet16f& a, Packet16f& e // Extract exponent without existence of Packet8l. template<> -EIGEN_STRONG_INLINE +EIGEN_STRONG_INLINE Packet8d pfrexp_generic_get_biased_exponent(const Packet8d& a) { const Packet8d cst_exp_mask = pset1frombits(static_cast(0x7ff0000000000000ull)); #ifdef EIGEN_VECTORIZE_AVX512DQ @@ -924,11 +1178,11 @@ template<> EIGEN_STRONG_INLINE Packet8d pldexp(const Packet8d& a, cons // Clamp exponent to [-2099, 2099] const Packet8d max_exponent = pset1(2099.0); const Packet8i e = _mm512_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); - + // Split 2^e into four factors and multiply. const Packet8i bias = pset1(1023); Packet8i b = parithmetic_shift_right<2>(e); // floor(e/4) - + // 2^b const Packet8i permute_idx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); Packet8i hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx); @@ -936,7 +1190,7 @@ template<> EIGEN_STRONG_INLINE Packet8d pldexp(const Packet8d& a, cons hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52); Packet8d c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1)); Packet8d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b) - + // 2^(e - 3b) b = psub(psub(psub(e, b), b), b); // e - 3b hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx); @@ -952,6 +1206,11 @@ template<> EIGEN_STRONG_INLINE Packet8d pldexp(const Packet8d& a, cons #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \ __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0); \ __m256 OUTPUT##_1 = _mm512_extractf32x8_ps(INPUT, 1) + +// AVX512F does not define _mm512_extracti32x8_epi32 to extract _m256i from _m512i +#define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \ + __m256i OUTPUT##_0 = _mm512_extracti32x8_epi32(INPUT, 0); \ + __m256i OUTPUT##_1 = _mm512_extracti32x8_epi32(INPUT, 1) #else #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \ __m256 OUTPUT##_0 = _mm256_insertf128_ps( \ @@ -959,12 +1218,23 @@ template<> EIGEN_STRONG_INLINE Packet8d pldexp(const Packet8d& a, cons _mm512_extractf32x4_ps(INPUT, 1), 1); \ __m256 OUTPUT##_1 = _mm256_insertf128_ps( \ _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \ - _mm512_extractf32x4_ps(INPUT, 3), 1); + _mm512_extractf32x4_ps(INPUT, 3), 1) + +#define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \ + __m256i OUTPUT##_0 = _mm256_insertf128_si256( \ + _mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 0)), \ + _mm512_extracti32x4_epi32(INPUT, 1), 1); \ + __m256i OUTPUT##_1 = _mm256_insertf128_si256( \ + _mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 2)), \ + _mm512_extracti32x4_epi32(INPUT, 3), 1) #endif #ifdef EIGEN_VECTORIZE_AVX512DQ #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \ OUTPUT = _mm512_insertf32x8(_mm512_castps256_ps512(INPUTA), INPUTB, 1); + +#define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \ + OUTPUT = _mm512_inserti32x8(_mm512_castsi256_si512(INPUTA), INPUTB, 1); #else #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \ OUTPUT = _mm512_undefined_ps(); \ @@ -972,6 +1242,13 @@ template<> EIGEN_STRONG_INLINE Packet8d pldexp(const Packet8d& a, cons OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \ OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \ OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3); + +#define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \ + OUTPUT = _mm512_undefined_epi32(); \ + OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 0), 0); \ + OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 1), 1); \ + OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 0), 2); \ + OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 1), 3); #endif template <> @@ -1000,6 +1277,24 @@ EIGEN_STRONG_INLINE double predux(const Packet8d& a) { __m256d tmp0 = _mm256_hadd_pd(sum, _mm256_permute2f128_pd(sum, sum, 1)); return _mm_cvtsd_f64(_mm256_castpd256_pd128(_mm256_hadd_pd(tmp0, tmp0))); } +template <> +EIGEN_STRONG_INLINE int predux(const Packet16i& a) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + __m256i lane0 = _mm512_extracti32x8_epi32(a, 0); + __m256i lane1 = _mm512_extracti32x8_epi32(a, 1); + Packet8i x = _mm256_add_epi32(lane0, lane1); + return predux(x); +#else + __m128i lane0 = _mm512_extracti32x4_epi32(a, 0); + __m128i lane1 = _mm512_extracti32x4_epi32(a, 1); + __m128i lane2 = _mm512_extracti32x4_epi32(a, 2); + __m128i lane3 = _mm512_extracti32x4_epi32(a, 3); + __m128i sum = _mm_add_epi32(_mm_add_epi32(lane0, lane1), _mm_add_epi32(lane2, lane3)); + sum = _mm_hadd_epi32(sum, sum); + sum = _mm_hadd_epi32(sum, _mm_castps_si128(_mm_permute_ps(_mm_castsi128_ps(sum), 1))); + return _mm_cvtsi128_si32(sum); +#endif +} template <> EIGEN_STRONG_INLINE Packet8f predux_half_dowto4(const Packet16f& a) { @@ -1023,6 +1318,22 @@ EIGEN_STRONG_INLINE Packet4d predux_half_dowto4(const Packet8d& a) { __m256d lane1 = _mm512_extractf64x4_pd(a, 1); return _mm256_add_pd(lane0, lane1); } +template <> +EIGEN_STRONG_INLINE Packet8i predux_half_dowto4(const Packet16i& a) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + __m256i lane0 = _mm512_extracti32x8_epi32(a, 0); + __m256i lane1 = _mm512_extracti32x8_epi32(a, 1); + return _mm256_add_epi32(lane0, lane1); +#else + __m128i lane0 = _mm512_extracti32x4_epi32(a, 0); + __m128i lane1 = _mm512_extracti32x4_epi32(a, 1); + __m128i lane2 = _mm512_extracti32x4_epi32(a, 2); + __m128i lane3 = _mm512_extracti32x4_epi32(a, 3); + __m128i sum0 = _mm_add_epi32(lane0, lane2); + __m128i sum1 = _mm_add_epi32(lane1, lane3); + return _mm256_inserti128_si256(_mm256_castsi128_si256(sum0), sum1, 1); +#endif +} template <> EIGEN_STRONG_INLINE float predux_mul(const Packet16f& a) { @@ -1099,7 +1410,11 @@ template<> EIGEN_STRONG_INLINE bool predux_any(const Packet16f& x) return !_mm512_kortestz(tmp,tmp); } - +template<> EIGEN_STRONG_INLINE bool predux_any(const Packet16i& x) +{ + __mmask16 tmp = _mm512_test_epi32_mask(x,x); + return !_mm512_kortestz(tmp,tmp); +} #define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \ EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]); @@ -1219,6 +1534,44 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], \ INPUT[2 * INDEX + STRIDE]); +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { + __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0],kernel.packet[1]); + __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0],kernel.packet[1]); + __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2],kernel.packet[3]); + __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2],kernel.packet[3]); + __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4],kernel.packet[5]); + __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4],kernel.packet[5]); + __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6],kernel.packet[7]); + __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6],kernel.packet[7]); + + kernel.packet[0] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T0),_mm512_castps_pd(T2))); + kernel.packet[1] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T0),_mm512_castps_pd(T2))); + kernel.packet[2] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T1),_mm512_castps_pd(T3))); + kernel.packet[3] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T1),_mm512_castps_pd(T3))); + kernel.packet[4] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T4),_mm512_castps_pd(T6))); + kernel.packet[5] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T4),_mm512_castps_pd(T6))); + kernel.packet[6] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T5),_mm512_castps_pd(T7))); + kernel.packet[7] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T5),_mm512_castps_pd(T7))); + + T0 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0x44); + T1 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0xee); + T2 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0x44); + T3 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0xee); + T4 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0x44); + T5 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0xee); + T6 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0x44); + T7 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0xee); + + kernel.packet[0] = _mm512_shuffle_f32x4(T0, T2, 0x88); + kernel.packet[2] = _mm512_shuffle_f32x4(T0, T2, 0xdd); + kernel.packet[1] = _mm512_shuffle_f32x4(T4, T6, 0x88); + kernel.packet[3] = _mm512_shuffle_f32x4(T4, T6, 0xdd); + kernel.packet[4] = _mm512_shuffle_f32x4(T1, T3, 0x88); + kernel.packet[6] = _mm512_shuffle_f32x4(T1, T3, 0xdd); + kernel.packet[5] = _mm512_shuffle_f32x4(T5, T7, 0x88); + kernel.packet[7] = _mm512_shuffle_f32x4(T5, T7, 0xdd); +} + EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]); __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]); @@ -1295,69 +1648,222 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { - __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]); - __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]); - __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]); - __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]); - __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]); - __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]); - __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]); - __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]); - - PacketBlock tmp; + __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0],kernel.packet[1]); + __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0],kernel.packet[1]); + __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2],kernel.packet[3]); + __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2],kernel.packet[3]); + __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4],kernel.packet[5]); + __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4],kernel.packet[5]); + __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6],kernel.packet[7]); + __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6],kernel.packet[7]); + + kernel.packet[0] = _mm512_permutex_pd(T2, 0x4E); + kernel.packet[0] = _mm512_mask_blend_pd(0xCC, T0, kernel.packet[0]); + kernel.packet[2] = _mm512_permutex_pd(T0, 0x4E); + kernel.packet[2] = _mm512_mask_blend_pd(0xCC, kernel.packet[2], T2); + kernel.packet[1] = _mm512_permutex_pd(T3, 0x4E); + kernel.packet[1] = _mm512_mask_blend_pd(0xCC, T1, kernel.packet[1]); + kernel.packet[3] = _mm512_permutex_pd(T1, 0x4E); + kernel.packet[3] = _mm512_mask_blend_pd(0xCC, kernel.packet[3], T3); + kernel.packet[4] = _mm512_permutex_pd(T6, 0x4E); + kernel.packet[4] = _mm512_mask_blend_pd(0xCC, T4, kernel.packet[4]); + kernel.packet[6] = _mm512_permutex_pd(T4, 0x4E); + kernel.packet[6] = _mm512_mask_blend_pd(0xCC, kernel.packet[6], T6); + kernel.packet[5] = _mm512_permutex_pd(T7, 0x4E); + kernel.packet[5] = _mm512_mask_blend_pd(0xCC, T5, kernel.packet[5]); + kernel.packet[7] = _mm512_permutex_pd(T5, 0x4E); + kernel.packet[7] = _mm512_mask_blend_pd(0xCC, kernel.packet[7], T7); + + T0 = _mm512_shuffle_f64x2(kernel.packet[4], kernel.packet[4], 0x4E); + T0 = _mm512_mask_blend_pd(0xF0, kernel.packet[0], T0); + T4 = _mm512_shuffle_f64x2(kernel.packet[0], kernel.packet[0], 0x4E); + T4 = _mm512_mask_blend_pd(0xF0, T4, kernel.packet[4]); + T1 = _mm512_shuffle_f64x2(kernel.packet[5], kernel.packet[5], 0x4E); + T1 = _mm512_mask_blend_pd(0xF0, kernel.packet[1], T1); + T5 = _mm512_shuffle_f64x2(kernel.packet[1], kernel.packet[1], 0x4E); + T5 = _mm512_mask_blend_pd(0xF0, T5, kernel.packet[5]); + T2 = _mm512_shuffle_f64x2(kernel.packet[6], kernel.packet[6], 0x4E); + T2 = _mm512_mask_blend_pd(0xF0, kernel.packet[2], T2); + T6 = _mm512_shuffle_f64x2(kernel.packet[2], kernel.packet[2], 0x4E); + T6 = _mm512_mask_blend_pd(0xF0, T6, kernel.packet[6]); + T3 = _mm512_shuffle_f64x2(kernel.packet[7], kernel.packet[7], 0x4E); + T3 = _mm512_mask_blend_pd(0xF0, kernel.packet[3], T3); + T7 = _mm512_shuffle_f64x2(kernel.packet[3], kernel.packet[3], 0x4E); + T7 = _mm512_mask_blend_pd(0xF0, T7, kernel.packet[7]); + + kernel.packet[0] = T0; kernel.packet[1] = T1; + kernel.packet[2] = T2; kernel.packet[3] = T3; + kernel.packet[4] = T4; kernel.packet[5] = T5; + kernel.packet[6] = T6; kernel.packet[7] = T7; +} + +#define PACK_OUTPUT_I32(OUTPUT, INPUT, INDEX, STRIDE) \ + EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]); + +#define PACK_OUTPUT_I32_2(OUTPUT, INPUT, INDEX, STRIDE) \ + EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[2 * INDEX], \ + INPUT[2 * INDEX + STRIDE]); - tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), - _mm512_extractf64x4_pd(T2, 0), 0x20); - tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), - _mm512_extractf64x4_pd(T3, 0), 0x20); - tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), - _mm512_extractf64x4_pd(T2, 0), 0x31); - tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), - _mm512_extractf64x4_pd(T3, 0), 0x31); +#define SHUFFLE_EPI32(A, B, M) \ + _mm512_castps_si512(_mm512_shuffle_ps(_mm512_castsi512_ps(A), _mm512_castsi512_ps(B), M)) + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { + __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]); + __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]); + __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]); + __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]); + __m512i T4 = _mm512_unpacklo_epi32(kernel.packet[4], kernel.packet[5]); + __m512i T5 = _mm512_unpackhi_epi32(kernel.packet[4], kernel.packet[5]); + __m512i T6 = _mm512_unpacklo_epi32(kernel.packet[6], kernel.packet[7]); + __m512i T7 = _mm512_unpackhi_epi32(kernel.packet[6], kernel.packet[7]); + __m512i T8 = _mm512_unpacklo_epi32(kernel.packet[8], kernel.packet[9]); + __m512i T9 = _mm512_unpackhi_epi32(kernel.packet[8], kernel.packet[9]); + __m512i T10 = _mm512_unpacklo_epi32(kernel.packet[10], kernel.packet[11]); + __m512i T11 = _mm512_unpackhi_epi32(kernel.packet[10], kernel.packet[11]); + __m512i T12 = _mm512_unpacklo_epi32(kernel.packet[12], kernel.packet[13]); + __m512i T13 = _mm512_unpackhi_epi32(kernel.packet[12], kernel.packet[13]); + __m512i T14 = _mm512_unpacklo_epi32(kernel.packet[14], kernel.packet[15]); + __m512i T15 = _mm512_unpackhi_epi32(kernel.packet[14], kernel.packet[15]); + __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0)); + __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2)); + __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0)); + __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2)); + __m512i S4 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(1, 0, 1, 0)); + __m512i S5 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(3, 2, 3, 2)); + __m512i S6 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(1, 0, 1, 0)); + __m512i S7 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(3, 2, 3, 2)); + __m512i S8 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(1, 0, 1, 0)); + __m512i S9 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(3, 2, 3, 2)); + __m512i S10 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(1, 0, 1, 0)); + __m512i S11 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(3, 2, 3, 2)); + __m512i S12 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(1, 0, 1, 0)); + __m512i S13 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(3, 2, 3, 2)); + __m512i S14 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(1, 0, 1, 0)); + __m512i S15 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(3, 2, 3, 2)); + + EIGEN_EXTRACT_8i_FROM_16i(S0, S0); + EIGEN_EXTRACT_8i_FROM_16i(S1, S1); + EIGEN_EXTRACT_8i_FROM_16i(S2, S2); + EIGEN_EXTRACT_8i_FROM_16i(S3, S3); + EIGEN_EXTRACT_8i_FROM_16i(S4, S4); + EIGEN_EXTRACT_8i_FROM_16i(S5, S5); + EIGEN_EXTRACT_8i_FROM_16i(S6, S6); + EIGEN_EXTRACT_8i_FROM_16i(S7, S7); + EIGEN_EXTRACT_8i_FROM_16i(S8, S8); + EIGEN_EXTRACT_8i_FROM_16i(S9, S9); + EIGEN_EXTRACT_8i_FROM_16i(S10, S10); + EIGEN_EXTRACT_8i_FROM_16i(S11, S11); + EIGEN_EXTRACT_8i_FROM_16i(S12, S12); + EIGEN_EXTRACT_8i_FROM_16i(S13, S13); + EIGEN_EXTRACT_8i_FROM_16i(S14, S14); + EIGEN_EXTRACT_8i_FROM_16i(S15, S15); + + PacketBlock tmp; + + tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S4_0, 0x20); + tmp.packet[1] = _mm256_permute2f128_si256(S1_0, S5_0, 0x20); + tmp.packet[2] = _mm256_permute2f128_si256(S2_0, S6_0, 0x20); + tmp.packet[3] = _mm256_permute2f128_si256(S3_0, S7_0, 0x20); + tmp.packet[4] = _mm256_permute2f128_si256(S0_0, S4_0, 0x31); + tmp.packet[5] = _mm256_permute2f128_si256(S1_0, S5_0, 0x31); + tmp.packet[6] = _mm256_permute2f128_si256(S2_0, S6_0, 0x31); + tmp.packet[7] = _mm256_permute2f128_si256(S3_0, S7_0, 0x31); + + tmp.packet[8] = _mm256_permute2f128_si256(S0_1, S4_1, 0x20); + tmp.packet[9] = _mm256_permute2f128_si256(S1_1, S5_1, 0x20); + tmp.packet[10] = _mm256_permute2f128_si256(S2_1, S6_1, 0x20); + tmp.packet[11] = _mm256_permute2f128_si256(S3_1, S7_1, 0x20); + tmp.packet[12] = _mm256_permute2f128_si256(S0_1, S4_1, 0x31); + tmp.packet[13] = _mm256_permute2f128_si256(S1_1, S5_1, 0x31); + tmp.packet[14] = _mm256_permute2f128_si256(S2_1, S6_1, 0x31); + tmp.packet[15] = _mm256_permute2f128_si256(S3_1, S7_1, 0x31); - tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), - _mm512_extractf64x4_pd(T2, 1), 0x20); - tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), - _mm512_extractf64x4_pd(T3, 1), 0x20); - tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), - _mm512_extractf64x4_pd(T2, 1), 0x31); - tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), - _mm512_extractf64x4_pd(T3, 1), 0x31); + // Second set of _m256 outputs + tmp.packet[16] = _mm256_permute2f128_si256(S8_0, S12_0, 0x20); + tmp.packet[17] = _mm256_permute2f128_si256(S9_0, S13_0, 0x20); + tmp.packet[18] = _mm256_permute2f128_si256(S10_0, S14_0, 0x20); + tmp.packet[19] = _mm256_permute2f128_si256(S11_0, S15_0, 0x20); + tmp.packet[20] = _mm256_permute2f128_si256(S8_0, S12_0, 0x31); + tmp.packet[21] = _mm256_permute2f128_si256(S9_0, S13_0, 0x31); + tmp.packet[22] = _mm256_permute2f128_si256(S10_0, S14_0, 0x31); + tmp.packet[23] = _mm256_permute2f128_si256(S11_0, S15_0, 0x31); + + tmp.packet[24] = _mm256_permute2f128_si256(S8_1, S12_1, 0x20); + tmp.packet[25] = _mm256_permute2f128_si256(S9_1, S13_1, 0x20); + tmp.packet[26] = _mm256_permute2f128_si256(S10_1, S14_1, 0x20); + tmp.packet[27] = _mm256_permute2f128_si256(S11_1, S15_1, 0x20); + tmp.packet[28] = _mm256_permute2f128_si256(S8_1, S12_1, 0x31); + tmp.packet[29] = _mm256_permute2f128_si256(S9_1, S13_1, 0x31); + tmp.packet[30] = _mm256_permute2f128_si256(S10_1, S14_1, 0x31); + tmp.packet[31] = _mm256_permute2f128_si256(S11_1, S15_1, 0x31); - tmp.packet[8] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0), - _mm512_extractf64x4_pd(T6, 0), 0x20); - tmp.packet[9] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0), - _mm512_extractf64x4_pd(T7, 0), 0x20); - tmp.packet[10] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0), - _mm512_extractf64x4_pd(T6, 0), 0x31); - tmp.packet[11] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0), - _mm512_extractf64x4_pd(T7, 0), 0x31); - - tmp.packet[12] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1), - _mm512_extractf64x4_pd(T6, 1), 0x20); - tmp.packet[13] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1), - _mm512_extractf64x4_pd(T7, 1), 0x20); - tmp.packet[14] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1), - _mm512_extractf64x4_pd(T6, 1), 0x31); - tmp.packet[15] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1), - _mm512_extractf64x4_pd(T7, 1), 0x31); - - PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 0, 8); - PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 1, 8); - PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 2, 8); - PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 3, 8); - - PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 4, 8); - PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 5, 8); - PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 6, 8); - PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 7, 8); -} -template <> -EIGEN_STRONG_INLINE Packet16f pblend(const Selector<16>& /*ifPacket*/, - const Packet16f& /*thenPacket*/, - const Packet16f& /*elsePacket*/) { - assert(false && "To be implemented"); - return Packet16f(); + // Pack them into the output + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 0, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 1, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 2, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 3, 16); + + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 4, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 5, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 6, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 7, 16); + + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 8, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 9, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 10, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 11, 16); + + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 12, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 13, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 14, 16); + PACK_OUTPUT_I32(kernel.packet, tmp.packet, 15, 16); +} + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { + __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]); + __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]); + __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]); + __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]); + + __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0)); + __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2)); + __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0)); + __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2)); + + EIGEN_EXTRACT_8i_FROM_16i(S0, S0); + EIGEN_EXTRACT_8i_FROM_16i(S1, S1); + EIGEN_EXTRACT_8i_FROM_16i(S2, S2); + EIGEN_EXTRACT_8i_FROM_16i(S3, S3); + + PacketBlock tmp; + + tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S1_0, 0x20); + tmp.packet[1] = _mm256_permute2f128_si256(S2_0, S3_0, 0x20); + tmp.packet[2] = _mm256_permute2f128_si256(S0_0, S1_0, 0x31); + tmp.packet[3] = _mm256_permute2f128_si256(S2_0, S3_0, 0x31); + + tmp.packet[4] = _mm256_permute2f128_si256(S0_1, S1_1, 0x20); + tmp.packet[5] = _mm256_permute2f128_si256(S2_1, S3_1, 0x20); + tmp.packet[6] = _mm256_permute2f128_si256(S0_1, S1_1, 0x31); + tmp.packet[7] = _mm256_permute2f128_si256(S2_1, S3_1, 0x31); + + PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 0, 1); + PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 1, 1); + PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 2, 1); + PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 3, 1); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pblend(const Selector<16>& ifPacket, + const Packet16f& thenPacket, + const Packet16f& elsePacket) { + __mmask16 m = (ifPacket.select[0]) | (ifPacket.select[1] << 1) | (ifPacket.select[2] << 2) | + (ifPacket.select[3] << 3) | (ifPacket.select[4] << 4) | (ifPacket.select[5] << 5) | + (ifPacket.select[6] << 6) | (ifPacket.select[7] << 7) | (ifPacket.select[8] << 8) | + (ifPacket.select[9] << 9) | (ifPacket.select[10] << 10) | (ifPacket.select[11] << 11) | + (ifPacket.select[12] << 12) | (ifPacket.select[13] << 13) | (ifPacket.select[14] << 14) | + (ifPacket.select[15] << 15); + return _mm512_mask_blend_ps(m, elsePacket, thenPacket); } template <> EIGEN_STRONG_INLINE Packet8d pblend(const Selector<8>& ifPacket, @@ -1426,64 +1932,15 @@ ploadquad(const Eigen::half* from) { } EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) { -#ifdef EIGEN_HAS_FP16_C return _mm512_cvtph_ps(a); -#else - EIGEN_ALIGN64 half aux[16]; - pstore(aux, a); - float f0(aux[0]); - float f1(aux[1]); - float f2(aux[2]); - float f3(aux[3]); - float f4(aux[4]); - float f5(aux[5]); - float f6(aux[6]); - float f7(aux[7]); - float f8(aux[8]); - float f9(aux[9]); - float fa(aux[10]); - float fb(aux[11]); - float fc(aux[12]); - float fd(aux[13]); - float fe(aux[14]); - float ff(aux[15]); - - return _mm512_set_ps( - ff, fe, fd, fc, fb, fa, f9, f8, f7, f6, f5, f4, f3, f2, f1, f0); -#endif } EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) { -#ifdef EIGEN_HAS_FP16_C return _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC); -#else - EIGEN_ALIGN64 float aux[16]; - pstore(aux, a); - half h0(aux[0]); - half h1(aux[1]); - half h2(aux[2]); - half h3(aux[3]); - half h4(aux[4]); - half h5(aux[5]); - half h6(aux[6]); - half h7(aux[7]); - half h8(aux[8]); - half h9(aux[9]); - half ha(aux[10]); - half hb(aux[11]); - half hc(aux[12]); - half hd(aux[13]); - half he(aux[14]); - half hf(aux[15]); - - return _mm256_set_epi16( - hf.x, he.x, hd.x, hc.x, hb.x, ha.x, h9.x, h8.x, - h7.x, h6.x, h5.x, h4.x, h3.x, h2.x, h1.x, h0.x); -#endif } template<> EIGEN_STRONG_INLINE Packet16h ptrue(const Packet16h& a) { - return ptrue(Packet8i(a)); + return Packet16h(ptrue(Packet8i(a))); } template <> @@ -1512,16 +1969,16 @@ EIGEN_STRONG_INLINE Packet16h plset(const half& a) { template<> EIGEN_STRONG_INLINE Packet16h por(const Packet16h& a,const Packet16h& b) { // in some cases Packet8i is a wrapper around __m256i, so we need to // cast to Packet8i to call the correct overload. - return por(Packet8i(a),Packet8i(b)); + return Packet16h(por(Packet8i(a),Packet8i(b))); } template<> EIGEN_STRONG_INLINE Packet16h pxor(const Packet16h& a,const Packet16h& b) { - return pxor(Packet8i(a),Packet8i(b)); + return Packet16h(pxor(Packet8i(a),Packet8i(b))); } template<> EIGEN_STRONG_INLINE Packet16h pand(const Packet16h& a,const Packet16h& b) { - return pand(Packet8i(a),Packet8i(b)); + return Packet16h(pand(Packet8i(a),Packet8i(b))); } template<> EIGEN_STRONG_INLINE Packet16h pandnot(const Packet16h& a,const Packet16h& b) { - return pandnot(Packet8i(a),Packet8i(b)); + return Packet16h(pandnot(Packet8i(a),Packet8i(b))); } template<> EIGEN_STRONG_INLINE Packet16h pselect(const Packet16h& mask, const Packet16h& a, const Packet16h& b) { @@ -1569,6 +2026,7 @@ template<> EIGEN_STRONG_INLINE Packet16h pnegate(const Packet16h& a) { return _mm256_xor_si256(a, sign_mask); } +#ifndef EIGEN_VECTORIZE_AVX512FP16 template<> EIGEN_STRONG_INLINE Packet16h padd(const Packet16h& a, const Packet16h& b) { Packet16f af = half2float(a); Packet16f bf = half2float(b); @@ -1602,6 +2060,8 @@ template<> EIGEN_STRONG_INLINE half predux(const Packet16h& from) { return half(predux(from_float)); } +#endif + template <> EIGEN_STRONG_INLINE Packet8h predux_half_dowto4(const Packet16h& a) { Packet8h lane0 = _mm256_extractf128_si256(a, 0); @@ -1847,25 +2307,22 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 16, - HasHalfPacket = 1, HasBlend = 0, HasInsert = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, -#if EIGEN_GNUC_AT_LEAST(5, 3) || (!EIGEN_COMP_GNUC_STRICT) + HasSqrt = 1, + HasRsqrt = 1, #ifdef EIGEN_VECTORIZE_AVX512DQ HasLog = 1, // Currently fails test with bad accuracy. HasLog1p = 1, HasExpm1 = 1, HasNdtri = 1, - HasBessel = 1, + HasBessel = 1, #endif HasExp = 1, - HasSqrt = EIGEN_FAST_MATH, - HasRsqrt = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, -#endif HasCmp = 1, HasDiv = 1 }; @@ -1915,7 +2372,6 @@ EIGEN_STRONG_INLINE void pstoreu(bfloat16* to, template<> EIGEN_STRONG_INLINE Packet16bf ploaddup(const bfloat16* from) { - Packet16bf r; unsigned short a = from[0].value; unsigned short b = from[1].value; unsigned short c = from[2].value; @@ -1929,7 +2385,6 @@ ploaddup(const bfloat16* from) { template<> EIGEN_STRONG_INLINE Packet16bf ploadquad(const bfloat16* from) { - Packet16bf r; unsigned short a = from[0].value; unsigned short b = from[1].value; unsigned short c = from[2].value; @@ -1945,9 +2400,9 @@ EIGEN_STRONG_INLINE Packet16f Bf16ToF32(const Packet16bf& a) { EIGEN_STRONG_INLINE Packet16bf F32ToBf16(const Packet16f& a) { Packet16bf r; -#if defined(EIGEN_VECTORIZE_AVX512BF16) && EIGEN_GNUC_AT_LEAST(10, 1) +#if defined(EIGEN_VECTORIZE_AVX512BF16) && EIGEN_GNUC_STRICT_AT_LEAST(10,1,0) // Since GCC 10.1 supports avx512bf16 and C style explicit cast - // (C++ static_cast is not supported yet), do converion via intrinsic + // (C++ static_cast is not supported yet), do conversion via intrinsic // and register path for performance. r = (__m256i)(_mm512_cvtneps_pbh(a)); @@ -1978,28 +2433,28 @@ EIGEN_STRONG_INLINE Packet16bf F32ToBf16(const Packet16f& a) { template <> EIGEN_STRONG_INLINE Packet16bf ptrue(const Packet16bf& a) { - return ptrue(a); + return Packet16bf(ptrue(Packet8i(a))); } template <> EIGEN_STRONG_INLINE Packet16bf por(const Packet16bf& a, const Packet16bf& b) { - return por(a, b); + return Packet16bf(por(Packet8i(a), Packet8i(b))); } template <> EIGEN_STRONG_INLINE Packet16bf pxor(const Packet16bf& a, const Packet16bf& b) { - return pxor(a, b); + return Packet16bf(pxor(Packet8i(a), Packet8i(b))); } template <> EIGEN_STRONG_INLINE Packet16bf pand(const Packet16bf& a, const Packet16bf& b) { - return pand(a, b); + return Packet16bf(pand(Packet8i(a), Packet8i(b))); } template <> EIGEN_STRONG_INLINE Packet16bf pandnot(const Packet16bf& a, const Packet16bf& b) { - return pandnot(a, b); + return Packet16bf(pandnot(Packet8i(a), Packet8i(b))); } template <> diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/PacketMathFP16.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/PacketMathFP16.h new file mode 100644 index 00000000000..faa385310dc --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/PacketMathFP16.h @@ -0,0 +1,870 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_PACKET_MATH_FP16_AVX512_H +#define EIGEN_PACKET_MATH_FP16_AVX512_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +typedef __m512h Packet32h; +typedef eigen_packet_wrapper<__m256i, 1> Packet16h; +typedef eigen_packet_wrapper<__m128i, 2> Packet8h; + +template <> +struct is_arithmetic { + enum { value = true }; +}; + +template <> +struct packet_traits : default_packet_traits { + typedef Packet32h type; + typedef Packet16h half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 32, + + HasCmp = 1, + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasAbs = 1, + HasAbs2 = 0, + HasMin = 1, + HasMax = 1, + HasConj = 1, + HasSetLinear = 0, + HasLog = 1, + HasLog1p = 1, + HasExp = 1, + HasExpm1 = 1, + HasSqrt = 1, + HasRsqrt = 1, + // These ones should be implemented in future + HasBessel = 0, + HasNdtri = 0, + HasSin = EIGEN_FAST_MATH, + HasCos = EIGEN_FAST_MATH, + HasTanh = EIGEN_FAST_MATH, + HasErf = 0, // EIGEN_FAST_MATH, + HasBlend = 0, + HasRound = 1, + HasFloor = 1, + HasCeil = 1, + HasRint = 1 + }; +}; + +template <> +struct unpacket_traits { + typedef Eigen::half type; + typedef Packet16h half; + enum { + size = 32, + alignment = Aligned64, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +template <> +struct unpacket_traits { + typedef Eigen::half type; + typedef Packet8h half; + enum { + size = 16, + alignment = Aligned32, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +template <> +struct unpacket_traits { + typedef Eigen::half type; + typedef Packet8h half; + enum { + size = 8, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +// Memory functions + +// pset1 + +template <> +EIGEN_STRONG_INLINE Packet32h pset1(const Eigen::half& from) { + return _mm512_set1_ph(static_cast<_Float16>(from)); +} + +// pset1frombits +template <> +EIGEN_STRONG_INLINE Packet32h pset1frombits(unsigned short from) { + return _mm512_castsi512_ph(_mm512_set1_epi16(from)); +} + +// pfirst + +template <> +EIGEN_STRONG_INLINE Eigen::half pfirst(const Packet32h& from) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + return half_impl::raw_uint16_to_half( + static_cast(_mm256_extract_epi16(_mm512_extracti32x8_epi32(_mm512_castph_si512(from), 0), 0))); +#else + Eigen::half dest[32]; + _mm512_storeu_ph(dest, from); + return dest[0]; +#endif +} + +// pload + +template <> +EIGEN_STRONG_INLINE Packet32h pload(const Eigen::half* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ph(from); +} + +// ploadu + +template <> +EIGEN_STRONG_INLINE Packet32h ploadu(const Eigen::half* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ph(from); +} + +// pstore + +template <> +EIGEN_STRONG_INLINE void pstore(Eigen::half* to, const Packet32h& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ph(to, from); +} + +// pstoreu + +template <> +EIGEN_STRONG_INLINE void pstoreu(Eigen::half* to, const Packet32h& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ph(to, from); +} + +// ploaddup +template <> +EIGEN_STRONG_INLINE Packet32h ploaddup(const Eigen::half* from) { + __m512h a = _mm512_castph256_ph512(_mm256_loadu_ph(from)); + return _mm512_permutexvar_ph(_mm512_set_epi16(15, 15, 14, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, 6, 6, + 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0), + a); +} + +// ploadquad +template <> +EIGEN_STRONG_INLINE Packet32h ploadquad(const Eigen::half* from) { + __m512h a = _mm512_castph128_ph512(_mm_loadu_ph(from)); + return _mm512_permutexvar_ph( + _mm512_set_epi16(7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0), + a); +} + +// pabs + +template <> +EIGEN_STRONG_INLINE Packet32h pabs(const Packet32h& a) { + return _mm512_abs_ph(a); +} + +// psignbit + +template <> +EIGEN_STRONG_INLINE Packet32h psignbit(const Packet32h& a) { + return _mm512_castsi512_ph(_mm512_srai_epi16(_mm512_castph_si512(a), 15)); +} + +// pmin + +template <> +EIGEN_STRONG_INLINE Packet32h pmin(const Packet32h& a, const Packet32h& b) { + return _mm512_min_ph(a, b); +} + +// pmax + +template <> +EIGEN_STRONG_INLINE Packet32h pmax(const Packet32h& a, const Packet32h& b) { + return _mm512_max_ph(a, b); +} + +// plset +template <> +EIGEN_STRONG_INLINE Packet32h plset(const half& a) { + return _mm512_add_ph(_mm512_set1_ph(a), + _mm512_set_ph(31.0f, 30.0f, 29.0f, 28.0f, 27.0f, 26.0f, 25.0f, 24.0f, 23.0f, 22.0f, 21.0f, 20.0f, + 19.0f, 18.0f, 17.0f, 16.0f, 15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, + 7.0f, 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f, 0.0f)); +} + +// por + +template <> +EIGEN_STRONG_INLINE Packet32h por(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_or_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); +} + +// pxor + +template <> +EIGEN_STRONG_INLINE Packet32h pxor(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_xor_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); +} + +// pand + +template <> +EIGEN_STRONG_INLINE Packet32h pand(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_and_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); +} + +// pandnot + +template <> +EIGEN_STRONG_INLINE Packet32h pandnot(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_andnot_si512(_mm512_castph_si512(b), _mm512_castph_si512(a))); +} + +// pselect + +template <> +EIGEN_DEVICE_FUNC inline Packet32h pselect(const Packet32h& mask, const Packet32h& a, const Packet32h& b) { + __mmask32 mask32 = _mm512_cmp_epi16_mask(_mm512_castph_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ); + return _mm512_mask_blend_ph(mask32, a, b); +} + +// pcmp_eq + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_eq(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_EQ_OQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, 0xffffu)); +} + +// pcmp_le + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_le(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LE_OQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, 0xffffu)); +} + +// pcmp_lt + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_lt(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LT_OQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, 0xffffu)); +} + +// pcmp_lt_or_nan + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_lt_or_nan(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_NGE_UQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi16(0), mask, 0xffffu)); +} + +// padd + +template <> +EIGEN_STRONG_INLINE Packet32h padd(const Packet32h& a, const Packet32h& b) { + return _mm512_add_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h padd(const Packet16h& a, const Packet16h& b) { + return _mm256_castph_si256(_mm256_add_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h padd(const Packet8h& a, const Packet8h& b) { + return _mm_castph_si128(_mm_add_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b))); +} + +// psub + +template <> +EIGEN_STRONG_INLINE Packet32h psub(const Packet32h& a, const Packet32h& b) { + return _mm512_sub_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h psub(const Packet16h& a, const Packet16h& b) { + return _mm256_castph_si256(_mm256_sub_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h psub(const Packet8h& a, const Packet8h& b) { + return _mm_castph_si128(_mm_sub_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b))); +} + +// pmul + +template <> +EIGEN_STRONG_INLINE Packet32h pmul(const Packet32h& a, const Packet32h& b) { + return _mm512_mul_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmul(const Packet16h& a, const Packet16h& b) { + return _mm256_castph_si256(_mm256_mul_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmul(const Packet8h& a, const Packet8h& b) { + return _mm_castph_si128(_mm_mul_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b))); +} + +// pdiv + +template <> +EIGEN_STRONG_INLINE Packet32h pdiv(const Packet32h& a, const Packet32h& b) { + return _mm512_div_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pdiv(const Packet16h& a, const Packet16h& b) { + return _mm256_castph_si256(_mm256_div_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pdiv(const Packet8h& a, const Packet8h& b) { + return _mm_castph_si128(_mm_div_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b))); +} + +// pround + +template <> +EIGEN_STRONG_INLINE Packet32h pround(const Packet32h& a) { + // Work-around for default std::round rounding mode. + + // Mask for the sign bit + const Packet32h signMask = pset1frombits(static_cast(0x8000u)); + // The largest half-preicision float less than 0.5 + const Packet32h prev0dot5 = pset1frombits(static_cast(0x37FFu)); + + return _mm512_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO); +} + +// print + +template <> +EIGEN_STRONG_INLINE Packet32h print(const Packet32h& a) { + return _mm512_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION); +} + +// pceil + +template <> +EIGEN_STRONG_INLINE Packet32h pceil(const Packet32h& a) { + return _mm512_roundscale_ph(a, _MM_FROUND_TO_POS_INF); +} + +// pfloor + +template <> +EIGEN_STRONG_INLINE Packet32h pfloor(const Packet32h& a) { + return _mm512_roundscale_ph(a, _MM_FROUND_TO_NEG_INF); +} + +// predux +template <> +EIGEN_STRONG_INLINE half predux(const Packet32h& a) { + return (half)_mm512_reduce_add_ph(a); +} + +template <> +EIGEN_STRONG_INLINE half predux(const Packet16h& a) { + return (half)_mm256_reduce_add_ph(_mm256_castsi256_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux(const Packet8h& a) { + return (half)_mm_reduce_add_ph(_mm_castsi128_ph(a)); +} + +// predux_half_dowto4 +template <> +EIGEN_STRONG_INLINE Packet16h predux_half_dowto4(const Packet32h& a) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + __m256i lowHalf = _mm256_castps_si256(_mm512_extractf32x8_ps(_mm512_castph_ps(a), 0)); + __m256i highHalf = _mm256_castps_si256(_mm512_extractf32x8_ps(_mm512_castph_ps(a), 1)); + + return Packet16h(padd(lowHalf, highHalf)); +#else + Eigen::half data[32]; + _mm512_storeu_ph(data, a); + + __m256i lowHalf = _mm256_castph_si256(_mm256_loadu_ph(data)); + __m256i highHalf = _mm256_castph_si256(_mm256_loadu_ph(data + 16)); + + return Packet16h(padd(lowHalf, highHalf)); +#endif +} + +// predux_max + +// predux_min + +// predux_mul + +#ifdef EIGEN_VECTORIZE_FMA + +// pmadd + +template <> +EIGEN_STRONG_INLINE Packet32h pmadd(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fmadd_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmadd(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_castph_si256(_mm256_fmadd_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b), _mm256_castsi256_ph(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmadd(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_castph_si128(_mm_fmadd_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b), _mm_castsi128_ph(c))); +} + +// pmsub + +template <> +EIGEN_STRONG_INLINE Packet32h pmsub(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fmsub_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmsub(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_castph_si256(_mm256_fmsub_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b), _mm256_castsi256_ph(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmsub(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_castph_si128(_mm_fmsub_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b), _mm_castsi128_ph(c))); +} + +// pnmadd + +template <> +EIGEN_STRONG_INLINE Packet32h pnmadd(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fnmadd_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pnmadd(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_castph_si256(_mm256_fnmadd_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b), _mm256_castsi256_ph(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pnmadd(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_castph_si128(_mm_fnmadd_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b), _mm_castsi128_ph(c))); +} + +// pnmsub + +template <> +EIGEN_STRONG_INLINE Packet32h pnmsub(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fnmsub_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pnmsub(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_castph_si256(_mm256_fnmsub_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b), _mm256_castsi256_ph(c))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pnmsub(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_castph_si128(_mm_fnmsub_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b), _mm_castsi128_ph(c))); +} + +#endif + +// pnegate + +template <> +EIGEN_STRONG_INLINE Packet32h pnegate(const Packet32h& a) { + return _mm512_sub_ph(_mm512_set1_ph(0.0), a); +} + +// pconj + +template <> +EIGEN_STRONG_INLINE Packet32h pconj(const Packet32h& a) { + return a; +} + +// psqrt + +template <> +EIGEN_STRONG_INLINE Packet32h psqrt(const Packet32h& a) { + return _mm512_sqrt_ph(a); +} + +// prsqrt + +template <> +EIGEN_STRONG_INLINE Packet32h prsqrt(const Packet32h& a) { + return _mm512_rsqrt_ph(a); +} + +// preciprocal + +template <> +EIGEN_STRONG_INLINE Packet32h preciprocal(const Packet32h& a) { + return _mm512_rcp_ph(a); +} + +// ptranspose + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& a) { + __m512i t[32]; + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 16; i++) { + t[2 * i] = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1])); + t[2 * i + 1] = + _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1])); + } + + __m512i p[32]; + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 8; i++) { + p[4 * i] = _mm512_unpacklo_epi32(t[4 * i], t[4 * i + 2]); + p[4 * i + 1] = _mm512_unpackhi_epi32(t[4 * i], t[4 * i + 2]); + p[4 * i + 2] = _mm512_unpacklo_epi32(t[4 * i + 1], t[4 * i + 3]); + p[4 * i + 3] = _mm512_unpackhi_epi32(t[4 * i + 1], t[4 * i + 3]); + } + + __m512i q[32]; + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 4; i++) { + q[8 * i] = _mm512_unpacklo_epi64(p[8 * i], p[8 * i + 4]); + q[8 * i + 1] = _mm512_unpackhi_epi64(p[8 * i], p[8 * i + 4]); + q[8 * i + 2] = _mm512_unpacklo_epi64(p[8 * i + 1], p[8 * i + 5]); + q[8 * i + 3] = _mm512_unpackhi_epi64(p[8 * i + 1], p[8 * i + 5]); + q[8 * i + 4] = _mm512_unpacklo_epi64(p[8 * i + 2], p[8 * i + 6]); + q[8 * i + 5] = _mm512_unpackhi_epi64(p[8 * i + 2], p[8 * i + 6]); + q[8 * i + 6] = _mm512_unpacklo_epi64(p[8 * i + 3], p[8 * i + 7]); + q[8 * i + 7] = _mm512_unpackhi_epi64(p[8 * i + 3], p[8 * i + 7]); + } + + __m512i f[32]; + +#define PACKET32H_TRANSPOSE_HELPER(X, Y) \ + do { \ + f[Y * 8] = _mm512_inserti32x4(f[Y * 8], _mm512_extracti32x4_epi32(q[X * 8], Y), X); \ + f[Y * 8 + 1] = _mm512_inserti32x4(f[Y * 8 + 1], _mm512_extracti32x4_epi32(q[X * 8 + 1], Y), X); \ + f[Y * 8 + 2] = _mm512_inserti32x4(f[Y * 8 + 2], _mm512_extracti32x4_epi32(q[X * 8 + 2], Y), X); \ + f[Y * 8 + 3] = _mm512_inserti32x4(f[Y * 8 + 3], _mm512_extracti32x4_epi32(q[X * 8 + 3], Y), X); \ + f[Y * 8 + 4] = _mm512_inserti32x4(f[Y * 8 + 4], _mm512_extracti32x4_epi32(q[X * 8 + 4], Y), X); \ + f[Y * 8 + 5] = _mm512_inserti32x4(f[Y * 8 + 5], _mm512_extracti32x4_epi32(q[X * 8 + 5], Y), X); \ + f[Y * 8 + 6] = _mm512_inserti32x4(f[Y * 8 + 6], _mm512_extracti32x4_epi32(q[X * 8 + 6], Y), X); \ + f[Y * 8 + 7] = _mm512_inserti32x4(f[Y * 8 + 7], _mm512_extracti32x4_epi32(q[X * 8 + 7], Y), X); \ + } while (false); + + PACKET32H_TRANSPOSE_HELPER(0, 0); + PACKET32H_TRANSPOSE_HELPER(1, 1); + PACKET32H_TRANSPOSE_HELPER(2, 2); + PACKET32H_TRANSPOSE_HELPER(3, 3); + + PACKET32H_TRANSPOSE_HELPER(1, 0); + PACKET32H_TRANSPOSE_HELPER(2, 0); + PACKET32H_TRANSPOSE_HELPER(3, 0); + PACKET32H_TRANSPOSE_HELPER(2, 1); + PACKET32H_TRANSPOSE_HELPER(3, 1); + PACKET32H_TRANSPOSE_HELPER(3, 2); + + PACKET32H_TRANSPOSE_HELPER(0, 1); + PACKET32H_TRANSPOSE_HELPER(0, 2); + PACKET32H_TRANSPOSE_HELPER(0, 3); + PACKET32H_TRANSPOSE_HELPER(1, 2); + PACKET32H_TRANSPOSE_HELPER(1, 3); + PACKET32H_TRANSPOSE_HELPER(2, 3); + +#undef PACKET32H_TRANSPOSE_HELPER + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 32; i++) { + a.packet[i] = _mm512_castsi512_ph(f[i]); + } +} + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& a) { + __m512i p0, p1, p2, p3, t0, t1, t2, t3, a0, a1, a2, a3; + t0 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1])); + t1 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1])); + t2 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3])); + t3 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3])); + + p0 = _mm512_unpacklo_epi32(t0, t2); + p1 = _mm512_unpackhi_epi32(t0, t2); + p2 = _mm512_unpacklo_epi32(t1, t3); + p3 = _mm512_unpackhi_epi32(t1, t3); + + a0 = p0; + a1 = p1; + a2 = p2; + a3 = p3; + + a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p1, 0), 1); + a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p0, 1), 0); + + a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p2, 0), 2); + a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p0, 2), 0); + + a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p3, 0), 3); + a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p0, 3), 0); + + a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p2, 1), 2); + a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p1, 2), 1); + + a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p3, 2), 3); + a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p2, 3), 2); + + a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p3, 1), 3); + a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p1, 3), 1); + + a.packet[0] = _mm512_castsi512_ph(a0); + a.packet[1] = _mm512_castsi512_ph(a1); + a.packet[2] = _mm512_castsi512_ph(a2); + a.packet[3] = _mm512_castsi512_ph(a3); +} + +// preverse + +template <> +EIGEN_STRONG_INLINE Packet32h preverse(const Packet32h& a) { + return _mm512_permutexvar_ph(_mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31), + a); +} + +// pscatter + +template <> +EIGEN_STRONG_INLINE void pscatter(half* to, const Packet32h& from, Index stride) { + EIGEN_ALIGN64 half aux[32]; + pstore(aux, from); + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 32; i++) { + to[stride * i] = aux[i]; + } +} + +// pgather + +template <> +EIGEN_STRONG_INLINE Packet32h pgather(const Eigen::half* from, Index stride) { + return _mm512_castsi512_ph(_mm512_set_epi16( + from[31 * stride].x, from[30 * stride].x, from[29 * stride].x, from[28 * stride].x, from[27 * stride].x, + from[26 * stride].x, from[25 * stride].x, from[24 * stride].x, from[23 * stride].x, from[22 * stride].x, + from[21 * stride].x, from[20 * stride].x, from[19 * stride].x, from[18 * stride].x, from[17 * stride].x, + from[16 * stride].x, from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x, + from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x, from[7 * stride].x, + from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, from[3 * stride].x, from[2 * stride].x, + from[1 * stride].x, from[0 * stride].x)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pcos(const Packet16h&); +template <> +EIGEN_STRONG_INLINE Packet16h psin(const Packet16h&); +template <> +EIGEN_STRONG_INLINE Packet16h plog(const Packet16h&); +template <> +EIGEN_STRONG_INLINE Packet16h plog2(const Packet16h&); +template <> +EIGEN_STRONG_INLINE Packet16h plog1p(const Packet16h&); +template <> +EIGEN_STRONG_INLINE Packet16h pexp(const Packet16h&); +template <> +EIGEN_STRONG_INLINE Packet16h pexpm1(const Packet16h&); +template <> +EIGEN_STRONG_INLINE Packet16h ptanh(const Packet16h&); +template <> +EIGEN_STRONG_INLINE Packet16h pfrexp(const Packet16h&, Packet16h&); +template <> +EIGEN_STRONG_INLINE Packet16h pldexp(const Packet16h&, const Packet16h&); + +EIGEN_STRONG_INLINE Packet32h combine2Packet16h(const Packet16h& a, const Packet16h& b) { + __m512d result = _mm512_undefined_pd(); + result = _mm512_insertf64x4(result, _mm256_castsi256_pd(a), 0); + result = _mm512_insertf64x4(result, _mm256_castsi256_pd(b), 1); + return _mm512_castpd_ph(result); +} + +EIGEN_STRONG_INLINE void extract2Packet16h(const Packet32h& x, Packet16h& a, Packet16h& b) { + a = _mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castph_pd(x), 0)); + b = _mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castph_pd(x), 1)); +} + +// psin +template <> +EIGEN_STRONG_INLINE Packet32h psin(const Packet32h& a) { + Packet16h low; + Packet16h high; + extract2Packet16h(a, low, high); + + Packet16h lowOut = psin(low); + Packet16h highOut = psin(high); + + return combine2Packet16h(lowOut, highOut); +} + +// pcos +template <> +EIGEN_STRONG_INLINE Packet32h pcos(const Packet32h& a) { + Packet16h low; + Packet16h high; + extract2Packet16h(a, low, high); + + Packet16h lowOut = pcos(low); + Packet16h highOut = pcos(high); + + return combine2Packet16h(lowOut, highOut); +} + +// plog +template <> +EIGEN_STRONG_INLINE Packet32h plog(const Packet32h& a) { + Packet16h low; + Packet16h high; + extract2Packet16h(a, low, high); + + Packet16h lowOut = plog(low); + Packet16h highOut = plog(high); + + return combine2Packet16h(lowOut, highOut); +} + +// plog2 +template <> +EIGEN_STRONG_INLINE Packet32h plog2(const Packet32h& a) { + Packet16h low; + Packet16h high; + extract2Packet16h(a, low, high); + + Packet16h lowOut = plog2(low); + Packet16h highOut = plog2(high); + + return combine2Packet16h(lowOut, highOut); +} + +// plog1p +template <> +EIGEN_STRONG_INLINE Packet32h plog1p(const Packet32h& a) { + Packet16h low; + Packet16h high; + extract2Packet16h(a, low, high); + + Packet16h lowOut = plog1p(low); + Packet16h highOut = plog1p(high); + + return combine2Packet16h(lowOut, highOut); +} + +// pexp +template <> +EIGEN_STRONG_INLINE Packet32h pexp(const Packet32h& a) { + Packet16h low; + Packet16h high; + extract2Packet16h(a, low, high); + + Packet16h lowOut = pexp(low); + Packet16h highOut = pexp(high); + + return combine2Packet16h(lowOut, highOut); +} + +// pexpm1 +template <> +EIGEN_STRONG_INLINE Packet32h pexpm1(const Packet32h& a) { + Packet16h low; + Packet16h high; + extract2Packet16h(a, low, high); + + Packet16h lowOut = pexpm1(low); + Packet16h highOut = pexpm1(high); + + return combine2Packet16h(lowOut, highOut); +} + +// ptanh +template <> +EIGEN_STRONG_INLINE Packet32h ptanh(const Packet32h& a) { + Packet16h low; + Packet16h high; + extract2Packet16h(a, low, high); + + Packet16h lowOut = ptanh(low); + Packet16h highOut = ptanh(high); + + return combine2Packet16h(lowOut, highOut); +} + +// pfrexp +template <> +EIGEN_STRONG_INLINE Packet32h pfrexp(const Packet32h& a, Packet32h& exponent) { + Packet16h low; + Packet16h high; + extract2Packet16h(a, low, high); + + Packet16h exp1 = _mm256_undefined_si256(); + Packet16h exp2 = _mm256_undefined_si256(); + + Packet16h lowOut = pfrexp(low, exp1); + Packet16h highOut = pfrexp(high, exp2); + + exponent = combine2Packet16h(exp1, exp2); + + return combine2Packet16h(lowOut, highOut); +} + +// pldexp +template <> +EIGEN_STRONG_INLINE Packet32h pldexp(const Packet32h& a, const Packet32h& exponent) { + Packet16h low; + Packet16h high; + extract2Packet16h(a, low, high); + + Packet16h exp1; + Packet16h exp2; + extract2Packet16h(exponent, exp1, exp2); + + Packet16h lowOut = pldexp(low, exp1); + Packet16h highOut = pldexp(high, exp2); + + return combine2Packet16h(lowOut, highOut); +} + +} // end namespace internal +} // end namespace Eigen + +#endif // EIGEN_PACKET_MATH_FP16_AVX512_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/TrsmKernel.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/TrsmKernel.h new file mode 100644 index 00000000000..a3025ec2c7f --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/TrsmKernel.h @@ -0,0 +1,1165 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2022 Intel Corporation +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CORE_ARCH_AVX512_TRSM_KERNEL_H +#define EIGEN_CORE_ARCH_AVX512_TRSM_KERNEL_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +#if !defined(EIGEN_USE_AVX512_TRSM_KERNELS) +#define EIGEN_USE_AVX512_TRSM_KERNELS 1 +#endif + +// TRSM kernels currently unconditionally rely on malloc with AVX512. +// Disable them if malloc is explicitly disabled at compile-time. +#ifdef EIGEN_NO_MALLOC +#undef EIGEN_USE_AVX512_TRSM_KERNELS +#define EIGEN_USE_AVX512_TRSM_KERNELS 0 +#endif + +#if EIGEN_USE_AVX512_TRSM_KERNELS +#if !defined(EIGEN_USE_AVX512_TRSM_R_KERNELS) +#define EIGEN_USE_AVX512_TRSM_R_KERNELS 1 +#endif +#if !defined(EIGEN_USE_AVX512_TRSM_L_KERNELS) +#define EIGEN_USE_AVX512_TRSM_L_KERNELS 1 +#endif +#else // EIGEN_USE_AVX512_TRSM_KERNELS == 0 +#define EIGEN_USE_AVX512_TRSM_R_KERNELS 0 +#define EIGEN_USE_AVX512_TRSM_L_KERNELS 0 +#endif + +// Need this for some std::min calls. +#ifdef min +#undef min +#endif + +namespace Eigen { +namespace internal { + +#define EIGEN_AVX_MAX_NUM_ACC (int64_t(24)) +#define EIGEN_AVX_MAX_NUM_ROW (int64_t(8)) // Denoted L in code. +#define EIGEN_AVX_MAX_K_UNROL (int64_t(4)) +#define EIGEN_AVX_B_LOAD_SETS (int64_t(2)) +#define EIGEN_AVX_MAX_A_BCAST (int64_t(2)) +typedef Packet16f vecFullFloat; +typedef Packet8d vecFullDouble; +typedef Packet8f vecHalfFloat; +typedef Packet4d vecHalfDouble; + +// Compile-time unrolls are implemented here. +// Note: this depends on macros and typedefs above. +#include "TrsmUnrolls.inc" + +#if (EIGEN_USE_AVX512_TRSM_KERNELS) && (EIGEN_COMP_CLANG != 0) +/** + * For smaller problem sizes, and certain compilers, using the optimized kernels trsmKernelL/R directly + * is faster than the packed versions in TriangularSolverMatrix.h. + * + * The current heuristic is based on having having all arrays used in the largest gemm-update + * in triSolve fit in roughly L2Cap (percentage) of the L2 cache. These cutoffs are a bit conservative and could be + * larger for some trsm cases. + * The formula: + * + * (L*M + M*N + L*N)*sizeof(Scalar) < L2Cache*L2Cap + * + * L = number of rows to solve at a time + * N = number of rhs + * M = Dimension of triangular matrix + * + */ +#if !defined(EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS) +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS 1 +#endif + +#if EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS + +#if EIGEN_USE_AVX512_TRSM_R_KERNELS +#if !defined(EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS) +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS 1 +#endif // !defined(EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS) +#endif + +#if EIGEN_USE_AVX512_TRSM_L_KERNELS +#if !defined(EIGEN_ENABLE_AVX512_NOCOPY_TRSM_L_CUTOFFS) +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_L_CUTOFFS 1 +#endif +#endif // EIGEN_USE_AVX512_TRSM_L_KERNELS + +#else // EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS == 0 +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS 0 +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_L_CUTOFFS 0 +#endif // EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS + +template +int64_t avx512_trsm_cutoff(int64_t L2Size, int64_t N, double L2Cap) { + const int64_t U3 = 3 * packet_traits::size; + const int64_t MaxNb = 5 * U3; + int64_t Nb = std::min(MaxNb, N); + double cutoff_d = + (((L2Size * L2Cap) / (sizeof(Scalar))) - (EIGEN_AVX_MAX_NUM_ROW)*Nb) / ((EIGEN_AVX_MAX_NUM_ROW) + Nb); + int64_t cutoff_l = static_cast(cutoff_d); + return (cutoff_l / EIGEN_AVX_MAX_NUM_ROW) * EIGEN_AVX_MAX_NUM_ROW; +} +#else // !(EIGEN_USE_AVX512_TRSM_KERNELS) || !(EIGEN_COMP_CLANG != 0) +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS 0 +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS 0 +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_L_CUTOFFS 0 +#endif + +/** + * Used by gemmKernel for the case A/B row-major and C col-major. + */ +template +EIGEN_ALWAYS_INLINE void transStoreC(PacketBlock &zmm, + Scalar *C_arr, int64_t LDC, int64_t remM_ = 0, int64_t remN_ = 0) { + EIGEN_UNUSED_VARIABLE(remN_); + EIGEN_UNUSED_VARIABLE(remM_); + using urolls = unrolls::trans; + + constexpr int64_t U3 = urolls::PacketSize * 3; + constexpr int64_t U2 = urolls::PacketSize * 2; + constexpr int64_t U1 = urolls::PacketSize * 1; + + static_assert(unrollN == U1 || unrollN == U2 || unrollN == U3, "unrollN should be a multiple of PacketSize"); + static_assert(unrollM == EIGEN_AVX_MAX_NUM_ROW, "unrollM should be equal to EIGEN_AVX_MAX_NUM_ROW"); + + urolls::template transpose(zmm); + EIGEN_IF_CONSTEXPR(unrollN > U2) urolls::template transpose(zmm); + EIGEN_IF_CONSTEXPR(unrollN > U1) urolls::template transpose(zmm); + + static_assert((remN && unrollN == U1) || !remN, "When handling N remainder set unrollN=U1"); + EIGEN_IF_CONSTEXPR(!remN) { + urolls::template storeC(C_arr, LDC, zmm, remM_); + EIGEN_IF_CONSTEXPR(unrollN > U1) { + constexpr int64_t unrollN_ = std::min(unrollN - U1, U1); + urolls::template storeC(C_arr + U1 * LDC, LDC, zmm, remM_); + } + EIGEN_IF_CONSTEXPR(unrollN > U2) { + constexpr int64_t unrollN_ = std::min(unrollN - U2, U1); + urolls::template storeC(C_arr + U2 * LDC, LDC, zmm, remM_); + } + } + else { + EIGEN_IF_CONSTEXPR((std::is_same::value)) { + // Note: without "if constexpr" this section of code will also be + // parsed by the compiler so each of the storeC will still be instantiated. + // We use enable_if in aux_storeC to set it to an empty function for + // these cases. + if (remN_ == 15) + urolls::template storeC<15, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 14) + urolls::template storeC<14, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 13) + urolls::template storeC<13, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 12) + urolls::template storeC<12, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 11) + urolls::template storeC<11, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 10) + urolls::template storeC<10, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 9) + urolls::template storeC<9, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 8) + urolls::template storeC<8, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 7) + urolls::template storeC<7, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 6) + urolls::template storeC<6, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 5) + urolls::template storeC<5, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 4) + urolls::template storeC<4, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 3) + urolls::template storeC<3, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 2) + urolls::template storeC<2, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 1) + urolls::template storeC<1, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + } + else { + if (remN_ == 7) + urolls::template storeC<7, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 6) + urolls::template storeC<6, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 5) + urolls::template storeC<5, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 4) + urolls::template storeC<4, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 3) + urolls::template storeC<3, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 2) + urolls::template storeC<2, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 1) + urolls::template storeC<1, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + } + } +} + +/** + * GEMM like operation for trsm panel updates. + * Computes: C -= A*B + * K must be multipe of 4. + * + * Unrolls used are {1,2,4,8}x{U1,U2,U3}; + * For good performance we want K to be large with M/N relatively small, but also large enough + * to use the {8,U3} unroll block. + * + * isARowMajor: is A_arr row-major? + * isCRowMajor: is C_arr row-major? (B_arr is assumed to be row-major). + * isAdd: C += A*B or C -= A*B (used by trsm) + * handleKRem: Handle arbitrary K? This is not needed for trsm. + */ +template +void gemmKernel(Scalar *A_arr, Scalar *B_arr, Scalar *C_arr, int64_t M, int64_t N, int64_t K, int64_t LDA, int64_t LDB, + int64_t LDC) { + using urolls = unrolls::gemm; + constexpr int64_t U3 = urolls::PacketSize * 3; + constexpr int64_t U2 = urolls::PacketSize * 2; + constexpr int64_t U1 = urolls::PacketSize * 1; + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + int64_t N_ = (N / U3) * U3; + int64_t M_ = (M / EIGEN_AVX_MAX_NUM_ROW) * EIGEN_AVX_MAX_NUM_ROW; + int64_t K_ = (K / EIGEN_AVX_MAX_K_UNROL) * EIGEN_AVX_MAX_K_UNROL; + int64_t j = 0; + for (; j < N_; j += U3) { + constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 3; + int64_t i = 0; + for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<3, EIGEN_AVX_MAX_NUM_ROW>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC); + } + } + if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<3, 4>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<3, 4>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<3, 4>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4); + } + i += 4; + } + if (M - i >= 2) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<3, 2>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<3, 2>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<3, 2>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2); + } + i += 2; + } + if (M - i > 0) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<3, 1>(zmm); + { + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<3, 1>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<3, 1>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1); + } + } + } + } + if (N - j >= U2) { + constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 2; + int64_t i = 0; + for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j]; + EIGEN_IF_CONSTEXPR(isCRowMajor) B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<2, EIGEN_AVX_MAX_NUM_ROW>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC); + } + } + if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<2, 4>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<2, 4>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<2, 4>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4); + } + i += 4; + } + if (M - i >= 2) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<2, 2>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<2, 2>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<2, 2>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2); + } + i += 2; + } + if (M - i > 0) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<2, 1>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<2, 1>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<2, 1>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1); + } + } + j += U2; + } + if (N - j >= U1) { + constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 1; + int64_t i = 0; + for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, EIGEN_AVX_MAX_NUM_ROW>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC); + } + } + if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 4>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 4>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<1, 4>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4); + } + i += 4; + } + if (M - i >= 2) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 2>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 2>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<1, 2>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2); + } + i += 2; + } + if (M - i > 0) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 1>(zmm); + { + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 1>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<1, 1>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1); + } + } + } + j += U1; + } + if (N - j > 0) { + constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 1; + int64_t i = 0; + for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, EIGEN_AVX_MAX_NUM_ROW>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 0, N - j); + } + } + if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 4>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm, N - j); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + urolls::template storeC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4, N - j); + } + i += 4; + } + if (M - i >= 2) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 2>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm, N - j); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + urolls::template storeC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2, N - j); + } + i += 2; + } + if (M - i > 0) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 1>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm, N - j); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, + N - j); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + urolls::template storeC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1, N - j); + } + } + } +} + +/** + * Triangular solve kernel with A on left with K number of rhs. dim(A) = unrollM + * + * unrollM: dimension of A matrix (triangular matrix). unrollM should be <= EIGEN_AVX_MAX_NUM_ROW + * isFWDSolve: is forward solve? + * isUnitDiag: is the diagonal of A all ones? + * The B matrix (RHS) is assumed to be row-major + */ +template +EIGEN_ALWAYS_INLINE void triSolveKernel(Scalar *A_arr, Scalar *B_arr, int64_t K, int64_t LDA, int64_t LDB) { + static_assert(unrollM <= EIGEN_AVX_MAX_NUM_ROW, "unrollM should be equal to EIGEN_AVX_MAX_NUM_ROW"); + using urolls = unrolls::trsm; + constexpr int64_t U3 = urolls::PacketSize * 3; + constexpr int64_t U2 = urolls::PacketSize * 2; + constexpr int64_t U1 = urolls::PacketSize * 1; + + PacketBlock RHSInPacket; + PacketBlock AInPacket; + + int64_t k = 0; + while (K - k >= U3) { + urolls::template loadRHS(B_arr + k, LDB, RHSInPacket); + urolls::template triSolveMicroKernel(A_arr, LDA, RHSInPacket, + AInPacket); + urolls::template storeRHS(B_arr + k, LDB, RHSInPacket); + k += U3; + } + if (K - k >= U2) { + urolls::template loadRHS(B_arr + k, LDB, RHSInPacket); + urolls::template triSolveMicroKernel(A_arr, LDA, RHSInPacket, + AInPacket); + urolls::template storeRHS(B_arr + k, LDB, RHSInPacket); + k += U2; + } + if (K - k >= U1) { + urolls::template loadRHS(B_arr + k, LDB, RHSInPacket); + urolls::template triSolveMicroKernel(A_arr, LDA, RHSInPacket, + AInPacket); + urolls::template storeRHS(B_arr + k, LDB, RHSInPacket); + k += U1; + } + if (K - k > 0) { + // Handle remaining number of RHS + urolls::template loadRHS(B_arr + k, LDB, RHSInPacket, K - k); + urolls::template triSolveMicroKernel(A_arr, LDA, RHSInPacket, + AInPacket); + urolls::template storeRHS(B_arr + k, LDB, RHSInPacket, K - k); + } +} + +/** + * Triangular solve routine with A on left and dimension of at most L with K number of rhs. This is essentially + * a wrapper for triSolveMicrokernel for M = {1,2,3,4,5,6,7,8}. + * + * isFWDSolve: is forward solve? + * isUnitDiag: is the diagonal of A all ones? + * The B matrix (RHS) is assumed to be row-major + */ +template +void triSolveKernelLxK(Scalar *A_arr, Scalar *B_arr, int64_t M, int64_t K, int64_t LDA, int64_t LDB) { + // Note: this assumes EIGEN_AVX_MAX_NUM_ROW = 8. Unrolls should be adjusted + // accordingly if EIGEN_AVX_MAX_NUM_ROW is smaller. + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + if (M == 8) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 7) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 6) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 5) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 4) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 3) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 2) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 1) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + return; +} + +/** + * This routine is used to copy B to/from a temporary array (row-major) for cases where B is column-major. + * + * toTemp: true => copy to temporary array, false => copy from temporary array + * remM: true = need to handle remainder values for M (M < EIGEN_AVX_MAX_NUM_ROW) + * + */ +template +EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar *B_arr, int64_t LDB, int64_t K, Scalar *B_temp, int64_t LDB_, + int64_t remM_ = 0) { + EIGEN_UNUSED_VARIABLE(remM_); + using urolls = unrolls::transB; + using vecHalf = typename std::conditional::value, vecHalfFloat, vecFullDouble>::type; + PacketBlock ymm; + constexpr int64_t U3 = urolls::PacketSize * 3; + constexpr int64_t U2 = urolls::PacketSize * 2; + constexpr int64_t U1 = urolls::PacketSize * 1; + int64_t K_ = K / U3 * U3; + int64_t k = 0; + + for (; k < K_; k += U3) { + urolls::template transB_kernel(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += U3; + } + if (K - k >= U2) { + urolls::template transB_kernel(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += U2; + k += U2; + } + if (K - k >= U1) { + urolls::template transB_kernel(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += U1; + k += U1; + } + EIGEN_IF_CONSTEXPR(U1 > 8) { + // Note: without "if constexpr" this section of code will also be + // parsed by the compiler so there is an additional check in {load/store}BBlock + // to make sure the counter is not non-negative. + if (K - k >= 8) { + urolls::template transB_kernel<8, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += 8; + k += 8; + } + } + EIGEN_IF_CONSTEXPR(U1 > 4) { + // Note: without "if constexpr" this section of code will also be + // parsed by the compiler so there is an additional check in {load/store}BBlock + // to make sure the counter is not non-negative. + if (K - k >= 4) { + urolls::template transB_kernel<4, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += 4; + k += 4; + } + } + if (K - k >= 2) { + urolls::template transB_kernel<2, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += 2; + k += 2; + } + if (K - k >= 1) { + urolls::template transB_kernel<1, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += 1; + k += 1; + } +} + +/** + * Main triangular solve driver + * + * Triangular solve with A on the left. + * Scalar: Scalar precision, only float/double is supported. + * isARowMajor: is A row-major? + * isBRowMajor: is B row-major? + * isFWDSolve: is this forward solve or backward (true => forward)? + * isUnitDiag: is diagonal of A unit or nonunit (true => A has unit diagonal)? + * + * M: dimension of A + * numRHS: number of right hand sides (coincides with K dimension for gemm updates) + * + * Here are the mapping between the different TRSM cases (col-major) and triSolve: + * + * LLN (left , lower, A non-transposed) :: isARowMajor=false, isBRowMajor=false, isFWDSolve=true + * LUT (left , upper, A transposed) :: isARowMajor=true, isBRowMajor=false, isFWDSolve=true + * LUN (left , upper, A non-transposed) :: isARowMajor=false, isBRowMajor=false, isFWDSolve=false + * LLT (left , lower, A transposed) :: isARowMajor=true, isBRowMajor=false, isFWDSolve=false + * RUN (right, upper, A non-transposed) :: isARowMajor=true, isBRowMajor=true, isFWDSolve=true + * RLT (right, lower, A transposed) :: isARowMajor=false, isBRowMajor=true, isFWDSolve=true + * RUT (right, upper, A transposed) :: isARowMajor=false, isBRowMajor=true, isFWDSolve=false + * RLN (right, lower, A non-transposed) :: isARowMajor=true, isBRowMajor=true, isFWDSolve=false + * + * Note: For RXX cases M,numRHS should be swapped. + * + */ +template +void triSolve(Scalar *A_arr, Scalar *B_arr, int64_t M, int64_t numRHS, int64_t LDA, int64_t LDB) { + constexpr int64_t psize = packet_traits::size; + /** + * The values for kB, numM were determined experimentally. + * kB: Number of RHS we process at a time. + * numM: number of rows of B we will store in a temporary array (see below.) This should be a multiple of L. + * + * kB was determined by initially setting kB = numRHS and benchmarking triSolve (TRSM-RUN case) + * performance with M=numRHS. + * It was observed that performance started to drop around M=numRHS=240. This is likely machine dependent. + * + * numM was chosen "arbitrarily". It should be relatively small so B_temp is not too large, but it should be + * large enough to allow GEMM updates to have larger "K"s (see below.) No benchmarking has been done so far to + * determine optimal values for numM. + */ + constexpr int64_t kB = (3 * psize) * 5; // 5*U3 + constexpr int64_t numM = 8 * EIGEN_AVX_MAX_NUM_ROW; + + int64_t sizeBTemp = 0; + Scalar *B_temp = NULL; + EIGEN_IF_CONSTEXPR(!isBRowMajor) { + /** + * If B is col-major, we copy it to a fixed-size temporary array of size at most ~numM*kB and + * transpose it to row-major. Call the solve routine, and copy+transpose it back to the original array. + * The updated row-major copy of B is reused in the GEMM updates. + */ + sizeBTemp = (((std::min(kB, numRHS) + psize - 1) / psize + 4) * psize) * numM; + } + + EIGEN_IF_CONSTEXPR(!isBRowMajor) B_temp = (Scalar *)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64); + + for (int64_t k = 0; k < numRHS; k += kB) { + int64_t bK = numRHS - k > kB ? kB : numRHS - k; + int64_t M_ = (M / EIGEN_AVX_MAX_NUM_ROW) * EIGEN_AVX_MAX_NUM_ROW, gemmOff = 0; + + // bK rounded up to next multiple of L=EIGEN_AVX_MAX_NUM_ROW. When B_temp is used, we solve for bkL RHS + // instead of bK RHS in triSolveKernelLxK. + int64_t bkL = ((bK + (EIGEN_AVX_MAX_NUM_ROW - 1)) / EIGEN_AVX_MAX_NUM_ROW) * EIGEN_AVX_MAX_NUM_ROW; + const int64_t numScalarPerCache = 64 / sizeof(Scalar); + // Leading dimension of B_temp, will be a multiple of the cache line size. + int64_t LDT = ((bkL + (numScalarPerCache - 1)) / numScalarPerCache) * numScalarPerCache; + int64_t offsetBTemp = 0; + for (int64_t i = 0; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { + EIGEN_IF_CONSTEXPR(!isBRowMajor) { + int64_t indA_i = isFWDSolve ? i : M - 1 - i; + int64_t indB_i = isFWDSolve ? i : M - (i + EIGEN_AVX_MAX_NUM_ROW); + int64_t offB_1 = isFWDSolve ? offsetBTemp : sizeBTemp - EIGEN_AVX_MAX_NUM_ROW * LDT - offsetBTemp; + int64_t offB_2 = isFWDSolve ? offsetBTemp : sizeBTemp - LDT - offsetBTemp; + // Copy values from B to B_temp. + copyBToRowMajor(B_arr + indB_i + k * LDB, LDB, bK, B_temp + offB_1, LDT); + // Triangular solve with a small block of A and long horizontal blocks of B (or B_temp if B col-major) + triSolveKernelLxK( + &A_arr[idA(indA_i, indA_i, LDA)], B_temp + offB_2, EIGEN_AVX_MAX_NUM_ROW, bkL, LDA, LDT); + // Copy values from B_temp back to B. B_temp will be reused in gemm call below. + copyBToRowMajor(B_arr + indB_i + k * LDB, LDB, bK, B_temp + offB_1, LDT); + + offsetBTemp += EIGEN_AVX_MAX_NUM_ROW * LDT; + } + else { + int64_t ind = isFWDSolve ? i : M - 1 - i; + triSolveKernelLxK( + &A_arr[idA(ind, ind, LDA)], B_arr + k + ind * LDB, EIGEN_AVX_MAX_NUM_ROW, bK, LDA, LDB); + } + if (i + EIGEN_AVX_MAX_NUM_ROW < M_) { + /** + * For the GEMM updates, we want "K" (K=i+8 in this case) to be large as soon as possible + * to reuse the accumulators in GEMM as much as possible. So we only update 8xbK blocks of + * B as follows: + * + * A B + * __ + * |__|__ |__| + * |__|__|__ |__| + * |__|__|__|__ |__| + * |********|__| |**| + */ + EIGEN_IF_CONSTEXPR(isBRowMajor) { + int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW); + int64_t indA_j = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW); + int64_t indB_i = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW); + int64_t indB_i2 = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW); + gemmKernel( + &A_arr[idA(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB, + EIGEN_AVX_MAX_NUM_ROW, bK, i + EIGEN_AVX_MAX_NUM_ROW, LDA, LDB, LDB); + } + else { + if (offsetBTemp + EIGEN_AVX_MAX_NUM_ROW * LDT > sizeBTemp) { + /** + * Similar idea as mentioned above, but here we are limited by the number of updated values of B + * that can be stored (row-major) in B_temp. + * + * If there is not enough space to store the next batch of 8xbK of B in B_temp, we call GEMM + * update and partially update the remaining old values of B which depends on the new values + * of B stored in B_temp. These values are then no longer needed and can be overwritten. + */ + int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : 0; + int64_t indA_j = isFWDSolve ? gemmOff : M - (i + EIGEN_AVX_MAX_NUM_ROW); + int64_t indB_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : 0; + int64_t offB_1 = isFWDSolve ? 0 : sizeBTemp - offsetBTemp; + gemmKernel( + &A_arr[idA(indA_i, indA_j, LDA)], B_temp + offB_1, B_arr + indB_i + (k)*LDB, + M - (i + EIGEN_AVX_MAX_NUM_ROW), bK, i + EIGEN_AVX_MAX_NUM_ROW - gemmOff, LDA, LDT, LDB); + offsetBTemp = 0; + gemmOff = i + EIGEN_AVX_MAX_NUM_ROW; + } else { + /** + * If there is enough space in B_temp, we only update the next 8xbK values of B. + */ + int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW); + int64_t indA_j = isFWDSolve ? gemmOff : M - (i + EIGEN_AVX_MAX_NUM_ROW); + int64_t indB_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW); + int64_t offB_1 = isFWDSolve ? 0 : sizeBTemp - offsetBTemp; + gemmKernel( + &A_arr[idA(indA_i, indA_j, LDA)], B_temp + offB_1, B_arr + indB_i + (k)*LDB, + EIGEN_AVX_MAX_NUM_ROW, bK, i + EIGEN_AVX_MAX_NUM_ROW - gemmOff, LDA, LDT, LDB); + } + } + } + } + // Handle M remainder.. + int64_t bM = M - M_; + if (bM > 0) { + if (M_ > 0) { + EIGEN_IF_CONSTEXPR(isBRowMajor) { + int64_t indA_i = isFWDSolve ? M_ : 0; + int64_t indA_j = isFWDSolve ? 0 : bM; + int64_t indB_i = isFWDSolve ? 0 : bM; + int64_t indB_i2 = isFWDSolve ? M_ : 0; + gemmKernel( + &A_arr[idA(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB, bM, + bK, M_, LDA, LDB, LDB); + } + else { + int64_t indA_i = isFWDSolve ? M_ : 0; + int64_t indA_j = isFWDSolve ? gemmOff : bM; + int64_t indB_i = isFWDSolve ? M_ : 0; + int64_t offB_1 = isFWDSolve ? 0 : sizeBTemp - offsetBTemp; + gemmKernel(&A_arr[idA(indA_i, indA_j, LDA)], + B_temp + offB_1, B_arr + indB_i + (k)*LDB, bM, bK, + M_ - gemmOff, LDA, LDT, LDB); + } + } + EIGEN_IF_CONSTEXPR(!isBRowMajor) { + int64_t indA_i = isFWDSolve ? M_ : M - 1 - M_; + int64_t indB_i = isFWDSolve ? M_ : 0; + int64_t offB_1 = isFWDSolve ? 0 : (bM - 1) * bkL; + copyBToRowMajor(B_arr + indB_i + k * LDB, LDB, bK, B_temp, bkL, bM); + triSolveKernelLxK(&A_arr[idA(indA_i, indA_i, LDA)], + B_temp + offB_1, bM, bkL, LDA, bkL); + copyBToRowMajor(B_arr + indB_i + k * LDB, LDB, bK, B_temp, bkL, bM); + } + else { + int64_t ind = isFWDSolve ? M_ : M - 1 - M_; + triSolveKernelLxK(&A_arr[idA(ind, ind, LDA)], + B_arr + k + ind * LDB, bM, bK, LDA, LDB); + } + } + } + + EIGEN_IF_CONSTEXPR(!isBRowMajor) handmade_aligned_free(B_temp); +} + +// Template specializations of trsmKernelL/R for float/double and inner strides of 1. +#if (EIGEN_USE_AVX512_TRSM_KERNELS) +#if (EIGEN_USE_AVX512_TRSM_R_KERNELS) +template +struct trsmKernelR; + +template +struct trsmKernelR { + static void kernel(Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + Index otherStride); +}; + +template +struct trsmKernelR { + static void kernel(Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + Index otherStride); +}; + +template +EIGEN_DONT_INLINE void trsmKernelR::kernel( + Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + Index otherStride) { + EIGEN_UNUSED_VARIABLE(otherIncr); +#ifdef EIGEN_RUNTIME_NO_MALLOC + if (!is_malloc_allowed()) { + trsmKernelR::kernel( + size, otherSize, _tri, triStride, _other, otherIncr, otherStride); + return; + } +#endif + triSolve( + const_cast(_tri), _other, size, otherSize, triStride, otherStride); +} + +template +EIGEN_DONT_INLINE void trsmKernelR::kernel( + Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + Index otherStride) { + EIGEN_UNUSED_VARIABLE(otherIncr); +#ifdef EIGEN_RUNTIME_NO_MALLOC + if (!is_malloc_allowed()) { + trsmKernelR::kernel( + size, otherSize, _tri, triStride, _other, otherIncr, otherStride); + return; + } +#endif + triSolve( + const_cast(_tri), _other, size, otherSize, triStride, otherStride); +} +#endif // (EIGEN_USE_AVX512_TRSM_R_KERNELS) + +// These trsm kernels require temporary memory allocation +#if (EIGEN_USE_AVX512_TRSM_L_KERNELS) +template +struct trsmKernelL; + +template +struct trsmKernelL { + static void kernel(Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + Index otherStride); +}; + +template +struct trsmKernelL { + static void kernel(Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + Index otherStride); +}; + +template +EIGEN_DONT_INLINE void trsmKernelL::kernel( + Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + Index otherStride) { + EIGEN_UNUSED_VARIABLE(otherIncr); +#ifdef EIGEN_RUNTIME_NO_MALLOC + if (!is_malloc_allowed()) { + trsmKernelL::kernel( + size, otherSize, _tri, triStride, _other, otherIncr, otherStride); + return; + } +#endif + triSolve( + const_cast(_tri), _other, size, otherSize, triStride, otherStride); +} + +template +EIGEN_DONT_INLINE void trsmKernelL::kernel( + Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + Index otherStride) { + EIGEN_UNUSED_VARIABLE(otherIncr); +#ifdef EIGEN_RUNTIME_NO_MALLOC + if (!is_malloc_allowed()) { + trsmKernelL::kernel( + size, otherSize, _tri, triStride, _other, otherIncr, otherStride); + return; + } +#endif + triSolve( + const_cast(_tri), _other, size, otherSize, triStride, otherStride); +} +#endif // EIGEN_USE_AVX512_TRSM_L_KERNELS +#endif // EIGEN_USE_AVX512_TRSM_KERNELS +} // namespace internal +} // namespace Eigen +#endif // EIGEN_CORE_ARCH_AVX512_TRSM_KERNEL_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc new file mode 100644 index 00000000000..4c6116c6922 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc @@ -0,0 +1,1218 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2022 Intel Corporation +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CORE_ARCH_AVX512_TRSM_UNROLLS_H +#define EIGEN_CORE_ARCH_AVX512_TRSM_UNROLLS_H + +template +EIGEN_ALWAYS_INLINE int64_t idA(int64_t i, int64_t j, int64_t LDA) { + EIGEN_IF_CONSTEXPR(isARowMajor) return i * LDA + j; + else return i + j * LDA; +} + +/** + * This namespace contains various classes used to generate compile-time unrolls which are + * used throughout the trsm/gemm kernels. The unrolls are characterized as for-loops (1-D), nested + * for-loops (2-D), or triple nested for-loops (3-D). Unrolls are generated using template recursion + * + * Example, the 2-D for-loop is unrolled recursively by first flattening to a 1-D loop. + * + * for(startI = 0; startI < endI; startI++) for(startC = 0; startC < endI*endJ; startC++) + * for(startJ = 0; startJ < endJ; startJ++) ----> startI = (startC)/(endJ) + * func(startI,startJ) startJ = (startC)%(endJ) + * func(...) + * + * The 1-D loop can be unrolled recursively by using enable_if and defining an auxillary function + * with a template parameter used as a counter. + * + * template + * std::enable_if_t<(counter <= 0)> <---- tail case. + * aux_func {} + * + * template + * std::enable_if_t<(counter > 0)> <---- actual for-loop + * aux_func { + * startC = endI*endJ - counter + * startI = (startC)/(endJ) + * startJ = (startC)%(endJ) + * func(startI, startJ) + * aux_func() + * } + * + * Note: Additional wrapper functions are provided for aux_func which hides the counter template + * parameter since counter usually depends on endI, endJ, etc... + * + * Conventions: + * 1) endX: specifies the terminal value for the for-loop, (ex: for(startX = 0; startX < endX; startX++)) + * + * 2) rem, remM, remK template parameters are used for deciding whether to use masked operations for + * handling remaining tails (when sizes are not multiples of PacketSize or EIGEN_AVX_MAX_NUM_ROW) + */ +namespace unrolls { + +template +EIGEN_ALWAYS_INLINE auto remMask(int64_t m) { + EIGEN_IF_CONSTEXPR(N == 16) { return 0xFFFF >> (16 - m); } + else EIGEN_IF_CONSTEXPR(N == 8) { + return 0xFF >> (8 - m); + } + else EIGEN_IF_CONSTEXPR(N == 4) { + return 0x0F >> (4 - m); + } + return 0; +} + +template +EIGEN_ALWAYS_INLINE void trans8x8blocks(PacketBlock &kernel); + +template <> +EIGEN_ALWAYS_INLINE void trans8x8blocks(PacketBlock &kernel) { + __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]); + __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]); + __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]); + __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]); + __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]); + __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]); + __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]); + __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]); + + kernel.packet[0] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2))); + kernel.packet[1] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2))); + kernel.packet[2] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3))); + kernel.packet[3] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3))); + kernel.packet[4] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6))); + kernel.packet[5] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6))); + kernel.packet[6] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7))); + kernel.packet[7] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7))); + + T0 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[4]), 0x4E)); + T0 = _mm512_mask_blend_ps(0xF0F0, kernel.packet[0], T0); + T4 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[0]), 0x4E)); + T4 = _mm512_mask_blend_ps(0xF0F0, T4, kernel.packet[4]); + T1 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[5]), 0x4E)); + T1 = _mm512_mask_blend_ps(0xF0F0, kernel.packet[1], T1); + T5 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[1]), 0x4E)); + T5 = _mm512_mask_blend_ps(0xF0F0, T5, kernel.packet[5]); + T2 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[6]), 0x4E)); + T2 = _mm512_mask_blend_ps(0xF0F0, kernel.packet[2], T2); + T6 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[2]), 0x4E)); + T6 = _mm512_mask_blend_ps(0xF0F0, T6, kernel.packet[6]); + T3 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[7]), 0x4E)); + T3 = _mm512_mask_blend_ps(0xF0F0, kernel.packet[3], T3); + T7 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[3]), 0x4E)); + T7 = _mm512_mask_blend_ps(0xF0F0, T7, kernel.packet[7]); + + kernel.packet[0] = T0; + kernel.packet[1] = T1; + kernel.packet[2] = T2; + kernel.packet[3] = T3; + kernel.packet[4] = T4; + kernel.packet[5] = T5; + kernel.packet[6] = T6; + kernel.packet[7] = T7; +} + +template <> +EIGEN_ALWAYS_INLINE void trans8x8blocks(PacketBlock &kernel) { + ptranspose(kernel); +} + +/*** + * Unrolls for tranposed C stores + */ +template +class trans { + public: + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + using vecHalf = typename std::conditional::value, vecHalfFloat, vecFullDouble>::type; + static constexpr int64_t PacketSize = packet_traits::size; + + /*********************************** + * Auxillary Functions for: + * - storeC + *********************************** + */ + + /** + * aux_storeC + * + * 1-D unroll + * for(startN = 0; startN < endN; startN++) + * + * (endN <= PacketSize) is required to handle the fp32 case, see comments in transStoreC + * + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0 && endN <= PacketSize)> aux_storeC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t remM_ = 0) { + constexpr int64_t counterReverse = endN - counter; + constexpr int64_t startN = counterReverse; + + EIGEN_IF_CONSTEXPR(startN < EIGEN_AVX_MAX_NUM_ROW) { + EIGEN_IF_CONSTEXPR(remM) { + pstoreu( + C_arr + LDC * startN, + padd(ploadu((const Scalar *)C_arr + LDC * startN, remMask(remM_)), + preinterpret(zmm.packet[packetIndexOffset + (unrollN / PacketSize) * startN]), + remMask(remM_)), + remMask(remM_)); + } + else { + pstoreu(C_arr + LDC * startN, + padd(ploadu((const Scalar *)C_arr + LDC * startN), + preinterpret(zmm.packet[packetIndexOffset + (unrollN / PacketSize) * startN]))); + } + } + else { // This block is only needed for fp32 case + // Reinterpret as __m512 for _mm512_shuffle_f32x4 + vecFullFloat zmm2vecFullFloat = preinterpret( + zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)]); + // Swap lower and upper half of avx register. + zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)] = + preinterpret(_mm512_shuffle_f32x4(zmm2vecFullFloat, zmm2vecFullFloat, 0b01001110)); + + EIGEN_IF_CONSTEXPR(remM) { + pstoreu( + C_arr + LDC * startN, + padd(ploadu((const Scalar *)C_arr + LDC * startN, remMask(remM_)), + preinterpret( + zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)])), + remMask(remM_)); + } + else { + pstoreu( + C_arr + LDC * startN, + padd(ploadu((const Scalar *)C_arr + LDC * startN), + preinterpret( + zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)]))); + } + } + aux_storeC(C_arr, LDC, zmm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t 0 && endN <= PacketSize)> aux_storeC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t remM_ = 0) { + EIGEN_UNUSED_VARIABLE(C_arr); + EIGEN_UNUSED_VARIABLE(LDC); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(remM_); + } + + template + static EIGEN_ALWAYS_INLINE void storeC(Scalar *C_arr, int64_t LDC, + PacketBlock &zmm, + int64_t remM_ = 0) { + aux_storeC(C_arr, LDC, zmm, remM_); + } + + /** + * Transposes LxunrollN row major block of matrices stored EIGEN_AVX_MAX_NUM_ACC zmm registers to + * "unrollN"xL ymm registers to be stored col-major into C. + * + * For 8x48, the 8x48 block (row-major) is stored in zmm as follows: + * + * row0: zmm0 zmm1 zmm2 + * row1: zmm3 zmm4 zmm5 + * . + * . + * row7: zmm21 zmm22 zmm23 + * + * For 8x32, the 8x32 block (row-major) is stored in zmm as follows: + * + * row0: zmm0 zmm1 + * row1: zmm2 zmm3 + * . + * . + * row7: zmm14 zmm15 + * + * + * In general we will have {1,2,3} groups of avx registers each of size + * EIGEN_AVX_MAX_NUM_ROW. packetIndexOffset is used to select which "block" of + * avx registers are being transposed. + */ + template + static EIGEN_ALWAYS_INLINE void transpose(PacketBlock &zmm) { + // Note: this assumes EIGEN_AVX_MAX_NUM_ROW = 8. Unrolls should be adjusted + // accordingly if EIGEN_AVX_MAX_NUM_ROW is smaller. + constexpr int64_t zmmStride = unrollN / PacketSize; + PacketBlock r; + r.packet[0] = zmm.packet[packetIndexOffset + zmmStride * 0]; + r.packet[1] = zmm.packet[packetIndexOffset + zmmStride * 1]; + r.packet[2] = zmm.packet[packetIndexOffset + zmmStride * 2]; + r.packet[3] = zmm.packet[packetIndexOffset + zmmStride * 3]; + r.packet[4] = zmm.packet[packetIndexOffset + zmmStride * 4]; + r.packet[5] = zmm.packet[packetIndexOffset + zmmStride * 5]; + r.packet[6] = zmm.packet[packetIndexOffset + zmmStride * 6]; + r.packet[7] = zmm.packet[packetIndexOffset + zmmStride * 7]; + trans8x8blocks(r); + zmm.packet[packetIndexOffset + zmmStride * 0] = r.packet[0]; + zmm.packet[packetIndexOffset + zmmStride * 1] = r.packet[1]; + zmm.packet[packetIndexOffset + zmmStride * 2] = r.packet[2]; + zmm.packet[packetIndexOffset + zmmStride * 3] = r.packet[3]; + zmm.packet[packetIndexOffset + zmmStride * 4] = r.packet[4]; + zmm.packet[packetIndexOffset + zmmStride * 5] = r.packet[5]; + zmm.packet[packetIndexOffset + zmmStride * 6] = r.packet[6]; + zmm.packet[packetIndexOffset + zmmStride * 7] = r.packet[7]; + } +}; + +/** + * Unrolls for copyBToRowMajor + * + * Idea: + * 1) Load a block of right-hand sides to registers (using loadB). + * 2) Convert the block from column-major to row-major (transposeLxL) + * 3) Store the blocks from register either to a temp array (toTemp == true), or back to B (toTemp == false). + * + * We use at most EIGEN_AVX_MAX_NUM_ACC avx registers to store the blocks of B. The remaining registers are + * used as temps for transposing. + * + * Blocks will be of size Lx{U1,U2,U3}. packetIndexOffset is used to index between these subblocks + * For fp32, PacketSize = 2*EIGEN_AVX_MAX_NUM_ROW, so we reinterpret packets as packets half the size (zmm -> ymm). + */ +template +class transB { + public: + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + using vecHalf = typename std::conditional::value, vecHalfFloat, vecFullDouble>::type; + static constexpr int64_t PacketSize = packet_traits::size; + + /*********************************** + * Auxillary Functions for: + * - loadB + * - storeB + * - loadBBlock + * - storeBBlock + *********************************** + */ + + /** + * aux_loadB + * + * 1-D unroll + * for(startN = 0; startN < endN; startN++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_loadB( + Scalar *B_arr, int64_t LDB, PacketBlock &ymm, + int64_t remM_ = 0) { + constexpr int64_t counterReverse = endN - counter; + constexpr int64_t startN = counterReverse; + + EIGEN_IF_CONSTEXPR(remM) { + ymm.packet[packetIndexOffset + startN] = + ploadu((const Scalar *)&B_arr[startN * LDB], remMask(remM_)); + } + else { + EIGEN_IF_CONSTEXPR(remN_ == 0) { + ymm.packet[packetIndexOffset + startN] = ploadu((const Scalar *)&B_arr[startN * LDB]); + } + else ymm.packet[packetIndexOffset + startN] = + ploadu((const Scalar *)&B_arr[startN * LDB], remMask(remN_)); + } + + aux_loadB(B_arr, LDB, ymm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_loadB( + Scalar *B_arr, int64_t LDB, PacketBlock &ymm, + int64_t remM_ = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(ymm); + EIGEN_UNUSED_VARIABLE(remM_); + } + + /** + * aux_storeB + * + * 1-D unroll + * for(startN = 0; startN < endN; startN++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_storeB( + Scalar *B_arr, int64_t LDB, PacketBlock &ymm, int64_t rem_ = 0) { + constexpr int64_t counterReverse = endN - counter; + constexpr int64_t startN = counterReverse; + + EIGEN_IF_CONSTEXPR(remK || remM) { + pstoreu(&B_arr[startN * LDB], ymm.packet[packetIndexOffset + startN], + remMask(rem_)); + } + else { + pstoreu(&B_arr[startN * LDB], ymm.packet[packetIndexOffset + startN]); + } + + aux_storeB(B_arr, LDB, ymm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_storeB( + Scalar *B_arr, int64_t LDB, PacketBlock &ymm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(ymm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /** + * aux_loadBBlock + * + * 1-D unroll + * for(startN = 0; startN < endN; startN += EIGEN_AVX_MAX_NUM_ROW) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_loadBBlock( + Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, int64_t remM_ = 0) { + constexpr int64_t counterReverse = endN - counter; + constexpr int64_t startN = counterReverse; + transB::template loadB(&B_temp[startN], LDB_, ymm); + aux_loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_loadBBlock( + Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, int64_t remM_ = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(B_temp); + EIGEN_UNUSED_VARIABLE(LDB_); + EIGEN_UNUSED_VARIABLE(ymm); + EIGEN_UNUSED_VARIABLE(remM_); + } + + /** + * aux_storeBBlock + * + * 1-D unroll + * for(startN = 0; startN < endN; startN += EIGEN_AVX_MAX_NUM_ROW) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_storeBBlock( + Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, int64_t remM_ = 0) { + constexpr int64_t counterReverse = endN - counter; + constexpr int64_t startN = counterReverse; + + EIGEN_IF_CONSTEXPR(toTemp) { + transB::template storeB(&B_temp[startN], LDB_, ymm, remK_); + } + else { + transB::template storeB(&B_arr[0 + startN * LDB], LDB, + ymm, remM_); + } + aux_storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_storeBBlock( + Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, int64_t remM_ = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(B_temp); + EIGEN_UNUSED_VARIABLE(LDB_); + EIGEN_UNUSED_VARIABLE(ymm); + EIGEN_UNUSED_VARIABLE(remM_); + } + + /******************************************************** + * Wrappers for aux_XXXX to hide counter parameter + ********************************************************/ + + template + static EIGEN_ALWAYS_INLINE void loadB(Scalar *B_arr, int64_t LDB, + PacketBlock &ymm, + int64_t remM_ = 0) { + aux_loadB(B_arr, LDB, ymm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE void storeB(Scalar *B_arr, int64_t LDB, + PacketBlock &ymm, + int64_t rem_ = 0) { + aux_storeB(B_arr, LDB, ymm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE void loadBBlock(Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, + int64_t remM_ = 0) { + EIGEN_IF_CONSTEXPR(toTemp) { transB::template loadB(&B_arr[0], LDB, ymm, remM_); } + else { + aux_loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + } + + template + static EIGEN_ALWAYS_INLINE void storeBBlock(Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, + int64_t remM_ = 0) { + aux_storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE void transposeLxL(PacketBlock &ymm) { + // Note: this assumes EIGEN_AVX_MAX_NUM_ROW = 8. Unrolls should be adjusted + // accordingly if EIGEN_AVX_MAX_NUM_ROW is smaller. + PacketBlock r; + r.packet[0] = ymm.packet[packetIndexOffset + 0]; + r.packet[1] = ymm.packet[packetIndexOffset + 1]; + r.packet[2] = ymm.packet[packetIndexOffset + 2]; + r.packet[3] = ymm.packet[packetIndexOffset + 3]; + r.packet[4] = ymm.packet[packetIndexOffset + 4]; + r.packet[5] = ymm.packet[packetIndexOffset + 5]; + r.packet[6] = ymm.packet[packetIndexOffset + 6]; + r.packet[7] = ymm.packet[packetIndexOffset + 7]; + ptranspose(r); + ymm.packet[packetIndexOffset + 0] = r.packet[0]; + ymm.packet[packetIndexOffset + 1] = r.packet[1]; + ymm.packet[packetIndexOffset + 2] = r.packet[2]; + ymm.packet[packetIndexOffset + 3] = r.packet[3]; + ymm.packet[packetIndexOffset + 4] = r.packet[4]; + ymm.packet[packetIndexOffset + 5] = r.packet[5]; + ymm.packet[packetIndexOffset + 6] = r.packet[6]; + ymm.packet[packetIndexOffset + 7] = r.packet[7]; + } + + template + static EIGEN_ALWAYS_INLINE void transB_kernel(Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, + int64_t remM_ = 0) { + constexpr int64_t U3 = PacketSize * 3; + constexpr int64_t U2 = PacketSize * 2; + constexpr int64_t U1 = PacketSize * 1; + /** + * Unrolls needed for each case: + * - AVX512 fp32 48 32 16 8 4 2 1 + * - AVX512 fp64 24 16 8 4 2 1 + * + * For fp32 L and U1 are 1:2 so for U3/U2 cases the loads/stores need to be split up. + */ + EIGEN_IF_CONSTEXPR(unrollN == U3) { + // load LxU3 B col major, transpose LxU3 row major + constexpr int64_t maxUBlock = std::min(3 * EIGEN_AVX_MAX_NUM_ROW, U3); + transB::template loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + + EIGEN_IF_CONSTEXPR(maxUBlock < U3) { + transB::template loadBBlock(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_, + ymm, remM_); + transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template storeBBlock(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_, + ymm, remM_); + } + } + else EIGEN_IF_CONSTEXPR(unrollN == U2) { + // load LxU2 B col major, transpose LxU2 row major + constexpr int64_t maxUBlock = std::min(3 * EIGEN_AVX_MAX_NUM_ROW, U2); + transB::template loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + EIGEN_IF_CONSTEXPR(maxUBlock < U2) transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + + EIGEN_IF_CONSTEXPR(maxUBlock < U2) { + transB::template loadBBlock(&B_arr[maxUBlock * LDB], LDB, + &B_temp[maxUBlock], LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + transB::template storeBBlock(&B_arr[maxUBlock * LDB], LDB, + &B_temp[maxUBlock], LDB_, ymm, remM_); + } + } + else EIGEN_IF_CONSTEXPR(unrollN == U1) { + // load LxU1 B col major, transpose LxU1 row major + transB::template loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + EIGEN_IF_CONSTEXPR(EIGEN_AVX_MAX_NUM_ROW < U1) { transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); } + transB::template storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + else EIGEN_IF_CONSTEXPR(unrollN == 8 && U1 > 8) { + // load Lx4 B col major, transpose Lx4 row major + transB::template loadBBlock<8, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + transB::template storeBBlock<8, toTemp, remM, 8>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + else EIGEN_IF_CONSTEXPR(unrollN == 4 && U1 > 4) { + // load Lx4 B col major, transpose Lx4 row major + transB::template loadBBlock<4, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + transB::template storeBBlock<4, toTemp, remM, 4>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + else EIGEN_IF_CONSTEXPR(unrollN == 2) { + // load Lx2 B col major, transpose Lx2 row major + transB::template loadBBlock<2, toTemp, remM, 2>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + transB::template storeBBlock<2, toTemp, remM, 2>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + else EIGEN_IF_CONSTEXPR(unrollN == 1) { + // load Lx1 B col major, transpose Lx1 row major + transB::template loadBBlock<1, toTemp, remM, 1>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + transB::template storeBBlock<1, toTemp, remM, 1>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + } +}; + +/** + * Unrolls for triSolveKernel + * + * Idea: + * 1) Load a block of right-hand sides to registers in RHSInPacket (using loadRHS). + * 2) Do triangular solve with RHSInPacket and a small block of A (triangular matrix) + * stored in AInPacket (using triSolveMicroKernel). + * 3) Store final results (in avx registers) back into memory (using storeRHS). + * + * RHSInPacket uses at most EIGEN_AVX_MAX_NUM_ACC avx registers and AInPacket uses at most + * EIGEN_AVX_MAX_NUM_ROW registers. + */ +template +class trsm { + public: + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + static constexpr int64_t PacketSize = packet_traits::size; + + /*********************************** + * Auxillary Functions for: + * - loadRHS + * - storeRHS + * - divRHSByDiag + * - updateRHS + * - triSolveMicroKernel + ************************************/ + /** + * aux_loadRHS + * + * 2-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startK = 0; startK < endK; startK++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_loadRHS( + Scalar *B_arr, int64_t LDB, PacketBlock &RHSInPacket, int64_t rem = 0) { + constexpr int64_t counterReverse = endM * endK - counter; + constexpr int64_t startM = counterReverse / (endK); + constexpr int64_t startK = counterReverse % endK; + + constexpr int64_t packetIndex = startM * endK + startK; + constexpr int64_t startM_ = isFWDSolve ? startM : -startM; + const int64_t rhsIndex = (startK * PacketSize) + startM_ * LDB; + EIGEN_IF_CONSTEXPR(krem) { + RHSInPacket.packet[packetIndex] = ploadu(&B_arr[rhsIndex], remMask(rem)); + } + else { + RHSInPacket.packet[packetIndex] = ploadu(&B_arr[rhsIndex]); + } + aux_loadRHS(B_arr, LDB, RHSInPacket, rem); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_loadRHS( + Scalar *B_arr, int64_t LDB, PacketBlock &RHSInPacket, int64_t rem = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(RHSInPacket); + EIGEN_UNUSED_VARIABLE(rem); + } + + /** + * aux_storeRHS + * + * 2-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startK = 0; startK < endK; startK++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_storeRHS( + Scalar *B_arr, int64_t LDB, PacketBlock &RHSInPacket, int64_t rem = 0) { + constexpr int64_t counterReverse = endM * endK - counter; + constexpr int64_t startM = counterReverse / (endK); + constexpr int64_t startK = counterReverse % endK; + + constexpr int64_t packetIndex = startM * endK + startK; + constexpr int64_t startM_ = isFWDSolve ? startM : -startM; + const int64_t rhsIndex = (startK * PacketSize) + startM_ * LDB; + EIGEN_IF_CONSTEXPR(krem) { + pstoreu(&B_arr[rhsIndex], RHSInPacket.packet[packetIndex], remMask(rem)); + } + else { + pstoreu(&B_arr[rhsIndex], RHSInPacket.packet[packetIndex]); + } + aux_storeRHS(B_arr, LDB, RHSInPacket, rem); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_storeRHS( + Scalar *B_arr, int64_t LDB, PacketBlock &RHSInPacket, int64_t rem = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(RHSInPacket); + EIGEN_UNUSED_VARIABLE(rem); + } + + /** + * aux_divRHSByDiag + * + * currM may be -1, (currM >=0) in enable_if checks for this + * + * 1-D unroll + * for(startK = 0; startK < endK; startK++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0 && currM >= 0)> aux_divRHSByDiag( + PacketBlock &RHSInPacket, PacketBlock &AInPacket) { + constexpr int64_t counterReverse = endK - counter; + constexpr int64_t startK = counterReverse; + + constexpr int64_t packetIndex = currM * endK + startK; + RHSInPacket.packet[packetIndex] = pmul(AInPacket.packet[currM], RHSInPacket.packet[packetIndex]); + aux_divRHSByDiag(RHSInPacket, AInPacket); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t 0 && currM >= 0)> aux_divRHSByDiag( + PacketBlock &RHSInPacket, PacketBlock &AInPacket) { + EIGEN_UNUSED_VARIABLE(RHSInPacket); + EIGEN_UNUSED_VARIABLE(AInPacket); + } + + /** + * aux_updateRHS + * + * 2-D unroll + * for(startM = initM; startM < endM; startM++) + * for(startK = 0; startK < endK; startK++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_updateRHS( + Scalar *A_arr, int64_t LDA, PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + constexpr int64_t counterReverse = (endM - initM) * endK - counter; + constexpr int64_t startM = initM + counterReverse / (endK); + constexpr int64_t startK = counterReverse % endK; + + // For each row of A, first update all corresponding RHS + constexpr int64_t packetIndex = startM * endK + startK; + EIGEN_IF_CONSTEXPR(currentM > 0) { + RHSInPacket.packet[packetIndex] = + pnmadd(AInPacket.packet[startM], RHSInPacket.packet[(currentM - 1) * endK + startK], + RHSInPacket.packet[packetIndex]); + } + + EIGEN_IF_CONSTEXPR(startK == endK - 1) { + // Once all RHS for previous row of A is updated, we broadcast the next element in the column A_{i, currentM}. + EIGEN_IF_CONSTEXPR(startM == currentM && !isUnitDiag) { + // If diagonal is not unit, we broadcast reciprocals of diagonals AinPacket.packet[currentM]. + // This will be used in divRHSByDiag + EIGEN_IF_CONSTEXPR(isFWDSolve) + AInPacket.packet[currentM] = pset1(Scalar(1) / A_arr[idA(currentM, currentM, LDA)]); + else AInPacket.packet[currentM] = pset1(Scalar(1) / A_arr[idA(-currentM, -currentM, LDA)]); + } + else { + // Broadcast next off diagonal element of A + EIGEN_IF_CONSTEXPR(isFWDSolve) + AInPacket.packet[startM] = pset1(A_arr[idA(startM, currentM, LDA)]); + else AInPacket.packet[startM] = pset1(A_arr[idA(-startM, -currentM, LDA)]); + } + } + + aux_updateRHS( + A_arr, LDA, RHSInPacket, AInPacket); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_updateRHS( + Scalar *A_arr, int64_t LDA, PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + EIGEN_UNUSED_VARIABLE(A_arr); + EIGEN_UNUSED_VARIABLE(LDA); + EIGEN_UNUSED_VARIABLE(RHSInPacket); + EIGEN_UNUSED_VARIABLE(AInPacket); + } + + /** + * aux_triSolverMicroKernel + * + * 1-D unroll + * for(startM = 0; startM < endM; startM++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_triSolveMicroKernel( + Scalar *A_arr, int64_t LDA, PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + constexpr int64_t counterReverse = endM - counter; + constexpr int64_t startM = counterReverse; + + constexpr int64_t currentM = startM; + // Divides the right-hand side in row startM, by digonal value of A + // broadcasted to AInPacket.packet[startM-1] in the previous iteration. + // + // Without "if constexpr" the compiler instantiates the case <-1, numK> + // this is handled with enable_if to prevent out-of-bound warnings + // from the compiler + EIGEN_IF_CONSTEXPR(!isUnitDiag && startM > 0) + trsm::template divRHSByDiag(RHSInPacket, AInPacket); + + // After division, the rhs corresponding to subsequent rows of A can be partially updated + // We also broadcast the reciprocal of the next diagonal to AInPacket.packet[currentM] (if needed) + // to be used in the next iteration. + trsm::template updateRHS(A_arr, LDA, RHSInPacket, + AInPacket); + + // Handle division for the RHS corresponding to the final row of A. + EIGEN_IF_CONSTEXPR(!isUnitDiag && startM == endM - 1) + trsm::template divRHSByDiag(RHSInPacket, AInPacket); + + aux_triSolveMicroKernel(A_arr, LDA, RHSInPacket, + AInPacket); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_triSolveMicroKernel( + Scalar *A_arr, int64_t LDA, PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + EIGEN_UNUSED_VARIABLE(A_arr); + EIGEN_UNUSED_VARIABLE(LDA); + EIGEN_UNUSED_VARIABLE(RHSInPacket); + EIGEN_UNUSED_VARIABLE(AInPacket); + } + + /******************************************************** + * Wrappers for aux_XXXX to hide counter parameter + ********************************************************/ + + /** + * Load endMxendK block of B to RHSInPacket + * Masked loads are used for cases where endK is not a multiple of PacketSize + */ + template + static EIGEN_ALWAYS_INLINE void loadRHS(Scalar *B_arr, int64_t LDB, + PacketBlock &RHSInPacket, int64_t rem = 0) { + aux_loadRHS(B_arr, LDB, RHSInPacket, rem); + } + + /** + * Load endMxendK block of B to RHSInPacket + * Masked loads are used for cases where endK is not a multiple of PacketSize + */ + template + static EIGEN_ALWAYS_INLINE void storeRHS(Scalar *B_arr, int64_t LDB, + PacketBlock &RHSInPacket, int64_t rem = 0) { + aux_storeRHS(B_arr, LDB, RHSInPacket, rem); + } + + /** + * Only used if Triangular matrix has non-unit diagonal values + */ + template + static EIGEN_ALWAYS_INLINE void divRHSByDiag(PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + aux_divRHSByDiag(RHSInPacket, AInPacket); + } + + /** + * Update right-hand sides (stored in avx registers) + * Traversing along the column A_{i,currentM}, where currentM <= i <= endM, and broadcasting each value to AInPacket. + **/ + template + static EIGEN_ALWAYS_INLINE void updateRHS(Scalar *A_arr, int64_t LDA, + PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + aux_updateRHS( + A_arr, LDA, RHSInPacket, AInPacket); + } + + /** + * endM: dimension of A. 1 <= endM <= EIGEN_AVX_MAX_NUM_ROW + * numK: number of avx registers to use for each row of B (ex fp32: 48 rhs => 3 avx reg used). 1 <= endK <= 3. + * isFWDSolve: true => forward substitution, false => backwards substitution + * isUnitDiag: true => triangular matrix has unit diagonal. + */ + template + static EIGEN_ALWAYS_INLINE void triSolveMicroKernel(Scalar *A_arr, int64_t LDA, + PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + static_assert(numK >= 1 && numK <= 3, "numK out of range"); + aux_triSolveMicroKernel(A_arr, LDA, RHSInPacket, AInPacket); + } +}; + +/** + * Unrolls for gemm kernel + * + * isAdd: true => C += A*B, false => C -= A*B + */ +template +class gemm { + public: + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + static constexpr int64_t PacketSize = packet_traits::size; + + /*********************************** + * Auxillary Functions for: + * - setzero + * - updateC + * - storeC + * - startLoadB + * - triSolveMicroKernel + ************************************/ + + /** + * aux_setzero + * + * 2-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startN = 0; startN < endN; startN++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_setzero( + PacketBlock &zmm) { + constexpr int64_t counterReverse = endM * endN - counter; + constexpr int64_t startM = counterReverse / (endN); + constexpr int64_t startN = counterReverse % endN; + + zmm.packet[startN * endM + startM] = pzero(zmm.packet[startN * endM + startM]); + aux_setzero(zmm); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_setzero( + PacketBlock &zmm) { + EIGEN_UNUSED_VARIABLE(zmm); + } + + /** + * aux_updateC + * + * 2-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startN = 0; startN < endN; startN++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_updateC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + constexpr int64_t counterReverse = endM * endN - counter; + constexpr int64_t startM = counterReverse / (endN); + constexpr int64_t startN = counterReverse % endN; + + EIGEN_IF_CONSTEXPR(rem) + zmm.packet[startN * endM + startM] = + padd(ploadu(&C_arr[(startN)*LDC + startM * PacketSize], remMask(rem_)), + zmm.packet[startN * endM + startM], remMask(rem_)); + else zmm.packet[startN * endM + startM] = + padd(ploadu(&C_arr[(startN)*LDC + startM * PacketSize]), zmm.packet[startN * endM + startM]); + aux_updateC(C_arr, LDC, zmm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_updateC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(C_arr); + EIGEN_UNUSED_VARIABLE(LDC); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /** + * aux_storeC + * + * 2-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startN = 0; startN < endN; startN++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_storeC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + constexpr int64_t counterReverse = endM * endN - counter; + constexpr int64_t startM = counterReverse / (endN); + constexpr int64_t startN = counterReverse % endN; + + EIGEN_IF_CONSTEXPR(rem) + pstoreu(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM], + remMask(rem_)); + else pstoreu(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM]); + aux_storeC(C_arr, LDC, zmm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_storeC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(C_arr); + EIGEN_UNUSED_VARIABLE(LDC); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /** + * aux_startLoadB + * + * 1-D unroll + * for(startL = 0; startL < endL; startL++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_startLoadB( + Scalar *B_t, int64_t LDB, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + constexpr int64_t counterReverse = endL - counter; + constexpr int64_t startL = counterReverse; + + EIGEN_IF_CONSTEXPR(rem) + zmm.packet[unrollM * unrollN + startL] = + ploadu(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize], remMask(rem_)); + else zmm.packet[unrollM * unrollN + startL] = + ploadu(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize]); + + aux_startLoadB(B_t, LDB, zmm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_startLoadB( + Scalar *B_t, int64_t LDB, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(B_t); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /** + * aux_startBCastA + * + * 1-D unroll + * for(startB = 0; startB < endB; startB++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_startBCastA( + Scalar *A_t, int64_t LDA, PacketBlock &zmm) { + constexpr int64_t counterReverse = endB - counter; + constexpr int64_t startB = counterReverse; + + zmm.packet[unrollM * unrollN + numLoad + startB] = pload1(&A_t[idA(startB, 0, LDA)]); + + aux_startBCastA(A_t, LDA, zmm); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_startBCastA( + Scalar *A_t, int64_t LDA, PacketBlock &zmm) { + EIGEN_UNUSED_VARIABLE(A_t); + EIGEN_UNUSED_VARIABLE(LDA); + EIGEN_UNUSED_VARIABLE(zmm); + } + + /** + * aux_loadB + * currK: current K + * + * 1-D unroll + * for(startM = 0; startM < endM; startM++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_loadB( + Scalar *B_t, int64_t LDB, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + if ((numLoad / endM + currK < unrollK)) { + constexpr int64_t counterReverse = endM - counter; + constexpr int64_t startM = counterReverse; + + EIGEN_IF_CONSTEXPR(rem) { + zmm.packet[endM * unrollN + (startM + currK * endM) % numLoad] = + ploadu(&B_t[(numLoad / endM + currK) * LDB + startM * PacketSize], remMask(rem_)); + } + else { + zmm.packet[endM * unrollN + (startM + currK * endM) % numLoad] = + ploadu(&B_t[(numLoad / endM + currK) * LDB + startM * PacketSize]); + } + + aux_loadB(B_t, LDB, zmm, rem_); + } + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_loadB( + Scalar *B_t, int64_t LDB, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(B_t); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /** + * aux_microKernel + * + * 3-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startN = 0; startN < endN; startN++) + * for(startK = 0; startK < endK; startK++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_microKernel( + Scalar *B_t, Scalar *A_t, int64_t LDB, int64_t LDA, PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + constexpr int64_t counterReverse = endM * endN * endK - counter; + constexpr int startK = counterReverse / (endM * endN); + constexpr int startN = (counterReverse / (endM)) % endN; + constexpr int startM = counterReverse % endM; + + EIGEN_IF_CONSTEXPR(startK == 0 && startM == 0 && startN == 0) { + gemm::template startLoadB(B_t, LDB, zmm, rem_); + gemm::template startBCastA(A_t, LDA, zmm); + } + + { + // Interleave FMA and Bcast + EIGEN_IF_CONSTEXPR(isAdd) { + zmm.packet[startN * endM + startM] = + pmadd(zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast], + zmm.packet[endM * endN + (startM + startK * endM) % numLoad], zmm.packet[startN * endM + startM]); + } + else { + zmm.packet[startN * endM + startM] = + pnmadd(zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast], + zmm.packet[endM * endN + (startM + startK * endM) % numLoad], zmm.packet[startN * endM + startM]); + } + // Bcast + EIGEN_IF_CONSTEXPR(startM == endM - 1 && (numBCast + startN + startK * endN < endK * endN)) { + zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast] = pload1(&A_t[idA( + (numBCast + startN + startK * endN) % endN, (numBCast + startN + startK * endN) / endN, LDA)]); + } + } + + // We have updated all accumlators, time to load next set of B's + EIGEN_IF_CONSTEXPR((startN == endN - 1) && (startM == endM - 1)) { + gemm::template loadB(B_t, LDB, zmm, rem_); + } + aux_microKernel(B_t, A_t, LDB, LDA, zmm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_microKernel( + Scalar *B_t, Scalar *A_t, int64_t LDB, int64_t LDA, PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(B_t); + EIGEN_UNUSED_VARIABLE(A_t); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(LDA); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /******************************************************** + * Wrappers for aux_XXXX to hide counter parameter + ********************************************************/ + + template + static EIGEN_ALWAYS_INLINE void setzero(PacketBlock &zmm) { + aux_setzero(zmm); + } + + /** + * Ideally the compiler folds these into vaddp{s,d} with an embedded memory load. + */ + template + static EIGEN_ALWAYS_INLINE void updateC(Scalar *C_arr, int64_t LDC, + PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + aux_updateC(C_arr, LDC, zmm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE void storeC(Scalar *C_arr, int64_t LDC, + PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + aux_storeC(C_arr, LDC, zmm, rem_); + } + + /** + * Use numLoad registers for loading B at start of microKernel + */ + template + static EIGEN_ALWAYS_INLINE void startLoadB(Scalar *B_t, int64_t LDB, + PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + aux_startLoadB(B_t, LDB, zmm, rem_); + } + + /** + * Use numBCast registers for broadcasting A at start of microKernel + */ + template + static EIGEN_ALWAYS_INLINE void startBCastA(Scalar *A_t, int64_t LDA, + PacketBlock &zmm) { + aux_startBCastA(A_t, LDA, zmm); + } + + /** + * Loads next set of B into vector registers between each K unroll. + */ + template + static EIGEN_ALWAYS_INLINE void loadB(Scalar *B_t, int64_t LDB, + PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + aux_loadB(B_t, LDB, zmm, rem_); + } + + /** + * Generates a microkernel for gemm (row-major) with unrolls {1,2,4,8}x{U1,U2,U3} to compute C -= A*B. + * A matrix can be row/col-major. B matrix is assumed row-major. + * + * isARowMajor: is A row major + * endM: Number registers per row + * endN: Number of rows + * endK: Loop unroll for K. + * numLoad: Number of registers for loading B. + * numBCast: Number of registers for broadcasting A. + * + * Ex: microkernel: 8x48 unroll (24 accumulators), k unrolled 4 times, + * 6 register for loading B, 2 for broadcasting A. + * + * Note: Ideally the microkernel should not have any register spilling. + * The avx instruction counts should be: + * - endK*endN vbroadcasts{s,d} + * - endK*endM vmovup{s,d} + * - endK*endN*endM FMAs + * + * From testing, there are no register spills with clang. There are register spills with GNU, which + * causes a performance hit. + */ + template + static EIGEN_ALWAYS_INLINE void microKernel(Scalar *B_t, Scalar *A_t, int64_t LDB, int64_t LDA, + PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + aux_microKernel(B_t, A_t, LDB, LDA, zmm, + rem_); + } +}; +} // namespace unrolls + +#endif // EIGEN_CORE_ARCH_AVX512_TRSM_UNROLLS_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/TypeCasting.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/TypeCasting.h index 330412729ea..5053230353b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/TypeCasting.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AVX512/TypeCasting.h @@ -10,18 +10,79 @@ #ifndef EIGEN_TYPE_CASTING_AVX512_H #define EIGEN_TYPE_CASTING_AVX512_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; + +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; + +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; + +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; + +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; + +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; + +template<> EIGEN_STRONG_INLINE Packet16b pcast(const Packet16f& a) { + __mmask16 mask = _mm512_cmpneq_ps_mask(a, pzero(a)); + return _mm512_maskz_cvtepi32_epi8(mask, _mm512_set1_epi32(1)); +} + +template<> EIGEN_STRONG_INLINE Packet16f pcast(const Packet16b& a) { + return _mm512_cvtepi32_ps(_mm512_and_si512(_mm512_cvtepi8_epi32(a), _mm512_set1_epi32(1))); +} + template<> EIGEN_STRONG_INLINE Packet16i pcast(const Packet16f& a) { return _mm512_cvttps_epi32(a); } +template<> EIGEN_STRONG_INLINE Packet8d pcast(const Packet16f& a) { + return _mm512_cvtps_pd(_mm512_castps512_ps256(a)); +} + +template<> EIGEN_STRONG_INLINE Packet8d pcast(const Packet8f& a) { + return _mm512_cvtps_pd(a); +} + template<> EIGEN_STRONG_INLINE Packet16f pcast(const Packet16i& a) { return _mm512_cvtepi32_ps(a); } +template<> EIGEN_STRONG_INLINE Packet8d pcast(const Packet16i& a) { + return _mm512_cvtepi32_pd(_mm512_castsi512_si256(a)); +} + +template<> EIGEN_STRONG_INLINE Packet8d pcast(const Packet8i& a) { + return _mm512_cvtepi32_pd(a); +} + +template<> EIGEN_STRONG_INLINE Packet16f pcast(const Packet8d& a, const Packet8d& b) { + return cat256(_mm512_cvtpd_ps(a), _mm512_cvtpd_ps(b)); +} + +template<> EIGEN_STRONG_INLINE Packet16i pcast(const Packet8d& a, const Packet8d& b) { + return cat256i(_mm512_cvttpd_epi32(a), _mm512_cvttpd_epi32(b)); +} + +template<> EIGEN_STRONG_INLINE Packet8i pcast(const Packet8d& a) { + return _mm512_cvtpd_epi32(a); +} +template<> EIGEN_STRONG_INLINE Packet8f pcast(const Packet8d& a) { + return _mm512_cvtpd_ps(a); +} + template<> EIGEN_STRONG_INLINE Packet16i preinterpret(const Packet16f& a) { return _mm512_castps_si512(a); } @@ -30,58 +91,141 @@ template<> EIGEN_STRONG_INLINE Packet16f preinterpret(cons return _mm512_castsi512_ps(a); } -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; +template<> EIGEN_STRONG_INLINE Packet8d preinterpret(const Packet16f& a) { + return _mm512_castps_pd(a); +} + +template<> EIGEN_STRONG_INLINE Packet16f preinterpret(const Packet8d& a) { + return _mm512_castpd_ps(a); +} + +template<> EIGEN_STRONG_INLINE Packet8f preinterpret(const Packet16f& a) { + return _mm512_castps512_ps256(a); +} + +template<> EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet16f& a) { + return _mm512_castps512_ps128(a); +} + +template<> EIGEN_STRONG_INLINE Packet4d preinterpret(const Packet8d& a) { + return _mm512_castpd512_pd256(a); +} + +template<> EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet8d& a) { + return _mm512_castpd512_pd128(a); +} + +template<> EIGEN_STRONG_INLINE Packet16f preinterpret(const Packet8f& a) { + return _mm512_castps256_ps512(a); +} + +template<> EIGEN_STRONG_INLINE Packet16f preinterpret(const Packet4f& a) { + return _mm512_castps128_ps512(a); +} + +template<> EIGEN_STRONG_INLINE Packet8d preinterpret(const Packet4d& a) { + return _mm512_castpd256_pd512(a); +} + +template<> EIGEN_STRONG_INLINE Packet8d preinterpret(const Packet2d& a) { + return _mm512_castpd128_pd512(a); +} + +template<> EIGEN_STRONG_INLINE Packet8i preinterpret(const Packet16i& a) { + return _mm512_castsi512_si256(a); +} +template<> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet16i& a) { + return _mm512_castsi512_si128(a); +} + +template<> EIGEN_STRONG_INLINE Packet8h preinterpret(const Packet16h& a) { + return _mm256_castsi256_si128(a); +} + +template<> EIGEN_STRONG_INLINE Packet8bf preinterpret(const Packet16bf& a) { + return _mm256_castsi256_si128(a); +} + +#ifndef EIGEN_VECTORIZE_AVX512FP16 template<> EIGEN_STRONG_INLINE Packet16f pcast(const Packet16h& a) { return half2float(a); } -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; - template<> EIGEN_STRONG_INLINE Packet16h pcast(const Packet16f& a) { return float2half(a); } -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; +#endif template<> EIGEN_STRONG_INLINE Packet16f pcast(const Packet16bf& a) { return Bf16ToF32(a); } -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; - template<> EIGEN_STRONG_INLINE Packet16bf pcast(const Packet16f& a) { return F32ToBf16(a); } +#ifdef EIGEN_VECTORIZE_AVX512FP16 + +template<> EIGEN_STRONG_INLINE Packet16h preinterpret(const Packet32h& a) { + return _mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castph_pd(a), 0)); +} +template<> EIGEN_STRONG_INLINE Packet8h preinterpret(const Packet32h& a) { + return _mm256_castsi256_si128(preinterpret(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pcast(const Packet32h& a) { + // Discard second-half of input. + Packet16h low = _mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castph_pd(a), 0)); + return _mm512_cvtxph_ps(_mm256_castsi256_ph(low)); +} + + +template <> +EIGEN_STRONG_INLINE Packet32h pcast(const Packet16f& a, const Packet16f& b) { + __m512d result = _mm512_undefined_pd(); + result = _mm512_insertf64x4(result, _mm256_castsi256_pd(_mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC)), 0); + result = _mm512_insertf64x4(result, _mm256_castsi256_pd(_mm512_cvtps_ph(b, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC)), 1); + return _mm512_castpd_ph(result); +} + +template <> +EIGEN_STRONG_INLINE Packet8f pcast(const Packet16h& a) { + // Discard second-half of input. + Packet8h low = _mm_castps_si128(_mm256_extractf32x4_ps(_mm256_castsi256_ps(a), 0)); + return _mm256_cvtxph_ps(_mm_castsi128_ph(low)); +} + + +template <> +EIGEN_STRONG_INLINE Packet16h pcast(const Packet8f& a, const Packet8f& b) { + __m256d result = _mm256_undefined_pd(); + result = _mm256_insertf64x2(result, _mm_castsi128_pd(_mm256_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC)), 0); + result = _mm256_insertf64x2(result, _mm_castsi128_pd(_mm256_cvtps_ph(b, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC)), 1); + return _mm256_castpd_si256(result); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet8h& a) { + Packet8f full = _mm256_cvtxph_ps(_mm_castsi128_ph(a)); + // Discard second-half of input. + return _mm256_extractf32x4_ps(full, 0); +} + + +template <> +EIGEN_STRONG_INLINE Packet8h pcast(const Packet4f& a, const Packet4f& b) { + __m256 result = _mm256_undefined_ps(); + result = _mm256_insertf128_ps(result, a, 0); + result = _mm256_insertf128_ps(result, b, 1); + return _mm256_cvtps_ph(result, _MM_FROUND_TO_NEAREST_INT); +} + + +#endif + } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/Complex.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/Complex.h index b3932998c1b..915b01baaec 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/Complex.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/Complex.h @@ -11,12 +11,15 @@ #ifndef EIGEN_COMPLEX32_ALTIVEC_H #define EIGEN_COMPLEX32_ALTIVEC_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { static Packet4ui p4ui_CONJ_XOR = vec_mergeh((Packet4ui)p4i_ZERO, (Packet4ui)p4f_MZERO);//{ 0x00000000, 0x80000000, 0x00000000, 0x80000000 }; -#ifdef __VSX__ +#ifdef EIGEN_VECTORIZE_VSX #if defined(_BIG_ENDIAN) static Packet2ul p2ul_CONJ_XOR1 = (Packet2ul) vec_sld((Packet4ui) p2d_MZERO, (Packet4ui) p2l_ZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; static Packet2ul p2ul_CONJ_XOR2 = (Packet2ul) vec_sld((Packet4ui) p2l_ZERO, (Packet4ui) p2d_MZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; @@ -89,7 +92,6 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 2, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -100,7 +102,8 @@ template<> struct packet_traits > : default_packet_traits HasAbs2 = 0, HasMin = 0, HasMax = 0, -#ifdef __VSX__ + HasSqrt = 1, +#ifdef EIGEN_VECTORIZE_VSX HasBlend = 1, #endif HasSetLinear = 0 @@ -112,25 +115,44 @@ template<> struct unpacket_traits { typedef std::complex type; template<> EIGEN_STRONG_INLINE Packet2cf pset1(const std::complex& from) { Packet2cf res; +#ifdef EIGEN_VECTORIZE_VSX + // Load a single std::complex from memory and duplicate + // + // Using pload would read past the end of the reference in this case + // Using vec_xl_len + vec_splat, generates poor assembly + __asm__ ("lxvdsx %x0,%y1" : "=wa" (res.v) : "Z" (from)); +#else if((std::ptrdiff_t(&from) % 16) == 0) res.v = pload((const float *)&from); else res.v = ploadu((const float *)&from); res.v = vec_perm(res.v, res.v, p16uc_PSET64_HI); +#endif return res; } template<> EIGEN_STRONG_INLINE Packet2cf pload(const std::complex* from) { return Packet2cf(pload((const float *) from)); } template<> EIGEN_STRONG_INLINE Packet2cf ploadu(const std::complex* from) { return Packet2cf(ploadu((const float*) from)); } +template<> EIGEN_ALWAYS_INLINE Packet2cf pload_partial(const std::complex* from, const Index n, const Index offset) +{ + return Packet2cf(pload_partial((const float *) from, n * 2, offset * 2)); +} +template<> EIGEN_ALWAYS_INLINE Packet2cf ploadu_partial(const std::complex* from, const Index n, const Index offset) +{ + return Packet2cf(ploadu_partial((const float*) from, n * 2, offset * 2)); +} template<> EIGEN_STRONG_INLINE Packet2cf ploaddup(const std::complex* from) { return pset1(*from); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex * to, const Packet2cf& from) { pstore((float*)to, from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex * to, const Packet2cf& from) { pstoreu((float*)to, from.v); } +template<> EIGEN_ALWAYS_INLINE void pstore_partial >(std::complex * to, const Packet2cf& from, const Index n, const Index offset) { pstore_partial((float*)to, from.v, n * 2, offset * 2); } +template<> EIGEN_ALWAYS_INLINE void pstoreu_partial >(std::complex * to, const Packet2cf& from, const Index n, const Index offset) { pstoreu_partial((float*)to, from.v, n * 2, offset * 2); } EIGEN_STRONG_INLINE Packet2cf pload2(const std::complex& from0, const std::complex& from1) { Packet4f res0, res1; -#ifdef __VSX__ +#ifdef EIGEN_VECTORIZE_VSX + // Load two std::complex from memory and combine __asm__ ("lxsdx %x0,%y1" : "=wa" (res0) : "Z" (from0)); __asm__ ("lxsdx %x0,%y1" : "=wa" (res1) : "Z" (from1)); #ifdef _BIG_ENDIAN @@ -146,19 +168,46 @@ EIGEN_STRONG_INLINE Packet2cf pload2(const std::complex& from0, const std return Packet2cf(res0); } -template<> EIGEN_DEVICE_FUNC inline Packet2cf pgather, Packet2cf>(const std::complex* from, Index stride) +template<> EIGEN_ALWAYS_INLINE Packet2cf pload_ignore(const std::complex* from) +{ + Packet2cf res; + res.v = pload_ignore(reinterpret_cast(from)); + return res; +} + +template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pgather_complex_size2(const Scalar* from, Index stride, const Index n = 2) { - EIGEN_ALIGN16 std::complex af[2]; - af[0] = from[0*stride]; - af[1] = from[1*stride]; - return pload(af); + eigen_internal_assert(n <= unpacket_traits::size && "number of elements will gather past end of packet"); + EIGEN_ALIGN16 Scalar af[2]; + for (Index i = 0; i < n; i++) { + af[i] = from[i*stride]; + } + return pload_ignore(af); } -template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet2cf>(std::complex* to, const Packet2cf& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2cf pgather, Packet2cf>(const std::complex* from, Index stride) { - EIGEN_ALIGN16 std::complex af[2]; - pstore >((std::complex *) af, from); - to[0*stride] = af[0]; - to[1*stride] = af[1]; + return pgather_complex_size2, Packet2cf>(from, stride); +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2cf pgather_partial, Packet2cf>(const std::complex* from, Index stride, const Index n) +{ + return pgather_complex_size2, Packet2cf>(from, stride, n); +} +template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_complex_size2(Scalar* to, const Packet& from, Index stride, const Index n = 2) +{ + eigen_internal_assert(n <= unpacket_traits::size && "number of elements will scatter past end of packet"); + EIGEN_ALIGN16 Scalar af[2]; + pstore((Scalar *) af, from); + for (Index i = 0; i < n; i++) { + to[i*stride] = af[i]; + } +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter, Packet2cf>(std::complex* to, const Packet2cf& from, Index stride) +{ + pscatter_complex_size2, Packet2cf>(to, from, stride); +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial, Packet2cf>(std::complex* to, const Packet2cf& from, Index stride, const Index n) +{ + pscatter_complex_size2, Packet2cf>(to, from, stride, n); } template<> EIGEN_STRONG_INLINE Packet2cf padd(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(a.v + b.v); } @@ -184,7 +233,7 @@ template<> EIGEN_STRONG_INLINE std::complex pfirst(const Pack template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { Packet4f rev_a; - rev_a = vec_perm(a.v, a.v, p16uc_COMPLEX32_REV2); + rev_a = vec_sld(a.v, a.v, 8); return Packet2cf(rev_a); } @@ -210,10 +259,7 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) template<> EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { - // TODO optimize it for AltiVec - Packet2cf res = pmul(a, pconj(b)); - Packet4f s = pmul(b.v, b.v); - return Packet2cf(pdiv(res.v, padd(s, vec_perm(s, s, p16uc_COMPLEX32_REV)))); + return pdiv_complex(a, b); } template<> EIGEN_STRONG_INLINE Packet2cf pcplxflip(const Packet2cf& x) @@ -223,8 +269,13 @@ template<> EIGEN_STRONG_INLINE Packet2cf pcplxflip(const Packet2cf& x EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { +#ifdef EIGEN_VECTORIZE_VSX + Packet4f tmp = reinterpret_cast(vec_mergeh(reinterpret_cast(kernel.packet[0].v), reinterpret_cast(kernel.packet[1].v))); + kernel.packet[1].v = reinterpret_cast(vec_mergel(reinterpret_cast(kernel.packet[0].v), reinterpret_cast(kernel.packet[1].v))); +#else Packet4f tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_HI); kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO); +#endif kernel.packet[0].v = tmp; } @@ -233,7 +284,7 @@ template<> EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf& a, const Packe return Packet2cf(vec_and(eq, vec_perm(eq, eq, p16uc_COMPLEX32_REV))); } -#ifdef __VSX__ +#ifdef EIGEN_VECTORIZE_VSX template<> EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket, const Packet2cf& elsePacket) { Packet2cf result; result.v = reinterpret_cast(pblend(ifPacket, reinterpret_cast(thenPacket.v), reinterpret_cast(elsePacket.v))); @@ -247,7 +298,7 @@ template<> EIGEN_STRONG_INLINE Packet2cf psqrt(const Packet2cf& a) } //---------- double ---------- -#ifdef __VSX__ +#ifdef EIGEN_VECTORIZE_VSX struct Packet1cd { EIGEN_STRONG_INLINE Packet1cd() {} @@ -309,7 +360,6 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 0, size = 1, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -320,6 +370,7 @@ template<> struct packet_traits > : default_packet_traits HasAbs2 = 0, HasMin = 0, HasMax = 0, + HasSqrt = 1, HasSetLinear = 0 }; }; @@ -328,17 +379,35 @@ template<> struct unpacket_traits { typedef std::complex type template<> EIGEN_STRONG_INLINE Packet1cd pload (const std::complex* from) { return Packet1cd(pload((const double*)from)); } template<> EIGEN_STRONG_INLINE Packet1cd ploadu(const std::complex* from) { return Packet1cd(ploadu((const double*)from)); } +template<> EIGEN_ALWAYS_INLINE Packet1cd pload_partial(const std::complex* from, const Index n, const Index offset) +{ + return Packet1cd(pload_partial((const double*)from, n * 2, offset * 2)); +} +template<> EIGEN_ALWAYS_INLINE Packet1cd ploadu_partial(const std::complex* from, const Index n, const Index offset) +{ + return Packet1cd(ploadu_partial((const double*)from, n * 2, offset * 2)); +} template<> EIGEN_STRONG_INLINE void pstore >(std::complex * to, const Packet1cd& from) { pstore((double*)to, from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex * to, const Packet1cd& from) { pstoreu((double*)to, from.v); } +template<> EIGEN_ALWAYS_INLINE void pstore_partial >(std::complex * to, const Packet1cd& from, const Index n, const Index offset) { pstore_partial((double*)to, from.v, n * 2, offset * 2); } +template<> EIGEN_ALWAYS_INLINE void pstoreu_partial >(std::complex * to, const Packet1cd& from, const Index n, const Index offset) { pstoreu_partial((double*)to, from.v, n * 2, offset * 2); } template<> EIGEN_STRONG_INLINE Packet1cd pset1(const std::complex& from) { /* here we really have to use unaligned loads :( */ return ploadu(&from); } -template<> EIGEN_DEVICE_FUNC inline Packet1cd pgather, Packet1cd>(const std::complex* from, Index) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet1cd pgather, Packet1cd>(const std::complex* from, Index) +{ + return pload(from); +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet1cd pgather_partial, Packet1cd>(const std::complex* from, Index, const Index) { return pload(from); } -template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet1cd>(std::complex* to, const Packet1cd& from, Index) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter, Packet1cd>(std::complex* to, const Packet1cd& from, Index) +{ + pstore >(to, from); +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial, Packet1cd>(std::complex* to, const Packet1cd& from, Index, const Index) { pstore >(to, from); } @@ -359,7 +428,7 @@ template<> EIGEN_STRONG_INLINE void prefetch >(const std::c template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet1cd& a) { - EIGEN_ALIGN16 std::complex res[2]; + EIGEN_ALIGN16 std::complex res[1]; pstore >(res, a); return res[0]; @@ -375,10 +444,7 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) template<> EIGEN_STRONG_INLINE Packet1cd pdiv(const Packet1cd& a, const Packet1cd& b) { - // TODO optimize it for AltiVec - Packet1cd res = pmul(a,pconj(b)); - Packet2d s = pmul(b.v, b.v); - return Packet1cd(pdiv(res.v, padd(s, vec_perm(s, s, p16uc_REVERSE64)))); + return pdiv_complex(a, b); } EIGEN_STRONG_INLINE Packet1cd pcplxflip/**/(const Packet1cd& x) @@ -388,8 +454,8 @@ EIGEN_STRONG_INLINE Packet1cd pcplxflip/**/(const Packet1cd& x) EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { - Packet2d tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_HI); - kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO); + Packet2d tmp = vec_mergeh(kernel.packet[0].v, kernel.packet[1].v); + kernel.packet[1].v = vec_mergel(kernel.packet[0].v, kernel.packet[1].v); kernel.packet[0].v = tmp; } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MathFunctions.h index 3a7a329361e..a8a2309c617 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MathFunctions.h @@ -12,77 +12,71 @@ #ifndef EIGEN_MATH_FUNCTIONS_ALTIVEC_H #define EIGEN_MATH_FUNCTIONS_ALTIVEC_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f plog(const Packet4f& _x) -{ - return plog_float(_x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f pexp(const Packet4f& _x) -{ - return pexp_float(_x); -} +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet4f) +#ifdef EIGEN_VECTORIZE_VSX +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE(Packet2d) +#endif -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f psin(const Packet4f& _x) +#ifdef EIGEN_VECTORIZE_VSX +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet4f psqrt(const Packet4f& x) { - return psin_float(_x); + return vec_sqrt(x); } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f pcos(const Packet4f& _x) +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet2d psqrt(const Packet2d& x) { - return pcos_float(_x); + return vec_sqrt(x); } -#ifndef EIGEN_COMP_CLANG -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +#if !EIGEN_COMP_CLANG +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f prsqrt(const Packet4f& x) { - return vec_rsqrt(x); + return pset1(1.0f) / psqrt(x); +// vec_rsqrt returns different results from the generic version +// return vec_rsqrt(x); } -#endif -#ifdef __VSX__ -#ifndef EIGEN_COMP_CLANG -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d prsqrt(const Packet2d& x) { - return vec_rsqrt(x); + return pset1(1.0) / psqrt(x); +// vec_rsqrt returns different results from the generic version +// return vec_rsqrt(x); } + #endif -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f psqrt(const Packet4f& x) -{ - return vec_sqrt(x); +template<> EIGEN_STRONG_INLINE Packet8bf psqrt (const Packet8bf& a){ + BF16_TO_F32_UNARY_OP_WRAPPER(psqrt, a); } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet2d psqrt(const Packet2d& x) -{ - return vec_sqrt(x); +#if !EIGEN_COMP_CLANG +template<> EIGEN_STRONG_INLINE Packet8bf prsqrt (const Packet8bf& a){ + BF16_TO_F32_UNARY_OP_WRAPPER(prsqrt, a); } - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet2d pexp(const Packet2d& _x) +#endif +#else +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet4f psqrt(const Packet4f& x) { - return pexp_double(_x); + Packet4f a; + for (Index i = 0; i < packet_traits::size; i++) { + a[i] = numext::sqrt(x[i]); + } + return a; } #endif -// Hyperbolic Tangent function. -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f -ptanh(const Packet4f& x) { - return internal::generic_fast_tanh_float(x); -} - } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProduct.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProduct.h index 8feb88ea704..e9a930711e7 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProduct.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProduct.h @@ -15,34 +15,43 @@ #define EIGEN_ALTIVEC_USE_CUSTOM_PACK 1 #endif -#include "MatrixProductCommon.h" - -// Since LLVM doesn't support dynamic dispatching, force either always MMA or VSX -#if EIGEN_COMP_LLVM -#if !defined(EIGEN_ALTIVEC_DISABLE_MMA) && !defined(EIGEN_ALTIVEC_MMA_ONLY) -#ifdef __MMA__ -#define EIGEN_ALTIVEC_MMA_ONLY -#else -#define EIGEN_ALTIVEC_DISABLE_MMA +#if !defined(EIGEN_ALTIVEC_DISABLE_MMA) +#define EIGEN_ALTIVEC_DISABLE_MMA 0 #endif + +// Check for MMA builtin support. +#if !EIGEN_ALTIVEC_DISABLE_MMA && defined(__has_builtin) +#if __has_builtin(__builtin_mma_assemble_acc) + #define EIGEN_ALTIVEC_MMA_SUPPORT #endif #endif -#ifdef __has_builtin -#if __has_builtin(__builtin_mma_assemble_acc) - #define ALTIVEC_MMA_SUPPORT +// Check if and how we should actually use MMA if supported. +#if defined(EIGEN_ALTIVEC_MMA_SUPPORT) + +#if !defined(EIGEN_ALTIVEC_ENABLE_MMA_DYNAMIC_DISPATCH) +#define EIGEN_ALTIVEC_ENABLE_MMA_DYNAMIC_DISPATCH 0 #endif + +// Check if we want to enable dynamic dispatch. Not supported by LLVM. +#if EIGEN_ALTIVEC_ENABLE_MMA_DYNAMIC_DISPATCH && !EIGEN_COMP_LLVM +#define EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH 1 +// Otherwise, use MMA by default if available. +#elif defined(__MMA__) +#define EIGEN_ALTIVEC_MMA_ONLY 1 #endif -#if defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) +#endif // EIGEN_ALTIVEC_MMA_SUPPORT + +#include "MatrixProductCommon.h" + +#if defined(EIGEN_ALTIVEC_MMA_ONLY) || defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH) #include "MatrixProductMMA.h" #endif -/************************************************************************************************** - * TODO * - * - Check StorageOrder on dhs_pack (the innermost second loop seems unvectorized when it could). * - * - Check the possibility of transposing as GETREAL and GETIMAG when needed. * - **************************************************************************************************/ +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -78,6 +87,20 @@ struct quad_traits }; }; +template<> +struct quad_traits +{ + typedef Packet8bf vectortype; + typedef PacketBlock type; + typedef vectortype rhstype; + enum + { + vectorsize = packet_traits::size, + size = 8, + rows = 4 + }; +}; + // MatrixProduct decomposes real/imaginary vectors into a real vector and an imaginary vector, this turned out // to be faster than Eigen's usual approach of having real/imaginary pairs on a single vector. This constants then // are responsible to extract from convert between Eigen's and MatrixProduct approach. @@ -91,12 +114,16 @@ const static Packet16uc p16uc_GETIMAG32 = { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31}; -const static Packet16uc p16uc_GETREAL64 = { 0, 1, 2, 3, 4, 5, 6, 7, - 16, 17, 18, 19, 20, 21, 22, 23}; -//[a,ai],[b,bi] = [ai,bi] -const static Packet16uc p16uc_GETIMAG64 = { 8, 9, 10, 11, 12, 13, 14, 15, - 24, 25, 26, 27, 28, 29, 30, 31}; +const static Packet16uc p16uc_GETREAL32b = { 0, 1, 2, 3, + 16, 17, 18, 19, + 8, 9, 10, 11, + 24, 25, 26, 27}; + +const static Packet16uc p16uc_GETIMAG32b = { 4, 5, 6, 7, + 20, 21, 22, 23, + 12, 13, 14, 15, + 28, 29, 30, 31}; /********************************************* * Single precision real and complex packing * @@ -116,7 +143,7 @@ const static Packet16uc p16uc_GETIMAG64 = { 8, 9, 10, 11, 12, 13, 14, 15, * reason why packing for complex is broken down into several different parts, also the reason why we endup having a * float32/64 and complex float32/64 version. **/ -template +template EIGEN_ALWAYS_INLINE std::complex getAdjointVal(Index i, Index j, const_blas_data_mapper, Index, StorageOrder>& dt) { std::complex v; @@ -135,7 +162,7 @@ EIGEN_ALWAYS_INLINE std::complex getAdjointVal(Index i, Index j, const_b return v; } -template +template EIGEN_STRONG_INLINE void symm_pack_complex_rhs_helper(std::complex* blockB, const std::complex* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { const Index depth = k2 + rows; @@ -153,7 +180,7 @@ EIGEN_STRONG_INLINE void symm_pack_complex_rhs_helper(std::complex* bloc { for(Index k = 0; k < vectorSize; k++) { - std::complex v = getAdjointVal(i, j + k, rhs); + std::complex v = getAdjointVal(i, j + k, rhs); blockBf[rir + k] = v.real(); blockBf[rii + k] = v.imag(); @@ -171,7 +198,7 @@ EIGEN_STRONG_INLINE void symm_pack_complex_rhs_helper(std::complex* bloc for(Index i = k2; i < depth; i++) { - std::complex v = getAdjointVal(i, j, rhs); + std::complex v = getAdjointVal(i, j, rhs); blockBf[rir] = v.real(); blockBf[rii] = v.imag(); @@ -184,14 +211,14 @@ EIGEN_STRONG_INLINE void symm_pack_complex_rhs_helper(std::complex* bloc } } -template +template EIGEN_STRONG_INLINE void symm_pack_complex_lhs_helper(std::complex* blockA, const std::complex* _lhs, Index lhsStride, Index cols, Index rows) { const Index depth = cols; const_blas_data_mapper, Index, StorageOrder> lhs(_lhs, lhsStride); const Index vectorSize = quad_traits::vectorsize; const Index vectorDelta = vectorSize * depth; - Scalar* blockAf = (Scalar *)(blockA); + Scalar* blockAf = reinterpret_cast(blockA); Index rir = 0, rii, j = 0; for(; j + vectorSize <= rows; j+=vectorSize) @@ -202,7 +229,7 @@ EIGEN_STRONG_INLINE void symm_pack_complex_lhs_helper(std::complex* bloc { for(Index k = 0; k < vectorSize; k++) { - std::complex v = getAdjointVal(j+k, i, lhs); + std::complex v = getAdjointVal(j+k, i, lhs); blockAf[rir + k] = v.real(); blockAf[rii + k] = v.imag(); @@ -223,7 +250,7 @@ EIGEN_STRONG_INLINE void symm_pack_complex_lhs_helper(std::complex* bloc Index k = j; for(; k < rows; k++) { - std::complex v = getAdjointVal(k, i, lhs); + std::complex v = getAdjointVal(k, i, lhs); blockAf[rir] = v.real(); blockAf[rii] = v.imag(); @@ -235,7 +262,7 @@ EIGEN_STRONG_INLINE void symm_pack_complex_lhs_helper(std::complex* bloc } } -template +template EIGEN_STRONG_INLINE void symm_pack_rhs_helper(Scalar* blockB, const Scalar* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { const Index depth = k2 + rows; @@ -272,7 +299,7 @@ EIGEN_STRONG_INLINE void symm_pack_rhs_helper(Scalar* blockB, const Scalar* _rhs } } -template +template EIGEN_STRONG_INLINE void symm_pack_lhs_helper(Scalar* blockA, const Scalar* _lhs, Index lhsStride, Index cols, Index rows) { const Index depth = cols; @@ -319,7 +346,7 @@ struct symm_pack_rhs, Index, nr, StorageOrder> { void operator()(std::complex* blockB, const std::complex* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { - symm_pack_complex_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); + symm_pack_complex_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); } }; @@ -328,7 +355,7 @@ struct symm_pack_lhs, Index, Pack1, Pack2_dummy, StorageOrde { void operator()(std::complex* blockA, const std::complex* _lhs, Index lhsStride, Index cols, Index rows) { - symm_pack_complex_lhs_helper(blockA, _lhs, lhsStride, cols, rows); + symm_pack_complex_lhs_helper(blockA, _lhs, lhsStride, cols, rows); } }; @@ -339,7 +366,7 @@ struct symm_pack_rhs, Index, nr, StorageOrder> { void operator()(std::complex* blockB, const std::complex* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { - symm_pack_complex_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); + symm_pack_complex_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); } }; @@ -348,7 +375,7 @@ struct symm_pack_lhs, Index, Pack1, Pack2_dummy, StorageOrd { void operator()(std::complex* blockA, const std::complex* _lhs, Index lhsStride, Index cols, Index rows) { - symm_pack_complex_lhs_helper(blockA, _lhs, lhsStride, cols, rows); + symm_pack_complex_lhs_helper(blockA, _lhs, lhsStride, cols, rows); } }; @@ -358,7 +385,7 @@ struct symm_pack_rhs { void operator()(float* blockB, const float* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { - symm_pack_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); + symm_pack_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); } }; @@ -367,7 +394,7 @@ struct symm_pack_lhs { void operator()(float* blockA, const float* _lhs, Index lhsStride, Index cols, Index rows) { - symm_pack_lhs_helper(blockA, _lhs, lhsStride, cols, rows); + symm_pack_lhs_helper(blockA, _lhs, lhsStride, cols, rows); } }; @@ -377,7 +404,7 @@ struct symm_pack_rhs { void operator()(double* blockB, const double* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { - symm_pack_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); + symm_pack_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); } }; @@ -386,7 +413,7 @@ struct symm_pack_lhs { void operator()(double* blockA, const double* _lhs, Index lhsStride, Index cols, Index rows) { - symm_pack_lhs_helper(blockA, _lhs, lhsStride, cols, rows); + symm_pack_lhs_helper(blockA, _lhs, lhsStride, cols, rows); } }; @@ -401,7 +428,7 @@ struct symm_pack_lhs * and offset and behaves accordingly. **/ -template +template EIGEN_ALWAYS_INLINE void storeBlock(Scalar* to, PacketBlock& block) { const Index size = 16 / sizeof(Scalar); @@ -416,8 +443,80 @@ EIGEN_ALWAYS_INLINE void storeBlock(Scalar* to, PacketBlock& block) } // General template for lhs & rhs complex packing. -template +template struct dhs_cpack { + template + EIGEN_ALWAYS_INLINE void dhs_cblock(PacketBlock& cblock, PacketBlock& block, Packet16uc permute) + { + if (transpose) { + block.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, permute); + block.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, permute); + block.packet[2] = vec_perm(cblock.packet[4].v, cblock.packet[5].v, permute); + block.packet[3] = vec_perm(cblock.packet[6].v, cblock.packet[7].v, permute); + + Packet4f t0, t1, t2, t3; +#ifdef EIGEN_VECTORIZE_VSX + t0 = reinterpret_cast(vec_mergeh(reinterpret_cast(block.packet[0]), reinterpret_cast(block.packet[1]))); + t1 = reinterpret_cast(vec_mergel(reinterpret_cast(block.packet[0]), reinterpret_cast(block.packet[1]))); + t2 = reinterpret_cast(vec_mergeh(reinterpret_cast(block.packet[2]), reinterpret_cast(block.packet[3]))); + t3 = reinterpret_cast(vec_mergel(reinterpret_cast(block.packet[2]), reinterpret_cast(block.packet[3]))); +#else + t0 = reinterpret_cast(vec_perm(block.packet[0], block.packet[1], p16uc_TRANSPOSE64_HI)); + t1 = reinterpret_cast(vec_perm(block.packet[0], block.packet[1], p16uc_TRANSPOSE64_LO)); + t2 = reinterpret_cast(vec_perm(block.packet[2], block.packet[3], p16uc_TRANSPOSE64_HI)); + t3 = reinterpret_cast(vec_perm(block.packet[2], block.packet[3], p16uc_TRANSPOSE64_LO)); +#endif + + block.packet[0] = t0; + block.packet[1] = t1; + block.packet[2] = t2; + block.packet[3] = t3; + } else { + block.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[4].v, permute); + block.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[5].v, permute); + block.packet[2] = vec_perm(cblock.packet[2].v, cblock.packet[6].v, permute); + block.packet[3] = vec_perm(cblock.packet[3].v, cblock.packet[7].v, permute); + } + } + + EIGEN_ALWAYS_INLINE void dhs_ccopy(Scalar* blockAt, const DataMapper& lhs2, Index& i, Index& rir, Index& rii, Index depth, const Index vectorSize) + { + PacketBlock blockr, blocki; + PacketBlock cblock; + + for(; i + vectorSize <= depth; i+=vectorSize) + { + if (UseLhs) { + bload(cblock, lhs2, 0, i); + } else { + bload(cblock, lhs2, i, 0); + } + + if(((StorageOrder == RowMajor) && UseLhs) || (((StorageOrder == ColMajor) && !UseLhs))) + { + dhs_cblock(cblock, blockr, p16uc_GETREAL32b); + dhs_cblock(cblock, blocki, p16uc_GETIMAG32b); + } else { + dhs_cblock(cblock, blockr, p16uc_GETREAL32); + dhs_cblock(cblock, blocki, p16uc_GETIMAG32); + } + + if(Conjugate) + { + blocki.packet[0] = -blocki.packet[0]; + blocki.packet[1] = -blocki.packet[1]; + blocki.packet[2] = -blocki.packet[2]; + blocki.packet[3] = -blocki.packet[3]; + } + + storeBlock(blockAt + rir, blockr); + storeBlock(blockAt + rii, blocki); + + rir += 4*vectorSize; + rii += 4*vectorSize; + } + } + EIGEN_STRONG_INLINE void operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { const Index vectorSize = quad_traits::vectorsize; @@ -428,51 +527,13 @@ struct dhs_cpack { for(; j + vectorSize <= rows; j+=vectorSize) { + const DataMapper lhs2 = UseLhs ? lhs.getSubMapper(j, 0) : lhs.getSubMapper(0, j); Index i = 0; rii = rir + vectorDelta; - for(; i + vectorSize <= depth; i+=vectorSize) - { - PacketBlock blockr, blocki; - PacketBlock cblock; - - if (UseLhs) { - bload(cblock, lhs, j, i); - } else { - bload(cblock, lhs, i, j); - } - - blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[4].v, p16uc_GETREAL32); - blockr.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[5].v, p16uc_GETREAL32); - blockr.packet[2] = vec_perm(cblock.packet[2].v, cblock.packet[6].v, p16uc_GETREAL32); - blockr.packet[3] = vec_perm(cblock.packet[3].v, cblock.packet[7].v, p16uc_GETREAL32); - - blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[4].v, p16uc_GETIMAG32); - blocki.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[5].v, p16uc_GETIMAG32); - blocki.packet[2] = vec_perm(cblock.packet[2].v, cblock.packet[6].v, p16uc_GETIMAG32); - blocki.packet[3] = vec_perm(cblock.packet[3].v, cblock.packet[7].v, p16uc_GETIMAG32); - - if(Conjugate) - { - blocki.packet[0] = -blocki.packet[0]; - blocki.packet[1] = -blocki.packet[1]; - blocki.packet[2] = -blocki.packet[2]; - blocki.packet[3] = -blocki.packet[3]; - } - - if(((StorageOrder == RowMajor) && UseLhs) || (((StorageOrder == ColMajor) && !UseLhs))) - { - ptranspose(blockr); - ptranspose(blocki); - } + dhs_ccopy(blockAt, lhs2, i, rir, rii, depth, vectorSize); - storeBlock(blockAt + rir, blockr); - storeBlock(blockAt + rii, blocki); - - rir += 4*vectorSize; - rii += 4*vectorSize; - } for(; i < depth; i++) { PacketBlock blockr, blocki; @@ -481,19 +542,19 @@ struct dhs_cpack { if(((StorageOrder == ColMajor) && UseLhs) || (((StorageOrder == RowMajor) && !UseLhs))) { if (UseLhs) { - cblock.packet[0] = lhs.template loadPacket(j + 0, i); - cblock.packet[1] = lhs.template loadPacket(j + 2, i); + cblock.packet[0] = lhs2.template loadPacket(0, i); + cblock.packet[1] = lhs2.template loadPacket(2, i); } else { - cblock.packet[0] = lhs.template loadPacket(i, j + 0); - cblock.packet[1] = lhs.template loadPacket(i, j + 2); + cblock.packet[0] = lhs2.template loadPacket(i, 0); + cblock.packet[1] = lhs2.template loadPacket(i, 2); } } else { if (UseLhs) { - cblock.packet[0] = pload2(lhs(j + 0, i), lhs(j + 1, i)); - cblock.packet[1] = pload2(lhs(j + 2, i), lhs(j + 3, i)); + cblock.packet[0] = pload2(lhs2(0, i), lhs2(1, i)); + cblock.packet[1] = pload2(lhs2(2, i), lhs2(3, i)); } else { - cblock.packet[0] = pload2(lhs(i, j + 0), lhs(i, j + 1)); - cblock.packet[1] = pload2(lhs(i, j + 2), lhs(i, j + 3)); + cblock.packet[0] = pload2(lhs2(i, 0), lhs2(i, 1)); + cblock.packet[1] = pload2(lhs2(i, 2), lhs2(i, 3)); } } @@ -521,16 +582,17 @@ struct dhs_cpack { for(; j < rows; j++) { + const DataMapper lhs2 = lhs.getSubMapper(0, j); rii = rir + ((PanelMode) ? stride : depth); for(Index i = 0; i < depth; i++) { - blockAt[rir] = lhs(i, j).real(); + blockAt[rir] = lhs2(i, 0).real(); if(Conjugate) - blockAt[rii] = -lhs(i, j).imag(); + blockAt[rii] = -lhs2(i, 0).imag(); else - blockAt[rii] = lhs(i, j).imag(); + blockAt[rii] = lhs2(i, 0).imag(); rir += 1; rii += 1; @@ -566,8 +628,38 @@ struct dhs_cpack { }; // General template for lhs & rhs packing. -template +template struct dhs_pack{ + template + EIGEN_ALWAYS_INLINE void dhs_copy(Scalar* blockA, const DataMapper& lhs2, Index& i, Index& ri, Index depth, const Index vectorSize) + { + PacketBlock block[n]; + + for(; i + n*vectorSize <= depth; i+=n*vectorSize) + { + for (Index k = 0; k < n; k++) { + if (UseLhs) { + bload(block[k], lhs2, 0, i + k*vectorSize); + } else { + bload(block[k], lhs2, i + k*vectorSize, 0); + } + } + + if(((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) + { + for (Index k = 0; k < n; k++) { + ptranspose(block[k]); + } + } + + for (Index k = 0; k < n; k++) { + storeBlock(blockA + ri + k*4*vectorSize, block[k]); + } + + ri += n*4*vectorSize; + } + } + EIGEN_STRONG_INLINE void operator()(Scalar* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { const Index vectorSize = quad_traits::vectorsize; @@ -575,49 +667,36 @@ struct dhs_pack{ for(; j + vectorSize <= rows; j+=vectorSize) { + const DataMapper lhs2 = UseLhs ? lhs.getSubMapper(j, 0) : lhs.getSubMapper(0, j); Index i = 0; if(PanelMode) ri += vectorSize*offset; - for(; i + vectorSize <= depth; i+=vectorSize) - { - PacketBlock block; - - if (UseLhs) { - bload(block, lhs, j, i); - } else { - bload(block, lhs, i, j); - } - if(((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) - { - ptranspose(block); - } - - storeBlock(blockA + ri, block); + dhs_copy<4>(blockA, lhs2, i, ri, depth, vectorSize); + dhs_copy<2>(blockA, lhs2, i, ri, depth, vectorSize); + dhs_copy<1>(blockA, lhs2, i, ri, depth, vectorSize); - ri += 4*vectorSize; - } for(; i < depth; i++) { if(((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) { if (UseLhs) { - blockA[ri+0] = lhs(j+0, i); - blockA[ri+1] = lhs(j+1, i); - blockA[ri+2] = lhs(j+2, i); - blockA[ri+3] = lhs(j+3, i); + blockA[ri+0] = lhs2(0, i); + blockA[ri+1] = lhs2(1, i); + blockA[ri+2] = lhs2(2, i); + blockA[ri+3] = lhs2(3, i); } else { - blockA[ri+0] = lhs(i, j+0); - blockA[ri+1] = lhs(i, j+1); - blockA[ri+2] = lhs(i, j+2); - blockA[ri+3] = lhs(i, j+3); + blockA[ri+0] = lhs2(i, 0); + blockA[ri+1] = lhs2(i, 1); + blockA[ri+2] = lhs2(i, 2); + blockA[ri+3] = lhs2(i, 3); } } else { Packet lhsV; if (UseLhs) { - lhsV = lhs.template loadPacket(j, i); + lhsV = lhs2.template loadPacket(0, i); } else { - lhsV = lhs.template loadPacket(i, j); + lhsV = lhs2.template loadPacket(i, 0); } pstore(blockA + ri, lhsV); } @@ -634,9 +713,10 @@ struct dhs_pack{ for(; j < rows; j++) { + const DataMapper lhs2 = lhs.getSubMapper(0, j); for(Index i = 0; i < depth; i++) { - blockA[ri] = lhs(i, j); + blockA[ri] = lhs2(i, 0); ri += 1; } @@ -662,9 +742,42 @@ struct dhs_pack{ }; // General template for lhs packing, float64 specialization. -template -struct dhs_pack +template +struct dhs_pack { + template + EIGEN_ALWAYS_INLINE void dhs_copy(double* blockA, const DataMapper& lhs2, Index& i, Index& ri, Index depth, const Index vectorSize) + { + PacketBlock block[n]; + + for(; i + n*vectorSize <= depth; i+=n*vectorSize) + { + for (Index k = 0; k < n; k++) { + if(StorageOrder == RowMajor) + { + block[k].packet[0] = lhs2.template loadPacket(0, i + k*vectorSize); + block[k].packet[1] = lhs2.template loadPacket(1, i + k*vectorSize); + } else { + block[k].packet[0] = lhs2.template loadPacket(0, i + k*vectorSize + 0); + block[k].packet[1] = lhs2.template loadPacket(0, i + k*vectorSize + 1); + } + } + + if(StorageOrder == RowMajor) + { + for (Index k = 0; k < n; k++) { + ptranspose(block[k]); + } + } + + for (Index k = 0; k < n; k++) { + storeBlock(blockA + ri + k*2*vectorSize, block[k]); + } + + ri += n*2*vectorSize; + } + } + EIGEN_STRONG_INLINE void operator()(double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { const Index vectorSize = quad_traits::vectorsize; @@ -672,36 +785,23 @@ struct dhs_pack block; - if(StorageOrder == RowMajor) - { - block.packet[0] = lhs.template loadPacket(j + 0, i); - block.packet[1] = lhs.template loadPacket(j + 1, i); - - ptranspose(block); - } else { - block.packet[0] = lhs.template loadPacket(j, i + 0); - block.packet[1] = lhs.template loadPacket(j, i + 1); - } + dhs_copy<4>(blockA, lhs2, i, ri, depth, vectorSize); + dhs_copy<2>(blockA, lhs2, i, ri, depth, vectorSize); + dhs_copy<1>(blockA, lhs2, i, ri, depth, vectorSize); - storeBlock(blockA + ri, block); - - ri += 2*vectorSize; - } for(; i < depth; i++) { if(StorageOrder == RowMajor) { - blockA[ri+0] = lhs(j+0, i); - blockA[ri+1] = lhs(j+1, i); + blockA[ri+0] = lhs2(0, i); + blockA[ri+1] = lhs2(1, i); } else { - Packet2d lhsV = lhs.template loadPacket(j, i); + Packet2d lhsV = lhs2.template loadPacket(0, i); pstore(blockA + ri, lhsV); } @@ -729,9 +829,56 @@ struct dhs_pack -struct dhs_pack +template +struct dhs_pack { + template + EIGEN_ALWAYS_INLINE void dhs_copy(double* blockB, const DataMapper& rhs2, Index& i, Index& ri, Index depth, const Index vectorSize) + { + PacketBlock block1[n], block2[n]; + PacketBlock block3[n]; + + for(; i + n*vectorSize <= depth; i+=n*vectorSize) + { + for (Index k = 0; k < n; k++) { + if(StorageOrder == ColMajor) + { + block1[k].packet[0] = rhs2.template loadPacket(i + k*vectorSize, 0); + block1[k].packet[1] = rhs2.template loadPacket(i + k*vectorSize, 1); + block2[k].packet[0] = rhs2.template loadPacket(i + k*vectorSize, 2); + block2[k].packet[1] = rhs2.template loadPacket(i + k*vectorSize, 3); + } else { + block3[k].packet[0] = rhs2.template loadPacket(i + k*vectorSize + 0, 0); //[a1 a2] + block3[k].packet[1] = rhs2.template loadPacket(i + k*vectorSize + 0, 2); //[a3 a4] + block3[k].packet[2] = rhs2.template loadPacket(i + k*vectorSize + 1, 0); //[b1 b2] + block3[k].packet[3] = rhs2.template loadPacket(i + k*vectorSize + 1, 2); //[b3 b4] + } + } + + if(StorageOrder == ColMajor) + { + for (Index k = 0; k < n; k++) { + ptranspose(block1[k]); + ptranspose(block2[k]); + } + } + + for (Index k = 0; k < n; k++) { + if(StorageOrder == ColMajor) + { + pstore(blockB + ri + k*4*vectorSize , block1[k].packet[0]); + pstore(blockB + ri + k*4*vectorSize + 2, block2[k].packet[0]); + pstore(blockB + ri + k*4*vectorSize + 4, block1[k].packet[1]); + pstore(blockB + ri + k*4*vectorSize + 6, block2[k].packet[1]); + } else { + storeBlock(blockB + ri + k*4*vectorSize, block3[k]); + } + } + + ri += n*4*vectorSize; + } + } + EIGEN_STRONG_INLINE void operator()(double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { const Index vectorSize = quad_traits::vectorsize; @@ -739,57 +886,33 @@ struct dhs_pack block; - if(StorageOrder == ColMajor) - { - PacketBlock block1, block2; - block1.packet[0] = rhs.template loadPacket(i, j + 0); - block1.packet[1] = rhs.template loadPacket(i, j + 1); - block2.packet[0] = rhs.template loadPacket(i, j + 2); - block2.packet[1] = rhs.template loadPacket(i, j + 3); - - ptranspose(block1); - ptranspose(block2); - - pstore(blockB + ri , block1.packet[0]); - pstore(blockB + ri + 2, block2.packet[0]); - pstore(blockB + ri + 4, block1.packet[1]); - pstore(blockB + ri + 6, block2.packet[1]); - } else { - block.packet[0] = rhs.template loadPacket(i + 0, j + 0); //[a1 a2] - block.packet[1] = rhs.template loadPacket(i + 0, j + 2); //[a3 a4] - block.packet[2] = rhs.template loadPacket(i + 1, j + 0); //[b1 b2] - block.packet[3] = rhs.template loadPacket(i + 1, j + 2); //[b3 b4] - - storeBlock(blockB + ri, block); - } + dhs_copy<4>(blockB, rhs2, i, ri, depth, vectorSize); + dhs_copy<2>(blockB, rhs2, i, ri, depth, vectorSize); + dhs_copy<1>(blockB, rhs2, i, ri, depth, vectorSize); - ri += 4*vectorSize; - } for(; i < depth; i++) { if(StorageOrder == ColMajor) { - blockB[ri+0] = rhs(i, j+0); - blockB[ri+1] = rhs(i, j+1); + blockB[ri+0] = rhs2(i, 0); + blockB[ri+1] = rhs2(i, 1); ri += vectorSize; - blockB[ri+0] = rhs(i, j+2); - blockB[ri+1] = rhs(i, j+3); + blockB[ri+0] = rhs2(i, 2); + blockB[ri+1] = rhs2(i, 3); } else { - Packet2d rhsV = rhs.template loadPacket(i, j); + Packet2d rhsV = rhs2.template loadPacket(i, 0); pstore(blockB + ri, rhsV); ri += vectorSize; - rhsV = rhs.template loadPacket(i, j + 2); + rhsV = rhs2.template loadPacket(i, 2); pstore(blockB + ri, rhsV); } ri += vectorSize; @@ -802,9 +925,10 @@ struct dhs_pack -struct dhs_cpack +// General template for lhs packing, bfloat16 specialization. +template +struct dhs_pack { - EIGEN_STRONG_INLINE void operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) + EIGEN_STRONG_INLINE void operator()(bfloat16* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { - const Index vectorSize = quad_traits::vectorsize; - const Index vectorDelta = vectorSize * ((PanelMode) ? stride : depth); - Index rir = ((PanelMode) ? (vectorSize*offset) : 0), rii; - double* blockAt = reinterpret_cast(blockA); - Index j = 0; + const Index vectorSize = quad_traits::vectorsize; + Index ri = 0, j = 0; - for(; j + vectorSize <= rows; j+=vectorSize) + for(; j + 2*vectorSize <= rows; j+=2*vectorSize) { + const DataMapper lhs2 = lhs.getSubMapper(j, 0); Index i = 0; - rii = rir + vectorDelta; + if(PanelMode) ri += 2*vectorSize*offset; - for(; i + vectorSize <= depth; i+=vectorSize) + if(StorageOrder == ColMajor) { - PacketBlock blockr, blocki; - PacketBlock cblock; - - if(StorageOrder == ColMajor) + for(; i + 2 <= depth; i+=2) { - cblock.packet[0] = lhs.template loadPacket(j, i + 0); //[a1 a1i] - cblock.packet[1] = lhs.template loadPacket(j, i + 1); //[b1 b1i] + PacketBlock block; - cblock.packet[2] = lhs.template loadPacket(j + 1, i + 0); //[a2 a2i] - cblock.packet[3] = lhs.template loadPacket(j + 1, i + 1); //[b2 b2i] + block.packet[0] = lhs2.template loadPacket(0 * vectorSize, i + 0); + block.packet[1] = lhs2.template loadPacket(1 * vectorSize, i + 0); + block.packet[2] = lhs2.template loadPacket(0 * vectorSize, i + 1); + block.packet[3] = lhs2.template loadPacket(1 * vectorSize, i + 1); - blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[2].v, p16uc_GETREAL64); //[a1 a2] - blockr.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[3].v, p16uc_GETREAL64); //[b1 b2] + Packet8bf t0, t1; + t0 = vec_mergeh(block.packet[0].m_val, block.packet[2].m_val); + t1 = vec_mergel(block.packet[0].m_val, block.packet[2].m_val); + block.packet[2] = vec_mergeh(block.packet[1].m_val, block.packet[3].m_val); + block.packet[3] = vec_mergel(block.packet[1].m_val, block.packet[3].m_val); + block.packet[0] = t0; + block.packet[1] = t1; - blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[2].v, p16uc_GETIMAG64); - blocki.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[3].v, p16uc_GETIMAG64); - } else { - cblock.packet[0] = lhs.template loadPacket(j + 0, i); //[a1 a1i] - cblock.packet[1] = lhs.template loadPacket(j + 1, i); //[a2 a2i] + storeBlock(blockA + ri, block); + + ri += 2*2*vectorSize; + } + if (depth & 1) + { + PacketBlock block; - cblock.packet[2] = lhs.template loadPacket(j + 0, i + 1); //[b1 b1i] - cblock.packet[3] = lhs.template loadPacket(j + 1, i + 1); //[b2 b2i + block.packet[0] = lhs2.template loadPacket(0 * vectorSize, i + 0); + block.packet[1] = lhs2.template loadPacket(1 * vectorSize, i + 0); - blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL64); //[a1 a2] - blockr.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETREAL64); //[b1 b2] + storeBlock(blockA + ri, block); - blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG64); - blocki.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETIMAG64); + ri += 2*vectorSize; } - - if(Conjugate) + } else { + for(; i + vectorSize <= depth; i+=vectorSize) { - blocki.packet[0] = -blocki.packet[0]; - blocki.packet[1] = -blocki.packet[1]; - } + PacketBlock block1, block2; + + bload(block1, lhs2, 0 * vectorSize, i); + bload(block2, lhs2, 1 * vectorSize, i); + + Packet4ui v1[8], v2[8]; + + v1[0] = vec_mergeh(reinterpret_cast(block1.packet[0].m_val), reinterpret_cast(block1.packet[1].m_val)); + v1[1] = vec_mergel(reinterpret_cast(block1.packet[0].m_val), reinterpret_cast(block1.packet[1].m_val)); + v1[2] = vec_mergeh(reinterpret_cast(block1.packet[2].m_val), reinterpret_cast(block1.packet[3].m_val)); + v1[3] = vec_mergel(reinterpret_cast(block1.packet[2].m_val), reinterpret_cast(block1.packet[3].m_val)); + v1[4] = vec_mergeh(reinterpret_cast(block1.packet[4].m_val), reinterpret_cast(block1.packet[5].m_val)); + v1[5] = vec_mergel(reinterpret_cast(block1.packet[4].m_val), reinterpret_cast(block1.packet[5].m_val)); + v1[6] = vec_mergeh(reinterpret_cast(block1.packet[6].m_val), reinterpret_cast(block1.packet[7].m_val)); + v1[7] = vec_mergel(reinterpret_cast(block1.packet[6].m_val), reinterpret_cast(block1.packet[7].m_val)); + v2[0] = vec_mergeh(reinterpret_cast(block2.packet[0].m_val), reinterpret_cast(block2.packet[1].m_val)); + v2[1] = vec_mergel(reinterpret_cast(block2.packet[0].m_val), reinterpret_cast(block2.packet[1].m_val)); + v2[2] = vec_mergeh(reinterpret_cast(block2.packet[2].m_val), reinterpret_cast(block2.packet[3].m_val)); + v2[3] = vec_mergel(reinterpret_cast(block2.packet[2].m_val), reinterpret_cast(block2.packet[3].m_val)); + v2[4] = vec_mergeh(reinterpret_cast(block2.packet[4].m_val), reinterpret_cast(block2.packet[5].m_val)); + v2[5] = vec_mergel(reinterpret_cast(block2.packet[4].m_val), reinterpret_cast(block2.packet[5].m_val)); + v2[6] = vec_mergeh(reinterpret_cast(block2.packet[6].m_val), reinterpret_cast(block2.packet[7].m_val)); + v2[7] = vec_mergel(reinterpret_cast(block2.packet[6].m_val), reinterpret_cast(block2.packet[7].m_val)); + +#ifdef EIGEN_VECTORIZE_VSX + block1.packet[0] = reinterpret_cast(vec_mergeh(reinterpret_cast(v1[0]),reinterpret_cast(v1[2]))); + block1.packet[2] = reinterpret_cast(vec_mergel(reinterpret_cast(v1[0]),reinterpret_cast(v1[2]))); + block1.packet[4] = reinterpret_cast(vec_mergeh(reinterpret_cast(v1[1]),reinterpret_cast(v1[3]))); + block1.packet[6] = reinterpret_cast(vec_mergel(reinterpret_cast(v1[1]),reinterpret_cast(v1[3]))); + block1.packet[1] = reinterpret_cast(vec_mergeh(reinterpret_cast(v1[4]),reinterpret_cast(v1[6]))); + block1.packet[3] = reinterpret_cast(vec_mergel(reinterpret_cast(v1[4]),reinterpret_cast(v1[6]))); + block1.packet[5] = reinterpret_cast(vec_mergeh(reinterpret_cast(v1[5]),reinterpret_cast(v1[7]))); + block1.packet[7] = reinterpret_cast(vec_mergel(reinterpret_cast(v1[5]),reinterpret_cast(v1[7]))); + block2.packet[0] = reinterpret_cast(vec_mergeh(reinterpret_cast(v2[0]),reinterpret_cast(v2[2]))); + block2.packet[2] = reinterpret_cast(vec_mergel(reinterpret_cast(v2[0]),reinterpret_cast(v2[2]))); + block2.packet[4] = reinterpret_cast(vec_mergeh(reinterpret_cast(v2[1]),reinterpret_cast(v2[3]))); + block2.packet[6] = reinterpret_cast(vec_mergel(reinterpret_cast(v2[1]),reinterpret_cast(v2[3]))); + block2.packet[1] = reinterpret_cast(vec_mergeh(reinterpret_cast(v2[4]),reinterpret_cast(v2[6]))); + block2.packet[3] = reinterpret_cast(vec_mergel(reinterpret_cast(v2[4]),reinterpret_cast(v2[6]))); + block2.packet[5] = reinterpret_cast(vec_mergeh(reinterpret_cast(v2[5]),reinterpret_cast(v2[7]))); + block2.packet[7] = reinterpret_cast(vec_mergel(reinterpret_cast(v2[5]),reinterpret_cast(v2[7]))); +#else + block1.packet[0] = reinterpret_cast(vec_perm(v1[0],v1[2],p16uc_TRANSPOSE64_HI)); + block1.packet[2] = reinterpret_cast(vec_perm(v1[0],v1[2],p16uc_TRANSPOSE64_LO)); + block1.packet[4] = reinterpret_cast(vec_perm(v1[1],v1[3],p16uc_TRANSPOSE64_HI)); + block1.packet[6] = reinterpret_cast(vec_perm(v1[1],v1[3],p16uc_TRANSPOSE64_LO)); + block1.packet[1] = reinterpret_cast(vec_perm(v1[4],v1[6],p16uc_TRANSPOSE64_HI)); + block1.packet[3] = reinterpret_cast(vec_perm(v1[4],v1[6],p16uc_TRANSPOSE64_LO)); + block1.packet[5] = reinterpret_cast(vec_perm(v1[5],v1[7],p16uc_TRANSPOSE64_HI)); + block1.packet[7] = reinterpret_cast(vec_perm(v1[5],v1[7],p16uc_TRANSPOSE64_LO)); + block2.packet[0] = reinterpret_cast(vec_perm(v2[0],v2[2],p16uc_TRANSPOSE64_HI)); + block2.packet[2] = reinterpret_cast(vec_perm(v2[0],v2[2],p16uc_TRANSPOSE64_LO)); + block2.packet[4] = reinterpret_cast(vec_perm(v2[1],v2[3],p16uc_TRANSPOSE64_HI)); + block2.packet[6] = reinterpret_cast(vec_perm(v2[1],v2[3],p16uc_TRANSPOSE64_LO)); + block2.packet[1] = reinterpret_cast(vec_perm(v2[4],v2[6],p16uc_TRANSPOSE64_HI)); + block2.packet[3] = reinterpret_cast(vec_perm(v2[4],v2[6],p16uc_TRANSPOSE64_LO)); + block2.packet[5] = reinterpret_cast(vec_perm(v2[5],v2[7],p16uc_TRANSPOSE64_HI)); + block2.packet[7] = reinterpret_cast(vec_perm(v2[5],v2[7],p16uc_TRANSPOSE64_LO)); +#endif + + for(Index M = 0; M < 8; M+=2) { + pstore(blockA + ri + (0 * vectorSize) + (2*vectorSize * M), block1.packet[M+0]); + pstore(blockA + ri + (1 * vectorSize) + (2*vectorSize * M), block1.packet[M+1]); + pstore(blockA + ri + (2 * vectorSize) + (2*vectorSize * M), block2.packet[M+0]); + pstore(blockA + ri + (3 * vectorSize) + (2*vectorSize * M), block2.packet[M+1]); + } - storeBlock(blockAt + rir, blockr); - storeBlock(blockAt + rii, blocki); + ri += 2*vectorSize*vectorSize; + } + for(; i + 2 <= depth; i+=2) + { + for(Index M = 0; M < 2*vectorSize; M++) { + blockA[ri + (M * 2) + 0] = lhs2(M, i + 0); + blockA[ri + (M * 2) + 1] = lhs2(M, i + 1); + } - rir += 2*vectorSize; - rii += 2*vectorSize; + ri += 2*2*vectorSize; + } + if (depth & 1) + { + for(Index M = 0; M < 2*vectorSize; M++) { + blockA[ri + M] = lhs2(M, i); + } + ri += 2*vectorSize; + } } - for(; i < depth; i++) - { - PacketBlock blockr, blocki; - PacketBlock cblock; - cblock.packet[0] = lhs.template loadPacket(j + 0, i); - cblock.packet[1] = lhs.template loadPacket(j + 1, i); + if(PanelMode) ri += 2*vectorSize*(stride - offset - depth); + } + for(; j + vectorSize <= rows; j+=vectorSize) + { + const DataMapper lhs2 = lhs.getSubMapper(j, 0); + Index i = 0; - blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL64); - blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG64); + if(PanelMode) ri += vectorSize*offset; - if(Conjugate) + if(StorageOrder == ColMajor) + { + for(; i + 2 <= depth; i+=2) { - blocki.packet[0] = -blocki.packet[0]; - } + PacketBlock block; - pstore(blockAt + rir, blockr.packet[0]); - pstore(blockAt + rii, blocki.packet[0]); + block.packet[0] = lhs2.template loadPacket(0 * vectorSize, i + 0); + block.packet[1] = lhs2.template loadPacket(0 * vectorSize, i + 1); - rir += vectorSize; - rii += vectorSize; - } + Packet8bf t0; + t0 = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val); + block.packet[1] = vec_mergel(block.packet[0].m_val, block.packet[1].m_val); + block.packet[0] = t0; - rir += ((PanelMode) ? (vectorSize*(2*stride - depth)) : vectorDelta); - } + storeBlock(blockA + ri, block); - if (j < rows) - { - if(PanelMode) rir += (offset*(rows - j - vectorSize)); - rii = rir + (((PanelMode) ? stride : depth) * (rows - j)); + ri += 2*vectorSize; + } + if (depth & 1) + { + Packet8bf lhsV = lhs2.template loadPacket(0 * vectorSize, i + 0); + pstore(blockA + ri, lhsV); - for(Index i = 0; i < depth; i++) - { - Index k = j; - for(; k < rows; k++) + ri += vectorSize; + } + } else { + for(; i + vectorSize <= depth; i+=vectorSize) { - blockAt[rir] = lhs(k, i).real(); + PacketBlock block1; + + bload(block1, lhs2, 0 * vectorSize, i); + + Packet4ui v1[8]; + + // This is transposing and interleaving data + v1[0] = vec_mergeh(reinterpret_cast(block1.packet[0].m_val), reinterpret_cast(block1.packet[1].m_val)); + v1[1] = vec_mergel(reinterpret_cast(block1.packet[0].m_val), reinterpret_cast(block1.packet[1].m_val)); + v1[2] = vec_mergeh(reinterpret_cast(block1.packet[2].m_val), reinterpret_cast(block1.packet[3].m_val)); + v1[3] = vec_mergel(reinterpret_cast(block1.packet[2].m_val), reinterpret_cast(block1.packet[3].m_val)); + v1[4] = vec_mergeh(reinterpret_cast(block1.packet[4].m_val), reinterpret_cast(block1.packet[5].m_val)); + v1[5] = vec_mergel(reinterpret_cast(block1.packet[4].m_val), reinterpret_cast(block1.packet[5].m_val)); + v1[6] = vec_mergeh(reinterpret_cast(block1.packet[6].m_val), reinterpret_cast(block1.packet[7].m_val)); + v1[7] = vec_mergel(reinterpret_cast(block1.packet[6].m_val), reinterpret_cast(block1.packet[7].m_val)); + +#ifdef EIGEN_VECTORIZE_VSX + block1.packet[0] = reinterpret_cast(vec_mergeh(reinterpret_cast(v1[0]),reinterpret_cast(v1[2]))); + block1.packet[2] = reinterpret_cast(vec_mergel(reinterpret_cast(v1[0]),reinterpret_cast(v1[2]))); + block1.packet[4] = reinterpret_cast(vec_mergeh(reinterpret_cast(v1[1]),reinterpret_cast(v1[3]))); + block1.packet[6] = reinterpret_cast(vec_mergel(reinterpret_cast(v1[1]),reinterpret_cast(v1[3]))); + block1.packet[1] = reinterpret_cast(vec_mergeh(reinterpret_cast(v1[4]),reinterpret_cast(v1[6]))); + block1.packet[3] = reinterpret_cast(vec_mergel(reinterpret_cast(v1[4]),reinterpret_cast(v1[6]))); + block1.packet[5] = reinterpret_cast(vec_mergeh(reinterpret_cast(v1[5]),reinterpret_cast(v1[7]))); + block1.packet[7] = reinterpret_cast(vec_mergel(reinterpret_cast(v1[5]),reinterpret_cast(v1[7]))); +#else + block1.packet[0] = reinterpret_cast(vec_perm(v1[0],v1[2],p16uc_TRANSPOSE64_HI)); + block1.packet[2] = reinterpret_cast(vec_perm(v1[0],v1[2],p16uc_TRANSPOSE64_LO)); + block1.packet[4] = reinterpret_cast(vec_perm(v1[1],v1[3],p16uc_TRANSPOSE64_HI)); + block1.packet[6] = reinterpret_cast(vec_perm(v1[1],v1[3],p16uc_TRANSPOSE64_LO)); + block1.packet[1] = reinterpret_cast(vec_perm(v1[4],v1[6],p16uc_TRANSPOSE64_HI)); + block1.packet[3] = reinterpret_cast(vec_perm(v1[4],v1[6],p16uc_TRANSPOSE64_LO)); + block1.packet[5] = reinterpret_cast(vec_perm(v1[5],v1[7],p16uc_TRANSPOSE64_HI)); + block1.packet[7] = reinterpret_cast(vec_perm(v1[5],v1[7],p16uc_TRANSPOSE64_LO)); +#endif - if(Conjugate) - blockAt[rii] = -lhs(k, i).imag(); - else - blockAt[rii] = lhs(k, i).imag(); + for(Index M = 0; M < 8; M++) { + pstore(blockA + ri + (vectorSize * M), block1.packet[M]); + } - rir += 1; - rii += 1; + ri += vectorSize*vectorSize; } - } - } - } -}; + for(; i + 2 <= depth; i+=2) + { + for(Index M = 0; M < vectorSize; M++) { + blockA[ri + (M * 2) + 0] = lhs2(M, i + 0); + blockA[ri + (M * 2) + 1] = lhs2(M, i + 1); + } -// General template for rhs complex packing, float64 specialization. -template -struct dhs_cpack -{ - EIGEN_STRONG_INLINE void operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) - { - const Index vectorSize = quad_traits::vectorsize; - const Index vectorDelta = 2*vectorSize * ((PanelMode) ? stride : depth); - Index rir = ((PanelMode) ? (2*vectorSize*offset) : 0), rii; - double* blockBt = reinterpret_cast(blockB); - Index j = 0; + ri += 2*vectorSize; + } + if (depth & 1) + { + for(Index M = 0; M < vectorSize; M++) { + blockA[ri + M] = lhs2(M, i); + } - for(; j + 2*vectorSize <= cols; j+=2*vectorSize) + ri += vectorSize; + } + } + + if(PanelMode) ri += vectorSize*(stride - offset - depth); + } + if(j + 4 <= rows) { + const DataMapper lhs2 = lhs.getSubMapper(j, 0); Index i = 0; - rii = rir + vectorDelta; + if(PanelMode) ri += 4*offset; - for(; i < depth; i++) + for(; i + 2 <= depth; i+=2) { - PacketBlock cblock; - PacketBlock blockr, blocki; - - bload(cblock, rhs, i, j); + if(StorageOrder == ColMajor) + { + PacketBlock block; - blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL64); - blockr.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETREAL64); + block.packet[0] = lhs2.template loadPacketPartial(0, i + 0, 4); + block.packet[1] = lhs2.template loadPacketPartial(0, i + 1, 4); - blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG64); - blocki.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETIMAG64); + block.packet[0] = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val); + + pstore(blockA + ri, block.packet[0]); + } else { + blockA[ri+0] = lhs2(0, i + 0); + blockA[ri+1] = lhs2(0, i + 1); + blockA[ri+2] = lhs2(1, i + 0); + blockA[ri+3] = lhs2(1, i + 1); + blockA[ri+4] = lhs2(2, i + 0); + blockA[ri+5] = lhs2(2, i + 1); + blockA[ri+6] = lhs2(3, i + 0); + blockA[ri+7] = lhs2(3, i + 1); + } + + ri += 2*4; + } + if (depth & 1) + { + if(StorageOrder == ColMajor) + { + Packet8bf lhsV = lhs2.template loadPacketPartial(0, i + 0, 4); + + pstore_partial(blockA + ri, lhsV, 4); + } else { + blockA[ri+0] = lhs2(0, i); + blockA[ri+1] = lhs2(1, i); + blockA[ri+2] = lhs2(2, i); + blockA[ri+3] = lhs2(3, i); + } + + ri += 4; + } + + if(PanelMode) ri += 4*(stride - offset - depth); + j += 4; + } + + if (j < rows) + { + if(PanelMode) ri += offset*(rows - j); + + Index i = 0; + for(; i + 2 <= depth; i+=2) + { + Index k = j; + for(; k < rows; k++) + { + blockA[ri+0] = lhs(k, i + 0); + blockA[ri+1] = lhs(k, i + 1); + ri += 2; + } + } + if (depth & 1) + { + for(; j < rows; j++) + { + blockA[ri] = lhs(j, i); + ri += 1; + } + } + } + } +}; + +// General template for rhs packing, bfloat16 specialization. +template +struct dhs_pack +{ + EIGEN_STRONG_INLINE void operator()(bfloat16* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) + { + const Index vectorSize = quad_traits::vectorsize; + Index ri = 0, j = 0; + + for(; j + 4 <= cols; j+=4) + { + const DataMapper rhs2 = rhs.getSubMapper(0, j); + Index i = 0; + + if(PanelMode) ri += 4*offset; + + for(; i + vectorSize <= depth; i+=vectorSize) + { + if(StorageOrder == ColMajor) + { + PacketBlock block; + + bload(block, rhs2, i, 0); + + Packet4ui t0, t1, t2, t3; + + t0 = vec_mergeh(reinterpret_cast(block.packet[0].m_val), reinterpret_cast(block.packet[1].m_val)); + t1 = vec_mergel(reinterpret_cast(block.packet[0].m_val), reinterpret_cast(block.packet[1].m_val)); + t2 = vec_mergeh(reinterpret_cast(block.packet[2].m_val), reinterpret_cast(block.packet[3].m_val)); + t3 = vec_mergel(reinterpret_cast(block.packet[2].m_val), reinterpret_cast(block.packet[3].m_val)); + +#ifdef EIGEN_VECTORIZE_VSX + block.packet[0] = reinterpret_cast(vec_mergeh(reinterpret_cast(t0),reinterpret_cast(t2))); + block.packet[1] = reinterpret_cast(vec_mergel(reinterpret_cast(t0),reinterpret_cast(t2))); + block.packet[2] = reinterpret_cast(vec_mergeh(reinterpret_cast(t1),reinterpret_cast(t3))); + block.packet[3] = reinterpret_cast(vec_mergel(reinterpret_cast(t1),reinterpret_cast(t3))); +#else + block.packet[0] = reinterpret_cast(vec_perm(t0,t2,p16uc_TRANSPOSE64_HI)); + block.packet[1] = reinterpret_cast(vec_perm(t0,t2,p16uc_TRANSPOSE64_LO)); + block.packet[2] = reinterpret_cast(vec_perm(t1,t3,p16uc_TRANSPOSE64_HI)); + block.packet[3] = reinterpret_cast(vec_perm(t1,t3,p16uc_TRANSPOSE64_LO)); +#endif + + storeBlock(blockB + ri, block); + } else { + PacketBlock block; + + for (int M = 0; M < 8; M++) { + block.packet[M] = rhs2.template loadPacketPartial(i + M, 0, 4); + } + + block.packet[0] = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val); + block.packet[1] = vec_mergeh(block.packet[2].m_val, block.packet[3].m_val); + block.packet[2] = vec_mergeh(block.packet[4].m_val, block.packet[5].m_val); + block.packet[3] = vec_mergeh(block.packet[6].m_val, block.packet[7].m_val); + + const Index size = 16 / sizeof(bfloat16); + + for (int M = 0; M < 4; M++) { + pstore(blockB + ri + (M * size), block.packet[M]); + } + } + + ri += 4*vectorSize; + } + for (; i + 2 <= depth; i += 2) { + if(StorageOrder == ColMajor) + { + blockB[ri+0] = rhs2(i + 0, 0); + blockB[ri+1] = rhs2(i + 1, 0); + blockB[ri+2] = rhs2(i + 0, 1); + blockB[ri+3] = rhs2(i + 1, 1); + blockB[ri+4] = rhs2(i + 0, 2); + blockB[ri+5] = rhs2(i + 1, 2); + blockB[ri+6] = rhs2(i + 0, 3); + blockB[ri+7] = rhs2(i + 1, 3); + } else { + PacketBlock block; + + for (int M = 0; M < 2; M++) { + block.packet[M] = rhs2.template loadPacketPartial(i + M, 0, 4); + } + + block.packet[0] = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val); + + pstore(blockB + ri, block.packet[0]); + } + + ri += 4*2; + } + if (depth & 1) + { + blockB[ri+0] = rhs2(i, 0); + blockB[ri+1] = rhs2(i, 1); + blockB[ri+2] = rhs2(i, 2); + blockB[ri+3] = rhs2(i, 3); + + ri += 4; + } + + if(PanelMode) ri += 4*(stride - offset - depth); + } + + if (j < cols) + { + if(PanelMode) ri += offset*(cols - j); + + Index i = 0; + for(; i + 2 <= depth; i+=2) + { + Index k = j; + for(; k < cols; k++) + { + blockB[ri+0] = rhs(i + 0, k); + blockB[ri+1] = rhs(i + 1, k); + ri += 2; + } + } + if (depth & 1) + { + for(; j < cols; j++) + { + blockB[ri] = rhs(i, j); + ri += 1; + } + } + } + } +}; + +// General template for lhs complex packing, float64 specialization. +template +struct dhs_cpack +{ + EIGEN_ALWAYS_INLINE void dhs_ccopy(double* blockAt, const DataMapper& lhs2, Index& i, Index& rir, Index& rii, Index depth, const Index vectorSize) + { + PacketBlock blockr, blocki; + PacketBlock cblock; + + for(; i + vectorSize <= depth; i+=vectorSize) + { + if(StorageOrder == ColMajor) + { + cblock.packet[0] = lhs2.template loadPacket(0, i + 0); //[a1 a1i] + cblock.packet[1] = lhs2.template loadPacket(0, i + 1); //[b1 b1i] + + cblock.packet[2] = lhs2.template loadPacket(1, i + 0); //[a2 a2i] + cblock.packet[3] = lhs2.template loadPacket(1, i + 1); //[b2 b2i] + + blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[2].v); //[a1 a2] + blockr.packet[1] = vec_mergeh(cblock.packet[1].v, cblock.packet[3].v); //[b1 b2] + + blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[2].v); + blocki.packet[1] = vec_mergel(cblock.packet[1].v, cblock.packet[3].v); + } else { + cblock.packet[0] = lhs2.template loadPacket(0, i); //[a1 a1i] + cblock.packet[1] = lhs2.template loadPacket(1, i); //[a2 a2i] + + cblock.packet[2] = lhs2.template loadPacket(0, i + 1); //[b1 b1i] + cblock.packet[3] = lhs2.template loadPacket(1, i + 1); //[b2 b2i + + blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[1].v); //[a1 a2] + blockr.packet[1] = vec_mergeh(cblock.packet[2].v, cblock.packet[3].v); //[b1 b2] + + blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[1].v); + blocki.packet[1] = vec_mergel(cblock.packet[2].v, cblock.packet[3].v); + } + + if(Conjugate) + { + blocki.packet[0] = -blocki.packet[0]; + blocki.packet[1] = -blocki.packet[1]; + } + + storeBlock(blockAt + rir, blockr); + storeBlock(blockAt + rii, blocki); + + rir += 2*vectorSize; + rii += 2*vectorSize; + } + } + + EIGEN_STRONG_INLINE void operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) + { + const Index vectorSize = quad_traits::vectorsize; + const Index vectorDelta = vectorSize * ((PanelMode) ? stride : depth); + Index rir = ((PanelMode) ? (vectorSize*offset) : 0), rii; + double* blockAt = reinterpret_cast(blockA); + Index j = 0; + + for(; j + vectorSize <= rows; j+=vectorSize) + { + const DataMapper lhs2 = lhs.getSubMapper(j, 0); + Index i = 0; + + rii = rir + vectorDelta; + + dhs_ccopy(blockAt, lhs2, i, rir, rii, depth, vectorSize); + + for(; i < depth; i++) + { + PacketBlock blockr, blocki; + PacketBlock cblock; + + cblock.packet[0] = lhs2.template loadPacket(0, i); + cblock.packet[1] = lhs2.template loadPacket(1, i); + + blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[1].v); + blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[1].v); if(Conjugate) { blocki.packet[0] = -blocki.packet[0]; - blocki.packet[1] = -blocki.packet[1]; } - storeBlock(blockBt + rir, blockr); - storeBlock(blockBt + rii, blocki); + pstore(blockAt + rir, blockr.packet[0]); + pstore(blockAt + rii, blocki.packet[0]); - rir += 2*vectorSize; - rii += 2*vectorSize; + rir += vectorSize; + rii += vectorSize; } + rir += ((PanelMode) ? (vectorSize*(2*stride - depth)) : vectorDelta); + } + + if (j < rows) + { + if(PanelMode) rir += (offset*(rows - j - vectorSize)); + rii = rir + (((PanelMode) ? stride : depth) * (rows - j)); + + for(Index i = 0; i < depth; i++) + { + Index k = j; + for(; k < rows; k++) + { + blockAt[rir] = lhs(k, i).real(); + + if(Conjugate) + blockAt[rii] = -lhs(k, i).imag(); + else + blockAt[rii] = lhs(k, i).imag(); + + rir += 1; + rii += 1; + } + } + } + } +}; + +// General template for rhs complex packing, float64 specialization. +template +struct dhs_cpack +{ + EIGEN_ALWAYS_INLINE void dhs_ccopy(double* blockBt, const DataMapper& rhs2, Index& i, Index& rir, Index& rii, Index depth, const Index vectorSize) + { + for(; i < depth; i++) + { + PacketBlock cblock; + PacketBlock blockr, blocki; + + bload(cblock, rhs2, i, 0); + + blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[1].v); + blockr.packet[1] = vec_mergeh(cblock.packet[2].v, cblock.packet[3].v); + + blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[1].v); + blocki.packet[1] = vec_mergel(cblock.packet[2].v, cblock.packet[3].v); + + if(Conjugate) + { + blocki.packet[0] = -blocki.packet[0]; + blocki.packet[1] = -blocki.packet[1]; + } + + storeBlock(blockBt + rir, blockr); + storeBlock(blockBt + rii, blocki); + + rir += 2*vectorSize; + rii += 2*vectorSize; + } + } + + EIGEN_STRONG_INLINE void operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) + { + const Index vectorSize = quad_traits::vectorsize; + const Index vectorDelta = 2*vectorSize * ((PanelMode) ? stride : depth); + Index rir = ((PanelMode) ? (2*vectorSize*offset) : 0), rii; + double* blockBt = reinterpret_cast(blockB); + Index j = 0; + + for(; j + 2*vectorSize <= cols; j+=2*vectorSize) + { + const DataMapper rhs2 = rhs.getSubMapper(0, j); + Index i = 0; + + rii = rir + vectorDelta; + + dhs_ccopy(blockBt, rhs2, i, rir, rii, depth, vectorSize); + rir += ((PanelMode) ? (2*vectorSize*(2*stride - depth)) : vectorDelta); } @@ -977,16 +1564,17 @@ struct dhs_cpack* acc, const Packet& l { if(NegativeAccumulate) { - acc->packet[0] = vec_nmsub(lhsV, rhsV[0], acc->packet[0]); - if (N > 1) { - acc->packet[1] = vec_nmsub(lhsV, rhsV[1], acc->packet[1]); - } - if (N > 2) { - acc->packet[2] = vec_nmsub(lhsV, rhsV[2], acc->packet[2]); - } - if (N > 3) { - acc->packet[3] = vec_nmsub(lhsV, rhsV[3], acc->packet[3]); + for (int M = 0; M < N; M++) { + acc->packet[M] = vec_nmsub(lhsV, rhsV[M], acc->packet[M]); } } else { - acc->packet[0] = vec_madd(lhsV, rhsV[0], acc->packet[0]); - if (N > 1) { - acc->packet[1] = vec_madd(lhsV, rhsV[1], acc->packet[1]); - } - if (N > 2) { - acc->packet[2] = vec_madd(lhsV, rhsV[2], acc->packet[2]); - } - if (N > 3) { - acc->packet[3] = vec_madd(lhsV, rhsV[3], acc->packet[3]); + for (int M = 0; M < N; M++) { + acc->packet[M] = vec_madd(lhsV, rhsV[M], acc->packet[M]); } } } @@ -1039,31 +1613,9 @@ EIGEN_ALWAYS_INLINE void pger(PacketBlock* acc, const Scalar* lhs, con pger_common(acc, lhsV, rhsV); } -template -EIGEN_ALWAYS_INLINE void loadPacketRemaining(const Scalar* lhs, Packet &lhsV) -{ -#ifdef _ARCH_PWR9 - lhsV = vec_xl_len((Scalar *)lhs, remaining_rows * sizeof(Scalar)); -#else - Index i = 0; - do { - lhsV[i] = lhs[i]; - } while (++i < remaining_rows); -#endif -} - -template -EIGEN_ALWAYS_INLINE void pger(PacketBlock* acc, const Scalar* lhs, const Packet* rhsV) -{ - Packet lhsV; - loadPacketRemaining(lhs, lhsV); - - pger_common(acc, lhsV, rhsV); -} - // 512-bits rank1-update of complex acc. It takes decoupled accumulators as entries. It also takes cares of mixed types real * complex and complex * real. template -EIGEN_ALWAYS_INLINE void pgerc_common(PacketBlock* accReal, PacketBlock* accImag, const Packet &lhsV, const Packet &lhsVi, const Packet* rhsV, const Packet* rhsVi) +EIGEN_ALWAYS_INLINE void pgerc_common(PacketBlock* accReal, PacketBlock* accImag, const Packet &lhsV, Packet &lhsVi, const Packet* rhsV, const Packet* rhsVi) { pger_common(accReal, lhsV, rhsV); if(LhsIsReal) @@ -1084,97 +1636,56 @@ EIGEN_ALWAYS_INLINE void pgerc_common(PacketBlock* accReal, PacketBloc template EIGEN_ALWAYS_INLINE void pgerc(PacketBlock* accReal, PacketBlock* accImag, const Scalar* lhs_ptr, const Scalar* lhs_ptr_imag, const Packet* rhsV, const Packet* rhsVi) { - Packet lhsV = ploadLhs(lhs_ptr); + Packet lhsV = ploadLhs(lhs_ptr); Packet lhsVi; - if(!LhsIsReal) lhsVi = ploadLhs(lhs_ptr_imag); - else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); - - pgerc_common(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi); -} - -template -EIGEN_ALWAYS_INLINE void loadPacketRemaining(const Scalar* lhs_ptr, const Scalar* lhs_ptr_imag, Packet &lhsV, Packet &lhsVi) -{ -#ifdef _ARCH_PWR9 - lhsV = vec_xl_len((Scalar *)lhs_ptr, remaining_rows * sizeof(Scalar)); - if(!LhsIsReal) lhsVi = vec_xl_len((Scalar *)lhs_ptr_imag, remaining_rows * sizeof(Scalar)); + if(!LhsIsReal) lhsVi = ploadLhs(lhs_ptr_imag); else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); -#else - Index i = 0; - do { - lhsV[i] = lhs_ptr[i]; - if(!LhsIsReal) lhsVi[i] = lhs_ptr_imag[i]; - } while (++i < remaining_rows); - if(LhsIsReal) EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); -#endif -} - -template -EIGEN_ALWAYS_INLINE void pgerc(PacketBlock* accReal, PacketBlock* accImag, const Scalar* lhs_ptr, const Scalar* lhs_ptr_imag, const Packet* rhsV, const Packet* rhsVi) -{ - Packet lhsV, lhsVi; - loadPacketRemaining(lhs_ptr, lhs_ptr_imag, lhsV, lhsVi); pgerc_common(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi); } -template -EIGEN_ALWAYS_INLINE Packet ploadLhs(const Scalar* lhs) +template +EIGEN_ALWAYS_INLINE Packet ploadLhs(const __UNPACK_TYPE__(Packet)* lhs) { return ploadu(lhs); } // Zero the accumulator on PacketBlock. -template +template EIGEN_ALWAYS_INLINE void bsetzero(PacketBlock& acc) { - acc.packet[0] = pset1((Scalar)0); - if (N > 1) { - acc.packet[1] = pset1((Scalar)0); - } - if (N > 2) { - acc.packet[2] = pset1((Scalar)0); - } - if (N > 3) { - acc.packet[3] = pset1((Scalar)0); + for (int M = 0; M < N; M++) { + acc.packet[M] = pset1((__UNPACK_TYPE__(Packet))0); } } -// Scale the PacketBlock vectors by alpha. template -EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha) +EIGEN_ALWAYS_INLINE void bscalec_common(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha) { - acc.packet[0] = pmadd(pAlpha, accZ.packet[0], acc.packet[0]); - if (N > 1) { - acc.packet[1] = pmadd(pAlpha, accZ.packet[1], acc.packet[1]); - } - if (N > 2) { - acc.packet[2] = pmadd(pAlpha, accZ.packet[2], acc.packet[2]); - } - if (N > 3) { - acc.packet[3] = pmadd(pAlpha, accZ.packet[3], acc.packet[3]); + for (int M = 0; M < N; M++) { + acc.packet[M] = vec_mul(accZ.packet[M], pAlpha); } } template -EIGEN_ALWAYS_INLINE void bscalec_common(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha) +EIGEN_ALWAYS_INLINE void band(PacketBlock& acc, const Packet& pMask) { - acc.packet[0] = pmul(accZ.packet[0], pAlpha); - if (N > 1) { - acc.packet[1] = pmul(accZ.packet[1], pAlpha); - } - if (N > 2) { - acc.packet[2] = pmul(accZ.packet[2], pAlpha); - } - if (N > 3) { - acc.packet[3] = pmul(accZ.packet[3], pAlpha); + for (int M = 0; M < N; M++) { + acc.packet[M] = pand(acc.packet[M], pMask); } } // Complex version of PacketBlock scaling. -template -EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock& aImag, const Packet& bReal, const Packet& bImag, PacketBlock& cReal, PacketBlock& cImag) +template +EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock& aImag, const Packet& bReal, const Packet& bImag, PacketBlock& cReal, PacketBlock& cImag, const Packet& pMask) { + if (mask && (sizeof(__UNPACK_TYPE__(Packet)) == sizeof(float))) { + band(aReal, pMask); + band(aImag, pMask); + } else { + EIGEN_UNUSED_VARIABLE(pMask); + } + bscalec_common(cReal, aReal, bReal); bscalec_common(cImag, aImag, bReal); @@ -1184,213 +1695,282 @@ EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock(&cImag, bImag, aReal.packet); } -template -EIGEN_ALWAYS_INLINE void band(PacketBlock& acc, const Packet& pMask) -{ - acc.packet[0] = pand(acc.packet[0], pMask); - if (N > 1) { - acc.packet[1] = pand(acc.packet[1], pMask); - } - if (N > 2) { - acc.packet[2] = pand(acc.packet[2], pMask); - } - if (N > 3) { - acc.packet[3] = pand(acc.packet[3], pMask); - } -} - -template -EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock& aImag, const Packet& bReal, const Packet& bImag, PacketBlock& cReal, PacketBlock& cImag, const Packet& pMask) -{ - band(aReal, pMask); - band(aImag, pMask); - - bscalec(aReal, aImag, bReal, bImag, cReal, cImag); -} - // Load a PacketBlock, the N parameters make tunning gemm easier so we can add more accumulators as needed. -template +// +// full = operate (load) on the entire PacketBlock or only half +template EIGEN_ALWAYS_INLINE void bload(PacketBlock& acc, const DataMapper& res, Index row, Index col) { if (StorageOrder == RowMajor) { - acc.packet[0] = res.template loadPacket(row + 0, col); - if (N > 1) { - acc.packet[1] = res.template loadPacket(row + 1, col); - } - if (N > 2) { - acc.packet[2] = res.template loadPacket(row + 2, col); - } - if (N > 3) { - acc.packet[3] = res.template loadPacket(row + 3, col); + for (int M = 0; M < N; M++) { + acc.packet[M] = res.template loadPacket(row + M, col); } if (Complex) { - acc.packet[0+N] = res.template loadPacket(row + 0, col + accCols); - if (N > 1) { - acc.packet[1+N] = res.template loadPacket(row + 1, col + accCols); - } - if (N > 2) { - acc.packet[2+N] = res.template loadPacket(row + 2, col + accCols); - } - if (N > 3) { - acc.packet[3+N] = res.template loadPacket(row + 3, col + accCols); + for (int M = 0; M < N; M++) { + acc.packet[M+N] = res.template loadPacket(row + M, col + accCols); } } } else { - acc.packet[0] = res.template loadPacket(row, col + 0); - if (N > 1) { - acc.packet[1] = res.template loadPacket(row, col + 1); - } - if (N > 2) { - acc.packet[2] = res.template loadPacket(row, col + 2); - } - if (N > 3) { - acc.packet[3] = res.template loadPacket(row, col + 3); + for (int M = 0; M < N; M++) { + acc.packet[M] = res.template loadPacket(row, col + M); } - if (Complex) { - acc.packet[0+N] = res.template loadPacket(row + accCols, col + 0); - if (N > 1) { - acc.packet[1+N] = res.template loadPacket(row + accCols, col + 1); - } - if (N > 2) { - acc.packet[2+N] = res.template loadPacket(row + accCols, col + 2); - } - if (N > 3) { - acc.packet[3+N] = res.template loadPacket(row + accCols, col + 3); + if (Complex && full) { + for (int M = 0; M < N; M++) { + acc.packet[M+N] = res.template loadPacket(row + accCols, col + M); } } } } -const static Packet4i mask41 = { -1, 0, 0, 0 }; -const static Packet4i mask42 = { -1, -1, 0, 0 }; -const static Packet4i mask43 = { -1, -1, -1, 0 }; - -const static Packet2l mask21 = { -1, 0 }; +template +EIGEN_ALWAYS_INLINE void bstore(PacketBlock& acc, const DataMapper& res, Index row) +{ + for (int M = 0; M < N; M++) { + res.template storePacket(row, M, acc.packet[M]); + } +} -template -EIGEN_ALWAYS_INLINE Packet bmask(const int remaining_rows) +#ifdef USE_PARTIAL_PACKETS +template +EIGEN_ALWAYS_INLINE void bload_partial(PacketBlock& acc, const DataMapper& res, Index row, Index elements) { - if (remaining_rows == 0) { - return pset1(float(0.0)); // Not used - } else { - switch (remaining_rows) { - case 1: return Packet(mask41); - case 2: return Packet(mask42); - default: return Packet(mask43); + for (Index M = 0; M < N; M++) { + acc.packet[M] = res.template loadPacketPartial(row, M, elements); + } + if (Complex && full) { + for (Index M = 0; M < N; M++) { + acc.packet[M+N] = res.template loadPacketPartial(row + accCols, M, elements); } } } -template<> -EIGEN_ALWAYS_INLINE Packet2d bmask(const int remaining_rows) +template +EIGEN_ALWAYS_INLINE void bstore_partial(PacketBlock& acc, const DataMapper& res, Index row, Index elements) { - if (remaining_rows == 0) { - return pset1(double(0.0)); // Not used - } else { - return Packet2d(mask21); + for (Index M = 0; M < N; M++) { + res.template storePacketPartial(row, M, acc.packet[M], elements); } } +#endif -template -EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha, const Packet& pMask) +#ifdef _ARCH_PWR10 +#define USE_P10_AND_PVIPR2_0 (EIGEN_COMP_LLVM || (__GNUC__ >= 11)) +#else +#define USE_P10_AND_PVIPR2_0 0 +#endif + +#if !USE_P10_AND_PVIPR2_0 +const static Packet4i mask4[4] = { { 0, 0, 0, 0 }, { -1, 0, 0, 0 }, { -1, -1, 0, 0 }, { -1, -1, -1, 0 } }; +#endif + +template +EIGEN_ALWAYS_INLINE Packet bmask(const Index remaining_rows) { - band(accZ, pMask); +#if USE_P10_AND_PVIPR2_0 +#ifdef _BIG_ENDIAN + return Packet(vec_reve(vec_genwm((1 << remaining_rows) - 1))); +#else + return Packet(vec_genwm((1 << remaining_rows) - 1)); +#endif +#else + return Packet(mask4[remaining_rows]); +#endif +} - bscale(acc, accZ, pAlpha); +template<> +EIGEN_ALWAYS_INLINE Packet2d bmask(const Index remaining_rows) +{ +#if USE_P10_AND_PVIPR2_0 + Packet2d mask2 = Packet2d(vec_gendm(remaining_rows)); +#ifdef _BIG_ENDIAN + return preverse(mask2); +#else + return mask2; +#endif +#else + Packet2l ret = { -remaining_rows, 0 }; + return Packet2d(ret); +#endif } -template EIGEN_ALWAYS_INLINE void -pbroadcastN_old(const __UNPACK_TYPE__(Packet) *a, - Packet& a0, Packet& a1, Packet& a2, Packet& a3) +template +EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha) { - a0 = pset1(a[0]); - if (N > 1) { - a1 = pset1(a[1]); - } else { - EIGEN_UNUSED_VARIABLE(a1); + for (int M = 0; M < N; M++) { + acc.packet[M] = pmadd(pAlpha, accZ.packet[M], acc.packet[M]); } - if (N > 2) { - a2 = pset1(a[2]); +} + +// Scale the PacketBlock vectors by alpha. +template +EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha, const Packet& pMask) +{ + if (mask) { + band(accZ, pMask); } else { - EIGEN_UNUSED_VARIABLE(a2); + EIGEN_UNUSED_VARIABLE(pMask); } - if (N > 3) { - a3 = pset1(a[3]); + + bscale(acc, accZ, pAlpha); +} + +template +EIGEN_ALWAYS_INLINE void pbroadcastN(const __UNPACK_TYPE__(Packet) *ap0, + const __UNPACK_TYPE__(Packet) *ap1, const __UNPACK_TYPE__(Packet) *ap2, + Packet& a0, Packet& a1, Packet& a2, Packet& a3) +{ + a0 = pset1(ap0[0]); + if (N == 4) { + a1 = pset1(ap0[1]); + a2 = pset1(ap0[2]); + a3 = pset1(ap0[3]); + EIGEN_UNUSED_VARIABLE(ap1); + EIGEN_UNUSED_VARIABLE(ap2); } else { - EIGEN_UNUSED_VARIABLE(a3); + if (N > 1) { + a1 = pset1(ap1[0]); + } else { + EIGEN_UNUSED_VARIABLE(a1); + EIGEN_UNUSED_VARIABLE(ap1); + } + if (N > 2) { + a2 = pset1(ap2[0]); + } else { + EIGEN_UNUSED_VARIABLE(a2); + EIGEN_UNUSED_VARIABLE(ap2); + } } } -template<> -EIGEN_ALWAYS_INLINE void pbroadcastN_old(const float* a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) +template<> EIGEN_ALWAYS_INLINE void +pbroadcastN(const float *ap0, const float *, const float *, + Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) +{ + pbroadcast4(ap0, a0, a1, a2, a3); +} + +template<> EIGEN_ALWAYS_INLINE void +pbroadcastN(const float *ap0, const float *ap1, const float *ap2, + Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) { - pbroadcast4(a, a0, a1, a2, a3); + pbroadcastN(ap0, ap1, ap2, a0, a1, a2, a3); } template<> -EIGEN_ALWAYS_INLINE void pbroadcastN_old(const double* a, Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3) +EIGEN_ALWAYS_INLINE void pbroadcastN(const double* ap0, const double *, + const double *, Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3) { - a1 = pload(a); - a3 = pload(a + 2); + a1 = pload(ap0); + a3 = pload(ap0 + 2); a0 = vec_splat(a1, 0); a1 = vec_splat(a1, 1); a2 = vec_splat(a3, 0); a3 = vec_splat(a3, 1); } -template EIGEN_ALWAYS_INLINE void -pbroadcastN(const __UNPACK_TYPE__(Packet) *a, - Packet& a0, Packet& a1, Packet& a2, Packet& a3) +// Grab two decouples real/imaginary PacketBlocks and return two coupled (real/imaginary pairs) PacketBlocks. +template +EIGEN_ALWAYS_INLINE void bcouple_common(PacketBlock& taccReal, PacketBlock& taccImag, PacketBlock& acc1, PacketBlock& acc2) { - a0 = pset1(a[0]); - if (N > 1) { - a1 = pset1(a[1]); - } else { - EIGEN_UNUSED_VARIABLE(a1); + for (int M = 0; M < N; M++) { + acc1.packet[M].v = vec_mergeh(taccReal.packet[M], taccImag.packet[M]); } - if (N > 2) { - a2 = pset1(a[2]); - } else { - EIGEN_UNUSED_VARIABLE(a2); - } - if (N > 3) { - a3 = pset1(a[3]); - } else { - EIGEN_UNUSED_VARIABLE(a3); + + if (full) { + for (int M = 0; M < N; M++) { + acc2.packet[M].v = vec_mergel(taccReal.packet[M], taccImag.packet[M]); + } } } -template<> EIGEN_ALWAYS_INLINE void -pbroadcastN(const float *a, - Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) +template +EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock& taccImag, PacketBlock& tRes, PacketBlock& acc1, PacketBlock& acc2) { - a3 = pload(a); - a0 = vec_splat(a3, 0); - a1 = vec_splat(a3, 1); - a2 = vec_splat(a3, 2); - a3 = vec_splat(a3, 3); + bcouple_common(taccReal, taccImag, acc1, acc2); + + for (int M = 0; M < N; M++) { + acc1.packet[M] = padd(tRes.packet[M], acc1.packet[M]); + } + + if (full) { + for (int M = 0; M < N; M++) { + acc2.packet[M] = padd(tRes.packet[M+N], acc2.packet[M]); + } + } } // PEEL loop factor. #define PEEL 7 #define PEEL_ROW 7 -#define MICRO_UNROLL_PEEL(func) \ +#define MICRO_UNROLL(func) \ func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7) +#define MICRO_NORMAL_ROWS \ + accRows == quad_traits::rows || accRows == 1 + +#define MICRO_NEW_ROWS ((MICRO_NORMAL_ROWS) ? accRows : 1) + +#define MICRO_RHS(ptr, N) rhs_##ptr##N + #define MICRO_ZERO_PEEL(peel) \ if ((PEEL_ROW > peel) && (peel != 0)) { \ - bsetzero(accZero##peel); \ + bsetzero(accZero##peel); \ } else { \ EIGEN_UNUSED_VARIABLE(accZero##peel); \ } -#define MICRO_ZERO_PEEL_ROW \ - MICRO_UNROLL_PEEL(MICRO_ZERO_PEEL); +#define MICRO_ADD(ptr, N) \ + if (MICRO_NORMAL_ROWS) { \ + MICRO_RHS(ptr,0) += (accRows * N); \ + } else { \ + MICRO_RHS(ptr,0) += N; \ + MICRO_RHS(ptr,1) += N; \ + if (accRows == 3) { \ + MICRO_RHS(ptr,2) += N; \ + } \ + } + +#define MICRO_ADD_ROWS(N) MICRO_ADD(ptr, N) + +#define MICRO_BROADCAST1(peel, ptr, rhsV, real) \ + if (MICRO_NORMAL_ROWS) { \ + pbroadcastN(MICRO_RHS(ptr,0) + (accRows * peel), MICRO_RHS(ptr,0), MICRO_RHS(ptr,0), rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ + } else { \ + pbroadcastN(MICRO_RHS(ptr,0) + peel, MICRO_RHS(ptr,1) + peel, MICRO_RHS(ptr,2) + peel, rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ + } + +#define MICRO_BROADCAST(peel) MICRO_BROADCAST1(peel, ptr, rhsV, true) + +#define MICRO_BROADCAST_EXTRA1(ptr, rhsV, real) \ + pbroadcastN(MICRO_RHS(ptr,0), MICRO_RHS(ptr,1), MICRO_RHS(ptr,2), rhsV[0], rhsV[1], rhsV[2], rhsV[3]); + +#define MICRO_BROADCAST_EXTRA \ + Packet rhsV[4]; \ + MICRO_BROADCAST_EXTRA1(ptr, rhsV, true) \ + MICRO_ADD_ROWS(1) + +#define MICRO_SRC2(ptr, N, M) \ + if (MICRO_NORMAL_ROWS) { \ + EIGEN_UNUSED_VARIABLE(strideB); \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr,1)); \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr,2)); \ + } else { \ + MICRO_RHS(ptr,1) = rhs_base + N + M; \ + if (accRows == 3) { \ + MICRO_RHS(ptr,2) = rhs_base + N*2 + M; \ + } else { \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr,2)); \ + } \ + } + +#define MICRO_SRC2_PTR MICRO_SRC2(ptr, strideB, 0) + +#define MICRO_ZERO_PEEL_ROW MICRO_UNROLL(MICRO_ZERO_PEEL) #define MICRO_WORK_PEEL(peel) \ if (PEEL_ROW > peel) { \ - pbroadcastN(rhs_ptr + (accRows * peel), rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ + MICRO_BROADCAST(peel) \ pger(&accZero##peel, lhs_ptr + (remaining_rows * peel), rhsV##peel); \ } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ @@ -1398,9 +1978,9 @@ pbroadcastN(const float *a, #define MICRO_WORK_PEEL_ROW \ Packet rhsV0[4], rhsV1[4], rhsV2[4], rhsV3[4], rhsV4[4], rhsV5[4], rhsV6[4], rhsV7[4]; \ - MICRO_UNROLL_PEEL(MICRO_WORK_PEEL); \ + MICRO_UNROLL(MICRO_WORK_PEEL) \ lhs_ptr += (remaining_rows * PEEL_ROW); \ - rhs_ptr += (accRows * PEEL_ROW); + MICRO_ADD_ROWS(PEEL_ROW) #define MICRO_ADD_PEEL(peel, sum) \ if (PEEL_ROW > peel) { \ @@ -1413,20 +1993,37 @@ pbroadcastN(const float *a, MICRO_ADD_PEEL(4, 0) MICRO_ADD_PEEL(5, 1) MICRO_ADD_PEEL(6, 2) MICRO_ADD_PEEL(7, 3) \ MICRO_ADD_PEEL(2, 0) MICRO_ADD_PEEL(3, 1) MICRO_ADD_PEEL(1, 0) -template +#define MICRO_PREFETCHN1(ptr, N) \ + EIGEN_POWER_PREFETCH(MICRO_RHS(ptr,0)); \ + if (N == 2 || N == 3) { \ + EIGEN_POWER_PREFETCH(MICRO_RHS(ptr,1)); \ + if (N == 3) { \ + EIGEN_POWER_PREFETCH(MICRO_RHS(ptr,2)); \ + } \ + } + +#define MICRO_PREFETCHN(N) MICRO_PREFETCHN1(ptr, N) + +#define MICRO_COMPLEX_PREFETCHN(N) \ + MICRO_PREFETCHN1(ptr_real, N); \ + if(!RhsIsReal) { \ + MICRO_PREFETCHN1(ptr_imag, N); \ + } + +template EIGEN_ALWAYS_INLINE void MICRO_EXTRA_ROW( const Scalar* &lhs_ptr, - const Scalar* &rhs_ptr, + const Scalar* &rhs_ptr0, + const Scalar* &rhs_ptr1, + const Scalar* &rhs_ptr2, PacketBlock &accZero) { - Packet rhsV[4]; - pbroadcastN(rhs_ptr, rhsV[0], rhsV[1], rhsV[2], rhsV[3]); + MICRO_BROADCAST_EXTRA pger(&accZero, lhs_ptr, rhsV); lhs_ptr += remaining_rows; - rhs_ptr += accRows; } -template +template EIGEN_ALWAYS_INLINE void gemm_unrolled_row_iteration( const DataMapper& res, const Scalar* lhs_base, @@ -1434,61 +2031,80 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_row_iteration( Index depth, Index strideA, Index offsetA, + Index strideB, Index row, - Index col, Index rows, - Index cols, const Packet& pAlpha, const Packet& pMask) { - const Scalar* rhs_ptr = rhs_base; + const Scalar* rhs_ptr0 = rhs_base, * rhs_ptr1 = NULL, * rhs_ptr2 = NULL; const Scalar* lhs_ptr = lhs_base + row*strideA + remaining_rows*offsetA; PacketBlock accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7, acc; - bsetzero(accZero0); + MICRO_SRC2_PTR + bsetzero(accZero0); - Index remaining_depth = (col + quad_traits::rows < cols) ? depth : (depth & -quad_traits::rows); + Index remaining_depth = depth & -quad_traits::rows; Index k = 0; if (remaining_depth >= PEEL_ROW) { MICRO_ZERO_PEEL_ROW do { - EIGEN_POWER_PREFETCH(rhs_ptr); + MICRO_PREFETCHN(accRows) EIGEN_POWER_PREFETCH(lhs_ptr); MICRO_WORK_PEEL_ROW } while ((k += PEEL_ROW) + PEEL_ROW <= remaining_depth); MICRO_ADD_PEEL_ROW } - for(; k < remaining_depth; k++) + for(; k < depth; k++) { - MICRO_EXTRA_ROW(lhs_ptr, rhs_ptr, accZero0); + MICRO_EXTRA_ROW(lhs_ptr, rhs_ptr0, rhs_ptr1, rhs_ptr2, accZero0); } - if ((remaining_depth == depth) && (rows >= accCols)) +#ifdef USE_PARTIAL_PACKETS + EIGEN_UNUSED_VARIABLE(rows); + EIGEN_UNUSED_VARIABLE(pMask); + bload_partial(acc, res, row, remaining_rows); + bscale(acc, accZero0, pAlpha); + bstore_partial(acc, res, row, remaining_rows); +#else + bload(acc, res, row, 0); + if ((accRows == 1) || (rows >= accCols)) { - bload(acc, res, row, 0); - bscale(acc, accZero0, pAlpha, pMask); - res.template storePacketBlock(row, 0, acc); + bscale(acc, accZero0, pAlpha, pMask); + bstore(acc, res, row); } else { - for(; k < depth; k++) - { - Packet rhsV[4]; - pbroadcastN(rhs_ptr, rhsV[0], rhsV[1], rhsV[2], rhsV[3]); - pger(&accZero0, lhs_ptr, rhsV); - lhs_ptr += remaining_rows; - rhs_ptr += accRows; - } - + bscale(acc, accZero0, pAlpha, pMask); for(Index j = 0; j < accRows; j++) { - accZero0.packet[j] = vec_mul(pAlpha, accZero0.packet[j]); for(Index i = 0; i < remaining_rows; i++) { - res(row + i, j) += accZero0.packet[j][i]; + res(row + i, j) = acc.packet[j][i]; } } } +#endif } -template +#define MICRO_EXTRA(MICRO_EXTRA_UNROLL, value, is_col) \ + switch(value) { \ + default: \ + MICRO_EXTRA_UNROLL(1) \ + break; \ + case 2: \ + if (is_col || (sizeof(Scalar) == sizeof(float))) { \ + MICRO_EXTRA_UNROLL(2) \ + } \ + break; \ + case 3: \ + if (is_col || (sizeof(Scalar) == sizeof(float))) { \ + MICRO_EXTRA_UNROLL(3) \ + } \ + break; \ + } + +#define MICRO_EXTRA_ROWS(N) \ + gemm_unrolled_row_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, pAlpha, pMask); + +template EIGEN_ALWAYS_INLINE void gemm_extra_row( const DataMapper& res, const Scalar* lhs_base, @@ -1496,46 +2112,20 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row( Index depth, Index strideA, Index offsetA, + Index strideB, Index row, - Index col, Index rows, - Index cols, Index remaining_rows, const Packet& pAlpha, const Packet& pMask) { - switch(remaining_rows) { - case 1: - gemm_unrolled_row_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, rows, cols, pAlpha, pMask); - break; - case 2: - if (sizeof(Scalar) == sizeof(float)) { - gemm_unrolled_row_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, rows, cols, pAlpha, pMask); - } - break; - default: - if (sizeof(Scalar) == sizeof(float)) { - gemm_unrolled_row_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, rows, cols, pAlpha, pMask); - } - break; - } + MICRO_EXTRA(MICRO_EXTRA_ROWS, remaining_rows, false) } -#define MICRO_UNROLL(func) \ - func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7) - #define MICRO_UNROLL_WORK(func, func2, peel) \ - MICRO_UNROLL(func2); \ - func(0,peel) func(1,peel) func(2,peel) func(3,peel) \ - func(4,peel) func(5,peel) func(6,peel) func(7,peel) - -#define MICRO_LOAD_ONE(iter) \ - if (unroll_factor > iter) { \ - lhsV##iter = ploadLhs(lhs_ptr##iter); \ - lhs_ptr##iter += accCols; \ - } else { \ - EIGEN_UNUSED_VARIABLE(lhsV##iter); \ - } + MICRO_UNROLL(func2); \ + func(0,peel) func(1,peel) func(2,peel) func(3,peel) \ + func(4,peel) func(5,peel) func(6,peel) func(7,peel) #define MICRO_WORK_ONE(iter, peel) \ if (unroll_factor > iter) { \ @@ -1545,7 +2135,7 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row( #define MICRO_TYPE_PEEL4(func, func2, peel) \ if (PEEL > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \ - pbroadcastN(rhs_ptr + (accRows * peel), rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ + MICRO_BROADCAST(peel) \ MICRO_UNROLL_WORK(func, func2, peel) \ } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ @@ -1553,79 +2143,95 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row( #define MICRO_UNROLL_TYPE_PEEL(M, func, func1, func2) \ Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M], rhsV4[M], rhsV5[M], rhsV6[M], rhsV7[M]; \ - func(func1,func2,0); func(func1,func2,1); \ - func(func1,func2,2); func(func1,func2,3); \ - func(func1,func2,4); func(func1,func2,5); \ - func(func1,func2,6); func(func1,func2,7); + func(func1,func2,0) func(func1,func2,1) \ + func(func1,func2,2) func(func1,func2,3) \ + func(func1,func2,4) func(func1,func2,5) \ + func(func1,func2,6) func(func1,func2,7) #define MICRO_UNROLL_TYPE_ONE(M, func, func1, func2) \ Packet rhsV0[M]; \ - func(func1,func2,0); + func(func1,func2,0) + +#define MICRO_UNROLL_TYPE(MICRO_TYPE, size) \ + MICRO_TYPE(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_ONE) \ + MICRO_ADD_ROWS(size) -#define MICRO_ONE_PEEL4 \ - MICRO_UNROLL_TYPE_PEEL(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_ONE); \ - rhs_ptr += (accRows * PEEL); +#define MICRO_ONE_PEEL4 MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_PEEL, PEEL) -#define MICRO_ONE4 \ - MICRO_UNROLL_TYPE_ONE(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_ONE); \ - rhs_ptr += accRows; +#define MICRO_ONE4 MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_ONE, 1) #define MICRO_DST_PTR_ONE(iter) \ if (unroll_factor > iter) { \ - bsetzero(accZero##iter); \ + bsetzero(accZero##iter); \ } else { \ EIGEN_UNUSED_VARIABLE(accZero##iter); \ } #define MICRO_DST_PTR MICRO_UNROLL(MICRO_DST_PTR_ONE) -#define MICRO_SRC_PTR_ONE(iter) \ - if (unroll_factor > iter) { \ - lhs_ptr##iter = lhs_base + ( (row/accCols) + iter )*strideA*accCols; \ - } else { \ - EIGEN_UNUSED_VARIABLE(lhs_ptr##iter); \ - } - #define MICRO_SRC_PTR MICRO_UNROLL(MICRO_SRC_PTR_ONE) -#define MICRO_PREFETCH_ONE(iter) \ - if (unroll_factor > iter) { \ - EIGEN_POWER_PREFETCH(lhs_ptr##iter); \ - } - #define MICRO_PREFETCH MICRO_UNROLL(MICRO_PREFETCH_ONE) +#ifdef USE_PARTIAL_PACKETS #define MICRO_STORE_ONE(iter) \ if (unroll_factor > iter) { \ - bload(acc, res, row + iter*accCols, 0); \ - bscale(acc, accZero##iter, pAlpha); \ - res.template storePacketBlock(row + iter*accCols, 0, acc); \ + if (MICRO_NORMAL_PARTIAL(iter)) { \ + bload(acc, res, row + iter*accCols, 0); \ + bscale(acc, accZero##iter, pAlpha); \ + bstore(acc, res, row + iter*accCols); \ + } else { \ + bload_partial(acc, res, row + iter*accCols, accCols2); \ + bscale(acc, accZero##iter, pAlpha); \ + bstore_partial(acc, res, row + iter*accCols, accCols2); \ + } \ } +#else +#define MICRO_STORE_ONE(iter) \ + if (unroll_factor > iter) { \ + bload(acc, res, row + iter*accCols, 0); \ + bscale(acc, accZero##iter, pAlpha, pMask); \ + bstore(acc, res, row + iter*accCols); \ + } +#endif #define MICRO_STORE MICRO_UNROLL(MICRO_STORE_ONE) -template -EIGEN_STRONG_INLINE void gemm_unrolled_iteration( +#ifdef USE_PARTIAL_PACKETS +template +#else +template +#endif +EIGEN_ALWAYS_INLINE void gemm_unrolled_iteration( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, + Index offsetA, + Index strideB, Index& row, - const Packet& pAlpha) + const Packet& pAlpha, +#ifdef USE_PARTIAL_PACKETS + Index accCols2 +#else + const Packet& pMask +#endif + ) { - const Scalar* rhs_ptr = rhs_base; - const Scalar* lhs_ptr0 = NULL, * lhs_ptr1 = NULL, * lhs_ptr2 = NULL, * lhs_ptr3 = NULL, * lhs_ptr4 = NULL, * lhs_ptr5 = NULL, * lhs_ptr6 = NULL, * lhs_ptr7 = NULL; + const Scalar* rhs_ptr0 = rhs_base, * rhs_ptr1 = NULL, * rhs_ptr2 = NULL; + const Scalar* lhs_ptr0 = NULL, * lhs_ptr1 = NULL, * lhs_ptr2 = NULL, * lhs_ptr3 = NULL, * lhs_ptr4 = NULL, * lhs_ptr5 = NULL, * lhs_ptr6 = NULL, * lhs_ptr7 = NULL; PacketBlock accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7; PacketBlock acc; + MICRO_SRC2_PTR MICRO_SRC_PTR MICRO_DST_PTR Index k = 0; for(; k + PEEL <= depth; k+= PEEL) { - EIGEN_POWER_PREFETCH(rhs_ptr); + MICRO_PREFETCHN(accRows) MICRO_PREFETCH MICRO_ONE_PEEL4 } @@ -1635,10 +2241,20 @@ EIGEN_STRONG_INLINE void gemm_unrolled_iteration( } MICRO_STORE - row += unroll_factor*accCols; + MICRO_UPDATE } -template +#ifdef USE_PARTIAL_PACKETS +#define MICRO_UNROLL_ITER2(N, M) \ + gemm_unrolled_iteration(res3, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlpha, M ? remaining_rows : accCols); \ + if (M) return; +#else +#define MICRO_UNROLL_ITER2(N, M) \ + gemm_unrolled_iteration(res3, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlpha, pMask); \ + if (M) return; +#endif + +template EIGEN_ALWAYS_INLINE void gemm_cols( const DataMapper& res, const Scalar* blockA, @@ -1650,55 +2266,54 @@ EIGEN_ALWAYS_INLINE void gemm_cols( Index offsetB, Index col, Index rows, - Index cols, Index remaining_rows, const Packet& pAlpha, const Packet& pMask) { const DataMapper res3 = res.getSubMapper(0, col); - const Scalar* rhs_base = blockB + col*strideB + accRows*offsetB; + const Scalar* rhs_base = blockB + col*strideB + MICRO_NEW_ROWS*offsetB; const Scalar* lhs_base = blockA + accCols*offsetA; Index row = 0; -#define MAX_UNROLL 6 +#define MAX_UNROLL 7 while(row + MAX_UNROLL*accCols <= rows) { - gemm_unrolled_iteration(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + MICRO_UNROLL_ITER2(MAX_UNROLL, 0); } switch( (rows-row)/accCols ) { #if MAX_UNROLL > 7 case 7: - gemm_unrolled_iteration<7, Scalar, Packet, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 7) break; #endif #if MAX_UNROLL > 6 case 6: - gemm_unrolled_iteration<6, Scalar, Packet, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 6) break; #endif #if MAX_UNROLL > 5 case 5: - gemm_unrolled_iteration<5, Scalar, Packet, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 5) break; #endif #if MAX_UNROLL > 4 case 4: - gemm_unrolled_iteration<4, Scalar, Packet, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 4) break; #endif #if MAX_UNROLL > 3 case 3: - gemm_unrolled_iteration<3, Scalar, Packet, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 3) break; #endif #if MAX_UNROLL > 2 case 2: - gemm_unrolled_iteration<2, Scalar, Packet, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 2) break; #endif #if MAX_UNROLL > 1 case 1: - gemm_unrolled_iteration<1, Scalar, Packet, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 1) break; #endif default: @@ -1708,12 +2323,15 @@ EIGEN_ALWAYS_INLINE void gemm_cols( if(remaining_rows > 0) { - gemm_extra_row(res3, blockA, rhs_base, depth, strideA, offsetA, row, col, rows, cols, remaining_rows, pAlpha, pMask); + gemm_extra_row(res3, blockA, rhs_base, depth, strideA, offsetA, strideB, row, rows, remaining_rows, pAlpha, pMask); } } -template -EIGEN_STRONG_INLINE void gemm_extra_cols( +#define MICRO_EXTRA_COLS(N) \ + gemm_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlpha, pMask); + +template +EIGEN_ALWAYS_INLINE void gemm_extra_cols( const DataMapper& res, const Scalar* blockA, const Scalar* blockB, @@ -1729,15 +2347,13 @@ EIGEN_STRONG_INLINE void gemm_extra_cols( const Packet& pAlpha, const Packet& pMask) { - for (; col < cols; col++) { - gemm_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlpha, pMask); - } + MICRO_EXTRA(MICRO_EXTRA_COLS, cols-col, true) } /**************** * GEMM kernels * * **************/ -template +template EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index remaining_rows = rows % accCols; @@ -1746,15 +2362,18 @@ EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const if( strideB == -1 ) strideB = depth; const Packet pAlpha = pset1(alpha); - const Packet pMask = bmask((const int)(remaining_rows)); + const Packet pMask = bmask(remaining_rows); Index col = 0; for(; col + accRows <= cols; col += accRows) { - gemm_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlpha, pMask); + gemm_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlpha, pMask); } - gemm_extra_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlpha, pMask); + if (col != cols) + { + gemm_extra_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlpha, pMask); + } } #define accColsC (accCols / 2) @@ -1765,41 +2384,79 @@ EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const #define PEEL_COMPLEX 3 #define PEEL_COMPLEX_ROW 3 -#define MICRO_COMPLEX_UNROLL_PEEL(func) \ +#define MICRO_COMPLEX_UNROLL(func) \ func(0) func(1) func(2) func(3) #define MICRO_COMPLEX_ZERO_PEEL(peel) \ if ((PEEL_COMPLEX_ROW > peel) && (peel != 0)) { \ - bsetzero(accReal##peel); \ - bsetzero(accImag##peel); \ + bsetzero(accReal##peel); \ + bsetzero(accImag##peel); \ } else { \ EIGEN_UNUSED_VARIABLE(accReal##peel); \ EIGEN_UNUSED_VARIABLE(accImag##peel); \ } -#define MICRO_COMPLEX_ZERO_PEEL_ROW \ - MICRO_COMPLEX_UNROLL_PEEL(MICRO_COMPLEX_ZERO_PEEL); +#define MICRO_COMPLEX_ADD_ROWS(N, used) \ + MICRO_ADD(ptr_real, N) \ + if (!RhsIsReal) { \ + MICRO_ADD(ptr_imag, N) \ + } else if (used) { \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag,0)); \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag,1)); \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag,2)); \ + } + +#define MICRO_COMPLEX_BROADCAST(peel) \ + MICRO_BROADCAST1(peel, ptr_real, rhsV, false) \ + if (!RhsIsReal) { \ + MICRO_BROADCAST1(peel, ptr_imag, rhsVi, false) \ + } else { \ + EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ + } + +#define MICRO_COMPLEX_BROADCAST_EXTRA \ + Packet rhsV[4], rhsVi[4]; \ + MICRO_BROADCAST_EXTRA1(ptr_real, rhsV, false) \ + if(!RhsIsReal) { \ + MICRO_BROADCAST_EXTRA1(ptr_imag, rhsVi, false) \ + } else { \ + EIGEN_UNUSED_VARIABLE(rhsVi); \ + } \ + MICRO_COMPLEX_ADD_ROWS(1, true) + +#define MICRO_COMPLEX_SRC2_PTR \ + MICRO_SRC2(ptr_real, strideB*advanceCols, 0) \ + if (!RhsIsReal) { \ + MICRO_RHS(ptr_imag,0) = rhs_base + MICRO_NEW_ROWS*strideB; \ + MICRO_SRC2(ptr_imag, strideB*advanceCols, strideB) \ + } else { \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag,0)); \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag,1)); \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag,2)); \ + } + +#define MICRO_COMPLEX_ZERO_PEEL_ROW MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_ZERO_PEEL) #define MICRO_COMPLEX_WORK_PEEL(peel) \ if (PEEL_COMPLEX_ROW > peel) { \ - pbroadcastN_old(rhs_ptr_real + (accRows * peel), rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ - if(!RhsIsReal) pbroadcastN_old(rhs_ptr_imag + (accRows * peel), rhsVi##peel[0], rhsVi##peel[1], rhsVi##peel[2], rhsVi##peel[3]); \ + MICRO_COMPLEX_BROADCAST(peel) \ pgerc(&accReal##peel, &accImag##peel, lhs_ptr_real + (remaining_rows * peel), lhs_ptr_imag + (remaining_rows * peel), rhsV##peel, rhsVi##peel); \ } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } +#define MICRO_COMPLEX_ADD_COLS(size) \ + lhs_ptr_real += (remaining_rows * size); \ + if(!LhsIsReal) lhs_ptr_imag += (remaining_rows * size); \ + else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); + #define MICRO_COMPLEX_WORK_PEEL_ROW \ Packet rhsV0[4], rhsV1[4], rhsV2[4], rhsV3[4]; \ Packet rhsVi0[4], rhsVi1[4], rhsVi2[4], rhsVi3[4]; \ - MICRO_COMPLEX_UNROLL_PEEL(MICRO_COMPLEX_WORK_PEEL); \ - lhs_ptr_real += (remaining_rows * PEEL_COMPLEX_ROW); \ - if(!LhsIsReal) lhs_ptr_imag += (remaining_rows * PEEL_COMPLEX_ROW); \ - else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); \ - rhs_ptr_real += (accRows * PEEL_COMPLEX_ROW); \ - if(!RhsIsReal) rhs_ptr_imag += (accRows * PEEL_COMPLEX_ROW); \ - else EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); + MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_WORK_PEEL) \ + MICRO_COMPLEX_ADD_COLS(PEEL_COMPLEX_ROW) \ + MICRO_COMPLEX_ADD_ROWS(PEEL_COMPLEX_ROW, false) #define MICRO_COMPLEX_ADD_PEEL(peel, sum) \ if (PEEL_COMPLEX_ROW > peel) { \ @@ -1813,25 +2470,19 @@ EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const MICRO_COMPLEX_ADD_PEEL(2, 0) MICRO_COMPLEX_ADD_PEEL(3, 1) \ MICRO_COMPLEX_ADD_PEEL(1, 0) -template +template EIGEN_ALWAYS_INLINE void MICRO_COMPLEX_EXTRA_ROW( const Scalar* &lhs_ptr_real, const Scalar* &lhs_ptr_imag, - const Scalar* &rhs_ptr_real, const Scalar* &rhs_ptr_imag, + const Scalar* &rhs_ptr_real0, const Scalar* &rhs_ptr_real1, const Scalar* &rhs_ptr_real2, + const Scalar* &rhs_ptr_imag0, const Scalar* &rhs_ptr_imag1, const Scalar* &rhs_ptr_imag2, PacketBlock &accReal, PacketBlock &accImag) { - Packet rhsV[4], rhsVi[4]; - pbroadcastN_old(rhs_ptr_real, rhsV[0], rhsV[1], rhsV[2], rhsV[3]); - if(!RhsIsReal) pbroadcastN_old(rhs_ptr_imag, rhsVi[0], rhsVi[1], rhsVi[2], rhsVi[3]); + MICRO_COMPLEX_BROADCAST_EXTRA pgerc(&accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi); - lhs_ptr_real += remaining_rows; - if(!LhsIsReal) lhs_ptr_imag += remaining_rows; - else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); - rhs_ptr_real += accRows; - if(!RhsIsReal) rhs_ptr_imag += accRows; - else EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); + MICRO_COMPLEX_ADD_COLS(1) } -template +template EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration( const DataMapper& res, const Scalar* lhs_base, @@ -1841,17 +2492,13 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration( Index offsetA, Index strideB, Index row, - Index col, Index rows, - Index cols, const Packet& pAlphaReal, const Packet& pAlphaImag, const Packet& pMask) { - const Scalar* rhs_ptr_real = rhs_base; - const Scalar* rhs_ptr_imag = NULL; - if(!RhsIsReal) rhs_ptr_imag = rhs_base + accRows*strideB; - else EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); + const Scalar* rhs_ptr_real0 = rhs_base, * rhs_ptr_real1 = NULL, * rhs_ptr_real2 = NULL; + const Scalar* rhs_ptr_imag0 = NULL, * rhs_ptr_imag1 = NULL, * rhs_ptr_imag2 = NULL; const Scalar* lhs_ptr_real = lhs_base + advanceRows*row*strideA + remaining_rows*offsetA; const Scalar* lhs_ptr_imag = NULL; if(!LhsIsReal) lhs_ptr_imag = lhs_ptr_real + remaining_rows*strideA; @@ -1861,19 +2508,18 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration( PacketBlock acc0, acc1; PacketBlock tRes; - bsetzero(accReal0); - bsetzero(accImag0); + MICRO_COMPLEX_SRC2_PTR - Index remaining_depth = (col + quad_traits::rows < cols) ? depth : (depth & -quad_traits::rows); + bsetzero(accReal0); + bsetzero(accImag0); + + Index remaining_depth = depth & -quad_traits::rows; Index k = 0; if (remaining_depth >= PEEL_COMPLEX_ROW) { MICRO_COMPLEX_ZERO_PEEL_ROW do { - EIGEN_POWER_PREFETCH(rhs_ptr_real); - if(!RhsIsReal) { - EIGEN_POWER_PREFETCH(rhs_ptr_imag); - } + MICRO_COMPLEX_PREFETCHN(accRows) EIGEN_POWER_PREFETCH(lhs_ptr_real); if(!LhsIsReal) { EIGEN_POWER_PREFETCH(lhs_ptr_imag); @@ -1882,53 +2528,45 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration( } while ((k += PEEL_COMPLEX_ROW) + PEEL_COMPLEX_ROW <= remaining_depth); MICRO_COMPLEX_ADD_PEEL_ROW } - for(; k < remaining_depth; k++) + for(; k < depth; k++) { - MICRO_COMPLEX_EXTRA_ROW(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real, rhs_ptr_imag, accReal0, accImag0); + MICRO_COMPLEX_EXTRA_ROW(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real0, rhs_ptr_real1, rhs_ptr_real2, rhs_ptr_imag0, rhs_ptr_imag1, rhs_ptr_imag2, accReal0, accImag0); } - if ((remaining_depth == depth) && (rows >= accCols)) + constexpr bool full = (remaining_rows > accColsC); + bload(tRes, res, row, 0); + if ((accRows == 1) || (rows >= accCols)) { - bload(tRes, res, row, 0); - bscalec(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); - bcouple(taccReal, taccImag, tRes, acc0, acc1); - res.template storePacketBlock(row + 0, 0, acc0); - res.template storePacketBlock(row + accColsC, 0, acc1); - } else { - for(; k < depth; k++) - { - Packet rhsV[4], rhsVi[4]; - pbroadcastN_old(rhs_ptr_real, rhsV[0], rhsV[1], rhsV[2], rhsV[3]); - if(!RhsIsReal) pbroadcastN_old(rhs_ptr_imag, rhsVi[0], rhsVi[1], rhsVi[2], rhsVi[3]); - pgerc(&accReal0, &accImag0, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi); - lhs_ptr_real += remaining_rows; - if(!LhsIsReal) lhs_ptr_imag += remaining_rows; - rhs_ptr_real += accRows; - if(!RhsIsReal) rhs_ptr_imag += accRows; + bscalec(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); + bcouple(taccReal, taccImag, tRes, acc0, acc1); + bstore(acc0, res, row + 0); + if (full) { + bstore(acc1, res, row + accColsC); } - - bscalec(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag); - bcouple_common(taccReal, taccImag, acc0, acc1); + } else { + bscalec(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); + bcouple(taccReal, taccImag, tRes, acc0, acc1); if ((sizeof(Scalar) == sizeof(float)) && (remaining_rows == 1)) { for(Index j = 0; j < accRows; j++) { - res(row + 0, j) += pfirst(acc0.packet[j]); + res(row + 0, j) = pfirst(acc0.packet[j]); } } else { - for(Index j = 0; j < accRows; j++) { - PacketBlock acc2; - acc2.packet[0] = res.template loadPacket(row + 0, j) + acc0.packet[j]; - res.template storePacketBlock(row + 0, j, acc2); - if(remaining_rows > accColsC) { - res(row + accColsC, j) += pfirst(acc1.packet[j]); + bstore(acc0, res, row + 0); + if (full) { + for(Index j = 0; j < accRows; j++) { + res(row + accColsC, j) = pfirst(acc1.packet[j]); } } } } } -template +#define MICRO_COMPLEX_EXTRA_ROWS(N) \ + gemm_unrolled_complex_row_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, pAlphaReal, pAlphaImag, pMask); + +template EIGEN_ALWAYS_INLINE void gemm_complex_extra_row( const DataMapper& res, const Scalar* lhs_base, @@ -1938,51 +2576,18 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row( Index offsetA, Index strideB, Index row, - Index col, Index rows, - Index cols, Index remaining_rows, const Packet& pAlphaReal, const Packet& pAlphaImag, const Packet& pMask) { - switch(remaining_rows) { - case 1: - gemm_unrolled_complex_row_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, rows, cols, pAlphaReal, pAlphaImag, pMask); - break; - case 2: - if (sizeof(Scalar) == sizeof(float)) { - gemm_unrolled_complex_row_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, rows, cols, pAlphaReal, pAlphaImag, pMask); - } - break; - default: - if (sizeof(Scalar) == sizeof(float)) { - gemm_unrolled_complex_row_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, rows, cols, pAlphaReal, pAlphaImag, pMask); - } - break; - } + MICRO_EXTRA(MICRO_COMPLEX_EXTRA_ROWS, remaining_rows, false) } -#define MICRO_COMPLEX_UNROLL(func) \ - func(0) func(1) func(2) func(3) - #define MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \ - MICRO_COMPLEX_UNROLL(func2); \ - func(0,peel) func(1,peel) func(2,peel) func(3,peel) - -#define MICRO_COMPLEX_LOAD_ONE(iter) \ - if (unroll_factor > iter) { \ - lhsV##iter = ploadLhs(lhs_ptr_real##iter); \ - if(!LhsIsReal) { \ - lhsVi##iter = ploadLhs(lhs_ptr_real##iter + imag_delta); \ - } else { \ - EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ - } \ - lhs_ptr_real##iter += accCols; \ - } else { \ - EIGEN_UNUSED_VARIABLE(lhsV##iter); \ - EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ - } + MICRO_COMPLEX_UNROLL(func2); \ + func(0,peel) func(1,peel) func(2,peel) func(3,peel) #define MICRO_COMPLEX_WORK_ONE4(iter, peel) \ if (unroll_factor > iter) { \ @@ -1993,12 +2598,7 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row( if (PEEL_COMPLEX > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3; \ Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3; \ - pbroadcastN_old(rhs_ptr_real + (accRows * peel), rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ - if(!RhsIsReal) { \ - pbroadcastN_old(rhs_ptr_imag + (accRows * peel), rhsVi##peel[0], rhsVi##peel[1], rhsVi##peel[2], rhsVi##peel[3]); \ - } else { \ - EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ - } \ + MICRO_COMPLEX_BROADCAST(peel) \ MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \ } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ @@ -2008,27 +2608,25 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row( #define MICRO_COMPLEX_UNROLL_TYPE_PEEL(M, func, func1, func2) \ Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M]; \ Packet rhsVi0[M], rhsVi1[M], rhsVi2[M], rhsVi3[M]; \ - func(func1,func2,0); func(func1,func2,1); \ - func(func1,func2,2); func(func1,func2,3); + func(func1,func2,0) func(func1,func2,1) \ + func(func1,func2,2) func(func1,func2,3) #define MICRO_COMPLEX_UNROLL_TYPE_ONE(M, func, func1, func2) \ Packet rhsV0[M], rhsVi0[M];\ - func(func1,func2,0); + func(func1,func2,0) + +#define MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_TYPE, size) \ + MICRO_COMPLEX_TYPE(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_ONE) \ + MICRO_COMPLEX_ADD_ROWS(size, false) -#define MICRO_COMPLEX_ONE_PEEL4 \ - MICRO_COMPLEX_UNROLL_TYPE_PEEL(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_ONE); \ - rhs_ptr_real += (accRows * PEEL_COMPLEX); \ - if(!RhsIsReal) rhs_ptr_imag += (accRows * PEEL_COMPLEX); +#define MICRO_COMPLEX_ONE_PEEL4 MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_PEEL, PEEL_COMPLEX) -#define MICRO_COMPLEX_ONE4 \ - MICRO_COMPLEX_UNROLL_TYPE_ONE(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_ONE); \ - rhs_ptr_real += accRows; \ - if(!RhsIsReal) rhs_ptr_imag += accRows; +#define MICRO_COMPLEX_ONE4 MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_ONE, 1) #define MICRO_COMPLEX_DST_PTR_ONE(iter) \ if (unroll_factor > iter) { \ - bsetzero(accReal##iter); \ - bsetzero(accImag##iter); \ + bsetzero(accReal##iter); \ + bsetzero(accImag##iter); \ } else { \ EIGEN_UNUSED_VARIABLE(accReal##iter); \ EIGEN_UNUSED_VARIABLE(accImag##iter); \ @@ -2036,53 +2634,42 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row( #define MICRO_COMPLEX_DST_PTR MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_DST_PTR_ONE) -#define MICRO_COMPLEX_SRC_PTR_ONE(iter) \ - if (unroll_factor > iter) { \ - lhs_ptr_real##iter = lhs_base + ( ((advanceRows*row)/accCols) + iter*advanceRows )*strideA*accCols; \ - } else { \ - EIGEN_UNUSED_VARIABLE(lhs_ptr_real##iter); \ - } - #define MICRO_COMPLEX_SRC_PTR MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE) -#define MICRO_COMPLEX_PREFETCH_ONE(iter) \ - if (unroll_factor > iter) { \ - EIGEN_POWER_PREFETCH(lhs_ptr_real##iter); \ - } - #define MICRO_COMPLEX_PREFETCH MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_PREFETCH_ONE) #define MICRO_COMPLEX_STORE_ONE(iter) \ if (unroll_factor > iter) { \ - bload(tRes, res, row + iter*accCols, 0); \ - bscalec(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, taccImag); \ - bcouple(taccReal, taccImag, tRes, acc0, acc1); \ - res.template storePacketBlock(row + iter*accCols + 0, 0, acc0); \ - res.template storePacketBlock(row + iter*accCols + accColsC, 0, acc1); \ + constexpr bool full = ((MICRO_NORMAL(iter)) || (accCols2 > accColsC)); \ + bload(tRes, res, row + iter*accCols, 0); \ + bscalec(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); \ + bcouple(taccReal, taccImag, tRes, acc0, acc1); \ + bstore(acc0, res, row + iter*accCols + 0); \ + if (full) { \ + bstore(acc1, res, row + iter*accCols + accColsC); \ + } \ } #define MICRO_COMPLEX_STORE MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_STORE_ONE) -template -EIGEN_STRONG_INLINE void gemm_complex_unrolled_iteration( +template +EIGEN_ALWAYS_INLINE void gemm_complex_unrolled_iteration( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, + Index offsetA, Index strideB, Index& row, const Packet& pAlphaReal, - const Packet& pAlphaImag) + const Packet& pAlphaImag, + const Packet& pMask) { - const Scalar* rhs_ptr_real = rhs_base; - const Scalar* rhs_ptr_imag = NULL; + const Scalar* rhs_ptr_real0 = rhs_base, * rhs_ptr_real1 = NULL, * rhs_ptr_real2 = NULL; + const Scalar* rhs_ptr_imag0 = NULL, * rhs_ptr_imag1 = NULL, * rhs_ptr_imag2 = NULL; const Index imag_delta = accCols*strideA; - if(!RhsIsReal) { - rhs_ptr_imag = rhs_base + accRows*strideB; - } else { - EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); - } + const Index imag_delta2 = accCols2*strideA; const Scalar* lhs_ptr_real0 = NULL, * lhs_ptr_real1 = NULL; const Scalar* lhs_ptr_real2 = NULL, * lhs_ptr_real3 = NULL; PacketBlock accReal0, accImag0, accReal1, accImag1; @@ -2091,16 +2678,14 @@ EIGEN_STRONG_INLINE void gemm_complex_unrolled_iteration( PacketBlock acc0, acc1; PacketBlock tRes; + MICRO_COMPLEX_SRC2_PTR MICRO_COMPLEX_SRC_PTR MICRO_COMPLEX_DST_PTR Index k = 0; for(; k + PEEL_COMPLEX <= depth; k+= PEEL_COMPLEX) { - EIGEN_POWER_PREFETCH(rhs_ptr_real); - if(!RhsIsReal) { - EIGEN_POWER_PREFETCH(rhs_ptr_imag); - } + MICRO_COMPLEX_PREFETCHN(accRows) MICRO_COMPLEX_PREFETCH MICRO_COMPLEX_ONE_PEEL4 } @@ -2110,10 +2695,14 @@ EIGEN_STRONG_INLINE void gemm_complex_unrolled_iteration( } MICRO_COMPLEX_STORE - row += unroll_factor*accCols; + MICRO_COMPLEX_UPDATE } -template +#define MICRO_COMPLEX_UNROLL_ITER2(N, M) \ + gemm_complex_unrolled_iteration(res3, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlphaReal, pAlphaImag, pMask); \ + if (M) return; + +template EIGEN_ALWAYS_INLINE void gemm_complex_cols( const DataMapper& res, const Scalar* blockA, @@ -2125,7 +2714,6 @@ EIGEN_ALWAYS_INLINE void gemm_complex_cols( Index offsetB, Index col, Index rows, - Index cols, Index remaining_rows, const Packet& pAlphaReal, const Packet& pAlphaImag, @@ -2133,33 +2721,33 @@ EIGEN_ALWAYS_INLINE void gemm_complex_cols( { const DataMapper res3 = res.getSubMapper(0, col); - const Scalar* rhs_base = blockB + advanceCols*col*strideB + accRows*offsetB; + const Scalar* rhs_base = blockB + advanceCols*col*strideB + MICRO_NEW_ROWS*offsetB; const Scalar* lhs_base = blockA + accCols*offsetA; Index row = 0; -#define MAX_COMPLEX_UNROLL 3 +#define MAX_COMPLEX_UNROLL 4 while(row + MAX_COMPLEX_UNROLL*accCols <= rows) { - gemm_complex_unrolled_iteration(res3, lhs_base, rhs_base, depth, strideA, strideB, row, pAlphaReal, pAlphaImag); + MICRO_COMPLEX_UNROLL_ITER2(MAX_COMPLEX_UNROLL, 0); } switch( (rows-row)/accCols ) { #if MAX_COMPLEX_UNROLL > 4 case 4: - gemm_complex_unrolled_iteration<4, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res3, lhs_base, rhs_base, depth, strideA, strideB, row, pAlphaReal, pAlphaImag); + MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 4) break; #endif #if MAX_COMPLEX_UNROLL > 3 case 3: - gemm_complex_unrolled_iteration<3, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res3, lhs_base, rhs_base, depth, strideA, strideB, row, pAlphaReal, pAlphaImag); + MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 3) break; #endif #if MAX_COMPLEX_UNROLL > 2 case 2: - gemm_complex_unrolled_iteration<2, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res3, lhs_base, rhs_base, depth, strideA, strideB, row, pAlphaReal, pAlphaImag); + MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 2) break; #endif #if MAX_COMPLEX_UNROLL > 1 case 1: - gemm_complex_unrolled_iteration<1, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res3, lhs_base, rhs_base, depth, strideA, strideB, row, pAlphaReal, pAlphaImag); + MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 1) break; #endif default: @@ -2169,12 +2757,15 @@ EIGEN_ALWAYS_INLINE void gemm_complex_cols( if(remaining_rows > 0) { - gemm_complex_extra_row(res3, blockA, rhs_base, depth, strideA, offsetA, strideB, row, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask); + gemm_complex_extra_row(res3, blockA, rhs_base, depth, strideA, offsetA, strideB, row, rows, remaining_rows, pAlphaReal, pAlphaImag, pMask); } } -template -EIGEN_STRONG_INLINE void gemm_complex_extra_cols( +#define MICRO_COMPLEX_EXTRA_COLS(N) \ + gemm_complex_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlphaReal, pAlphaImag, pMask); + +template +EIGEN_ALWAYS_INLINE void gemm_complex_extra_cols( const DataMapper& res, const Scalar* blockA, const Scalar* blockB, @@ -2191,12 +2782,10 @@ EIGEN_STRONG_INLINE void gemm_complex_extra_cols( const Packet& pAlphaImag, const Packet& pMask) { - for (; col < cols; col++) { - gemm_complex_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask); - } + MICRO_EXTRA(MICRO_COMPLEX_EXTRA_COLS, cols-col, true) } -template +template EIGEN_STRONG_INLINE void gemm_complex(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index remaining_rows = rows % accCols; @@ -2206,7 +2795,7 @@ EIGEN_STRONG_INLINE void gemm_complex(const DataMapper& res, const LhsScalar* bl const Packet pAlphaReal = pset1(alpha.real()); const Packet pAlphaImag = pset1(alpha.imag()); - const Packet pMask = bmask((const int)(remaining_rows)); + const Packet pMask = bmask(remaining_rows); const Scalar* blockA = (Scalar *) blockAc; const Scalar* blockB = (Scalar *) blockBc; @@ -2214,16 +2803,566 @@ EIGEN_STRONG_INLINE void gemm_complex(const DataMapper& res, const LhsScalar* bl Index col = 0; for(; col + accRows <= cols; col += accRows) { - gemm_complex_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask); + gemm_complex_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlphaReal, pAlphaImag, pMask); } - gemm_complex_extra_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask); + if (col != cols) + { + gemm_complex_extra_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask); + } } #undef accColsC #undef advanceCols #undef advanceRows +EIGEN_ALWAYS_INLINE bool supportsMMA() +{ +#if defined(EIGEN_ALTIVEC_MMA_ONLY) + return true; +#elif defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH) && defined(__BUILTIN_CPU_SUPPORTS__) + return __builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma"); +#else + return false; // No dynamic dispatch for LLVM or older GCC +#endif +} + +EIGEN_ALWAYS_INLINE Packet4f loadAndMultiplyF32(Packet4f acc, const Packet4f pAlpha, float* result) +{ + Packet4f result_block = ploadu(result); + return pmadd(acc, pAlpha, result_block); +} + +template +EIGEN_ALWAYS_INLINE void storeF32(float*& result, Packet4f result_block, Index rows, Index extra_rows) +{ + if (lhsExtraRows) { + pstoreu_partial(result, result_block, extra_rows); + } else { + pstoreu(result, result_block); + } + result += rows; +} + +template +EIGEN_ALWAYS_INLINE void storeResults(Packet4f (&acc)[4], Index rows, const Packet4f pAlpha, float* result, Index extra_cols, Index extra_rows) +{ + Index x = 0; + if (rhsExtraCols) { + do{ + Packet4f result_block = loadAndMultiplyF32(acc[x], pAlpha, result); + storeF32(result, result_block, rows, extra_rows); + } while (++x < extra_cols); + } else { + Packet4f result_block[4]; + float *result2 = result; + do{ + result_block[x] = loadAndMultiplyF32(acc[x], pAlpha, result); + result += rows; + } while (++x < 4); + x = 0; + do{ + storeF32(result2, result_block[x], rows, extra_rows); + } while (++x < 4); + } +} + +EIGEN_ALWAYS_INLINE Packet4f oneConvertBF16Hi(Packet8us data) +{ + Packet8us z = pset1(0); +#ifdef _BIG_ENDIAN + return reinterpret_cast(vec_mergeh(data, z)); +#else + return reinterpret_cast(vec_mergeh(z, data)); +#endif +} + +EIGEN_ALWAYS_INLINE Packet4f oneConvertBF16Lo(Packet8us data) +{ + Packet8us z = pset1(0); +#ifdef _BIG_ENDIAN + return reinterpret_cast(vec_mergel(data, z)); +#else + return reinterpret_cast(vec_mergel(z, data)); +#endif +} + +template +EIGEN_ALWAYS_INLINE void storeConvertTwoBF16(float* to, PacketBlock& block, Index extra = 0) +{ + if (N < 4) { + pstoreu_partial(to + 0, oneConvertBF16Hi(block.packet[0].m_val), extra); + } else if (N >= (M*8+4)) { + pstoreu(to + 0, oneConvertBF16Hi(block.packet[M].m_val)); + if (N >= 8) { + pstoreu(to + 4, oneConvertBF16Lo(block.packet[M].m_val)); + } + } +} + +template +EIGEN_ALWAYS_INLINE void storeConvertBlockBF16(float* to, PacketBlock& block, Index extra) +{ + storeConvertTwoBF16(to + 0, block, extra); + if (N >= 16) { + storeConvertTwoBF16(to + 8, block); + } + if (N >= 32) { + storeConvertTwoBF16(to + 16, block); + storeConvertTwoBF16(to + 24, block); + } +} + +template +EIGEN_ALWAYS_INLINE Packet8bf loadBF16fromResult(bfloat16* src, Index resInc) +{ + if (non_unit_stride) { + return pgather(src + delta*resInc, resInc); + } else { + return ploadu(src + delta); + } +} + +static Packet16uc p16uc_MERGE16_32_1 = { 0, 1, 16,17, 2, 3, 18,19, 0, 1, 16,17, 2, 3, 18,19 }; +static Packet16uc p16uc_MERGE16_32_2 = { 4, 5, 20,21, 6, 7, 22,23, 4, 5, 20,21, 6, 7, 22,23 }; +static Packet16uc p16uc_MERGE16_32_3 = { 8, 9, 24,25, 10,11, 26,27, 8, 9, 24,25, 10,11, 26,27 }; +static Packet16uc p16uc_MERGE16_32_4 = { 12,13, 28,29, 14,15, 30,31, 12,13, 28,29, 14,15, 30,31 }; + +static Packet16uc p16uc_MERGE16_32_5 = { 0,1, 16,17, 16,17, 16,17, 0,1, 16,17, 16,17, 16,17 }; +static Packet16uc p16uc_MERGE16_32_6 = { 2,3, 18,19, 18,19, 18,19, 2,3, 18,19, 18,19, 18,19 }; +static Packet16uc p16uc_MERGE16_32_7 = { 4,5, 20,21, 20,21, 20,21, 4,5, 20,21, 20,21, 20,21 }; +static Packet16uc p16uc_MERGE16_32_8 = { 6,7, 22,23, 22,23, 22,23, 6,7, 22,23, 22,23, 22,23 }; + +EIGEN_ALWAYS_INLINE Packet4f oneConvertBF16Perm(Packet8us data, Packet16uc mask) +{ + Packet8us z = pset1(0); +#ifdef _BIG_ENDIAN + return reinterpret_cast(vec_perm(data, z, mask)); +#else + return reinterpret_cast(vec_perm(z, data, mask)); +#endif +} + +template +EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32DupOne(float *result, Index rows, const bfloat16* src, Index extra_rows) +{ + Packet4f dup[4*4]; + Packet8bf data[4]; + + for (Index i = 0; i < size; i++) { + data[i] = ploadu(src + rows*i); + } + + for (Index i = 0, j = 0; i < size; i++, j += 4) { + dup[j+0] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_5 : p16uc_MERGE16_32_1); + dup[j+1] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_6 : p16uc_MERGE16_32_2); + dup[j+2] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_7 : p16uc_MERGE16_32_3); + dup[j+3] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_8 : p16uc_MERGE16_32_4); + } + + for (Index j = 0; j < 4*size; j += 4) { + if (lhsExtraRows) { + Packet4f z = pset1(float(0)); + Index i = 0; + do { + pstoreu(result + (j+i)*4, dup[j+i]); + } while (++i < extra_rows); + do { + pstoreu(result + (j+i)*4, z); + } while (++i < 4); + } else { + for (Index i = 0; i < 4; i++) { + pstoreu(result + (j+i)*4, dup[j+i]); + } + } + } +} + +template +EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32Dup(float *result, Index cols, Index rows, const bfloat16* src, Index delta, Index extra_rows) +{ + Index col = 0; + src += delta*2; + for(; col + 4*2 <= cols; col += 4*2, result += 4*4*4, src += 4*rows) { + convertArrayPointerBF16toF32DupOne(result, rows, src, extra_rows); + } + for(; col + 2 <= cols; col += 2, result += 4*4, src += rows) { + convertArrayPointerBF16toF32DupOne(result, rows, src, extra_rows); + } + if (cols & 1) { + convertArrayPointerBF16toF32DupOne(result, rows, src - delta, extra_rows); + } +} + +template +EIGEN_ALWAYS_INLINE void convertPointerBF16toF32(Index& i, float *result, Index rows, bfloat16*& src, Index resInc) +{ + constexpr Index extra = ((size < 4) ? 4 : size); + while (i + size <= rows) { + PacketBlock r32; + r32.packet[0] = loadBF16fromResult(src, resInc); + if (size >= 16) { + r32.packet[1] = loadBF16fromResult(src, resInc); + } + if (size >= 32) { + r32.packet[2] = loadBF16fromResult(src, resInc); + r32.packet[3] = loadBF16fromResult(src, resInc); + } + storeConvertBlockBF16(result + i, r32, rows & 3); + i += extra; src += extra*resInc; + if (size != 32) break; + } +} + +template +EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32(float *result, Index cols, Index rows, bfloat16* src, Index resInc) +{ + for(Index col = 0; col < cols; col++, src += (rows*resInc), result += rows) { + Index i = 0; + bfloat16* src2 = src; + convertPointerBF16toF32<32, non_unit_stride>(i, result, rows, src2, resInc); + convertPointerBF16toF32<16, non_unit_stride>(i, result, rows, src2, resInc); + convertPointerBF16toF32<8, non_unit_stride>(i, result, rows, src2, resInc); + convertPointerBF16toF32<4, non_unit_stride>(i, result, rows, src2, resInc); + convertPointerBF16toF32<1, non_unit_stride>(i, result, rows, src2, resInc); + } +} + +template +EIGEN_ALWAYS_INLINE void zeroAccumulators(Packet4f (&acc)[num_acc][size]) +{ + Packet4f z = pset1(float(0)); + + for(Index k = 0; k < num_acc; k++) { + for(Index j = 0; j < size; j++) { + acc[k][j] = z; + } + } +} + +template +EIGEN_ALWAYS_INLINE void tranposeResults(Packet4f (&acc)[num_acc][4]) +{ + for(Index i = 0; i < num_acc; i++) { + Packet4ui t0, t1, t2, t3; + t0 = vec_mergeh(reinterpret_cast(acc[i][0]), reinterpret_cast(acc[i][2])); + t1 = vec_mergel(reinterpret_cast(acc[i][0]), reinterpret_cast(acc[i][2])); + t2 = vec_mergeh(reinterpret_cast(acc[i][1]), reinterpret_cast(acc[i][3])); + t3 = vec_mergel(reinterpret_cast(acc[i][1]), reinterpret_cast(acc[i][3])); + acc[i][0] = reinterpret_cast(vec_mergeh(t0, t2)); + acc[i][1] = reinterpret_cast(vec_mergel(t0, t2)); + acc[i][2] = reinterpret_cast(vec_mergeh(t1, t3)); + acc[i][3] = reinterpret_cast(vec_mergel(t1, t3)); + } +} + +template +EIGEN_ALWAYS_INLINE void addResults(Packet4f (&acc)[num_acc][4]) +{ + for(Index i = 0, j = 0; j < num_acc; i++, j += 2) { + for(Index x = 0, y = 0; x < 2; x++, y += 2) { + for(Index w = 0, z = 0; w < 2; w++, z += 2) { + acc[i][y+w] = acc[j+x][z+0] + acc[j+x][z+1]; + } + } + } +} + +template +EIGEN_ALWAYS_INLINE void outputResultsVSX(Packet4f (&acc)[num_acc][4], Index rows, const Packet4f pAlpha, float* result, const Index extra_cols, Index extra_rows) +{ + tranposeResults(acc); + addResults(acc); + + constexpr Index real_rhs = ((num_rhs / 2) - (rhsExtraCols ? 1 : 0)); + Index k = 0; + for(Index i = 0; i < real_rhs; i++, result += 4*rows, k++){ + storeResults(acc[k], rows, pAlpha, result, extra_cols, extra_rows); + } + if(rhsExtraCols) { + storeResults(acc[k], rows, pAlpha, result, extra_cols, extra_rows); + } +} + +template +EIGEN_ALWAYS_INLINE void loadTwoRhsFloat32(const float* block, Index strideB, Index i, Packet4f& dhs0, Packet4f &dhs1) +{ + dhs0 = ploadu(block + strideB*i + 0); + if (zero) { + Packet4f dhs2 = pset1(float(0)); + dhs1 = vec_mergel(dhs0, dhs2); + dhs0 = vec_mergeh(dhs0, dhs2); + } else { + dhs1 = ploadu(block + strideB*i + 4); + } +} + +template +EIGEN_ALWAYS_INLINE void KLoop +( + const float* indexA, + const float* indexB, + Packet4f (&acc)[num_acc][4], + Index strideB, + Index k, + Index offsetB, + Index extra_cols +) +{ + constexpr Index num_lhs = 4; + Packet4f lhs[num_lhs], rhs[num_rhs]; + + constexpr Index real_rhs = (num_rhs - (rhsExtraCols ? 2 : 0)); + for(Index i = 0; i < real_rhs; i += 2){ + loadTwoRhsFloat32(indexB + k*4, strideB, i, rhs[i + 0], rhs[i + 1]); + } + if(rhsExtraCols) { + loadTwoRhsFloat32(indexB + k*extra_cols - offsetB, strideB, real_rhs, rhs[real_rhs + 0], rhs[real_rhs + 1]); + } + + indexA += 2*k*4; + for(Index j = 0; j < num_lhs; j++) { + lhs[j] = ploadu(indexA + j*4); + } + + for(Index j = 0; j < num_rhs; j++) { + for(Index i = 0; i < num_lhs; i++) { + acc[j][i] = pmadd(rhs[j], lhs[i], acc[j][i]); + } + } +} + +template +EIGEN_ALWAYS_INLINE void colVSXLoopBodyIter(Index depth, Index rows, const Packet4f pAlpha, const float* indexA, const float* indexB, Index strideB, Index offsetB, float* result, const Index extra_cols, const Index extra_rows) +{ + constexpr Index num_rhs = num_acc; + + Packet4f acc[num_acc][4]; + + zeroAccumulators(acc); + + Index k; + for(k = 0; k + 2 <= depth; k += 2){ + KLoop(indexA, indexB, acc, strideB, k, offsetB, extra_cols); + } + if(depth&1){ + KLoop(indexA, indexB, acc, strideB, k, offsetB, extra_cols); + } + + outputResultsVSX(acc, rows, pAlpha, result, extra_cols, extra_rows); +} + +// No more than 4 (uses 2X the accumulators or 8X the number of VSX registers) +#define MAX_BFLOAT16_ACC_VSX 4 + +template +void colVSXLoopBody(Index& col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float* indexA, const float* indexB, Index strideB, Index offsetB, float* result) +{ + constexpr Index step = (num_acc * 4); // each accumulator has 4 elements + const Index extra_cols = (rhsExtraCols) ? (cols & 3) : 0; + const Index extra_rows = (lhsExtraRows) ? (rows & 3) : 0; + constexpr bool multiIters = !rhsExtraCols && (num_acc == MAX_BFLOAT16_ACC_VSX); + + do{ + colVSXLoopBodyIter(depth, rows, pAlpha, indexA, indexB, strideB, offsetB, result, extra_cols, extra_rows); + + indexB += strideB*(num_acc * 2); + result += rows*step; + } while(multiIters && (step <= cols - (col += step))); +} + +template +EIGEN_ALWAYS_INLINE void colVSXLoopBodyExtraN(Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float* indexA, const float* blockB, Index strideB, Index offsetB, float* result) +{ + if (MAX_BFLOAT16_ACC_VSX > num_acc) { + colVSXLoopBody(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + } +} + +template +void colVSXLoopBodyExtra(Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float* indexA, const float* blockB, Index strideB, Index offsetB, float* result) +{ + switch ((cols - col) >> 2) { + case 3: + colVSXLoopBodyExtraN<3, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + break; + case 2: + colVSXLoopBodyExtraN<2, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + break; + case 1: + colVSXLoopBodyExtraN<1, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + break; + default: + if (rhsExtraCols) { + colVSXLoopBody<1, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + } + break; + } +} + +template +EIGEN_ALWAYS_INLINE void colVSXLoops(Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexA, const float* indexA2, const float* blockB2, Index strideA, Index strideB, Index offsetB, float* result2) +{ + Index delta_rows = 2*(lhsExtraRows ? (rows & 3) : size); + for (Index row = 0; row < size; row += 4) { + convertArrayPointerBF16toF32Dup(const_cast(indexA2), strideA, delta_rows, indexA, row, rows & 3); + + const float *blockB = blockB2; + float *result = result2 + row; + + Index col = 0; + if (cols >= (MAX_BFLOAT16_ACC_VSX * 4)) { + colVSXLoopBody(col, depth, cols, rows, pAlpha, indexA2, blockB, strideB, 0, result); + blockB += (strideB >> 1)*col; + result += rows*col; + } + if (cols & 3) { + colVSXLoopBodyExtra(col, depth, cols, rows, pAlpha, indexA2, blockB, strideB, offsetB, result); + } else { + colVSXLoopBodyExtra(col, depth, cols, rows, pAlpha, indexA2, blockB, strideB, 0, result); + } + } +} + +template +EIGEN_ALWAYS_INLINE void calcVSXColLoops(const bfloat16*& indexA, const float* indexA2, Index& row, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float* indexB, Index strideA, Index strideB, Index offsetA, Index offsetB, Index bigSuffix, float *result) +{ + if ((size == 16) || (rows & size)) { + indexA += size*offsetA; + colVSXLoops(depth, cols, rows, pAlpha, indexA, indexA2, indexB, strideA, strideB, offsetB, result + row); + row += size; + indexA += bigSuffix*size/16; + } +} + +template +EIGEN_ALWAYS_INLINE void convertBF16toF32(Index& i, float *result, Index rows, const DataMapper& src) +{ + constexpr Index extra = ((size < 4) ? 4 : size); + while (i + size <= rows) { + PacketBlock r32; + r32.packet[0] = src.template loadPacket(i + 0); + if (size >= 16) { + r32.packet[1] = src.template loadPacket(i + 8); + } + if (size >= 32) { + r32.packet[2] = src.template loadPacket(i + 16); + r32.packet[3] = src.template loadPacket(i + 24); + } + storeConvertBlockBF16(result + i, r32, rows & 3); + i += extra; + if (size != 32) break; + } +} + +template +EIGEN_ALWAYS_INLINE void convertArrayBF16toF32(float *result, Index cols, Index rows, const DataMapper& src) +{ + typedef typename DataMapper::LinearMapper LinearMapper; + for(Index j = 0; j < cols; j++, result += rows){ + const LinearMapper src2 = src.getLinearMapper(0, j); + Index i = 0; + convertBF16toF32<32, LinearMapper>(i, result, rows, src2); + convertBF16toF32<16, LinearMapper>(i, result, rows, src2); + convertBF16toF32<8, LinearMapper>(i, result, rows, src2); + convertBF16toF32<4, LinearMapper>(i, result, rows, src2); + convertBF16toF32<1, LinearMapper>(i, result, rows, src2); + } +} + +EIGEN_ALWAYS_INLINE Packet8bf convertF32toBF16VSX(const float *res) +{ + return F32ToBf16Both(ploadu(res + 0), ploadu(res + 4)); +} + +template +EIGEN_ALWAYS_INLINE void convertArrayF32toBF16ColVSX(float *result, Index col, Index rows, const DataMapper& res) +{ + const DataMapper res2 = res.getSubMapper(0, col); + Index row; + float *result2 = result + col*rows; + for(row = 0; row + 8 <= rows; row += 8, result2 += 8){ + // get and save block + PacketBlock block; + for(Index j = 0; j < size; j++){ + block.packet[j] = convertF32toBF16VSX(result2 + j*rows); + } + res2.template storePacketBlock(row, 0, block); + } + // extra rows + if(row < rows){ + for(Index j = 0; j < size; j++){ + Packet8bf fp16 = convertF32toBF16VSX(result2 + j*rows); + res2.template storePacketPartial(row, j, fp16, rows & 7); + } + } +} + +template +EIGEN_ALWAYS_INLINE void convertArrayF32toBF16VSX(float *result, Index cols, Index rows, const DataMapper& res) +{ + Index col; + for(col = 0; col + 4 <= cols; col += 4){ + convertArrayF32toBF16ColVSX(result, col, rows, res); + } + // extra cols + switch (cols - col) { + case 1: + convertArrayF32toBF16ColVSX(result, col, rows, res); + break; + case 2: + convertArrayF32toBF16ColVSX(result, col, rows, res); + break; + case 3: + convertArrayF32toBF16ColVSX(result, col, rows, res); + break; + } +} + +template +void gemmbfloat16(const DataMapper& res, const bfloat16* indexA, const bfloat16* indexB, Index rows, Index depth, Index cols, bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) +{ + float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha); + const Packet4f pAlpha = pset1(falpha); + + if( strideA == -1 ) strideA = depth; + if( strideB == -1 ) strideB = depth; + + ei_declare_aligned_stack_constructed_variable(float, result, cols*rows, 0); + ei_declare_aligned_stack_constructed_variable(float, indexB2, strideB*cols, 0); + ei_declare_aligned_stack_constructed_variable(float, indexA2, ((strideA + 1) & -2)*4*2, 0); + + convertArrayBF16toF32(result, cols, rows, res); + convertArrayPointerBF16toF32(indexB2, cols, strideB, const_cast(indexB)); + + Index bigSuffix = 2*8*(strideA-offsetA); + float* indexBF32 = indexB2 + 4*offsetB; + offsetB *= 3; + strideB *= 2; + + Index row = 0; + // LHS (8x16) block + while(row + 16 <= rows){ + calcVSXColLoops<16>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB, bigSuffix, result); + } + // LHS (8x8) block + calcVSXColLoops<8>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB, bigSuffix, result); + // LHS (8x4) block + calcVSXColLoops<4>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB, bigSuffix, result); + // extra rows + if(rows & 3){ + // This index is the beginning of remaining block. + colVSXLoops<4, true>(depth, cols, rows, pAlpha, indexA, indexA2, indexBF32, strideA, strideB, offsetB, result + row); + } + + // Convert back to bfloat16 + convertArrayF32toBF16VSX(result, cols, rows, res); +} + +#undef MAX_BFLOAT16_ACC_VSX + +#include "MatrixVectorProduct.h" + /************************************ * ppc64le template specializations * * **********************************/ @@ -2237,7 +3376,7 @@ template ::operator()(double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { - dhs_pack pack; + dhs_pack pack; pack(blockA, lhs, depth, rows, stride, offset); } @@ -2251,7 +3390,7 @@ template ::operator()(double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { - dhs_pack pack; + dhs_pack pack; pack(blockA, lhs, depth, rows, stride, offset); } @@ -2266,7 +3405,7 @@ template ::operator()(double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { - dhs_pack pack; + dhs_pack pack; pack(blockB, rhs, depth, cols, stride, offset); } @@ -2280,11 +3419,67 @@ template ::operator()(double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { - dhs_pack pack; + dhs_pack pack; + pack(blockB, rhs, depth, cols, stride, offset); +} + +template +struct gemm_pack_rhs +{ + void operator()(bfloat16* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0); +}; + +template +void gemm_pack_rhs + ::operator()(bfloat16* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) +{ + dhs_pack pack; + pack(blockB, rhs, depth, cols, stride, offset); +} + +template +struct gemm_pack_rhs +{ + void operator()(bfloat16* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0); +}; + +template +void gemm_pack_rhs + ::operator()(bfloat16* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) +{ + dhs_pack pack; pack(blockB, rhs, depth, cols, stride, offset); } #endif +template +struct gemm_pack_lhs +{ + void operator()(bfloat16* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); +}; + +template +void gemm_pack_lhs + ::operator()(bfloat16* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) +{ + dhs_pack pack; + pack(blockA, lhs, depth, rows, stride, offset); +} + +template +struct gemm_pack_lhs +{ + void operator()(bfloat16* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); +}; + +template +void gemm_pack_lhs + ::operator()(bfloat16* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) +{ + dhs_pack pack; + pack(blockA, lhs, depth, rows, stride, offset); +} + template struct gemm_pack_lhs { @@ -2295,7 +3490,7 @@ template ::operator()(float* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { - dhs_pack pack; + dhs_pack pack; pack(blockA, lhs, depth, rows, stride, offset); } @@ -2309,7 +3504,7 @@ template ::operator()(float* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { - dhs_pack pack; + dhs_pack pack; pack(blockA, lhs, depth, rows, stride, offset); } @@ -2323,7 +3518,7 @@ template, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> ::operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { - dhs_cpack pack; + dhs_cpack pack; pack(blockA, lhs, depth, rows, stride, offset); } @@ -2337,7 +3532,7 @@ template, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> ::operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { - dhs_cpack pack; + dhs_cpack pack; pack(blockA, lhs, depth, rows, stride, offset); } @@ -2352,7 +3547,7 @@ template ::operator()(float* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { - dhs_pack pack; + dhs_pack pack; pack(blockB, rhs, depth, cols, stride, offset); } @@ -2366,7 +3561,7 @@ template ::operator()(float* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { - dhs_pack pack; + dhs_pack pack; pack(blockB, rhs, depth, cols, stride, offset); } #endif @@ -2381,7 +3576,7 @@ template, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> ::operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { - dhs_cpack pack; + dhs_cpack pack; pack(blockB, rhs, depth, cols, stride, offset); } @@ -2395,7 +3590,7 @@ template, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> ::operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { - dhs_cpack pack; + dhs_cpack pack; pack(blockB, rhs, depth, cols, stride, offset); } @@ -2409,7 +3604,7 @@ template, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> ::operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { - dhs_cpack pack; + dhs_cpack pack; pack(blockA, lhs, depth, rows, stride, offset); } @@ -2423,7 +3618,7 @@ template, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> ::operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { - dhs_cpack pack; + dhs_cpack pack; pack(blockA, lhs, depth, rows, stride, offset); } @@ -2437,7 +3632,7 @@ template, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> ::operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { - dhs_cpack pack; + dhs_cpack pack; pack(blockB, rhs, depth, cols, stride, offset); } @@ -2451,7 +3646,7 @@ template, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> ::operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { - dhs_cpack pack; + dhs_cpack pack; pack(blockB, rhs, depth, cols, stride, offset); } @@ -2475,22 +3670,13 @@ void gebp_kernel::rows; const Index accCols = quad_traits::size; - void (*gemm_function)(const DataMapper&, const float*, const float*, Index, Index, Index, float, Index, Index, Index, Index); - - #ifdef EIGEN_ALTIVEC_MMA_ONLY - //generate with MMA only - gemm_function = &Eigen::internal::gemmMMA; - #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) - if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ - gemm_function = &Eigen::internal::gemmMMA; - } - else{ - gemm_function = &Eigen::internal::gemm; - } - #else - gemm_function = &Eigen::internal::gemm; + static void (*gemm_function)(const DataMapper&, const float*, const float*, Index, Index, Index, float, Index, Index, Index, Index) = + #ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H + (supportsMMA()) ? + &Eigen::internal::gemmMMA : #endif - gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); + &Eigen::internal::gemm; + gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template @@ -2513,23 +3699,14 @@ void gebp_kernel, std::complex, Index, DataMapper, mr { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; - void (*gemm_function)(const DataMapper&, const std::complex*, const std::complex*, - Index, Index, Index, std::complex, Index, Index, Index, Index); - - #ifdef EIGEN_ALTIVEC_MMA_ONLY - //generate with MMA only - gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; - #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) - if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ - gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; - } - else{ - gemm_function = &Eigen::internal::gemm_complex, std::complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; - } - #else - gemm_function = &Eigen::internal::gemm_complex, std::complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; - #endif - gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); + static void (*gemm_function)(const DataMapper&, const std::complex*, const std::complex*, + Index, Index, Index, std::complex, Index, Index, Index, Index) = + #ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H + (supportsMMA()) ? + &Eigen::internal::gemm_complexMMA, std::complex, std::complex, float, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false> : + #endif + &Eigen::internal::gemm_complex, std::complex, std::complex, float, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; + gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template @@ -2552,22 +3729,14 @@ void gebp_kernel, Index, DataMapper, mr, nr, Conjugat { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; - void (*gemm_function)(const DataMapper&, const float*, const std::complex*, - Index, Index, Index, std::complex, Index, Index, Index, Index); - #ifdef EIGEN_ALTIVEC_MMA_ONLY - //generate with MMA only - gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; - #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) - if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ - gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; - } - else{ - gemm_function = &Eigen::internal::gemm_complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; - } - #else - gemm_function = &Eigen::internal::gemm_complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; - #endif - gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); + static void (*gemm_function)(const DataMapper&, const float*, const std::complex*, + Index, Index, Index, std::complex, Index, Index, Index, Index) = + #ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H + (supportsMMA()) ? + &Eigen::internal::gemm_complexMMA, std::complex, float, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false> : + #endif + &Eigen::internal::gemm_complex, std::complex, float, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; + gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template @@ -2590,22 +3759,14 @@ void gebp_kernel, float, Index, DataMapper, mr, nr, Conjugat { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; - void (*gemm_function)(const DataMapper&, const std::complex*, const float*, - Index, Index, Index, std::complex, Index, Index, Index, Index); - #ifdef EIGEN_ALTIVEC_MMA_ONLY - //generate with MMA only - gemm_function = &Eigen::internal::gemm_complexMMA, float, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; - #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) - if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ - gemm_function = &Eigen::internal::gemm_complexMMA, float, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; - } - else{ - gemm_function = &Eigen::internal::gemm_complex, float, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; - } - #else - gemm_function = &Eigen::internal::gemm_complex, float, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; - #endif - gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); + static void (*gemm_function)(const DataMapper&, const std::complex*, const float*, + Index, Index, Index, std::complex, Index, Index, Index, Index) = + #ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H + (supportsMMA()) ? + &Eigen::internal::gemm_complexMMA, float, std::complex, float, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true> : + #endif + &Eigen::internal::gemm_complex, float, std::complex, float, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; + gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template @@ -2627,22 +3788,13 @@ void gebp_kernel::rows; const Index accCols = quad_traits::size; - void (*gemm_function)(const DataMapper&, const double*, const double*, Index, Index, Index, double, Index, Index, Index, Index); - - #ifdef EIGEN_ALTIVEC_MMA_ONLY - //generate with MMA only - gemm_function = &Eigen::internal::gemmMMA; - #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) - if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ - gemm_function = &Eigen::internal::gemmMMA; - } - else{ - gemm_function = &Eigen::internal::gemm; - } - #else - gemm_function = &Eigen::internal::gemm; + static void (*gemm_function)(const DataMapper&, const double*, const double*, Index, Index, Index, double, Index, Index, Index, Index) = + #ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H + (supportsMMA()) ? + &Eigen::internal::gemmMMA : #endif - gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); + &Eigen::internal::gemm; + gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template @@ -2665,22 +3817,14 @@ void gebp_kernel, std::complex, Index, DataMapper, { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; - void (*gemm_function)(const DataMapper&, const std::complex*, const std::complex*, - Index, Index, Index, std::complex, Index, Index, Index, Index); - #ifdef EIGEN_ALTIVEC_MMA_ONLY - //generate with MMA only - gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; - #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) - if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ - gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; - } - else{ - gemm_function = &Eigen::internal::gemm_complex, std::complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; - } - #else - gemm_function = &Eigen::internal::gemm_complex, std::complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; - #endif - gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); + static void (*gemm_function)(const DataMapper&, const std::complex*, const std::complex*, + Index, Index, Index, std::complex, Index, Index, Index, Index) = + #ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H + (supportsMMA()) ? + &Eigen::internal::gemm_complexMMA, std::complex, std::complex, double, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false> : + #endif + &Eigen::internal::gemm_complex, std::complex, std::complex, double, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; + gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template @@ -2703,22 +3847,14 @@ void gebp_kernel, double, Index, DataMapper, mr, nr, Conjug { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; - void (*gemm_function)(const DataMapper&, const std::complex*, const double*, - Index, Index, Index, std::complex, Index, Index, Index, Index); - #ifdef EIGEN_ALTIVEC_MMA_ONLY - //generate with MMA only - gemm_function = &Eigen::internal::gemm_complexMMA, double, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; - #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) - if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ - gemm_function = &Eigen::internal::gemm_complexMMA, double, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; - } - else{ - gemm_function = &Eigen::internal::gemm_complex, double, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; - } - #else - gemm_function = &Eigen::internal::gemm_complex, double, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; - #endif - gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); + static void (*gemm_function)(const DataMapper&, const std::complex*, const double*, + Index, Index, Index, std::complex, Index, Index, Index, Index) = + #ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H + (supportsMMA()) ? + &Eigen::internal::gemm_complexMMA, double, std::complex, double, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true> : + #endif + &Eigen::internal::gemm_complex, double, std::complex, double, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; + gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template @@ -2741,22 +3877,40 @@ void gebp_kernel, Index, DataMapper, mr, nr, Conjug { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; - void (*gemm_function)(const DataMapper&, const double*, const std::complex*, - Index, Index, Index, std::complex, Index, Index, Index, Index); - #ifdef EIGEN_ALTIVEC_MMA_ONLY - //generate with MMA only - gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; - #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) - if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ - gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; - } - else{ - gemm_function = &Eigen::internal::gemm_complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; - } - #else - gemm_function = &Eigen::internal::gemm_complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; - #endif - gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); + static void (*gemm_function)(const DataMapper&, const double*, const std::complex*, + Index, Index, Index, std::complex, Index, Index, Index, Index) = + #ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H + (supportsMMA()) ? + &Eigen::internal::gemm_complexMMA, std::complex, double, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false> : + #endif + &Eigen::internal::gemm_complex, std::complex, double, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; + gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); + } + +template +struct gebp_kernel +{ + typedef typename quad_traits::vectortype Packet; + typedef typename quad_traits::rhstype RhsPacket; + + void operator()(const DataMapper& res, const bfloat16* blockA, const bfloat16* blockB, + Index rows, Index depth, Index cols, bfloat16 alpha, + Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0); +}; + +template +void gebp_kernel + ::operator()(const DataMapper& res, const bfloat16* blockA, const bfloat16* blockB, + Index rows, Index depth, Index cols, bfloat16 alpha, + Index strideA, Index strideB, Index offsetA, Index offsetB) + { + static void (*gemm_function)(const DataMapper&, const bfloat16*, const bfloat16*, Index, Index, Index, bfloat16, Index, Index, Index, Index) = + #ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H + (supportsMMA()) ? + &Eigen::internal::gemmMMAbfloat16 : + #endif + &Eigen::internal::gemmbfloat16; + gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } } // end namespace internal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h index bf01dba1cad..fa1755f94e3 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h @@ -5,11 +5,18 @@ #define EIGEN_POWER_PREFETCH(p) #endif +#if defined(_ARCH_PWR9) || defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH) +#define USE_PARTIAL_PACKETS +#endif + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template +template EIGEN_ALWAYS_INLINE void gemm_extra_row( const DataMapper& res, const Scalar* lhs_base, @@ -17,16 +24,15 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row( Index depth, Index strideA, Index offsetA, + Index strideB, Index row, - Index col, Index rows, - Index cols, Index remaining_rows, const Packet& pAlpha, const Packet& pMask); -template -EIGEN_STRONG_INLINE void gemm_extra_cols( +template +EIGEN_ALWAYS_INLINE void gemm_extra_cols( const DataMapper& res, const Scalar* blockA, const Scalar* blockB, @@ -43,9 +49,9 @@ EIGEN_STRONG_INLINE void gemm_extra_cols( const Packet& pMask); template -EIGEN_ALWAYS_INLINE Packet bmask(const int remaining_rows); +EIGEN_ALWAYS_INLINE Packet bmask(const Index remaining_rows); -template +template EIGEN_ALWAYS_INLINE void gemm_complex_extra_row( const DataMapper& res, const Scalar* lhs_base, @@ -55,16 +61,14 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row( Index offsetA, Index strideB, Index row, - Index col, Index rows, - Index cols, Index remaining_rows, const Packet& pAlphaReal, const Packet& pAlphaImag, const Packet& pMask); -template -EIGEN_STRONG_INLINE void gemm_complex_extra_cols( +template +EIGEN_ALWAYS_INLINE void gemm_complex_extra_cols( const DataMapper& res, const Scalar* blockA, const Scalar* blockB, @@ -81,79 +85,154 @@ EIGEN_STRONG_INLINE void gemm_complex_extra_cols( const Packet& pAlphaImag, const Packet& pMask); -template -EIGEN_ALWAYS_INLINE Packet ploadLhs(const Scalar* lhs); +template +EIGEN_ALWAYS_INLINE void convertArrayBF16toF32(float *result, Index cols, Index rows, const DataMapper& src); + +template +EIGEN_ALWAYS_INLINE void storeBF16fromResult(bfloat16* dst, Packet8bf data, Index resInc, Index extra = 0); + +template +EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32(float *result, Index cols, Index rows, bfloat16* src, Index resInc = 1); + +template +EIGEN_ALWAYS_INLINE void storeResults(Packet4f (&acc)[4], Index rows, const Packet4f pAlpha, float* result, Index extra_cols, Index extra_rows); -template -EIGEN_ALWAYS_INLINE void bload(PacketBlock& acc, const DataMapper& res, Index row, Index col); +template +EIGEN_ALWAYS_INLINE void outputVecColResults(Packet4f (&acc)[num_acc][size], float *result, Packet4f pAlpha, Index extra_rows); + +template +EIGEN_ALWAYS_INLINE void outputVecResults(Packet4f (&acc)[num_acc][size], float *result, Packet4f pAlpha); + +template +EIGEN_ALWAYS_INLINE Packet8bf loadColData(RhsMapper& rhs, Index j); + +template +EIGEN_ALWAYS_INLINE Packet ploadLhs(const __UNPACK_TYPE__(Packet)* lhs); + +template +EIGEN_ALWAYS_INLINE void bload(PacketBlock& acc, const DataMapper& res, Index row, Index col); + +template +EIGEN_ALWAYS_INLINE void bstore(PacketBlock& acc, const DataMapper& res, Index row); + +#ifdef USE_PARTIAL_PACKETS +template +EIGEN_ALWAYS_INLINE void bload_partial(PacketBlock& acc, const DataMapper& res, Index row, Index elements); + +template +EIGEN_ALWAYS_INLINE void bstore_partial(PacketBlock& acc, const DataMapper& res, Index row, Index elements); +#endif template EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha); -template -EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock& aImag, const Packet& bReal, const Packet& bImag, PacketBlock& cReal, PacketBlock& cImag); - -// Grab two decouples real/imaginary PacketBlocks and return two coupled (real/imaginary pairs) PacketBlocks. -template -EIGEN_ALWAYS_INLINE void bcouple_common(PacketBlock& taccReal, PacketBlock& taccImag, PacketBlock& acc1, PacketBlock& acc2) -{ - acc1.packet[0].v = vec_mergeh(taccReal.packet[0], taccImag.packet[0]); - if (N > 1) { - acc1.packet[1].v = vec_mergeh(taccReal.packet[1], taccImag.packet[1]); - } - if (N > 2) { - acc1.packet[2].v = vec_mergeh(taccReal.packet[2], taccImag.packet[2]); - } - if (N > 3) { - acc1.packet[3].v = vec_mergeh(taccReal.packet[3], taccImag.packet[3]); +template +EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha, const Packet& pMask); + +template +EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock& aImag, const Packet& bReal, const Packet& bImag, PacketBlock& cReal, PacketBlock& cImag, const Packet& pMask); + +template +EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock& taccImag, PacketBlock& tRes, PacketBlock& acc1, PacketBlock& acc2); + +#define MICRO_NORMAL(iter) \ + (accCols == accCols2) || (unroll_factor != (iter + 1)) + +#define MICRO_UNROLL_ITER1(func, N) \ + switch (remaining_rows) { \ + default: \ + func(N, 0) \ + break; \ + case 1: \ + func(N, 1) \ + break; \ + case 2: \ + if (sizeof(Scalar) == sizeof(float)) { \ + func(N, 2) \ + } \ + break; \ + case 3: \ + if (sizeof(Scalar) == sizeof(float)) { \ + func(N, 3) \ + } \ + break; \ } - acc2.packet[0].v = vec_mergel(taccReal.packet[0], taccImag.packet[0]); - if (N > 1) { - acc2.packet[1].v = vec_mergel(taccReal.packet[1], taccImag.packet[1]); - } - if (N > 2) { - acc2.packet[2].v = vec_mergel(taccReal.packet[2], taccImag.packet[2]); +#ifdef USE_PARTIAL_PACKETS +#define MICRO_UNROLL_ITER(func, N) \ + if (remaining_rows) { \ + func(N, true); \ + } else { \ + func(N, false); \ } - if (N > 3) { - acc2.packet[3].v = vec_mergel(taccReal.packet[3], taccImag.packet[3]); - } -} -template -EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock& taccImag, PacketBlock& tRes, PacketBlock& acc1, PacketBlock& acc2) -{ - bcouple_common(taccReal, taccImag, acc1, acc2); +#define MICRO_NORMAL_PARTIAL(iter) \ + full || (unroll_factor != (iter + 1)) +#else +#define MICRO_UNROLL_ITER(func, N) MICRO_UNROLL_ITER1(func, N) +#endif + +#define MICRO_COMPLEX_UNROLL_ITER(func, N) MICRO_UNROLL_ITER1(func, N) - acc1.packet[0] = padd(tRes.packet[0], acc1.packet[0]); - if (N > 1) { - acc1.packet[1] = padd(tRes.packet[1], acc1.packet[1]); - } - if (N > 2) { - acc1.packet[2] = padd(tRes.packet[2], acc1.packet[2]); +#define MICRO_NORMAL_COLS(iter, a, b) ((MICRO_NORMAL(iter)) ? a : b) + +#define MICRO_LOAD1(lhs_ptr, iter) \ + if (unroll_factor > iter) { \ + lhsV##iter = ploadLhs(lhs_ptr##iter); \ + lhs_ptr##iter += MICRO_NORMAL_COLS(iter, accCols, accCols2); \ + } else { \ + EIGEN_UNUSED_VARIABLE(lhsV##iter); \ } - if (N > 3) { - acc1.packet[3] = padd(tRes.packet[3], acc1.packet[3]); + +#define MICRO_LOAD_ONE(iter) MICRO_LOAD1(lhs_ptr, iter) + +#define MICRO_COMPLEX_LOAD_ONE(iter) \ + if (!LhsIsReal && (unroll_factor > iter)) { \ + lhsVi##iter = ploadLhs(lhs_ptr_real##iter + MICRO_NORMAL_COLS(iter, imag_delta, imag_delta2)); \ + } else { \ + EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ + } \ + MICRO_LOAD1(lhs_ptr_real, iter) \ + +#define MICRO_SRC_PTR1(lhs_ptr, advRows, iter) \ + if (unroll_factor > iter) { \ + lhs_ptr##iter = lhs_base + (row+(iter*accCols))*strideA*advRows - MICRO_NORMAL_COLS(iter, 0, (accCols-accCols2)*offsetA); \ + } else { \ + EIGEN_UNUSED_VARIABLE(lhs_ptr##iter); \ } - acc2.packet[0] = padd(tRes.packet[0+N], acc2.packet[0]); - if (N > 1) { - acc2.packet[1] = padd(tRes.packet[1+N], acc2.packet[1]); +#define MICRO_SRC_PTR_ONE(iter) MICRO_SRC_PTR1(lhs_ptr, 1, iter) + +#define MICRO_COMPLEX_SRC_PTR_ONE(iter) MICRO_SRC_PTR1(lhs_ptr_real, advanceRows, iter) + +#define MICRO_PREFETCH1(lhs_ptr, iter) \ + if (unroll_factor > iter) { \ + EIGEN_POWER_PREFETCH(lhs_ptr##iter); \ } - if (N > 2) { - acc2.packet[2] = padd(tRes.packet[2+N], acc2.packet[2]); + +#define MICRO_PREFETCH_ONE(iter) MICRO_PREFETCH1(lhs_ptr, iter) + +#define MICRO_COMPLEX_PREFETCH_ONE(iter) MICRO_PREFETCH1(lhs_ptr_real, iter) + +#ifdef USE_PARTIAL_PACKETS +#define MICRO_UPDATE_MASK +#else +#define MICRO_UPDATE_MASK EIGEN_UNUSED_VARIABLE(pMask); +#endif + +#define MICRO_UPDATE \ + if (accCols == accCols2) { \ + MICRO_UPDATE_MASK \ + EIGEN_UNUSED_VARIABLE(offsetA); \ + row += unroll_factor*accCols; \ } - if (N > 3) { - acc2.packet[3] = padd(tRes.packet[3+N], acc2.packet[3]); + +#define MICRO_COMPLEX_UPDATE \ + MICRO_UPDATE \ + if(LhsIsReal || (accCols == accCols2)) { \ + EIGEN_UNUSED_VARIABLE(imag_delta2); \ } -} - -// This is necessary because ploadRhs for double returns a pair of vectors when MMA is enabled. -template -EIGEN_ALWAYS_INLINE Packet ploadRhs(const Scalar* rhs) -{ - return ploadu(rhs); -} + } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h index 5b44495379e..72e8c31a2f4 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h @@ -11,56 +11,77 @@ #ifndef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H #define EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H +// If using dynamic dispatch, set the CPU target. +#if defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH) +#pragma GCC push_options #pragma GCC target("cpu=power10,htm") +#endif #ifdef __has_builtin #if !__has_builtin(__builtin_vsx_assemble_pair) #define __builtin_vsx_assemble_pair __builtin_mma_assemble_pair #endif +#if !__has_builtin(__builtin_vsx_disassemble_pair) +#define __builtin_vsx_disassemble_pair __builtin_mma_disassemble_pair +#endif #endif +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +#include "MatrixProductMMAbfloat16.h" + namespace Eigen { namespace internal { -template +#define accColsC (accCols / 2) + EIGEN_ALWAYS_INLINE void bsetzeroMMA(__vector_quad* acc) { __builtin_mma_xxsetaccz(acc); } -template -EIGEN_ALWAYS_INLINE void storeAccumulator(Index i, const DataMapper& data, const Packet& alpha, __vector_quad* acc) +template +EIGEN_ALWAYS_INLINE void storeAccumulator(Index i, const DataMapper& data, const Packet& alpha, const Index elements, __vector_quad* acc) { PacketBlock result; __builtin_mma_disassemble_acc(&result.packet, acc); PacketBlock tRes; - bload(tRes, data, i, 0); - - bscale(tRes, result, alpha); - - data.template storePacketBlock(i, 0, tRes); + if (full) { + EIGEN_UNUSED_VARIABLE(elements); + bload(tRes, data, i, 0); + bscale(tRes, result, alpha); + bstore(tRes, data, i); + } else { + bload_partial(tRes, data, i, elements); + bscale(tRes, result, alpha); + bstore_partial(tRes, data, i, elements); + } } -template -EIGEN_ALWAYS_INLINE void storeComplexAccumulator(Index i, const DataMapper& data, const Packet& alphaReal, const Packet& alphaImag, __vector_quad* accReal, __vector_quad* accImag) +template +EIGEN_ALWAYS_INLINE void storeComplexAccumulator(Index i, const DataMapper& data, const Packet& alphaReal, const Packet& alphaImag, const Packet& pMask, __vector_quad* accReal, __vector_quad* accImag) { + constexpr bool full = (accCols2 > accColsC); PacketBlock resultReal, resultImag; __builtin_mma_disassemble_acc(&resultReal.packet, accReal); __builtin_mma_disassemble_acc(&resultImag.packet, accImag); PacketBlock tRes; - bload(tRes, data, i, 0); + bload(tRes, data, i, 0); - PacketBlock taccReal, taccImag; - bscalec(resultReal, resultImag, alphaReal, alphaImag, taccReal, taccImag); + PacketBlock taccReal, taccImag; + bscalec(resultReal, resultImag, alphaReal, alphaImag, taccReal, taccImag, pMask); PacketBlock acc1, acc2; - bcouple(taccReal, taccImag, tRes, acc1, acc2); + bcouple(taccReal, taccImag, tRes, acc1, acc2); - data.template storePacketBlock(i, 0, acc1); - data.template storePacketBlock(i + accColsC, 0, acc2); + bstore(acc1, data, i); + if (full) { + bstore(acc2, data, i + accColsC); + } } // Defaults to float32, since Eigen still supports C++03 we can't use default template arguments @@ -75,18 +96,6 @@ EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const RhsPacket& a, const L } } -template -EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const PacketBlock& a, const Packet2d& b) -{ - __vector_pair* a0 = (__vector_pair *)(&a.packet[0]); - if(NegativeAccumulate) - { - __builtin_mma_xvf64gernp(acc, *a0, (__vector unsigned char)b); - } else { - __builtin_mma_xvf64gerpp(acc, *a0, (__vector unsigned char)b); - } -} - template EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const __vector_pair& a, const Packet2d& b) { @@ -98,18 +107,13 @@ EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const __vector_pair& a, con } } -template -EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad*, const __vector_pair&, const Packet4f&) -{ - // Just for compilation -} - -template -EIGEN_ALWAYS_INLINE void pgercMMA(__vector_quad* accReal, __vector_quad* accImag, const Packet& lhsV, const Packet& lhsVi, const RhsPacket& rhsV, const RhsPacket& rhsVi) +template +EIGEN_ALWAYS_INLINE void pgercMMA(__vector_quad* accReal, __vector_quad* accImag, const Packet& lhsV, Packet& lhsVi, const RhsPacket& rhsV, RhsPacket& rhsVi) { pgerMMA(accReal, rhsV, lhsV); if(LhsIsReal) { pgerMMA(accImag, rhsVi, lhsV); + EIGEN_UNUSED_VARIABLE(lhsVi); } else { if(!RhsIsReal) { pgerMMA(accReal, rhsVi, lhsVi); @@ -122,145 +126,272 @@ EIGEN_ALWAYS_INLINE void pgercMMA(__vector_quad* accReal, __vector_quad* accImag } // This is necessary because ploadRhs for double returns a pair of vectors when MMA is enabled. +template +EIGEN_ALWAYS_INLINE Packet ploadRhs(const __UNPACK_TYPE__(Packet)* rhs) +{ + return ploadu(rhs); +} + template EIGEN_ALWAYS_INLINE void ploadRhsMMA(const Scalar* rhs, Packet& rhsV) { - rhsV = ploadRhs(rhs); + rhsV = ploadRhs(rhs); } template<> -EIGEN_ALWAYS_INLINE void ploadRhsMMA >(const double* rhs, PacketBlock& rhsV) -{ - rhsV.packet[0] = ploadRhs((const double *)((Packet2d *)rhs )); - rhsV.packet[1] = ploadRhs((const double *)(((Packet2d *)rhs) + 1)); -} - -template<> -EIGEN_ALWAYS_INLINE void ploadRhsMMA(const double* rhs, __vector_pair& rhsV) +EIGEN_ALWAYS_INLINE void ploadRhsMMA(const double* rhs, __vector_pair& rhsV) { #if EIGEN_COMP_LLVM __builtin_vsx_assemble_pair(&rhsV, - (__vector unsigned char)(ploadRhs((const double *)(((Packet2d *)rhs) + 1))), - (__vector unsigned char)(ploadRhs((const double *)((Packet2d *)rhs )))); + reinterpret_cast<__vector unsigned char>(ploadRhs(rhs + (sizeof(Packet2d) / sizeof(double)))), + reinterpret_cast<__vector unsigned char>(ploadRhs(rhs))); #else - __asm__ ("lxvp %x0,%1" : "=wa" (rhsV) : "Y" (*rhs)); + rhsV = *reinterpret_cast<__vector_pair *>(const_cast(rhs)); #endif } -template<> -EIGEN_ALWAYS_INLINE void ploadRhsMMA(const float*, __vector_pair&) +EIGEN_ALWAYS_INLINE void ploadLhsMMA(const double* lhs, __vector_pair& lhsV) { - // Just for compilation + ploadRhsMMA(lhs, lhsV); } +#define GEMM_MULTIPLE_COLS + +// Disable in GCC until unnecessary register moves are fixed +//#if (EIGEN_COMP_LLVM || (__GNUC__ >= 11)) +#if EIGEN_COMP_LLVM +#define VECTOR_PAIR_LOADS_LHS +#endif + // PEEL_MMA loop factor. +#ifdef GEMM_MULTIPLE_COLS +#define PEEL_MMA 8 +#else +// Register spillage with GCC12+ +#if EIGEN_COMP_LLVM || (__GNUC__ < 12) || defined(VECTOR_PAIR_LOADS_LHS) #define PEEL_MMA 7 +#else +#define PEEL_MMA 6 +#endif +#endif #define MICRO_MMA_UNROLL(func) \ func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7) -#define MICRO_MMA_LOAD_ONE(iter) \ - if (unroll_factor > iter) { \ - lhsV##iter = ploadLhs(lhs_ptr##iter); \ - lhs_ptr##iter += accCols; \ +#define MICRO_MMA_WORK(func, type, peel) \ + if (accItr == 1) { \ + func(0,type,peel,0,0) func(1,type,peel,1,0) func(2,type,peel,2,0) func(3,type,peel,3,0) \ + func(4,type,peel,4,0) func(5,type,peel,5,0) func(6,type,peel,6,0) func(7,type,peel,7,0) \ + } else if (accItr == 2) { \ + func(0,type,peel,0,0) func(1,type,peel,0,1) func(2,type,peel,1,0) func(3,type,peel,1,1) \ + func(4,type,peel,2,0) func(5,type,peel,2,1) func(6,type,peel,3,0) func(7,type,peel,3,1) \ } else { \ - EIGEN_UNUSED_VARIABLE(lhsV##iter); \ + func(0,type,peel,0,0) func(1,type,peel,0,1) func(2,type,peel,0,2) func(3,type,peel,0,3) \ + func(4,type,peel,1,0) func(5,type,peel,1,1) func(6,type,peel,1,2) func(7,type,peel,1,3) \ } -#define MICRO_MMA_WORK_ONE(iter, type, peel) \ - if (unroll_factor > iter) { \ - pgerMMA(&accZero##iter, rhsV##peel, lhsV##iter); \ +#define MICRO_MMA_WORK_ONE(iter, type, peel, left, right) \ + if (unroll_factor > left) { \ + pgerMMA(&accZero##iter, rhsV##right[peel], lhsV##left); \ } -#define MICRO_MMA_TYPE_PEEL(func, func2, type, peel) \ +#ifdef VECTOR_PAIR_LOADS_LHS +#define MICRO_MMA_WORK_TWO(iter, type, peel, left, right) \ + if (unroll_factor > left) { \ + pgerMMA(&accZero##iter, rhsV##right[peel], lhsV2##left.packet[peel & 1]); \ + } + +#define MICRO_MMA_LOAD1_TWO(lhs_ptr, left) \ + if (unroll_factor > left) { \ + if (MICRO_NORMAL(left)) { \ + ploadLhsMMA(reinterpret_cast(lhs_ptr##left), plhsV##left); \ + __builtin_vsx_disassemble_pair(reinterpret_cast(&lhsV2##left.packet), &plhsV##left); \ + lhs_ptr##left += accCols*2; \ + } else { \ + lhsV2##left.packet[0] = ploadLhs(lhs_ptr##left); \ + lhsV2##left.packet[1] = ploadLhs(lhs_ptr##left + accCols2); \ + lhs_ptr##left += accCols2*2; \ + EIGEN_UNUSED_VARIABLE(plhsV##left); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(lhsV2##left); \ + EIGEN_UNUSED_VARIABLE(plhsV##left); \ + } + +#define MICRO_MMA_LOAD_TWO(left) MICRO_MMA_LOAD1_TWO(lhs_ptr, left) +#endif + +#define MICRO_MMA_UNROLL_ITER(func, val) \ + func(val,0) \ + if (accItr > 1) { \ + func(val,1) \ + if (accItr > 2) { \ + func(val,2) \ + func(val,3) \ + } \ + } + +#define MICRO_MMA_LOAD_ONE_RHS1(peel, right) \ + ploadRhsMMA(rhs_ptr##right + (accRows * peel), rhsV##right[peel]); + +#define MICRO_MMA_LOAD_ONE_RHS(peel) \ + MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_ONE_RHS1, peel) + +#define MICRO_MMA_TYPE_PEEL(funcw, funcl, type, peel) \ if (PEEL_MMA > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \ - ploadRhsMMA(rhs_ptr + (accRows * peel), rhsV##peel); \ - MICRO_MMA_UNROLL(func2); \ - func(0,type,peel) func(1,type,peel) func(2,type,peel) func(3,type,peel) \ - func(4,type,peel) func(5,type,peel) func(6,type,peel) func(7,type,peel) \ + MICRO_MMA_LOAD_ONE_RHS(peel) \ + MICRO_MMA_UNROLL(funcl) \ + MICRO_MMA_WORK(funcw, type, peel) \ + } + +#ifndef VECTOR_PAIR_LOADS_LHS +#define MICRO_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type) \ + type rhsV0[8], rhsV1[(accItr > 1) ? 8 : 1], rhsV2[(accItr > 2) ? 8 : 1], rhsV3[(accItr > 2) ? 8 : 1]; \ + MICRO_MMA_TYPE_PEEL(funcw,funcl,type,0) MICRO_MMA_TYPE_PEEL(funcw,funcl,type,1) \ + MICRO_MMA_TYPE_PEEL(funcw,funcl,type,2) MICRO_MMA_TYPE_PEEL(funcw,funcl,type,3) \ + MICRO_MMA_TYPE_PEEL(funcw,funcl,type,4) MICRO_MMA_TYPE_PEEL(funcw,funcl,type,5) \ + MICRO_MMA_TYPE_PEEL(funcw,funcl,type,6) MICRO_MMA_TYPE_PEEL(funcw,funcl,type,7) +#else +#define MICRO_MMA_LOAD_TWO_RHS(peel1, right) \ + ploadRhsMMA(reinterpret_cast(rhs_ptr##right + (accRows * peel1)), prhsV##peel1); \ + __builtin_vsx_disassemble_pair(reinterpret_cast(&rhsV##right[peel1]), &prhsV##peel1); + +#define MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, peel1, peel2) \ + if (PEEL_MMA > peel2) { \ + PacketBlock lhsV20, lhsV21, lhsV22, lhsV23, lhsV24, lhsV25, lhsV26, lhsV27; \ + __vector_pair plhsV0, plhsV1, plhsV2, plhsV3, plhsV4, plhsV5, plhsV6, plhsV7; \ + if (sizeof(type) == 16) { \ + MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_TWO_RHS, peel1) \ + } else { \ + EIGEN_UNUSED_VARIABLE(prhsV##peel1); \ + MICRO_MMA_LOAD_ONE_RHS(peel1) \ + MICRO_MMA_LOAD_ONE_RHS(peel2) \ + } \ + MICRO_MMA_UNROLL(funcl2) \ + MICRO_MMA_WORK(funcw2, type, peel1) \ + MICRO_MMA_WORK(funcw2, type, peel2) \ } else { \ - EIGEN_UNUSED_VARIABLE(rhsV##peel); \ + EIGEN_UNUSED_VARIABLE(prhsV##peel1); \ + MICRO_MMA_TYPE_PEEL(funcw1, funcl1, type, peel1) \ } -#define MICRO_MMA_UNROLL_TYPE_PEEL(func, func2, type) \ - type rhsV0, rhsV1, rhsV2, rhsV3, rhsV4, rhsV5, rhsV6, rhsV7; \ - MICRO_MMA_TYPE_PEEL(func,func2,type,0); MICRO_MMA_TYPE_PEEL(func,func2,type,1); \ - MICRO_MMA_TYPE_PEEL(func,func2,type,2); MICRO_MMA_TYPE_PEEL(func,func2,type,3); \ - MICRO_MMA_TYPE_PEEL(func,func2,type,4); MICRO_MMA_TYPE_PEEL(func,func2,type,5); \ - MICRO_MMA_TYPE_PEEL(func,func2,type,6); MICRO_MMA_TYPE_PEEL(func,func2,type,7); +#define MICRO_MMA_UNROLL_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type) \ + type rhsV0[8], rhsV1[(accItr > 1) ? 8 : 1], rhsV2[(accItr > 2) ? 8 : 1], rhsV3[(accItr > 2) ? 8 : 1]; \ + __vector_pair prhsV0, prhsV2, prhsV4, prhsV6; \ + MICRO_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,0,1) \ + MICRO_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,2,3) \ + MICRO_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,4,5) \ + MICRO_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,6,7) +#endif -#define MICRO_MMA_UNROLL_TYPE_ONE(func, func2, type) \ - type rhsV0; \ - MICRO_MMA_TYPE_PEEL(func,func2,type,0); +#define MICRO_MMA_UNROLL_TYPE_ONE(funcw, funcl, type) \ + type rhsV0[1], rhsV1[1], rhsV2[1], rhsV3[1]; \ + MICRO_MMA_TYPE_PEEL(funcw,funcl,type,0) -#define MICRO_MMA_ONE_PEEL \ - if (sizeof(Scalar) == sizeof(float)) { \ - MICRO_MMA_UNROLL_TYPE_PEEL(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, RhsPacket); \ - } else { \ - MICRO_MMA_UNROLL_TYPE_PEEL(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, __vector_pair); \ - } \ - rhs_ptr += (accRows * PEEL_MMA); +#define MICRO_MMA_UPDATE_RHS1(size, right) \ + rhs_ptr##right += (accRows * size); -#define MICRO_MMA_ONE \ - if (sizeof(Scalar) == sizeof(float)) { \ - MICRO_MMA_UNROLL_TYPE_ONE(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, RhsPacket); \ - } else { \ - MICRO_MMA_UNROLL_TYPE_ONE(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, __vector_pair); \ - } \ - rhs_ptr += accRows; +#define MICRO_MMA_UPDATE_RHS(size) \ + MICRO_MMA_UNROLL_ITER(MICRO_MMA_UPDATE_RHS1, size) + +#define MICRO_MMA_UNROLL_TYPE(MICRO_MMA_TYPE, size) \ + MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, RhsPacket) \ + MICRO_MMA_UPDATE_RHS(size) + +#ifndef VECTOR_PAIR_LOADS_LHS +#define MICRO_MMA_ONE_PEEL MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_PEEL, PEEL_MMA) +#else +#define MICRO_MMA_UNROLL_TYPE2(MICRO_MMA_TYPE, size) \ + MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, MICRO_MMA_WORK_TWO, MICRO_MMA_LOAD_TWO, RhsPacket) \ + MICRO_MMA_UPDATE_RHS(size) + +#define MICRO_MMA_ONE_PEEL MICRO_MMA_UNROLL_TYPE2(MICRO_MMA_UNROLL_TYPE_PEEL2, PEEL_MMA) +#endif + +#define MICRO_MMA_ONE MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_ONE, 1) #define MICRO_MMA_DST_PTR_ONE(iter) \ - if (unroll_factor > iter) { \ - bsetzeroMMA(&accZero##iter); \ + if (unroll_factor * accItr > iter) { \ + bsetzeroMMA(&accZero##iter); \ } else { \ EIGEN_UNUSED_VARIABLE(accZero##iter); \ } #define MICRO_MMA_DST_PTR MICRO_MMA_UNROLL(MICRO_MMA_DST_PTR_ONE) -#define MICRO_MMA_SRC_PTR_ONE(iter) \ - if (unroll_factor > iter) { \ - lhs_ptr##iter = lhs_base + ( (row/accCols) + iter )*strideA*accCols; \ - } else { \ - EIGEN_UNUSED_VARIABLE(lhs_ptr##iter); \ - } +#define MICRO_MMA_SRC_PTR MICRO_MMA_UNROLL(MICRO_SRC_PTR_ONE) -#define MICRO_MMA_SRC_PTR MICRO_MMA_UNROLL(MICRO_MMA_SRC_PTR_ONE) +#define MICRO_MMA_PREFETCH MICRO_MMA_UNROLL(MICRO_PREFETCH_ONE) -#define MICRO_MMA_PREFETCH_ONE(iter) \ - if (unroll_factor > iter) { \ - EIGEN_POWER_PREFETCH(lhs_ptr##iter); \ +#define MICRO_MMA_STORE_ONE(iter, left, right) \ + if (unroll_factor > left) { \ + storeAccumulator(row + left*accCols, res##right, pAlpha, accCols2, &accZero##iter); \ } -#define MICRO_MMA_PREFETCH MICRO_MMA_UNROLL(MICRO_MMA_PREFETCH_ONE) - -#define MICRO_MMA_STORE_ONE(iter) \ - if (unroll_factor > iter) { \ - storeAccumulator(row + iter*accCols, res, pAlpha, &accZero##iter); \ +#define MICRO_MMA_ITER_UNROLL(func) \ + if (accItr == 1) { \ + func(0,0,0) func(1,1,0) func(2,2,0) func(3,3,0) \ + func(4,4,0) func(5,5,0) func(6,6,0) func(7,7,0) \ + } else if (accItr == 2) { \ + func(0,0,0) func(1,0,1) func(2,1,0) func(3,1,1) \ + func(4,2,0) func(5,2,1) func(6,3,0) func(7,3,1) \ + } else { \ + func(0,0,0) func(1,0,1) func(2,0,2) func(3,0,3) \ + func(4,1,0) func(5,1,1) func(6,1,2) func(7,1,3) \ } -#define MICRO_MMA_STORE MICRO_MMA_UNROLL(MICRO_MMA_STORE_ONE) +#define MICRO_MMA_STORE MICRO_MMA_ITER_UNROLL(MICRO_MMA_STORE_ONE) + +#define MICRO_MMA_EXTRA_ROWS(right) \ + gemm_extra_row(res3##right, blockA, rhs_base + right*accRows*strideB, depth, strideA, offsetA, strideB, row, rows, remaining_rows, pAlpha, pMask); + +#define MICRO_MMA_EXTRA_ROWS1(val, right) \ + MICRO_MMA_EXTRA_ROWS(right); -template +template EIGEN_ALWAYS_INLINE void gemm_unrolled_MMA_iteration( - const DataMapper& res, + const DataMapper& res0, + const DataMapper& res1, + const DataMapper& res2, + const DataMapper& res3, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, + Index strideB, + Index offsetA, Index& row, - const Packet& pAlpha) + const Packet& pAlpha, + Index accCols2 + ) { - const Scalar* rhs_ptr = rhs_base; + const Scalar* rhs_ptr0 = rhs_base, * rhs_ptr1 = NULL, * rhs_ptr2 = NULL, * rhs_ptr3 = NULL; const Scalar* lhs_ptr0 = NULL, * lhs_ptr1 = NULL, * lhs_ptr2 = NULL, * lhs_ptr3 = NULL, * lhs_ptr4 = NULL, * lhs_ptr5 = NULL, * lhs_ptr6 = NULL, * lhs_ptr7 = NULL; __vector_quad accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7; + if (accItr > 1) { + rhs_ptr1 = rhs_base + (accRows * strideB); + } else { + EIGEN_UNUSED_VARIABLE(strideB); + EIGEN_UNUSED_VARIABLE(rhs_ptr1); + EIGEN_UNUSED_VARIABLE(res1); + } + if (accItr > 2) { + rhs_ptr2 = rhs_base + (2 * accRows * strideB); + rhs_ptr3 = rhs_base + (3 * accRows * strideB); + } else { + EIGEN_UNUSED_VARIABLE(rhs_ptr2); + EIGEN_UNUSED_VARIABLE(rhs_ptr3); + EIGEN_UNUSED_VARIABLE(res2); + EIGEN_UNUSED_VARIABLE(res3); + } + MICRO_MMA_SRC_PTR MICRO_MMA_DST_PTR - Index k = 0; - for(; k + PEEL_MMA <= depth; k+= PEEL_MMA) + Index k = 0, depth2 = depth - PEEL_MMA; + for(; k <= depth2; k += PEEL_MMA) { EIGEN_POWER_PREFETCH(rhs_ptr); MICRO_MMA_PREFETCH @@ -272,10 +403,19 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_MMA_iteration( } MICRO_MMA_STORE - row += unroll_factor*accCols; + MICRO_UPDATE } -template +#define MICRO_MMA_UNROLL_ITER2(N, M) \ + gemm_unrolled_MMA_iteration(res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, strideB, offsetA, row, pAlpha, M ? remaining_rows : accCols); \ + if (M) return; + +#define MICRO_MMA_ROWS(n) \ + while(row + n*accCols <= rows) { \ + MICRO_MMA_UNROLL_ITER2(n, 0); \ + } + +template EIGEN_ALWAYS_INLINE void gemmMMA_cols( const DataMapper& res, const Scalar* blockA, @@ -287,55 +427,80 @@ EIGEN_ALWAYS_INLINE void gemmMMA_cols( Index offsetB, Index col, Index rows, - Index cols, Index remaining_rows, const Packet& pAlpha, const Packet& pMask) { - const DataMapper res3 = res.getSubMapper(0, col); + const DataMapper res30 = res.getSubMapper(0, col); + const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows*1) : res30; + const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows*2) : res30; + const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows*3) : res30; const Scalar* rhs_base = blockB + col*strideB + accRows*offsetB; const Scalar* lhs_base = blockA + accCols*offsetA; Index row = 0; #define MAX_MMA_UNROLL 7 - while(row + MAX_MMA_UNROLL*accCols <= rows) { - gemm_unrolled_MMA_iteration(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + +#if MAX_MMA_UNROLL < 2 + if (1) { +#elif MAX_MMA_UNROLL < 4 + if (accItr <= 2) { +#else + if (accItr == 1) { +#endif + MICRO_MMA_ROWS(MAX_MMA_UNROLL); + } else if (accItr == 2) { + MICRO_MMA_ROWS(4); + } else { + MICRO_MMA_ROWS(2); } switch( (rows-row)/accCols ) { #if MAX_MMA_UNROLL > 7 case 7: - gemm_unrolled_MMA_iteration<7, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + if (accItr == 1) { + MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 7) + } break; #endif #if MAX_MMA_UNROLL > 6 case 6: - gemm_unrolled_MMA_iteration<6, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + if (accItr == 1) { + MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 6) + } break; #endif #if MAX_MMA_UNROLL > 5 case 5: - gemm_unrolled_MMA_iteration<5, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + if (accItr == 1) { + MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 5) + } break; #endif #if MAX_MMA_UNROLL > 4 case 4: - gemm_unrolled_MMA_iteration<4, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + if (accItr == 1) { + MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 4) + } break; #endif #if MAX_MMA_UNROLL > 3 case 3: - gemm_unrolled_MMA_iteration<3, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + if (accItr <= 2) { + MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 3) + } break; #endif #if MAX_MMA_UNROLL > 2 case 2: - gemm_unrolled_MMA_iteration<2, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + if (accItr <= 2) { + MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 2) + } break; #endif #if MAX_MMA_UNROLL > 1 case 1: - gemm_unrolled_MMA_iteration<1, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res3, lhs_base, rhs_base, depth, strideA, row, pAlpha); + MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 1) break; #endif default: @@ -345,11 +510,17 @@ EIGEN_ALWAYS_INLINE void gemmMMA_cols( if(remaining_rows > 0) { - gemm_extra_row(res3, blockA, rhs_base, depth, strideA, offsetA, row, col, rows, cols, remaining_rows, pAlpha, pMask); + MICRO_MMA_UNROLL_ITER(MICRO_MMA_EXTRA_ROWS1, 0) } } -template +#define MICRO_MMA_COLS(n) \ + for(; col + n*accRows <= cols; col += n*accRows) \ + { \ + gemmMMA_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlpha, pMask); \ + } + +template void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index remaining_rows = rows % accCols; @@ -358,95 +529,171 @@ void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, if( strideB == -1 ) strideB = depth; const Packet pAlpha = pset1(alpha); - const Packet pMask = bmask((const int)(remaining_rows)); + const Packet pMask = bmask(remaining_rows); + + typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2; Index col = 0; - for(; col + accRows <= cols; col += accRows) +#ifdef GEMM_MULTIPLE_COLS + MICRO_MMA_COLS(4); + MICRO_MMA_COLS(2); +#endif + MICRO_MMA_COLS(1); + + if (col != cols) { - gemmMMA_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlpha, pMask); + gemm_extra_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlpha, pMask); } - - gemm_extra_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlpha, pMask); } -#define accColsC (accCols / 2) #define advanceRows ((LhsIsReal) ? 1 : 2) #define advanceCols ((RhsIsReal) ? 1 : 2) // PEEL_COMPLEX_MMA loop factor. +#ifdef GEMM_MULTIPLE_COLS +#define PEEL_COMPLEX_MMA 4 +#else #define PEEL_COMPLEX_MMA 3 +#endif #define MICRO_COMPLEX_MMA_UNROLL(func) \ func(0) func(1) func(2) func(3) -#define MICRO_COMPLEX_MMA_LOAD_ONE(iter) \ - if (unroll_factor > iter) { \ - lhsV##iter = ploadLhs(lhs_ptr_real##iter); \ - if(!LhsIsReal) { \ - lhsVi##iter = ploadLhs(lhs_ptr_real##iter + imag_delta); \ +#define MICRO_COMPLEX_MMA_WORK(func, type, peel) \ + if (accItr == 1) { \ + func(0,type,peel,0,0) func(1,type,peel,1,0) func(2,type,peel,2,0) func(3,type,peel,3,0) \ + } else if (accItr == 2) { \ + func(0,type,peel,0,0) func(1,type,peel,0,1) func(2,type,peel,1,0) func(3,type,peel,1,1) \ + } else { \ + func(0,type,peel,0,0) func(1,type,peel,0,1) func(2,type,peel,0,2) func(3,type,peel,0,3) \ + } + +#define MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel, left, right) \ + if (unroll_factor > left) { \ + pgercMMA(&accReal##iter, &accImag##iter, lhsV##left, lhsVi##left, rhsV##right[peel], rhsVi##right[peel]); \ + } + +#ifdef VECTOR_PAIR_LOADS_LHS +#define MICRO_COMPLEX_MMA_WORK_TWO(iter, type, peel, left, right) \ + if (unroll_factor > left) { \ + pgercMMA(&accReal##iter, &accImag##iter, lhsV2##left.packet[peel & 1], lhsVi2##left.packet[peel & 1], rhsV##right[peel], rhsVi##right[peel]); \ + } + +#define MICRO_COMPLEX_MMA_LOAD1_TWO(lhs_ptr, left) \ + if (!LhsIsReal && (unroll_factor > left)) { \ + if (MICRO_NORMAL(left)) { \ + ploadLhsMMA(reinterpret_cast(lhs_ptr_real##left + imag_delta), plhsVi##left); \ + __builtin_vsx_disassemble_pair(reinterpret_cast(&lhsVi2##left.packet), &plhsVi##left); \ } else { \ - EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ + lhsVi2##left.packet[0] = ploadLhs(lhs_ptr_real##left + imag_delta2); \ + lhsVi2##left.packet[1] = ploadLhs(lhs_ptr_real##left + imag_delta2 + accCols2); \ + EIGEN_UNUSED_VARIABLE(plhsVi##left); \ } \ - lhs_ptr_real##iter += accCols; \ } else { \ - EIGEN_UNUSED_VARIABLE(lhsV##iter); \ - EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ - } + EIGEN_UNUSED_VARIABLE(lhsVi2##left); \ + EIGEN_UNUSED_VARIABLE(plhsVi##left); \ + } \ + MICRO_MMA_LOAD1_TWO(lhs_ptr_real, left) + +#define MICRO_COMPLEX_MMA_LOAD_TWO(left) MICRO_COMPLEX_MMA_LOAD1_TWO(lhs_ptr, left) +#endif -#define MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel) \ - if (unroll_factor > iter) { \ - pgercMMA(&accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \ +#define MICRO_COMPLEX_MMA_LOAD_RHS1(peel, right) \ + ploadRhsMMA(rhs_ptr_real##right + (accRows * peel), rhsV##right[peel]); \ + if (!RhsIsReal) { \ + ploadRhsMMA(rhs_ptr_imag##right + (accRows * peel), rhsVi##right[peel]); \ } -#define MICRO_COMPLEX_MMA_TYPE_PEEL(func, func2, type, peel) \ +#define MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel) \ + MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_RHS1, peel) + +#define MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, peel) \ if (PEEL_COMPLEX_MMA > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3; \ Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3; \ - ploadRhsMMA(rhs_ptr_real + (accRows * peel), rhsV##peel); \ - if(!RhsIsReal) { \ - ploadRhsMMA(rhs_ptr_imag + (accRows * peel), rhsVi##peel); \ + MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel) \ + MICRO_COMPLEX_MMA_UNROLL(funcl) \ + MICRO_COMPLEX_MMA_WORK(funcw, type, peel) \ + } + +#ifndef VECTOR_PAIR_LOADS_LHS +#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type) \ + type rhsV0[4], rhsVi0[4], rhsV1[(accItr > 1) ? 4 : 1], rhsVi1[(accItr > 1) ? 4 : 1], rhsV2[(accItr > 2) ? 4 : 1], rhsVi2[(accItr > 2) ? 4 : 1], rhsV3[(accItr > 2) ? 4 : 1], rhsVi3[(accItr > 2) ? 4 : 1]; \ + MICRO_COMPLEX_MMA_TYPE_PEEL(funcw,funcl,type,0) MICRO_COMPLEX_MMA_TYPE_PEEL(funcw,funcl,type,1) \ + MICRO_COMPLEX_MMA_TYPE_PEEL(funcw,funcl,type,2) MICRO_COMPLEX_MMA_TYPE_PEEL(funcw,funcl,type,3) +#else +#define MICRO_COMPLEX_MMA_LOAD_TWO_RHS(peel1, right) \ + ploadRhsMMA(reinterpret_cast(rhs_ptr_real##right + (accRows * peel1)), prhsV##peel1); \ + __builtin_vsx_disassemble_pair(reinterpret_cast(&rhsV##right[peel1]), &prhsV##peel1); \ + if(!RhsIsReal) { \ + ploadRhsMMA(reinterpret_cast(rhs_ptr_imag##right + (accRows * peel1)), prhsVi##peel1); \ + __builtin_vsx_disassemble_pair(reinterpret_cast(&rhsVi##right[peel1]), &prhsVi##peel1); \ + } else { \ + EIGEN_UNUSED_VARIABLE(prhsVi##peel1); \ + } + +#define MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, peel1, peel2) \ + if (PEEL_COMPLEX_MMA > peel2) { \ + PacketBlock lhsV20, lhsV21, lhsV22, lhsV23; \ + PacketBlock lhsVi20, lhsVi21, lhsVi22, lhsVi23; \ + __vector_pair plhsV0, plhsV1, plhsV2, plhsV3; \ + __vector_pair plhsVi0, plhsVi1, plhsVi2, plhsVi3; \ + if (sizeof(type) == 16) { \ + MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_TWO_RHS, peel1) \ } else { \ - EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ + EIGEN_UNUSED_VARIABLE(prhsV##peel1); \ + EIGEN_UNUSED_VARIABLE(prhsVi##peel1); \ + MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel1); \ + MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel2); \ } \ - MICRO_COMPLEX_MMA_UNROLL(func2); \ - func(0,type,peel) func(1,type,peel) func(2,type,peel) func(3,type,peel) \ + MICRO_COMPLEX_MMA_UNROLL(funcl2) \ + MICRO_COMPLEX_MMA_WORK(funcw2, type, peel1) \ + MICRO_COMPLEX_MMA_WORK(funcw2, type, peel2) \ } else { \ - EIGEN_UNUSED_VARIABLE(rhsV##peel); \ - EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ + EIGEN_UNUSED_VARIABLE(prhsV##peel1); \ + EIGEN_UNUSED_VARIABLE(prhsVi##peel1); \ + MICRO_COMPLEX_MMA_TYPE_PEEL(funcw1, funcl1, type, peel1) \ } -#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(func, func2, type) \ - type rhsV0, rhsV1, rhsV2, rhsV3; \ - type rhsVi0, rhsVi1, rhsVi2, rhsVi3; \ - MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,0); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,1); \ - MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,2); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,3); +#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type) \ + type rhsV0[4], rhsVi0[4], rhsV1[(accItr > 1) ? 4 : 1], rhsVi1[(accItr > 1) ? 4 : 1], rhsV2[(accItr > 2) ? 4 : 1], rhsVi2[(accItr > 2) ? 4 : 1], rhsV3[(accItr > 2) ? 4 : 1], rhsVi3[(accItr > 2) ? 4 : 1]; \ + __vector_pair prhsV0, prhsV2; \ + __vector_pair prhsVi0, prhsVi2; \ + MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,0,1) \ + MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,2,3) +#endif -#define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(func, func2, type) \ - type rhsV0, rhsVi0; \ - MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,0); +#define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(funcw, funcl, type) \ + type rhsV0[1], rhsVi0[1], rhsV1[1], rhsVi1[1], rhsV2[1], rhsVi2[1], rhsV3[1], rhsVi3[1]; \ + MICRO_COMPLEX_MMA_TYPE_PEEL(funcw,funcl,type,0) -#define MICRO_COMPLEX_MMA_ONE_PEEL \ - if (sizeof(Scalar) == sizeof(float)) { \ - MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, RhsPacket); \ - } else { \ - MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, __vector_pair); \ - } \ - rhs_ptr_real += (accRows * PEEL_COMPLEX_MMA); \ - if(!RhsIsReal) rhs_ptr_imag += (accRows * PEEL_COMPLEX_MMA); +#define MICRO_COMPLEX_MMA_UPDATE_RHS1(size, right) \ + rhs_ptr_real##right += (accRows * size); \ + if(!RhsIsReal) rhs_ptr_imag##right += (accRows * size); -#define MICRO_COMPLEX_MMA_ONE \ - if (sizeof(Scalar) == sizeof(float)) { \ - MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, RhsPacket); \ - } else { \ - MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, __vector_pair); \ - } \ - rhs_ptr_real += accRows; \ - if(!RhsIsReal) rhs_ptr_imag += accRows; +#define MICRO_COMPLEX_MMA_UPDATE_RHS(size) \ + MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_UPDATE_RHS1, size) + +#define MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_TYPE, size) \ + MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, RhsPacket) \ + MICRO_COMPLEX_MMA_UPDATE_RHS(size); + +#ifndef VECTOR_PAIR_LOADS_LHS +#define MICRO_COMPLEX_MMA_ONE_PEEL MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL, PEEL_COMPLEX_MMA) +#else +#define MICRO_COMPLEX_MMA_UNROLL_TYPE2(MICRO_COMPLEX_MMA_TYPE, size) \ + MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, MICRO_COMPLEX_MMA_WORK_TWO, MICRO_COMPLEX_MMA_LOAD_TWO, RhsPacket) \ + MICRO_COMPLEX_MMA_UPDATE_RHS(size); + +#define MICRO_COMPLEX_MMA_ONE_PEEL MICRO_COMPLEX_MMA_UNROLL_TYPE2(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL2, PEEL_COMPLEX_MMA) +#endif + +#define MICRO_COMPLEX_MMA_ONE MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE, 1) #define MICRO_COMPLEX_MMA_DST_PTR_ONE(iter) \ - if (unroll_factor > iter) { \ - bsetzeroMMA(&accReal##iter); \ - bsetzeroMMA(&accImag##iter); \ + if (unroll_factor * accItr > iter) { \ + bsetzeroMMA(&accReal##iter); \ + bsetzeroMMA(&accImag##iter); \ } else { \ EIGEN_UNUSED_VARIABLE(accReal##iter); \ EIGEN_UNUSED_VARIABLE(accImag##iter); \ @@ -454,48 +701,91 @@ void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, #define MICRO_COMPLEX_MMA_DST_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_DST_PTR_ONE) -#define MICRO_COMPLEX_MMA_SRC_PTR_ONE(iter) \ - if (unroll_factor > iter) { \ - lhs_ptr_real##iter = lhs_base + ( ((advanceRows*row)/accCols) + iter*advanceRows )*strideA*accCols; \ - } else { \ - EIGEN_UNUSED_VARIABLE(lhs_ptr_real##iter); \ - } +#define MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE) -#define MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_SRC_PTR_ONE) +#define MICRO_COMPLEX_MMA_PREFETCH MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_PREFETCH_ONE) -#define MICRO_COMPLEX_MMA_PREFETCH_ONE(iter) \ - if (unroll_factor > iter) { \ - EIGEN_POWER_PREFETCH(lhs_ptr_real##iter); \ +#define MICRO_COMPLEX_MMA_STORE_ONE(iter, left, right) \ + if (unroll_factor > left) { \ + storeComplexAccumulator(row + left*accCols, res##right, pAlphaReal, pAlphaImag, pMask, &accReal##iter, &accImag##iter); \ } -#define MICRO_COMPLEX_MMA_PREFETCH MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_PREFETCH_ONE) - -#define MICRO_COMPLEX_MMA_STORE_ONE(iter) \ - if (unroll_factor > iter) { \ - storeComplexAccumulator(row + iter*accCols, res, pAlphaReal, pAlphaImag, &accReal##iter, &accImag##iter); \ +#define MICRO_COMPLEX_MMA_ITER_UNROLL(func) \ + if (accItr == 1) { \ + func(0,0,0) func(1,1,0) func(2,2,0) func(3,3,0) \ + } else if (accItr == 2) { \ + func(0,0,0) func(1,0,1) func(2,1,0) func(3,1,1) \ + } else { \ + func(0,0,0) func(1,0,1) func(2,0,2) func(3,0,3) \ } -#define MICRO_COMPLEX_MMA_STORE MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE) +#define MICRO_COMPLEX_MMA_STORE MICRO_COMPLEX_MMA_ITER_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE) -template +#define MICRO_COMPLEX_MMA_EXTRA_ROWS(right) \ + gemm_complex_extra_row(res3##right, blockA, rhs_base + right*accRows*(RhsIsReal ? 1 : 2)*strideB, depth, strideA, offsetA, strideB, row, rows, remaining_rows, pAlphaReal, pAlphaImag, pMask); + +#define MICRO_COMPLEX_MMA_EXTRA_ROWS1(val, right) \ + MICRO_COMPLEX_MMA_EXTRA_ROWS(right); + +template EIGEN_ALWAYS_INLINE void gemm_complex_unrolled_MMA_iteration( - const DataMapper& res, + const DataMapper& res0, + const DataMapper& res1, + const DataMapper& res2, + const DataMapper& res3, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, + Index offsetA, Index strideB, Index& row, const Packet& pAlphaReal, - const Packet& pAlphaImag) + const Packet& pAlphaImag, + const Packet& pMask) { - const Scalar* rhs_ptr_real = rhs_base; - const Scalar* rhs_ptr_imag = NULL; + const Scalar* rhs_ptr_real0 = rhs_base, * rhs_ptr_real1 = NULL, * rhs_ptr_real2 = NULL, * rhs_ptr_real3 = NULL; + const Scalar* rhs_ptr_imag0 = NULL, * rhs_ptr_imag1 = NULL, * rhs_ptr_imag2 = NULL, * rhs_ptr_imag3 = NULL; const Index imag_delta = accCols*strideA; + const Index imag_delta2 = accCols2*strideA; + if(!RhsIsReal) { - rhs_ptr_imag = rhs_base + accRows*strideB; + rhs_ptr_imag0 = rhs_base + accRows*strideB; + } else { + EIGEN_UNUSED_VARIABLE(rhs_ptr_imag0); + } + if (accItr > 1) { + if(!RhsIsReal) { + rhs_ptr_real1 = rhs_base + (2*accRows*strideB); + rhs_ptr_imag1 = rhs_base + (3*accRows*strideB); + } else { + rhs_ptr_real1 = rhs_base + accRows*strideB; + EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1); + } + } else { + EIGEN_UNUSED_VARIABLE(rhs_ptr_real1); + EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1); + EIGEN_UNUSED_VARIABLE(res1); + } + if (accItr > 2) { + if(!RhsIsReal) { + rhs_ptr_real2 = rhs_base + (4*accRows*strideB); + rhs_ptr_imag2 = rhs_base + (5*accRows*strideB); + rhs_ptr_real3 = rhs_base + (6*accRows*strideB); + rhs_ptr_imag3 = rhs_base + (7*accRows*strideB); + } else { + rhs_ptr_real2 = rhs_base + (2*accRows*strideB); + rhs_ptr_real3 = rhs_base + (3*accRows*strideB); + EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2); + EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3); + } } else { - EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); + EIGEN_UNUSED_VARIABLE(rhs_ptr_real2); + EIGEN_UNUSED_VARIABLE(rhs_ptr_real3); + EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2); + EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3); + EIGEN_UNUSED_VARIABLE(res2); + EIGEN_UNUSED_VARIABLE(res3); } const Scalar* lhs_ptr_real0 = NULL, * lhs_ptr_real1 = NULL; const Scalar* lhs_ptr_real2 = NULL, * lhs_ptr_real3 = NULL; @@ -504,8 +794,8 @@ EIGEN_ALWAYS_INLINE void gemm_complex_unrolled_MMA_iteration( MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_DST_PTR - Index k = 0; - for(; k + PEEL_COMPLEX_MMA <= depth; k+= PEEL_COMPLEX_MMA) + Index k = 0, depth2 = depth - PEEL_COMPLEX_MMA; + for(; k <= depth2; k += PEEL_COMPLEX_MMA) { EIGEN_POWER_PREFETCH(rhs_ptr_real); if(!RhsIsReal) { @@ -520,10 +810,19 @@ EIGEN_ALWAYS_INLINE void gemm_complex_unrolled_MMA_iteration( } MICRO_COMPLEX_MMA_STORE - row += unroll_factor*accCols; + MICRO_COMPLEX_UPDATE } -template +#define MICRO_COMPLEX_MMA_UNROLL_ITER2(N, M) \ + gemm_complex_unrolled_MMA_iteration(res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlphaReal, pAlphaImag, pMask); \ + if (M) return; + +#define MICRO_COMPLEX_MMA_ROWS(n) \ + while(row + n*accCols <= rows) { \ + MICRO_COMPLEX_MMA_UNROLL_ITER2(n, 0); \ + } + +template EIGEN_ALWAYS_INLINE void gemmMMA_complex_cols( const DataMapper& res, const Scalar* blockA, @@ -535,41 +834,55 @@ EIGEN_ALWAYS_INLINE void gemmMMA_complex_cols( Index offsetB, Index col, Index rows, - Index cols, Index remaining_rows, const Packet& pAlphaReal, const Packet& pAlphaImag, const Packet& pMask) { - const DataMapper res3 = res.getSubMapper(0, col); + const DataMapper res30 = res.getSubMapper(0, col); + const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows*1) : res30; + const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows*2) : res30; + const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows*3) : res30; const Scalar* rhs_base = blockB + advanceCols*col*strideB + accRows*offsetB; const Scalar* lhs_base = blockA + accCols*offsetA; Index row = 0; #define MAX_COMPLEX_MMA_UNROLL 4 - while(row + MAX_COMPLEX_MMA_UNROLL*accCols <= rows) { - gemm_complex_unrolled_MMA_iteration(res3, lhs_base, rhs_base, depth, strideA, strideB, row, pAlphaReal, pAlphaImag); + +#if MAX_COMPLEX_MMA_UNROLL < 2 + if (1) { +#elif MAX_COMPLEX_MMA_UNROLL < 4 + if (accItr <= 2) { +#else + if (accItr == 1) { +#endif + MICRO_COMPLEX_MMA_ROWS(MAX_COMPLEX_MMA_UNROLL); + } else if (accItr == 2) { + MICRO_COMPLEX_MMA_ROWS(2); + } else { + MICRO_COMPLEX_MMA_ROWS(1); } switch( (rows-row)/accCols ) { -#if MAX_COMPLEX_MMA_UNROLL > 4 - case 4: - gemm_complex_unrolled_MMA_iteration<4, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res3, lhs_base, rhs_base, depth, strideA, strideB, row, pAlphaReal, pAlphaImag); - break; -#endif #if MAX_COMPLEX_MMA_UNROLL > 3 case 3: - gemm_complex_unrolled_MMA_iteration<3, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res3, lhs_base, rhs_base, depth, strideA, strideB, row, pAlphaReal, pAlphaImag); + if (accItr == 1) { + MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 3) + } break; #endif #if MAX_COMPLEX_MMA_UNROLL > 2 case 2: - gemm_complex_unrolled_MMA_iteration<2, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res3, lhs_base, rhs_base, depth, strideA, strideB, row, pAlphaReal, pAlphaImag); + if (accItr == 1) { + MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 2) + } break; #endif #if MAX_COMPLEX_MMA_UNROLL > 1 case 1: - gemm_complex_unrolled_MMA_iteration<1, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res3, lhs_base, rhs_base, depth, strideA, strideB, row, pAlphaReal, pAlphaImag); + if (accItr <= 2) { + MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 1) + } break; #endif default: @@ -579,11 +892,17 @@ EIGEN_ALWAYS_INLINE void gemmMMA_complex_cols( if(remaining_rows > 0) { - gemm_complex_extra_row(res3, blockA, rhs_base, depth, strideA, offsetA, strideB, row, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask); + MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_EXTRA_ROWS1, 0) } } -template +#define MICRO_COMPLEX_MMA_COLS(n) \ + for(; col + n*accRows <= cols; col += n*accRows) \ + { \ + gemmMMA_complex_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlphaReal, pAlphaImag, pMask); \ + } + +template void gemm_complexMMA(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index remaining_rows = rows % accCols; @@ -593,28 +912,37 @@ void gemm_complexMMA(const DataMapper& res, const LhsScalar* blockAc, const RhsS const Packet pAlphaReal = pset1(alpha.real()); const Packet pAlphaImag = pset1(alpha.imag()); - const Packet pMask = bmask((const int)(remaining_rows)); + const Packet pMask = bmask(remaining_rows); const Scalar* blockA = (Scalar *) blockAc; const Scalar* blockB = (Scalar *) blockBc; + typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2; + Index col = 0; - for(; col + accRows <= cols; col += accRows) +#ifdef GEMM_MULTIPLE_COLS + MICRO_COMPLEX_MMA_COLS(4); + MICRO_COMPLEX_MMA_COLS(2); +#endif + MICRO_COMPLEX_MMA_COLS(1); + + if (col != cols) { - gemmMMA_complex_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask); + gemm_complex_extra_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask); } - - gemm_complex_extra_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask); } #undef accColsC #undef advanceRows #undef advanceCols -#pragma GCC reset_options } // end namespace internal } // end namespace Eigen +#if defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH) +#pragma GCC pop_options +#endif + #endif // EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h new file mode 100644 index 00000000000..50941183931 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h @@ -0,0 +1,749 @@ +#ifndef EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H +#define EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H + +#if EIGEN_COMP_LLVM +#define BFLOAT16_UNROLL _Pragma("unroll 8") +#else +#define BFLOAT16_UNROLL _Pragma("GCC unroll(8)") +#endif + +namespace Eigen { + +namespace internal { + +template +EIGEN_ALWAYS_INLINE Packet8bf loadBfloat16(const bfloat16* indexA) +{ + Packet8bf lhs1 = ploadu(indexA); + if(zero){ + Packet8bf lhs2 = pset1(Eigen::bfloat16(0)); + return vec_mergeh(lhs1.m_val, lhs2.m_val); + } else { + return lhs1; + } +} + +template +EIGEN_ALWAYS_INLINE Packet8bf loadRhsBfloat16(const bfloat16* blockB, Index strideB, Index i) +{ + return loadBfloat16(blockB + strideB*i); +} + +template +EIGEN_ALWAYS_INLINE void KLoop +( + const bfloat16* indexA, + const bfloat16* indexB, + __vector_quad (&quad_acc)[num_acc], + Index strideB, + Index k, + Index offsetB, + Index extra_cols, + Index extra_rows +) +{ + Packet8bf lhs[num_lhs], rhs[num_rhs]; + + BFLOAT16_UNROLL + for(Index i = 0; i < (num_rhs - (rhsExtraCols ? 1 : 0)); i++){ + rhs[i] = loadRhsBfloat16(indexB + k*4, strideB, i); + } + if(rhsExtraCols) { + rhs[num_rhs - 1] = loadRhsBfloat16(indexB + k*extra_cols - offsetB, strideB, num_rhs - 1); + } + + indexA += k*(lhsExtraRows ? extra_rows : num_packets); + if (num_lhs == 1) { + lhs[0] = loadBfloat16(indexA); + } else { + BFLOAT16_UNROLL + for(Index j = 0; j < num_lhs; j += 2) { + Packet8bf lhs1 = ploadu(indexA + (j + 0)*(zero ? 4 : 8)); + if (zero) { + Packet8bf lhs2 = pset1(Eigen::bfloat16(0)); + lhs[j + 0] = vec_mergeh(lhs1.m_val, lhs2.m_val); + lhs[j + 1] = vec_mergel(lhs1.m_val, lhs2.m_val); + } else { + lhs[j + 0] = lhs1; + lhs[j + 1] = ploadu(indexA + (j + 1)*8); + } + } + } + + BFLOAT16_UNROLL + for(Index i = 0, x = 0; i < num_rhs; i++) { + BFLOAT16_UNROLL + for(Index j = 0; j < num_lhs; j++, x++) { + __builtin_mma_xvbf16ger2pp(&(quad_acc[x]), reinterpret_cast(rhs[i].m_val), reinterpret_cast(lhs[j].m_val)); + } + } +} + +template +EIGEN_ALWAYS_INLINE void zeroAccumulators(__vector_quad (&quad_acc)[num_acc]) +{ + BFLOAT16_UNROLL + for(Index k = 0; k < num_acc; k++) + __builtin_mma_xxsetaccz(&(quad_acc[k])); +} + +template +EIGEN_ALWAYS_INLINE void disassembleAccumulators(__vector_quad (&quad_acc)[num_acc], Packet4f (&acc)[num_acc][4]) +{ + BFLOAT16_UNROLL + for(Index k = 0; k < num_acc; k++) + __builtin_mma_disassemble_acc((void*)acc[k], &(quad_acc[k])); +} + +template +EIGEN_ALWAYS_INLINE void outputResults(Packet4f (&acc)[num_acc][4], Index rows, const Packet4f pAlpha, float* result, const Index extra_cols, Index extra_rows) +{ + BFLOAT16_UNROLL + for(Index i = 0, k = 0; i < num_rhs - (rhsExtraCols ? 1 : 0); i++, result += 4*rows){ + BFLOAT16_UNROLL + for(Index j = 0; j < num_lhs; j++, k++) { + storeResults(acc[k], rows, pAlpha, result + j*4, extra_cols, extra_rows); + } + } + if(rhsExtraCols) { + storeResults(acc[num_acc - 1], rows, pAlpha, result, extra_cols, extra_rows); + } +} + +template +EIGEN_ALWAYS_INLINE void colLoopBodyIter(Index depth, Index rows, const Packet4f pAlpha, const bfloat16* indexA, const bfloat16* indexB, Index strideB, Index offsetB, float* result, const Index extra_cols, const Index extra_rows) +{ + constexpr Index num_lhs = multiIter ? (num_packets / 4) : 1; + constexpr Index num_rhs = (num_acc + num_lhs - 1) / num_lhs; + + for(Index offset_row = 0; offset_row < num_packets; offset_row += 4, indexA += (multiIter ? 0 : 8), indexB += (multiIter ? (num_rhs*strideB) : 0), result += (multiIter ? (4*rows*num_rhs) : 4)) { + Packet4f acc[num_acc][4]; + __vector_quad quad_acc[num_acc]; + + zeroAccumulators(quad_acc); + + Index k; + for(k = 0; k + 2 <= depth; k += 2){ + KLoop(indexA, indexB, quad_acc, strideB, k, offsetB, extra_cols, extra_rows); + } + if(depth&1){ + KLoop(indexA - (multiIter ? 0 : offset_row), indexB, quad_acc, strideB, k, offsetB, extra_cols, extra_rows); + } + + disassembleAccumulators(quad_acc, acc); + + outputResults(acc, rows, pAlpha, result, extra_cols, extra_rows); + } +} + +#define MAX_BFLOAT16_ACC 8 + +template +void colLoopBody(Index& col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexA, const bfloat16* indexB, Index strideB, Index offsetB, float* result) +{ + constexpr Index step = (num_acc * 4); // each accumulator has 4 elements + const Index extra_cols = (rhsExtraCols) ? (cols & 3) : 0; + const Index extra_rows = (lhsExtraRows) ? (rows & 3) : 0; + constexpr bool multiIters = !rhsExtraCols && (num_acc == MAX_BFLOAT16_ACC); + constexpr bool normIters = multiIters && ((num_acc % (num_packets / 4)) == 0); + + do{ + colLoopBodyIter(depth, rows, pAlpha, indexA, indexB, strideB, offsetB, result, extra_cols, extra_rows); + + indexB += strideB*num_acc; + result += rows*step; + } while(multiIters && (step <= cols - (col += step))); +} + +template +EIGEN_ALWAYS_INLINE void colLoopBodyExtraN(Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexA, const bfloat16* blockB, Index strideB, Index offsetB, float* result) +{ + if (MAX_BFLOAT16_ACC > num_acc) { + colLoopBody(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + } +} + +template +void colLoopBodyExtra(Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexA, const bfloat16* blockB, Index strideB, Index offsetB, float* result) +{ + switch ((cols - col) >> 2) { + case 7: + colLoopBodyExtraN<7, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + break; + case 6: + colLoopBodyExtraN<6, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + break; + case 5: + colLoopBodyExtraN<5, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + break; + case 4: + colLoopBodyExtraN<4, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + break; + case 3: + colLoopBodyExtraN<3, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + break; + case 2: + colLoopBodyExtraN<2, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + break; + case 1: + colLoopBodyExtraN<1, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + break; + default: + if (rhsExtraCols) { + colLoopBody<1, num_packets, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + } + break; + } +} + +template +EIGEN_ALWAYS_INLINE void colLoops(Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexA, const bfloat16* blockB, Index strideB, Index offsetB, float* result) +{ + Index col = 0; + if (cols >= (MAX_BFLOAT16_ACC * 4)) { + colLoopBody(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, 0, result); + blockB += (strideB >> 2)*col; + result += rows*col; + } + if (cols & 3) { + colLoopBodyExtra(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + } else { + colLoopBodyExtra(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, 0, result); + } +} + +EIGEN_ALWAYS_INLINE Packet8bf convertF32toBF16(const float *res) +{ + Packet16uc fp16[2]; + __vector_pair fp16_vp = *reinterpret_cast<__vector_pair *>(const_cast(res)); + __builtin_vsx_disassemble_pair(reinterpret_cast(fp16), &fp16_vp); + fp16[0] = __builtin_vsx_xvcvspbf16(fp16[0]); + fp16[1] = __builtin_vsx_xvcvspbf16(fp16[1]); + return vec_pack(reinterpret_cast(fp16[0]), reinterpret_cast(fp16[1])); +} + +template +EIGEN_ALWAYS_INLINE void convertArrayF32toBF16Col(float *result, Index col, Index rows, const DataMapper& res) +{ + const DataMapper res2 = res.getSubMapper(0, col); + Index row; + float *result2 = result + col*rows; + for(row = 0; row + 8 <= rows; row += 8, result2 += 8){ + // get and save block + PacketBlock block; + BFLOAT16_UNROLL + for(Index j = 0; j < size; j++){ + block.packet[j] = convertF32toBF16(result2 + j*rows); + } + res2.template storePacketBlock(row, 0, block); + } + // extra rows + if(row < rows){ + BFLOAT16_UNROLL + for(Index j = 0; j < size; j++){ + Packet8bf fp16 = convertF32toBF16(result2 + j*rows); + res2.template storePacketPartial(row, j, fp16, rows & 7); + } + } +} + +template +EIGEN_ALWAYS_INLINE void convertPointerF32toBF16(Index& i, float* result, Index rows, bfloat16*& dst, Index resInc = 1) +{ + constexpr Index extra = ((size < 8) ? 8 : size); + while (i + size <= rows){ + PacketBlock r32; + r32.packet[0] = convertF32toBF16(result + i + 0); + if (size >= 16) { + r32.packet[1] = convertF32toBF16(result + i + 8); + } + if (size >= 32) { + r32.packet[2] = convertF32toBF16(result + i + 16); + r32.packet[3] = convertF32toBF16(result + i + 24); + } + storeBF16fromResult(dst, r32.packet[0], resInc, rows & 7); + if (size >= 16) { + storeBF16fromResult(dst, r32.packet[1], resInc); + } + if (size >= 32) { + storeBF16fromResult(dst, r32.packet[2], resInc); + storeBF16fromResult(dst, r32.packet[3], resInc); + } + i += extra; dst += extra*resInc; + if (size != 32) break; + } +} + +template +EIGEN_ALWAYS_INLINE void convertArrayPointerF32toBF16(float *result, Index rows, bfloat16* dst, Index resInc = 1) +{ + Index i = 0; + convertPointerF32toBF16<32,non_unit_stride>(i, result, rows, dst, resInc); + convertPointerF32toBF16<16,non_unit_stride>(i, result, rows, dst, resInc); + convertPointerF32toBF16<8,non_unit_stride>(i, result, rows, dst, resInc); + convertPointerF32toBF16<1,non_unit_stride>(i, result, rows, dst, resInc); +} + +template +EIGEN_ALWAYS_INLINE void convertArrayF32toBF16(float *result, Index cols, Index rows, const DataMapper& res) +{ + Index col; + for(col = 0; col + 4 <= cols; col += 4){ + convertArrayF32toBF16Col(result, col, rows, res); + } + // extra cols + switch (cols - col) { + case 1: + convertArrayF32toBF16Col(result, col, rows, res); + break; + case 2: + convertArrayF32toBF16Col(result, col, rows, res); + break; + case 3: + convertArrayF32toBF16Col(result, col, rows, res); + break; + } +} + +template +EIGEN_ALWAYS_INLINE void calcColLoops(const bfloat16*& indexA, Index& row, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexB, Index strideB, Index offsetA, Index offsetB, Index bigSuffix, float *result) +{ + if ((size == 16) || (rows & size)) { + indexA += size*offsetA; + colLoops(depth, cols, rows, pAlpha, indexA, indexB, strideB, offsetB, result + row); + row += size; + indexA += bigSuffix*size/16; + } +} + +template +void gemmMMAbfloat16(const DataMapper& res, const bfloat16* indexA, const bfloat16* indexB, Index rows, Index depth, Index cols, bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) +{ + float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha); + const Packet4f pAlpha = pset1(falpha); + ei_declare_aligned_stack_constructed_variable(float, result, cols*rows, 0); + + convertArrayBF16toF32(result, cols, rows, res); + + if( strideA == -1 ) strideA = depth; + if( strideB == -1 ) strideB = depth; + // Packing is done in blocks. + // There's 4 possible sizes of blocks + // Blocks of 8 columns with 16 elements (8x16) + // Blocks of 8 columns with 8 elements (8x8). This happens when there's 16 > rows >= 8 + // Blocks of 8 columns with 4 elements (8x4). This happens when there's 8 > rows >= 4 + // Blocks of 8 columns with < 4 elements. This happens when there's less than 4 remaining rows + + // Loop for LHS standard block (8x16) + Index bigSuffix = (2*8) * (strideA-offsetA); + indexB += 4*offsetB; + strideB *= 4; + offsetB *= 3; + + Index row = 0; + while(row + 16 <= rows){ + calcColLoops<16>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result); + } + // LHS (8x8) block + calcColLoops<8>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result); + // LHS (8x4) block + calcColLoops<4>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result); + // extra rows + if(rows & 3){ + // This index is the beginning of remaining block. + colLoops<4, true>(depth, cols, rows, pAlpha, indexA, indexB, strideB, offsetB, result + row); + } + + // Convert back to bfloat16 + convertArrayF32toBF16(result, cols, rows, res); +} + +#undef MAX_BFLOAT16_ACC + +#if !EIGEN_ALTIVEC_DISABLE_MMA +template +EIGEN_ALWAYS_INLINE void loadVecLoop(Index k, LhsMapper& lhs, Packet8bf (&a0)[num_acc], Packet8bf b1) +{ + a0[k + 0] = lhs.template loadPacket(k*4, 0); + if (!zero) { + b1 = lhs.template loadPacket(k*4, 1); + } + if (num_acc > (k + 1)) { + a0[k + 1] = vec_mergel(a0[k + 0].m_val, b1.m_val); + } + a0[k + 0] = vec_mergeh(a0[k + 0].m_val, b1.m_val); +} + +template +EIGEN_ALWAYS_INLINE void multVec(__vector_quad (&quad_acc)[num_acc], Packet8bf (&a0)[num_acc], Packet8bf b0) +{ + BFLOAT16_UNROLL + for(Index k = 0; k < num_acc; k++) { + __builtin_mma_xvbf16ger2pp(&(quad_acc[k]), reinterpret_cast(b0.m_val), reinterpret_cast(a0[k].m_val)); + } +} + +template +EIGEN_ALWAYS_INLINE void vecColLoop(Index j, LhsMapper& lhs, RhsMapper& rhs, __vector_quad (&quad_acc)[num_acc]) +{ + Packet8bf a0[num_acc]; + Packet8bf b1 = pset1(Eigen::bfloat16(0)); + Packet8bf b0 = loadColData(rhs, j); + + if (zero) { + b0 = vec_mergeh(b0.m_val, b1.m_val); + } + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(0, j); + BFLOAT16_UNROLL + for(Index k = 0; k < num_acc; k += 2) { + loadVecLoop(k, lhs2, a0, b1); + } + + multVec(quad_acc, a0, b0); +} + +#define MAX_BFLOAT16_VEC_ACC 8 + +template +void colVecColLoopBody(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + constexpr Index step = (num_acc * 4); + const Index extra_rows = (extraRows) ? (rows & 3) : 0; + constexpr bool multiIters = !extraRows && (num_acc == MAX_BFLOAT16_VEC_ACC); + + do{ + Packet4f acc[num_acc][4]; + __vector_quad quad_acc[num_acc]; + + zeroAccumulators(quad_acc); + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(row, 0); + for(Index j = 0; j + 2 <= cend; j += 2) { + vecColLoop(j, lhs2, rhs, quad_acc); + } + if (cend & 1) { + vecColLoop(cend - 1, lhs2, rhs, quad_acc); + } + + disassembleAccumulators(quad_acc, acc); + + outputVecColResults(acc, result, pAlpha, extra_rows); + + result += step; + } while(multiIters && (step <= rows - (row += step))); +} + +template +EIGEN_ALWAYS_INLINE void colVecColLoopBodyExtraN(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + if (MAX_BFLOAT16_VEC_ACC > num_acc) { + colVecColLoopBody(row, cend, rows, lhs, rhs, pAlpha, result); + } +} + +template +EIGEN_ALWAYS_INLINE void colVecColLoopBodyExtra(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + switch ((rows - row) >> 2) { + case 7: + colVecColLoopBodyExtraN<7, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 6: + colVecColLoopBodyExtraN<6, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 5: + colVecColLoopBodyExtraN<5, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 4: + colVecColLoopBodyExtraN<4, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 3: + colVecColLoopBodyExtraN<3, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 2: + colVecColLoopBodyExtraN<2, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 1: + colVecColLoopBodyExtraN<1, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + default: + if (extraRows) { + colVecColLoopBody<1, LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + } + break; + } +} + +template +EIGEN_ALWAYS_INLINE void calcVecColLoops(Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + Index row = 0; + if (rows >= (MAX_BFLOAT16_VEC_ACC * 4)) { + colVecColLoopBody(row, cend, rows, lhs, rhs, pAlpha, result); + result += row; + } + if (rows & 3) { + colVecColLoopBodyExtra(row, cend, rows, lhs, rhs, pAlpha, result); + } else { + colVecColLoopBodyExtra(row, cend, rows, lhs, rhs, pAlpha, result); + } +} + +template +struct UseMMAStride : std::false_type { + static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs, Packet4f pAlpha, float *result) + { + using RhsSubMapper = typename RhsMapper::SubMapper; + + RhsSubMapper rhs2 = rhs.getSubMapper(j2, 0); + calcVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } +}; + +template +struct UseMMAStride::value>> : std::true_type { + static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs, Packet4f pAlpha, float *result) + { + using RhsSubMapper = typename RhsMapper::SubMapper; + + RhsSubMapper rhs2 = rhs.getSubMapper(j2, 0); + if (rhs.stride() == 1) { + calcVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } else { + calcVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } + } +}; + +template +void gemvMMA_bfloat16_col( + Index rows, Index cols, + const LhsMapper& alhs, + const RhsMapper& rhs, + bfloat16* res, Index resIncr, + bfloat16 alpha) +{ + EIGEN_UNUSED_VARIABLE(resIncr); + eigen_internal_assert(resIncr == 1); + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + RhsMapper rhs2(rhs); + + const Index lhsStride = lhs.stride(); + + // TODO: improve the following heuristic: + const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(bfloat16) < 16000 ? 16 : 8); + float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha); + Packet4f pAlpha = pset1(falpha); + + ei_declare_aligned_stack_constructed_variable(float, result, rows, 0); + + convertArrayPointerBF16toF32(result, 1, rows, res); + + for (Index j2 = 0; j2 < cols; j2 += block_cols) + { + Index jend = numext::mini(j2 + block_cols, cols); + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(0, j2); + UseMMAStride::run(j2, jend, rows, lhs2, rhs2, pAlpha, result); + } + + convertArrayPointerF32toBF16(result, rows, res); +} + +static Packet16uc p16uc_ELEMENT_VEC3 = { 0x0c,0x0d,0x0e,0x0f, 0x1c,0x1d,0x1e,0x1f, 0x0c,0x0d,0x0e,0x0f, 0x1c,0x1d,0x1e,0x1f }; + +template +EIGEN_ALWAYS_INLINE void preduxVecResults2(Packet4f (&acc)[num_acc][4], Index k) +{ + if (num_acc > (k + 1)) { + acc[k][0] = vec_mergeh(acc[k][0], acc[k + 1][0]); + acc[k][1] = vec_mergeo(acc[k][1], acc[k + 1][1]); + acc[k][2] = vec_mergel(acc[k][2], acc[k + 1][2]); + acc[k][3] = vec_perm(acc[k][3], acc[k + 1][3], p16uc_ELEMENT_VEC3); + + acc[k][0] = (acc[k][0] + acc[k][2]) + (acc[k][1] + acc[k][3]); + } else { + acc[k][0] = vec_mergeh(acc[k][0], acc[k][1]); + acc[k][0] += vec_mergel(acc[k][2], acc[k][3]); +#ifdef _BIG_ENDIAN + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 12); +#else + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 4); +#endif + } +} + +template +EIGEN_ALWAYS_INLINE void preduxVecResults(Packet4f (&acc)[num_acc][4]) +{ + BFLOAT16_UNROLL + for(Index k = 0; k < num_acc; k += 4) { + preduxVecResults2(acc, k + 0); + if (num_acc > (k + 2)) { + preduxVecResults2(acc, k + 2); + acc[k + 0][0] = reinterpret_cast(vec_mergeh(reinterpret_cast(acc[k + 0][0]), reinterpret_cast(acc[k + 2][0]))); + } + } +} + +template +EIGEN_ALWAYS_INLINE void multVecLoop(__vector_quad (&quad_acc)[num_acc], const LhsMapper& lhs, RhsMapper& rhs, Index j, Index extra_cols) +{ + Packet8bf a0[num_acc], b0; + + if (extra) { + b0 = rhs.template loadPacketPartial(j, extra_cols); + } else { + b0 = rhs.template loadPacket(j); + } + + const LhsMapper lhs2 = lhs.getSubMapper(0, j); + BFLOAT16_UNROLL + for(Index k = 0; k < num_acc; k++) { + if (extra) { + a0[k] = lhs2.template loadPacketPartial(k, 0, extra_cols); + } else { + a0[k] = lhs2.template loadPacket(k, 0); + } + } + + multVec(quad_acc, a0, b0); +} + +template +EIGEN_ALWAYS_INLINE void vecLoop(Index cols, const LhsMapper& lhs, RhsMapper& rhs, __vector_quad (&quad_acc)[num_acc], Index extra_cols) +{ + Index j = 0; + for(; j + 8 <= cols; j += 8){ + multVecLoop(quad_acc, lhs, rhs, j, extra_cols); + } + + if (extra_cols) { + multVecLoop(quad_acc, lhs, rhs, j, extra_cols); + } +} + +template +void colVecLoopBody(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + constexpr bool multiIters = (num_acc == MAX_BFLOAT16_VEC_ACC); + const Index extra_cols = (cols & 7); + + do{ + Packet4f acc[num_acc][4]; + __vector_quad quad_acc[num_acc]; + + zeroAccumulators(quad_acc); + + const LhsMapper lhs2 = lhs.getSubMapper(row, 0); + vecLoop(cols, lhs2, rhs, quad_acc, extra_cols); + + disassembleAccumulators(quad_acc, acc); + + preduxVecResults(acc); + + outputVecResults(acc, result, pAlpha); + + result += num_acc; + } while(multiIters && (num_acc <= rows - (row += num_acc))); +} + +template +EIGEN_ALWAYS_INLINE void colVecLoopBodyExtraN(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + if (MAX_BFLOAT16_VEC_ACC > num_acc) { + colVecLoopBody(row, cols, rows, lhs, rhs, pAlpha, result); + } +} + +template +EIGEN_ALWAYS_INLINE void colVecLoopBodyExtra(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + switch (rows - row) { + case 7: + colVecLoopBodyExtraN<7, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 6: + colVecLoopBodyExtraN<6, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 5: + colVecLoopBodyExtraN<5, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 4: + colVecLoopBodyExtraN<4, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 3: + colVecLoopBodyExtraN<3, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 2: + colVecLoopBodyExtraN<2, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 1: + colVecLoopBodyExtraN<1, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + } +} + +template +EIGEN_ALWAYS_INLINE void calcVecLoops(Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + Index row = 0; + if (rows >= MAX_BFLOAT16_VEC_ACC) { + colVecLoopBody(row, cols, rows, lhs, rhs, pAlpha, result); + result += row; + } + colVecLoopBodyExtra(row, cols, rows, lhs, rhs, pAlpha, result); +} + +template +EIGEN_STRONG_INLINE void gemvMMA_bfloat16_row( + Index rows, Index cols, + const LhsMapper& alhs, + const RhsMapper& rhs, + bfloat16* res, Index resIncr, + bfloat16 alpha) +{ + typedef typename RhsMapper::LinearMapper LinearMapper; + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + LinearMapper rhs2 = rhs.getLinearMapper(0, 0); + + eigen_internal_assert(rhs.stride() == 1); + + float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha); + const Packet4f pAlpha = pset1(falpha); + + ei_declare_aligned_stack_constructed_variable(float, result, rows, 0); + if (resIncr == 1) { + convertArrayPointerBF16toF32(result, 1, rows, res); + } else { + convertArrayPointerBF16toF32(result, 1, rows, res, resIncr); + } + calcVecLoops(cols, rows, lhs, rhs2, pAlpha, result); + if (resIncr == 1) { + convertArrayPointerF32toBF16(result, rows, res); + } else { + convertArrayPointerF32toBF16(result, rows, res, resIncr); + } +} +#endif + +#undef MAX_BFLOAT16_VEC_ACC +#undef BFLOAT16_UNROLL + +} +} +#endif //EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.h new file mode 100644 index 00000000000..66e10880132 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.h @@ -0,0 +1,2990 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2021 Chip Kerchner (chip.kerchner@ibm.com) +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_MATRIX_VECTOR_PRODUCT_ALTIVEC_H +#define EIGEN_MATRIX_VECTOR_PRODUCT_ALTIVEC_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +#if defined(__MMA__) && !EIGEN_ALTIVEC_DISABLE_MMA +#if EIGEN_COMP_LLVM || (__GNUC__ > 10 || __GNUC_MINOR__ >= 3) +#define USE_GEMV_MMA +#endif + +#if !EIGEN_COMP_LLVM && (__GNUC__ < 11) +// Only allow one vector_pair in buggy gcc - gcc 10.x has a bug +#define GCC_ONE_VECTORPAIR_BUG +#endif +#endif + +//#define USE_SLOWER_GEMV_MMA // MMA is currently not as fast as VSX in complex double GEMV (revisit when gcc is improved) + +//#define EIGEN_POWER_USE_GEMV_PREFETCH +#ifdef EIGEN_POWER_USE_GEMV_PREFETCH +#define EIGEN_POWER_GEMV_PREFETCH(p) prefetch(p) +#else +#define EIGEN_POWER_GEMV_PREFETCH(p) +#endif + +#ifdef __has_builtin +#if !__has_builtin(__builtin_vsx_assemble_pair) +#define __builtin_vsx_assemble_pair __builtin_mma_assemble_pair +#endif +#if !__has_builtin(__builtin_vsx_disassemble_pair) +#define __builtin_vsx_disassemble_pair __builtin_mma_disassemble_pair +#endif +#endif + +#if EIGEN_COMP_LLVM +#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \ + __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src2, (__vector unsigned char)src1) +#else +#if (__GNUC__ <= 10) +#if (__GNUC_MINOR__ > 3) +#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \ + __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src2, (__vector unsigned char)src1) +#else +#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \ + __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src1, (__vector unsigned char)src2) +#endif +#else +#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \ + __builtin_vsx_build_pair(&dst, (__vector unsigned char)src1, (__vector unsigned char)src2) +#endif +#endif + +#define GEMV_IS_COMPLEX_COMPLEX ((sizeof(LhsPacket) == 16) && (sizeof(RhsPacket) == 16)) +#define GEMV_IS_FLOAT (ResPacketSize == (16 / sizeof(float))) +#define GEMV_IS_SCALAR (sizeof(ResPacket) != 16) +#define GEMV_IS_COMPLEX_FLOAT (ResPacketSize == (16 / sizeof(std::complex))) + +/** \internal multiply and add and store results */ +template +EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResPacket& palpha, ResPacket& data) +{ + pstoreu(res, pmadd(data, palpha, ploadu(res))); +} + +template +EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScalar& data) +{ + *res += (alpha * data); +} + +#define GEMV_UNROLL(func, N) \ + func(0, N) func(1, N) func(2, N) func(3, N) \ + func(4, N) func(5, N) func(6, N) func(7, N) + +#define GEMV_UNROLL_HALF(func, N) \ + func(0, 0, 1, N) func(1, 2, 3, N) func(2, 4, 5, N) func(3, 6, 7, N) + +#define GEMV_GETN(N) (((N) * ResPacketSize) >> 2) + +#define GEMV_LOADPACKET_COL(iter) \ + lhs.template load(i + ((iter) * LhsPacketSize), j) + +#ifdef USE_GEMV_MMA +#define GEMV_UNROLL3(func, N, which) \ + func(0, N, which) func(1, N, which) func(2, N, which) func(3, N, which) \ + func(4, N, which) func(5, N, which) func(6, N, which) func(7, N, which) + +#define GEMV_UNUSED_VAR(iter, N, which) \ + if (GEMV_GETN(N) <= iter) { \ + EIGEN_UNUSED_VARIABLE(which##iter); \ + } + +#define GEMV_UNUSED_EXTRA_VAR(iter, N, which) \ + if (N <= iter) { \ + EIGEN_UNUSED_VARIABLE(which##iter); \ + } + +#define GEMV_UNUSED_EXTRA(N, which) \ + GEMV_UNROLL3(GEMV_UNUSED_EXTRA_VAR, N, which) + +#define GEMV_UNUSED(N, which) \ + GEMV_UNROLL3(GEMV_UNUSED_VAR, N, which) + +#define GEMV_INIT_MMA(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + __builtin_mma_xxsetaccz(&e##iter); \ + } + +#if EIGEN_COMP_LLVM +#define GEMV_LOADPAIR_COL_MMA(iter1, iter2) \ + GEMV_BUILDPAIR_MMA(b##iter1, GEMV_LOADPACKET_COL(iter2), GEMV_LOADPACKET_COL((iter2) + 1)); +#else +#define GEMV_LOADPAIR_COL_MMA(iter1, iter2) \ + const LhsScalar& src##iter1 = lhs(i + ((iter1 * 32) / sizeof(LhsScalar)), j); \ + b##iter1 = *reinterpret_cast<__vector_pair *>(const_cast(&src##iter1)); +#endif + +#define GEMV_LOAD1A_COL_MMA(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + if (GEMV_IS_FLOAT) { \ + g##iter = GEMV_LOADPACKET_COL(iter); \ + EIGEN_UNUSED_VARIABLE(b##iter); \ + } else { \ + GEMV_LOADPAIR_COL_MMA(iter, iter << 1) \ + EIGEN_UNUSED_VARIABLE(g##iter); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(b##iter); \ + EIGEN_UNUSED_VARIABLE(g##iter); \ + } + +#define GEMV_WORK1A_COL_MMA(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + if (GEMV_IS_FLOAT) { \ + pger_vecMMA_acc(&e##iter, a0, g##iter); \ + } else { \ + pger_vecMMA_acc(&e##iter, b##iter, a0); \ + } \ + } + +#define GEMV_LOAD1B_COL_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN(N) > iter1) { \ + if (GEMV_IS_FLOAT) { \ + GEMV_LOADPAIR_COL_MMA(iter2, iter2) \ + EIGEN_UNUSED_VARIABLE(b##iter3); \ + } else { \ + GEMV_LOADPAIR_COL_MMA(iter2, iter2 << 1) \ + GEMV_LOADPAIR_COL_MMA(iter3, iter3 << 1) \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(b##iter2); \ + EIGEN_UNUSED_VARIABLE(b##iter3); \ + } \ + EIGEN_UNUSED_VARIABLE(g##iter2); \ + EIGEN_UNUSED_VARIABLE(g##iter3); + +#define GEMV_WORK1B_COL_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN(N) > iter1) { \ + if (GEMV_IS_FLOAT) { \ + LhsPacket h[2]; \ + __builtin_vsx_disassemble_pair(reinterpret_cast(h), &b##iter2); \ + pger_vecMMA_acc(&e##iter2, a0, h[0]); \ + pger_vecMMA_acc(&e##iter3, a0, h[1]); \ + } else { \ + pger_vecMMA_acc(&e##iter2, b##iter2, a0); \ + pger_vecMMA_acc(&e##iter3, b##iter3, a0); \ + } \ + } + +#if EIGEN_COMP_LLVM +#define GEMV_LOAD_COL_MMA(N) \ + if (GEMV_GETN(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_LOAD1B_COL_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) \ + } + +#define GEMV_WORK_COL_MMA(N) \ + if (GEMV_GETN(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_WORK1B_COL_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N) \ + } +#else +#define GEMV_LOAD_COL_MMA(N) \ + GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) + +#define GEMV_WORK_COL_MMA(N) \ + GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N) +#endif + +#define GEMV_DISASSEMBLE_MMA(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + __builtin_mma_disassemble_acc(&result##iter.packet, &e##iter); \ + if (!GEMV_IS_FLOAT) { \ + result##iter.packet[0][1] = result##iter.packet[1][0]; \ + result##iter.packet[2][1] = result##iter.packet[3][0]; \ + } \ + } + +#define GEMV_LOADPAIR2_COL_MMA(iter1, iter2) \ + b##iter1 = *reinterpret_cast<__vector_pair *>(res + i + ((iter2) * ResPacketSize)); + +#define GEMV_LOAD2_COL_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN(N) > iter1) { \ + if (GEMV_IS_FLOAT) { \ + GEMV_LOADPAIR2_COL_MMA(iter2, iter2); \ + EIGEN_UNUSED_VARIABLE(b##iter3); \ + } else { \ + GEMV_LOADPAIR2_COL_MMA(iter2, iter2 << 1); \ + GEMV_LOADPAIR2_COL_MMA(iter3, iter3 << 1); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(b##iter2); \ + EIGEN_UNUSED_VARIABLE(b##iter3); \ + } + +#if EIGEN_COMP_LLVM +#define GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter4) \ + ResPacket f##iter2[2]; \ + __builtin_vsx_disassemble_pair(reinterpret_cast(f##iter2), &b##iter2); \ + f##iter2[0] = pmadd(result##iter2.packet[0], palpha, f##iter2[0]); \ + f##iter2[1] = pmadd(result##iter3.packet[(iter2 == iter3) ? 2 : 0], palpha, f##iter2[1]); \ + GEMV_BUILDPAIR_MMA(b##iter2, f##iter2[0], f##iter2[1]); +#else +#define GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter4) \ + if (GEMV_IS_FLOAT) { \ + __asm__ ("xvmaddasp %0,%x1,%x3\n\txvmaddasp %L0,%x2,%x3" : "+&d" (b##iter2) : "wa" (result##iter3.packet[0]), "wa" (result##iter2.packet[0]), "wa" (palpha)); \ + } else { \ + __asm__ ("xvmaddadp %0,%x1,%x3\n\txvmaddadp %L0,%x2,%x3" : "+&d" (b##iter2) : "wa" (result##iter2.packet[2]), "wa" (result##iter2.packet[0]), "wa" (palpha)); \ + } +#endif + +#define GEMV_WORK2_COL_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN(N) > iter1) { \ + if (GEMV_IS_FLOAT) { \ + GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter2); \ + } else { \ + GEMV_WORKPAIR2_COL_MMA(iter2, iter2, iter2 << 1); \ + GEMV_WORKPAIR2_COL_MMA(iter3, iter3, iter3 << 1); \ + } \ + } + +#define GEMV_STOREPAIR2_COL_MMA(iter1, iter2) \ + *reinterpret_cast<__vector_pair *>(res + i + ((iter2) * ResPacketSize)) = b##iter1; + +#define GEMV_STORE_COL_MMA(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + if (GEMV_IS_FLOAT) { \ + storeMaddData(res + i + (iter * ResPacketSize), palpha, result##iter.packet[0]); \ + } else { \ + GEMV_LOADPAIR2_COL_MMA(iter, iter << 1) \ + GEMV_WORKPAIR2_COL_MMA(iter, iter, iter << 1) \ + GEMV_STOREPAIR2_COL_MMA(iter, iter << 1) \ + } \ + } + +#define GEMV_STORE2_COL_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN(N) > iter1) { \ + if (GEMV_IS_FLOAT) { \ + GEMV_STOREPAIR2_COL_MMA(iter2, iter2); \ + } else { \ + GEMV_STOREPAIR2_COL_MMA(iter2, iter2 << 1) \ + GEMV_STOREPAIR2_COL_MMA(iter3, iter3 << 1) \ + } \ + } + +#define GEMV_PROCESS_COL_ONE_MMA(N) \ + GEMV_UNROLL(GEMV_INIT_MMA, N) \ + Index j = j2; \ + __vector_pair b0, b1, b2, b3, b4, b5, b6, b7; \ + do { \ + LhsPacket g0, g1, g2, g3, g4, g5, g6, g7; \ + RhsPacket a0 = pset1(rhs2(j, 0)); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_LOAD_COL_MMA(N) \ + GEMV_WORK_COL_MMA(N) \ + } while (++j < jend); \ + GEMV_UNROLL(GEMV_DISASSEMBLE_MMA, N) \ + if (GEMV_GETN(N) <= 1) { \ + GEMV_UNROLL(GEMV_STORE_COL_MMA, N) \ + } else { \ + GEMV_UNROLL_HALF(GEMV_LOAD2_COL_MMA, (N >> 1)) \ + GEMV_UNROLL_HALF(GEMV_WORK2_COL_MMA, (N >> 1)) \ + GEMV_UNROLL_HALF(GEMV_STORE2_COL_MMA, (N >> 1)) \ + } \ + i += (ResPacketSize * N); +#endif + +#define GEMV_INIT(iter, N) \ + if (N > iter) { \ + c##iter = pset1(ResScalar(0)); \ + } else { \ + EIGEN_UNUSED_VARIABLE(c##iter); \ + } + +#ifdef EIGEN_POWER_USE_GEMV_PREFETCH +#define GEMV_PREFETCH(iter, N) \ + if (GEMV_GETN(N) > ((iter >> 1) + ((N >> 1) * (iter & 1)))) { \ + lhs.prefetch(i + (iter * LhsPacketSize) + prefetch_dist, j); \ + } +#else +#define GEMV_PREFETCH(iter, N) +#endif + +#define GEMV_WORK_COL(iter, N) \ + if (N > iter) { \ + c##iter = pcj.pmadd(GEMV_LOADPACKET_COL(iter), a0, c##iter); \ + } + +#define GEMV_STORE_COL(iter, N) \ + if (N > iter) { \ + pstoreu(res + i + (iter * ResPacketSize), pmadd(c##iter, palpha, ploadu(res + i + (iter * ResPacketSize)))); \ + } + +/** \internal main macro for gemv_col - initialize accumulators, multiply and add inputs, and store results */ +#define GEMV_PROCESS_COL_ONE(N) \ + GEMV_UNROLL(GEMV_INIT, N) \ + Index j = j2; \ + do { \ + RhsPacket a0 = pset1(rhs2(j, 0)); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_UNROLL(GEMV_WORK_COL, N) \ + } while (++j < jend); \ + GEMV_UNROLL(GEMV_STORE_COL, N) \ + i += (ResPacketSize * N); + +#ifdef USE_GEMV_MMA +#define GEMV_PROCESS_COL(N) \ + GEMV_PROCESS_COL_ONE_MMA(N) +#else +#define GEMV_PROCESS_COL(N) \ + GEMV_PROCESS_COL_ONE(N) +#endif + +/** \internal perform a matrix multiply and accumulate of packet a and packet b */ +#ifdef USE_GEMV_MMA +template +EIGEN_ALWAYS_INLINE void pger_vecMMA_acc(__vector_quad* acc, const RhsPacket& a, const LhsPacket& b) +{ + if (accumulate) + { + __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b); + } + else + { + __builtin_mma_xvf32ger(acc, (__vector unsigned char)a, (__vector unsigned char)b); + } +} + +/** \internal perform a matrix multiply and accumulate of vector_pair a and packet b */ +template +EIGEN_ALWAYS_INLINE void pger_vecMMA_acc(__vector_quad* acc, __vector_pair& a, const LhsPacket& b) +{ + if (accumulate) + { + __builtin_mma_xvf64gerpp(acc, a, (__vector unsigned char)b); + } + else + { + __builtin_mma_xvf64ger(acc, a, (__vector unsigned char)b); + } +} +#endif + +template +EIGEN_STRONG_INLINE void gemv_col( + Index rows, Index cols, + const LhsMapper& alhs, + const RhsMapper& rhs, + ResScalar* res, Index resIncr, + ResScalar alpha) +{ + typedef gemv_traits Traits; + + typedef typename Traits::LhsPacket LhsPacket; + typedef typename Traits::RhsPacket RhsPacket; + typedef typename Traits::ResPacket ResPacket; + + EIGEN_UNUSED_VARIABLE(resIncr); + eigen_internal_assert(resIncr == 1); + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + RhsMapper rhs2(rhs); + + conj_helper cj; + conj_helper pcj; + + const Index lhsStride = lhs.stride(); + // TODO: for padded aligned inputs, we could enable aligned reads + enum { + LhsAlignment = Unaligned, + ResPacketSize = Traits::ResPacketSize, + LhsPacketSize = Traits::LhsPacketSize, + RhsPacketSize = Traits::RhsPacketSize, + }; + +#ifndef GCC_ONE_VECTORPAIR_BUG + const Index n8 = rows - 8 * ResPacketSize + 1; + const Index n4 = rows - 4 * ResPacketSize + 1; + const Index n2 = rows - 2 * ResPacketSize + 1; +#endif + const Index n1 = rows - 1 * ResPacketSize + 1; +#ifdef EIGEN_POWER_USE_GEMV_PREFETCH + const Index prefetch_dist = 64 * LhsPacketSize; +#endif + + // TODO: improve the following heuristic: + const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(LhsScalar) < 16000 ? 16 : 8); + ResPacket palpha = pset1(alpha); + + for (Index j2 = 0; j2 < cols; j2 += block_cols) + { + Index jend = numext::mini(j2 + block_cols, cols); + Index i = 0; + ResPacket c0, c1, c2, c3, c4, c5, c6, c7; +#ifdef USE_GEMV_MMA + __vector_quad e0, e1, e2, e3, e4, e5, e6, e7; + PacketBlock result0, result1, result2, result3, result4, result5, result6, result7; + GEMV_UNUSED(8, e) + GEMV_UNUSED(8, result) + GEMV_UNUSED_EXTRA(1, c) +#endif +#ifndef GCC_ONE_VECTORPAIR_BUG + while (i < n8) + { + GEMV_PROCESS_COL(8) + } + if (i < n4) + { + GEMV_PROCESS_COL(4) + } + if (i < n2) + { + GEMV_PROCESS_COL(2) + } + if (i < n1) +#else + while (i < n1) +#endif + { + GEMV_PROCESS_COL_ONE(1) + } + for (;i < rows;++i) + { + ResScalar d0(0); + Index j = j2; + do { + d0 += cj.pmul(lhs(i, j), rhs2(j, 0)); + } while (++j < jend); + res[i] += alpha * d0; + } + } +} + +template +EIGEN_ALWAYS_INLINE void outputVecCol(Packet4f acc, float *result, Packet4f pAlpha, Index extra_rows) +{ + Packet4f d0 = ploadu(result); + d0 = pmadd(acc, pAlpha, d0); + if (extraRows) { + pstoreu_partial(result, d0, extra_rows); + } else { + pstoreu(result, d0); + } +} + +template +EIGEN_ALWAYS_INLINE void outputVecColResults(Packet4f (&acc)[num_acc][size], float *result, Packet4f pAlpha, Index extra_rows) +{ + constexpr Index real_acc = (num_acc - (extraRows ? 1 : 0)); + for(Index k = 0; k < real_acc; k++) { + outputVecCol(acc[k][0], result + k*4, pAlpha, extra_rows); + } + if (extraRows) { + outputVecCol(acc[real_acc][0], result + real_acc*4, pAlpha, extra_rows); + } +} + +static Packet16uc p16uc_MERGE16_32_V1 = { 0, 1, 16,17, 0, 1, 16,17, 0, 1, 16,17, 0, 1, 16,17 }; +static Packet16uc p16uc_MERGE16_32_V2 = { 2, 3, 18,19, 2, 3, 18,19, 2, 3, 18,19, 2, 3, 18,19 }; + +template +EIGEN_ALWAYS_INLINE void loadVecLoopVSX(Index k, LhsMapper& lhs, Packet4f (&a0)[num_acc][2]) +{ + Packet8bf c0 = lhs.template loadPacket(k*4, 0); + Packet8bf b1; + if (!zero) { + b1 = lhs.template loadPacket(k*4, 1); + + a0[k + 0][1] = oneConvertBF16Hi(b1.m_val); + } + a0[k + 0][0] = oneConvertBF16Hi(c0.m_val); + + if (num_acc > (k + 1)) { + a0[k + 1][0] = oneConvertBF16Lo(c0.m_val); + if (!zero) { + a0[k + 1][1] = oneConvertBF16Lo(b1.m_val); + } + } +} + +template +EIGEN_ALWAYS_INLINE void multVecVSX(Packet4f (&acc)[num_acc][2], Packet4f (&a0)[num_acc][2], Packet4f (&b0)[2]) +{ + for(Index k = 0; k < num_acc; k++) { + for(Index i = 0; i < (zero ? 1 : 2); i++) { + acc[k][i] = pmadd(b0[i], a0[k][i], acc[k][i]); + } + } +} + +template +struct loadColData_impl +{ + // linear == false + static EIGEN_ALWAYS_INLINE Packet8bf run(RhsMapper& rhs, Index j) + { + const Index n = unpacket_traits::size; + EIGEN_ALIGN16 bfloat16 to[n]; + LOAD_STORE_UNROLL_16 + for (Index i = 0; i < n; i++) { + to[i] = rhs(j + i, 0); + } + return pload(to); + } +}; + +template +struct loadColData_impl +{ + // linear == true + static EIGEN_ALWAYS_INLINE Packet8bf run(RhsMapper& rhs, Index j) + { + return rhs.template loadPacket(j + 0, 0); + } +}; + +template +EIGEN_ALWAYS_INLINE Packet8bf loadColData(RhsMapper& rhs, Index j) +{ + return loadColData_impl::run(rhs, j); +} + +template +EIGEN_ALWAYS_INLINE void vecColLoopVSX(Index j, LhsMapper& lhs, RhsMapper& rhs, Packet4f (&acc)[num_acc][2]) +{ + Packet4f a0[num_acc][2], b0[2]; + Packet8bf b2 = loadColData(rhs, j); + + b0[0] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V1); + if (!zero) { + b0[1] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V2); + } + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(0, j); + for(Index k = 0; k < num_acc; k += 2) { + loadVecLoopVSX(k, lhs2, a0); + } + + multVecVSX(acc, a0, b0); +} + +template +EIGEN_ALWAYS_INLINE void addResultsVSX(Packet4f (&acc)[num_acc][2]) +{ + for(Index i = 0; i < num_acc; i++) { + acc[i][0] = acc[i][0] + acc[i][1]; + } +} + +// Uses 2X the accumulators or 4X the number of VSX registers +#define MAX_BFLOAT16_VEC_ACC_VSX 8 + +template +void colVSXVecColLoopBody(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + constexpr Index step = (num_acc * 4); + const Index extra_rows = (extraRows) ? (rows & 3) : 0; + constexpr bool multiIters = !extraRows && (num_acc == MAX_BFLOAT16_VEC_ACC_VSX); + + do{ + Packet4f acc[num_acc][2]; + + zeroAccumulators(acc); + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(row, 0); + for(Index j = 0; j + 2 <= cend; j += 2) { + vecColLoopVSX(j, lhs2, rhs, acc); + } + if (cend & 1) { + vecColLoopVSX(cend - 1, lhs2, rhs, acc); + } + + addResultsVSX(acc); + + outputVecColResults(acc, result, pAlpha, extra_rows); + + result += step; + } while(multiIters && (step <= rows - (row += step))); +} + +template +EIGEN_ALWAYS_INLINE void colVSXVecColLoopBodyExtraN(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + if (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { + colVSXVecColLoopBody(row, cend, rows, lhs, rhs, pAlpha, result); + } +} + +template +EIGEN_ALWAYS_INLINE void colVSXVecColLoopBodyExtra(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + switch ((rows - row) >> 2) { + case 7: + colVSXVecColLoopBodyExtraN<7, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 6: + colVSXVecColLoopBodyExtraN<6, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 5: + colVSXVecColLoopBodyExtraN<5, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 4: + colVSXVecColLoopBodyExtraN<4, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 3: + colVSXVecColLoopBodyExtraN<3, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 2: + colVSXVecColLoopBodyExtraN<2, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 1: + colVSXVecColLoopBodyExtraN<1, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + default: + if (extraRows) { + colVSXVecColLoopBody<1, LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + } + break; + } +} + +template +EIGEN_ALWAYS_INLINE void calcVSXVecColLoops(Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + Index row = 0; + if (rows >= (MAX_BFLOAT16_VEC_ACC_VSX * 4)) { + colVSXVecColLoopBody(row, cend, rows, lhs, rhs, pAlpha, result); + result += row; + } + if (rows & 3) { + colVSXVecColLoopBodyExtra(row, cend, rows, lhs, rhs, pAlpha, result); + } else { + colVSXVecColLoopBodyExtra(row, cend, rows, lhs, rhs, pAlpha, result); + } +} + +template +EIGEN_ALWAYS_INLINE void storeBF16fromResult(bfloat16* dst, Packet8bf data, Index resInc, Index extra) +{ + if (inc) { + if (size < 8) { + pscatter_partial(dst + delta*resInc, data, resInc, extra); + } else { + pscatter(dst + delta*resInc, data, resInc); + } + } else { + if (size < 8) { + pstoreu_partial(dst + delta, data, extra); + } else { + pstoreu(dst + delta, data); + } + } +} + +template +EIGEN_ALWAYS_INLINE void convertPointerF32toBF16VSX(Index& i, float* result, Index rows, bfloat16*& dst, Index resInc = 1) +{ + constexpr Index extra = ((size < 8) ? 8 : size); + while (i + size <= rows) { + PacketBlock r32; + r32.packet[0] = convertF32toBF16VSX(result + i + 0); + if (size >= 16) { + r32.packet[1] = convertF32toBF16VSX(result + i + 8); + } + if (size >= 32) { + r32.packet[2] = convertF32toBF16VSX(result + i + 16); + r32.packet[3] = convertF32toBF16VSX(result + i + 24); + } + storeBF16fromResult(dst, r32.packet[0], resInc, rows & 7); + if (size >= 16) { + storeBF16fromResult(dst, r32.packet[1], resInc); + } + if (size >= 32) { + storeBF16fromResult(dst, r32.packet[2], resInc); + storeBF16fromResult(dst, r32.packet[3], resInc); + } + i += extra; dst += extra*resInc; + if (size != 32) break; + } +} + +template +EIGEN_ALWAYS_INLINE void convertArrayPointerF32toBF16VSX(float *result, Index rows, bfloat16* dst, Index resInc = 1) +{ + Index i = 0; + convertPointerF32toBF16VSX<32,inc>(i, result, rows, dst, resInc); + convertPointerF32toBF16VSX<16,inc>(i, result, rows, dst, resInc); + convertPointerF32toBF16VSX<8,inc>(i, result, rows, dst, resInc); + convertPointerF32toBF16VSX<1,inc>(i, result, rows, dst, resInc); +} + +template +struct UseStride : std::false_type { + static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs, Packet4f pAlpha, float *result) + { + using RhsSubMapper = typename RhsMapper::SubMapper; + + RhsSubMapper rhs2 = rhs.getSubMapper(j2, 0); + calcVSXVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } +}; + +template +struct UseStride::value>> : std::true_type { + static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs, Packet4f pAlpha, float *result) + { + using RhsSubMapper = typename RhsMapper::SubMapper; + + RhsSubMapper rhs2 = rhs.getSubMapper(j2, 0); + if (rhs.stride() == 1) { + calcVSXVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } else { + calcVSXVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } + } +}; + +template +void gemv_bfloat16_col( + Index rows, Index cols, + const LhsMapper& alhs, + const RhsMapper& rhs, + bfloat16* res, Index resIncr, + bfloat16 alpha) +{ + EIGEN_UNUSED_VARIABLE(resIncr); + eigen_internal_assert(resIncr == 1); + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + RhsMapper rhs2(rhs); + + const Index lhsStride = lhs.stride(); + + // TODO: improve the following heuristic: + const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(bfloat16) < 16000 ? 16 : 8); + float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha); + Packet4f pAlpha = pset1(falpha); + + ei_declare_aligned_stack_constructed_variable(float, result, rows, 0); + + convertArrayPointerBF16toF32(result, 1, rows, res); + + for (Index j2 = 0; j2 < cols; j2 += block_cols) + { + Index jend = numext::mini(j2 + block_cols, cols); + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(0, j2); + UseStride::run(j2, jend, rows, lhs2, rhs2, pAlpha, result); + } + + convertArrayPointerF32toBF16VSX(result, rows, res); +} + +template +EIGEN_ALWAYS_INLINE void outputVecResults(Packet4f (&acc)[num_acc][size], float *result, Packet4f pAlpha) +{ + constexpr Index extra = num_acc & 3; + + for(Index k = 0; k < num_acc; k += 4) { + Packet4f d0 = ploadu(result + k); + d0 = pmadd(acc[k + 0][0], pAlpha, d0); + + if (num_acc > (k + 3)) { + pstoreu(result + k, d0); + } else { + if (extra == 3) { + pstoreu_partial(result + k, d0, extra); + } else { + memcpy((void *)(result + k), (void *)(&d0), sizeof(float) * extra); + } + } + } +} + +template +EIGEN_ALWAYS_INLINE void preduxVecResults2VSX(Packet4f (&acc)[num_acc][2], Index k) +{ + if (num_acc > (k + 1)) { + acc[k][1] = vec_mergel(acc[k + 0][0], acc[k + 1][0]); + acc[k][0] = vec_mergeh(acc[k + 0][0], acc[k + 1][0]); + acc[k][0] = acc[k][0] + acc[k][1]; + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8); + } else { + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8); +#ifdef _BIG_ENDIAN + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 12); +#else + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 4); +#endif + } +} + +template +EIGEN_ALWAYS_INLINE void preduxVecResultsVSX(Packet4f (&acc)[num_acc][2]) +{ + for(Index k = 0; k < num_acc; k += 4) { + preduxVecResults2VSX(acc, k + 0); + if (num_acc > (k + 2)) { + preduxVecResults2VSX(acc, k + 2); +#ifdef EIGEN_VECTORIZE_VSX + acc[k + 0][0] = reinterpret_cast(vec_mergeh(reinterpret_cast(acc[k + 0][0]), reinterpret_cast(acc[k + 2][0]))); +#else + acc[k + 0][0] = reinterpret_cast(vec_perm(acc[k + 0][0],acc[k + 2][0],p16uc_TRANSPOSE64_HI)); +#endif + } + } +} + +#ifndef _ARCH_PWR9 +EIGEN_ALWAYS_INLINE Packet8us loadPacketPartialZero(Packet8us data, Index extra_cols) +{ + Packet16uc shift = pset1(8 * 2 * (8 - extra_cols)); +#ifdef _BIG_ENDIAN + return reinterpret_cast(vec_slo(vec_sro(reinterpret_cast(data), shift), shift)); +#else + return reinterpret_cast(vec_sro(vec_slo(reinterpret_cast(data), shift), shift)); +#endif +} +#endif + +template +EIGEN_ALWAYS_INLINE void multVSXVecLoop(Packet4f (&acc)[num_acc][2], const LhsMapper& lhs, RhsMapper& rhs, Index j, Index extra_cols) +{ + Packet4f a0[num_acc][2], b0[2]; + Packet8bf a1, b1; + + if (extra) { + b1 = rhs.template loadPacketPartial(j, extra_cols); +#ifndef _ARCH_PWR9 + b1 = loadPacketPartialZero(b1.m_val, extra_cols); +#endif + } else { + b1 = rhs.template loadPacket(j); + } + b0[0] = oneConvertBF16Hi(b1.m_val); + b0[1] = oneConvertBF16Lo(b1.m_val); + + const LhsMapper lhs2 = lhs.getSubMapper(0, j); + for(Index k = 0; k < num_acc; k++) { + if (extra) { + a1 = lhs2.template loadPacketPartial(k, 0, extra_cols); +#ifndef _ARCH_PWR9 + a1 = loadPacketPartialZero(a1.m_val, extra_cols); +#endif + } else { + a1 = lhs2.template loadPacket(k, 0); + } + a0[k][0] = oneConvertBF16Hi(a1.m_val); + a0[k][1] = oneConvertBF16Lo(a1.m_val); + } + + multVecVSX(acc, a0, b0); +} + +template +EIGEN_ALWAYS_INLINE void vecVSXLoop(Index cols, const LhsMapper& lhs, RhsMapper& rhs, Packet4f (&acc)[num_acc][2], Index extra_cols) +{ + Index j = 0; + for(; j + 8 <= cols; j += 8){ + multVSXVecLoop(acc, lhs, rhs, j, extra_cols); + } + + if (extra_cols) { + multVSXVecLoop(acc, lhs, rhs, j, extra_cols); + } +} + +template +void colVSXVecLoopBody(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + constexpr bool multiIters = (num_acc == MAX_BFLOAT16_VEC_ACC_VSX); + const Index extra_cols = (cols & 7); + + do{ + Packet4f acc[num_acc][2]; + + zeroAccumulators(acc); + + const LhsMapper lhs2 = lhs.getSubMapper(row, 0); + vecVSXLoop(cols, lhs2, rhs, acc, extra_cols); + + addResultsVSX(acc); + + preduxVecResultsVSX(acc); + + outputVecResults(acc, result, pAlpha); + + result += num_acc; + } while(multiIters && (num_acc <= rows - (row += num_acc))); +} + +template +EIGEN_ALWAYS_INLINE void colVSXVecLoopBodyExtraN(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + if (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { + colVSXVecLoopBody(row, cols, rows, lhs, rhs, pAlpha, result); + } +} + +template +EIGEN_ALWAYS_INLINE void colVSXVecLoopBodyExtra(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + switch (rows - row) { + case 7: + colVSXVecLoopBodyExtraN<7, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 6: + colVSXVecLoopBodyExtraN<6, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 5: + colVSXVecLoopBodyExtraN<5, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 4: + colVSXVecLoopBodyExtraN<4, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 3: + colVSXVecLoopBodyExtraN<3, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 2: + colVSXVecLoopBodyExtraN<2, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 1: + colVSXVecLoopBodyExtraN<1, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + } +} + +template +EIGEN_ALWAYS_INLINE void calcVSXVecLoops(Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + Index row = 0; + if (rows >= MAX_BFLOAT16_VEC_ACC_VSX) { + colVSXVecLoopBody(row, cols, rows, lhs, rhs, pAlpha, result); + result += row; + } + colVSXVecLoopBodyExtra(row, cols, rows, lhs, rhs, pAlpha, result); +} + +template +EIGEN_STRONG_INLINE void gemv_bfloat16_row( + Index rows, Index cols, + const LhsMapper& alhs, + const RhsMapper& rhs, + bfloat16* res, Index resIncr, + bfloat16 alpha) +{ + typedef typename RhsMapper::LinearMapper LinearMapper; + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + LinearMapper rhs2 = rhs.getLinearMapper(0, 0); + + eigen_internal_assert(rhs.stride() == 1); + + float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha); + const Packet4f pAlpha = pset1(falpha); + + ei_declare_aligned_stack_constructed_variable(float, result, rows, 0); + if (resIncr == 1) { + convertArrayPointerBF16toF32(result, 1, rows, res); + } else { + convertArrayPointerBF16toF32(result, 1, rows, res, resIncr); + } + calcVSXVecLoops(cols, rows, lhs, rhs2, pAlpha, result); + if (resIncr == 1) { + convertArrayPointerF32toBF16VSX(result, rows, res); + } else { + convertArrayPointerF32toBF16VSX(result, rows, res, resIncr); + } +} + +#undef MAX_BFLOAT16_VEC_ACC_VSX + +const Packet16uc p16uc_COMPLEX32_XORFLIP = { 0x44,0x55,0x66,0x77, 0x00,0x11,0x22,0x33, 0xcc,0xdd,0xee,0xff, 0x88,0x99,0xaa,0xbb }; +const Packet16uc p16uc_COMPLEX64_XORFLIP = { 0x88,0x99,0xaa,0xbb, 0xcc,0xdd,0xee,0xff, 0x00,0x11,0x22,0x33, 0x44,0x55,0x66,0x77 }; + +#ifdef _BIG_ENDIAN +const Packet16uc p16uc_COMPLEX32_CONJ_XOR = { 0x00,0x00,0x00,0x00, 0x80,0x00,0x00,0x00, 0x00,0x00,0x00,0x00, 0x80,0x00,0x00,0x00 }; +const Packet16uc p16uc_COMPLEX64_CONJ_XOR = { 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00, 0x80,0x00,0x00,0x00, 0x00,0x00,0x00,0x00 }; +const Packet16uc p16uc_COMPLEX32_CONJ_XOR2 = { 0x80,0x00,0x00,0x00, 0x00,0x00,0x00,0x00, 0x80,0x00,0x00,0x00, 0x00,0x00,0x00,0x00 }; +const Packet16uc p16uc_COMPLEX64_CONJ_XOR2 = { 0x80,0x00,0x00,0x00, 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00 }; +const Packet16uc p16uc_COMPLEX32_NEGATE = { 0x80,0x00,0x00,0x00, 0x80,0x00,0x00,0x00, 0x80,0x00,0x00,0x00, 0x80,0x00,0x00,0x00 }; +const Packet16uc p16uc_COMPLEX64_NEGATE = { 0x80,0x00,0x00,0x00, 0x00,0x00,0x00,0x00, 0x80,0x00,0x00,0x00, 0x00,0x00,0x00,0x00 }; +#else +const Packet16uc p16uc_COMPLEX32_CONJ_XOR = { 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x80, 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x80 }; +const Packet16uc p16uc_COMPLEX64_CONJ_XOR = { 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x80 }; +const Packet16uc p16uc_COMPLEX32_CONJ_XOR2 = { 0x00,0x00,0x00,0x80, 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x80, 0x00,0x00,0x00,0x00 }; +const Packet16uc p16uc_COMPLEX64_CONJ_XOR2 = { 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x80, 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00 }; +const Packet16uc p16uc_COMPLEX32_NEGATE = { 0x00,0x00,0x00,0x80, 0x00,0x00,0x00,0x80, 0x00,0x00,0x00,0x80, 0x00,0x00,0x00,0x80 }; +const Packet16uc p16uc_COMPLEX64_NEGATE = { 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x80, 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x80 }; +#endif + +#ifdef _BIG_ENDIAN +#define COMPLEX_DELTA 0 +#else +#define COMPLEX_DELTA 2 +#endif + +/** \internal packet conjugate (same as pconj but uses the constants in pcplxflipconj for better code generation) */ +EIGEN_ALWAYS_INLINE Packet2cf pconj2(const Packet2cf& a) { + return Packet2cf(pxor(a.v, reinterpret_cast(p16uc_COMPLEX32_CONJ_XOR))); +} + +EIGEN_ALWAYS_INLINE Packet1cd pconj2(const Packet1cd& a) { + return Packet1cd(pxor(a.v, reinterpret_cast(p16uc_COMPLEX64_CONJ_XOR))); +} + +/** \internal packet conjugate with real & imaginary operation inverted */ +EIGEN_ALWAYS_INLINE Packet2cf pconjinv(const Packet2cf& a) { +#ifdef __POWER8_VECTOR__ + return Packet2cf(Packet4f(vec_neg(Packet2d(a.v)))); +#else + return Packet2cf(pxor(a.v, reinterpret_cast(p16uc_COMPLEX32_CONJ_XOR2))); +#endif +} + +EIGEN_ALWAYS_INLINE Packet1cd pconjinv(const Packet1cd& a) { + return Packet1cd(pxor(a.v, reinterpret_cast(p16uc_COMPLEX64_CONJ_XOR2))); +} + +#if defined(_ARCH_PWR8) && (!EIGEN_COMP_LLVM || __clang_major__ >= 12) +#define PERMXOR_GOOD // Clang had a bug with vec_permxor and endianness prior to version 12 +#endif + +/** \internal flip the real & imaginary results and packet conjugate */ +EIGEN_ALWAYS_INLINE Packet2cf pcplxflipconj(Packet2cf a) +{ +#ifdef PERMXOR_GOOD + return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_CONJ_XOR, p16uc_COMPLEX32_XORFLIP))); +#else + return pcplxflip(pconj2(a)); +#endif +} + +EIGEN_ALWAYS_INLINE Packet1cd pcplxflipconj(Packet1cd a) +{ +#ifdef PERMXOR_GOOD + return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_CONJ_XOR, p16uc_COMPLEX64_XORFLIP))); +#else + return pcplxflip(pconj2(a)); +#endif +} + +/** \internal packet conjugate and flip the real & imaginary results */ +EIGEN_ALWAYS_INLINE Packet2cf pcplxconjflip(Packet2cf a) +{ +#ifdef PERMXOR_GOOD + return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_CONJ_XOR2, p16uc_COMPLEX32_XORFLIP))); +#else + return pconj2(pcplxflip(a)); +#endif +} + +EIGEN_ALWAYS_INLINE Packet1cd pcplxconjflip(Packet1cd a) +{ +#ifdef PERMXOR_GOOD + return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_CONJ_XOR2, p16uc_COMPLEX64_XORFLIP))); +#else + return pconj2(pcplxflip(a)); +#endif +} + +/** \internal packet negate */ +EIGEN_ALWAYS_INLINE Packet2cf pnegate2(Packet2cf a) +{ +#ifdef __POWER8_VECTOR__ + return Packet2cf(vec_neg(a.v)); +#else + return Packet2cf(pxor(a.v, reinterpret_cast(p16uc_COMPLEX32_NEGATE))); +#endif +} + +EIGEN_ALWAYS_INLINE Packet1cd pnegate2(Packet1cd a) +{ +#ifdef __POWER8_VECTOR__ + return Packet1cd(vec_neg(a.v)); +#else + return Packet1cd(pxor(a.v, reinterpret_cast(p16uc_COMPLEX64_NEGATE))); +#endif +} + +/** \internal flip the real & imaginary results and negate */ +EIGEN_ALWAYS_INLINE Packet2cf pcplxflipnegate(Packet2cf a) +{ +#ifdef PERMXOR_GOOD + return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_NEGATE, p16uc_COMPLEX32_XORFLIP))); +#else + return pcplxflip(pnegate2(a)); +#endif +} + +EIGEN_ALWAYS_INLINE Packet1cd pcplxflipnegate(Packet1cd a) +{ +#ifdef PERMXOR_GOOD + return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_NEGATE, p16uc_COMPLEX64_XORFLIP))); +#else + return pcplxflip(pnegate2(a)); +#endif +} + +/** \internal flip the real & imaginary results */ +EIGEN_ALWAYS_INLINE Packet2cf pcplxflip2(Packet2cf a) +{ + return Packet2cf(Packet4f(vec_perm(Packet16uc(a.v), Packet16uc(a.v), p16uc_COMPLEX32_XORFLIP))); +} + +EIGEN_ALWAYS_INLINE Packet1cd pcplxflip2(Packet1cd a) +{ +#ifdef EIGEN_VECTORIZE_VSX + return Packet1cd(__builtin_vsx_xxpermdi(a.v, a.v, 2)); +#else + return Packet1cd(Packet2d(vec_perm(Packet16uc(a.v), Packet16uc(a.v), p16uc_COMPLEX64_XORFLIP))); +#endif +} + +/** \internal load half a vector with one complex value */ +EIGEN_ALWAYS_INLINE Packet4f pload_complex_half(std::complex* src) +{ + Packet4f t; +#ifdef EIGEN_VECTORIZE_VSX + // Load float64/two float32 (doubleword alignment) + __asm__("lxsdx %x0,%y1" : "=wa" (t) : "Z" (*src)); +#else + *reinterpret_cast*>(reinterpret_cast(&t) + COMPLEX_DELTA) = *src; +#endif + return t; +} + +/** \internal load two vectors from the real and imaginary portions of a complex value */ +template +EIGEN_ALWAYS_INLINE void pload_realimag(RhsScalar* src, Packet4f& r, Packet4f& i) +{ +#ifdef _ARCH_PWR9 + __asm__("lxvwsx %x0,%y1" : "=wa" (r) : "Z" (*(reinterpret_cast(src) + 0))); + __asm__("lxvwsx %x0,%y1" : "=wa" (i) : "Z" (*(reinterpret_cast(src) + 1))); +#else + Packet4f t = pload_complex_half(src); + r = vec_splat(t, COMPLEX_DELTA + 0); + i = vec_splat(t, COMPLEX_DELTA + 1); +#endif +} + +template +EIGEN_ALWAYS_INLINE void pload_realimag(RhsScalar* src, Packet2d& r, Packet2d& i) +{ +#ifdef EIGEN_VECTORIZE_VSX + __asm__("lxvdsx %x0,%y1" : "=wa" (r) : "Z" (*(reinterpret_cast(src) + 0))); + __asm__("lxvdsx %x0,%y1" : "=wa" (i) : "Z" (*(reinterpret_cast(src) + 1))); +#else + Packet2d t = ploadu(reinterpret_cast(src)); + r = vec_splat(t, 0); + i = vec_splat(t, 1); +#endif +} + +#ifndef __POWER8_VECTOR__ +const Packet16uc p16uc_MERGEE = { 0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13, 0x08, 0x09, 0x0A, 0x0B, 0x18, 0x19, 0x1A, 0x1B }; + +const Packet16uc p16uc_MERGEO = { 0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17, 0x0C, 0x0D, 0x0E, 0x0F, 0x1C, 0x1D, 0x1E, 0x1F }; +#endif + +/** \internal load two vectors from the interleaved real & imaginary values of src */ +template +EIGEN_ALWAYS_INLINE void pload_realimag_row(RhsScalar* src, Packet4f& r, Packet4f& i) +{ + Packet4f t = ploadu(reinterpret_cast(src)); +#ifdef __POWER8_VECTOR__ + r = vec_mergee(t, t); + i = vec_mergeo(t, t); +#else + r = vec_perm(t, t, p16uc_MERGEE); + i = vec_perm(t, t, p16uc_MERGEO); +#endif +} + +template +EIGEN_ALWAYS_INLINE void pload_realimag_row(RhsScalar* src, Packet2d& r, Packet2d& i) +{ + return pload_realimag(src, r, i); +} + +/** \internal load and splat a complex value into a vector - column-wise */ +EIGEN_ALWAYS_INLINE Packet4f pload_realimag_combine(std::complex* src) +{ +#ifdef EIGEN_VECTORIZE_VSX + Packet4f ret; + __asm__("lxvdsx %x0,%y1" : "=wa" (ret) : "Z" (*(reinterpret_cast(src) + 0))); + return ret; +#else + return Packet4f(ploaddup(reinterpret_cast(src))); +#endif +} + +EIGEN_ALWAYS_INLINE Packet2d pload_realimag_combine(std::complex* src) +{ + return ploadu(src).v; +} + +/** \internal load a complex value into a vector - row-wise */ +EIGEN_ALWAYS_INLINE Packet4f pload_realimag_combine_row(std::complex* src) +{ + return ploadu(src).v; +} + +EIGEN_ALWAYS_INLINE Packet2d pload_realimag_combine_row(std::complex* src) +{ + return ploadu(src).v; +} + +/** \internal load a scalar or a vector from complex location */ +template +EIGEN_ALWAYS_INLINE Packet4f pload_complex(std::complex* src) +{ + if (GEMV_IS_SCALAR) { + return pload_complex_half(src); + } + else + { + return ploadu(reinterpret_cast(src)); + } +} + +template +EIGEN_ALWAYS_INLINE Packet2d pload_complex(std::complex* src) +{ + return ploadu(reinterpret_cast(src)); +} + +/** \internal load from a complex vector and convert to a real vector */ +template +EIGEN_ALWAYS_INLINE Packet4f pload_complex(Packet2cf* src) +{ + return src->v; +} + +template +EIGEN_ALWAYS_INLINE Packet2d pload_complex(Packet1cd* src) +{ + return src->v; +} + +/** \internal load a full vector from complex location - column-wise */ +EIGEN_ALWAYS_INLINE Packet4f pload_complex_full(std::complex* src) +{ + return Packet4f(ploaddup(reinterpret_cast(src))); +} + +EIGEN_ALWAYS_INLINE Packet2d pload_complex_full(std::complex* src) +{ + return ploadu(src).v; +} + +/** \internal load a full vector from complex location - row-wise */ +EIGEN_ALWAYS_INLINE Packet4f pload_complex_full_row(std::complex* src) +{ + return ploadu(src).v; +} + +EIGEN_ALWAYS_INLINE Packet2d pload_complex_full_row(std::complex* src) +{ + return pload_complex_full(src); +} + +/** \internal load a vector from a real-only scalar location - column-wise */ +EIGEN_ALWAYS_INLINE Packet4f pload_real(float* src) +{ + return pset1(*src); +} + +EIGEN_ALWAYS_INLINE Packet2d pload_real(double* src) +{ + return pset1(*src); +} + +EIGEN_ALWAYS_INLINE Packet4f pload_real(Packet4f& src) +{ + return src; +} + +EIGEN_ALWAYS_INLINE Packet2d pload_real(Packet2d& src) +{ + return src; +} + +/** \internal load a vector from a real-only vector location */ +EIGEN_ALWAYS_INLINE Packet4f pload_real_full(float* src) +{ + Packet4f ret = ploadu(src); + return vec_mergeh(ret, ret); +} + +EIGEN_ALWAYS_INLINE Packet2d pload_real_full(double* src) +{ + return pload_real(src); +} + +EIGEN_ALWAYS_INLINE Packet4f pload_real_full(std::complex* src) +{ + return pload_complex_full(src); // Just for compilation +} + +EIGEN_ALWAYS_INLINE Packet2d pload_real_full(std::complex* src) +{ + return pload_complex_full(src); // Just for compilation +} + +/** \internal load a vector from a real-only scalar location - row-wise */ +template +EIGEN_ALWAYS_INLINE Packet4f pload_real_row(float* src) +{ + if (GEMV_IS_SCALAR) { + return pload_real_full(src); + } + else { + return ploadu(src); + } +} + +template +EIGEN_ALWAYS_INLINE Packet2d pload_real_row(double* src) +{ + return pload_real(src); +} + +EIGEN_ALWAYS_INLINE Packet2cf padd(Packet2cf& a, std::complex& b) +{ + EIGEN_UNUSED_VARIABLE(b); + return a; // Just for compilation +} + +EIGEN_ALWAYS_INLINE Packet1cd padd(Packet1cd& a, std::complex& b) +{ + EIGEN_UNUSED_VARIABLE(b); + return a; // Just for compilation +} + +/** \internal set a scalar from complex location */ +template +EIGEN_ALWAYS_INLINE Scalar pset1_realimag(ResScalar& alpha, int which, int conj) +{ + return (which) ? ((conj) ? -alpha.real() : alpha.real()) : ((conj) ? -alpha.imag() : alpha.imag()); +} + +/** \internal set a vector from complex location */ +template +EIGEN_ALWAYS_INLINE Packet2cf pset1_complex(std::complex& alpha) +{ + Packet2cf ret; + ret.v[COMPLEX_DELTA + 0] = pset1_realimag(alpha, (which & 0x01), (which & 0x04)); + ret.v[COMPLEX_DELTA + 1] = pset1_realimag(alpha, (which & 0x02), (which & 0x08)); + ret.v[2 - COMPLEX_DELTA] = ret.v[COMPLEX_DELTA + 0]; + ret.v[3 - COMPLEX_DELTA] = ret.v[COMPLEX_DELTA + 1]; + return ret; +} + +template +EIGEN_ALWAYS_INLINE Packet1cd pset1_complex(std::complex& alpha) +{ + Packet1cd ret; + ret.v[0] = pset1_realimag(alpha, (which & 0x01), (which & 0x04)); + ret.v[1] = pset1_realimag(alpha, (which & 0x02), (which & 0x08)); + return ret; +} + +/** \internal zero out a vector for real or complex forms */ +template +EIGEN_ALWAYS_INLINE Packet pset_zero() +{ + return pset1(__UNPACK_TYPE__(Packet)(0)); +} + +template<> +EIGEN_ALWAYS_INLINE Packet2cf pset_zero() +{ + return Packet2cf(pset1(float(0))); +} + +template<> +EIGEN_ALWAYS_INLINE Packet1cd pset_zero() +{ + return Packet1cd(pset1(double(0))); +} + +/** \internal initialize a vector from another vector */ +template +EIGEN_ALWAYS_INLINE Packet pset_init(Packet& c1) +{ + if (GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_UNUSED_VARIABLE(c1); + return pset_zero(); + } + else + { + return c1; // Intentionally left uninitialized + } +} + +template +struct alpha_store +{ + alpha_store(ResScalar& alpha) { + separate.r = pset1_complex(alpha); + separate.i = pset1_complex(alpha); + } + struct ri { + PResPacket r; + PResPacket i; + } separate; +}; + +/** \internal multiply and add for complex math */ +template +EIGEN_ALWAYS_INLINE ScalarPacket pmadd_complex(ScalarPacket& c0, ScalarPacket& c2, ScalarPacket& c4, AlphaData& b0) +{ + return pmadd(c2, b0.separate.i.v, pmadd(c0, b0.separate.r.v, c4)); +} + +/** \internal store and madd for complex math */ +template +EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(PResPacket& c0, AlphaData& b0, ResScalar* res) +{ + PResPacket c2 = pcplxflipconj(c0); + if (GEMV_IS_SCALAR) { + ScalarPacket c4 = ploadu(reinterpret_cast(res)); + ScalarPacket c3 = pmadd_complex(c0.v, c2.v, c4, b0); + pstoreu(reinterpret_cast(res), c3); + } else { + ScalarPacket c4 = pload_complex(res); + PResPacket c3 = PResPacket(pmadd_complex(c0.v, c2.v, c4, b0)); + pstoreu(res, c3); + } +} + +template +EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(PResPacket& c0, PResPacket& c1, AlphaData& b0, ResScalar* res) +{ + PResPacket c2 = pcplxflipconj(c0); + PResPacket c3 = pcplxflipconj(c1); +#if !defined(_ARCH_PWR10) + ScalarPacket c4 = pload_complex(res + (iter2 * ResPacketSize)); + ScalarPacket c5 = pload_complex(res + ((iter2 + 1) * ResPacketSize)); + PResPacket c6 = PResPacket(pmadd_complex(c0.v, c2.v, c4, b0)); + PResPacket c7 = PResPacket(pmadd_complex(c1.v, c3.v, c5, b0)); + pstoreu(res + (iter2 * ResPacketSize), c6); + pstoreu(res + ((iter2 + 1) * ResPacketSize), c7); +#else + __vector_pair a = *reinterpret_cast<__vector_pair *>(res + (iter2 * ResPacketSize)); +#if EIGEN_COMP_LLVM + PResPacket c6[2]; + __builtin_vsx_disassemble_pair(reinterpret_cast(c6), &a); + c6[0] = PResPacket(pmadd_complex(c0.v, c2.v, c6[0].v, b0)); + c6[1] = PResPacket(pmadd_complex(c1.v, c3.v, c6[1].v, b0)); + GEMV_BUILDPAIR_MMA(a, c6[0].v, c6[1].v); +#else + if (GEMV_IS_COMPLEX_FLOAT) { + __asm__ ("xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" : "+&d" (a) : "wa" (b0.separate.r.v), "wa" (c0.v), "wa" (c1.v)); + __asm__ ("xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" : "+&d" (a) : "wa" (b0.separate.i.v), "wa" (c2.v), "wa" (c3.v)); + } else { + __asm__ ("xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" : "+&d" (a) : "wa" (b0.separate.r.v), "wa" (c0.v), "wa" (c1.v)); + __asm__ ("xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" : "+&d" (a) : "wa" (b0.separate.i.v), "wa" (c2.v), "wa" (c3.v)); + } +#endif + *reinterpret_cast<__vector_pair *>(res + (iter2 * ResPacketSize)) = a; +#endif +} + +/** \internal load lhs packet */ +template +EIGEN_ALWAYS_INLINE LhsPacket loadLhsPacket(LhsMapper& lhs, Index i, Index j) +{ + if (sizeof(Scalar) == sizeof(LhsScalar)) { + const LhsScalar& src = lhs(i + 0, j); + return LhsPacket(pload_real_full(const_cast(&src))); + } + return lhs.template load(i + 0, j); +} + +/** \internal madd for complex times complex */ +template +EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_complex(RealPacket& a, RealPacket& b, RealPacket& c) +{ + if (ConjugateLhs && ConjugateRhs) { + return vec_madd(a, pconj2(ComplexPacket(b)).v, c); + } + else if (Negate && !ConjugateLhs && ConjugateRhs) { + return vec_nmsub(a, b, c); + } + else { + return vec_madd(a, b, c); + } +} + +/** \internal madd for complex times real */ +template +EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_real(RealPacket& a, RealPacket& b, RealPacket& c) +{ + if (Conjugate) { + return vec_madd(a, pconj2(ComplexPacket(b)).v, c); + } + else { + return vec_madd(a, b, c); + } +} + +template +EIGEN_ALWAYS_INLINE void gemv_mult_generic(LhsPacket& a0, RhsScalar* b, PResPacket& c0) +{ + conj_helper pcj; + RhsPacket b0; + if (StorageOrder == ColMajor) { + b0 = pset1(*b); + } + else { + b0 = ploadu(b); + } + c0 = pcj.pmadd(a0, b0, c0); +} + +/** \internal core multiply operation for vectors - complex times complex */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_complex_complex(LhsPacket& a0, RhsScalar* b, PResPacket& c0, ResPacket& c1) +{ + ScalarPacket br, bi; + if (StorageOrder == ColMajor) { + pload_realimag(b, br, bi); + } + else { + pload_realimag_row(b, br, bi); + } + if (ConjugateLhs && !ConjugateRhs) a0 = pconj2(a0); + LhsPacket a1 = pcplxflipconj(a0); + ScalarPacket cr = pmadd_complex_complex(a0.v, br, c0.v); + ScalarPacket ci = pmadd_complex_complex(a1.v, bi, c1.v); + c1 = ResPacket(ci); + c0 = PResPacket(cr); +} + +/** \internal core multiply operation for vectors - real times complex */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_real_complex(LhsPacket& a0, RhsScalar* b, PResPacket& c0) +{ + ScalarPacket b0; + if (StorageOrder == ColMajor) { + b0 = pload_complex_full(b); + } + else { + b0 = pload_complex_full_row(b); + } + ScalarPacket cri = pmadd_complex_real(a0, b0, c0.v); + c0 = PResPacket(cri); +} + +/** \internal core multiply operation for vectors - complex times real */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_complex_real(LhsPacket& a0, RhsScalar* b, PResPacket& c0) +{ + ScalarPacket a1 = pload_complex(&a0); + ScalarPacket b0; + if (StorageOrder == ColMajor) { + b0 = pload_real(b); + } + else { + b0 = pload_real_row(b); + } + ScalarPacket cri = pmadd_complex_real(a1, b0, c0.v); + c0 = PResPacket(cri); +} + +#define GEMV_MULT_COMPLEX_COMPLEX(LhsType, RhsType, ResType) \ +template \ +EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType& c0, ResType& c1) \ +{ \ + gemv_mult_complex_complex(a0, b, c0, c1); \ +} + +GEMV_MULT_COMPLEX_COMPLEX(Packet2cf, std::complex, Packet2cf) +GEMV_MULT_COMPLEX_COMPLEX(Packet1cd, std::complex, Packet1cd) + +#define GEMV_MULT_REAL_COMPLEX(LhsType, RhsType, ResType) \ +template \ +EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType& c0, RhsType&) \ +{ \ + gemv_mult_real_complex(a0, b, c0); \ +} + +GEMV_MULT_REAL_COMPLEX(float, std::complex, Packet2cf) +GEMV_MULT_REAL_COMPLEX(double, std::complex, Packet1cd) +GEMV_MULT_REAL_COMPLEX(Packet4f, std::complex, Packet2cf) +GEMV_MULT_REAL_COMPLEX(Packet2d, std::complex, Packet1cd) + +#define GEMV_MULT_COMPLEX_REAL(LhsType, RhsType, ResType1, ResType2) \ +template \ +EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType1& c0, ResType2&) \ +{ \ + gemv_mult_complex_real(a0, b, c0); \ +} + +GEMV_MULT_COMPLEX_REAL(Packet2cf, float, Packet2cf, std::complex) +GEMV_MULT_COMPLEX_REAL(Packet1cd, double, Packet1cd, std::complex) +GEMV_MULT_COMPLEX_REAL(std::complex, float, Packet2cf, std::complex) +GEMV_MULT_COMPLEX_REAL(std::complex, double, Packet1cd, std::complex) + +#ifdef USE_GEMV_MMA +/** \internal convert packet to real form */ +template +EIGEN_ALWAYS_INLINE T convertReal(T a) +{ + return a; +} + +EIGEN_ALWAYS_INLINE Packet4f convertReal(Packet2cf a) +{ + return a.v; +} + +EIGEN_ALWAYS_INLINE Packet2d convertReal(Packet1cd a) +{ + return a.v; +} + +/** \internal convert packet to complex form */ +template +EIGEN_ALWAYS_INLINE T convertComplex(T a) +{ + return a; +} + +EIGEN_ALWAYS_INLINE Packet2cf convertComplex(Packet4f a) +{ + return Packet2cf(a); +} + +EIGEN_ALWAYS_INLINE Packet1cd convertComplex(Packet2d a) +{ + return Packet1cd(a); +} + +/** \internal load a vector from a complex location (for MMA version) */ +template +EIGEN_ALWAYS_INLINE void pload_complex_MMA(SLhsPacket& a) +{ + a = SLhsPacket(pload_complex(&a)); +} + +template +EIGEN_ALWAYS_INLINE void pload_complex_MMA(__vector_pair&) +{ + // Pass thru +} + +/** \internal perform a matrix multiply and accumulate (positive and negative) of packet a and packet b */ +template +EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, RhsPacket& a, LhsPacket& b) +{ + if (NegativeAccumulate) + { + __builtin_mma_xvf32gernp(acc, (__vector unsigned char)a, (__vector unsigned char)b); + } + else { + __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b); + } +} + +/** \internal perform a matrix multiply and accumulate (positive and negative) of vector_pair a and packet b */ +template +EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, __vector_pair& a, Packet2d& b) +{ + if (NegativeAccumulate) + { + __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b); + } + else { + __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b); + } +} + +template +EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad*, __vector_pair&, Packet4f&) +{ + // Just for compilation +} + +/** \internal madd for complex times complex (MMA version) */ +template +EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(LhsPacket& a, RealPacket& b, __vector_quad* c) +{ + if (ConjugateLhs && ConjugateRhs) { + RealPacket b2 = pconj2(convertComplex(b)).v; + return pger_vecMMA(c, b2, a.v); + } + else if (Negate && !ConjugateLhs && ConjugateRhs) { + return pger_vecMMA(c, b, a.v); + } + else { + return pger_vecMMA(c, b, a.v); + } +} + +template +EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(__vector_pair& a, RealPacket& b, __vector_quad* c) +{ + if (ConjugateLhs && ConjugateRhs) { + RealPacket b2 = pconj2(convertComplex(b)).v; + return pger_vecMMA(c, a, b2); + } + else if (Negate && !ConjugateLhs && ConjugateRhs) { + return pger_vecMMA(c, a, b); + } + else { + return pger_vecMMA(c, a, b); + } +} + +/** \internal madd for complex times real (MMA version) */ +template +EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(LhsPacket& a, RealPacket& b, __vector_quad* c) +{ + RealPacket a2 = convertReal(a); + if (Conjugate) { + RealPacket b2 = pconj2(convertComplex(b)).v; + if (StorageOrder == ColMajor) { + return pger_vecMMA(c, b2, a2); + } else { + return pger_vecMMA(c, a2, b2); + } + } + else { + if (StorageOrder == ColMajor) { + return pger_vecMMA(c, b, a2); + } else { + return pger_vecMMA(c, a2, b); + } + } +} + +/** \internal madd for real times complex (MMA version) */ +template +EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(__vector_pair& a, RealPacket& b, __vector_quad* c) +{ + if (Conjugate) { + RealPacket b2 = pconj2(convertComplex(b)).v; + return pger_vecMMA(c, a, b2); + } + else { + return pger_vecMMA(c, a, b); + } +} + +/** \internal core multiply operation for vectors (MMA version) - complex times complex */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_complex_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) +{ + ScalarPacket b0; + if (StorageOrder == ColMajor) { + b0 = pload_realimag_combine(b); + } else { + b0 = pload_realimag_combine_row(b); + } + pmadd_complex_complex_MMA(a0, b0, c0); +} + +/** \internal core multiply operation for vectors (MMA version) - complex times real */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_complex_real_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) +{ + pload_complex_MMA(a0); + ScalarPacket b0; + if (StorageOrder == ColMajor) { + b0 = pload_real(b); + } + else { + b0 = pload_real_row(b); + } + pmadd_complex_real_MMA(a0, b0, c0); +} + +/** \internal core multiply operation for vectors (MMA version) - real times complex */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_real_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) +{ + ScalarPacket b0; + if (StorageOrder == ColMajor) { + b0 = pload_complex_full(b); + } + else { + b0 = pload_complex_full_row(b); + } + pmadd_complex_real_MMA)) ? StorageOrder : ColMajor>(a0, b0, c0); +} + +#define GEMV_MULT_COMPLEX_COMPLEX_MMA(LhsType, RhsType) \ +template \ +EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) \ +{ \ + gemv_mult_complex_complex_MMA(a0, b, c0); \ +} + +GEMV_MULT_COMPLEX_COMPLEX_MMA(Packet2cf, std::complex) +GEMV_MULT_COMPLEX_COMPLEX_MMA(__vector_pair, std::complex) +GEMV_MULT_COMPLEX_COMPLEX_MMA(Packet1cd, std::complex) + +/** \internal core multiply operation for vectors (MMA version) - complex times complex */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(__vector_pair& a0, std::complex* b, __vector_quad* c0) +{ + if (sizeof(LhsScalar) == 16) { + gemv_mult_complex_complex_MMA(a0, b, c0); + } + else { + gemv_mult_real_complex_MMA(a0, b, c0); + } +} + +#define GEMV_MULT_REAL_COMPLEX_MMA(LhsType, RhsType) \ +template \ +EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) \ +{ \ + gemv_mult_real_complex_MMA(a0, b, c0); \ +} + +GEMV_MULT_REAL_COMPLEX_MMA(Packet4f, std::complex) +GEMV_MULT_REAL_COMPLEX_MMA(Packet2d, std::complex) + +#define GEMV_MULT_COMPLEX_REAL_MMA(LhsType, RhsType) \ +template \ +EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) \ +{ \ + gemv_mult_complex_real_MMA(a0, b, c0); \ +} + +GEMV_MULT_COMPLEX_REAL_MMA(Packet2cf, float) +GEMV_MULT_COMPLEX_REAL_MMA(Packet1cd, double) +GEMV_MULT_COMPLEX_REAL_MMA(__vector_pair, float) +GEMV_MULT_COMPLEX_REAL_MMA(__vector_pair, double) + +/** \internal disassemble MMA accumulator results into packets */ +template +EIGEN_ALWAYS_INLINE void disassembleResults2(__vector_quad* c0, PacketBlock& result0) +{ + __builtin_mma_disassemble_acc(&result0.packet, c0); + if (sizeof(LhsPacket) == 16) { + if (sizeof(RhsPacket) == 16) { + ScalarPacket tmp0, tmp2; + tmp2 = vec_mergeh(result0.packet[2], result0.packet[3]); + tmp0 = vec_mergeh(result0.packet[0], result0.packet[1]); + result0.packet[3] = vec_mergel(result0.packet[3], result0.packet[2]); + result0.packet[1] = vec_mergel(result0.packet[1], result0.packet[0]); + result0.packet[2] = tmp2; + result0.packet[0] = tmp0; + + if (ConjugateLhs) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + result0.packet[2] = pconj2(convertComplex(result0.packet[2])).v; + } else if (ConjugateRhs) { + result0.packet[1] = pconj2(convertComplex(result0.packet[1])).v; + result0.packet[3] = pconj2(convertComplex(result0.packet[3])).v; + } else { + result0.packet[1] = pconjinv(convertComplex(result0.packet[1])).v; + result0.packet[3] = pconjinv(convertComplex(result0.packet[3])).v; + } + result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]); + result0.packet[2] = vec_add(result0.packet[2], result0.packet[3]); + } else { + result0.packet[0][1] = result0.packet[1][1]; + result0.packet[2][1] = result0.packet[3][1]; + } + } +} + +template +EIGEN_ALWAYS_INLINE void disassembleResults4(__vector_quad* c0, PacketBlock& result0) +{ + __builtin_mma_disassemble_acc(&result0.packet, c0); + if (GEMV_IS_COMPLEX_COMPLEX) { + if (ConjugateLhs) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v; + } else { + if (ConjugateRhs) { + result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v; + } else { + result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v; + } + } + result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]); + } else if (sizeof(LhsPacket) == sizeof(std::complex)) { + if (ConjugateLhs) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + } + } else { + result0.packet[0] = vec_mergee(result0.packet[0], result0.packet[1]); + } +} + +template +EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock& result0) +{ + if (!GEMV_IS_COMPLEX_FLOAT) { + disassembleResults2(c0, result0); + } else { + disassembleResults4(c0, result0); + } +} +#endif + +#define GEMV_GETN_COMPLEX(N) (((N) * ResPacketSize) >> 1) + +#define GEMV_LOADPACKET_COL_COMPLEX(iter) \ + loadLhsPacket(lhs, i + ((iter) * ResPacketSize), j) + +#define GEMV_LOADPACKET_COL_COMPLEX_DATA(iter) \ + convertReal(GEMV_LOADPACKET_COL_COMPLEX(iter)) + +#ifdef USE_GEMV_MMA +#define GEMV_INIT_COL_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + __builtin_mma_xxsetaccz(&e0##iter); \ + } + +#if EIGEN_COMP_LLVM +#define GEMV_LOADPAIR_COL_COMPLEX_MMA(iter1, iter2) \ + GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_COL_COMPLEX_DATA(iter2), GEMV_LOADPACKET_COL_COMPLEX_DATA((iter2) + 1)); \ + EIGEN_UNUSED_VARIABLE(f##iter1); +#else +#define GEMV_LOADPAIR_COL_COMPLEX_MMA(iter1, iter2) \ + if (sizeof(LhsPacket) == 16) { \ + const LhsScalar& src = lhs(i + ((32 * iter1) / sizeof(LhsScalar)), j); \ + a##iter1 = *reinterpret_cast<__vector_pair *>(const_cast(&src)); \ + EIGEN_UNUSED_VARIABLE(f##iter1); \ + } else { \ + f##iter1 = lhs.template load(i + ((iter2) * ResPacketSize), j); \ + GEMV_BUILDPAIR_MMA(a##iter1, vec_splat(convertReal(f##iter1), 0), vec_splat(convertReal(f##iter1), 1)); \ + } +#endif + +#define GEMV_LOAD1_COL_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \ + EIGEN_UNUSED_VARIABLE(a##iter); \ + } else { \ + GEMV_LOADPAIR_COL_COMPLEX_MMA(iter, iter << 1) \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(a##iter); \ + EIGEN_UNUSED_VARIABLE(f##iter); \ + } + +#define GEMV_WORK1_COL_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + gemv_mult_complex_MMA(f##iter, b, &e0##iter); \ + } else { \ + gemv_mult_complex_MMA(a##iter, b, &e0##iter); \ + } \ + } + +#define GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter1, iter2) \ + GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_COL_COMPLEX_DATA(iter2), GEMV_LOADPACKET_COL_COMPLEX_DATA((iter2) + 1)); + +#define GEMV_LOAD2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN_COMPLEX(N) > iter1) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2); \ + EIGEN_UNUSED_VARIABLE(a##iter3) \ + } else { \ + GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2 << 1); \ + GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter3, iter3 << 1); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(a##iter2); \ + EIGEN_UNUSED_VARIABLE(a##iter3); \ + } \ + EIGEN_UNUSED_VARIABLE(f##iter2); \ + EIGEN_UNUSED_VARIABLE(f##iter3); + +#define GEMV_WORK2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN_COMPLEX(N) > iter1) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + PLhsPacket g[2]; \ + __builtin_vsx_disassemble_pair(reinterpret_cast(g), &a##iter2); \ + gemv_mult_complex_MMA(g[0], b, &e0##iter2); \ + gemv_mult_complex_MMA(g[1], b, &e0##iter3); \ + } else { \ + gemv_mult_complex_MMA(a##iter2, b, &e0##iter2); \ + gemv_mult_complex_MMA(a##iter3, b, &e0##iter3); \ + } \ + } + +#if EIGEN_COMP_LLVM +#define GEMV_LOAD_COL_COMPLEX_MMA(N) \ + if (GEMV_GETN_COMPLEX(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_LOAD2_COL_COMPLEX_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) \ + } + +#define GEMV_WORK_COL_COMPLEX_MMA(N) \ + if (GEMV_GETN_COMPLEX(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_WORK2_COL_COMPLEX_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N) \ + } +#else +#define GEMV_LOAD_COL_COMPLEX_MMA(N) \ + GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) + +#define GEMV_WORK_COL_COMPLEX_MMA(N) \ + GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N) +#endif + +#define GEMV_DISASSEMBLE_COMPLEX_MMA(iter) \ + disassembleResults(&e0##iter, result0##iter); + +#define GEMV_STORE_COL_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + GEMV_DISASSEMBLE_COMPLEX_MMA(iter); \ + c0##iter = PResPacket(result0##iter.packet[0]); \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + pstoreu_pmadd_complex(c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \ + } else { \ + pstoreu_pmadd_complex(c0##iter, alpha_data, res + i + ((iter << 1) * ResPacketSize)); \ + c0##iter = PResPacket(result0##iter.packet[2]); \ + pstoreu_pmadd_complex(c0##iter, alpha_data, res + i + (((iter << 1) + 1) * ResPacketSize)); \ + } \ + } + +#define GEMV_STORE2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN_COMPLEX(N) > iter1) { \ + GEMV_DISASSEMBLE_COMPLEX_MMA(iter2); \ + GEMV_DISASSEMBLE_COMPLEX_MMA(iter3); \ + c0##iter2 = PResPacket(result0##iter2.packet[0]); \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + c0##iter3 = PResPacket(result0##iter3.packet[0]); \ + pstoreu_pmadd_complex(c0##iter2, c0##iter3, alpha_data, res + i); \ + } else { \ + c0##iter3 = PResPacket(result0##iter2.packet[2]); \ + pstoreu_pmadd_complex(c0##iter2, c0##iter3, alpha_data, res + i); \ + c0##iter2 = PResPacket(result0##iter3.packet[0]); \ + c0##iter3 = PResPacket(result0##iter3.packet[2]); \ + pstoreu_pmadd_complex(c0##iter2, c0##iter3, alpha_data, res + i); \ + } \ + } + +#define GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \ + GEMV_UNROLL(GEMV_INIT_COL_COMPLEX_MMA, N) \ + Index j = j2; \ + do { \ + const RhsScalar& b1 = rhs2(j, 0); \ + RhsScalar* b = const_cast(&b1); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_LOAD_COL_COMPLEX_MMA(N) \ + GEMV_WORK_COL_COMPLEX_MMA(N) \ + } while (++j < jend); \ + if (GEMV_GETN(N) <= 2) { \ + GEMV_UNROLL(GEMV_STORE_COL_COMPLEX_MMA, N) \ + } else { \ + GEMV_UNROLL_HALF(GEMV_STORE2_COL_COMPLEX_MMA, (N >> 1)) \ + } \ + i += (ResPacketSize * N); +#endif + +#define GEMV_INIT_COMPLEX(iter, N) \ + if (N > iter) { \ + c0##iter = pset_zero(); \ + c1##iter = pset_init(c1##iter); \ + } else { \ + EIGEN_UNUSED_VARIABLE(c0##iter); \ + EIGEN_UNUSED_VARIABLE(c1##iter); \ + } + +#define GEMV_WORK_COL_COMPLEX(iter, N) \ + if (N > iter) { \ + f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \ + gemv_mult_complex(f##iter, b, c0##iter, c1##iter); \ + } else { \ + EIGEN_UNUSED_VARIABLE(f##iter); \ + } + +#define GEMV_STORE_COL_COMPLEX(iter, N) \ + if (N > iter) { \ + if (GEMV_IS_COMPLEX_COMPLEX) { \ + c0##iter = padd(c0##iter, c1##iter); \ + } \ + pstoreu_pmadd_complex(c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \ + } + +/** \internal main macro for gemv_complex_col - initialize accumulators, multiply and add inputs, and store results */ +#define GEMV_PROCESS_COL_COMPLEX_ONE(N) \ + GEMV_UNROLL(GEMV_INIT_COMPLEX, N) \ + Index j = j2; \ + do { \ + const RhsScalar& b1 = rhs2(j, 0); \ + RhsScalar* b = const_cast(&b1); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_UNROLL(GEMV_WORK_COL_COMPLEX, N) \ + } while (++j < jend); \ + GEMV_UNROLL(GEMV_STORE_COL_COMPLEX, N) \ + i += (ResPacketSize * N); + +#if defined(USE_GEMV_MMA) && (EIGEN_COMP_LLVM || defined(USE_SLOWER_GEMV_MMA)) +#define USE_GEMV_COL_COMPLEX_MMA +#endif + +#ifdef USE_GEMV_COL_COMPLEX_MMA +#define GEMV_PROCESS_COL_COMPLEX(N) \ + GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) +#else +#if defined(USE_GEMV_MMA) && (__GNUC__ > 10) +#define GEMV_PROCESS_COL_COMPLEX(N) \ + if (sizeof(Scalar) != sizeof(LhsPacket)) { \ + GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \ + } else { \ + GEMV_PROCESS_COL_COMPLEX_ONE(N) \ + } +#else +#define GEMV_PROCESS_COL_COMPLEX(N) \ + GEMV_PROCESS_COL_COMPLEX_ONE(N) +#endif +#endif + +template +EIGEN_STRONG_INLINE void gemv_complex_col( + Index rows, Index cols, + const LhsMapper& alhs, + const RhsMapper& rhs, + ResScalar* res, Index resIncr, + ResScalar alpha) +{ + typedef gemv_traits Traits; + + typedef typename Traits::LhsPacket LhsPacket; + typedef typename Traits::RhsPacket RhsPacket; + typedef typename Traits::ResPacket ResPacket; + + typedef typename packet_traits::type ScalarPacket; + typedef typename packet_traits::type PLhsPacket; + typedef typename packet_traits::type PResPacket; + typedef gemv_traits PTraits; + + EIGEN_UNUSED_VARIABLE(resIncr); + eigen_internal_assert(resIncr == 1); + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + RhsMapper rhs2(rhs); + + conj_helper cj; + + const Index lhsStride = lhs.stride(); + // TODO: for padded aligned inputs, we could enable aligned reads + enum { + LhsAlignment = Unaligned, + ResPacketSize = PTraits::ResPacketSize, + LhsPacketSize = PTraits::LhsPacketSize, + RhsPacketSize = PTraits::RhsPacketSize, + }; +#ifdef EIGEN_POWER_USE_GEMV_PREFETCH + const Index prefetch_dist = 64 * LhsPacketSize; +#endif + +#ifndef GCC_ONE_VECTORPAIR_BUG + const Index n8 = rows - 8 * ResPacketSize + 1; + const Index n4 = rows - 4 * ResPacketSize + 1; + const Index n2 = rows - 2 * ResPacketSize + 1; +#endif + const Index n1 = rows - 1 * ResPacketSize + 1; + + // TODO: improve the following heuristic: + const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(LhsScalar) < 16000 ? 16 : 8); + + typedef alpha_store AlphaData; + AlphaData alpha_data(alpha); + + for (Index j2 = 0; j2 < cols; j2 += block_cols) + { + Index jend = numext::mini(j2 + block_cols, cols); + Index i = 0; + PResPacket c00, c01, c02, c03, c04, c05, c06, c07; + ResPacket c10, c11, c12, c13, c14, c15, c16, c17; + PLhsPacket f0, f1, f2, f3, f4, f5, f6, f7; +#ifdef USE_GEMV_MMA + __vector_quad e00, e01, e02, e03, e04, e05, e06, e07; + __vector_pair a0, a1, a2, a3, a4, a5, a6, a7; + PacketBlock result00, result01, result02, result03, result04, result05, result06, result07; + GEMV_UNUSED(8, e0) + GEMV_UNUSED(8, result0) + GEMV_UNUSED(8, a) + GEMV_UNUSED(8, f) +#if !defined(GCC_ONE_VECTORPAIR_BUG) && defined(USE_GEMV_COL_COMPLEX_MMA) + if (GEMV_IS_COMPLEX_COMPLEX || !GEMV_IS_COMPLEX_FLOAT) +#endif +#endif +#ifndef GCC_ONE_VECTORPAIR_BUG + { + while (i < n8) + { + GEMV_PROCESS_COL_COMPLEX(8) + } + } + while (i < n4) + { + GEMV_PROCESS_COL_COMPLEX(4) + } + if (i < n2) + { + GEMV_PROCESS_COL_COMPLEX(2) + } + if (i < n1) +#else + while (i < n1) +#endif + { + GEMV_PROCESS_COL_COMPLEX_ONE(1) + } + for (;i < rows;++i) + { + ResScalar d0(0); + Index j = j2; + do { + d0 += cj.pmul(lhs(i, j), rhs2(j, 0)); + } while (++j < jend); + res[i] += alpha * d0; + } + } +} + +template struct ScalarBlock { + Scalar scalar[N]; +}; + +#ifdef USE_GEMV_MMA +static Packet16uc p16uc_ELEMENT_3 = { 0x0c,0x0d,0x0e,0x0f, 0x1c,0x1d,0x1e,0x1f, 0x0c,0x0d,0x0e,0x0f, 0x1c,0x1d,0x1e,0x1f }; + +/** \internal predux (add elements of a vector) from a MMA accumulator - real results */ +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_real(__vector_quad* acc0, __vector_quad* acc1) +{ + PacketBlock result0, result1; + __builtin_mma_disassemble_acc(&result0.packet, acc0); + __builtin_mma_disassemble_acc(&result1.packet, acc1); + result0.packet[0] = vec_mergeh(result0.packet[0], result1.packet[0]); + result0.packet[1] = vec_mergeo(result0.packet[1], result1.packet[1]); + result0.packet[2] = vec_mergel(result0.packet[2], result1.packet[2]); + result0.packet[3] = vec_perm(result0.packet[3], result1.packet[3], p16uc_ELEMENT_3); + result0.packet[0] = vec_add(vec_add(result0.packet[0], result0.packet[2]), vec_add(result0.packet[1], result0.packet[3])); + return *reinterpret_cast *>(&result0.packet[0]); +} + +template<> +EIGEN_ALWAYS_INLINE ScalarBlock predux_real(__vector_quad* acc0, __vector_quad* acc1) +{ + PacketBlock result0, result1; + __builtin_mma_disassemble_acc(&result0.packet, acc0); + __builtin_mma_disassemble_acc(&result1.packet, acc1); + result0.packet[0] = vec_add(vec_mergeh(result0.packet[0], result1.packet[0]), vec_mergel(result0.packet[1], result1.packet[1])); + return *reinterpret_cast *>(&result0.packet[0]); +} + +/** \internal add complex results together */ +template +EIGEN_ALWAYS_INLINE ScalarBlock, 2> addComplexResults(PacketBlock& result0, PacketBlock& result1) +{ + ScalarBlock, 2> cc0; + result0.packet[0] = reinterpret_cast(vec_mergeh(reinterpret_cast(result0.packet[0]), reinterpret_cast(result1.packet[0]))); + result0.packet[2] = reinterpret_cast(vec_mergel(reinterpret_cast(result0.packet[2]), reinterpret_cast(result1.packet[2]))); + result0.packet[0] = vec_add(result0.packet[0], result0.packet[2]); + if (GEMV_IS_COMPLEX_COMPLEX) { + result0.packet[1] = reinterpret_cast(vec_mergeh(reinterpret_cast(result0.packet[1]), reinterpret_cast(result1.packet[1]))); + result0.packet[3] = reinterpret_cast(vec_mergel(reinterpret_cast(result0.packet[3]), reinterpret_cast(result1.packet[3]))); + result0.packet[1] = vec_add(result0.packet[1], result0.packet[3]); + if (ConjugateLhs) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v; + } else if (ConjugateRhs) { + result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v; + } else { + result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v; + } + result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]); + } else { + if (ConjugateLhs && (sizeof(LhsPacket) == sizeof(std::complex))) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + } + } + cc0.scalar[0].real(result0.packet[0][0]); + cc0.scalar[0].imag(result0.packet[0][1]); + cc0.scalar[1].real(result0.packet[0][2]); + cc0.scalar[1].imag(result0.packet[0][3]); + return cc0; +} + +template +EIGEN_ALWAYS_INLINE ScalarBlock, 2> addComplexResults(PacketBlock&, PacketBlock&) +{ + ScalarBlock, 2> cc0; + EIGEN_UNUSED_VARIABLE(cc0); + return cc0; // Just for compilation +} + +/** \internal predux (add elements of a vector) from a MMA accumulator - complex results */ +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(__vector_quad* acc0, __vector_quad* acc1) +{ + PacketBlock result0, result1; + __builtin_mma_disassemble_acc(&result0.packet, acc0); + __builtin_mma_disassemble_acc(&result1.packet, acc1); + return addComplexResults(result0, result1); +} + +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_real(__vector_quad* acc0) +{ + PacketBlock result0; + __builtin_mma_disassemble_acc(&result0.packet, acc0); + result0.packet[0] = vec_add(vec_mergeh(result0.packet[0], result0.packet[2]), vec_mergel(result0.packet[1], result0.packet[3])); + return *reinterpret_cast *>(&result0.packet[0]); +} + +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(__vector_quad* acc0) +{ + ScalarBlock cc0; + PacketBlock result0; + __builtin_mma_disassemble_acc(&result0.packet, acc0); + if (GEMV_IS_COMPLEX_COMPLEX) { + if (ConjugateLhs) { + result0.packet[1] = pconjinv(convertComplex(result0.packet[1])).v; + result0.packet[3] = pconjinv(convertComplex(result0.packet[3])).v; + } else if (ConjugateRhs) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + result0.packet[2] = pconj2(convertComplex(result0.packet[2])).v; + } else { + result0.packet[1] = pconj2(convertComplex(result0.packet[1])).v; + result0.packet[3] = pconj2(convertComplex(result0.packet[3])).v; + } + result0.packet[0] = vec_add(result0.packet[0], __builtin_vsx_xxpermdi(result0.packet[1], result0.packet[1], 2)); + result0.packet[2] = vec_add(result0.packet[2], __builtin_vsx_xxpermdi(result0.packet[3], result0.packet[3], 2)); + } else { + result0.packet[0] = __builtin_vsx_xxpermdi(result0.packet[0], result0.packet[1], 1); + result0.packet[2] = __builtin_vsx_xxpermdi(result0.packet[2], result0.packet[3], 1); + } + cc0.scalar[0].real(result0.packet[0][0]); + cc0.scalar[0].imag(result0.packet[0][1]); + cc0.scalar[1].real(result0.packet[2][0]); + cc0.scalar[1].imag(result0.packet[2][1]); + return cc0; +} +#endif + +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_real(ResPacket& a, ResPacket& b) +{ + ScalarBlock cc0; + cc0.scalar[0] = predux(a); + cc0.scalar[1] = predux(b); + return cc0; +} + +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(ResPacket& a, ResPacket& b) +{ + return predux_real(a, b); +} + +#define GEMV_UNROLL_ROW(func, N) \ + func(0, N) func(1, N) func(2, N) func(3, N) func(4, N) func(5, N) func(6, N) func(7, N) + +#define GEMV_UNROLL_ROW_HALF(func, N) \ + func(0, 0, 1, N) func(1, 2, 3, N) func(2, 4, 5, N) func(3, 6, 7, N) + +#define GEMV_LOADPACKET_ROW(iter) \ + lhs.template load(i + (iter), j) + +#ifdef USE_GEMV_MMA +#define GEMV_UNROLL3_ROW(func, N, which) \ + func(0, N, which) func(1, N, which) func(2, N, which) func(3, N, which) \ + func(4, N, which) func(5, N, which) func(6, N, which) func(7, N, which) + +#define GEMV_UNUSED_ROW(N, which) \ + GEMV_UNROLL3_ROW(GEMV_UNUSED_VAR, N, which) + +#define GEMV_INIT_ROW(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + __builtin_mma_xxsetaccz(&c##iter); \ + } + +#define GEMV_LOADPAIR_ROW(iter1, iter2) \ + GEMV_BUILDPAIR_MMA(b##iter1, GEMV_LOADPACKET_ROW(iter2), GEMV_LOADPACKET_ROW((iter2) + 1)); + +#define GEMV_WORK_ROW(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + if (GEMV_IS_FLOAT) { \ + pger_vecMMA_acc(&c##iter, a0, GEMV_LOADPACKET_ROW(iter)); \ + } else { \ + __vector_pair b##iter; \ + GEMV_LOADPAIR_ROW(iter, iter << 1) \ + pger_vecMMA_acc(&c##iter, b##iter, a0); \ + } \ + } + +#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + if (GEMV_IS_FLOAT) { \ + cc##iter1 = predux_real(&c##iter2, &c##iter3); \ + } else { \ + cc##iter1 = predux_real(&c##iter1); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ + } +#else +#define GEMV_INIT_ROW(iter, N) \ + if (N > iter) { \ + c##iter = pset1(ResScalar(0)); \ + } else { \ + EIGEN_UNUSED_VARIABLE(c##iter); \ + } + +#define GEMV_WORK_ROW(iter, N) \ + if (N > iter) { \ + c##iter = pcj.pmadd(GEMV_LOADPACKET_ROW(iter), a0, c##iter); \ + } + +#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + cc##iter1 = predux_real(c##iter2, c##iter3); \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ + } +#endif + +#define GEMV_MULT(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), a0); \ + cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), a0); \ + } + +#define GEMV_STORE_ROW(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + storeMaddData(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \ + storeMaddData(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \ + } + +/** \internal main macro for gemv_row - initialize accumulators, multiply and add inputs, predux and store results */ +#define GEMV_PROCESS_ROW(N) \ + for (; i < n##N; i += N) { \ + GEMV_UNROLL_ROW(GEMV_INIT_ROW, N) \ + Index j = 0; \ + for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \ + RhsPacket a0 = rhs2.template load(j); \ + GEMV_UNROLL_ROW(GEMV_WORK_ROW, N) \ + } \ + GEMV_UNROLL_ROW_HALF(GEMV_PREDUX2, (N >> 1)) \ + for (; j < cols; ++j) { \ + RhsScalar a0 = rhs2(j); \ + GEMV_UNROLL_ROW_HALF(GEMV_MULT, (N >> 1)) \ + } \ + GEMV_UNROLL_ROW_HALF(GEMV_STORE_ROW, (N >> 1)) \ + } + +template +EIGEN_STRONG_INLINE void gemv_row( + Index rows, Index cols, + const LhsMapper& alhs, + const RhsMapper& rhs, + ResScalar* res, Index resIncr, + ResScalar alpha) +{ + typedef gemv_traits Traits; + + typedef typename Traits::LhsPacket LhsPacket; + typedef typename Traits::RhsPacket RhsPacket; + typedef typename Traits::ResPacket ResPacket; + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + typename RhsMapper::LinearMapper rhs2 = rhs.getLinearMapper(0, 0); + + eigen_internal_assert(rhs.stride() == 1); + conj_helper cj; + conj_helper pcj; + + // TODO: fine tune the following heuristic. The rationale is that if the matrix is very large, + // processing 8 rows at once might be counter productive wrt cache. +#ifndef GCC_ONE_VECTORPAIR_BUG + const Index n8 = lhs.stride() * sizeof(LhsScalar) > 32000 ? (rows - 7) : (rows - 7); + const Index n4 = rows - 3; + const Index n2 = rows - 1; +#endif + + // TODO: for padded aligned inputs, we could enable aligned reads + enum { + LhsAlignment = Unaligned, + ResPacketSize = Traits::ResPacketSize, + LhsPacketSize = Traits::LhsPacketSize, + RhsPacketSize = Traits::RhsPacketSize, + }; + + Index i = 0; +#ifdef USE_GEMV_MMA + __vector_quad c0, c1, c2, c3, c4, c5, c6, c7; + GEMV_UNUSED_ROW(8, c) +#else + ResPacket c0, c1, c2, c3, c4, c5, c6, c7; +#endif +#ifndef GCC_ONE_VECTORPAIR_BUG + ScalarBlock cc0, cc1, cc2, cc3; + GEMV_PROCESS_ROW(8) + GEMV_PROCESS_ROW(4) + GEMV_PROCESS_ROW(2) +#endif + for (; i < rows; ++i) + { + ResPacket d0 = pset1(ResScalar(0)); + Index j = 0; + for (; j + LhsPacketSize <= cols; j += LhsPacketSize) + { + RhsPacket b0 = rhs2.template load(j); + + d0 = pcj.pmadd(lhs.template load(i + 0, j), b0, d0); + } + ResScalar dd0 = predux(d0); + for (; j < cols; ++j) + { + dd0 += cj.pmul(lhs(i, j), rhs2(j)); + } + res[i * resIncr] += alpha * dd0; + } +} + +#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(Scalar) \ +template \ +struct general_matrix_vector_product \ +{ \ + typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; \ +\ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run( \ + Index rows, Index cols, \ + const LhsMapper& lhs, \ + const RhsMapper& rhs, \ + ResScalar* res, Index resIncr, \ + ResScalar alpha) { \ + gemv_col(rows, cols, lhs, rhs, res, resIncr, alpha); \ + } \ +}; + +#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(Scalar) \ +template \ +struct general_matrix_vector_product \ +{ \ + typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; \ +\ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run( \ + Index rows, Index cols, \ + const LhsMapper& lhs, \ + const RhsMapper& rhs, \ + ResScalar* res, Index resIncr, \ + ResScalar alpha) { \ + gemv_row(rows, cols, lhs, rhs, res, resIncr, alpha); \ + } \ +}; + +EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(float) +EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(double) +EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(float) +EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(double) + +#ifdef USE_GEMV_MMA +#define gemv_bf16_col gemvMMA_bfloat16_col +#define gemv_bf16_row gemvMMA_bfloat16_row +#else +#define gemv_bf16_col gemv_bfloat16_col +#define gemv_bf16_row gemv_bfloat16_row +#endif + +#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL_BFLOAT16() \ +template \ +struct general_matrix_vector_product \ +{ \ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run( \ + Index rows, Index cols, \ + const LhsMapper& lhs, \ + const RhsMapper& rhs, \ + bfloat16* res, Index resIncr, \ + bfloat16 alpha) { \ + gemv_bf16_col(rows, cols, lhs, rhs, res, resIncr, alpha); \ + } \ +}; + +#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW_BFLOAT16() \ +template \ +struct general_matrix_vector_product \ +{ \ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run( \ + Index rows, Index cols, \ + const LhsMapper& lhs, \ + const RhsMapper& rhs, \ + bfloat16* res, Index resIncr, \ + bfloat16 alpha) { \ + gemv_bf16_row(rows, cols, lhs, rhs, res, resIncr, alpha); \ + } \ +}; + +EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL_BFLOAT16() +EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW_BFLOAT16() + +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PResPacket& b0, ResPacket& a1, ResPacket& b1) +{ + if (GEMV_IS_COMPLEX_COMPLEX) { + a0 = padd(a0, a1); + b0 = padd(b0, b1); + } + return predux_complex(a0, b0); +} + +#define GEMV_LOADPACKET_ROW_COMPLEX(iter) \ + loadLhsPacket(lhs, i + (iter), j) + +#define GEMV_LOADPACKET_ROW_COMPLEX_DATA(iter) \ + convertReal(GEMV_LOADPACKET_ROW_COMPLEX(iter)) + +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(which, N) \ + j = 0; \ + for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \ + const RhsScalar& b1 = rhs2(j); \ + RhsScalar* b = const_cast(&b1); \ + GEMV_UNROLL_ROW(which, N) \ + } + +#define GEMV_PROCESS_END_ROW_COMPLEX(N) \ + for (; j < cols; ++j) { \ + RhsScalar b0 = rhs2(j); \ + GEMV_UNROLL_ROW_HALF(GEMV_MULT_COMPLEX, (N >> 1)) \ + } \ + GEMV_UNROLL_ROW_HALF(GEMV_STORE_ROW_COMPLEX, (N >> 1)) + +#ifdef USE_GEMV_MMA +#define GEMV_INIT_ROW_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + __builtin_mma_xxsetaccz(&e0##iter); \ + } + +#define GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter1, iter2) \ + GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_ROW_COMPLEX_DATA(iter2), GEMV_LOADPACKET_ROW_COMPLEX_DATA((iter2) + 1)); + +#define GEMV_WORK_ROW_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \ + gemv_mult_complex_MMA(a##iter, b, &e0##iter); \ + } else { \ + __vector_pair a##iter; \ + GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter, iter << 1) \ + gemv_mult_complex_MMA(a##iter, b, &e0##iter); \ + } \ + } + +#define GEMV_PREDUX4_COMPLEX_MMA(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + cc##iter1 = predux_complex(&e0##iter2, &e0##iter3); \ + } else { \ + cc##iter1 = predux_complex(&e0##iter1); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_MMA(N) \ + GEMV_UNROLL_ROW(GEMV_INIT_ROW_COMPLEX_MMA, N) \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(GEMV_WORK_ROW_COMPLEX_MMA, N) + +#define GEMV_PROCESS_ROW_COMPLEX_ONE_MMA(N) \ + for (; i < n##N; i += N) { \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_MMA(N) \ + GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX_MMA, (N >> 1)) \ + GEMV_PROCESS_END_ROW_COMPLEX(N); \ + } +#endif + +#define GEMV_WORK_ROW_COMPLEX(iter, N) \ + if (N > iter) { \ + PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \ + gemv_mult_complex(a##iter, b, c0##iter, c1##iter); \ + } + +#define GEMV_PREDUX4_COMPLEX(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + cc##iter1 = predux_complex(c0##iter2, c0##iter3, c1##iter2, c1##iter3); \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ + } + +#define GEMV_MULT_COMPLEX(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), b0); \ + cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), b0); \ + } + +#define GEMV_STORE_ROW_COMPLEX(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + storeMaddData(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \ + storeMaddData(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \ + GEMV_UNROLL_ROW(GEMV_INIT_COMPLEX, N) \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(GEMV_WORK_ROW_COMPLEX, N) + +/** \internal main macro for gemv_complex_row - initialize accumulators, multiply and add inputs, predux and store results */ +#define GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N) \ + for (; i < n##N; i += N) { \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \ + GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX, (N >> 1)) \ + GEMV_PROCESS_END_ROW_COMPLEX(N); \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \ + if (GEMV_IS_COMPLEX_COMPLEX) { \ + c0##iter = padd(c0##iter, c1##iter); \ + } \ + dd0 = predux(c0##iter); + +#if EIGEN_COMP_LLVM +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE(N) \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) + +#define GEMV_PROCESS_ROW_COMPLEX_ONE(N) \ + GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N) + +#define GEMV_PROCESS_ROW_COMPLEX_PREDUX(iter) \ + GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) +#else +// gcc seems to be reading and writing registers unnecessarily to memory. +// Use the old way for complex double until it is fixed. + +#define GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter) \ + lhs.template load(i + (iter), j) + +#define GEMV_INIT_COMPLEX_OLD(iter, N) \ + EIGEN_UNUSED_VARIABLE(c0##iter); \ + if (N > iter) { \ + c1##iter = pset_zero(); \ + } else { \ + EIGEN_UNUSED_VARIABLE(c1##iter); \ + } + +#define GEMV_WORK_ROW_COMPLEX_OLD(iter, N) \ + if (N > iter) { \ + LhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter); \ + c1##iter = pcj.pmadd(a##iter, b0, c1##iter); \ + } + +#define GEMV_PREDUX4_COMPLEX_OLD(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + cc##iter1.scalar[0] = predux(c1##iter2); \ + cc##iter1.scalar[1] = predux(c1##iter3); \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \ + GEMV_UNROLL_ROW(GEMV_INIT_COMPLEX_OLD, N) \ + j = 0; \ + for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \ + RhsPacket b0 = rhs2.template load(j); \ + GEMV_UNROLL_ROW(GEMV_WORK_ROW_COMPLEX_OLD, N) \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_ONE_OLD(N) \ + for (; i < n##N; i += N) { \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \ + GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX_OLD, (N >> 1)) \ + GEMV_PROCESS_END_ROW_COMPLEX(N) \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_OLD(iter) \ + dd0 = predux(c1##iter); + +#if (__GNUC__ > 10) +#define GEMV_PROCESS_ROW_COMPLEX_IS_NEW 1 +#else +#define GEMV_PROCESS_ROW_COMPLEX_IS_NEW \ + (sizeof(Scalar) == sizeof(float)) || GEMV_IS_COMPLEX_COMPLEX +#endif + +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE(N) \ + if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \ + } else { \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_ONE(N) \ + if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \ + GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N) \ + } else { \ + GEMV_PROCESS_ROW_COMPLEX_ONE_OLD(N) \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_PREDUX(iter) \ + if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \ + GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \ + } else { \ + GEMV_PROCESS_ROW_COMPLEX_PREDUX_OLD(iter) \ + } +#endif + +#ifdef USE_GEMV_MMA +#define GEMV_PROCESS_ROW_COMPLEX(N) \ + GEMV_PROCESS_ROW_COMPLEX_ONE_MMA(N) +#else +#define GEMV_PROCESS_ROW_COMPLEX(N) \ + GEMV_PROCESS_ROW_COMPLEX_ONE(N) +#endif + +template +EIGEN_STRONG_INLINE void gemv_complex_row( + Index rows, Index cols, + const LhsMapper& alhs, + const RhsMapper& rhs, + ResScalar* res, Index resIncr, + ResScalar alpha) +{ + typedef gemv_traits Traits; + + typedef typename Traits::LhsPacket LhsPacket; + typedef typename Traits::RhsPacket RhsPacket; + typedef typename Traits::ResPacket ResPacket; + + typedef typename packet_traits::type ScalarPacket; + typedef typename packet_traits::type PLhsPacket; + typedef typename packet_traits::type PResPacket; + typedef gemv_traits PTraits; + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + typename RhsMapper::LinearMapper rhs2 = rhs.getLinearMapper(0, 0); + + eigen_internal_assert(rhs.stride() == 1); + conj_helper cj; +#if !EIGEN_COMP_LLVM + conj_helper pcj; +#endif + + // TODO: fine tune the following heuristic. The rationale is that if the matrix is very large, + // processing 8 rows at once might be counter productive wrt cache. +#ifndef GCC_ONE_VECTORPAIR_BUG + const Index n8 = lhs.stride() * sizeof(LhsScalar) > 32000 ? (rows - 7) : (rows - 7); + const Index n4 = rows - 3; + const Index n2 = rows - 1; +#endif + + // TODO: for padded aligned inputs, we could enable aligned reads + enum { + LhsAlignment = Unaligned, + ResPacketSize = PTraits::ResPacketSize, + LhsPacketSize = PTraits::LhsPacketSize, + RhsPacketSize = PTraits::RhsPacketSize, + }; + + Index i = 0, j; + PResPacket c00, c01, c02, c03, c04, c05, c06, c07; + ResPacket c10, c11, c12, c13, c14, c15, c16, c17; +#ifdef USE_GEMV_MMA + __vector_quad e00, e01, e02, e03, e04, e05, e06, e07; + GEMV_UNUSED_ROW(8, e0) + GEMV_UNUSED_EXTRA(1, c0) + GEMV_UNUSED_EXTRA(1, c1) +#endif + ResScalar dd0; +#ifndef GCC_ONE_VECTORPAIR_BUG + ScalarBlock cc0, cc1, cc2, cc3; +#ifdef USE_GEMV_MMA + if (!GEMV_IS_COMPLEX_COMPLEX) +#endif + { + GEMV_PROCESS_ROW_COMPLEX(8) + } + GEMV_PROCESS_ROW_COMPLEX(4) + GEMV_PROCESS_ROW_COMPLEX(2) +#endif + for (; i < rows; ++i) + { + GEMV_PROCESS_ROW_COMPLEX_SINGLE(1) + GEMV_PROCESS_ROW_COMPLEX_PREDUX(0) + for (; j < cols; ++j) + { + dd0 += cj.pmul(lhs(i, j), rhs2(j)); + } + res[i * resIncr] += alpha * dd0; + } +} + +#define EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(Scalar, LhsScalar, RhsScalar) \ +template \ +struct general_matrix_vector_product \ +{ \ + typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; \ +\ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run( \ + Index rows, Index cols, \ + const LhsMapper& lhs, \ + const RhsMapper& rhs, \ + ResScalar* res, Index resIncr, \ + ResScalar alpha) { \ + gemv_complex_col(rows, cols, lhs, rhs, res, resIncr, alpha); \ + } \ +}; + +#define EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(Scalar, LhsScalar, RhsScalar) \ +template \ +struct general_matrix_vector_product \ +{ \ + typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; \ +\ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run( \ + Index rows, Index cols, \ + const LhsMapper& lhs, \ + const RhsMapper& rhs, \ + ResScalar* res, Index resIncr, \ + ResScalar alpha) { \ + gemv_complex_row(rows, cols, lhs, rhs, res, resIncr, alpha); \ + } \ +}; + +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(float, float, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(float, std::complex, float) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(float, std::complex, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(double, double, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(double, std::complex, double) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(double, std::complex, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(float, float, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(float, std::complex, float) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(float, std::complex, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(double, double, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(double, std::complex, double) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(double, std::complex, std::complex) + +#endif // EIGEN_MATRIX_VECTOR_PRODUCT_ALTIVEC_H + diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/PacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/PacketMath.h index 2a440545b20..b945b3344dc 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/PacketMath.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/PacketMath.h @@ -10,6 +10,9 @@ #ifndef EIGEN_PACKET_MATH_ALTIVEC_H #define EIGEN_PACKET_MATH_ALTIVEC_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -33,40 +36,41 @@ typedef __vector unsigned int Packet4ui; typedef __vector __bool int Packet4bi; typedef __vector short int Packet8s; typedef __vector unsigned short int Packet8us; +typedef __vector __bool short Packet8bi; typedef __vector signed char Packet16c; typedef __vector unsigned char Packet16uc; typedef eigen_packet_wrapper<__vector unsigned short int,0> Packet8bf; // We don't want to write the same code all the time, but we need to reuse the constants // and it doesn't really work to declare them global, so we define macros instead -#define _EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \ +#define EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \ Packet4f p4f_##NAME = {X, X, X, X} -#define _EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \ +#define EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \ Packet4i p4i_##NAME = vec_splat_s32(X) -#define _EIGEN_DECLARE_CONST_FAST_Packet4ui(NAME,X) \ +#define EIGEN_DECLARE_CONST_FAST_Packet4ui(NAME,X) \ Packet4ui p4ui_##NAME = {X, X, X, X} -#define _EIGEN_DECLARE_CONST_FAST_Packet8us(NAME,X) \ +#define EIGEN_DECLARE_CONST_FAST_Packet8us(NAME,X) \ Packet8us p8us_##NAME = {X, X, X, X, X, X, X, X} -#define _EIGEN_DECLARE_CONST_FAST_Packet16uc(NAME,X) \ +#define EIGEN_DECLARE_CONST_FAST_Packet16uc(NAME,X) \ Packet16uc p16uc_##NAME = {X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X} -#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ Packet4f p4f_##NAME = pset1(X) -#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ Packet4i p4i_##NAME = pset1(X) -#define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ Packet2d p2d_##NAME = pset1(X) -#define _EIGEN_DECLARE_CONST_Packet2l(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet2l(NAME,X) \ Packet2l p2l_##NAME = pset1(X) -#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ const Packet4f p4f_##NAME = reinterpret_cast(pset1(X)) #define DST_CHAN 1 @@ -74,15 +78,14 @@ typedef eigen_packet_wrapper<__vector unsigned short int,0> Packet8bf; #define __UNPACK_TYPE__(PACKETNAME) typename unpacket_traits::type // These constants are endian-agnostic -static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0} -static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,} -static _EIGEN_DECLARE_CONST_FAST_Packet4i(ONE,1); //{ 1, 1, 1, 1} -static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS16,-16); //{ -16, -16, -16, -16} -static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1); //{ -1, -1, -1, -1} -static _EIGEN_DECLARE_CONST_FAST_Packet4ui(SIGN, 0x80000000u); -static _EIGEN_DECLARE_CONST_FAST_Packet4ui(PREV0DOT5, 0x3EFFFFFFu); -static _EIGEN_DECLARE_CONST_FAST_Packet8us(ONE,1); //{ 1, 1, 1, 1, 1, 1, 1, 1} -static _EIGEN_DECLARE_CONST_FAST_Packet16uc(ONE,1); +static EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0} +static EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,} +static EIGEN_DECLARE_CONST_FAST_Packet4i(ONE,1); //{ 1, 1, 1, 1} +static EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS16,-16); //{ -16, -16, -16, -16} +static EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1); //{ -1, -1, -1, -1} +static EIGEN_DECLARE_CONST_FAST_Packet4ui(SIGN, 0x80000000u); +static EIGEN_DECLARE_CONST_FAST_Packet4ui(PREV0DOT5, 0x3EFFFFFFu); +static EIGEN_DECLARE_CONST_FAST_Packet8us(ONE,1); //{ 1, 1, 1, 1, 1, 1, 1, 1} static Packet4f p4f_MZERO = (Packet4f) vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1); //{ 0x80000000, 0x80000000, 0x80000000, 0x80000000} #ifndef __VSX__ static Packet4f p4f_ONE = vec_ctf(p4i_ONE, 0); //{ 1.0, 1.0, 1.0, 1.0} @@ -100,29 +103,35 @@ static Packet16uc p16uc_COUNTDOWN = { 0, 1, 2, 3, 4, 5, 6, 7, static Packet16uc p16uc_REVERSE32 = { 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3 }; static Packet16uc p16uc_REVERSE16 = { 14,15, 12,13, 10,11, 8,9, 6,7, 4,5, 2,3, 0,1 }; +#ifndef _ARCH_PWR9 static Packet16uc p16uc_REVERSE8 = { 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 }; +#endif +#ifdef _BIG_ENDIAN static Packet16uc p16uc_DUPLICATE32_HI = { 0,1,2,3, 0,1,2,3, 4,5,6,7, 4,5,6,7 }; -static Packet16uc p16uc_DUPLICATE16_HI = { 0,1,0,1, 2,3,2,3, 4,5,4,5, 6,7,6,7 }; -static Packet16uc p16uc_DUPLICATE8_HI = { 0,0, 1,1, 2,2, 3,3, 4,4, 5,5, 6,6, 7,7 }; +#endif static const Packet16uc p16uc_DUPLICATE16_EVEN= { 0,1 ,0,1, 4,5, 4,5, 8,9, 8,9, 12,13, 12,13 }; static const Packet16uc p16uc_DUPLICATE16_ODD = { 2,3 ,2,3, 6,7, 6,7, 10,11, 10,11, 14,15, 14,15 }; static Packet16uc p16uc_QUADRUPLICATE16_HI = { 0,1,0,1,0,1,0,1, 2,3,2,3,2,3,2,3 }; +static Packet16uc p16uc_MERGEE16 = { 0,1, 16,17, 4,5, 20,21, 8,9, 24,25, 12,13, 28,29 }; +static Packet16uc p16uc_MERGEO16 = { 2,3, 18,19, 6,7, 22,23, 10,11, 26,27, 14,15, 30,31 }; +#ifdef _BIG_ENDIAN +static Packet16uc p16uc_MERGEH16 = { 0,1, 4,5, 8,9, 12,13, 16,17, 20,21, 24,25, 28,29 }; +#else +static Packet16uc p16uc_MERGEL16 = { 2,3, 6,7, 10,11, 14,15, 18,19, 22,23, 26,27, 30,31 }; +#endif + // Handle endianness properly while loading constants // Define global static constants: #ifdef _BIG_ENDIAN static Packet16uc p16uc_FORWARD = vec_lvsl(0, (float*)0); -#ifdef __VSX__ -static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; -#endif static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 }; static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 }; static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3), 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16}; #else static Packet16uc p16uc_FORWARD = p16uc_REVERSE32; -static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 1), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 }; static Packet16uc p16uc_PSET32_WEVEN = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 }; static Packet16uc p16uc_HALF64_0_16 = vec_sld(vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 0), (Packet16uc)p4i_ZERO, 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16}; @@ -135,18 +144,18 @@ static Packet16uc p16uc_TRANSPOSE64_LO = p16uc_PSET64_LO + p16uc_HALF64_0_16; static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 }; -#ifdef _BIG_ENDIAN -static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; -#else -static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_PSET64_HI, p16uc_PSET64_LO, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; -#endif // _BIG_ENDIAN - #if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC #define EIGEN_PPC_PREFETCH(ADDR) __builtin_prefetch(ADDR); #else #define EIGEN_PPC_PREFETCH(ADDR) asm( " dcbt [%[addr]]\n" :: [addr] "r" (ADDR) : "cc" ); #endif +#if EIGEN_COMP_LLVM +#define LOAD_STORE_UNROLL_16 _Pragma("unroll 16") +#else +#define LOAD_STORE_UNROLL_16 _Pragma("GCC unroll(16)") +#endif + template <> struct packet_traits : default_packet_traits { typedef Packet4f type; @@ -155,7 +164,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 1, HasAdd = 1, HasSub = 1, @@ -166,25 +174,32 @@ struct packet_traits : default_packet_traits { HasAbs = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, + HasACos = 1, + HasASin = 1, + HasATan = 1, + HasATanh = 1, HasLog = 1, HasExp = 1, -#ifdef __VSX__ +#ifdef EIGEN_VECTORIZE_VSX HasSqrt = 1, #if !EIGEN_COMP_CLANG HasRsqrt = 1, #else HasRsqrt = 0, #endif + HasTanh = EIGEN_FAST_MATH, + HasErf = EIGEN_FAST_MATH, + HasRint = 1, #else HasSqrt = 0, HasRsqrt = 0, - HasTanh = EIGEN_FAST_MATH, - HasErf = EIGEN_FAST_MATH, + HasTanh = 0, + HasErf = 0, + HasRint = 0, #endif HasRound = 1, HasFloor = 1, HasCeil = 1, - HasRint = 1, HasNegate = 1, HasBlend = 1 }; @@ -197,7 +212,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -210,23 +224,24 @@ struct packet_traits : default_packet_traits { HasCos = EIGEN_FAST_MATH, HasLog = 1, HasExp = 1, -#ifdef __VSX__ +#ifdef EIGEN_VECTORIZE_VSX HasSqrt = 1, #if !EIGEN_COMP_CLANG HasRsqrt = 1, #else HasRsqrt = 0, #endif + HasRint = 1, #else HasSqrt = 0, HasRsqrt = 0, - HasTanh = EIGEN_FAST_MATH, - HasErf = EIGEN_FAST_MATH, + HasRint = 0, #endif + HasTanh = 0, + HasErf = 0, HasRound = 1, HasFloor = 1, HasCeil = 1, - HasRint = 1, HasNegate = 1, HasBlend = 1 }; @@ -240,14 +255,18 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, +#if defined(_ARCH_PWR10) && (EIGEN_COMP_LLVM || EIGEN_GNUC_STRICT_AT_LEAST(11,0,0)) + HasDiv = 1, +#else HasDiv = 0, - HasBlend = 1 +#endif + HasBlend = 1, + HasCmp = 1 }; }; @@ -259,13 +278,13 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 0, - HasBlend = 1 + HasBlend = 1, + HasCmp = 1 }; }; @@ -277,13 +296,13 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 0, - HasBlend = 1 + HasBlend = 1, + HasCmp = 1 }; }; @@ -295,13 +314,13 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 16, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 0, - HasBlend = 1 + HasBlend = 1, + HasCmp = 1 }; }; @@ -313,13 +332,13 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 16, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 0, - HasBlend = 1 + HasBlend = 1, + HasCmp = 1 }; }; @@ -432,7 +451,7 @@ EIGEN_STRONG_INLINE Packet pload_common(const __UNPACK_TYPE__(Packet)* from) // ignoring these warnings for now. EIGEN_UNUSED_VARIABLE(from); EIGEN_DEBUG_ALIGNED_LOAD -#ifdef __VSX__ +#ifdef EIGEN_VECTORIZE_VSX return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from)); #else return vec_ld(0, from); @@ -475,13 +494,115 @@ template<> EIGEN_STRONG_INLINE Packet8bf pload(const bfloat16* fr return pload_common(reinterpret_cast(from)); } +template +EIGEN_ALWAYS_INLINE Packet pload_ignore(const __UNPACK_TYPE__(Packet)* from) +{ + // some versions of GCC throw "unused-but-set-parameter". + // ignoring these warnings for now. + EIGEN_UNUSED_VARIABLE(from); + EIGEN_DEBUG_ALIGNED_LOAD + // Ignore partial input memory initialized +#if !EIGEN_COMP_LLVM + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#ifdef EIGEN_VECTORIZE_VSX + return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from)); +#else + return vec_ld(0, from); +#endif +#if !EIGEN_COMP_LLVM + #pragma GCC diagnostic pop +#endif +} + +template<> EIGEN_ALWAYS_INLINE Packet8bf pload_ignore(const bfloat16* from) +{ + return pload_ignore(reinterpret_cast(from)); +} + +template +EIGEN_ALWAYS_INLINE Packet pload_partial_common(const __UNPACK_TYPE__(Packet)* from, const Index n, const Index offset) +{ + // some versions of GCC throw "unused-but-set-parameter". + // ignoring these warnings for now. + const Index packet_size = unpacket_traits::size; + eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet"); + const Index size = sizeof(__UNPACK_TYPE__(Packet)); +#ifdef _ARCH_PWR9 + EIGEN_UNUSED_VARIABLE(packet_size); + EIGEN_DEBUG_ALIGNED_LOAD + EIGEN_UNUSED_VARIABLE(from); + Packet load = vec_xl_len(const_cast<__UNPACK_TYPE__(Packet)*>(from), n * size); + if (offset) { + Packet16uc shift = pset1(offset * 8 * size); +#ifdef _BIG_ENDIAN + load = Packet(vec_sro(Packet16uc(load), shift)); +#else + load = Packet(vec_slo(Packet16uc(load), shift)); +#endif + } + return load; +#else + if (n) { + EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) load[packet_size]; + unsigned char* load2 = reinterpret_cast(load + offset); + unsigned char* from2 = reinterpret_cast(const_cast<__UNPACK_TYPE__(Packet)*>(from)); + Index n2 = n * size; + if (16 <= n2) { + pstoreu(load2, ploadu(from2)); + } else { + memcpy((void *)load2, (void *)from2, n2); + } + return pload_ignore(load); + } else { + return Packet(pset1(0)); + } +#endif +} + +template<> EIGEN_ALWAYS_INLINE Packet4f pload_partial(const float* from, const Index n, const Index offset) +{ + return pload_partial_common(from, n, offset); +} + +template<> EIGEN_ALWAYS_INLINE Packet4i pload_partial(const int* from, const Index n, const Index offset) +{ + return pload_partial_common(from, n, offset); +} + +template<> EIGEN_ALWAYS_INLINE Packet8s pload_partial(const short int* from, const Index n, const Index offset) +{ + return pload_partial_common(from, n, offset); +} + +template<> EIGEN_ALWAYS_INLINE Packet8us pload_partial(const unsigned short int* from, const Index n, const Index offset) +{ + return pload_partial_common(from, n, offset); +} + +template<> EIGEN_ALWAYS_INLINE Packet8bf pload_partial(const bfloat16* from, const Index n, const Index offset) +{ + return pload_partial_common(reinterpret_cast(from), n, offset); +} + +template<> EIGEN_ALWAYS_INLINE Packet16c pload_partial(const signed char* from, const Index n, const Index offset) +{ + return pload_partial_common(from, n, offset); +} + +template<> EIGEN_ALWAYS_INLINE Packet16uc pload_partial(const unsigned char* from, const Index n, const Index offset) +{ + return pload_partial_common(from, n, offset); +} + template EIGEN_STRONG_INLINE void pstore_common(__UNPACK_TYPE__(Packet)* to, const Packet& from){ // some versions of GCC throw "unused-but-set-parameter" (float *to). // ignoring these warnings for now. EIGEN_UNUSED_VARIABLE(to); EIGEN_DEBUG_ALIGNED_STORE -#ifdef __VSX__ +#ifdef EIGEN_VECTORIZE_VSX vec_xst(from, 0, to); #else vec_st(from, 0, to); @@ -510,7 +631,7 @@ template<> EIGEN_STRONG_INLINE void pstore(unsigned short in template<> EIGEN_STRONG_INLINE void pstore(bfloat16* to, const Packet8bf& from) { - pstore_common(reinterpret_cast(to), from); + pstore_common(reinterpret_cast(to), from.m_val); } template<> EIGEN_STRONG_INLINE void pstore(signed char* to, const Packet16c& from) @@ -523,6 +644,78 @@ template<> EIGEN_STRONG_INLINE void pstore(unsigned char* t pstore_common(to, from); } +template EIGEN_ALWAYS_INLINE void pstore_partial_common(__UNPACK_TYPE__(Packet)* to, const Packet& from, const Index n, const Index offset) +{ + // some versions of GCC throw "unused-but-set-parameter" (float *to). + // ignoring these warnings for now. + const Index packet_size = unpacket_traits::size; + eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet"); + const Index size = sizeof(__UNPACK_TYPE__(Packet)); +#ifdef _ARCH_PWR9 + EIGEN_UNUSED_VARIABLE(packet_size); + EIGEN_UNUSED_VARIABLE(to); + EIGEN_DEBUG_ALIGNED_STORE + Packet store = from; + if (offset) { + Packet16uc shift = pset1(offset * 8 * size); +#ifdef _BIG_ENDIAN + store = Packet(vec_slo(Packet16uc(store), shift)); +#else + store = Packet(vec_sro(Packet16uc(store), shift)); +#endif + } + vec_xst_len(store, to, n * size); +#else + if (n) { + EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) store[packet_size]; + pstore(store, from); + unsigned char* store2 = reinterpret_cast(store + offset); + unsigned char* to2 = reinterpret_cast(to); + Index n2 = n * size; + if (16 <= n2) { + pstore(to2, ploadu(store2)); + } else { + memcpy((void *)to2, (void *)store2, n2); + } + } +#endif +} + +template<> EIGEN_ALWAYS_INLINE void pstore_partial(float* to, const Packet4f& from, const Index n, const Index offset) +{ + pstore_partial_common(to, from, n, offset); +} + +template<> EIGEN_ALWAYS_INLINE void pstore_partial(int* to, const Packet4i& from, const Index n, const Index offset) +{ + pstore_partial_common(to, from, n, offset); +} + +template<> EIGEN_ALWAYS_INLINE void pstore_partial(short int* to, const Packet8s& from, const Index n, const Index offset) +{ + pstore_partial_common(to, from, n, offset); +} + +template<> EIGEN_ALWAYS_INLINE void pstore_partial(unsigned short int* to, const Packet8us& from, const Index n, const Index offset) +{ + pstore_partial_common(to, from, n, offset); +} + +template<> EIGEN_ALWAYS_INLINE void pstore_partial(bfloat16* to, const Packet8bf& from, const Index n, const Index offset) +{ + pstore_partial_common(reinterpret_cast(to), from.m_val, n, offset); +} + +template<> EIGEN_ALWAYS_INLINE void pstore_partial(signed char* to, const Packet16c& from, const Index n, const Index offset) +{ + pstore_partial_common(to, from, n, offset); +} + +template<> EIGEN_ALWAYS_INLINE void pstore_partial(unsigned char* to, const Packet16uc& from, const Index n, const Index offset) +{ + pstore_partial_common(to, from, n, offset); +} + template EIGEN_STRONG_INLINE Packet pset1_size4(const __UNPACK_TYPE__(Packet)& from) { @@ -600,168 +793,183 @@ pbroadcast4(const int *a, pbroadcast4_common(a, a0, a1, a2, a3); } -template EIGEN_DEVICE_FUNC inline Packet pgather_common(const __UNPACK_TYPE__(Packet)* from, Index stride) -{ - EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[4]; - a[0] = from[0*stride]; - a[1] = from[1*stride]; - a[2] = from[2*stride]; - a[3] = from[3*stride]; - return pload(a); +template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pgather_common(const __UNPACK_TYPE__(Packet)* from, Index stride, const Index n = unpacket_traits::size) +{ + EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[unpacket_traits::size]; + eigen_internal_assert(n <= unpacket_traits::size && "number of elements will gather past end of packet"); + if (stride == 1) { + if (n == unpacket_traits::size) { + return ploadu(from); + } else { + return ploadu_partial(from, n); + } + } else { + LOAD_STORE_UNROLL_16 + for (Index i = 0; i < n; i++) { + a[i] = from[i*stride]; + } + // Leave rest of the array uninitialized + return pload_ignore(a); + } } -template<> EIGEN_DEVICE_FUNC inline Packet4f pgather(const float* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4f pgather(const float* from, Index stride) { return pgather_common(from, stride); } -template<> EIGEN_DEVICE_FUNC inline Packet4i pgather(const int* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4i pgather(const int* from, Index stride) { return pgather_common(from, stride); } -template EIGEN_DEVICE_FUNC inline Packet pgather_size8(const __UNPACK_TYPE__(Packet)* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8s pgather(const short int* from, Index stride) +{ + return pgather_common(from, stride); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8us pgather(const unsigned short int* from, Index stride) +{ + return pgather_common(from, stride); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8bf pgather(const bfloat16* from, Index stride) { - EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[8]; - a[0] = from[0*stride]; - a[1] = from[1*stride]; - a[2] = from[2*stride]; - a[3] = from[3*stride]; - a[4] = from[4*stride]; - a[5] = from[5*stride]; - a[6] = from[6*stride]; - a[7] = from[7*stride]; - return pload(a); + return pgather_common(from, stride); } -template<> EIGEN_DEVICE_FUNC inline Packet8s pgather(const short int* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16c pgather(const signed char* from, Index stride) { - return pgather_size8(from, stride); + return pgather_common(from, stride); } -template<> EIGEN_DEVICE_FUNC inline Packet8us pgather(const unsigned short int* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16uc pgather(const unsigned char* from, Index stride) { - return pgather_size8(from, stride); + return pgather_common(from, stride); } -template<> EIGEN_DEVICE_FUNC inline Packet8bf pgather(const bfloat16* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4f pgather_partial(const float* from, Index stride, const Index n) { - return pgather_size8(from, stride); + return pgather_common(from, stride, n); } -template EIGEN_DEVICE_FUNC inline Packet pgather_size16(const __UNPACK_TYPE__(Packet)* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4i pgather_partial(const int* from, Index stride, const Index n) { - EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[16]; - a[0] = from[0*stride]; - a[1] = from[1*stride]; - a[2] = from[2*stride]; - a[3] = from[3*stride]; - a[4] = from[4*stride]; - a[5] = from[5*stride]; - a[6] = from[6*stride]; - a[7] = from[7*stride]; - a[8] = from[8*stride]; - a[9] = from[9*stride]; - a[10] = from[10*stride]; - a[11] = from[11*stride]; - a[12] = from[12*stride]; - a[13] = from[13*stride]; - a[14] = from[14*stride]; - a[15] = from[15*stride]; - return pload(a); + return pgather_common(from, stride, n); } +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8s pgather_partial(const short int* from, Index stride, const Index n) +{ + return pgather_common(from, stride, n); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8us pgather_partial(const unsigned short int* from, Index stride, const Index n) +{ + return pgather_common(from, stride, n); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8bf pgather_partial(const bfloat16* from, Index stride, const Index n) +{ + return pgather_common(from, stride, n); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16c pgather_partial(const signed char* from, Index stride, const Index n) +{ + return pgather_common(from, stride, n); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16uc pgather_partial(const unsigned char* from, Index stride, const Index n) +{ + return pgather_common(from, stride, n); +} -template<> EIGEN_DEVICE_FUNC inline Packet16c pgather(const signed char* from, Index stride) +template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_common(__UNPACK_TYPE__(Packet)* to, const Packet& from, Index stride, const Index n = unpacket_traits::size) { - return pgather_size16(from, stride); + EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[unpacket_traits::size]; + eigen_internal_assert(n <= unpacket_traits::size && "number of elements will scatter past end of packet"); + if (stride == 1) { + if (n == unpacket_traits::size) { + return pstoreu(to, from); + } else { + return pstoreu_partial(to, from, n); + } + } else { + pstore<__UNPACK_TYPE__(Packet)>(a, from); + LOAD_STORE_UNROLL_16 + for (Index i = 0; i < n; i++) { + to[i*stride] = a[i]; + } + } } -template<> EIGEN_DEVICE_FUNC inline Packet16uc pgather(const unsigned char* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter(float* to, const Packet4f& from, Index stride) { - return pgather_size16(from, stride); + pscatter_common(to, from, stride); } -template EIGEN_DEVICE_FUNC inline void pscatter_size4(__UNPACK_TYPE__(Packet)* to, const Packet& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter(int* to, const Packet4i& from, Index stride) { - EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[4]; - pstore<__UNPACK_TYPE__(Packet)>(a, from); - to[0*stride] = a[0]; - to[1*stride] = a[1]; - to[2*stride] = a[2]; - to[3*stride] = a[3]; + pscatter_common(to, from, stride); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const Packet4f& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter(short int* to, const Packet8s& from, Index stride) { - pscatter_size4(to, from, stride); + pscatter_common(to, from, stride); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(int* to, const Packet4i& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter(unsigned short int* to, const Packet8us& from, Index stride) { - pscatter_size4(to, from, stride); + pscatter_common(to, from, stride); } -template EIGEN_DEVICE_FUNC inline void pscatter_size8(__UNPACK_TYPE__(Packet)* to, const Packet& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter(bfloat16* to, const Packet8bf& from, Index stride) { - EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[8]; - pstore<__UNPACK_TYPE__(Packet)>(a, from); - to[0*stride] = a[0]; - to[1*stride] = a[1]; - to[2*stride] = a[2]; - to[3*stride] = a[3]; - to[4*stride] = a[4]; - to[5*stride] = a[5]; - to[6*stride] = a[6]; - to[7*stride] = a[7]; + pscatter_common(to, from, stride); } +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter(signed char* to, const Packet16c& from, Index stride) +{ + pscatter_common(to, from, stride); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter(unsigned char* to, const Packet16uc& from, Index stride) +{ + pscatter_common(to, from, stride); +} -template<> EIGEN_DEVICE_FUNC inline void pscatter(short int* to, const Packet8s& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial(float* to, const Packet4f& from, Index stride, const Index n) { - pscatter_size8(to, from, stride); + pscatter_common(to, from, stride, n); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(unsigned short int* to, const Packet8us& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial(int* to, const Packet4i& from, Index stride, const Index n) { - pscatter_size8(to, from, stride); + pscatter_common(to, from, stride, n); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(bfloat16* to, const Packet8bf& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial(short int* to, const Packet8s& from, Index stride, const Index n) { - pscatter_size8(to, from, stride); + pscatter_common(to, from, stride, n); } -template EIGEN_DEVICE_FUNC inline void pscatter_size16(__UNPACK_TYPE__(Packet)* to, const Packet& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial(unsigned short int* to, const Packet8us& from, Index stride, const Index n) { - EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[16]; - pstore<__UNPACK_TYPE__(Packet)>(a, from); - to[0*stride] = a[0]; - to[1*stride] = a[1]; - to[2*stride] = a[2]; - to[3*stride] = a[3]; - to[4*stride] = a[4]; - to[5*stride] = a[5]; - to[6*stride] = a[6]; - to[7*stride] = a[7]; - to[8*stride] = a[8]; - to[9*stride] = a[9]; - to[10*stride] = a[10]; - to[11*stride] = a[11]; - to[12*stride] = a[12]; - to[13*stride] = a[13]; - to[14*stride] = a[14]; - to[15*stride] = a[15]; + pscatter_common(to, from, stride, n); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(signed char* to, const Packet16c& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial(bfloat16* to, const Packet8bf& from, Index stride, const Index n) { - pscatter_size16(to, from, stride); + pscatter_common(to, from, stride, n); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(unsigned char* to, const Packet16uc& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial(signed char* to, const Packet16c& from, Index stride, const Index n) { - pscatter_size16(to, from, stride); + pscatter_common(to, from, stride, n); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial(unsigned char* to, const Packet16uc& from, Index stride, const Index n) +{ + pscatter_common(to, from, stride, n); } template<> EIGEN_STRONG_INLINE Packet4f plset(const float& a) { return pset1(a) + p4f_COUNTDOWN; } @@ -786,8 +994,38 @@ template<> EIGEN_STRONG_INLINE Packet8us psub (const Packet8us& a, template<> EIGEN_STRONG_INLINE Packet16c psub (const Packet16c& a, const Packet16c& b) { return a - b; } template<> EIGEN_STRONG_INLINE Packet16uc psub(const Packet16uc& a, const Packet16uc& b) { return a - b; } -template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return p4f_ZERO - a; } -template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return p4i_ZERO - a; } +template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) +{ +#ifdef __POWER8_VECTOR__ + return vec_neg(a); +#else + return vec_xor(a, p4f_MZERO); +#endif +} +template<> EIGEN_STRONG_INLINE Packet16c pnegate(const Packet16c& a) +{ +#ifdef __POWER8_VECTOR__ + return vec_neg(a); +#else + return reinterpret_cast(p4i_ZERO) - a; +#endif +} +template<> EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) +{ +#ifdef __POWER8_VECTOR__ + return vec_neg(a); +#else + return reinterpret_cast(p4i_ZERO) - a; +#endif +} +template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) +{ +#ifdef __POWER8_VECTOR__ + return vec_neg(a); +#else + return p4i_ZERO - a; +#endif +} template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; } @@ -818,9 +1056,16 @@ template<> EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const #endif } -template<> EIGEN_STRONG_INLINE Packet4i pdiv(const Packet4i& /*a*/, const Packet4i& /*b*/) -{ eigen_assert(false && "packet integer division are not supported by AltiVec"); +template<> EIGEN_STRONG_INLINE Packet4i pdiv(const Packet4i& a, const Packet4i& b) +{ +#if defined(_ARCH_PWR10) && (EIGEN_COMP_LLVM || EIGEN_GNUC_STRICT_AT_LEAST(11,0,0)) + return vec_div(a, b); +#else + EIGEN_UNUSED_VARIABLE(a); + EIGEN_UNUSED_VARIABLE(b); + eigen_assert(false && "packet integer division are not supported by AltiVec"); return pset1(0); +#endif } // for some weird raisons, it has to be overloaded for packet of integers @@ -829,9 +1074,15 @@ template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& template<> EIGEN_STRONG_INLINE Packet8s pmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) { return vec_madd(a,b,c); } template<> EIGEN_STRONG_INLINE Packet8us pmadd(const Packet8us& a, const Packet8us& b, const Packet8us& c) { return vec_madd(a,b,c); } +#ifdef EIGEN_VECTORIZE_VSX +template<> EIGEN_STRONG_INLINE Packet4f pmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vec_msub(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet4f pnmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vec_nmsub(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet4f pnmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vec_nmadd(a,b,c); } +#endif + template<> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { - #ifdef __VSX__ + #ifdef EIGEN_VECTORIZE_VSX // NOTE: about 10% slower than vec_min, but consistent with std::min and SSE regarding NaN Packet4f ret; __asm__ ("xvcmpgesp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b)); @@ -849,7 +1100,7 @@ template<> EIGEN_STRONG_INLINE Packet16uc pmin(const Packet16uc& a, template<> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { - #ifdef __VSX__ + #ifdef EIGEN_VECTORIZE_VSX // NOTE: about 10% slower than vec_max, but consistent with std::max and SSE regarding NaN Packet4f ret; __asm__ ("xvcmpgtsp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b)); @@ -865,26 +1116,39 @@ template<> EIGEN_STRONG_INLINE Packet16c pmax(const Packet16c& a, con template<> EIGEN_STRONG_INLINE Packet16uc pmax(const Packet16uc& a, const Packet16uc& b) { return vec_max(a, b); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) { return reinterpret_cast(vec_cmple(a,b)); } +// To fix bug with vec_cmplt on older versions +#ifdef EIGEN_VECTORIZE_VSX template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) { return reinterpret_cast(vec_cmplt(a,b)); } +#endif template<> EIGEN_STRONG_INLINE Packet4f pcmp_eq(const Packet4f& a, const Packet4f& b) { return reinterpret_cast(vec_cmpeq(a,b)); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f& a, const Packet4f& b) { Packet4f c = reinterpret_cast(vec_cmpge(a,b)); return vec_nor(c,c); } +#ifdef EIGEN_VECTORIZE_VSX template<> EIGEN_STRONG_INLINE Packet4i pcmp_le(const Packet4i& a, const Packet4i& b) { return reinterpret_cast(vec_cmple(a,b)); } +#endif template<> EIGEN_STRONG_INLINE Packet4i pcmp_lt(const Packet4i& a, const Packet4i& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) { return reinterpret_cast(vec_cmpeq(a,b)); } +#ifdef EIGEN_VECTORIZE_VSX template<> EIGEN_STRONG_INLINE Packet8s pcmp_le(const Packet8s& a, const Packet8s& b) { return reinterpret_cast(vec_cmple(a,b)); } +#endif template<> EIGEN_STRONG_INLINE Packet8s pcmp_lt(const Packet8s& a, const Packet8s& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet8s pcmp_eq(const Packet8s& a, const Packet8s& b) { return reinterpret_cast(vec_cmpeq(a,b)); } +#ifdef EIGEN_VECTORIZE_VSX template<> EIGEN_STRONG_INLINE Packet8us pcmp_le(const Packet8us& a, const Packet8us& b) { return reinterpret_cast(vec_cmple(a,b)); } +#endif template<> EIGEN_STRONG_INLINE Packet8us pcmp_lt(const Packet8us& a, const Packet8us& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet8us pcmp_eq(const Packet8us& a, const Packet8us& b) { return reinterpret_cast(vec_cmpeq(a,b)); } +#ifdef EIGEN_VECTORIZE_VSX template<> EIGEN_STRONG_INLINE Packet16c pcmp_le(const Packet16c& a, const Packet16c& b) { return reinterpret_cast(vec_cmple(a,b)); } +#endif template<> EIGEN_STRONG_INLINE Packet16c pcmp_lt(const Packet16c& a, const Packet16c& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet16c pcmp_eq(const Packet16c& a, const Packet16c& b) { return reinterpret_cast(vec_cmpeq(a,b)); } +#ifdef EIGEN_VECTORIZE_VSX template<> EIGEN_STRONG_INLINE Packet16uc pcmp_le(const Packet16uc& a, const Packet16uc& b) { return reinterpret_cast(vec_cmple(a,b)); } +#endif template<> EIGEN_STRONG_INLINE Packet16uc pcmp_lt(const Packet16uc& a, const Packet16uc& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet16uc pcmp_eq(const Packet16uc& a, const Packet16uc& b) { return reinterpret_cast(vec_cmpeq(a,b)); } @@ -907,6 +1171,7 @@ template<> EIGEN_STRONG_INLINE Packet8bf por(const Packet8bf& a, cons template<> EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b) { return vec_xor(a, b); } template<> EIGEN_STRONG_INLINE Packet4i pxor(const Packet4i& a, const Packet4i& b) { return vec_xor(a, b); } +template<> EIGEN_STRONG_INLINE Packet8us pxor(const Packet8us& a, const Packet8us& b) { return vec_xor(a, b); } template<> EIGEN_STRONG_INLINE Packet8bf pxor(const Packet8bf& a, const Packet8bf& b) { return pxor(a, b); } @@ -923,7 +1188,7 @@ template<> EIGEN_STRONG_INLINE Packet4f pround(const Packet4f& a) Packet4f t = vec_add(reinterpret_cast(vec_or(vec_and(reinterpret_cast(a), p4ui_SIGN), p4ui_PREV0DOT5)), a); Packet4f res; -#ifdef __VSX__ +#ifdef EIGEN_VECTORIZE_VSX __asm__("xvrspiz %x0, %x1\n\t" : "=&wa" (res) : "wa" (t)); @@ -937,6 +1202,7 @@ template<> EIGEN_STRONG_INLINE Packet4f pround(const Packet4f& a) } template<> EIGEN_STRONG_INLINE Packet4f pceil(const Packet4f& a) { return vec_ceil(a); } template<> EIGEN_STRONG_INLINE Packet4f pfloor(const Packet4f& a) { return vec_floor(a); } +#ifdef EIGEN_VECTORIZE_VSX template<> EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) { Packet4f res; @@ -947,11 +1213,15 @@ template<> EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) return res; } +#endif template EIGEN_STRONG_INLINE Packet ploadu_common(const __UNPACK_TYPE__(Packet)* from) { EIGEN_DEBUG_ALIGNED_LOAD -#ifdef _BIG_ENDIAN +#if defined(EIGEN_VECTORIZE_VSX) || !defined(_BIG_ENDIAN) + EIGEN_DEBUG_UNALIGNED_LOAD + return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from)); +#else Packet16uc MSQ, LSQ; Packet16uc mask; MSQ = vec_ld(0, (unsigned char *)from); // most significant quadword @@ -959,9 +1229,6 @@ template EIGEN_STRONG_INLINE Packet ploadu_common(const __UNPAC mask = vec_lvsl(0, from); // create the permute mask //TODO: Add static_cast here return (Packet) vec_perm(MSQ, LSQ, mask); // align the data -#else - EIGEN_DEBUG_UNALIGNED_LOAD - return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from)); #endif } @@ -994,12 +1261,78 @@ template<> EIGEN_STRONG_INLINE Packet16uc ploadu(const unsigned char return ploadu_common(from); } +template EIGEN_ALWAYS_INLINE Packet ploadu_partial_common(const __UNPACK_TYPE__(Packet)* from, const Index n, const Index offset) +{ + const Index packet_size = unpacket_traits::size; + eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet"); + const Index size = sizeof(__UNPACK_TYPE__(Packet)); +#ifdef _ARCH_PWR9 + EIGEN_UNUSED_VARIABLE(packet_size); + EIGEN_DEBUG_ALIGNED_LOAD + EIGEN_DEBUG_UNALIGNED_LOAD + Packet load = vec_xl_len(const_cast<__UNPACK_TYPE__(Packet)*>(from), n * size); + if (offset) { + Packet16uc shift = pset1(offset * 8 * size); +#ifdef _BIG_ENDIAN + load = Packet(vec_sro(Packet16uc(load), shift)); +#else + load = Packet(vec_slo(Packet16uc(load), shift)); +#endif + } + return load; +#else + if (n) { + EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) load[packet_size]; + unsigned char* load2 = reinterpret_cast(load + offset); + unsigned char* from2 = reinterpret_cast(const_cast<__UNPACK_TYPE__(Packet)*>(from)); + Index n2 = n * size; + if (16 <= n2) { + pstoreu(load2, ploadu(from2)); + } else { + memcpy((void *)load2, (void *)from2, n2); + } + return pload_ignore(load); + } else { + return Packet(pset1(0)); + } +#endif +} + +template<> EIGEN_ALWAYS_INLINE Packet4f ploadu_partial(const float* from, const Index n, const Index offset) +{ + return ploadu_partial_common(from, n, offset); +} +template<> EIGEN_ALWAYS_INLINE Packet4i ploadu_partial(const int* from, const Index n, const Index offset) +{ + return ploadu_partial_common(from, n, offset); +} +template<> EIGEN_ALWAYS_INLINE Packet8s ploadu_partial(const short int* from, const Index n, const Index offset) +{ + return ploadu_partial_common(from, n, offset); +} +template<> EIGEN_ALWAYS_INLINE Packet8us ploadu_partial(const unsigned short int* from, const Index n, const Index offset) +{ + return ploadu_partial_common(from, n, offset); +} +template<> EIGEN_ALWAYS_INLINE Packet8bf ploadu_partial(const bfloat16* from, const Index n, const Index offset) +{ + return ploadu_partial_common(reinterpret_cast(from), n, offset); +} +template<> EIGEN_ALWAYS_INLINE Packet16c ploadu_partial(const signed char* from, const Index n, const Index offset) +{ + return ploadu_partial_common(from, n, offset); +} +template<> EIGEN_ALWAYS_INLINE Packet16uc ploadu_partial(const unsigned char* from, const Index n, const Index offset) +{ + return ploadu_partial_common(from, n, offset); +} + template EIGEN_STRONG_INLINE Packet ploaddup_common(const __UNPACK_TYPE__(Packet)* from) { Packet p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); - return vec_perm(p, p, p16uc_DUPLICATE32_HI); + return vec_mergeh(p, p); } template<> EIGEN_STRONG_INLINE Packet4f ploaddup(const float* from) { @@ -1015,7 +1348,7 @@ template<> EIGEN_STRONG_INLINE Packet8s ploaddup(const short int* Packet8s p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); - return vec_perm(p, p, p16uc_DUPLICATE16_HI); + return vec_mergeh(p, p); } template<> EIGEN_STRONG_INLINE Packet8us ploaddup(const unsigned short int* from) @@ -1023,7 +1356,7 @@ template<> EIGEN_STRONG_INLINE Packet8us ploaddup(const unsigned shor Packet8us p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); - return vec_perm(p, p, p16uc_DUPLICATE16_HI); + return vec_mergeh(p, p); } template<> EIGEN_STRONG_INLINE Packet8s ploadquad(const short int* from) @@ -1052,7 +1385,7 @@ template<> EIGEN_STRONG_INLINE Packet16c ploaddup(const signed char* Packet16c p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); - return vec_perm(p, p, p16uc_DUPLICATE8_HI); + return vec_mergeh(p, p); } template<> EIGEN_STRONG_INLINE Packet16uc ploaddup(const unsigned char* from) @@ -1060,13 +1393,15 @@ template<> EIGEN_STRONG_INLINE Packet16uc ploaddup(const unsigned ch Packet16uc p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); - return vec_perm(p, p, p16uc_DUPLICATE8_HI); + return vec_mergeh(p, p); } template EIGEN_STRONG_INLINE void pstoreu_common(__UNPACK_TYPE__(Packet)* to, const Packet& from) { EIGEN_DEBUG_UNALIGNED_STORE -#ifdef _BIG_ENDIAN +#if defined(EIGEN_VECTORIZE_VSX) || !defined(_BIG_ENDIAN) + vec_xst(from, 0, to); +#else // Taken from http://developer.apple.com/hardwaredrivers/ve/alignment.html // Warning: not thread safe! Packet16uc MSQ, LSQ, edges; @@ -1081,8 +1416,6 @@ template EIGEN_STRONG_INLINE void pstoreu_common(__UNPACK_TYPE_ LSQ = vec_perm((Packet16uc)from,edges,align); // misalign the data (LSQ) vec_st( LSQ, 15, (unsigned char *)to ); // Store the LSQ part first vec_st( MSQ, 0, (unsigned char *)to ); // Store the MSQ part second -#else - vec_xst(from, 0, to); #endif } template<> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet4f& from) @@ -1103,7 +1436,7 @@ template<> EIGEN_STRONG_INLINE void pstoreu(unsigned short i } template<> EIGEN_STRONG_INLINE void pstoreu(bfloat16* to, const Packet8bf& from) { - pstoreu_common(reinterpret_cast(to), from); + pstoreu_common(reinterpret_cast(to), from.m_val); } template<> EIGEN_STRONG_INLINE void pstoreu(signed char* to, const Packet16c& from) { @@ -1114,6 +1447,69 @@ template<> EIGEN_STRONG_INLINE void pstoreu(unsigned char* t pstoreu_common(to, from); } +template EIGEN_ALWAYS_INLINE void pstoreu_partial_common(__UNPACK_TYPE__(Packet)* to, const Packet& from, const Index n, const Index offset) +{ + const Index packet_size = unpacket_traits::size; + eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet"); + const Index size = sizeof(__UNPACK_TYPE__(Packet)); +#ifdef _ARCH_PWR9 + EIGEN_UNUSED_VARIABLE(packet_size); + EIGEN_DEBUG_UNALIGNED_STORE + Packet store = from; + if (offset) { + Packet16uc shift = pset1(offset * 8 * size); +#ifdef _BIG_ENDIAN + store = Packet(vec_slo(Packet16uc(store), shift)); +#else + store = Packet(vec_sro(Packet16uc(store), shift)); +#endif + } + vec_xst_len(store, to, n * size); +#else + if (n) { + EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) store[packet_size]; + pstore(store, from); + unsigned char* store2 = reinterpret_cast(store + offset); + unsigned char* to2 = reinterpret_cast(to); + Index n2 = n * size; + if (16 <= n2) { + pstoreu(to2, ploadu(store2)); + } else { + memcpy((void *)to2, (void *)store2, n2); + } + } +#endif +} + +template<> EIGEN_ALWAYS_INLINE void pstoreu_partial(float* to, const Packet4f& from, const Index n, const Index offset) +{ + pstoreu_partial_common(to, from, n, offset); +} +template<> EIGEN_ALWAYS_INLINE void pstoreu_partial(int* to, const Packet4i& from, const Index n, const Index offset) +{ + pstoreu_partial_common(to, from, n, offset); +} +template<> EIGEN_ALWAYS_INLINE void pstoreu_partial(short int* to, const Packet8s& from, const Index n, const Index offset) +{ + pstoreu_partial_common(to, from, n, offset); +} +template<> EIGEN_ALWAYS_INLINE void pstoreu_partial(unsigned short int* to, const Packet8us& from, const Index n, const Index offset) +{ + pstoreu_partial_common(to, from, n, offset); +} +template<> EIGEN_ALWAYS_INLINE void pstoreu_partial(bfloat16* to, const Packet8bf& from, const Index n, const Index offset) +{ + pstoreu_partial_common(reinterpret_cast(to), from, n, offset); +} +template<> EIGEN_ALWAYS_INLINE void pstoreu_partial(signed char* to, const Packet16c& from, const Index n, const Index offset) +{ + pstoreu_partial_common(to, from, n, offset); +} +template<> EIGEN_ALWAYS_INLINE void pstoreu_partial(unsigned char* to, const Packet16uc& from, const Index n, const Index offset) +{ + pstoreu_partial_common(to, from, n, offset); +} + template<> EIGEN_STRONG_INLINE void prefetch(const float* addr) { EIGEN_PPC_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE void prefetch(const int* addr) { EIGEN_PPC_PREFETCH(addr); } @@ -1162,11 +1558,19 @@ template<> EIGEN_STRONG_INLINE Packet8us preverse(const Packet8us& a) } template<> EIGEN_STRONG_INLINE Packet16c preverse(const Packet16c& a) { +#ifdef _ARCH_PWR9 + return vec_revb(a); +#else return vec_perm(a, a, p16uc_REVERSE8); +#endif } template<> EIGEN_STRONG_INLINE Packet16uc preverse(const Packet16uc& a) { +#ifdef _ARCH_PWR9 + return vec_revb(a); +#else return vec_perm(a, a, p16uc_REVERSE8); +#endif } template<> EIGEN_STRONG_INLINE Packet8bf preverse(const Packet8bf& a) { @@ -1180,10 +1584,13 @@ template<> EIGEN_STRONG_INLINE Packet8us pabs(const Packet8us& a) { return a; } template<> EIGEN_STRONG_INLINE Packet16c pabs(const Packet16c& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE Packet16uc pabs(const Packet16uc& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8bf pabs(const Packet8bf& a) { - _EIGEN_DECLARE_CONST_FAST_Packet8us(abs_mask,0x7FFF); + EIGEN_DECLARE_CONST_FAST_Packet8us(abs_mask,0x7FFF); return pand(p8us_abs_mask, a); } +template<> EIGEN_STRONG_INLINE Packet8bf psignbit(const Packet8bf& a) { return vec_sra(a.m_val, vec_splat_u16(15)); } +template<> EIGEN_STRONG_INLINE Packet4f psignbit(const Packet4f& a) { return (Packet4f)vec_sra((Packet4i)a, vec_splats((unsigned int)(31))); } + template EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i& a) { return vec_sra(a,reinterpret_cast(pset1(N))); } template EIGEN_STRONG_INLINE Packet4i plogical_shift_right(const Packet4i& a) @@ -1192,38 +1599,38 @@ template EIGEN_STRONG_INLINE Packet4i plogical_shift_left(const Packet4i& { return vec_sl(a,reinterpret_cast(pset1(N))); } template EIGEN_STRONG_INLINE Packet4f plogical_shift_left(const Packet4f& a) { - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); + const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); Packet4ui r = vec_sl(reinterpret_cast(a), p4ui_mask); return reinterpret_cast(r); } template EIGEN_STRONG_INLINE Packet4f plogical_shift_right(const Packet4f& a) { - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); + const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); Packet4ui r = vec_sr(reinterpret_cast(a), p4ui_mask); return reinterpret_cast(r); } template EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(const Packet4ui& a) { - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); + const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); return vec_sr(a, p4ui_mask); } template EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(const Packet4ui& a) { - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); + const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); return vec_sl(a, p4ui_mask); } template EIGEN_STRONG_INLINE Packet8us plogical_shift_left(const Packet8us& a) { - const _EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N); + const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N); return vec_sl(a, p8us_mask); } template EIGEN_STRONG_INLINE Packet8us plogical_shift_right(const Packet8us& a) { - const _EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N); + const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N); return vec_sr(a, p8us_mask); } @@ -1232,69 +1639,248 @@ EIGEN_STRONG_INLINE Packet4f Bf16ToF32Even(const Packet8bf& bf){ } EIGEN_STRONG_INLINE Packet4f Bf16ToF32Odd(const Packet8bf& bf){ - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(high_mask, 0xFFFF0000); + const EIGEN_DECLARE_CONST_FAST_Packet4ui(high_mask, 0xFFFF0000); return pand( reinterpret_cast(bf.m_val), reinterpret_cast(p4ui_high_mask) ); } +EIGEN_ALWAYS_INLINE Packet8us pmerge(Packet4ui even, Packet4ui odd) { +#ifdef _BIG_ENDIAN + return vec_perm(reinterpret_cast(odd), reinterpret_cast(even), p16uc_MERGEO16); +#else + return vec_perm(reinterpret_cast(even), reinterpret_cast(odd), p16uc_MERGEE16); +#endif +} + // Simple interleaving of bool masks, prevents true values from being // converted to NaNs. EIGEN_STRONG_INLINE Packet8bf F32ToBf16Bool(Packet4f even, Packet4f odd) { - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(high_mask, 0xFFFF0000); - Packet4f bf_odd, bf_even; - bf_odd = pand(reinterpret_cast(p4ui_high_mask), odd); - bf_even = plogical_shift_right<16>(even); - return reinterpret_cast(por(bf_even, bf_odd)); + return pmerge(reinterpret_cast(even), reinterpret_cast(odd)); } +//#define SUPPORT_BF16_SUBNORMALS + +#ifndef __VEC_CLASS_FP_NAN +#define __VEC_CLASS_FP_NAN (1<<6) +#endif + +#if defined(SUPPORT_BF16_SUBNORMALS) && !defined(__VEC_CLASS_FP_SUBNORMAL) +#define __VEC_CLASS_FP_SUBNORMAL_P (1<<1) +#define __VEC_CLASS_FP_SUBNORMAL_N (1<<0) + +#define __VEC_CLASS_FP_SUBNORMAL (__VEC_CLASS_FP_SUBNORMAL_P | __VEC_CLASS_FP_SUBNORMAL_N) +#endif + EIGEN_STRONG_INLINE Packet8bf F32ToBf16(Packet4f p4f){ +#ifdef _ARCH_PWR10 + return reinterpret_cast(__builtin_vsx_xvcvspbf16(reinterpret_cast(p4f))); +#else Packet4ui input = reinterpret_cast(p4f); Packet4ui lsb = plogical_shift_right<16>(input); lsb = pand(lsb, reinterpret_cast(p4i_ONE)); - _EIGEN_DECLARE_CONST_FAST_Packet4ui(BIAS,0x7FFFu); + EIGEN_DECLARE_CONST_FAST_Packet4ui(BIAS,0x7FFFu); Packet4ui rounding_bias = padd(lsb, p4ui_BIAS); input = padd(input, rounding_bias); - //Test NaN and Subnormal - Begin - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(exp_mask, 0x7F800000); + const EIGEN_DECLARE_CONST_FAST_Packet4ui(nan, 0x7FC00000); +#ifdef _ARCH_PWR9 + Packet4bi nan_selector = vec_test_data_class(p4f, __VEC_CLASS_FP_NAN); + input = vec_sel(input, p4ui_nan, nan_selector); + +#ifdef SUPPORT_BF16_SUBNORMALS + Packet4bi subnormal_selector = vec_test_data_class(p4f, __VEC_CLASS_FP_SUBNORMAL); + input = vec_sel(input, reinterpret_cast(p4f), subnormal_selector); +#endif +#else +#ifdef SUPPORT_BF16_SUBNORMALS + //Test NaN and Subnormal + const EIGEN_DECLARE_CONST_FAST_Packet4ui(exp_mask, 0x7F800000); Packet4ui exp = pand(p4ui_exp_mask, reinterpret_cast(p4f)); - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(mantissa_mask, 0x7FFFFF); + const EIGEN_DECLARE_CONST_FAST_Packet4ui(mantissa_mask, 0x7FFFFF); Packet4ui mantissa = pand(p4ui_mantissa_mask, reinterpret_cast(p4f)); - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(max_exp, 0x7F800000); - Packet4bi is_max_exp = vec_cmpeq(exp, p4ui_max_exp); - Packet4bi is_zero_exp = vec_cmpeq(exp, reinterpret_cast(p4i_ZERO)); - + Packet4bi is_max_exp = vec_cmpeq(exp, p4ui_exp_mask); Packet4bi is_mant_zero = vec_cmpeq(mantissa, reinterpret_cast(p4i_ZERO)); + Packet4ui nan_selector = pandnot( reinterpret_cast(is_max_exp), reinterpret_cast(is_mant_zero) ); + Packet4bi is_zero_exp = vec_cmpeq(exp, reinterpret_cast(p4i_ZERO)); + Packet4ui subnormal_selector = pandnot( reinterpret_cast(is_zero_exp), reinterpret_cast(is_mant_zero) ); - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(nan, 0x7FC00000); input = vec_sel(input, p4ui_nan, nan_selector); input = vec_sel(input, reinterpret_cast(p4f), subnormal_selector); - //Test NaN and Subnormal - End +#else + //Test only NaN + Packet4bi nan_selector = vec_cmpeq(p4f, p4f); + + input = vec_sel(p4ui_nan, input, nan_selector); +#endif +#endif input = plogical_shift_right<16>(input); return reinterpret_cast(input); +#endif } +#ifdef _BIG_ENDIAN +/** + * Pack the high portion of two float Packets into one bfloat16 Packet + * + * @param lohi to expect either a low & high OR odd & even order + */ +template +EIGEN_ALWAYS_INLINE Packet8bf Bf16PackHigh(Packet4f lo, Packet4f hi) +{ + if (lohi) { + return vec_perm(reinterpret_cast(lo), reinterpret_cast(hi), p16uc_MERGEH16); + } else { + return vec_perm(reinterpret_cast(hi), reinterpret_cast(lo), p16uc_MERGEE16); + } +} + +/** + * Pack the low portion of two float Packets into one bfloat16 Packet + * + * @param lohi to expect either a low & high OR odd & even order + */ +template +EIGEN_ALWAYS_INLINE Packet8bf Bf16PackLow(Packet4f lo, Packet4f hi) +{ + if (lohi) { + return vec_pack(reinterpret_cast(lo), reinterpret_cast(hi)); + } else { + return vec_perm(reinterpret_cast(hi), reinterpret_cast(lo), p16uc_MERGEO16); + } +} +#else +template +EIGEN_ALWAYS_INLINE Packet8bf Bf16PackLow(Packet4f hi, Packet4f lo) +{ + if (lohi) { + return vec_pack(reinterpret_cast(hi), reinterpret_cast(lo)); + } else { + return vec_perm(reinterpret_cast(hi), reinterpret_cast(lo), p16uc_MERGEE16); + } +} + +template +EIGEN_ALWAYS_INLINE Packet8bf Bf16PackHigh(Packet4f hi, Packet4f lo) +{ + if (lohi) { + return vec_perm(reinterpret_cast(hi), reinterpret_cast(lo), p16uc_MERGEL16); + } else { + return vec_perm(reinterpret_cast(hi), reinterpret_cast(lo), p16uc_MERGEO16); + } +} +#endif + +/** + * Convert and pack two float Packets into one bfloat16 Packet + * + * @param lohi to expect either a low & high OR odd & even order + */ +template +EIGEN_ALWAYS_INLINE Packet8bf F32ToBf16Two(Packet4f lo, Packet4f hi) +{ + Packet8us p4f = Bf16PackHigh(lo, hi); + Packet8us p4f2 = Bf16PackLow(lo, hi); + + Packet8us lsb = pand(p4f, p8us_ONE); + EIGEN_DECLARE_CONST_FAST_Packet8us(BIAS,0x7FFFu); + lsb = padd(lsb, p8us_BIAS); + lsb = padd(lsb, p4f2); + + Packet8bi rounding_bias = vec_cmplt(lsb, p4f2); + Packet8us input = psub(p4f, reinterpret_cast(rounding_bias)); + +#ifdef _ARCH_PWR9 + Packet4bi nan_selector_lo = vec_test_data_class(lo, __VEC_CLASS_FP_NAN); + Packet4bi nan_selector_hi = vec_test_data_class(hi, __VEC_CLASS_FP_NAN); + Packet8us nan_selector = Bf16PackLow(reinterpret_cast(nan_selector_lo), reinterpret_cast(nan_selector_hi)); + + input = vec_sel(input, p8us_BIAS, nan_selector); + +#ifdef SUPPORT_BF16_SUBNORMALS + Packet4bi subnormal_selector_lo = vec_test_data_class(lo, __VEC_CLASS_FP_SUBNORMAL); + Packet4bi subnormal_selector_hi = vec_test_data_class(hi, __VEC_CLASS_FP_SUBNORMAL); + Packet8us subnormal_selector = Bf16PackLow(reinterpret_cast(subnormal_selector_lo), reinterpret_cast(subnormal_selector_hi)); + + input = vec_sel(input, reinterpret_cast(p4f), subnormal_selector); +#endif +#else +#ifdef SUPPORT_BF16_SUBNORMALS + //Test NaN and Subnormal + const EIGEN_DECLARE_CONST_FAST_Packet8us(exp_mask, 0x7F80); + Packet8us exp = pand(p8us_exp_mask, p4f); + + const EIGEN_DECLARE_CONST_FAST_Packet8us(mantissa_mask, 0x7Fu); + Packet8us mantissa = pand(p8us_mantissa_mask, p4f); + + Packet8bi is_max_exp = vec_cmpeq(exp, p8us_exp_mask); + Packet8bi is_mant_zero = vec_cmpeq(mantissa, reinterpret_cast(p4i_ZERO)); + + Packet8us nan_selector = pandnot( + reinterpret_cast(is_max_exp), + reinterpret_cast(is_mant_zero) + ); + + Packet8bi is_zero_exp = vec_cmpeq(exp, reinterpret_cast(p4i_ZERO)); + + Packet8us subnormal_selector = pandnot( + reinterpret_cast(is_zero_exp), + reinterpret_cast(is_mant_zero) + ); + + // Using BIAS as NaN (since any or all of the last 7 bits can be set) + input = vec_sel(input, p8us_BIAS, nan_selector); + input = vec_sel(input, reinterpret_cast(p4f), subnormal_selector); +#else + //Test only NaN + Packet4bi nan_selector_lo = vec_cmpeq(lo, lo); + Packet4bi nan_selector_hi = vec_cmpeq(hi, hi); + Packet8us nan_selector = Bf16PackLow(reinterpret_cast(nan_selector_lo), reinterpret_cast(nan_selector_hi)); + + input = vec_sel(p8us_BIAS, input, nan_selector); +#endif +#endif + + return input; +} + +/** + * Convert and pack two float Packets into one bfloat16 Packet - low & high order + */ +EIGEN_STRONG_INLINE Packet8bf F32ToBf16Both(Packet4f lo, Packet4f hi) +{ +#ifdef _ARCH_PWR10 + Packet8bf fp16_0 = F32ToBf16(lo); + Packet8bf fp16_1 = F32ToBf16(hi); + return vec_pack(reinterpret_cast(fp16_0.m_val), reinterpret_cast(fp16_1.m_val)); +#else + return F32ToBf16Two(lo, hi); +#endif +} + +/** + * Convert and pack two float Packets into one bfloat16 Packet - odd & even order + */ EIGEN_STRONG_INLINE Packet8bf F32ToBf16(Packet4f even, Packet4f odd){ - Packet4f bf_odd, bf_even; - bf_odd = reinterpret_cast(F32ToBf16(odd).m_val); - bf_odd = plogical_shift_left<16>(bf_odd); - bf_even = reinterpret_cast(F32ToBf16(even).m_val); - return reinterpret_cast(por(bf_even, bf_odd)); +#ifdef _ARCH_PWR10 + return pmerge(reinterpret_cast(F32ToBf16(even).m_val), reinterpret_cast(F32ToBf16(odd).m_val)); +#else + return F32ToBf16Two(even, odd); +#endif } #define BF16_TO_F32_UNARY_OP_WRAPPER(OP, A) \ Packet4f a_even = Bf16ToF32Even(A);\ @@ -1334,19 +1920,14 @@ template<> EIGEN_STRONG_INLINE Packet8bf pdiv(const Packet8bf& a, con } template<> EIGEN_STRONG_INLINE Packet8bf pnegate(const Packet8bf& a) { - BF16_TO_F32_UNARY_OP_WRAPPER(pnegate, a); + EIGEN_DECLARE_CONST_FAST_Packet8us(neg_mask,0x8000); + return pxor(p8us_neg_mask, a); } template<> EIGEN_STRONG_INLINE Packet8bf psub(const Packet8bf& a, const Packet8bf& b) { BF16_TO_F32_BINARY_OP_WRAPPER(psub, a, b); } -template<> EIGEN_STRONG_INLINE Packet8bf psqrt (const Packet8bf& a){ - BF16_TO_F32_UNARY_OP_WRAPPER(vec_sqrt, a); -} -template<> EIGEN_STRONG_INLINE Packet8bf prsqrt (const Packet8bf& a){ - BF16_TO_F32_UNARY_OP_WRAPPER(prsqrt, a); -} template<> EIGEN_STRONG_INLINE Packet8bf pexp (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(pexp_float, a); } @@ -1390,9 +1971,11 @@ template<> EIGEN_STRONG_INLINE Packet8bf pceil (const Packet8bf& a){ template<> EIGEN_STRONG_INLINE Packet8bf pround (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(pround, a); } +#ifdef EIGEN_VECTORIZE_VSX template<> EIGEN_STRONG_INLINE Packet8bf print (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(print, a); } +#endif template<> EIGEN_STRONG_INLINE Packet8bf pmadd(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) { Packet4f a_even = Bf16ToF32Even(a); Packet4f a_odd = Bf16ToF32Odd(a); @@ -2100,7 +2683,7 @@ ptranspose(PacketBlock& kernel) { template EIGEN_STRONG_INLINE Packet pblend4(const Selector<4>& ifPacket, const Packet& thenPacket, const Packet& elsePacket) { Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] }; - Packet4ui mask = reinterpret_cast(vec_cmpeq(reinterpret_cast(select), reinterpret_cast(p4i_ONE))); + Packet4ui mask = reinterpret_cast(pnegate(reinterpret_cast(select))); return vec_sel(elsePacket, thenPacket, mask); } @@ -2115,7 +2698,7 @@ template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, cons template<> EIGEN_STRONG_INLINE Packet8s pblend(const Selector<8>& ifPacket, const Packet8s& thenPacket, const Packet8s& elsePacket) { Packet8us select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3], ifPacket.select[4], ifPacket.select[5], ifPacket.select[6], ifPacket.select[7] }; - Packet8us mask = reinterpret_cast(vec_cmpeq(select, p8us_ONE)); + Packet8us mask = reinterpret_cast(pnegate(reinterpret_cast(select))); Packet8s result = vec_sel(elsePacket, thenPacket, mask); return result; } @@ -2123,7 +2706,7 @@ template<> EIGEN_STRONG_INLINE Packet8s pblend(const Selector<8>& ifPacket, cons template<> EIGEN_STRONG_INLINE Packet8us pblend(const Selector<8>& ifPacket, const Packet8us& thenPacket, const Packet8us& elsePacket) { Packet8us select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3], ifPacket.select[4], ifPacket.select[5], ifPacket.select[6], ifPacket.select[7] }; - Packet8us mask = reinterpret_cast(vec_cmpeq(reinterpret_cast(select), p8us_ONE)); + Packet8us mask = reinterpret_cast(pnegate(reinterpret_cast(select))); return vec_sel(elsePacket, thenPacket, mask); } @@ -2137,7 +2720,7 @@ template<> EIGEN_STRONG_INLINE Packet16c pblend(const Selector<16>& ifPacket, co ifPacket.select[8], ifPacket.select[9], ifPacket.select[10], ifPacket.select[11], ifPacket.select[12], ifPacket.select[13], ifPacket.select[14], ifPacket.select[15] }; - Packet16uc mask = reinterpret_cast(vec_cmpeq(reinterpret_cast(select), p16uc_ONE)); + Packet16uc mask = reinterpret_cast(pnegate(reinterpret_cast(select))); return vec_sel(elsePacket, thenPacket, mask); } @@ -2147,112 +2730,13 @@ template<> EIGEN_STRONG_INLINE Packet16uc pblend(const Selector<16>& ifPacket, c ifPacket.select[8], ifPacket.select[9], ifPacket.select[10], ifPacket.select[11], ifPacket.select[12], ifPacket.select[13], ifPacket.select[14], ifPacket.select[15] }; - Packet16uc mask = reinterpret_cast(vec_cmpeq(reinterpret_cast(select), p16uc_ONE)); + Packet16uc mask = reinterpret_cast(pnegate(reinterpret_cast(select))); return vec_sel(elsePacket, thenPacket, mask); } -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; - -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; - -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; - -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; - -template<> EIGEN_STRONG_INLINE Packet4i pcast(const Packet4f& a) { - return vec_cts(a,0); -} - -template<> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4f& a) { - return vec_ctu(a,0); -} - -template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4i& a) { - return vec_ctf(a,0); -} - -template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4ui& a) { - return vec_ctf(a,0); -} - -template<> EIGEN_STRONG_INLINE Packet8us pcast(const Packet8bf& a) { - Packet4f float_even = Bf16ToF32Even(a); - Packet4f float_odd = Bf16ToF32Odd(a); - Packet4ui int_even = pcast(float_even); - Packet4ui int_odd = pcast(float_odd); - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF); - Packet4ui low_even = pand(int_even, p4ui_low_mask); - Packet4ui low_odd = pand(int_odd, p4ui_low_mask); - - //Check values that are bigger than USHRT_MAX (0xFFFF) - Packet4bi overflow_selector; - if(vec_any_gt(int_even, p4ui_low_mask)){ - overflow_selector = vec_cmpgt(int_even, p4ui_low_mask); - low_even = vec_sel(low_even, p4ui_low_mask, overflow_selector); - } - if(vec_any_gt(int_odd, p4ui_low_mask)){ - overflow_selector = vec_cmpgt(int_odd, p4ui_low_mask); - low_odd = vec_sel(low_even, p4ui_low_mask, overflow_selector); - } - - low_odd = plogical_shift_left<16>(low_odd); - - Packet4ui int_final = por(low_even, low_odd); - return reinterpret_cast(int_final); -} - -template<> EIGEN_STRONG_INLINE Packet8bf pcast(const Packet8us& a) { - //short -> int -> float -> bfloat16 - const _EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF); - Packet4ui int_cast = reinterpret_cast(a); - Packet4ui int_even = pand(int_cast, p4ui_low_mask); - Packet4ui int_odd = plogical_shift_right<16>(int_cast); - Packet4f float_even = pcast(int_even); - Packet4f float_odd = pcast(int_odd); - return F32ToBf16(float_even, float_odd); -} - - -template<> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4f& a) { - return reinterpret_cast(a); -} - -template<> EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4i& a) { - return reinterpret_cast(a); -} - - //---------- double ---------- -#ifdef __VSX__ +#ifdef EIGEN_VECTORIZE_VSX typedef __vector double Packet2d; typedef __vector unsigned long long Packet2ul; typedef __vector long long Packet2l; @@ -2262,7 +2746,6 @@ typedef Packet2ul Packet2bl; typedef __vector __bool long Packet2bl; #endif -static Packet2l p2l_ONE = { 1, 1 }; static Packet2l p2l_ZERO = reinterpret_cast(p4i_ZERO); static Packet2ul p2ul_SIGN = { 0x8000000000000000ull, 0x8000000000000000ull }; static Packet2ul p2ul_PREV0DOT5 = { 0x3FDFFFFFFFFFFFFFull, 0x3FDFFFFFFFFFFFFFull }; @@ -2290,7 +2773,6 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size=2, - HasHalfPacket = 1, HasAdd = 1, HasSub = 1, @@ -2301,10 +2783,15 @@ template<> struct packet_traits : default_packet_traits HasAbs = 1, HasSin = 0, HasCos = 0, + HasATan = 0, HasLog = 0, HasExp = 1, HasSqrt = 1, +#if !EIGEN_COMP_CLANG HasRsqrt = 1, +#else + HasRsqrt = 0, +#endif HasRound = 1, HasFloor = 1, HasCeil = 1, @@ -2345,12 +2832,22 @@ template<> EIGEN_STRONG_INLINE Packet2d pload(const double* from) return vec_xl(0, const_cast(from)); // cast needed by Clang } +template<> EIGEN_ALWAYS_INLINE Packet2d pload_partial(const double* from, const Index n, const Index offset) +{ + return pload_partial_common(from, n, offset); +} + template<> EIGEN_STRONG_INLINE void pstore(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE vec_xst(from, 0, to); } +template<> EIGEN_ALWAYS_INLINE void pstore_partial(double* to, const Packet2d& from, const Index n, const Index offset) +{ + pstore_partial_common(to, from, n, offset); +} + template<> EIGEN_STRONG_INLINE Packet2d pset1(const double& from) { Packet2d v = {from, from}; return v; @@ -2372,19 +2869,21 @@ pbroadcast4(const double *a, a3 = pset1(a[3]); } -template<> EIGEN_DEVICE_FUNC inline Packet2d pgather(const double* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2d pgather(const double* from, Index stride) { - EIGEN_ALIGN16 double af[2]; - af[0] = from[0*stride]; - af[1] = from[1*stride]; - return pload(af); + return pgather_common(from, stride); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(double* to, const Packet2d& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2d pgather_partial(const double* from, Index stride, const Index n) { - EIGEN_ALIGN16 double af[2]; - pstore(af, from); - to[0*stride] = af[0]; - to[1*stride] = af[1]; + return pgather_common(from, stride, n); +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter(double* to, const Packet2d& from, Index stride) +{ + pscatter_common(to, from, stride); +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial(double* to, const Packet2d& from, Index stride, const Index n) +{ + pscatter_common(to, from, stride, n); } template<> EIGEN_STRONG_INLINE Packet2d plset(const double& a) { return pset1(a) + p2d_COUNTDOWN; } @@ -2393,7 +2892,14 @@ template<> EIGEN_STRONG_INLINE Packet2d padd(const Packet2d& a, const template<> EIGEN_STRONG_INLINE Packet2d psub(const Packet2d& a, const Packet2d& b) { return a - b; } -template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { return p2d_ZERO - a; } +template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) +{ +#ifdef __POWER8_VECTOR__ + return vec_neg(a); +#else + return vec_xor(a, p2d_MZERO); +#endif +} template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; } @@ -2402,6 +2908,9 @@ template<> EIGEN_STRONG_INLINE Packet2d pdiv(const Packet2d& a, const // for some weird raisons, it has to be overloaded for packet of integers template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vec_madd(a, b, c); } +template<> EIGEN_STRONG_INLINE Packet2d pmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vec_msub(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet2d pnmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vec_nmsub(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet2d pnmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vec_nmadd(a,b,c); } template<> EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { @@ -2465,6 +2974,11 @@ template<> EIGEN_STRONG_INLINE Packet2d ploadu(const double* from) return vec_xl(0, const_cast(from)); } +template<> EIGEN_ALWAYS_INLINE Packet2d ploadu_partial(const double* from, const Index n, const Index offset) +{ + return ploadu_partial_common(from, n, offset); +} + template<> EIGEN_STRONG_INLINE Packet2d ploaddup(const double* from) { Packet2d p; @@ -2479,45 +2993,39 @@ template<> EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet2d& vec_xst(from, 0, to); } +template<> EIGEN_ALWAYS_INLINE void pstoreu_partial(double* to, const Packet2d& from, const Index n, const Index offset) +{ + pstoreu_partial_common(to, from, n, offset); +} + template<> EIGEN_STRONG_INLINE void prefetch(const double* addr) { EIGEN_PPC_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { EIGEN_ALIGN16 double x[2]; pstore(x, a); return x[0]; } template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { - return reinterpret_cast(vec_perm(reinterpret_cast(a), reinterpret_cast(a), p16uc_REVERSE64)); + return vec_sld(a, a, 8); } template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vec_abs(a); } - -// VSX support varies between different compilers and even different -// versions of the same compiler. For gcc version >= 4.9.3, we can use -// vec_cts to efficiently convert Packet2d to Packet2l. Otherwise, use -// a slow version that works with older compilers. -// Update: apparently vec_cts/vec_ctf intrinsics for 64-bit doubles -// are buggy, https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70963 -template<> -inline Packet2l pcast(const Packet2d& x) { -#if EIGEN_GNUC_AT_LEAST(5, 4) || \ - (EIGEN_GNUC_AT(6, 1) && __GNUC_PATCHLEVEL__ >= 1) - return vec_cts(x, 0); // TODO: check clang version. +#ifdef __POWER8_VECTOR__ +template<> EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) { return (Packet2d)vec_sra((Packet2l)a, vec_splats((unsigned long long)(63))); } +#else +#ifdef _BIG_ENDIAN +static Packet16uc p16uc_DUPSIGN = { 0,0,0,0, 0,0,0,0, 8,8,8,8, 8,8,8,8 }; #else - double tmp[2]; - memcpy(tmp, &x, sizeof(tmp)); - Packet2l l = { static_cast(tmp[0]), - static_cast(tmp[1]) }; - return l; +static Packet16uc p16uc_DUPSIGN = { 7,7,7,7, 7,7,7,7, 15,15,15,15, 15,15,15,15 }; #endif -} -template<> -inline Packet2d pcast(const Packet2l& x) { - unsigned long long tmp[2]; - memcpy(tmp, &x, sizeof(tmp)); - Packet2d d = { static_cast(tmp[0]), - static_cast(tmp[1]) }; - return d; +template<> EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) +{ + Packet16c tmp = vec_sra(reinterpret_cast(a), vec_splats((unsigned char)(7))); + return reinterpret_cast(vec_perm(tmp, tmp, p16uc_DUPSIGN)); } +#endif + +template<> inline Packet2l pcast(const Packet2d& x); +template<> inline Packet2d pcast(const Packet2l& x); // Packet2l shifts. // For POWER8 we simply use vec_sr/l. @@ -2571,7 +3079,7 @@ template struct plogical_shift_left_impl; template -struct plogical_shift_left_impl= 0)>::type> { +struct plogical_shift_left_impl= 0)>> { static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) { static const unsigned n = static_cast(N); const Packet4ui shift = {n, n, n, n}; @@ -2585,7 +3093,7 @@ struct plogical_shift_left_impl= 0)>::typ }; template -struct plogical_shift_left_impl= 32)>::type> { +struct plogical_shift_left_impl= 32)>> { static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) { static const unsigned m = static_cast(N - 32); const Packet4ui shift = {m, m, m, m}; @@ -2603,7 +3111,7 @@ template struct plogical_shift_right_impl; template -struct plogical_shift_right_impl= 0)>::type> { +struct plogical_shift_right_impl= 0)>> { static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) { static const unsigned n = static_cast(N); const Packet4ui shift = {n, n, n, n}; @@ -2617,7 +3125,7 @@ struct plogical_shift_right_impl= 0)>::ty }; template -struct plogical_shift_right_impl= 32)>::type> { +struct plogical_shift_right_impl= 32)>> { static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) { static const unsigned m = static_cast(N - 32); const Packet4ui shift = {m, m, m, m}; @@ -2690,15 +3198,15 @@ template<> EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet2d t0, t1; - t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI); - t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO); + t0 = vec_mergeh(kernel.packet[0], kernel.packet[1]); + t1 = vec_mergel(kernel.packet[0], kernel.packet[1]); kernel.packet[0] = t0; kernel.packet[1] = t1; } template<> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) { Packet2l select = { ifPacket.select[0], ifPacket.select[1] }; - Packet2bl mask = reinterpret_cast( vec_cmpeq(reinterpret_cast(select), reinterpret_cast(p2l_ONE)) ); + Packet2ul mask = reinterpret_cast(pnegate(reinterpret_cast(select))); return vec_sel(elsePacket, thenPacket, mask); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/TypeCasting.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/TypeCasting.h new file mode 100644 index 00000000000..361c69f38cc --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/AltiVec/TypeCasting.h @@ -0,0 +1,179 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2019 Rasmus Munk Larsen +// Copyright (C) 2023 Chip Kerchner (chip.kerchner@ibm.com) +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_TYPE_CASTING_ALTIVEC_H +#define EIGEN_TYPE_CASTING_ALTIVEC_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { +template <> +struct type_casting_traits { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template <> +struct type_casting_traits { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template <> +struct type_casting_traits { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template <> +struct type_casting_traits { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_STRONG_INLINE Packet4i pcast(const Packet4f& a) { + return vec_cts(a,0); +} + +template<> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4f& a) { + return vec_ctu(a,0); +} + +template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4i& a) { + return vec_ctf(a,0); +} + +template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4ui& a) { + return vec_ctf(a,0); +} + +template<> EIGEN_STRONG_INLINE Packet8us pcast(const Packet8bf& a) { + Packet4f float_even = Bf16ToF32Even(a); + Packet4f float_odd = Bf16ToF32Odd(a); + Packet4ui int_even = pcast(float_even); + Packet4ui int_odd = pcast(float_odd); + const EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF); + Packet4ui low_even = pand(int_even, p4ui_low_mask); + Packet4ui low_odd = pand(int_odd, p4ui_low_mask); + + //Check values that are bigger than USHRT_MAX (0xFFFF) + Packet4bi overflow_selector; + if(vec_any_gt(int_even, p4ui_low_mask)){ + overflow_selector = vec_cmpgt(int_even, p4ui_low_mask); + low_even = vec_sel(low_even, p4ui_low_mask, overflow_selector); + } + if(vec_any_gt(int_odd, p4ui_low_mask)){ + overflow_selector = vec_cmpgt(int_odd, p4ui_low_mask); + low_odd = vec_sel(low_even, p4ui_low_mask, overflow_selector); + } + + return pmerge(low_even, low_odd); +} + +template<> EIGEN_STRONG_INLINE Packet8bf pcast(const Packet8us& a) { + //short -> int -> float -> bfloat16 + const EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF); + Packet4ui int_cast = reinterpret_cast(a); + Packet4ui int_even = pand(int_cast, p4ui_low_mask); + Packet4ui int_odd = plogical_shift_right<16>(int_cast); + Packet4f float_even = pcast(int_even); + Packet4f float_odd = pcast(int_odd); + return F32ToBf16(float_even, float_odd); +} + +template <> +struct type_casting_traits { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 2 + }; +}; + +template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet8bf& a) { + Packet8us z = pset1(0); +#ifdef _BIG_ENDIAN + return reinterpret_cast(vec_mergeh(a.m_val, z)); +#else + return reinterpret_cast(vec_mergeh(z, a.m_val)); +#endif +} + +template <> +struct type_casting_traits { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 2, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_STRONG_INLINE Packet8bf pcast(const Packet4f& a, const Packet4f &b) { + return F32ToBf16Both(a, b); +} + +template<> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4f& a) { + return reinterpret_cast(a); +} + +template<> EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4i& a) { + return reinterpret_cast(a); +} + +#ifdef EIGEN_VECTORIZE_VSX +// VSX support varies between different compilers and even different +// versions of the same compiler. For gcc version >= 4.9.3, we can use +// vec_cts to efficiently convert Packet2d to Packet2l. Otherwise, use +// a slow version that works with older compilers. +// Update: apparently vec_cts/vec_ctf intrinsics for 64-bit doubles +// are buggy, https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70963 +template<> +inline Packet2l pcast(const Packet2d& x) { +#if EIGEN_GNUC_STRICT_AT_LEAST(7,1,0) + return vec_cts(x, 0); // TODO: check clang version. +#else + double tmp[2]; + memcpy(tmp, &x, sizeof(tmp)); + Packet2l l = { static_cast(tmp[0]), + static_cast(tmp[1]) }; + return l; +#endif +} + +template<> +inline Packet2d pcast(const Packet2l& x) { + unsigned long long tmp[2]; + memcpy(tmp, &x, sizeof(tmp)); + Packet2d d = { static_cast(tmp[0]), + static_cast(tmp[1]) }; + return d; +} +#endif + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_TYPE_CASTING_ALTIVEC_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/BFloat16.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/BFloat16.h index f21d1a0a32c..93e87140d93 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/BFloat16.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/BFloat16.h @@ -16,6 +16,21 @@ limitations under the License. #ifndef EIGEN_BFLOAT16_H #define EIGEN_BFLOAT16_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +#if defined(EIGEN_HAS_HIP_BF16) +// When compiling with GPU support, the "hip_bfloat16" base class as well as +// some other routines are defined in the GPU compiler header files +// (hip_bfloat16.h), and they are not tagged constexpr +// As a consequence, we get compile failures when compiling Eigen with +// GPU support. Hence the need to disable EIGEN_CONSTEXPR when building +// Eigen with GPU support + #pragma push_macro("EIGEN_CONSTEXPR") + #undef EIGEN_CONSTEXPR + #define EIGEN_CONSTEXPR +#endif + #define BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, METHOD) \ template <> \ EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED \ @@ -23,19 +38,47 @@ limitations under the License. return F32ToBf16(METHOD(Bf16ToF32(_x))); \ } +// Only use HIP GPU bf16 in kernels +#if defined(EIGEN_HAS_HIP_BF16) && defined(EIGEN_GPU_COMPILE_PHASE) +#define EIGEN_USE_HIP_BF16 +#endif + namespace Eigen { struct bfloat16; +namespace numext { +template <> +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::bfloat16 bit_cast(const uint16_t& src); + +template <> +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC uint16_t bit_cast(const Eigen::bfloat16& src); +} // namespace numext namespace bfloat16_impl { +#if defined(EIGEN_USE_HIP_BF16) + +struct __bfloat16_raw : public hip_bfloat16 { + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw() {} + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(hip_bfloat16 hb) : hip_bfloat16(hb) {} + explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(unsigned short raw) : hip_bfloat16(raw) {} +}; + +#else + // Make our own __bfloat16_raw definition. struct __bfloat16_raw { +#if defined(EIGEN_HAS_HIP_BF16) && !defined(EIGEN_GPU_COMPILE_PHASE) + EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw() {} +#else EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw() : value(0) {} +#endif explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(unsigned short raw) : value(raw) {} unsigned short value; }; +#endif // defined(EIGEN_USE_HIP_BF16) + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(unsigned short value); template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(float ff); @@ -83,57 +126,114 @@ struct bfloat16 : public bfloat16_impl::bfloat16_base { return bfloat16_impl::bfloat16_to_float(*this); } }; -} // namespace Eigen -namespace std { -template<> -struct numeric_limits { - static const bool is_specialized = true; - static const bool is_signed = true; - static const bool is_integer = false; - static const bool is_exact = false; - static const bool has_infinity = true; - static const bool has_quiet_NaN = true; - static const bool has_signaling_NaN = true; - static const float_denorm_style has_denorm = std::denorm_absent; - static const bool has_denorm_loss = false; - static const std::float_round_style round_style = numeric_limits::round_style; - static const bool is_iec559 = false; - static const bool is_bounded = true; - static const bool is_modulo = false; - static const int digits = 8; - static const int digits10 = 2; - static const int max_digits10 = 4; - static const int radix = 2; - static const int min_exponent = numeric_limits::min_exponent; - static const int min_exponent10 = numeric_limits::min_exponent10; - static const int max_exponent = numeric_limits::max_exponent; - static const int max_exponent10 = numeric_limits::max_exponent10; - static const bool traps = numeric_limits::traps; - static const bool tinyness_before = numeric_limits::tinyness_before; - - static Eigen::bfloat16 (min)() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0080); } - static Eigen::bfloat16 lowest() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0xff7f); } - static Eigen::bfloat16 (max)() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f7f); } - static Eigen::bfloat16 epsilon() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x3c00); } - static Eigen::bfloat16 round_error() { return Eigen::bfloat16(0x3f00); } - static Eigen::bfloat16 infinity() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f80); } - static Eigen::bfloat16 quiet_NaN() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7fc0); } - static Eigen::bfloat16 signaling_NaN() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f81); } - static Eigen::bfloat16 denorm_min() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0001); } +// TODO(majnemer): Get rid of this once we can rely on C++17 inline variables do +// solve the ODR issue. +namespace bfloat16_impl { +template +struct numeric_limits_bfloat16_impl { + static EIGEN_CONSTEXPR const bool is_specialized = true; + static EIGEN_CONSTEXPR const bool is_signed = true; + static EIGEN_CONSTEXPR const bool is_integer = false; + static EIGEN_CONSTEXPR const bool is_exact = false; + static EIGEN_CONSTEXPR const bool has_infinity = true; + static EIGEN_CONSTEXPR const bool has_quiet_NaN = true; + static EIGEN_CONSTEXPR const bool has_signaling_NaN = true; + static EIGEN_CONSTEXPR const std::float_denorm_style has_denorm = std::denorm_present; + static EIGEN_CONSTEXPR const bool has_denorm_loss = false; + static EIGEN_CONSTEXPR const std::float_round_style round_style = std::numeric_limits::round_style; + static EIGEN_CONSTEXPR const bool is_iec559 = true; + // The C++ standard defines this as "true if the set of values representable + // by the type is finite." BFloat16 has finite precision. + static EIGEN_CONSTEXPR const bool is_bounded = true; + static EIGEN_CONSTEXPR const bool is_modulo = false; + static EIGEN_CONSTEXPR const int digits = 8; + static EIGEN_CONSTEXPR const int digits10 = 2; + static EIGEN_CONSTEXPR const int max_digits10 = 4; + static EIGEN_CONSTEXPR const int radix = std::numeric_limits::radix; + static EIGEN_CONSTEXPR const int min_exponent = std::numeric_limits::min_exponent; + static EIGEN_CONSTEXPR const int min_exponent10 = std::numeric_limits::min_exponent10; + static EIGEN_CONSTEXPR const int max_exponent = std::numeric_limits::max_exponent; + static EIGEN_CONSTEXPR const int max_exponent10 = std::numeric_limits::max_exponent10; + static EIGEN_CONSTEXPR const bool traps = std::numeric_limits::traps; + // IEEE754: "The implementer shall choose how tininess is detected, but shall + // detect tininess in the same way for all operations in radix two" + static EIGEN_CONSTEXPR const bool tinyness_before = std::numeric_limits::tinyness_before; + + static EIGEN_CONSTEXPR Eigen::bfloat16 (min)() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0080); } + static EIGEN_CONSTEXPR Eigen::bfloat16 lowest() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0xff7f); } + static EIGEN_CONSTEXPR Eigen::bfloat16 (max)() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f7f); } + static EIGEN_CONSTEXPR Eigen::bfloat16 epsilon() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x3c00); } + static EIGEN_CONSTEXPR Eigen::bfloat16 round_error() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x3f00); } + static EIGEN_CONSTEXPR Eigen::bfloat16 infinity() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f80); } + static EIGEN_CONSTEXPR Eigen::bfloat16 quiet_NaN() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7fc0); } + static EIGEN_CONSTEXPR Eigen::bfloat16 signaling_NaN() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7fa0); } + static EIGEN_CONSTEXPR Eigen::bfloat16 denorm_min() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0001); } }; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::is_specialized; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::is_signed; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::is_integer; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::is_exact; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::has_infinity; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::has_quiet_NaN; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::has_signaling_NaN; +template +EIGEN_CONSTEXPR const std::float_denorm_style numeric_limits_bfloat16_impl::has_denorm; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::has_denorm_loss; +template +EIGEN_CONSTEXPR const std::float_round_style numeric_limits_bfloat16_impl::round_style; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::is_iec559; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::is_bounded; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::is_modulo; +template +EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl::digits; +template +EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl::digits10; +template +EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl::max_digits10; +template +EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl::radix; +template +EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl::min_exponent; +template +EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl::min_exponent10; +template +EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl::max_exponent; +template +EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl::max_exponent10; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::traps; +template +EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl::tinyness_before; +} // end namespace bfloat16_impl +} // end namespace Eigen + +namespace std { // If std::numeric_limits is specialized, should also specialize // std::numeric_limits, std::numeric_limits, and // std::numeric_limits // https://stackoverflow.com/a/16519653/ template<> -struct numeric_limits : numeric_limits {}; +class numeric_limits : public Eigen::bfloat16_impl::numeric_limits_bfloat16_impl<> {}; template<> -struct numeric_limits : numeric_limits {}; +class numeric_limits : public numeric_limits {}; template<> -struct numeric_limits : numeric_limits {}; -} // namespace std +class numeric_limits : public numeric_limits {}; +template<> +class numeric_limits : public numeric_limits {}; +} // end namespace std namespace Eigen { @@ -148,7 +248,7 @@ namespace bfloat16_impl { // We need to provide emulated *host-side* BF16 operators for clang. #pragma push_macro("EIGEN_DEVICE_FUNC") #undef EIGEN_DEVICE_FUNC -#if defined(EIGEN_HAS_CUDA_BF16) && defined(EIGEN_HAS_NATIVE_BF16) +#if (defined(EIGEN_HAS_GPU_BF16) && defined(EIGEN_HAS_NATIVE_BF16)) #define EIGEN_DEVICE_FUNC __host__ #else // both host and device need emulated ops. #define EIGEN_DEVICE_FUNC __host__ __device__ @@ -177,9 +277,8 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator / (const bfloat16& a, co return bfloat16(float(a) / float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator - (const bfloat16& a) { - bfloat16 result; - result.value = a.value ^ 0x8000; - return result; + numext::uint16_t x = numext::bit_cast(a) ^ 0x8000; + return numext::bit_cast(x); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator += (bfloat16& a, const bfloat16& b) { a = bfloat16(float(a) + float(b)); @@ -246,33 +345,47 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator / (const bfloat16& a, In } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw truncate_to_bfloat16(const float v) { +#if defined(EIGEN_USE_HIP_BF16) + return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(v, __bfloat16_raw::truncate)); +#else __bfloat16_raw output; - if (Eigen::numext::isnan EIGEN_NOT_A_MACRO(v)) { + if (numext::isnan EIGEN_NOT_A_MACRO(v)) { output.value = std::signbit(v) ? 0xFFC0: 0x7FC0; return output; } output.value = static_cast(numext::bit_cast(v) >> 16); return output; +#endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(numext::uint16_t value) { +#if defined(EIGEN_USE_HIP_BF16) + __bfloat16_raw bf; + bf.data = value; + return bf; +#else return __bfloat16_raw(value); +#endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR numext::uint16_t raw_bfloat16_as_uint16(const __bfloat16_raw& bf) { +#if defined(EIGEN_USE_HIP_BF16) + return bf.data; +#else return bf.value; +#endif } // float_to_bfloat16_rtne template specialization that does not make any // assumption about the value of its function argument (ff). template <> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(float ff) { -#if (defined(EIGEN_HAS_CUDA_BF16) && defined(EIGEN_HAS_HIP_BF16)) - // Nothing to do here +#if defined(EIGEN_USE_HIP_BF16) + return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(ff)); #else __bfloat16_raw output; - if (Eigen::numext::isnan EIGEN_NOT_A_MACRO(ff)) { + if (numext::isnan EIGEN_NOT_A_MACRO(ff)) { // If the value is a NaN, squash it to a qNaN with msb of fraction set, // this makes sure after truncation we don't end up with an inf. // @@ -441,8 +554,8 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(float ff) { -#if (defined(EIGEN_HAS_CUDA_BF16) && defined(EIGEN_HAS_HIP_BF16)) - // Nothing to do here +#if defined(EIGEN_USE_HIP_BF16) + return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(ff)); #else numext::uint32_t input = numext::bit_cast(ff); __bfloat16_raw output; @@ -457,29 +570,41 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(h); +#else return numext::bit_cast(static_cast(h.value) << 16); +#endif } + // --- standard functions --- EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isinf)(const bfloat16& a) { EIGEN_USING_STD(isinf); +#if defined(EIGEN_USE_HIP_BF16) + return (isinf)(a); // Uses HIP hip_bfloat16 isinf operator +#else return (isinf)(float(a)); +#endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isnan)(const bfloat16& a) { EIGEN_USING_STD(isnan); +#if defined(EIGEN_USE_HIP_BF16) + return (isnan)(a); // Uses HIP hip_bfloat16 isnan operator +#else return (isnan)(float(a)); +#endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isfinite)(const bfloat16& a) { return !(isinf EIGEN_NOT_A_MACRO (a)) && !(isnan EIGEN_NOT_A_MACRO (a)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 abs(const bfloat16& a) { - bfloat16 result; - result.value = a.value & 0x7FFF; - return result; + numext::uint16_t x = numext::bit_cast(a) & 0x7FFF; + return numext::bit_cast(x); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 exp(const bfloat16& a) { - return bfloat16(::expf(float(a))); + return bfloat16(::expf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 expm1(const bfloat16& a) { return bfloat16(numext::expm1(float(a))); @@ -497,11 +622,14 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log2(const bfloat16& a) { return bfloat16(static_cast(EIGEN_LOG2E) * ::logf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sqrt(const bfloat16& a) { - return bfloat16(::sqrtf(float(a))); + return bfloat16(::sqrtf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 pow(const bfloat16& a, const bfloat16& b) { return bfloat16(::powf(float(a), float(b))); } +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atan2(const bfloat16& a, const bfloat16& b) { + return bfloat16(::atan2f(float(a), float(b))); +} EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sin(const bfloat16& a) { return bfloat16(::sinf(float(a))); } @@ -529,7 +657,6 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 cosh(const bfloat16& a) { EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 tanh(const bfloat16& a) { return bfloat16(::tanhf(float(a))); } -#if EIGEN_HAS_CXX11_MATH EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 asinh(const bfloat16& a) { return bfloat16(::asinhf(float(a))); } @@ -539,7 +666,6 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 acosh(const bfloat16& a) { EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atanh(const bfloat16& a) { return bfloat16(::atanhf(float(a))); } -#endif EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 floor(const bfloat16& a) { return bfloat16(::floorf(float(a))); } @@ -561,6 +687,7 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 (min)(const bfloat16& a, const bf const float f2 = static_cast(b); return f2 < f1 ? b : a; } + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 (max)(const bfloat16& a, const bfloat16& b) { const float f1 = static_cast(a); const float f2 = static_cast(b); @@ -572,6 +699,7 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmin(const bfloat16& a, const bfl const float f2 = static_cast(b); return bfloat16(::fminf(f1, f2)); } + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmax(const bfloat16& a, const bfloat16& b) { const float f1 = static_cast(a); const float f2 = static_cast(b); @@ -621,7 +749,6 @@ template<> struct NumTraits } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 dummy_precision() { return bfloat16_impl::raw_uint16_to_bfloat16(0x3D4D); // bfloat16(5e-2f); - } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 highest() { return bfloat16_impl::raw_uint16_to_bfloat16(0x7F7F); @@ -639,6 +766,11 @@ template<> struct NumTraits } // namespace Eigen + +#if defined(EIGEN_HAS_HIP_BF16) + #pragma pop_macro("EIGEN_CONSTEXPR") +#endif + namespace Eigen { namespace numext { @@ -662,7 +794,7 @@ bool (isfinite)(const Eigen::bfloat16& h) { template <> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::bfloat16 bit_cast(const uint16_t& src) { - return Eigen::bfloat16(Eigen::bfloat16_impl::raw_uint16_to_bfloat16(src)); + return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(src); } template <> @@ -684,5 +816,49 @@ struct hash { } // namespace std #endif +// Add the missing shfl* intrinsics. +// The __shfl* functions are only valid on HIP or _CUDA_ARCH_ >= 300. +// CUDA defines them for (__CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__)) +// +// HIP and CUDA prior to SDK 9.0 define +// __shfl, __shfl_up, __shfl_down, __shfl_xor for int and float +// CUDA since 9.0 deprecates those and instead defines +// __shfl_sync, __shfl_up_sync, __shfl_down_sync, __shfl_xor_sync, +// with native support for __half and __nv_bfloat16 +// +// Note that the following are __device__ - only functions. +#if defined(EIGEN_HIPCC) + +#if defined(EIGEN_HAS_HIP_BF16) + +__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl(Eigen::bfloat16 var, int srcLane, int width=warpSize) { + const int ivar = static_cast(Eigen::numext::bit_cast(var)); + return Eigen::numext::bit_cast(static_cast(__shfl(ivar, srcLane, width))); +} + +__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_up(Eigen::bfloat16 var, unsigned int delta, int width=warpSize) { + const int ivar = static_cast(Eigen::numext::bit_cast(var)); + return Eigen::numext::bit_cast(static_cast(__shfl_up(ivar, delta, width))); +} + +__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_down(Eigen::bfloat16 var, unsigned int delta, int width=warpSize) { + const int ivar = static_cast(Eigen::numext::bit_cast(var)); + return Eigen::numext::bit_cast(static_cast(__shfl_down(ivar, delta, width))); +} + +__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_xor(Eigen::bfloat16 var, int laneMask, int width=warpSize) { + const int ivar = static_cast(Eigen::numext::bit_cast(var)); + return Eigen::numext::bit_cast(static_cast(__shfl_xor(ivar, laneMask, width))); +} + +#endif // HIP + +#endif // __shfl* + +#if defined(EIGEN_HIPCC) +EIGEN_STRONG_INLINE __device__ Eigen::bfloat16 __ldg(const Eigen::bfloat16* ptr) { + return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(__ldg(Eigen::numext::bit_cast(ptr))); +} +#endif // __ldg #endif // EIGEN_BFLOAT16_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/ConjHelper.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/ConjHelper.h index 53830b5a274..84da47f918c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/ConjHelper.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/ConjHelper.h @@ -38,6 +38,9 @@ } \ }; +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h index c9fbaf68b49..3d4a2a5d1ae 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h @@ -16,6 +16,9 @@ #ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_H #define EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -26,11 +29,11 @@ template<> struct make_integer { typedef numext::int64_t type; }; template<> struct make_integer { typedef numext::int16_t type; }; template<> struct make_integer { typedef numext::int16_t type; }; -template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC +template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pfrexp_generic_get_biased_exponent(const Packet& a) { typedef typename unpacket_traits::type Scalar; typedef typename unpacket_traits::integer_packet PacketI; - enum { mantissa_bits = numext::numeric_limits::digits - 1}; + static constexpr int mantissa_bits = numext::numeric_limits::digits - 1; return pcast(plogical_shift_right(preinterpret(pabs(a)))); } @@ -40,42 +43,41 @@ template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pfrexp_generic(const Packet& a, Packet& exponent) { typedef typename unpacket_traits::type Scalar; typedef typename make_unsigned::type>::type ScalarUI; - enum { + static constexpr int TotalBits = sizeof(Scalar) * CHAR_BIT, MantissaBits = numext::numeric_limits::digits - 1, - ExponentBits = int(TotalBits) - int(MantissaBits) - 1 - }; + ExponentBits = TotalBits - MantissaBits - 1; - EIGEN_CONSTEXPR ScalarUI scalar_sign_mantissa_mask = - ~(((ScalarUI(1) << int(ExponentBits)) - ScalarUI(1)) << int(MantissaBits)); // ~0x7f800000 - const Packet sign_mantissa_mask = pset1frombits(static_cast(scalar_sign_mantissa_mask)); + EIGEN_CONSTEXPR ScalarUI scalar_sign_mantissa_mask = + ~(((ScalarUI(1) << ExponentBits) - ScalarUI(1)) << MantissaBits); // ~0x7f800000 + const Packet sign_mantissa_mask = pset1frombits(static_cast(scalar_sign_mantissa_mask)); const Packet half = pset1(Scalar(0.5)); const Packet zero = pzero(a); const Packet normal_min = pset1((numext::numeric_limits::min)()); // Minimum normal value, 2^-126 - + // To handle denormals, normalize by multiplying by 2^(int(MantissaBits)+1). const Packet is_denormal = pcmp_lt(pabs(a), normal_min); - EIGEN_CONSTEXPR ScalarUI scalar_normalization_offset = ScalarUI(int(MantissaBits) + 1); // 24 + EIGEN_CONSTEXPR ScalarUI scalar_normalization_offset = ScalarUI(MantissaBits + 1); // 24 // The following cannot be constexpr because bfloat16(uint16_t) is not constexpr. const Scalar scalar_normalization_factor = Scalar(ScalarUI(1) << int(scalar_normalization_offset)); // 2^24 - const Packet normalization_factor = pset1(scalar_normalization_factor); + const Packet normalization_factor = pset1(scalar_normalization_factor); const Packet normalized_a = pselect(is_denormal, pmul(a, normalization_factor), a); - + // Determine exponent offset: -126 if normal, -126-24 if denormal - const Scalar scalar_exponent_offset = -Scalar((ScalarUI(1)<<(int(ExponentBits)-1)) - ScalarUI(2)); // -126 + const Scalar scalar_exponent_offset = -Scalar((ScalarUI(1)<<(ExponentBits-1)) - ScalarUI(2)); // -126 Packet exponent_offset = pset1(scalar_exponent_offset); const Packet normalization_offset = pset1(-Scalar(scalar_normalization_offset)); // -24 exponent_offset = pselect(is_denormal, padd(exponent_offset, normalization_offset), exponent_offset); - + // Determine exponent and mantissa from normalized_a. exponent = pfrexp_generic_get_biased_exponent(normalized_a); // Zero, Inf and NaN return 'a' unmodified, exponent is zero // (technically the exponent is unspecified for inf/NaN, but GCC/Clang set it to zero) - const Scalar scalar_non_finite_exponent = Scalar((ScalarUI(1) << int(ExponentBits)) - ScalarUI(1)); // 255 + const Scalar scalar_non_finite_exponent = Scalar((ScalarUI(1) << ExponentBits) - ScalarUI(1)); // 255 const Packet non_finite_exponent = pset1(scalar_non_finite_exponent); const Packet is_zero_or_not_finite = por(pcmp_eq(a, zero), pcmp_eq(exponent, non_finite_exponent)); const Packet m = pselect(is_zero_or_not_finite, a, por(pand(normalized_a, sign_mantissa_mask), half)); - exponent = pselect(is_zero_or_not_finite, zero, padd(exponent, exponent_offset)); + exponent = pselect(is_zero_or_not_finite, zero, padd(exponent, exponent_offset)); return m; } @@ -108,25 +110,24 @@ Packet pldexp_generic(const Packet& a, const Packet& exponent) { typedef typename unpacket_traits::integer_packet PacketI; typedef typename unpacket_traits::type Scalar; typedef typename unpacket_traits::type ScalarI; - enum { + static constexpr int TotalBits = sizeof(Scalar) * CHAR_BIT, MantissaBits = numext::numeric_limits::digits - 1, - ExponentBits = int(TotalBits) - int(MantissaBits) - 1 - }; + ExponentBits = TotalBits - MantissaBits - 1; - const Packet max_exponent = pset1(Scalar((ScalarI(1)<((ScalarI(1)<<(int(ExponentBits)-1)) - ScalarI(1)); // 127 + const Packet max_exponent = pset1(Scalar((ScalarI(1)<((ScalarI(1)<<(ExponentBits-1)) - ScalarI(1)); // 127 const PacketI e = pcast(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); PacketI b = parithmetic_shift_right<2>(e); // floor(e/4); - Packet c = preinterpret(plogical_shift_left(padd(b, bias))); // 2^b + Packet c = preinterpret(plogical_shift_left(padd(b, bias))); // 2^b Packet out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b) b = psub(psub(psub(e, b), b), b); // e - 3b - c = preinterpret(plogical_shift_left(padd(b, bias))); // 2^(e-3*b) + c = preinterpret(plogical_shift_left(padd(b, bias))); // 2^(e-3*b) out = pmul(out, c); return out; } -// Explicitly multiplies +// Explicitly multiplies // a * (2^e) // clamping e to the range // [NumTraits::min_exponent()-2, NumTraits::max_exponent()] @@ -140,20 +141,19 @@ struct pldexp_fast_impl { typedef typename unpacket_traits::integer_packet PacketI; typedef typename unpacket_traits::type Scalar; typedef typename unpacket_traits::type ScalarI; - enum { + static constexpr int TotalBits = sizeof(Scalar) * CHAR_BIT, MantissaBits = numext::numeric_limits::digits - 1, - ExponentBits = int(TotalBits) - int(MantissaBits) - 1 - }; - + ExponentBits = TotalBits - MantissaBits - 1; + static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet run(const Packet& a, const Packet& exponent) { - const Packet bias = pset1(Scalar((ScalarI(1)<<(int(ExponentBits)-1)) - ScalarI(1))); // 127 - const Packet limit = pset1(Scalar((ScalarI(1)<(Scalar((ScalarI(1)<<(ExponentBits-1)) - ScalarI(1))); // 127 + const Packet limit = pset1(Scalar((ScalarI(1)<(pmin(pmax(padd(exponent, bias), pzero(limit)), limit)); // exponent + 127 // return a * (2^e) - return pmul(a, preinterpret(plogical_shift_left(e))); + return pmul(a, preinterpret(plogical_shift_left(e))); } }; @@ -165,36 +165,16 @@ struct pldexp_fast_impl { // polynomial interpolants -> ... -> profit! template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet plog_impl_float(const Packet _x) { - Packet x = _x; - const Packet cst_1 = pset1(1.0f); - const Packet cst_neg_half = pset1(-0.5f); - // The smallest non denormalized float number. - const Packet cst_min_norm_pos = pset1frombits( 0x00800000u); - const Packet cst_minus_inf = pset1frombits( 0xff800000u); - const Packet cst_pos_inf = pset1frombits( 0x7f800000u); + const Packet cst_minus_inf = pset1frombits(static_cast(0xff800000u)); + const Packet cst_pos_inf = pset1frombits(static_cast(0x7f800000u)); - // Polynomial coefficients. const Packet cst_cephes_SQRTHF = pset1(0.707106781186547524f); - const Packet cst_cephes_log_p0 = pset1(7.0376836292E-2f); - const Packet cst_cephes_log_p1 = pset1(-1.1514610310E-1f); - const Packet cst_cephes_log_p2 = pset1(1.1676998740E-1f); - const Packet cst_cephes_log_p3 = pset1(-1.2420140846E-1f); - const Packet cst_cephes_log_p4 = pset1(+1.4249322787E-1f); - const Packet cst_cephes_log_p5 = pset1(-1.6668057665E-1f); - const Packet cst_cephes_log_p6 = pset1(+2.0000714765E-1f); - const Packet cst_cephes_log_p7 = pset1(-2.4999993993E-1f); - const Packet cst_cephes_log_p8 = pset1(+3.3333331174E-1f); - - // Truncate input values to the minimum positive normal. - x = pmax(x, cst_min_norm_pos); - - Packet e; + Packet e, x; // extract significant in the range [0.5,1) and exponent - x = pfrexp(x,e); + x = pfrexp(_x,e); // part2: Shift the inputs from the range [0.5,1) to [sqrt(1/2),sqrt(2)) // and shift by -1. The values are then centered around 0, which improves @@ -209,24 +189,22 @@ Packet plog_impl_float(const Packet _x) e = psub(e, pand(cst_1, mask)); x = padd(x, tmp); - Packet x2 = pmul(x, x); - Packet x3 = pmul(x2, x); - - // Evaluate the polynomial approximant of degree 8 in three parts, probably - // to improve instruction-level parallelism. - Packet y, y1, y2; - y = pmadd(cst_cephes_log_p0, x, cst_cephes_log_p1); - y1 = pmadd(cst_cephes_log_p3, x, cst_cephes_log_p4); - y2 = pmadd(cst_cephes_log_p6, x, cst_cephes_log_p7); - y = pmadd(y, x, cst_cephes_log_p2); - y1 = pmadd(y1, x, cst_cephes_log_p5); - y2 = pmadd(y2, x, cst_cephes_log_p8); - y = pmadd(y, x3, y1); - y = pmadd(y, x3, y2); - y = pmul(y, x3); - - y = pmadd(cst_neg_half, x2, y); - x = padd(x, y); + // Polynomial coefficients for rational (3,3) r(x) = p(x)/q(x) + // approximating log(1+x) on [sqrt(0.5)-1;sqrt(2)-1]. + const Packet cst_p1 = pset1(1.0000000190281136f); + const Packet cst_p2 = pset1(1.0000000190281063f); + const Packet cst_p3 = pset1(0.18256296349849254f); + const Packet cst_q1 = pset1(1.4999999999999927f); + const Packet cst_q2 = pset1(0.59923249590823520f); + const Packet cst_q3 = pset1(0.049616247954120038f); + + Packet p = pmadd(x, cst_p3, cst_p2); + p = pmadd(x, p, cst_p1); + p = pmul(x, p); + Packet q = pmadd(x, cst_q3, cst_q2); + q = pmadd(x, q, cst_q1); + q = pmadd(x, q, cst_1); + x = pdiv(p, q); // Add the logarithm of the exponent back to the result of the interpolation. if (base2) { @@ -250,7 +228,6 @@ Packet plog_impl_float(const Packet _x) template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet plog_float(const Packet _x) { return plog_impl_float(_x); @@ -258,7 +235,6 @@ Packet plog_float(const Packet _x) template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet plog2_float(const Packet _x) { return plog_impl_float(_x); @@ -275,15 +251,12 @@ Packet plog2_float(const Packet _x) */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet plog_impl_double(const Packet _x) { Packet x = _x; const Packet cst_1 = pset1(1.0); const Packet cst_neg_half = pset1(-0.5); - // The smallest non denormalized double. - const Packet cst_min_norm_pos = pset1frombits( static_cast(0x0010000000000000ull)); const Packet cst_minus_inf = pset1frombits( static_cast(0xfff0000000000000ull)); const Packet cst_pos_inf = pset1frombits( static_cast(0x7ff0000000000000ull)); @@ -305,9 +278,6 @@ Packet plog_impl_double(const Packet _x) const Packet cst_cephes_log_q4 = pset1(7.11544750618563894466E1); const Packet cst_cephes_log_q5 = pset1(2.31251620126765340583E1); - // Truncate input values to the minimum positive normal. - x = pmax(x, cst_min_norm_pos); - Packet e; // extract significant in the range [0.5,1) and exponent x = pfrexp(x,e); @@ -371,7 +341,6 @@ Packet plog_impl_double(const Packet _x) template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet plog_double(const Packet _x) { return plog_impl_double(_x); @@ -379,7 +348,6 @@ Packet plog_double(const Packet _x) template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet plog2_double(const Packet _x) { return plog_impl_double(_x); @@ -433,26 +401,27 @@ Packet generic_expm1(const Packet& x) // Exponential function. Works by writing "x = m*log(2) + r" where // "m = floor(x/log(2)+1/2)" and "r" is the remainder. The result is then // "exp(x) = 2^m*exp(r)" where exp(r) is in the range [-1,1). +// exp(r) is computed using a 6th order minimax polynomial approximation. template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet pexp_float(const Packet _x) { - const Packet cst_1 = pset1(1.0f); + const Packet cst_zero = pset1(0.0f); + const Packet cst_one = pset1(1.0f); const Packet cst_half = pset1(0.5f); - const Packet cst_exp_hi = pset1( 88.723f); - const Packet cst_exp_lo = pset1(-88.723f); + const Packet cst_exp_hi = pset1(88.723f); + const Packet cst_exp_lo = pset1(-104.f); const Packet cst_cephes_LOG2EF = pset1(1.44269504088896341f); - const Packet cst_cephes_exp_p0 = pset1(1.9875691500E-4f); - const Packet cst_cephes_exp_p1 = pset1(1.3981999507E-3f); - const Packet cst_cephes_exp_p2 = pset1(8.3334519073E-3f); - const Packet cst_cephes_exp_p3 = pset1(4.1665795894E-2f); - const Packet cst_cephes_exp_p4 = pset1(1.6666665459E-1f); - const Packet cst_cephes_exp_p5 = pset1(5.0000001201E-1f); + const Packet cst_p2 = pset1(0.49999988079071044921875f); + const Packet cst_p3 = pset1(0.16666518151760101318359375f); + const Packet cst_p4 = pset1(4.166965186595916748046875e-2f); + const Packet cst_p5 = pset1(8.36894474923610687255859375e-3f); + const Packet cst_p6 = pset1(1.37449637986719608306884765625e-3f); // Clamp x. - Packet x = pmax(pmin(_x, cst_exp_hi), cst_exp_lo); + Packet zero_mask = pcmp_lt(_x, cst_exp_lo); + Packet x = pmin(_x, cst_exp_hi); // Express exp(x) as exp(m*ln(2) + r), start by extracting // m = floor(x/ln(2) + 0.5). @@ -466,31 +435,27 @@ Packet pexp_float(const Packet _x) Packet r = pmadd(m, cst_cephes_exp_C1, x); r = pmadd(m, cst_cephes_exp_C2, r); - Packet r2 = pmul(r, r); - Packet r3 = pmul(r2, r); - - // Evaluate the polynomial approximant,improved by instruction-level parallelism. - Packet y, y1, y2; - y = pmadd(cst_cephes_exp_p0, r, cst_cephes_exp_p1); - y1 = pmadd(cst_cephes_exp_p3, r, cst_cephes_exp_p4); - y2 = padd(r, cst_1); - y = pmadd(y, r, cst_cephes_exp_p2); - y1 = pmadd(y1, r, cst_cephes_exp_p5); - y = pmadd(y, r3, y1); - y = pmadd(y, r2, y2); + // Evaluate the 6th order polynomial approximation to exp(r) + // with r in the interval [-ln(2)/2;ln(2)/2]. + const Packet r2 = pmul(r, r); + Packet p_even = pmadd(r2, cst_p6, cst_p4); + const Packet p_odd = pmadd(r2, cst_p5, cst_p3); + p_even = pmadd(r2, p_even, cst_p2); + const Packet p_low = padd(r, cst_one); + Packet y = pmadd(r, p_odd, p_even); + y = pmadd(r2, y, p_low); // Return 2^m * exp(r). // TODO: replace pldexp with faster implementation since y in [-1, 1). - return pmax(pldexp(y,m), _x); + return pselect(zero_mask, cst_zero, pmax(pldexp(y,m), _x)); } template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet pexp_double(const Packet _x) { Packet x = _x; - + const Packet cst_zero = pset1(0.0); const Packet cst_1 = pset1(1.0); const Packet cst_2 = pset1(2.0); const Packet cst_half = pset1(0.5); @@ -512,7 +477,8 @@ Packet pexp_double(const Packet _x) Packet tmp, fx; // clamp x - x = pmax(pmin(x, cst_exp_hi), cst_exp_lo); + Packet zero_mask = pcmp_lt(_x, cst_exp_lo); + x = pmin(x, cst_exp_hi); // Express exp(x) as exp(g + n*log(2)). fx = pmadd(cst_cephes_LOG2EF, x, cst_half); @@ -550,7 +516,7 @@ Packet pexp_double(const Packet _x) // Construct the result 2^n * exp(g) = e * x. The max is used to catch // non-finite values in the input. // TODO: replace pldexp with faster implementation since x in [-1, 1). - return pmax(pldexp(x,fx), _x); + return pselect(zero_mask, cst_zero, pmax(pldexp(x,fx), _x)); } // The following code is inspired by the following stack-overflow answer: @@ -562,7 +528,7 @@ Packet pexp_double(const Packet _x) // aligned on 8-bits, and (2) replicating the storage of the bits of 2/pi. // - Avoid a branch in rounding and extraction of the remaining fractional part. // Overall, I measured a speed up higher than x2 on x86-64. -inline float trig_reduce_huge (float xf, int *quadrant) +inline float trig_reduce_huge (float xf, Eigen::numext::int32_t *quadrant) { using Eigen::numext::int32_t; using Eigen::numext::uint32_t; @@ -570,7 +536,7 @@ inline float trig_reduce_huge (float xf, int *quadrant) using Eigen::numext::uint64_t; const double pio2_62 = 3.4061215800865545e-19; // pi/2 * 2^-62 - const uint64_t zero_dot_five = uint64_t(1) << 61; // 0.5 in 2.62-bit fixed-point foramt + const uint64_t zero_dot_five = uint64_t(1) << 61; // 0.5 in 2.62-bit fixed-point format // 192 bits of 2/pi for Payne-Hanek reduction // Bits are introduced by packet of 8 to enable aligned reads. @@ -618,8 +584,7 @@ inline float trig_reduce_huge (float xf, int *quadrant) template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED -#if EIGEN_GNUC_AT_LEAST(4,4) && EIGEN_COMP_GNUC_STRICT +#if EIGEN_COMP_GNUC_STRICT __attribute__((optimize("-fno-unsafe-math-optimizations"))) #endif Packet psincos_float(const Packet& _x) @@ -629,20 +594,20 @@ Packet psincos_float(const Packet& _x) const Packet cst_2oPI = pset1(0.636619746685028076171875f); // 2/PI const Packet cst_rounding_magic = pset1(12582912); // 2^23 for rounding const PacketI csti_1 = pset1(1); - const Packet cst_sign_mask = pset1frombits(0x80000000u); + const Packet cst_sign_mask = pset1frombits(static_cast(0x80000000u)); Packet x = pabs(_x); // Scale x by 2/Pi to find x's octant. Packet y = pmul(x, cst_2oPI); - // Rounding trick: + // Rounding trick to find nearest integer: Packet y_round = padd(y, cst_rounding_magic); EIGEN_OPTIMIZATION_BARRIER(y_round) PacketI y_int = preinterpret(y_round); // last 23 digits represent integer (if abs(x)<2^24) - y = psub(y_round, cst_rounding_magic); // nearest integer to x*4/pi + y = psub(y_round, cst_rounding_magic); // nearest integer to x * (2/pi) - // Reduce x by y octants to get: -Pi/4 <= x <= +Pi/4 + // Subtract y * Pi/2 to reduce x to the interval -Pi/4 <= x <= +Pi/4 // using "Extended precision modular arithmetic" #if defined(EIGEN_HAS_SINGLE_INSTRUCTION_MADD) // This version requires true FMA for high accuracy @@ -685,7 +650,7 @@ Packet psincos_float(const Packet& _x) const int PacketSize = unpacket_traits::size; EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) float vals[PacketSize]; EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) float x_cpy[PacketSize]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) int y_int2[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Eigen::numext::int32_t y_int2[PacketSize]; pstoreu(vals, pabs(_x)); pstoreu(x_cpy, x); pstoreu(y_int2, y_int); @@ -743,7 +708,6 @@ Packet psincos_float(const Packet& _x) template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet psin_float(const Packet& x) { return psincos_float(x); @@ -751,16 +715,310 @@ Packet psin_float(const Packet& x) template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet pcos_float(const Packet& x) { return psincos_float(x); } +// Generic implementation of acos(x). +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet pacos_float(const Packet& x_in) { + typedef typename unpacket_traits::type Scalar; + static_assert(std::is_same::value, "Scalar type must be float"); + + const Packet cst_one = pset1(Scalar(1)); + const Packet cst_pi = pset1(Scalar(EIGEN_PI)); + const Packet p6 = pset1(Scalar(2.36423197202384471893310546875e-3)); + const Packet p5 = pset1(Scalar(-1.1368644423782825469970703125e-2)); + const Packet p4 = pset1(Scalar(2.717843465507030487060546875e-2)); + const Packet p3 = pset1(Scalar(-4.8969544470310211181640625e-2)); + const Packet p2 = pset1(Scalar(8.8804088532924652099609375e-2)); + const Packet p1 = pset1(Scalar(-0.214591205120086669921875)); + const Packet p0 = pset1(Scalar(1.57079637050628662109375)); + + // For x in [0:1], we approximate acos(x)/sqrt(1-x), which is a smooth + // function, by a 6'th order polynomial. + // For x in [-1:0) we use that acos(-x) = pi - acos(x). + const Packet neg_mask = psignbit(x_in); + const Packet abs_x = pabs(x_in); + + // Evaluate the polynomial using Horner's rule: + // P(x) = p0 + x * (p1 + x * (p2 + ... (p5 + x * p6)) ... ) . + // We evaluate even and odd terms independently to increase + // instruction level parallelism. + Packet x2 = pmul(x_in,x_in); + Packet p_even = pmadd(p6, x2, p4); + Packet p_odd = pmadd(p5, x2, p3); + p_even = pmadd(p_even, x2, p2); + p_odd = pmadd(p_odd, x2, p1); + p_even = pmadd(p_even, x2, p0); + Packet p = pmadd(p_odd, abs_x, p_even); + + // The polynomial approximates acos(x)/sqrt(1-x), so + // multiply by sqrt(1-x) to get acos(x). + // Conveniently returns NaN for arguments outside [-1:1]. + Packet denom = psqrt(psub(cst_one, abs_x)); + Packet result = pmul(denom, p); + // Undo mapping for negative arguments. + return pselect(neg_mask, psub(cst_pi, result), result); +} + +// Generic implementation of asin(x). +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet pasin_float(const Packet& x_in) { + typedef typename unpacket_traits::type Scalar; + static_assert(std::is_same::value, "Scalar type must be float"); + + constexpr float kPiOverTwo = static_cast(EIGEN_PI / 2); + + const Packet cst_half = pset1(0.5f); + const Packet cst_one = pset1(1.0f); + const Packet cst_two = pset1(2.0f); + const Packet cst_pi_over_two = pset1(kPiOverTwo); + // For |x| < 0.5 approximate asin(x)/x by an 8th order polynomial with + // even terms only. + const Packet p9 = pset1(5.08838854730129241943359375e-2f); + const Packet p7 = pset1(3.95139865577220916748046875e-2f); + const Packet p5 = pset1(7.550220191478729248046875e-2f); + const Packet p3 = pset1(0.16664917767047882080078125f); + const Packet p1 = pset1(1.00000011920928955078125f); + + const Packet abs_x = pabs(x_in); + const Packet sign_mask = pandnot(x_in, abs_x); + const Packet invalid_mask = pcmp_lt(cst_one, abs_x); + + // For arguments |x| > 0.5, we map x back to [0:0.5] using + // the transformation x_large = sqrt(0.5*(1-x)), and use the + // identity + // asin(x) = pi/2 - 2 * asin( sqrt( 0.5 * (1 - x))) + + const Packet x_large = psqrt(pnmadd(cst_half, abs_x, cst_half)); + const Packet large_mask = pcmp_lt(cst_half, abs_x); + const Packet x = pselect(large_mask, x_large, abs_x); + const Packet x2 = pmul(x, x); + + // Compute polynomial. + // x * (p1 + x^2*(p3 + x^2*(p5 + x^2*(p7 + x^2*p9)))) + + Packet p = pmadd(p9, x2, p7); + p = pmadd(p, x2, p5); + p = pmadd(p, x2, p3); + p = pmadd(p, x2, p1); + p = pmul(p, x); + + const Packet p_large = pnmadd(cst_two, p, cst_pi_over_two); + p = pselect(large_mask, p_large, p); + // Flip the sign for negative arguments. + p = pxor(p, sign_mask); + // Return NaN for arguments outside [-1:1]. + return por(invalid_mask, p); +} + +// Computes elementwise atan(x) for x in [-1:1] with 2 ulp accuracy. +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet patan_reduced_float(const Packet& x) { + const Packet q0 = pset1(-0.3333314359188079833984375f); + const Packet q2 = pset1(0.19993579387664794921875f); + const Packet q4 = pset1(-0.14209578931331634521484375f); + const Packet q6 = pset1(0.1066047251224517822265625f); + const Packet q8 = pset1(-7.5408883392810821533203125e-2f); + const Packet q10 = pset1(4.3082617223262786865234375e-2f); + const Packet q12 = pset1(-1.62907354533672332763671875e-2f); + const Packet q14 = pset1(2.90188402868807315826416015625e-3f); + + // Approximate atan(x) by a polynomial of the form + // P(x) = x + x^3 * Q(x^2), + // where Q(x^2) is a 7th order polynomial in x^2. + // We evaluate even and odd terms in x^2 in parallel + // to take advantage of instruction level parallelism + // and hardware with multiple FMA units. + + // note: if x == -0, this returns +0 + const Packet x2 = pmul(x, x); + const Packet x4 = pmul(x2, x2); + Packet q_odd = pmadd(q14, x4, q10); + Packet q_even = pmadd(q12, x4, q8); + q_odd = pmadd(q_odd, x4, q6); + q_even = pmadd(q_even, x4, q4); + q_odd = pmadd(q_odd, x4, q2); + q_even = pmadd(q_even, x4, q0); + const Packet q = pmadd(q_odd, x2, q_even); + return pmadd(q, pmul(x, x2), x); +} + +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet patan_float(const Packet& x_in) { + typedef typename unpacket_traits::type Scalar; + static_assert(std::is_same::value, "Scalar type must be float"); + + constexpr float kPiOverTwo = static_cast(EIGEN_PI / 2); + + const Packet cst_signmask = pset1(-0.0f); + const Packet cst_one = pset1(1.0f); + const Packet cst_pi_over_two = pset1(kPiOverTwo); + + // "Large": For |x| > 1, use atan(1/x) = sign(x)*pi/2 - atan(x). + // "Small": For |x| <= 1, approximate atan(x) directly by a polynomial + // calculated using Sollya. + + const Packet abs_x = pabs(x_in); + const Packet x_signmask = pand(x_in, cst_signmask); + const Packet large_mask = pcmp_lt(cst_one, abs_x); + const Packet x = pselect(large_mask, preciprocal(abs_x), abs_x); + const Packet p = patan_reduced_float(x); + // Apply transformations according to the range reduction masks. + Packet result = pselect(large_mask, psub(cst_pi_over_two, p), p); + // Return correct sign + return pxor(result, x_signmask); +} + +// Computes elementwise atan(x) for x in [-tan(pi/8):tan(pi/8)] +// with 2 ulp accuracy. +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet +patan_reduced_double(const Packet& x) { + const Packet q0 = + pset1(-0.33333333333330028569463365784031338989734649658203); + const Packet q2 = + pset1(0.199999999990664090177006073645316064357757568359375); + const Packet q4 = + pset1(-0.142857141937123677255527809393242932856082916259766); + const Packet q6 = + pset1(0.111111065991039953404495577160560060292482376098633); + const Packet q8 = + pset1(-9.0907812986129224452902519715280504897236824035645e-2); + const Packet q10 = + pset1(7.6900542950704739442180368769186316058039665222168e-2); + const Packet q12 = + pset1(-6.6410112986494976294871150912513257935643196105957e-2); + const Packet q14 = + pset1(5.6920144995467943094258345126945641823112964630127e-2); + const Packet q16 = + pset1(-4.3577020814990513608577771265117917209863662719727e-2); + const Packet q18 = + pset1(2.1244050233624342527427586446719942614436149597168e-2); + + // Approximate atan(x) on [0:tan(pi/8)] by a polynomial of the form + // P(x) = x + x^3 * Q(x^2), + // where Q(x^2) is a 9th order polynomial in x^2. + // We evaluate even and odd terms in x^2 in parallel + // to take advantage of instruction level parallelism + // and hardware with multiple FMA units. + const Packet x2 = pmul(x, x); + const Packet x4 = pmul(x2, x2); + Packet q_odd = pmadd(q18, x4, q14); + Packet q_even = pmadd(q16, x4, q12); + q_odd = pmadd(q_odd, x4, q10); + q_even = pmadd(q_even, x4, q8); + q_odd = pmadd(q_odd, x4, q6); + q_even = pmadd(q_even, x4, q4); + q_odd = pmadd(q_odd, x4, q2); + q_even = pmadd(q_even, x4, q0); + const Packet p = pmadd(q_odd, x2, q_even); + return pmadd(p, pmul(x, x2), x); +} + +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet patan_double(const Packet& x_in) { + typedef typename unpacket_traits::type Scalar; + static_assert(std::is_same::value, "Scalar type must be double"); + + constexpr double kPiOverTwo = static_cast(EIGEN_PI / 2); + constexpr double kPiOverFour = static_cast(EIGEN_PI / 4); + constexpr double kTanPiOverEight = 0.4142135623730950488016887; + constexpr double kTan3PiOverEight = 2.4142135623730950488016887; + + const Packet cst_signmask = pset1(-0.0); + const Packet cst_one = pset1(1.0); + const Packet cst_pi_over_two = pset1(kPiOverTwo); + const Packet cst_pi_over_four = pset1(kPiOverFour); + const Packet cst_large = pset1(kTan3PiOverEight); + const Packet cst_medium = pset1(kTanPiOverEight); + + // Use the same range reduction strategy (to [0:tan(pi/8)]) as the + // Cephes library: + // "Large": For x >= tan(3*pi/8), use atan(1/x) = pi/2 - atan(x). + // "Medium": For x in [tan(pi/8) : tan(3*pi/8)), + // use atan(x) = pi/4 + atan((x-1)/(x+1)). + // "Small": For x < tan(pi/8), approximate atan(x) directly by a polynomial + // calculated using Sollya. + + const Packet abs_x = pabs(x_in); + const Packet x_signmask = pand(x_in, cst_signmask); + const Packet large_mask = pcmp_lt(cst_large, abs_x); + const Packet medium_mask = pandnot(pcmp_lt(cst_medium, abs_x), large_mask); + + Packet x = abs_x; + x = pselect(large_mask, preciprocal(abs_x), x); + x = pselect(medium_mask, pdiv(psub(abs_x, cst_one), padd(abs_x, cst_one)), x); + + // Compute approximation of p ~= atan(x') where x' is the argument reduced to + // [0:tan(pi/8)]. + Packet p = patan_reduced_double(x); + + // Apply transformations according to the range reduction masks. + p = pselect(large_mask, psub(cst_pi_over_two, p), p); + p = pselect(medium_mask, padd(cst_pi_over_four, p), p); + // Return the correct sign + return pxor(p, x_signmask); +} + +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet patanh_float(const Packet& x) { + typedef typename unpacket_traits::type Scalar; + static_assert(std::is_same::value, "Scalar type must be float"); + const Packet half = pset1(0.5f); + const Packet x_gt_half = pcmp_le(half, pabs(x)); + // For |x| in [0:0.5] we use a polynomial approximation of the form + // P(x) = x + x^3*(c3 + x^2 * (c5 + x^2 * (... x^2 * c11) ... )). + const Packet C3 = pset1(0.3333373963832855224609375f); + const Packet C5 = pset1(0.1997792422771453857421875f); + const Packet C7 = pset1(0.14672131836414337158203125f); + const Packet C9 = pset1(8.2311116158962249755859375e-2f); + const Packet C11 = pset1(0.1819281280040740966796875f); + const Packet x2 = pmul(x,x); + Packet p = pmadd(C11, x2, C9); + p = pmadd(x2, p, C7); + p = pmadd(x2, p, C5); + p = pmadd(x2, p, C3); + p = pmadd(pmul(x,x2), p, x); + + // For |x| in ]0.5:1.0] we use atanh = 0.5*ln((1+x)/(1-x)); + const Packet one = pset1(1.0f); + Packet r = pdiv(padd(one, x), psub(one, x)); + r = pmul(half, plog(r)); + return pselect(x_gt_half, r, p); +} + +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet pdiv_complex(const Packet& x, const Packet& y) { + typedef typename unpacket_traits::as_real RealPacket; + // In the following we annotate the code for the case where the inputs + // are a pair length-2 SIMD vectors representing a single pair of complex + // numbers x = a + i*b, y = c + i*d. + const RealPacket y_abs = pabs(y.v); // |c|, |d| + const RealPacket y_abs_flip = pcplxflip(Packet(y_abs)).v; // |d|, |c| + const RealPacket y_max = pmax(y_abs, y_abs_flip); // max(|c|, |d|), max(|c|, |d|) + const RealPacket y_scaled = pdiv(y.v, y_max); // c / max(|c|, |d|), d / max(|c|, |d|) + // Compute scaled denominator. + const RealPacket y_scaled_sq = pmul(y_scaled, y_scaled); // c'**2, d'**2 + const RealPacket denom = padd(y_scaled_sq, pcplxflip(Packet(y_scaled_sq)).v); + Packet result_scaled = pmul(x, pconj(Packet(y_scaled))); // a * c' + b * d', -a * d + b * c + // Divide elementwise by denom. + result_scaled = Packet(pdiv(result_scaled.v, denom)); + // Rescale result + return Packet(pdiv(result_scaled.v, y_max)); +} template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet psqrt_complex(const Packet& a) { typedef typename unpacket_traits::type Scalar; typedef typename Scalar::value_type RealScalar; @@ -832,8 +1090,8 @@ Packet psqrt_complex(const Packet& a) { // Step 4. Compute solution for inputs with negative real part: // [|eta0|, sign(y0)*rho0, |eta1|, sign(y1)*rho1] - const RealScalar neg_zero = RealScalar(numext::bit_cast(0x80000000u)); - const RealPacket cst_imag_sign_mask = pset1(Scalar(RealScalar(0.0), neg_zero)).v; + const RealPacket cst_imag_sign_mask = + pset1(Scalar(RealScalar(0.0), RealScalar(-0.0))).v; RealPacket imag_signs = pand(a.v, cst_imag_sign_mask); Packet negative_real_result; // Notice that rho is positive, so taking it's absolute value is a noop. @@ -866,11 +1124,97 @@ Packet psqrt_complex(const Packet& a) { is_imag_inf = por(is_imag_inf, pcplxflip(is_imag_inf)); Packet imag_inf_result; imag_inf_result.v = por(pand(cst_pos_inf, real_mask), pandnot(a.v, real_mask)); + // unless otherwise specified, if either the real or imaginary component is nan, the entire result is nan + Packet result_is_nan = pisnan(result); + result = por(result_is_nan, result); - return pselect(is_imag_inf, imag_inf_result, - pselect(is_real_inf, real_inf_result,result)); + return pselect(is_imag_inf, imag_inf_result, pselect(is_real_inf, real_inf_result, result)); } + +template +struct psign_impl::type>::IsComplex && + !NumTraits::type>::IsInteger>> { + static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { + using Scalar = typename unpacket_traits::type; + const Packet cst_one = pset1(Scalar(1)); + const Packet cst_zero = pzero(a); + + const Packet abs_a = pabs(a); + const Packet sign_mask = pandnot(a, abs_a); + const Packet nonzero_mask = pcmp_lt(cst_zero, abs_a); + + return pselect(nonzero_mask, por(sign_mask, cst_one), abs_a); + } +}; + +template +struct psign_impl::type>::IsComplex && + NumTraits::type>::IsSigned && + NumTraits::type>::IsInteger>> { + static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { + using Scalar = typename unpacket_traits::type; + const Packet cst_one = pset1(Scalar(1)); + const Packet cst_minus_one = pset1(Scalar(-1)); + const Packet cst_zero = pzero(a); + + const Packet positive_mask = pcmp_lt(cst_zero, a); + const Packet positive = pand(positive_mask, cst_one); + const Packet negative_mask = pcmp_lt(a, cst_zero); + const Packet negative = pand(negative_mask, cst_minus_one); + + return por(positive, negative); + } +}; + +template +struct psign_impl::type>::IsComplex && + !NumTraits::type>::IsSigned && + NumTraits::type>::IsInteger>> { + static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { + using Scalar = typename unpacket_traits::type; + const Packet cst_one = pset1(Scalar(1)); + const Packet cst_zero = pzero(a); + + const Packet zero_mask = pcmp_eq(cst_zero, a); + return pandnot(cst_one, zero_mask); + } +}; + +// \internal \returns the the sign of a complex number z, defined as z / abs(z). +template +struct psign_impl::type>::IsComplex && + unpacket_traits::vectorizable>> { + static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { + typedef typename unpacket_traits::type Scalar; + typedef typename Scalar::value_type RealScalar; + typedef typename unpacket_traits::as_real RealPacket; + + // Step 1. Compute (for each element z = x + i*y in a) + // l = abs(z) = sqrt(x^2 + y^2). + // To avoid over- and underflow, we use the stable formula for each hypotenuse + // l = (zmin == 0 ? zmax : zmax * sqrt(1 + (zmin/zmax)**2)), + // where zmax = max(|x|, |y|), zmin = min(|x|, |y|), + RealPacket a_abs = pabs(a.v); + RealPacket a_abs_flip = pcplxflip(Packet(a_abs)).v; + RealPacket a_max = pmax(a_abs, a_abs_flip); + RealPacket a_min = pmin(a_abs, a_abs_flip); + RealPacket a_min_zero_mask = pcmp_eq(a_min, pzero(a_min)); + RealPacket a_max_zero_mask = pcmp_eq(a_max, pzero(a_max)); + RealPacket r = pdiv(a_min, a_max); + const RealPacket cst_one = pset1(RealScalar(1)); + RealPacket l = pmul(a_max, psqrt(padd(cst_one, pmul(r, r)))); // [l0, l0, l1, l1] + // Set l to a_max if a_min is zero, since the roundtrip sqrt(a_max^2) may be + // lossy. + l = pselect(a_min_zero_mask, a_max, l); + // Step 2 compute a / abs(a). + RealPacket sign_as_real = pandnot(pdiv(a.v, l), a_max_zero_mask); + Packet sign; + sign.v = sign_as_real; + return sign; + } +}; + // TODO(rmlarsen): The following set of utilities for double word arithmetic // should perhaps be refactored as a separate file, since it would be generally // useful for special function implementation etc. Writing the algorithms in @@ -905,7 +1249,7 @@ EIGEN_STRONG_INLINE void twoprod(const Packet& x, const Packet& y, Packet& p_hi, Packet& p_lo) { p_hi = pmul(x, y); - p_lo = pmadd(x, y, pnegate(p_hi)); + p_lo = pmsub(x, y, p_hi); } #else @@ -1040,32 +1384,23 @@ void twoprod(const Packet& x_hi, const Packet& x_lo, fast_twosum(p_hi_hi, p_hi_lo, p_lo_hi, p_lo_lo, p_hi, p_lo); } -// This function computes the reciprocal of a floating point number -// with extra precision and returns the result as a double word. +// This function implements the division of double word {x_hi, x_lo} +// by float y. This is Algorithm 15 from "Tight and rigourous error bounds +// for basic building blocks of double-word arithmetic", Joldes, Muller, & Popescu, +// 2017. https://hal.archives-ouvertes.fr/hal-01351529 template -void doubleword_reciprocal(const Packet& x, Packet& recip_hi, Packet& recip_lo) { - typedef typename unpacket_traits::type Scalar; - // 1. Approximate the reciprocal as the reciprocal of the high order element. - Packet approx_recip = prsqrt(x); - approx_recip = pmul(approx_recip, approx_recip); - - // 2. Run one step of Newton-Raphson iteration in double word arithmetic - // to get the bottom half. The NR iteration for reciprocal of 'a' is - // x_{i+1} = x_i * (2 - a * x_i) - - // -a*x_i - Packet t1_hi, t1_lo; - twoprod(pnegate(x), approx_recip, t1_hi, t1_lo); - // 2 - a*x_i - Packet t2_hi, t2_lo; - fast_twosum(pset1(Scalar(2)), t1_hi, t2_hi, t2_lo); - Packet t3_hi, t3_lo; - fast_twosum(t2_hi, padd(t2_lo, t1_lo), t3_hi, t3_lo); - // x_i * (2 - a * x_i) - twoprod(t3_hi, t3_lo, approx_recip, recip_hi, recip_lo); +void doubleword_div_fp(const Packet& x_hi, const Packet& x_lo, const Packet& y, + Packet& z_hi, Packet& z_lo) { + const Packet t_hi = pdiv(x_hi, y); + Packet pi_hi, pi_lo; + twoprod(t_hi, y, pi_hi, pi_lo); + const Packet delta_hi = psub(x_hi, pi_hi); + const Packet delta_t = psub(delta_hi, pi_lo); + const Packet delta = padd(delta_t, x_lo); + const Packet t_lo = pdiv(delta, y); + fast_twosum(t_hi, t_lo, z_hi, z_lo); } - // This function computes log2(x) and returns the result as a double word. template struct accurate_log2 { @@ -1204,16 +1539,13 @@ struct accurate_log2 { const Packet cst_2_log2e_hi = pset1(2.88539008177792677); const Packet cst_2_log2e_lo = pset1(4.07660016854549667e-17); // c * (x - 1) - Packet num_hi, num_lo; - twoprod(cst_2_log2e_hi, cst_2_log2e_lo, psub(x, one), num_hi, num_lo); - // TODO(rmlarsen): Investigate if using the division algorithm by - // Muller et al. is faster/more accurate. - // 1 / (x + 1) - Packet denom_hi, denom_lo; - doubleword_reciprocal(padd(x, one), denom_hi, denom_lo); - // r = c * (x-1) / (x+1), + Packet t_hi, t_lo; + // t = c * (x-1) + twoprod(cst_2_log2e_hi, cst_2_log2e_lo, psub(x, one), t_hi, t_lo); + // r = c * (x-1) / (x+1), Packet r_hi, r_lo; - twoprod(num_hi, num_lo, denom_hi, denom_lo, r_hi, r_lo); + doubleword_div_fp(t_hi, t_lo, padd(x, one), r_hi, r_lo); + // r2 = r * r Packet r2_hi, r2_lo; twoprod(r_hi, r_lo, r_hi, r_lo, r2_hi, r2_lo); @@ -1443,39 +1775,40 @@ EIGEN_STRONG_INLINE Packet generic_pow_impl(const Packet& x, const Packet& y) { } // Generic implementation of pow(x,y). -template -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED -Packet generic_pow(const Packet& x, const Packet& y) { +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet generic_pow(const Packet& x, const Packet& y) { typedef typename unpacket_traits::type Scalar; const Packet cst_pos_inf = pset1(NumTraits::infinity()); + const Packet cst_neg_inf = pset1(-NumTraits::infinity()); const Packet cst_zero = pset1(Scalar(0)); const Packet cst_one = pset1(Scalar(1)); const Packet cst_nan = pset1(NumTraits::quiet_NaN()); const Packet abs_x = pabs(x); // Predicates for sign and magnitude of x. - const Packet x_is_zero = pcmp_eq(x, cst_zero); - const Packet x_is_neg = pcmp_lt(x, cst_zero); + const Packet abs_x_is_zero = pcmp_eq(abs_x, cst_zero); + const Packet x_has_signbit = psignbit(x); + const Packet x_is_neg = pandnot(x_has_signbit, abs_x_is_zero); + const Packet x_is_neg_zero = pand(x_has_signbit, abs_x_is_zero); const Packet abs_x_is_inf = pcmp_eq(abs_x, cst_pos_inf); - const Packet abs_x_is_one = pcmp_eq(abs_x, cst_one); + const Packet abs_x_is_one = pcmp_eq(abs_x, cst_one); const Packet abs_x_is_gt_one = pcmp_lt(cst_one, abs_x); const Packet abs_x_is_lt_one = pcmp_lt(abs_x, cst_one); - const Packet x_is_one = pandnot(abs_x_is_one, x_is_neg); - const Packet x_is_neg_one = pand(abs_x_is_one, x_is_neg); - const Packet x_is_nan = pandnot(ptrue(x), pcmp_eq(x, x)); + const Packet x_is_one = pandnot(abs_x_is_one, x_is_neg); + const Packet x_is_neg_one = pand(abs_x_is_one, x_is_neg); + const Packet x_is_nan = pisnan(x); // Predicates for sign and magnitude of y. + const Packet abs_y = pabs(y); const Packet y_is_one = pcmp_eq(y, cst_one); - const Packet y_is_zero = pcmp_eq(y, cst_zero); + const Packet abs_y_is_zero = pcmp_eq(abs_y, cst_zero); const Packet y_is_neg = pcmp_lt(y, cst_zero); - const Packet y_is_pos = pandnot(ptrue(y), por(y_is_zero, y_is_neg)); - const Packet y_is_nan = pandnot(ptrue(y), pcmp_eq(y, y)); - const Packet abs_y_is_inf = pcmp_eq(pabs(y), cst_pos_inf); + const Packet y_is_pos = pandnot(ptrue(y), por(abs_y_is_zero, y_is_neg)); + const Packet y_is_nan = pisnan(y); + const Packet abs_y_is_inf = pcmp_eq(abs_y, cst_pos_inf); EIGEN_CONSTEXPR Scalar huge_exponent = - (NumTraits::max_exponent() * Scalar(EIGEN_LN2)) / - NumTraits::epsilon(); + (NumTraits::max_exponent() * Scalar(EIGEN_LN2)) / NumTraits::epsilon(); const Packet abs_y_is_huge = pcmp_le(pset1(huge_exponent), pabs(y)); // Predicates for whether y is integer and/or even. @@ -1484,39 +1817,33 @@ Packet generic_pow(const Packet& x, const Packet& y) { const Packet y_is_even = pcmp_eq(pround(y_div_2), y_div_2); // Predicates encoding special cases for the value of pow(x,y) - const Packet invalid_negative_x = pandnot(pandnot(pandnot(x_is_neg, abs_x_is_inf), - y_is_int), - abs_y_is_inf); - const Packet pow_is_one = por(por(x_is_one, y_is_zero), - pand(x_is_neg_one, - por(abs_y_is_inf, pandnot(y_is_even, invalid_negative_x)))); + const Packet invalid_negative_x = pandnot(pandnot(pandnot(x_is_neg, abs_x_is_inf), y_is_int), abs_y_is_inf); const Packet pow_is_nan = por(invalid_negative_x, por(x_is_nan, y_is_nan)); - const Packet pow_is_zero = por(por(por(pand(x_is_zero, y_is_pos), - pand(abs_x_is_inf, y_is_neg)), - pand(pand(abs_x_is_lt_one, abs_y_is_huge), - y_is_pos)), - pand(pand(abs_x_is_gt_one, abs_y_is_huge), - y_is_neg)); - const Packet pow_is_inf = por(por(por(pand(x_is_zero, y_is_neg), - pand(abs_x_is_inf, y_is_pos)), - pand(pand(abs_x_is_lt_one, abs_y_is_huge), - y_is_neg)), - pand(pand(abs_x_is_gt_one, abs_y_is_huge), - y_is_pos)); - + const Packet pow_is_one = + por(por(x_is_one, abs_y_is_zero), pand(x_is_neg_one, por(abs_y_is_inf, pandnot(y_is_even, invalid_negative_x)))); + const Packet pow_is_zero = por(por(por(pand(abs_x_is_zero, y_is_pos), pand(abs_x_is_inf, y_is_neg)), + pand(pand(abs_x_is_lt_one, abs_y_is_huge), y_is_pos)), + pand(pand(abs_x_is_gt_one, abs_y_is_huge), y_is_neg)); + const Packet pow_is_inf = por(por(por(pand(abs_x_is_zero, y_is_neg), pand(abs_x_is_inf, y_is_pos)), + pand(pand(abs_x_is_lt_one, abs_y_is_huge), y_is_neg)), + pand(pand(abs_x_is_gt_one, abs_y_is_huge), y_is_pos)); + const Packet pow_is_neg_zero = pand(pandnot(y_is_int, y_is_even), + por(pand(y_is_neg, pand(abs_x_is_inf, x_is_neg)), pand(y_is_pos, x_is_neg_zero))); + const Packet inf_val = + pselect(pandnot(pand(por(pand(abs_x_is_inf, x_is_neg), pand(x_is_neg_zero, y_is_neg)), y_is_int), y_is_even), + cst_neg_inf, cst_pos_inf); // General computation of pow(x,y) for positive x or negative x and integer y. const Packet negate_pow_abs = pandnot(x_is_neg, y_is_even); const Packet pow_abs = generic_pow_impl(abs_x, y); return pselect(y_is_one, x, pselect(pow_is_one, cst_one, pselect(pow_is_nan, cst_nan, - pselect(pow_is_inf, cst_pos_inf, - pselect(pow_is_zero, cst_zero, - pselect(negate_pow_abs, pnegate(pow_abs), pow_abs)))))); + pselect(pow_is_inf, inf_val, + pselect(pow_is_neg_zero, pnegate(cst_zero), + pselect(pow_is_zero, cst_zero, + pselect(negate_pow_abs, pnegate(pow_abs), pow_abs))))))); } - - /* polevl (modified for Eigen) * * Evaluate polynomial @@ -1643,6 +1970,241 @@ struct pchebevl { } }; +namespace unary_pow { + +template ::IsInteger> +struct exponent_helper { + using safe_abs_type = ScalarExponent; + static constexpr ScalarExponent one_half = ScalarExponent(0.5); + // these routines assume that exp is an integer stored as a floating point type + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ScalarExponent safe_abs(const ScalarExponent& exp) { + return numext::abs(exp); + } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_odd(const ScalarExponent& exp) { + eigen_assert(((numext::isfinite)(exp) && exp == numext::floor(exp)) && "exp must be an integer"); + ScalarExponent exp_div_2 = exp * one_half; + ScalarExponent floor_exp_div_2 = numext::floor(exp_div_2); + return exp_div_2 != floor_exp_div_2; + } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ScalarExponent floor_div_two(const ScalarExponent& exp) { + ScalarExponent exp_div_2 = exp * one_half; + return numext::floor(exp_div_2); + } +}; + +template +struct exponent_helper { + // if `exp` is a signed integer type, cast it to its unsigned counterpart to safely store its absolute value + // consider the (rare) case where `exp` is an int32_t: abs(-2147483648) != 2147483648 + using safe_abs_type = typename numext::get_integer_by_size::unsigned_type; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE safe_abs_type safe_abs(const ScalarExponent& exp) { + ScalarExponent mask = numext::signbit(exp); + safe_abs_type result = safe_abs_type(exp ^ mask); + return result + safe_abs_type(ScalarExponent(1) & mask); + } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_odd(const safe_abs_type& exp) { + return exp % safe_abs_type(2) != safe_abs_type(0); + } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE safe_abs_type floor_div_two(const safe_abs_type& exp) { + return exp >> safe_abs_type(1); + } +}; + +template ::type>::IsInteger && NumTraits::IsSigned> +struct reciprocate { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) { + using Scalar = typename unpacket_traits::type; + const Packet cst_pos_one = pset1(Scalar(1)); + return exponent < 0 ? pdiv(cst_pos_one, x) : x; + } +}; + +template +struct reciprocate { + // pdiv not defined, nor necessary for integer base types + // if the exponent is unsigned, then the exponent cannot be negative + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent&) { return x; } +}; + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet int_pow(const Packet& x, const ScalarExponent& exponent) { + using Scalar = typename unpacket_traits::type; + using ExponentHelper = exponent_helper; + using AbsExponentType = typename ExponentHelper::safe_abs_type; + const Packet cst_pos_one = pset1(Scalar(1)); + if (exponent == ScalarExponent(0)) return cst_pos_one; + + Packet result = reciprocate::run(x, exponent); + Packet y = cst_pos_one; + AbsExponentType m = ExponentHelper::safe_abs(exponent); + + while (m > 1) { + bool odd = ExponentHelper::is_odd(m); + if (odd) y = pmul(y, result); + result = pmul(result, result); + m = ExponentHelper::floor_div_two(m); + } + + return pmul(y, result); +} + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet gen_pow(const Packet& x, + const typename unpacket_traits::type& exponent) { + const Packet exponent_packet = pset1(exponent); + return generic_pow_impl(x, exponent_packet); +} + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet handle_nonint_nonint_errors(const Packet& x, const Packet& powx, + const ScalarExponent& exponent) { + using Scalar = typename unpacket_traits::type; + + // non-integer base and exponent case + + const Scalar pos_zero = Scalar(0); + const Scalar all_ones = ptrue(Scalar()); + const Scalar pos_one = Scalar(1); + const Scalar pos_inf = NumTraits::infinity(); + + const Packet cst_pos_zero = pzero(x); + const Packet cst_pos_one = pset1(pos_one); + const Packet cst_pos_inf = pset1(pos_inf); + + const bool exponent_is_not_fin = !(numext::isfinite)(exponent); + const bool exponent_is_neg = exponent < ScalarExponent(0); + const bool exponent_is_pos = exponent > ScalarExponent(0); + + const Packet exp_is_not_fin = pset1(exponent_is_not_fin ? all_ones : pos_zero); + const Packet exp_is_neg = pset1(exponent_is_neg ? all_ones : pos_zero); + const Packet exp_is_pos = pset1(exponent_is_pos ? all_ones : pos_zero); + const Packet exp_is_inf = pand(exp_is_not_fin, por(exp_is_neg, exp_is_pos)); + const Packet exp_is_nan = pandnot(exp_is_not_fin, por(exp_is_neg, exp_is_pos)); + + const Packet x_is_le_zero = pcmp_le(x, cst_pos_zero); + const Packet x_is_ge_zero = pcmp_le(cst_pos_zero, x); + const Packet x_is_zero = pand(x_is_le_zero, x_is_ge_zero); + + const Packet abs_x = pabs(x); + const Packet abs_x_is_le_one = pcmp_le(abs_x, cst_pos_one); + const Packet abs_x_is_ge_one = pcmp_le(cst_pos_one, abs_x); + const Packet abs_x_is_inf = pcmp_eq(abs_x, cst_pos_inf); + const Packet abs_x_is_one = pand(abs_x_is_le_one, abs_x_is_ge_one); + + Packet pow_is_inf_if_exp_is_neg = por(x_is_zero, pand(abs_x_is_le_one, exp_is_inf)); + Packet pow_is_inf_if_exp_is_pos = por(abs_x_is_inf, pand(abs_x_is_ge_one, exp_is_inf)); + Packet pow_is_one = pand(abs_x_is_one, por(exp_is_inf, x_is_ge_zero)); + + Packet result = powx; + result = por(x_is_le_zero, result); + result = pselect(pow_is_inf_if_exp_is_neg, pand(cst_pos_inf, exp_is_neg), result); + result = pselect(pow_is_inf_if_exp_is_pos, pand(cst_pos_inf, exp_is_pos), result); + result = por(exp_is_nan, result); + result = pselect(pow_is_one, cst_pos_one, result); + return result; +} + +template ::type>::IsSigned, bool> = true> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet handle_negative_exponent(const Packet& x, const ScalarExponent& exponent) { + using Scalar = typename unpacket_traits::type; + + // singed integer base, signed integer exponent case + + // This routine handles negative exponents. + // The return value is either 0, 1, or -1. + + const Scalar pos_zero = Scalar(0); + const Scalar all_ones = ptrue(Scalar()); + const Scalar pos_one = Scalar(1); + + const Packet cst_pos_one = pset1(pos_one); + + const bool exponent_is_odd = exponent % ScalarExponent(2) != ScalarExponent(0); + + const Packet exp_is_odd = pset1(exponent_is_odd ? all_ones : pos_zero); + + const Packet abs_x = pabs(x); + const Packet abs_x_is_one = pcmp_eq(abs_x, cst_pos_one); + + Packet result = pselect(exp_is_odd, x, abs_x); + result = pand(abs_x_is_one, result); + return result; +} + +template ::type>::IsSigned, bool> = true> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet handle_negative_exponent(const Packet& x, const ScalarExponent&) { + using Scalar = typename unpacket_traits::type; + + // unsigned integer base, signed integer exponent case + + // This routine handles negative exponents. + // The return value is either 0 or 1 + + const Scalar pos_one = Scalar(1); + + const Packet cst_pos_one = pset1(pos_one); + + const Packet x_is_one = pcmp_eq(x, cst_pos_one); + + return pand(x_is_one, x); +} + + +} // end namespace unary_pow + +template ::type>::IsInteger, + bool ExponentIsIntegerType = NumTraits::IsInteger, + bool ExponentIsSigned = NumTraits::IsSigned> +struct unary_pow_impl; + +template +struct unary_pow_impl { + typedef typename unpacket_traits::type Scalar; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) { + const bool exponent_is_integer = (numext::isfinite)(exponent) && numext::round(exponent) == exponent; + if (exponent_is_integer) { + return unary_pow::int_pow(x, exponent); + } else { + Packet result = unary_pow::gen_pow(x, exponent); + result = unary_pow::handle_nonint_nonint_errors(x, result, exponent); + return result; + } + } +}; + +template +struct unary_pow_impl { + typedef typename unpacket_traits::type Scalar; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) { + return unary_pow::int_pow(x, exponent); + } +}; + +template +struct unary_pow_impl { + typedef typename unpacket_traits::type Scalar; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) { + if (exponent < ScalarExponent(0)) { + return unary_pow::handle_negative_exponent(x, exponent); + } else { + return unary_pow::int_pow(x, exponent); + } + } +}; + +template +struct unary_pow_impl { + typedef typename unpacket_traits::type Scalar; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) { + return unary_pow::int_pow(x, exponent); + } +}; + } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h index 177a04e93e6..9e038aba2b2 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h @@ -10,6 +10,9 @@ #ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_FWD_H #define EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_FWD_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -42,25 +45,21 @@ Packet pldexp_generic(const Packet& a, const Packet& exponent); /** \internal \returns log(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet plog_float(const Packet _x); /** \internal \returns log2(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet plog2_float(const Packet _x); /** \internal \returns log(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet plog_double(const Packet _x); /** \internal \returns log2(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet plog2_double(const Packet _x); /** \internal \returns log(1 + x) */ @@ -74,35 +73,98 @@ Packet generic_expm1(const Packet& x); /** \internal \returns exp(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet pexp_float(const Packet _x); /** \internal \returns exp(x) for double precision real numbers */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet pexp_double(const Packet _x); /** \internal \returns sin(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet psin_float(const Packet& x); /** \internal \returns cos(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet pcos_float(const Packet& x); +/** \internal \returns asin(x) for single precision float */ +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet pasin_float(const Packet& x); + +/** \internal \returns acos(x) for single precision float */ +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet pacos_float(const Packet& x); + +/** \internal \returns atan(x) for single precision float */ +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet patan_float(const Packet& x); + +/** \internal \returns atan(x) for double precision float */ +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet patan_double(const Packet& x); + +/** \internal \returns atanh(x) for single precision float */ +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet patanh_float(const Packet& x); + /** \internal \returns sqrt(x) for complex types */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -EIGEN_UNUSED Packet psqrt_complex(const Packet& a); +/** \internal \returns x / y for complex types */ +template +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet pdiv_complex(const Packet& x, const Packet& y); + template struct ppolevl; +// Macros for instantiating these generic functions for different backends. +#define EIGEN_PACKET_FUNCTION(METHOD, SCALAR, PACKET) \ + template <> \ + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_UNUSED PACKET p##METHOD(const PACKET& _x) { \ + return p##METHOD##_##SCALAR(_x); \ + } + +#define EIGEN_FLOAT_PACKET_FUNCTION(METHOD, PACKET) EIGEN_PACKET_FUNCTION(METHOD, float, PACKET) +#define EIGEN_DOUBLE_PACKET_FUNCTION(METHOD, PACKET) EIGEN_PACKET_FUNCTION(METHOD, double, PACKET) + +#define EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(PACKET) \ + EIGEN_FLOAT_PACKET_FUNCTION(sin, PACKET) \ + EIGEN_FLOAT_PACKET_FUNCTION(cos, PACKET) \ + EIGEN_FLOAT_PACKET_FUNCTION(asin, PACKET) \ + EIGEN_FLOAT_PACKET_FUNCTION(acos, PACKET) \ + EIGEN_FLOAT_PACKET_FUNCTION(atan, PACKET) \ + EIGEN_FLOAT_PACKET_FUNCTION(atanh, PACKET) \ + EIGEN_FLOAT_PACKET_FUNCTION(log, PACKET) \ + EIGEN_FLOAT_PACKET_FUNCTION(log2, PACKET) \ + EIGEN_FLOAT_PACKET_FUNCTION(exp, PACKET) \ + template <> \ + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_UNUSED PACKET pexpm1(const PACKET& _x) { \ + return internal::generic_expm1(_x); \ + } \ + template <> \ + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_UNUSED PACKET plog1p(const PACKET& _x) { \ + return internal::generic_plog1p(_x); \ + } \ + template <> \ + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_UNUSED PACKET ptanh(const PACKET& _x) { \ + return internal::generic_fast_tanh_float(_x); \ + } + +#define EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE(PACKET) \ + EIGEN_DOUBLE_PACKET_FUNCTION(atan, PACKET) \ + EIGEN_DOUBLE_PACKET_FUNCTION(log, PACKET) \ + EIGEN_DOUBLE_PACKET_FUNCTION(log2, PACKET) \ + EIGEN_DOUBLE_PACKET_FUNCTION(exp, PACKET) } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/Half.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/Half.h index 9f8e8cc1e7a..c65231850d1 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/Half.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/Half.h @@ -36,7 +36,8 @@ #ifndef EIGEN_HALF_H #define EIGEN_HALF_H -#include +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" #if defined(EIGEN_HAS_GPU_FP16) || defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) // When compiling with GPU support, the "__half_raw" base class as well as @@ -202,57 +203,113 @@ struct half : public half_impl::half_base { #endif }; -} // end namespace Eigen - -namespace std { -template<> -struct numeric_limits { - static const bool is_specialized = true; - static const bool is_signed = true; - static const bool is_integer = false; - static const bool is_exact = false; - static const bool has_infinity = true; - static const bool has_quiet_NaN = true; - static const bool has_signaling_NaN = true; - static const float_denorm_style has_denorm = denorm_present; - static const bool has_denorm_loss = false; - static const std::float_round_style round_style = std::round_to_nearest; - static const bool is_iec559 = false; - static const bool is_bounded = false; - static const bool is_modulo = false; - static const int digits = 11; - static const int digits10 = 3; // according to http://half.sourceforge.net/structstd_1_1numeric__limits_3_01half__float_1_1half_01_4.html - static const int max_digits10 = 5; // according to http://half.sourceforge.net/structstd_1_1numeric__limits_3_01half__float_1_1half_01_4.html - static const int radix = 2; - static const int min_exponent = -13; - static const int min_exponent10 = -4; - static const int max_exponent = 16; - static const int max_exponent10 = 4; - static const bool traps = true; - static const bool tinyness_before = false; - - static Eigen::half (min)() { return Eigen::half_impl::raw_uint16_to_half(0x400); } - static Eigen::half lowest() { return Eigen::half_impl::raw_uint16_to_half(0xfbff); } - static Eigen::half (max)() { return Eigen::half_impl::raw_uint16_to_half(0x7bff); } - static Eigen::half epsilon() { return Eigen::half_impl::raw_uint16_to_half(0x0800); } - static Eigen::half round_error() { return Eigen::half(0.5); } - static Eigen::half infinity() { return Eigen::half_impl::raw_uint16_to_half(0x7c00); } - static Eigen::half quiet_NaN() { return Eigen::half_impl::raw_uint16_to_half(0x7e00); } - static Eigen::half signaling_NaN() { return Eigen::half_impl::raw_uint16_to_half(0x7d00); } - static Eigen::half denorm_min() { return Eigen::half_impl::raw_uint16_to_half(0x1); } +// TODO(majnemer): Get rid of this once we can rely on C++17 inline variables do +// solve the ODR issue. +namespace half_impl { +template +struct numeric_limits_half_impl { + static EIGEN_CONSTEXPR const bool is_specialized = true; + static EIGEN_CONSTEXPR const bool is_signed = true; + static EIGEN_CONSTEXPR const bool is_integer = false; + static EIGEN_CONSTEXPR const bool is_exact = false; + static EIGEN_CONSTEXPR const bool has_infinity = true; + static EIGEN_CONSTEXPR const bool has_quiet_NaN = true; + static EIGEN_CONSTEXPR const bool has_signaling_NaN = true; + static EIGEN_CONSTEXPR const std::float_denorm_style has_denorm = std::denorm_present; + static EIGEN_CONSTEXPR const bool has_denorm_loss = false; + static EIGEN_CONSTEXPR const std::float_round_style round_style = std::round_to_nearest; + static EIGEN_CONSTEXPR const bool is_iec559 = true; + // The C++ standard defines this as "true if the set of values representable + // by the type is finite." Half has finite precision. + static EIGEN_CONSTEXPR const bool is_bounded = true; + static EIGEN_CONSTEXPR const bool is_modulo = false; + static EIGEN_CONSTEXPR const int digits = 11; + static EIGEN_CONSTEXPR const int digits10 = 3; // according to http://half.sourceforge.net/structstd_1_1numeric__limits_3_01half__float_1_1half_01_4.html + static EIGEN_CONSTEXPR const int max_digits10 = 5; // according to http://half.sourceforge.net/structstd_1_1numeric__limits_3_01half__float_1_1half_01_4.html + static EIGEN_CONSTEXPR const int radix = std::numeric_limits::radix; + static EIGEN_CONSTEXPR const int min_exponent = -13; + static EIGEN_CONSTEXPR const int min_exponent10 = -4; + static EIGEN_CONSTEXPR const int max_exponent = 16; + static EIGEN_CONSTEXPR const int max_exponent10 = 4; + static EIGEN_CONSTEXPR const bool traps = std::numeric_limits::traps; + // IEEE754: "The implementer shall choose how tininess is detected, but shall + // detect tininess in the same way for all operations in radix two" + static EIGEN_CONSTEXPR const bool tinyness_before = std::numeric_limits::tinyness_before; + + static EIGEN_CONSTEXPR Eigen::half (min)() { return Eigen::half_impl::raw_uint16_to_half(0x0400); } + static EIGEN_CONSTEXPR Eigen::half lowest() { return Eigen::half_impl::raw_uint16_to_half(0xfbff); } + static EIGEN_CONSTEXPR Eigen::half (max)() { return Eigen::half_impl::raw_uint16_to_half(0x7bff); } + static EIGEN_CONSTEXPR Eigen::half epsilon() { return Eigen::half_impl::raw_uint16_to_half(0x1400); } + static EIGEN_CONSTEXPR Eigen::half round_error() { return Eigen::half_impl::raw_uint16_to_half(0x3800); } + static EIGEN_CONSTEXPR Eigen::half infinity() { return Eigen::half_impl::raw_uint16_to_half(0x7c00); } + static EIGEN_CONSTEXPR Eigen::half quiet_NaN() { return Eigen::half_impl::raw_uint16_to_half(0x7e00); } + static EIGEN_CONSTEXPR Eigen::half signaling_NaN() { return Eigen::half_impl::raw_uint16_to_half(0x7d00); } + static EIGEN_CONSTEXPR Eigen::half denorm_min() { return Eigen::half_impl::raw_uint16_to_half(0x0001); } }; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::is_specialized; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::is_signed; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::is_integer; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::is_exact; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::has_infinity; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::has_quiet_NaN; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::has_signaling_NaN; +template +EIGEN_CONSTEXPR const std::float_denorm_style numeric_limits_half_impl::has_denorm; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::has_denorm_loss; +template +EIGEN_CONSTEXPR const std::float_round_style numeric_limits_half_impl::round_style; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::is_iec559; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::is_bounded; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::is_modulo; +template +EIGEN_CONSTEXPR const int numeric_limits_half_impl::digits; +template +EIGEN_CONSTEXPR const int numeric_limits_half_impl::digits10; +template +EIGEN_CONSTEXPR const int numeric_limits_half_impl::max_digits10; +template +EIGEN_CONSTEXPR const int numeric_limits_half_impl::radix; +template +EIGEN_CONSTEXPR const int numeric_limits_half_impl::min_exponent; +template +EIGEN_CONSTEXPR const int numeric_limits_half_impl::min_exponent10; +template +EIGEN_CONSTEXPR const int numeric_limits_half_impl::max_exponent; +template +EIGEN_CONSTEXPR const int numeric_limits_half_impl::max_exponent10; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::traps; +template +EIGEN_CONSTEXPR const bool numeric_limits_half_impl::tinyness_before; +} // end namespace half_impl +} // end namespace Eigen + +namespace std { // If std::numeric_limits is specialized, should also specialize // std::numeric_limits, std::numeric_limits, and // std::numeric_limits // https://stackoverflow.com/a/16519653/ template<> -struct numeric_limits : numeric_limits {}; +class numeric_limits : public Eigen::half_impl::numeric_limits_half_impl<> {}; template<> -struct numeric_limits : numeric_limits {}; +class numeric_limits : public numeric_limits {}; template<> -struct numeric_limits : numeric_limits {}; -} // end namespace std +class numeric_limits : public numeric_limits {}; +template<> +class numeric_limits : public numeric_limits {}; +} // end namespace std namespace Eigen { @@ -261,7 +318,7 @@ namespace half_impl { #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && \ EIGEN_CUDA_ARCH >= 530) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(HIP_DEVICE_COMPILE)) -// Note: We deliberatly do *not* define this to 1 even if we have Arm's native +// Note: We deliberately do *not* define this to 1 even if we have Arm's native // fp16 type since GPU halfs are rather different from native CPU halfs. // TODO: Rename to something like EIGEN_HAS_NATIVE_GPU_FP16 #define EIGEN_HAS_NATIVE_FP16 @@ -334,7 +391,7 @@ EIGEN_STRONG_INLINE __device__ bool operator >= (const half& a, const half& b) { } #endif -#if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) +#if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) && !defined(EIGEN_GPU_COMPILE_PHASE) EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator + (const half& a, const half& b) { return half(vaddh_f16(a.x, b.x)); } @@ -389,7 +446,7 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator >= (const half& a, const hal // of the functions, while the latter can only deal with one of them. #elif !defined(EIGEN_HAS_NATIVE_FP16) || (EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) // Emulate support for half floats -#if EIGEN_COMP_CLANG && defined(EIGEN_CUDACC) +#if EIGEN_COMP_CLANG && defined(EIGEN_GPUCC) // We need to provide emulated *host-side* FP16 operators for clang. #pragma push_macro("EIGEN_DEVICE_FUNC") #undef EIGEN_DEVICE_FUNC @@ -454,7 +511,7 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator >= (const half& a, const hal return float(a) >= float(b); } -#if defined(__clang__) && defined(__CUDA__) +#if EIGEN_COMP_CLANG && defined(EIGEN_GPUCC) #pragma pop_macro("EIGEN_DEVICE_FUNC") #endif #endif // Emulate support for half floats @@ -534,7 +591,12 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw float_to_half_rtne(float ff) { #elif defined(EIGEN_HAS_FP16_C) __half_raw h; - h.x = _cvtss_sh(ff, 0); + #if EIGEN_COMP_MSVC + // MSVC does not have scalar instructions. + h.x =_mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(ff), 0), 0); + #else + h.x = _cvtss_sh(ff, 0); + #endif return h; #elif defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) @@ -595,7 +657,12 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float half_to_float(__half_raw h) { (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) return __half2float(h); #elif defined(EIGEN_HAS_FP16_C) - return _cvtsh_ss(h.x); + #if EIGEN_COMP_MSVC + // MSVC does not have scalar instructions. + return _mm_cvtss_f32(_mm_cvtph_ps(_mm_set1_epi16(h.x))); + #else + return _cvtsh_ss(h.x); + #endif #elif defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) return static_cast(h.x); #else @@ -692,6 +759,9 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half sqrt(const half& a) { EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half pow(const half& a, const half& b) { return half(::powf(float(a), float(b))); } +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half atan2(const half& a, const half& b) { + return half(::atan2f(float(a), float(b))); +} EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half sin(const half& a) { return half(::sinf(float(a))); } @@ -710,6 +780,12 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half asin(const half& a) { EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half acos(const half& a) { return half(::acosf(float(a))); } +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half atan(const half& a) { + return half(::atanf(float(a))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half atanh(const half& a) { + return half(::atanhf(float(a))); +} EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half floor(const half& a) { #if (EIGEN_CUDA_SDK_VER >= 80000 && defined EIGEN_CUDA_ARCH && EIGEN_CUDA_ARCH >= 300) || \ defined(EIGEN_HIP_DEVICE_COMPILE) @@ -939,4 +1015,49 @@ struct hash { } // end namespace std #endif +namespace Eigen { +namespace internal { + +template <> +struct cast_impl { + EIGEN_DEVICE_FUNC + static inline half run(const float& a) { +#if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + return __float2half(a); +#else + return half(a); +#endif + } +}; + +template <> +struct cast_impl { + EIGEN_DEVICE_FUNC + static inline half run(const int& a) { +#if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + return __float2half(static_cast(a)); +#else + return half(static_cast(a)); +#endif + } +}; + +template <> +struct cast_impl { + EIGEN_DEVICE_FUNC + static inline float run(const half& a) { +#if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + return __half2float(a); +#else + return static_cast(a); +#endif + } +}; + +} // namespace internal +} // namespace Eigen + #endif // EIGEN_HALF_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/TypeCasting.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/TypeCasting.h deleted file mode 100644 index fb8183b78ec..00000000000 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/Default/TypeCasting.h +++ /dev/null @@ -1,120 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2016 Benoit Steiner -// Copyright (C) 2019 Rasmus Munk Larsen -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_GENERIC_TYPE_CASTING_H -#define EIGEN_GENERIC_TYPE_CASTING_H - -namespace Eigen { - -namespace internal { - -template<> -struct scalar_cast_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) - typedef Eigen::half result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half operator() (const float& a) const { - #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ - (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) - return __float2half(a); - #else - return Eigen::half(a); - #endif - } -}; - -template<> -struct functor_traits > -{ enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; - - -template<> -struct scalar_cast_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) - typedef Eigen::half result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half operator() (const int& a) const { - #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ - (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) - return __float2half(static_cast(a)); - #else - return Eigen::half(static_cast(a)); - #endif - } -}; - -template<> -struct functor_traits > -{ enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; - - -template<> -struct scalar_cast_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) - typedef float result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator() (const Eigen::half& a) const { - #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ - (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) - return __half2float(a); - #else - return static_cast(a); - #endif - } -}; - -template<> -struct functor_traits > -{ enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; - - -template<> -struct scalar_cast_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) - typedef Eigen::bfloat16 result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::bfloat16 operator() (const float& a) const { - return Eigen::bfloat16(a); - } -}; - -template<> -struct functor_traits > -{ enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; - - -template<> -struct scalar_cast_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) - typedef Eigen::bfloat16 result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::bfloat16 operator() (const int& a) const { - return Eigen::bfloat16(static_cast(a)); - } -}; - -template<> -struct functor_traits > -{ enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; - - -template<> -struct scalar_cast_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) - typedef float result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator() (const Eigen::bfloat16& a) const { - return static_cast(a); - } -}; - -template<> -struct functor_traits > -{ enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; - - -} -} - -#endif // EIGEN_GENERIC_TYPE_CASTING_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/CUDA/Complex.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/Complex.h similarity index 98% rename from Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/CUDA/Complex.h rename to Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/Complex.h index 45f6ddb949f..8a7869c2fe1 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/CUDA/Complex.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/Complex.h @@ -8,8 +8,8 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#ifndef EIGEN_COMPLEX_CUDA_H -#define EIGEN_COMPLEX_CUDA_H +#ifndef EIGEN_COMPLEX_GPU_H +#define EIGEN_COMPLEX_GPU_H // Many std::complex methods such as operator+, operator-, operator* and // operator/ are not constexpr. Due to this, GCC and older versions of clang do @@ -30,7 +30,7 @@ // - Compiling with ICC requires defining _USE_COMPLEX_SPECIALIZATION_ prior // to the first inclusion of . -#if defined(EIGEN_CUDACC) && defined(EIGEN_GPU_COMPILE_PHASE) +#if defined(EIGEN_GPUCC) && defined(EIGEN_GPU_COMPILE_PHASE) // ICC already specializes std::complex and std::complex // operators, preventing us from making them device functions here. @@ -39,7 +39,7 @@ // To allow std::complex operator use on device, define _OVERRIDE_COMPLEX_SPECIALIZATION_ // prior to first inclusion of . This prevents ICC from adding // its own specializations, so our custom ones below can be used instead. -#if !(defined(EIGEN_COMP_ICC) && defined(_USE_COMPLEX_SPECIALIZATION_)) +#if !(EIGEN_COMP_ICC && defined(_USE_COMPLEX_SPECIALIZATION_)) // Import Eigen's internal operator specializations. #define EIGEN_USING_STD_COMPLEX_OPERATORS \ @@ -54,6 +54,9 @@ using Eigen::complex_operator_detail::operator==; \ using Eigen::complex_operator_detail::operator!=; +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { // Specialized std::complex overloads. @@ -264,6 +267,6 @@ EIGEN_USING_STD_COMPLEX_OPERATORS #endif // !(EIGEN_COMP_ICC && _USE_COMPLEX_SPECIALIZATION_) -#endif // EIGEN_CUDACC && EIGEN_GPU_COMPILE_PHASE +#endif // EIGEN_GPUCC && EIGEN_GPU_COMPILE_PHASE -#endif // EIGEN_COMPLEX_CUDA_H +#endif // EIGEN_COMPLEX_GPU_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/MathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/MathFunctions.h index d2b3a25684a..f8191db607c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/MathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/MathFunctions.h @@ -10,6 +10,9 @@ #ifndef EIGEN_MATH_FUNCTIONS_GPU_H #define EIGEN_MATH_FUNCTIONS_GPU_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/PacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/PacketMath.h index 25c45fd3557..5c959edc61a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/PacketMath.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/PacketMath.h @@ -10,6 +10,9 @@ #ifndef EIGEN_PACKET_MATH_GPU_H #define EIGEN_PACKET_MATH_GPU_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -44,7 +47,6 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size=4, - HasHalfPacket = 0, HasDiv = 1, HasSin = 0, @@ -80,7 +82,6 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size=2, - HasHalfPacket = 0, HasDiv = 1, HasLog = 1, @@ -121,7 +122,6 @@ template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pset1(const do // invoked by NVCC’ (e.g. on MacOS). The former needs to see both host and device implementation // of the functions, while the latter can only deal with one of them. #if defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIPCC) || (defined(EIGEN_CUDACC) && EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) -namespace { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float bitwise_and(const float& a, const float& b) { @@ -175,12 +175,21 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float lt_mask(const float& a, const float& b) { return __int_as_float(a < b ? 0xffffffffu : 0u); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double lt_mask(const double& a, const double& b) { return __longlong_as_double(a < b ? 0xffffffffffffffffull : 0ull); } -} // namespace +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float le_mask(const float& a, + const float& b) { + return __int_as_float(a <= b ? 0xffffffffu : 0u); +} + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double le_mask(const double& a, + const double& b) { + return __longlong_as_double(a <= b ? 0xffffffffffffffffull : 0ull); +} template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pand(const float4& a, @@ -243,6 +252,12 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_lt(const float4& a, lt_mask(a.w, b.w)); } template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_le(const float4& a, + const float4& b) { + return make_float4(le_mask(a.x, b.x), le_mask(a.y, b.y), le_mask(a.z, b.z), + le_mask(a.w, b.w)); +} +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_eq(const double2& a, const double2& b) { return make_double2(eq_mask(a.x, b.x), eq_mask(a.y, b.y)); @@ -252,6 +267,11 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_lt(const double2& a, const double2& b) { return make_double2(lt_mask(a.x, b.x), lt_mask(a.y, b.y)); } +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 +pcmp_le(const double2& a, const double2& b) { + return make_double2(le_mask(a.x, b.x), le_mask(a.y, b.y)); +} #endif // defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIPCC) || (defined(EIGEN_CUDACC) && EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 plset(const float& a) { @@ -358,7 +378,7 @@ template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstoreu(double* to template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float4 ploadt_ro(const float* from) { #if defined(EIGEN_GPU_HAS_LDG) - return __ldg((const float4*)from); + return __ldg(reinterpret_cast(from)); #else return make_float4(from[0], from[1], from[2], from[3]); #endif @@ -366,7 +386,7 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float4 ploadt_ro(const fl template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double2 ploadt_ro(const double* from) { #if defined(EIGEN_GPU_HAS_LDG) - return __ldg((const double2*)from); + return __ldg(reinterpret_cast(from)); #else return make_double2(from[0], from[1]); #endif @@ -513,7 +533,6 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size=8, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, @@ -677,6 +696,19 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcmp_lt(const half2& a, return __halves2half2(eq1, eq2); } +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcmp_le(const half2& a, + const half2& b) { + half true_half = half_impl::raw_uint16_to_half(0xffffu); + half false_half = half_impl::raw_uint16_to_half(0x0000u); + half a1 = __low2half(a); + half a2 = __high2half(a); + half b1 = __low2half(b); + half b2 = __high2half(b); + half eq1 = __half2float(a1) <= __half2float(b1) ? true_half : false_half; + half eq2 = __half2float(a2) <= __half2float(b2) ? true_half : false_half; + return __halves2half2(eq1, eq2); +} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pand(const half2& a, const half2& b) { half a1 = __low2half(a); @@ -1258,6 +1290,34 @@ pcmp_eq(const Packet4h2& a, const Packet4h2& b) { return r; } +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 +pcmp_lt(const Packet4h2& a, const Packet4h2& b) { + Packet4h2 r; + half2* r_alias = reinterpret_cast(&r); + const half2* a_alias = reinterpret_cast(&a); + const half2* b_alias = reinterpret_cast(&b); + r_alias[0] = pcmp_lt(a_alias[0], b_alias[0]); + r_alias[1] = pcmp_lt(a_alias[1], b_alias[1]); + r_alias[2] = pcmp_lt(a_alias[2], b_alias[2]); + r_alias[3] = pcmp_lt(a_alias[3], b_alias[3]); + return r; +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 +pcmp_le(const Packet4h2& a, const Packet4h2& b) { + Packet4h2 r; + half2* r_alias = reinterpret_cast(&r); + const half2* a_alias = reinterpret_cast(&a); + const half2* b_alias = reinterpret_cast(&b); + r_alias[0] = pcmp_le(a_alias[0], b_alias[0]); + r_alias[1] = pcmp_le(a_alias[1], b_alias[1]); + r_alias[2] = pcmp_le(a_alias[2], b_alias[2]); + r_alias[3] = pcmp_le(a_alias[3], b_alias[3]); + return r; +} + template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pand( const Packet4h2& a, const Packet4h2& b) { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/Tuple.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/Tuple.h new file mode 100644 index 00000000000..e223ca1d2c1 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/Tuple.h @@ -0,0 +1,302 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2021 The Eigen Team +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_TUPLE_GPU +#define EIGEN_TUPLE_GPU + +#include +#include + +// This is a replacement of std::tuple that can be used in device code. + +namespace Eigen { +namespace internal { +namespace tuple_impl { + +// Internal tuple implementation. +template +class TupleImpl; + +// Generic recursive tuple. +template +class TupleImpl { + public: + // Tuple may contain Eigen types. + EIGEN_MAKE_ALIGNED_OPERATOR_NEW + + // Default constructor, enable if all types are default-constructible. + template::value + && reduce_all::value...>::value + >> + EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC + TupleImpl() : head_{}, tail_{} {} + + // Element constructor. + template 1 || std::is_convertible::value) + >> + EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC + TupleImpl(U1&& arg1, Us&&... args) + : head_(std::forward(arg1)), tail_(std::forward(args)...) {} + + // The first stored value. + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE + T1& head() { + return head_; + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE + const T1& head() const { + return head_; + } + + // The tail values. + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE + TupleImpl& tail() { + return tail_; + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE + const TupleImpl& tail() const { + return tail_; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void swap(TupleImpl& other) { + using numext::swap; + swap(head_, other.head_); + swap(tail_, other.tail_); + } + + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + TupleImpl& operator=(const TupleImpl& other) { + head_ = other.head_; + tail_ = other.tail_; + return *this; + } + + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + TupleImpl& operator=(TupleImpl&& other) { + head_ = std::move(other.head_); + tail_ = std::move(other.tail_); + return *this; + } + + private: + // Allow related tuples to reference head_/tail_. + template + friend class TupleImpl; + + T1 head_; + TupleImpl tail_; +}; + +// Empty tuple specialization. +template<> +class TupleImpl {}; + +template +struct is_tuple : std::false_type {}; + +template +struct is_tuple< TupleImpl > : std::true_type {}; + +// Gets an element from a tuple. +template +struct tuple_get_impl { + using TupleType = TupleImpl; + using ReturnType = typename tuple_get_impl::ReturnType; + + static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE + ReturnType& run(TupleType& tuple) { + return tuple_get_impl::run(tuple.tail()); + } + + static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE + const ReturnType& run(const TupleType& tuple) { + return tuple_get_impl::run(tuple.tail()); + } +}; + +// Base case, getting the head element. +template +struct tuple_get_impl<0, T1, Ts...> { + using TupleType = TupleImpl; + using ReturnType = T1; + + static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE + T1& run(TupleType& tuple) { + return tuple.head(); + } + + static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE + const T1& run(const TupleType& tuple) { + return tuple.head(); + } +}; + +// Concatenates N Tuples. +template +struct tuple_cat_impl; + +template +struct tuple_cat_impl, TupleImpl, Tuples...> { + using TupleType1 = TupleImpl; + using TupleType2 = TupleImpl; + using MergedTupleType = TupleImpl; + + using ReturnType = typename tuple_cat_impl::ReturnType; + + // Uses the index sequences to extract and merge elements from tuple1 and tuple2, + // then recursively calls again. + template + static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + ReturnType run(Tuple1&& tuple1, std::index_sequence, + Tuple2&& tuple2, std::index_sequence, + MoreTuples&&... tuples) { + return tuple_cat_impl::run( + MergedTupleType(tuple_get_impl::run(std::forward(tuple1))..., + tuple_get_impl::run(std::forward(tuple2))...), + std::forward(tuples)...); + } + + // Concatenates the first two tuples. + template + static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + ReturnType run(Tuple1&& tuple1, Tuple2&& tuple2, MoreTuples&&... tuples) { + return run(std::forward(tuple1), std::make_index_sequence{}, + std::forward(tuple2), std::make_index_sequence{}, + std::forward(tuples)...); + } +}; + +// Base case with a single tuple. +template +struct tuple_cat_impl<1, TupleImpl > { + using ReturnType = TupleImpl; + + template + static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + ReturnType run(Tuple1&& tuple1) { + return tuple1; + } +}; + +// Special case of no tuples. +template<> +struct tuple_cat_impl<0> { + using ReturnType = TupleImpl<0>; + static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + ReturnType run() {return ReturnType{}; } +}; + +// For use in make_tuple, unwraps a reference_wrapper. +template +struct unwrap_reference_wrapper { using type = T; }; + +template +struct unwrap_reference_wrapper > { using type = T&; }; + +// For use in make_tuple, decays a type and unwraps a reference_wrapper. +template +struct unwrap_decay { + using type = typename unwrap_reference_wrapper::type>::type; +}; + +/** + * Utility for determining a tuple's size. + */ +template +struct tuple_size; + +template +struct tuple_size< TupleImpl > : std::integral_constant {}; + +/** + * Gets an element of a tuple. + * \tparam Idx index of the element. + * \tparam Types ... tuple element types. + * \param tuple the tuple. + * \return a reference to the desired element. + */ +template +EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const typename tuple_get_impl::ReturnType& +get(const TupleImpl& tuple) { + return tuple_get_impl::run(tuple); +} + +template +EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename tuple_get_impl::ReturnType& +get(TupleImpl& tuple) { + return tuple_get_impl::run(tuple); +} + +/** + * Concatenate multiple tuples. + * \param tuples ... list of tuples. + * \return concatenated tuple. + */ +template::type>::value...>::value>> +EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename tuple_cat_impl::type...>::ReturnType +tuple_cat(Tuples&&... tuples) { + return tuple_cat_impl::type...>::run(std::forward(tuples)...); +} + +/** + * Tie arguments together into a tuple. + */ +template > +EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +ReturnType tie(Args&... args) EIGEN_NOEXCEPT { + return ReturnType{args...}; +} + +/** + * Create a tuple of l-values with the supplied arguments. + */ +template ::type...> > +EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +ReturnType make_tuple(Args&&... args) { + return ReturnType{std::forward(args)...}; +} + +/** + * Forward a set of arguments as a tuple. + */ +template +EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +TupleImpl forward_as_tuple(Args&&... args) { + return TupleImpl(std::forward(args)...); +} + +/** + * Alternative to std::tuple that can be used on device. + */ +template +using tuple = TupleImpl; + +} // namespace tuple_impl +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_TUPLE_GPU diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/TypeCasting.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/TypeCasting.h index c8195bb2b09..aa89cd2bc44 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/TypeCasting.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/GPU/TypeCasting.h @@ -10,6 +10,9 @@ #ifndef EIGEN_TYPE_CASTING_GPU_H #define EIGEN_TYPE_CASTING_GPU_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/HVX/GeneralBlockPanelKernel.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/HVX/GeneralBlockPanelKernel.h new file mode 100644 index 00000000000..51f37fa4148 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/HVX/GeneralBlockPanelKernel.h @@ -0,0 +1,46 @@ +#ifndef EIGEN_HVX_GENERAL_BLOCK_KERNEL_H +#define EIGEN_HVX_GENERAL_BLOCK_KERNEL_H + +// Only support 128B HVX now. +// Floating-point operations are only supported since V68. +#if defined __HVX__ && (__HVX_LENGTH__ == 128) && __HVX_ARCH__ >= 68 + +namespace Eigen { +namespace internal { + +template +class gebp_traits + : public gebp_traits { + public: + typedef Packet32qf AccPacket; + + EIGEN_STRONG_INLINE void initAcc(Packet32qf& p) { p = pzero(p); } + + template + EIGEN_STRONG_INLINE void madd(const Packet32f& a, const Packet32f& b, + Packet32qf& c, Packet32f& /*tmp*/, + const LaneIdType&) const { + c = pmadd_f32_to_qf32(a, b, c); + } + + template + EIGEN_STRONG_INLINE void madd(const Packet32f& a, + const QuadPacket& b, Packet32qf& c, + Packet32f& tmp, const LaneIdType& lane) const { + madd(a, b.get(lane), c, tmp, lane); + } + + EIGEN_STRONG_INLINE void acc(const Packet32qf& c, const Packet32f& alpha, + Packet32f& r) const { + r = pmadd_qf32_to_f32(c, alpha, r); + } +}; + +} // end namespace internal +} // end namespace Eigen + +#endif // __HVX__ && (__HVX_LENGTH__ == 128) && __HVX_ARCH__ >= 68 + +#endif // EIGEN_HVX_GENERAL_BLOCK_KERNEL_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/HVX/PacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/HVX/PacketMath.h new file mode 100644 index 00000000000..cc8722fdda9 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/HVX/PacketMath.h @@ -0,0 +1,548 @@ + +#ifndef EIGEN_HVX_PACKET_MATH_H +#define EIGEN_HVX_PACKET_MATH_H + +// Only support 128B HVX now. +// Floating-point operations are supported only since V68. +#if defined __HVX__ && (__HVX_LENGTH__ == 128) && __HVX_ARCH__ >= 68 + +// All the floating-point operations do not support IEEE standard. +// From HVX document: +// There is no concept of infinity or NaN. QFloat saturates to maximum +// exponent with maximum positive or minimum negative significand. + +#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS +#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 +#endif + +namespace Eigen { +namespace internal { + +EIGEN_STRONG_INLINE HVX_Vector HVX_load(const void* mem) { + return *((HVX_Vector*)mem); +} + +EIGEN_STRONG_INLINE HVX_Vector HVX_loadu(const void* mem) { + return *((HVX_UVector*)mem); +} + +EIGEN_STRONG_INLINE void HVX_store(void* mem, HVX_Vector v) { + *((HVX_Vector*)mem) = v; +} + +EIGEN_STRONG_INLINE void HVX_storeu(void* mem, HVX_Vector v) { + *((HVX_UVector*)mem) = v; +} + +// Hexagon compiler uses same HVX_Vector to represent all HVX vector types. +// Wrap different vector type (float32, int32, etc) to different class with +// explicit constructor and casting back-and-force to HVX_Vector. +template +class HVXPacket { + public: + HVXPacket() = default; + static HVXPacket Create(HVX_Vector v) { return HVXPacket(v); } + HVX_Vector Get() const { return m_val; } + + private: + explicit HVXPacket(HVX_Vector v) : m_val(v) {} + HVX_Vector m_val = Q6_V_vzero(); +}; + +typedef HVXPacket<0> Packet32f; // float32 +typedef HVXPacket<1> Packet32qf; // qfloat32 + +template <> +struct packet_traits : default_packet_traits { + typedef Packet32f type; + typedef Packet32f half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 32, + }; +}; + +template <> +struct unpacket_traits { + typedef float type; + typedef Packet32f half; + enum { + size = 32, + alignment = Aligned128, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +// float32 operations. +template <> +EIGEN_STRONG_INLINE Packet32f pset1(const float& from) { + union { + float f; + int32_t i; + } u; + u.f = from; + return Packet32f::Create(Q6_V_vsplat_R(u.i)); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pload(const float* from) { + return Packet32f::Create(HVX_load(from)); +} +template <> +EIGEN_STRONG_INLINE Packet32f ploadu(const float* from) { + return Packet32f::Create(HVX_loadu(from)); +} + +template <> +EIGEN_STRONG_INLINE void pstore(float* to, const Packet32f& from) { + HVX_store(to, from.Get()); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet32f& from) { + HVX_storeu(to, from.Get()); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pmul(const Packet32f& a, + const Packet32f& b) { + return Packet32f::Create( + Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a.Get(), b.Get()))); +} + +template <> +EIGEN_STRONG_INLINE Packet32f padd(const Packet32f& a, + const Packet32f& b) { + return Packet32f::Create( + Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a.Get(), b.Get()))); +} + +template <> +EIGEN_STRONG_INLINE Packet32f psub(const Packet32f& a, + const Packet32f& b) { + return Packet32f::Create( + Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(a.Get(), b.Get()))); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pnegate(const Packet32f& a) { + return psub(Packet32f::Create(Q6_V_vzero()), a); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pcmp_le(const Packet32f& a, const Packet32f& b) { + HVX_Vector v_true = Q6_Vb_vsplat_R(0xff); + HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(a.Get(), b.Get()); + return Packet32f::Create(Q6_V_vmux_QVV(pred, Q6_V_vzero(), v_true)); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pcmp_eq(const Packet32f& a, const Packet32f& b) { + HVX_Vector v_true = Q6_Vb_vsplat_R(0xff); + HVX_VectorPred pred = Q6_Q_vcmp_eq_VwVw(a.Get(), b.Get()); + return Packet32f::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero())); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pcmp_lt(const Packet32f& a, const Packet32f& b) { + HVX_Vector v_true = Q6_Vb_vsplat_R(0xff); + HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(b.Get(), a.Get()); + return Packet32f::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero())); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pcmp_lt_or_nan(const Packet32f& a, + const Packet32f& b) { + HVX_Vector v_true = Q6_Vb_vsplat_R(0xff); + HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(b.Get(), a.Get()); + return Packet32f::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero())); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pabs(const Packet32f& a) { + HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(a.Get(), Q6_V_vzero()); + return Packet32f::Create(Q6_V_vmux_QVV(pred, a.Get(), pnegate(a).Get())); +} + +template <> +EIGEN_STRONG_INLINE float pfirst(const Packet32f& a) { + float vsf[32] __attribute__((aligned(128))); + pstore(vsf, a); + return vsf[0]; +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + // Shuffle the 32-bit lanes. + HVX_VectorPair v_0_1_0 = + Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4); + HVX_VectorPair v_0_3_2 = + Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4); + + // Shuffle the 64-bit lanes. + HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), + HEXAGON_HVX_GET_V0(v_0_1_0), -8); + HVX_VectorPair v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_3_2), + HEXAGON_HVX_GET_V1(v_0_1_0), -8); + + kernel.packet[0] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_1_1_0)); + kernel.packet[1] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_1_1_0)); + kernel.packet[2] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_1_3_2)); + kernel.packet[3] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_1_3_2)); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + // Shuffle the 32-bit lanes. + HVX_VectorPair v_0_1_0 = + Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4); + HVX_VectorPair v_0_3_2 = + Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4); + HVX_VectorPair v_0_5_4 = + Q6_W_vshuff_VVR(kernel.packet[5].Get(), kernel.packet[4].Get(), -4); + HVX_VectorPair v_0_7_6 = + Q6_W_vshuff_VVR(kernel.packet[7].Get(), kernel.packet[6].Get(), -4); + HVX_VectorPair v_0_9_8 = + Q6_W_vshuff_VVR(kernel.packet[9].Get(), kernel.packet[8].Get(), -4); + HVX_VectorPair v_0_11_10 = + Q6_W_vshuff_VVR(kernel.packet[11].Get(), kernel.packet[10].Get(), -4); + HVX_VectorPair v_0_13_12 = + Q6_W_vshuff_VVR(kernel.packet[13].Get(), kernel.packet[12].Get(), -4); + HVX_VectorPair v_0_15_14 = + Q6_W_vshuff_VVR(kernel.packet[15].Get(), kernel.packet[14].Get(), -4); + HVX_VectorPair v_0_17_16 = + Q6_W_vshuff_VVR(kernel.packet[17].Get(), kernel.packet[16].Get(), -4); + HVX_VectorPair v_0_19_18 = + Q6_W_vshuff_VVR(kernel.packet[19].Get(), kernel.packet[18].Get(), -4); + HVX_VectorPair v_0_21_20 = + Q6_W_vshuff_VVR(kernel.packet[21].Get(), kernel.packet[20].Get(), -4); + HVX_VectorPair v_0_23_22 = + Q6_W_vshuff_VVR(kernel.packet[23].Get(), kernel.packet[22].Get(), -4); + HVX_VectorPair v_0_25_24 = + Q6_W_vshuff_VVR(kernel.packet[25].Get(), kernel.packet[24].Get(), -4); + HVX_VectorPair v_0_27_26 = + Q6_W_vshuff_VVR(kernel.packet[27].Get(), kernel.packet[26].Get(), -4); + HVX_VectorPair v_0_29_28 = + Q6_W_vshuff_VVR(kernel.packet[29].Get(), kernel.packet[28].Get(), -4); + HVX_VectorPair v_0_31_30 = + Q6_W_vshuff_VVR(kernel.packet[31].Get(), kernel.packet[30].Get(), -4); + + // Shuffle the 64-bit lanes. + HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), + HEXAGON_HVX_GET_V0(v_0_1_0), -8); + HVX_VectorPair v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_3_2), + HEXAGON_HVX_GET_V1(v_0_1_0), -8); + HVX_VectorPair v_1_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_7_6), + HEXAGON_HVX_GET_V0(v_0_5_4), -8); + HVX_VectorPair v_1_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_7_6), + HEXAGON_HVX_GET_V1(v_0_5_4), -8); + HVX_VectorPair v_1_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_11_10), + HEXAGON_HVX_GET_V0(v_0_9_8), -8); + HVX_VectorPair v_1_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_11_10), + HEXAGON_HVX_GET_V1(v_0_9_8), -8); + HVX_VectorPair v_1_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_15_14), + HEXAGON_HVX_GET_V0(v_0_13_12), -8); + HVX_VectorPair v_1_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_15_14), + HEXAGON_HVX_GET_V1(v_0_13_12), -8); + HVX_VectorPair v_1_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_19_18), + HEXAGON_HVX_GET_V0(v_0_17_16), -8); + HVX_VectorPair v_1_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_19_18), + HEXAGON_HVX_GET_V1(v_0_17_16), -8); + HVX_VectorPair v_1_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_23_22), + HEXAGON_HVX_GET_V0(v_0_21_20), -8); + HVX_VectorPair v_1_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_23_22), + HEXAGON_HVX_GET_V1(v_0_21_20), -8); + HVX_VectorPair v_1_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_27_26), + HEXAGON_HVX_GET_V0(v_0_25_24), -8); + HVX_VectorPair v_1_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_27_26), + HEXAGON_HVX_GET_V1(v_0_25_24), -8); + HVX_VectorPair v_1_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_31_30), + HEXAGON_HVX_GET_V0(v_0_29_28), -8); + HVX_VectorPair v_1_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_31_30), + HEXAGON_HVX_GET_V1(v_0_29_28), -8); + + // Shuffle the 128-bit lanes. + v_0_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_5_4), + HEXAGON_HVX_GET_V0(v_1_1_0), -16); + v_0_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_5_4), + HEXAGON_HVX_GET_V1(v_1_1_0), -16); + v_0_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_7_6), + HEXAGON_HVX_GET_V0(v_1_3_2), -16); + v_0_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_7_6), + HEXAGON_HVX_GET_V1(v_1_3_2), -16); + v_0_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_13_12), + HEXAGON_HVX_GET_V0(v_1_9_8), -16); + v_0_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_13_12), + HEXAGON_HVX_GET_V1(v_1_9_8), -16); + v_0_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_15_14), + HEXAGON_HVX_GET_V0(v_1_11_10), -16); + v_0_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_15_14), + HEXAGON_HVX_GET_V1(v_1_11_10), -16); + v_0_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_21_20), + HEXAGON_HVX_GET_V0(v_1_17_16), -16); + v_0_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_21_20), + HEXAGON_HVX_GET_V1(v_1_17_16), -16); + v_0_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_23_22), + HEXAGON_HVX_GET_V0(v_1_19_18), -16); + v_0_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_23_22), + HEXAGON_HVX_GET_V1(v_1_19_18), -16); + v_0_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_29_28), + HEXAGON_HVX_GET_V0(v_1_25_24), -16); + v_0_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_29_28), + HEXAGON_HVX_GET_V1(v_1_25_24), -16); + v_0_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_31_30), + HEXAGON_HVX_GET_V0(v_1_27_26), -16); + v_0_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_31_30), + HEXAGON_HVX_GET_V1(v_1_27_26), -16); + + // Shuffle the 256-bit lanes. + v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_9_8), + HEXAGON_HVX_GET_V0(v_0_1_0), -32); + v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_9_8), + HEXAGON_HVX_GET_V1(v_0_1_0), -32); + v_1_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_11_10), + HEXAGON_HVX_GET_V0(v_0_3_2), -32); + v_1_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_11_10), + HEXAGON_HVX_GET_V1(v_0_3_2), -32); + v_1_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_13_12), + HEXAGON_HVX_GET_V0(v_0_5_4), -32); + v_1_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_13_12), + HEXAGON_HVX_GET_V1(v_0_5_4), -32); + v_1_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_15_14), + HEXAGON_HVX_GET_V0(v_0_7_6), -32); + v_1_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_15_14), + HEXAGON_HVX_GET_V1(v_0_7_6), -32); + v_1_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_25_24), + HEXAGON_HVX_GET_V0(v_0_17_16), -32); + v_1_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_25_24), + HEXAGON_HVX_GET_V1(v_0_17_16), -32); + v_1_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_27_26), + HEXAGON_HVX_GET_V0(v_0_19_18), -32); + v_1_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_27_26), + HEXAGON_HVX_GET_V1(v_0_19_18), -32); + v_1_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_29_28), + HEXAGON_HVX_GET_V0(v_0_21_20), -32); + v_1_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_29_28), + HEXAGON_HVX_GET_V1(v_0_21_20), -32); + v_1_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_31_30), + HEXAGON_HVX_GET_V0(v_0_23_22), -32); + v_1_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_31_30), + HEXAGON_HVX_GET_V1(v_0_23_22), -32); + + // Shuffle the 512-bit lanes. + v_0_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_17_16), + HEXAGON_HVX_GET_V0(v_1_1_0), -64); + v_0_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_17_16), + HEXAGON_HVX_GET_V1(v_1_1_0), -64); + v_0_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_19_18), + HEXAGON_HVX_GET_V0(v_1_3_2), -64); + v_0_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_19_18), + HEXAGON_HVX_GET_V1(v_1_3_2), -64); + v_0_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_21_20), + HEXAGON_HVX_GET_V0(v_1_5_4), -64); + v_0_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_21_20), + HEXAGON_HVX_GET_V1(v_1_5_4), -64); + v_0_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_23_22), + HEXAGON_HVX_GET_V0(v_1_7_6), -64); + v_0_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_23_22), + HEXAGON_HVX_GET_V1(v_1_7_6), -64); + v_0_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_25_24), + HEXAGON_HVX_GET_V0(v_1_9_8), -64); + v_0_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_25_24), + HEXAGON_HVX_GET_V1(v_1_9_8), -64); + v_0_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_27_26), + HEXAGON_HVX_GET_V0(v_1_11_10), -64); + v_0_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_27_26), + HEXAGON_HVX_GET_V1(v_1_11_10), -64); + v_0_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_29_28), + HEXAGON_HVX_GET_V0(v_1_13_12), -64); + v_0_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_29_28), + HEXAGON_HVX_GET_V1(v_1_13_12), -64); + v_0_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_31_30), + HEXAGON_HVX_GET_V0(v_1_15_14), -64); + v_0_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_31_30), + HEXAGON_HVX_GET_V1(v_1_15_14), -64); + + kernel.packet[0] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_1_0)); + kernel.packet[1] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_1_0)); + kernel.packet[2] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_3_2)); + kernel.packet[3] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_3_2)); + kernel.packet[4] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_5_4)); + kernel.packet[5] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_5_4)); + kernel.packet[6] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_7_6)); + kernel.packet[7] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_7_6)); + kernel.packet[8] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_9_8)); + kernel.packet[9] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_9_8)); + kernel.packet[10] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_11_10)); + kernel.packet[11] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_11_10)); + kernel.packet[12] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_13_12)); + kernel.packet[13] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_13_12)); + kernel.packet[14] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_15_14)); + kernel.packet[15] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_15_14)); + kernel.packet[16] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_17_16)); + kernel.packet[17] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_17_16)); + kernel.packet[18] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_19_18)); + kernel.packet[19] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_19_18)); + kernel.packet[20] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_21_20)); + kernel.packet[21] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_21_20)); + kernel.packet[22] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_23_22)); + kernel.packet[23] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_23_22)); + kernel.packet[24] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_25_24)); + kernel.packet[25] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_25_24)); + kernel.packet[26] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_27_26)); + kernel.packet[27] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_27_26)); + kernel.packet[28] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_29_28)); + kernel.packet[29] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_29_28)); + kernel.packet[30] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_31_30)); + kernel.packet[31] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_31_30)); +} + +template <> +EIGEN_STRONG_INLINE float predux(const Packet32f& a) { + HVX_Vector vsum_4 = Q6_Vqf32_vadd_VsfVsf(Q6_V_vror_VR(a.Get(), 4), a.Get()); + HVX_Vector vsum_8 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_vror_VR(vsum_4, 8), vsum_4); + HVX_Vector vsum_16 = + Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_vror_VR(vsum_8, 16), vsum_8); + HVX_Vector vsum_32 = + Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_vror_VR(vsum_16, 32), vsum_16); + HVX_Vector vsum_64 = + Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_vror_VR(vsum_32, 64), vsum_32); + return pfirst(Packet32f::Create(Q6_Vsf_equals_Vqf32(vsum_64))); +} + +template <> +EIGEN_STRONG_INLINE Packet32f ploaddup(const float* from) { + HVX_Vector load = HVX_loadu(from); + HVX_VectorPair dup = Q6_W_vshuff_VVR(load, load, -4); + return Packet32f::Create(HEXAGON_HVX_GET_V0(dup)); +} + +template <> +EIGEN_STRONG_INLINE Packet32f ploadquad(const float* from) { + HVX_Vector load = HVX_loadu(from); + HVX_VectorPair dup = Q6_W_vshuff_VVR(load, load, -4); + HVX_VectorPair quad = + Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(dup), HEXAGON_HVX_GET_V0(dup), -8); + return Packet32f::Create(HEXAGON_HVX_GET_V0(quad)); +} + +template <> +EIGEN_STRONG_INLINE Packet32f preverse(const Packet32f& a) { + HVX_Vector delta = Q6_Vb_vsplat_R(0x7c); + return Packet32f::Create(Q6_V_vdelta_VV(a.Get(), delta)); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pmin(const Packet32f& a, const Packet32f& b) { + return Packet32f::Create(Q6_Vsf_vmin_VsfVsf(a.Get(), b.Get())); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pmax(const Packet32f& a, const Packet32f& b) { + return Packet32f::Create(Q6_Vsf_vmax_VsfVsf(a.Get(), b.Get())); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pand(const Packet32f& a, const Packet32f& b) { + return Packet32f::Create(a.Get() & b.Get()); +} + +template <> +EIGEN_STRONG_INLINE Packet32f por(const Packet32f& a, const Packet32f& b) { + return Packet32f::Create(a.Get() | b.Get()); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pxor(const Packet32f& a, const Packet32f& b) { + return Packet32f::Create(a.Get() ^ b.Get()); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pnot(const Packet32f& a) { + return Packet32f::Create(~a.Get()); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pselect(const Packet32f& mask, const Packet32f& a, + const Packet32f& b) { + HVX_VectorPred pred = Q6_Q_vcmp_eq_VwVw(mask.Get(), Q6_V_vzero()); + return Packet32f::Create(Q6_V_vmux_QVV(pred, b.Get(), a.Get())); +} + +template +EIGEN_STRONG_INLINE float predux_generic(const Packet32f& a, Op op) { + Packet32f vredux_4 = op(Packet32f::Create(Q6_V_vror_VR(a.Get(), 4)), a); + Packet32f vredux_8 = + op(Packet32f::Create(Q6_V_vror_VR(vredux_4.Get(), 8)), vredux_4); + Packet32f vredux_16 = + op(Packet32f::Create(Q6_V_vror_VR(vredux_8.Get(), 16)), vredux_8); + Packet32f vredux_32 = + op(Packet32f::Create(Q6_V_vror_VR(vredux_16.Get(), 32)), vredux_16); + Packet32f vredux_64 = + op(Packet32f::Create(Q6_V_vror_VR(vredux_32.Get(), 64)), vredux_32); + return pfirst(vredux_64); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet32f& a) { + return predux_generic(a, pmax); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet32f& a) { + return predux_generic(a, pmin); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet32f& a) { + return predux_generic(a, por) != 0.0f; +} + +static const float index_vsf[32] __attribute__((aligned(128))) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}; + +template <> +EIGEN_STRONG_INLINE Packet32f plset(const float& a) { + return padd(pload(index_vsf), pset1(a)); +} + +// qfloat32 operations. +template <> +EIGEN_STRONG_INLINE Packet32qf pzero(const Packet32qf&) { + return Packet32qf::Create(Q6_V_vzero()); +} + +template <> +EIGEN_STRONG_INLINE Packet32qf pmul(const Packet32qf& a, + const Packet32qf& b) { + return Packet32qf::Create(Q6_Vqf32_vmpy_Vqf32Vqf32(a.Get(), b.Get())); +} + +template <> +EIGEN_STRONG_INLINE Packet32qf padd(const Packet32qf& a, + const Packet32qf& b) { + return Packet32qf::Create(Q6_Vqf32_vadd_Vqf32Vqf32(a.Get(), b.Get())); +} + +// Mixed float32 and qfloat32 operations. +EIGEN_STRONG_INLINE Packet32qf pmadd_f32_to_qf32(const Packet32f& a, + const Packet32f& b, + const Packet32qf& c) { + return Packet32qf::Create(Q6_Vqf32_vadd_Vqf32Vqf32( + Q6_Vqf32_vmpy_VsfVsf(a.Get(), b.Get()), c.Get())); +} + +EIGEN_STRONG_INLINE Packet32f pmadd_qf32_to_f32(const Packet32qf& a, + const Packet32f& b, + const Packet32f& c) { + return Packet32f::Create(Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf( + Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(a.Get()), b.Get()), c.Get()))); +} + +} // end namespace internal +} // end namespace Eigen + +#endif // __HVX__ && (__HVX_LENGTH__ == 128) && __HVX_ARCH__ >= 68 + +#endif // EIGEN_HVX_PACKET_MATH_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/Complex.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/Complex.h index 53dacfa43d8..b64bd8d3aeb 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/Complex.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/Complex.h @@ -15,6 +15,9 @@ #include +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -75,16 +78,13 @@ struct Packet2cf { EIGEN_STRONG_INLINE Packet2cf operator-(const Packet2cf& b) const { return Packet2cf(*this) -= b; } + EIGEN_STRONG_INLINE Packet2cf operator/(const Packet2cf& b) const { + return pdiv_complex(Packet2cf(*this), b); + } EIGEN_STRONG_INLINE Packet2cf& operator/=(const Packet2cf& b) { - *this *= b.conjugate(); - Packet4f s = pmul(b.v, b.v); - s = padd(s, (Packet4f)__builtin_msa_shf_w((v4i32)s, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); - v = pdiv(v, s); + *this = Packet2cf(*this) / b; return *this; } - EIGEN_STRONG_INLINE Packet2cf operator/(const Packet2cf& b) const { - return Packet2cf(*this) /= b; - } EIGEN_STRONG_INLINE Packet2cf operator-(void) const { return Packet2cf(pnegate(v)); } @@ -108,7 +108,6 @@ struct packet_traits > : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 2, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -423,7 +422,6 @@ struct packet_traits > : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 0, size = 1, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/MathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/MathFunctions.h index f5181b90ec5..3e7732922c2 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/MathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/MathFunctions.h @@ -26,27 +26,30 @@ #ifndef EIGEN_MATH_FUNCTIONS_MSA_H #define EIGEN_MATH_FUNCTIONS_MSA_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f plog(const Packet4f& _x) { - static _EIGEN_DECLARE_CONST_Packet4f(cephes_SQRTHF, 0.707106781186547524f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p0, 7.0376836292e-2f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p1, -1.1514610310e-1f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p2, 1.1676998740e-1f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p3, -1.2420140846e-1f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p4, +1.4249322787e-1f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p5, -1.6668057665e-1f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p6, +2.0000714765e-1f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p7, -2.4999993993e-1f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p8, +3.3333331174e-1f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q1, -2.12194440e-4f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q2, 0.693359375f); - static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); - static _EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_SQRTHF, 0.707106781186547524f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p0, 7.0376836292e-2f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p1, -1.1514610310e-1f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p2, 1.1676998740e-1f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p3, -1.2420140846e-1f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p4, +1.4249322787e-1f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p5, -1.6668057665e-1f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p6, +2.0000714765e-1f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p7, -2.4999993993e-1f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p8, +3.3333331174e-1f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_log_q1, -2.12194440e-4f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_log_q2, 0.693359375f); + static EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); + static EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); // Convert negative argument into NAN (quiet negative, to be specific). Packet4f zero = (Packet4f)__builtin_msa_ldi_w(0); @@ -119,23 +122,23 @@ plog(const Packet4f& _x) { } template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f pexp(const Packet4f& _x) { // Limiting single-precision pexp's argument to [-128, +128] lets pexp // reach 0 and INFINITY naturally. - static _EIGEN_DECLARE_CONST_Packet4f(exp_lo, -128.0f); - static _EIGEN_DECLARE_CONST_Packet4f(exp_hi, +128.0f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500e-4f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507e-3f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073e-3f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894e-2f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459e-1f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201e-1f); - static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); - static _EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); + static EIGEN_DECLARE_CONST_Packet4f(exp_lo, -128.0f); + static EIGEN_DECLARE_CONST_Packet4f(exp_hi, +128.0f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500e-4f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507e-3f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073e-3f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894e-2f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459e-1f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201e-1f); + static EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); + static EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); Packet4f x = _x; @@ -172,23 +175,23 @@ pexp(const Packet4f& _x) { } template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f ptanh(const Packet4f& _x) { - static _EIGEN_DECLARE_CONST_Packet4f(tanh_tiny, 1e-4f); - static _EIGEN_DECLARE_CONST_Packet4f(tanh_hi, 9.0f); + static EIGEN_DECLARE_CONST_Packet4f(tanh_tiny, 1e-4f); + static EIGEN_DECLARE_CONST_Packet4f(tanh_hi, 9.0f); // The monomial coefficients of the numerator polynomial (odd). - static _EIGEN_DECLARE_CONST_Packet4f(alpha_1, 4.89352455891786e-3f); - static _EIGEN_DECLARE_CONST_Packet4f(alpha_3, 6.37261928875436e-4f); - static _EIGEN_DECLARE_CONST_Packet4f(alpha_5, 1.48572235717979e-5f); - static _EIGEN_DECLARE_CONST_Packet4f(alpha_7, 5.12229709037114e-8f); - static _EIGEN_DECLARE_CONST_Packet4f(alpha_9, -8.60467152213735e-11f); - static _EIGEN_DECLARE_CONST_Packet4f(alpha_11, 2.00018790482477e-13f); - static _EIGEN_DECLARE_CONST_Packet4f(alpha_13, -2.76076847742355e-16f); + static EIGEN_DECLARE_CONST_Packet4f(alpha_1, 4.89352455891786e-3f); + static EIGEN_DECLARE_CONST_Packet4f(alpha_3, 6.37261928875436e-4f); + static EIGEN_DECLARE_CONST_Packet4f(alpha_5, 1.48572235717979e-5f); + static EIGEN_DECLARE_CONST_Packet4f(alpha_7, 5.12229709037114e-8f); + static EIGEN_DECLARE_CONST_Packet4f(alpha_9, -8.60467152213735e-11f); + static EIGEN_DECLARE_CONST_Packet4f(alpha_11, 2.00018790482477e-13f); + static EIGEN_DECLARE_CONST_Packet4f(alpha_13, -2.76076847742355e-16f); // The monomial coefficients of the denominator polynomial (even). - static _EIGEN_DECLARE_CONST_Packet4f(beta_0, 4.89352518554385e-3f); - static _EIGEN_DECLARE_CONST_Packet4f(beta_2, 2.26843463243900e-3f); - static _EIGEN_DECLARE_CONST_Packet4f(beta_4, 1.18534705686654e-4f); - static _EIGEN_DECLARE_CONST_Packet4f(beta_6, 1.19825839466702e-6f); + static EIGEN_DECLARE_CONST_Packet4f(beta_0, 4.89352518554385e-3f); + static EIGEN_DECLARE_CONST_Packet4f(beta_2, 2.26843463243900e-3f); + static EIGEN_DECLARE_CONST_Packet4f(beta_4, 1.18534705686654e-4f); + static EIGEN_DECLARE_CONST_Packet4f(beta_6, 1.19825839466702e-6f); Packet4f x = pabs(_x); Packet4i tiny_mask = __builtin_msa_fclt_w(x, p4f_tanh_tiny); @@ -229,19 +232,19 @@ ptanh(const Packet4f& _x) { template Packet4f psincos_inner_msa_float(const Packet4f& _x) { - static _EIGEN_DECLARE_CONST_Packet4f(sincos_max_arg, 13176795.0f); // Approx. (2**24) / (4/Pi). - static _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP1, -0.78515625f); - static _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP2, -2.4187564849853515625e-4f); - static _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP3, -3.77489497744594108e-8f); - static _EIGEN_DECLARE_CONST_Packet4f(sincof_p0, -1.9515295891e-4f); - static _EIGEN_DECLARE_CONST_Packet4f(sincof_p1, 8.3321608736e-3f); - static _EIGEN_DECLARE_CONST_Packet4f(sincof_p2, -1.6666654611e-1f); - static _EIGEN_DECLARE_CONST_Packet4f(coscof_p0, 2.443315711809948e-5f); - static _EIGEN_DECLARE_CONST_Packet4f(coscof_p1, -1.388731625493765e-3f); - static _EIGEN_DECLARE_CONST_Packet4f(coscof_p2, 4.166664568298827e-2f); - static _EIGEN_DECLARE_CONST_Packet4f(cephes_FOPI, 1.27323954473516f); // 4/Pi. - static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); - static _EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); + static EIGEN_DECLARE_CONST_Packet4f(sincos_max_arg, 13176795.0f); // Approx. (2**24) / (4/Pi). + static EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP1, -0.78515625f); + static EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP2, -2.4187564849853515625e-4f); + static EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP3, -3.77489497744594108e-8f); + static EIGEN_DECLARE_CONST_Packet4f(sincof_p0, -1.9515295891e-4f); + static EIGEN_DECLARE_CONST_Packet4f(sincof_p1, 8.3321608736e-3f); + static EIGEN_DECLARE_CONST_Packet4f(sincof_p2, -1.6666654611e-1f); + static EIGEN_DECLARE_CONST_Packet4f(coscof_p0, 2.443315711809948e-5f); + static EIGEN_DECLARE_CONST_Packet4f(coscof_p1, -1.388731625493765e-3f); + static EIGEN_DECLARE_CONST_Packet4f(coscof_p2, 4.166664568298827e-2f); + static EIGEN_DECLARE_CONST_Packet4f(cephes_FOPI, 1.27323954473516f); // 4/Pi. + static EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); + static EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); Packet4f x = pabs(_x); @@ -310,37 +313,37 @@ Packet4f psincos_inner_msa_float(const Packet4f& _x) { } template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f psin(const Packet4f& x) { return psincos_inner_msa_float(x); } template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f pcos(const Packet4f& x) { return psincos_inner_msa_float(x); } template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d pexp(const Packet2d& _x) { // Limiting double-precision pexp's argument to [-1024, +1024] lets pexp // reach 0 and INFINITY naturally. - static _EIGEN_DECLARE_CONST_Packet2d(exp_lo, -1024.0); - static _EIGEN_DECLARE_CONST_Packet2d(exp_hi, +1024.0); - static _EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); - static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); - static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); - static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); - static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); - static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); - static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); - static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); - static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); - static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); - static _EIGEN_DECLARE_CONST_Packet2d(half, 0.5); - static _EIGEN_DECLARE_CONST_Packet2d(1, 1.0); - static _EIGEN_DECLARE_CONST_Packet2d(2, 2.0); + static EIGEN_DECLARE_CONST_Packet2d(exp_lo, -1024.0); + static EIGEN_DECLARE_CONST_Packet2d(exp_hi, +1024.0); + static EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); + static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); + static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); + static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); + static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); + static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); + static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); + static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); + static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); + static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); + static EIGEN_DECLARE_CONST_Packet2d(half, 0.5); + static EIGEN_DECLARE_CONST_Packet2d(1, 1.0); + static EIGEN_DECLARE_CONST_Packet2d(2, 2.0); Packet2d x = _x; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/PacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/PacketMath.h index afe8f3375ba..b36f0248053 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/PacketMath.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/MSA/PacketMath.h @@ -16,6 +16,9 @@ #include #include +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -51,9 +54,9 @@ typedef v4f32 Packet4f; typedef v4i32 Packet4i; typedef v4u32 Packet4ui; -#define _EIGEN_DECLARE_CONST_Packet4f(NAME, X) const Packet4f p4f_##NAME = { X, X, X, X } -#define _EIGEN_DECLARE_CONST_Packet4i(NAME, X) const Packet4i p4i_##NAME = { X, X, X, X } -#define _EIGEN_DECLARE_CONST_Packet4ui(NAME, X) const Packet4ui p4ui_##NAME = { X, X, X, X } +#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) const Packet4f p4f_##NAME = { X, X, X, X } +#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) const Packet4i p4i_##NAME = { X, X, X, X } +#define EIGEN_DECLARE_CONST_Packet4ui(NAME, X) const Packet4ui p4ui_##NAME = { X, X, X, X } inline std::ostream& operator<<(std::ostream& os, const Packet4f& value) { os << "[ " << value[0] << ", " << value[1] << ", " << value[2] << ", " << value[3] << " ]"; @@ -78,7 +81,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 0, // Packet2f intrinsics not implemented yet // FIXME check the Has* HasDiv = 1, HasSin = EIGEN_FAST_MATH, @@ -104,7 +106,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 0, // Packet2i intrinsics not implemented yet // FIXME check the Has* HasDiv = 1, HasBlend = 1 @@ -821,9 +822,9 @@ typedef v2f64 Packet2d; typedef v2i64 Packet2l; typedef v2u64 Packet2ul; -#define _EIGEN_DECLARE_CONST_Packet2d(NAME, X) const Packet2d p2d_##NAME = { X, X } -#define _EIGEN_DECLARE_CONST_Packet2l(NAME, X) const Packet2l p2l_##NAME = { X, X } -#define _EIGEN_DECLARE_CONST_Packet2ul(NAME, X) const Packet2ul p2ul_##NAME = { X, X } +#define EIGEN_DECLARE_CONST_Packet2d(NAME, X) const Packet2d p2d_##NAME = { X, X } +#define EIGEN_DECLARE_CONST_Packet2l(NAME, X) const Packet2l p2l_##NAME = { X, X } +#define EIGEN_DECLARE_CONST_Packet2ul(NAME, X) const Packet2ul p2ul_##NAME = { X, X } inline std::ostream& operator<<(std::ostream& os, const Packet2d& value) { os << "[ " << value[0] << ", " << value[1] << " ]"; @@ -848,7 +849,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 2, - HasHalfPacket = 0, // FIXME check the Has* HasDiv = 1, HasExp = 1, diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/Complex.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/Complex.h index f40af7f87f2..98b76da5fb5 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/Complex.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/Complex.h @@ -11,6 +11,9 @@ #ifndef EIGEN_COMPLEX_NEON_H #define EIGEN_COMPLEX_NEON_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -57,13 +60,13 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 2, - HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, + HasSqrt = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, @@ -129,12 +132,12 @@ template<> EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) { return Pa template<> EIGEN_STRONG_INLINE Packet1cf pconj(const Packet1cf& a) { - const Packet2ui b = vreinterpret_u32_f32(a.v); + const Packet2ui b = Packet2ui(vreinterpret_u32_f32(a.v)); return Packet1cf(vreinterpret_f32_u32(veor_u32(b, p2ui_CONJ_XOR()))); } template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { - const Packet4ui b = vreinterpretq_u32_f32(a.v); + const Packet4ui b = Packet4ui(vreinterpretq_u32_f32(a.v)); return Packet2cf(vreinterpretq_f32_u32(veorq_u32(b, p4ui_CONJ_XOR()))); } @@ -347,27 +350,11 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) template<> EIGEN_STRONG_INLINE Packet1cf pdiv(const Packet1cf& a, const Packet1cf& b) { - // TODO optimize it for NEON - Packet1cf res = pmul(a, pconj(b)); - Packet2f s, rev_s; - - // this computes the norm - s = vmul_f32(b.v, b.v); - rev_s = vrev64_f32(s); - - return Packet1cf(pdiv(res.v, vadd_f32(s, rev_s))); + return pdiv_complex(a, b); } template<> EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { - // TODO optimize it for NEON - Packet2cf res = pmul(a,pconj(b)); - Packet4f s, rev_s; - - // this computes the norm - s = vmulq_f32(b.v, b.v); - rev_s = vrev64q_f32(s); - - return Packet2cf(pdiv(res.v, vaddq_f32(s, rev_s))); + return pdiv_complex(a, b); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& /*kernel*/) {} @@ -390,7 +377,7 @@ template<> EIGEN_STRONG_INLINE Packet2cf psqrt(const Packet2cf& a) { #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG // See bug 1325, clang fails to call vld1q_u64. -#if EIGEN_COMP_CLANG || EIGEN_COMP_CASTXML +#if EIGEN_COMP_CLANG || EIGEN_COMP_CASTXML || EIGEN_COMP_CPE static uint64x2_t p2ul_CONJ_XOR = {0x0, 0x8000000000000000}; #else const uint64_t p2ul_conj_XOR_DATA[] = { 0x0, 0x8000000000000000 }; @@ -413,13 +400,13 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 0, size = 1, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, + HasSqrt = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, @@ -553,12 +540,7 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) template<> EIGEN_STRONG_INLINE Packet1cd pdiv(const Packet1cd& a, const Packet1cd& b) { - // TODO optimize it for NEON - Packet1cd res = pmul(a,pconj(b)); - Packet2d s = pmul(b.v, b.v); - Packet2d rev_s = preverse(s); - - return Packet1cd(pdiv(res.v, padd(s,rev_s))); + return pdiv_complex(a, b); } EIGEN_STRONG_INLINE Packet1cd pcplxflip/**/(const Packet1cd& x) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/GeneralBlockPanelKernel.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/GeneralBlockPanelKernel.h index 3481f337e37..48410c5e533 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/GeneralBlockPanelKernel.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/GeneralBlockPanelKernel.h @@ -1,6 +1,9 @@ +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { - + #if EIGEN_ARCH_ARM && EIGEN_COMP_CLANG // Clang seems to excessively spill registers in the GEBP kernel on 32-bit arm. @@ -24,7 +27,7 @@ struct gebp_traits template EIGEN_STRONG_INLINE void madd(const Packet4f& a, const Packet4f& b, - Packet4f& c, Packet4f& tmp, + Packet4f& c, Packet4f&, const LaneIdType&) const { acc(a, b, c); } @@ -41,15 +44,18 @@ struct gebp_traits #if EIGEN_ARCH_ARM64 +#ifndef EIGEN_NEON_GEBP_NR +#define EIGEN_NEON_GEBP_NR 8 +#endif + template<> struct gebp_traits : gebp_traits { typedef float RhsPacket; typedef float32x4_t RhsPacketx4; - - EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const - { + enum { nr = EIGEN_NEON_GEBP_NR }; + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; } @@ -75,7 +81,6 @@ struct gebp_traits { c = vfmaq_n_f32(c, a, b); } - // NOTE: Template parameter inference failed when compiled with Android NDK: // "candidate template ignored: could not match 'FixedInt' against 'Eigen::internal::FixedInt<0>". @@ -92,9 +97,10 @@ struct gebp_traits template EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const { - #if EIGEN_COMP_GNUC_STRICT && !(EIGEN_GNUC_AT_LEAST(9,0)) - // workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101 - // vfmaq_laneq_f32 is implemented through a costly dup + #if EIGEN_GNUC_STRICT_LESS_THAN(9,0,0) + // 1. workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101 + // vfmaq_laneq_f32 is implemented through a costly dup, which was fixed in gcc9 + // 2. workaround the gcc register split problem on arm64-neon if(LaneID==0) asm("fmla %0.4s, %1.4s, %2.s[0]\n" : "+w" (c) : "w" (a), "w" (b) : ); else if(LaneID==1) asm("fmla %0.4s, %1.4s, %2.s[1]\n" : "+w" (c) : "w" (a), "w" (b) : ); else if(LaneID==2) asm("fmla %0.4s, %1.4s, %2.s[2]\n" : "+w" (c) : "w" (a), "w" (b) : ); @@ -111,7 +117,7 @@ struct gebp_traits : gebp_traits { typedef double RhsPacket; - + enum { nr = EIGEN_NEON_GEBP_NR }; struct RhsPacketx4 { float64x2_t B_0, B_1; }; @@ -161,9 +167,10 @@ struct gebp_traits template EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const { - #if EIGEN_COMP_GNUC_STRICT && !(EIGEN_GNUC_AT_LEAST(9,0)) - // workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101 - // vfmaq_laneq_f64 is implemented through a costly dup + #if EIGEN_GNUC_STRICT_LESS_THAN(9,0,0) + // 1. workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101 + // vfmaq_laneq_f64 is implemented through a costly dup, which was fixed in gcc9 + // 2. workaround the gcc register split problem on arm64-neon if(LaneID==0) asm("fmla %0.2d, %1.2d, %2.d[0]\n" : "+w" (c) : "w" (a), "w" (b.B_0) : ); else if(LaneID==1) asm("fmla %0.2d, %1.2d, %2.d[1]\n" : "+w" (c) : "w" (a), "w" (b.B_0) : ); else if(LaneID==2) asm("fmla %0.2d, %1.2d, %2.d[0]\n" : "+w" (c) : "w" (a), "w" (b.B_1) : ); @@ -177,6 +184,73 @@ struct gebp_traits } }; +// The register at operand 3 of fmla for data type half must be v0~v15, the compiler may not +// allocate a required register for the '%2' of inline asm 'fmla %0.8h, %1.8h, %2.h[id]', +// so inline assembly can't be used here to advoid the bug that vfmaq_lane_f16 is implemented +// through a costly dup in gcc compiler. +#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC && EIGEN_COMP_CLANG + +template<> +struct gebp_traits + : gebp_traits +{ + typedef half RhsPacket; + typedef float16x4_t RhsPacketx4; + typedef float16x4_t PacketHalf; + enum { nr = EIGEN_NEON_GEBP_NR }; + + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const + { + dest = *b; + } + + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const + { + dest = vld1_f16((const __fp16 *)b); + } + + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const + { + dest = *b; + } + + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const + {} + + EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar*, RhsPacket&) const + { + // If LHS is a Packet8h, we cannot correctly mimic a ploadquad of the RHS + // using a single scalar value. + eigen_assert(false && "Cannot loadRhsQuad for a scalar RHS."); + } + + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const + { + c = vfmaq_n_f16(c, a, b); + } + EIGEN_STRONG_INLINE void madd(const PacketHalf& a, const RhsPacket& b, PacketHalf& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const + { + c = vfma_n_f16(c, a, b); + } + + // NOTE: Template parameter inference failed when compiled with Android NDK: + // "candidate template ignored: could not match 'FixedInt' against 'Eigen::internal::FixedInt<0>". + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const + { madd_helper<0>(a, b, c); } + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<1>&) const + { madd_helper<1>(a, b, c); } + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<2>&) const + { madd_helper<2>(a, b, c); } + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<3>&) const + { madd_helper<3>(a, b, c); } + private: + template + EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const + { + c = vfmaq_lane_f16(c, a, b, LaneID); + } +}; +#endif // EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC && EIGEN_COMP_CLANG #endif // EIGEN_ARCH_ARM64 } // namespace internal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/MathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/MathFunctions.h index fa6615a8519..861181078e9 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/MathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/MathFunctions.h @@ -8,35 +8,33 @@ #ifndef EIGEN_MATH_FUNCTIONS_NEON_H #define EIGEN_MATH_FUNCTIONS_NEON_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f pexp(const Packet2f& x) -{ return pexp_float(x); } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pexp(const Packet4f& x) -{ return pexp_float(x); } - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f plog(const Packet2f& x) -{ return plog_float(x); } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f plog(const Packet4f& x) -{ return plog_float(x); } +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet2f) +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet4f) -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f psin(const Packet2f& x) -{ return psin_float(x); } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psin(const Packet4f& x) -{ return psin_float(x); } - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f pcos(const Packet2f& x) -{ return pcos_float(x); } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pcos(const Packet4f& x) -{ return pcos_float(x); } +#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC +template <> +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC +Packet4hf ptanh(const Packet4hf& x) { + // Convert to float, call the float ptanh, and then convert back. + return vcvt_f16_f32(ptanh(vcvt_f32_f16(x))); +} -// Hyperbolic Tangent function. -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f ptanh(const Packet2f& x) -{ return internal::generic_fast_tanh_float(x); } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f ptanh(const Packet4f& x) -{ return internal::generic_fast_tanh_float(x); } +template <> +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC +Packet8hf ptanh(const Packet8hf& x) { + // Convert each 4 halfs to float, call the float ptanh, and then convert back. + return vcombine_f16( + vcvt_f16_f32(ptanh(vcvt_f32_f16(vget_low_f16(x)))), + vcvt_f16_f32(ptanh(vcvt_high_f32_f16(x)))); +} +#endif // EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC BF16_PACKET_FUNCTION(Packet4f, Packet4bf, psin) BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pcos) @@ -60,11 +58,8 @@ EIGEN_STRONG_INLINE Packet4bf pldexp(const Packet4bf& a, const Packet4bf& expone //---------- double ---------- #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d pexp(const Packet2d& x) -{ return pexp_double(x); } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d plog(const Packet2d& x) -{ return plog_double(x); } +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE(Packet2d) #endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/PacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/PacketMath.h index ad648b04ab8..e70f8b0e2c5 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/PacketMath.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/PacketMath.h @@ -12,6 +12,9 @@ #ifndef EIGEN_PACKET_MATH_NEON_H #define EIGEN_PACKET_MATH_NEON_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -57,6 +60,16 @@ typedef eigen_packet_wrapper Packet4ui; typedef eigen_packet_wrapper Packet2l; typedef eigen_packet_wrapper Packet2ul; +EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d) { + float from[4] = {a, b, c, d}; + return vld1q_f32(from); +} + +EIGEN_ALWAYS_INLINE Packet2f make_packet2f(float a, float b) { + float from[2] = {a, b}; + return vld1_f32(from); +} + #else typedef float32x2_t Packet2f; @@ -78,11 +91,14 @@ typedef uint32x4_t Packet4ui; typedef int64x2_t Packet2l; typedef uint64x2_t Packet2ul; +EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d) { return Packet4f{a, b, c, d}; } +EIGEN_ALWAYS_INLINE Packet2f make_packet2f(float a, float b) { return Packet2f{a, b}; } + #endif // EIGEN_COMP_MSVC_STRICT EIGEN_STRONG_INLINE Packet4f shuffle1(const Packet4f& m, int mask){ const float* a = reinterpret_cast(&m); - Packet4f res = {*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3 )), *(a + ((mask >> 6) & 3))}; + Packet4f res = make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3 )), *(a + ((mask >> 6) & 3))); return res; } @@ -95,7 +111,7 @@ EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f &m, const Packet4f &n, int { const float* a = reinterpret_cast(&m); const float* b = reinterpret_cast(&n); - Packet4f res = {*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))}; + Packet4f res = make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))); return res; } @@ -104,7 +120,7 @@ EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f &m, const Packet4f &n { const float* a = reinterpret_cast(&m); const float* b = reinterpret_cast(&n); - Packet4f res = {*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))}; + Packet4f res = make_packet4f(*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))); return res; } @@ -135,18 +151,18 @@ EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f& a, const Packet4f& b return shuffle2(a,b,eigen_neon_shuffle_mask(2, 2, 3, 3)); } #define vec4f_duplane(a, p) \ - vdupq_lane_f32(vget_low_f32(a), p) + Packet4f(vdupq_lane_f32(vget_low_f32(a), p)) -#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ const Packet4f p4f_##NAME = pset1(X) -#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1(X)) -#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ const Packet4i p4i_##NAME = pset1(X) -#if EIGEN_ARCH_ARM64 +#if EIGEN_ARCH_ARM64 && EIGEN_COMP_GNUC // __builtin_prefetch tends to do nothing on ARM64 compilers because the // prefetch instructions there are too detailed for __builtin_prefetch to map // meaningfully to them. @@ -172,8 +188,8 @@ struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 1, + HasCmp = 1, HasAdd = 1, HasSub = 1, HasShift = 1, @@ -186,7 +202,7 @@ struct packet_traits : default_packet_traits HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0, HasDiv = 1, @@ -196,6 +212,10 @@ struct packet_traits : default_packet_traits HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, + HasACos = 1, + HasASin = 1, + HasATan = 1, + HasATanh = 1, HasLog = 1, HasExp = 1, HasSqrt = 1, @@ -217,8 +237,8 @@ struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 16, - HasHalfPacket = 1, + HasCmp = 1, HasAdd = 1, HasSub = 1, HasShift = 1, @@ -231,7 +251,7 @@ struct packet_traits : default_packet_traits HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0 }; }; @@ -246,8 +266,8 @@ struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 16, - HasHalfPacket = 1, + HasCmp = 1, HasAdd = 1, HasSub = 1, HasShift = 1, @@ -260,7 +280,7 @@ struct packet_traits : default_packet_traits HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0, HasSqrt = 1 @@ -277,8 +297,8 @@ struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasHalfPacket = 1, + HasCmp = 1, HasAdd = 1, HasSub = 1, HasShift = 1, @@ -291,7 +311,7 @@ struct packet_traits : default_packet_traits HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0 }; }; @@ -306,21 +326,21 @@ struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasHalfPacket = 1, + HasCmp = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 0, - HasAbs = 0, + HasAbs = 1, HasAbsDiff = 1, HasArg = 0, HasAbs2 = 1, HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0, HasSqrt = 1 }; @@ -336,8 +356,8 @@ struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 1, + HasCmp = 1, HasAdd = 1, HasSub = 1, HasShift = 1, @@ -350,7 +370,7 @@ struct packet_traits : default_packet_traits HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0 }; }; @@ -365,21 +385,21 @@ struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 1, + HasCmp = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 0, - HasAbs = 0, + HasAbs = 1, HasArg = 0, HasAbs2 = 1, HasAbsDiff = 1, HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0, HasSqrt = 1 @@ -396,7 +416,6 @@ struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 2, - HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, @@ -411,7 +430,7 @@ struct packet_traits : default_packet_traits HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0 }; }; @@ -426,7 +445,6 @@ struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 2, - HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, @@ -434,27 +452,18 @@ struct packet_traits : default_packet_traits HasShift = 1, HasMul = 1, HasNegate = 0, - HasAbs = 0, + HasAbs = 1, HasArg = 0, HasAbs2 = 1, HasAbsDiff = 1, HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0 }; }; -#if EIGEN_GNUC_AT_MOST(4, 4) && !EIGEN_COMP_LLVM -// workaround gcc 4.2, 4.3 and 4.4 compilation issue -EIGEN_STRONG_INLINE float32x4_t vld1q_f32(const float* x) { return ::vld1q_f32((const float32_t*)x); } -EIGEN_STRONG_INLINE float32x2_t vld1_f32(const float* x) { return ::vld1_f32 ((const float32_t*)x); } -EIGEN_STRONG_INLINE float32x2_t vld1_dup_f32(const float* x) { return ::vld1_dup_f32 ((const float32_t*)x); } -EIGEN_STRONG_INLINE void vst1q_f32(float* to, float32x4_t from) { ::vst1q_f32((float32_t*)to,from); } -EIGEN_STRONG_INLINE void vst1_f32 (float* to, float32x2_t from) { ::vst1_f32 ((float32_t*)to,from); } -#endif - template<> struct unpacket_traits { typedef float type; @@ -712,9 +721,9 @@ template<> EIGEN_STRONG_INLINE Packet4ui pset1(const uint32_t& from) template<> EIGEN_STRONG_INLINE Packet2l pset1(const int64_t& from) { return vdupq_n_s64(from); } template<> EIGEN_STRONG_INLINE Packet2ul pset1(const uint64_t& from) { return vdupq_n_u64(from); } -template<> EIGEN_STRONG_INLINE Packet2f pset1frombits(unsigned int from) +template<> EIGEN_STRONG_INLINE Packet2f pset1frombits(uint32_t from) { return vreinterpret_f32_u32(vdup_n_u32(from)); } -template<> EIGEN_STRONG_INLINE Packet4f pset1frombits(unsigned int from) +template<> EIGEN_STRONG_INLINE Packet4f pset1frombits(uint32_t from) { return vreinterpretq_f32_u32(vdupq_n_u32(from)); } template<> EIGEN_STRONG_INLINE Packet2f plset(const float& a) @@ -862,12 +871,12 @@ template<> EIGEN_STRONG_INLINE Packet2ul psub(const Packet2ul& a, con template<> EIGEN_STRONG_INLINE Packet2f pxor(const Packet2f& a, const Packet2f& b); template<> EIGEN_STRONG_INLINE Packet2f paddsub(const Packet2f& a, const Packet2f & b) { - Packet2f mask = {numext::bit_cast(0x80000000u), 0.0f}; + Packet2f mask = make_packet2f(numext::bit_cast(0x80000000u), 0.0f); return padd(a, pxor(mask, b)); } template<> EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b); template<> EIGEN_STRONG_INLINE Packet4f paddsub(const Packet4f& a, const Packet4f& b) { - Packet4f mask = {numext::bit_cast(0x80000000u), 0.0f, numext::bit_cast(0x80000000u), 0.0f}; + Packet4f mask = make_packet4f(numext::bit_cast(0x80000000u), 0.0f, numext::bit_cast(0x80000000u), 0.0f); return padd(a, pxor(mask, b)); } @@ -947,57 +956,6 @@ template<> EIGEN_STRONG_INLINE Packet2ul pmul(const Packet2ul& a, con vdup_n_u64(vgetq_lane_u64(a, 1)*vgetq_lane_u64(b, 1))); } -template<> EIGEN_STRONG_INLINE Packet2f pdiv(const Packet2f& a, const Packet2f& b) -{ -#if EIGEN_ARCH_ARM64 - return vdiv_f32(a,b); -#else - Packet2f inv, restep, div; - - // NEON does not offer a divide instruction, we have to do a reciprocal approximation - // However NEON in contrast to other SIMD engines (AltiVec/SSE), offers - // a reciprocal estimate AND a reciprocal step -which saves a few instructions - // vrecpeq_f32() returns an estimate to 1/b, which we will finetune with - // Newton-Raphson and vrecpsq_f32() - inv = vrecpe_f32(b); - - // This returns a differential, by which we will have to multiply inv to get a better - // approximation of 1/b. - restep = vrecps_f32(b, inv); - inv = vmul_f32(restep, inv); - - // Finally, multiply a by 1/b and get the wanted result of the division. - div = vmul_f32(a, inv); - - return div; -#endif -} -template<> EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) -{ -#if EIGEN_ARCH_ARM64 - return vdivq_f32(a,b); -#else - Packet4f inv, restep, div; - - // NEON does not offer a divide instruction, we have to do a reciprocal approximation - // However NEON in contrast to other SIMD engines (AltiVec/SSE), offers - // a reciprocal estimate AND a reciprocal step -which saves a few instructions - // vrecpeq_f32() returns an estimate to 1/b, which we will finetune with - // Newton-Raphson and vrecpsq_f32() - inv = vrecpeq_f32(b); - - // This returns a differential, by which we will have to multiply inv to get a better - // approximation of 1/b. - restep = vrecpsq_f32(b, inv); - inv = vmulq_f32(restep, inv); - - // Finally, multiply a by 1/b and get the wanted result of the division. - div = vmulq_f32(a, inv); - - return div; -#endif -} - template<> EIGEN_STRONG_INLINE Packet4c pdiv(const Packet4c& /*a*/, const Packet4c& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); @@ -1668,7 +1626,7 @@ template<> EIGEN_STRONG_INLINE Packet4f pload(const float* from) template<> EIGEN_STRONG_INLINE Packet4c pload(const int8_t* from) { Packet4c res; - memcpy((void *)&res, from, sizeof(Packet4c)); + memcpy(&res, from, sizeof(Packet4c)); return res; } template<> EIGEN_STRONG_INLINE Packet8c pload(const int8_t* from) @@ -1678,7 +1636,7 @@ template<> EIGEN_STRONG_INLINE Packet16c pload(const int8_t* from) template<> EIGEN_STRONG_INLINE Packet4uc pload(const uint8_t* from) { Packet4uc res; - memcpy((void *)&res, from, sizeof(Packet4uc)); + memcpy(&res, from, sizeof(Packet4uc)); return res; } template<> EIGEN_STRONG_INLINE Packet8uc pload(const uint8_t* from) @@ -1713,7 +1671,7 @@ template<> EIGEN_STRONG_INLINE Packet4f ploadu(const float* from) template<> EIGEN_STRONG_INLINE Packet4c ploadu(const int8_t* from) { Packet4c res; - memcpy((void *)&res, from, sizeof(Packet4c)); + memcpy(&res, from, sizeof(Packet4c)); return res; } template<> EIGEN_STRONG_INLINE Packet8c ploadu(const int8_t* from) @@ -1723,7 +1681,7 @@ template<> EIGEN_STRONG_INLINE Packet16c ploadu(const int8_t* from) template<> EIGEN_STRONG_INLINE Packet4uc ploadu(const uint8_t* from) { Packet4uc res; - memcpy((void *)&res, from, sizeof(Packet4uc)); + memcpy(&res, from, sizeof(Packet4uc)); return res; } template<> EIGEN_STRONG_INLINE Packet8uc ploadu(const uint8_t* from) @@ -2374,6 +2332,15 @@ template<> EIGEN_STRONG_INLINE Packet2l pabs(const Packet2l& a) { } template<> EIGEN_STRONG_INLINE Packet2ul pabs(const Packet2ul& a) { return a; } +template <> +EIGEN_STRONG_INLINE Packet2f psignbit(const Packet2f& a) { + return vreinterpret_f32_s32(vshr_n_s32(vreinterpret_s32_f32(a), 31)); +} +template <> +EIGEN_STRONG_INLINE Packet4f psignbit(const Packet4f& a) { + return vreinterpretq_f32_s32(vshrq_n_s32(vreinterpretq_s32_f32(a), 31)); +} + template<> EIGEN_STRONG_INLINE Packet2f pfrexp(const Packet2f& a, Packet2f& exponent) { return pfrexp_generic(a,exponent); } template<> EIGEN_STRONG_INLINE Packet4f pfrexp(const Packet4f& a, Packet4f& exponent) @@ -2384,12 +2351,17 @@ template<> EIGEN_STRONG_INLINE Packet2f pldexp(const Packet2f& a, cons template<> EIGEN_STRONG_INLINE Packet4f pldexp(const Packet4f& a, const Packet4f& exponent) { return pldexp_generic(a,exponent); } +#if EIGEN_ARCH_ARM64 +template<> EIGEN_STRONG_INLINE float predux(const Packet2f& a) { return vaddv_f32(a); } +template<> EIGEN_STRONG_INLINE float predux(const Packet4f& a) { return vaddvq_f32(a); } +#else template<> EIGEN_STRONG_INLINE float predux(const Packet2f& a) { return vget_lane_f32(vpadd_f32(a,a), 0); } template<> EIGEN_STRONG_INLINE float predux(const Packet4f& a) { const float32x2_t sum = vadd_f32(vget_low_f32(a), vget_high_f32(a)); return vget_lane_f32(vpadd_f32(sum, sum), 0); } +#endif template<> EIGEN_STRONG_INLINE int8_t predux(const Packet4c& a) { const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a)); @@ -2397,6 +2369,10 @@ template<> EIGEN_STRONG_INLINE int8_t predux(const Packet4c& a) sum = vpadd_s8(sum, sum); return vget_lane_s8(sum, 0); } +#if EIGEN_ARCH_ARM64 +template<> EIGEN_STRONG_INLINE int8_t predux(const Packet8c& a) { return vaddv_s8(a); } +template<> EIGEN_STRONG_INLINE int8_t predux(const Packet16c& a) { return vaddvq_s8(a); } +#else template<> EIGEN_STRONG_INLINE int8_t predux(const Packet8c& a) { int8x8_t sum = vpadd_s8(a,a); @@ -2412,6 +2388,7 @@ template<> EIGEN_STRONG_INLINE int8_t predux(const Packet16c& a) sum = vpadd_s8(sum, sum); return vget_lane_s8(sum, 0); } +#endif template<> EIGEN_STRONG_INLINE uint8_t predux(const Packet4uc& a) { const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a)); @@ -2419,6 +2396,20 @@ template<> EIGEN_STRONG_INLINE uint8_t predux(const Packet4uc& a) sum = vpadd_u8(sum, sum); return vget_lane_u8(sum, 0); } +#if EIGEN_ARCH_ARM64 +template<> EIGEN_STRONG_INLINE uint8_t predux(const Packet8uc& a) { return vaddv_u8(a); } +template<> EIGEN_STRONG_INLINE uint8_t predux(const Packet16uc& a) { return vaddvq_u8(a); } +template<> EIGEN_STRONG_INLINE int16_t predux(const Packet4s& a) { return vaddv_s16(a); } +template<> EIGEN_STRONG_INLINE int16_t predux(const Packet8s& a) { return vaddvq_s16(a); } +template<> EIGEN_STRONG_INLINE uint16_t predux(const Packet4us& a) { return vaddv_u16(a); } +template<> EIGEN_STRONG_INLINE uint16_t predux(const Packet8us& a) { return vaddvq_u16(a); } +template<> EIGEN_STRONG_INLINE int32_t predux(const Packet2i& a) { return vaddv_s32(a); } +template<> EIGEN_STRONG_INLINE int32_t predux(const Packet4i& a) { return vaddvq_s32(a); } +template<> EIGEN_STRONG_INLINE uint32_t predux(const Packet2ui& a) { return vaddv_u32(a); } +template<> EIGEN_STRONG_INLINE uint32_t predux(const Packet4ui& a) { return vaddvq_u32(a); } +template<> EIGEN_STRONG_INLINE int64_t predux(const Packet2l& a) { return vaddvq_s64(a); } +template<> EIGEN_STRONG_INLINE uint64_t predux(const Packet2ul& a) { return vaddvq_u64(a); } +#else template<> EIGEN_STRONG_INLINE uint8_t predux(const Packet8uc& a) { uint8x8_t sum = vpadd_u8(a,a); @@ -2474,6 +2465,7 @@ template<> EIGEN_STRONG_INLINE int64_t predux(const Packet2l& a) { return vgetq_lane_s64(a, 0) + vgetq_lane_s64(a, 1); } template<> EIGEN_STRONG_INLINE uint64_t predux(const Packet2ul& a) { return vgetq_lane_u64(a, 0) + vgetq_lane_u64(a, 1); } +#endif template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c predux_half_dowto4(const Packet8c& a) { @@ -2499,7 +2491,7 @@ template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us predux_half_dowto4(co template<> EIGEN_STRONG_INLINE float predux_mul(const Packet2f& a) { return vget_lane_f32(a, 0) * vget_lane_f32(a, 1); } template<> EIGEN_STRONG_INLINE float predux_mul(const Packet4f& a) -{ return predux_mul(vmul_f32(vget_low_f32(a), vget_high_f32(a))); } +{ return predux_mul(vmul_f32(vget_low_f32(a), vget_high_f32(a))); } template<> EIGEN_STRONG_INLINE int8_t predux_mul(const Packet4c& a) { int8x8_t prod = vreinterpret_s8_s32(vdup_n_s32(a)); @@ -2513,7 +2505,7 @@ template<> EIGEN_STRONG_INLINE int8_t predux_mul(const Packet8c& a) return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 4); } template<> EIGEN_STRONG_INLINE int8_t predux_mul(const Packet16c& a) -{ return predux_mul(vmul_s8(vget_low_s8(a), vget_high_s8(a))); } +{ return predux_mul(vmul_s8(vget_low_s8(a), vget_high_s8(a))); } template<> EIGEN_STRONG_INLINE uint8_t predux_mul(const Packet4uc& a) { uint8x8_t prod = vreinterpret_u8_u32(vdup_n_u32(a)); @@ -2527,7 +2519,7 @@ template<> EIGEN_STRONG_INLINE uint8_t predux_mul(const Packet8uc& a) return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 4); } template<> EIGEN_STRONG_INLINE uint8_t predux_mul(const Packet16uc& a) -{ return predux_mul(vmul_u8(vget_low_u8(a), vget_high_u8(a))); } +{ return predux_mul(vmul_u8(vget_low_u8(a), vget_high_u8(a))); } template<> EIGEN_STRONG_INLINE int16_t predux_mul(const Packet4s& a) { const int16x4_t prod = vmul_s16(a, vrev32_s16(a)); @@ -2563,17 +2555,21 @@ template<> EIGEN_STRONG_INLINE uint16_t predux_mul(const Packet8us& a template<> EIGEN_STRONG_INLINE int32_t predux_mul(const Packet2i& a) { return vget_lane_s32(a, 0) * vget_lane_s32(a, 1); } template<> EIGEN_STRONG_INLINE int32_t predux_mul(const Packet4i& a) -{ return predux_mul(vmul_s32(vget_low_s32(a), vget_high_s32(a))); } +{ return predux_mul(vmul_s32(vget_low_s32(a), vget_high_s32(a))); } template<> EIGEN_STRONG_INLINE uint32_t predux_mul(const Packet2ui& a) { return vget_lane_u32(a, 0) * vget_lane_u32(a, 1); } template<> EIGEN_STRONG_INLINE uint32_t predux_mul(const Packet4ui& a) -{ return predux_mul(vmul_u32(vget_low_u32(a), vget_high_u32(a))); } +{ return predux_mul(vmul_u32(vget_low_u32(a), vget_high_u32(a))); } template<> EIGEN_STRONG_INLINE int64_t predux_mul(const Packet2l& a) { return vgetq_lane_s64(a, 0) * vgetq_lane_s64(a, 1); } template<> EIGEN_STRONG_INLINE uint64_t predux_mul(const Packet2ul& a) { return vgetq_lane_u64(a, 0) * vgetq_lane_u64(a, 1); } // min +#if EIGEN_ARCH_ARM64 +template<> EIGEN_STRONG_INLINE float predux_min(const Packet2f& a) { return vminv_f32(a); } +template<> EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) { return vminvq_f32(a); } +#else template<> EIGEN_STRONG_INLINE float predux_min(const Packet2f& a) { return vget_lane_f32(vpmin_f32(a,a), 0); } template<> EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) @@ -2581,6 +2577,7 @@ template<> EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) const float32x2_t min = vmin_f32(vget_low_f32(a), vget_high_f32(a)); return vget_lane_f32(vpmin_f32(min, min), 0); } +#endif template<> EIGEN_STRONG_INLINE int8_t predux_min(const Packet4c& a) { const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a)); @@ -2588,6 +2585,10 @@ template<> EIGEN_STRONG_INLINE int8_t predux_min(const Packet4c& a) min = vpmin_s8(min, min); return vget_lane_s8(min, 0); } +#if EIGEN_ARCH_ARM64 +template<> EIGEN_STRONG_INLINE int8_t predux_min(const Packet8c& a) { return vminv_s8(a); } +template<> EIGEN_STRONG_INLINE int8_t predux_min(const Packet16c& a) { return vminvq_s8(a); } +#else template<> EIGEN_STRONG_INLINE int8_t predux_min(const Packet8c& a) { int8x8_t min = vpmin_s8(a,a); @@ -2603,6 +2604,7 @@ template<> EIGEN_STRONG_INLINE int8_t predux_min(const Packet16c& a) min = vpmin_s8(min, min); return vget_lane_s8(min, 0); } +#endif template<> EIGEN_STRONG_INLINE uint8_t predux_min(const Packet4uc& a) { const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a)); @@ -2610,6 +2612,18 @@ template<> EIGEN_STRONG_INLINE uint8_t predux_min(const Packet4uc& a) min = vpmin_u8(min, min); return vget_lane_u8(min, 0); } +#if EIGEN_ARCH_ARM64 +template<> EIGEN_STRONG_INLINE uint8_t predux_min(const Packet8uc& a) { return vminv_u8(a); } +template<> EIGEN_STRONG_INLINE uint8_t predux_min(const Packet16uc& a) { return vminvq_u8(a); } +template<> EIGEN_STRONG_INLINE int16_t predux_min(const Packet4s& a) { return vminv_s16(a); } +template<> EIGEN_STRONG_INLINE int16_t predux_min(const Packet8s& a) { return vminvq_s16(a); } +template<> EIGEN_STRONG_INLINE uint16_t predux_min(const Packet4us& a) { return vminv_u16(a); } +template<> EIGEN_STRONG_INLINE uint16_t predux_min(const Packet8us& a) { return vminvq_u16(a); } +template<> EIGEN_STRONG_INLINE int32_t predux_min(const Packet2i& a) { return vminv_s32(a); } +template<> EIGEN_STRONG_INLINE int32_t predux_min(const Packet4i& a) { return vminvq_s32(a); } +template<> EIGEN_STRONG_INLINE uint32_t predux_min(const Packet2ui& a) { return vminv_u32(a); } +template<> EIGEN_STRONG_INLINE uint32_t predux_min(const Packet4ui& a) { return vminvq_u32(a); } +#else template<> EIGEN_STRONG_INLINE uint8_t predux_min(const Packet8uc& a) { uint8x8_t min = vpmin_u8(a,a); @@ -2663,12 +2677,17 @@ template<> EIGEN_STRONG_INLINE uint32_t predux_min(const Packet4ui& a const uint32x2_t min = vmin_u32(vget_low_u32(a), vget_high_u32(a)); return vget_lane_u32(vpmin_u32(min, min), 0); } +#endif template<> EIGEN_STRONG_INLINE int64_t predux_min(const Packet2l& a) { return (std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1)); } template<> EIGEN_STRONG_INLINE uint64_t predux_min(const Packet2ul& a) { return (std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1)); } // max +#if EIGEN_ARCH_ARM64 +template<> EIGEN_STRONG_INLINE float predux_max(const Packet2f& a) { return vmaxv_f32(a); } +template<> EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) { return vmaxvq_f32(a); } +#else template<> EIGEN_STRONG_INLINE float predux_max(const Packet2f& a) { return vget_lane_f32(vpmax_f32(a,a), 0); } template<> EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) @@ -2676,6 +2695,7 @@ template<> EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) const float32x2_t max = vmax_f32(vget_low_f32(a), vget_high_f32(a)); return vget_lane_f32(vpmax_f32(max, max), 0); } +#endif template<> EIGEN_STRONG_INLINE int8_t predux_max(const Packet4c& a) { const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a)); @@ -2683,6 +2703,10 @@ template<> EIGEN_STRONG_INLINE int8_t predux_max(const Packet4c& a) max = vpmax_s8(max, max); return vget_lane_s8(max, 0); } +#if EIGEN_ARCH_ARM64 +template<> EIGEN_STRONG_INLINE int8_t predux_max(const Packet8c& a) { return vmaxv_s8(a); } +template<> EIGEN_STRONG_INLINE int8_t predux_max(const Packet16c& a) { return vmaxvq_s8(a); } +#else template<> EIGEN_STRONG_INLINE int8_t predux_max(const Packet8c& a) { int8x8_t max = vpmax_s8(a,a); @@ -2698,6 +2722,7 @@ template<> EIGEN_STRONG_INLINE int8_t predux_max(const Packet16c& a) max = vpmax_s8(max, max); return vget_lane_s8(max, 0); } +#endif template<> EIGEN_STRONG_INLINE uint8_t predux_max(const Packet4uc& a) { const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a)); @@ -2705,6 +2730,18 @@ template<> EIGEN_STRONG_INLINE uint8_t predux_max(const Packet4uc& a) max = vpmax_u8(max, max); return vget_lane_u8(max, 0); } +#if EIGEN_ARCH_ARM64 +template<> EIGEN_STRONG_INLINE uint8_t predux_max(const Packet8uc& a) { return vmaxv_u8(a); } +template<> EIGEN_STRONG_INLINE uint8_t predux_max(const Packet16uc& a) { return vmaxvq_u8(a); } +template<> EIGEN_STRONG_INLINE int16_t predux_max(const Packet4s& a) { return vmaxv_s16(a); } +template<> EIGEN_STRONG_INLINE int16_t predux_max(const Packet8s& a) { return vmaxvq_s16(a); } +template<> EIGEN_STRONG_INLINE uint16_t predux_max(const Packet4us& a) { return vmaxv_u16(a); } +template<> EIGEN_STRONG_INLINE uint16_t predux_max(const Packet8us& a) { return vmaxvq_u16(a); } +template<> EIGEN_STRONG_INLINE int32_t predux_max(const Packet2i& a) { return vmaxv_s32(a); } +template<> EIGEN_STRONG_INLINE int32_t predux_max(const Packet4i& a) { return vmaxvq_s32(a); } +template<> EIGEN_STRONG_INLINE uint32_t predux_max(const Packet2ui& a) { return vmaxv_u32(a); } +template<> EIGEN_STRONG_INLINE uint32_t predux_max(const Packet4ui& a) { return vmaxvq_u32(a); } +#else template<> EIGEN_STRONG_INLINE uint8_t predux_max(const Packet8uc& a) { uint8x8_t max = vpmax_u8(a,a); @@ -2758,6 +2795,7 @@ template<> EIGEN_STRONG_INLINE uint32_t predux_max(const Packet4ui& a const uint32x2_t max = vmax_u32(vget_low_u32(a), vget_high_u32(a)); return vget_lane_u32(vpmax_u32(max, max), 0); } +#endif template<> EIGEN_STRONG_INLINE int64_t predux_max(const Packet2l& a) { return (std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1)); } template<> EIGEN_STRONG_INLINE uint64_t predux_max(const Packet2ul& a) @@ -3273,40 +3311,115 @@ template<> EIGEN_STRONG_INLINE Packet4ui psqrt(const Packet4ui& a) { return res; } -template<> EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) { +EIGEN_STRONG_INLINE Packet4f prsqrt_float_unsafe(const Packet4f& a) { // Compute approximate reciprocal sqrt. - Packet4f x = vrsqrteq_f32(a); - // Do Newton iterations for 1/sqrt(x). - x = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, x), x), x); - x = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, x), x), x); - const Packet4f infinity = pset1(NumTraits::infinity()); - return pselect(pcmp_eq(a, pzero(a)), infinity, x); + // Does not correctly handle +/- 0 or +inf + float32x4_t result = vrsqrteq_f32(a); + result = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, result), result), result); + result = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, result), result), result); + return result; } -template<> EIGEN_STRONG_INLINE Packet2f prsqrt(const Packet2f& a) { +EIGEN_STRONG_INLINE Packet2f prsqrt_float_unsafe(const Packet2f& a) { // Compute approximate reciprocal sqrt. - Packet2f x = vrsqrte_f32(a); - // Do Newton iterations for 1/sqrt(x). - x = vmul_f32(vrsqrts_f32(vmul_f32(a, x), x), x); - x = vmul_f32(vrsqrts_f32(vmul_f32(a, x), x), x); - const Packet2f infinity = pset1(NumTraits::infinity()); - return pselect(pcmp_eq(a, pzero(a)), infinity, x); + // Does not correctly handle +/- 0 or +inf + float32x2_t result = vrsqrte_f32(a); + result = vmul_f32(vrsqrts_f32(vmul_f32(a, result), result), result); + result = vmul_f32(vrsqrts_f32(vmul_f32(a, result), result), result); + return result; +} + +template Packet prsqrt_float_common(const Packet& a) { + const Packet cst_zero = pzero(a); + const Packet cst_inf = pset1(NumTraits::infinity()); + Packet return_zero = pcmp_eq(a, cst_inf); + Packet return_inf = pcmp_eq(a, cst_zero); + Packet result = prsqrt_float_unsafe(a); + result = pselect(return_inf, por(cst_inf, a), result); + result = pandnot(result, return_zero); + return result; +} + +template<> EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) { + return prsqrt_float_common(a); +} + +template<> EIGEN_STRONG_INLINE Packet2f prsqrt(const Packet2f& a) { + return prsqrt_float_common(a); +} + +template<> EIGEN_STRONG_INLINE Packet4f preciprocal(const Packet4f& a) +{ + // Compute approximate reciprocal. + float32x4_t result = vrecpeq_f32(a); + result = vmulq_f32(vrecpsq_f32(a, result), result); + result = vmulq_f32(vrecpsq_f32(a, result), result); + return result; +} + +template<> EIGEN_STRONG_INLINE Packet2f preciprocal(const Packet2f& a) +{ + // Compute approximate reciprocal. + float32x2_t result = vrecpe_f32(a); + result = vmul_f32(vrecps_f32(a, result), result); + result = vmul_f32(vrecps_f32(a, result), result); + return result; } // Unfortunately vsqrt_f32 is only available for A64. #if EIGEN_ARCH_ARM64 -template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& _x){return vsqrtq_f32(_x);} -template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& _x){return vsqrt_f32(_x); } +template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) { return vsqrtq_f32(a); } + +template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& a) { return vsqrt_f32(a); } + +template<> EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) { return vdivq_f32(a, b); } + +template<> EIGEN_STRONG_INLINE Packet2f pdiv(const Packet2f& a, const Packet2f& b) { return vdiv_f32(a, b); } #else +template +EIGEN_STRONG_INLINE Packet psqrt_float_common(const Packet& a) { + const Packet cst_zero = pzero(a); + const Packet cst_inf = pset1(NumTraits::infinity()); + + Packet result = pmul(a, prsqrt_float_unsafe(a)); + Packet a_is_zero = pcmp_eq(a, cst_zero); + Packet a_is_inf = pcmp_eq(a, cst_inf); + Packet return_a = por(a_is_zero, a_is_inf); + + result = pselect(return_a, a, result); + return result; +} + template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) { - const Packet4f infinity = pset1(NumTraits::infinity()); - const Packet4f is_zero_or_inf = por(pcmp_eq(a, pzero(a)), pcmp_eq(a, infinity)); - return pselect(is_zero_or_inf, a, pmul(a, prsqrt(a))); + return psqrt_float_common(a); } + template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& a) { - const Packet2f infinity = pset1(NumTraits::infinity()); - const Packet2f is_zero_or_inf = por(pcmp_eq(a, pzero(a)), pcmp_eq(a, infinity)); - return pselect(is_zero_or_inf, a, pmul(a, prsqrt(a))); + return psqrt_float_common(a); +} + +template +EIGEN_STRONG_INLINE Packet pdiv_float_common(const Packet& a, const Packet& b) { + // if b is large, NEON intrinsics will flush preciprocal(b) to zero + // avoid underflow with the following manipulation: + // a / b = f * (a * reciprocal(f * b)) + + const Packet cst_one = pset1(1.0f); + const Packet cst_quarter = pset1(0.25f); + const Packet cst_thresh = pset1(NumTraits::highest() / 4.0f); + + Packet b_will_underflow = pcmp_le(cst_thresh, pabs(b)); + Packet f = pselect(b_will_underflow, cst_quarter, cst_one); + Packet result = pmul(f, pmul(a, preciprocal(pmul(b, f)))); + return result; +} + +template<> EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) { + return pdiv_float_common(a, b); +} + +template<> EIGEN_STRONG_INLINE Packet2f pdiv(const Packet2f& a, const Packet2f& b) { + return pdiv_float_common(a, b); } #endif @@ -3327,7 +3440,6 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, @@ -3342,7 +3454,7 @@ template<> struct packet_traits : default_packet_traits HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0, HasDiv = 1, HasFloor = 1, @@ -3386,9 +3498,9 @@ EIGEN_ALWAYS_INLINE void zip_in_place(Packet4bf& p1, Packet4bf& p2) { EIGEN_STRONG_INLINE Packet4bf F32ToBf16(const Packet4f& p) { - // See the scalar implemention in BFloat16.h for a comprehensible explanation + // See the scalar implementation in BFloat16.h for a comprehensible explanation // of this fast rounding algorithm - Packet4ui input = reinterpret_cast(p); + Packet4ui input = Packet4ui(vreinterpretq_u32_f32(p)); // lsb = (input >> 16) & 1 Packet4ui lsb = vandq_u32(vshrq_n_u32(input, 16), vdupq_n_u32(1)); @@ -3413,7 +3525,7 @@ EIGEN_STRONG_INLINE Packet4bf F32ToBf16(const Packet4f& p) EIGEN_STRONG_INLINE Packet4f Bf16ToF32(const Packet4bf& p) { - return reinterpret_cast(vshlq_n_u32(vmovl_u16(p), 16)); + return Packet4f(vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(p), 16))); } EIGEN_STRONG_INLINE Packet4bf F32MaskToBf16Mask(const Packet4f& p) { @@ -3421,21 +3533,21 @@ EIGEN_STRONG_INLINE Packet4bf F32MaskToBf16Mask(const Packet4f& p) { } template<> EIGEN_STRONG_INLINE Packet4bf pset1(const bfloat16& from) { - return pset1(from.value); + return Packet4bf(pset1(from.value)); } template<> EIGEN_STRONG_INLINE bfloat16 pfirst(const Packet4bf& from) { - return bfloat16_impl::raw_uint16_to_bfloat16(static_cast(pfirst(from))); + return bfloat16_impl::raw_uint16_to_bfloat16(static_cast(pfirst(Packet4us(from)))); } template<> EIGEN_STRONG_INLINE Packet4bf pload(const bfloat16* from) { - return pload(reinterpret_cast(from)); + return Packet4bf(pload(reinterpret_cast(from))); } template<> EIGEN_STRONG_INLINE Packet4bf ploadu(const bfloat16* from) { - return ploadu(reinterpret_cast(from)); + return Packet4bf(ploadu(reinterpret_cast(from))); } template<> EIGEN_STRONG_INLINE void pstore(bfloat16* to, const Packet4bf& from) @@ -3450,7 +3562,7 @@ template<> EIGEN_STRONG_INLINE void pstoreu(bfloat16* to, const Packet template<> EIGEN_STRONG_INLINE Packet4bf ploaddup(const bfloat16* from) { - return ploaddup(reinterpret_cast(from)); + return Packet4bf(ploaddup(reinterpret_cast(from))); } template <> EIGEN_STRONG_INLINE Packet4bf pabs(const Packet4bf& a) { @@ -3497,25 +3609,25 @@ template<> EIGEN_STRONG_INLINE Packet4bf plset(const bfloat16& a) } template<> EIGEN_STRONG_INLINE Packet4bf por(const Packet4bf& a,const Packet4bf& b) { - return por(a, b); + return Packet4bf(por(Packet4us(a), Packet4us(b))); } template<> EIGEN_STRONG_INLINE Packet4bf pxor(const Packet4bf& a,const Packet4bf& b) { - return pxor(a, b); + return Packet4bf(pxor(Packet4us(a), Packet4us(b))); } template<> EIGEN_STRONG_INLINE Packet4bf pand(const Packet4bf& a,const Packet4bf& b) { - return pand(a, b); + return Packet4bf(pand(Packet4us(a), Packet4us(b))); } template<> EIGEN_STRONG_INLINE Packet4bf pandnot(const Packet4bf& a,const Packet4bf& b) { - return pandnot(a, b); + return Packet4bf(pandnot(Packet4us(a), Packet4us(b))); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf pselect(const Packet4bf& mask, const Packet4bf& a, const Packet4bf& b) { - return pselect(mask, a, b); + return Packet4bf(pselect(Packet4us(mask), Packet4us(a), Packet4us(b))); } template<> EIGEN_STRONG_INLINE Packet4bf print(const Packet4bf& a) @@ -3554,13 +3666,13 @@ template<> EIGEN_STRONG_INLINE Packet4bf pdiv(const Packet4bf& a, con template<> EIGEN_STRONG_INLINE Packet4bf pgather(const bfloat16* from, Index stride) { - return pgather(reinterpret_cast(from), stride); + return Packet4bf(pgather(reinterpret_cast(from), stride)); } template<> EIGEN_STRONG_INLINE void pscatter(bfloat16* to, const Packet4bf& from, Index stride) { - pscatter(reinterpret_cast(to), from, stride); + pscatter(reinterpret_cast(to), Packet4us(from), stride); } template<> EIGEN_STRONG_INLINE bfloat16 predux(const Packet4bf& a) @@ -3585,7 +3697,7 @@ template<> EIGEN_STRONG_INLINE bfloat16 predux_mul(const Packet4bf& a template<> EIGEN_STRONG_INLINE Packet4bf preverse(const Packet4bf& a) { - return preverse(a); + return Packet4bf(preverse(Packet4us(a))); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) @@ -3620,35 +3732,52 @@ template<> EIGEN_STRONG_INLINE Packet4bf pcmp_le(const Packet4bf& a, template<> EIGEN_STRONG_INLINE Packet4bf pnegate(const Packet4bf& a) { - return pxor(a, pset1(static_cast(0x8000))); + return Packet4bf(pxor(Packet4us(a), pset1(static_cast(0x8000)))); } //---------- double ---------- // Clang 3.5 in the iOS toolchain has an ICE triggered by NEON intrisics for double. // Confirmed at least with __apple_build_version__ = 6000054. -#ifdef __apple_build_version__ +#if EIGEN_COMP_CLANGAPPLE // Let's hope that by the time __apple_build_version__ hits the 601* range, the bug will be fixed. // https://gist.github.com/yamaya/2924292 suggests that the 3 first digits are only updated with // major toolchain updates. -#define EIGEN_APPLE_DOUBLE_NEON_BUG (__apple_build_version__ < 6010000) +#define EIGEN_APPLE_DOUBLE_NEON_BUG (EIGEN_COMP_CLANGAPPLE < 6010000) #else #define EIGEN_APPLE_DOUBLE_NEON_BUG 0 #endif #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG +#if EIGEN_COMP_GNUC // Bug 907: workaround missing declarations of the following two functions in the ADK // Defining these functions as templates ensures that if these intrinsics are // already defined in arm_neon.h, then our workaround doesn't cause a conflict // and has lower priority in overload resolution. +// This doesn't work with MSVC though, since the function names are macros. template uint64x2_t vreinterpretq_u64_f64(T a) { return (uint64x2_t) a; } template float64x2_t vreinterpretq_f64_u64(T a) { return (float64x2_t) a; } +#endif + +#if EIGEN_COMP_MSVC_STRICT +typedef eigen_packet_wrapper Packet2d; +typedef eigen_packet_wrapper Packet1d; +EIGEN_ALWAYS_INLINE Packet2d make_packet2d(double a, double b) { + double from[2] = {a, b}; + return vld1q_f64(from); +} + +#else typedef float64x2_t Packet2d; typedef float64x1_t Packet1d; +EIGEN_ALWAYS_INLINE Packet2d make_packet2d(double a, double b) { return Packet2d{a, b}; } +#endif + + // fuctionally equivalent to _mm_shuffle_pd in SSE (i.e. shuffle(m, n, mask) equals _mm_shuffle_pd(m,n,mask)) // Currently used in LU/arch/InverseSize4.h to enable a shared implementation // for fast inversion of matrices of size 4. @@ -3656,7 +3785,7 @@ EIGEN_STRONG_INLINE Packet2d shuffle(const Packet2d& m, const Packet2d& n, int m { const double* a = reinterpret_cast(&m); const double* b = reinterpret_cast(&n); - Packet2d res = {*(a + (mask & 1)), *(b + ((mask >> 1) & 1))}; + Packet2d res = make_packet2d(*(a + (mask & 1)), *(b + ((mask >> 1) & 1))); return res; } @@ -3673,7 +3802,7 @@ EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d& a,const Packet2d& b) return shuffle(a, b, 3); } #define vec2d_duplane(a, p) \ - vdupq_laneq_f64(a, p) + Packet2d(vdupq_laneq_f64(a, p)) template<> struct packet_traits : default_packet_traits { @@ -3684,7 +3813,6 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 2, - HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, @@ -3699,7 +3827,7 @@ template<> struct packet_traits : default_packet_traits HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0, HasDiv = 1, @@ -3707,10 +3835,13 @@ template<> struct packet_traits : default_packet_traits HasCeil = 1, HasRint = 1, +#if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG + HasExp = 1, + HasLog = 1, + HasATan = 1, +#endif HasSin = 0, HasCos = 0, - HasLog = 1, - HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasTanh = 0, @@ -3747,7 +3878,7 @@ template<> EIGEN_STRONG_INLINE Packet2d psub(const Packet2d& a, const template<> EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& , const Packet2d& ); template<> EIGEN_STRONG_INLINE Packet2d paddsub(const Packet2d& a, const Packet2d& b){ - const Packet2d mask = {numext::bit_cast(0x8000000000000000ull),0.0}; + const Packet2d mask = make_packet2d(numext::bit_cast(0x8000000000000000ull), 0.0); return padd(a, pxor(mask, b)); } @@ -3846,32 +3977,31 @@ template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vabsq_f64(a); } -#if EIGEN_COMP_CLANG && defined(__apple_build_version__) -// workaround ICE, see bug 907 -template<> EIGEN_STRONG_INLINE double predux(const Packet2d& a) -{ return (vget_low_f64(a) + vget_high_f64(a))[0]; } -#else +template <> +EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) { + return vreinterpretq_f64_s64(vshrq_n_s64(vreinterpretq_s64_f64(a), 63)); +} + template<> EIGEN_STRONG_INLINE double predux(const Packet2d& a) -{ return vget_lane_f64(vget_low_f64(a) + vget_high_f64(a), 0); } -#endif +{ return vaddvq_f64(a); } // Other reduction functions: // mul -#if EIGEN_COMP_CLANG && defined(__apple_build_version__) +#if EIGEN_COMP_CLANGAPPLE template<> EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) { return (vget_low_f64(a) * vget_high_f64(a))[0]; } #else template<> EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) -{ return vget_lane_f64(vget_low_f64(a) * vget_high_f64(a), 0); } +{ return vget_lane_f64(vmul_f64(vget_low_f64(a), vget_high_f64(a)), 0); } #endif // min template<> EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) -{ return vgetq_lane_f64(vpminq_f64(a,a), 0); } +{ return vminvq_f64(a); } // max template<> EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) -{ return vgetq_lane_f64(vpmaxq_f64(a,a), 0); } +{ return vmaxvq_f64(a); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void @@ -3906,18 +4036,14 @@ template<> EIGEN_STRONG_INLINE Packet2d pset1frombits(uint64_t from) { return vreinterpretq_f64_u64(vdupq_n_u64(from)); } template<> EIGEN_STRONG_INLINE Packet2d prsqrt(const Packet2d& a) { - // Compute approximate reciprocal sqrt. - Packet2d x = vrsqrteq_f64(a); // Do Newton iterations for 1/sqrt(x). - x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x); - x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x); - x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x); - const Packet2d infinity = pset1(NumTraits::infinity()); - return pselect(pcmp_eq(a, pzero(a)), infinity, x); + return generic_rsqrt_newton_step::run(a, vrsqrteq_f64(a)); } template<> EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& _x){ return vsqrtq_f64(_x); } +#endif // EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG + // Do we have an fp16 types and supporting Neon intrinsics? #if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC typedef float16x4_t Packet4hf; @@ -3931,7 +4057,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasHalfPacket = 1, HasCmp = 1, HasCast = 1, @@ -3947,7 +4072,7 @@ struct packet_traits : default_packet_traits { HasMin = 1, HasMax = 1, HasConj = 1, - HasSetLinear = 0, + HasSetLinear = 1, HasBlend = 0, HasInsert = 1, HasReduxp = 1, @@ -3959,6 +4084,7 @@ struct packet_traits : default_packet_traits { HasCos = 0, HasLog = 0, HasExp = 0, + HasTanh = packet_traits::HasTanh, // tanh calls tanh HasSqrt = 1, HasRsqrt = 1, HasErf = EIGEN_FAST_MATH, @@ -4418,11 +4544,21 @@ EIGEN_STRONG_INLINE Packet8hf pabs(const Packet8hf& a) { return vabsq_f16(a); } +template<> +EIGEN_STRONG_INLINE Packet8hf psignbit(const Packet8hf& a) { + return vreinterpretq_f16_s16(vshrq_n_s16(vreinterpretq_s16_f16(a), 15)); +} + template <> EIGEN_STRONG_INLINE Packet4hf pabs(const Packet4hf& a) { return vabs_f16(a); } +template <> +EIGEN_STRONG_INLINE Packet4hf psignbit(const Packet4hf& a) { + return vreinterpret_f16_s16( vshr_n_s16( vreinterpret_s16_f16(a), 15)); +} + template <> EIGEN_STRONG_INLINE Eigen::half predux(const Packet8hf& a) { float16x4_t a_lo, a_hi, sum; @@ -4474,51 +4610,29 @@ EIGEN_STRONG_INLINE Eigen::half predux_mul(const Packet4hf& a) { template <> EIGEN_STRONG_INLINE Eigen::half predux_min(const Packet8hf& a) { - float16x4_t a_lo, a_hi, min; - - a_lo = vget_low_f16(a); - a_hi = vget_high_f16(a); - min = vpmin_f16(a_lo, a_hi); - min = vpmin_f16(min, min); - min = vpmin_f16(min, min); - Eigen::half h; - h.x = vget_lane_f16(min, 0); + h.x = vminvq_f16(a); return h; } template <> EIGEN_STRONG_INLINE Eigen::half predux_min(const Packet4hf& a) { - Packet4hf tmp; - tmp = vpmin_f16(a, a); - tmp = vpmin_f16(tmp, tmp); Eigen::half h; - h.x = vget_lane_f16(tmp, 0); + h.x = vminv_f16(a); return h; } template <> EIGEN_STRONG_INLINE Eigen::half predux_max(const Packet8hf& a) { - float16x4_t a_lo, a_hi, max; - - a_lo = vget_low_f16(a); - a_hi = vget_high_f16(a); - max = vpmax_f16(a_lo, a_hi); - max = vpmax_f16(max, max); - max = vpmax_f16(max, max); - Eigen::half h; - h.x = vget_lane_f16(max, 0); + h.x = vmaxvq_f16(a); return h; } template <> EIGEN_STRONG_INLINE Eigen::half predux_max(const Packet4hf& a) { - Packet4hf tmp; - tmp = vpmax_f16(a, a); - tmp = vpmax_f16(tmp, tmp); Eigen::half h; - h.x = vget_lane_f16(tmp, 0); + h.x = vmaxv_f16(a); return h; } @@ -4578,8 +4692,6 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& } #endif // end EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC -#endif // EIGEN_ARCH_ARM64 - } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/TypeCasting.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/TypeCasting.h index 54f97336e03..a265e4d10ff 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/TypeCasting.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/TypeCasting.h @@ -11,26 +11,181 @@ #ifndef EIGEN_TYPE_CASTING_NEON_H #define EIGEN_TYPE_CASTING_NEON_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { + //============================================================================== -// pcast, SrcType = float +// preinterpret (truncation operations) //============================================================================== + template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; +EIGEN_STRONG_INLINE Packet8c preinterpret(const Packet16c& a) { + return Packet8c(vget_low_s8(a)); +} +template <> +EIGEN_STRONG_INLINE Packet4c preinterpret(const Packet8c& a) { + return Packet4c(vget_lane_s32(vreinterpret_s32_s8(a), 0)); +} +template <> +EIGEN_STRONG_INLINE Packet4c preinterpret(const Packet16c& a) { + return preinterpret(preinterpret(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet8uc preinterpret(const Packet16uc& a) { + return Packet8uc(vget_low_u8(a)); +} +template <> +EIGEN_STRONG_INLINE Packet4uc preinterpret(const Packet8uc& a) { + return Packet4uc(vget_lane_u32(vreinterpret_u32_u8(a), 0)); +} +template <> +EIGEN_STRONG_INLINE Packet4uc preinterpret(const Packet16uc& a) { + return preinterpret(preinterpret(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet4s preinterpret(const Packet8s& a) { + return Packet4s(vget_low_s16(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet4us preinterpret(const Packet8us& a) { + return Packet4us(vget_low_u16(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet2i preinterpret(const Packet4i& a) { + return Packet2i(vget_low_s32(a)); +} +template <> +EIGEN_STRONG_INLINE Packet2ui preinterpret(const Packet4ui& a) { + return Packet2ui(vget_low_u32(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet2f preinterpret(const Packet4f& a) { + return Packet2f(vget_low_f32(a)); +} + +//============================================================================== +// preinterpret +//============================================================================== +template <> +EIGEN_STRONG_INLINE Packet2f preinterpret(const Packet2i& a) { + return Packet2f(vreinterpret_f32_s32(a)); +} +template <> +EIGEN_STRONG_INLINE Packet2f preinterpret(const Packet2ui& a) { + return Packet2f(vreinterpret_f32_u32(a)); +} +template <> +EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4i& a) { + return Packet4f(vreinterpretq_f32_s32(a)); +} +template <> +EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4ui& a) { + return Packet4f(vreinterpretq_f32_u32(a)); +} + + +template <> +EIGEN_STRONG_INLINE Packet4c preinterpret(const Packet4uc& a) { + return static_cast(a); +} +template <> +EIGEN_STRONG_INLINE Packet8c preinterpret(const Packet8uc& a) { + return Packet8c(vreinterpret_s8_u8(a)); +} +template <> +EIGEN_STRONG_INLINE Packet16c preinterpret(const Packet16uc& a) { + return Packet16c(vreinterpretq_s8_u8(a)); +} + + +template <> +EIGEN_STRONG_INLINE Packet4uc preinterpret(const Packet4c& a) { + return static_cast(a); +} +template <> +EIGEN_STRONG_INLINE Packet8uc preinterpret(const Packet8c& a) { + return Packet8uc(vreinterpret_u8_s8(a)); +} +template <> +EIGEN_STRONG_INLINE Packet16uc preinterpret(const Packet16c& a) { + return Packet16uc(vreinterpretq_u8_s8(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet4s preinterpret(const Packet4us& a) { + return Packet4s(vreinterpret_s16_u16(a)); +} +template <> +EIGEN_STRONG_INLINE Packet8s preinterpret(const Packet8us& a) { + return Packet8s(vreinterpretq_s16_u16(a)); +} template <> -EIGEN_STRONG_INLINE Packet4f pcast(const Packet4f& a) { - return a; +EIGEN_STRONG_INLINE Packet4us preinterpret(const Packet4s& a) { + return Packet4us(vreinterpret_u16_s16(a)); +} +template <> +EIGEN_STRONG_INLINE Packet8us preinterpret(const Packet8s& a) { + return Packet8us(vreinterpretq_u16_s16(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet2i preinterpret(const Packet2f& a) { + return Packet2i(vreinterpret_s32_f32(a)); } template <> -EIGEN_STRONG_INLINE Packet2f pcast(const Packet2f& a) { - return a; +EIGEN_STRONG_INLINE Packet2i preinterpret(const Packet2ui& a) { + return Packet2i(vreinterpret_s32_u32(a)); +} +template <> +EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4f& a) { + return Packet4i(vreinterpretq_s32_f32(a)); +} +template <> +EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4ui& a) { + return Packet4i(vreinterpretq_s32_u32(a)); } +template <> +EIGEN_STRONG_INLINE Packet2ui preinterpret(const Packet2f& a) { + return Packet2ui(vreinterpret_u32_f32(a)); +} +template <> +EIGEN_STRONG_INLINE Packet2ui preinterpret(const Packet2i& a) { + return Packet2ui(vreinterpret_u32_s32(a)); +} +template <> +EIGEN_STRONG_INLINE Packet4ui preinterpret(const Packet4f& a) { + return Packet4ui(vreinterpretq_u32_f32(a)); +} +template <> +EIGEN_STRONG_INLINE Packet4ui preinterpret(const Packet4i& a) { + return Packet4ui(vreinterpretq_u32_s32(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet2l preinterpret(const Packet2ul& a) { + return Packet2l(vreinterpretq_s64_u64(a)); +} +template <> +EIGEN_STRONG_INLINE Packet2ul preinterpret(const Packet2l& a) { + return Packet2ul(vreinterpretq_u64_s64(a)); +} + +//============================================================================== +// pcast, SrcType = float +//============================================================================== + + template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; @@ -47,10 +202,18 @@ EIGEN_STRONG_INLINE Packet2l pcast(const Packet4f& a) { return vcvtq_s64_f64(vcvt_f64_f32(vget_low_f32(a))); } template <> +EIGEN_STRONG_INLINE Packet2l pcast(const Packet2f& a) { + return vcvtq_s64_f64(vcvt_f64_f32(a)); +} +template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet4f& a) { // Discard second half of input. return vcvtq_u64_f64(vcvt_f64_f32(vget_low_f32(a))); } +template <> +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet2f& a) { + return vcvtq_u64_f64(vcvt_f64_f32(a)); +} #else template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet4f& a) { @@ -58,10 +221,19 @@ EIGEN_STRONG_INLINE Packet2l pcast(const Packet4f& a) { return vmovl_s32(vget_low_s32(vcvtq_s32_f32(a))); } template <> +EIGEN_STRONG_INLINE Packet2l pcast(const Packet2f& a) { + return vmovl_s32(vcvt_s32_f32(a)); +} +template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet4f& a) { // Discard second half of input. return vmovl_u32(vget_low_u32(vcvtq_u32_f32(a))); } +template <> +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet2f& a) { + // Discard second half of input. + return vmovl_u32(vcvt_u32_f32(a)); +} #endif // EIGEN_ARCH_ARM64 template <> @@ -99,6 +271,10 @@ EIGEN_STRONG_INLINE Packet8s pcast(const Packet4f& a, const return vcombine_s16(vmovn_s32(vcvtq_s32_f32(a)), vmovn_s32(vcvtq_s32_f32(b))); } template <> +EIGEN_STRONG_INLINE Packet4s pcast(const Packet4f& a) { + return vmovn_s32(vcvtq_s32_f32(a)); +} +template <> EIGEN_STRONG_INLINE Packet4s pcast(const Packet2f& a, const Packet2f& b) { return vmovn_s32(vcombine_s32(vcvt_s32_f32(a), vcvt_s32_f32(b))); } @@ -112,6 +288,10 @@ EIGEN_STRONG_INLINE Packet8us pcast(const Packet4f& a, cons return vcombine_u16(vmovn_u32(vcvtq_u32_f32(a)), vmovn_u32(vcvtq_u32_f32(b))); } template <> +EIGEN_STRONG_INLINE Packet4us pcast(const Packet4f& a) { + return vmovn_u32(vcvtq_u32_f32(a)); +} +template <> EIGEN_STRONG_INLINE Packet4us pcast(const Packet2f& a, const Packet2f& b) { return vmovn_u32(vcombine_u32(vcvt_u32_f32(a), vcvt_u32_f32(b))); } @@ -128,12 +308,25 @@ EIGEN_STRONG_INLINE Packet16c pcast(const Packet4f& a, cons return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16)); } template <> +EIGEN_STRONG_INLINE Packet8c pcast(const Packet4f& a, const Packet4f& b) { + const int16x8_t ab_s16 = pcast(a, b); + return vmovn_s16(ab_s16); +} +template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet2f& a, const Packet2f& b, const Packet2f& c, const Packet2f& d) { const int16x4_t ab_s16 = pcast(a, b); const int16x4_t cd_s16 = pcast(c, d); return vmovn_s16(vcombine_s16(ab_s16, cd_s16)); } +template <> +EIGEN_STRONG_INLINE Packet4c pcast(const Packet4f& a) { + const int32x4_t a_s32x4 = vcvtq_s32_f32(a); + const int16x4_t a_s16x4 = vmovn_s32(a_s32x4); + const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4); + const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8); + return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0); +} template <> struct type_casting_traits { @@ -142,16 +335,20 @@ struct type_casting_traits { template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet4f& a, const Packet4f& b, const Packet4f& c, const Packet4f& d) { - const uint16x8_t ab_u16 = pcast(a, b); - const uint16x8_t cd_u16 = pcast(c, d); - return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16)); + return preinterpret(pcast(a, b, c, d)); +} +template <> +EIGEN_STRONG_INLINE Packet8uc pcast(const Packet4f& a, const Packet4f& b) { + return preinterpret(pcast(a, b)); } template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet2f& a, const Packet2f& b, const Packet2f& c, const Packet2f& d) { - const uint16x4_t ab_u16 = pcast(a, b); - const uint16x4_t cd_u16 = pcast(c, d); - return vmovn_u16(vcombine_u16(ab_u16, cd_u16)); + return preinterpret(pcast(a, b, c, d)); +} +template <> +EIGEN_STRONG_INLINE Packet4uc pcast(const Packet4f& a) { + return static_cast(pcast(a)); } //============================================================================== @@ -167,6 +364,10 @@ EIGEN_STRONG_INLINE Packet4f pcast(const Packet16c& a) { return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))))); } template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet4c& a) { + return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_s32(vdup_n_s32(a)))))); +} +template <> EIGEN_STRONG_INLINE Packet2f pcast(const Packet8c& a) { // Discard all but first 2 bytes. return vcvt_f32_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a))))); @@ -188,7 +389,7 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet16c& a) { - return vreinterpretq_u64_s64(pcast(a)); + return preinterpret(pcast(a)); } template <> @@ -201,22 +402,35 @@ EIGEN_STRONG_INLINE Packet4i pcast(const Packet16c& a) { return vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))); } template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet8c& a) { + return vmovl_s16(vget_low_s16(vmovl_s8(a))); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet4c& a) { + return pcast(vreinterpret_s8_s32(vdup_n_s32(a))); +} +template <> EIGEN_STRONG_INLINE Packet2i pcast(const Packet8c& a) { // Discard all but first 2 bytes. return vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a)))); } + template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet16c& a) { - return vreinterpretq_u32_s32(pcast(a)); + return preinterpret(pcast(a)); } template <> EIGEN_STRONG_INLINE Packet2ui pcast(const Packet8c& a) { - return vreinterpret_u32_s32(pcast(a)); + return preinterpret(pcast(a)); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4c& a) { + return preinterpret(pcast(a)); } template <> @@ -229,10 +443,18 @@ EIGEN_STRONG_INLINE Packet8s pcast(const Packet16c& a) { return vmovl_s8(vget_low_s8(a)); } template <> +EIGEN_STRONG_INLINE Packet8s pcast(const Packet8c& a) { + return vmovl_s8(a); +} +template <> EIGEN_STRONG_INLINE Packet4s pcast(const Packet8c& a) { // Discard second half of input. return vget_low_s16(vmovl_s8(a)); } +template <> +EIGEN_STRONG_INLINE Packet4s pcast(const Packet4c& a) { + return pcast(vreinterpret_s8_s32(vdup_n_s32(a))); +} template <> struct type_casting_traits { @@ -240,46 +462,21 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet16c& a) { - return vreinterpretq_u16_s16(pcast(a)); + return preinterpret(pcast(a)); } template <> -EIGEN_STRONG_INLINE Packet4us pcast(const Packet8c& a) { - return vreinterpret_u16_s16(pcast(a)); +EIGEN_STRONG_INLINE Packet8us pcast(const Packet8c& a) { + return preinterpret(pcast(a)); } - template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet16c pcast(const Packet16c& a) { - return a; -} -template <> -EIGEN_STRONG_INLINE Packet8c pcast(const Packet8c& a) { - return a; +EIGEN_STRONG_INLINE Packet4us pcast(const Packet8c& a) { + return preinterpret(pcast(a)); } template <> -EIGEN_STRONG_INLINE Packet4c pcast(const Packet4c& a) { - return a; +EIGEN_STRONG_INLINE Packet4us pcast(const Packet4c& a) { + return preinterpret(pcast(a)); } -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet16uc pcast(const Packet16c& a) { - return vreinterpretq_u8_s8(a); -} -template <> -EIGEN_STRONG_INLINE Packet8uc pcast(const Packet8c& a) { - return vreinterpret_u8_s8(a); -} -template <> -EIGEN_STRONG_INLINE Packet4uc pcast(const Packet4c& a) { - return static_cast(a); -} //============================================================================== // pcast, SrcType = uint8_t @@ -294,6 +491,10 @@ EIGEN_STRONG_INLINE Packet4f pcast(const Packet16uc& a) { return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))))); } template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet4uc& a) { + return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a)))))); +} +template <> EIGEN_STRONG_INLINE Packet2f pcast(const Packet8uc& a) { // Discard all but first 2 bytes. return vcvt_f32_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a))))); @@ -315,7 +516,7 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet16uc& a) { - return vreinterpretq_s64_u64(pcast(a)); + return preinterpret(pcast(a)); } template <> @@ -328,10 +529,18 @@ EIGEN_STRONG_INLINE Packet4ui pcast(const Packet16uc& a) return vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a)))); } template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet8uc& a) { + return vmovl_u16(vget_low_u16(vmovl_u8(a))); +} +template <> EIGEN_STRONG_INLINE Packet2ui pcast(const Packet8uc& a) { // Discard all but first 2 bytes. return vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a)))); } +template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4uc& a) { + return pcast(vreinterpret_u8_u32(vdup_n_u32(a))); +} template <> struct type_casting_traits { @@ -339,11 +548,15 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet16uc& a) { - return vreinterpretq_s32_u32(pcast(a)); + return preinterpret(pcast(a)); } template <> EIGEN_STRONG_INLINE Packet2i pcast(const Packet8uc& a) { - return vreinterpret_s32_u32(pcast(a)); + return preinterpret(pcast(a)); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet4uc& a) { + return preinterpret(pcast(a)); } template <> @@ -356,57 +569,32 @@ EIGEN_STRONG_INLINE Packet8us pcast(const Packet16uc& a) return vmovl_u8(vget_low_u8(a)); } template <> -EIGEN_STRONG_INLINE Packet4us pcast(const Packet8uc& a) { - // Discard second half of input. - return vget_low_u16(vmovl_u8(a)); +EIGEN_STRONG_INLINE Packet8us pcast(const Packet8uc& a) { + return vmovl_u8(a); +} +template <> +EIGEN_STRONG_INLINE Packet4us pcast(const Packet4uc& a) { + return vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a)))); } + template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet16uc& a) { - return vreinterpretq_s16_u16(pcast(a)); -} -template <> -EIGEN_STRONG_INLINE Packet4s pcast(const Packet8uc& a) { - return vreinterpret_s16_u16(pcast(a)); -} - -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet16uc pcast(const Packet16uc& a) { - return a; + return preinterpret(pcast(a)); } template <> -EIGEN_STRONG_INLINE Packet8uc pcast(const Packet8uc& a) { - return a; +EIGEN_STRONG_INLINE Packet8s pcast(const Packet8uc& a) { + return preinterpret(pcast(a)); } template <> -EIGEN_STRONG_INLINE Packet4uc pcast(const Packet4uc& a) { - return a; +EIGEN_STRONG_INLINE Packet4s pcast(const Packet4uc& a) { + return preinterpret(pcast(a)); } -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet16c pcast(const Packet16uc& a) { - return vreinterpretq_s8_u8(a); -} -template <> -EIGEN_STRONG_INLINE Packet8c pcast(const Packet8uc& a) { - return vreinterpret_s8_u8(a); -} -template <> -EIGEN_STRONG_INLINE Packet4c pcast(const Packet4uc& a) { - return static_cast(a); -} //============================================================================== // pcast, SrcType = int16_t @@ -421,6 +609,10 @@ EIGEN_STRONG_INLINE Packet4f pcast(const Packet8s& a) { return vcvtq_f32_s32(vmovl_s16(vget_low_s16(a))); } template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet4s& a) { + return vcvtq_f32_s32(vmovl_s16(a)); +} +template <> EIGEN_STRONG_INLINE Packet2f pcast(const Packet4s& a) { // Discard second half of input. return vcvt_f32_s32(vget_low_s32(vmovl_s16(a))); @@ -442,7 +634,7 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet8s& a) { - return vreinterpretq_u64_s64(pcast(a)); + return preinterpret(pcast(a)); } template <> @@ -455,6 +647,10 @@ EIGEN_STRONG_INLINE Packet4i pcast(const Packet8s& a) { return vmovl_s16(vget_low_s16(a)); } template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet4s& a) { + return vmovl_s16(a); +} +template <> EIGEN_STRONG_INLINE Packet2i pcast(const Packet4s& a) { // Discard second half of input. return vget_low_s32(vmovl_s16(a)); @@ -466,38 +662,17 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet8s& a) { - return vreinterpretq_u32_s32(pcast(a)); -} -template <> -EIGEN_STRONG_INLINE Packet2ui pcast(const Packet4s& a) { - return vreinterpret_u32_s32(pcast(a)); + return preinterpret(pcast(a)); } - -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; template <> -EIGEN_STRONG_INLINE Packet8s pcast(const Packet8s& a) { - return a; +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4s& a) { + return preinterpret(pcast(a)); } template <> -EIGEN_STRONG_INLINE Packet4s pcast(const Packet4s& a) { - return a; +EIGEN_STRONG_INLINE Packet2ui pcast(const Packet4s& a) { + return preinterpret(pcast(a)); } -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet8us pcast(const Packet8s& a) { - return vreinterpretq_u16_s16(a); -} -template <> -EIGEN_STRONG_INLINE Packet4us pcast(const Packet4s& a) { - return vreinterpret_u16_s16(a); -} template <> struct type_casting_traits { @@ -508,9 +683,18 @@ EIGEN_STRONG_INLINE Packet16c pcast(const Packet8s& a, cons return vcombine_s8(vmovn_s16(a), vmovn_s16(b)); } template <> +EIGEN_STRONG_INLINE Packet8c pcast(const Packet8s& a) { + return vmovn_s16(a); +} +template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet4s& a, const Packet4s& b) { return vmovn_s16(vcombine_s16(a, b)); } +template <> +EIGEN_STRONG_INLINE Packet4c pcast(const Packet4s& a) { + const int8x8_t aa_s8x8 = pcast(a, a); + return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0); +} template <> struct type_casting_traits { @@ -518,11 +702,19 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet8s& a, const Packet8s& b) { - return vcombine_u8(vmovn_u16(vreinterpretq_u16_s16(a)), vmovn_u16(vreinterpretq_u16_s16(b))); + return preinterpret(pcast(a, b)); +} +template <> +EIGEN_STRONG_INLINE Packet8uc pcast(const Packet8s& a) { + return preinterpret(pcast(a)); } template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet4s& a, const Packet4s& b) { - return vmovn_u16(vcombine_u16(vreinterpret_u16_s16(a), vreinterpret_u16_s16(b))); + return preinterpret(pcast(a, b)); +} +template <> +EIGEN_STRONG_INLINE Packet4uc pcast(const Packet4s& a) { + return static_cast(pcast(a)); } //============================================================================== @@ -538,6 +730,10 @@ EIGEN_STRONG_INLINE Packet4f pcast(const Packet8us& a) { return vcvtq_f32_u32(vmovl_u16(vget_low_u16(a))); } template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet4us& a) { + return vcvtq_f32_u32(vmovl_u16(a)); +} +template <> EIGEN_STRONG_INLINE Packet2f pcast(const Packet4us& a) { // Discard second half of input. return vcvt_f32_u32(vget_low_u32(vmovl_u16(a))); @@ -559,7 +755,7 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet8us& a) { - return vreinterpretq_s64_u64(pcast(a)); + return preinterpret(pcast(a)); } template <> @@ -572,6 +768,10 @@ EIGEN_STRONG_INLINE Packet4ui pcast(const Packet8us& a) { return vmovl_u16(vget_low_u16(a)); } template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4us& a) { + return vmovl_u16(a); +} +template <> EIGEN_STRONG_INLINE Packet2ui pcast(const Packet4us& a) { // Discard second half of input. return vget_low_u32(vmovl_u16(a)); @@ -583,38 +783,17 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet8us& a) { - return vreinterpretq_s32_u32(pcast(a)); + return preinterpret(pcast(a)); } template <> -EIGEN_STRONG_INLINE Packet2i pcast(const Packet4us& a) { - return vreinterpret_s32_u32(pcast(a)); +EIGEN_STRONG_INLINE Packet4i pcast(const Packet4us& a) { + return preinterpret(pcast(a)); } - template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet8us pcast(const Packet8us& a) { - return a; -} -template <> -EIGEN_STRONG_INLINE Packet4us pcast(const Packet4us& a) { - return a; +EIGEN_STRONG_INLINE Packet2i pcast(const Packet4us& a) { + return preinterpret(pcast(a)); } -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet8s pcast(const Packet8us& a) { - return vreinterpretq_s16_u16(a); -} -template <> -EIGEN_STRONG_INLINE Packet4s pcast(const Packet4us& a) { - return vreinterpret_s16_u16(a); -} template <> struct type_casting_traits { @@ -625,9 +804,18 @@ EIGEN_STRONG_INLINE Packet16uc pcast(const Packet8us& a, return vcombine_u8(vmovn_u16(a), vmovn_u16(b)); } template <> +EIGEN_STRONG_INLINE Packet8uc pcast(const Packet8us& a) { + return vmovn_u16(a); +} +template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet4us& a, const Packet4us& b) { return vmovn_u16(vcombine_u16(a, b)); } +template <> +EIGEN_STRONG_INLINE Packet4uc pcast(const Packet4us& a) { + uint8x8_t aa_u8x8 = pcast(a, a); + return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0); +} template <> struct type_casting_traits { @@ -635,11 +823,19 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet8us& a, const Packet8us& b) { - return vreinterpretq_s8_u8(pcast(a, b)); + return preinterpret(pcast(a, b)); +} +template <> +EIGEN_STRONG_INLINE Packet8c pcast(const Packet8us& a) { + return preinterpret(pcast(a)); } template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet4us& a, const Packet4us& b) { - return vreinterpret_s8_u8(pcast(a, b)); + return preinterpret(pcast(a, b)); +} +template <> +EIGEN_STRONG_INLINE Packet4c pcast(const Packet4us& a) { + return static_cast(pcast(a)); } //============================================================================== @@ -667,6 +863,10 @@ EIGEN_STRONG_INLINE Packet2l pcast(const Packet4i& a) { // Discard second half of input. return vmovl_s32(vget_low_s32(a)); } +template <> +EIGEN_STRONG_INLINE Packet2l pcast(const Packet2i& a) { + return vmovl_s32(a); +} template <> struct type_casting_traits { @@ -674,34 +874,13 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet4i& a) { - return vreinterpretq_u64_s64(pcast(a)); + return preinterpret(pcast(a)); } - -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; template <> -EIGEN_STRONG_INLINE Packet4i pcast(const Packet4i& a) { - return a; -} -template <> -EIGEN_STRONG_INLINE Packet2i pcast(const Packet2i& a) { - return a; +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet2i& a) { + return preinterpret(pcast(a)); } -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4i& a) { - return vreinterpretq_u32_s32(a); -} -template <> -EIGEN_STRONG_INLINE Packet2ui pcast(const Packet2i& a) { - return vreinterpret_u32_s32(a); -} template <> struct type_casting_traits { @@ -712,6 +891,10 @@ EIGEN_STRONG_INLINE Packet8s pcast(const Packet4i& a, const return vcombine_s16(vmovn_s32(a), vmovn_s32(b)); } template <> +EIGEN_STRONG_INLINE Packet4s pcast(const Packet4i& a) { + return vmovn_s32(a); +} +template <> EIGEN_STRONG_INLINE Packet4s pcast(const Packet2i& a, const Packet2i& b) { return vmovn_s32(vcombine_s32(a, b)); } @@ -725,6 +908,10 @@ EIGEN_STRONG_INLINE Packet8us pcast(const Packet4i& a, cons return vcombine_u16(vmovn_u32(vreinterpretq_u32_s32(a)), vmovn_u32(vreinterpretq_u32_s32(b))); } template <> +EIGEN_STRONG_INLINE Packet4us pcast(const Packet4i& a) { + return vmovn_u32(vreinterpretq_u32_s32(a)); +} +template <> EIGEN_STRONG_INLINE Packet4us pcast(const Packet2i& a, const Packet2i& b) { return vmovn_u32(vreinterpretq_u32_s32(vcombine_s32(a, b))); } @@ -741,12 +928,24 @@ EIGEN_STRONG_INLINE Packet16c pcast(const Packet4i& a, cons return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16)); } template <> +EIGEN_STRONG_INLINE Packet8c pcast(const Packet4i& a, const Packet4i& b) { + const int16x8_t ab_s16 = pcast(a, b); + return vmovn_s16(ab_s16); +} +template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet2i& a, const Packet2i& b, const Packet2i& c, const Packet2i& d) { const int16x4_t ab_s16 = vmovn_s32(vcombine_s32(a, b)); const int16x4_t cd_s16 = vmovn_s32(vcombine_s32(c, d)); return vmovn_s16(vcombine_s16(ab_s16, cd_s16)); } +template <> +EIGEN_STRONG_INLINE Packet4c pcast(const Packet4i& a) { + const int16x4_t a_s16x4 = vmovn_s32(a); + const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4); + const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8); + return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0); +} template <> struct type_casting_traits { @@ -755,16 +954,20 @@ struct type_casting_traits { template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet4i& a, const Packet4i& b, const Packet4i& c, const Packet4i& d) { - const uint16x8_t ab_u16 = pcast(a, b); - const uint16x8_t cd_u16 = pcast(c, d); - return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16)); + return preinterpret(pcast(a, b, c, d)); +} +template <> +EIGEN_STRONG_INLINE Packet8uc pcast(const Packet4i& a, const Packet4i& b) { + return preinterpret(pcast(a, b)); } template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet2i& a, const Packet2i& b, const Packet2i& c, const Packet2i& d) { - const uint16x4_t ab_u16 = pcast(a, b); - const uint16x4_t cd_u16 = pcast(c, d); - return vmovn_u16(vcombine_u16(ab_u16, cd_u16)); + return preinterpret(pcast(a, b, c, d)); +} +template <> +EIGEN_STRONG_INLINE Packet4uc pcast(const Packet4i& a) { + return static_cast(pcast(a)); } //============================================================================== @@ -792,6 +995,10 @@ EIGEN_STRONG_INLINE Packet2ul pcast(const Packet4ui& a) { // Discard second half of input. return vmovl_u32(vget_low_u32(a)); } +template <> +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet2ui& a) { + return vmovl_u32(a); +} template <> struct type_casting_traits { @@ -799,34 +1006,13 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet4ui& a) { - return vreinterpretq_s64_u64(pcast(a)); -} - -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4ui& a) { - return a; + return preinterpret(pcast(a)); } template <> -EIGEN_STRONG_INLINE Packet2ui pcast(const Packet2ui& a) { - return a; +EIGEN_STRONG_INLINE Packet2l pcast(const Packet2ui& a) { + return preinterpret(pcast(a)); } -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet4i pcast(const Packet4ui& a) { - return vreinterpretq_s32_u32(a); -} -template <> -EIGEN_STRONG_INLINE Packet2i pcast(const Packet2ui& a) { - return vreinterpret_s32_u32(a); -} template <> struct type_casting_traits { @@ -840,6 +1026,10 @@ template <> EIGEN_STRONG_INLINE Packet4us pcast(const Packet2ui& a, const Packet2ui& b) { return vmovn_u32(vcombine_u32(a, b)); } +template <> +EIGEN_STRONG_INLINE Packet4us pcast(const Packet4ui& a) { + return vmovn_u32(a); +} template <> struct type_casting_traits { @@ -847,11 +1037,15 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet4ui& a, const Packet4ui& b) { - return vreinterpretq_s16_u16(pcast(a, b)); + return preinterpret(pcast(a, b)); } template <> EIGEN_STRONG_INLINE Packet4s pcast(const Packet2ui& a, const Packet2ui& b) { - return vreinterpret_s16_u16(pcast(a, b)); + return preinterpret(pcast(a, b)); +} +template <> +EIGEN_STRONG_INLINE Packet4s pcast(const Packet4ui& a) { + return preinterpret(pcast(a)); } template <> @@ -866,12 +1060,24 @@ EIGEN_STRONG_INLINE Packet16uc pcast(const Packet4ui& a, return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16)); } template <> +EIGEN_STRONG_INLINE Packet8uc pcast(const Packet4ui& a, const Packet4ui& b) { + const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b)); + return vmovn_u16(ab_u16); +} +template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c, const Packet2ui& d) { const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(a, b)); const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(c, d)); return vmovn_u16(vcombine_u16(ab_u16, cd_u16)); } +template <> +EIGEN_STRONG_INLINE Packet4uc pcast(const Packet4ui& a) { + const uint16x4_t a_u16x4 = vmovn_u32(a); + const uint16x8_t aa_u16x8 = vcombine_u16(a_u16x4, a_u16x4); + const uint8x8_t aa_u8x8 = vmovn_u16(aa_u16x8); + return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0); +} template <> struct type_casting_traits { @@ -880,12 +1086,20 @@ struct type_casting_traits { template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c, const Packet4ui& d) { - return vreinterpretq_s8_u8(pcast(a, b, c, d)); + return preinterpret(pcast(a, b, c, d)); +} +template <> +EIGEN_STRONG_INLINE Packet8c pcast(const Packet4ui& a, const Packet4ui& b) { + return preinterpret(pcast(a, b)); } template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c, const Packet2ui& d) { - return vreinterpret_s8_u8(pcast(a, b, c, d)); + return preinterpret(pcast(a, b, c, d)); +} +template <> +EIGEN_STRONG_INLINE Packet4c pcast(const Packet4ui& a) { + return static_cast(pcast(a)); } //============================================================================== @@ -899,24 +1113,11 @@ template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet2l& a, const Packet2l& b) { return vcvtq_f32_s32(vcombine_s32(vmovn_s64(a), vmovn_s64(b))); } - -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; template <> -EIGEN_STRONG_INLINE Packet2l pcast(const Packet2l& a) { - return a; +EIGEN_STRONG_INLINE Packet2f pcast(const Packet2l& a) { + return vcvt_f32_s32(vmovn_s64(a)); } -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet2ul pcast(const Packet2l& a) { - return vreinterpretq_u64_s64(a); -} template <> struct type_casting_traits { @@ -926,6 +1127,10 @@ template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet2l& a, const Packet2l& b) { return vcombine_s32(vmovn_s64(a), vmovn_s64(b)); } +template <> +EIGEN_STRONG_INLINE Packet2i pcast(const Packet2l& a) { + return vmovn_s64(a); +} template <> struct type_casting_traits { @@ -935,6 +1140,10 @@ template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet2l& a, const Packet2l& b) { return vcombine_u32(vmovn_u64(vreinterpretq_u64_s64(a)), vmovn_u64(vreinterpretq_u64_s64(b))); } +template <> +EIGEN_STRONG_INLINE Packet2ui pcast(const Packet2l& a) { + return vmovn_u64(vreinterpretq_u64_s64(a)); +} template <> struct type_casting_traits { @@ -947,6 +1156,11 @@ EIGEN_STRONG_INLINE Packet8s pcast(const Packet2l& a, const const int32x4_t cd_s32 = pcast(c, d); return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32)); } +template <> +EIGEN_STRONG_INLINE Packet4s pcast(const Packet2l& a, const Packet2l& b) { + const int32x4_t ab_s32 = pcast(a, b); + return vmovn_s32(ab_s32); +} template <> struct type_casting_traits { @@ -955,9 +1169,11 @@ struct type_casting_traits { template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet2l& a, const Packet2l& b, const Packet2l& c, const Packet2l& d) { - const uint32x4_t ab_u32 = pcast(a, b); - const uint32x4_t cd_u32 = pcast(c, d); - return vcombine_u16(vmovn_u32(ab_u32), vmovn_u32(cd_u32)); + return preinterpret(pcast(a, b, c, d)); +} +template <> +EIGEN_STRONG_INLINE Packet4us pcast(const Packet2l& a, const Packet2l& b) { + return preinterpret(pcast(a, b)); } template <> @@ -972,6 +1188,19 @@ EIGEN_STRONG_INLINE Packet16c pcast(const Packet2l& a, cons const int16x8_t efgh_s16 = pcast(e, f, g, h); return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16)); } +template <> +EIGEN_STRONG_INLINE Packet8c pcast(const Packet2l& a, const Packet2l& b, const Packet2l& c, + const Packet2l& d) { + const int16x8_t abcd_s16 = pcast(a, b, c, d); + return vmovn_s16(abcd_s16); +} +template <> +EIGEN_STRONG_INLINE Packet4c pcast(const Packet2l& a, const Packet2l& b) { + const int16x4_t ab_s16 = pcast(a, b); + const int16x8_t abab_s16 = vcombine_s16(ab_s16, ab_s16); + const int8x8_t abab_s8 = vmovn_s16(abab_s16); + return vget_lane_s32(vreinterpret_s32_s8(abab_s8), 0); +} template <> struct type_casting_traits { @@ -985,6 +1214,15 @@ EIGEN_STRONG_INLINE Packet16uc pcast(const Packet2l& a, co const uint16x8_t efgh_u16 = pcast(e, f, g, h); return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16)); } +template <> +EIGEN_STRONG_INLINE Packet8uc pcast(const Packet2l& a, const Packet2l& b, const Packet2l& c, + const Packet2l& d) { + return preinterpret(pcast(a, b, c, d)); +} +template <> +EIGEN_STRONG_INLINE Packet4uc pcast(const Packet2l& a, const Packet2l& b) { + return static_cast(pcast(a, b)); +} //============================================================================== // pcast, SrcType = uint64_t @@ -997,24 +1235,11 @@ template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet2ul& a, const Packet2ul& b) { return vcvtq_f32_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b))); } - template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet2ul pcast(const Packet2ul& a) { - return a; +EIGEN_STRONG_INLINE Packet2f pcast(const Packet2ul& a) { + return vcvt_f32_u32(vmovn_u64(a)); } -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet2l pcast(const Packet2ul& a) { - return vreinterpretq_s64_u64(a); -} template <> struct type_casting_traits { @@ -1024,6 +1249,10 @@ template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet2ul& a, const Packet2ul& b) { return vcombine_u32(vmovn_u64(a), vmovn_u64(b)); } +template <> +EIGEN_STRONG_INLINE Packet2ui pcast(const Packet2ul& a) { + return vmovn_u64(a); +} template <> struct type_casting_traits { @@ -1031,7 +1260,11 @@ struct type_casting_traits { }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet2ul& a, const Packet2ul& b) { - return vreinterpretq_s32_u32(pcast(a, b)); + return preinterpret(pcast(a, b)); +} +template <> +EIGEN_STRONG_INLINE Packet2i pcast(const Packet2ul& a) { + return preinterpret(pcast(a)); } template <> @@ -1045,6 +1278,10 @@ EIGEN_STRONG_INLINE Packet8us pcast(const Packet2ul& a, co const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(vmovn_u64(c), vmovn_u64(d))); return vcombine_u16(ab_u16, cd_u16); } +template <> +EIGEN_STRONG_INLINE Packet4us pcast(const Packet2ul& a, const Packet2ul& b) { + return vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b))); +} template <> struct type_casting_traits { @@ -1053,7 +1290,11 @@ struct type_casting_traits { template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, const Packet2ul& d) { - return vreinterpretq_s16_u16(pcast(a, b, c, d)); + return preinterpret(pcast(a, b, c, d)); +} +template <> +EIGEN_STRONG_INLINE Packet4s pcast(const Packet2ul& a, const Packet2ul& b) { + return preinterpret(pcast(a, b)); } template <> @@ -1068,6 +1309,19 @@ EIGEN_STRONG_INLINE Packet16uc pcast(const Packet2ul& a, const uint16x8_t efgh_u16 = pcast(e, f, g, h); return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16)); } +template <> +EIGEN_STRONG_INLINE Packet8uc pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, + const Packet2ul& d) { + const uint16x8_t abcd_u16 = pcast(a, b, c, d); + return vmovn_u16(abcd_u16); +} +template <> +EIGEN_STRONG_INLINE Packet4uc pcast(const Packet2ul& a, const Packet2ul& b) { + const uint16x4_t ab_u16 = pcast(a, b); + const uint16x8_t abab_u16 = vcombine_u16(ab_u16, ab_u16); + const uint8x8_t abab_u8 = vmovn_u16(abab_u16); + return vget_lane_u32(vreinterpret_u32_u8(abab_u8), 0); +} template <> struct type_casting_traits { @@ -1077,130 +1331,49 @@ template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, const Packet2ul& d, const Packet2ul& e, const Packet2ul& f, const Packet2ul& g, const Packet2ul& h) { - return vreinterpretq_s8_u8(pcast(a, b, c, d, e, f, g, h)); -} - -//============================================================================== -// preinterpret -//============================================================================== -template <> -EIGEN_STRONG_INLINE Packet2f preinterpret(const Packet2i& a) { - return vreinterpret_f32_s32(a); -} -template <> -EIGEN_STRONG_INLINE Packet2f preinterpret(const Packet2ui& a) { - return vreinterpret_f32_u32(a); + return preinterpret(pcast(a, b, c, d, e, f, g, h)); } template <> -EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4i& a) { - return vreinterpretq_f32_s32(a); -} -template <> -EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4ui& a) { - return vreinterpretq_f32_u32(a); -} - -template <> -EIGEN_STRONG_INLINE Packet4c preinterpret(const Packet4uc& a) { - return static_cast(a); -} -template <> -EIGEN_STRONG_INLINE Packet8c preinterpret(const Packet8uc& a) { - return vreinterpret_s8_u8(a); -} -template <> -EIGEN_STRONG_INLINE Packet16c preinterpret(const Packet16uc& a) { - return vreinterpretq_s8_u8(a); -} - -template <> -EIGEN_STRONG_INLINE Packet4uc preinterpret(const Packet4c& a) { - return static_cast(a); -} -template <> -EIGEN_STRONG_INLINE Packet8uc preinterpret(const Packet8c& a) { - return vreinterpret_u8_s8(a); -} -template <> -EIGEN_STRONG_INLINE Packet16uc preinterpret(const Packet16c& a) { - return vreinterpretq_u8_s8(a); -} - -template <> -EIGEN_STRONG_INLINE Packet4s preinterpret(const Packet4us& a) { - return vreinterpret_s16_u16(a); +EIGEN_STRONG_INLINE Packet8c pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, + const Packet2ul& d) { + return preinterpret(pcast(a, b, c, d)); } template <> -EIGEN_STRONG_INLINE Packet8s preinterpret(const Packet8us& a) { - return vreinterpretq_s16_u16(a); +EIGEN_STRONG_INLINE Packet4c pcast(const Packet2ul& a, const Packet2ul& b) { + return static_cast(pcast(a, b)); } -template <> -EIGEN_STRONG_INLINE Packet4us preinterpret(const Packet4s& a) { - return vreinterpret_u16_s16(a); -} -template <> -EIGEN_STRONG_INLINE Packet8us preinterpret(const Packet8s& a) { - return vreinterpretq_u16_s16(a); -} +#if EIGEN_ARCH_ARM64 -template <> -EIGEN_STRONG_INLINE Packet2i preinterpret(const Packet2f& a) { - return vreinterpret_s32_f32(a); -} -template <> -EIGEN_STRONG_INLINE Packet2i preinterpret(const Packet2ui& a) { - return vreinterpret_s32_u32(a); -} -template <> -EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4f& a) { - return vreinterpretq_s32_f32(a); -} -template <> -EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4ui& a) { - return vreinterpretq_s32_u32(a); -} +//============================================================================== +// pcast/preinterpret, Double +//============================================================================== template <> -EIGEN_STRONG_INLINE Packet2ui preinterpret(const Packet2f& a) { - return vreinterpret_u32_f32(a); +EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet2l& a) { + return Packet2d(vreinterpretq_f64_s64(a)); } template <> -EIGEN_STRONG_INLINE Packet2ui preinterpret(const Packet2i& a) { - return vreinterpret_u32_s32(a); +EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet2ul& a) { + return Packet2d(vreinterpretq_f64_u64(a)); } template <> -EIGEN_STRONG_INLINE Packet4ui preinterpret(const Packet4f& a) { - return vreinterpretq_u32_f32(a); +EIGEN_STRONG_INLINE Packet2l preinterpret(const Packet2d& a) { + return Packet2l(vreinterpretq_s64_f64(a)); } template <> -EIGEN_STRONG_INLINE Packet4ui preinterpret(const Packet4i& a) { - return vreinterpretq_u32_s32(a); +EIGEN_STRONG_INLINE Packet2ul preinterpret(const Packet2d& a) { + return Packet2ul(vreinterpretq_u64_f64(a)); } - template <> -EIGEN_STRONG_INLINE Packet2l preinterpret(const Packet2ul& a) { - return vreinterpretq_s64_u64(a); +EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet4i& a) { + return Packet2d(vreinterpretq_f64_s32(a)); } template <> -EIGEN_STRONG_INLINE Packet2ul preinterpret(const Packet2l& a) { - return vreinterpretq_u64_s64(a); +EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet2d& a) { + return Packet4i(vreinterpretq_s32_f64(a)); } -#if EIGEN_ARCH_ARM64 - -//============================================================================== -// pcast/preinterpret, Double -//============================================================================== - -template <> -struct type_casting_traits { - enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; -}; -template <> -EIGEN_STRONG_INLINE Packet2d pcast(const Packet2d& a) { - return a; -} template <> struct type_casting_traits { @@ -1210,6 +1383,10 @@ template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet2d& a, const Packet2d& b) { return vcombine_f32(vcvt_f32_f64(a), vcvt_f32_f64(b)); } +template <> +EIGEN_STRONG_INLINE Packet2f pcast(const Packet2d& a) { + return vcvt_f32_f64(a); +} template <> struct type_casting_traits { @@ -1237,6 +1414,10 @@ template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet2d& a, const Packet2d& b) { return vcombine_s32(vmovn_s64(vcvtq_s64_f64(a)), vmovn_s64(vcvtq_s64_f64(b))); } +template <> +EIGEN_STRONG_INLINE Packet2i pcast(const Packet2d& a) { + return vmovn_s64(vcvtq_s64_f64(a)); +} template <> struct type_casting_traits { @@ -1246,6 +1427,10 @@ template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet2d& a, const Packet2d& b) { return vcombine_u32(vmovn_u64(vcvtq_u64_f64(a)), vmovn_u64(vcvtq_u64_f64(b))); } +template <> +EIGEN_STRONG_INLINE Packet2ui pcast(const Packet2d& a) { + return vmovn_u64(vcvtq_u64_f64(a)); +} template <> struct type_casting_traits { @@ -1258,6 +1443,11 @@ EIGEN_STRONG_INLINE Packet8s pcast(const Packet2d& a, const const int32x4_t cd_s32 = pcast(c, d); return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32)); } +template <> +EIGEN_STRONG_INLINE Packet4s pcast(const Packet2d& a, const Packet2d& b) { + const int32x4_t ab_s32 = pcast(a, b); + return vmovn_s32(ab_s32); +} template <> struct type_casting_traits { @@ -1266,9 +1456,11 @@ struct type_casting_traits { template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet2d& a, const Packet2d& b, const Packet2d& c, const Packet2d& d) { - const uint32x4_t ab_u32 = pcast(a, b); - const uint32x4_t cd_u32 = pcast(c, d); - return vcombine_u16(vmovn_u32(ab_u32), vmovn_u32(cd_u32)); + return preinterpret(pcast(a, b, c, d)); +} +template <> +EIGEN_STRONG_INLINE Packet4us pcast(const Packet2d& a, const Packet2d& b) { + return preinterpret(pcast(a, b)); } template <> @@ -1283,6 +1475,17 @@ EIGEN_STRONG_INLINE Packet16c pcast(const Packet2d& a, cons const int16x8_t efgh_s16 = pcast(e, f, g, h); return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16)); } +template <> +EIGEN_STRONG_INLINE Packet8c pcast(const Packet2d& a, const Packet2d& b, const Packet2d& c, + const Packet2d& d) { + const int16x8_t abcd_s16 = pcast(a, b, c, d); + return vmovn_s16(abcd_s16); +} +template <> +EIGEN_STRONG_INLINE Packet4c pcast(const Packet2d& a, const Packet2d& b) { + const int32x4_t ab_s32 = pcast(a, b); + return pcast(ab_s32); +} template <> struct type_casting_traits { @@ -1296,6 +1499,15 @@ EIGEN_STRONG_INLINE Packet16uc pcast(const Packet2d& a, co const uint16x8_t efgh_u16 = pcast(e, f, g, h); return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16)); } +template <> +EIGEN_STRONG_INLINE Packet8uc pcast(const Packet2d& a, const Packet2d& b, const Packet2d& c, + const Packet2d& d) { + return preinterpret(pcast(a, b, c, d)); +} +template <> +EIGEN_STRONG_INLINE Packet4uc pcast(const Packet2d& a, const Packet2d& b) { + return static_cast(pcast(a, b)); +} template <> struct type_casting_traits { @@ -1306,6 +1518,10 @@ EIGEN_STRONG_INLINE Packet2d pcast(const Packet4f& a) { // Discard second-half of input. return vcvt_f64_f32(vget_low_f32(a)); } +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet2f& a) { + return vcvt_f64_f32(a); +} template <> struct type_casting_traits { @@ -1314,7 +1530,9 @@ struct type_casting_traits { template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet16c& a) { // Discard all but first two values. - return vcvt_f64_f32(pcast(vget_low_s8(a))); + // MSVC defines most intrinsics as macros, so we need to do this in two lines for portability. + Packet2f tmp = pcast(vget_low_s8(a)); + return vcvt_f64_f32(tmp); } template <> @@ -1324,7 +1542,8 @@ struct type_casting_traits { template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet16uc& a) { // Discard all but first two values. - return vcvt_f64_f32(pcast(vget_low_u8(a))); + Packet2f tmp = pcast(vget_low_u8(a)); + return vcvt_f64_f32(tmp); } template <> @@ -1334,7 +1553,8 @@ struct type_casting_traits { template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet8s& a) { // Discard all but first two values. - return vcvt_f64_f32(pcast(vget_low_s16(a))); + Packet2f tmp = pcast(vget_low_s16(a)); + return vcvt_f64_f32(tmp); } template <> @@ -1344,7 +1564,8 @@ struct type_casting_traits { template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet8us& a) { // Discard all but first two values. - return vcvt_f64_f32(pcast(vget_low_u16(a))); + Packet2f tmp = pcast(vget_low_u16(a)); + return vcvt_f64_f32(tmp); } template <> @@ -1356,6 +1577,10 @@ EIGEN_STRONG_INLINE Packet2d pcast(const Packet4i& a) { // Discard second half of input. return vcvtq_f64_s64(vmovl_s32(vget_low_s32(a))); } +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet2i& a) { + return vcvtq_f64_s64(vmovl_s32(a)); +} template <> struct type_casting_traits { @@ -1366,6 +1591,10 @@ EIGEN_STRONG_INLINE Packet2d pcast(const Packet4ui& a) { // Discard second half of input. return vcvtq_f64_u64(vmovl_u32(vget_low_u32(a))); } +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet2ui& a) { + return vcvtq_f64_u64(vmovl_u32(a)); +} template <> struct type_casting_traits { @@ -1385,31 +1614,6 @@ EIGEN_STRONG_INLINE Packet2d pcast(const Packet2ul& a) { return vcvtq_f64_u64(a); } -template <> -EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet2l& a) { - return vreinterpretq_f64_s64(a); -} -template <> -EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet2ul& a) { - return vreinterpretq_f64_u64(a); -} -template <> -EIGEN_STRONG_INLINE Packet2l preinterpret(const Packet2d& a) { - return vreinterpretq_s64_f64(a); -} -template <> -EIGEN_STRONG_INLINE Packet2ul preinterpret(const Packet2d& a) { - return vreinterpretq_u64_f64(a); -} -template <> -EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet4i& a) { - return vreinterpretq_f64_s32(a); -} -template <> -EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet2d& a) { - return vreinterpretq_s32_f64(a); -} - #endif // EIGEN_ARCH_ARM64 } // end namespace internal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/UnaryFunctors.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/UnaryFunctors.h new file mode 100644 index 00000000000..09da91c5618 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/NEON/UnaryFunctors.h @@ -0,0 +1,64 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_NEON_UNARY_FUNCTORS_H +#define EIGEN_NEON_UNARY_FUNCTORS_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC +/** \internal + * \brief Template specialization of the logistic function for Eigen::half. + */ +template <> +struct scalar_logistic_op { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Eigen::half operator()(const Eigen::half& x) const { + // Convert to float and call scalar_logistic_op. + const scalar_logistic_op float_op; + return Eigen::half(float_op(float(x))); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Eigen::half packetOp(const Eigen::half& x) const { + return this->operator()(x); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Packet4hf packetOp(const Packet4hf& x) const { + const scalar_logistic_op float_op; + return vcvt_f16_f32(float_op.packetOp(vcvt_f32_f16(x))); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Packet8hf packetOp(const Packet8hf& x) const { + const scalar_logistic_op float_op; + return vcombine_f16( + vcvt_f16_f32(float_op.packetOp(vcvt_f32_f16(vget_low_f16(x)))), + vcvt_f16_f32(float_op.packetOp(vcvt_high_f32_f16(x)))); + } +}; + +template<> +struct functor_traits> { + enum { + Cost = functor_traits>::Cost, + PacketAccess = functor_traits>::PacketAccess, + }; +}; +#endif // EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_NEON_UNARY_FUNCTORS_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/Complex.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/Complex.h index 215bfd7bb6a..d068806ec53 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/Complex.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/Complex.h @@ -10,6 +10,9 @@ #ifndef EIGEN_COMPLEX_SSE_H #define EIGEN_COMPLEX_SSE_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -33,7 +36,6 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 2, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -135,17 +137,9 @@ template<> EIGEN_STRONG_INLINE void prefetch >(const std::co template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet2cf& a) { - #if EIGEN_GNUC_AT_MOST(4,3) - // Workaround gcc 4.2 ICE - this is not performance wise ideal, but who cares... - // This workaround also fix invalid code generation with gcc 4.3 - EIGEN_ALIGN16 std::complex res[2]; - _mm_store_ps((float*)res, a.v); - return res[0]; - #else - std::complex res; + alignas(alignof(__m64)) std::complex res; _mm_storel_pi((__m64*)&res, a.v); return res; - #endif } template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { return Packet2cf(_mm_castpd_ps(preverse(Packet2d(_mm_castps_pd(a.v))))); } @@ -169,14 +163,9 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) template<> EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { - // TODO optimize it for SSE3 and 4 - Packet2cf res = pmul(a, pconj(b)); - __m128 s = _mm_mul_ps(b.v,b.v); - return Packet2cf(_mm_div_ps(res.v,_mm_add_ps(s,vec4f_swizzle1(s, 1, 0, 3, 2)))); + return pdiv_complex(a, b); } - - //---------- double ---------- struct Packet1cd { @@ -196,7 +185,6 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 0, size = 1, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -294,10 +282,7 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) template<> EIGEN_STRONG_INLINE Packet1cd pdiv(const Packet1cd& a, const Packet1cd& b) { - // TODO optimize it for SSE3 and 4 - Packet1cd res = pmul(a,pconj(b)); - __m128d s = _mm_mul_pd(b.v,b.v); - return Packet1cd(_mm_div_pd(res.v, _mm_add_pd(s,_mm_shuffle_pd(s, s, 0x1)))); + return pdiv_complex(a, b); } EIGEN_STRONG_INLINE Packet1cd pcplxflip/* */(const Packet1cd& x) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/MathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/MathFunctions.h index 8736d0d6b50..0f86bcf8b5e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/MathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/MathFunctions.h @@ -15,158 +15,44 @@ #ifndef EIGEN_MATH_FUNCTIONS_SSE_H #define EIGEN_MATH_FUNCTIONS_SSE_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f plog(const Packet4f& _x) { - return plog_float(_x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet2d plog(const Packet2d& _x) { - return plog_double(_x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f plog2(const Packet4f& _x) { - return plog2_float(_x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet2d plog2(const Packet2d& _x) { - return plog2_double(_x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f plog1p(const Packet4f& _x) { - return generic_plog1p(_x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f pexpm1(const Packet4f& _x) { - return generic_expm1(_x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f pexp(const Packet4f& _x) -{ - return pexp_float(_x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet2d pexp(const Packet2d& x) -{ - return pexp_double(x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f psin(const Packet4f& _x) -{ - return psin_float(_x); -} - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f pcos(const Packet4f& _x) -{ - return pcos_float(_x); -} - -#if EIGEN_FAST_MATH +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet4f) +EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE(Packet2d) -// Functions for sqrt. -// The EIGEN_FAST_MATH version uses the _mm_rsqrt_ps approximation and one step -// of Newton's method, at a cost of 1-2 bits of precision as opposed to the -// exact solution. It does not handle +inf, or denormalized numbers correctly. -// The main advantage of this approach is not just speed, but also the fact that -// it can be inlined and pipelined with other computations, further reducing its -// effective latency. This is similar to Quake3's fast inverse square root. -// For detail see here: http://www.beyond3d.com/content/articles/8/ -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f psqrt(const Packet4f& _x) -{ - Packet4f minus_half_x = pmul(_x, pset1(-0.5f)); - Packet4f denormal_mask = pandnot( - pcmp_lt(_x, pset1((std::numeric_limits::min)())), - pcmp_lt(_x, pzero(_x))); - - // Compute approximate reciprocal sqrt. - Packet4f x = _mm_rsqrt_ps(_x); - // Do a single step of Newton's iteration. - x = pmul(x, pmadd(minus_half_x, pmul(x,x), pset1(1.5f))); - // Flush results for denormals to zero. - return pandnot(pmul(_x,x), denormal_mask); -} - -#else - -template<>EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +// Notice that for newer processors, it is counterproductive to use Newton +// iteration for square root. In particular, Skylake and Zen2 processors +// have approximately doubled throughput of the _mm_sqrt_ps instruction +// compared to their predecessors. +template<>EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f psqrt(const Packet4f& x) { return _mm_sqrt_ps(x); } - -#endif - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d psqrt(const Packet2d& x) { return _mm_sqrt_pd(x); } - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16b psqrt(const Packet16b& x) { return x; } #if EIGEN_FAST_MATH - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f prsqrt(const Packet4f& _x) { - _EIGEN_DECLARE_CONST_Packet4f(one_point_five, 1.5f); - _EIGEN_DECLARE_CONST_Packet4f(minus_half, -0.5f); - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(inf, 0x7f800000u); - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(flt_min, 0x00800000u); - - Packet4f neg_half = pmul(_x, p4f_minus_half); - - // Identity infinite, zero, negative and denormal arguments. - Packet4f lt_min_mask = _mm_cmplt_ps(_x, p4f_flt_min); - Packet4f inf_mask = _mm_cmpeq_ps(_x, p4f_inf); - Packet4f not_normal_finite_mask = _mm_or_ps(lt_min_mask, inf_mask); - - // Compute an approximate result using the rsqrt intrinsic. - Packet4f y_approx = _mm_rsqrt_ps(_x); - - // Do a single step of Newton-Raphson iteration to improve the approximation. - // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n). - // It is essential to evaluate the inner term like this because forming - // y_n^2 may over- or underflow. - Packet4f y_newton = pmul( - y_approx, pmadd(y_approx, pmul(neg_half, y_approx), p4f_one_point_five)); - - // Select the result of the Newton-Raphson step for positive normal arguments. - // For other arguments, choose the output of the intrinsic. This will - // return rsqrt(+inf) = 0, rsqrt(x) = NaN if x < 0, and rsqrt(x) = +inf if - // x is zero or a positive denormalized float (equivalent to flushing positive - // denormalized inputs to zero). - return pselect(not_normal_finite_mask, y_approx, y_newton); -} - -#else - +// Even on Skylake, using Newton iteration is a win for reciprocal square root. template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f prsqrt(const Packet4f& x) { - // Unfortunately we can't use the much faster mm_rsqrt_ps since it only provides an approximation. - return _mm_div_ps(pset1(1.0f), _mm_sqrt_ps(x)); + return generic_rsqrt_newton_step::run(x, _mm_rsqrt_ps(x)); } -#endif - -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet2d prsqrt(const Packet2d& x) { - return _mm_div_pd(pset1(1.0), _mm_sqrt_pd(x)); +#ifdef EIGEN_VECTORIZE_FMA +// Trying to speed up reciprocal using Newton-Raphson is counterproductive +// unless FMA is available. Without FMA pdiv(pset1(Scalar(1),a)) is +// 30% faster. +template<> EIGEN_STRONG_INLINE Packet4f preciprocal(const Packet4f& x) { + return generic_reciprocal_newton_step::run(x, _mm_rcp_ps(x)); } +#endif -// Hyperbolic Tangent function. -template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f -ptanh(const Packet4f& x) { - return internal::generic_fast_tanh_float(x); -} +#endif } // end namespace internal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/PacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/PacketMath.h index db102c73a23..8dd553d8cf8 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/PacketMath.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/PacketMath.h @@ -10,6 +10,10 @@ #ifndef EIGEN_PACKET_MATH_SSE_H #define EIGEN_PACKET_MATH_SSE_H +#include +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -30,7 +34,7 @@ namespace internal { #endif #endif -#if ((defined EIGEN_VECTORIZE_AVX) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_MINGW) && (__GXX_ABI_VERSION < 1004)) || EIGEN_OS_QNX +#if ((defined EIGEN_VECTORIZE_AVX) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_MINGW || EIGEN_COMP_LCC) && (__GXX_ABI_VERSION < 1004)) || EIGEN_OS_QNX // With GCC's default ABI version, a __m128 or __m256 are the same types and therefore we cannot // have overloads for both types without linking error. // One solution is to increase ABI version using -fabi-version=4 (or greater). @@ -45,11 +49,16 @@ typedef __m128d Packet2d; typedef eigen_packet_wrapper<__m128i, 0> Packet4i; typedef eigen_packet_wrapper<__m128i, 1> Packet16b; +typedef eigen_packet_wrapper<__m128i, 4> Packet4ui; template<> struct is_arithmetic<__m128> { enum { value = true }; }; template<> struct is_arithmetic<__m128i> { enum { value = true }; }; template<> struct is_arithmetic<__m128d> { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; +// Note that `Packet4ui` uses the underlying type `__m128i`, which is +// interpreted as a vector of _signed_ `int32`s, which breaks some arithmetic +// operations used in `GenericPacketMath.h`. +template<> struct is_arithmetic { enum { value = false }; }; template<> struct is_arithmetic { enum { value = true }; }; template @@ -64,6 +73,9 @@ struct shuffle_mask{ #define vec4i_swizzle1(v,p,q,r,s) \ Packet4i(_mm_shuffle_epi32( v, (shuffle_mask::mask))) +#define vec4ui_swizzle1(v, p, q, r, s) \ + Packet4ui(vec4i_swizzle1(v,p,q,r,s)) + #define vec2d_swizzle1(v,p,q) \ Packet2d(_mm_castsi128_pd(_mm_shuffle_epi32( _mm_castpd_si128(v), (shuffle_mask<2*p,2*p+1,2*q,2*q+1>::mask)))) @@ -73,6 +85,9 @@ struct shuffle_mask{ #define vec4i_swizzle2(a,b,p,q,r,s) \ Packet4i(_mm_castps_si128( (_mm_shuffle_ps( _mm_castsi128_ps(a), _mm_castsi128_ps(b), (shuffle_mask::mask))))) +#define vec4ui_swizzle2(a,b,p,q,r,s) \ + Packet4i(vec4i_swizzle2(a,b,p,q,r,s)) + EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b) { return Packet4f(_mm_movelh_ps(a,b)); @@ -106,18 +121,19 @@ EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d& a, const Packet2d& b #define vec2d_duplane(a,p) \ vec2d_swizzle2(a,a,(p<<1)|p) -#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ const Packet4f p4f_##NAME = pset1(X) -#define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ const Packet2d p2d_##NAME = pset1(X) -#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ const Packet4f p4f_##NAME = pset1frombits(X) -#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ const Packet4i p4i_##NAME = pset1(X) +#define EIGEN_DECLARE_CONST_Packet4ui(NAME, X) const Packet4ui p4ui_##NAME = pset1(X) // Use the packet_traits defined in AVX/PacketMath.h instead if we're going // to leverage AVX instructions. @@ -130,12 +146,16 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 0, HasCmp = 1, HasDiv = 1, + HasReciprocal = EIGEN_FAST_MATH, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, + HasACos = 1, + HasASin = 1, + HasATan = 1, + HasATanh = 1, HasLog = 1, HasLog1p = 1, HasExpm1 = 1, @@ -152,7 +172,8 @@ struct packet_traits : default_packet_traits { #ifdef EIGEN_VECTORIZE_SSE4_1 HasRound = 1, #endif - HasRint = 1 + HasRint = 1, + HasSign = 0 // The manually vectorized version is slightly slower for SSE. }; }; template <> @@ -163,7 +184,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size=2, - HasHalfPacket = 0, HasCmp = 1, HasDiv = 1, @@ -171,6 +191,7 @@ struct packet_traits : default_packet_traits { HasExp = 1, HasSqrt = 1, HasRsqrt = 1, + HasATan = 1, HasBlend = 1, HasFloor = 1, HasCeil = 1, @@ -180,7 +201,6 @@ struct packet_traits : default_packet_traits { HasRint = 1 }; }; -#endif template<> struct packet_traits : default_packet_traits { typedef Packet4i type; @@ -188,13 +208,34 @@ template<> struct packet_traits : default_packet_traits enum { Vectorizable = 1, AlignedOnScalar = 1, + HasCmp = 1, + HasDiv=1, size=4, HasShift = 1, HasBlend = 1 }; }; +template<> struct packet_traits : default_packet_traits +{ + typedef Packet4ui type; + typedef Packet4ui half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 4, + HasDiv = 0, + HasNegate = 0, + HasSqrt = 0, + HasCmp = 1, + HasMin = 1, + HasMax = 1, + HasShift = 1, + HasBlend = 1 + }; +}; +#endif template<> struct packet_traits : default_packet_traits { typedef Packet16b type; @@ -202,11 +243,11 @@ template<> struct packet_traits : default_packet_traits enum { Vectorizable = 1, AlignedOnScalar = 1, - HasHalfPacket = 0, size=16, HasAdd = 1, HasSub = 1, + HasCmp = 1, // note -- only pcmp_eq is defined HasShift = 0, HasMul = 1, HasNegate = 1, @@ -215,7 +256,8 @@ template<> struct packet_traits : default_packet_traits HasMin = 0, HasMax = 0, HasConj = 0, - HasSqrt = 1 + HasSqrt = 1, + HasSign = 0 // Don't try to vectorize psign = identity. }; }; @@ -233,7 +275,12 @@ template<> struct unpacket_traits { template<> struct unpacket_traits { typedef int type; typedef Packet4i half; - enum {size=4, alignment=Aligned16, vectorizable=false, masked_load_available=false, masked_store_available=false}; + enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; +}; +template<> struct unpacket_traits { + typedef uint32_t type; + typedef Packet4ui half; + enum {size = 4, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false}; }; template<> struct unpacket_traits { typedef bool type; @@ -246,18 +293,10 @@ template<> struct scalar_div_cost { enum { value = 7 }; }; template<> struct scalar_div_cost { enum { value = 8 }; }; #endif -#if EIGEN_COMP_MSVC==1500 -// Workaround MSVC 9 internal compiler error. -// TODO: It has been detected with win64 builds (amd64), so let's check whether it also happens in 32bits+SSE mode -// TODO: let's check whether there does not exist a better fix, like adding a pset0() function. (it crashed on pset1(0)). -template<> EIGEN_STRONG_INLINE Packet4f pset1(const float& from) { return _mm_set_ps(from,from,from,from); } -template<> EIGEN_STRONG_INLINE Packet2d pset1(const double& from) { return _mm_set_pd(from,from); } -template<> EIGEN_STRONG_INLINE Packet4i pset1(const int& from) { return _mm_set_epi32(from,from,from,from); } -#else template<> EIGEN_STRONG_INLINE Packet4f pset1(const float& from) { return _mm_set_ps1(from); } template<> EIGEN_STRONG_INLINE Packet2d pset1(const double& from) { return _mm_set1_pd(from); } template<> EIGEN_STRONG_INLINE Packet4i pset1(const int& from) { return _mm_set1_epi32(from); } -#endif +template<> EIGEN_STRONG_INLINE Packet4ui pset1(const uint32_t& from) { return _mm_set1_epi32(numext::bit_cast(from)); } template<> EIGEN_STRONG_INLINE Packet16b pset1(const bool& from) { return _mm_set1_epi8(static_cast(from)); } template<> EIGEN_STRONG_INLINE Packet4f pset1frombits(unsigned int from) { return _mm_castsi128_ps(pset1(from)); } @@ -265,11 +304,13 @@ template<> EIGEN_STRONG_INLINE Packet2d pset1frombits(uint64_t from) { template<> EIGEN_STRONG_INLINE Packet4f peven_mask(const Packet4f& /*a*/) { return _mm_castsi128_ps(_mm_set_epi32(0, -1, 0, -1)); } template<> EIGEN_STRONG_INLINE Packet4i peven_mask(const Packet4i& /*a*/) { return _mm_set_epi32(0, -1, 0, -1); } +template<> EIGEN_STRONG_INLINE Packet4ui peven_mask(const Packet4ui& /*a*/) { return _mm_set_epi32(0, -1, 0, -1); } template<> EIGEN_STRONG_INLINE Packet2d peven_mask(const Packet2d& /*a*/) { return _mm_castsi128_pd(_mm_set_epi32(0, 0, -1, -1)); } template<> EIGEN_STRONG_INLINE Packet4f pzero(const Packet4f& /*a*/) { return _mm_setzero_ps(); } template<> EIGEN_STRONG_INLINE Packet2d pzero(const Packet2d& /*a*/) { return _mm_setzero_pd(); } template<> EIGEN_STRONG_INLINE Packet4i pzero(const Packet4i& /*a*/) { return _mm_setzero_si128(); } +template<> EIGEN_STRONG_INLINE Packet4ui pzero(const Packet4ui& /*a*/) { return _mm_setzero_si128(); } // GCC generates a shufps instruction for _mm_set1_ps/_mm_load1_ps instead of the more efficient pshufd instruction. // However, using inrinsics for pset1 makes gcc to generate crappy code in some cases (see bug 203) @@ -285,16 +326,23 @@ template<> EIGEN_STRONG_INLINE Packet4f pload1(const float *from) { template<> EIGEN_STRONG_INLINE Packet4f plset(const float& a) { return _mm_add_ps(pset1(a), _mm_set_ps(3,2,1,0)); } template<> EIGEN_STRONG_INLINE Packet2d plset(const double& a) { return _mm_add_pd(pset1(a),_mm_set_pd(1,0)); } template<> EIGEN_STRONG_INLINE Packet4i plset(const int& a) { return _mm_add_epi32(pset1(a),_mm_set_epi32(3,2,1,0)); } +template<> EIGEN_STRONG_INLINE Packet4ui plset(const uint32_t& a) { return _mm_add_epi32(pset1(a), _mm_set_epi32(3, 2, 1, 0)); } template<> EIGEN_STRONG_INLINE Packet4f padd(const Packet4f& a, const Packet4f& b) { return _mm_add_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d padd(const Packet2d& a, const Packet2d& b) { return _mm_add_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i padd(const Packet4i& a, const Packet4i& b) { return _mm_add_epi32(a,b); } +template<> EIGEN_STRONG_INLINE Packet4ui padd(const Packet4ui& a, const Packet4ui& b) { return _mm_add_epi32(a, b); } template<> EIGEN_STRONG_INLINE Packet16b padd(const Packet16b& a, const Packet16b& b) { return _mm_or_si128(a,b); } +template EIGEN_STRONG_INLINE Packet padds(const Packet& a, const Packet& b); +template<> EIGEN_STRONG_INLINE Packet4f padds(const Packet4f& a, const Packet4f& b) { return _mm_add_ss(a,b); } +template<> EIGEN_STRONG_INLINE Packet2d padds(const Packet2d& a, const Packet2d& b) { return _mm_add_sd(a,b); } + template<> EIGEN_STRONG_INLINE Packet4f psub(const Packet4f& a, const Packet4f& b) { return _mm_sub_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d psub(const Packet2d& a, const Packet2d& b) { return _mm_sub_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i psub(const Packet4i& a, const Packet4i& b) { return _mm_sub_epi32(a,b); } +template<> EIGEN_STRONG_INLINE Packet4ui psub(const Packet4ui& a, const Packet4ui& b) { return _mm_sub_epi32(a, b); } template<> EIGEN_STRONG_INLINE Packet16b psub(const Packet16b& a, const Packet16b& b) { return _mm_xor_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b); @@ -311,7 +359,7 @@ template<> EIGEN_STRONG_INLINE Packet4f paddsub(const Packet4f& a, con template<> EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& , const Packet2d& ); template<> EIGEN_STRONG_INLINE Packet2d paddsub(const Packet2d& a, const Packet2d& b) { -#ifdef EIGEN_VECTORIZE_SSE3 +#ifdef EIGEN_VECTORIZE_SSE3 return _mm_addsub_pd(a,b); #else const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x80000000,0x0,0x0)); @@ -360,17 +408,59 @@ template<> EIGEN_STRONG_INLINE Packet4i pmul(const Packet4i& a, const 0,2,1,3); #endif } +template<> EIGEN_STRONG_INLINE Packet4ui pmul(const Packet4ui& a, const Packet4ui& b) +{ +#ifdef EIGEN_VECTORIZE_SSE4_1 + return _mm_mullo_epi32(a,b); +#else + // this version is slightly faster than 4 scalar products + return vec4ui_swizzle1( + vec4ui_swizzle2( + _mm_mul_epu32(a,b), + _mm_mul_epu32(vec4ui_swizzle1(a,1,0,3,2), + vec4ui_swizzle1(b,1,0,3,2)), + 0,2,0,2), + 0,2,1,3); +#endif +} template<> EIGEN_STRONG_INLINE Packet16b pmul(const Packet16b& a, const Packet16b& b) { return _mm_and_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) { return _mm_div_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pdiv(const Packet2d& a, const Packet2d& b) { return _mm_div_pd(a,b); } +template <> +EIGEN_STRONG_INLINE Packet4i pdiv(const Packet4i& a, + const Packet4i& b) { +#ifdef EIGEN_VECTORIZE_AVX + return _mm256_cvttpd_epi32( + _mm256_div_pd(_mm256_cvtepi32_pd(a), _mm256_cvtepi32_pd(b))); +#else + __m128i q_lo = _mm_cvttpd_epi32(_mm_div_pd(_mm_cvtepi32_pd(a), _mm_cvtepi32_pd(b))); + __m128i q_hi = + _mm_cvttpd_epi32(_mm_div_pd(_mm_cvtepi32_pd(vec4i_swizzle1(a, 2, 3, 0, 1)), + _mm_cvtepi32_pd(vec4i_swizzle1(b, 2, 3, 0, 1)))); + return vec4i_swizzle1(_mm_unpacklo_epi32(q_lo, q_hi), 0, 2, 1, 3); +#endif +} + + // for some weird raisons, it has to be overloaded for packet of integers template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd(pmul(a,b), c); } +template<> EIGEN_STRONG_INLINE Packet4ui pmadd(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c) { return padd(pmul(a, b), c); } #ifdef EIGEN_VECTORIZE_FMA template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return _mm_fmadd_ps(a,b,c); } template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return _mm_fmadd_pd(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet4f pmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return _mm_fmsub_ps(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet2d pmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return _mm_fmsub_pd(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet4f pnmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return _mm_fnmadd_ps(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet2d pnmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return _mm_fnmadd_pd(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet4f pnmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return _mm_fnmsub_ps(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet2d pnmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return _mm_fnmsub_pd(a,b,c); } + +template EIGEN_STRONG_INLINE Packet pmadds(const Packet& a, const Packet& b, const Packet& c); +template<> EIGEN_STRONG_INLINE Packet4f pmadds(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return _mm_fmadd_ss(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet2d pmadds(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return _mm_fmadd_sd(a,b,c); } #endif #ifdef EIGEN_VECTORIZE_SSE4_1 @@ -382,6 +472,10 @@ template<> EIGEN_DEVICE_FUNC inline Packet4i pselect(const Packet4i& mask, const return _mm_castps_si128(_mm_blendv_ps(_mm_castsi128_ps(b),_mm_castsi128_ps(a),_mm_castsi128_ps(mask))); } +template<> EIGEN_DEVICE_FUNC inline Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b) { + return _mm_castps_si128(_mm_blendv_ps(_mm_castsi128_ps(b),_mm_castsi128_ps(a),_mm_castsi128_ps(mask))); +} + template<> EIGEN_DEVICE_FUNC inline Packet2d pselect(const Packet2d& mask, const Packet2d& a, const Packet2d& b) { return _mm_blendv_pd(b,a,mask); } template<> EIGEN_DEVICE_FUNC inline Packet16b pselect(const Packet16b& mask, const Packet16b& a, const Packet16b& b) { @@ -412,21 +506,25 @@ ptrue(const Packet2d& a) { template<> EIGEN_STRONG_INLINE Packet4f pand(const Packet4f& a, const Packet4f& b) { return _mm_and_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pand(const Packet2d& a, const Packet2d& b) { return _mm_and_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pand(const Packet4i& a, const Packet4i& b) { return _mm_and_si128(a,b); } +template<> EIGEN_STRONG_INLINE Packet4ui pand(const Packet4ui& a, const Packet4ui& b) { return _mm_and_si128(a, b); } template<> EIGEN_STRONG_INLINE Packet16b pand(const Packet16b& a, const Packet16b& b) { return _mm_and_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet4f por(const Packet4f& a, const Packet4f& b) { return _mm_or_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d por(const Packet2d& a, const Packet2d& b) { return _mm_or_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i por(const Packet4i& a, const Packet4i& b) { return _mm_or_si128(a,b); } +template<> EIGEN_STRONG_INLINE Packet4ui por(const Packet4ui& a, const Packet4ui& b) { return _mm_or_si128(a, b); } template<> EIGEN_STRONG_INLINE Packet16b por(const Packet16b& a, const Packet16b& b) { return _mm_or_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b) { return _mm_xor_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& a, const Packet2d& b) { return _mm_xor_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pxor(const Packet4i& a, const Packet4i& b) { return _mm_xor_si128(a,b); } +template<> EIGEN_STRONG_INLINE Packet4ui pxor(const Packet4ui& a, const Packet4ui& b) { return _mm_xor_si128(a, b); } template<> EIGEN_STRONG_INLINE Packet16b pxor(const Packet16b& a, const Packet16b& b) { return _mm_xor_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pandnot(const Packet4f& a, const Packet4f& b) { return _mm_andnot_ps(b,a); } template<> EIGEN_STRONG_INLINE Packet2d pandnot(const Packet2d& a, const Packet2d& b) { return _mm_andnot_pd(b,a); } template<> EIGEN_STRONG_INLINE Packet4i pandnot(const Packet4i& a, const Packet4i& b) { return _mm_andnot_si128(b,a); } +template<> EIGEN_STRONG_INLINE Packet4ui pandnot(const Packet4ui& a, const Packet4ui& b) { return _mm_andnot_si128(b, a); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) { return _mm_cmple_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) { return _mm_cmplt_ps(a,b); } @@ -440,22 +538,23 @@ template<> EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2 template<> EIGEN_STRONG_INLINE Packet4i pcmp_lt(const Packet4i& a, const Packet4i& b) { return _mm_cmplt_epi32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) { return _mm_cmpeq_epi32(a,b); } +template<> EIGEN_STRONG_INLINE Packet4ui pcmp_eq(const Packet4ui& a, const Packet4ui& b) { return _mm_cmpeq_epi32(a, b); } template<> EIGEN_STRONG_INLINE Packet16b pcmp_eq(const Packet16b& a, const Packet16b& b) { return _mm_cmpeq_epi8(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_le(const Packet4i& a, const Packet4i& b) { return por(pcmp_lt(a,b), pcmp_eq(a,b)); } template<> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { -#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 - // There appears to be a bug in GCC, by which the optimizer may - // flip the argument order in calls to _mm_min_ps, so we have to - // resort to inline ASM here. This is supposed to be fixed in gcc6.3, - // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 - #ifdef EIGEN_VECTORIZE_AVX +#if EIGEN_GNUC_STRICT_LESS_THAN(6,3,0) +// There appears to be a bug in GCC, by which the optimizer may +// flip the argument order in calls to _mm_min_ps, so we have to +// resort to inline ASM here. This is supposed to be fixed in gcc6.3, +// see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 +#ifdef EIGEN_VECTORIZE_AVX Packet4f res; asm("vminps %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); - #else +#else Packet4f res = b; asm("minps %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); - #endif +#endif return res; #else // Arguments are reversed to match NaN propagation behavior of std::min. @@ -463,18 +562,18 @@ template<> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const #endif } template<> EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { -#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 - // There appears to be a bug in GCC, by which the optimizer may - // flip the argument order in calls to _mm_min_pd, so we have to - // resort to inline ASM here. This is supposed to be fixed in gcc6.3, - // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 - #ifdef EIGEN_VECTORIZE_AVX +#if EIGEN_GNUC_STRICT_LESS_THAN(6,3,0) +// There appears to be a bug in GCC, by which the optimizer may +// flip the argument order in calls to _mm_min_pd, so we have to +// resort to inline ASM here. This is supposed to be fixed in gcc6.3, +// see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 +#ifdef EIGEN_VECTORIZE_AVX Packet2d res; asm("vminpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); - #else +#else Packet2d res = b; asm("minpd %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); - #endif +#endif return res; #else // Arguments are reversed to match NaN propagation behavior of std::min. @@ -491,21 +590,30 @@ template<> EIGEN_STRONG_INLINE Packet4i pmin(const Packet4i& a, const return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b)); #endif } +template<> EIGEN_STRONG_INLINE Packet4ui pmin(const Packet4ui& a, const Packet4ui& b) { +#ifdef EIGEN_VECTORIZE_SSE4_1 + return _mm_min_epu32(a, b); +#else + return padd((Packet4ui)pmin((Packet4i)psub(a, pset1(0x80000000UL)), + (Packet4i)psub(b, pset1(0x80000000UL))), + pset1(0x80000000UL)); +#endif +} template<> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { -#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 - // There appears to be a bug in GCC, by which the optimizer may - // flip the argument order in calls to _mm_max_ps, so we have to - // resort to inline ASM here. This is supposed to be fixed in gcc6.3, - // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 - #ifdef EIGEN_VECTORIZE_AVX +#if EIGEN_GNUC_STRICT_LESS_THAN(6,3,0) +// There appears to be a bug in GCC, by which the optimizer may +// flip the argument order in calls to _mm_max_ps, so we have to +// resort to inline ASM here. This is supposed to be fixed in gcc6.3, +// see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 +#ifdef EIGEN_VECTORIZE_AVX Packet4f res; asm("vmaxps %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); - #else +#else Packet4f res = b; asm("maxps %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); - #endif +#endif return res; #else // Arguments are reversed to match NaN propagation behavior of std::max. @@ -513,18 +621,18 @@ template<> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const #endif } template<> EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { -#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 - // There appears to be a bug in GCC, by which the optimizer may - // flip the argument order in calls to _mm_max_pd, so we have to - // resort to inline ASM here. This is supposed to be fixed in gcc6.3, - // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 - #ifdef EIGEN_VECTORIZE_AVX +#if EIGEN_GNUC_STRICT_LESS_THAN(6,3,0) +// There appears to be a bug in GCC, by which the optimizer may +// flip the argument order in calls to _mm_max_pd, so we have to +// resort to inline ASM here. This is supposed to be fixed in gcc6.3, +// see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 +#ifdef EIGEN_VECTORIZE_AVX Packet2d res; asm("vmaxpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); - #else +#else Packet2d res = b; asm("maxpd %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); - #endif +#endif return res; #else // Arguments are reversed to match NaN propagation behavior of std::max. @@ -541,6 +649,32 @@ template<> EIGEN_STRONG_INLINE Packet4i pmax(const Packet4i& a, const return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b)); #endif } +template<> EIGEN_STRONG_INLINE Packet4ui pmax(const Packet4ui& a, const Packet4ui& b) { +#ifdef EIGEN_VECTORIZE_SSE4_1 + return _mm_max_epu32(a, b); +#else + return padd((Packet4ui)pmax((Packet4i)psub(a, pset1(0x80000000UL)), + (Packet4i)psub(b, pset1(0x80000000UL))), + pset1(0x80000000UL)); +#endif +} + +template<> EIGEN_STRONG_INLINE Packet4ui pcmp_lt(const Packet4ui& a, const Packet4ui& b) { +#ifdef EIGEN_VECTORIZE_SSE4_1 + return pxor(pcmp_eq(a, pmax(a, b)), ptrue(a)); +#else + return (Packet4ui)pcmp_lt((Packet4i)psub(a, pset1(0x80000000UL)), + (Packet4i)psub(b, pset1(0x80000000UL))); +#endif +} +template<> EIGEN_STRONG_INLINE Packet4ui pcmp_le(const Packet4ui& a, const Packet4ui& b) { +#ifdef EIGEN_VECTORIZE_SSE4_1 + return pcmp_eq(a, pmin(a, b)); +#else + return (Packet4ui)pcmp_le((Packet4i)psub(a, pset1(0x80000000UL)), + (Packet4i)psub(b, pset1(0x80000000UL))); +#endif +} template EIGEN_STRONG_INLINE Packet pminmax_propagate_numbers(const Packet& a, const Packet& b, Op op) { @@ -598,6 +732,10 @@ template EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packe template EIGEN_STRONG_INLINE Packet4i plogical_shift_right (const Packet4i& a) { return _mm_srli_epi32(a,N); } template EIGEN_STRONG_INLINE Packet4i plogical_shift_left (const Packet4i& a) { return _mm_slli_epi32(a,N); } +template EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(const Packet4ui& a) { return _mm_srli_epi32(a,N); } +template EIGEN_STRONG_INLINE Packet4ui plogical_shift_right (const Packet4ui& a) { return _mm_srli_epi32(a,N); } +template EIGEN_STRONG_INLINE Packet4ui plogical_shift_left (const Packet4ui& a) { return _mm_slli_epi32(a,N); } + template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF)); @@ -610,18 +748,31 @@ template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) } template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { - #ifdef EIGEN_VECTORIZE_SSSE3 +#ifdef EIGEN_VECTORIZE_SSSE3 return _mm_abs_epi32(a); - #else +#else Packet4i aux = _mm_srai_epi32(a,31); return _mm_sub_epi32(_mm_xor_si128(a,aux),aux); - #endif +#endif } +template<> EIGEN_STRONG_INLINE Packet4ui pabs(const Packet4ui& a) { return a; } + +template<> EIGEN_STRONG_INLINE Packet4f psignbit(const Packet4f& a) { return _mm_castsi128_ps(_mm_srai_epi32(_mm_castps_si128(a), 31)); } +template<> EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) +{ + Packet4f tmp = psignbit(_mm_castpd_ps(a)); +#ifdef EIGEN_VECTORIZE_AVX + return _mm_castps_pd(_mm_permute_ps(tmp, (shuffle_mask<1, 1, 3, 3>::mask))); +#else + return _mm_castps_pd(_mm_shuffle_ps(tmp, tmp, (shuffle_mask<1, 1, 3, 3>::mask))); +#endif // EIGEN_VECTORIZE_AVX +} +template<> EIGEN_STRONG_INLINE Packet4ui psignbit(const Packet4ui& a) { return pzero(a); } #ifdef EIGEN_VECTORIZE_SSE4_1 template<> EIGEN_STRONG_INLINE Packet4f pround(const Packet4f& a) { - // Unfortunatly _mm_round_ps doesn't have a rounding mode to implement numext::round. + // Unfortunately _mm_round_ps doesn't have a rounding mode to implement numext::round. const Packet4f mask = pset1frombits(0x80000000u); const Packet4f prev0dot5 = pset1frombits(0x3EFFFFFFu); return _mm_round_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO); @@ -715,21 +866,14 @@ template<> EIGEN_STRONG_INLINE Packet2d pceil(const Packet2d& a) template<> EIGEN_STRONG_INLINE Packet4f pload(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ps(from); } template<> EIGEN_STRONG_INLINE Packet2d pload(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_pd(from); } template<> EIGEN_STRONG_INLINE Packet4i pload(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast(from)); } +template<> EIGEN_STRONG_INLINE Packet4ui pload(const uint32_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet16b pload(const bool* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast(from)); } #if EIGEN_COMP_MSVC template<> EIGEN_STRONG_INLINE Packet4f ploadu(const float* from) { - EIGEN_DEBUG_UNALIGNED_LOAD - #if (EIGEN_COMP_MSVC==1600) - // NOTE Some version of MSVC10 generates bad code when using _mm_loadu_ps - // (i.e., it does not generate an unaligned load!! - __m128 res = _mm_loadl_pi(_mm_set1_ps(0.0f), (const __m64*)(from)); - res = _mm_loadh_pi(res, (const __m64*)(from+2)); - return res; - #else - return _mm_loadu_ps(from); - #endif - } + EIGEN_DEBUG_UNALIGNED_LOAD + return _mm_loadu_ps(from); +} #else // NOTE: with the code below, MSVC's compiler crashes! @@ -750,11 +894,25 @@ template<> EIGEN_STRONG_INLINE Packet4i ploadu(const int* from) EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_si128(reinterpret_cast(from)); } +template<> EIGEN_STRONG_INLINE Packet4ui ploadu(const uint32_t* from) +{ + EIGEN_DEBUG_UNALIGNED_LOAD + return _mm_loadu_si128(reinterpret_cast(from)); +} template<> EIGEN_STRONG_INLINE Packet16b ploadu(const bool* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_si128(reinterpret_cast(from)); } +// Load lower part of packet zero extending. +template EIGEN_STRONG_INLINE Packet ploadl(const typename unpacket_traits::type* from); +template<> EIGEN_STRONG_INLINE Packet4f ploadl(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_castpd_ps(_mm_load_sd(reinterpret_cast(from))); } +template<> EIGEN_STRONG_INLINE Packet2d ploadl(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_load_sd(from); } + +// Load scalar +template EIGEN_STRONG_INLINE Packet ploads(const typename unpacket_traits::type* from); +template<> EIGEN_STRONG_INLINE Packet4f ploads(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_load_ss(from); } +template<> EIGEN_STRONG_INLINE Packet2d ploads(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_load_sd(from); } template<> EIGEN_STRONG_INLINE Packet4f ploaddup(const float* from) { @@ -768,6 +926,12 @@ template<> EIGEN_STRONG_INLINE Packet4i ploaddup(const int* from) tmp = _mm_loadl_epi64(reinterpret_cast(from)); return vec4i_swizzle1(tmp, 0, 0, 1, 1); } +template<> EIGEN_STRONG_INLINE Packet4ui ploaddup(const uint32_t* from) +{ + Packet4ui tmp; + tmp = _mm_loadl_epi64(reinterpret_cast(from)); + return vec4ui_swizzle1(tmp, 0, 0, 1, 1); +} // Loads 8 bools from memory and returns the packet // {b0, b0, b1, b1, b2, b2, b3, b3, b4, b4, b5, b5, b6, b6, b7, b7} @@ -789,13 +953,23 @@ ploadquad(const bool* from) { template<> EIGEN_STRONG_INLINE void pstore(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_ps(to, from); } template<> EIGEN_STRONG_INLINE void pstore(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_pd(to, from); } template<> EIGEN_STRONG_INLINE void pstore(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from); } +template<> EIGEN_STRONG_INLINE void pstore(uint32_t* to, const Packet4ui& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstore(bool* to, const Packet16b& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet2d& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_pd(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_ps(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(int* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from); } +template<> EIGEN_STRONG_INLINE void pstoreu(uint32_t* to, const Packet4ui& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(bool* to, const Packet16b& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from); } +template EIGEN_STRONG_INLINE void pstorel(Scalar* to, const Packet& from); +template<> EIGEN_STRONG_INLINE void pstorel(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storel_pi(reinterpret_cast<__m64*>(to), from); } +template<> EIGEN_STRONG_INLINE void pstorel(double* to, const Packet2d& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storel_pd(to, from); } + +template EIGEN_STRONG_INLINE void pstores(Scalar* to, const Packet& from); +template<> EIGEN_STRONG_INLINE void pstores(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_store_ss(to, from); } +template<> EIGEN_STRONG_INLINE void pstores(double* to, const Packet2d& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_store_sd(to, from); } + template<> EIGEN_DEVICE_FUNC inline Packet4f pgather(const float* from, Index stride) { return _mm_set_ps(from[3*stride], from[2*stride], from[1*stride], from[0*stride]); @@ -808,6 +982,11 @@ template<> EIGEN_DEVICE_FUNC inline Packet4i pgather(const int* f { return _mm_set_epi32(from[3*stride], from[2*stride], from[1*stride], from[0*stride]); } +template<> EIGEN_DEVICE_FUNC inline Packet4ui pgather(const uint32_t* from, Index stride) +{ + return _mm_set_epi32(numext::bit_cast(from[3 * stride]), numext::bit_cast(from[2 * stride]), + numext::bit_cast(from[1 * stride]), numext::bit_cast(from[0 * stride])); +} template<> EIGEN_DEVICE_FUNC inline Packet16b pgather(const bool* from, Index stride) { @@ -836,6 +1015,13 @@ template<> EIGEN_DEVICE_FUNC inline void pscatter(int* to, const to[stride*2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 2)); to[stride*3] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 3)); } +template<> EIGEN_DEVICE_FUNC inline void pscatter(uint32_t* to, const Packet4ui& from, Index stride) +{ + to[stride * 0] = numext::bit_cast(_mm_cvtsi128_si32(from)); + to[stride * 1] = numext::bit_cast(_mm_cvtsi128_si32(_mm_shuffle_epi32(from, 1))); + to[stride * 2] = numext::bit_cast(_mm_cvtsi128_si32(_mm_shuffle_epi32(from, 2))); + to[stride * 3] = numext::bit_cast(_mm_cvtsi128_si32(_mm_shuffle_epi32(from, 3))); +} template<> EIGEN_DEVICE_FUNC inline void pscatter(bool* to, const Packet16b& from, Index stride) { to[4*stride*0] = _mm_cvtsi128_si32(from); @@ -868,6 +1054,7 @@ typedef const char * SsePrefetchPtrType; template<> EIGEN_STRONG_INLINE void prefetch(const float* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE void prefetch(const double* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE void prefetch(const int* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } +template<> EIGEN_STRONG_INLINE void prefetch(const uint32_t* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } #endif #if EIGEN_COMP_MSVC_STRICT && EIGEN_OS_WIN64 @@ -876,21 +1063,25 @@ template<> EIGEN_STRONG_INLINE void prefetch(const int* addr) { _mm_p template<> EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { return a.m128_f32[0]; } template<> EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { return a.m128d_f64[0]; } template<> EIGEN_STRONG_INLINE int pfirst(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; } +template<> EIGEN_STRONG_INLINE uint32_t pfirst(const Packet4ui& a) { uint32_t x = numext::bit_cast(_mm_cvtsi128_si32(a)); return x; } #elif EIGEN_COMP_MSVC_STRICT // The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010 template<> EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { float x = _mm_cvtss_f32(a); return x; } template<> EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { double x = _mm_cvtsd_f64(a); return x; } template<> EIGEN_STRONG_INLINE int pfirst(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; } +template<> EIGEN_STRONG_INLINE uint32_t pfirst(const Packet4ui& a) { uint32_t x = numext::bit_cast(_mm_cvtsi128_si32(a)); return x; } #else template<> EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { return _mm_cvtss_f32(a); } template<> EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { return _mm_cvtsd_f64(a); } template<> EIGEN_STRONG_INLINE int pfirst(const Packet4i& a) { return _mm_cvtsi128_si32(a); } +template<> EIGEN_STRONG_INLINE uint32_t pfirst(const Packet4ui& a) { return numext::bit_cast(_mm_cvtsi128_si32(a)); } #endif template<> EIGEN_STRONG_INLINE bool pfirst(const Packet16b& a) { int x = _mm_cvtsi128_si32(a); return static_cast(x & 1); } template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { return _mm_shuffle_ps(a,a,0x1B); } template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { return _mm_shuffle_pd(a,a,0x1); } template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { return _mm_shuffle_epi32(a,0x1B); } +template<> EIGEN_STRONG_INLINE Packet4ui preverse(const Packet4ui& a) { return _mm_shuffle_epi32(a, 0x1B); } template<> EIGEN_STRONG_INLINE Packet16b preverse(const Packet16b& a) { #ifdef EIGEN_VECTORIZE_SSSE3 __m128i mask = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); @@ -929,10 +1120,10 @@ template<> EIGEN_STRONG_INLINE Packet2d pldexp(const Packet2d& a, cons // Clamp exponent to [-2099, 2099] const Packet2d max_exponent = pset1(2099.0); const Packet2d e = pmin(pmax(exponent, pnegate(max_exponent)), max_exponent); - + // Convert e to integer and swizzle to low-order bits. const Packet4i ei = vec4i_swizzle1(_mm_cvtpd_epi32(e), 0, 3, 1, 3); - + // Split 2^e into four factors and multiply: const Packet4i bias = _mm_set_epi32(0, 1023, 0, 1023); Packet4i b = parithmetic_shift_right<2>(ei); // floor(e/4) @@ -988,24 +1179,24 @@ template<> EIGEN_STRONG_INLINE float predux(const Packet4f& a) { // Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures // (from Nehalem to Haswell) -// #ifdef EIGEN_VECTORIZE_SSE3 -// Packet4f tmp = _mm_add_ps(a, vec4f_swizzle1(a,2,3,2,3)); -// return pfirst(_mm_hadd_ps(tmp, tmp)); -// #else + // #ifdef EIGEN_VECTORIZE_SSE3 + // Packet4f tmp = _mm_add_ps(a, vec4f_swizzle1(a,2,3,2,3)); + // return pfirst(_mm_hadd_ps(tmp, tmp)); + // #else Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a,a)); return pfirst(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); -// #endif + // #endif } template<> EIGEN_STRONG_INLINE double predux(const Packet2d& a) { // Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures // (from Nehalem to Haswell) -// #ifdef EIGEN_VECTORIZE_SSE3 -// return pfirst(_mm_hadd_pd(a, a)); -// #else + // #ifdef EIGEN_VECTORIZE_SSE3 + // return pfirst(_mm_hadd_pd(a, a)); + // #else return pfirst(_mm_add_sd(a, _mm_unpackhi_pd(a,a))); -// #endif + // #endif } #ifdef EIGEN_VECTORIZE_SSSE3 @@ -1014,6 +1205,11 @@ template<> EIGEN_STRONG_INLINE int predux(const Packet4i& a) Packet4i tmp0 = _mm_hadd_epi32(a,a); return pfirst(_mm_hadd_epi32(tmp0,tmp0)); } +template<> EIGEN_STRONG_INLINE uint32_t predux(const Packet4ui& a) +{ + Packet4ui tmp0 = _mm_hadd_epi32(a, a); + return pfirst(_mm_hadd_epi32(tmp0, tmp0)); +} #else template<> EIGEN_STRONG_INLINE int predux(const Packet4i& a) @@ -1021,6 +1217,10 @@ template<> EIGEN_STRONG_INLINE int predux(const Packet4i& a) Packet4i tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a,a)); return pfirst(tmp) + pfirst(_mm_shuffle_epi32(tmp, 1)); } +template<> EIGEN_STRONG_INLINE uint32_t predux(const Packet4ui& a) { + Packet4ui tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a, a)); + return pfirst(tmp) + pfirst(_mm_shuffle_epi32(tmp, 1)); +} #endif template<> EIGEN_STRONG_INLINE bool predux(const Packet16b& a) { @@ -1050,6 +1250,15 @@ template<> EIGEN_STRONG_INLINE int predux_mul(const Packet4i& a) pstore(aux, a); return (aux[0] * aux[1]) * (aux[2] * aux[3]); } +template<> EIGEN_STRONG_INLINE uint32_t predux_mul(const Packet4ui& a) +{ + // after some experiments, it is seems this is the fastest way to implement it + // for GCC (eg., reusing pmul is very slow !) + // TODO try to call _mm_mul_epu32 directly + EIGEN_ALIGN16 uint32_t aux[4]; + pstore(aux, a); + return (aux[0] * aux[1]) * (aux[2] * aux[3]); +} template<> EIGEN_STRONG_INLINE bool predux_mul(const Packet16b& a) { Packet4i tmp = _mm_and_si128(a, _mm_unpackhi_epi64(a,a)); @@ -1082,6 +1291,21 @@ template<> EIGEN_STRONG_INLINE int predux_min(const Packet4i& a) return aux0 EIGEN_STRONG_INLINE uint32_t predux_min(const Packet4ui& a) +{ +#ifdef EIGEN_VECTORIZE_SSE4_1 + Packet4ui tmp = _mm_min_epu32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0,0,3,2))); + return pfirst(_mm_min_epu32(tmp,_mm_shuffle_epi32(tmp, 1))); +#else + // after some experiments, it is seems this is the fastest way to implement it + // for GCC (eg., it does not like using std::min after the pstore !!) + EIGEN_ALIGN16 uint32_t aux[4]; + pstore(aux, a); + uint32_t aux0 = aux[0] EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) @@ -1108,6 +1332,21 @@ template<> EIGEN_STRONG_INLINE int predux_max(const Packet4i& a) return aux0>aux2 ? aux0 : aux2; #endif // EIGEN_VECTORIZE_SSE4_1 } +template<> EIGEN_STRONG_INLINE uint32_t predux_max(const Packet4ui& a) +{ +#ifdef EIGEN_VECTORIZE_SSE4_1 + Packet4ui tmp = _mm_max_epu32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0,0,3,2))); + return pfirst(_mm_max_epu32(tmp,_mm_shuffle_epi32(tmp, 1))); +#else + // after some experiments, it is seems this is the fastest way to implement it + // for GCC (eg., it does not like using std::min after the pstore !!) + EIGEN_ALIGN16 uint32_t aux[4]; + pstore(aux, a); + uint32_t aux0 = aux[0]>aux[1] ? aux[0] : aux[1]; + uint32_t aux2 = aux[2]>aux[3] ? aux[2] : aux[3]; + return aux0>aux2 ? aux0 : aux2; +#endif // EIGEN_VECTORIZE_SSE4_1 +} // not needed yet // template<> EIGEN_STRONG_INLINE bool predux_all(const Packet4f& x) @@ -1120,6 +1359,15 @@ template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) return _mm_movemask_ps(x) != 0x0; } +template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4i& x) +{ + return _mm_movemask_ps(_mm_castsi128_ps(x)) != 0x0; +} +template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4ui& x) +{ + return _mm_movemask_ps(_mm_castsi128_ps(x)) != 0x0; +} + EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { _MM_TRANSPOSE4_PS(kernel.packet[0], kernel.packet[1], kernel.packet[2], kernel.packet[3]); @@ -1144,6 +1392,9 @@ ptranspose(PacketBlock& kernel) { kernel.packet[2] = _mm_unpacklo_epi64(T2, T3); kernel.packet[3] = _mm_unpackhi_epi64(T2, T3); } +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { + ptranspose((PacketBlock&)kernel); +} EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { @@ -1249,6 +1500,10 @@ template<> EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, cons return _mm_or_si128(_mm_andnot_si128(false_mask, thenPacket), _mm_and_si128(false_mask, elsePacket)); #endif } +template<> EIGEN_STRONG_INLINE Packet4ui pblend(const Selector<4>& ifPacket, const Packet4ui& thenPacket, + const Packet4ui& elsePacket) { + return (Packet4ui)pblend(ifPacket, (Packet4i)thenPacket, (Packet4i)elsePacket); +} template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) { const __m128 zero = _mm_setzero_ps(); const __m128 select = _mm_set_ps(ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]); @@ -1278,8 +1533,126 @@ template<> EIGEN_STRONG_INLINE float pmadd(const float& a, const float& b, const template<> EIGEN_STRONG_INLINE double pmadd(const double& a, const double& b, const double& c) { return ::fma(a,b,c); } +template<> EIGEN_STRONG_INLINE float pmsub(const float& a, const float& b, const float& c) { + return ::fmaf(a,b,-c); +} +template<> EIGEN_STRONG_INLINE double pmsub(const double& a, const double& b, const double& c) { + return ::fma(a,b,-c); +} +template<> EIGEN_STRONG_INLINE float pnmadd(const float& a, const float& b, const float& c) { + return ::fmaf(-a,b,c); +} +template<> EIGEN_STRONG_INLINE double pnmadd(const double& a, const double& b, const double& c) { + return ::fma(-a,b,c); +} +template<> EIGEN_STRONG_INLINE float pnmsub(const float& a, const float& b, const float& c) { + return ::fmaf(-a,b,-c); +} +template<> EIGEN_STRONG_INLINE double pnmsub(const double& a, const double& b, const double& c) { + return ::fma(-a,b,-c); +} #endif +#ifdef EIGEN_VECTORIZE_SSE4_1 +// Helpers for half->float and float->half conversions. +// Currently only used by the AVX code. +EIGEN_STRONG_INLINE __m128i half2floatsse(__m128i h) { + __m128i input = _mm_cvtepu16_epi32(h); + + // Direct vectorization of half_to_float, C parts in the comments. + __m128i shifted_exp = _mm_set1_epi32(0x7c00 << 13); + // o.u = (h.x & 0x7fff) << 13; // exponent/mantissa bits + __m128i ou = _mm_slli_epi32(_mm_and_si128(input, _mm_set1_epi32(0x7fff)), 13); + // exp = shifted_exp & o.u; // just the exponent + __m128i exp = _mm_and_si128(ou, shifted_exp); + // o.u += (127 - 15) << 23; + ou = _mm_add_epi32(ou, _mm_set1_epi32((127 - 15) << 23)); + + // Inf/NaN? + __m128i naninf_mask = _mm_cmpeq_epi32(exp, shifted_exp); + // Inf/NaN adjust + __m128i naninf_adj = + _mm_and_si128(_mm_set1_epi32((128 - 16) << 23), naninf_mask); + // extra exp adjust for Inf/NaN + ou = _mm_add_epi32(ou, naninf_adj); + + // Zero/Denormal? + __m128i zeroden_mask = _mm_cmpeq_epi32(exp, _mm_setzero_si128()); + __m128i zeroden_adj = _mm_and_si128(zeroden_mask, _mm_set1_epi32(1 << 23)); + // o.u += 1 << 23; + ou = _mm_add_epi32(ou, zeroden_adj); + // magic.u = 113 << 23 + __m128i magic = _mm_and_si128(zeroden_mask, _mm_set1_epi32(113 << 23)); + // o.f -= magic.f + ou = _mm_castps_si128( + _mm_sub_ps(_mm_castsi128_ps(ou), _mm_castsi128_ps(magic))); + + __m128i sign = + _mm_slli_epi32(_mm_and_si128(input, _mm_set1_epi32(0x8000)), 16); + // o.u |= (h.x & 0x8000) << 16; // sign bit + ou = _mm_or_si128(ou, sign); + // return o.f; + // We are actually returning uint version, to make + // _mm256_insertf128_si256 work. + return ou; +} + +EIGEN_STRONG_INLINE __m128i float2half(__m128 f) { + __m128i o = _mm_setzero_si128(); + + // unsigned int sign_mask = 0x80000000u; + __m128i sign = _mm_set1_epi32(0x80000000u); + // unsigned int sign = f.u & sign_mask; + sign = _mm_and_si128(sign, _mm_castps_si128(f)); + // f.u ^= sign; + f = _mm_xor_ps(f, _mm_castsi128_ps(sign)); + + __m128i fu = _mm_castps_si128(f); + + __m128i f16max = _mm_set1_epi32((127 + 16) << 23); + __m128i f32infty = _mm_set1_epi32(255 << 23); + // if (f.u >= f16max.u) // result is Inf or NaN (all exponent bits set) + // there is no _mm_cmpge_epi32, so use lt and swap operands + __m128i infnan_mask = _mm_cmplt_epi32(f16max, _mm_castps_si128(f)); + __m128i inf_mask = _mm_cmpgt_epi32(_mm_castps_si128(f), f32infty); + __m128i nan_mask = _mm_andnot_si128(inf_mask, infnan_mask); + __m128i inf_value = _mm_and_si128(inf_mask, _mm_set1_epi32(0x7e00)); + __m128i nan_value = _mm_and_si128(nan_mask, _mm_set1_epi32(0x7c00)); + // o.x = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf + __m128i naninf_value = _mm_or_si128(inf_value, nan_value); + + __m128i denorm_magic = _mm_set1_epi32(((127 - 15) + (23 - 10) + 1) << 23); + __m128i subnorm_mask = + _mm_cmplt_epi32(_mm_castps_si128(f), _mm_set1_epi32(113 << 23)); + // f.f += denorm_magic.f; + f = _mm_add_ps(f, _mm_castsi128_ps(denorm_magic)); + // f.u - denorm_magic.u + o = _mm_sub_epi32(_mm_castps_si128(f), denorm_magic); + o = _mm_and_si128(o, subnorm_mask); + // Correct result for inf/nan/zero/subnormal, 0 otherwise + o = _mm_or_si128(o, naninf_value); + + __m128i mask = _mm_or_si128(infnan_mask, subnorm_mask); + o = _mm_and_si128(o, mask); + + // mant_odd = (f.u >> 13) & 1; + __m128i mand_odd = _mm_and_si128(_mm_srli_epi32(fu, 13), _mm_set1_epi32(0x1)); + // f.u += 0xc8000fffU; + fu = _mm_add_epi32(fu, _mm_set1_epi32(0xc8000fffU)); + // f.u += mant_odd; + fu = _mm_add_epi32(fu, mand_odd); + fu = _mm_andnot_si128(mask, fu); + // f.u >> 13 + fu = _mm_srli_epi32(fu, 13); + o = _mm_or_si128(fu, o); + + // o.x |= static_cast(sign >> 16); + o = _mm_or_si128(o, _mm_srli_epi32(sign, 16)); + + // 16 bit values + return _mm_and_si128(o, _mm_set1_epi32(0xffff)); +} +#endif // Packet math for Eigen::half // Disable the following code since it's broken on too many platforms / compilers. @@ -1302,7 +1675,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/TypeCasting.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/TypeCasting.h index d2a0037e014..7e3099bfb12 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/TypeCasting.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SSE/TypeCasting.h @@ -10,52 +10,68 @@ #ifndef EIGEN_TYPE_CASTING_SSE_H #define EIGEN_TYPE_CASTING_SSE_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { #ifndef EIGEN_VECTORIZE_AVX -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; -template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 1 - }; -}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; + +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; + +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +template<> struct type_casting_traits : vectorized_type_casting_traits {}; +#endif template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 2, - TgtCoeffRatio = 1 - }; -}; +EIGEN_STRONG_INLINE Packet16b pcast(const Packet4f& a, + const Packet4f& b, + const Packet4f& c, + const Packet4f& d) { + __m128 zero = pzero(a); + __m128 nonzero_a = _mm_cmpneq_ps(a, zero); + __m128 nonzero_b = _mm_cmpneq_ps(b, zero); + __m128 nonzero_c = _mm_cmpneq_ps(c, zero); + __m128 nonzero_d = _mm_cmpneq_ps(d, zero); + __m128i ab_bytes = _mm_packs_epi32(_mm_castps_si128(nonzero_a), _mm_castps_si128(nonzero_b)); + __m128i cd_bytes = _mm_packs_epi32(_mm_castps_si128(nonzero_c), _mm_castps_si128(nonzero_d)); + __m128i merged = _mm_packs_epi16(ab_bytes, cd_bytes); + return _mm_and_si128(merged, _mm_set1_epi8(1)); +} template <> -struct type_casting_traits { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 2 - }; -}; -#endif +EIGEN_STRONG_INLINE Packet4f pcast(const Packet16b& a) { + const __m128 cst_one = _mm_set_ps1(1.0f); + #ifdef EIGEN_VECTORIZE_SSE4_1 + __m128i a_extended = _mm_cvtepi8_epi32(a); + __m128i abcd = _mm_cmpeq_epi32(a_extended, _mm_setzero_si128()); + #else + __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128()); + __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop); + __m128i abcd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh); + #endif + __m128 result = _mm_andnot_ps(_mm_castsi128_ps(abcd), cst_one); + return result; +} template<> EIGEN_STRONG_INLINE Packet4i pcast(const Packet4f& a) { return _mm_cvttps_epi32(a); } +template<> EIGEN_STRONG_INLINE Packet4i pcast(const Packet2d& a, const Packet2d& b) { + return _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(_mm_cvttpd_epi32(a)), + _mm_castsi128_ps(_mm_cvttpd_epi32(b)), + (1 << 2) | (1 << 6))); +} + template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4i& a) { return _mm_cvtepi32_ps(a); } @@ -64,11 +80,24 @@ template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet2d return _mm_shuffle_ps(_mm_cvtpd_ps(a), _mm_cvtpd_ps(b), (1 << 2) | (1 << 6)); } +template<> EIGEN_STRONG_INLINE Packet2d pcast(const Packet4i& a) { + // Simply discard the second half of the input + return _mm_cvtepi32_pd(a); +} + template<> EIGEN_STRONG_INLINE Packet2d pcast(const Packet4f& a) { // Simply discard the second half of the input return _mm_cvtps_pd(a); } +template<> EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet4f& a) { + return _mm_castps_pd(a); +} + +template<> EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet2d& a) { + return _mm_castpd_ps(a); +} + template<> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4f& a) { return _mm_castps_si128(a); } @@ -85,6 +114,13 @@ template<> EIGEN_STRONG_INLINE Packet4i preinterpret(const Pa return _mm_castpd_si128(a); } +template<> EIGEN_STRONG_INLINE Packet4ui preinterpret(const Packet4i& a) { + return Packet4ui(a); +} + +template<> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4ui& a) { + return Packet4i(a); +} // Disable the following code since it's broken on too many platforms / compilers. //#elif defined(EIGEN_VECTORIZE_SSE) && (!EIGEN_ARCH_x86_64) && (!EIGEN_COMP_MSVC) #if 0 diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/MathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/MathFunctions.h index b139ea2e4e1..b09527545db 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/MathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/MathFunctions.h @@ -10,32 +10,35 @@ #ifndef EIGEN_MATH_FUNCTIONS_SVE_H #define EIGEN_MATH_FUNCTIONS_SVE_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { template <> -EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf pexp(const PacketXf& x) { +EIGEN_STRONG_INLINE PacketXf pexp(const PacketXf& x) { return pexp_float(x); } template <> -EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf plog(const PacketXf& x) { +EIGEN_STRONG_INLINE PacketXf plog(const PacketXf& x) { return plog_float(x); } template <> -EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf psin(const PacketXf& x) { +EIGEN_STRONG_INLINE PacketXf psin(const PacketXf& x) { return psin_float(x); } template <> -EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf pcos(const PacketXf& x) { +EIGEN_STRONG_INLINE PacketXf pcos(const PacketXf& x) { return pcos_float(x); } // Hyperbolic Tangent function. template <> -EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf ptanh(const PacketXf& x) { +EIGEN_STRONG_INLINE PacketXf ptanh(const PacketXf& x) { return internal::generic_fast_tanh_float(x); } } // end namespace internal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/PacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/PacketMath.h index 9060b372ff5..64b710f8287 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/PacketMath.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/PacketMath.h @@ -10,6 +10,9 @@ #ifndef EIGEN_PACKET_MATH_SVE_H #define EIGEN_PACKET_MATH_SVE_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal @@ -40,7 +43,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = sve_packet_size_selector::size, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -149,7 +151,7 @@ EIGEN_STRONG_INLINE PacketXi pmax(const PacketXi& a, const PacketXi& b template <> EIGEN_STRONG_INLINE PacketXi pcmp_le(const PacketXi& a, const PacketXi& b) { - return svdup_n_s32_z(svcmplt_s32(svptrue_b32(), a, b), 0xffffffffu); + return svdup_n_s32_z(svcmple_s32(svptrue_b32(), a, b), 0xffffffffu); } template <> @@ -209,13 +211,13 @@ EIGEN_STRONG_INLINE PacketXi parithmetic_shift_right(PacketXi a) template EIGEN_STRONG_INLINE PacketXi plogical_shift_right(PacketXi a) { - return svreinterpret_s32_u32(svlsr_u32_z(svptrue_b32(), svreinterpret_u32_s32(a), svdup_n_u32_z(svptrue_b32(), N))); + return svreinterpret_s32_u32(svlsr_n_u32_z(svptrue_b32(), svreinterpret_u32_s32(a), N)); } template EIGEN_STRONG_INLINE PacketXi plogical_shift_left(PacketXi a) { - return svlsl_s32_z(svptrue_b32(), a, svdup_n_u32_z(svptrue_b32(), N)); + return svlsl_n_s32_z(svptrue_b32(), a, N); } template <> @@ -375,7 +377,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = sve_packet_size_selector::size, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -523,7 +524,7 @@ EIGEN_STRONG_INLINE PacketXf pmax(const PacketXf& a, template <> EIGEN_STRONG_INLINE PacketXf pcmp_le(const PacketXf& a, const PacketXf& b) { - return svreinterpret_f32_u32(svdup_n_u32_z(svcmplt_f32(svptrue_b32(), a, b), 0xffffffffu)); + return svreinterpret_f32_u32(svdup_n_u32_z(svcmple_f32(svptrue_b32(), a, b), 0xffffffffu)); } template <> diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/TypeCasting.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/TypeCasting.h index 7ba5d9cd119..068ff48b9cb 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/TypeCasting.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SVE/TypeCasting.h @@ -10,6 +10,9 @@ #ifndef EIGEN_TYPE_CASTING_SVE_H #define EIGEN_TYPE_CASTING_SVE_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/InteropHeaders.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/InteropHeaders.h index 10856ff5e5d..27d9a82d378 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/InteropHeaders.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/InteropHeaders.h @@ -21,6 +21,9 @@ #ifndef EIGEN_INTEROP_HEADERS_SYCL_H #define EIGEN_INTEROP_HEADERS_SYCL_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { #if !defined(EIGEN_DONT_VECTORIZE_SYCL) @@ -33,7 +36,6 @@ struct sycl_packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = lengths, - HasHalfPacket = 0, HasDiv = 1, HasLog = 1, HasExp = 1, @@ -84,6 +86,8 @@ struct sycl_packet_traits : default_packet_traits { typedef packet_type half; \ }; +SYCL_PACKET_TRAITS(cl::sycl::cl_half8, 1, Eigen::half, 8) +SYCL_PACKET_TRAITS(cl::sycl::cl_half8, 1, const Eigen::half, 8) SYCL_PACKET_TRAITS(cl::sycl::cl_float4, 1, float, 4) SYCL_PACKET_TRAITS(cl::sycl::cl_float4, 1, const float, 4) SYCL_PACKET_TRAITS(cl::sycl::cl_double2, 0, double, 2) @@ -98,6 +102,7 @@ SYCL_PACKET_TRAITS(cl::sycl::cl_double2, 0, const double, 2) struct is_arithmetic { \ enum { value = true }; \ }; +SYCL_ARITHMETIC(cl::sycl::cl_half8) SYCL_ARITHMETIC(cl::sycl::cl_float4) SYCL_ARITHMETIC(cl::sycl::cl_double2) #undef SYCL_ARITHMETIC @@ -109,6 +114,7 @@ SYCL_ARITHMETIC(cl::sycl::cl_double2) enum { size = lengths, vectorizable = true, alignment = Aligned16 }; \ typedef packet_type half; \ }; +SYCL_UNPACKET_TRAITS(cl::sycl::cl_half8, Eigen::half, 8) SYCL_UNPACKET_TRAITS(cl::sycl::cl_float4, float, 4) SYCL_UNPACKET_TRAITS(cl::sycl::cl_double2, double, 2) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/MathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/MathFunctions.h index 2ab0f2a76b7..a8adc46c573 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/MathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/MathFunctions.h @@ -20,6 +20,9 @@ #ifndef EIGEN_MATH_FUNCTIONS_SYCL_H #define EIGEN_MATH_FUNCTIONS_SYCL_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -35,6 +38,7 @@ namespace internal { return cl::sycl::log(a); \ } +SYCL_PLOG(cl::sycl::cl_half8) SYCL_PLOG(cl::sycl::cl_float4) SYCL_PLOG(cl::sycl::cl_double2) #undef SYCL_PLOG @@ -46,6 +50,7 @@ SYCL_PLOG(cl::sycl::cl_double2) return cl::sycl::log1p(a); \ } +SYCL_PLOG1P(cl::sycl::cl_half8) SYCL_PLOG1P(cl::sycl::cl_float4) SYCL_PLOG1P(cl::sycl::cl_double2) #undef SYCL_PLOG1P @@ -57,6 +62,7 @@ SYCL_PLOG1P(cl::sycl::cl_double2) return cl::sycl::log10(a); \ } +SYCL_PLOG10(cl::sycl::cl_half8) SYCL_PLOG10(cl::sycl::cl_float4) SYCL_PLOG10(cl::sycl::cl_double2) #undef SYCL_PLOG10 @@ -68,6 +74,8 @@ SYCL_PLOG10(cl::sycl::cl_double2) return cl::sycl::exp(a); \ } +SYCL_PEXP(cl::sycl::cl_half8) +SYCL_PEXP(cl::sycl::cl_half) SYCL_PEXP(cl::sycl::cl_float4) SYCL_PEXP(cl::sycl::cl_float) SYCL_PEXP(cl::sycl::cl_double2) @@ -80,6 +88,7 @@ SYCL_PEXP(cl::sycl::cl_double2) return cl::sycl::expm1(a); \ } +SYCL_PEXPM1(cl::sycl::cl_half8) SYCL_PEXPM1(cl::sycl::cl_float4) SYCL_PEXPM1(cl::sycl::cl_double2) #undef SYCL_PEXPM1 @@ -91,6 +100,7 @@ SYCL_PEXPM1(cl::sycl::cl_double2) return cl::sycl::sqrt(a); \ } +SYCL_PSQRT(cl::sycl::cl_half8) SYCL_PSQRT(cl::sycl::cl_float4) SYCL_PSQRT(cl::sycl::cl_double2) #undef SYCL_PSQRT @@ -102,6 +112,7 @@ SYCL_PSQRT(cl::sycl::cl_double2) return cl::sycl::rsqrt(a); \ } +SYCL_PRSQRT(cl::sycl::cl_half8) SYCL_PRSQRT(cl::sycl::cl_float4) SYCL_PRSQRT(cl::sycl::cl_double2) #undef SYCL_PRSQRT @@ -114,6 +125,7 @@ SYCL_PRSQRT(cl::sycl::cl_double2) return cl::sycl::sin(a); \ } +SYCL_PSIN(cl::sycl::cl_half8) SYCL_PSIN(cl::sycl::cl_float4) SYCL_PSIN(cl::sycl::cl_double2) #undef SYCL_PSIN @@ -126,6 +138,7 @@ SYCL_PSIN(cl::sycl::cl_double2) return cl::sycl::cos(a); \ } +SYCL_PCOS(cl::sycl::cl_half8) SYCL_PCOS(cl::sycl::cl_float4) SYCL_PCOS(cl::sycl::cl_double2) #undef SYCL_PCOS @@ -138,6 +151,7 @@ SYCL_PCOS(cl::sycl::cl_double2) return cl::sycl::tan(a); \ } +SYCL_PTAN(cl::sycl::cl_half8) SYCL_PTAN(cl::sycl::cl_float4) SYCL_PTAN(cl::sycl::cl_double2) #undef SYCL_PTAN @@ -150,6 +164,7 @@ SYCL_PTAN(cl::sycl::cl_double2) return cl::sycl::asin(a); \ } +SYCL_PASIN(cl::sycl::cl_half8) SYCL_PASIN(cl::sycl::cl_float4) SYCL_PASIN(cl::sycl::cl_double2) #undef SYCL_PASIN @@ -162,6 +177,7 @@ SYCL_PASIN(cl::sycl::cl_double2) return cl::sycl::acos(a); \ } +SYCL_PACOS(cl::sycl::cl_half8) SYCL_PACOS(cl::sycl::cl_float4) SYCL_PACOS(cl::sycl::cl_double2) #undef SYCL_PACOS @@ -174,6 +190,7 @@ SYCL_PACOS(cl::sycl::cl_double2) return cl::sycl::atan(a); \ } +SYCL_PATAN(cl::sycl::cl_half8) SYCL_PATAN(cl::sycl::cl_float4) SYCL_PATAN(cl::sycl::cl_double2) #undef SYCL_PATAN @@ -186,6 +203,7 @@ SYCL_PATAN(cl::sycl::cl_double2) return cl::sycl::sinh(a); \ } +SYCL_PSINH(cl::sycl::cl_half8) SYCL_PSINH(cl::sycl::cl_float4) SYCL_PSINH(cl::sycl::cl_double2) #undef SYCL_PSINH @@ -198,6 +216,7 @@ SYCL_PSINH(cl::sycl::cl_double2) return cl::sycl::cosh(a); \ } +SYCL_PCOSH(cl::sycl::cl_half8) SYCL_PCOSH(cl::sycl::cl_float4) SYCL_PCOSH(cl::sycl::cl_double2) #undef SYCL_PCOSH @@ -210,6 +229,7 @@ SYCL_PCOSH(cl::sycl::cl_double2) return cl::sycl::tanh(a); \ } +SYCL_PTANH(cl::sycl::cl_half8) SYCL_PTANH(cl::sycl::cl_float4) SYCL_PTANH(cl::sycl::cl_double2) #undef SYCL_PTANH @@ -221,6 +241,7 @@ SYCL_PTANH(cl::sycl::cl_double2) return cl::sycl::ceil(a); \ } +SYCL_PCEIL(cl::sycl::cl_half) SYCL_PCEIL(cl::sycl::cl_float4) SYCL_PCEIL(cl::sycl::cl_double2) #undef SYCL_PCEIL @@ -232,6 +253,7 @@ SYCL_PCEIL(cl::sycl::cl_double2) return cl::sycl::round(a); \ } +SYCL_PROUND(cl::sycl::cl_half8) SYCL_PROUND(cl::sycl::cl_float4) SYCL_PROUND(cl::sycl::cl_double2) #undef SYCL_PROUND @@ -243,6 +265,7 @@ SYCL_PROUND(cl::sycl::cl_double2) return cl::sycl::rint(a); \ } +SYCL_PRINT(cl::sycl::cl_half8) SYCL_PRINT(cl::sycl::cl_float4) SYCL_PRINT(cl::sycl::cl_double2) #undef SYCL_PRINT @@ -254,6 +277,7 @@ SYCL_PRINT(cl::sycl::cl_double2) return cl::sycl::floor(a); \ } +SYCL_FLOOR(cl::sycl::cl_half8) SYCL_FLOOR(cl::sycl::cl_float4) SYCL_FLOOR(cl::sycl::cl_double2) #undef SYCL_FLOOR @@ -265,6 +289,7 @@ SYCL_FLOOR(cl::sycl::cl_double2) return expr; \ } +SYCL_PMIN(cl::sycl::cl_half8, cl::sycl::fmin(a, b)) SYCL_PMIN(cl::sycl::cl_float4, cl::sycl::fmin(a, b)) SYCL_PMIN(cl::sycl::cl_double2, cl::sycl::fmin(a, b)) #undef SYCL_PMIN @@ -276,6 +301,7 @@ SYCL_PMIN(cl::sycl::cl_double2, cl::sycl::fmin(a, b)) return expr; \ } +SYCL_PMAX(cl::sycl::cl_half8, cl::sycl::fmax(a, b)) SYCL_PMAX(cl::sycl::cl_float4, cl::sycl::fmax(a, b)) SYCL_PMAX(cl::sycl::cl_double2, cl::sycl::fmax(a, b)) #undef SYCL_PMAX @@ -289,6 +315,7 @@ SYCL_PMAX(cl::sycl::cl_double2, cl::sycl::fmax(a, b)) cl::sycl::rounding_mode::automatic>()); \ } +SYCL_PLDEXP(cl::sycl::cl_half8) SYCL_PLDEXP(cl::sycl::cl_float4) SYCL_PLDEXP(cl::sycl::cl_double2) #undef SYCL_PLDEXP diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/PacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/PacketMath.h index 87badc07662..4b0b1c6c0e5 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/PacketMath.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/PacketMath.h @@ -21,190 +21,90 @@ #ifndef EIGEN_PACKET_MATH_SYCL_H #define EIGEN_PACKET_MATH_SYCL_H #include -namespace Eigen { - -namespace internal { -#ifdef SYCL_DEVICE_ONLY -#define SYCL_PLOADT_RO(address_space_target) \ - template \ - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type ploadt_ro( \ - typename cl::sycl::multi_ptr< \ - const typename unpacket_traits::type, \ - cl::sycl::access::address_space::address_space_target>::pointer_t \ - from) { \ - typedef typename unpacket_traits::type scalar; \ - typedef cl::sycl::multi_ptr< \ - scalar, cl::sycl::access::address_space::address_space_target> \ - multi_ptr; \ - auto res = packet_type( \ - static_cast::type>(0)); \ - res.load(0, multi_ptr(const_cast(from))); \ - return res; \ - } - -SYCL_PLOADT_RO(global_space) -SYCL_PLOADT_RO(local_space) -#undef SYCL_PLOADT_RO -#endif +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" -template -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type -ploadt_ro(const Eigen::TensorSycl::internal::RangeAccess< - cl::sycl::access::mode::read_write, T>& from) { - return ploadt_ro(from.get_pointer()); -} +namespace Eigen { +namespace internal { #ifdef SYCL_DEVICE_ONLY -#define SYCL_PLOAD(address_space_target, Alignment, AlignedType) \ - template \ - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type pload##AlignedType( \ - typename cl::sycl::multi_ptr< \ - const typename unpacket_traits::type, \ - cl::sycl::access::address_space::address_space_target>::pointer_t \ - from) { \ - return ploadt_ro(from); \ +#define SYCL_PLOAD(packet_type, AlignedType) \ + template <> \ + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type \ + pload##AlignedType( \ + const typename unpacket_traits::type* from) { \ + auto ptr = cl::sycl::address_space_cast(from);\ + packet_type res{}; \ + res.load(0, ptr); \ + return res; \ } -// global space -SYCL_PLOAD(global_space, Unaligned, u) -SYCL_PLOAD(global_space, Aligned, ) -// local space -SYCL_PLOAD(local_space, Unaligned, u) -SYCL_PLOAD(local_space, Aligned, ) - +SYCL_PLOAD(cl::sycl::cl_float4, u) +SYCL_PLOAD(cl::sycl::cl_float4, ) +SYCL_PLOAD(cl::sycl::cl_double2, u) +SYCL_PLOAD(cl::sycl::cl_double2, ) #undef SYCL_PLOAD -#endif - -#define SYCL_PLOAD(Alignment, AlignedType) \ - template \ - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type pload##AlignedType( \ - const Eigen::TensorSycl::internal::RangeAccess< \ - cl::sycl::access::mode::read_write, \ - typename unpacket_traits::type> \ - from) { \ - return ploadt_ro(from); \ - } -SYCL_PLOAD(Unaligned, u) -SYCL_PLOAD(Aligned, ) -#undef SYCL_PLOAD - -#ifdef SYCL_DEVICE_ONLY -/** \internal \returns a packet version of \a *from. - * The pointer \a from must be aligned on a \a Alignment bytes boundary. */ -#define SYCL_PLOADT(address_space_target) \ - template \ - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type ploadt( \ - typename cl::sycl::multi_ptr< \ - const typename unpacket_traits::type, \ - cl::sycl::access::address_space::address_space_target>::pointer_t \ - from) { \ - if (Alignment >= unpacket_traits::alignment) \ - return pload(from); \ - else \ - return ploadu(from); \ - } -// global space -SYCL_PLOADT(global_space) -// local space -SYCL_PLOADT(local_space) -#undef SYCL_PLOADT -#endif - -template -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type -ploadt(const Eigen::TensorSycl::internal::RangeAccess< - cl::sycl::access::mode::read_write, - typename unpacket_traits::type>& from) { - return ploadt(from.get_pointer()); +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_half8 + pload( + const typename unpacket_traits::type* from) { + auto ptr = cl::sycl::address_space_cast< + cl::sycl::access::address_space::generic_space, + cl::sycl::access::decorated::no>( + reinterpret_cast(from)); + cl::sycl::cl_half8 res{}; + res.load(0, ptr); + return res; } -#ifdef SYCL_DEVICE_ONLY -// private_space -#define SYCL_PLOADT_RO_SPECIAL(packet_type, Alignment) \ - template <> \ - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type \ - ploadt_ro( \ - const typename unpacket_traits::type* from) { \ - typedef typename unpacket_traits::type scalar; \ - auto res = packet_type(static_cast(0)); \ - res.template load( \ - 0, const_cast(from)); \ - return res; \ - } - -SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_float4, Aligned) -SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_double2, Aligned) -SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_float4, Unaligned) -SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_double2, Unaligned) - -#define SYCL_PLOAD_SPECIAL(packet_type, alignment_type) \ - template <> \ - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type pload##alignment_type( \ - const typename unpacket_traits::type* from) { \ - typedef typename unpacket_traits::type scalar; \ - auto res = packet_type(static_cast(0)); \ - res.template load( \ - 0, const_cast(from)); \ - return res; \ - } -SYCL_PLOAD_SPECIAL(cl::sycl::cl_float4, ) -SYCL_PLOAD_SPECIAL(cl::sycl::cl_double2, ) -SYCL_PLOAD_SPECIAL(cl::sycl::cl_float4, u) -SYCL_PLOAD_SPECIAL(cl::sycl::cl_double2, u) - -#undef SYCL_PLOAD_SPECIAL +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_half8 +ploadu( + const typename unpacket_traits::type* from) { + auto ptr = cl::sycl::address_space_cast< + cl::sycl::access::address_space::generic_space, + cl::sycl::access::decorated::no>( + reinterpret_cast(from)); + cl::sycl::cl_half8 res{}; + res.load(0, ptr); + return res; +} -#define SYCL_PSTORE(scalar, packet_type, address_space_target, alignment) \ - template <> \ - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstore##alignment( \ - typename cl::sycl::multi_ptr< \ - scalar, \ - cl::sycl::access::address_space::address_space_target>::pointer_t \ - to, \ - const packet_type& from) { \ - typedef cl::sycl::multi_ptr< \ - scalar, cl::sycl::access::address_space::address_space_target> \ - multi_ptr; \ - from.store(0, multi_ptr(to)); \ +#define SYCL_PSTORE(scalar, packet_type, alignment) \ + template <> \ + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstore##alignment( \ + scalar* to, const packet_type& from) { \ + auto ptr = cl::sycl::address_space_cast(to);\ + from.store(0, ptr); \ } -// global space -SYCL_PSTORE(float, cl::sycl::cl_float4, global_space, ) -SYCL_PSTORE(float, cl::sycl::cl_float4, global_space, u) -SYCL_PSTORE(double, cl::sycl::cl_double2, global_space, ) -SYCL_PSTORE(double, cl::sycl::cl_double2, global_space, u) -SYCL_PSTORE(float, cl::sycl::cl_float4, local_space, ) -SYCL_PSTORE(float, cl::sycl::cl_float4, local_space, u) -SYCL_PSTORE(double, cl::sycl::cl_double2, local_space, ) -SYCL_PSTORE(double, cl::sycl::cl_double2, local_space, u) - -SYCL_PSTORE(float, cl::sycl::cl_float4, private_space, ) -SYCL_PSTORE(float, cl::sycl::cl_float4, private_space, u) -SYCL_PSTORE(double, cl::sycl::cl_double2, private_space, ) -SYCL_PSTORE(double, cl::sycl::cl_double2, private_space, u) +SYCL_PSTORE(float, cl::sycl::cl_float4, ) +SYCL_PSTORE(float, cl::sycl::cl_float4, u) +SYCL_PSTORE(double, cl::sycl::cl_double2, ) +SYCL_PSTORE(double, cl::sycl::cl_double2, u) #undef SYCL_PSTORE -#define SYCL_PSTORE_T(address_space_target) \ - template \ - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret( \ - typename cl::sycl::multi_ptr< \ - scalar, \ - cl::sycl::access::address_space::address_space_target>::pointer_t \ - to, \ - const packet_type& from) { \ - if (Alignment) \ - pstore(to, from); \ - else \ - pstoreu(to, from); \ - } - -SYCL_PSTORE_T(global_space) - -SYCL_PSTORE_T(local_space) +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoreu( + Eigen::half* to, const cl::sycl::cl_half8& from) { + auto ptr = cl::sycl::address_space_cast< + cl::sycl::access::address_space::generic_space, + cl::sycl::access::decorated::no>( + reinterpret_cast(to)); + from.store(0, ptr); +} -#undef SYCL_PSTORE_T +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstore( + Eigen::half* to, const cl::sycl::cl_half8& from) { + auto ptr = cl::sycl::address_space_cast< + cl::sycl::access::address_space::generic_space, + cl::sycl::access::decorated::no>( + reinterpret_cast(to)); + from.store(0, ptr); +} #define SYCL_PSET1(packet_type) \ template <> \ @@ -214,6 +114,7 @@ SYCL_PSTORE_T(local_space) } // global space +SYCL_PSET1(cl::sycl::cl_half8) SYCL_PSET1(cl::sycl::cl_float4) SYCL_PSET1(cl::sycl::cl_double2) @@ -230,6 +131,58 @@ struct get_base_packet { get_pgather(sycl_multi_pointer, Index) {} }; +template <> +struct get_base_packet { + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_half8 get_ploaddup( + sycl_multi_pointer from) { + return cl::sycl::cl_half8(static_cast(from[0]), + static_cast(from[0]), + static_cast(from[1]), + static_cast(from[1]), + static_cast(from[2]), + static_cast(from[2]), + static_cast(from[3]), + static_cast(from[3])); + } + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_half8 get_pgather( + sycl_multi_pointer from, Index stride) { + return cl::sycl::cl_half8(static_cast(from[0 * stride]), + static_cast(from[1 * stride]), + static_cast(from[2 * stride]), + static_cast(from[3 * stride]), + static_cast(from[4 * stride]), + static_cast(from[5 * stride]), + static_cast(from[6 * stride]), + static_cast(from[7 * stride])); + } + + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void set_pscatter( + sycl_multi_pointer to, const cl::sycl::cl_half8& from, Index stride) { + auto tmp = stride; + to[0] = Eigen::half(from.s0()); + to[tmp] = Eigen::half(from.s1()); + to[tmp += stride] = Eigen::half(from.s2()); + to[tmp += stride] = Eigen::half(from.s3()); + to[tmp += stride] = Eigen::half(from.s4()); + to[tmp += stride] = Eigen::half(from.s5()); + to[tmp += stride] = Eigen::half(from.s6()); + to[tmp += stride] = Eigen::half(from.s7()); + } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_half8 set_plset( + const cl::sycl::half& a) { + return cl::sycl::cl_half8(static_cast(a), static_cast(a + 1), + static_cast(a + 2), + static_cast(a + 3), + static_cast(a + 4), + static_cast(a + 5), + static_cast(a + 6), + static_cast(a + 7)); + } +}; + template <> struct get_base_packet { template @@ -289,22 +242,6 @@ struct get_base_packet { } }; -#define SYCL_PLOAD_DUP(address_space_target) \ - template \ - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type ploaddup( \ - typename cl::sycl::multi_ptr< \ - const typename unpacket_traits::type, \ - cl::sycl::access::address_space::address_space_target>::pointer_t \ - from) { \ - return get_base_packet::get_ploaddup(from); \ - } - -// global space -SYCL_PLOAD_DUP(global_space) -// local_space -SYCL_PLOAD_DUP(local_space) -#undef SYCL_PLOAD_DUP - #define SYCL_PLOAD_DUP_SPECILIZE(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type ploaddup( \ @@ -312,6 +249,7 @@ SYCL_PLOAD_DUP(local_space) return get_base_packet::get_ploaddup(from); \ } +SYCL_PLOAD_DUP_SPECILIZE(cl::sycl::cl_half8) SYCL_PLOAD_DUP_SPECILIZE(cl::sycl::cl_float4) SYCL_PLOAD_DUP_SPECILIZE(cl::sycl::cl_double2) @@ -323,29 +261,15 @@ SYCL_PLOAD_DUP_SPECILIZE(cl::sycl::cl_double2) const typename unpacket_traits::type& a) { \ return get_base_packet::set_plset(a); \ } - SYCL_PLSET(cl::sycl::cl_float4) SYCL_PLSET(cl::sycl::cl_double2) - #undef SYCL_PLSET -#define SYCL_PGATHER(address_space_target) \ - template \ - EIGEN_DEVICE_FUNC inline packet_type pgather( \ - typename cl::sycl::multi_ptr< \ - const typename unpacket_traits::type, \ - cl::sycl::access::address_space::address_space_target>::pointer_t \ - from, \ - Index stride) { \ - return get_base_packet::get_pgather(from, stride); \ - } - -// global space -SYCL_PGATHER(global_space) -// local space -SYCL_PGATHER(local_space) - -#undef SYCL_PGATHER +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_half8 plset( + const typename unpacket_traits::type& a) { + return get_base_packet::set_plset((const cl::sycl::half &) a); +} #define SYCL_PGATHER_SPECILIZE(scalar, packet_type) \ template <> \ @@ -355,29 +279,11 @@ SYCL_PGATHER(local_space) return get_base_packet::get_pgather(from, stride); \ } +SYCL_PGATHER_SPECILIZE(Eigen::half, cl::sycl::cl_half8) SYCL_PGATHER_SPECILIZE(float, cl::sycl::cl_float4) SYCL_PGATHER_SPECILIZE(double, cl::sycl::cl_double2) - #undef SYCL_PGATHER_SPECILIZE -#define SYCL_PSCATTER(address_space_target) \ - template \ - EIGEN_DEVICE_FUNC inline void pscatter( \ - typename cl::sycl::multi_ptr< \ - typename unpacket_traits::type, \ - cl::sycl::access::address_space::address_space_target>::pointer_t \ - to, \ - const packet_type& from, Index stride) { \ - get_base_packet::set_pscatter(to, from, stride); \ - } - -// global space -SYCL_PSCATTER(global_space) -// local space -SYCL_PSCATTER(local_space) - -#undef SYCL_PSCATTER - #define SYCL_PSCATTER_SPECILIZE(scalar, packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter( \ @@ -386,6 +292,7 @@ SYCL_PSCATTER(local_space) get_base_packet::set_pscatter(to, from, stride); \ } +SYCL_PSCATTER_SPECILIZE(Eigen::half, cl::sycl::cl_half8) SYCL_PSCATTER_SPECILIZE(float, cl::sycl::cl_float4) SYCL_PSCATTER_SPECILIZE(double, cl::sycl::cl_double2) @@ -398,10 +305,16 @@ SYCL_PSCATTER_SPECILIZE(double, cl::sycl::cl_double2) return cl::sycl::mad(a, b, c); \ } +SYCL_PMAD(cl::sycl::cl_half8) SYCL_PMAD(cl::sycl::cl_float4) SYCL_PMAD(cl::sycl::cl_double2) #undef SYCL_PMAD +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Eigen::half pfirst( + const cl::sycl::cl_half8& a) { + return Eigen::half(a.s0()); +} template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float pfirst( const cl::sycl::cl_float4& a) { @@ -413,6 +326,13 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double pfirst( return a.x(); } +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Eigen::half predux( + const cl::sycl::cl_half8& a) { + return Eigen::half(a.s0() + a.s1() + a.s2() + a.s3() + a.s4() + a.s5() + + a.s6() + a.s7()); +} + template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux( const cl::sycl::cl_float4& a) { @@ -425,6 +345,17 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux( return a.x() + a.y(); } +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Eigen::half predux_max( + const cl::sycl::cl_half8& a) { + return Eigen::half(cl::sycl::fmax( + cl::sycl::fmax( + cl::sycl::fmax(a.s0(), a.s1()), + cl::sycl::fmax(a.s2(), a.s3())), + cl::sycl::fmax( + cl::sycl::fmax(a.s4(), a.s5()), + cl::sycl::fmax(a.s6(), a.s7())))); +} template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux_max( const cl::sycl::cl_float4& a) { @@ -437,6 +368,17 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux_max( return cl::sycl::fmax(a.x(), a.y()); } +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Eigen::half predux_min( + const cl::sycl::cl_half8& a) { + return Eigen::half(cl::sycl::fmin( + cl::sycl::fmin( + cl::sycl::fmin(a.s0(), a.s1()), + cl::sycl::fmin(a.s2(), a.s3())), + cl::sycl::fmin( + cl::sycl::fmin(a.s4(), a.s5()), + cl::sycl::fmin(a.s6(), a.s7())))); +} template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux_min( const cl::sycl::cl_float4& a) { @@ -449,6 +391,12 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux_min( return cl::sycl::fmin(a.x(), a.y()); } +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Eigen::half predux_mul( + const cl::sycl::cl_half8& a) { + return Eigen::half(a.s0() * a.s1() * a.s2() * a.s3() * a.s4() * a.s5() * + a.s6() * a.s7()); +} template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux_mul( const cl::sycl::cl_float4& a) { @@ -460,6 +408,14 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux_mul( return a.x() * a.y(); } +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_half8 +pabs(const cl::sycl::cl_half8& a) { + return cl::sycl::cl_half8(cl::sycl::fabs(a.s0()), cl::sycl::fabs(a.s1()), + cl::sycl::fabs(a.s2()), cl::sycl::fabs(a.s3()), + cl::sycl::fabs(a.s4()), cl::sycl::fabs(a.s5()), + cl::sycl::fabs(a.s6()), cl::sycl::fabs(a.s7())); +} template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 pabs(const cl::sycl::cl_float4& a) { @@ -475,25 +431,19 @@ pabs(const cl::sycl::cl_double2& a) { template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet sycl_pcmp_le(const Packet &a, const Packet &b) { - return ((a <= b) - .template convert::type, - cl::sycl::rounding_mode::automatic>()); + return (a <= b).template as(); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet sycl_pcmp_lt(const Packet &a, const Packet &b) { - return ((a < b) - .template convert::type, - cl::sycl::rounding_mode::automatic>()); + return (a < b).template as(); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet sycl_pcmp_eq(const Packet &a, const Packet &b) { - return ((a == b) - .template convert::type, - cl::sycl::rounding_mode::automatic>()); + return (a == b).template as(); } #define SYCL_PCMP(OP, TYPE) \ @@ -503,6 +453,9 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet sycl_pcmp_eq(const Packet &a, return sycl_pcmp_##OP(a, b); \ } +SYCL_PCMP(le, cl::sycl::cl_half8) +SYCL_PCMP(lt, cl::sycl::cl_half8) +SYCL_PCMP(eq, cl::sycl::cl_half8) SYCL_PCMP(le, cl::sycl::cl_float4) SYCL_PCMP(lt, cl::sycl::cl_float4) SYCL_PCMP(eq, cl::sycl::cl_float4) @@ -511,75 +464,120 @@ SYCL_PCMP(lt, cl::sycl::cl_double2) SYCL_PCMP(eq, cl::sycl::cl_double2) #undef SYCL_PCMP -template struct convert_to_integer; - -template <> struct convert_to_integer { - using type = std::int32_t; - using packet_type = cl::sycl::cl_int4; -}; -template <> struct convert_to_integer { - using type = std::int64_t; - using packet_type = cl::sycl::cl_long2; -}; - -template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename convert_to_integer< - typename unpacket_traits::type>::packet_type -vector_as_int(const PacketIn &p) { - return ( - p.template convert::type>::type, - cl::sycl::rounding_mode::automatic>()); -} - -template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packetOut -convert_vector(const PacketIn &p) { - return (p.template convert::type, - cl::sycl::rounding_mode::automatic>()); -} - -#define SYCL_PAND(TYPE) \ - template <> \ - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TYPE pand(const TYPE &a, \ - const TYPE &b) { \ - return convert_vector(vector_as_int(a) & vector_as_int(b)); \ - } -SYCL_PAND(cl::sycl::cl_float4) -SYCL_PAND(cl::sycl::cl_double2) -#undef SYCL_PAND - -#define SYCL_POR(TYPE) \ - template <> \ - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TYPE por(const TYPE &a, \ - const TYPE &b) { \ - return convert_vector(vector_as_int(a) | vector_as_int(b)); \ - } +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void ptranspose( + PacketBlock& kernel) { + cl::sycl::cl_half tmp = kernel.packet[0].s1(); + kernel.packet[0].s1() = kernel.packet[1].s0(); + kernel.packet[1].s0() = tmp; -SYCL_POR(cl::sycl::cl_float4) -SYCL_POR(cl::sycl::cl_double2) -#undef SYCL_POR + tmp = kernel.packet[0].s2(); + kernel.packet[0].s2() = kernel.packet[2].s0(); + kernel.packet[2].s0() = tmp; -#define SYCL_PXOR(TYPE) \ - template <> \ - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TYPE pxor(const TYPE &a, \ - const TYPE &b) { \ - return convert_vector(vector_as_int(a) ^ vector_as_int(b)); \ - } + tmp = kernel.packet[0].s3(); + kernel.packet[0].s3() = kernel.packet[3].s0(); + kernel.packet[3].s0() = tmp; + + tmp = kernel.packet[0].s4(); + kernel.packet[0].s4() = kernel.packet[4].s0(); + kernel.packet[4].s0() = tmp; + + tmp = kernel.packet[0].s5(); + kernel.packet[0].s5() = kernel.packet[5].s0(); + kernel.packet[5].s0() = tmp; + + tmp = kernel.packet[0].s6(); + kernel.packet[0].s6() = kernel.packet[6].s0(); + kernel.packet[6].s0() = tmp; + + tmp = kernel.packet[0].s7(); + kernel.packet[0].s7() = kernel.packet[7].s0(); + kernel.packet[7].s0() = tmp; + + tmp = kernel.packet[1].s2(); + kernel.packet[1].s2() = kernel.packet[2].s1(); + kernel.packet[2].s1() = tmp; + + tmp = kernel.packet[1].s3(); + kernel.packet[1].s3() = kernel.packet[3].s1(); + kernel.packet[3].s1() = tmp; + + tmp = kernel.packet[1].s4(); + kernel.packet[1].s4() = kernel.packet[4].s1(); + kernel.packet[4].s1() = tmp; + + tmp = kernel.packet[1].s5(); + kernel.packet[1].s5() = kernel.packet[5].s1(); + kernel.packet[5].s1() = tmp; + + tmp = kernel.packet[1].s6(); + kernel.packet[1].s6() = kernel.packet[6].s1(); + kernel.packet[6].s1() = tmp; + + tmp = kernel.packet[1].s7(); + kernel.packet[1].s7() = kernel.packet[7].s1(); + kernel.packet[7].s1() = tmp; + + tmp = kernel.packet[2].s3(); + kernel.packet[2].s3() = kernel.packet[3].s2(); + kernel.packet[3].s2() = tmp; + + tmp = kernel.packet[2].s4(); + kernel.packet[2].s4() = kernel.packet[4].s2(); + kernel.packet[4].s2() = tmp; + + tmp = kernel.packet[2].s5(); + kernel.packet[2].s5() = kernel.packet[5].s2(); + kernel.packet[5].s2() = tmp; + + tmp = kernel.packet[2].s6(); + kernel.packet[2].s6() = kernel.packet[6].s2(); + kernel.packet[6].s2() = tmp; + + tmp = kernel.packet[2].s7(); + kernel.packet[2].s7() = kernel.packet[7].s2(); + kernel.packet[7].s2() = tmp; + + tmp = kernel.packet[3].s4(); + kernel.packet[3].s4() = kernel.packet[4].s3(); + kernel.packet[4].s3() = tmp; + + tmp = kernel.packet[3].s5(); + kernel.packet[3].s5() = kernel.packet[5].s3(); + kernel.packet[5].s3() = tmp; + + tmp = kernel.packet[3].s6(); + kernel.packet[3].s6() = kernel.packet[6].s3(); + kernel.packet[6].s3() = tmp; + + tmp = kernel.packet[3].s7(); + kernel.packet[3].s7() = kernel.packet[7].s3(); + kernel.packet[7].s3() = tmp; + + tmp = kernel.packet[4].s5(); + kernel.packet[4].s5() = kernel.packet[5].s4(); + kernel.packet[5].s4() = tmp; + + tmp = kernel.packet[4].s6(); + kernel.packet[4].s6() = kernel.packet[6].s4(); + kernel.packet[6].s4() = tmp; + + tmp = kernel.packet[4].s7(); + kernel.packet[4].s7() = kernel.packet[7].s4(); + kernel.packet[7].s4() = tmp; + + tmp = kernel.packet[5].s6(); + kernel.packet[5].s6() = kernel.packet[6].s5(); + kernel.packet[6].s5() = tmp; -SYCL_PXOR(cl::sycl::cl_float4) -SYCL_PXOR(cl::sycl::cl_double2) -#undef SYCL_PXOR + tmp = kernel.packet[5].s7(); + kernel.packet[5].s7() = kernel.packet[7].s5(); + kernel.packet[7].s5() = tmp; -#define SYCL_PANDNOT(TYPE) \ - template <> \ - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TYPE pandnot(const TYPE &a, \ - const TYPE &b) { \ - return convert_vector(vector_as_int(a) & (~vector_as_int(b))); \ - } -SYCL_PANDNOT(cl::sycl::cl_float4) -SYCL_PANDNOT(cl::sycl::cl_double2) -#undef SYCL_PANDNOT + tmp = kernel.packet[6].s7(); + kernel.packet[6].s7() = kernel.packet[7].s6(); + kernel.packet[7].s6() = tmp; +} EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void ptranspose( PacketBlock& kernel) { @@ -615,6 +613,19 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void ptranspose( kernel.packet[1].x() = tmp; } +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_half8 pblend( + const Selector::size>& ifPacket, + const cl::sycl::cl_half8& thenPacket, + const cl::sycl::cl_half8& elsePacket) { + cl::sycl::cl_short8 condition( + ifPacket.select[0] ? 0 : -1, ifPacket.select[1] ? 0 : -1, + ifPacket.select[2] ? 0 : -1, ifPacket.select[3] ? 0 : -1, + ifPacket.select[4] ? 0 : -1, ifPacket.select[5] ? 0 : -1, + ifPacket.select[6] ? 0 : -1, ifPacket.select[7] ? 0 : -1); + return cl::sycl::select(thenPacket, elsePacket, condition); +} + template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 pblend( const Selector::size>& ifPacket, @@ -637,32 +648,6 @@ inline cl::sycl::cl_double2 pblend( } #endif // SYCL_DEVICE_ONLY -#define SYCL_PSTORE(alignment) \ - template \ - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstore##alignment( \ - const Eigen::TensorSycl::internal::RangeAccess< \ - cl::sycl::access::mode::read_write, \ - typename unpacket_traits::type>& to, \ - const packet_type& from) { \ - pstore##alignment(to.get_pointer(), from); \ - } - -// global space -SYCL_PSTORE() -SYCL_PSTORE(u) - -#undef SYCL_PSTORE - -template -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret( - Eigen::TensorSycl::internal::RangeAccess< - cl::sycl::access::mode::read_write, - typename unpacket_traits::type> - to, - const packet_type& from) { - pstoret(to.get_pointer(), from); -} - } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/SyclMemoryModel.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/SyclMemoryModel.h deleted file mode 100644 index f81e59db583..00000000000 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/SyclMemoryModel.h +++ /dev/null @@ -1,694 +0,0 @@ -/*************************************************************************** - * Copyright (C) 2017 Codeplay Software Limited - * This Source Code Form is subject to the terms of the Mozilla - * Public License v. 2.0. If a copy of the MPL was not distributed - * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - * - * - * SyclMemoryModel.h - * - * Description: - * Interface for SYCL buffers to behave as a non-dereferenceable pointer - * Interface for Placeholder accessor to behave as a pointer on both host - * and device - * - * Authors: - * - * Ruyman Reyes Codeplay Software Ltd. - * Mehdi Goli Codeplay Software Ltd. - * Vanya Yaneva Codeplay Software Ltd. - * - **************************************************************************/ - -#if defined(EIGEN_USE_SYCL) && \ - !defined(EIGEN_CXX11_TENSOR_TENSOR_SYCL_STORAGE_MEMORY_H) -#define EIGEN_CXX11_TENSOR_TENSOR_SYCL_STORAGE_MEMORY_H - -#include -#ifdef EIGEN_EXCEPTIONS -#include -#endif -#include -#include -#include -#include - -namespace Eigen { -namespace TensorSycl { -namespace internal { - -using sycl_acc_target = cl::sycl::access::target; -using sycl_acc_mode = cl::sycl::access::mode; - -/** - * Default values for template arguments - */ -using buffer_data_type_t = uint8_t; -const sycl_acc_target default_acc_target = sycl_acc_target::global_buffer; -const sycl_acc_mode default_acc_mode = sycl_acc_mode::read_write; - -/** - * PointerMapper - * Associates fake pointers with buffers. - * - */ -class PointerMapper { - public: - using base_ptr_t = std::intptr_t; - - /* Structure of a virtual pointer - * - * |================================================| - * | POINTER ADDRESS | - * |================================================| - */ - struct virtual_pointer_t { - /* Type for the pointers - */ - base_ptr_t m_contents; - - /** Conversions from virtual_pointer_t to - * void * should just reinterpret_cast the integer number - */ - operator void *() const { return reinterpret_cast(m_contents); } - - /** - * Convert back to the integer number. - */ - operator base_ptr_t() const { return m_contents; } - - /** - * Add a certain value to the pointer to create a - * new pointer to that offset - */ - virtual_pointer_t operator+(size_t off) { return m_contents + off; } - - /* Numerical order for sorting pointers in containers. */ - bool operator<(virtual_pointer_t rhs) const { - return (static_cast(m_contents) < - static_cast(rhs.m_contents)); - } - - bool operator>(virtual_pointer_t rhs) const { - return (static_cast(m_contents) > - static_cast(rhs.m_contents)); - } - - /** - * Numerical order for sorting pointers in containers - */ - bool operator==(virtual_pointer_t rhs) const { - return (static_cast(m_contents) == - static_cast(rhs.m_contents)); - } - - /** - * Simple forward to the equality overload. - */ - bool operator!=(virtual_pointer_t rhs) const { - return !(this->operator==(rhs)); - } - - /** - * Converts a void * into a virtual pointer structure. - * Note that this will only work if the void * was - * already a virtual_pointer_t, but we have no way of - * checking - */ - virtual_pointer_t(const void *ptr) - : m_contents(reinterpret_cast(ptr)){}; - - /** - * Creates a virtual_pointer_t from the given integer - * number - */ - virtual_pointer_t(base_ptr_t u) : m_contents(u){}; - }; - - /* Definition of a null pointer - */ - const virtual_pointer_t null_virtual_ptr = nullptr; - - /** - * Whether if a pointer is null or not. - * A pointer is nullptr if the value is of null_virtual_ptr - */ - static inline bool is_nullptr(virtual_pointer_t ptr) { - return (static_cast(ptr) == nullptr); - } - - /* basic type for all buffers - */ - using buffer_t = cl::sycl::buffer_mem; - - /** - * Node that stores information about a device allocation. - * Nodes are sorted by size to organise a free list of nodes - * that can be recovered. - */ - struct pMapNode_t { - buffer_t m_buffer; - size_t m_size; - bool m_free; - - pMapNode_t(buffer_t b, size_t size, bool f) - : m_buffer{b}, m_size{size}, m_free{f} { - m_buffer.set_final_data(nullptr); - } - - bool operator<=(const pMapNode_t &rhs) { return (m_size <= rhs.m_size); } - }; - - /** Storage of the pointer / buffer tree - */ - using pointerMap_t = std::map; - - /** - * Obtain the insertion point in the pointer map for - * a pointer of the given size. - * \param requiredSize Size attemted to reclaim - */ - typename pointerMap_t::iterator get_insertion_point(size_t requiredSize) { - typename pointerMap_t::iterator retVal; - bool reuse = false; - if (!m_freeList.empty()) { - // try to re-use an existing block - for (auto freeElem : m_freeList) { - if (freeElem->second.m_size >= requiredSize) { - retVal = freeElem; - reuse = true; - // Element is not going to be free anymore - m_freeList.erase(freeElem); - break; - } - } - } - if (!reuse) { - retVal = std::prev(m_pointerMap.end()); - } - return retVal; - } - - /** - * Returns an iterator to the node that stores the information - * of the given virtual pointer from the given pointer map structure. - * If pointer is not found, throws std::out_of_range. - * If the pointer map structure is empty, throws std::out_of_range - * - * \param pMap the pointerMap_t structure storing all the pointers - * \param virtual_pointer_ptr The virtual pointer to obtain the node of - * \throws std::out:of_range if the pointer is not found or pMap is empty - */ - typename pointerMap_t::iterator get_node(const virtual_pointer_t ptr) { - if (this->count() == 0) { - m_pointerMap.clear(); - EIGEN_THROW_X(std::out_of_range("There are no pointers allocated\n")); - - } - if (is_nullptr(ptr)) { - m_pointerMap.clear(); - EIGEN_THROW_X(std::out_of_range("Cannot access null pointer\n")); - } - // The previous element to the lower bound is the node that - // holds this memory address - auto node = m_pointerMap.lower_bound(ptr); - // If the value of the pointer is not the one of the node - // then we return the previous one - if (node == std::end(m_pointerMap)) { - --node; - } else if (node->first != ptr) { - if (node == std::begin(m_pointerMap)) { - m_pointerMap.clear(); - EIGEN_THROW_X( - std::out_of_range("The pointer is not registered in the map\n")); - - } - --node; - } - - return node; - } - - /* get_buffer. - * Returns a buffer from the map using the pointer address - */ - template - cl::sycl::buffer get_buffer( - const virtual_pointer_t ptr) { - using sycl_buffer_t = cl::sycl::buffer; - - // get_node() returns a `buffer_mem`, so we need to cast it to a `buffer<>`. - // We can do this without the `buffer_mem` being a pointer, as we - // only declare member variables in the base class (`buffer_mem`) and not in - // the child class (`buffer<>). - auto node = get_node(ptr); - eigen_assert(node->first == ptr || node->first < ptr); - eigen_assert(ptr < static_cast(node->second.m_size + - node->first)); - return *(static_cast(&node->second.m_buffer)); - } - - /** - * @brief Returns an accessor to the buffer of the given virtual pointer - * @param accessMode - * @param accessTarget - * @param ptr The virtual pointer - */ - template - cl::sycl::accessor - get_access(const virtual_pointer_t ptr) { - auto buf = get_buffer(ptr); - return buf.template get_access(); - } - - /** - * @brief Returns an accessor to the buffer of the given virtual pointer - * in the given command group scope - * @param accessMode - * @param accessTarget - * @param ptr The virtual pointer - * @param cgh Reference to the command group scope - */ - template - cl::sycl::accessor - get_access(const virtual_pointer_t ptr, cl::sycl::handler &cgh) { - auto buf = get_buffer(ptr); - return buf.template get_access(cgh); - } - - /* - * Returns the offset from the base address of this pointer. - */ - inline std::ptrdiff_t get_offset(const virtual_pointer_t ptr) { - // The previous element to the lower bound is the node that - // holds this memory address - auto node = get_node(ptr); - auto start = node->first; - eigen_assert(start == ptr || start < ptr); - eigen_assert(ptr < start + node->second.m_size); - return (ptr - start); - } - - /* - * Returns the number of elements by which the given pointer is offset from - * the base address. - */ - template - inline size_t get_element_offset(const virtual_pointer_t ptr) { - return get_offset(ptr) / sizeof(buffer_data_type); - } - - /** - * Constructs the PointerMapper structure. - */ - PointerMapper(base_ptr_t baseAddress = 4096) - : m_pointerMap{}, m_freeList{}, m_baseAddress{baseAddress} { - if (m_baseAddress == 0) { - EIGEN_THROW_X(std::invalid_argument("Base address cannot be zero\n")); - } - }; - - /** - * PointerMapper cannot be copied or moved - */ - PointerMapper(const PointerMapper &) = delete; - - /** - * Empty the pointer list - */ - inline void clear() { - m_freeList.clear(); - m_pointerMap.clear(); - } - - /* add_pointer. - * Adds an existing pointer to the map and returns the virtual pointer id. - */ - inline virtual_pointer_t add_pointer(const buffer_t &b) { - return add_pointer_impl(b); - } - - /* add_pointer. - * Adds a pointer to the map and returns the virtual pointer id. - */ - inline virtual_pointer_t add_pointer(buffer_t &&b) { - return add_pointer_impl(b); - } - - /** - * @brief Fuses the given node with the previous nodes in the - * pointer map if they are free - * - * @param node A reference to the free node to be fused - */ - void fuse_forward(typename pointerMap_t::iterator &node) { - while (node != std::prev(m_pointerMap.end())) { - // if following node is free - // remove it and extend the current node with its size - auto fwd_node = std::next(node); - if (!fwd_node->second.m_free) { - break; - } - auto fwd_size = fwd_node->second.m_size; - m_freeList.erase(fwd_node); - m_pointerMap.erase(fwd_node); - - node->second.m_size += fwd_size; - } - } - - /** - * @brief Fuses the given node with the following nodes in the - * pointer map if they are free - * - * @param node A reference to the free node to be fused - */ - void fuse_backward(typename pointerMap_t::iterator &node) { - while (node != m_pointerMap.begin()) { - // if previous node is free, extend it - // with the size of the current one - auto prev_node = std::prev(node); - if (!prev_node->second.m_free) { - break; - } - prev_node->second.m_size += node->second.m_size; - - // remove the current node - m_freeList.erase(node); - m_pointerMap.erase(node); - - // point to the previous node - node = prev_node; - } - } - - /* remove_pointer. - * Removes the given pointer from the map. - * The pointer is allowed to be reused only if ReUse if true. - */ - template - void remove_pointer(const virtual_pointer_t ptr) { - if (is_nullptr(ptr)) { - return; - } - auto node = this->get_node(ptr); - - node->second.m_free = true; - m_freeList.emplace(node); - - // Fuse the node - // with free nodes before and after it - fuse_forward(node); - fuse_backward(node); - - // If after fusing the node is the last one - // simply remove it (since it is free) - if (node == std::prev(m_pointerMap.end())) { - m_freeList.erase(node); - m_pointerMap.erase(node); - } - } - - /* count. - * Return the number of active pointers (i.e, pointers that - * have been malloc but not freed). - */ - size_t count() const { return (m_pointerMap.size() - m_freeList.size()); } - - private: - /* add_pointer_impl. - * Adds a pointer to the map and returns the virtual pointer id. - * BufferT is either a const buffer_t& or a buffer_t&&. - */ - template - virtual_pointer_t add_pointer_impl(BufferT b) { - virtual_pointer_t retVal = nullptr; - size_t bufSize = b.get_count(); - pMapNode_t p{b, bufSize, false}; - // If this is the first pointer: - if (m_pointerMap.empty()) { - virtual_pointer_t initialVal{m_baseAddress}; - m_pointerMap.emplace(initialVal, p); - return initialVal; - } - - auto lastElemIter = get_insertion_point(bufSize); - // We are recovering an existing free node - if (lastElemIter->second.m_free) { - lastElemIter->second.m_buffer = b; - lastElemIter->second.m_free = false; - - // If the recovered node is bigger than the inserted one - // add a new free node with the remaining space - if (lastElemIter->second.m_size > bufSize) { - // create a new node with the remaining space - auto remainingSize = lastElemIter->second.m_size - bufSize; - pMapNode_t p2{b, remainingSize, true}; - - // update size of the current node - lastElemIter->second.m_size = bufSize; - - // add the new free node - auto newFreePtr = lastElemIter->first + bufSize; - auto freeNode = m_pointerMap.emplace(newFreePtr, p2).first; - m_freeList.emplace(freeNode); - } - - retVal = lastElemIter->first; - } else { - size_t lastSize = lastElemIter->second.m_size; - retVal = lastElemIter->first + lastSize; - m_pointerMap.emplace(retVal, p); - } - return retVal; - } - - /** - * Compare two iterators to pointer map entries according to - * the size of the allocation on the device. - */ - struct SortBySize { - bool operator()(typename pointerMap_t::iterator a, - typename pointerMap_t::iterator b) const { - return ((a->first < b->first) && (a->second <= b->second)) || - ((a->first < b->first) && (b->second <= a->second)); - } - }; - - /* Maps the pointer addresses to buffer and size pairs. - */ - pointerMap_t m_pointerMap; - - /* List of free nodes available for re-using - */ - std::set m_freeList; - - /* Base address used when issuing the first virtual pointer, allows users - * to specify alignment. Cannot be zero. */ - std::intptr_t m_baseAddress; -}; - -/* remove_pointer. - * Removes the given pointer from the map. - * The pointer is allowed to be reused only if ReUse if true. - */ -template <> -inline void PointerMapper::remove_pointer(const virtual_pointer_t ptr) { - if (is_nullptr(ptr)) { - return; - } - m_pointerMap.erase(this->get_node(ptr)); -} - -/** - * Malloc-like interface to the pointer-mapper. - * Given a size, creates a byte-typed buffer and returns a - * fake pointer to keep track of it. - * \param size Size in bytes of the desired allocation - * \throw cl::sycl::exception if error while creating the buffer - */ -inline void *SYCLmalloc(size_t size, PointerMapper &pMap) { - if (size == 0) { - return nullptr; - } - // Create a generic buffer of the given size - using buffer_t = cl::sycl::buffer; - auto thePointer = pMap.add_pointer(buffer_t(cl::sycl::range<1>{size})); - // Store the buffer on the global list - return static_cast(thePointer); -} - -/** - * Free-like interface to the pointer mapper. - * Given a fake-pointer created with the virtual-pointer malloc, - * destroys the buffer and remove it from the list. - * If ReUse is false, the pointer is not added to the freeList, - * it should be false only for sub-buffers. - */ -template -inline void SYCLfree(void *ptr, PointerMapper &pMap) { - pMap.template remove_pointer(ptr); -} - -/** - * Clear all the memory allocated by SYCL. - */ -template -inline void SYCLfreeAll(PointerMapper &pMap) { - pMap.clear(); -} - -template -struct RangeAccess { - static const auto global_access = cl::sycl::access::target::global_buffer; - static const auto is_place_holder = cl::sycl::access::placeholder::true_t; - typedef T scalar_t; - typedef scalar_t &ref_t; - typedef typename cl::sycl::global_ptr::pointer_t ptr_t; - - // the accessor type does not necessarily the same as T - typedef cl::sycl::accessor - accessor; - - typedef RangeAccess self_t; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE RangeAccess(accessor access, - size_t offset, - std::intptr_t virtual_ptr) - : access_(access), offset_(offset), virtual_ptr_(virtual_ptr) {} - - RangeAccess(cl::sycl::buffer buff = - cl::sycl::buffer(cl::sycl::range<1>(1))) - : access_{accessor{buff}}, offset_(0), virtual_ptr_(-1) {} - - // This should be only used for null constructor on the host side - RangeAccess(std::nullptr_t) : RangeAccess() {} - // This template parameter must be removed and scalar_t should be replaced - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ptr_t get_pointer() const { - return (access_.get_pointer().get() + offset_); - } - template - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t &operator+=(Index offset) { - offset_ += (offset); - return *this; - } - template - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t operator+(Index offset) const { - return self_t(access_, offset_ + offset, virtual_ptr_); - } - template - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t operator-(Index offset) const { - return self_t(access_, offset_ - offset, virtual_ptr_); - } - template - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t &operator-=(Index offset) { - offset_ -= offset; - return *this; - } - - // THIS IS FOR NULL COMPARISON ONLY - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend bool operator==( - const RangeAccess &lhs, std::nullptr_t) { - return ((lhs.virtual_ptr_ == -1)); - } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend bool operator!=( - const RangeAccess &lhs, std::nullptr_t i) { - return !(lhs == i); - } - - // THIS IS FOR NULL COMPARISON ONLY - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend bool operator==( - std::nullptr_t, const RangeAccess &rhs) { - return ((rhs.virtual_ptr_ == -1)); - } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend bool operator!=( - std::nullptr_t i, const RangeAccess &rhs) { - return !(i == rhs); - } - // Prefix operator (Increment and return value) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t &operator++() { - offset_++; - return (*this); - } - - // Postfix operator (Return value and increment) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t operator++(int i) { - EIGEN_UNUSED_VARIABLE(i); - self_t temp_iterator(*this); - offset_++; - return temp_iterator; - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t get_size() const { - return (access_.get_count() - offset_); - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t get_offset() const { - return offset_; - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void set_offset(std::ptrdiff_t offset) { - offset_ = offset; - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ref_t operator*() const { - return *get_pointer(); - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ref_t operator*() { - return *get_pointer(); - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ptr_t operator->() = delete; - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ref_t operator[](int x) { - return *(get_pointer() + x); - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ref_t operator[](int x) const { - return *(get_pointer() + x); - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_t *get_virtual_pointer() const { - return reinterpret_cast(virtual_ptr_ + - (offset_ * sizeof(scalar_t))); - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit operator bool() const { - return (virtual_ptr_ != -1); - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE operator RangeAccess() { - return RangeAccess(access_, offset_, virtual_ptr_); - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - operator RangeAccess() const { - return RangeAccess(access_, offset_, virtual_ptr_); - } - // binding placeholder accessors to a command group handler for SYCL - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void bind( - cl::sycl::handler &cgh) const { - cgh.require(access_); - } - - private: - accessor access_; - size_t offset_; - std::intptr_t virtual_ptr_; // the location of the buffer in the map -}; - -template -struct RangeAccess : RangeAccess { - typedef RangeAccess Base; - using Base::Base; -}; - -} // namespace internal -} // namespace TensorSycl -} // namespace Eigen - -#endif // EIGEN_CXX11_TENSOR_TENSOR_SYCL_STORAGE_MEMORY_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/TypeCasting.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/TypeCasting.h index 9208ab21d94..9f193c140aa 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/TypeCasting.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/SYCL/TypeCasting.h @@ -21,6 +21,9 @@ #ifndef EIGEN_TYPE_CASTING_SYCL_H #define EIGEN_TYPE_CASTING_SYCL_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -62,7 +65,7 @@ pcast( cl::sycl::rounding_mode::automatic>(); auto b1 = b.template convert(); - return cl::sycl::float4(a1.x(), a1.y(), b1.x(), b1.y()); + return cl::sycl::cl_float4(a1.x(), a1.y(), b1.x(), b1.y()); } template <> diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/Complex.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/Complex.h index 6c67cfe0584..4d74d3d5400 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/Complex.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/Complex.h @@ -8,8 +8,11 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#ifndef EIGEN_COMPLEX32_ALTIVEC_H -#define EIGEN_COMPLEX32_ALTIVEC_H +#ifndef EIGEN_COMPLEX32_ZVECTOR_H +#define EIGEN_COMPLEX32_ZVECTOR_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" namespace Eigen { @@ -51,7 +54,6 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 2, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -76,7 +78,6 @@ template<> struct packet_traits > : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 1, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -179,10 +180,7 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) template<> EIGEN_STRONG_INLINE Packet1cd pdiv(const Packet1cd& a, const Packet1cd& b) { - // TODO optimize it for AltiVec - Packet1cd res = pmul(a,pconj(b)); - Packet2d s = vec_madd(b.v, b.v, p2d_ZERO_); - return Packet1cd(pdiv(res.v, s + vec_perm(s, s, p16uc_REVERSE64))); + return pdiv_complex(a, b); } EIGEN_STRONG_INLINE Packet1cd pcplxflip/**/(const Packet1cd& x) @@ -318,11 +316,7 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) template<> EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { - // TODO optimize it for AltiVec - Packet2cf res; - res.cd[0] = pdiv(a.cd[0], b.cd[0]); - res.cd[1] = pdiv(a.cd[1], b.cd[1]); - return res; + return pdiv_complex(a, b); } EIGEN_STRONG_INLINE Packet2cf pcplxflip/**/(const Packet2cf& x) @@ -404,10 +398,7 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) template<> EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { - // TODO optimize it for AltiVec - Packet2cf res = pmul(a, pconj(b)); - Packet4f s = pmul(b.v, b.v); - return Packet2cf(pdiv(res.v, padd(s, vec_perm(s, s, p16uc_COMPLEX32_REV)))); + return pdiv_complex(a, b); } template<> EIGEN_STRONG_INLINE Packet2cf pcplxflip(const Packet2cf& x) @@ -433,4 +424,4 @@ template<> EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, con } // end namespace Eigen -#endif // EIGEN_COMPLEX32_ALTIVEC_H +#endif // EIGEN_COMPLEX32_ZVECTOR_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/MathFunctions.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/MathFunctions.h index 1635e128c81..1b43878f630 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/MathFunctions.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/MathFunctions.h @@ -13,79 +13,82 @@ * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ */ -#ifndef EIGEN_MATH_FUNCTIONS_ALTIVEC_H -#define EIGEN_MATH_FUNCTIONS_ALTIVEC_H +#ifndef EIGEN_MATH_FUNCTIONS_ZVECTOR_H +#define EIGEN_MATH_FUNCTIONS_ZVECTOR_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" namespace Eigen { namespace internal { #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) -static _EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f); -static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); -static _EIGEN_DECLARE_CONST_Packet4i(0x7f, 0x7f); -static _EIGEN_DECLARE_CONST_Packet4i(23, 23); +static EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f); +static EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); +static EIGEN_DECLARE_CONST_Packet4i(0x7f, 0x7f); +static EIGEN_DECLARE_CONST_Packet4i(23, 23); -static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(inv_mant_mask, ~0x7f800000); +static EIGEN_DECLARE_CONST_Packet4f_FROM_INT(inv_mant_mask, ~0x7f800000); /* the smallest non denormalized float number */ -static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(min_norm_pos, 0x00800000); -static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_inf, 0xff800000); // -1.f/0.f -static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_nan, 0xffffffff); +static EIGEN_DECLARE_CONST_Packet4f_FROM_INT(min_norm_pos, 0x00800000); +static EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_inf, 0xff800000); // -1.f/0.f +static EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_nan, 0xffffffff); /* natural logarithm computed for 4 simultaneous float return NaN for x <= 0 */ -static _EIGEN_DECLARE_CONST_Packet4f(cephes_SQRTHF, 0.707106781186547524f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p0, 7.0376836292E-2f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p1, - 1.1514610310E-1f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p2, 1.1676998740E-1f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p3, - 1.2420140846E-1f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p4, + 1.4249322787E-1f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p5, - 1.6668057665E-1f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p6, + 2.0000714765E-1f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p7, - 2.4999993993E-1f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p8, + 3.3333331174E-1f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q1, -2.12194440e-4f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q2, 0.693359375f); - -static _EIGEN_DECLARE_CONST_Packet4f(exp_hi, 88.3762626647950f); -static _EIGEN_DECLARE_CONST_Packet4f(exp_lo, -88.3762626647949f); - -static _EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f); - -static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500E-4f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507E-3f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073E-3f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894E-2f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459E-1f); -static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201E-1f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_SQRTHF, 0.707106781186547524f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p0, 7.0376836292E-2f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p1, - 1.1514610310E-1f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p2, 1.1676998740E-1f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p3, - 1.2420140846E-1f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p4, + 1.4249322787E-1f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p5, - 1.6668057665E-1f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p6, + 2.0000714765E-1f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p7, - 2.4999993993E-1f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p8, + 3.3333331174E-1f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_log_q1, -2.12194440e-4f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_log_q2, 0.693359375f); + +static EIGEN_DECLARE_CONST_Packet4f(exp_hi, 88.3762626647950f); +static EIGEN_DECLARE_CONST_Packet4f(exp_lo, -88.3762626647949f); + +static EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f); + +static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500E-4f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507E-3f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073E-3f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894E-2f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459E-1f); +static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201E-1f); #endif -static _EIGEN_DECLARE_CONST_Packet2d(1 , 1.0); -static _EIGEN_DECLARE_CONST_Packet2d(2 , 2.0); -static _EIGEN_DECLARE_CONST_Packet2d(half, 0.5); +static EIGEN_DECLARE_CONST_Packet2d(1 , 1.0); +static EIGEN_DECLARE_CONST_Packet2d(2 , 2.0); +static EIGEN_DECLARE_CONST_Packet2d(half, 0.5); -static _EIGEN_DECLARE_CONST_Packet2d(exp_hi, 709.437); -static _EIGEN_DECLARE_CONST_Packet2d(exp_lo, -709.436139303); +static EIGEN_DECLARE_CONST_Packet2d(exp_hi, 709.437); +static EIGEN_DECLARE_CONST_Packet2d(exp_lo, -709.436139303); -static _EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); +static EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); -static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); -static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); -static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); +static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); +static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); +static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); -static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); -static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); -static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); -static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); +static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); +static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); +static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); +static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); -static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); -static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); +static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); +static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d pexp(const Packet2d& _x) { Packet2d x = _x; @@ -136,7 +139,7 @@ Packet2d pexp(const Packet2d& _x) isnumber_mask); } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f pexp(const Packet4f& _x) { #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) @@ -183,13 +186,13 @@ Packet4f pexp(const Packet4f& _x) #endif } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d psqrt(const Packet2d& x) { return vec_sqrt(x); } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f psqrt(const Packet4f& x) { Packet4f res; @@ -202,12 +205,12 @@ Packet4f psqrt(const Packet4f& x) return res; } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d prsqrt(const Packet2d& x) { return pset1(1.0) / psqrt(x); } -template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f prsqrt(const Packet4f& x) { Packet4f res; #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) @@ -221,7 +224,7 @@ Packet4f prsqrt(const Packet4f& x) { // Hyperbolic Tangent function. template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f ptanh(const Packet4f& x) { return internal::generic_fast_tanh_float(x); } @@ -230,4 +233,4 @@ ptanh(const Packet4f& x) { } // end namespace Eigen -#endif // EIGEN_MATH_FUNCTIONS_ALTIVEC_H +#endif // EIGEN_MATH_FUNCTIONS_ZVECTOR_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/PacketMath.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/PacketMath.h index a7b59c80edf..07de778fbfa 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/PacketMath.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/arch/ZVector/PacketMath.h @@ -10,6 +10,9 @@ #ifndef EIGEN_PACKET_MATH_ZVECTOR_H #define EIGEN_PACKET_MATH_ZVECTOR_H +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -64,48 +67,48 @@ typedef union { // We don't want to write the same code all the time, but we need to reuse the constants // and it doesn't really work to declare them global, so we define macros instead -#define _EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \ +#define EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \ Packet4i p4i_##NAME = reinterpret_cast(vec_splat_s32(X)) -#define _EIGEN_DECLARE_CONST_FAST_Packet2d(NAME,X) \ +#define EIGEN_DECLARE_CONST_FAST_Packet2d(NAME,X) \ Packet2d p2d_##NAME = reinterpret_cast(vec_splat_s64(X)) -#define _EIGEN_DECLARE_CONST_FAST_Packet2l(NAME,X) \ +#define EIGEN_DECLARE_CONST_FAST_Packet2l(NAME,X) \ Packet2l p2l_##NAME = reinterpret_cast(vec_splat_s64(X)) -#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ Packet4i p4i_##NAME = pset1(X) -#define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ Packet2d p2d_##NAME = pset1(X) -#define _EIGEN_DECLARE_CONST_Packet2l(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet2l(NAME,X) \ Packet2l p2l_##NAME = pset1(X) // These constants are endian-agnostic -static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,} -static _EIGEN_DECLARE_CONST_FAST_Packet4i(ONE, 1); //{ 1, 1, 1, 1} +static EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,} +static EIGEN_DECLARE_CONST_FAST_Packet4i(ONE, 1); //{ 1, 1, 1, 1} -static _EIGEN_DECLARE_CONST_FAST_Packet2d(ZERO, 0); -static _EIGEN_DECLARE_CONST_FAST_Packet2l(ZERO, 0); -static _EIGEN_DECLARE_CONST_FAST_Packet2l(ONE, 1); +static EIGEN_DECLARE_CONST_FAST_Packet2d(ZERO, 0); +static EIGEN_DECLARE_CONST_FAST_Packet2l(ZERO, 0); +static EIGEN_DECLARE_CONST_FAST_Packet2l(ONE, 1); static Packet2d p2d_ONE = { 1.0, 1.0 }; static Packet2d p2d_ZERO_ = { numext::bit_cast(0x8000000000000000ull), numext::bit_cast(0x8000000000000000ull) }; #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) -#define _EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \ +#define EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \ Packet4f p4f_##NAME = reinterpret_cast(vec_splat_s32(X)) -#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ Packet4f p4f_##NAME = pset1(X) -#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ +#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ const Packet4f p4f_##NAME = reinterpret_cast(pset1(X)) -static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0} -static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1); //{ -1, -1, -1, -1} +static EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0} +static EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1); //{ -1, -1, -1, -1} static Packet4f p4f_MZERO = { 0x80000000, 0x80000000, 0x80000000, 0x80000000}; #endif @@ -117,9 +120,9 @@ static Packet16uc p16uc_PSET64_HI = { 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 }; static Packet16uc p16uc_DUPLICATE32_HI = { 0,1,2,3, 0,1,2,3, 4,5,6,7, 4,5,6,7 }; // Mask alignment -#define _EIGEN_MASK_ALIGNMENT 0xfffffffffffffff0 +#define EIGEN_MASK_ALIGNMENT 0xfffffffffffffff0 -#define _EIGEN_ALIGNED_PTR(x) ((std::ptrdiff_t)(x) & _EIGEN_MASK_ALIGNMENT) +#define EIGEN_ALIGNED_PTR(x) ((std::ptrdiff_t)(x) & EIGEN_MASK_ALIGNMENT) // Handle endianness properly while loading constants // Define global static constants: @@ -158,7 +161,6 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -176,7 +178,6 @@ struct packet_traits : default_packet_traits { Vectorizable = 1, AlignedOnScalar = 1, size = 4, - HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -209,7 +210,6 @@ template<> struct packet_traits : default_packet_traits Vectorizable = 1, AlignedOnScalar = 1, size=2, - HasHalfPacket = 1, HasAdd = 1, HasSub = 1, diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/AssignmentFunctors.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/AssignmentFunctors.h index bf64ef4ede6..8688fbd2c0b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/AssignmentFunctors.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/AssignmentFunctors.h @@ -10,6 +10,9 @@ #ifndef EIGEN_ASSIGNMENT_FUNCTORS_H #define EIGEN_ASSIGNMENT_FUNCTORS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -20,9 +23,8 @@ namespace internal { */ template struct assign_op { - EIGEN_EMPTY_STRUCT_CTOR(assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a = b; } - + template EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { internal::pstoret(a,b); } @@ -45,9 +47,8 @@ struct functor_traits > { */ template struct add_assign_op { - EIGEN_EMPTY_STRUCT_CTOR(add_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a += b; } - + template EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { internal::pstoret(a,internal::padd(internal::ploadt(a),b)); } @@ -66,9 +67,8 @@ struct functor_traits > { */ template struct sub_assign_op { - EIGEN_EMPTY_STRUCT_CTOR(sub_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a -= b; } - + template EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { internal::pstoret(a,internal::psub(internal::ploadt(a),b)); } @@ -88,9 +88,8 @@ struct functor_traits > { template struct mul_assign_op { - EIGEN_EMPTY_STRUCT_CTOR(mul_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a *= b; } - + template EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { internal::pstoret(a,internal::pmul(internal::ploadt(a),b)); } @@ -109,9 +108,8 @@ struct functor_traits > { */ template struct div_assign_op { - EIGEN_EMPTY_STRUCT_CTOR(div_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a /= b; } - + template EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { internal::pstoret(a,internal::pdiv(internal::ploadt(a),b)); } @@ -141,7 +139,6 @@ struct functor_traits > { */ template struct swap_assign_op { - EIGEN_EMPTY_STRUCT_CTOR(swap_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Scalar& a, const Scalar& b) const { #ifdef EIGEN_GPUCC @@ -158,7 +155,7 @@ struct functor_traits > { enum { Cost = 3 * NumTraits::ReadCost, PacketAccess = - #if defined(EIGEN_VECTORIZE_AVX) && EIGEN_COMP_CLANG && (EIGEN_COMP_CLANG<800 || defined(__apple_build_version__)) + #if defined(EIGEN_VECTORIZE_AVX) && (EIGEN_CLANG_STRICT_LESS_THAN(8,0,0) || EIGEN_COMP_CLANGAPPLE) // This is a partial workaround for a bug in clang generating bad code // when mixing 256/512 bits loads and 128 bits moves. // See http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1684 diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/BinaryFunctors.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/BinaryFunctors.h index 63f09ab9317..ce8cf1ace86 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/BinaryFunctors.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/BinaryFunctors.h @@ -10,6 +10,9 @@ #ifndef EIGEN_BINARY_FUNCTORS_H #define EIGEN_BINARY_FUNCTORS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -32,9 +35,7 @@ template struct scalar_sum_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; -#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN - EIGEN_EMPTY_STRUCT_CTOR(scalar_sum_op) -#else +#ifdef EIGEN_SCALAR_BINARY_OP_PLUGIN scalar_sum_op() { EIGEN_SCALAR_BINARY_OP_PLUGIN } @@ -70,9 +71,7 @@ template struct scalar_product_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; -#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN - EIGEN_EMPTY_STRUCT_CTOR(scalar_product_op) -#else +#ifdef EIGEN_SCALAR_BINARY_OP_PLUGIN scalar_product_op() { EIGEN_SCALAR_BINARY_OP_PLUGIN } @@ -110,13 +109,12 @@ struct scalar_conj_product_op : binary_op_base enum { Conj = NumTraits::IsComplex }; - + typedef typename ScalarBinaryOpTraits::ReturnType result_type; - - EIGEN_EMPTY_STRUCT_CTOR(scalar_conj_product_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return conj_helper().pmul(a,b); } - + template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { return conj_helper().pmul(a,b); } @@ -138,7 +136,6 @@ template struct scalar_min_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; - EIGEN_EMPTY_STRUCT_CTOR(scalar_min_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return internal::pmin(a, b); } @@ -171,7 +168,6 @@ template struct scalar_max_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; - EIGEN_EMPTY_STRUCT_CTOR(scalar_max_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return internal::pmax(a,b); } @@ -196,73 +192,122 @@ struct functor_traits > { }; /** \internal - * \brief Template functors for comparison of two scalars - * \todo Implement packet-comparisons - */ -template struct scalar_cmp_op; + * \brief Template functors for comparison of two scalars + * \todo Implement packet-comparisons + */ +template +struct scalar_cmp_op; -template -struct functor_traits > { +template +struct functor_traits> { enum { - Cost = (NumTraits::AddCost+NumTraits::AddCost)/2, - PacketAccess = false + Cost = (NumTraits::AddCost + NumTraits::AddCost) / 2, + PacketAccess = (UseTypedComparators || is_same::value) && is_same::value && + packet_traits::HasCmp }; }; -template -struct result_of(LhsScalar,RhsScalar)> { - typedef bool type; +template +struct typed_cmp_helper { + static constexpr bool SameType = is_same::value; + static constexpr bool IsNumeric = is_arithmetic::Real>::value; + static constexpr bool UseTyped = UseTypedComparators && SameType && IsNumeric; + using type = typename conditional::type; }; +template +using cmp_return_t = typename typed_cmp_helper::type; -template -struct scalar_cmp_op : binary_op_base -{ - typedef bool result_type; - EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a==b;} +template +struct scalar_cmp_op : binary_op_base { + using result_type = cmp_return_t; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator()(const LhsScalar& a, const RhsScalar& b) const { + return a == b ? result_type(1) : result_type(0); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + const Packet cst_one = pset1(result_type(1)); + return pand(pcmp_eq(a, b), cst_one); + } }; -template -struct scalar_cmp_op : binary_op_base -{ - typedef bool result_type; - EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a +struct scalar_cmp_op : binary_op_base { + using result_type = cmp_return_t; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator()(const LhsScalar& a, const RhsScalar& b) const { + return a < b ? result_type(1) : result_type(0); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + const Packet cst_one = pset1(result_type(1)); + return pand(pcmp_lt(a, b), cst_one); + } }; -template -struct scalar_cmp_op : binary_op_base -{ - typedef bool result_type; - EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a<=b;} + +template +struct scalar_cmp_op : binary_op_base { + using result_type = cmp_return_t; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator()(const LhsScalar& a, const RhsScalar& b) const { + return a <= b ? result_type(1) : result_type(0); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + const Packet cst_one = pset1(result_type(1)); + return pand(cst_one, pcmp_le(a, b)); + } }; -template -struct scalar_cmp_op : binary_op_base -{ - typedef bool result_type; - EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a>b;} + +template +struct scalar_cmp_op : binary_op_base { + using result_type = cmp_return_t; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator()(const LhsScalar& a, const RhsScalar& b) const { + return a > b ? result_type(1) : result_type(0); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + const Packet cst_one = pset1(result_type(1)); + return pand(cst_one, pcmp_lt(b, a)); + } }; -template -struct scalar_cmp_op : binary_op_base -{ - typedef bool result_type; - EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a>=b;} + +template +struct scalar_cmp_op : binary_op_base { + using result_type = cmp_return_t; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator()(const LhsScalar& a, const RhsScalar& b) const { + return a >= b ? result_type(1) : result_type(0); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + const Packet cst_one = pset1(result_type(1)); + return pand(cst_one, pcmp_le(b, a)); + } }; -template -struct scalar_cmp_op : binary_op_base -{ - typedef bool result_type; - EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return !(a<=b || b<=a);} + +template +struct scalar_cmp_op : binary_op_base { + using result_type = cmp_return_t; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator()(const LhsScalar& a, const RhsScalar& b) const { + return !(a <= b || b <= a) ? result_type(1) : result_type(0); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + const Packet cst_one = pset1(result_type(1)); + return pandnot(cst_one, por(pcmp_le(a, b), pcmp_le(b, a))); + } }; -template -struct scalar_cmp_op : binary_op_base -{ - typedef bool result_type; - EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a!=b;} + +template +struct scalar_cmp_op : binary_op_base { + using result_type = cmp_return_t; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator()(const LhsScalar& a, const RhsScalar& b) const { + return a != b ? result_type(1) : result_type(0); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + const Packet cst_one = pset1(result_type(1)); + return pandnot(cst_one, pcmp_eq(a, b)); + } }; /** \internal @@ -273,8 +318,6 @@ struct scalar_cmp_op : binary_op_base struct scalar_hypot_op : binary_op_base { - EIGEN_EMPTY_STRUCT_CTOR(scalar_hypot_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar &x, const Scalar &y) const { // This functor is used by hypotNorm only for which it is faster to first apply abs @@ -304,9 +347,7 @@ template struct scalar_pow_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; -#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN - EIGEN_EMPTY_STRUCT_CTOR(scalar_pow_op) -#else +#ifdef EIGEN_SCALAR_BINARY_OP_PLUGIN scalar_pow_op() { typedef Scalar LhsScalar; typedef Exponent RhsScalar; @@ -331,7 +372,7 @@ struct functor_traits > { PacketAccess = (!NumTraits::IsComplex && !NumTraits::IsInteger && packet_traits::HasExp && packet_traits::HasLog && packet_traits::HasRound && packet_traits::HasCmp && - // Temporarly disable packet access for half/bfloat16 until + // Temporarily disable packet access for half/bfloat16 until // accuracy is improved. !is_same::value && !is_same::value ) @@ -349,9 +390,7 @@ template struct scalar_difference_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; -#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN - EIGEN_EMPTY_STRUCT_CTOR(scalar_difference_op) -#else +#ifdef EIGEN_SCALAR_BINARY_OP_PLUGIN scalar_difference_op() { EIGEN_SCALAR_BINARY_OP_PLUGIN } @@ -369,6 +408,28 @@ struct functor_traits > { }; }; +template ::type>::IsInteger> +struct maybe_raise_div_by_zero { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Packet x) { + EIGEN_UNUSED_VARIABLE(x); + } +}; + +#ifndef EIGEN_GPU_COMPILE_PHASE +template +struct maybe_raise_div_by_zero { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Packet x) { + if (EIGEN_PREDICT_FALSE(predux_any(pcmp_eq(x, pzero(x))))) { + // Use volatile variables to force a division by zero, which will + // result in the default platform behaviour (usually SIGFPE). + volatile typename unpacket_traits::type zero = 0; + volatile typename unpacket_traits::type val = 1; + val = val / zero; + } + } +}; +#endif + /** \internal * \brief Template functor to compute the quotient of two scalars * @@ -378,17 +439,17 @@ template struct scalar_quotient_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; -#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN - EIGEN_EMPTY_STRUCT_CTOR(scalar_quotient_op) -#else +#ifdef EIGEN_SCALAR_BINARY_OP_PLUGIN scalar_quotient_op() { EIGEN_SCALAR_BINARY_OP_PLUGIN } #endif EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a / b; } template - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const - { return internal::pdiv(a,b); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { + maybe_raise_div_by_zero::run(b); + return internal::pdiv(a,b); + } }; template struct functor_traits > { @@ -399,63 +460,203 @@ struct functor_traits > { }; }; - - /** \internal - * \brief Template functor to compute the and of two booleans + * \brief Template functor to compute the and of two scalars as if they were booleans * * \sa class CwiseBinaryOp, ArrayBase::operator&& */ +template struct scalar_boolean_and_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_and_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a, const bool& b) const { return a && b; } - template - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const - { return internal::pand(a,b); } + using result_type = Scalar; + // `false` any value `a` that satisfies `a == Scalar(0)` + // `true` is the complement of `false` + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a, const Scalar& b) const { + return (a != Scalar(0)) && (b != Scalar(0)) ? Scalar(1) : Scalar(0); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + const Packet cst_one = pset1(Scalar(1)); + // and(a,b) == !or(!a,!b) + Packet not_a = pcmp_eq(a, pzero(a)); + Packet not_b = pcmp_eq(b, pzero(b)); + Packet a_nand_b = por(not_a, not_b); + return pandnot(cst_one, a_nand_b); + } }; -template<> struct functor_traits { - enum { - Cost = NumTraits::AddCost, - PacketAccess = true - }; +template +struct functor_traits> { + enum { Cost = NumTraits::AddCost, PacketAccess = packet_traits::HasCmp }; }; /** \internal - * \brief Template functor to compute the or of two booleans + * \brief Template functor to compute the or of two scalars as if they were booleans * * \sa class CwiseBinaryOp, ArrayBase::operator|| */ +template struct scalar_boolean_or_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_or_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a, const bool& b) const { return a || b; } - template - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const - { return internal::por(a,b); } + using result_type = Scalar; + // `false` any value `a` that satisfies `a == Scalar(0)` + // `true` is the complement of `false` + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a, const Scalar& b) const { + return (a != Scalar(0)) || (b != Scalar(0)) ? Scalar(1) : Scalar(0); + } + template + EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + const Packet cst_one = pset1(Scalar(1)); + // if or(a,b) == 0, then a == 0 and b == 0 + // or(a,b) == !nor(a,b) + Packet a_nor_b = pcmp_eq(por(a, b), pzero(a)); + return pandnot(cst_one, a_nor_b); + } }; -template<> struct functor_traits { - enum { - Cost = NumTraits::AddCost, - PacketAccess = true - }; +template +struct functor_traits> { + enum { Cost = NumTraits::AddCost, PacketAccess = packet_traits::HasCmp }; }; /** \internal - * \brief Template functor to compute the xor of two booleans + * \brief Template functor to compute the xor of two scalars as if they were booleans * * \sa class CwiseBinaryOp, ArrayBase::operator^ */ +template struct scalar_boolean_xor_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_xor_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a, const bool& b) const { return a ^ b; } - template - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const - { return internal::pxor(a,b); } + using result_type = Scalar; + // `false` any value `a` that satisfies `a == Scalar(0)` + // `true` is the complement of `false` + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a, const Scalar& b) const { + return (a != Scalar(0)) != (b != Scalar(0)) ? Scalar(1) : Scalar(0); + } + template + EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + const Packet cst_one = pset1(Scalar(1)); + // xor(a,b) == xor(!a,!b) + Packet not_a = pcmp_eq(a, pzero(a)); + Packet not_b = pcmp_eq(b, pzero(b)); + Packet a_xor_b = pxor(not_a, not_b); + return pand(cst_one, a_xor_b); + } }; -template<> struct functor_traits { - enum { - Cost = NumTraits::AddCost, - PacketAccess = true - }; +template +struct functor_traits> { + enum { Cost = NumTraits::AddCost, PacketAccess = packet_traits::HasCmp }; +}; + +template ::IsComplex> +struct bitwise_binary_impl { + static constexpr size_t Size = sizeof(Scalar); + using uint_t = typename numext::get_integer_by_size::unsigned_type; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_and(const Scalar& a, const Scalar& b) { + uint_t a_as_uint = numext::bit_cast(a); + uint_t b_as_uint = numext::bit_cast(b); + uint_t result = a_as_uint & b_as_uint; + return numext::bit_cast(result); + } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_or(const Scalar& a, const Scalar& b) { + uint_t a_as_uint = numext::bit_cast(a); + uint_t b_as_uint = numext::bit_cast(b); + uint_t result = a_as_uint | b_as_uint; + return numext::bit_cast(result); + } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_xor(const Scalar& a, const Scalar& b) { + uint_t a_as_uint = numext::bit_cast(a); + uint_t b_as_uint = numext::bit_cast(b); + uint_t result = a_as_uint ^ b_as_uint; + return numext::bit_cast(result); + } +}; + +template +struct bitwise_binary_impl { + using Real = typename NumTraits::Real; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_and(const Scalar& a, const Scalar& b) { + Real real_result = bitwise_binary_impl::run_and(numext::real(a), numext::real(b)); + Real imag_result = bitwise_binary_impl::run_and(numext::imag(a), numext::imag(b)); + return Scalar(real_result, imag_result); + } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_or(const Scalar& a, const Scalar& b) { + Real real_result = bitwise_binary_impl::run_or(numext::real(a), numext::real(b)); + Real imag_result = bitwise_binary_impl::run_or(numext::imag(a), numext::imag(b)); + return Scalar(real_result, imag_result); + } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_xor(const Scalar& a, const Scalar& b) { + Real real_result = bitwise_binary_impl::run_xor(numext::real(a), numext::real(b)); + Real imag_result = bitwise_binary_impl::run_xor(numext::imag(a), numext::imag(b)); + return Scalar(real_result, imag_result); + } +}; + +/** \internal + * \brief Template functor to compute the bitwise and of two scalars + * + * \sa class CwiseBinaryOp, ArrayBase::operator& + */ +template +struct scalar_bitwise_and_op { + EIGEN_STATIC_ASSERT(!NumTraits::RequireInitialization, + BITWISE OPERATIONS MAY ONLY BE PERFORMED ON PLAIN DATA TYPES) + EIGEN_STATIC_ASSERT((!internal::is_same::value), DONT USE BITWISE OPS ON BOOLEAN TYPES) + using result_type = Scalar; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a, const Scalar& b) const { + return bitwise_binary_impl::run_and(a, b); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + return pand(a, b); + } +}; +template +struct functor_traits> { + enum { Cost = NumTraits::AddCost, PacketAccess = true }; +}; + +/** \internal + * \brief Template functor to compute the bitwise or of two scalars + * + * \sa class CwiseBinaryOp, ArrayBase::operator| + */ +template +struct scalar_bitwise_or_op { + EIGEN_STATIC_ASSERT(!NumTraits::RequireInitialization, + BITWISE OPERATIONS MAY ONLY BE PERFORMED ON PLAIN DATA TYPES) + EIGEN_STATIC_ASSERT((!internal::is_same::value), DONT USE BITWISE OPS ON BOOLEAN TYPES) + using result_type = Scalar; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a, const Scalar& b) const { + return bitwise_binary_impl::run_or(a, b); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + return por(a, b); + } +}; +template +struct functor_traits> { + enum { Cost = NumTraits::AddCost, PacketAccess = true }; +}; + +/** \internal + * \brief Template functor to compute the bitwise xor of two scalars + * + * \sa class CwiseBinaryOp, ArrayBase::operator^ + */ +template +struct scalar_bitwise_xor_op { + EIGEN_STATIC_ASSERT(!NumTraits::RequireInitialization, + BITWISE OPERATIONS MAY ONLY BE PERFORMED ON PLAIN DATA TYPES) + EIGEN_STATIC_ASSERT((!internal::is_same::value), DONT USE BITWISE OPS ON BOOLEAN TYPES) + using result_type = Scalar; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a, const Scalar& b) const { + return bitwise_binary_impl::run_xor(a, b); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { + return pxor(a, b); + } +}; +template +struct functor_traits> { + enum { Cost = NumTraits::AddCost, PacketAccess = true }; }; /** \internal @@ -467,9 +668,7 @@ template struct scalar_absolute_difference_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; -#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN - EIGEN_EMPTY_STRUCT_CTOR(scalar_absolute_difference_op) -#else +#ifdef EIGEN_SCALAR_BINARY_OP_PLUGIN scalar_absolute_difference_op() { EIGEN_SCALAR_BINARY_OP_PLUGIN } @@ -489,6 +688,30 @@ struct functor_traits > { }; +template +struct scalar_atan2_op { + using Scalar = LhsScalar; + + static constexpr bool Enable = is_same::value && !NumTraits::IsInteger && !NumTraits::IsComplex; + EIGEN_STATIC_ASSERT(Enable, "LhsScalar and RhsScalar must be the same non-integer, non-complex type") + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& y, const Scalar& x) const { + return numext::atan2(y, x); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& y, const Packet& x) const { + return internal::patan2(y, x); + } +}; + +template + struct functor_traits> { + using Scalar = LhsScalar; + enum { + PacketAccess = is_same::value && packet_traits::HasATan && packet_traits::HasDiv && !NumTraits::IsInteger && !NumTraits::IsComplex, + Cost = int(scalar_div_cost::value) + int(functor_traits>::Cost) + }; +}; //---------- binary functors bound to a constant, thus appearing as a unary functor ---------- diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/NullaryFunctors.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/NullaryFunctors.h index 192f225ddaf..fde36bedb98 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/NullaryFunctors.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/NullaryFunctors.h @@ -10,6 +10,9 @@ #ifndef EIGEN_NULLARY_FUNCTORS_H #define EIGEN_NULLARY_FUNCTORS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -29,7 +32,6 @@ struct functor_traits > PacketAccess = packet_traits::Vectorizable, IsRepeatable = true }; }; template struct scalar_identity_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_identity_op) template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (IndexType row, IndexType col) const { return row==col ? Scalar(1) : Scalar(0); } }; @@ -144,6 +146,39 @@ template struct linspaced_op const linspaced_op_impl::IsInteger> impl; }; +template +struct equalspaced_op { + typedef typename NumTraits::Real RealScalar; + + EIGEN_DEVICE_FUNC equalspaced_op(const Scalar& start, const Scalar& step) : m_start(start), m_step(step) {} + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator()(IndexType i) const { + return m_start + m_step * static_cast(i); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(IndexType i) const { + const Packet cst_start = pset1(m_start); + const Packet cst_step = pset1(m_step); + const Packet cst_lin0 = plset(Scalar(0)); + const Packet cst_offset = pmadd(cst_lin0, cst_step, cst_start); + + Packet i_packet = pset1(static_cast(i)); + return pmadd(i_packet, cst_step, cst_offset); + } + const Scalar m_start; + const Scalar m_step; +}; + +template +struct functor_traits > { + enum { + Cost = NumTraits::AddCost + NumTraits::MulCost, + PacketAccess = + packet_traits::HasSetLinear && packet_traits::HasMul && packet_traits::HasAdd, + IsRepeatable = true + }; +}; + // Linear access is automatically determined from the operator() prototypes available for the given functor. // If it exposes an operator()(i,j), then we assume the i and j coefficients are required independently // and linear access is not possible. In all other cases, linear access is enabled. @@ -152,7 +187,7 @@ template struct functor_has_linear_access { enum { ret = !has_ // For unreliable compilers, let's specialize the has_*ary_operator // helpers so that at least built-in nullary functors work fine. -#if !( (EIGEN_COMP_MSVC>1600) || (EIGEN_GNUC_AT_LEAST(4,8)) || (EIGEN_COMP_ICC>=1600)) +#if !( EIGEN_COMP_MSVC || EIGEN_COMP_GNUC || (EIGEN_COMP_ICC>=1600)) template struct has_nullary_operator,IndexType> { enum { value = 1}; }; template diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/StlFunctors.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/StlFunctors.h index 4570c9b634b..8ee1de5895c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/StlFunctors.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/StlFunctors.h @@ -10,30 +10,10 @@ #ifndef EIGEN_STL_FUNCTORS_H #define EIGEN_STL_FUNCTORS_H -namespace Eigen { - -// Portable replacements for certain functors. -namespace numext { - -template -struct equal_to { - typedef bool result_type; - EIGEN_DEVICE_FUNC bool operator()(const T& lhs, const T& rhs) const { - return lhs == rhs; - } -}; - -template -struct not_equal_to { - typedef bool result_type; - EIGEN_DEVICE_FUNC bool operator()(const T& lhs, const T& rhs) const { - return lhs != rhs; - } -}; - -} - +// IWYU pragma: private +#include "../InternalHeaderCheck.h" +namespace Eigen { namespace internal { // default functor traits for STL functors: @@ -90,29 +70,10 @@ template struct functor_traits > { enum { Cost = 1, PacketAccess = false }; }; -template -struct functor_traits > - : functor_traits > {}; - template struct functor_traits > { enum { Cost = 1, PacketAccess = false }; }; -template -struct functor_traits > - : functor_traits > {}; - -#if (EIGEN_COMP_CXXVER < 11) -// std::binder* are deprecated since c++11 and will be removed in c++17 -template -struct functor_traits > -{ enum { Cost = functor_traits::Cost, PacketAccess = false }; }; - -template -struct functor_traits > -{ enum { Cost = functor_traits::Cost, PacketAccess = false }; }; -#endif - #if (EIGEN_COMP_CXXVER < 17) // std::unary_negate is deprecated since c++17 and will be removed in c++20 template diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/TernaryFunctors.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/TernaryFunctors.h index b254e96c6ad..859cd197fe7 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/TernaryFunctors.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/TernaryFunctors.h @@ -10,13 +10,38 @@ #ifndef EIGEN_TERNARY_FUNCTORS_H #define EIGEN_TERNARY_FUNCTORS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { //---------- associative ternary functors ---------- - +template +struct scalar_boolean_select_op { + static constexpr bool ThenElseAreSame = is_same::value; + EIGEN_STATIC_ASSERT(ThenElseAreSame, THEN AND ELSE MUST BE SAME TYPE) + using Scalar = ThenScalar; + using result_type = Scalar; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const ThenScalar& a, const ElseScalar& b, const ConditionScalar& cond) const { + return cond == ConditionScalar(0) ? b : a; + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b, const Packet& cond) const { + return pselect(pcmp_eq(cond, pzero(cond)), b, a); + } +}; + +template + struct functor_traits> { + using Scalar = ThenScalar; + enum { + Cost = 1, + PacketAccess = is_same::value && is_same::value && packet_traits::HasCmp + }; +}; } // end namespace internal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/UnaryFunctors.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/UnaryFunctors.h index 16136d185aa..d988eb1d728 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/UnaryFunctors.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/functors/UnaryFunctors.h @@ -10,6 +10,9 @@ #ifndef EIGEN_UNARY_FUNCTORS_H #define EIGEN_UNARY_FUNCTORS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -20,7 +23,6 @@ namespace internal { * \sa class CwiseUnaryOp, MatrixBase::operator- */ template struct scalar_opposite_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_opposite_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return -a; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const @@ -39,7 +41,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, Cwise::abs */ template struct scalar_abs_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_abs_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a) const { return numext::abs(a); } template @@ -70,14 +71,12 @@ struct functor_traits > : functor_traits struct abs_knowing_score { - EIGEN_EMPTY_STRUCT_CTOR(abs_knowing_score) typedef typename NumTraits::Real result_type; template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a, const Score&) const { return numext::abs(a); } }; template struct abs_knowing_score::Score_is_abs> { - EIGEN_EMPTY_STRUCT_CTOR(abs_knowing_score) typedef typename NumTraits::Real result_type; template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scal&, const result_type& a) const { return a; } @@ -89,7 +88,6 @@ template struct abs_knowing_score struct scalar_abs2_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_abs2_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a) const { return numext::abs2(a); } @@ -107,7 +105,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, MatrixBase::conjugate() */ template struct scalar_conjugate_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_conjugate_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::conj(a); } template @@ -136,7 +133,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, Cwise::arg */ template struct scalar_arg_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_arg_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a) const { return numext::arg(a); } template @@ -151,6 +147,28 @@ struct functor_traits > PacketAccess = packet_traits::HasArg }; }; + +/** \internal + * \brief Template functor to compute the complex argument, returned as a complex type + * + * \sa class CwiseUnaryOp, Cwise::carg + */ +template +struct scalar_carg_op { + using result_type = Scalar; + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const Scalar operator()(const Scalar& a) const { return Scalar(numext::arg(a)); } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { + return pcarg(a); + } +}; +template +struct functor_traits> { + using RealScalar = typename NumTraits::Real; + enum { Cost = functor_traits>::Cost, PacketAccess = packet_traits::HasATan }; +}; + /** \internal * \brief Template functor to cast a scalar to another type * @@ -158,14 +176,32 @@ struct functor_traits > */ template struct scalar_cast_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) typedef NewType result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const NewType operator() (const Scalar& a) const { return cast(a); } }; + template struct functor_traits > { enum { Cost = is_same::value ? 0 : NumTraits::AddCost, PacketAccess = false }; }; +/** \internal + * `core_cast_op` serves to distinguish the vectorized implementation from that of the legacy `scalar_cast_op` for backwards + * compatibility. The manner in which packet ops are handled is defined by the specialized unary_evaluator: + * `unary_evaluator, ArgType>, IndexBased>` in CoreEvaluators.h + * Otherwise, the non-vectorized behavior is identical to that of `scalar_cast_op` + */ +template +struct core_cast_op : scalar_cast_op {}; + +template +struct functor_traits> { + using CastingTraits = type_casting_traits; + enum { + Cost = is_same::value ? 0 : NumTraits::AddCost, + PacketAccess = CastingTraits::VectorizedCast && (CastingTraits::SrcCoeffRatio <= 8) + }; +}; + /** \internal * \brief Template functor to arithmetically shift a scalar right by a number of bits * @@ -173,7 +209,6 @@ struct functor_traits > */ template struct scalar_shift_right_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_shift_right_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return a >> N; } @@ -192,8 +227,6 @@ struct functor_traits > */ template struct scalar_shift_left_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_shift_left_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return a << N; } template @@ -211,7 +244,6 @@ struct functor_traits > */ template struct scalar_real_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_real_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { return numext::real(a); } @@ -227,7 +259,6 @@ struct functor_traits > */ template struct scalar_imag_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_imag_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { return numext::imag(a); } @@ -243,7 +274,6 @@ struct functor_traits > */ template struct scalar_real_ref_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_real_ref_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type& operator() (const Scalar& a) const { return numext::real_ref(*const_cast(&a)); } @@ -259,7 +289,6 @@ struct functor_traits > */ template struct scalar_imag_ref_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_imag_ref_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type& operator() (const Scalar& a) const { return numext::imag_ref(*const_cast(&a)); } @@ -275,8 +304,7 @@ struct functor_traits > * \sa class CwiseUnaryOp, Cwise::exp() */ template struct scalar_exp_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_exp_op) - EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::exp(a); } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return internal::pexp(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pexp(a); } }; @@ -315,7 +343,6 @@ struct functor_traits > { * \sa class CwiseUnaryOp, ArrayBase::expm1() */ template struct scalar_expm1_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_expm1_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::expm1(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pexpm1(a); } @@ -335,7 +362,6 @@ struct functor_traits > { * \sa class CwiseUnaryOp, ArrayBase::log() */ template struct scalar_log_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_log_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::log(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog(a); } @@ -366,7 +392,6 @@ struct functor_traits > { * \sa class CwiseUnaryOp, ArrayBase::log1p() */ template struct scalar_log1p_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_log1p_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::log1p(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog1p(a); } @@ -386,7 +411,6 @@ struct functor_traits > { * \sa class CwiseUnaryOp, Cwise::log10() */ template struct scalar_log10_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_log10_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { EIGEN_USING_STD(log10) return log10(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog10(a); } @@ -402,7 +426,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, Cwise::log2() */ template struct scalar_log2_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_log2_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return Scalar(EIGEN_LOG2E) * numext::log(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog2(a); } @@ -416,7 +439,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, Cwise::sqrt() */ template struct scalar_sqrt_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_sqrt_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::sqrt(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psqrt(a); } @@ -440,7 +462,6 @@ struct functor_traits > { // Boolean specialization to eliminate -Wimplicit-conversion-floating-point-to-bool warnings. template<> struct scalar_sqrt_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_sqrt_op) EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline bool operator() (const bool& a) const { return a; } template EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return a; } @@ -450,12 +471,25 @@ struct functor_traits > { enum { Cost = 1, PacketAccess = packet_traits::Vectorizable }; }; +/** \internal + * \brief Template functor to compute the cube root of a scalar + * \sa class CwiseUnaryOp, Cwise::sqrt() + */ +template +struct scalar_cbrt_op { + EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::cbrt(a); } +}; + +template +struct functor_traits > { + enum { Cost = 5 * NumTraits::MulCost, PacketAccess = false }; +}; + /** \internal * \brief Template functor to compute the reciprocal square root of a scalar * \sa class CwiseUnaryOp, Cwise::rsqrt() */ template struct scalar_rsqrt_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_rsqrt_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::rsqrt(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::prsqrt(a); } @@ -474,7 +508,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::cos() */ template struct scalar_cos_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cos_op) EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return numext::cos(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pcos(a); } @@ -493,7 +526,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::sin() */ template struct scalar_sin_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_sin_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::sin(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psin(a); } @@ -513,7 +545,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::tan() */ template struct scalar_tan_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_tan_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::tan(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::ptan(a); } @@ -532,7 +563,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::acos() */ template struct scalar_acos_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_acos_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::acos(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pacos(a); } @@ -551,7 +581,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::asin() */ template struct scalar_asin_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_asin_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::asin(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pasin(a); } @@ -571,7 +600,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::atan() */ template struct scalar_atan_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_atan_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::atan(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::patan(a); } @@ -591,7 +619,6 @@ struct functor_traits > */ template struct scalar_tanh_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_tanh_op) EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::tanh(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& x) const { return ptanh(x); } @@ -622,29 +649,30 @@ struct functor_traits > { }; }; -#if EIGEN_HAS_CXX11_MATH /** \internal * \brief Template functor to compute the atanh of a scalar * \sa class CwiseUnaryOp, ArrayBase::atanh() */ template struct scalar_atanh_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_atanh_op) EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::atanh(a); } + template + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& x) const { return patanh(x); } }; template struct functor_traits > { - enum { Cost = 5 * NumTraits::MulCost, PacketAccess = false }; + enum { + Cost = 5 * NumTraits::MulCost, + PacketAccess = packet_traits::HasATanh + }; }; -#endif /** \internal * \brief Template functor to compute the sinh of a scalar * \sa class CwiseUnaryOp, ArrayBase::sinh() */ template struct scalar_sinh_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_sinh_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::sinh(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psinh(a); } @@ -658,14 +686,12 @@ struct functor_traits > }; }; -#if EIGEN_HAS_CXX11_MATH /** \internal * \brief Template functor to compute the asinh of a scalar * \sa class CwiseUnaryOp, ArrayBase::asinh() */ template struct scalar_asinh_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_asinh_op) EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::asinh(a); } }; @@ -673,14 +699,12 @@ template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = false }; }; -#endif /** \internal * \brief Template functor to compute the cosh of a scalar * \sa class CwiseUnaryOp, ArrayBase::cosh() */ template struct scalar_cosh_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cosh_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::cosh(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pcosh(a); } @@ -694,14 +718,12 @@ struct functor_traits > }; }; -#if EIGEN_HAS_CXX11_MATH /** \internal * \brief Template functor to compute the acosh of a scalar * \sa class CwiseUnaryOp, ArrayBase::acosh() */ template struct scalar_acosh_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_acosh_op) EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::acosh(a); } }; @@ -709,7 +731,6 @@ template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = false }; }; -#endif /** \internal * \brief Template functor to compute the inverse of a scalar @@ -717,17 +738,21 @@ struct functor_traits > { */ template struct scalar_inverse_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_inverse_op) EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return Scalar(1)/a; } template EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const - { return internal::pdiv(pset1(Scalar(1)),a); } + { return internal::preciprocal(a); } }; template struct functor_traits > { enum { PacketAccess = packet_traits::HasDiv, - Cost = scalar_div_cost::value + // If packet_traits::HasReciprocal then the Estimated cost is that + // of computing an approximation plus a single Newton-Raphson step, which + // consists of 1 pmul + 1 pmadd. + Cost = (packet_traits::HasReciprocal + ? 4 * NumTraits::MulCost + : scalar_div_cost::value) }; }; @@ -737,7 +762,6 @@ struct functor_traits > { */ template struct scalar_square_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_square_op) EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return a*a; } template EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const @@ -750,7 +774,6 @@ struct functor_traits > // Boolean specialization to avoid -Wint-in-bool-context warnings on GCC. template<> struct scalar_square_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_square_op) EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline bool operator() (const bool& a) const { return a; } template EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const @@ -766,7 +789,6 @@ struct functor_traits > */ template struct scalar_cube_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cube_op) EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return a*a*a; } template EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const @@ -779,7 +801,6 @@ struct functor_traits > // Boolean specialization to avoid -Wint-in-bool-context warnings on GCC. template<> struct scalar_cube_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cube_op) EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline bool operator() (const bool& a) const { return a; } template EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const @@ -794,7 +815,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::round() */ template struct scalar_round_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_round_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::round(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pround(a); } @@ -804,7 +824,7 @@ struct functor_traits > { enum { Cost = NumTraits::MulCost, - PacketAccess = packet_traits::HasRound + PacketAccess = packet_traits::HasRound || NumTraits::IsInteger }; }; @@ -813,7 +833,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::floor() */ template struct scalar_floor_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_floor_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::floor(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pfloor(a); } @@ -823,7 +842,7 @@ struct functor_traits > { enum { Cost = NumTraits::MulCost, - PacketAccess = packet_traits::HasFloor + PacketAccess = packet_traits::HasFloor || NumTraits::IsInteger }; }; @@ -832,7 +851,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::rint() */ template struct scalar_rint_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_rint_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::rint(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::print(a); } @@ -842,7 +860,7 @@ struct functor_traits > { enum { Cost = NumTraits::MulCost, - PacketAccess = packet_traits::HasRint + PacketAccess = packet_traits::HasRint || NumTraits::IsInteger }; }; @@ -851,7 +869,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::ceil() */ template struct scalar_ceil_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_ceil_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::ceil(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pceil(a); } @@ -861,7 +878,7 @@ struct functor_traits > { enum { Cost = NumTraits::MulCost, - PacketAccess = packet_traits::HasCeil + PacketAccess = packet_traits::HasCeil || NumTraits::IsInteger }; }; @@ -869,23 +886,39 @@ struct functor_traits > * \brief Template functor to compute whether a scalar is NaN * \sa class CwiseUnaryOp, ArrayBase::isnan() */ -template struct scalar_isnan_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_isnan_op) - typedef bool result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { +template +struct scalar_isnan_op { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const Scalar& a) const { #if defined(SYCL_DEVICE_ONLY) return numext::isnan(a); #else - return (numext::isnan)(a); + return numext::isnan EIGEN_NOT_A_MACRO (a); #endif } }; + + template -struct functor_traits > +struct scalar_isnan_op { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator() (const Scalar& a) const { +#if defined(SYCL_DEVICE_ONLY) + return (numext::isnan(a) ? ptrue(a) : pzero(a)); +#else + return (numext::isnan EIGEN_NOT_A_MACRO (a) ? ptrue(a) : pzero(a)); +#endif + } + template + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { + return pisnan(a); + } +}; + +template +struct functor_traits > { enum { Cost = NumTraits::MulCost, - PacketAccess = false + PacketAccess = packet_traits::HasCmp && UseTypedPredicate }; }; @@ -894,7 +927,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::isinf() */ template struct scalar_isinf_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_isinf_op) typedef bool result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { #if defined(SYCL_DEVICE_ONLY) @@ -918,7 +950,6 @@ struct functor_traits > * \sa class CwiseUnaryOp, ArrayBase::isfinite() */ template struct scalar_isfinite_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_isfinite_op) typedef bool result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { #if defined(SYCL_DEVICE_ONLY) @@ -938,67 +969,91 @@ struct functor_traits > }; /** \internal - * \brief Template functor to compute the logical not of a boolean + * \brief Template functor to compute the logical not of a scalar as if it were a boolean * * \sa class CwiseUnaryOp, ArrayBase::operator! */ -template struct scalar_boolean_not_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_not_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a) const { return !a; } +template +struct scalar_boolean_not_op { + using result_type = Scalar; + // `false` any value `a` that satisfies `a == Scalar(0)` + // `true` is the complement of `false` + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a) const { + return a == Scalar(0) ? Scalar(1) : Scalar(0); + } + template + EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { + const Packet cst_one = pset1(Scalar(1)); + Packet not_a = pcmp_eq(a, pzero(a)); + return pand(not_a, cst_one); + } }; -template -struct functor_traits > { - enum { - Cost = NumTraits::AddCost, - PacketAccess = false - }; +template +struct functor_traits> { + enum { Cost = NumTraits::AddCost, PacketAccess = packet_traits::HasCmp }; +}; + +template ::IsComplex> +struct bitwise_unary_impl { + static constexpr size_t Size = sizeof(Scalar); + using uint_t = typename numext::get_integer_by_size::unsigned_type; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_not(const Scalar& a) { + uint_t a_as_uint = numext::bit_cast(a); + uint_t result = ~a_as_uint; + return numext::bit_cast(result); + } +}; + +template +struct bitwise_unary_impl { + using Real = typename NumTraits::Real; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_not(const Scalar& a) { + Real real_result = bitwise_unary_impl::run_not(numext::real(a)); + Real imag_result = bitwise_unary_impl::run_not(numext::imag(a)); + return Scalar(real_result, imag_result); + } +}; + +/** \internal + * \brief Template functor to compute the bitwise not of a scalar + * + * \sa class CwiseUnaryOp, ArrayBase::operator~ + */ +template +struct scalar_bitwise_not_op { + EIGEN_STATIC_ASSERT(!NumTraits::RequireInitialization, BITWISE OPERATIONS MAY ONLY BE PERFORMED ON PLAIN DATA TYPES) + EIGEN_STATIC_ASSERT((!internal::is_same::value), DONT USE BITWISE OPS ON BOOLEAN TYPES) + using result_type = Scalar; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a) const { + return bitwise_unary_impl::run_not(a); + } + template + EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { + return pandnot(ptrue(a), a); + } +}; +template +struct functor_traits> { + enum { Cost = NumTraits::AddCost, PacketAccess = true }; }; /** \internal * \brief Template functor to compute the signum of a scalar * \sa class CwiseUnaryOp, Cwise::sign() */ -template::IsComplex!=0), bool is_integer=(NumTraits::IsInteger!=0) > struct scalar_sign_op; template -struct scalar_sign_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_sign_op) +struct scalar_sign_op { EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { - return Scalar( (a>Scalar(0)) - (a - //EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psign(a); } -}; -template -struct scalar_sign_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_sign_op) - EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const - { - return (numext::isnan)(a) ? a : Scalar( (a>Scalar(0)) - (a + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { + return internal::psign(a); } - //TODO - //template - //EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psign(a); } }; -template -struct scalar_sign_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_sign_op) - EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const - { - typedef typename NumTraits::Real real_type; - real_type aa = numext::abs(a); - if (aa==real_type(0)) - return Scalar(0); - aa = real_type(1)/aa; - return Scalar(a.real()*aa, a.imag()*aa ); - } - //TODO - //template - //EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psign(a); } -}; template struct functor_traits > { enum { @@ -1006,7 +1061,7 @@ struct functor_traits > NumTraits::IsComplex ? ( 8*NumTraits::MulCost ) // roughly : ( 3*NumTraits::AddCost), - PacketAccess = packet_traits::HasSign + PacketAccess = packet_traits::HasSign && packet_traits::Vectorizable }; }; @@ -1016,7 +1071,6 @@ struct functor_traits > */ template struct scalar_logistic_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_logistic_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& x) const { return packetOp(x); } @@ -1024,87 +1078,104 @@ struct scalar_logistic_op { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& x) const { const Packet one = pset1(T(1)); - return pdiv(one, padd(one, pexp(pnegate(x)))); + const Packet inf = pset1(NumTraits::infinity()); + const Packet e = pexp(x); + const Packet inf_mask = pcmp_eq(e, inf); + return pselect(inf_mask, one, pdiv(e, padd(one, e))); } }; -#ifndef EIGEN_GPU_COMPILE_PHASE +// TODO(rmlarsen): Enable the following on host when integer_packet is defined +// for the relevant packet types. +#ifdef EIGEN_GPU_CC + /** \internal * \brief Template specialization of the logistic function for float. - * - * Uses just a 9/10-degree rational interpolant which - * interpolates 1/(1+exp(-x)) - 0.5 up to a couple of ulps in the range - * [-9, 18]. Below -9 we use the more accurate approximation - * 1/(1+exp(-x)) ~= exp(x), and above 18 the logistic function is 1 withing - * one ulp. The shifted logistic is interpolated because it was easier to - * make the fit converge. - * + * Computes S(x) = exp(x) / (1 + exp(x)), where exp(x) is implemented + * using an algorithm partly adopted from the implementation of + * pexp_float. See the individual steps described in the code below. + * Note that compared to pexp, we use an additional outer multiplicative + * range reduction step using the identity exp(x) = exp(x/2)^2. + * This prevert us from having to call ldexp on values that could produce + * a denormal result, which allows us to call the faster implementation in + * pldexp_fast_impl::run(p, m). + * The final squaring, however, doubles the error bound on the final + * approximation. Exhaustive testing shows that we have a worst case error + * of 4.5 ulps (compared to computing S(x) in double precision), which is + * acceptable. */ template <> struct scalar_logistic_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_logistic_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator()(const float& x) const { - return packetOp(x); + // Truncate at the first point where the interpolant is exactly one. + const float cst_exp_hi = 16.6355324f; + const float e = numext::exp(numext::mini(x, cst_exp_hi)); + return e / (1.0f + e); } - template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - Packet packetOp(const Packet& _x) const { - const Packet cutoff_lower = pset1(-9.f); - const Packet lt_mask = pcmp_lt(_x, cutoff_lower); - const bool any_small = predux_any(lt_mask); - - // The upper cut-off is the smallest x for which the rational approximation evaluates to 1. - // Choosing this value saves us a few instructions clamping the results at the end. -#ifdef EIGEN_VECTORIZE_FMA - const Packet cutoff_upper = pset1(15.7243833541870117f); -#else - const Packet cutoff_upper = pset1(15.6437711715698242f); -#endif - const Packet x = pmin(_x, cutoff_upper); - - // The monomial coefficients of the numerator polynomial (odd). - const Packet alpha_1 = pset1(2.48287947061529e-01f); - const Packet alpha_3 = pset1(8.51377133304701e-03f); - const Packet alpha_5 = pset1(6.08574864600143e-05f); - const Packet alpha_7 = pset1(1.15627324459942e-07f); - const Packet alpha_9 = pset1(4.37031012579801e-11f); - - // The monomial coefficients of the denominator polynomial (even). - const Packet beta_0 = pset1(9.93151921023180e-01f); - const Packet beta_2 = pset1(1.16817656904453e-01f); - const Packet beta_4 = pset1(1.70198817374094e-03f); - const Packet beta_6 = pset1(6.29106785017040e-06f); - const Packet beta_8 = pset1(5.76102136993427e-09f); - const Packet beta_10 = pset1(6.10247389755681e-13f); - - // Since the polynomials are odd/even, we need x^2. - const Packet x2 = pmul(x, x); - - // Evaluate the numerator polynomial p. - Packet p = pmadd(x2, alpha_9, alpha_7); - p = pmadd(x2, p, alpha_5); - p = pmadd(x2, p, alpha_3); - p = pmadd(x2, p, alpha_1); - p = pmul(x, p); - - // Evaluate the denominator polynomial q. - Packet q = pmadd(x2, beta_10, beta_8); - q = pmadd(x2, q, beta_6); - q = pmadd(x2, q, beta_4); - q = pmadd(x2, q, beta_2); - q = pmadd(x2, q, beta_0); - // Divide the numerator by the denominator and shift it up. - const Packet logistic = padd(pdiv(p, q), pset1(0.5f)); - if (EIGEN_PREDICT_FALSE(any_small)) { - const Packet exponential = pexp(_x); - return pselect(lt_mask, exponential, logistic); - } else { - return logistic; - } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet + packetOp(const Packet& _x) const { + const Packet cst_zero = pset1(0.0f); + const Packet cst_one = pset1(1.0f); + const Packet cst_half = pset1(0.5f); + // Truncate at the first point where the interpolant is exactly one. + const Packet cst_exp_hi = pset1(16.6355324f); + const Packet cst_exp_lo = pset1(-104.f); + + // Clamp x to the non-trivial range where S(x). Outside this + // interval the correctly rounded value of S(x) is either zero + // or one. + Packet zero_mask = pcmp_lt(_x, cst_exp_lo); + Packet x = pmin(_x, cst_exp_hi); + + // 1. Multiplicative range reduction: + // Reduce the range of x by a factor of 2. This avoids having + // to compute exp(x) accurately where the result is a denormalized + // value. + x = pmul(x, cst_half); + + // 2. Subtractive range reduction: + // Express exp(x) as exp(m*ln(2) + r) = 2^m*exp(r), start by extracting + // m = floor(x/ln(2) + 0.5), such that x = m*ln(2) + r. + const Packet cst_cephes_LOG2EF = pset1(1.44269504088896341f); + Packet m = pfloor(pmadd(x, cst_cephes_LOG2EF, cst_half)); + // Get r = x - m*ln(2). We use a trick from Cephes where the term + // m*ln(2) is subtracted out in two parts, m*C1+m*C2 = m*ln(2), + // to avoid accumulating truncation errors. + const Packet cst_cephes_exp_C1 = pset1(-0.693359375f); + const Packet cst_cephes_exp_C2 = pset1(2.12194440e-4f); + Packet r = pmadd(m, cst_cephes_exp_C1, x); + r = pmadd(m, cst_cephes_exp_C2, r); + + // 3. Compute an approximation to exp(r) using a degree 5 minimax polynomial. + // We compute even and odd terms separately to increase instruction level + // parallelism. + Packet r2 = pmul(r, r); + const Packet cst_p2 = pset1(0.49999141693115234375f); + const Packet cst_p3 = pset1(0.16666877269744873046875f); + const Packet cst_p4 = pset1(4.1898667812347412109375e-2f); + const Packet cst_p5 = pset1(8.33471305668354034423828125e-3f); + + const Packet p_even = pmadd(r2, cst_p4, cst_p2); + const Packet p_odd = pmadd(r2, cst_p5, cst_p3); + const Packet p_low = padd(r, cst_one); + Packet p = pmadd(r, p_odd, p_even); + p = pmadd(r2, p, p_low); + + // 4. Undo subtractive range reduction exp(m*ln(2) + r) = 2^m * exp(r). + Packet e = pldexp_fast_impl::run(p, m); + + // 5. Undo multiplicative range reduction by using exp(r) = exp(r/2)^2. + e = pmul(e, e); + + // Return exp(x) / (1 + exp(x)) + return pselect(zero_mask, cst_zero, pdiv(e, padd(cst_one, e))); } }; #endif // #ifndef EIGEN_GPU_COMPILE_PHASE + template struct functor_traits > { enum { @@ -1124,6 +1195,97 @@ struct functor_traits > { }; }; +template ::IsInteger, + bool IsExponentInteger = NumTraits::IsInteger, + bool IsBaseComplex = NumTraits::IsComplex, + bool IsExponentComplex = NumTraits::IsComplex> +struct scalar_unary_pow_op { + typedef typename internal::promote_scalar_arg< + Scalar, ExponentScalar, + internal::has_ReturnType >::value>::type PromotedExponent; + typedef typename ScalarBinaryOpTraits::ReturnType result_type; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_unary_pow_op(const ExponentScalar& exponent) : m_exponent(exponent) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator()(const Scalar& a) const { + EIGEN_USING_STD(pow); + return static_cast(pow(a, m_exponent)); + } + + private: + const ExponentScalar m_exponent; + scalar_unary_pow_op() {} +}; + +template +constexpr int exponent_digits() { + return CHAR_BIT * sizeof(T) - NumTraits::digits() - NumTraits::IsSigned; +} + +template +struct is_floating_exactly_representable { + // TODO(rmlarsen): Add radix to NumTraits and enable this check. + // (NumTraits::radix == NumTraits::radix) && + static constexpr bool value = (exponent_digits() >= exponent_digits() && + NumTraits::digits() >= NumTraits::digits()); +}; + + +// Specialization for real, non-integer types, non-complex types. +template +struct scalar_unary_pow_op { + template ::value> + std::enable_if_t check_is_representable() const {} + + // Issue a deprecation warning if we do a narrowing conversion on the exponent. + template ::value> + EIGEN_DEPRECATED std::enable_if_t check_is_representable() const {} + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + scalar_unary_pow_op(const ExponentScalar& exponent) : m_exponent(static_cast(exponent)) { + check_is_representable(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a) const { + EIGEN_USING_STD(pow); + return static_cast(pow(a, m_exponent)); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { + return unary_pow_impl::run(a, m_exponent); + } + + private: + const Scalar m_exponent; + scalar_unary_pow_op() {} +}; + +template +struct scalar_unary_pow_op { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_unary_pow_op(const ExponentScalar& exponent) : m_exponent(exponent) {} + // TODO: error handling logic for complex^real_integer + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a) const { + return unary_pow_impl::run(a, m_exponent); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { + return unary_pow_impl::run(a, m_exponent); + } + + private: + const ExponentScalar m_exponent; + scalar_unary_pow_op() {} +}; + +template +struct functor_traits> { + enum { + GenPacketAccess = functor_traits>::PacketAccess, + IntPacketAccess = !NumTraits::IsComplex && packet_traits::HasMul && (packet_traits::HasDiv || NumTraits::IsInteger) && packet_traits::HasCmp, + PacketAccess = NumTraits::IsInteger ? IntPacketAccess : (IntPacketAccess && GenPacketAccess), + Cost = functor_traits>::Cost + }; +}; + } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralBlockPanelKernel.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralBlockPanelKernel.h index f35b760c1df..0a349b028c3 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralBlockPanelKernel.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralBlockPanelKernel.h @@ -11,6 +11,9 @@ #define EIGEN_GENERAL_BLOCK_PANEL_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -21,7 +24,7 @@ enum GEBPPacketSizeType { GEBPPacketQuarter }; -template +template class gebp_traits; @@ -55,8 +58,13 @@ const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(256*10 const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(2*1024*1024); #elif EIGEN_ARCH_PPC const std::ptrdiff_t defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(64*1024); +#ifdef _ARCH_PWR10 +const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(2*1024*1024); +const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(8*1024*1024); +#else const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(512*1024); const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(4*1024*1024); +#endif #else const std::ptrdiff_t defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(16*1024); const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(512*1024); @@ -352,9 +360,9 @@ inline void computeProductBlockingSizes(Index& k, Index& m, Index& n, Index num_ template struct RhsPanelHelper { private: - static const int remaining_registers = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS - registers_taken; + static constexpr int remaining_registers = (std::max)(int(EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS) - registers_taken, 0); public: - typedef typename conditional=4, RhsPacketx4, RhsPacket>::type type; + typedef std::conditional_t=4, RhsPacketx4, RhsPacket> type; }; template @@ -376,12 +384,12 @@ struct packet_conditional { typedef T1 type; }; template struct packet_conditional { typedef T2 type; }; -#define PACKET_DECL_COND_PREFIX(prefix, name, packet_size) \ +#define PACKET_DECL_COND_POSTFIX(postfix, name, packet_size) \ typedef typename packet_conditional::type, \ typename packet_traits::half, \ typename unpacket_traits::half>::half>::type \ - prefix ## name ## Packet + name ## Packet ## postfix #define PACKET_DECL_COND(name, packet_size) \ typedef typename packet_conditional { typedef T2 type; }; typename unpacket_traits::half>::half>::type \ name ## Packet -#define PACKET_DECL_COND_SCALAR_PREFIX(prefix, packet_size) \ +#define PACKET_DECL_COND_SCALAR_POSTFIX(postfix, packet_size) \ typedef typename packet_conditional::type, \ typename packet_traits::half, \ typename unpacket_traits::half>::half>::type \ - prefix ## ScalarPacket + ScalarPacket ## postfix #define PACKET_DECL_COND_SCALAR(packet_size) \ typedef typename packet_conditional { typedef T2 type; }; * cplx*real : unpack rhs to constant packets, ... * real*cplx : load lhs as (a0,a0,a1,a1), and mul as usual */ -template +template class gebp_traits { public: - typedef _LhsScalar LhsScalar; - typedef _RhsScalar RhsScalar; + typedef LhsScalar_ LhsScalar; + typedef RhsScalar_ RhsScalar; typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; - PACKET_DECL_COND_PREFIX(_, Lhs, _PacketSize); - PACKET_DECL_COND_PREFIX(_, Rhs, _PacketSize); - PACKET_DECL_COND_PREFIX(_, Res, _PacketSize); + PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_); + PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_); + PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_); enum { - ConjLhs = _ConjLhs, - ConjRhs = _ConjRhs, - Vectorizable = unpacket_traits<_LhsPacket>::vectorizable && unpacket_traits<_RhsPacket>::vectorizable, - LhsPacketSize = Vectorizable ? unpacket_traits<_LhsPacket>::size : 1, - RhsPacketSize = Vectorizable ? unpacket_traits<_RhsPacket>::size : 1, - ResPacketSize = Vectorizable ? unpacket_traits<_ResPacket>::size : 1, + ConjLhs = ConjLhs_, + ConjRhs = ConjRhs_, + Vectorizable = unpacket_traits::vectorizable && unpacket_traits::vectorizable, + LhsPacketSize = Vectorizable ? unpacket_traits::size : 1, + RhsPacketSize = Vectorizable ? unpacket_traits::size : 1, + ResPacketSize = Vectorizable ? unpacket_traits::size : 1, NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS, @@ -440,7 +448,7 @@ class gebp_traits nr = 4, // register block size along the M direction (currently, this one cannot be modified) - default_mr = (EIGEN_PLAIN_ENUM_MIN(16,NumberOfRegisters)/2/nr)*LhsPacketSize, + default_mr = (plain_enum_min(16, NumberOfRegisters)/2/nr)*LhsPacketSize, #if defined(EIGEN_HAS_SINGLE_INSTRUCTION_MADD) && !defined(EIGEN_VECTORIZE_ALTIVEC) && !defined(EIGEN_VECTORIZE_VSX) \ && ((!EIGEN_COMP_MSVC) || (EIGEN_COMP_MSVC>=1914)) // we assume 16 registers or more @@ -457,9 +465,9 @@ class gebp_traits }; - typedef typename conditional::type LhsPacket; - typedef typename conditional::type RhsPacket; - typedef typename conditional::type ResPacket; + typedef std::conditional_t LhsPacket; + typedef std::conditional_t RhsPacket; + typedef std::conditional_t ResPacket; typedef LhsPacket LhsPacket4Packing; typedef QuadPacket RhsPacketx4; @@ -543,25 +551,25 @@ class gebp_traits }; -template -class gebp_traits, RealScalar, _ConjLhs, false, Arch, _PacketSize> +template +class gebp_traits, RealScalar, ConjLhs_, false, Arch, PacketSize_> { public: typedef std::complex LhsScalar; typedef RealScalar RhsScalar; typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; - PACKET_DECL_COND_PREFIX(_, Lhs, _PacketSize); - PACKET_DECL_COND_PREFIX(_, Rhs, _PacketSize); - PACKET_DECL_COND_PREFIX(_, Res, _PacketSize); + PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_); + PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_); + PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_); enum { - ConjLhs = _ConjLhs, + ConjLhs = ConjLhs_, ConjRhs = false, - Vectorizable = unpacket_traits<_LhsPacket>::vectorizable && unpacket_traits<_RhsPacket>::vectorizable, - LhsPacketSize = Vectorizable ? unpacket_traits<_LhsPacket>::size : 1, - RhsPacketSize = Vectorizable ? unpacket_traits<_RhsPacket>::size : 1, - ResPacketSize = Vectorizable ? unpacket_traits<_ResPacket>::size : 1, + Vectorizable = unpacket_traits::vectorizable && unpacket_traits::vectorizable, + LhsPacketSize = Vectorizable ? unpacket_traits::size : 1, + RhsPacketSize = Vectorizable ? unpacket_traits::size : 1, + ResPacketSize = Vectorizable ? unpacket_traits::size : 1, NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS, nr = 4, @@ -569,16 +577,16 @@ class gebp_traits, RealScalar, _ConjLhs, false, Arch, _ // we assume 16 registers mr = 3*LhsPacketSize, #else - mr = (EIGEN_PLAIN_ENUM_MIN(16,NumberOfRegisters)/2/nr)*LhsPacketSize, + mr = (plain_enum_min(16, NumberOfRegisters)/2/nr)*LhsPacketSize, #endif LhsProgress = LhsPacketSize, RhsProgress = 1 }; - typedef typename conditional::type LhsPacket; - typedef typename conditional::type RhsPacket; - typedef typename conditional::type ResPacket; + typedef std::conditional_t LhsPacket; + typedef std::conditional_t RhsPacket; + typedef std::conditional_t ResPacket; typedef LhsPacket LhsPacket4Packing; typedef QuadPacket RhsPacketx4; @@ -612,7 +620,7 @@ class gebp_traits, RealScalar, _ConjLhs, false, Arch, _ EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { - loadRhsQuad_impl(b,dest, typename conditional::type()); + loadRhsQuad_impl(b,dest, std::conditional_t()); } EIGEN_STRONG_INLINE void loadRhsQuad_impl(const RhsScalar* b, RhsPacket& dest, const true_type&) const @@ -643,7 +651,7 @@ class gebp_traits, RealScalar, _ConjLhs, false, Arch, _ template EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp, const LaneIdType&) const { - madd_impl(a, b, c, tmp, typename conditional::type()); + madd_impl(a, b, c, tmp, std::conditional_t()); } template @@ -701,7 +709,7 @@ DoublePacket padd(const DoublePacket &a, const DoublePacket const DoublePacket& predux_half_dowto4(const DoublePacket &a, - typename enable_if::size<=8>::type* = 0) + std::enable_if_t::size<=8>* = 0) { return a; } @@ -709,7 +717,7 @@ predux_half_dowto4(const DoublePacket &a, template DoublePacket::half> predux_half_dowto4(const DoublePacket &a, - typename enable_if::size==16>::type* = 0) + std::enable_if_t::size==16>* = 0) { // yes, that's pretty hackish :( DoublePacket::half> res; @@ -723,7 +731,7 @@ predux_half_dowto4(const DoublePacket &a, // same here, "quad" actually means "8" in terms of real coefficients template void loadQuadToDoublePacket(const Scalar* b, DoublePacket& dest, - typename enable_if::size<=8>::type* = 0) + std::enable_if_t::size<=8>* = 0) { dest.first = pset1(numext::real(*b)); dest.second = pset1(numext::imag(*b)); @@ -731,7 +739,7 @@ void loadQuadToDoublePacket(const Scalar* b, DoublePacket& dest, template void loadQuadToDoublePacket(const Scalar* b, DoublePacket& dest, - typename enable_if::size==16>::type* = 0) + std::enable_if_t::size==16>* = 0) { // yes, that's pretty hackish too :( typedef typename NumTraits::Real RealScalar; @@ -744,6 +752,9 @@ void loadQuadToDoublePacket(const Scalar* b, DoublePacket& dest, template struct unpacket_traits > { typedef DoublePacket::half> half; + enum{ + size = 2 * unpacket_traits::size + }; }; // template // DoublePacket pmadd(const DoublePacket &a, const DoublePacket &b) @@ -754,8 +765,8 @@ template struct unpacket_traits > { // return res; // } -template -class gebp_traits, std::complex, _ConjLhs, _ConjRhs, Arch, _PacketSize > +template +class gebp_traits, std::complex, ConjLhs_, ConjRhs_, Arch, PacketSize_ > { public: typedef std::complex Scalar; @@ -763,19 +774,19 @@ class gebp_traits, std::complex, _ConjLhs, typedef std::complex RhsScalar; typedef std::complex ResScalar; - PACKET_DECL_COND_PREFIX(_, Lhs, _PacketSize); - PACKET_DECL_COND_PREFIX(_, Rhs, _PacketSize); - PACKET_DECL_COND_PREFIX(_, Res, _PacketSize); - PACKET_DECL_COND(Real, _PacketSize); - PACKET_DECL_COND_SCALAR(_PacketSize); + PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_); + PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_); + PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_); + PACKET_DECL_COND(Real, PacketSize_); + PACKET_DECL_COND_SCALAR(PacketSize_); enum { - ConjLhs = _ConjLhs, - ConjRhs = _ConjRhs, + ConjLhs = ConjLhs_, + ConjRhs = ConjRhs_, Vectorizable = unpacket_traits::vectorizable && unpacket_traits::vectorizable, - ResPacketSize = Vectorizable ? unpacket_traits<_ResPacket>::size : 1, - LhsPacketSize = Vectorizable ? unpacket_traits<_LhsPacket>::size : 1, + ResPacketSize = Vectorizable ? unpacket_traits::size : 1, + LhsPacketSize = Vectorizable ? unpacket_traits::size : 1, RhsPacketSize = Vectorizable ? unpacket_traits::size : 1, RealPacketSize = Vectorizable ? unpacket_traits::size : 1, @@ -789,13 +800,13 @@ class gebp_traits, std::complex, _ConjLhs, typedef DoublePacket DoublePacketType; - typedef typename conditional::type LhsPacket4Packing; - typedef typename conditional::type LhsPacket; - typedef typename conditional::type RhsPacket; - typedef typename conditional::type ResPacket; - typedef typename conditional::type AccPacket; + typedef std::conditional_t LhsPacket4Packing; + typedef std::conditional_t LhsPacket; + typedef std::conditional_t RhsPacket; + typedef std::conditional_t ResPacket; + typedef std::conditional_t AccPacket; - // this actualy holds 8 packets! + // this actually holds 8 packets! typedef QuadPacket RhsPacketx4; EIGEN_STRONG_INLINE void initAcc(Scalar& p) { p = Scalar(0); } @@ -866,7 +877,7 @@ class gebp_traits, std::complex, _ConjLhs, template EIGEN_STRONG_INLINE - typename enable_if::value>::type + std::enable_if_t::value> madd(const LhsPacketType& a, const RhsPacketType& b, DoublePacket& c, TmpType& /*tmp*/, const LaneIdType&) const { c.first = padd(pmul(a,b.first), c.first); @@ -920,8 +931,8 @@ class gebp_traits, std::complex, _ConjLhs, conj_helper cj; }; -template -class gebp_traits, false, _ConjRhs, Arch, _PacketSize > +template +class gebp_traits, false, ConjRhs_, Arch, PacketSize_ > { public: typedef std::complex Scalar; @@ -929,38 +940,38 @@ class gebp_traits, false, _ConjRhs, Arch, _ typedef Scalar RhsScalar; typedef Scalar ResScalar; - PACKET_DECL_COND_PREFIX(_, Lhs, _PacketSize); - PACKET_DECL_COND_PREFIX(_, Rhs, _PacketSize); - PACKET_DECL_COND_PREFIX(_, Res, _PacketSize); - PACKET_DECL_COND_PREFIX(_, Real, _PacketSize); - PACKET_DECL_COND_SCALAR_PREFIX(_, _PacketSize); + PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_); + PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_); + PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_); + PACKET_DECL_COND_POSTFIX(_, Real, PacketSize_); + PACKET_DECL_COND_SCALAR_POSTFIX(_, PacketSize_); -#undef PACKET_DECL_COND_SCALAR_PREFIX -#undef PACKET_DECL_COND_PREFIX +#undef PACKET_DECL_COND_SCALAR_POSTFIX +#undef PACKET_DECL_COND_POSTFIX #undef PACKET_DECL_COND_SCALAR #undef PACKET_DECL_COND enum { ConjLhs = false, - ConjRhs = _ConjRhs, - Vectorizable = unpacket_traits<_RealPacket>::vectorizable - && unpacket_traits<_ScalarPacket>::vectorizable, - LhsPacketSize = Vectorizable ? unpacket_traits<_LhsPacket>::size : 1, - RhsPacketSize = Vectorizable ? unpacket_traits<_RhsPacket>::size : 1, - ResPacketSize = Vectorizable ? unpacket_traits<_ResPacket>::size : 1, + ConjRhs = ConjRhs_, + Vectorizable = unpacket_traits::vectorizable + && unpacket_traits::vectorizable, + LhsPacketSize = Vectorizable ? unpacket_traits::size : 1, + RhsPacketSize = Vectorizable ? unpacket_traits::size : 1, + ResPacketSize = Vectorizable ? unpacket_traits::size : 1, NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS, // FIXME: should depend on NumberOfRegisters nr = 4, - mr = (EIGEN_PLAIN_ENUM_MIN(16,NumberOfRegisters)/2/nr)*ResPacketSize, + mr = (plain_enum_min(16, NumberOfRegisters)/2/nr)*ResPacketSize, LhsProgress = ResPacketSize, RhsProgress = 1 }; - typedef typename conditional::type LhsPacket; - typedef typename conditional::type RhsPacket; - typedef typename conditional::type ResPacket; + typedef std::conditional_t LhsPacket; + typedef std::conditional_t RhsPacket; + typedef std::conditional_t ResPacket; typedef LhsPacket LhsPacket4Packing; typedef QuadPacket RhsPacketx4; typedef ResPacket AccPacket; @@ -1009,7 +1020,7 @@ class gebp_traits, false, _ConjRhs, Arch, _ template EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp, const LaneIdType&) const { - madd_impl(a, b, c, tmp, typename conditional::type()); + madd_impl(a, b, c, tmp, std::conditional_t()); } template @@ -1068,6 +1079,7 @@ struct gebp_kernel typedef typename Traits::RhsPacketx4 RhsPacketx4; typedef typename RhsPanelHelper::type RhsPanel15; + typedef typename RhsPanelHelper::type RhsPanel27; typedef gebp_traits SwappedTraits; @@ -1201,7 +1213,7 @@ struct lhs_process_one_packet traits.madd(*A0, *rhs_panel, *C1, *T0, fix<1>); traits.madd(*A0, *rhs_panel, *C2, *T0, fix<2>); traits.madd(*A0, *rhs_panel, *C3, *T0, fix<3>); - #if EIGEN_GNUC_AT_LEAST(6,0) && defined(EIGEN_VECTORIZE_SSE) + #if EIGEN_GNUC_STRICT_AT_LEAST(6,0,0) && defined(EIGEN_VECTORIZE_SSE) && !(EIGEN_COMP_LCC) __asm__ ("" : "+x,m" (*A0)); #endif EIGEN_ASM_COMMENT("end step of gebp micro kernel 1X4"); @@ -1213,13 +1225,140 @@ struct lhs_process_one_packet int prefetch_res_offset, Index peeled_kc, Index pk, Index cols, Index depth, Index packet_cols4) { GEBPTraits traits; - + Index packet_cols8 = nr>=8 ? (cols/8) * 8 : 0; // loops on each largest micro horizontal panel of lhs // (LhsProgress x depth) for(Index i=peelStart; i=8) { + for(Index j2=0; j2); \ + traits.updateRhs(&blB[(1 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C1, T0, fix<1>); \ + traits.updateRhs(&blB[(2 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C2, T0, fix<2>); \ + traits.updateRhs(&blB[(3 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C3, T0, fix<3>); \ + traits.loadRhs(&blB[(4 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C4, T0, fix<0>); \ + traits.updateRhs(&blB[(5 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C5, T0, fix<1>); \ + traits.updateRhs(&blB[(6 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C6, T0, fix<2>); \ + traits.updateRhs(&blB[(7 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C7, T0, fix<3>); \ + EIGEN_ASM_COMMENT("end step of gebp micro kernel 1pX8"); \ + } while (false) + + EIGEN_ASM_COMMENT("begin gebp micro kernel 1pX8"); + + EIGEN_GEBGP_ONESTEP(0); + EIGEN_GEBGP_ONESTEP(1); + EIGEN_GEBGP_ONESTEP(2); + EIGEN_GEBGP_ONESTEP(3); + EIGEN_GEBGP_ONESTEP(4); + EIGEN_GEBGP_ONESTEP(5); + EIGEN_GEBGP_ONESTEP(6); + EIGEN_GEBGP_ONESTEP(7); + + blB += pk*8*RhsProgress; + blA += pk*(1*LhsProgress); + + EIGEN_ASM_COMMENT("end gebp micro kernel 1pX8"); + } + // process remaining peeled loop + for(Index k=peeled_kc; k(alpha); + + R0 = r0.template loadPacket(0); + R1 = r1.template loadPacket(0); + traits.acc(C0, alphav, R0); + traits.acc(C1, alphav, R1); + r0.storePacket(0, R0); + r1.storePacket(0, R1); + + R0 = r2.template loadPacket(0); + R1 = r3.template loadPacket(0); + traits.acc(C2, alphav, R0); + traits.acc(C3, alphav, R1); + r2.storePacket(0, R0); + r3.storePacket(0, R1); + + R0 = r4.template loadPacket(0); + R1 = r5.template loadPacket(0); + traits.acc(C4, alphav, R0); + traits.acc(C5, alphav, R1); + r4.storePacket(0, R0); + r5.storePacket(0, R1); + + R0 = r6.template loadPacket(0); + R1 = r7.template loadPacket(0); + traits.acc(C6, alphav, R0); + traits.acc(C7, alphav, R1); + r6.storePacket(0, R0); + r7.storePacket(0, R1); + } + } +#endif + // loops on each largest micro vertical panel of rhs (depth * nr) - for(Index j2=0; j2 cj; Index packet_cols4 = nr>=4 ? (cols/4) * 4 : 0; + Index packet_cols8 = nr>=8 ? (cols/8) * 8 : 0; const Index peeled_mc3 = mr>=3*Traits::LhsProgress ? (rows/(3*LhsProgress))*(3*LhsProgress) : 0; const Index peeled_mc2 = mr>=2*Traits::LhsProgress ? peeled_mc3+((rows-peeled_mc3)/(2*LhsProgress))*(2*LhsProgress) : 0; const Index peeled_mc1 = mr>=1*Traits::LhsProgress ? peeled_mc2+((rows-peeled_mc2)/(1*LhsProgress))*(1*LhsProgress) : 0; @@ -1441,7 +1581,220 @@ void gebp_kernel=8) { + for(Index j2=0; j2); \ + traits.madd(A1, rhs_panel, C8, T0, fix<0>); \ + traits.madd(A2, rhs_panel, C16, T0, fix<0>); \ + traits.updateRhs(blB + (1 + 8 * K) * Traits::RhsProgress, rhs_panel); \ + traits.madd(A0, rhs_panel, C1, T0, fix<1>); \ + traits.madd(A1, rhs_panel, C9, T0, fix<1>); \ + traits.madd(A2, rhs_panel, C17, T0, fix<1>); \ + traits.updateRhs(blB + (2 + 8 * K) * Traits::RhsProgress, rhs_panel); \ + traits.madd(A0, rhs_panel, C2, T0, fix<2>); \ + traits.madd(A1, rhs_panel, C10, T0, fix<2>); \ + traits.madd(A2, rhs_panel, C18, T0, fix<2>); \ + traits.updateRhs(blB + (3 + 8 * K) * Traits::RhsProgress, rhs_panel); \ + traits.madd(A0, rhs_panel, C3, T0, fix<3>); \ + traits.madd(A1, rhs_panel, C11, T0, fix<3>); \ + traits.madd(A2, rhs_panel, C19, T0, fix<3>); \ + traits.loadRhs(blB + (4 + 8 * K) * Traits::RhsProgress, rhs_panel); \ + traits.madd(A0, rhs_panel, C4, T0, fix<0>); \ + traits.madd(A1, rhs_panel, C12, T0, fix<0>); \ + traits.madd(A2, rhs_panel, C20, T0, fix<0>); \ + traits.updateRhs(blB + (5 + 8 * K) * Traits::RhsProgress, rhs_panel); \ + traits.madd(A0, rhs_panel, C5, T0, fix<1>); \ + traits.madd(A1, rhs_panel, C13, T0, fix<1>); \ + traits.madd(A2, rhs_panel, C21, T0, fix<1>); \ + traits.updateRhs(blB + (6 + 8 * K) * Traits::RhsProgress, rhs_panel); \ + traits.madd(A0, rhs_panel, C6, T0, fix<2>); \ + traits.madd(A1, rhs_panel, C14, T0, fix<2>); \ + traits.madd(A2, rhs_panel, C22, T0, fix<2>); \ + traits.updateRhs(blB + (7 + 8 * K) * Traits::RhsProgress, rhs_panel); \ + traits.madd(A0, rhs_panel, C7, T0, fix<3>); \ + traits.madd(A1, rhs_panel, C15, T0, fix<3>); \ + traits.madd(A2, rhs_panel, C23, T0, fix<3>); \ + EIGEN_ASM_COMMENT("end step of gebp micro kernel 3pX8"); \ + } while (false) + + EIGEN_GEBP_ONESTEP(0); + EIGEN_GEBP_ONESTEP(1); + EIGEN_GEBP_ONESTEP(2); + EIGEN_GEBP_ONESTEP(3); + EIGEN_GEBP_ONESTEP(4); + EIGEN_GEBP_ONESTEP(5); + EIGEN_GEBP_ONESTEP(6); + EIGEN_GEBP_ONESTEP(7); + + blB += pk * 8 * RhsProgress; + blA += pk * 3 * Traits::LhsProgress; + EIGEN_ASM_COMMENT("end gebp micro kernel 3pX8"); + } + + // process remaining peeled loop + for (Index k = peeled_kc; k < depth; k++) + { + + RhsPanel27 rhs_panel; + RhsPacket T0; + LhsPacket A2; + EIGEN_GEBP_ONESTEP(0); + blB += 8 * RhsProgress; + blA += 3 * Traits::LhsProgress; + } + + #undef EIGEN_GEBP_ONESTEP + + ResPacket R0, R1, R2; + ResPacket alphav = pset1(alpha); + + R0 = r0.template loadPacket(0 * Traits::ResPacketSize); + R1 = r0.template loadPacket(1 * Traits::ResPacketSize); + R2 = r0.template loadPacket(2 * Traits::ResPacketSize); + traits.acc(C0, alphav, R0); + traits.acc(C8, alphav, R1); + traits.acc(C16, alphav, R2); + r0.storePacket(0 * Traits::ResPacketSize, R0); + r0.storePacket(1 * Traits::ResPacketSize, R1); + r0.storePacket(2 * Traits::ResPacketSize, R2); + + R0 = r1.template loadPacket(0 * Traits::ResPacketSize); + R1 = r1.template loadPacket(1 * Traits::ResPacketSize); + R2 = r1.template loadPacket(2 * Traits::ResPacketSize); + traits.acc(C1, alphav, R0); + traits.acc(C9, alphav, R1); + traits.acc(C17, alphav, R2); + r1.storePacket(0 * Traits::ResPacketSize, R0); + r1.storePacket(1 * Traits::ResPacketSize, R1); + r1.storePacket(2 * Traits::ResPacketSize, R2); + + R0 = r2.template loadPacket(0 * Traits::ResPacketSize); + R1 = r2.template loadPacket(1 * Traits::ResPacketSize); + R2 = r2.template loadPacket(2 * Traits::ResPacketSize); + traits.acc(C2, alphav, R0); + traits.acc(C10, alphav, R1); + traits.acc(C18, alphav, R2); + r2.storePacket(0 * Traits::ResPacketSize, R0); + r2.storePacket(1 * Traits::ResPacketSize, R1); + r2.storePacket(2 * Traits::ResPacketSize, R2); + + R0 = r3.template loadPacket(0 * Traits::ResPacketSize); + R1 = r3.template loadPacket(1 * Traits::ResPacketSize); + R2 = r3.template loadPacket(2 * Traits::ResPacketSize); + traits.acc(C3, alphav, R0); + traits.acc(C11, alphav, R1); + traits.acc(C19, alphav, R2); + r3.storePacket(0 * Traits::ResPacketSize, R0); + r3.storePacket(1 * Traits::ResPacketSize, R1); + r3.storePacket(2 * Traits::ResPacketSize, R2); + + R0 = r4.template loadPacket(0 * Traits::ResPacketSize); + R1 = r4.template loadPacket(1 * Traits::ResPacketSize); + R2 = r4.template loadPacket(2 * Traits::ResPacketSize); + traits.acc(C4, alphav, R0); + traits.acc(C12, alphav, R1); + traits.acc(C20, alphav, R2); + r4.storePacket(0 * Traits::ResPacketSize, R0); + r4.storePacket(1 * Traits::ResPacketSize, R1); + r4.storePacket(2 * Traits::ResPacketSize, R2); + + R0 = r5.template loadPacket(0 * Traits::ResPacketSize); + R1 = r5.template loadPacket(1 * Traits::ResPacketSize); + R2 = r5.template loadPacket(2 * Traits::ResPacketSize); + traits.acc(C5, alphav, R0); + traits.acc(C13, alphav, R1); + traits.acc(C21, alphav, R2); + r5.storePacket(0 * Traits::ResPacketSize, R0); + r5.storePacket(1 * Traits::ResPacketSize, R1); + r5.storePacket(2 * Traits::ResPacketSize, R2); + + R0 = r6.template loadPacket(0 * Traits::ResPacketSize); + R1 = r6.template loadPacket(1 * Traits::ResPacketSize); + R2 = r6.template loadPacket(2 * Traits::ResPacketSize); + traits.acc(C6, alphav, R0); + traits.acc(C14, alphav, R1); + traits.acc(C22, alphav, R2); + r6.storePacket(0 * Traits::ResPacketSize, R0); + r6.storePacket(1 * Traits::ResPacketSize, R1); + r6.storePacket(2 * Traits::ResPacketSize, R2); + + R0 = r7.template loadPacket(0 * Traits::ResPacketSize); + R1 = r7.template loadPacket(1 * Traits::ResPacketSize); + R2 = r7.template loadPacket(2 * Traits::ResPacketSize); + traits.acc(C7, alphav, R0); + traits.acc(C15, alphav, R1); + traits.acc(C23, alphav, R2); + r7.storePacket(0 * Traits::ResPacketSize, R0); + r7.storePacket(1 * Traits::ResPacketSize, R1); + r7.storePacket(2 * Traits::ResPacketSize, R2); + } + } + } +#endif + for(Index j2=packet_cols8; j2=8) { + for(Index j2=0; j2=6 without FMA (bug 1637) + #if EIGEN_GNUC_STRICT_AT_LEAST(6,0,0) && defined(EIGEN_VECTORIZE_SSE) + #define EIGEN_GEBP_2Px8_SPILLING_WORKAROUND __asm__ ("" : [a0] "+x,m" (A0),[a1] "+x,m" (A1)); + #else + #define EIGEN_GEBP_2Px8_SPILLING_WORKAROUND + #endif +#define EIGEN_GEBGP_ONESTEP(K) \ + do { \ + EIGEN_ASM_COMMENT("begin step of gebp micro kernel 2pX8"); \ + traits.loadLhs(&blA[(0 + 2 * K) * LhsProgress], A0); \ + traits.loadLhs(&blA[(1 + 2 * K) * LhsProgress], A1); \ + traits.loadRhs(&blB[(0 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C0, T0, fix<0>); \ + traits.madd(A1, rhs_panel, C8, T0, fix<0>); \ + traits.updateRhs(&blB[(1 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C1, T0, fix<1>); \ + traits.madd(A1, rhs_panel, C9, T0, fix<1>); \ + traits.updateRhs(&blB[(2 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C2, T0, fix<2>); \ + traits.madd(A1, rhs_panel, C10, T0, fix<2>); \ + traits.updateRhs(&blB[(3 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C3, T0, fix<3>); \ + traits.madd(A1, rhs_panel, C11, T0, fix<3>); \ + traits.loadRhs(&blB[(4 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C4, T0, fix<0>); \ + traits.madd(A1, rhs_panel, C12, T0, fix<0>); \ + traits.updateRhs(&blB[(5 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C5, T0, fix<1>); \ + traits.madd(A1, rhs_panel, C13, T0, fix<1>); \ + traits.updateRhs(&blB[(6 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C6, T0, fix<2>); \ + traits.madd(A1, rhs_panel, C14, T0, fix<2>); \ + traits.updateRhs(&blB[(7 + 8 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C7, T0, fix<3>); \ + traits.madd(A1, rhs_panel, C15, T0, fix<3>); \ + EIGEN_GEBP_2Px8_SPILLING_WORKAROUND \ + EIGEN_ASM_COMMENT("end step of gebp micro kernel 2pX8"); \ + } while (false) + + EIGEN_ASM_COMMENT("begin gebp micro kernel 2pX8"); + + EIGEN_GEBGP_ONESTEP(0); + EIGEN_GEBGP_ONESTEP(1); + EIGEN_GEBGP_ONESTEP(2); + EIGEN_GEBGP_ONESTEP(3); + EIGEN_GEBGP_ONESTEP(4); + EIGEN_GEBGP_ONESTEP(5); + EIGEN_GEBGP_ONESTEP(6); + EIGEN_GEBGP_ONESTEP(7); + + blB += pk*8*RhsProgress; + blA += pk*(2*Traits::LhsProgress); + + EIGEN_ASM_COMMENT("end gebp micro kernel 2pX8"); + } + // process remaining peeled loop + for(Index k=peeled_kc; k(alpha); + + R0 = r0.template loadPacket(0 * Traits::ResPacketSize); + R1 = r0.template loadPacket(1 * Traits::ResPacketSize); + R2 = r1.template loadPacket(0 * Traits::ResPacketSize); + R3 = r1.template loadPacket(1 * Traits::ResPacketSize); + traits.acc(C0, alphav, R0); + traits.acc(C8, alphav, R1); + traits.acc(C1, alphav, R2); + traits.acc(C9, alphav, R3); + r0.storePacket(0 * Traits::ResPacketSize, R0); + r0.storePacket(1 * Traits::ResPacketSize, R1); + r1.storePacket(0 * Traits::ResPacketSize, R2); + r1.storePacket(1 * Traits::ResPacketSize, R3); + + R0 = r2.template loadPacket(0 * Traits::ResPacketSize); + R1 = r2.template loadPacket(1 * Traits::ResPacketSize); + R2 = r3.template loadPacket(0 * Traits::ResPacketSize); + R3 = r3.template loadPacket(1 * Traits::ResPacketSize); + traits.acc(C2, alphav, R0); + traits.acc(C10, alphav, R1); + traits.acc(C3, alphav, R2); + traits.acc(C11, alphav, R3); + r2.storePacket(0 * Traits::ResPacketSize, R0); + r2.storePacket(1 * Traits::ResPacketSize, R1); + r3.storePacket(0 * Traits::ResPacketSize, R2); + r3.storePacket(1 * Traits::ResPacketSize, R3); + + R0 = r4.template loadPacket(0 * Traits::ResPacketSize); + R1 = r4.template loadPacket(1 * Traits::ResPacketSize); + R2 = r5.template loadPacket(0 * Traits::ResPacketSize); + R3 = r5.template loadPacket(1 * Traits::ResPacketSize); + traits.acc(C4, alphav, R0); + traits.acc(C12, alphav, R1); + traits.acc(C5, alphav, R2); + traits.acc(C13, alphav, R3); + r4.storePacket(0 * Traits::ResPacketSize, R0); + r4.storePacket(1 * Traits::ResPacketSize, R1); + r5.storePacket(0 * Traits::ResPacketSize, R2); + r5.storePacket(1 * Traits::ResPacketSize, R3); + + R0 = r6.template loadPacket(0 * Traits::ResPacketSize); + R1 = r6.template loadPacket(1 * Traits::ResPacketSize); + R2 = r7.template loadPacket(0 * Traits::ResPacketSize); + R3 = r7.template loadPacket(1 * Traits::ResPacketSize); + traits.acc(C6, alphav, R0); + traits.acc(C14, alphav, R1); + traits.acc(C7, alphav, R2); + traits.acc(C15, alphav, R3); + r6.storePacket(0 * Traits::ResPacketSize, R0); + r6.storePacket(1 * Traits::ResPacketSize, R1); + r7.storePacket(0 * Traits::ResPacketSize, R2); + r7.storePacket(1 * Traits::ResPacketSize, R3); + } + } + } +#endif + for(Index j2=packet_cols8; j2=6 without FMA (bug 1637) - #if EIGEN_GNUC_AT_LEAST(6,0) && defined(EIGEN_VECTORIZE_SSE) + #if EIGEN_GNUC_STRICT_AT_LEAST(6,0,0) && defined(EIGEN_VECTORIZE_SSE) && !(EIGEN_COMP_LCC) #define EIGEN_GEBP_2PX4_SPILLING_WORKAROUND __asm__ ("" : [a0] "+x,m" (A0),[a1] "+x,m" (A1)); #else #define EIGEN_GEBP_2PX4_SPILLING_WORKAROUND @@ -1904,22 +2423,84 @@ void gebp_kernel=8) { // loop on each panel of the rhs - for(Index j2=0; j2::half>::size; const int SResPacketQuarterSize = unpacket_traits::half>::half>::size; - if ((SwappedTraits::LhsProgress % 4) == 0 && + // The following code assumes we can load SRhsPacket in such a way that + // it multiplies blocks of 4 elements in SLhsPacket. This is not the + // case for some customized kernels (i.e. NEON fp16). If the assumption + // fails, drop down to the scalar path. + constexpr bool kCanLoadSRhsQuad = (unpacket_traits::size < 4) || (unpacket_traits::size % (unpacket_traits::size / 4)) == 0; + if (kCanLoadSRhsQuad && + (SwappedTraits::LhsProgress % 4) == 0 && (SwappedTraits::LhsProgress<=16) && (SwappedTraits::LhsProgress!=8 || SResPacketHalfSize==nr) && (SwappedTraits::LhsProgress!=16 || SResPacketQuarterSize==nr)) @@ -1974,10 +2555,10 @@ void gebp_kernel=8,typename unpacket_traits::half,SResPacket>::type SResPacketHalf; - typedef typename conditional=8,typename unpacket_traits::half,SLhsPacket>::type SLhsPacketHalf; - typedef typename conditional=8,typename unpacket_traits::half,SRhsPacket>::type SRhsPacketHalf; - typedef typename conditional=8,typename unpacket_traits::half,SAccPacket>::type SAccPacketHalf; + typedef std::conditional_t=8,typename unpacket_traits::half,SResPacket> SResPacketHalf; + typedef std::conditional_t=8,typename unpacket_traits::half,SLhsPacket> SLhsPacketHalf; + typedef std::conditional_t=8,typename unpacket_traits::half,SRhsPacket> SRhsPacketHalf; + typedef std::conditional_t=8,typename unpacket_traits::half,SAccPacket> SAccPacketHalf; SResPacketHalf R = res.template gatherPacket(i, j2); SResPacketHalf alphav = pset1(alpha); @@ -2269,8 +2850,8 @@ EIGEN_DONT_INLINE void gemm_pack_lhs0) { Index remaining_rows = rows-i; @@ -2290,21 +2871,21 @@ EIGEN_DONT_INLINE void gemm_pack_lhs kernel; - for (int p = 0; p < psize; ++p) kernel.packet[p] = lhs.template loadPacket(i+p+m, k); + for (Index p = 0; p < psize; ++p) kernel.packet[p] = lhs.template loadPacket(i+p+m, k); ptranspose(kernel); - for (int p = 0; p < psize; ++p) pstore(blockA+count+m+(pack)*p, cj.pconj(kernel.packet[p])); + for (Index p = 0; p < psize; ++p) pstore(blockA+count+m+(pack)*p, cj.pconj(kernel.packet[p])); } else if (HasHalf && psize == HalfPacketSize) { gone_half = true; PacketBlock kernel_half; - for (int p = 0; p < psize; ++p) kernel_half.packet[p] = lhs.template loadPacket(i+p+m, k); + for (Index p = 0; p < psize; ++p) kernel_half.packet[p] = lhs.template loadPacket(i+p+m, k); ptranspose(kernel_half); - for (int p = 0; p < psize; ++p) pstore(blockA+count+m+(pack)*p, cj.pconj(kernel_half.packet[p])); + for (Index p = 0; p < psize; ++p) pstore(blockA+count+m+(pack)*p, cj.pconj(kernel_half.packet[p])); } else if (HasQuarter && psize == QuarterPacketSize) { gone_quarter = true; PacketBlock kernel_quarter; - for (int p = 0; p < psize; ++p) kernel_quarter.packet[p] = lhs.template loadPacket(i+p+m, k); + for (Index p = 0; p < psize; ++p) kernel_quarter.packet[p] = lhs.template loadPacket(i+p+m, k); ptranspose(kernel_quarter); - for (int p = 0; p < psize; ++p) pstore(blockA+count+m+(pack)*p, cj.pconj(kernel_quarter.packet[p])); + for (Index p = 0; p < psize; ++p) pstore(blockA+count+m+(pack)*p, cj.pconj(kernel_quarter.packet[p])); } } count += psize*pack; @@ -2395,53 +2976,125 @@ EIGEN_DONT_INLINE void gemm_pack_rhs=4 ? (cols/4) * 4 : 0; Index count = 0; const Index peeled_k = (depth/PacketSize)*PacketSize; -// if(nr>=8) -// { -// for(Index j2=0; j2 kernel; -// for (int p = 0; p < PacketSize; ++p) { -// kernel.packet[p] = ploadu(&rhs[(j2+p)*rhsStride+k]); -// } -// ptranspose(kernel); -// for (int p = 0; p < PacketSize; ++p) { -// pstoreu(blockB+count, cj.pconj(kernel.packet[p])); -// count+=PacketSize; -// } -// } -// } -// for(; k=4) + +#if EIGEN_ARCH_ARM64 + EIGEN_IF_CONSTEXPR(nr>=8) + { + for(Index j2=0; j2 kernel0, kernel1, kernel2, kernel3; + kernel0.packet[0%PacketSize] = dm0.template loadPacket(k); + kernel0.packet[1%PacketSize] = dm1.template loadPacket(k); + kernel1.packet[0%PacketSize] = dm2.template loadPacket(k); + kernel1.packet[1%PacketSize] = dm3.template loadPacket(k); + kernel2.packet[0%PacketSize] = dm4.template loadPacket(k); + kernel2.packet[1%PacketSize] = dm5.template loadPacket(k); + kernel3.packet[0%PacketSize] = dm6.template loadPacket(k); + kernel3.packet[1%PacketSize] = dm7.template loadPacket(k); + ptranspose(kernel0); + ptranspose(kernel1); + ptranspose(kernel2); + ptranspose(kernel3); + + pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel0.packet[0 % PacketSize])); + pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel1.packet[0 % PacketSize])); + pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel2.packet[0 % PacketSize])); + pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel3.packet[0 % PacketSize])); + + pstoreu(blockB + count + 4 * PacketSize, cj.pconj(kernel0.packet[1 % PacketSize])); + pstoreu(blockB + count + 5 * PacketSize, cj.pconj(kernel1.packet[1 % PacketSize])); + pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel2.packet[1 % PacketSize])); + pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel3.packet[1 % PacketSize])); + count+=8*PacketSize; + } + else if (PacketSize == 4) + { + PacketBlock kernel0, kernel1; + + kernel0.packet[0%PacketSize] = dm0.template loadPacket(k); + kernel0.packet[1%PacketSize] = dm1.template loadPacket(k); + kernel0.packet[2%PacketSize] = dm2.template loadPacket(k); + kernel0.packet[3%PacketSize] = dm3.template loadPacket(k); + kernel1.packet[0%PacketSize] = dm4.template loadPacket(k); + kernel1.packet[1%PacketSize] = dm5.template loadPacket(k); + kernel1.packet[2%PacketSize] = dm6.template loadPacket(k); + kernel1.packet[3%PacketSize] = dm7.template loadPacket(k); + ptranspose(kernel0); + ptranspose(kernel1); + + pstoreu(blockB+count+0*PacketSize, cj.pconj(kernel0.packet[0%PacketSize])); + pstoreu(blockB+count+1*PacketSize, cj.pconj(kernel1.packet[0%PacketSize])); + pstoreu(blockB+count+2*PacketSize, cj.pconj(kernel0.packet[1%PacketSize])); + pstoreu(blockB+count+3*PacketSize, cj.pconj(kernel1.packet[1%PacketSize])); + pstoreu(blockB+count+4*PacketSize, cj.pconj(kernel0.packet[2%PacketSize])); + pstoreu(blockB+count+5*PacketSize, cj.pconj(kernel1.packet[2%PacketSize])); + pstoreu(blockB+count+6*PacketSize, cj.pconj(kernel0.packet[3%PacketSize])); + pstoreu(blockB+count+7*PacketSize, cj.pconj(kernel1.packet[3%PacketSize])); + count+=8*PacketSize; + } + else if (PacketSize == 8) + { + PacketBlock kernel0; + + kernel0.packet[0%PacketSize] = dm0.template loadPacket(k); + kernel0.packet[1%PacketSize] = dm1.template loadPacket(k); + kernel0.packet[2%PacketSize] = dm2.template loadPacket(k); + kernel0.packet[3%PacketSize] = dm3.template loadPacket(k); + kernel0.packet[4%PacketSize] = dm4.template loadPacket(k); + kernel0.packet[5%PacketSize] = dm5.template loadPacket(k); + kernel0.packet[6%PacketSize] = dm6.template loadPacket(k); + kernel0.packet[7%PacketSize] = dm7.template loadPacket(k); + ptranspose(kernel0); + + pstoreu(blockB+count+0*PacketSize, cj.pconj(kernel0.packet[0%PacketSize])); + pstoreu(blockB+count+1*PacketSize, cj.pconj(kernel0.packet[1%PacketSize])); + pstoreu(blockB+count+2*PacketSize, cj.pconj(kernel0.packet[2%PacketSize])); + pstoreu(blockB+count+3*PacketSize, cj.pconj(kernel0.packet[3%PacketSize])); + pstoreu(blockB+count+4*PacketSize, cj.pconj(kernel0.packet[4%PacketSize])); + pstoreu(blockB+count+5*PacketSize, cj.pconj(kernel0.packet[5%PacketSize])); + pstoreu(blockB+count+6*PacketSize, cj.pconj(kernel0.packet[6%PacketSize])); + pstoreu(blockB+count+7*PacketSize, cj.pconj(kernel0.packet[7%PacketSize])); + count+=8*PacketSize; + } + } + } + + for(; k=4) { for(Index j2=packet_cols8; j2=4 ? (cols/4) * 4 : 0; Index count = 0; - // if(nr>=8) - // { - // for(Index j2=0; j2(&rhs[k*rhsStride + j2]); - // pstoreu(blockB+count, cj.pconj(A)); - // } else if (PacketSize==4) { - // Packet A = ploadu(&rhs[k*rhsStride + j2]); - // Packet B = ploadu(&rhs[k*rhsStride + j2 + PacketSize]); - // pstoreu(blockB+count, cj.pconj(A)); - // pstoreu(blockB+count+PacketSize, cj.pconj(B)); - // } else { - // const Scalar* b0 = &rhs[k*rhsStride + j2]; - // blockB[count+0] = cj(b0[0]); - // blockB[count+1] = cj(b0[1]); - // blockB[count+2] = cj(b0[2]); - // blockB[count+3] = cj(b0[3]); - // blockB[count+4] = cj(b0[4]); - // blockB[count+5] = cj(b0[5]); - // blockB[count+6] = cj(b0[6]); - // blockB[count+7] = cj(b0[7]); - // } - // count += 8; - // } - // // skip what we have after - // if(PanelMode) count += 8 * (stride-offset-depth); - // } - // } +#if EIGEN_ARCH_ARM64 + EIGEN_IF_CONSTEXPR(nr>=8) + { + for(Index j2=0; j2(k, j2); + pstoreu(blockB+count, cj.pconj(A)); + count += PacketSize; + } else if (PacketSize==4) { + Packet A = rhs.template loadPacket(k, j2); + Packet B = rhs.template loadPacket(k, j2 + 4); + pstoreu(blockB+count, cj.pconj(A)); + pstoreu(blockB+count+PacketSize, cj.pconj(B)); + count += 2*PacketSize; + } else { + const LinearMapper dm0 = rhs.getLinearMapper(k, j2); + blockB[count+0] = cj(dm0(0)); + blockB[count+1] = cj(dm0(1)); + blockB[count+2] = cj(dm0(2)); + blockB[count+3] = cj(dm0(3)); + blockB[count+4] = cj(dm0(4)); + blockB[count+5] = cj(dm0(5)); + blockB[count+6] = cj(dm0(6)); + blockB[count+7] = cj(dm0(7)); + count += 8; + } + } + // skip what we have after + if(PanelMode) count += 8 * (stride-offset-depth); + } + } +#endif + if(nr>=4) { for(Index j2=packet_cols8; j2 class level3_blocking; +template class level3_blocking; /* Specialization for a row-major destination matrix => simple transposition of the product */ template< @@ -59,9 +62,9 @@ typedef gebp_traits Traits; typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; static void run(Index rows, Index cols, Index depth, - const LhsScalar* _lhs, Index lhsStride, - const RhsScalar* _rhs, Index rhsStride, - ResScalar* _res, Index resIncr, Index resStride, + const LhsScalar* lhs_, Index lhsStride, + const RhsScalar* rhs_, Index rhsStride, + ResScalar* res_, Index resIncr, Index resStride, ResScalar alpha, level3_blocking& blocking, GemmParallelInfo* info = 0) @@ -69,9 +72,9 @@ static void run(Index rows, Index cols, Index depth, typedef const_blas_data_mapper LhsMapper; typedef const_blas_data_mapper RhsMapper; typedef blas_data_mapper ResMapper; - LhsMapper lhs(_lhs, lhsStride); - RhsMapper rhs(_rhs, rhsStride); - ResMapper res(_res, resStride, resIncr); + LhsMapper lhs(lhs_, lhsStride); + RhsMapper rhs(rhs_, rhsStride); + ResMapper res(res_, resStride, resIncr); Index kc = blocking.kc(); // cache block size along the K direction Index mc = (std::min)(rows,blocking.mc()); // cache block size along the M direction @@ -148,9 +151,6 @@ static void run(Index rows, Index cols, Index depth, // Release all the sub blocks A'_i of A' for the current thread, // i.e., we simply decrement the number of users by 1 for(Index i=0; i class gemm_blocking_space; -template +template class level3_blocking { - typedef _LhsScalar LhsScalar; - typedef _RhsScalar RhsScalar; + typedef LhsScalar_ LhsScalar; + typedef RhsScalar_ RhsScalar; protected: LhsScalar* m_blockA; @@ -275,20 +275,19 @@ class level3_blocking inline RhsScalar* blockB() { return m_blockB; } }; -template -class gemm_blocking_space +template +class gemm_blocking_space : public level3_blocking< - typename conditional::type, - typename conditional::type> + std::conditional_t, + std::conditional_t> { enum { Transpose = StorageOrder==RowMajor, ActualRows = Transpose ? MaxCols : MaxRows, ActualCols = Transpose ? MaxRows : MaxCols }; - typedef typename conditional::type LhsScalar; - typedef typename conditional::type RhsScalar; - typedef gebp_traits Traits; + typedef std::conditional_t LhsScalar; + typedef std::conditional_t RhsScalar; enum { SizeA = ActualRows * MaxDepth, SizeB = ActualCols * MaxDepth @@ -313,8 +312,8 @@ class gemm_blocking_spacem_blockA = m_staticA; this->m_blockB = m_staticB; #else - this->m_blockA = reinterpret_cast((internal::UIntPtr(m_staticA) + (EIGEN_DEFAULT_ALIGN_BYTES-1)) & ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1)); - this->m_blockB = reinterpret_cast((internal::UIntPtr(m_staticB) + (EIGEN_DEFAULT_ALIGN_BYTES-1)) & ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1)); + this->m_blockA = reinterpret_cast((std::uintptr_t(m_staticA) + (EIGEN_DEFAULT_ALIGN_BYTES-1)) & ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1)); + this->m_blockB = reinterpret_cast((std::uintptr_t(m_staticB) + (EIGEN_DEFAULT_ALIGN_BYTES-1)) & ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1)); #endif } @@ -326,18 +325,17 @@ class gemm_blocking_space -class gemm_blocking_space +template +class gemm_blocking_space : public level3_blocking< - typename conditional::type, - typename conditional::type> + std::conditional_t, + std::conditional_t> { enum { Transpose = StorageOrder==RowMajor }; - typedef typename conditional::type LhsScalar; - typedef typename conditional::type RhsScalar; - typedef gebp_traits Traits; + typedef std::conditional_t LhsScalar; + typedef std::conditional_t RhsScalar; Index m_sizeA; Index m_sizeB; @@ -416,14 +414,14 @@ struct generic_product_impl typedef internal::blas_traits LhsBlasTraits; typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; - typedef typename internal::remove_all::type ActualLhsTypeCleaned; + typedef internal::remove_all_t ActualLhsTypeCleaned; typedef internal::blas_traits RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; - typedef typename internal::remove_all::type ActualRhsTypeCleaned; + typedef internal::remove_all_t ActualRhsTypeCleaned; enum { - MaxDepthAtCompileTime = EIGEN_SIZE_MIN_PREFER_FIXED(Lhs::MaxColsAtCompileTime,Rhs::MaxRowsAtCompileTime) + MaxDepthAtCompileTime = min_size_prefer_fixed(Lhs::MaxColsAtCompileTime, Rhs::MaxRowsAtCompileTime) }; typedef generic_product_impl lazyproduct; @@ -486,8 +484,8 @@ struct generic_product_impl ::scaleAndAddTo(dst_vec, a_lhs.row(0), a_rhs, alpha); } - typename internal::add_const_on_value_type::type lhs = LhsBlasTraits::extract(a_lhs); - typename internal::add_const_on_value_type::type rhs = RhsBlasTraits::extract(a_rhs); + add_const_on_value_type_t lhs = LhsBlasTraits::extract(a_lhs); + add_const_on_value_type_t rhs = RhsBlasTraits::extract(a_rhs); Scalar actualAlpha = combine_scalar_factors(alpha, a_lhs, a_rhs); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h index 6ba0d9bdb8d..55b637ba8e7 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h @@ -10,6 +10,9 @@ #ifndef EIGEN_GENERAL_MATRIX_MATRIX_TRIANGULAR_H #define EIGEN_GENERAL_MATRIX_MATRIX_TRIANGULAR_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { template @@ -60,9 +63,9 @@ template { typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; - static EIGEN_STRONG_INLINE void run(Index size, Index depth,const LhsScalar* _lhs, Index lhsStride, - const RhsScalar* _rhs, Index rhsStride, - ResScalar* _res, Index resIncr, Index resStride, + static EIGEN_STRONG_INLINE void run(Index size, Index depth,const LhsScalar* lhs_, Index lhsStride, + const RhsScalar* rhs_, Index rhsStride, + ResScalar* res_, Index resIncr, Index resStride, const ResScalar& alpha, level3_blocking& blocking) { typedef gebp_traits Traits; @@ -70,9 +73,9 @@ struct general_matrix_matrix_triangular_product LhsMapper; typedef const_blas_data_mapper RhsMapper; typedef blas_data_mapper ResMapper; - LhsMapper lhs(_lhs,lhsStride); - RhsMapper rhs(_rhs,rhsStride); - ResMapper res(_res, resStride, resIncr); + LhsMapper lhs(lhs_,lhsStride); + RhsMapper rhs(rhs_,rhsStride); + ResMapper res(res_, resStride, resIncr); Index kc = blocking.kc(); Index mc = (std::min)(size,blocking.mc()); @@ -113,7 +116,7 @@ struct general_matrix_matrix_triangular_product::ret + BlockSize = meta_least_common_multiple::ret }; - void operator()(ResScalar* _res, Index resIncr, Index resStride, const LhsScalar* blockA, const RhsScalar* blockB, Index size, Index depth, const ResScalar& alpha) + void operator()(ResScalar* res_, Index resIncr, Index resStride, const LhsScalar* blockA, const RhsScalar* blockB, Index size, Index depth, const ResScalar& alpha) { typedef blas_data_mapper ResMapper; typedef blas_data_mapper BufferMapper; - ResMapper res(_res, resStride, resIncr); + ResMapper res(res_, resStride, resIncr); gebp_kernel gebp_kernel1; gebp_kernel gebp_kernel2; @@ -208,17 +211,17 @@ struct general_product_to_triangular_selector { typedef typename MatrixType::Scalar Scalar; - typedef typename internal::remove_all::type Lhs; + typedef internal::remove_all_t Lhs; typedef internal::blas_traits LhsBlasTraits; typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhs; - typedef typename internal::remove_all::type _ActualLhs; - typename internal::add_const_on_value_type::type actualLhs = LhsBlasTraits::extract(prod.lhs()); + typedef internal::remove_all_t ActualLhs_; + internal::add_const_on_value_type_t actualLhs = LhsBlasTraits::extract(prod.lhs()); - typedef typename internal::remove_all::type Rhs; + typedef internal::remove_all_t Rhs; typedef internal::blas_traits RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhs; - typedef typename internal::remove_all::type _ActualRhs; - typename internal::add_const_on_value_type::type actualRhs = RhsBlasTraits::extract(prod.rhs()); + typedef internal::remove_all_t ActualRhs_; + internal::add_const_on_value_type_t actualRhs = RhsBlasTraits::extract(prod.rhs()); Scalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(prod.lhs().derived()) * RhsBlasTraits::extractScalarFactor(prod.rhs().derived()); @@ -227,19 +230,19 @@ struct general_product_to_triangular_selector enum { StorageOrder = (internal::traits::Flags&RowMajorBit) ? RowMajor : ColMajor, - UseLhsDirectly = _ActualLhs::InnerStrideAtCompileTime==1, - UseRhsDirectly = _ActualRhs::InnerStrideAtCompileTime==1 + UseLhsDirectly = ActualLhs_::InnerStrideAtCompileTime==1, + UseRhsDirectly = ActualRhs_::InnerStrideAtCompileTime==1 }; internal::gemv_static_vector_if static_lhs; ei_declare_aligned_stack_constructed_variable(Scalar, actualLhsPtr, actualLhs.size(), (UseLhsDirectly ? const_cast(actualLhs.data()) : static_lhs.data())); - if(!UseLhsDirectly) Map(actualLhsPtr, actualLhs.size()) = actualLhs; + if(!UseLhsDirectly) Map(actualLhsPtr, actualLhs.size()) = actualLhs; internal::gemv_static_vector_if static_rhs; ei_declare_aligned_stack_constructed_variable(Scalar, actualRhsPtr, actualRhs.size(), (UseRhsDirectly ? const_cast(actualRhs.data()) : static_rhs.data())); - if(!UseRhsDirectly) Map(actualRhsPtr, actualRhs.size()) = actualRhs; + if(!UseRhsDirectly) Map(actualRhsPtr, actualRhs.size()) = actualRhs; selfadjoint_rank1_update { static void run(MatrixType& mat, const ProductType& prod, const typename MatrixType::Scalar& alpha, bool beta) { - typedef typename internal::remove_all::type Lhs; + typedef internal::remove_all_t Lhs; typedef internal::blas_traits LhsBlasTraits; typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhs; - typedef typename internal::remove_all::type _ActualLhs; - typename internal::add_const_on_value_type::type actualLhs = LhsBlasTraits::extract(prod.lhs()); + typedef internal::remove_all_t ActualLhs_; + internal::add_const_on_value_type_t actualLhs = LhsBlasTraits::extract(prod.lhs()); - typedef typename internal::remove_all::type Rhs; + typedef internal::remove_all_t Rhs; typedef internal::blas_traits RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhs; - typedef typename internal::remove_all::type _ActualRhs; - typename internal::add_const_on_value_type::type actualRhs = RhsBlasTraits::extract(prod.rhs()); + typedef internal::remove_all_t ActualRhs_; + internal::add_const_on_value_type_t actualRhs = RhsBlasTraits::extract(prod.rhs()); typename ProductType::Scalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(prod.lhs().derived()) * RhsBlasTraits::extractScalarFactor(prod.rhs().derived()); @@ -273,8 +276,8 @@ struct general_product_to_triangular_selector enum { IsRowMajor = (internal::traits::Flags&RowMajorBit) ? 1 : 0, - LhsIsRowMajor = _ActualLhs::Flags&RowMajorBit ? 1 : 0, - RhsIsRowMajor = _ActualRhs::Flags&RowMajorBit ? 1 : 0, + LhsIsRowMajor = ActualLhs_::Flags&RowMajorBit ? 1 : 0, + RhsIsRowMajor = ActualRhs_::Flags&RowMajorBit ? 1 : 0, SkipDiag = (UpLo&(UnitDiag|ZeroDiag))!=0 }; @@ -284,7 +287,7 @@ struct general_product_to_triangular_selector Index depth = actualLhs.cols(); typedef internal::gemm_blocking_space BlockingType; + MatrixType::MaxColsAtCompileTime, MatrixType::MaxColsAtCompileTime, ActualRhs_::MaxColsAtCompileTime> BlockingType; BlockingType blocking(size, size, depth, 1, false); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h index 9a650ec23d5..8379f5b5784 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h @@ -33,6 +33,9 @@ #ifndef EIGEN_GENERAL_MATRIX_MATRIX_TRIANGULAR_BLAS_H #define EIGEN_GENERAL_MATRIX_MATRIX_TRIANGULAR_BLAS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h index 71abf4013d1..b40bcec3e5c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h @@ -33,6 +33,9 @@ #ifndef EIGEN_GENERAL_MATRIX_MATRIX_BLAS_H #define EIGEN_GENERAL_MATRIX_MATRIX_BLAS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixVector.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixVector.h index dfb6aebced4..475ac85e255 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixVector.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/GeneralMatrixVector.h @@ -10,6 +10,9 @@ #ifndef EIGEN_GENERAL_MATRIX_VECTOR_H #define EIGEN_GENERAL_MATRIX_VECTOR_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -29,42 +32,42 @@ struct gemv_packet_cond { typedef T1 type; }; template struct gemv_packet_cond { typedef T2 type; }; -template +template class gemv_traits { typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; -#define PACKET_DECL_COND_PREFIX(prefix, name, packet_size) \ +#define PACKET_DECL_COND_POSTFIX(postfix, name, packet_size) \ typedef typename gemv_packet_cond::type, \ typename packet_traits::half, \ typename unpacket_traits::half>::half>::type \ - prefix ## name ## Packet + name ## Packet ## postfix - PACKET_DECL_COND_PREFIX(_, Lhs, _PacketSize); - PACKET_DECL_COND_PREFIX(_, Rhs, _PacketSize); - PACKET_DECL_COND_PREFIX(_, Res, _PacketSize); -#undef PACKET_DECL_COND_PREFIX + PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_); + PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_); + PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_); +#undef PACKET_DECL_COND_POSTFIX public: enum { - Vectorizable = unpacket_traits<_LhsPacket>::vectorizable && - unpacket_traits<_RhsPacket>::vectorizable && - int(unpacket_traits<_LhsPacket>::size)==int(unpacket_traits<_RhsPacket>::size), - LhsPacketSize = Vectorizable ? unpacket_traits<_LhsPacket>::size : 1, - RhsPacketSize = Vectorizable ? unpacket_traits<_RhsPacket>::size : 1, - ResPacketSize = Vectorizable ? unpacket_traits<_ResPacket>::size : 1 + Vectorizable = unpacket_traits::vectorizable && + unpacket_traits::vectorizable && + int(unpacket_traits::size)==int(unpacket_traits::size), + LhsPacketSize = Vectorizable ? unpacket_traits::size : 1, + RhsPacketSize = Vectorizable ? unpacket_traits::size : 1, + ResPacketSize = Vectorizable ? unpacket_traits::size : 1 }; - typedef typename conditional::type LhsPacket; - typedef typename conditional::type RhsPacket; - typedef typename conditional::type ResPacket; + typedef std::conditional_t LhsPacket; + typedef std::conditional_t RhsPacket; + typedef std::conditional_t ResPacket; }; /* Optimized col-major matrix * vector product: * This algorithm processes the matrix per vertical panels, - * which are then processed horizontaly per chunck of 8*PacketSize x 1 vertical segments. + * which are then processed horizontally per chunck of 8*PacketSize x 1 vertical segments. * * Mixing type logic: C += alpha * A * B * | A | B |alpha| comments @@ -359,6 +362,10 @@ EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void general_matrix_vector_product(ResScalar(0)), c7 = pset1(ResScalar(0)); - Index j=0; - for(; j+LhsPacketSize<=cols; j+=LhsPacketSize) + for (Index j = 0; j < fullColBlockEnd; j += LhsPacketSize) { RhsPacket b0 = rhs.template load(j,0); @@ -393,7 +399,8 @@ EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void general_matrix_vector_product(ResScalar(0)), c3 = pset1(ResScalar(0)); - Index j=0; - for(; j+LhsPacketSize<=cols; j+=LhsPacketSize) + for (Index j = 0; j < fullColBlockEnd; j += LhsPacketSize) { RhsPacket b0 = rhs.template load(j,0); @@ -436,7 +442,8 @@ EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void general_matrix_vector_product(ResScalar(0)), c1 = pset1(ResScalar(0)); - Index j=0; - for(; j+LhsPacketSize<=cols; j+=LhsPacketSize) + for (Index j = 0; j < fullColBlockEnd; j += LhsPacketSize) { RhsPacket b0 = rhs.template load(j,0); @@ -465,7 +471,8 @@ EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void general_matrix_vector_product(ResScalar(0)); ResPacketHalf c0_h = pset1(ResScalar(0)); ResPacketQuarter c0_q = pset1(ResScalar(0)); - Index j=0; - for(; j+LhsPacketSize<=cols; j+=LhsPacketSize) + + for (Index j = 0; j < fullColBlockEnd; j += LhsPacketSize) { RhsPacket b0 = rhs.template load(j,0); c0 = pcj.pmadd(lhs.template load(i,j),b0,c0); } ResScalar cc0 = predux(c0); if (HasHalf) { - for(; j+LhsPacketSizeHalf<=cols; j+=LhsPacketSizeHalf) + for (Index j = fullColBlockEnd; j < halfColBlockEnd; j += LhsPacketSizeHalf) { RhsPacketHalf b0 = rhs.template load(j,0); c0_h = pcj_half.pmadd(lhs.template load(i,j),b0,c0_h); @@ -496,14 +503,14 @@ EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void general_matrix_vector_product(j,0); c0_q = pcj_quarter.pmadd(lhs.template load(i,j),b0,c0_q); } cc0 += predux(c0_q); } - for(; j -#endif +// IWYU pragma: private +#include "../InternalHeaderCheck.h" namespace Eigen { @@ -78,18 +77,13 @@ namespace internal { template struct GemmParallelInfo { - GemmParallelInfo() : sync(-1), users(0), lhs_start(0), lhs_length(0) {} - // volatile is not enough on all architectures (see bug 1572) - // to guarantee that when thread A says to thread B that it is - // done with packing a block, then all writes have been really - // carried out... C++11 memory model+atomic guarantees this. -#if EIGEN_HAS_CXX11_ATOMIC +#ifdef EIGEN_HAS_OPENMP + GemmParallelInfo() : sync(-1), users(0), lhs_start(0), lhs_length(0) {} std::atomic sync; std::atomic users; #else - Index volatile sync; - int volatile users; + GemmParallelInfo() : lhs_start(0), lhs_length(0) {} #endif Index lhs_start; @@ -104,7 +98,7 @@ void parallelize_gemm(const Functor& func, Index rows, Index cols, Index depth, // Without C++11, we have to disable GEMM's parallelization on // non x86 architectures because there volatile is not enough for our purpose. // See bug 1572. -#if (! defined(EIGEN_HAS_OPENMP)) || defined(EIGEN_USE_BLAS) || ((!EIGEN_HAS_CXX11_ATOMIC) && !(EIGEN_ARCH_i386_OR_x86_64)) +#if (! defined(EIGEN_HAS_OPENMP)) || defined(EIGEN_USE_BLAS) // FIXME the transpose variable is only needed to properly split // the matrix product when multithreading is enabled. This is a temporary // fix to support row-major destination matrices. This whole diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixMatrix.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixMatrix.h index 33ecf10f610..8133880ca15 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixMatrix.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixMatrix.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SELFADJOINT_MATRIX_MATRIX_H #define EIGEN_SELFADJOINT_MATRIX_MATRIX_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -43,7 +46,7 @@ struct symm_pack_lhs for(Index w=0; w::type>::half HalfPacket; typedef typename unpacket_traits::type>::half>::half QuarterPacket; @@ -53,7 +56,7 @@ struct symm_pack_lhs HasHalf = (int)HalfPacketSize < (int)PacketSize, HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize}; - const_blas_data_mapper lhs(_lhs,lhsStride); + const_blas_data_mapper lhs(lhs_,lhsStride); Index count = 0; //Index peeled_mc3 = (rows/Pack1)*Pack1; @@ -101,11 +104,11 @@ template struct symm_pack_rhs { enum { PacketSize = packet_traits::size }; - void operator()(Scalar* blockB, const Scalar* _rhs, Index rhsStride, Index rows, Index cols, Index k2) + void operator()(Scalar* blockB, const Scalar* rhs_, Index rhsStride, Index rows, Index cols, Index k2) { Index end_k = k2 + rows; Index count = 0; - const_blas_data_mapper rhs(_rhs,rhsStride); + const_blas_data_mapper rhs(rhs_,rhsStride); Index packet_cols8 = nr>=8 ? (cols/8) * 8 : 0; Index packet_cols4 = nr>=4 ? (cols/4) * 4 : 0; @@ -312,10 +315,10 @@ struct product_selfadjoint_matrix& blocking) { product_selfadjoint_matrix::IsComplex && EIGEN_LOGICAL_XOR(RhsSelfAdjoint,ConjugateRhs), - EIGEN_LOGICAL_XOR(LhsSelfAdjoint,LhsStorageOrder==RowMajor) ? ColMajor : RowMajor, - LhsSelfAdjoint, NumTraits::IsComplex && EIGEN_LOGICAL_XOR(LhsSelfAdjoint,ConjugateLhs), + logical_xor(RhsSelfAdjoint,RhsStorageOrder==RowMajor) ? ColMajor : RowMajor, + RhsSelfAdjoint, NumTraits::IsComplex && logical_xor(RhsSelfAdjoint, ConjugateRhs), + logical_xor(LhsSelfAdjoint,LhsStorageOrder==RowMajor) ? ColMajor : RowMajor, + LhsSelfAdjoint, NumTraits::IsComplex && logical_xor(LhsSelfAdjoint, ConjugateLhs), ColMajor,ResInnerStride> ::run(cols, rows, rhs, rhsStride, lhs, lhsStride, res, resIncr, resStride, alpha, blocking); } @@ -330,8 +333,8 @@ struct product_selfadjoint_matrix& blocking); }; @@ -342,9 +345,9 @@ template EIGEN_DONT_INLINE void product_selfadjoint_matrix::run( Index rows, Index cols, - const Scalar* _lhs, Index lhsStride, - const Scalar* _rhs, Index rhsStride, - Scalar* _res, Index resIncr, Index resStride, + const Scalar* lhs_, Index lhsStride, + const Scalar* rhs_, Index rhsStride, + Scalar* res_, Index resIncr, Index resStride, const Scalar& alpha, level3_blocking& blocking) { Index size = rows; @@ -355,10 +358,10 @@ EIGEN_DONT_INLINE void product_selfadjoint_matrix LhsTransposeMapper; typedef const_blas_data_mapper RhsMapper; typedef blas_data_mapper ResMapper; - LhsMapper lhs(_lhs,lhsStride); - LhsTransposeMapper lhs_transpose(_lhs,lhsStride); - RhsMapper rhs(_rhs,rhsStride); - ResMapper res(_res, resStride, resIncr); + LhsMapper lhs(lhs_,lhsStride); + LhsTransposeMapper lhs_transpose(lhs_,lhsStride); + RhsMapper rhs(rhs_,rhsStride); + ResMapper res(res_, resStride, resIncr); Index kc = blocking.kc(); // cache block size along the K direction Index mc = (std::min)(rows,blocking.mc()); // cache block size along the M direction @@ -425,8 +428,8 @@ struct product_selfadjoint_matrix& blocking); }; @@ -437,9 +440,9 @@ template EIGEN_DONT_INLINE void product_selfadjoint_matrix::run( Index rows, Index cols, - const Scalar* _lhs, Index lhsStride, - const Scalar* _rhs, Index rhsStride, - Scalar* _res, Index resIncr, Index resStride, + const Scalar* lhs_, Index lhsStride, + const Scalar* rhs_, Index rhsStride, + Scalar* res_, Index resIncr, Index resStride, const Scalar& alpha, level3_blocking& blocking) { Index size = cols; @@ -448,8 +451,8 @@ EIGEN_DONT_INLINE void product_selfadjoint_matrix LhsMapper; typedef blas_data_mapper ResMapper; - LhsMapper lhs(_lhs,lhsStride); - ResMapper res(_res,resStride, resIncr); + LhsMapper lhs(lhs_,lhsStride); + ResMapper res(res_,resStride, resIncr); Index kc = blocking.kc(); // cache block size along the K direction Index mc = (std::min)(rows,blocking.mc()); // cache block size along the M direction @@ -466,7 +469,7 @@ EIGEN_DONT_INLINE void product_selfadjoint_matrix GEPP for(Index i2=0; i2 { eigen_assert(dst.rows()==a_lhs.rows() && dst.cols()==a_rhs.cols()); - typename internal::add_const_on_value_type::type lhs = LhsBlasTraits::extract(a_lhs); - typename internal::add_const_on_value_type::type rhs = RhsBlasTraits::extract(a_rhs); + add_const_on_value_type_t lhs = LhsBlasTraits::extract(a_lhs); + add_const_on_value_type_t rhs = RhsBlasTraits::extract(a_rhs); Scalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(a_lhs) * RhsBlasTraits::extractScalarFactor(a_rhs); @@ -521,10 +524,10 @@ struct selfadjoint_product_impl BlockingType blocking(lhs.rows(), rhs.cols(), lhs.cols(), 1, false); internal::product_selfadjoint_matrix::Flags &RowMajorBit) ? RowMajor : ColMajor, LhsIsSelfAdjoint, - NumTraits::IsComplex && EIGEN_LOGICAL_XOR(LhsIsUpper,bool(LhsBlasTraits::NeedToConjugate)), - EIGEN_LOGICAL_XOR(RhsIsUpper,internal::traits::Flags &RowMajorBit) ? RowMajor : ColMajor, RhsIsSelfAdjoint, - NumTraits::IsComplex && EIGEN_LOGICAL_XOR(RhsIsUpper,bool(RhsBlasTraits::NeedToConjugate)), + internal::logical_xor(LhsIsUpper, internal::traits::Flags &RowMajorBit) ? RowMajor : ColMajor, LhsIsSelfAdjoint, + NumTraits::IsComplex && internal::logical_xor(LhsIsUpper, bool(LhsBlasTraits::NeedToConjugate)), + internal::logical_xor(RhsIsUpper, internal::traits::Flags &RowMajorBit) ? RowMajor : ColMajor, RhsIsSelfAdjoint, + NumTraits::IsComplex && internal::logical_xor(RhsIsUpper, bool(RhsBlasTraits::NeedToConjugate)), internal::traits::Flags&RowMajorBit ? RowMajor : ColMajor, Dest::InnerStrideAtCompileTime> ::run( diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixMatrix_BLAS.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixMatrix_BLAS.h index 61396dbdf6f..9aa56a208d7 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixMatrix_BLAS.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixMatrix_BLAS.h @@ -33,6 +33,9 @@ #ifndef EIGEN_SELFADJOINT_MATRIX_MATRIX_BLAS_H #define EIGEN_SELFADJOINT_MATRIX_MATRIX_BLAS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixVector.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixVector.h index d38fd72b227..0aac52e7f76 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixVector.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixVector.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SELFADJOINT_MATRIX_VECTOR_H #define EIGEN_SELFADJOINT_MATRIX_VECTOR_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -55,12 +58,12 @@ void selfadjoint_matrix_vector_product::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, IsRowMajor), ConjugateRhs> cj0; - conj_helper::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> cj1; + conj_helper::IsComplex && logical_xor(ConjugateLhs, IsRowMajor), ConjugateRhs> cj0; + conj_helper::IsComplex && logical_xor(ConjugateLhs, !IsRowMajor), ConjugateRhs> cj1; conj_helper cjd; - conj_helper::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, IsRowMajor), ConjugateRhs> pcj0; - conj_helper::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> pcj1; + conj_helper::IsComplex && logical_xor(ConjugateLhs, IsRowMajor), ConjugateRhs> pcj0; + conj_helper::IsComplex && logical_xor(ConjugateLhs, !IsRowMajor), ConjugateRhs> pcj1; Scalar cjAlpha = ConjugateRhs ? numext::conj(alpha) : alpha; @@ -167,11 +170,11 @@ struct selfadjoint_product_impl typedef internal::blas_traits LhsBlasTraits; typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; - typedef typename internal::remove_all::type ActualLhsTypeCleaned; + typedef internal::remove_all_t ActualLhsTypeCleaned; typedef internal::blas_traits RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; - typedef typename internal::remove_all::type ActualRhsTypeCleaned; + typedef internal::remove_all_t ActualRhsTypeCleaned; enum { LhsUpLo = LhsMode&(Upper|Lower) }; @@ -181,12 +184,12 @@ struct selfadjoint_product_impl { typedef typename Dest::Scalar ResScalar; typedef typename Rhs::Scalar RhsScalar; - typedef Map, EIGEN_PLAIN_ENUM_MIN(AlignedMax,internal::packet_traits::size)> MappedDest; + typedef Map, plain_enum_min(AlignedMax,internal::packet_traits::size)> MappedDest; eigen_assert(dest.rows()==a_lhs.rows() && dest.cols()==a_rhs.cols()); - typename internal::add_const_on_value_type::type lhs = LhsBlasTraits::extract(a_lhs); - typename internal::add_const_on_value_type::type rhs = RhsBlasTraits::extract(a_rhs); + add_const_on_value_type_t lhs = LhsBlasTraits::extract(a_lhs); + add_const_on_value_type_t rhs = RhsBlasTraits::extract(a_rhs); Scalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(a_lhs) * RhsBlasTraits::extractScalarFactor(a_rhs); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h index 1238345e3fa..177ea091540 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h @@ -33,6 +33,9 @@ #ifndef EIGEN_SELFADJOINT_MATRIX_VECTOR_BLAS_H #define EIGEN_SELFADJOINT_MATRIX_VECTOR_BLAS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointProduct.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointProduct.h index a21be80504f..a208a1542d0 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointProduct.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointProduct.h @@ -16,6 +16,9 @@ * It corresponds to the level 3 SYRK and level 2 SYR Blas routines. **********************************************************************/ +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { @@ -26,7 +29,7 @@ struct selfadjoint_rank1_update { internal::conj_if cj; typedef Map > OtherMap; - typedef typename internal::conditional::type ConjLhsType; + typedef std::conditional_t ConjLhsType; for (Index i=0; i >(mat+stride*i+(UpLo==Lower ? i : 0), (UpLo==Lower ? size-i : (i+1))) @@ -55,14 +58,14 @@ struct selfadjoint_product_selector typedef typename MatrixType::Scalar Scalar; typedef internal::blas_traits OtherBlasTraits; typedef typename OtherBlasTraits::DirectLinearAccessType ActualOtherType; - typedef typename internal::remove_all::type _ActualOtherType; - typename internal::add_const_on_value_type::type actualOther = OtherBlasTraits::extract(other.derived()); + typedef internal::remove_all_t ActualOtherType_; + internal::add_const_on_value_type_t actualOther = OtherBlasTraits::extract(other.derived()); Scalar actualAlpha = alpha * OtherBlasTraits::extractScalarFactor(other.derived()); enum { StorageOrder = (internal::traits::Flags&RowMajorBit) ? RowMajor : ColMajor, - UseOtherDirectly = _ActualOtherType::InnerStrideAtCompileTime==1 + UseOtherDirectly = ActualOtherType_::InnerStrideAtCompileTime==1 }; internal::gemv_static_vector_if static_other; @@ -70,7 +73,7 @@ struct selfadjoint_product_selector (UseOtherDirectly ? const_cast(actualOther.data()) : static_other.data())); if(!UseOtherDirectly) - Map(actualOtherPtr, actualOther.size()) = actualOther; + Map(actualOtherPtr, actualOther.size()) = actualOther; selfadjoint_rank1_update::IsComplex, @@ -87,21 +90,21 @@ struct selfadjoint_product_selector typedef typename MatrixType::Scalar Scalar; typedef internal::blas_traits OtherBlasTraits; typedef typename OtherBlasTraits::DirectLinearAccessType ActualOtherType; - typedef typename internal::remove_all::type _ActualOtherType; - typename internal::add_const_on_value_type::type actualOther = OtherBlasTraits::extract(other.derived()); + typedef internal::remove_all_t ActualOtherType_; + internal::add_const_on_value_type_t actualOther = OtherBlasTraits::extract(other.derived()); Scalar actualAlpha = alpha * OtherBlasTraits::extractScalarFactor(other.derived()); enum { IsRowMajor = (internal::traits::Flags&RowMajorBit) ? 1 : 0, - OtherIsRowMajor = _ActualOtherType::Flags&RowMajorBit ? 1 : 0 + OtherIsRowMajor = ActualOtherType_::Flags&RowMajorBit ? 1 : 0 }; Index size = mat.cols(); Index depth = actualOther.cols(); typedef internal::gemm_blocking_space BlockingType; + MatrixType::MaxColsAtCompileTime, MatrixType::MaxColsAtCompileTime, ActualOtherType_::MaxColsAtCompileTime> BlockingType; BlockingType blocking(size, size, depth, 1, false); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointRank2Update.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointRank2Update.h index f752a0bf093..f433a21fff2 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointRank2Update.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/SelfadjointRank2Update.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SELFADJOINTRANK2UPTADE_H #define EIGEN_SELFADJOINTRANK2UPTADE_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -50,9 +53,8 @@ struct selfadjoint_rank2_update_selector } }; -template struct conj_expr_if - : conditional::Scalar>,T> > {}; +template +using conj_expr_if = std::conditional::Scalar>,T>>; } // end namespace internal @@ -63,13 +65,13 @@ ::rankUpdate(const MatrixBase& u, const MatrixBase& v, const { typedef internal::blas_traits UBlasTraits; typedef typename UBlasTraits::DirectLinearAccessType ActualUType; - typedef typename internal::remove_all::type _ActualUType; - typename internal::add_const_on_value_type::type actualU = UBlasTraits::extract(u.derived()); + typedef internal::remove_all_t ActualUType_; + internal::add_const_on_value_type_t actualU = UBlasTraits::extract(u.derived()); typedef internal::blas_traits VBlasTraits; typedef typename VBlasTraits::DirectLinearAccessType ActualVType; - typedef typename internal::remove_all::type _ActualVType; - typename internal::add_const_on_value_type::type actualV = VBlasTraits::extract(v.derived()); + typedef internal::remove_all_t ActualVType_; + internal::add_const_on_value_type_t actualV = VBlasTraits::extract(v.derived()); // If MatrixType is row major, then we use the routine for lower triangular in the upper triangular case and // vice versa, and take the complex conjugate of all coefficients and vector entries. @@ -80,8 +82,8 @@ ::rankUpdate(const MatrixBase& u, const MatrixBase& v, const if (IsRowMajor) actualAlpha = numext::conj(actualAlpha); - typedef typename internal::remove_all::type>::type UType; - typedef typename internal::remove_all::type>::type VType; + typedef internal::remove_all_t::type> UType; + typedef internal::remove_all_t::type> VType; internal::selfadjoint_rank2_update_selector ::run(_expression().const_cast_derived().data(),_expression().outerStride(),UType(actualU),VType(actualV),actualAlpha); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixMatrix.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixMatrix.h index f0c60507ab1..22e9375571d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixMatrix.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixMatrix.h @@ -10,6 +10,9 @@ #ifndef EIGEN_TRIANGULAR_MATRIX_MATRIX_H #define EIGEN_TRIANGULAR_MATRIX_MATRIX_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -18,10 +21,10 @@ namespace internal { // struct gemm_pack_lhs_triangular // { // Matrix::IsComplex && Conjugate> cj; -// const_blas_data_mapper lhs(_lhs,lhsStride); +// const_blas_data_mapper lhs(lhs_,lhsStride); // int count = 0; // const int peeled_mc = (rows/mr)*mr; // for(int i=0; i Traits; enum { - SmallPanelWidth = 2 * EIGEN_PLAIN_ENUM_MAX(Traits::mr,Traits::nr), + SmallPanelWidth = 2 * plain_enum_max(Traits::mr, Traits::nr), IsLower = (Mode&Lower) == Lower, SetDiag = (Mode&(ZeroDiag|UnitDiag)) ? 0 : 1 }; static EIGEN_DONT_INLINE void run( Index _rows, Index _cols, Index _depth, - const Scalar* _lhs, Index lhsStride, - const Scalar* _rhs, Index rhsStride, + const Scalar* lhs_, Index lhsStride, + const Scalar* rhs_, Index rhsStride, Scalar* res, Index resIncr, Index resStride, const Scalar& alpha, level3_blocking& blocking); }; @@ -110,9 +113,9 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix::run( Index _rows, Index _cols, Index _depth, - const Scalar* _lhs, Index lhsStride, - const Scalar* _rhs, Index rhsStride, - Scalar* _res, Index resIncr, Index resStride, + const Scalar* lhs_, Index lhsStride, + const Scalar* rhs_, Index rhsStride, + Scalar* res_, Index resIncr, Index resStride, const Scalar& alpha, level3_blocking& blocking) { // strip zeros @@ -124,9 +127,9 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix LhsMapper; typedef const_blas_data_mapper RhsMapper; typedef blas_data_mapper ResMapper; - LhsMapper lhs(_lhs,lhsStride); - RhsMapper rhs(_rhs,rhsStride); - ResMapper res(_res, resStride, resIncr); + LhsMapper lhs(lhs_,lhsStride); + RhsMapper rhs(rhs_,rhsStride); + ResMapper res(res_, resStride, resIncr); Index kc = blocking.kc(); // cache block size along the K direction Index mc = (std::min)(rows,blocking.mc()); // cache block size along the M direction @@ -247,15 +250,15 @@ struct product_triangular_matrix_matrix Traits; enum { - SmallPanelWidth = EIGEN_PLAIN_ENUM_MAX(Traits::mr,Traits::nr), + SmallPanelWidth = plain_enum_max(Traits::mr, Traits::nr), IsLower = (Mode&Lower) == Lower, SetDiag = (Mode&(ZeroDiag|UnitDiag)) ? 0 : 1 }; static EIGEN_DONT_INLINE void run( Index _rows, Index _cols, Index _depth, - const Scalar* _lhs, Index lhsStride, - const Scalar* _rhs, Index rhsStride, + const Scalar* lhs_, Index lhsStride, + const Scalar* rhs_, Index rhsStride, Scalar* res, Index resIncr, Index resStride, const Scalar& alpha, level3_blocking& blocking); }; @@ -268,9 +271,9 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix::run( Index _rows, Index _cols, Index _depth, - const Scalar* _lhs, Index lhsStride, - const Scalar* _rhs, Index rhsStride, - Scalar* _res, Index resIncr, Index resStride, + const Scalar* lhs_, Index lhsStride, + const Scalar* rhs_, Index rhsStride, + Scalar* res_, Index resIncr, Index resStride, const Scalar& alpha, level3_blocking& blocking) { const Index PacketBytes = packet_traits::size*sizeof(Scalar); @@ -283,9 +286,9 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix LhsMapper; typedef const_blas_data_mapper RhsMapper; typedef blas_data_mapper ResMapper; - LhsMapper lhs(_lhs,lhsStride); - RhsMapper rhs(_rhs,rhsStride); - ResMapper res(_res, resStride, resIncr); + LhsMapper lhs(lhs_,lhsStride); + RhsMapper rhs(rhs_,rhsStride); + ResMapper res(res_, resStride, resIncr); Index kc = blocking.kc(); // cache block size along the K direction Index mc = (std::min)(rows,blocking.mc()); // cache block size along the M direction @@ -412,13 +415,19 @@ struct triangular_product_impl typedef internal::blas_traits LhsBlasTraits; typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; - typedef typename internal::remove_all::type ActualLhsTypeCleaned; + typedef internal::remove_all_t ActualLhsTypeCleaned; typedef internal::blas_traits RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; - typedef typename internal::remove_all::type ActualRhsTypeCleaned; - - typename internal::add_const_on_value_type::type lhs = LhsBlasTraits::extract(a_lhs); - typename internal::add_const_on_value_type::type rhs = RhsBlasTraits::extract(a_rhs); + typedef internal::remove_all_t ActualRhsTypeCleaned; + + internal::add_const_on_value_type_t lhs = LhsBlasTraits::extract(a_lhs); + internal::add_const_on_value_type_t rhs = RhsBlasTraits::extract(a_rhs); + + // Empty product, return early. Otherwise, we get `nullptr` use errors below when we try to access + // coeffRef(0,0). + if (lhs.size() == 0 || rhs.size() == 0) { + return; + } LhsScalar lhs_alpha = LhsBlasTraits::extractScalarFactor(a_lhs); RhsScalar rhs_alpha = RhsBlasTraits::extractScalarFactor(a_rhs); @@ -451,12 +460,12 @@ struct triangular_product_impl // Apply correction if the diagonal is unit and a scalar factor was nested: if ((Mode&UnitDiag)==UnitDiag) { - if (LhsIsTriangular && lhs_alpha!=LhsScalar(1)) + if (LhsIsTriangular && !numext::is_exactly_one(lhs_alpha)) { Index diagSize = (std::min)(lhs.rows(),lhs.cols()); dst.topRows(diagSize) -= ((lhs_alpha-LhsScalar(1))*a_rhs).topRows(diagSize); } - else if ((!LhsIsTriangular) && rhs_alpha!=RhsScalar(1)) + else if ((!LhsIsTriangular) && !numext::is_exactly_one(rhs_alpha)) { Index diagSize = (std::min)(rhs.rows(),rhs.cols()); dst.leftCols(diagSize) -= (rhs_alpha-RhsScalar(1))*a_lhs.leftCols(diagSize); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixMatrix_BLAS.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixMatrix_BLAS.h index a98d12e4ae1..c556a24f068 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixMatrix_BLAS.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixMatrix_BLAS.h @@ -33,6 +33,9 @@ #ifndef EIGEN_TRIANGULAR_MATRIX_MATRIX_BLAS_H #define EIGEN_TRIANGULAR_MATRIX_MATRIX_BLAS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixVector.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixVector.h index 76bfa159ced..bd30dc3ca9c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixVector.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixVector.h @@ -10,6 +10,9 @@ #ifndef EIGEN_TRIANGULARMATRIXVECTOR_H #define EIGEN_TRIANGULARMATRIXVECTOR_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -21,19 +24,18 @@ template { typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; - enum { - IsLower = ((Mode&Lower)==Lower), - HasUnitDiag = (Mode & UnitDiag)==UnitDiag, - HasZeroDiag = (Mode & ZeroDiag)==ZeroDiag - }; - static EIGEN_DONT_INLINE void run(Index _rows, Index _cols, const LhsScalar* _lhs, Index lhsStride, - const RhsScalar* _rhs, Index rhsIncr, ResScalar* _res, Index resIncr, const RhsScalar& alpha); + static constexpr bool IsLower = ((Mode & Lower) == Lower); + static constexpr bool HasUnitDiag = (Mode & UnitDiag) == UnitDiag; + static constexpr bool HasZeroDiag = (Mode & ZeroDiag) == ZeroDiag; + static EIGEN_DONT_INLINE void run(Index _rows, Index _cols, const LhsScalar* lhs_, Index lhsStride, + const RhsScalar* rhs_, Index rhsIncr, ResScalar* res_, Index resIncr, + const RhsScalar& alpha); }; template EIGEN_DONT_INLINE void triangular_matrix_vector_product - ::run(Index _rows, Index _cols, const LhsScalar* _lhs, Index lhsStride, - const RhsScalar* _rhs, Index rhsIncr, ResScalar* _res, Index resIncr, const RhsScalar& alpha) + ::run(Index _rows, Index _cols, const LhsScalar* lhs_, Index lhsStride, + const RhsScalar* rhs_, Index rhsIncr, ResScalar* res_, Index resIncr, const RhsScalar& alpha) { static const Index PanelWidth = EIGEN_TUNE_TRIANGULAR_PANEL_WIDTH; Index size = (std::min)(_rows,_cols); @@ -41,15 +43,15 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product, 0, OuterStride<> > LhsMap; - const LhsMap lhs(_lhs,rows,cols,OuterStride<>(lhsStride)); + const LhsMap lhs(lhs_,rows,cols,OuterStride<>(lhsStride)); typename conj_expr_if::type cjLhs(lhs); typedef Map, 0, InnerStride<> > RhsMap; - const RhsMap rhs(_rhs,cols,InnerStride<>(rhsIncr)); + const RhsMap rhs(rhs_,cols,InnerStride<>(rhsIncr)); typename conj_expr_if::type cjRhs(rhs); typedef Map > ResMap; - ResMap res(_res,rows); + ResMap res(res_,rows); typedef const_blas_data_mapper LhsMapper; typedef const_blas_data_mapper RhsMapper; @@ -84,7 +86,7 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product { typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; - enum { - IsLower = ((Mode&Lower)==Lower), - HasUnitDiag = (Mode & UnitDiag)==UnitDiag, - HasZeroDiag = (Mode & ZeroDiag)==ZeroDiag - }; - static EIGEN_DONT_INLINE void run(Index _rows, Index _cols, const LhsScalar* _lhs, Index lhsStride, - const RhsScalar* _rhs, Index rhsIncr, ResScalar* _res, Index resIncr, const ResScalar& alpha); + static constexpr bool IsLower = ((Mode & Lower) == Lower); + static constexpr bool HasUnitDiag = (Mode & UnitDiag) == UnitDiag; + static constexpr bool HasZeroDiag = (Mode & ZeroDiag) == ZeroDiag; + static EIGEN_DONT_INLINE void run(Index _rows, Index _cols, const LhsScalar* lhs_, Index lhsStride, + const RhsScalar* rhs_, Index rhsIncr, ResScalar* res_, Index resIncr, + const ResScalar& alpha); }; template EIGEN_DONT_INLINE void triangular_matrix_vector_product - ::run(Index _rows, Index _cols, const LhsScalar* _lhs, Index lhsStride, - const RhsScalar* _rhs, Index rhsIncr, ResScalar* _res, Index resIncr, const ResScalar& alpha) + ::run(Index _rows, Index _cols, const LhsScalar* lhs_, Index lhsStride, + const RhsScalar* rhs_, Index rhsIncr, ResScalar* res_, Index resIncr, const ResScalar& alpha) { static const Index PanelWidth = EIGEN_TUNE_TRIANGULAR_PANEL_WIDTH; Index diagSize = (std::min)(_rows,_cols); @@ -112,15 +113,15 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product, 0, OuterStride<> > LhsMap; - const LhsMap lhs(_lhs,rows,cols,OuterStride<>(lhsStride)); + const LhsMap lhs(lhs_,rows,cols,OuterStride<>(lhsStride)); typename conj_expr_if::type cjLhs(lhs); typedef Map > RhsMap; - const RhsMap rhs(_rhs,cols); + const RhsMap rhs(rhs_,cols); typename conj_expr_if::type cjRhs(rhs); typedef Map, 0, InnerStride<> > ResMap; - ResMap res(_res,rows,InnerStride<>(resIncr)); + ResMap res(res_,rows,InnerStride<>(resIncr)); typedef const_blas_data_mapper LhsMapper; typedef const_blas_data_mapper RhsMapper; @@ -200,7 +201,7 @@ struct triangular_product_impl namespace internal { // TODO: find a way to factorize this piece of code with gemv_selector since the logic is exactly the same. - + template struct trmv_selector { template @@ -209,33 +210,31 @@ template struct trmv_selector typedef typename Lhs::Scalar LhsScalar; typedef typename Rhs::Scalar RhsScalar; typedef typename Dest::Scalar ResScalar; - typedef typename Dest::RealScalar RealScalar; - + typedef internal::blas_traits LhsBlasTraits; typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; typedef internal::blas_traits RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; - - typedef Map, EIGEN_PLAIN_ENUM_MIN(AlignedMax,internal::packet_traits::size)> MappedDest; + constexpr int Alignment = (std::min)(int(AlignedMax), int(internal::packet_traits::size)); + + typedef Map, Alignment> MappedDest; - typename internal::add_const_on_value_type::type actualLhs = LhsBlasTraits::extract(lhs); - typename internal::add_const_on_value_type::type actualRhs = RhsBlasTraits::extract(rhs); + add_const_on_value_type_t actualLhs = LhsBlasTraits::extract(lhs); + add_const_on_value_type_t actualRhs = RhsBlasTraits::extract(rhs); LhsScalar lhs_alpha = LhsBlasTraits::extractScalarFactor(lhs); RhsScalar rhs_alpha = RhsBlasTraits::extractScalarFactor(rhs); ResScalar actualAlpha = alpha * lhs_alpha * rhs_alpha; - enum { - // FIXME find a way to allow an inner stride on the result if packet_traits::size==1 - // on, the other hand it is good for the cache to pack the vector anyways... - EvalToDestAtCompileTime = Dest::InnerStrideAtCompileTime==1, - ComplexByReal = (NumTraits::IsComplex) && (!NumTraits::IsComplex), - MightCannotUseDest = (Dest::InnerStrideAtCompileTime!=1) || ComplexByReal - }; + // FIXME find a way to allow an inner stride on the result if packet_traits::size==1 + // on, the other hand it is good for the cache to pack the vector anyways... + constexpr bool EvalToDestAtCompileTime = Dest::InnerStrideAtCompileTime==1; + constexpr bool ComplexByReal = (NumTraits::IsComplex) && (!NumTraits::IsComplex); + constexpr bool MightCannotUseDest = (Dest::InnerStrideAtCompileTime!=1) || ComplexByReal; gemv_static_vector_if static_dest; - bool alphaIsCompatible = (!ComplexByReal) || (numext::imag(actualAlpha)==RealScalar(0)); + bool alphaIsCompatible = (!ComplexByReal) || numext::is_exactly_zero(numext::imag(actualAlpha)); bool evalToDest = EvalToDestAtCompileTime && alphaIsCompatible; RhsScalar compatibleAlpha = get_factor::run(actualAlpha); @@ -276,7 +275,7 @@ template struct trmv_selector dest = MappedDest(actualDestPtr, dest.size()); } - if ( ((Mode&UnitDiag)==UnitDiag) && (lhs_alpha!=LhsScalar(1)) ) + if ( ((Mode&UnitDiag)==UnitDiag) && !numext::is_exactly_one(lhs_alpha) ) { Index diagSize = (std::min)(lhs.rows(),lhs.cols()); dest.head(diagSize) -= (lhs_alpha-LhsScalar(1))*rhs.head(diagSize); @@ -297,18 +296,16 @@ template struct trmv_selector typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; typedef internal::blas_traits RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; - typedef typename internal::remove_all::type ActualRhsTypeCleaned; + typedef internal::remove_all_t ActualRhsTypeCleaned; - typename add_const::type actualLhs = LhsBlasTraits::extract(lhs); - typename add_const::type actualRhs = RhsBlasTraits::extract(rhs); + std::add_const_t actualLhs = LhsBlasTraits::extract(lhs); + std::add_const_t actualRhs = RhsBlasTraits::extract(rhs); LhsScalar lhs_alpha = LhsBlasTraits::extractScalarFactor(lhs); RhsScalar rhs_alpha = RhsBlasTraits::extractScalarFactor(rhs); ResScalar actualAlpha = alpha * lhs_alpha * rhs_alpha; - enum { - DirectlyUseRhs = ActualRhsTypeCleaned::InnerStrideAtCompileTime==1 - }; + constexpr bool DirectlyUseRhs = ActualRhsTypeCleaned::InnerStrideAtCompileTime==1; gemv_static_vector_if static_rhs; @@ -335,7 +332,7 @@ template struct trmv_selector dest.data(),dest.innerStride(), actualAlpha); - if ( ((Mode&UnitDiag)==UnitDiag) && (lhs_alpha!=LhsScalar(1)) ) + if ( ((Mode&UnitDiag)==UnitDiag) && !numext::is_exactly_one(lhs_alpha) ) { Index diagSize = (std::min)(lhs.rows(),lhs.cols()); dest.head(diagSize) -= (lhs_alpha-LhsScalar(1))*rhs.head(diagSize); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixVector_BLAS.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixVector_BLAS.h index 3d47a2b94cb..f62a28a0956 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixVector_BLAS.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularMatrixVector_BLAS.h @@ -33,6 +33,9 @@ #ifndef EIGEN_TRIANGULAR_MATRIX_VECTOR_BLAS_H #define EIGEN_TRIANGULAR_MATRIX_VECTOR_BLAS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -50,18 +53,18 @@ struct triangular_matrix_vector_product_trmv : #define EIGEN_BLAS_TRMV_SPECIALIZE(Scalar) \ template \ struct triangular_matrix_vector_product { \ - static void run(Index _rows, Index _cols, const Scalar* _lhs, Index lhsStride, \ - const Scalar* _rhs, Index rhsIncr, Scalar* _res, Index resIncr, Scalar alpha) { \ + static void run(Index rows_, Index cols_, const Scalar* lhs_, Index lhsStride, \ + const Scalar* rhs_, Index rhsIncr, Scalar* res_, Index resIncr, Scalar alpha) { \ triangular_matrix_vector_product_trmv::run( \ - _rows, _cols, _lhs, lhsStride, _rhs, rhsIncr, _res, resIncr, alpha); \ + rows_, cols_, lhs_, lhsStride, rhs_, rhsIncr, res_, resIncr, alpha); \ } \ }; \ template \ struct triangular_matrix_vector_product { \ - static void run(Index _rows, Index _cols, const Scalar* _lhs, Index lhsStride, \ - const Scalar* _rhs, Index rhsIncr, Scalar* _res, Index resIncr, Scalar alpha) { \ + static void run(Index rows_, Index cols_, const Scalar* lhs_, Index lhsStride, \ + const Scalar* rhs_, Index rhsIncr, Scalar* res_, Index resIncr, Scalar alpha) { \ triangular_matrix_vector_product_trmv::run( \ - _rows, _cols, _lhs, lhsStride, _rhs, rhsIncr, _res, resIncr, alpha); \ + rows_, cols_, lhs_, lhsStride, rhs_, rhsIncr, res_, resIncr, alpha); \ } \ }; @@ -81,23 +84,23 @@ struct triangular_matrix_vector_product_trmv::run( \ - _rows, _cols, _lhs, lhsStride, _rhs, rhsIncr, _res, resIncr, alpha); \ + rows_, cols_, lhs_, lhsStride, rhs_, rhsIncr, res_, resIncr, alpha); \ return; \ }\ - Index size = (std::min)(_rows,_cols); \ - Index rows = IsLower ? _rows : size; \ - Index cols = IsLower ? size : _cols; \ + Index size = (std::min)(rows_,cols_); \ + Index rows = IsLower ? rows_ : size; \ + Index cols = IsLower ? size : cols_; \ \ typedef VectorX##EIGPREFIX VectorRhs; \ EIGTYPE *x, *y;\ \ /* Set x*/ \ - Map > rhs(_rhs,cols,InnerStride<>(rhsIncr)); \ + Map > rhs(rhs_,cols,InnerStride<>(rhsIncr)); \ VectorRhs x_tmp; \ if (ConjRhs) x_tmp = rhs.conjugate(); else x_tmp = rhs; \ x = x_tmp.data(); \ @@ -121,24 +124,24 @@ struct triangular_matrix_vector_product_trmv(rows-size); \ n = convert_index(size); \ } \ else { \ x += size; \ - y = _res; \ - a = _lhs + size*lda; \ + y = res_; \ + a = lhs_ + size*lda; \ m = convert_index(size); \ n = convert_index(cols-size); \ } \ @@ -170,23 +173,23 @@ struct triangular_matrix_vector_product_trmv::run( \ - _rows, _cols, _lhs, lhsStride, _rhs, rhsIncr, _res, resIncr, alpha); \ + rows_, cols_, lhs_, lhsStride, rhs_, rhsIncr, res_, resIncr, alpha); \ return; \ }\ - Index size = (std::min)(_rows,_cols); \ - Index rows = IsLower ? _rows : size; \ - Index cols = IsLower ? size : _cols; \ + Index size = (std::min)(rows_,cols_); \ + Index rows = IsLower ? rows_ : size; \ + Index cols = IsLower ? size : cols_; \ \ typedef VectorX##EIGPREFIX VectorRhs; \ EIGTYPE *x, *y;\ \ /* Set x*/ \ - Map > rhs(_rhs,cols,InnerStride<>(rhsIncr)); \ + Map > rhs(rhs_,cols,InnerStride<>(rhsIncr)); \ VectorRhs x_tmp; \ if (ConjRhs) x_tmp = rhs.conjugate(); else x_tmp = rhs; \ x = x_tmp.data(); \ @@ -210,24 +213,24 @@ struct triangular_matrix_vector_product_trmv(rows-size); \ n = convert_index(size); \ } \ else { \ x += size; \ - y = _res; \ - a = _lhs + size; \ + y = res_; \ + a = lhs_ + size; \ m = convert_index(size); \ n = convert_index(cols-size); \ } \ diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularSolverMatrix.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularSolverMatrix.h index 6d879ba00fd..e21d5565c13 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularSolverMatrix.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/products/TriangularSolverMatrix.h @@ -2,6 +2,7 @@ // for linear algebra. // // Copyright (C) 2009 Gael Guennebaud +// Modifications Copyright (C) 2022 Intel Corporation // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -10,10 +11,124 @@ #ifndef EIGEN_TRIANGULAR_SOLVER_MATRIX_H #define EIGEN_TRIANGULAR_SOLVER_MATRIX_H -namespace Eigen { +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + +namespace Eigen { namespace internal { +template +struct trsmKernelL { + // Generic Implementation of triangular solve for triangular matrix on left and multiple rhs. + // Handles non-packed matrices. + static void kernel( + Index size, Index otherSize, + const Scalar* _tri, Index triStride, + Scalar* _other, Index otherIncr, Index otherStride); +}; + +template +struct trsmKernelR { + // Generic Implementation of triangular solve for triangular matrix on right and multiple lhs. + // Handles non-packed matrices. + static void kernel( + Index size, Index otherSize, + const Scalar* _tri, Index triStride, + Scalar* _other, Index otherIncr, Index otherStride); +}; + +template +EIGEN_STRONG_INLINE void trsmKernelL::kernel( + Index size, Index otherSize, + const Scalar* _tri, Index triStride, + Scalar* _other, Index otherIncr, Index otherStride) + { + typedef const_blas_data_mapper TriMapper; + typedef blas_data_mapper OtherMapper; + TriMapper tri(_tri, triStride); + OtherMapper other(_other, otherStride, otherIncr); + + enum { IsLower = (Mode&Lower) == Lower }; + conj_if conj; + + // tr solve + for (Index k=0; k +EIGEN_STRONG_INLINE void trsmKernelR::kernel( + Index size, Index otherSize, + const Scalar* _tri, Index triStride, + Scalar* _other, Index otherIncr, Index otherStride) +{ + typedef typename NumTraits::Real RealScalar; + typedef blas_data_mapper LhsMapper; + typedef const_blas_data_mapper RhsMapper; + LhsMapper lhs(_other, otherStride, otherIncr); + RhsMapper rhs(_tri, triStride); + + enum { + RhsStorageOrder = TriStorageOrder, + IsLower = (Mode&Lower) == Lower + }; + conj_if conj; + + for (Index k=0; k struct triangular_solve_matrix @@ -44,6 +159,7 @@ struct triangular_solve_matrix& blocking); }; + template EIGEN_DONT_INLINE void triangular_solve_matrix::run( Index size, Index otherSize, @@ -53,6 +169,25 @@ EIGEN_DONT_INLINE void triangular_solve_matrix::value || + std::is_same::value)) ) { + // Very rough cutoffs to determine when to call trsm w/o packing + // For small problem sizes trsmKernel compiled with clang is generally faster. + // TODO: Investigate better heuristics for cutoffs. + double L2Cap = 0.5; // 50% of L2 size + if (size < avx512_trsm_cutoff(l2, cols, L2Cap)) { + trsmKernelL::kernel( + size, cols, _tri, triStride, _other, 1, otherStride); + return; + } + } +#endif + typedef const_blas_data_mapper TriMapper; typedef blas_data_mapper OtherMapper; TriMapper tri(_tri, triStride); @@ -61,7 +196,7 @@ EIGEN_DONT_INLINE void triangular_solve_matrix Traits; enum { - SmallPanelWidth = EIGEN_PLAIN_ENUM_MAX(Traits::mr,Traits::nr), + SmallPanelWidth = plain_enum_max(Traits::mr, Traits::nr), IsLower = (Mode&Lower) == Lower }; @@ -74,15 +209,12 @@ EIGEN_DONT_INLINE void triangular_solve_matrix conj; gebp_kernel gebp_kernel; gemm_pack_lhs pack_lhs; gemm_pack_rhs pack_rhs; // the goal here is to subdivise the Rhs panels such that we keep some cache // coherence when accessing the rhs elements - std::ptrdiff_t l1, l2, l3; - manage_caching_sizes(GetAction, &l1, &l2, &l3); Index subcols = cols>0 ? l2/(4 * sizeof(Scalar) * std::max(otherStride,size)) : 0; subcols = std::max((subcols/Traits::nr)*Traits::nr, Traits::nr); @@ -113,38 +245,19 @@ EIGEN_DONT_INLINE void triangular_solve_matrix(actual_kc-k1, SmallPanelWidth); // tr solve - for (Index k=0; k::value || + std::is_same::value)) ) { + i = IsLower ? k2 + k1: k2 - k1 - actualPanelWidth; } +#endif + trsmKernelL::kernel( + actualPanelWidth, actual_cols, + _tri + i + (i)*triStride, triStride, + _other + i*OtherInnerStride + j2*otherStride, otherIncr, otherStride); } Index lengthTarget = actual_kc-k1-actualPanelWidth; @@ -166,7 +279,7 @@ EIGEN_DONT_INLINE void triangular_solve_matrix GEPP { Index start = IsLower ? k2+kc : 0; @@ -196,6 +309,7 @@ struct triangular_solve_matrix& blocking); }; + template EIGEN_DONT_INLINE void triangular_solve_matrix::run( Index size, Index otherSize, @@ -204,7 +318,22 @@ EIGEN_DONT_INLINE void triangular_solve_matrix& blocking) { Index rows = otherSize; - typedef typename NumTraits::Real RealScalar; + +#if defined(EIGEN_VECTORIZE_AVX512) && EIGEN_USE_AVX512_TRSM_R_KERNELS && EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS + EIGEN_IF_CONSTEXPR( (OtherInnerStride == 1 && + (std::is_same::value || + std::is_same::value)) ) { + // TODO: Investigate better heuristics for cutoffs. + std::ptrdiff_t l1, l2, l3; + manage_caching_sizes(GetAction, &l1, &l2, &l3); + double L2Cap = 0.5; // 50% of L2 size + if (size < avx512_trsm_cutoff(l2, rows, L2Cap)) { + trsmKernelR:: + kernel(size, rows, _tri, triStride, _other, 1, otherStride); + return; + } + } +#endif typedef blas_data_mapper LhsMapper; typedef const_blas_data_mapper RhsMapper; @@ -214,7 +343,7 @@ EIGEN_DONT_INLINE void triangular_solve_matrix Traits; enum { RhsStorageOrder = TriStorageOrder, - SmallPanelWidth = EIGEN_PLAIN_ENUM_MAX(Traits::mr,Traits::nr), + SmallPanelWidth = plain_enum_max(Traits::mr, Traits::nr), IsLower = (Mode&Lower) == Lower }; @@ -227,7 +356,6 @@ EIGEN_DONT_INLINE void triangular_solve_matrix conj; gebp_kernel gebp_kernel; gemm_pack_rhs pack_rhs; gemm_pack_rhs pack_rhs_panel; @@ -294,27 +422,13 @@ EIGEN_DONT_INLINE void triangular_solve_matrix:: + kernel(actualPanelWidth, actual_mc, + _tri + absolute_j2 + absolute_j2*triStride, triStride, + _other + i2*OtherInnerStride + absolute_j2*otherStride, otherIncr, otherStride); } - // pack the just computed part of lhs to A pack_lhs_panel(blockA, lhs.getSubMapper(i2,absolute_j2), actualPanelWidth, actual_mc, @@ -329,7 +443,6 @@ EIGEN_DONT_INLINE void triangular_solve_matrix LhsMapper; typedef const_blas_data_mapper RhsMapper; - typename internal::conditional< - Conjugate, - const CwiseUnaryOp,LhsMap>, - const LhsMap&> - ::type cjLhs(lhs); + std::conditional_t< + Conjugate, + const CwiseUnaryOp,LhsMap>, + const LhsMap&> cjLhs(lhs); static const Index PanelWidth = EIGEN_TUNE_TRIANGULAR_PANEL_WIDTH; for(Index pi=IsLower ? 0 : size; IsLower ? pi0; @@ -77,7 +79,7 @@ struct triangular_solve_vector0) rhs[i] -= (cjLhs.row(i).segment(s,k).transpose().cwiseProduct(Map >(rhs+s,k))).sum(); - if((!(Mode & UnitDiag)) && numext::not_equal_strict(rhs[i],RhsScalar(0))) + if((!(Mode & UnitDiag)) && !is_identically_zero(rhs[i])) rhs[i] /= cjLhs(i,i); } } @@ -97,10 +99,10 @@ struct triangular_solve_vector(lhsStride)); typedef const_blas_data_mapper LhsMapper; typedef const_blas_data_mapper RhsMapper; - typename internal::conditional,LhsMap>, - const LhsMap& - >::type cjLhs(lhs); + std::conditional_t,LhsMap>, + const LhsMap& + > cjLhs(lhs); static const Index PanelWidth = EIGEN_TUNE_TRIANGULAR_PANEL_WIDTH; for(Index pi=IsLower ? 0 : size; @@ -114,7 +116,7 @@ struct triangular_solve_vector + +#ifndef EIGEN_USE_CUSTOM_PLAIN_ASSERT +// Disable new custom asserts by default for now. +#define EIGEN_USE_CUSTOM_PLAIN_ASSERT 0 +#endif + +#if EIGEN_USE_CUSTOM_PLAIN_ASSERT + +#ifndef EIGEN_HAS_BUILTIN_FILE +// Clang can check if __builtin_FILE() is supported. +// GCC > 5, MSVC 2019 14.26 (1926) all have __builtin_FILE(). +// +// For NVCC, it's more complicated. Through trial-and-error: +// - nvcc+gcc supports __builtin_FILE() on host, and on device after CUDA 11. +// - nvcc+msvc supports __builtin_FILE() only after CUDA 11. +#if (EIGEN_HAS_BUILTIN(__builtin_FILE) && (EIGEN_COMP_CLANG || !defined(EIGEN_CUDA_ARCH))) || \ + (EIGEN_GNUC_STRICT_AT_LEAST(5, 0, 0) && (EIGEN_COMP_NVCC >= 110000 || !defined(EIGEN_CUDA_ARCH))) || \ + (EIGEN_COMP_MSVC >= 1926 && (!EIGEN_COMP_NVCC || EIGEN_COMP_NVCC >= 110000)) +#define EIGEN_HAS_BUILTIN_FILE 1 +#else +#define EIGEN_HAS_BUILTIN_FILE 0 +#endif +#endif // EIGEN_HAS_BUILTIN_FILE + +#if EIGEN_HAS_BUILTIN_FILE +# define EIGEN_BUILTIN_FILE __builtin_FILE() +# define EIGEN_BUILTIN_LINE __builtin_LINE() +#else +// Default (potentially unsafe) values. +# define EIGEN_BUILTIN_FILE __FILE__ +# define EIGEN_BUILTIN_LINE __LINE__ +#endif + +// Use __PRETTY_FUNCTION__ when available, since it is more descriptive, as +// __builtin_FUNCTION() only returns the undecorated function name. +// This should still be okay ODR-wise since it is a compiler-specific fixed +// value. Mixing compilers will likely lead to ODR violations anyways. +#if EIGEN_COMP_MSVC +# define EIGEN_BUILTIN_FUNCTION __FUNCSIG__ +#elif EIGEN_COMP_GNUC +# define EIGEN_BUILTIN_FUNCTION __PRETTY_FUNCTION__ +#else +# define EIGEN_BUILTIN_FUNCTION __func__ +#endif + +namespace Eigen { +namespace internal { + +// Generic default assert handler. +template +struct assert_handler_impl { + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE + static inline void run(const char* expression, const char* file, unsigned line, const char* function) { +#ifdef EIGEN_GPU_COMPILE_PHASE + // GPU device code doesn't allow stderr or abort, so use printf and raise an + // illegal instruction exception to trigger a kernel failure. +#ifndef EIGEN_NO_IO + printf("Assertion failed at %s:%u in %s: %s\n", + file == nullptr ? "" : file, + line, + function == nullptr ? "" : function, + expression); +#endif + __trap(); + +#else // EIGEN_GPU_COMPILE_PHASE + + // Print to stderr and abort, as specified in . +#ifndef EIGEN_NO_IO + fprintf(stderr, "Assertion failed at %s:%u in %s: %s\n", + file == nullptr ? "" : file, + line, + function == nullptr ? "" : function, + expression); +#endif + std::abort(); + +#endif // EIGEN_GPU_COMPILE_PHASE + } +}; + +// Use POSIX __assert_fail handler when available. +// +// This allows us to integrate with systems that have custom handlers. +// +// NOTE: this handler is not always available on all POSIX systems (otherwise +// we could simply test for __unix__ or similar). The handler function name +// seems to depend on the specific toolchain implementation, and differs between +// compilers, platforms, OSes, etc. Hence, we detect support via SFINAE. +template +struct assert_handler_impl< + void_t()... // Empty substitution required for SFINAE. + ))>, EmptyArgs... > { + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE + static inline void run(const char* expression, const char* file, unsigned line, const char* function) { + // GCC requires this call to be dependent on the template parameters. + __assert_fail(expression, file, line, function, std::declval()...); + } +}; + +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE +inline void __assert_handler(const char* expression, const char* file, unsigned line, const char* function) { + assert_handler_impl<>::run(expression, file, line, function); +} + +} // namespace internal +} // namespace Eigen + +#define eigen_plain_assert(expression) \ + (EIGEN_PREDICT_FALSE(!(expression)) ? \ + Eigen::internal::__assert_handler(#expression, \ + EIGEN_BUILTIN_FILE, \ + EIGEN_BUILTIN_LINE, \ + EIGEN_BUILTIN_FUNCTION) : (void)0) + +#else // EIGEN_USE_CUSTOM_PLAIN_ASSERT + +// Use regular assert. +#define eigen_plain_assert(condition) assert(condition) + +#endif // EIGEN_USE_CUSTOM_PLAIN_ASSERT + +#else // EIGEN_NO_DEBUG + +#define eigen_plain_assert(condition) ((void)0) + +#endif // EIGEN_NO_DEBUG + +#endif // EIGEN_CORE_UTIL_ASSERT_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/BlasUtil.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/BlasUtil.h index e16a5649807..c56b92593cd 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/BlasUtil.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/BlasUtil.h @@ -13,6 +13,9 @@ // This file contains many lightweight helper classes used to // implement and control fast level 2 and level 3 BLAS-like routines. +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -64,7 +67,7 @@ class BlasVectorMapper { template EIGEN_DEVICE_FUNC bool aligned(Index i) const { - return (UIntPtr(m_data+i)%sizeof(Packet))==0; + return (std::uintptr_t(m_data+i)%sizeof(Packet))==0; } protected: @@ -85,7 +88,7 @@ class BlasLinearMapper eigen_assert(incr==1); } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void prefetch(int i) const { + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void prefetch(Index i) const { internal::prefetch(&operator()(i)); } @@ -98,11 +101,26 @@ class BlasLinearMapper return ploadt(m_data + i); } + template + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index n, Index offset = 0) const { + return ploadt_partial(m_data + i, n, offset); + } + + template + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType load(Index i) const { + return ploadt(m_data + i); + } + template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, const PacketType &p) const { pstoret(m_data + i, p); } + template + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketPartial(Index i, const PacketType &p, Index n, Index offset = 0) const { + pstoret_partial(m_data + i, p, n, offset); + } + protected: Scalar *m_data; }; @@ -165,6 +183,7 @@ class blas_data_mapper { public: typedef BlasLinearMapper LinearMapper; + typedef blas_data_mapper SubMapper; typedef BlasVectorMapper VectorMapper; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE blas_data_mapper(Scalar* data, Index stride, Index incr=1) @@ -174,9 +193,9 @@ class blas_data_mapper eigen_assert(incr==1); } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE blas_data_mapper + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SubMapper getSubMapper(Index i, Index j) const { - return blas_data_mapper(&operator()(i, j), m_stride); + return SubMapper(&operator()(i, j), m_stride); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j) const { @@ -187,6 +206,9 @@ class blas_data_mapper return VectorMapper(&operator()(i, j)); } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void prefetch(Index i, Index j) const { + internal::prefetch(&operator()(i, j)); + } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar& operator()(Index i, Index j) const { @@ -198,11 +220,26 @@ class blas_data_mapper return ploadt(&operator()(i, j)); } + template + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index j, Index n, Index offset = 0) const { + return ploadt_partial(&operator()(i, j), n, offset); + } + template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i, Index j) const { return ploadt(&operator()(i, j)); } + template + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, Index j, const PacketType &p) const { + pstoret(&operator()(i, j), p); + } + + template + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketPartial(Index i, Index j, const PacketType &p, Index n, Index offset = 0) const { + pstoret_partial(&operator()(i, j), p, n, offset); + } + template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void scatterPacket(Index i, Index j, const SubPacket &p) const { pscatter(&operator()(i, j), p, m_stride); @@ -214,10 +251,11 @@ class blas_data_mapper } EIGEN_DEVICE_FUNC const Index stride() const { return m_stride; } + EIGEN_DEVICE_FUNC const Index incr() const { return 1; } EIGEN_DEVICE_FUNC const Scalar* data() const { return m_data; } EIGEN_DEVICE_FUNC Index firstAligned(Index size) const { - if (UIntPtr(m_data)%sizeof(Scalar)) { + if (std::uintptr_t(m_data)%sizeof(Scalar)) { return -1; } return internal::first_default_aligned(m_data, size); @@ -255,11 +293,21 @@ class BlasLinearMapper return pgather(m_data + i*m_incr.value(), m_incr.value()); } + template + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index n, Index /*offset*/ = 0) const { + return pgather_partial(m_data + i*m_incr.value(), m_incr.value(), n); + } + template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, const PacketType &p) const { pscatter(m_data + i*m_incr.value(), p, m_incr.value()); } + template + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketPartial(Index i, const PacketType &p, Index n, Index /*offset*/ = 0) const { + pscatter_partial(m_data + i*m_incr.value(), p, m_incr.value(), n); + } + protected: Scalar *m_data; const internal::variable_if_dynamic m_incr; @@ -270,18 +318,23 @@ class blas_data_mapper { public: typedef BlasLinearMapper LinearMapper; + typedef blas_data_mapper SubMapper; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE blas_data_mapper(Scalar* data, Index stride, Index incr) : m_data(data), m_stride(stride), m_incr(incr) {} - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE blas_data_mapper + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SubMapper getSubMapper(Index i, Index j) const { - return blas_data_mapper(&operator()(i, j), m_stride, m_incr.value()); + return SubMapper(&operator()(i, j), m_stride, m_incr.value()); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j) const { return LinearMapper(&operator()(i, j), m_incr.value()); } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void prefetch(Index i, Index j) const { + internal::prefetch(&operator()(i, j)); + } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar& operator()(Index i, Index j) const { return m_data[StorageOrder==RowMajor ? j*m_incr.value() + i*m_stride : i*m_incr.value() + j*m_stride]; @@ -292,11 +345,26 @@ class blas_data_mapper return pgather(&operator()(i, j),m_incr.value()); } + template + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacketPartial(Index i, Index j, Index n, Index /*offset*/ = 0) const { + return pgather_partial(&operator()(i, j),m_incr.value(),n); + } + template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i, Index j) const { return pgather(&operator()(i, j),m_incr.value()); } + template + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, Index j, const PacketType &p) const { + pscatter(&operator()(i, j), p, m_incr.value()); + } + + template + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketPartial(Index i, Index j, const PacketType &p, Index n, Index /*offset*/ = 0) const { + pscatter_partial(&operator()(i, j), p, m_incr.value(), n); + } + template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void scatterPacket(Index i, Index j, const SubPacket &p) const { pscatter(&operator()(i, j), p, m_stride); @@ -308,17 +376,13 @@ class blas_data_mapper } // storePacketBlock_helper defines a way to access values inside the PacketBlock, this is essentially required by the Complex types. - template + template struct storePacketBlock_helper { - storePacketBlock_helper spbh; + storePacketBlock_helper spbh; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(const blas_data_mapper* sup, Index i, Index j, const PacketBlock& block) const { spbh.store(sup, i,j,block); - for(int l = 0; l < unpacket_traits::size; l++) - { - ScalarT *v = &sup->operator()(i+l, j+idx); - *v = block.packet[idx][l]; - } + sup->template storePacket(i, j+idx, block.packet[idx]); } }; @@ -328,12 +392,7 @@ class blas_data_mapper storePacketBlock_helper, n, idx-1> spbh; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(const blas_data_mapper* sup, Index i, Index j, const PacketBlock& block) const { spbh.store(sup,i,j,block); - for(int l = 0; l < unpacket_traits::size; l++) - { - std::complex *v = &sup->operator()(i+l, j+idx); - v->real(block.packet[idx].v[2*l+0]); - v->imag(block.packet[idx].v[2*l+1]); - } + sup->template storePacket(i, j+idx, block.packet[idx]); } }; @@ -352,8 +411,8 @@ class blas_data_mapper } }; - template - struct storePacketBlock_helper + template + struct storePacketBlock_helper { EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(const blas_data_mapper*, Index, Index, const PacketBlock& ) const { } @@ -378,6 +437,10 @@ class blas_data_mapper storePacketBlock_helper spb; spb.store(this, i,j,block); } + + EIGEN_DEVICE_FUNC const Index stride() const { return m_stride; } + EIGEN_DEVICE_FUNC const Index incr() const { return m_incr.value(); } + EIGEN_DEVICE_FUNC Scalar* data() const { return m_data; } protected: Scalar* EIGEN_RESTRICT m_data; const Index m_stride; @@ -388,10 +451,12 @@ class blas_data_mapper template class const_blas_data_mapper : public blas_data_mapper { public: + typedef const_blas_data_mapper SubMapper; + EIGEN_ALWAYS_INLINE const_blas_data_mapper(const Scalar *data, Index stride) : blas_data_mapper(data, stride) {} - EIGEN_ALWAYS_INLINE const_blas_data_mapper getSubMapper(Index i, Index j) const { - return const_blas_data_mapper(&(this->operator()(i, j)), this->m_stride); + EIGEN_ALWAYS_INLINE SubMapper getSubMapper(Index i, Index j) const { + return SubMapper(&(this->operator()(i, j)), this->m_stride); } }; @@ -403,7 +468,7 @@ template struct blas_traits { typedef typename traits::Scalar Scalar; typedef const XprType& ExtractType; - typedef XprType _ExtractType; + typedef XprType ExtractType_; enum { IsComplex = NumTraits::IsComplex, IsTransposed = false, @@ -414,12 +479,12 @@ template struct blas_traits ) ? 1 : 0, HasScalarFactor = false }; - typedef typename conditional::type DirectLinearAccessType; - static inline EIGEN_DEVICE_FUNC ExtractType extract(const XprType& x) { return x; } - static inline EIGEN_DEVICE_FUNC const Scalar extractScalarFactor(const XprType&) { return Scalar(1); } + typename ExtractType_::PlainObject + > DirectLinearAccessType; + EIGEN_DEVICE_FUNC static inline EIGEN_DEVICE_FUNC ExtractType extract(const XprType& x) { return x; } + EIGEN_DEVICE_FUNC static inline EIGEN_DEVICE_FUNC const Scalar extractScalarFactor(const XprType&) { return Scalar(1); } }; // pop conjugate @@ -435,8 +500,8 @@ struct blas_traits, NestedXpr> > IsComplex = NumTraits::IsComplex, NeedToConjugate = Base::NeedToConjugate ? 0 : IsComplex }; - static inline ExtractType extract(const XprType& x) { return Base::extract(x.nestedExpression()); } - static inline Scalar extractScalarFactor(const XprType& x) { return conj(Base::extractScalarFactor(x.nestedExpression())); } + EIGEN_DEVICE_FUNC static inline ExtractType extract(const XprType& x) { return Base::extract(x.nestedExpression()); } + EIGEN_DEVICE_FUNC static inline Scalar extractScalarFactor(const XprType& x) { return conj(Base::extractScalarFactor(x.nestedExpression())); } }; // pop scalar multiple @@ -450,8 +515,8 @@ struct blas_traits, const CwiseNullaryOp typedef blas_traits Base; typedef CwiseBinaryOp, const CwiseNullaryOp,Plain>, NestedXpr> XprType; typedef typename Base::ExtractType ExtractType; - static inline EIGEN_DEVICE_FUNC ExtractType extract(const XprType& x) { return Base::extract(x.rhs()); } - static inline EIGEN_DEVICE_FUNC Scalar extractScalarFactor(const XprType& x) + EIGEN_DEVICE_FUNC static inline EIGEN_DEVICE_FUNC ExtractType extract(const XprType& x) { return Base::extract(x.rhs()); } + EIGEN_DEVICE_FUNC static inline EIGEN_DEVICE_FUNC Scalar extractScalarFactor(const XprType& x) { return x.lhs().functor().m_other * Base::extractScalarFactor(x.rhs()); } }; template @@ -464,8 +529,8 @@ struct blas_traits, NestedXpr, const Cwi typedef blas_traits Base; typedef CwiseBinaryOp, NestedXpr, const CwiseNullaryOp,Plain> > XprType; typedef typename Base::ExtractType ExtractType; - static inline ExtractType extract(const XprType& x) { return Base::extract(x.lhs()); } - static inline Scalar extractScalarFactor(const XprType& x) + EIGEN_DEVICE_FUNC static inline ExtractType extract(const XprType& x) { return Base::extract(x.lhs()); } + EIGEN_DEVICE_FUNC static inline Scalar extractScalarFactor(const XprType& x) { return Base::extractScalarFactor(x.lhs()) * x.rhs().functor().m_other; } }; template @@ -485,8 +550,8 @@ struct blas_traits, NestedXpr> > typedef blas_traits Base; typedef CwiseUnaryOp, NestedXpr> XprType; typedef typename Base::ExtractType ExtractType; - static inline ExtractType extract(const XprType& x) { return Base::extract(x.nestedExpression()); } - static inline Scalar extractScalarFactor(const XprType& x) + EIGEN_DEVICE_FUNC static inline ExtractType extract(const XprType& x) { return Base::extract(x.nestedExpression()); } + EIGEN_DEVICE_FUNC static inline Scalar extractScalarFactor(const XprType& x) { return - Base::extractScalarFactor(x.nestedExpression()); } }; @@ -498,17 +563,17 @@ struct blas_traits > typedef typename NestedXpr::Scalar Scalar; typedef blas_traits Base; typedef Transpose XprType; - typedef Transpose ExtractType; // const to get rid of a compile error; anyway blas traits are only used on the RHS - typedef Transpose _ExtractType; - typedef typename conditional ExtractType; // const to get rid of a compile error; anyway blas traits are only used on the RHS + typedef Transpose ExtractType_; + typedef std::conditional_t::type DirectLinearAccessType; + > DirectLinearAccessType; enum { IsTransposed = Base::IsTransposed ? 0 : 1 }; - static inline ExtractType extract(const XprType& x) { return ExtractType(Base::extract(x.nestedExpression())); } - static inline Scalar extractScalarFactor(const XprType& x) { return Base::extractScalarFactor(x.nestedExpression()); } + EIGEN_DEVICE_FUNC static inline ExtractType extract(const XprType& x) { return ExtractType(Base::extract(x.nestedExpression())); } + EIGEN_DEVICE_FUNC static inline Scalar extractScalarFactor(const XprType& x) { return Base::extractScalarFactor(x.nestedExpression()); } }; template @@ -526,7 +591,7 @@ struct extract_data_selector { template struct extract_data_selector { - static typename T::Scalar* run(const T&) { return 0; } + EIGEN_DEVICE_FUNC static typename T::Scalar* run(const T&) { return 0; } }; template diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ConfigureVectorization.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ConfigureVectorization.h index 2d12e1d173f..ffd93291fd3 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ConfigureVectorization.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ConfigureVectorization.h @@ -30,27 +30,13 @@ * * If we made alignment depend on whether or not EIGEN_VECTORIZE is defined, it would be impossible to link * vectorized and non-vectorized code. - * - * FIXME: this code can be cleaned up once we switch to proper C++11 only. */ #if (defined EIGEN_CUDACC) #define EIGEN_ALIGN_TO_BOUNDARY(n) __align__(n) #define EIGEN_ALIGNOF(x) __alignof(x) -#elif EIGEN_HAS_ALIGNAS +#else #define EIGEN_ALIGN_TO_BOUNDARY(n) alignas(n) #define EIGEN_ALIGNOF(x) alignof(x) -#elif EIGEN_COMP_GNUC || EIGEN_COMP_PGI || EIGEN_COMP_IBM || EIGEN_COMP_ARM - #define EIGEN_ALIGN_TO_BOUNDARY(n) __attribute__((aligned(n))) - #define EIGEN_ALIGNOF(x) __alignof(x) -#elif EIGEN_COMP_MSVC - #define EIGEN_ALIGN_TO_BOUNDARY(n) __declspec(align(n)) - #define EIGEN_ALIGNOF(x) __alignof(x) -#elif EIGEN_COMP_SUNCC - // FIXME not sure about this one: - #define EIGEN_ALIGN_TO_BOUNDARY(n) __attribute__((aligned(n))) - #define EIGEN_ALIGNOF(x) __alignof(x) -#else - #error Please tell me what is the equivalent of alignas(n) and alignof(x) for your compiler #endif // If the user explicitly disable vectorization, then we also disable alignment @@ -68,6 +54,8 @@ #elif defined(__AVX__) // 32 bytes static alignment is preferred only if really required #define EIGEN_IDEAL_MAX_ALIGN_BYTES 32 +#elif defined __HVX__ && (__HVX_LENGTH__ == 128) + #define EIGEN_IDEAL_MAX_ALIGN_BYTES 128 #else #define EIGEN_IDEAL_MAX_ALIGN_BYTES 16 #endif @@ -105,18 +93,12 @@ // try to keep heap alignment even when we have to disable static alignment. #if EIGEN_COMP_GNUC && !(EIGEN_ARCH_i386_OR_x86_64 || EIGEN_ARCH_ARM_OR_ARM64 || EIGEN_ARCH_PPC || EIGEN_ARCH_IA64 || EIGEN_ARCH_MIPS) #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1 - #elif EIGEN_ARCH_ARM_OR_ARM64 && EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_AT_MOST(4, 6) - // Old versions of GCC on ARM, at least 4.4, were once seen to have buggy static alignment support. - // Not sure which version fixed it, hopefully it doesn't affect 4.7, which is still somewhat in use. - // 4.8 and newer seem definitely unaffected. - #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1 #else #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 0 #endif // static alignment is completely disabled with GCC 3, Sun Studio, and QCC/QNX #if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT \ - && !EIGEN_GCC3_OR_OLDER \ && !EIGEN_COMP_SUNCC \ && !EIGEN_OS_QNX #define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 1 @@ -201,21 +183,19 @@ // removed as gcc 4.1 and msvc 2008 are not supported anyways. #if EIGEN_COMP_MSVC #include // for _aligned_malloc -- need it regardless of whether vectorization is enabled - #if (EIGEN_COMP_MSVC >= 1500) // 2008 or later - // a user reported that in 64-bit mode, MSVC doesn't care to define _M_IX86_FP. - #if (defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64 - #define EIGEN_SSE2_ON_MSVC_2008_OR_LATER - #endif + // a user reported that in 64-bit mode, MSVC doesn't care to define _M_IX86_FP. + #if (defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64 + #define EIGEN_SSE2_ON_MSVC_2008_OR_LATER #endif #else - #if (defined __SSE2__) && ( (!EIGEN_COMP_GNUC) || EIGEN_COMP_ICC || EIGEN_GNUC_AT_LEAST(4,2) ) - #define EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC + #if defined(__SSE2__) + #define EIGEN_SSE2_ON_NON_MSVC #endif #endif #if !(defined(EIGEN_DONT_VECTORIZE) || defined(EIGEN_GPUCC)) - #if defined (EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC) || defined(EIGEN_SSE2_ON_MSVC_2008_OR_LATER) + #if defined (EIGEN_SSE2_ON_NON_MSVC) || defined(EIGEN_SSE2_ON_MSVC_2008_OR_LATER) // Defines symbols for compile-time detection of which instructions are // used. @@ -292,11 +272,22 @@ #ifdef __AVX512BF16__ #define EIGEN_VECTORIZE_AVX512BF16 #endif + #ifdef __AVX512FP16__ + #ifdef __AVX512VL__ + #define EIGEN_VECTORIZE_AVX512FP16 + #else + #if EIGEN_COMP_GNUC + #error Please add -mavx512vl to your compiler flags: compiling with -mavx512fp16 alone without AVX512-VL is not supported. + #else + #error Please enable AVX512-VL in your compiler flags (e.g. -mavx512vl): compiling with AVX512-FP16 alone without AVX512-VL is not supported. + #endif + #endif + #endif #endif #endif // Disable AVX support on broken xcode versions - #if defined(__apple_build_version__) && (__apple_build_version__ == 11000033 ) && ( __MAC_OS_X_VERSION_MIN_REQUIRED == 101500 ) + #if ( EIGEN_COMP_CLANGAPPLE == 11000033 ) && ( __MAC_OS_X_VERSION_MIN_REQUIRED == 101500 ) // A nasty bug in the clang compiler shipped with xcode in a common compilation situation // when XCode 11.0 and Mac deployment target macOS 10.15 is https://trac.macports.org/ticket/58776#no1 #ifdef EIGEN_VECTORIZE_AVX @@ -339,7 +330,7 @@ extern "C" { // In theory we should only include immintrin.h and not the other *mmintrin.h header files directly. // Doing so triggers some issues with ICC. However old gcc versions seems to not have this file, thus: - #if EIGEN_COMP_ICC >= 1110 + #if EIGEN_COMP_ICC >= 1110 || EIGEN_COMP_EMSCRIPTEN #include #else #include @@ -363,10 +354,10 @@ #endif } // end extern "C" - #elif defined __VSX__ + #elif defined(__VSX__) && !defined(__APPLE__) #define EIGEN_VECTORIZE - #define EIGEN_VECTORIZE_VSX + #define EIGEN_VECTORIZE_VSX 1 #include // We need to #undef all these ugly tokens defined in // => use __vector instead of vector @@ -428,6 +419,12 @@ #include #endif +#elif defined __HVX__ && (__HVX_LENGTH__ == 128) + +#define EIGEN_VECTORIZE +#define EIGEN_VECTORIZE_HVX +#include + #endif #endif @@ -438,7 +435,7 @@ #include #endif -#if defined(__F16C__) && (!defined(EIGEN_GPUCC) && (!EIGEN_COMP_CLANG || EIGEN_COMP_CLANG>=380)) +#if defined(__F16C__) && !defined(EIGEN_GPUCC) && (!EIGEN_COMP_CLANG_STRICT || EIGEN_CLANG_STRICT_AT_LEAST(3,8,0)) // We can use the optimized fp16 to float and float to fp16 conversion routines #define EIGEN_HAS_FP16_C @@ -470,10 +467,15 @@ #include #define EIGEN_HAS_HIP_FP16 #include + #define EIGEN_HAS_HIP_BF16 + #include #endif /** \brief Namespace containing all symbols from the %Eigen library. */ +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { inline static const char *SimdInstructionSetsInUse(void) { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Constants.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Constants.h index 35dcaa7b386..3496c6cc3d7 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Constants.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Constants.h @@ -12,6 +12,9 @@ #ifndef EIGEN_CONSTANTS_H #define EIGEN_CONSTANTS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { /** This value means that a positive quantity (e.g., a size) is not known at compile-time, and that instead the value is @@ -134,7 +137,7 @@ const unsigned int LinearAccessBit = 0x10; * Means the expression has a coeffRef() method, i.e. is writable as its individual coefficients are directly addressable. * This rules out read-only expressions. * - * Note that DirectAccessBit and LvalueBit are mutually orthogonal, as there are examples of expression having one but note + * Note that DirectAccessBit and LvalueBit are mutually orthogonal, as there are examples of expression having one but not * the other: * \li writable expressions that don't have a very simple memory layout as a strided array, have LvalueBit but not DirectAccessBit * \li Map-to-const expressions, for example Map, have DirectAccessBit but not LvalueBit @@ -312,7 +315,7 @@ enum SpecializedType { }; /** \ingroup enums - * Enum containing possible values for the \p _Options template parameter of + * Enum containing possible values for the \p Options_ template parameter of * Matrix, Array and BandMatrix. */ enum StorageOptions { /** Storage order is column major (see \ref TopicStorageOrders). */ @@ -421,14 +424,16 @@ enum DecompositionOptions { /** \ingroup enums * Possible values for the \p QRPreconditioner template parameter of JacobiSVD. */ enum QRPreconditioners { + /** Use a QR decomposition with column pivoting as the first step. */ + ColPivHouseholderQRPreconditioner = 0x0, /** Do not specify what is to be done if the SVD of a non-square matrix is asked for. */ - NoQRPreconditioner, + NoQRPreconditioner = 0x40, /** Use a QR decomposition without pivoting as the first step. */ - HouseholderQRPreconditioner, - /** Use a QR decomposition with column pivoting as the first step. */ - ColPivHouseholderQRPreconditioner, + HouseholderQRPreconditioner = 0x80, /** Use a QR decomposition with full pivoting as the first step. */ - FullPivHouseholderQRPreconditioner + FullPivHouseholderQRPreconditioner = 0xC0, + /** Used to disable the QR Preconditioner in BDCSVD. */ + DisableQRDecomposition = NoQRPreconditioner }; #ifdef Success @@ -476,6 +481,7 @@ namespace Architecture NEON = 0x4, MSA = 0x5, SVE = 0x6, + HVX = 0x7, #if defined EIGEN_VECTORIZE_SSE Target = SSE #elif defined EIGEN_VECTORIZE_ALTIVEC @@ -488,6 +494,8 @@ namespace Architecture Target = SVE #elif defined EIGEN_VECTORIZE_MSA Target = MSA +#elif defined EIGEN_VECTORIZE_HVX + Target = HVX #else Target = Generic #endif @@ -529,6 +537,7 @@ struct DenseShape { static std::string debugName() { return "DenseSh struct SolverShape { static std::string debugName() { return "SolverShape"; } }; struct HomogeneousShape { static std::string debugName() { return "HomogeneousShape"; } }; struct DiagonalShape { static std::string debugName() { return "DiagonalShape"; } }; +struct SkewSymmetricShape { static std::string debugName() { return "SkewSymmetricShape"; } }; struct BandShape { static std::string debugName() { return "BandShape"; } }; struct TriangularShape { static std::string debugName() { return "TriangularShape"; } }; struct SelfAdjointShape { static std::string debugName() { return "SelfAdjointShape"; } }; @@ -547,7 +556,7 @@ struct IteratorBased {}; /** \internal * Constants for comparison functors */ -enum ComparisonName { +enum ComparisonName : unsigned int { cmp_EQ = 0, cmp_LT = 1, cmp_LE = 2, diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/DisableStupidWarnings.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/DisableStupidWarnings.h index a281f9767e0..7d1766c2426 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/DisableStupidWarnings.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/DisableStupidWarnings.h @@ -1,7 +1,7 @@ #ifndef EIGEN_WARNINGS_DISABLED #define EIGEN_WARNINGS_DISABLED -#ifdef _MSC_VER +#if defined(_MSC_VER) // 4100 - unreferenced formal parameter (occurred e.g. in aligned_allocator::destroy(pointer p)) // 4101 - unreferenced local variable // 4127 - conditional expression is constant @@ -21,6 +21,11 @@ #pragma warning( push ) #endif #pragma warning( disable : 4100 4101 4127 4181 4211 4244 4273 4324 4503 4512 4522 4700 4714 4717 4800) + // We currently rely on has_denorm in tests, and need it defined correctly for half/bfloat16. + #ifndef _SILENCE_CXX23_DENORM_DEPRECATION_WARNING + #define EIGEN_REENABLE_CXX23_DENORM_DEPRECATION_WARNING 1 + #define _SILENCE_CXX23_DENORM_DEPRECATION_WARNING + #endif #elif defined __INTEL_COMPILER // 2196 - routine is both "inline" and "noinline" ("noinline" assumed) @@ -48,7 +53,7 @@ #if __has_warning("-Wimplicit-int-float-conversion") #pragma clang diagnostic ignored "-Wimplicit-int-float-conversion" #endif - #if ( defined(__ALTIVEC__) || defined(__VSX__) ) && __cplusplus < 201103L + #if ( defined(__ALTIVEC__) || defined(__VSX__) ) && ( !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 201112L) ) // warning: generic selections are a C11-specific feature // ignoring warnings thrown at vec_ctf in Altivec/PacketMath.h #if __has_warning("-Wc11-extensions") @@ -57,7 +62,7 @@ #endif #endif -#elif defined __GNUC__ +#elif defined __GNUC__ && !defined(__FUJITSU) #if (!defined(EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS)) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) #pragma GCC diagnostic push @@ -78,32 +83,54 @@ #endif #if defined __NVCC__ - #pragma diag_suppress boolean_controlling_expr_is_constant + // MSVC 14.16 (required by CUDA 9.*) does not support the _Pragma keyword, so + // we instead use Microsoft's __pragma extension. + #if defined _MSC_VER + #define EIGEN_MAKE_PRAGMA(X) __pragma(#X) + #else + #define EIGEN_MAKE_PRAGMA(X) _Pragma(#X) + #endif + #if defined __NVCC_DIAG_PRAGMA_SUPPORT__ + #define EIGEN_NV_DIAG_SUPPRESS(X) EIGEN_MAKE_PRAGMA(nv_diag_suppress X) + #else + #define EIGEN_NV_DIAG_SUPPRESS(X) EIGEN_MAKE_PRAGMA(diag_suppress X) + #endif + + EIGEN_NV_DIAG_SUPPRESS(boolean_controlling_expr_is_constant) // Disable the "statement is unreachable" message - #pragma diag_suppress code_is_unreachable + EIGEN_NV_DIAG_SUPPRESS(code_is_unreachable) // Disable the "dynamic initialization in unreachable code" message - #pragma diag_suppress initialization_not_reachable + EIGEN_NV_DIAG_SUPPRESS(initialization_not_reachable) // Disable the "invalid error number" message that we get with older versions of nvcc - #pragma diag_suppress 1222 + EIGEN_NV_DIAG_SUPPRESS(1222) // Disable the "calling a __host__ function from a __host__ __device__ function is not allowed" messages (yes, there are many of them and they seem to change with every version of the compiler) - #pragma diag_suppress 2527 - #pragma diag_suppress 2529 - #pragma diag_suppress 2651 - #pragma diag_suppress 2653 - #pragma diag_suppress 2668 - #pragma diag_suppress 2669 - #pragma diag_suppress 2670 - #pragma diag_suppress 2671 - #pragma diag_suppress 2735 - #pragma diag_suppress 2737 - #pragma diag_suppress 2739 - #pragma diag_suppress 2976 - #pragma diag_suppress 2979 + EIGEN_NV_DIAG_SUPPRESS(2527) + EIGEN_NV_DIAG_SUPPRESS(2529) + EIGEN_NV_DIAG_SUPPRESS(2651) + EIGEN_NV_DIAG_SUPPRESS(2653) + EIGEN_NV_DIAG_SUPPRESS(2668) + EIGEN_NV_DIAG_SUPPRESS(2669) + EIGEN_NV_DIAG_SUPPRESS(2670) + EIGEN_NV_DIAG_SUPPRESS(2671) + EIGEN_NV_DIAG_SUPPRESS(2735) + EIGEN_NV_DIAG_SUPPRESS(2737) + EIGEN_NV_DIAG_SUPPRESS(2739) + EIGEN_NV_DIAG_SUPPRESS(2885) + EIGEN_NV_DIAG_SUPPRESS(2888) + EIGEN_NV_DIAG_SUPPRESS(2976) + EIGEN_NV_DIAG_SUPPRESS(2979) + EIGEN_NV_DIAG_SUPPRESS(20011) + EIGEN_NV_DIAG_SUPPRESS(20014) // Disable the "// __device__ annotation is ignored on a function(...) that is // explicitly defaulted on its first declaration" message. // The __device__ annotation seems to actually be needed in some cases, // otherwise resulting in kernel runtime errors. - #pragma diag_suppress 2977 + EIGEN_NV_DIAG_SUPPRESS(2886) + EIGEN_NV_DIAG_SUPPRESS(2929) + EIGEN_NV_DIAG_SUPPRESS(2977) + EIGEN_NV_DIAG_SUPPRESS(20012) + #undef EIGEN_NV_DIAG_SUPPRESS + #undef EIGEN_MAKE_PRAGMA #endif #else diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/EmulateArray.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/EmulateArray.h new file mode 100644 index 00000000000..a4b1d0c215b --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/EmulateArray.h @@ -0,0 +1,286 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_EMULATE_ARRAY_H +#define EIGEN_EMULATE_ARRAY_H + +// CUDA doesn't support the STL containers, so we use our own instead. +#if defined(EIGEN_GPUCC) || defined(EIGEN_AVOID_STL_ARRAY) + +namespace Eigen { +template class array { + + public: + typedef T value_type; + typedef T* iterator; + typedef const T* const_iterator; + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE iterator begin() { return values; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const_iterator begin() const { return values; } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE iterator end() { return values + n; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const_iterator end() const { return values + n; } + + +#if !defined(EIGEN_GPUCC) + typedef std::reverse_iterator reverse_iterator; + typedef std::reverse_iterator const_reverse_iterator; + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE reverse_iterator rbegin() { return reverse_iterator(end());} + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE reverse_iterator rend() { return reverse_iterator(begin()); } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } +#endif + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& operator[] (size_t index) { eigen_internal_assert(index < size()); return values[index]; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& operator[] (size_t index) const { eigen_internal_assert(index < size()); return values[index]; } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& at(size_t index) { eigen_assert(index < size()); return values[index]; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& at(size_t index) const { eigen_assert(index < size()); return values[index]; } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& front() { return values[0]; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& front() const { return values[0]; } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& back() { return values[n-1]; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& back() const { return values[n-1]; } + + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE + static std::size_t size() { return n; } + + T values[n]; + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE array() { } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE array(const T& v) { + EIGEN_STATIC_ASSERT(n==1, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE array(const T& v1, const T& v2) { + EIGEN_STATIC_ASSERT(n==2, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE array(const T& v1, const T& v2, const T& v3) { + EIGEN_STATIC_ASSERT(n==3, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE array(const T& v1, const T& v2, const T& v3, + const T& v4) { + EIGEN_STATIC_ASSERT(n==4, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + values[3] = v4; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE array(const T& v1, const T& v2, const T& v3, const T& v4, + const T& v5) { + EIGEN_STATIC_ASSERT(n==5, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + values[3] = v4; + values[4] = v5; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE array(const T& v1, const T& v2, const T& v3, const T& v4, + const T& v5, const T& v6) { + EIGEN_STATIC_ASSERT(n==6, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + values[3] = v4; + values[4] = v5; + values[5] = v6; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE array(const T& v1, const T& v2, const T& v3, const T& v4, + const T& v5, const T& v6, const T& v7) { + EIGEN_STATIC_ASSERT(n==7, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + values[3] = v4; + values[4] = v5; + values[5] = v6; + values[6] = v7; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE array( + const T& v1, const T& v2, const T& v3, const T& v4, + const T& v5, const T& v6, const T& v7, const T& v8) { + EIGEN_STATIC_ASSERT(n==8, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + values[3] = v4; + values[4] = v5; + values[5] = v6; + values[6] = v7; + values[7] = v8; + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE array(std::initializer_list l) { + eigen_assert(l.size() == n); + internal::smart_copy(l.begin(), l.end(), values); + } +}; + + +// Specialize array for zero size +template class array { + public: + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& operator[] (size_t) { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& operator[] (size_t) const { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& front() { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& front() const { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& back() { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& back() const { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::size_t size() { return 0; } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE array() : dummy() { } + + EIGEN_DEVICE_FUNC array(std::initializer_list l) : dummy() { + EIGEN_UNUSED_VARIABLE(l); + eigen_assert(l.size() == 0); + } + + private: + T dummy; +}; + +// Comparison operator +// Todo: implement !=, <, <=, >, and >= +template +EIGEN_DEVICE_FUNC bool operator==(const array& lhs, const array& rhs) { + for (std::size_t i = 0; i < N; ++i) { + if (lhs[i] != rhs[i]) { + return false; + } + } + return true; +} + + +namespace internal { +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& array_get(array& a) { + return a[I_]; +} +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& array_get(const array& a) { + return a[I_]; +} + +template struct array_size > { + enum { value = N }; +}; +template struct array_size& > { + enum { value = N }; +}; +template struct array_size > { + enum { value = N }; +}; +template struct array_size& > { + enum { value = N }; +}; + +} // end namespace internal +} // end namespace Eigen + +#else + +// The compiler supports c++11, and we're not targeting cuda: use std::array as Eigen::array +#include + +namespace Eigen { + +template using array = std::array; + +namespace internal { +/* std::get is only constexpr in C++14, not yet in C++11 + * - libstdc++ from version 4.7 onwards has it nevertheless, + * so use that + * - libstdc++ older versions: use _M_instance directly + * - libc++ all versions so far: use __elems_ directly + * - all other libs: use std::get to be portable, but + * this may not be constexpr + */ +#if defined(__GLIBCXX__) && __GLIBCXX__ < 20120322 +#define STD_GET_ARR_HACK a._M_instance[I_] +#elif defined(_LIBCPP_VERSION) +#define STD_GET_ARR_HACK a.__elems_[I_] +#else +#define STD_GET_ARR_HACK std::template get(a) +#endif + +template constexpr inline T& array_get(std::array& a) { return (T&) STD_GET_ARR_HACK; } +template constexpr inline T&& array_get(std::array&& a) { return (T&&) STD_GET_ARR_HACK; } +template constexpr inline T const& array_get(std::array const& a) { return (T const&) STD_GET_ARR_HACK; } + +#undef STD_GET_ARR_HACK + +} // end namespace internal +} // end namespace Eigen + +#endif + +#endif // EIGEN_EMULATE_ARRAY_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ForwardDeclarations.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ForwardDeclarations.h index 2f9cc449176..8bff87dd249 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ForwardDeclarations.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ForwardDeclarations.h @@ -11,6 +11,9 @@ #ifndef EIGEN_FORWARDDECLARATIONS_H #define EIGEN_FORWARDDECLARATIONS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -49,24 +52,13 @@ template class DenseBase; template class PlainObjectBase; template class DenseCoeffsBase; -template class Matrix; template class MatrixBase; @@ -87,7 +79,6 @@ template class Transpose; template class Conjugate; template class CwiseNullaryOp; template class CwiseUnaryOp; -template class CwiseUnaryView; template class CwiseBinaryOp; template class CwiseTernaryOp; template class Solve; @@ -96,16 +87,19 @@ template class Inverse; template class Product; template class DiagonalBase; -template class DiagonalWrapper; -template class DiagonalMatrix; +template class DiagonalWrapper; +template class DiagonalMatrix; template class DiagonalProduct; template class Diagonal; +template class SkewSymmetricBase; +template class SkewSymmetricWrapper; +template class SkewSymmetricMatrix3; template class PermutationMatrix; template class Transpositions; template class PermutationBase; template class TranspositionsBase; -template class PermutationWrapper; -template class TranspositionsWrapper; +template class PermutationWrapper; +template class TranspositionsWrapper; template::has_write_access ? WriteAccessors : ReadOnlyAccessors @@ -116,7 +110,8 @@ template class OuterStride; template > class Map; template class RefBase; template,OuterStride<> >::type > class Ref; + typename StrideType = typename std::conditional_t,OuterStride<> > > class Ref; +template> class CwiseUnaryView; template class TriangularBase; template class TriangularView; @@ -142,7 +137,7 @@ template struct image_retval; } // end namespace internal namespace internal { -template class BandMatrix; +template class BandMatrix; } namespace internal { @@ -190,6 +185,7 @@ template struct scalar_abs_op; template struct scalar_abs2_op; template struct scalar_absolute_difference_op; template struct scalar_sqrt_op; +template struct scalar_cbrt_op; template struct scalar_rsqrt_op; template struct scalar_exp_op; template struct scalar_log_op; @@ -198,6 +194,8 @@ template struct scalar_sin_op; template struct scalar_acos_op; template struct scalar_asin_op; template struct scalar_tan_op; +template struct scalar_atan_op; +template struct scalar_atan2_op; template struct scalar_inverse_op; template struct scalar_square_op; template struct scalar_cube_op; @@ -205,11 +203,24 @@ template struct scalar_cast_op; template struct scalar_random_op; template struct scalar_constant_op; template struct scalar_identity_op; -template struct scalar_sign_op; -template struct scalar_pow_op; +template struct scalar_sign_op; +template +struct scalar_pow_op; +template +struct scalar_unary_pow_op; template struct scalar_hypot_op; template struct scalar_product_op; template struct scalar_quotient_op; +// logical and bitwise operations +template struct scalar_boolean_and_op; +template struct scalar_boolean_or_op; +template struct scalar_boolean_xor_op; +template struct scalar_boolean_not_op; +template struct scalar_bitwise_and_op; +template struct scalar_bitwise_or_op; +template struct scalar_bitwise_xor_op; +template struct scalar_bitwise_not_op; // SpecialFunctions module template struct scalar_lgamma_op; @@ -242,58 +253,56 @@ template struct scalar_bessel_k1e_op; struct IOFormat; // Array module -template class Array; + int MaxRows_ = Rows_, int MaxCols_ = Cols_> class Array; template class Select; template class PartialReduxExpr; template class VectorwiseOp; template class Replicate; template class Reverse; -template class FullPivLU; -template class PartialPivLU; +#if defined(EIGEN_USE_LAPACKE) && defined(lapack_int) +// Lapacke interface requires StorageIndex to be lapack_int +typedef lapack_int DefaultPermutationIndex; +#else +typedef int DefaultPermutationIndex; +#endif + +template class FullPivLU; +template class PartialPivLU; namespace internal { template struct inverse_impl; } template class HouseholderQR; -template class ColPivHouseholderQR; -template class FullPivHouseholderQR; -template class CompleteOrthogonalDecomposition; +template class ColPivHouseholderQR; +template class FullPivHouseholderQR; +template class CompleteOrthogonalDecomposition; template class SVDBase; -template class JacobiSVD; -template class BDCSVD; +template class JacobiSVD; +template class BDCSVD; template class LLT; template class LDLT; template class HouseholderSequence; template class JacobiRotation; // Geometry module: -template class RotationBase; -template class Cross; +namespace internal { +template::SizeAtCompileTime> struct cross_impl; +} +template class RotationBase; template class QuaternionBase; template class Rotation2D; template class AngleAxis; template class Translation; template class AlignedBox; template class Quaternion; -template class Transform; -template class ParametrizedLine; -template class Hyperplane; +template class Transform; +template class ParametrizedLine; +template class Hyperplane; template class UniformScaling; template class Homogeneous; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/IndexedViewHelper.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/IndexedViewHelper.h index f85de305f21..29cefd55bb1 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/IndexedViewHelper.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/IndexedViewHelper.h @@ -11,11 +11,18 @@ #ifndef EIGEN_INDEXED_VIEW_HELPER_H #define EIGEN_INDEXED_VIEW_HELPER_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { struct symbolic_last_tag {}; -} +} // namespace internal + +namespace placeholders { + +typedef symbolic::SymbolExpr last_t; /** \var last * \ingroup Core_Module @@ -28,38 +35,20 @@ struct symbolic_last_tag {}; * A typical usage example would be: * \code * using namespace Eigen; - * using Eigen::last; + * using Eigen::placeholders::last; * VectorXd v(n); * v(seq(2,last-2)).setOnes(); * \endcode * * \sa end */ -static const symbolic::SymbolExpr last; // PLEASE use Eigen::last instead of Eigen::placeholders::last +static const last_t last; -/** \var lastp1 - * \ingroup Core_Module - * - * Can be used as a parameter to Eigen::seq and Eigen::seqN functions to symbolically - * reference the last+1 element/row/columns of the underlying vector or matrix once - * passed to DenseBase::operator()(const RowIndices&, const ColIndices&). - * - * This symbolic placeholder supports standard arithmetic operations. - * It is essentially an alias to last+fix<1>. - * - * \sa last - */ -#ifdef EIGEN_PARSED_BY_DOXYGEN -static const auto lastp1 = last+fix<1>; -#else -// Using a FixedExpr<1> expression is important here to make sure the compiler -// can fully optimize the computation starting indices with zero overhead. -static const symbolic::AddExpr,symbolic::ValueExpr > > lastp1(last+fix<1>()); -#endif +} // namespace placeholders namespace internal { - // Replace symbolic last/end "keywords" by their true runtime value +// Replace symbolic last/end "keywords" by their true runtime value inline Index eval_expr_given_size(Index x, Index /* size */) { return x; } template @@ -68,7 +57,7 @@ FixedInt eval_expr_given_size(FixedInt x, Index /*size*/) { return x; } template Index eval_expr_given_size(const symbolic::BaseExpr &x, Index size) { - return x.derived().eval(last=size-1); + return x.derived().eval(Eigen::placeholders::last=size-1); } // Extract increment/step at compile time @@ -111,7 +100,7 @@ template<> struct get_compile_time_incr { // Turn a single index into something that looks like an array (i.e., that exposes a .size(), and operator[](int) methods) template -struct IndexedViewCompatibleType::value>::type> { +struct IndexedViewCompatibleType::value>> { // Here we could simply use Array, but maybe it's less work for the compiler to use // a simpler wrapper as SingleRange //typedef Eigen::Array type; @@ -119,13 +108,13 @@ struct IndexedViewCompatibleType -struct IndexedViewCompatibleType::value>::type> { +struct IndexedViewCompatibleType::value>> { typedef SingleRange type; }; template -typename enable_if::value,SingleRange>::type +std::enable_if_t::value,SingleRange> makeIndexedViewCompatible(const T& id, Index size, SpecializedType) { return eval_expr_given_size(id,size); } @@ -163,23 +152,44 @@ template struct get_compile_time_incr > { } // end namespace internal +namespace placeholders { -/** \var all +typedef symbolic::AddExpr,symbolic::ValueExpr > > lastp1_t; +typedef Eigen::internal::all_t all_t; + +/** \var lastp1 * \ingroup Core_Module - * Can be used as a parameter to DenseBase::operator()(const RowIndices&, const ColIndices&) to index all rows or columns + * + * Can be used as a parameter to Eigen::seq and Eigen::seqN functions to symbolically + * reference the last+1 element/row/columns of the underlying vector or matrix once + * passed to DenseBase::operator()(const RowIndices&, const ColIndices&). + * + * This symbolic placeholder supports standard arithmetic operations. + * It is essentially an alias to last+fix<1>. + * + * \sa last */ -static const Eigen::internal::all_t all; // PLEASE use Eigen::all instead of Eigen::placeholders::all +#ifdef EIGEN_PARSED_BY_DOXYGEN +static const auto lastp1 = last+fix<1>; +#else +// Using a FixedExpr<1> expression is important here to make sure the compiler +// can fully optimize the computation starting indices with zero overhead. +static const lastp1_t lastp1(last+fix<1>()); +#endif +/** \var end + * \ingroup Core_Module + * \sa lastp1 + */ +static const lastp1_t end = lastp1; -namespace placeholders { - typedef symbolic::SymbolExpr last_t; - typedef symbolic::AddExpr,symbolic::ValueExpr > > end_t; - typedef Eigen::internal::all_t all_t; +/** \var all + * \ingroup Core_Module + * Can be used as a parameter to DenseBase::operator()(const RowIndices&, const ColIndices&) to index all rows or columns + */ +static const Eigen::internal::all_t all; - EIGEN_DEPRECATED static const all_t all = Eigen::all; // PLEASE use Eigen::all instead of Eigen::placeholders::all - EIGEN_DEPRECATED static const last_t last = Eigen::last; // PLEASE use Eigen::last instead of Eigen::placeholders::last - EIGEN_DEPRECATED static const end_t end = Eigen::lastp1; // PLEASE use Eigen::lastp1 instead of Eigen::placeholders::end -} +} // namespace placeholders } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/IntegralConstant.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/IntegralConstant.h index e0092f654d9..66c031d233d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/IntegralConstant.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/IntegralConstant.h @@ -11,6 +11,9 @@ #ifndef EIGEN_INTEGRAL_CONSTANT_H #define EIGEN_INTEGRAL_CONSTANT_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -29,10 +32,8 @@ template class VariableAndFixedInt; * - arithmetic and some bitwise operators: -, +, *, /, %, &, | * - c++98/14 compatibility with fix and fix() syntax to define integral constants. * - * It is strongly discouraged to directly deal with this class FixedInt. Instances are expcected to - * be created by the user using Eigen::fix or Eigen::fix(). In C++98-11, the former syntax does - * not create a FixedInt instance but rather a point to function that needs to be \em cleaned-up - * using the generic helper: + * It is strongly discouraged to directly deal with this class FixedInt. Instances are expected to + * be created by the user using Eigen::fix or Eigen::fix(). * \code * internal::cleanup_index_type::type * internal::cleanup_index_type::type @@ -53,7 +54,14 @@ template class FixedInt public: static const int value = N; EIGEN_CONSTEXPR operator int() const { return value; } - FixedInt() {} + + EIGEN_CONSTEXPR + FixedInt() = default; + + EIGEN_CONSTEXPR + FixedInt(std::integral_constant) {} + + EIGEN_CONSTEXPR FixedInt( VariableAndFixedInt other) { #ifndef EIGEN_INTERNAL_DEBUGGING EIGEN_UNUSED_VARIABLE(other); @@ -61,34 +69,41 @@ template class FixedInt eigen_internal_assert(int(other)==N); } + EIGEN_CONSTEXPR FixedInt<-N> operator-() const { return FixedInt<-N>(); } + template + EIGEN_CONSTEXPR FixedInt operator+( FixedInt) const { return FixedInt(); } + template + EIGEN_CONSTEXPR FixedInt operator-( FixedInt) const { return FixedInt(); } + template + EIGEN_CONSTEXPR FixedInt operator*( FixedInt) const { return FixedInt(); } + template + EIGEN_CONSTEXPR FixedInt operator/( FixedInt) const { return FixedInt(); } + template + EIGEN_CONSTEXPR FixedInt operator%( FixedInt) const { return FixedInt(); } + template + EIGEN_CONSTEXPR FixedInt operator|( FixedInt) const { return FixedInt(); } + template + EIGEN_CONSTEXPR FixedInt operator&( FixedInt) const { return FixedInt(); } -#if EIGEN_HAS_CXX14_VARIABLE_TEMPLATES // Needed in C++14 to allow fix(): - FixedInt operator() () const { return *this; } + EIGEN_CONSTEXPR FixedInt operator() () const { return *this; } VariableAndFixedInt operator() (int val) const { return VariableAndFixedInt(val); } -#else - FixedInt ( FixedInt (*)() ) {} -#endif - -#if EIGEN_HAS_CXX11 - FixedInt(std::integral_constant) {} -#endif }; /** \internal @@ -138,12 +153,6 @@ template struct get_fixed_value,Default> { static const int value = N; }; -#if !EIGEN_HAS_CXX14_VARIABLE_TEMPLATES -template struct get_fixed_value (*)(),Default> { - static const int value = N; -}; -#endif - template struct get_fixed_value,Default> { static const int value = N ; }; @@ -154,9 +163,6 @@ struct get_fixed_value,Default> { }; template EIGEN_DEVICE_FUNC Index get_runtime_value(const T &x) { return x; } -#if !EIGEN_HAS_CXX14_VARIABLE_TEMPLATES -template EIGEN_DEVICE_FUNC Index get_runtime_value(FixedInt (*)()) { return N; } -#endif // Cleanup integer/FixedInt/VariableAndFixedInt/etc types: @@ -164,38 +170,21 @@ template EIGEN_DEVICE_FUNC Index get_runtime_value(FixedInt (*)()) { r template struct cleanup_index_type { typedef T type; }; // Convert any integral type (e.g., short, int, unsigned int, etc.) to Eigen::Index -template struct cleanup_index_type::value>::type> { typedef Index type; }; - -#if !EIGEN_HAS_CXX14_VARIABLE_TEMPLATES -// In c++98/c++11, fix is a pointer to function that we better cleanup to a true FixedInt: -template struct cleanup_index_type (*)(), DynamicKey> { typedef FixedInt type; }; -#endif +template struct cleanup_index_type::value>> { typedef Index type; }; // If VariableAndFixedInt does not match DynamicKey, then we turn it to a pure compile-time value: template struct cleanup_index_type, DynamicKey> { typedef FixedInt type; }; // If VariableAndFixedInt matches DynamicKey, then we turn it to a pure runtime-value (aka Index): template struct cleanup_index_type, DynamicKey> { typedef Index type; }; -#if EIGEN_HAS_CXX11 template struct cleanup_index_type, DynamicKey> { typedef FixedInt type; }; -#endif } // end namespace internal #ifndef EIGEN_PARSED_BY_DOXYGEN -#if EIGEN_HAS_CXX14_VARIABLE_TEMPLATES -template -static const internal::FixedInt fix{}; -#else template -inline internal::FixedInt fix() { return internal::FixedInt(); } - -// The generic typename T is mandatory. Otherwise, a code like fix could refer to either the function above or this next overload. -// This way a code like fix can only refer to the previous function. -template -inline internal::VariableAndFixedInt fix(T val) { return internal::VariableAndFixedInt(internal::convert_index(val)); } -#endif +constexpr internal::FixedInt fix{}; #else // EIGEN_PARSED_BY_DOXYGEN @@ -221,14 +210,6 @@ inline internal::VariableAndFixedInt fix(T val) { return internal::VariableAn * \c std::integral_constant * Here, \c fix is thus an object of type \c internal::FixedInt. * - * In c++98/11, it is implemented as a function: - * \code - * template inline internal::FixedInt fix(); - * \endcode - * Here internal::FixedInt is thus a pointer to function. - * - * If for some reason you want a true object in c++98 then you can write: \code fix() \endcode which is also valid in c++14. - * * \sa fix(int), seq, seqN */ template diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/MKL_support.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/MKL_support.h index 17963fad4d4..9a91352cd76 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/MKL_support.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/MKL_support.h @@ -120,6 +120,9 @@ #include "../../misc/blas.h" #endif +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { typedef std::complex dcomplex; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Macros.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Macros.h index b436dfad3d7..69bbf2e73a8 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Macros.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Macros.h @@ -10,6 +10,8 @@ #ifndef EIGEN_MACROS_H #define EIGEN_MACROS_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" //------------------------------------------------------------------------------------------ // Eigen version and basic defaults @@ -17,7 +19,7 @@ #define EIGEN_WORLD_VERSION 3 #define EIGEN_MAJOR_VERSION 4 -#define EIGEN_MINOR_VERSION 0 +#define EIGEN_MINOR_VERSION 90 #define EIGEN_VERSION_AT_LEAST(x,y,z) (EIGEN_WORLD_VERSION>x || (EIGEN_WORLD_VERSION>=x && \ (EIGEN_MAJOR_VERSION>y || (EIGEN_MAJOR_VERSION>=y && \ @@ -58,20 +60,27 @@ // Compiler identification, EIGEN_COMP_* //------------------------------------------------------------------------------------------ -/// \internal EIGEN_COMP_GNUC set to 1 for all compilers compatible with GCC +/// \internal EIGEN_COMP_GNUC set to version (e.g., 951 for GCC 9.5.1) for all compilers compatible with GCC #ifdef __GNUC__ - #define EIGEN_COMP_GNUC (__GNUC__*10+__GNUC_MINOR__) + #define EIGEN_COMP_GNUC (__GNUC__*100+__GNUC_MINOR__*10+__GNUC_PATCHLEVEL__) #else #define EIGEN_COMP_GNUC 0 #endif -/// \internal EIGEN_COMP_CLANG set to major+minor version (e.g., 307 for clang 3.7) if the compiler is clang +/// \internal EIGEN_COMP_CLANG set to version (e.g., 372 for clang 3.7.2) if the compiler is clang #if defined(__clang__) - #define EIGEN_COMP_CLANG (__clang_major__*100+__clang_minor__) + #define EIGEN_COMP_CLANG (__clang_major__*100+__clang_minor__*10+__clang_patchlevel__) #else #define EIGEN_COMP_CLANG 0 #endif +/// \internal EIGEN_COMP_CLANGAPPLE set to the version number (e.g. 9000000 for AppleClang 9.0) if the compiler is AppleClang +#if defined(__clang__) && defined(__apple_build_version__) + #define EIGEN_COMP_CLANGAPPLE __apple_build_version__ +#else + #define EIGEN_COMP_CLANGAPPLE 0 +#endif + /// \internal EIGEN_COMP_CASTXML set to 1 if being preprocessed by CastXML #if defined(__castxml__) #define EIGEN_COMP_CASTXML 1 @@ -86,13 +95,20 @@ #define EIGEN_COMP_LLVM 0 #endif -/// \internal EIGEN_COMP_ICC set to __INTEL_COMPILER if the compiler is Intel compiler, 0 otherwise +/// \internal EIGEN_COMP_ICC set to __INTEL_COMPILER if the compiler is Intel icc compiler, 0 otherwise #if defined(__INTEL_COMPILER) #define EIGEN_COMP_ICC __INTEL_COMPILER #else #define EIGEN_COMP_ICC 0 #endif +/// \internal EIGEN_COMP_CLANGICC set to __INTEL_CLANG_COMPILER if the compiler is Intel icx compiler, 0 otherwise +#if defined(__INTEL_CLANG_COMPILER) + #define EIGEN_COMP_CLANGICC __INTEL_CLANG_COMPILER +#else + #define EIGEN_COMP_CLANGICC 0 +#endif + /// \internal EIGEN_COMP_MINGW set to 1 if the compiler is mingw #if defined(__MINGW32__) #define EIGEN_COMP_MINGW 1 @@ -128,10 +144,6 @@ // For the record, here is a table summarizing the possible values for EIGEN_COMP_MSVC: // name ver MSC_VER -// 2008 9 1500 -// 2010 10 1600 -// 2012 11 1700 -// 2013 12 1800 // 2015 14 1900 // "15" 15 1900 // 2017-14.1 15.0 1910 @@ -139,6 +151,9 @@ // 2017-14.12 15.5 1912 // 2017-14.13 15.6 1913 // 2017-14.14 15.7 1914 +// 2017 15.8 1915 +// 2017 15.9 1916 +// 2019 RTW 16.0 1920 /// \internal EIGEN_COMP_MSVC_LANG set to _MSVC_LANG if the compiler is Microsoft Visual C++, 0 otherwise. #if defined(_MSVC_LANG) @@ -193,34 +208,92 @@ #define EIGEN_COMP_EMSCRIPTEN 0 #endif +/// \internal EIGEN_COMP_FCC set to FCC version if the compiler is Fujitsu Compiler (traditional mode) +/// \note The Fujitsu C/C++ compiler uses the traditional mode based +/// on EDG g++ 6.1 by default or if envoked with the -Nnoclang flag +#if defined(__FUJITSU) + #define EIGEN_COMP_FCC (__FCC_major__*100+__FCC_minor__*10+__FCC_patchlevel__) +#else + #define EIGEN_COMP_FCC 0 +#endif -/// \internal EIGEN_GNUC_STRICT set to 1 if the compiler is really GCC and not a compatible compiler (e.g., ICC, clang, mingw, etc.) -#if EIGEN_COMP_GNUC && !(EIGEN_COMP_CLANG || EIGEN_COMP_ICC || EIGEN_COMP_MINGW || EIGEN_COMP_PGI || EIGEN_COMP_IBM || EIGEN_COMP_ARM || EIGEN_COMP_EMSCRIPTEN) - #define EIGEN_COMP_GNUC_STRICT 1 +/// \internal EIGEN_COMP_CLANGFCC set to FCC version if the compiler is Fujitsu Compiler (Clang mode) +/// \note The Fujitsu C/C++ compiler uses the non-traditional mode +/// based on Clang 7.1.0 if envoked with the -Nclang flag +#if defined(__CLANG_FUJITSU) + #define EIGEN_COMP_CLANGFCC (__FCC_major__*100+__FCC_minor__*10+__FCC_patchlevel__) #else - #define EIGEN_COMP_GNUC_STRICT 0 + #define EIGEN_COMP_CLANGFCC 0 +#endif + +/// \internal EIGEN_COMP_CPE set to CPE version if the compiler is HPE Cray Compiler (GCC based) +/// \note This is the SVE-enabled C/C++ compiler from the HPE Cray +/// Programming Environment (CPE) based on Cray GCC 8.1 +#if defined(_CRAYC) && !defined(__clang__) + #define EIGEN_COMP_CPE (_RELEASE_MAJOR*100+_RELEASE_MINOR*10+_RELEASE_PATCHLEVEL) +#else + #define EIGEN_COMP_CPE 0 #endif +/// \internal EIGEN_COMP_CLANGCPE set to CPE version if the compiler is HPE Cray Compiler (Clang based) +/// \note This is the C/C++ compiler from the HPE Cray Programming +/// Environment (CPE) based on Cray Clang 11.0 without SVE-support +#if defined(_CRAYC) && defined(__clang__) + #define EIGEN_COMP_CLANGCPE (_RELEASE_MAJOR*100+_RELEASE_MINOR*10+_RELEASE_PATCHLEVEL) +#else + #define EIGEN_COMP_CLANGCPE 0 +#endif -#if EIGEN_COMP_GNUC - #define EIGEN_GNUC_AT_LEAST(x,y) ((__GNUC__==x && __GNUC_MINOR__>=y) || __GNUC__>x) - #define EIGEN_GNUC_AT_MOST(x,y) ((__GNUC__==x && __GNUC_MINOR__<=y) || __GNUC__ x) || \ + (__GNUC__ == x && __GNUC_MINOR__ > y) || \ + (__GNUC__ == x && __GNUC_MINOR__ == y && __GNUC_PATCHLEVEL__ >= z)) + #define EIGEN_GNUC_STRICT_LESS_THAN(x,y,z) ((__GNUC__ < x) || \ + (__GNUC__ == x && __GNUC_MINOR__ < y) || \ + (__GNUC__ == x && __GNUC_MINOR__ == y && __GNUC_PATCHLEVEL__ < z)) #else -#define EIGEN_GCC3_OR_OLDER 0 + #define EIGEN_GNUC_STRICT_AT_LEAST(x,y,z) 0 + #define EIGEN_GNUC_STRICT_LESS_THAN(x,y,z) 0 #endif +/// \internal EIGEN_COMP_CLANG_STRICT set to 1 if the compiler is really Clang and not a compatible compiler (e.g., AppleClang, etc.) +#if EIGEN_COMP_CLANG && !(EIGEN_COMP_CLANGAPPLE || EIGEN_COMP_CLANGICC || EIGEN_COMP_CLANGFCC || EIGEN_COMP_CLANGCPE) + #define EIGEN_COMP_CLANG_STRICT 1 +#else + #define EIGEN_COMP_CLANG_STRICT 0 +#endif + +// Clang, and compilers forked from it, have different version schemes, so this only makes sense to use with the real Clang. +#if EIGEN_COMP_CLANG_STRICT + #define EIGEN_CLANG_STRICT_AT_LEAST(x,y,z) ((__clang_major__ > x) || \ + (__clang_major__ == x && __clang_minor__ > y) || \ + (__clang_major__ == x && __clang_minor__ == y && __clang_patchlevel__ >= z)) + #define EIGEN_CLANG_STRICT_LESS_THAN(x,y,z) ((__clang_major__ < x) || \ + (__clang_major__ == x && __clang_minor__ < y) || \ + (__clang_major__ == x && __clang_minor__ == y && __clang_patchlevel__ < z)) +#else + #define EIGEN_CLANG_STRICT_AT_LEAST(x,y,z) 0 + #define EIGEN_CLANG_STRICT_LESS_THAN(x,y,z) 0 +#endif + //------------------------------------------------------------------------------------------ // Architecture identification, EIGEN_ARCH_* //------------------------------------------------------------------------------------------ @@ -275,7 +348,7 @@ /// \internal EIGEN_HAS_ARM64_FP16 set to 1 if the architecture provides an IEEE /// compliant Arm fp16 type -#if EIGEN_ARCH_ARM64 +#if EIGEN_ARCH_ARM_OR_ARM64 #ifndef EIGEN_HAS_ARM64_FP16 #if defined(__ARM_FP16_FORMAT_IEEE) #define EIGEN_HAS_ARM64_FP16 1 @@ -285,28 +358,6 @@ #endif #endif -/// \internal EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC set to 1 if the architecture -/// supports Neon vector intrinsics for fp16. -#if EIGEN_ARCH_ARM64 - #ifndef EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC - #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) - #define EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC 1 - #else - #define EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC 0 - #endif - #endif -#endif - -/// \internal EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC set to 1 if the architecture -/// supports Neon scalar intrinsics for fp16. -#if EIGEN_ARCH_ARM64 - #ifndef EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC - #if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) - #define EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC 1 - #endif - #endif -#endif - /// \internal EIGEN_ARCH_MIPS set to 1 if the architecture is MIPS #if defined(__mips__) || defined(__mips) #define EIGEN_ARCH_MIPS 1 @@ -329,7 +380,7 @@ #endif /// \internal EIGEN_ARCH_PPC set to 1 if the architecture is PowerPC -#if defined(__powerpc__) || defined(__ppc__) || defined(_M_PPC) +#if defined(__powerpc__) || defined(__ppc__) || defined(_M_PPC) || defined(__POWERPC__) #define EIGEN_ARCH_PPC 1 #else #define EIGEN_ARCH_PPC 0 @@ -459,12 +510,12 @@ #error "NVCC as the target platform for HIPCC is currently not supported." #endif -#if defined(__CUDACC__) && !defined(EIGEN_NO_CUDA) +#if defined(__CUDACC__) && !defined(EIGEN_NO_CUDA) && !defined(__SYCL_DEVICE_ONLY__) // Means the compiler is either nvcc or clang with CUDA enabled #define EIGEN_CUDACC __CUDACC__ #endif -#if defined(__CUDA_ARCH__) && !defined(EIGEN_NO_CUDA) +#if defined(__CUDA_ARCH__) && !defined(EIGEN_NO_CUDA) && !defined(__SYCL_DEVICE_ONLY__) // Means we are generating code for the device #define EIGEN_CUDA_ARCH __CUDA_ARCH__ #endif @@ -476,7 +527,7 @@ #define EIGEN_CUDA_SDK_VER 0 #endif -#if defined(__HIPCC__) && !defined(EIGEN_NO_HIP) +#if defined(__HIPCC__) && !defined(EIGEN_NO_HIP) && !defined(__SYCL_DEVICE_ONLY__) // Means the compiler is HIPCC (analogous to EIGEN_CUDACC, but for HIP) #define EIGEN_HIPCC __HIPCC__ @@ -485,7 +536,7 @@ // ++ host_defines.h which contains the defines for the __host__ and __device__ macros #include - #if defined(__HIP_DEVICE_COMPILE__) + #if defined(__HIP_DEVICE_COMPILE__) && !defined(__SYCL_DEVICE_ONLY__) // analogous to EIGEN_CUDA_ARCH, but for HIP #define EIGEN_HIP_DEVICE_COMPILE __HIP_DEVICE_COMPILE__ #endif @@ -565,6 +616,32 @@ // #endif +/// \internal EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC set to 1 if the architecture +/// supports Neon vector intrinsics for fp16. +#if EIGEN_ARCH_ARM_OR_ARM64 + #ifndef EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC + // Clang only supports FP16 on aarch64, and not all intrinsics are available + // on A32 anyways even in GCC (e.g. vdiv_f16, vsqrt_f16). + #if EIGEN_ARCH_ARM64 && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(EIGEN_GPU_COMPILE_PHASE) + #define EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC 1 + #else + #define EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC 0 + #endif + #endif +#endif + +/// \internal EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC set to 1 if the architecture +/// supports Neon scalar intrinsics for fp16. +#if EIGEN_ARCH_ARM_OR_ARM64 + #ifndef EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC + // Clang only supports FP16 on aarch64, and not all intrinsics are available + // on A32 anyways, even in GCC (e.g. vceqh_f16). + #if EIGEN_ARCH_ARM64 && defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) && !defined(EIGEN_GPU_COMPILE_PHASE) + #define EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC 1 + #endif + #endif +#endif + #if defined(EIGEN_USE_SYCL) && defined(__SYCL_DEVICE_ONLY__) // EIGEN_USE_SYCL is a user-defined macro while __SYCL_DEVICE_ONLY__ is a compiler-defined macro. // In most cases we want to check if both macros are defined which can be done using the define below. @@ -575,13 +652,6 @@ // Detect Compiler/Architecture/OS specific features //------------------------------------------------------------------------------------------ -#if EIGEN_GNUC_AT_MOST(4,3) && !EIGEN_COMP_CLANG - // see bug 89 - #define EIGEN_SAFE_TO_USE_STANDARD_ASSERT_MACRO 0 -#else - #define EIGEN_SAFE_TO_USE_STANDARD_ASSERT_MACRO 1 -#endif - // Cross compiler wrapper around LLVM's __has_builtin #ifdef __has_builtin # define EIGEN_HAS_BUILTIN(x) __has_builtin(x) @@ -595,16 +665,6 @@ # define __has_feature(x) 0 #endif -// Some old compilers do not support template specializations like: -// template void foo(const T x[N]); -#if !( EIGEN_COMP_CLANG && ( (EIGEN_COMP_CLANG<309) \ - || (defined(__apple_build_version__) && (__apple_build_version__ < 9000000))) \ - || EIGEN_COMP_GNUC_STRICT && EIGEN_COMP_GNUC<49) -#define EIGEN_HAS_STATIC_ARRAY_TEMPLATE 1 -#else -#define EIGEN_HAS_STATIC_ARRAY_TEMPLATE 0 -#endif - // The macro EIGEN_CPLUSPLUS is a replacement for __cplusplus/_MSVC_LANG that // works for both platforms, indicating the C++ standard version number. // @@ -622,14 +682,14 @@ #define EIGEN_CPLUSPLUS 0 #endif -// The macro EIGEN_COMP_CXXVER defines the c++ verson expected by the compiler. +// The macro EIGEN_COMP_CXXVER defines the c++ version expected by the compiler. // For instance, if compiling with gcc and -std=c++17, then EIGEN_COMP_CXXVER // is defined to 17. -#if EIGEN_CPLUSPLUS > 201703L +#if EIGEN_CPLUSPLUS >= 202002L #define EIGEN_COMP_CXXVER 20 -#elif EIGEN_CPLUSPLUS > 201402L +#elif EIGEN_CPLUSPLUS >= 201703L #define EIGEN_COMP_CXXVER 17 -#elif EIGEN_CPLUSPLUS > 201103L +#elif EIGEN_CPLUSPLUS >= 201402L #define EIGEN_COMP_CXXVER 14 #elif EIGEN_CPLUSPLUS >= 201103L #define EIGEN_COMP_CXXVER 11 @@ -637,73 +697,40 @@ #define EIGEN_COMP_CXXVER 03 #endif -#ifndef EIGEN_HAS_CXX14_VARIABLE_TEMPLATES - #if defined(__cpp_variable_templates) && __cpp_variable_templates >= 201304 && EIGEN_MAX_CPP_VER>=14 - #define EIGEN_HAS_CXX14_VARIABLE_TEMPLATES 1 - #else - #define EIGEN_HAS_CXX14_VARIABLE_TEMPLATES 0 - #endif -#endif - // The macros EIGEN_HAS_CXX?? defines a rough estimate of available c++ features -// but in practice we should not rely on them but rather on the availabilty of +// but in practice we should not rely on them but rather on the availability of // individual features as defined later. // This is why there is no EIGEN_HAS_CXX17. -// FIXME: get rid of EIGEN_HAS_CXX14 and maybe even EIGEN_HAS_CXX11. -#if EIGEN_MAX_CPP_VER>=11 && EIGEN_COMP_CXXVER>=11 -#define EIGEN_HAS_CXX11 1 -#else -#define EIGEN_HAS_CXX11 0 -#endif - -#if EIGEN_MAX_CPP_VER>=14 && EIGEN_COMP_CXXVER>=14 -#define EIGEN_HAS_CXX14 1 -#else -#define EIGEN_HAS_CXX14 0 -#endif - -// Do we support r-value references? -#ifndef EIGEN_HAS_RVALUE_REFERENCES -#if EIGEN_MAX_CPP_VER>=11 && \ - (__has_feature(cxx_rvalue_references) || \ - (EIGEN_COMP_CXXVER >= 11) || (EIGEN_COMP_MSVC >= 1600)) - #define EIGEN_HAS_RVALUE_REFERENCES 1 -#else - #define EIGEN_HAS_RVALUE_REFERENCES 0 -#endif +#if EIGEN_MAX_CPP_VER < 14 || EIGEN_COMP_CXXVER < 14 || \ + (EIGEN_COMP_MSVC && EIGEN_COMP_MSVC < 1900) || \ + (EIGEN_COMP_ICC && EIGEN_COMP_ICC < 1500) || \ + (EIGEN_COMP_NVCC && EIGEN_COMP_NVCC < 80000) || \ + (EIGEN_COMP_CLANG_STRICT && EIGEN_COMP_CLANG < 390) || \ + (EIGEN_COMP_CLANGAPPLE && EIGEN_COMP_CLANGAPPLE < 9000000) || \ + (EIGEN_COMP_GNUC_STRICT && EIGEN_COMP_GNUC < 510) +#error This compiler appears to be too old to be supported by Eigen #endif // Does the compiler support C99? // Need to include to make sure _GLIBCXX_USE_C99 gets defined #include #ifndef EIGEN_HAS_C99_MATH -#if EIGEN_MAX_CPP_VER>=11 && \ - ((defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901)) \ +#if ((defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901)) \ || (defined(__GNUC__) && defined(_GLIBCXX_USE_C99)) \ || (defined(_LIBCPP_VERSION) && !defined(_MSC_VER)) \ - || (EIGEN_COMP_MSVC >= 1900) || defined(SYCL_DEVICE_ONLY)) + || (EIGEN_COMP_MSVC) || defined(SYCL_DEVICE_ONLY)) #define EIGEN_HAS_C99_MATH 1 #else #define EIGEN_HAS_C99_MATH 0 #endif #endif -// Does the compiler support result_of? -// result_of was deprecated in c++17 and removed in c++ 20 -#ifndef EIGEN_HAS_STD_RESULT_OF -#if EIGEN_HAS_CXX11 && EIGEN_COMP_CXXVER < 17 -#define EIGEN_HAS_STD_RESULT_OF 1 -#else -#define EIGEN_HAS_STD_RESULT_OF 0 -#endif -#endif - // Does the compiler support std::hash? #ifndef EIGEN_HAS_STD_HASH // The std::hash struct is defined in C++11 but is not labelled as a __device__ // function and is not constexpr, so cannot be used on device. -#if EIGEN_HAS_CXX11 && !defined(EIGEN_GPU_COMPILE_PHASE) +#if !defined(EIGEN_GPU_COMPILE_PHASE) #define EIGEN_HAS_STD_HASH 1 #else #define EIGEN_HAS_STD_HASH 0 @@ -718,147 +745,29 @@ #endif #endif -#ifndef EIGEN_HAS_ALIGNAS -#if EIGEN_MAX_CPP_VER>=11 && EIGEN_HAS_CXX11 && \ - ( __has_feature(cxx_alignas) \ - || EIGEN_HAS_CXX14 \ - || (EIGEN_COMP_MSVC >= 1800) \ - || (EIGEN_GNUC_AT_LEAST(4,8)) \ - || (EIGEN_COMP_CLANG>=305) \ - || (EIGEN_COMP_ICC>=1500) \ - || (EIGEN_COMP_PGI>=1500) \ - || (EIGEN_COMP_SUNCC>=0x5130)) -#define EIGEN_HAS_ALIGNAS 1 -#else -#define EIGEN_HAS_ALIGNAS 0 -#endif -#endif - -// Does the compiler support type_traits? -// - full support of type traits was added only to GCC 5.1.0. -// - 20150626 corresponds to the last release of 4.x libstdc++ -#ifndef EIGEN_HAS_TYPE_TRAITS -#if EIGEN_MAX_CPP_VER>=11 && (EIGEN_HAS_CXX11 || EIGEN_COMP_MSVC >= 1700) \ - && ((!EIGEN_COMP_GNUC_STRICT) || EIGEN_GNUC_AT_LEAST(5, 1)) \ - && ((!defined(__GLIBCXX__)) || __GLIBCXX__ > 20150626) -#define EIGEN_HAS_TYPE_TRAITS 1 -#define EIGEN_INCLUDE_TYPE_TRAITS -#else -#define EIGEN_HAS_TYPE_TRAITS 0 -#endif -#endif - -// Does the compiler support variadic templates? -#ifndef EIGEN_HAS_VARIADIC_TEMPLATES -#if EIGEN_MAX_CPP_VER>=11 && (EIGEN_COMP_CXXVER >= 11) \ - && (!defined(__NVCC__) || !EIGEN_ARCH_ARM_OR_ARM64 || (EIGEN_COMP_NVCC >= 80000) ) - // ^^ Disable the use of variadic templates when compiling with versions of nvcc older than 8.0 on ARM devices: - // this prevents nvcc from crashing when compiling Eigen on Tegra X1 -#define EIGEN_HAS_VARIADIC_TEMPLATES 1 -#elif EIGEN_MAX_CPP_VER>=11 && (EIGEN_COMP_CXXVER >= 11) && defined(SYCL_DEVICE_ONLY) -#define EIGEN_HAS_VARIADIC_TEMPLATES 1 -#else -#define EIGEN_HAS_VARIADIC_TEMPLATES 0 -#endif -#endif - -// Does the compiler fully support const expressions? (as in c++14) -#ifndef EIGEN_HAS_CONSTEXPR - #if defined(EIGEN_CUDACC) - // Const expressions are supported provided that c++11 is enabled and we're using either clang or nvcc 7.5 or above - #if EIGEN_MAX_CPP_VER>=14 && (EIGEN_COMP_CXXVER >= 11 && (EIGEN_COMP_CLANG || EIGEN_COMP_NVCC >= 70500)) - #define EIGEN_HAS_CONSTEXPR 1 - #endif - #elif EIGEN_MAX_CPP_VER>=14 && (__has_feature(cxx_relaxed_constexpr) || (EIGEN_COMP_CXXVER >= 14) || \ - (EIGEN_GNUC_AT_LEAST(4,8) && (EIGEN_COMP_CXXVER >= 11)) || \ - (EIGEN_COMP_CLANG >= 306 && (EIGEN_COMP_CXXVER >= 11))) - #define EIGEN_HAS_CONSTEXPR 1 - #endif - - #ifndef EIGEN_HAS_CONSTEXPR - #define EIGEN_HAS_CONSTEXPR 0 - #endif - -#endif // EIGEN_HAS_CONSTEXPR - -#if EIGEN_HAS_CONSTEXPR #define EIGEN_CONSTEXPR constexpr -#else -#define EIGEN_CONSTEXPR -#endif - -// Does the compiler support C++11 math? -// Let's be conservative and enable the default C++11 implementation only if we are sure it exists -#ifndef EIGEN_HAS_CXX11_MATH - #if EIGEN_MAX_CPP_VER>=11 && ((EIGEN_COMP_CXXVER > 11) || (EIGEN_COMP_CXXVER == 11) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_CLANG || EIGEN_COMP_MSVC || EIGEN_COMP_ICC) \ - && (EIGEN_ARCH_i386_OR_x86_64) && (EIGEN_OS_GNULINUX || EIGEN_OS_WIN_STRICT || EIGEN_OS_MAC)) - #define EIGEN_HAS_CXX11_MATH 1 - #else - #define EIGEN_HAS_CXX11_MATH 0 - #endif -#endif - -// Does the compiler support proper C++11 containers? -#ifndef EIGEN_HAS_CXX11_CONTAINERS - #if EIGEN_MAX_CPP_VER>=11 && \ - ((EIGEN_COMP_CXXVER > 11) \ - || ((EIGEN_COMP_CXXVER == 11) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_CLANG || EIGEN_COMP_MSVC || EIGEN_COMP_ICC>=1400))) - #define EIGEN_HAS_CXX11_CONTAINERS 1 - #else - #define EIGEN_HAS_CXX11_CONTAINERS 0 - #endif -#endif - -// Does the compiler support C++11 noexcept? -#ifndef EIGEN_HAS_CXX11_NOEXCEPT - #if EIGEN_MAX_CPP_VER>=11 && \ - (__has_feature(cxx_noexcept) \ - || (EIGEN_COMP_CXXVER > 11) \ - || ((EIGEN_COMP_CXXVER == 11) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_CLANG || EIGEN_COMP_MSVC || EIGEN_COMP_ICC>=1400))) - #define EIGEN_HAS_CXX11_NOEXCEPT 1 - #else - #define EIGEN_HAS_CXX11_NOEXCEPT 0 - #endif -#endif - -#ifndef EIGEN_HAS_CXX11_ATOMIC - #if EIGEN_MAX_CPP_VER>=11 && \ - (__has_feature(cxx_atomic) \ - || (EIGEN_COMP_CXXVER > 11) \ - || ((EIGEN_COMP_CXXVER == 11) && (EIGEN_COMP_MSVC==0 || EIGEN_COMP_MSVC >= 1700))) - #define EIGEN_HAS_CXX11_ATOMIC 1 - #else - #define EIGEN_HAS_CXX11_ATOMIC 0 - #endif -#endif - -#ifndef EIGEN_HAS_CXX11_OVERRIDE_FINAL - #if EIGEN_MAX_CPP_VER>=11 && \ - (EIGEN_COMP_CXXVER >= 11 || EIGEN_COMP_MSVC >= 1700) - #define EIGEN_HAS_CXX11_OVERRIDE_FINAL 1 - #else - #define EIGEN_HAS_CXX11_OVERRIDE_FINAL 0 - #endif -#endif // NOTE: the required Apple's clang version is very conservative // and it could be that XCode 9 works just fine. // NOTE: the MSVC version is based on https://en.cppreference.com/w/cpp/compiler_support // and not tested. +// NOTE: Intel C++ Compiler Classic (icc) Version 19.0 and later supports dynamic allocation +// for over-aligned data, but not in a manner that is compatible with Eigen. +// See https://gitlab.com/libeigen/eigen/-/issues/2575 #ifndef EIGEN_HAS_CXX17_OVERALIGN -#if EIGEN_MAX_CPP_VER>=17 && EIGEN_COMP_CXXVER>=17 && ( \ - (EIGEN_COMP_MSVC >= 1912) \ - || (EIGEN_GNUC_AT_LEAST(7,0)) \ - || ((!defined(__apple_build_version__)) && (EIGEN_COMP_CLANG>=500)) \ - || (( defined(__apple_build_version__)) && (__apple_build_version__>=10000000)) \ - ) +#if EIGEN_MAX_CPP_VER>=17 && EIGEN_COMP_CXXVER>=17 && ( \ + (EIGEN_COMP_MSVC >= 1912) \ + || (EIGEN_GNUC_STRICT_AT_LEAST(7,0,0)) \ + || (EIGEN_CLANG_STRICT_AT_LEAST(5,0,0)) \ + || (EIGEN_COMP_CLANGAPPLE && EIGEN_COMP_CLANGAPPLE >= 10000000) \ + ) && !EIGEN_COMP_ICC #define EIGEN_HAS_CXX17_OVERALIGN 1 #else #define EIGEN_HAS_CXX17_OVERALIGN 0 #endif #endif -#if defined(EIGEN_CUDACC) && EIGEN_HAS_CONSTEXPR +#if defined(EIGEN_CUDACC) // While available already with c++11, this is useful mostly starting with c++14 and relaxed constexpr rules #if defined(__NVCC__) // nvcc considers constexpr functions as __host__ __device__ with the option --expt-relaxed-constexpr @@ -918,15 +827,11 @@ #endif #endif -// EIGEN_ALWAYS_INLINE is the stronget, it has the effect of making the function inline and adding every possible +// EIGEN_ALWAYS_INLINE is the strongest, it has the effect of making the function inline and adding every possible // attribute to maximize inlining. This should only be used when really necessary: in particular, // it uses __attribute__((always_inline)) on GCC, which most of the time is useless and can severely harm compile times. // FIXME with the always_inline attribute, -// gcc 3.4.x and 4.1 reports the following compilation error: -// Eval.h:91: sorry, unimplemented: inlining failed in call to 'const Eigen::Eval Eigen::MatrixBase::eval() const' -// : function body not available -// See also bug 1367 -#if EIGEN_GNUC_AT_LEAST(4,2) && !defined(SYCL_DEVICE_ONLY) +#if EIGEN_COMP_GNUC && !defined(SYCL_DEVICE_ONLY) #define EIGEN_ALWAYS_INLINE __attribute__((always_inline)) inline #else #define EIGEN_ALWAYS_INLINE EIGEN_STRONG_INLINE @@ -948,8 +853,8 @@ // GPU stuff -// Disable some features when compiling with GPU compilers (NVCC/clang-cuda/SYCL/HIPCC) -#if defined(EIGEN_CUDACC) || defined(SYCL_DEVICE_ONLY) || defined(EIGEN_HIPCC) +// Disable some features when compiling with GPU compilers (SYCL/HIPCC) +#if defined(SYCL_DEVICE_ONLY) || defined(EIGEN_HIP_DEVICE_COMPILE) // Do not try asserts on device code #ifndef EIGEN_NO_DEBUG #define EIGEN_NO_DEBUG @@ -958,7 +863,10 @@ #ifdef EIGEN_INTERNAL_DEBUGGING #undef EIGEN_INTERNAL_DEBUGGING #endif +#endif +// No exceptions on device. +#if defined(SYCL_DEVICE_ONLY) || defined(EIGEN_GPU_COMPILE_PHASE) #ifdef EIGEN_EXCEPTIONS #undef EIGEN_EXCEPTIONS #endif @@ -990,48 +898,6 @@ # endif #endif -// eigen_plain_assert is where we implement the workaround for the assert() bug in GCC <= 4.3, see bug 89 -#ifdef EIGEN_NO_DEBUG - #ifdef SYCL_DEVICE_ONLY // used to silence the warning on SYCL device - #define eigen_plain_assert(x) EIGEN_UNUSED_VARIABLE(x) - #else - #define eigen_plain_assert(x) - #endif -#else - #if EIGEN_SAFE_TO_USE_STANDARD_ASSERT_MACRO - namespace Eigen { - namespace internal { - inline bool copy_bool(bool b) { return b; } - } - } - #define eigen_plain_assert(x) assert(x) - #else - // work around bug 89 - #include // for abort - #include // for std::cerr - - namespace Eigen { - namespace internal { - // trivial function copying a bool. Must be EIGEN_DONT_INLINE, so we implement it after including Eigen headers. - // see bug 89. - namespace { - EIGEN_DONT_INLINE bool copy_bool(bool b) { return b; } - } - inline void assert_fail(const char *condition, const char *function, const char *file, int line) - { - std::cerr << "assertion failed: " << condition << " in function " << function << " at " << file << ":" << line << std::endl; - abort(); - } - } - } - #define eigen_plain_assert(x) \ - do { \ - if(!Eigen::internal::copy_bool(x)) \ - Eigen::internal::assert_fail(EIGEN_MAKESTRING(x), __PRETTY_FUNCTION__, __FILE__, __LINE__); \ - } while(false) - #endif -#endif - // eigen_assert can be overridden #ifndef eigen_assert #define eigen_assert(x) eigen_plain_assert(x) @@ -1040,10 +906,10 @@ #ifdef EIGEN_INTERNAL_DEBUGGING #define eigen_internal_assert(x) eigen_assert(x) #else -#define eigen_internal_assert(x) +#define eigen_internal_assert(x) ((void)0) #endif -#ifdef EIGEN_NO_DEBUG +#if defined(EIGEN_NO_DEBUG) || (defined(EIGEN_GPU_COMPILE_PHASE) && defined(EIGEN_NO_DEBUG_GPU)) #define EIGEN_ONLY_USED_FOR_DEBUG(x) EIGEN_UNUSED_VARIABLE(x) #else #define EIGEN_ONLY_USED_FOR_DEBUG(x) @@ -1067,10 +933,26 @@ #define EIGEN_UNUSED #endif +#if EIGEN_COMP_GNUC + #define EIGEN_PRAGMA(tokens) _Pragma(#tokens) + #define EIGEN_DIAGNOSTICS(tokens) EIGEN_PRAGMA(GCC diagnostic tokens) + #define EIGEN_DIAGNOSTICS_OFF(msc, gcc) EIGEN_DIAGNOSTICS(gcc) +#elif EIGEN_COMP_MSVC + #define EIGEN_PRAGMA(tokens) __pragma(tokens) + #define EIGEN_DIAGNOSTICS(tokens) EIGEN_PRAGMA(warning(tokens)) + #define EIGEN_DIAGNOSTICS_OFF(msc, gcc) EIGEN_DIAGNOSTICS(msc) +#else + #define EIGEN_PRAGMA(tokens) + #define EIGEN_DIAGNOSTICS(tokens) + #define EIGEN_DIAGNOSTICS_OFF(msc, gcc) +#endif + +#define EIGEN_DISABLE_DEPRECATED_WARNING EIGEN_DIAGNOSTICS_OFF(disable : 4996, ignored "-Wdeprecated-declarations") + // Suppresses 'unused variable' warnings. namespace Eigen { namespace internal { - template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ignore_unused_variable(const T&) {} + template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void ignore_unused_variable(const T&) {} } } #define EIGEN_UNUSED_VARIABLE(var) Eigen::internal::ignore_unused_variable(var); @@ -1124,22 +1006,27 @@ namespace Eigen { // directly for std::complex, Eigen::half, Eigen::bfloat16. For these, // you will need to apply to the underlying POD type. #if EIGEN_ARCH_PPC && EIGEN_COMP_GNUC_STRICT - // This seems to be broken on clang. Packet4f is loaded into a single - // register rather than a vector, zeroing out some entries. Integer + // This seems to be broken on clang. Packet4f is loaded into a single + // register rather than a vector, zeroing out some entries. Integer // types also generate a compile error. - // General, Altivec, VSX. - #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__ ("" : "+r,v,wa" (X)); - #elif EIGEN_ARCH_ARM_OR_ARM64 - // General, NEON. - // Clang doesn't like "r", - // error: non-trivial scalar-to-vector conversion, possible invalid - // constraint for vector type - // GCC < 5 doesn't like "g", - // error: 'asm' operand requires impossible reload - #if EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_AT_MOST(5, 0) - #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__ ("" : "+r,w" (X)); + #if EIGEN_OS_MAC + // General, Altivec for Apple (VSX were added in ISA v2.06): + #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__ ("" : "+r,v" (X)); #else + // General, Altivec, VSX otherwise: + #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__ ("" : "+r,v,wa" (X)); + #endif + #elif EIGEN_ARCH_ARM_OR_ARM64 + #ifdef __ARM_FP + // General, VFP or NEON. + // Clang doesn't like "r", + // error: non-trivial scalar-to-vector conversion, possible invalid + // constraint for vector typ #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__ ("" : "+g,w" (X)); + #else + // Arm without VFP or NEON. + // "w" constraint will not compile. + #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__ ("" : "+g" (X)); #endif #elif EIGEN_ARCH_i386_OR_x86_64 // General, SSE. @@ -1194,8 +1081,8 @@ namespace Eigen { #define EIGEN_USING_STD(FUNC) using std::FUNC; #endif -#if EIGEN_COMP_MSVC_STRICT && (EIGEN_COMP_MSVC < 1900 || (EIGEN_COMP_MSVC == 1900 && EIGEN_COMP_NVCC)) - // For older MSVC versions, as well as 1900 && CUDA 8, using the base operator is necessary, +#if EIGEN_COMP_MSVC_STRICT && EIGEN_COMP_NVCC + // Wwhen compiling with NVCC, using the base operator is necessary, // otherwise we get duplicate definition errors // For later MSVC versions, we require explicit operator= definition, otherwise we get // use of implicitly deleted operator errors. @@ -1224,11 +1111,7 @@ namespace Eigen { * \brief Macro to explicitly define the default copy constructor. * This is necessary, because the implicit definition is deprecated if the copy-assignment is overridden. */ -#if EIGEN_HAS_CXX11 #define EIGEN_DEFAULT_COPY_CONSTRUCTOR(CLASS) EIGEN_DEVICE_FUNC CLASS(const CLASS&) = default; -#else -#define EIGEN_DEFAULT_COPY_CONSTRUCTOR(CLASS) -#endif @@ -1248,15 +1131,9 @@ namespace Eigen { * * Hiding the default destructor lead to problems in C++03 mode together with boost::multiprecision */ -#if EIGEN_HAS_CXX11 #define EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(Derived) \ EIGEN_DEVICE_FUNC Derived() = default; \ EIGEN_DEVICE_FUNC ~Derived() = default; -#else -#define EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(Derived) \ - EIGEN_DEVICE_FUNC Derived() {}; \ - /* EIGEN_DEVICE_FUNC ~Derived() {}; */ -#endif @@ -1294,35 +1171,6 @@ namespace Eigen { typedef typename Base::PacketScalar PacketScalar; -#define EIGEN_PLAIN_ENUM_MIN(a,b) (((int)a <= (int)b) ? (int)a : (int)b) -#define EIGEN_PLAIN_ENUM_MAX(a,b) (((int)a >= (int)b) ? (int)a : (int)b) - -// EIGEN_SIZE_MIN_PREFER_DYNAMIC gives the min between compile-time sizes. 0 has absolute priority, followed by 1, -// followed by Dynamic, followed by other finite values. The reason for giving Dynamic the priority over -// finite values is that min(3, Dynamic) should be Dynamic, since that could be anything between 0 and 3. -#define EIGEN_SIZE_MIN_PREFER_DYNAMIC(a,b) (((int)a == 0 || (int)b == 0) ? 0 \ - : ((int)a == 1 || (int)b == 1) ? 1 \ - : ((int)a == Dynamic || (int)b == Dynamic) ? Dynamic \ - : ((int)a <= (int)b) ? (int)a : (int)b) - -// EIGEN_SIZE_MIN_PREFER_FIXED is a variant of EIGEN_SIZE_MIN_PREFER_DYNAMIC comparing MaxSizes. The difference is that finite values -// now have priority over Dynamic, so that min(3, Dynamic) gives 3. Indeed, whatever the actual value is -// (between 0 and 3), it is not more than 3. -#define EIGEN_SIZE_MIN_PREFER_FIXED(a,b) (((int)a == 0 || (int)b == 0) ? 0 \ - : ((int)a == 1 || (int)b == 1) ? 1 \ - : ((int)a == Dynamic && (int)b == Dynamic) ? Dynamic \ - : ((int)a == Dynamic) ? (int)b \ - : ((int)b == Dynamic) ? (int)a \ - : ((int)a <= (int)b) ? (int)a : (int)b) - -// see EIGEN_SIZE_MIN_PREFER_DYNAMIC. No need for a separate variant for MaxSizes here. -#define EIGEN_SIZE_MAX(a,b) (((int)a == Dynamic || (int)b == Dynamic) ? Dynamic \ - : ((int)a >= (int)b) ? (int)a : (int)b) - -#define EIGEN_LOGICAL_XOR(a,b) (((a) || (b)) && !((a) && (b))) - -#define EIGEN_IMPLIES(a,b) (!(a) || (b)) - #if EIGEN_HAS_BUILTIN(__builtin_expect) || EIGEN_COMP_GNUC #define EIGEN_PREDICT_FALSE(x) (__builtin_expect(x, false)) #define EIGEN_PREDICT_TRUE(x) (__builtin_expect(false || (x), true)) @@ -1361,16 +1209,9 @@ namespace Eigen { CwiseBinaryOp::Scalar>, \ const typename internal::plain_constant_type::type, const EXPR> -// Workaround for MSVC 2010 (see ML thread "patch with compile for for MSVC 2010") -#if EIGEN_COMP_MSVC_STRICT && (EIGEN_COMP_MSVC_STRICT<=1600) -#define EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(X) typename internal::enable_if::type -#else -#define EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(X) X -#endif - #define EIGEN_MAKE_SCALAR_BINARY_OP_ONTHERIGHT(METHOD,OPNAME) \ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ - EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,typename internal::promote_scalar_arg::type,OPNAME))\ + const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,typename internal::promote_scalar_arg::type,OPNAME)\ (METHOD)(const T& scalar) const { \ typedef typename internal::promote_scalar_arg::type PromotedT; \ return EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,PromotedT,OPNAME)(derived(), \ @@ -1379,7 +1220,7 @@ namespace Eigen { #define EIGEN_MAKE_SCALAR_BINARY_OP_ONTHELEFT(METHOD,OPNAME) \ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend \ - EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(typename internal::promote_scalar_arg::type,Derived,OPNAME)) \ + const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(typename internal::promote_scalar_arg::type,Derived,OPNAME) \ (METHOD)(const T& scalar, const StorageBaseType& matrix) { \ typedef typename internal::promote_scalar_arg::type PromotedT; \ return EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(PromotedT,Derived,OPNAME)( \ @@ -1417,47 +1258,27 @@ namespace Eigen { #endif -#if EIGEN_HAS_CXX11_NOEXCEPT -# define EIGEN_INCLUDE_TYPE_TRAITS -# define EIGEN_NOEXCEPT noexcept -# define EIGEN_NOEXCEPT_IF(x) noexcept(x) -# define EIGEN_NO_THROW noexcept(true) -# define EIGEN_EXCEPTION_SPEC(X) noexcept(false) -#else -# define EIGEN_NOEXCEPT -# define EIGEN_NOEXCEPT_IF(x) -# define EIGEN_NO_THROW throw() -# if EIGEN_COMP_MSVC || EIGEN_COMP_CXXVER>=17 - // MSVC does not support exception specifications (warning C4290), - // and they are deprecated in c++11 anyway. This is even an error in c++17. -# define EIGEN_EXCEPTION_SPEC(X) throw() -# else -# define EIGEN_EXCEPTION_SPEC(X) throw(X) -# endif -#endif +#define EIGEN_NOEXCEPT noexcept +#define EIGEN_NOEXCEPT_IF(x) noexcept(x) +#define EIGEN_NO_THROW noexcept(true) +#define EIGEN_EXCEPTION_SPEC(X) noexcept(false) + -#if EIGEN_HAS_VARIADIC_TEMPLATES // The all function is used to enable a variadic version of eigen_assert which can take a parameter pack as its input. namespace Eigen { namespace internal { -inline bool all(){ return true; } +EIGEN_DEVICE_FUNC inline bool all(){ return true; } template -bool all(T t, Ts ... ts){ return t && all(ts...); } +EIGEN_DEVICE_FUNC bool all(T t, Ts ... ts){ return t && all(ts...); } } } -#endif -#if EIGEN_HAS_CXX11_OVERRIDE_FINAL // provide override and final specifiers if they are available: -# define EIGEN_OVERRIDE override -# define EIGEN_FINAL final -#else -# define EIGEN_OVERRIDE -# define EIGEN_FINAL -#endif +#define EIGEN_OVERRIDE override +#define EIGEN_FINAL final // Wrapping #pragma unroll in a macro since it is required for SYCL #if defined(SYCL_DEVICE_ONLY) @@ -1470,4 +1291,12 @@ bool all(T t, Ts ... ts){ return t && all(ts...); } #define EIGEN_UNROLL_LOOP #endif +// Notice: Use this macro with caution. The code in the if body should still +// compile with C++14. +#if defined(EIGEN_HAS_CXX17_IFCONSTEXPR) +#define EIGEN_IF_CONSTEXPR(X) if constexpr (X) +#else +#define EIGEN_IF_CONSTEXPR(X) if (X) +#endif + #endif // EIGEN_MACROS_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/MaxSizeVector.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/MaxSizeVector.h new file mode 100644 index 00000000000..ca0e3d1b913 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/MaxSizeVector.h @@ -0,0 +1,158 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_FIXEDSIZEVECTOR_H +#define EIGEN_FIXEDSIZEVECTOR_H + +namespace Eigen { + +/** \class MaxSizeVector + * \ingroup Core + * + * \brief The MaxSizeVector class. + * + * The %MaxSizeVector provides a subset of std::vector functionality. + * + * The goal is to provide basic std::vector operations when using + * std::vector is not an option (e.g. on GPU or when compiling using + * FMA/AVX, as this can cause either compilation failures or illegal + * instruction failures). + * + * Beware: The constructors are not API compatible with these of + * std::vector. + */ +template +class MaxSizeVector { + static const size_t alignment = internal::plain_enum_max(EIGEN_ALIGNOF(T), sizeof(void*)); + public: + // Construct a new MaxSizeVector, reserve n elements. + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit MaxSizeVector(size_t n) + : reserve_(n), size_(0), + data_(static_cast(internal::handmade_aligned_malloc(n * sizeof(T), alignment))) { + } + + // Construct a new MaxSizeVector, reserve and resize to n. + // Copy the init value to all elements. + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + MaxSizeVector(size_t n, const T& init) + : reserve_(n), size_(n), + data_(static_cast(internal::handmade_aligned_malloc(n * sizeof(T), alignment))) { + size_t i = 0; + EIGEN_TRY + { + for(; i < size_; ++i) { new (&data_[i]) T(init); } + } + EIGEN_CATCH(...) + { + // Construction failed, destruct in reverse order: + for(; (i+1) > 0; --i) { data_[i-1].~T(); } + internal::handmade_aligned_free(data_); + EIGEN_THROW; + } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + ~MaxSizeVector() { + for (size_t i = size_; i > 0; --i) { + data_[i-1].~T(); + } + internal::handmade_aligned_free(data_); + } + + void resize(size_t n) { + eigen_assert(n <= reserve_); + for (; size_ < n; ++size_) { + new (&data_[size_]) T; + } + for (; size_ > n; --size_) { + data_[size_-1].~T(); + } + eigen_assert(size_ == n); + } + + // Append new elements (up to reserved size). + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void push_back(const T& t) { + eigen_assert(size_ < reserve_); + new (&data_[size_++]) T(t); + } + + // For C++03 compatibility this only takes one argument + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void emplace_back(const X& x) { + eigen_assert(size_ < reserve_); + new (&data_[size_++]) T(x); + } + + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const T& operator[] (size_t i) const { + eigen_assert(i < size_); + return data_[i]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T& operator[] (size_t i) { + eigen_assert(i < size_); + return data_[i]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T& back() { + eigen_assert(size_ > 0); + return data_[size_ - 1]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const T& back() const { + eigen_assert(size_ > 0); + return data_[size_ - 1]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void pop_back() { + eigen_assert(size_ > 0); + data_[--size_].~T(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + size_t size() const { return size_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + bool empty() const { return size_ == 0; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T* data() { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const T* data() const { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T* begin() { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T* end() { return data_ + size_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const T* begin() const { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const T* end() const { return data_ + size_; } + + private: + size_t reserve_; + size_t size_; + T* data_; +}; + +} // namespace Eigen + +#endif // EIGEN_FIXEDSIZEVECTOR_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Memory.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Memory.h index 875318cdb1c..c87a5c3e08c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Memory.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Memory.h @@ -59,10 +59,61 @@ #endif +#ifndef EIGEN_MALLOC_CHECK_THREAD_LOCAL + +// Check whether we can use the thread_local keyword to allow or disallow +// allocating memory with per-thread granularity, by means of the +// set_is_malloc_allowed() function. +#ifndef EIGEN_AVOID_THREAD_LOCAL + +#if ((EIGEN_COMP_GNUC) || __has_feature(cxx_thread_local) || EIGEN_COMP_MSVC >= 1900) && !defined(EIGEN_GPU_COMPILE_PHASE) +#define EIGEN_MALLOC_CHECK_THREAD_LOCAL thread_local +#else +#define EIGEN_MALLOC_CHECK_THREAD_LOCAL +#endif + +#else // EIGEN_AVOID_THREAD_LOCAL +#define EIGEN_MALLOC_CHECK_THREAD_LOCAL +#endif // EIGEN_AVOID_THREAD_LOCAL + +#endif + +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { +/***************************************************************************** +*** Implementation of portable aligned versions of malloc/free/realloc *** +*****************************************************************************/ + +#ifdef EIGEN_NO_MALLOC +EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() +{ + eigen_assert(false && "heap allocation is forbidden (EIGEN_NO_MALLOC is defined)"); +} +#elif defined EIGEN_RUNTIME_NO_MALLOC +EIGEN_DEVICE_FUNC inline bool is_malloc_allowed_impl(bool update, bool new_value = false) +{ + EIGEN_MALLOC_CHECK_THREAD_LOCAL static bool value = true; + if (update == 1) + value = new_value; + return value; +} +EIGEN_DEVICE_FUNC inline bool is_malloc_allowed() { return is_malloc_allowed_impl(false); } +EIGEN_DEVICE_FUNC inline bool set_is_malloc_allowed(bool new_value) { return is_malloc_allowed_impl(true, new_value); } +EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() +{ + eigen_assert(is_malloc_allowed() && "heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and g_is_malloc_allowed is false)"); +} +#else +EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() +{} +#endif + + EIGEN_DEVICE_FUNC inline void throw_std_bad_alloc() { @@ -94,19 +145,20 @@ inline void throw_std_bad_alloc() /* ----- Hand made implementations of aligned malloc/free and realloc ----- */ -/** \internal Like malloc, but the returned pointer is guaranteed to be 16-byte aligned. - * Fast, but wastes 16 additional bytes of memory. Does not throw any exception. +/** \internal Like malloc, but the returned pointer is guaranteed to be aligned to `alignment`. + * Fast, but wastes `alignment` additional bytes of memory. Does not throw any exception. */ EIGEN_DEVICE_FUNC inline void* handmade_aligned_malloc(std::size_t size, std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES) { - eigen_assert(alignment >= sizeof(void*) && (alignment & (alignment-1)) == 0 && "Alignment must be at least sizeof(void*) and a power of 2"); + eigen_assert(alignment >= sizeof(void*) && alignment <= 128 && (alignment & (alignment-1)) == 0 && "Alignment must be at least sizeof(void*), less than or equal to 128, and a power of 2"); + check_that_malloc_is_allowed(); EIGEN_USING_STD(malloc) - void *original = malloc(size+alignment); - + void* original = malloc(size + alignment); if (original == 0) return 0; - void *aligned = reinterpret_cast((reinterpret_cast(original) & ~(std::size_t(alignment-1))) + alignment); - *(reinterpret_cast(aligned) - 1) = original; + uint8_t offset = static_cast(alignment - (reinterpret_cast(original) & (alignment - 1))); + void* aligned = static_cast(static_cast(original) + offset); + *(static_cast(aligned) - 1) = offset; return aligned; } @@ -114,8 +166,12 @@ EIGEN_DEVICE_FUNC inline void* handmade_aligned_malloc(std::size_t size, std::si EIGEN_DEVICE_FUNC inline void handmade_aligned_free(void *ptr) { if (ptr) { + uint8_t offset = static_cast(*(static_cast(ptr) - 1)); + void* original = static_cast(static_cast(ptr) - offset); + + check_that_malloc_is_allowed(); EIGEN_USING_STD(free) - free(*(reinterpret_cast(ptr) - 1)); + free(original); } } @@ -124,60 +180,39 @@ EIGEN_DEVICE_FUNC inline void handmade_aligned_free(void *ptr) * Since we know that our handmade version is based on std::malloc * we can use std::realloc to implement efficient reallocation. */ -inline void* handmade_aligned_realloc(void* ptr, std::size_t size, std::size_t = 0) +EIGEN_DEVICE_FUNC inline void* handmade_aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size, std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES) { - if (ptr == 0) return handmade_aligned_malloc(size); - void *original = *(reinterpret_cast(ptr) - 1); - std::ptrdiff_t previous_offset = static_cast(ptr)-static_cast(original); - original = std::realloc(original,size+EIGEN_DEFAULT_ALIGN_BYTES); - if (original == 0) return 0; - void *aligned = reinterpret_cast((reinterpret_cast(original) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1))) + EIGEN_DEFAULT_ALIGN_BYTES); - void *previous_aligned = static_cast(original)+previous_offset; - if(aligned!=previous_aligned) - std::memmove(aligned, previous_aligned, size); + if (ptr == nullptr) return handmade_aligned_malloc(new_size, alignment); + uint8_t old_offset = *(static_cast(ptr) - 1); + void* old_original = static_cast(ptr) - old_offset; - *(reinterpret_cast(aligned) - 1) = original; + check_that_malloc_is_allowed(); + EIGEN_USING_STD(realloc) + void* original = realloc(old_original, new_size + alignment); + if (original == nullptr) return nullptr; + if (original == old_original) return ptr; + uint8_t offset = static_cast(alignment - (reinterpret_cast(original) & (alignment - 1))); + void* aligned = static_cast(static_cast(original) + offset); + if (offset != old_offset) { + const void* src = static_cast(static_cast(original) + old_offset); + std::size_t count = (std::min)(new_size, old_size); + std::memmove(aligned, src, count); + } + *(static_cast(aligned) - 1) = offset; return aligned; } -/***************************************************************************** -*** Implementation of portable aligned versions of malloc/free/realloc *** -*****************************************************************************/ - -#ifdef EIGEN_NO_MALLOC -EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() -{ - eigen_assert(false && "heap allocation is forbidden (EIGEN_NO_MALLOC is defined)"); -} -#elif defined EIGEN_RUNTIME_NO_MALLOC -EIGEN_DEVICE_FUNC inline bool is_malloc_allowed_impl(bool update, bool new_value = false) -{ - static bool value = true; - if (update == 1) - value = new_value; - return value; -} -EIGEN_DEVICE_FUNC inline bool is_malloc_allowed() { return is_malloc_allowed_impl(false); } -EIGEN_DEVICE_FUNC inline bool set_is_malloc_allowed(bool new_value) { return is_malloc_allowed_impl(true, new_value); } -EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() -{ - eigen_assert(is_malloc_allowed() && "heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and g_is_malloc_allowed is false)"); -} -#else -EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() -{} -#endif - /** \internal Allocates \a size bytes. The returned pointer is guaranteed to have 16 or 32 bytes alignment depending on the requirements. * On allocation error, the returned pointer is null, and std::bad_alloc is thrown. */ EIGEN_DEVICE_FUNC inline void* aligned_malloc(std::size_t size) { - check_that_malloc_is_allowed(); - + if (size == 0) return nullptr; + void *result; #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED + check_that_malloc_is_allowed(); EIGEN_USING_STD(malloc) result = malloc(size); @@ -199,6 +234,8 @@ EIGEN_DEVICE_FUNC inline void aligned_free(void *ptr) { #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED + if(ptr) + check_that_malloc_is_allowed(); EIGEN_USING_STD(free) free(ptr); @@ -212,13 +249,19 @@ EIGEN_DEVICE_FUNC inline void aligned_free(void *ptr) * \brief Reallocates an aligned block of memory. * \throws std::bad_alloc on allocation failure */ -inline void* aligned_realloc(void *ptr, std::size_t new_size, std::size_t old_size) +EIGEN_DEVICE_FUNC inline void* aligned_realloc(void *ptr, std::size_t new_size, std::size_t old_size) { - EIGEN_UNUSED_VARIABLE(old_size) + if (ptr == nullptr) return aligned_malloc(new_size); + if (old_size == new_size) return ptr; + if (new_size == 0) { aligned_free(ptr); return nullptr; } void *result; #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED - result = std::realloc(ptr,new_size); + EIGEN_UNUSED_VARIABLE(old_size) + + check_that_malloc_is_allowed(); + EIGEN_USING_STD(realloc) + result = realloc(ptr,new_size); #else result = handmade_aligned_realloc(ptr,new_size,old_size); #endif @@ -243,8 +286,9 @@ template EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc(s template<> EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc(std::size_t size) { - check_that_malloc_is_allowed(); + if (size == 0) return nullptr; + check_that_malloc_is_allowed(); EIGEN_USING_STD(malloc) void *result = malloc(size); @@ -261,18 +305,26 @@ template EIGEN_DEVICE_FUNC inline void conditional_aligned_free(void template<> EIGEN_DEVICE_FUNC inline void conditional_aligned_free(void *ptr) { + if(ptr) + check_that_malloc_is_allowed(); EIGEN_USING_STD(free) free(ptr); } -template inline void* conditional_aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size) +template EIGEN_DEVICE_FUNC inline void* conditional_aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size) { return aligned_realloc(ptr, new_size, old_size); } -template<> inline void* conditional_aligned_realloc(void* ptr, std::size_t new_size, std::size_t) +template<> EIGEN_DEVICE_FUNC inline void* conditional_aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size) { - return std::realloc(ptr, new_size); + if (ptr == nullptr) return conditional_aligned_malloc(new_size); + if (old_size == new_size) return ptr; + if (new_size == 0) { conditional_aligned_free(ptr); return nullptr; } + + check_that_malloc_is_allowed(); + EIGEN_USING_STD(realloc) + return realloc(ptr, new_size); } /***************************************************************************** @@ -292,20 +344,55 @@ template EIGEN_DEVICE_FUNC inline void destruct_elements_of_array(T /** \internal Constructs the elements of an array. * The \a size parameter tells on how many objects to call the constructor of T. */ -template EIGEN_DEVICE_FUNC inline T* construct_elements_of_array(T *ptr, std::size_t size) +template EIGEN_DEVICE_FUNC inline T* default_construct_elements_of_array(T *ptr, std::size_t size) { - std::size_t i; + std::size_t i=0; EIGEN_TRY { for (i = 0; i < size; ++i) ::new (ptr + i) T; - return ptr; } EIGEN_CATCH(...) { destruct_elements_of_array(ptr, i); EIGEN_THROW; } - return NULL; + return ptr; +} + +/** \internal Copy-constructs the elements of an array. + * The \a size parameter tells on how many objects to copy. + */ +template EIGEN_DEVICE_FUNC inline T* copy_construct_elements_of_array(T *ptr, const T* src, std::size_t size) +{ + std::size_t i=0; + EIGEN_TRY + { + for (i = 0; i < size; ++i) ::new (ptr + i) T(*(src + i)); + } + EIGEN_CATCH(...) + { + destruct_elements_of_array(ptr, i); + EIGEN_THROW; + } + return ptr; +} + +/** \internal Move-constructs the elements of an array. + * The \a size parameter tells on how many objects to move. + */ +template EIGEN_DEVICE_FUNC inline T* move_construct_elements_of_array(T *ptr, T* src, std::size_t size) +{ + std::size_t i=0; + EIGEN_TRY + { + for (i = 0; i < size; ++i) ::new (ptr + i) T(std::move(*(src + i))); + } + EIGEN_CATCH(...) + { + destruct_elements_of_array(ptr, i); + EIGEN_THROW; + } + return ptr; } /***************************************************************************** @@ -326,10 +413,10 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void check_size_for_overflow(std::size_t s template EIGEN_DEVICE_FUNC inline T* aligned_new(std::size_t size) { check_size_for_overflow(size); - T *result = reinterpret_cast(aligned_malloc(sizeof(T)*size)); + T *result = static_cast(aligned_malloc(sizeof(T)*size)); EIGEN_TRY { - return construct_elements_of_array(result, size); + return default_construct_elements_of_array(result, size); } EIGEN_CATCH(...) { @@ -342,10 +429,10 @@ template EIGEN_DEVICE_FUNC inline T* aligned_new(std::size_t size) template EIGEN_DEVICE_FUNC inline T* conditional_aligned_new(std::size_t size) { check_size_for_overflow(size); - T *result = reinterpret_cast(conditional_aligned_malloc(sizeof(T)*size)); + T *result = static_cast(conditional_aligned_malloc(sizeof(T)*size)); EIGEN_TRY { - return construct_elements_of_array(result, size); + return default_construct_elements_of_array(result, size); } EIGEN_CATCH(...) { @@ -361,7 +448,7 @@ template EIGEN_DEVICE_FUNC inline T* conditional_aligned template EIGEN_DEVICE_FUNC inline void aligned_delete(T *ptr, std::size_t size) { destruct_elements_of_array(ptr, size); - Eigen::internal::aligned_free(ptr); + aligned_free(ptr); } /** \internal Deletes objects constructed with conditional_aligned_new @@ -377,21 +464,32 @@ template EIGEN_DEVICE_FUNC inline T* conditional_aligned { check_size_for_overflow(new_size); check_size_for_overflow(old_size); - if(new_size < old_size) - destruct_elements_of_array(pts+new_size, old_size-new_size); - T *result = reinterpret_cast(conditional_aligned_realloc(reinterpret_cast(pts), sizeof(T)*new_size, sizeof(T)*old_size)); - if(new_size > old_size) + + // If elements need to be explicitly initialized, we cannot simply realloc + // (or memcpy) the memory block - each element needs to be reconstructed. + // Otherwise, objects that contain internal pointers like mpfr or + // AnnoyingScalar can be pointing to the wrong thing. + T* result = static_cast(conditional_aligned_malloc(sizeof(T)*new_size)); + EIGEN_TRY { - EIGEN_TRY - { - construct_elements_of_array(result+old_size, new_size-old_size); - } - EIGEN_CATCH(...) - { - conditional_aligned_free(result); - EIGEN_THROW; + // Move-construct initial elements. + std::size_t copy_size = (std::min)(old_size, new_size); + move_construct_elements_of_array(result, pts, copy_size); + + // Default-construct remaining elements. + if (new_size > old_size) { + default_construct_elements_of_array(result + copy_size, new_size - old_size); } + + // Delete old elements. + conditional_aligned_delete(pts, old_size); + } + EIGEN_CATCH(...) + { + conditional_aligned_free(result); + EIGEN_THROW; } + return result; } @@ -401,12 +499,12 @@ template EIGEN_DEVICE_FUNC inline T* conditional_aligned if(size==0) return 0; // short-cut. Also fixes Bug 884 check_size_for_overflow(size); - T *result = reinterpret_cast(conditional_aligned_malloc(sizeof(T)*size)); + T *result = static_cast(conditional_aligned_malloc(sizeof(T)*size)); if(NumTraits::RequireInitialization) { EIGEN_TRY { - construct_elements_of_array(result, size); + default_construct_elements_of_array(result, size); } EIGEN_CATCH(...) { @@ -417,26 +515,15 @@ template EIGEN_DEVICE_FUNC inline T* conditional_aligned return result; } -template inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size) +template EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size) { + if (NumTraits::RequireInitialization) { + return conditional_aligned_realloc_new(pts, new_size, old_size); + } + check_size_for_overflow(new_size); check_size_for_overflow(old_size); - if(NumTraits::RequireInitialization && (new_size < old_size)) - destruct_elements_of_array(pts+new_size, old_size-new_size); - T *result = reinterpret_cast(conditional_aligned_realloc(reinterpret_cast(pts), sizeof(T)*new_size, sizeof(T)*old_size)); - if(NumTraits::RequireInitialization && (new_size > old_size)) - { - EIGEN_TRY - { - construct_elements_of_array(result+old_size, new_size-old_size); - } - EIGEN_CATCH(...) - { - conditional_aligned_free(result); - EIGEN_THROW; - } - } - return result; + return static_cast(conditional_aligned_realloc(static_cast(pts), sizeof(T)*new_size, sizeof(T)*old_size)); } template EIGEN_DEVICE_FUNC inline void conditional_aligned_delete_auto(T *ptr, std::size_t size) @@ -478,7 +565,7 @@ EIGEN_DEVICE_FUNC inline Index first_aligned(const Scalar* array, Index size) // so that all elements of the array have the same alignment. return 0; } - else if( (UIntPtr(array) & (sizeof(Scalar)-1)) || (Alignment%ScalarSize)!=0) + else if( (std::uintptr_t(array) & (sizeof(Scalar)-1)) || (Alignment%ScalarSize)!=0) { // The array is not aligned to the size of a single scalar, or the requested alignment is not a multiple of the scalar size. // Consequently, no element of the array is well aligned. @@ -486,7 +573,7 @@ EIGEN_DEVICE_FUNC inline Index first_aligned(const Scalar* array, Index size) } else { - Index first = (AlignmentSize - (Index((UIntPtr(array)/sizeof(Scalar))) & AlignmentMask)) & AlignmentMask; + Index first = (AlignmentSize - (Index((std::uintptr_t(array)/sizeof(Scalar))) & AlignmentMask)) & AlignmentMask; return (first < size) ? first : size; } } @@ -520,7 +607,7 @@ template EIGEN_DEVICE_FUNC void smart_copy(const T* start, const T* template struct smart_copy_helper { EIGEN_DEVICE_FUNC static inline void run(const T* start, const T* end, T* target) { - IntPtr size = IntPtr(end)-IntPtr(start); + std::intptr_t size = std::intptr_t(end)-std::intptr_t(start); if(size==0) return; eigen_internal_assert(start!=0 && end!=0 && target!=0); EIGEN_USING_STD(memcpy) @@ -544,7 +631,7 @@ template void smart_memmove(const T* start, const T* end, T* target) template struct smart_memmove_helper { static inline void run(const T* start, const T* end, T* target) { - IntPtr size = IntPtr(end)-IntPtr(start); + std::intptr_t size = std::intptr_t(end)-std::intptr_t(start); if(size==0) return; eigen_internal_assert(start!=0 && end!=0 && target!=0); std::memmove(target, start, size); @@ -554,7 +641,7 @@ template struct smart_memmove_helper { template struct smart_memmove_helper { static inline void run(const T* start, const T* end, T* target) { - if (UIntPtr(target) < UIntPtr(start)) + if (std::uintptr_t(target) < std::uintptr_t(start)) { std::copy(start, end, target); } @@ -566,17 +653,10 @@ template struct smart_memmove_helper { } }; -#if EIGEN_HAS_RVALUE_REFERENCES template EIGEN_DEVICE_FUNC T* smart_move(T* start, T* end, T* target) { return std::move(start, end, target); } -#else -template EIGEN_DEVICE_FUNC T* smart_move(T* start, T* end, T* target) -{ - return std::copy(start, end, target); -} -#endif /***************************************************************************** *** Implementation of runtime stack allocation (falling back to malloc) *** @@ -617,7 +697,7 @@ template class aligned_stack_memory_handler : noncopyable : m_ptr(ptr), m_size(size), m_deallocate(dealloc) { if(NumTraits::RequireInitialization && m_ptr) - Eigen::internal::construct_elements_of_array(m_ptr, size); + Eigen::internal::default_construct_elements_of_array(m_ptr, size); } EIGEN_DEVICE_FUNC ~aligned_stack_memory_handler() @@ -640,7 +720,7 @@ template struct local_nested_eval_wrapper { - static const bool NeedExternalBuffer = false; + static constexpr bool NeedExternalBuffer = false; typedef typename Xpr::Scalar Scalar; typedef typename nested_eval::type ObjectType; ObjectType object; @@ -656,7 +736,7 @@ struct local_nested_eval_wrapper template struct local_nested_eval_wrapper { - static const bool NeedExternalBuffer = true; + static constexpr bool NeedExternalBuffer = true; typedef typename Xpr::Scalar Scalar; typedef typename plain_object_eval::type PlainObject; typedef Map ObjectType; @@ -668,7 +748,7 @@ struct local_nested_eval_wrapper m_deallocate(ptr==0) { if(NumTraits::RequireInitialization && object.data()) - Eigen::internal::construct_elements_of_array(object.data(), object.size()); + Eigen::internal::default_construct_elements_of_array(object.data(), object.size()); object = xpr; } @@ -743,7 +823,7 @@ template void swap(scoped_array &a,scoped_array &b) #if EIGEN_DEFAULT_ALIGN_BYTES>0 // We always manually re-align the result of EIGEN_ALLOCA. // If alloca is already aligned, the compiler should be smart enough to optimize away the re-alignment. - #define EIGEN_ALIGNED_ALLOCA(SIZE) reinterpret_cast((internal::UIntPtr(EIGEN_ALLOCA(SIZE+EIGEN_DEFAULT_ALIGN_BYTES-1)) + EIGEN_DEFAULT_ALIGN_BYTES-1) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1))) + #define EIGEN_ALIGNED_ALLOCA(SIZE) reinterpret_cast((std::uintptr_t(EIGEN_ALLOCA(SIZE+EIGEN_DEFAULT_ALIGN_BYTES-1)) + EIGEN_DEFAULT_ALIGN_BYTES-1) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1))) #else #define EIGEN_ALIGNED_ALLOCA(SIZE) EIGEN_ALLOCA(SIZE) #endif @@ -853,7 +933,7 @@ template void swap(scoped_array &a,scoped_array &b) /** \class aligned_allocator * \ingroup Core_Module * -* \brief STL compatible allocator to use with types requiring a non standrad alignment. +* \brief STL compatible allocator to use with types requiring a non-standard alignment. * * The memory is aligned as for dynamically aligned matrix/array types such as MatrixXd. * By default, it will thus provide at least 16 bytes alignment and more in following cases: @@ -901,7 +981,7 @@ class aligned_allocator : public std::allocator ~aligned_allocator() {} - #if EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_AT_LEAST(7,0) + #if EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_STRICT_AT_LEAST(7,0,0) // In gcc std::allocator::max_size() is bugged making gcc triggers a warning: // eigen/Eigen/src/Core/util/Memory.h:189:12: warning: argument 1 value '18446744073709551612' exceeds maximum object size 9223372036854775807 // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87544 @@ -941,7 +1021,7 @@ class aligned_allocator : public std::allocator __asm__ __volatile__ ("cpuid": "=a" (abcd[0]), "=b" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "0" (func), "2" (id) ); # endif # elif EIGEN_COMP_MSVC -# if (EIGEN_COMP_MSVC > 1500) && EIGEN_ARCH_i386_OR_x86_64 +# if EIGEN_ARCH_i386_OR_x86_64 # define EIGEN_CPUID(abcd,func,id) __cpuidex((int*)abcd,func,id) # endif # endif @@ -1156,6 +1236,38 @@ inline int queryTopLevelCacheSize() return (std::max)(l2,l3); } + + +/** \internal + * This wraps C++20's std::construct_at, using placement new instead if it is not available. + */ + +#if EIGEN_COMP_CXXVER >= 20 +using std::construct_at; +#else +template +EIGEN_DEVICE_FUNC T* construct_at( T* p, Args&&... args ) +{ + return ::new (const_cast(static_cast(p))) + T(std::forward(args)...); +} +#endif + +/** \internal + * This wraps C++17's std::destroy_at. If it's not available it calls the destructor. + * The wrapper is not a full replacement for C++20's std::destroy_at as it cannot + * be applied to std::array. + */ +#if EIGEN_COMP_CXXVER >= 17 +using std::destroy_at; +#else +template +EIGEN_DEVICE_FUNC void destroy_at(T* p) +{ + p->~T(); +} +#endif + } // end namespace internal } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Meta.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Meta.h index 81ae2a32d1d..8e4c278ff11 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Meta.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Meta.h @@ -11,6 +11,9 @@ #ifndef EIGEN_META_H #define EIGEN_META_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + #if defined(EIGEN_GPU_COMPILE_PHASE) #include @@ -25,12 +28,9 @@ #endif -// Recent versions of ICC require for pointer types below. -#define EIGEN_ICC_NEEDS_CSTDINT (EIGEN_COMP_ICC>=1600 && EIGEN_COMP_CXXVER >= 11) - // Define portable (u)int{32,64} types -#if EIGEN_HAS_CXX11 || EIGEN_ICC_NEEDS_CSTDINT #include + namespace Eigen { namespace numext { typedef std::uint8_t uint8_t; @@ -41,25 +41,34 @@ typedef std::uint32_t uint32_t; typedef std::int32_t int32_t; typedef std::uint64_t uint64_t; typedef std::int64_t int64_t; + +template +struct get_integer_by_size { + typedef void signed_type; + typedef void unsigned_type; +}; +template <> +struct get_integer_by_size<1> { + typedef int8_t signed_type; + typedef uint8_t unsigned_type; +}; +template <> +struct get_integer_by_size<2> { + typedef int16_t signed_type; + typedef uint16_t unsigned_type; +}; +template <> +struct get_integer_by_size<4> { + typedef int32_t signed_type; + typedef uint32_t unsigned_type; +}; +template <> +struct get_integer_by_size<8> { + typedef int64_t signed_type; + typedef uint64_t unsigned_type; +}; } } -#else -// Without c++11, all compilers able to compile Eigen also -// provide the C99 stdint.h header file. -#include -namespace Eigen { -namespace numext { -typedef ::uint8_t uint8_t; -typedef ::int8_t int8_t; -typedef ::uint16_t uint16_t; -typedef ::int16_t int16_t; -typedef ::uint32_t uint32_t; -typedef ::int32_t int32_t; -typedef ::uint64_t uint64_t; -typedef ::int64_t int64_t; -} -} -#endif namespace Eigen { @@ -82,17 +91,6 @@ namespace internal { * we however don't want to add a dependency to Boost. */ -// Only recent versions of ICC complain about using ptrdiff_t to hold pointers, -// and older versions do not provide *intptr_t types. -#if EIGEN_ICC_NEEDS_CSTDINT -typedef std::intptr_t IntPtr; -typedef std::uintptr_t UIntPtr; -#else -typedef std::ptrdiff_t IntPtr; -typedef std::size_t UIntPtr; -#endif -#undef EIGEN_ICC_NEEDS_CSTDINT - struct true_type { enum { value = 1 }; }; struct false_type { enum { value = 0 }; }; @@ -105,23 +103,11 @@ struct bool_constant : true_type {}; template<> struct bool_constant : false_type {}; -template -struct conditional { typedef Then type; }; - -template -struct conditional { typedef Else type; }; - -template struct remove_reference { typedef T type; }; -template struct remove_reference { typedef T type; }; - -template struct remove_pointer { typedef T type; }; -template struct remove_pointer { typedef T type; }; -template struct remove_pointer { typedef T type; }; - -template struct remove_const { typedef T type; }; -template struct remove_const { typedef T type; }; -template struct remove_const { typedef T type[]; }; -template struct remove_const { typedef T type[Size]; }; +// Third-party libraries rely on these. +using std::conditional; +using std::remove_reference; +using std::remove_pointer; +using std::remove_const; template struct remove_all { typedef T type; }; template struct remove_all { typedef typename remove_all::type type; }; @@ -130,10 +116,16 @@ template struct remove_all { typedef typename remove_all< template struct remove_all { typedef typename remove_all::type type; }; template struct remove_all { typedef typename remove_all::type type; }; +template +using remove_all_t = typename remove_all::type; + template struct is_arithmetic { enum { value = false }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; +// GPU devices treat `long double` as `double`. +#ifndef EIGEN_GPU_COMPILE_PHASE template<> struct is_arithmetic { enum { value = true }; }; +#endif template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; @@ -149,64 +141,28 @@ template struct is_same { enum { value = 0 }; }; template struct is_same { enum { value = 1 }; }; template< class T > -struct is_void : is_same::type> {}; +struct is_void : is_same> {}; + +/** \internal + * Implementation of std::void_t for SFINAE. + * + * Pre C++17: + * Custom implementation. + * + * Post C++17: Uses std::void_t + */ +#if EIGEN_COMP_CXXVER >= 17 +using std::void_t; +#else +template +using void_t = void; +#endif -#if EIGEN_HAS_CXX11 template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; using std::is_integral; -#else -template struct is_integral { enum { value = false }; }; -template<> struct is_integral { enum { value = true }; }; -template<> struct is_integral { enum { value = true }; }; -template<> struct is_integral { enum { value = true }; }; -template<> struct is_integral { enum { value = true }; }; -template<> struct is_integral { enum { value = true }; }; -template<> struct is_integral { enum { value = true }; }; -template<> struct is_integral { enum { value = true }; }; -template<> struct is_integral { enum { value = true }; }; -template<> struct is_integral { enum { value = true }; }; -template<> struct is_integral { enum { value = true }; }; -#if EIGEN_COMP_MSVC -template<> struct is_integral { enum { value = true }; }; -template<> struct is_integral { enum { value = true }; }; -#endif -#endif -#if EIGEN_HAS_CXX11 using std::make_unsigned; -#else -// TODO: Possibly improve this implementation of make_unsigned. -// It is currently used only by -// template struct random_default_impl. -template struct make_unsigned; -template<> struct make_unsigned { typedef unsigned char type; }; -template<> struct make_unsigned { typedef unsigned char type; }; -template<> struct make_unsigned { typedef unsigned char type; }; -template<> struct make_unsigned { typedef unsigned short type; }; -template<> struct make_unsigned { typedef unsigned short type; }; -template<> struct make_unsigned { typedef unsigned int type; }; -template<> struct make_unsigned { typedef unsigned int type; }; -template<> struct make_unsigned { typedef unsigned long type; }; -template<> struct make_unsigned { typedef unsigned long type; }; -#if EIGEN_COMP_MSVC -template<> struct make_unsigned { typedef unsigned __int64 type; }; -template<> struct make_unsigned { typedef unsigned __int64 type; }; -#endif - -// Some platforms define int64_t as `long long` even for C++03, where -// `long long` is not guaranteed by the standard. In this case we are missing -// the definition for make_unsigned. If we just define it, we run into issues -// where `long long` doesn't exist in some compilers for C++03. We therefore add -// the specialization for these platforms only. -#if EIGEN_OS_MAC || EIGEN_COMP_MINGW -template<> struct make_unsigned { typedef unsigned long long type; }; -template<> struct make_unsigned { typedef unsigned long long type; }; -#endif -#endif - -template struct add_const { typedef const T type; }; -template struct add_const { typedef T& type; }; template struct is_const { enum { value = 0 }; }; template struct is_const { enum { value = 1 }; }; @@ -217,204 +173,10 @@ template struct add_const_on_value_type { typedef T const template struct add_const_on_value_type { typedef T const* const type; }; template struct add_const_on_value_type { typedef T const* const type; }; -#if EIGEN_HAS_CXX11 - -using std::is_convertible; - -#else - -template -struct is_convertible_impl -{ -private: - struct any_conversion - { - template any_conversion(const volatile T&); - template any_conversion(T&); - }; - struct yes {int a[1];}; - struct no {int a[2];}; - - template - static yes test(T, int); - - template - static no test(any_conversion, ...); - -public: - static typename internal::remove_reference::type* ms_from; -#ifdef __INTEL_COMPILER - #pragma warning push - #pragma warning ( disable : 2259 ) -#endif - enum { value = sizeof(test(*ms_from, 0))==sizeof(yes) }; -#ifdef __INTEL_COMPILER - #pragma warning pop -#endif -}; - -template -struct is_convertible -{ - enum { value = is_convertible_impl::value }; -}; - template -struct is_convertible { enum { value = false }; }; - -template -struct is_convertible { enum { value = true }; }; - -#endif - -/** \internal Allows to enable/disable an overload - * according to a compile time condition. - */ -template struct enable_if; - -template struct enable_if -{ typedef T type; }; - -#if defined(EIGEN_GPU_COMPILE_PHASE) && !EIGEN_HAS_CXX11 -#if !defined(__FLT_EPSILON__) -#define __FLT_EPSILON__ FLT_EPSILON -#define __DBL_EPSILON__ DBL_EPSILON -#endif - -namespace device { - -template struct numeric_limits -{ - EIGEN_DEVICE_FUNC - static EIGEN_CONSTEXPR T epsilon() { return 0; } - static T (max)() { assert(false && "Highest not supported for this type"); } - static T (min)() { assert(false && "Lowest not supported for this type"); } - static T infinity() { assert(false && "Infinity not supported for this type"); } - static T quiet_NaN() { assert(false && "quiet_NaN not supported for this type"); } -}; -template<> struct numeric_limits -{ - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static float epsilon() { return __FLT_EPSILON__; } - EIGEN_DEVICE_FUNC - static float (max)() { - #if defined(EIGEN_CUDA_ARCH) - return CUDART_MAX_NORMAL_F; - #else - return HIPRT_MAX_NORMAL_F; - #endif - } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static float (min)() { return FLT_MIN; } - EIGEN_DEVICE_FUNC - static float infinity() { - #if defined(EIGEN_CUDA_ARCH) - return CUDART_INF_F; - #else - return HIPRT_INF_F; - #endif - } - EIGEN_DEVICE_FUNC - static float quiet_NaN() { - #if defined(EIGEN_CUDA_ARCH) - return CUDART_NAN_F; - #else - return HIPRT_NAN_F; - #endif - } -}; -template<> struct numeric_limits -{ - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static double epsilon() { return __DBL_EPSILON__; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static double (max)() { return DBL_MAX; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static double (min)() { return DBL_MIN; } - EIGEN_DEVICE_FUNC - static double infinity() { - #if defined(EIGEN_CUDA_ARCH) - return CUDART_INF; - #else - return HIPRT_INF; - #endif - } - EIGEN_DEVICE_FUNC - static double quiet_NaN() { - #if defined(EIGEN_CUDA_ARCH) - return CUDART_NAN; - #else - return HIPRT_NAN; - #endif - } -}; -template<> struct numeric_limits -{ - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static int epsilon() { return 0; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static int (max)() { return INT_MAX; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static int (min)() { return INT_MIN; } -}; -template<> struct numeric_limits -{ - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static unsigned int epsilon() { return 0; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static unsigned int (max)() { return UINT_MAX; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static unsigned int (min)() { return 0; } -}; -template<> struct numeric_limits -{ - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static long epsilon() { return 0; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static long (max)() { return LONG_MAX; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static long (min)() { return LONG_MIN; } -}; -template<> struct numeric_limits -{ - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static unsigned long epsilon() { return 0; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static unsigned long (max)() { return ULONG_MAX; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static unsigned long (min)() { return 0; } -}; -template<> struct numeric_limits -{ - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static long long epsilon() { return 0; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static long long (max)() { return LLONG_MAX; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static long long (min)() { return LLONG_MIN; } -}; -template<> struct numeric_limits -{ - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static unsigned long long epsilon() { return 0; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static unsigned long long (max)() { return ULLONG_MAX; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static unsigned long long (min)() { return 0; } -}; -template<> struct numeric_limits -{ - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static bool epsilon() { return false; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static bool (max)() { return true; } - EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR - static bool (min)() { return false; } -}; - -} +using add_const_on_value_type_t = typename add_const_on_value_type::type; -#endif // defined(EIGEN_GPU_COMPILE_PHASE) && !EIGEN_HAS_CXX11 +using std::is_convertible; /** \internal * A base class do disable default copy ctor and copy assignment operator. @@ -446,7 +208,7 @@ template struct array_size { enum { value = Dynamic }; }; -template struct array_size::type> { +template struct array_size> { enum { value = T::SizeAtCompileTime }; }; @@ -457,44 +219,50 @@ template struct array_size { enum { value = N }; }; -#if EIGEN_HAS_CXX11 template struct array_size > { enum { value = N }; }; template struct array_size > { enum { value = N }; }; -#endif + /** \internal - * Analogue of the std::size free function. - * It returns the size of the container or view \a x of type \c T + * Analogue of the std::ssize free function. + * It returns the signed size of the container or view \a x of type \c T * * It currently supports: * - any types T defining a member T::size() const * - plain C arrays as T[N] * + * For C++20, this function just forwards to `std::ssize`, or any ADL discoverable `ssize` function. */ -template -EIGEN_CONSTEXPR Index size(const T& x) { return x.size(); } +#if EIGEN_COMP_CXXVER < 20 || EIGEN_GNUC_STRICT_LESS_THAN(10,0,0) +template +EIGEN_CONSTEXPR auto index_list_size(const T& x) { + using R = std::common_type_t>; + return static_cast(x.size()); +} -template -EIGEN_CONSTEXPR Index size(const T (&) [N]) { return N; } +template +EIGEN_CONSTEXPR std::ptrdiff_t index_list_size(const T (&)[N]) { return N; } +#else +template +EIGEN_CONSTEXPR auto index_list_size(T&& x) { + using std::ssize; + return ssize(std::forward(x)); +} +#endif // EIGEN_COMP_CXXVER /** \internal * Convenient struct to get the result type of a nullary, unary, binary, or * ternary functor. - * - * Pre C++11: - * Supports both a Func::result_type member and templated - * Func::result::type member. - * - * If none of these members is provided, then the type of the first - * argument is returned. - * - * Post C++11: + * + * Pre C++17: * This uses std::result_of. However, note the `type` member removes * const and converts references/pointers to their corresponding value type. + * + * Post C++17: Uses std::invoke_result */ #if EIGEN_HAS_STD_INVOKE_RESULT template struct result_of; @@ -502,151 +270,36 @@ template struct result_of; template struct result_of { typedef typename std::invoke_result::type type1; - typedef typename remove_all::type type; -}; -#elif EIGEN_HAS_STD_RESULT_OF -template struct result_of { - typedef typename std::result_of::type type1; - typedef typename remove_all::type type; -}; -#else -template struct result_of { }; - -struct has_none {int a[1];}; -struct has_std_result_type {int a[2];}; -struct has_tr1_result {int a[3];}; - -template -struct nullary_result_of_select {}; - -template -struct nullary_result_of_select {typedef typename Func::result_type type;}; - -template -struct nullary_result_of_select {typedef typename Func::template result::type type;}; - -template -struct result_of { - template - static has_std_result_type testFunctor(T const *, typename T::result_type const * = 0); - template - static has_tr1_result testFunctor(T const *, typename T::template result::type const * = 0); - static has_none testFunctor(...); - - // note that the following indirection is needed for gcc-3.3 - enum {FunctorType = sizeof(testFunctor(static_cast(0)))}; - typedef typename nullary_result_of_select::type type; -}; - -template -struct unary_result_of_select {typedef typename internal::remove_all::type type;}; - -template -struct unary_result_of_select {typedef typename Func::result_type type;}; - -template -struct unary_result_of_select {typedef typename Func::template result::type type;}; - -template -struct result_of { - template - static has_std_result_type testFunctor(T const *, typename T::result_type const * = 0); - template - static has_tr1_result testFunctor(T const *, typename T::template result::type const * = 0); - static has_none testFunctor(...); - - // note that the following indirection is needed for gcc-3.3 - enum {FunctorType = sizeof(testFunctor(static_cast(0)))}; - typedef typename unary_result_of_select::type type; + typedef remove_all_t type; }; -template -struct binary_result_of_select {typedef typename internal::remove_all::type type;}; - -template -struct binary_result_of_select -{typedef typename Func::result_type type;}; - -template -struct binary_result_of_select -{typedef typename Func::template result::type type;}; - -template -struct result_of { - template - static has_std_result_type testFunctor(T const *, typename T::result_type const * = 0); - template - static has_tr1_result testFunctor(T const *, typename T::template result::type const * = 0); - static has_none testFunctor(...); - - // note that the following indirection is needed for gcc-3.3 - enum {FunctorType = sizeof(testFunctor(static_cast(0)))}; - typedef typename binary_result_of_select::type type; -}; - -template -struct ternary_result_of_select {typedef typename internal::remove_all::type type;}; - -template -struct ternary_result_of_select -{typedef typename Func::result_type type;}; - -template -struct ternary_result_of_select -{typedef typename Func::template result::type type;}; - -template -struct result_of { - template - static has_std_result_type testFunctor(T const *, typename T::result_type const * = 0); - template - static has_tr1_result testFunctor(T const *, typename T::template result::type const * = 0); - static has_none testFunctor(...); - - // note that the following indirection is needed for gcc-3.3 - enum {FunctorType = sizeof(testFunctor(static_cast(0)))}; - typedef typename ternary_result_of_select::type type; -}; - -#endif - -#if EIGEN_HAS_STD_INVOKE_RESULT template struct invoke_result { typedef typename std::invoke_result::type type1; - typedef typename remove_all::type type; -}; -#elif EIGEN_HAS_CXX11 -template -struct invoke_result { - typedef typename result_of::type type1; - typedef typename remove_all::type type; + typedef remove_all_t type; }; #else -template -struct invoke_result { - typedef typename result_of::type type1; - typedef typename remove_all::type type; +template struct result_of { + typedef typename std::result_of::type type1; + typedef remove_all_t type; }; -template -struct invoke_result { - typedef typename result_of::type type1; - typedef typename remove_all::type type; +template +struct invoke_result { + typedef typename result_of::type type1; + typedef remove_all_t type; }; +#endif -template -struct invoke_result { - typedef typename result_of::type type1; - typedef typename remove_all::type type; -}; +// Reduces a sequence of bools to true if all are true, false otherwise. +template +using reduce_all = std::is_same, + std::integer_sequence >; -template -struct invoke_result { - typedef typename result_of::type type1; - typedef typename remove_all::type type; -}; -#endif +// Reduces a sequence of bools to true if any are true, false if all false. +template +using reduce_any = std::integral_constant, std::integer_sequence >::value>; struct meta_yes { char a[1]; }; struct meta_no { char a[2]; }; @@ -666,7 +319,7 @@ template const T* return_ptr(); template struct has_nullary_operator { - template static meta_yes testFunctor(C const *,typename enable_if<(sizeof(return_ptr()->operator()())>0)>::type * = 0); + template static meta_yes testFunctor(C const *,std::enable_if_t<(sizeof(return_ptr()->operator()())>0)> * = 0); static meta_no testFunctor(...); enum { value = sizeof(testFunctor(static_cast(0))) == sizeof(meta_yes) }; @@ -675,7 +328,7 @@ struct has_nullary_operator template struct has_unary_operator { - template static meta_yes testFunctor(C const *,typename enable_if<(sizeof(return_ptr()->operator()(IndexType(0)))>0)>::type * = 0); + template static meta_yes testFunctor(C const *,std::enable_if_t<(sizeof(return_ptr()->operator()(IndexType(0)))>0)> * = 0); static meta_no testFunctor(...); enum { value = sizeof(testFunctor(static_cast(0))) == sizeof(meta_yes) }; @@ -684,7 +337,7 @@ struct has_unary_operator template struct has_binary_operator { - template static meta_yes testFunctor(C const *,typename enable_if<(sizeof(return_ptr()->operator()(IndexType(0),IndexType(0)))>0)>::type * = 0); + template static meta_yes testFunctor(C const *,std::enable_if_t<(sizeof(return_ptr()->operator()(IndexType(0),IndexType(0)))>0)> * = 0); static meta_no testFunctor(...); enum { value = sizeof(testFunctor(static_cast(0))) == sizeof(meta_yes) }; @@ -696,8 +349,7 @@ struct has_binary_operator template Y))) > - // use ?: instead of || just to shut up a stupid gcc 4.3 warning + bool Done = ((SupX - InfX) <= 1 || ((SupX * SupX <= Y) && ((SupX + 1) * (SupX + 1) > Y)))> class meta_sqrt { enum { @@ -743,7 +395,7 @@ template struct scalar_product_traits // FIXME quick workaround around current limitation of result_of // template // struct result_of(ArgType0,ArgType1)> { -// typedef typename scalar_product_traits::type, typename remove_all::type>::ReturnType type; +// typedef typename scalar_product_traits, remove_all_t>::ReturnType type; // }; /** \internal Obtains a POD type suitable to use as storage for an object of a size @@ -758,6 +410,8 @@ struct aligned_storage { } // end namespace internal +template struct NumTraits; + namespace numext { #if defined(EIGEN_GPU_COMPILE_PHASE) @@ -766,25 +420,36 @@ template EIGEN_DEVICE_FUNC void swap(T &a, T &b) { T tmp = b; b = template EIGEN_STRONG_INLINE void swap(T &a, T &b) { std::swap(a,b); } #endif -#if defined(EIGEN_GPU_COMPILE_PHASE) && !EIGEN_HAS_CXX11 -using internal::device::numeric_limits; -#else using std::numeric_limits; -#endif -// Integer division with rounding up. -// T is assumed to be an integer type with a>=0, and b>0 -template -EIGEN_DEVICE_FUNC -T div_ceil(const T &a, const T &b) -{ - return (a+b-1) / b; -} +// Handle integer comparisons of different signedness. +template ::IsInteger, bool XIsSigned = NumTraits::IsSigned, + bool YIsInteger = NumTraits::IsInteger, bool YIsSigned = NumTraits::IsSigned> +struct equal_strict_impl { + static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool run(const X& x, const Y& y) { return x == y; } +}; +template +struct equal_strict_impl { + // X is an unsigned integer + // Y is a signed integer + // if Y is non-negative, it may be represented exactly as its unsigned counterpart. + using UnsignedY = typename internal::make_unsigned::type; + static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool run(const X& x, const Y& y) { + return y < Y(0) ? false : (x == static_cast(y)); + } +}; +template +struct equal_strict_impl { + // X is a signed integer + // Y is an unsigned integer + static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool run(const X& x, const Y& y) { + return equal_strict_impl::run(y, x); + } +}; // The aim of the following functions is to bypass -Wfloat-equal warnings // when we really want a strict equality comparison on floating points. -template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC -bool equal_strict(const X& x,const Y& y) { return x == y; } +template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool equal_strict(const X& x, const Y& y) { return equal_strict_impl::run(x, y); } #if !defined(EIGEN_GPU_COMPILE_PHASE) || (!defined(EIGEN_CUDA_ARCH) && defined(EIGEN_CONSTEXPR_ARE_DEVICE_FUNC)) template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC @@ -794,8 +459,22 @@ template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool equal_strict(const double& x,const double& y) { return std::equal_to()(x,y); } #endif +/** + * \internal Performs an exact comparison of x to zero, e.g. to decide whether a term can be ignored. + * Use this to to bypass -Wfloat-equal warnings when exact zero is what needs to be tested. +*/ +template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC +bool is_exactly_zero(const X& x) { return equal_strict(x, typename NumTraits::Literal{0}); } + +/** + * \internal Performs an exact comparison of x to one, e.g. to decide whether a factor needs to be multiplied. + * Use this to to bypass -Wfloat-equal warnings when exact one is what needs to be tested. +*/ +template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC +bool is_exactly_one(const X& x) { return equal_strict(x, typename NumTraits::Literal{1}); } + template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC -bool not_equal_strict(const X& x,const Y& y) { return x != y; } +bool not_equal_strict(const X& x,const Y& y) { return !equal_strict_impl::run(x, y); } #if !defined(EIGEN_GPU_COMPILE_PHASE) || (!defined(EIGEN_CUDA_ARCH) && defined(EIGEN_CONSTEXPR_ARE_DEVICE_FUNC)) template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC @@ -807,6 +486,101 @@ bool not_equal_strict(const double& x,const double& y) { return std::not_equal_t } // end namespace numext +namespace internal { + +template +struct is_identically_zero_impl { + static inline bool run(const Scalar& s) { + return numext::is_exactly_zero(s); + } +}; + +template EIGEN_STRONG_INLINE +bool is_identically_zero(const Scalar& s) { return is_identically_zero_impl::run(s); } + +/// \internal Returns true if its argument is of integer or enum type. +/// FIXME this has the same purpose as `is_valid_index_type` in XprHelper.h +template +constexpr bool is_int_or_enum_v = std::is_enum::value || std::is_integral::value; + +/// \internal Gets the minimum of two values which may be integers or enums +template +inline constexpr int plain_enum_min(A a, B b) { + static_assert(is_int_or_enum_v, "Argument a must be an integer or enum"); + static_assert(is_int_or_enum_v, "Argument b must be an integer or enum"); + return ((int) a <= (int) b) ? (int) a : (int) b; +} + +/// \internal Gets the maximum of two values which may be integers or enums +template +inline constexpr int plain_enum_max(A a, B b) { + static_assert(is_int_or_enum_v, "Argument a must be an integer or enum"); + static_assert(is_int_or_enum_v, "Argument b must be an integer or enum"); + return ((int) a >= (int) b) ? (int) a : (int) b; +} + +/** + * \internal + * `min_size_prefer_dynamic` gives the min between compile-time sizes. 0 has absolute priority, followed by 1, + * followed by Dynamic, followed by other finite values. The reason for giving Dynamic the priority over + * finite values is that min(3, Dynamic) should be Dynamic, since that could be anything between 0 and 3. + */ +template +inline constexpr int min_size_prefer_dynamic(A a, B b) { + static_assert(is_int_or_enum_v, "Argument a must be an integer or enum"); + static_assert(is_int_or_enum_v, "Argument b must be an integer or enum"); + if ((int) a == 0 || (int) b == 0) return 0; + if ((int) a == 1 || (int) b == 1) return 1; + if ((int) a == Dynamic || (int) b == Dynamic) return Dynamic; + return plain_enum_min(a, b); +} + +/** + * \internal + * min_size_prefer_fixed is a variant of `min_size_prefer_dynamic` comparing MaxSizes. The difference is that finite values + * now have priority over Dynamic, so that min(3, Dynamic) gives 3. Indeed, whatever the actual value is + * (between 0 and 3), it is not more than 3. + */ +template +inline constexpr int min_size_prefer_fixed(A a, B b) { + static_assert(is_int_or_enum_v, "Argument a must be an integer or enum"); + static_assert(is_int_or_enum_v, "Argument b must be an integer or enum"); + if ((int) a == 0 || (int) b == 0) return 0; + if ((int) a == 1 || (int) b == 1) return 1; + if ((int) a == Dynamic && (int) b == Dynamic) return Dynamic; + if ((int) a == Dynamic) return (int) b; + if ((int) b == Dynamic) return (int) a; + return plain_enum_min(a, b); +} + +/// \internal see `min_size_prefer_fixed`. No need for a separate variant for MaxSizes here. +template +inline constexpr int max_size_prefer_dynamic(A a, B b) { + static_assert(is_int_or_enum_v, "Argument a must be an integer or enum"); + static_assert(is_int_or_enum_v, "Argument b must be an integer or enum"); + if ((int) a == Dynamic || (int) b == Dynamic) return Dynamic; + return plain_enum_max(a, b); +} + +/// \internal Calculate logical XOR at compile time +inline constexpr bool logical_xor(bool a, bool b) { + return a != b; +} + +/// \internal Calculate logical IMPLIES at compile time +inline constexpr bool check_implication(bool a, bool b) { + return !a || b; +} + +/// \internal Provide fallback for std::is_constant_evaluated for pre-C++20. +#if EIGEN_COMP_CXXVER >= 20 +using std::is_constant_evaluated; +#else +constexpr bool is_constant_evaluated() { return false; } +#endif + +} // end namespace internal + } // end namespace Eigen #endif // EIGEN_META_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/MoreMeta.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/MoreMeta.h new file mode 100644 index 00000000000..403d7725c03 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/MoreMeta.h @@ -0,0 +1,532 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2008-2015 Gael Guennebaud +// Copyright (C) 2006-2008 Benoit Jacob +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_MOREMETA_H +#define EIGEN_MOREMETA_H + +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +template +struct type_list { constexpr static int count = sizeof...(tt); }; + +template +struct type_list { constexpr static int count = sizeof...(tt) + 1; typedef t first_type; }; + +template +struct numeric_list { constexpr static std::size_t count = sizeof...(nn); }; + +template +struct numeric_list { static constexpr std::size_t count = sizeof...(nn) + 1; + static constexpr T first_value = n; }; + +#ifndef EIGEN_PARSED_BY_DOXYGEN +/* numeric list constructors + * + * equivalencies: + * constructor result + * typename gen_numeric_list::type numeric_list + * typename gen_numeric_list_reversed::type numeric_list + * typename gen_numeric_list_swapped_pair::type numeric_list + * typename gen_numeric_list_repeated::type numeric_list + */ + +template struct gen_numeric_list : gen_numeric_list {}; +template struct gen_numeric_list { typedef numeric_list type; }; + +template struct gen_numeric_list_reversed : gen_numeric_list_reversed {}; +template struct gen_numeric_list_reversed { typedef numeric_list type; }; + +template struct gen_numeric_list_swapped_pair : gen_numeric_list_swapped_pair {}; +template struct gen_numeric_list_swapped_pair { typedef numeric_list type; }; + +template struct gen_numeric_list_repeated : gen_numeric_list_repeated {}; +template struct gen_numeric_list_repeated { typedef numeric_list type; }; + +/* list manipulation: concatenate */ + +template struct concat; + +template struct concat, type_list> { typedef type_list type; }; +template struct concat, numeric_list > { typedef numeric_list type; }; + +template struct mconcat; +template struct mconcat { typedef a type; }; +template struct mconcat : concat {}; +template struct mconcat : concat::type> {}; + +/* list manipulation: extract slices */ + +template struct take; +template struct take> : concat, typename take>::type> {}; +template struct take> { typedef type_list<> type; }; +template struct take<0, type_list> { typedef type_list<> type; }; +template<> struct take<0, type_list<>> { typedef type_list<> type; }; + +template struct take> : concat, typename take>::type> {}; +// XXX The following breaks in gcc-11, and is invalid anyways. +// template struct take> { typedef numeric_list type; }; +template struct take<0, numeric_list> { typedef numeric_list type; }; +template struct take<0, numeric_list> { typedef numeric_list type; }; + +template struct h_skip_helper_numeric; +template struct h_skip_helper_numeric : h_skip_helper_numeric {}; +template struct h_skip_helper_numeric { typedef numeric_list type; }; +template struct h_skip_helper_numeric { typedef numeric_list type; }; +template struct h_skip_helper_numeric { typedef numeric_list type; }; + +template struct h_skip_helper_type; +template struct h_skip_helper_type : h_skip_helper_type {}; +template struct h_skip_helper_type<0, t, tt...> { typedef type_list type; }; +template struct h_skip_helper_type { typedef type_list<> type; }; +template<> struct h_skip_helper_type<0> { typedef type_list<> type; }; +#endif //not EIGEN_PARSED_BY_DOXYGEN + +template +struct h_skip { + template + constexpr static EIGEN_STRONG_INLINE typename h_skip_helper_numeric::type helper(numeric_list) { return typename h_skip_helper_numeric::type(); } + template + constexpr static EIGEN_STRONG_INLINE typename h_skip_helper_type::type helper(type_list) { return typename h_skip_helper_type::type(); } +}; + +template struct skip { typedef decltype(h_skip::helper(a())) type; }; + +template struct slice : take::type> {}; + +/* list manipulation: retrieve single element from list */ + +template struct get; + +template struct get> : get> {}; +template struct get<0, type_list> { typedef a type; }; + +template struct get> : get> {}; +template struct get<0, numeric_list> { constexpr static T value = a; }; + +template constexpr T array_get(const numeric_list&) { + return get<(int)n, numeric_list>::value; +} + +/* always get type, regardless of dummy; good for parameter pack expansion */ + +template struct id_numeric { typedef t type; }; +template struct id_type { typedef t type; }; + +/* equality checking, flagged version */ + +template struct is_same_gf : is_same { constexpr static int global_flags = 0; }; + +/* apply_op to list */ + +template< + bool from_left, // false + template class op, + typename additional_param, + typename... values +> +struct h_apply_op_helper { typedef type_list::type...> type; }; +template< + template class op, + typename additional_param, + typename... values +> +struct h_apply_op_helper { typedef type_list::type...> type; }; + +template< + bool from_left, + template class op, + typename additional_param +> +struct h_apply_op +{ + template + constexpr static typename h_apply_op_helper::type helper(type_list) + { return typename h_apply_op_helper::type(); } +}; + +template< + template class op, + typename additional_param, + typename a +> +struct apply_op_from_left { typedef decltype(h_apply_op::helper(a())) type; }; + +template< + template class op, + typename additional_param, + typename a +> +struct apply_op_from_right { typedef decltype(h_apply_op::helper(a())) type; }; + +/* see if an element is in a list */ + +template< + template class test, + typename check_against, + typename h_list, + bool last_check_positive = false +> +struct contained_in_list; + +template< + template class test, + typename check_against, + typename h_list +> +struct contained_in_list +{ + constexpr static bool value = true; +}; + +template< + template class test, + typename check_against, + typename a, + typename... as +> +struct contained_in_list, false> : contained_in_list, test::value> {}; + +template< + template class test, + typename check_against, + typename... empty +> +struct contained_in_list, false> { constexpr static bool value = false; }; + +/* see if an element is in a list and check for global flags */ + +template< + template class test, + typename check_against, + typename h_list, + int default_flags = 0, + bool last_check_positive = false, + int last_check_flags = default_flags +> +struct contained_in_list_gf; + +template< + template class test, + typename check_against, + typename h_list, + int default_flags, + int last_check_flags +> +struct contained_in_list_gf +{ + constexpr static bool value = true; + constexpr static int global_flags = last_check_flags; +}; + +template< + template class test, + typename check_against, + typename a, + typename... as, + int default_flags, + int last_check_flags +> +struct contained_in_list_gf, default_flags, false, last_check_flags> : contained_in_list_gf, default_flags, test::value, test::global_flags> {}; + +template< + template class test, + typename check_against, + typename... empty, + int default_flags, + int last_check_flags +> +struct contained_in_list_gf, default_flags, false, last_check_flags> { constexpr static bool value = false; constexpr static int global_flags = default_flags; }; + +/* generic reductions */ + +template< + typename Reducer, + typename... Ts +> struct reduce; + +template< + typename Reducer +> struct reduce +{ + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE int run() { return Reducer::Identity; } +}; + +template< + typename Reducer, + typename A +> struct reduce +{ + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE A run(A a) { return a; } +}; + +template< + typename Reducer, + typename A, + typename... Ts +> struct reduce +{ + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE auto run(A a, Ts... ts) -> decltype(Reducer::run(a, reduce::run(ts...))) { + return Reducer::run(a, reduce::run(ts...)); + } +}; + +/* generic binary operations */ + +struct sum_op { + template EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a + b) { return a + b; } + static constexpr int Identity = 0; +}; +struct product_op { + template EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a * b) { return a * b; } + static constexpr int Identity = 1; +}; + +struct logical_and_op { template constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a && b) { return a && b; } }; +struct logical_or_op { template constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a || b) { return a || b; } }; + +struct equal_op { template constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a == b) { return a == b; } }; +struct not_equal_op { template constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a != b) { return a != b; } }; +struct lesser_op { template constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a < b) { return a < b; } }; +struct lesser_equal_op { template constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a <= b) { return a <= b; } }; +struct greater_op { template constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a > b) { return a > b; } }; +struct greater_equal_op { template constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a >= b) { return a >= b; } }; + +/* generic unary operations */ + +struct not_op { template constexpr static EIGEN_STRONG_INLINE auto run(A a) -> decltype(!a) { return !a; } }; +struct negation_op { template constexpr static EIGEN_STRONG_INLINE auto run(A a) -> decltype(-a) { return -a; } }; +struct greater_equal_zero_op { template constexpr static EIGEN_STRONG_INLINE auto run(A a) -> decltype(a >= 0) { return a >= 0; } }; + + +/* reductions for lists */ + +// using auto -> return value spec makes ICC 13.0 and 13.1 crash here, so we have to hack it +// together in front... (13.0 doesn't work with array_prod/array_reduce/... anyway, but 13.1 +// does... +template +EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE decltype(reduce::run((*((Ts*)0))...)) arg_prod(Ts... ts) +{ + return reduce::run(ts...); +} + +template +constexpr EIGEN_STRONG_INLINE decltype(reduce::run((*((Ts*)0))...)) arg_sum(Ts... ts) +{ + return reduce::run(ts...); +} + +/* reverse arrays */ + +template +constexpr EIGEN_STRONG_INLINE Array h_array_reverse(Array arr, numeric_list) +{ + return {{array_get(arr)...}}; +} + +template +constexpr EIGEN_STRONG_INLINE array array_reverse(array arr) +{ + return h_array_reverse(arr, typename gen_numeric_list::type()); +} + + +/* generic array reductions */ + +// can't reuse standard reduce() interface above because Intel's Compiler +// *really* doesn't like it, so we just reimplement the stuff +// (start from N - 1 and work down to 0 because specialization for +// n == N - 1 also doesn't work in Intel's compiler, so it goes into +// an infinite loop) +template +struct h_array_reduce { + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE auto run(array arr, T identity) -> decltype(Reducer::run(h_array_reduce::run(arr, identity), array_get(arr))) + { + return Reducer::run(h_array_reduce::run(arr, identity), array_get(arr)); + } +}; + +template +struct h_array_reduce +{ + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE T run(const array& arr, T) + { + return array_get<0>(arr); + } +}; + +template +struct h_array_reduce +{ + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE T run(const array&, T identity) + { + return identity; + } +}; + +template +EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE auto array_reduce(const array& arr, T identity) -> decltype(h_array_reduce::run(arr, identity)) +{ + return h_array_reduce::run(arr, identity); +} + +/* standard array reductions */ + +template +EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE auto array_sum(const array& arr) -> decltype(array_reduce(arr, static_cast(0))) +{ + return array_reduce(arr, static_cast(0)); +} + +template +EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE auto array_prod(const array& arr) -> decltype(array_reduce(arr, static_cast(1))) +{ + return array_reduce(arr, static_cast(1)); +} + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE t array_prod(const std::vector& a) { + eigen_assert(a.size() > 0); + t prod = 1; + for (size_t i = 0; i < a.size(); ++i) { prod *= a[i]; } + return prod; +} + +/* zip an array */ + +template +constexpr EIGEN_STRONG_INLINE array h_array_zip(array a, array b, numeric_list) +{ + return array{{ Op::run(array_get(a), array_get(b))... }}; +} + +template +constexpr EIGEN_STRONG_INLINE array array_zip(array a, array b) +{ + return h_array_zip(a, b, typename gen_numeric_list::type()); +} + +/* zip an array and reduce the result */ + +template +constexpr EIGEN_STRONG_INLINE auto h_array_zip_and_reduce(array a, array b, numeric_list) -> decltype(reduce::type...>::run(Op::run(array_get(a), array_get(b))...)) +{ + return reduce::type...>::run(Op::run(array_get(a), array_get(b))...); +} + +template +constexpr EIGEN_STRONG_INLINE auto array_zip_and_reduce(array a, array b) -> decltype(h_array_zip_and_reduce(a, b, typename gen_numeric_list::type())) +{ + return h_array_zip_and_reduce(a, b, typename gen_numeric_list::type()); +} + +/* apply stuff to an array */ + +template +constexpr EIGEN_STRONG_INLINE array h_array_apply(array a, numeric_list) +{ + return array{{ Op::run(array_get(a))... }}; +} + +template +constexpr EIGEN_STRONG_INLINE array array_apply(array a) +{ + return h_array_apply(a, typename gen_numeric_list::type()); +} + +/* apply stuff to an array and reduce */ + +template +constexpr EIGEN_STRONG_INLINE auto h_array_apply_and_reduce(array arr, numeric_list) -> decltype(reduce::type...>::run(Op::run(array_get(arr))...)) +{ + return reduce::type...>::run(Op::run(array_get(arr))...); +} + +template +constexpr EIGEN_STRONG_INLINE auto array_apply_and_reduce(array a) -> decltype(h_array_apply_and_reduce(a, typename gen_numeric_list::type())) +{ + return h_array_apply_and_reduce(a, typename gen_numeric_list::type()); +} + +/* repeat a value n times (and make an array out of it + * usage: + * array = repeat<16>(42); + */ + +template +struct h_repeat +{ + template + constexpr static EIGEN_STRONG_INLINE array run(t v, numeric_list) + { + return {{ typename id_numeric::type(v)... }}; + } +}; + +template +constexpr array repeat(t v) { return h_repeat::run(v, typename gen_numeric_list::type()); } + +/* instantiate a class by a C-style array */ +template +struct h_instantiate_by_c_array; + +template +struct h_instantiate_by_c_array +{ + static InstType run(ArrType* arr, Ps... args) + { + return h_instantiate_by_c_array::run(arr + 1, args..., arr[0]); + } +}; + +template +struct h_instantiate_by_c_array +{ + static InstType run(ArrType* arr, Ps... args) + { + return h_instantiate_by_c_array::run(arr + 1, arr[0], args...); + } +}; + +template +struct h_instantiate_by_c_array +{ + static InstType run(ArrType* arr, Ps... args) + { + (void)arr; + return InstType(args...); + } +}; + +template +struct h_instantiate_by_c_array +{ + static InstType run(ArrType* arr, Ps... args) + { + (void)arr; + return InstType(args...); + } +}; + +template +InstType instantiate_by_c_array(ArrType* arr) +{ + return h_instantiate_by_c_array::run(arr); +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_MOREMETA_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/NonMPL2.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/NonMPL2.h deleted file mode 100644 index 1af67cf18c7..00000000000 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/NonMPL2.h +++ /dev/null @@ -1,3 +0,0 @@ -#ifdef EIGEN_MPL2_ONLY -#error Including non-MPL2 code in EIGEN_MPL2_ONLY mode -#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ReenableStupidWarnings.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ReenableStupidWarnings.h index 1ce6fd1b009..c8238de842c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ReenableStupidWarnings.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ReenableStupidWarnings.h @@ -1,5 +1,5 @@ #ifdef EIGEN_WARNINGS_DISABLED_2 -// "DisableStupidWarnings.h" was included twice recursively: Do not reenable warnings yet! +// "DisableStupidWarnings.h" was included twice recursively: Do not re-enable warnings yet! # undef EIGEN_WARNINGS_DISABLED_2 #elif defined(EIGEN_WARNINGS_DISABLED) @@ -8,22 +8,35 @@ #ifndef EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS #ifdef _MSC_VER #pragma warning( pop ) + #ifdef EIGEN_REENABLE_CXX23_DENORM_DEPRECATION_WARNING + #undef EIGEN_REENABLE_CXX23_DENORM_DEPRECATION_WARNING + #undef _SILENCE_CXX23_DENORM_DEPRECATION_WARNING + #endif + #elif defined __INTEL_COMPILER #pragma warning pop #elif defined __clang__ #pragma clang diagnostic pop - #elif defined __GNUC__ && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) + #elif defined __GNUC__ && !defined(__FUJITSU) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) #pragma GCC diagnostic pop #endif #if defined __NVCC__ -// Don't reenable the diagnostic messages, as it turns out these messages need +// Don't re-enable the diagnostic messages, as it turns out these messages need // to be disabled at the point of the template instantiation (i.e the user code) // otherwise they'll be triggered by nvcc. -// #pragma diag_default code_is_unreachable -// #pragma diag_default initialization_not_reachable -// #pragma diag_default 2651 -// #pragma diag_default 2653 +// #define EIGEN_MAKE_PRAGMA(X) _Pragma(#X) +// #if __NVCC_DIAG_PRAGMA_SUPPORT__ +// #define EIGEN_NV_DIAG_DEFAULT(X) EIGEN_MAKE_PRAGMA(nv_diag_default X) +// #else +// #define EIGEN_NV_DIAG_DEFAULT(X) EIGEN_MAKE_PRAGMA(diag_default X) +// #endif +// EIGEN_NV_DIAG_DEFAULT(code_is_unreachable) +// EIGEN_NV_DIAG_DEFAULT(initialization_not_reachable) +// EIGEN_NV_DIAG_DEFAULT(2651) +// EIGEN_NV_DIAG_DEFAULT(2653) +// #undef EIGEN_NV_DIAG_DEFAULT +// #undef EIGEN_MAKE_PRAGMA #endif #endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ReshapedHelper.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ReshapedHelper.h index 412432132ca..6c818f5cf4d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ReshapedHelper.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/ReshapedHelper.h @@ -11,6 +11,9 @@ #ifndef EIGEN_RESHAPED_HELPER_H #define EIGEN_RESHAPED_HELPER_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { enum AutoSize_t { AutoSize }; @@ -39,10 +42,9 @@ inline Index get_runtime_reshape_size(AutoSize_t /*size*/, Index other, Index to return total/other; } -template -struct get_compiletime_reshape_order { - enum { value = Order == AutoOrder ? Flags & RowMajorBit : Order }; -}; +constexpr inline int get_compiletime_reshape_order(int flags, int order) { + return order == AutoOrder ? flags & RowMajorBit : order; +} } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Serializer.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Serializer.h new file mode 100644 index 00000000000..cbfc04acbab --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/Serializer.h @@ -0,0 +1,220 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2021 The Eigen Team +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SERIALIZER_H +#define EIGEN_SERIALIZER_H + +#include + +// The Serializer class encodes data into a memory buffer so it can be later +// reconstructed. This is mainly used to send objects back-and-forth between +// the CPU and GPU. + +namespace Eigen { + +/** + * Serializes an object to a memory buffer. + * + * Useful for transferring data (e.g. back-and-forth to a device). + */ +template +class Serializer; + +// Specialization for POD types. +template +class Serializer::value + && std::is_standard_layout::value>> { + public: + + /** + * Determines the required size of the serialization buffer for a value. + * + * \param value the value to serialize. + * \return the required size. + */ + EIGEN_DEVICE_FUNC size_t size(const T& value) const { + return sizeof(value); + } + + /** + * Serializes a value to a byte buffer. + * \param dest the destination buffer; if this is nullptr, does nothing. + * \param end the end of the destination buffer. + * \param value the value to serialize. + * \return the next memory address past the end of the serialized data. + */ + EIGEN_DEVICE_FUNC uint8_t* serialize(uint8_t* dest, uint8_t* end, const T& value) { + if (EIGEN_PREDICT_FALSE(dest == nullptr)) return nullptr; + if (EIGEN_PREDICT_FALSE(dest + sizeof(value) > end)) return nullptr; + EIGEN_USING_STD(memcpy) + memcpy(dest, &value, sizeof(value)); + return dest + sizeof(value); + } + + /** + * Deserializes a value from a byte buffer. + * \param src the source buffer; if this is nullptr, does nothing. + * \param end the end of the source buffer. + * \param value the value to populate. + * \return the next unprocessed memory address; nullptr if parsing errors are detected. + */ + EIGEN_DEVICE_FUNC const uint8_t* deserialize(const uint8_t* src, const uint8_t* end, T& value) const { + if (EIGEN_PREDICT_FALSE(src == nullptr)) return nullptr; + if (EIGEN_PREDICT_FALSE(src + sizeof(value) > end)) return nullptr; + EIGEN_USING_STD(memcpy) + memcpy(&value, src, sizeof(value)); + return src + sizeof(value); + } +}; + +// Specialization for DenseBase. +// Serializes [rows, cols, data...]. +template +class Serializer, void> { + public: + typedef typename Derived::Scalar Scalar; + + struct Header { + typename Derived::Index rows; + typename Derived::Index cols; + }; + + EIGEN_DEVICE_FUNC size_t size(const Derived& value) const { + return sizeof(Header) + sizeof(Scalar) * value.size(); + } + + EIGEN_DEVICE_FUNC uint8_t* serialize(uint8_t* dest, uint8_t* end, const Derived& value) { + if (EIGEN_PREDICT_FALSE(dest == nullptr)) return nullptr; + if (EIGEN_PREDICT_FALSE(dest + size(value) > end)) return nullptr; + const size_t header_bytes = sizeof(Header); + const size_t data_bytes = sizeof(Scalar) * value.size(); + Header header = {value.rows(), value.cols()}; + EIGEN_USING_STD(memcpy) + memcpy(dest, &header, header_bytes); + dest += header_bytes; + memcpy(dest, value.data(), data_bytes); + return dest + data_bytes; + } + + EIGEN_DEVICE_FUNC const uint8_t* deserialize(const uint8_t* src, const uint8_t* end, Derived& value) const { + if (EIGEN_PREDICT_FALSE(src == nullptr)) return nullptr; + if (EIGEN_PREDICT_FALSE(src + sizeof(Header) > end)) return nullptr; + const size_t header_bytes = sizeof(Header); + Header header; + EIGEN_USING_STD(memcpy) + memcpy(&header, src, header_bytes); + src += header_bytes; + const size_t data_bytes = sizeof(Scalar) * header.rows * header.cols; + if (EIGEN_PREDICT_FALSE(src + data_bytes > end)) return nullptr; + value.resize(header.rows, header.cols); + memcpy(value.data(), src, data_bytes); + return src + data_bytes; + } +}; + +template +class Serializer > : public + Serializer > > {}; + +template +class Serializer > : public + Serializer > > {}; + +namespace internal { + +// Recursive serialization implementation helper. +template +struct serialize_impl; + +template +struct serialize_impl { + using Serializer = Eigen::Serializer::type>; + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + size_t serialize_size(const T1& value, const Ts&... args) { + Serializer serializer; + size_t size = serializer.size(value); + return size + serialize_impl::serialize_size(args...); + } + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + uint8_t* serialize(uint8_t* dest, uint8_t* end, const T1& value, const Ts&... args) { + Serializer serializer; + dest = serializer.serialize(dest, end, value); + return serialize_impl::serialize(dest, end, args...); + } + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const uint8_t* deserialize(const uint8_t* src, const uint8_t* end, T1& value, Ts&... args) { + Serializer serializer; + src = serializer.deserialize(src, end, value); + return serialize_impl::deserialize(src, end, args...); + } +}; + +// Base case. +template<> +struct serialize_impl<0> { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + size_t serialize_size() { return 0; } + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + uint8_t* serialize(uint8_t* dest, uint8_t* /*end*/) { return dest; } + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const uint8_t* deserialize(const uint8_t* src, const uint8_t* /*end*/) { return src; } +}; + +} // namespace internal + + +/** + * Determine the buffer size required to serialize a set of values. + * + * \param args ... arguments to serialize in sequence. + * \return the total size of the required buffer. + */ +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +size_t serialize_size(const Args&... args) { + return internal::serialize_impl::serialize_size(args...); +} + +/** + * Serialize a set of values to the byte buffer. + * + * \param dest output byte buffer; if this is nullptr, does nothing. + * \param end the end of the output byte buffer. + * \param args ... arguments to serialize in sequence. + * \return the next address after all serialized values. + */ +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +uint8_t* serialize(uint8_t* dest, uint8_t* end, const Args&... args) { + return internal::serialize_impl::serialize(dest, end, args...); +} + +/** + * Deserialize a set of values from the byte buffer. + * + * \param src input byte buffer; if this is nullptr, does nothing. + * \param end the end of input byte buffer. + * \param args ... arguments to deserialize in sequence. + * \return the next address after all parsed values; nullptr if parsing errors are detected. + */ +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const uint8_t* deserialize(const uint8_t* src, const uint8_t* end, Args&... args) { + return internal::serialize_impl::deserialize(src, end, args...); +} + +} // namespace Eigen + +#endif // EIGEN_SERIALIZER_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/StaticAssert.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/StaticAssert.h index c45de59016e..c938eb800e5 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/StaticAssert.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/StaticAssert.h @@ -16,10 +16,6 @@ * - in EIGEN_STATIC_ASSERT(CONDITION,MSG) the parameter CONDITION must be a compile time boolean * expression, and MSG an enum listed in struct internal::static_assertion * - * - define EIGEN_NO_STATIC_ASSERT to disable them (and save compilation time) - * in that case, the static assertion is converted to the following runtime assert: - * eigen_assert(CONDITION && "MSG") - * * - currently EIGEN_STATIC_ASSERT can only be used in function scope * */ @@ -27,113 +23,11 @@ #ifndef EIGEN_STATIC_ASSERT #ifndef EIGEN_NO_STATIC_ASSERT - #if EIGEN_MAX_CPP_VER>=11 && (__has_feature(cxx_static_assert) || (EIGEN_COMP_CXXVER >= 11) || (EIGEN_COMP_MSVC >= 1600)) - - // if native static_assert is enabled, let's use it - #define EIGEN_STATIC_ASSERT(X,MSG) static_assert(X,#MSG); - - #else // not CXX0X - - namespace Eigen { - - namespace internal { - - template - struct static_assertion {}; - - template<> - struct static_assertion - { - enum { - YOU_TRIED_CALLING_A_VECTOR_METHOD_ON_A_MATRIX=1, - YOU_MIXED_VECTORS_OF_DIFFERENT_SIZES=1, - YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES=1, - THIS_METHOD_IS_ONLY_FOR_VECTORS_OF_A_SPECIFIC_SIZE=1, - THIS_METHOD_IS_ONLY_FOR_MATRICES_OF_A_SPECIFIC_SIZE=1, - THIS_METHOD_IS_ONLY_FOR_OBJECTS_OF_A_SPECIFIC_SIZE=1, - OUT_OF_RANGE_ACCESS=1, - YOU_MADE_A_PROGRAMMING_MISTAKE=1, - EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT=1, - EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE=1, - YOU_CALLED_A_FIXED_SIZE_METHOD_ON_A_DYNAMIC_SIZE_MATRIX_OR_VECTOR=1, - YOU_CALLED_A_DYNAMIC_SIZE_METHOD_ON_A_FIXED_SIZE_MATRIX_OR_VECTOR=1, - UNALIGNED_LOAD_AND_STORE_OPERATIONS_UNIMPLEMENTED_ON_ALTIVEC=1, - THIS_FUNCTION_IS_NOT_FOR_INTEGER_NUMERIC_TYPES=1, - FLOATING_POINT_ARGUMENT_PASSED__INTEGER_WAS_EXPECTED=1, - NUMERIC_TYPE_MUST_BE_REAL=1, - COEFFICIENT_WRITE_ACCESS_TO_SELFADJOINT_NOT_SUPPORTED=1, - WRITING_TO_TRIANGULAR_PART_WITH_UNIT_DIAGONAL_IS_NOT_SUPPORTED=1, - THIS_METHOD_IS_ONLY_FOR_FIXED_SIZE=1, - INVALID_MATRIX_PRODUCT=1, - INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS=1, - INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION=1, - YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY=1, - THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES=1, - THIS_METHOD_IS_ONLY_FOR_ROW_MAJOR_MATRICES=1, - INVALID_MATRIX_TEMPLATE_PARAMETERS=1, - INVALID_MATRIXBASE_TEMPLATE_PARAMETERS=1, - BOTH_MATRICES_MUST_HAVE_THE_SAME_STORAGE_ORDER=1, - THIS_METHOD_IS_ONLY_FOR_DIAGONAL_MATRIX=1, - THE_MATRIX_OR_EXPRESSION_THAT_YOU_PASSED_DOES_NOT_HAVE_THE_EXPECTED_TYPE=1, - THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_WITH_DIRECT_MEMORY_ACCESS_SUCH_AS_MAP_OR_PLAIN_MATRICES=1, - YOU_ALREADY_SPECIFIED_THIS_STRIDE=1, - INVALID_STORAGE_ORDER_FOR_THIS_VECTOR_EXPRESSION=1, - THE_BRACKET_OPERATOR_IS_ONLY_FOR_VECTORS__USE_THE_PARENTHESIS_OPERATOR_INSTEAD=1, - PACKET_ACCESS_REQUIRES_TO_HAVE_INNER_STRIDE_FIXED_TO_1=1, - THIS_METHOD_IS_ONLY_FOR_SPECIFIC_TRANSFORMATIONS=1, - YOU_CANNOT_MIX_ARRAYS_AND_MATRICES=1, - YOU_PERFORMED_AN_INVALID_TRANSFORMATION_CONVERSION=1, - THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY=1, - YOU_ARE_TRYING_TO_USE_AN_INDEX_BASED_ACCESSOR_ON_AN_EXPRESSION_THAT_DOES_NOT_SUPPORT_THAT=1, - THIS_METHOD_IS_ONLY_FOR_1x1_EXPRESSIONS=1, - THIS_METHOD_IS_ONLY_FOR_INNER_OR_LAZY_PRODUCTS=1, - THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL=1, - THIS_METHOD_IS_ONLY_FOR_ARRAYS_NOT_MATRICES=1, - YOU_PASSED_A_ROW_VECTOR_BUT_A_COLUMN_VECTOR_WAS_EXPECTED=1, - YOU_PASSED_A_COLUMN_VECTOR_BUT_A_ROW_VECTOR_WAS_EXPECTED=1, - THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE=1, - THE_STORAGE_ORDER_OF_BOTH_SIDES_MUST_MATCH=1, - OBJECT_ALLOCATED_ON_STACK_IS_TOO_BIG=1, - IMPLICIT_CONVERSION_TO_SCALAR_IS_FOR_INNER_PRODUCT_ONLY=1, - STORAGE_LAYOUT_DOES_NOT_MATCH=1, - EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT__INVALID_COST_VALUE=1, - THIS_COEFFICIENT_ACCESSOR_TAKING_ONE_ACCESS_IS_ONLY_FOR_EXPRESSIONS_ALLOWING_LINEAR_ACCESS=1, - MATRIX_FREE_CONJUGATE_GRADIENT_IS_COMPATIBLE_WITH_UPPER_UNION_LOWER_MODE_ONLY=1, - THIS_TYPE_IS_NOT_SUPPORTED=1, - STORAGE_KIND_MUST_MATCH=1, - STORAGE_INDEX_MUST_MATCH=1, - CHOLMOD_SUPPORTS_DOUBLE_PRECISION_ONLY=1, - SELFADJOINTVIEW_ACCEPTS_UPPER_AND_LOWER_MODE_ONLY=1, - INVALID_TEMPLATE_PARAMETER=1, - GPU_TENSOR_CONTRACTION_DOES_NOT_SUPPORT_OUTPUT_KERNELS=1, - THE_ARRAY_SIZE_SHOULD_EQUAL_WITH_PACKET_SIZE=1 - }; - }; - - } // end namespace internal - - } // end namespace Eigen - - // Specialized implementation for MSVC to avoid "conditional - // expression is constant" warnings. This implementation doesn't - // appear to work under GCC, hence the multiple implementations. - #if EIGEN_COMP_MSVC - - #define EIGEN_STATIC_ASSERT(CONDITION,MSG) \ - {Eigen::internal::static_assertion::MSG;} - - #else - // In some cases clang interprets bool(CONDITION) as function declaration - #define EIGEN_STATIC_ASSERT(CONDITION,MSG) \ - if (Eigen::internal::static_assertion(CONDITION)>::MSG) {} - - #endif - - #endif // not CXX0X +#define EIGEN_STATIC_ASSERT(X,MSG) static_assert(X,#MSG); #else // EIGEN_NO_STATIC_ASSERT - #define EIGEN_STATIC_ASSERT(CONDITION,MSG) eigen_assert((CONDITION) && #MSG); +#define EIGEN_STATIC_ASSERT(CONDITION,MSG) #endif // EIGEN_NO_STATIC_ASSERT #endif // EIGEN_STATIC_ASSERT diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/SymbolicIndex.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/SymbolicIndex.h index 354dd9add3b..a129b4da6bf 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/SymbolicIndex.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/SymbolicIndex.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SYMBOLIC_INDEX_H #define EIGEN_SYMBOLIC_INDEX_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { /** \namespace Eigen::symbolic @@ -30,12 +33,11 @@ namespace Eigen { * // And evaluate it: (c++14) * std::cout << expr.eval(x=6,y=3,z=-13) << "\n"; * - * // In c++98/11, only one symbol per expression is supported for now: - * auto expr98 = (3-x)/2; - * std::cout << expr98.eval(x=6) << "\n"; * \endcode * - * It is currently only used internally to define and manipulate the Eigen::last and Eigen::lastp1 symbols in Eigen::seq and Eigen::seqN. + * It is currently only used internally to define and manipulate the + * Eigen::placeholders::last and Eigen::placeholders::lastp1 symbols in + * Eigen::seq and Eigen::seqN. * */ namespace symbolic { @@ -88,10 +90,8 @@ class BaseExpr template Index eval(const T& values) const { return derived().eval_impl(values); } -#if EIGEN_HAS_CXX14 template Index eval(Types&&... values) const { return derived().eval_impl(std::make_tuple(values...)); } -#endif NegateExpr operator-() const { return NegateExpr(derived()); } @@ -139,34 +139,6 @@ class BaseExpr friend QuotientExpr >,Derived> operator/(internal::FixedInt, const BaseExpr& b) { return QuotientExpr > ,Derived>(ValueExpr >(),b.derived()); } -#if (!EIGEN_HAS_CXX14) - template - AddExpr > > operator+(internal::FixedInt (*)()) const - { return AddExpr > >(derived(), ValueExpr >()); } - template - AddExpr > > operator-(internal::FixedInt (*)()) const - { return AddExpr > >(derived(), ValueExpr >()); } - template - ProductExpr > > operator*(internal::FixedInt (*)()) const - { return ProductExpr > >(derived(),ValueExpr >()); } - template - QuotientExpr > > operator/(internal::FixedInt (*)()) const - { return QuotientExpr > >(derived(),ValueExpr >()); } - - template - friend AddExpr > > operator+(internal::FixedInt (*)(), const BaseExpr& b) - { return AddExpr > >(b.derived(), ValueExpr >()); } - template - friend AddExpr,ValueExpr > > operator-(internal::FixedInt (*)(), const BaseExpr& b) - { return AddExpr,ValueExpr > >(-b.derived(), ValueExpr >()); } - template - friend ProductExpr >,Derived> operator*(internal::FixedInt (*)(), const BaseExpr& b) - { return ProductExpr >,Derived>(ValueExpr >(),b.derived()); } - template - friend QuotientExpr >,Derived> operator/(internal::FixedInt (*)(), const BaseExpr& b) - { return QuotientExpr > ,Derived>(ValueExpr >(),b.derived()); } -#endif - template AddExpr operator+(const BaseExpr &b) const @@ -228,11 +200,9 @@ class SymbolExpr : public BaseExpr > Index eval_impl(const SymbolValue &values) const { return values.value(); } -#if EIGEN_HAS_CXX14 // C++14 versions suitable for multiple symbols template Index eval_impl(const std::tuple& values) const { return std::get >(values).value(); } -#endif }; template diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/XprHelper.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/XprHelper.h index 71c32b8a116..d05e7d1a532 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/XprHelper.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Core/util/XprHelper.h @@ -11,41 +11,65 @@ #ifndef EIGEN_XPRHELPER_H #define EIGEN_XPRHELPER_H -// just a workaround because GCC seems to not really like empty structs -// FIXME: gcc 4.3 generates bad code when strict-aliasing is enabled -// so currently we simply disable this optimization for gcc 4.3 -#if EIGEN_COMP_GNUC && !EIGEN_GNUC_AT(4,3) - #define EIGEN_EMPTY_STRUCT_CTOR(X) \ - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE X() {} \ - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE X(const X& ) {} -#else - #define EIGEN_EMPTY_STRUCT_CTOR(X) -#endif +// IWYU pragma: private +#include "../InternalHeaderCheck.h" namespace Eigen { namespace internal { -template -EIGEN_DEVICE_FUNC -inline IndexDest convert_index(const IndexSrc& idx) { - // for sizeof(IndexDest)>=sizeof(IndexSrc) compilers should be able to optimize this away: - eigen_internal_assert(idx <= NumTraits::highest() && "Index value to big for target type"); - return IndexDest(idx); + +// useful for unsigned / signed integer comparisons when idx is intended to be non-negative +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename make_unsigned::type returnUnsignedIndexValue( + const IndexType& idx) { + EIGEN_STATIC_ASSERT((NumTraits::IsInteger), THIS FUNCTION IS FOR INTEGER TYPES) + eigen_internal_assert(idx >= 0 && "Index value is negative and target type is unsigned"); + using UnsignedType = typename make_unsigned::type; + return static_cast(idx); +} + +template ::IsInteger, + bool IndexDestIsSigned = NumTraits::IsSigned, + bool IndexSrcIsInteger = NumTraits::IsInteger, + bool IndexSrcIsSigned = NumTraits::IsSigned> +struct convert_index_impl { + static inline EIGEN_DEVICE_FUNC IndexDest run(const IndexSrc& idx) { + eigen_internal_assert(idx <= NumTraits::highest() && "Index value is too big for target type"); + return static_cast(idx); + } +}; +template +struct convert_index_impl { + // IndexDest is a signed integer + // IndexSrc is an unsigned integer + static inline EIGEN_DEVICE_FUNC IndexDest run(const IndexSrc& idx) { + eigen_internal_assert(idx <= returnUnsignedIndexValue(NumTraits::highest()) && + "Index value is too big for target type"); + return static_cast(idx); + } +}; +template +struct convert_index_impl { + // IndexDest is an unsigned integer + // IndexSrc is a signed integer + static inline EIGEN_DEVICE_FUNC IndexDest run(const IndexSrc& idx) { + eigen_internal_assert(returnUnsignedIndexValue(idx) <= NumTraits::highest() && + "Index value is too big for target type"); + return static_cast(idx); + } +}; + +template +EIGEN_DEVICE_FUNC inline IndexDest convert_index(const IndexSrc& idx) { + return convert_index_impl::run(idx); } // true if T can be considered as an integral index (i.e., and integral type or enum) template struct is_valid_index_type { - enum { value = -#if EIGEN_HAS_TYPE_TRAITS - internal::is_integral::value || std::is_enum::value -#elif EIGEN_COMP_MSVC - internal::is_integral::value || __is_enum(T) -#else - // without C++11, we use is_convertible to Index instead of is_integral in order to treat enums as Index. - internal::is_convertible::value && !internal::is_same::value && !is_same::value -#endif + enum { value = internal::is_integral::value || std::is_enum::value }; }; @@ -119,7 +143,7 @@ class no_assignment_operator template struct promote_index_type { - typedef typename conditional<(sizeof(I1)::type type; + typedef std::conditional_t<(sizeof(I1) type; }; /** \internal If the template parameter Value is Dynamic, this class is just a wrapper around a T variable that @@ -154,7 +178,6 @@ template class variable_if_dynamic template class variable_if_dynamicindex { public: - EIGEN_EMPTY_STRUCT_CTOR(variable_if_dynamicindex) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit variable_if_dynamicindex(T v) { EIGEN_ONLY_USED_FOR_DEBUG(v); eigen_assert(v == T(Value)); } EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR T value() { return T(Value); } @@ -208,84 +231,95 @@ struct find_best_packet typedef typename find_best_packet_helper::type>::type type; }; -#if EIGEN_MAX_STATIC_ALIGN_BYTES>0 -template -struct compute_default_alignment_helper -{ - enum { value = 0 }; +template ::size) || + is_same::half>::value> +struct find_packet_by_size_helper; +template +struct find_packet_by_size_helper { + using type = PacketType; }; - -template -struct compute_default_alignment_helper // Match -{ - enum { value = AlignmentBytes }; +template +struct find_packet_by_size_helper { + using type = typename find_packet_by_size_helper::half>::type; }; -template -struct compute_default_alignment_helper // Try-half -{ - // current packet too large, try with an half-packet - enum { value = compute_default_alignment_helper::value }; +template +struct find_packet_by_size { + using type = typename find_packet_by_size_helper::type>::type; + static constexpr bool value = (Size == unpacket_traits::size); +}; +template +struct find_packet_by_size { + using type = typename unpacket_traits::type; + static constexpr bool value = (unpacket_traits::size == 1); }; + +#if EIGEN_MAX_STATIC_ALIGN_BYTES>0 +constexpr inline int compute_default_alignment_helper(int ArrayBytes, int AlignmentBytes) { + if((ArrayBytes % AlignmentBytes) == 0) { + return AlignmentBytes; + } else if (EIGEN_MIN_ALIGN_BYTES -struct compute_default_alignment_helper -{ - enum { value = 0 }; -}; +// This also avoids a division by zero +constexpr inline int compute_default_alignment_helper(int ArrayBytes, int AlignmentBytes) { + EIGEN_UNUSED_VARIABLE(ArrayBytes); + EIGEN_UNUSED_VARIABLE(AlignmentBytes); + return 0; +} #endif template struct compute_default_alignment { - enum { value = compute_default_alignment_helper::value }; + enum { value = compute_default_alignment_helper(Size*sizeof(T), EIGEN_MAX_STATIC_ALIGN_BYTES) }; }; template struct compute_default_alignment { enum { value = EIGEN_MAX_ALIGN_BYTES }; }; -template class make_proper_matrix_type { enum { - IsColVector = _Cols==1 && _Rows!=1, - IsRowVector = _Rows==1 && _Cols!=1, - Options = IsColVector ? (_Options | ColMajor) & ~RowMajor - : IsRowVector ? (_Options | RowMajor) & ~ColMajor - : _Options + IsColVector = Cols_==1 && Rows_!=1, + IsRowVector = Rows_==1 && Cols_!=1, + Options = IsColVector ? (Options_ | ColMajor) & ~RowMajor + : IsRowVector ? (Options_ | RowMajor) & ~ColMajor + : Options_ }; public: - typedef Matrix<_Scalar, _Rows, _Cols, Options, _MaxRows, _MaxCols> type; + typedef Matrix type; }; -template -class compute_matrix_flags -{ - enum { row_major_bit = Options&RowMajor ? RowMajorBit : 0 }; - public: - // FIXME currently we still have to handle DirectAccessBit at the expression level to handle DenseCoeffsBase<> - // and then propagate this information to the evaluator's flags. - // However, I (Gael) think that DirectAccessBit should only matter at the evaluation stage. - enum { ret = DirectAccessBit | LvalueBit | NestByRefBit | row_major_bit }; -}; +constexpr inline unsigned compute_matrix_flags(int Options) { + unsigned row_major_bit = Options&RowMajor ? RowMajorBit : 0; + // FIXME currently we still have to handle DirectAccessBit at the expression level to handle DenseCoeffsBase<> + // and then propagate this information to the evaluator's flags. + // However, I (Gael) think that DirectAccessBit should only matter at the evaluation stage. + return DirectAccessBit | LvalueBit | NestByRefBit | row_major_bit; +} -template struct size_at_compile_time -{ - enum { ret = (_Rows==Dynamic || _Cols==Dynamic) ? Dynamic : _Rows * _Cols }; -}; +constexpr inline int size_at_compile_time(int rows, int cols) { + if (rows == 0 || cols == 0) return 0; + if (rows == Dynamic || cols == Dynamic) return Dynamic; + return rows * cols; +} template struct size_of_xpr_at_compile_time { - enum { ret = size_at_compile_time::RowsAtCompileTime,traits::ColsAtCompileTime>::ret }; + enum { ret = size_at_compile_time(traits::RowsAtCompileTime, traits::ColsAtCompileTime) }; }; /* plain_matrix_type : the difference from eval is that plain_matrix_type is always a plain matrix type, @@ -303,6 +337,11 @@ template struct plain_matrix_type typedef typename T::PlainObject type; }; +template struct plain_matrix_type +{ + typedef typename T::PlainObject type; +}; + template struct plain_matrix_type_dense { typedef Matrix::Scalar, @@ -349,17 +388,22 @@ template struct eval typedef typename plain_matrix_type::type type; }; +template struct eval +{ + typedef typename plain_matrix_type::type type; +}; + // for matrices, no need to evaluate, just use a const reference to avoid a useless copy -template -struct eval, Dense> +template +struct eval, Dense> { - typedef const Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>& type; + typedef const Matrix& type; }; -template -struct eval, Dense> +template +struct eval, Dense> { - typedef const Array<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>& type; + typedef const Array& type; }; @@ -415,28 +459,28 @@ template struct plain_matrix_type_row_major template struct ref_selector { - typedef typename conditional< + typedef std::conditional_t< bool(traits::Flags & NestByRefBit), T const&, const T - >::type type; + > type; - typedef typename conditional< + typedef std::conditional_t< bool(traits::Flags & NestByRefBit), T &, T - >::type non_const_type; + > non_const_type; }; /** \internal Adds the const qualifier on the value-type of T2 if and only if T1 is a const type */ template struct transfer_constness { - typedef typename conditional< + typedef std::conditional_t< bool(internal::is_const::value), - typename internal::add_const_on_value_type::type, + add_const_on_value_type_t, T2 - >::type type; + > type; }; @@ -469,7 +513,7 @@ template Evaluate = (int(evaluator::Flags) & EvalBeforeNestingBit) || (int(CostEval) < int(CostNoEval)) }; - typedef typename conditional::type>::type type; + typedef std::conditional_t::type> type; }; template @@ -509,10 +553,10 @@ struct generic_xpr_base template struct cast_return_type { typedef typename XprType::Scalar CurrentScalarType; - typedef typename remove_all::type _CastType; - typedef typename _CastType::Scalar NewScalarType; - typedef typename conditional::value, - const XprType&,CastType>::type type; + typedef remove_all_t CastType_; + typedef typename CastType_::Scalar NewScalarType; + typedef std::conditional_t::value, + const XprType&,CastType> type; }; template struct promote_storage_type; @@ -587,6 +631,12 @@ template struct product_promote_storage_type struct product_promote_storage_type { typedef Dense ret; }; template struct product_promote_storage_type { typedef Dense ret; }; +template struct product_promote_storage_type { typedef A ret; }; +template struct product_promote_storage_type { typedef B ret; }; +template struct product_promote_storage_type { typedef Dense ret; }; +template struct product_promote_storage_type { typedef Dense ret; }; +template struct product_promote_storage_type { typedef Dense ret; }; + template struct product_promote_storage_type { typedef A ret; }; template struct product_promote_storage_type { typedef B ret; }; template struct product_promote_storage_type { typedef Dense ret; }; @@ -603,11 +653,11 @@ struct plain_row_type typedef Array ArrayRowType; - typedef typename conditional< + typedef std::conditional_t< is_same< typename traits::XprKind, MatrixXpr >::value, MatrixRowType, ArrayRowType - >::type type; + > type; }; template @@ -618,27 +668,28 @@ struct plain_col_type typedef Array ArrayColType; - typedef typename conditional< + typedef std::conditional_t< is_same< typename traits::XprKind, MatrixXpr >::value, MatrixColType, ArrayColType - >::type type; + > type; }; template struct plain_diag_type { - enum { diag_size = EIGEN_SIZE_MIN_PREFER_DYNAMIC(ExpressionType::RowsAtCompileTime, ExpressionType::ColsAtCompileTime), - max_diag_size = EIGEN_SIZE_MIN_PREFER_FIXED(ExpressionType::MaxRowsAtCompileTime, ExpressionType::MaxColsAtCompileTime) + enum { diag_size = internal::min_size_prefer_dynamic(ExpressionType::RowsAtCompileTime, ExpressionType::ColsAtCompileTime), + max_diag_size = min_size_prefer_fixed(ExpressionType::MaxRowsAtCompileTime, + ExpressionType::MaxColsAtCompileTime) }; typedef Matrix MatrixDiagType; typedef Array ArrayDiagType; - typedef typename conditional< + typedef std::conditional_t< is_same< typename traits::XprKind, MatrixXpr >::value, MatrixDiagType, ArrayDiagType - >::type type; + > type; }; template @@ -652,7 +703,7 @@ struct plain_constant_type typedef Matrix::RowsAtCompileTime, traits::ColsAtCompileTime, Options, traits::MaxRowsAtCompileTime,traits::MaxColsAtCompileTime> matrix_type; - typedef CwiseNullaryOp, const typename conditional::XprKind, MatrixXpr >::value, matrix_type, array_type>::type > type; + typedef CwiseNullaryOp, const std::conditional_t::XprKind, MatrixXpr >::value, matrix_type, array_type> > type; }; template @@ -692,14 +743,14 @@ struct possibly_same_dense { template EIGEN_DEVICE_FUNC -bool is_same_dense(const T1 &mat1, const T2 &mat2, typename enable_if::value>::type * = 0) +bool is_same_dense(const T1 &mat1, const T2 &mat2, std::enable_if_t::value> * = 0) { return (mat1.data()==mat2.data()) && (mat1.innerStride()==mat2.innerStride()) && (mat1.outerStride()==mat2.outerStride()); } template EIGEN_DEVICE_FUNC -bool is_same_dense(const T1 &, const T2 &, typename enable_if::value>::type * = 0) +bool is_same_dense(const T1 &, const T2 &, std::enable_if_t::value> * = 0) { return false; } @@ -721,9 +772,9 @@ struct scalar_div_cost, Vectorized> { template -struct scalar_div_cost::type> { enum { value = 24 }; }; +struct scalar_div_cost> { enum { value = 24 }; }; template -struct scalar_div_cost::type> { enum { value = 21 }; }; +struct scalar_div_cost> { enum { value = 21 }; }; #ifdef EIGEN_DEBUG_ASSIGN @@ -758,6 +809,54 @@ std::string demangle_flags(int f) } #endif +template +struct is_block_xpr : std::false_type {}; + +template +struct is_block_xpr> : std::true_type {}; + +template +struct is_block_xpr> : std::true_type {}; + +// Helper utility for constructing non-recursive block expressions. +template +struct block_xpr_helper { + using BaseType = XprType; + + // For regular block expressions, simply forward along the InnerPanel argument, + // which is set when calling row/column expressions. + static constexpr bool is_inner_panel(bool inner_panel) { return inner_panel; } + + // Only enable non-const base function if XprType is not const (otherwise we get a duplicate definition). + template::value>> + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE BaseType& base(XprType& xpr) { return xpr; } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE const BaseType& base(const XprType& xpr) { return xpr; } + static constexpr EIGEN_ALWAYS_INLINE Index row(const XprType& /*xpr*/, Index r) { return r; } + static constexpr EIGEN_ALWAYS_INLINE Index col(const XprType& /*xpr*/, Index c) { return c; } +}; + +template +struct block_xpr_helper> { + using BlockXprType = Block; + // Recursive helper in case of explicit block-of-block expression. + using NestedXprHelper = block_xpr_helper; + using BaseType = typename NestedXprHelper::BaseType; + + // For block-of-block expressions, we need to combine the InnerPannel trait + // with that of the block subexpression. + static constexpr bool is_inner_panel(bool inner_panel) { return InnerPanel && inner_panel; } + + // Only enable non-const base function if XprType is not const (otherwise we get a duplicates definition). + template::value>> + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE BaseType& base(BlockXprType& xpr) { return NestedXprHelper::base(xpr.nestedExpression()); } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE const BaseType& base(const BlockXprType& xpr) { return NestedXprHelper::base(xpr.nestedExpression()); } + static constexpr EIGEN_ALWAYS_INLINE Index row(const BlockXprType& xpr, Index r) { return xpr.startRow() + NestedXprHelper::row(xpr.nestedExpression(), r); } + static constexpr EIGEN_ALWAYS_INLINE Index col(const BlockXprType& xpr, Index c) { return xpr.startCol() + NestedXprHelper::col(xpr.nestedExpression(), c); } +}; + +template +struct block_xpr_helper> : block_xpr_helper> {}; + } // end namespace internal @@ -812,12 +911,12 @@ struct ScalarBinaryOpTraits }; template -struct ScalarBinaryOpTraits::IsComplex,T>::type>::Real, BinaryOp> +struct ScalarBinaryOpTraits::IsComplex,T>>::Real, BinaryOp> { typedef T ReturnType; }; template -struct ScalarBinaryOpTraits::IsComplex,T>::type>::Real, T, BinaryOp> +struct ScalarBinaryOpTraits::IsComplex,T>>::Real, T, BinaryOp> { typedef T ReturnType; }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexEigenSolver.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexEigenSolver.h index 081e918f132..bf8bc794230 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexEigenSolver.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexEigenSolver.h @@ -14,6 +14,9 @@ #include "./ComplexSchur.h" +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \eigenvalues_module \ingroup Eigenvalues_Module @@ -23,7 +26,7 @@ namespace Eigen { * * \brief Computes eigenvalues and eigenvectors of general complex matrices * - * \tparam _MatrixType the type of the matrix of which we are + * \tparam MatrixType_ the type of the matrix of which we are * computing the eigendecomposition; this is expected to be an * instantiation of the Matrix class template. * @@ -42,12 +45,12 @@ namespace Eigen { * * \sa class EigenSolver, class SelfAdjointEigenSolver */ -template class ComplexEigenSolver +template class ComplexEigenSolver { public: - /** \brief Synonym for the template parameter \p _MatrixType. */ - typedef _MatrixType MatrixType; + /** \brief Synonym for the template parameter \p MatrixType_. */ + typedef MatrixType_ MatrixType; enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, @@ -236,12 +239,9 @@ template class ComplexEigenSolver } protected: - - static void check_template_parameters() - { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - } - + + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) + EigenvectorType m_eivec; EigenvalueType m_eivalues; ComplexSchur m_schur; @@ -260,8 +260,6 @@ template ComplexEigenSolver& ComplexEigenSolver::compute(const EigenBase& matrix, bool computeEigenvectors) { - check_template_parameters(); - // this code is inspired from Jampack eigen_assert(matrix.cols() == matrix.rows()); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexSchur.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexSchur.h index fc71468f8df..cdff38abc4f 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexSchur.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexSchur.h @@ -14,6 +14,9 @@ #include "./HessenbergDecomposition.h" +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -27,7 +30,7 @@ template struct complex_schur_reduce_to_hes * * \brief Performs a complex Schur decomposition of a real or complex square matrix * - * \tparam _MatrixType the type of the matrix of which we are + * \tparam MatrixType_ the type of the matrix of which we are * computing the Schur decomposition; this is expected to be an * instantiation of the Matrix class template. * @@ -48,10 +51,10 @@ template struct complex_schur_reduce_to_hes * * \sa class RealSchur, class EigenSolver, class ComplexEigenSolver */ -template class ComplexSchur +template class ComplexSchur { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, @@ -60,12 +63,12 @@ template class ComplexSchur MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; - /** \brief Scalar type for matrices of type \p _MatrixType. */ + /** \brief Scalar type for matrices of type \p MatrixType_. */ typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits::Real RealScalar; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 - /** \brief Complex scalar type for \p _MatrixType. + /** \brief Complex scalar type for \p MatrixType_. * * This is \c std::complex if #Scalar is real (e.g., * \c float or \c double) and just \c Scalar if #Scalar is @@ -76,7 +79,7 @@ template class ComplexSchur /** \brief Type for the matrices in the Schur decomposition. * * This is a square matrix with entries of type #ComplexScalar. - * The size is the same as the size of \p _MatrixType. + * The size is the same as the size of \p MatrixType_. */ typedef Matrix ComplexMatrixType; @@ -259,7 +262,7 @@ template class ComplexSchur friend struct internal::complex_schur_reduce_to_hessenberg::IsComplex>; }; -/** If m_matT(i+1,i) is neglegible in floating point arithmetic +/** If m_matT(i+1,i) is negligible in floating point arithmetic * compared to m_matT(i,i) and m_matT(j,j), then set it to zero and * return true, else return false. */ template @@ -306,7 +309,7 @@ typename ComplexSchur::ComplexScalar ComplexSchur::compu // In this case, det==0, and all we have to do is checking that eival2_norm!=0 if(eival1_norm > eival2_norm) eival2 = det / eival1; - else if(eival2_norm!=RealScalar(0)) + else if(!numext::is_exactly_zero(eival2_norm)) eival1 = det / eival2; // choose the eigenvalue closest to the bottom entry of the diagonal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexSchur_LAPACKE.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexSchur_LAPACKE.h index 4980a3ede0a..3f14128df9e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexSchur_LAPACKE.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/ComplexSchur_LAPACKE.h @@ -33,6 +33,9 @@ #ifndef EIGEN_COMPLEX_SCHUR_LAPACKE_H #define EIGEN_COMPLEX_SCHUR_LAPACKE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \internal Specialization for the data types supported by LAPACKe */ diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/EigenSolver.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/EigenSolver.h index 572b29e4e98..04191920aa4 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/EigenSolver.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/EigenSolver.h @@ -13,6 +13,9 @@ #include "./RealSchur.h" +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \eigenvalues_module \ingroup Eigenvalues_Module @@ -22,7 +25,7 @@ namespace Eigen { * * \brief Computes eigenvalues and eigenvectors of general matrices * - * \tparam _MatrixType the type of the matrix of which we are computing the + * \tparam MatrixType_ the type of the matrix of which we are computing the * eigendecomposition; this is expected to be an instantiation of the Matrix * class template. Currently, only real matrices are supported. * @@ -61,12 +64,12 @@ namespace Eigen { * * \sa MatrixBase::eigenvalues(), class ComplexEigenSolver, class SelfAdjointEigenSolver */ -template class EigenSolver +template class EigenSolver { public: - /** \brief Synonym for the template parameter \p _MatrixType. */ - typedef _MatrixType MatrixType; + /** \brief Synonym for the template parameter \p MatrixType_. */ + typedef MatrixType_ MatrixType; enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/GeneralizedEigenSolver.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/GeneralizedEigenSolver.h index 87d789b3f4a..3f8b1369699 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/GeneralizedEigenSolver.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/GeneralizedEigenSolver.h @@ -14,6 +14,9 @@ #include "./RealQZ.h" +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \eigenvalues_module \ingroup Eigenvalues_Module @@ -23,7 +26,7 @@ namespace Eigen { * * \brief Computes the generalized eigenvalues and eigenvectors of a pair of general matrices * - * \tparam _MatrixType the type of the matrices of which we are computing the + * \tparam MatrixType_ the type of the matrices of which we are computing the * eigen-decomposition; this is expected to be an instantiation of the Matrix * class template. Currently, only real matrices are supported. * @@ -55,12 +58,12 @@ namespace Eigen { * * \sa MatrixBase::eigenvalues(), class ComplexEigenSolver, class SelfAdjointEigenSolver */ -template class GeneralizedEigenSolver +template class GeneralizedEigenSolver { public: - /** \brief Synonym for the template parameter \p _MatrixType. */ - typedef _MatrixType MatrixType; + /** \brief Synonym for the template parameter \p MatrixType_. */ + typedef MatrixType_ MatrixType; enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, @@ -119,8 +122,8 @@ template class GeneralizedEigenSolver : m_eivec(), m_alphas(), m_betas(), - m_valuesOkay(false), - m_vectorsOkay(false), + m_computeEigenvectors(false), + m_isInitialized(false), m_realQZ() {} @@ -134,8 +137,8 @@ template class GeneralizedEigenSolver : m_eivec(size, size), m_alphas(size), m_betas(size), - m_valuesOkay(false), - m_vectorsOkay(false), + m_computeEigenvectors(false), + m_isInitialized(false), m_realQZ(size), m_tmp(size) {} @@ -156,8 +159,8 @@ template class GeneralizedEigenSolver : m_eivec(A.rows(), A.cols()), m_alphas(A.cols()), m_betas(A.cols()), - m_valuesOkay(false), - m_vectorsOkay(false), + m_computeEigenvectors(false), + m_isInitialized(false), m_realQZ(A.cols()), m_tmp(A.cols()) { @@ -177,7 +180,8 @@ template class GeneralizedEigenSolver * \sa eigenvalues() */ EigenvectorsType eigenvectors() const { - eigen_assert(m_vectorsOkay && "Eigenvectors for GeneralizedEigenSolver were not calculated."); + eigen_assert(info() == Success && "GeneralizedEigenSolver failed to compute eigenvectors"); + eigen_assert(m_computeEigenvectors && "Eigenvectors for GeneralizedEigenSolver were not calculated"); return m_eivec; } @@ -201,7 +205,7 @@ template class GeneralizedEigenSolver */ EigenvalueType eigenvalues() const { - eigen_assert(m_valuesOkay && "GeneralizedEigenSolver is not initialized."); + eigen_assert(info() == Success && "GeneralizedEigenSolver failed to compute eigenvalues."); return EigenvalueType(m_alphas,m_betas); } @@ -210,9 +214,9 @@ template class GeneralizedEigenSolver * This vector permits to reconstruct the j-th eigenvalues as alphas(i)/betas(j). * * \sa betas(), eigenvalues() */ - ComplexVectorType alphas() const + const ComplexVectorType& alphas() const { - eigen_assert(m_valuesOkay && "GeneralizedEigenSolver is not initialized."); + eigen_assert(info() == Success && "GeneralizedEigenSolver failed to compute alphas."); return m_alphas; } @@ -221,9 +225,9 @@ template class GeneralizedEigenSolver * This vector permits to reconstruct the j-th eigenvalues as alphas(i)/betas(j). * * \sa alphas(), eigenvalues() */ - VectorType betas() const + const VectorType& betas() const { - eigen_assert(m_valuesOkay && "GeneralizedEigenSolver is not initialized."); + eigen_assert(info() == Success && "GeneralizedEigenSolver failed to compute betas."); return m_betas; } @@ -254,7 +258,7 @@ template class GeneralizedEigenSolver ComputationInfo info() const { - eigen_assert(m_valuesOkay && "EigenSolver is not initialized."); + eigen_assert(m_isInitialized && "EigenSolver is not initialized."); return m_realQZ.info(); } @@ -267,17 +271,15 @@ template class GeneralizedEigenSolver } protected: - - static void check_template_parameters() - { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - EIGEN_STATIC_ASSERT(!NumTraits::IsComplex, NUMERIC_TYPE_MUST_BE_REAL); - } - + + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) + EIGEN_STATIC_ASSERT(!NumTraits::IsComplex, NUMERIC_TYPE_MUST_BE_REAL) + EigenvectorsType m_eivec; ComplexVectorType m_alphas; VectorType m_betas; - bool m_valuesOkay, m_vectorsOkay; + bool m_computeEigenvectors; + bool m_isInitialized; RealQZ m_realQZ; ComplexVectorType m_tmp; }; @@ -286,14 +288,10 @@ template GeneralizedEigenSolver& GeneralizedEigenSolver::compute(const MatrixType& A, const MatrixType& B, bool computeEigenvectors) { - check_template_parameters(); - using std::sqrt; using std::abs; eigen_assert(A.cols() == A.rows() && B.cols() == A.rows() && B.cols() == B.rows()); Index size = A.cols(); - m_valuesOkay = false; - m_vectorsOkay = false; // Reduce to generalized real Schur form: // A = Q S Z and B = Q T Z m_realQZ.compute(A, B, computeEigenvectors); @@ -406,10 +404,9 @@ GeneralizedEigenSolver::compute(const MatrixType& A, const MatrixTyp i += 2; } } - - m_valuesOkay = true; - m_vectorsOkay = computeEigenvectors; } + m_computeEigenvectors = computeEigenvectors; + m_isInitialized = true; return *this; } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h index d0f9091bebe..941127b55c1 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h @@ -13,6 +13,9 @@ #include "./Tridiagonalization.h" +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \eigenvalues_module \ingroup Eigenvalues_Module @@ -22,7 +25,7 @@ namespace Eigen { * * \brief Computes eigenvalues and eigenvectors of the generalized selfadjoint eigen problem * - * \tparam _MatrixType the type of the matrix of which we are computing the + * \tparam MatrixType_ the type of the matrix of which we are computing the * eigendecomposition; this is expected to be an instantiation of the Matrix * class template. * @@ -44,19 +47,19 @@ namespace Eigen { * * \sa class SelfAdjointEigenSolver, class EigenSolver, class ComplexEigenSolver */ -template -class GeneralizedSelfAdjointEigenSolver : public SelfAdjointEigenSolver<_MatrixType> +template +class GeneralizedSelfAdjointEigenSolver : public SelfAdjointEigenSolver { - typedef SelfAdjointEigenSolver<_MatrixType> Base; + typedef SelfAdjointEigenSolver Base; public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; /** \brief Default constructor for fixed-size matrices. * * The default constructor is useful in cases in which the user intends to * perform decompositions via compute(). This constructor - * can only be used if \p _MatrixType is a fixed-size matrix; use + * can only be used if \p MatrixType_ is a fixed-size matrix; use * GeneralizedSelfAdjointEigenSolver(Index) for dynamic-size matrices. */ GeneralizedSelfAdjointEigenSolver() : Base() {} diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/HessenbergDecomposition.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/HessenbergDecomposition.h index 1f21139346e..2f2804d038e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/HessenbergDecomposition.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/HessenbergDecomposition.h @@ -11,6 +11,9 @@ #ifndef EIGEN_HESSENBERGDECOMPOSITION_H #define EIGEN_HESSENBERGDECOMPOSITION_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -31,7 +34,7 @@ struct traits > * * \brief Reduces a square matrix to Hessenberg form by an orthogonal similarity transformation * - * \tparam _MatrixType the type of the matrix of which we are computing the Hessenberg decomposition + * \tparam MatrixType_ the type of the matrix of which we are computing the Hessenberg decomposition * * This class performs an Hessenberg decomposition of a matrix \f$ A \f$. In * the real case, the Hessenberg decomposition consists of an orthogonal @@ -54,12 +57,12 @@ struct traits > * * \sa class ComplexSchur, class Tridiagonalization, \ref QR_Module "QR Module" */ -template class HessenbergDecomposition +template class HessenbergDecomposition { public: - /** \brief Synonym for the template parameter \p _MatrixType. */ - typedef _MatrixType MatrixType; + /** \brief Synonym for the template parameter \p MatrixType_. */ + typedef MatrixType_ MatrixType; enum { Size = MatrixType::RowsAtCompileTime, @@ -82,7 +85,7 @@ template class HessenbergDecomposition typedef Matrix CoeffVectorType; /** \brief Return type of matrixQ() */ - typedef HouseholderSequence::type> HouseholderSequenceType; + typedef HouseholderSequence> HouseholderSequenceType; typedef internal::HessenbergDecompositionMatrixHReturnType MatrixHReturnType; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/InternalHeaderCheck.h new file mode 100644 index 00000000000..374cbd45571 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_EIGENVALUES_MODULE_H +#error "Please include Eigen/Eigenvalues instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/MatrixBaseEigenvalues.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/MatrixBaseEigenvalues.h index 66e5a3dbb04..33748917afd 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/MatrixBaseEigenvalues.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/MatrixBaseEigenvalues.h @@ -11,6 +11,9 @@ #ifndef EIGEN_MATRIXBASEEIGENVALUES_H #define EIGEN_MATRIXBASEEIGENVALUES_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealQZ.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealQZ.h index 509130184b9..675ced08f42 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealQZ.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealQZ.h @@ -10,6 +10,9 @@ #ifndef EIGEN_REAL_QZ_H #define EIGEN_REAL_QZ_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \eigenvalues_module \ingroup Eigenvalues_Module @@ -19,7 +22,7 @@ namespace Eigen { * * \brief Performs a real QZ decomposition of a pair of square matrices * - * \tparam _MatrixType the type of the matrix of which we are computing the + * \tparam MatrixType_ the type of the matrix of which we are computing the * real QZ decomposition; this is expected to be an instantiation of the * Matrix class template. * @@ -54,10 +57,10 @@ namespace Eigen { * \sa class RealSchur, class ComplexSchur, class EigenSolver, class ComplexEigenSolver */ - template class RealQZ + template class RealQZ { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, @@ -237,7 +240,7 @@ namespace Eigen { for (Index i=dim-1; i>=j+2; i--) { JRs G; // kill S(i,j) - if(m_S.coeff(i,j) != 0) + if(!numext::is_exactly_zero(m_S.coeff(i, j))) { G.makeGivens(m_S.coeff(i-1,j), m_S.coeff(i,j), &m_S.coeffRef(i-1, j)); m_S.coeffRef(i,j) = Scalar(0.0); @@ -248,7 +251,7 @@ namespace Eigen { m_Q.applyOnTheRight(i-1,i,G); } // kill T(i,i-1) - if(m_T.coeff(i,i-1)!=Scalar(0)) + if(!numext::is_exactly_zero(m_T.coeff(i, i - 1))) { G.makeGivens(m_T.coeff(i,i), m_T.coeff(i,i-1), &m_T.coeffRef(i,i)); m_T.coeffRef(i,i-1) = Scalar(0.0); @@ -286,7 +289,7 @@ namespace Eigen { while (res > 0) { Scalar s = abs(m_S.coeff(res-1,res-1)) + abs(m_S.coeff(res,res)); - if (s == Scalar(0.0)) + if (numext::is_exactly_zero(s)) s = m_normOfS; if (abs(m_S.coeff(res,res-1)) < NumTraits::epsilon() * s) break; @@ -316,7 +319,7 @@ namespace Eigen { using std::abs; using std::sqrt; const Index dim=m_S.cols(); - if (abs(m_S.coeff(i+1,i))==Scalar(0)) + if (numext::is_exactly_zero(abs(m_S.coeff(i + 1, i)))) return; Index j = findSmallDiagEntry(i,i+1); if (j==i-1) @@ -627,7 +630,7 @@ namespace Eigen { { for(Index i=0; i j_left, j_right; internal::real_2x2_jacobi_svd(m_T, i, i+1, &j_left, &j_right); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealSchur.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealSchur.h index 7304ef3449e..12850cb64e6 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealSchur.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealSchur.h @@ -13,6 +13,9 @@ #include "./HessenbergDecomposition.h" +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \eigenvalues_module \ingroup Eigenvalues_Module @@ -22,7 +25,7 @@ namespace Eigen { * * \brief Performs a real Schur decomposition of a square matrix * - * \tparam _MatrixType the type of the matrix of which we are computing the + * \tparam MatrixType_ the type of the matrix of which we are computing the * real Schur decomposition; this is expected to be an instantiation of the * Matrix class template. * @@ -51,10 +54,10 @@ namespace Eigen { * * \sa class ComplexSchur, class EigenSolver, class ComplexEigenSolver */ -template class RealSchur +template class RealSchur { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, @@ -312,7 +315,7 @@ RealSchur& RealSchur::computeFromHessenberg(const HessMa Scalar considerAsZero = numext::maxi( norm * numext::abs2(NumTraits::epsilon()), (std::numeric_limits::min)() ); - if(norm!=Scalar(0)) + if(!numext::is_exactly_zero(norm)) { while (iu >= 0) { @@ -515,7 +518,7 @@ inline void RealSchur::performFrancisQRStep(Index il, Index im, Inde Matrix ess; v.makeHouseholder(ess, tau, beta); - if (beta != Scalar(0)) // if v is not zero + if (!numext::is_exactly_zero(beta)) // if v is not zero { if (firstIteration && k > il) m_matT.coeffRef(k,k-1) = -m_matT.coeff(k,k-1); @@ -535,7 +538,7 @@ inline void RealSchur::performFrancisQRStep(Index il, Index im, Inde Matrix ess; v.makeHouseholder(ess, tau, beta); - if (beta != Scalar(0)) // if v is not zero + if (!numext::is_exactly_zero(beta)) // if v is not zero { m_matT.coeffRef(iu-1, iu-2) = beta; m_matT.block(iu-1, iu-1, 2, size-iu+1).applyHouseholderOnTheLeft(ess, tau, workspace); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealSchur_LAPACKE.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealSchur_LAPACKE.h index 2c22517155e..562b8ac2573 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealSchur_LAPACKE.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/RealSchur_LAPACKE.h @@ -33,6 +33,9 @@ #ifndef EIGEN_REAL_SCHUR_LAPACKE_H #define EIGEN_REAL_SCHUR_LAPACKE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \internal Specialization for the data types supported by LAPACKe */ diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/SelfAdjointEigenSolver.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/SelfAdjointEigenSolver.h index 14692365ffb..fff98747b0b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/SelfAdjointEigenSolver.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/SelfAdjointEigenSolver.h @@ -13,9 +13,12 @@ #include "./Tridiagonalization.h" +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { -template +template class GeneralizedSelfAdjointEigenSolver; namespace internal { @@ -33,7 +36,7 @@ ComputationInfo computeFromTridiagonal_impl(DiagType& diag, SubDiagType& subdiag * * \brief Computes eigenvalues and eigenvectors of selfadjoint matrices * - * \tparam _MatrixType the type of the matrix of which we are computing the + * \tparam MatrixType_ the type of the matrix of which we are computing the * eigendecomposition; this is expected to be an instantiation of the Matrix * class template. * @@ -73,11 +76,11 @@ ComputationInfo computeFromTridiagonal_impl(DiagType& diag, SubDiagType& subdiag * * \sa MatrixBase::eigenvalues(), class EigenSolver, class ComplexEigenSolver */ -template class SelfAdjointEigenSolver +template class SelfAdjointEigenSolver { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; enum { Size = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, @@ -85,13 +88,13 @@ template class SelfAdjointEigenSolver MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; - /** \brief Scalar type for matrices of type \p _MatrixType. */ + /** \brief Scalar type for matrices of type \p MatrixType_. */ typedef typename MatrixType::Scalar Scalar; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 typedef Matrix EigenvectorsType; - /** \brief Real scalar type for \p _MatrixType. + /** \brief Real scalar type for \p MatrixType_. * * This is just \c Scalar if #Scalar is real (e.g., \c float or * \c double), and the type of the real part of \c Scalar if #Scalar is @@ -104,8 +107,9 @@ template class SelfAdjointEigenSolver /** \brief Type for vector of eigenvalues as returned by eigenvalues(). * * This is a column vector with entries of type #RealScalar. - * The length of the vector is the size of \p _MatrixType. + * The length of the vector is the size of \p MatrixType_. */ + typedef typename internal::plain_col_type::type VectorType; typedef typename internal::plain_col_type::type RealVectorType; typedef Tridiagonalization TridiagonalizationType; typedef typename TridiagonalizationType::SubDiagonalType SubDiagonalType; @@ -114,7 +118,7 @@ template class SelfAdjointEigenSolver * * The default constructor is useful in cases in which the user intends to * perform decompositions via compute(). This constructor - * can only be used if \p _MatrixType is a fixed-size matrix; use + * can only be used if \p MatrixType_ is a fixed-size matrix; use * SelfAdjointEigenSolver(Index) for dynamic-size matrices. * * Example: \include SelfAdjointEigenSolver_SelfAdjointEigenSolver.cpp @@ -123,6 +127,7 @@ template class SelfAdjointEigenSolver EIGEN_DEVICE_FUNC SelfAdjointEigenSolver() : m_eivec(), + m_workspace(), m_eivalues(), m_subdiag(), m_hcoeffs(), @@ -146,6 +151,7 @@ template class SelfAdjointEigenSolver EIGEN_DEVICE_FUNC explicit SelfAdjointEigenSolver(Index size) : m_eivec(size, size), + m_workspace(size), m_eivalues(size), m_subdiag(size > 1 ? size - 1 : 1), m_hcoeffs(size > 1 ? size - 1 : 1), @@ -172,6 +178,7 @@ template class SelfAdjointEigenSolver EIGEN_DEVICE_FUNC explicit SelfAdjointEigenSolver(const EigenBase& matrix, int options = ComputeEigenvectors) : m_eivec(matrix.rows(), matrix.cols()), + m_workspace(matrix.cols()), m_eivalues(matrix.cols()), m_subdiag(matrix.rows() > 1 ? matrix.rows() - 1 : 1), m_hcoeffs(matrix.cols() > 1 ? matrix.cols() - 1 : 1), @@ -372,13 +379,10 @@ template class SelfAdjointEigenSolver static const int m_maxIterations = 30; protected: - static EIGEN_DEVICE_FUNC - void check_template_parameters() - { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - } - + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) + EigenvectorsType m_eivec; + VectorType m_workspace; RealVectorType m_eivalues; typename TridiagonalizationType::SubDiagonalType m_subdiag; typename TridiagonalizationType::CoeffVectorType m_hcoeffs; @@ -419,10 +423,8 @@ EIGEN_DEVICE_FUNC SelfAdjointEigenSolver& SelfAdjointEigenSolver ::compute(const EigenBase& a_matrix, int options) { - check_template_parameters(); - const InputType &matrix(a_matrix.derived()); - + EIGEN_USING_STD(abs); eigen_assert(matrix.cols() == matrix.rows()); eigen_assert((options&~(EigVecMask|GenEigMask))==0 @@ -451,11 +453,11 @@ ::compute(const EigenBase& a_matrix, int options) // map the matrix coefficients to [-1:1] to avoid over- and underflow. mat = matrix.template triangularView(); RealScalar scale = mat.cwiseAbs().maxCoeff(); - if(scale==RealScalar(0)) scale = RealScalar(1); + if(numext::is_exactly_zero(scale)) scale = RealScalar(1); mat.template triangularView() /= scale; m_subdiag.resize(n-1); m_hcoeffs.resize(n-1); - internal::tridiagonalization_inplace(mat, diag, m_subdiag, m_hcoeffs, computeEigenvectors); + internal::tridiagonalization_inplace(mat, diag, m_subdiag, m_hcoeffs, m_workspace, computeEigenvectors); m_info = internal::computeFromTridiagonal_impl(diag, m_subdiag, m_maxIterations, computeEigenvectors, m_eivec); @@ -530,7 +532,7 @@ ComputationInfo computeFromTridiagonal_impl(DiagType& diag, SubDiagType& subdiag } // find the largest unreduced block at the end of the matrix. - while (end>0 && subdiag[end-1]==RealScalar(0)) + while (end>0 && numext::is_exactly_zero(subdiag[end - 1])) { end--; } @@ -542,7 +544,7 @@ ComputationInfo computeFromTridiagonal_impl(DiagType& diag, SubDiagType& subdiag if(iter > maxIterations * n) break; start = end - 1; - while (start>0 && subdiag[start-1]!=0) + while (start>0 && !numext::is_exactly_zero(subdiag[start - 1])) start--; internal::tridiagonal_qr_step(diag.data(), subdiag.data(), start, end, computeEigenvectors ? eivec.data() : (Scalar*)0, n); @@ -847,12 +849,12 @@ static void tridiagonal_qr_step(RealScalar* diag, RealScalar* subdiag, Index sta // RealScalar mu = diag[end] - e2 / (td + (td>0 ? 1 : -1) * sqrt(td*td + e2)); // This explain the following, somewhat more complicated, version: RealScalar mu = diag[end]; - if(td==RealScalar(0)) { + if(numext::is_exactly_zero(td)) { mu -= numext::abs(e); - } else if (e != RealScalar(0)) { + } else if (!numext::is_exactly_zero(e)) { const RealScalar e2 = numext::abs2(e); const RealScalar h = numext::hypot(td,e); - if(e2 == RealScalar(0)) { + if(numext::is_exactly_zero(e2)) { mu -= e / ((td + (td>RealScalar(0) ? h : -h)) / e); } else { mu -= e2 / (td + (td>RealScalar(0) ? h : -h)); @@ -863,7 +865,7 @@ static void tridiagonal_qr_step(RealScalar* diag, RealScalar* subdiag, Index sta RealScalar z = subdiag[start]; // If z ever becomes zero, the Givens rotation will be the identity and // z will stay zero for all future iterations. - for (Index k = start; k < end && z != RealScalar(0); ++k) + for (Index k = start; k < end && !numext::is_exactly_zero(z); ++k) { JacobiRotation rot; rot.makeGivens(x, z); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/SelfAdjointEigenSolver_LAPACKE.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/SelfAdjointEigenSolver_LAPACKE.h index b0c947dc07b..bfe044bbcbf 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/SelfAdjointEigenSolver_LAPACKE.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/SelfAdjointEigenSolver_LAPACKE.h @@ -33,6 +33,9 @@ #ifndef EIGEN_SAEIGENSOLVER_LAPACKE_H #define EIGEN_SAEIGENSOLVER_LAPACKE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \internal Specialization for the data types supported by LAPACKe */ diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/Tridiagonalization.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/Tridiagonalization.h index eda82794ab6..e884b7056bf 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/Tridiagonalization.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Eigenvalues/Tridiagonalization.h @@ -11,6 +11,9 @@ #ifndef EIGEN_TRIDIAGONALIZATION_H #define EIGEN_TRIDIAGONALIZATION_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -36,7 +39,7 @@ void tridiagonalization_inplace(MatrixType& matA, CoeffVectorType& hCoeffs); * * \brief Tridiagonal decomposition of a selfadjoint matrix * - * \tparam _MatrixType the type of the matrix of which we are computing the + * \tparam MatrixType_ the type of the matrix of which we are computing the * tridiagonal decomposition; this is expected to be an instantiation of the * Matrix class template. * @@ -61,12 +64,12 @@ void tridiagonalization_inplace(MatrixType& matA, CoeffVectorType& hCoeffs); * * \sa class HessenbergDecomposition, class SelfAdjointEigenSolver */ -template class Tridiagonalization +template class Tridiagonalization { public: - /** \brief Synonym for the template parameter \p _MatrixType. */ - typedef _MatrixType MatrixType; + /** \brief Synonym for the template parameter \p MatrixType_. */ + typedef MatrixType_ MatrixType; typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits::Real RealScalar; @@ -83,21 +86,21 @@ template class Tridiagonalization typedef Matrix CoeffVectorType; typedef typename internal::plain_col_type::type DiagonalType; typedef Matrix SubDiagonalType; - typedef typename internal::remove_all::type MatrixTypeRealView; + typedef internal::remove_all_t MatrixTypeRealView; typedef internal::TridiagonalizationMatrixTReturnType MatrixTReturnType; - typedef typename internal::conditional::IsComplex, - typename internal::add_const_on_value_type::RealReturnType>::type, + typedef std::conditional_t::IsComplex, + internal::add_const_on_value_type_t::RealReturnType>, const Diagonal - >::type DiagonalReturnType; + > DiagonalReturnType; - typedef typename internal::conditional::IsComplex, - typename internal::add_const_on_value_type::RealReturnType>::type, + typedef std::conditional_t::IsComplex, + internal::add_const_on_value_type_t::RealReturnType>, const Diagonal - >::type SubDiagonalReturnType; + > SubDiagonalReturnType; /** \brief Return type of matrixQ() */ - typedef HouseholderSequence::type> HouseholderSequenceType; + typedef HouseholderSequence> HouseholderSequenceType; /** \brief Default constructor. * @@ -425,13 +428,13 @@ struct tridiagonalization_inplace_selector; * * \sa class Tridiagonalization */ -template +template EIGEN_DEVICE_FUNC void tridiagonalization_inplace(MatrixType& mat, DiagonalType& diag, SubDiagonalType& subdiag, - CoeffVectorType& hcoeffs, bool extractQ) + CoeffVectorType& hcoeffs, WorkSpaceType& workspace, bool extractQ) { eigen_assert(mat.cols()==mat.rows() && diag.size()==mat.rows() && subdiag.size()==mat.rows()-1); - tridiagonalization_inplace_selector::run(mat, diag, subdiag, hcoeffs, extractQ); + tridiagonalization_inplace_selector::run(mat, diag, subdiag, hcoeffs, workspace, extractQ); } /** \internal @@ -441,17 +444,19 @@ template struct tridiagonalization_inplace_selector { typedef typename Tridiagonalization::HouseholderSequenceType HouseholderSequenceType; - template + template static EIGEN_DEVICE_FUNC - void run(MatrixType& mat, DiagonalType& diag, SubDiagonalType& subdiag, CoeffVectorType& hCoeffs, bool extractQ) + void run(MatrixType& mat, DiagonalType& diag, SubDiagonalType& subdiag, CoeffVectorType& hCoeffs, WorkSpaceType& workspace, bool extractQ) { tridiagonalization_inplace(mat, hCoeffs); diag = mat.diagonal().real(); subdiag = mat.template diagonal<-1>().real(); - if(extractQ) - mat = HouseholderSequenceType(mat, hCoeffs.conjugate()) - .setLength(mat.rows() - 1) - .setShift(1); + if (extractQ) { + HouseholderSequenceType(mat, hCoeffs.conjugate()) + .setLength(mat.rows() - 1) + .setShift(1) + .evalTo(mat, workspace); + } } }; @@ -465,8 +470,8 @@ struct tridiagonalization_inplace_selector typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; - template - static void run(MatrixType& mat, DiagonalType& diag, SubDiagonalType& subdiag, CoeffVectorType&, bool extractQ) + template + static void run(MatrixType& mat, DiagonalType& diag, SubDiagonalType& subdiag, CoeffVectorType&, WorkSpaceType&, bool extractQ) { using std::sqrt; const RealScalar tol = (std::numeric_limits::min)(); @@ -510,9 +515,9 @@ struct tridiagonalization_inplace_selector { typedef typename MatrixType::Scalar Scalar; - template + template static EIGEN_DEVICE_FUNC - void run(MatrixType& mat, DiagonalType& diag, SubDiagonalType&, CoeffVectorType&, bool extractQ) + void run(MatrixType& mat, DiagonalType& diag, SubDiagonalType&, CoeffVectorType&, WorkSpaceType&, bool extractQ) { diag(0,0) = numext::real(mat(0,0)); if(extractQ) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/AlignedBox.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/AlignedBox.h index 55a9d0ae134..3d51fc93ef1 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/AlignedBox.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/AlignedBox.h @@ -46,6 +46,9 @@ #ifndef EIGEN_ALIGNEDBOX_H #define EIGEN_ALIGNEDBOX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \geometry_module \ingroup Geometry_Module @@ -55,20 +58,20 @@ namespace Eigen { * * \brief An axis aligned box * - * \tparam _Scalar the type of the scalar coefficients - * \tparam _AmbientDim the dimension of the ambient space, can be a compile time value or Dynamic. + * \tparam Scalar_ the type of the scalar coefficients + * \tparam AmbientDim_ the dimension of the ambient space, can be a compile time value or Dynamic. * * This class represents an axis aligned box as a pair of the minimal and maximal corners. * \warning The result of most methods is undefined when applied to an empty box. You can check for empty boxes using isEmpty(). * \sa alignedboxtypedefs */ -template +template class AlignedBox { public: -EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(_Scalar,_AmbientDim) - enum { AmbientDimAtCompileTime = _AmbientDim }; - typedef _Scalar Scalar; +EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar_,AmbientDim_) + enum { AmbientDimAtCompileTime = AmbientDim_ }; + typedef Scalar_ Scalar; typedef NumTraits ScalarTraits; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 typedef typename ScalarTraits::Real RealScalar; @@ -181,7 +184,7 @@ EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(_Scalar,_AmbientDim) */ EIGEN_DEVICE_FUNC inline VectorType corner(CornerType corner) const { - EIGEN_STATIC_ASSERT(_AmbientDim <= 3, THIS_METHOD_IS_ONLY_FOR_VECTORS_OF_A_SPECIFIC_SIZE); + EIGEN_STATIC_ASSERT(AmbientDim_ <= 3, THIS_METHOD_IS_ONLY_FOR_VECTORS_OF_A_SPECIFIC_SIZE); VectorType res; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/AngleAxis.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/AngleAxis.h index 78328b6b572..75cb0eea07f 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/AngleAxis.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/AngleAxis.h @@ -10,6 +10,9 @@ #ifndef EIGEN_ANGLEAXIS_H #define EIGEN_ANGLEAXIS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \geometry_module \ingroup Geometry_Module @@ -18,7 +21,7 @@ namespace Eigen { * * \brief Represents a 3D rotation as a rotation angle around an arbitrary 3D axis * - * \param _Scalar the scalar type, i.e., the type of the coefficients. + * \param Scalar_ the scalar type, i.e., the type of the coefficients. * * \warning When setting up an AngleAxis object, the axis vector \b must \b be \b normalized. * @@ -39,16 +42,16 @@ namespace Eigen { */ namespace internal { -template struct traits > +template struct traits > { - typedef _Scalar Scalar; + typedef Scalar_ Scalar; }; } -template -class AngleAxis : public RotationBase,3> +template +class AngleAxis : public RotationBase,3> { - typedef RotationBase,3> Base; + typedef RotationBase,3> Base; public: @@ -56,7 +59,7 @@ class AngleAxis : public RotationBase,3> enum { Dim = 3 }; /** the scalar type of the coefficients */ - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef Matrix Matrix3; typedef Matrix Vector3; typedef Quaternion QuaternionType; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/EulerAngles.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/EulerAngles.h index 19b734ca7e0..6151c996327 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/EulerAngles.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/EulerAngles.h @@ -2,6 +2,7 @@ // for linear algebra. // // Copyright (C) 2008 Gael Guennebaud +// Copyright (C) 2023 Juraj Oršulić, University of Zagreb // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -10,12 +11,15 @@ #ifndef EIGEN_EULERANGLES_H #define EIGEN_EULERANGLES_H -namespace Eigen { +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { /** \geometry_module \ingroup Geometry_Module * * - * \returns the Euler-angles of the rotation matrix \c *this using the convention defined by the triplet (\a a0,\a a1,\a a2) + * \returns the canonical Euler-angles of the rotation matrix \c *this using the convention defined by the triplet (\a a0,\a a1,\a a2) * * Each of the three parameters \a a0,\a a1,\a a2 represents the respective rotation axis as an integer in {0,1,2}. * For instance, in: @@ -27,85 +31,190 @@ namespace Eigen { * * AngleAxisf(ea[1], Vector3f::UnitX()) * * AngleAxisf(ea[2], Vector3f::UnitZ()); \endcode * This corresponds to the right-multiply conventions (with right hand side frames). - * - * The returned angles are in the ranges [0:pi]x[-pi:pi]x[-pi:pi]. - * + * + * For Tait-Bryan angle configurations (a0 != a2), the returned angles are in the ranges [-pi:pi]x[-pi/2:pi/2]x[-pi:pi]. + * For proper Euler angle configurations (a0 == a2), the returned angles are in the ranges [-pi:pi]x[0:pi]x[-pi:pi]. + * + * The approach used is also described here: https://d3cw3dd2w32x2b.cloudfront.net/wp-content/uploads/2012/07/euler-angles.pdf + * * \sa class AngleAxis */ template EIGEN_DEVICE_FUNC inline Matrix::Scalar,3,1> +MatrixBase::canonicalEulerAngles(Index a0, Index a1, Index a2) const +{ + /* Implemented from Graphics Gems IV */ + EIGEN_STATIC_ASSERT_MATRIX_SPECIFIC_SIZE(Derived, 3, 3) + + Matrix res; + + const Index odd = ((a0 + 1) % 3 == a1) ? 0 : 1; + const Index i = a0; + const Index j = (a0 + 1 + odd) % 3; + const Index k = (a0 + 2 - odd) % 3; + + if (a0 == a2) + { + // Proper Euler angles (same first and last axis). + // The i, j, k indices enable addressing the input matrix as the XYX archetype matrix (see Graphics Gems IV), + // where e.g. coeff(k, i) means third column, first row in the XYX archetype matrix: + // c2 s2s1 s2c1 + // s2s3 -c2s1s3 + c1c3 -c2c1s3 - s1c3 + // -s2c3 c2s1c3 + c1s3 c2c1c3 - s1s3 + + // Note: s2 is always positive. + Scalar s2 = numext::hypot(coeff(j, i), coeff(k, i)); + if (odd) + { + res[0] = numext::atan2(coeff(j, i), coeff(k, i)); + // s2 is always positive, so res[1] will be within the canonical [0, pi] range + res[1] = numext::atan2(s2, coeff(i, i)); + } + else + { + // In the !odd case, signs of all three angles are flipped at the very end. To keep the solution within the canonical range, + // we flip the solution and make res[1] always negative here (since s2 is always positive, -atan2(s2, c2) will always be negative). + // The final flip at the end due to !odd will thus make res[1] positive and canonical. + // NB: in the general case, there are two correct solutions, but only one is canonical. For proper Euler angles, + // flipping from one solution to the other involves flipping the sign of the second angle res[1] and adding/subtracting pi + // to the first and third angles. The addition/subtraction of pi to the first angle res[0] is handled here by flipping + // the signs of arguments to atan2, while the calculation of the third angle does not need special adjustment since + // it uses the adjusted res[0] as the input and produces a correct result. + res[0] = numext::atan2(-coeff(j, i), -coeff(k, i)); + res[1] = -numext::atan2(s2, coeff(i, i)); + } + + // With a=(0,1,0), we have i=0; j=1; k=2, and after computing the first two angles, + // we can compute their respective rotation, and apply its inverse to M. Since the result must + // be a rotation around x, we have: + // + // c2 s1.s2 c1.s2 1 0 0 + // 0 c1 -s1 * M = 0 c3 s3 + // -s2 s1.c2 c1.c2 0 -s3 c3 + // + // Thus: m11.c1 - m21.s1 = c3 & m12.c1 - m22.s1 = s3 + + Scalar s1 = numext::sin(res[0]); + Scalar c1 = numext::cos(res[0]); + res[2] = numext::atan2(c1 * coeff(j, k) - s1 * coeff(k, k), c1 * coeff(j, j) - s1 * coeff(k, j)); + } + else + { + // Tait-Bryan angles (all three axes are different; typically used for yaw-pitch-roll calculations). + // The i, j, k indices enable addressing the input matrix as the XYZ archetype matrix (see Graphics Gems IV), + // where e.g. coeff(k, i) means third column, first row in the XYZ archetype matrix: + // c2c3 s2s1c3 - c1s3 s2c1c3 + s1s3 + // c2s3 s2s1s3 + c1c3 s2c1s3 - s1c3 + // -s2 c2s1 c2c1 + + res[0] = numext::atan2(coeff(j, k), coeff(k, k)); + + Scalar c2 = numext::hypot(coeff(i, i), coeff(i, j)); + // c2 is always positive, so the following atan2 will always return a result in the correct canonical middle angle range [-pi/2, pi/2] + res[1] = numext::atan2(-coeff(i, k), c2); + + Scalar s1 = numext::sin(res[0]); + Scalar c1 = numext::cos(res[0]); + res[2] = numext::atan2(s1 * coeff(k, i) - c1 * coeff(j, i), c1 * coeff(j, j) - s1 * coeff(k, j)); + } + if (!odd) + { + res = -res; + } + + return res; +} + +/** \geometry_module \ingroup Geometry_Module + * + * + * \returns the Euler-angles of the rotation matrix \c *this using the convention defined by the triplet (\a a0,\a a1,\a a2) + * + * NB: The returned angles are in non-canonical ranges [0:pi]x[-pi:pi]x[-pi:pi]. For canonical Tait-Bryan/proper Euler ranges, use canonicalEulerAngles. + * + * \sa MatrixBase::canonicalEulerAngles + * \sa class AngleAxis + */ +template +EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline Matrix::Scalar,3,1> MatrixBase::eulerAngles(Index a0, Index a1, Index a2) const { - EIGEN_USING_STD(atan2) - EIGEN_USING_STD(sin) - EIGEN_USING_STD(cos) /* Implemented from Graphics Gems IV */ - EIGEN_STATIC_ASSERT_MATRIX_SPECIFIC_SIZE(Derived,3,3) + EIGEN_STATIC_ASSERT_MATRIX_SPECIFIC_SIZE(Derived, 3, 3) - Matrix res; - typedef Matrix Vector2; + Matrix res; - const Index odd = ((a0+1)%3 == a1) ? 0 : 1; + const Index odd = ((a0 + 1) % 3 == a1) ? 0 : 1; const Index i = a0; - const Index j = (a0 + 1 + odd)%3; - const Index k = (a0 + 2 - odd)%3; - - if (a0==a2) + const Index j = (a0 + 1 + odd) % 3; + const Index k = (a0 + 2 - odd) % 3; + + if (a0 == a2) { - res[0] = atan2(coeff(j,i), coeff(k,i)); - if((odd && res[0]Scalar(0))) + res[0] = numext::atan2(coeff(j, i), coeff(k, i)); + if ((odd && res[0] < Scalar(0)) || ((!odd) && res[0] > Scalar(0))) { - if(res[0] > Scalar(0)) { + if (res[0] > Scalar(0)) + { res[0] -= Scalar(EIGEN_PI); } - else { + else + { res[0] += Scalar(EIGEN_PI); } - Scalar s2 = Vector2(coeff(j,i), coeff(k,i)).norm(); - res[1] = -atan2(s2, coeff(i,i)); + + Scalar s2 = numext::hypot(coeff(j, i), coeff(k, i)); + res[1] = -numext::atan2(s2, coeff(i, i)); } else { - Scalar s2 = Vector2(coeff(j,i), coeff(k,i)).norm(); - res[1] = atan2(s2, coeff(i,i)); + Scalar s2 = numext::hypot(coeff(j, i), coeff(k, i)); + res[1] = numext::atan2(s2, coeff(i, i)); } - + // With a=(0,1,0), we have i=0; j=1; k=2, and after computing the first two angles, // we can compute their respective rotation, and apply its inverse to M. Since the result must // be a rotation around x, we have: // - // c2 s1.s2 c1.s2 1 0 0 + // c2 s1.s2 c1.s2 1 0 0 // 0 c1 -s1 * M = 0 c3 s3 // -s2 s1.c2 c1.c2 0 -s3 c3 // // Thus: m11.c1 - m21.s1 = c3 & m12.c1 - m22.s1 = s3 - - Scalar s1 = sin(res[0]); - Scalar c1 = cos(res[0]); - res[2] = atan2(c1*coeff(j,k)-s1*coeff(k,k), c1*coeff(j,j) - s1 * coeff(k,j)); - } + + Scalar s1 = numext::sin(res[0]); + Scalar c1 = numext::cos(res[0]); + res[2] = numext::atan2(c1 * coeff(j, k) - s1 * coeff(k, k), c1 * coeff(j, j) - s1 * coeff(k, j)); + } else { - res[0] = atan2(coeff(j,k), coeff(k,k)); - Scalar c2 = Vector2(coeff(i,i), coeff(i,j)).norm(); - if((odd && res[0]Scalar(0))) { - if(res[0] > Scalar(0)) { + res[0] = numext::atan2(coeff(j, k), coeff(k, k)); + Scalar c2 = numext::hypot(coeff(i, i), coeff(i, j)); + if ((odd && res[0] < Scalar(0)) || ((!odd) && res[0] > Scalar(0))) + { + if (res[0] > Scalar(0)) + { res[0] -= Scalar(EIGEN_PI); } - else { + else + { res[0] += Scalar(EIGEN_PI); } - res[1] = atan2(-coeff(i,k), -c2); + res[1] = numext::atan2(-coeff(i, k), -c2); } else - res[1] = atan2(-coeff(i,k), c2); - Scalar s1 = sin(res[0]); - Scalar c1 = cos(res[0]); - res[2] = atan2(s1*coeff(k,i)-c1*coeff(j,i), c1*coeff(j,j) - s1 * coeff(k,j)); + { + res[1] = numext::atan2(-coeff(i, k), c2); + } + Scalar s1 = numext::sin(res[0]); + Scalar c1 = numext::cos(res[0]); + res[2] = numext::atan2(s1 * coeff(k, i) - c1 * coeff(j, i), c1 * coeff(j, j) - s1 * coeff(k, j)); } if (!odd) + { res = -res; - + } + return res; } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Homogeneous.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Homogeneous.h index 94083ac5413..292530b6f8c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Homogeneous.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Homogeneous.h @@ -10,6 +10,9 @@ #ifndef EIGEN_HOMOGENEOUS_H #define EIGEN_HOMOGENEOUS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \geometry_module \ingroup Geometry_Module @@ -35,7 +38,7 @@ struct traits > { typedef typename traits::StorageKind StorageKind; typedef typename ref_selector::type MatrixTypeNested; - typedef typename remove_reference::type _MatrixTypeNested; + typedef std::remove_reference_t MatrixTypeNested_; enum { RowsPlusOne = (MatrixType::RowsAtCompileTime != Dynamic) ? int(MatrixType::RowsAtCompileTime) + 1 : Dynamic, @@ -45,7 +48,7 @@ struct traits > ColsAtCompileTime = Direction==Horizontal ? ColsPlusOne : MatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = RowsAtCompileTime, MaxColsAtCompileTime = ColsAtCompileTime, - TmpFlags = _MatrixTypeNested::Flags & HereditaryBits, + TmpFlags = MatrixTypeNested_::Flags & HereditaryBits, Flags = ColsAtCompileTime==1 ? (TmpFlags & ~RowMajorBit) : RowsAtCompileTime==1 ? (TmpFlags | RowMajorBit) : TmpFlags @@ -57,13 +60,13 @@ template struct homogeneous_right_product_impl } // end namespace internal -template class Homogeneous - : public MatrixBase >, internal::no_assignment_operator +template class Homogeneous + : public MatrixBase >, internal::no_assignment_operator { public: typedef MatrixType NestedExpression; - enum { Direction = _Direction }; + enum { Direction = Direction_ }; typedef MatrixBase Base; EIGEN_DENSE_PUBLIC_INTERFACE(Homogeneous) @@ -225,7 +228,7 @@ template struct take_matrix_for_product > { typedef Transform TransformType; - typedef typename internal::add_const::type type; + typedef std::add_const_t type; EIGEN_DEVICE_FUNC static type run (const TransformType& x) { return x.affine(); } }; @@ -241,8 +244,8 @@ template struct traits,Lhs> > { typedef typename take_matrix_for_product::type LhsMatrixType; - typedef typename remove_all::type MatrixTypeCleaned; - typedef typename remove_all::type LhsMatrixTypeCleaned; + typedef remove_all_t MatrixTypeCleaned; + typedef remove_all_t LhsMatrixTypeCleaned; typedef typename make_proper_matrix_type< typename traits::Scalar, LhsMatrixTypeCleaned::RowsAtCompileTime, @@ -257,8 +260,8 @@ struct homogeneous_left_product_impl,Lhs> : public ReturnByValue,Lhs> > { typedef typename traits::LhsMatrixType LhsMatrixType; - typedef typename remove_all::type LhsMatrixTypeCleaned; - typedef typename remove_all::type LhsMatrixTypeNested; + typedef remove_all_t LhsMatrixTypeCleaned; + typedef remove_all_t LhsMatrixTypeNested; EIGEN_DEVICE_FUNC homogeneous_left_product_impl(const Lhs& lhs, const MatrixType& rhs) : m_lhs(take_matrix_for_product::run(lhs)), m_rhs(rhs) @@ -299,7 +302,7 @@ template struct homogeneous_right_product_impl,Rhs> : public ReturnByValue,Rhs> > { - typedef typename remove_all::type RhsNested; + typedef remove_all_t RhsNested; EIGEN_DEVICE_FUNC homogeneous_right_product_impl(const MatrixType& lhs, const Rhs& rhs) : m_lhs(lhs), m_rhs(rhs) {} @@ -343,7 +346,7 @@ struct unary_evaluator, IndexBased> EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& op) : Base(), m_temp(op) { - ::new (static_cast(this)) Base(m_temp); + internal::construct_at(this, m_temp); } protected: @@ -402,7 +405,7 @@ struct homogeneous_right_product_refactoring_helper Rows = Lhs::RowsAtCompileTime }; typedef typename Rhs::template ConstNRowsBlockXpr::Type LinearBlockConst; - typedef typename remove_const::type LinearBlock; + typedef std::remove_const_t LinearBlock; typedef typename Rhs::ConstRowXpr ConstantColumn; typedef Replicate ConstantBlock; typedef Product LinearProduct; @@ -455,7 +458,7 @@ struct homogeneous_left_product_refactoring_helper Cols = Rhs::ColsAtCompileTime }; typedef typename Lhs::template ConstNColsBlockXpr::Type LinearBlockConst; - typedef typename remove_const::type LinearBlock; + typedef std::remove_const_t LinearBlock; typedef typename Lhs::ConstColXpr ConstantColumn; typedef Replicate ConstantBlock; typedef Product LinearProduct; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Hyperplane.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Hyperplane.h index cebe0355702..c669a57dcbb 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Hyperplane.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Hyperplane.h @@ -11,6 +11,9 @@ #ifndef EIGEN_HYPERPLANE_H #define EIGEN_HYPERPLANE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \geometry_module \ingroup Geometry_Module @@ -22,24 +25,24 @@ namespace Eigen { * A hyperplane is an affine subspace of dimension n-1 in a space of dimension n. * For example, a hyperplane in a plane is a line; a hyperplane in 3-space is a plane. * - * \tparam _Scalar the scalar type, i.e., the type of the coefficients - * \tparam _AmbientDim the dimension of the ambient space, can be a compile time value or Dynamic. - * Notice that the dimension of the hyperplane is _AmbientDim-1. + * \tparam Scalar_ the scalar type, i.e., the type of the coefficients + * \tparam AmbientDim_ the dimension of the ambient space, can be a compile time value or Dynamic. + * Notice that the dimension of the hyperplane is AmbientDim_-1. * * This class represents an hyperplane as the zero set of the implicit equation * \f$ n \cdot x + d = 0 \f$ where \f$ n \f$ is a unit normal vector of the plane (linear part) * and \f$ d \f$ is the distance (offset) to the origin. */ -template +template class Hyperplane { public: - EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(_Scalar,_AmbientDim==Dynamic ? Dynamic : _AmbientDim+1) + EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar_,AmbientDim_==Dynamic ? Dynamic : AmbientDim_+1) enum { - AmbientDimAtCompileTime = _AmbientDim, - Options = _Options + AmbientDimAtCompileTime = AmbientDim_, + Options = Options_ }; - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef typename NumTraits::Real RealScalar; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 typedef Matrix VectorType; @@ -106,7 +109,7 @@ class Hyperplane if(norm <= v0.norm() * v1.norm() * NumTraits::epsilon()) { Matrix m; m << v0.transpose(), v1.transpose(); - JacobiSVD > svd(m, ComputeFullV); + JacobiSVD, ComputeFullV> svd(m); result.normal() = svd.matrixV().col(2); } else diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/InternalHeaderCheck.h new file mode 100644 index 00000000000..a1159a3c69c --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_GEOMETRY_MODULE_H +#error "Please include Eigen/Geometry instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/OrthoMethods.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/OrthoMethods.h index 524aebe1b96..2cf8310bd05 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/OrthoMethods.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/OrthoMethods.h @@ -11,41 +11,88 @@ #ifndef EIGEN_ORTHOMETHODS_H #define EIGEN_ORTHOMETHODS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { +namespace internal { + +// Vector3 version (default) +template +struct cross_impl +{ + typedef typename ScalarBinaryOpTraits::Scalar,typename internal::traits::Scalar>::ReturnType Scalar; + typedef Matrix::RowsAtCompileTime,MatrixBase::ColsAtCompileTime> return_type; + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + return_type run(const MatrixBase& first, const MatrixBase& second) + { + EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Derived,3) + EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(OtherDerived,3) + + // Note that there is no need for an expression here since the compiler + // optimize such a small temporary very well (even within a complex expression) + typename internal::nested_eval::type lhs(first.derived()); + typename internal::nested_eval::type rhs(second.derived()); + return return_type( + numext::conj(lhs.coeff(1) * rhs.coeff(2) - lhs.coeff(2) * rhs.coeff(1)), + numext::conj(lhs.coeff(2) * rhs.coeff(0) - lhs.coeff(0) * rhs.coeff(2)), + numext::conj(lhs.coeff(0) * rhs.coeff(1) - lhs.coeff(1) * rhs.coeff(0)) + ); + } +}; + +// Vector2 version +template +struct cross_impl +{ + typedef typename ScalarBinaryOpTraits::Scalar,typename internal::traits::Scalar>::ReturnType Scalar; + typedef Scalar return_type; + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + return_type run(const MatrixBase& first, const MatrixBase& second) + { + EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Derived,2); + EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(OtherDerived,2); + typename internal::nested_eval::type lhs(first.derived()); + typename internal::nested_eval::type rhs(second.derived()); + return numext::conj(lhs.coeff(0) * rhs.coeff(1) - lhs.coeff(1) * rhs.coeff(0)); + } +}; + +} // end namespace internal + /** \geometry_module \ingroup Geometry_Module * - * \returns the cross product of \c *this and \a other + * \returns the cross product of \c *this and \a other. This is either a scalar for size-2 vectors or a size-3 vector for size-3 vectors. * - * Here is a very good explanation of cross-product: http://xkcd.com/199/ + * This method is implemented for two different cases: between vectors of fixed size 2 and between vectors of fixed size 3. * - * With complex numbers, the cross product is implemented as - * \f$ (\mathbf{a}+i\mathbf{b}) \times (\mathbf{c}+i\mathbf{d}) = (\mathbf{a} \times \mathbf{c} - \mathbf{b} \times \mathbf{d}) - i(\mathbf{a} \times \mathbf{d} - \mathbf{b} \times \mathbf{c})\f$ + * For vectors of size 3, the output is simply the traditional cross product. + * + * For vectors of size 2, the output is a scalar. + * Given vectors \f$ v = \begin{bmatrix} v_1 & v_2 \end{bmatrix} \f$ and \f$ w = \begin{bmatrix} w_1 & w_2 \end{bmatrix} \f$, + * the result is simply \f$ v\times w = \overline{v_1 w_2 - v_2 w_1} = \text{conj}\left|\begin{smallmatrix} v_1 & w_1 \\ v_2 & w_2 \end{smallmatrix}\right| \f$; + * or, to put it differently, it is the third coordinate of the cross product of \f$ \begin{bmatrix} v_1 & v_2 & v_3 \end{bmatrix} \f$ and \f$ \begin{bmatrix} w_1 & w_2 & w_3 \end{bmatrix} \f$. + * For real-valued inputs, the result can be interpreted as the signed area of a parallelogram spanned by the two vectors. + * + * \note With complex numbers, the cross product is implemented as + * \f$ (\mathbf{a}+i\mathbf{b}) \times (\mathbf{c}+i\mathbf{d}) = (\mathbf{a} \times \mathbf{c} - \mathbf{b} \times \mathbf{d}) - i(\mathbf{a} \times \mathbf{d} + \mathbf{b} \times \mathbf{c})\f$ * * \sa MatrixBase::cross3() */ template template -#ifndef EIGEN_PARSED_BY_DOXYGEN EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -typename MatrixBase::template cross_product_return_type::type +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename internal::cross_impl::return_type #else -typename MatrixBase::PlainObject +inline std::conditional_t #endif MatrixBase::cross(const MatrixBase& other) const { - EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Derived,3) - EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(OtherDerived,3) - - // Note that there is no need for an expression here since the compiler - // optimize such a small temporary very well (even within a complex expression) - typename internal::nested_eval::type lhs(derived()); - typename internal::nested_eval::type rhs(other.derived()); - return typename cross_product_return_type::type( - numext::conj(lhs.coeff(1) * rhs.coeff(2) - lhs.coeff(2) * rhs.coeff(1)), - numext::conj(lhs.coeff(2) * rhs.coeff(0) - lhs.coeff(0) * rhs.coeff(2)), - numext::conj(lhs.coeff(0) * rhs.coeff(1) - lhs.coeff(1) * rhs.coeff(0)) - ); + return internal::cross_impl::run(*this, other); } namespace internal { @@ -91,8 +138,8 @@ MatrixBase::cross3(const MatrixBase& other) const OtherDerivedNested rhs(other.derived()); return internal::cross3_impl::type, - typename internal::remove_all::type>::run(lhs,rhs); + internal::remove_all_t, + internal::remove_all_t>::run(lhs,rhs); } /** \geometry_module \ingroup Geometry_Module diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/ParametrizedLine.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/ParametrizedLine.h index 584f50087bd..029796a0ac8 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/ParametrizedLine.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/ParametrizedLine.h @@ -11,6 +11,9 @@ #ifndef EIGEN_PARAMETRIZEDLINE_H #define EIGEN_PARAMETRIZEDLINE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \geometry_module \ingroup Geometry_Module @@ -23,19 +26,19 @@ namespace Eigen { * direction vector \f$ \mathbf{d} \f$ such that the line corresponds to * the set \f$ l(t) = \mathbf{o} + t \mathbf{d} \f$, \f$ t \in \mathbf{R} \f$. * - * \tparam _Scalar the scalar type, i.e., the type of the coefficients - * \tparam _AmbientDim the dimension of the ambient space, can be a compile time value or Dynamic. + * \tparam Scalar_ the scalar type, i.e., the type of the coefficients + * \tparam AmbientDim_ the dimension of the ambient space, can be a compile time value or Dynamic. */ -template +template class ParametrizedLine { public: - EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(_Scalar,_AmbientDim) + EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar_,AmbientDim_) enum { - AmbientDimAtCompileTime = _AmbientDim, - Options = _Options + AmbientDimAtCompileTime = AmbientDim_, + Options = Options_ }; - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef typename NumTraits::Real RealScalar; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 typedef Matrix VectorType; @@ -59,7 +62,7 @@ class ParametrizedLine : m_origin(origin), m_direction(direction) {} template - EIGEN_DEVICE_FUNC explicit ParametrizedLine(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane); + EIGEN_DEVICE_FUNC explicit ParametrizedLine(const Hyperplane& hyperplane); /** Constructs a parametrized line going from \a p0 to \a p1. */ EIGEN_DEVICE_FUNC static inline ParametrizedLine Through(const VectorType& p0, const VectorType& p1) @@ -96,13 +99,13 @@ class ParametrizedLine EIGEN_DEVICE_FUNC VectorType pointAt(const Scalar& t) const; template - EIGEN_DEVICE_FUNC Scalar intersectionParameter(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const; - + EIGEN_DEVICE_FUNC Scalar intersectionParameter(const Hyperplane& hyperplane) const; + template - EIGEN_DEVICE_FUNC Scalar intersection(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const; - + EIGEN_DEVICE_FUNC Scalar intersection(const Hyperplane& hyperplane) const; + template - EIGEN_DEVICE_FUNC VectorType intersectionPoint(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const; + EIGEN_DEVICE_FUNC VectorType intersectionPoint(const Hyperplane& hyperplane) const; /** Applies the transformation matrix \a mat to \c *this and returns a reference to \c *this. * @@ -178,9 +181,9 @@ class ParametrizedLine * * \warning the ambient space must have dimension 2 such that the hyperplane actually describes a line */ -template +template template -EIGEN_DEVICE_FUNC inline ParametrizedLine<_Scalar, _AmbientDim,_Options>::ParametrizedLine(const Hyperplane<_Scalar, _AmbientDim,OtherOptions>& hyperplane) +EIGEN_DEVICE_FUNC inline ParametrizedLine::ParametrizedLine(const Hyperplane& hyperplane) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(VectorType, 2) direction() = hyperplane.normal().unitOrthogonal(); @@ -189,18 +192,18 @@ EIGEN_DEVICE_FUNC inline ParametrizedLine<_Scalar, _AmbientDim,_Options>::Parame /** \returns the point at \a t along this line */ -template -EIGEN_DEVICE_FUNC inline typename ParametrizedLine<_Scalar, _AmbientDim,_Options>::VectorType -ParametrizedLine<_Scalar, _AmbientDim,_Options>::pointAt(const _Scalar& t) const +template +EIGEN_DEVICE_FUNC inline typename ParametrizedLine::VectorType +ParametrizedLine::pointAt(const Scalar_& t) const { return origin() + (direction()*t); } /** \returns the parameter value of the intersection between \c *this and the given \a hyperplane */ -template +template template -EIGEN_DEVICE_FUNC inline _Scalar ParametrizedLine<_Scalar, _AmbientDim,_Options>::intersectionParameter(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const +EIGEN_DEVICE_FUNC inline Scalar_ ParametrizedLine::intersectionParameter(const Hyperplane& hyperplane) const { return -(hyperplane.offset()+hyperplane.normal().dot(origin())) / hyperplane.normal().dot(direction()); @@ -210,19 +213,19 @@ EIGEN_DEVICE_FUNC inline _Scalar ParametrizedLine<_Scalar, _AmbientDim,_Options> /** \deprecated use intersectionParameter() * \returns the parameter value of the intersection between \c *this and the given \a hyperplane */ -template +template template -EIGEN_DEVICE_FUNC inline _Scalar ParametrizedLine<_Scalar, _AmbientDim,_Options>::intersection(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const +EIGEN_DEVICE_FUNC inline Scalar_ ParametrizedLine::intersection(const Hyperplane& hyperplane) const { return intersectionParameter(hyperplane); } /** \returns the point of the intersection between \c *this and the given hyperplane */ -template +template template -EIGEN_DEVICE_FUNC inline typename ParametrizedLine<_Scalar, _AmbientDim,_Options>::VectorType -ParametrizedLine<_Scalar, _AmbientDim,_Options>::intersectionPoint(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const +EIGEN_DEVICE_FUNC inline typename ParametrizedLine::VectorType +ParametrizedLine::intersectionPoint(const Hyperplane& hyperplane) const { return pointAt(intersectionParameter(hyperplane)); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Quaternion.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Quaternion.h index 3259e592df8..cd291388f91 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Quaternion.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Quaternion.h @@ -10,6 +10,9 @@ #ifndef EIGEN_QUATERNION_H #define EIGEN_QUATERNION_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { @@ -44,8 +47,8 @@ class QuaternionBase : public RotationBase typedef typename NumTraits::Real RealScalar; typedef typename internal::traits::Coefficients Coefficients; typedef typename Coefficients::CoeffReturnType CoeffReturnType; - typedef typename internal::conditional::Flags&LvalueBit), - Scalar&, CoeffReturnType>::type NonConstCoeffReturnType; + typedef std::conditional_t::Flags&LvalueBit), + Scalar&, CoeffReturnType> NonConstCoeffReturnType; enum { @@ -198,14 +201,14 @@ class QuaternionBase : public RotationBase template EIGEN_DEVICE_FUNC inline - typename internal::enable_if::value,const Derived&>::type cast() const + std::enable_if_t::value,const Derived&> cast() const { return derived(); } template EIGEN_DEVICE_FUNC inline - typename internal::enable_if::value,Quaternion >::type cast() const + std::enable_if_t::value,Quaternion > cast() const { return Quaternion(coeffs().template cast()); } @@ -236,8 +239,8 @@ class QuaternionBase : public RotationBase * * \brief The quaternion class used to represent 3D orientations and rotations * - * \tparam _Scalar the scalar type, i.e., the type of the coefficients - * \tparam _Options controls the memory alignment of the coefficients. Can be \# AutoAlign or \# DontAlign. Default is AutoAlign. + * \tparam Scalar_ the scalar type, i.e., the type of the coefficients + * \tparam Options_ controls the memory alignment of the coefficients. Can be \# AutoAlign or \# DontAlign. Default is AutoAlign. * * This class represents a quaternion \f$ w+xi+yj+zk \f$ that is a convenient representation of * orientations and rotations of objects in three dimensions. Compared to other representations @@ -256,12 +259,12 @@ class QuaternionBase : public RotationBase */ namespace internal { -template -struct traits > +template +struct traits > { - typedef Quaternion<_Scalar,_Options> PlainObject; - typedef _Scalar Scalar; - typedef Matrix<_Scalar,4,1,_Options> Coefficients; + typedef Quaternion PlainObject; + typedef Scalar_ Scalar; + typedef Matrix Coefficients; enum{ Alignment = internal::traits::Alignment, Flags = LvalueBit @@ -269,14 +272,14 @@ struct traits > }; } -template -class Quaternion : public QuaternionBase > +template +class Quaternion : public QuaternionBase > { public: - typedef QuaternionBase > Base; + typedef QuaternionBase > Base; enum { NeedsAlignment = internal::traits::Alignment>0 }; - typedef _Scalar Scalar; + typedef Scalar_ Scalar; EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Quaternion) using Base::operator*=; @@ -296,6 +299,15 @@ class Quaternion : public QuaternionBase > */ EIGEN_DEVICE_FUNC inline Quaternion(const Scalar& w, const Scalar& x, const Scalar& y, const Scalar& z) : m_coeffs(x, y, z, w){} + /** Constructs and initializes a quaternion from its real part as a scalar, + * and its imaginary part as a 3-vector [\c x, \c y, \c z] + */ + template + EIGEN_DEVICE_FUNC inline Quaternion(const Scalar& w, const Eigen::MatrixBase& vec) + : m_coeffs(vec.x(), vec.y(), vec.z(), w) { + EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Derived, 3); + } + /** Constructs and initialize a quaternion from the array data */ EIGEN_DEVICE_FUNC explicit inline Quaternion(const Scalar* data) : m_coeffs(data) {} @@ -307,7 +319,7 @@ class Quaternion : public QuaternionBase > /** Constructs and initializes a quaternion from either: * - a rotation matrix expression, - * - a 4D vector expression representing quaternion coefficients. + * - a 4D vector expression representing quaternion coefficients in the order [\c x, \c y, \c z, \c w]. */ template EIGEN_DEVICE_FUNC explicit inline Quaternion(const MatrixBase& other) { *this = other; } @@ -317,7 +329,6 @@ class Quaternion : public QuaternionBase > EIGEN_DEVICE_FUNC explicit inline Quaternion(const Quaternion& other) { m_coeffs = other.coeffs().template cast(); } -#if EIGEN_HAS_RVALUE_REFERENCES // We define a copy constructor, which means we don't get an implicit move constructor or assignment operator. /** Default move constructor */ EIGEN_DEVICE_FUNC inline Quaternion(Quaternion&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_constructible::value) @@ -330,7 +341,6 @@ class Quaternion : public QuaternionBase > m_coeffs = std::move(other.coeffs()); return *this; } -#endif EIGEN_DEVICE_FUNC static Quaternion UnitRandom(); @@ -341,20 +351,17 @@ class Quaternion : public QuaternionBase > EIGEN_DEVICE_FUNC inline const Coefficients& coeffs() const { return m_coeffs;} EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(bool(NeedsAlignment)) - + #ifdef EIGEN_QUATERNION_PLUGIN # include EIGEN_QUATERNION_PLUGIN #endif protected: Coefficients m_coeffs; - + #ifndef EIGEN_PARSED_BY_DOXYGEN - static EIGEN_STRONG_INLINE void _check_template_params() - { - EIGEN_STATIC_ASSERT( (_Options & DontAlign) == _Options, - INVALID_MATRIX_TEMPLATE_PARAMETERS) - } + EIGEN_STATIC_ASSERT( (Options_ & DontAlign) == Options_, + INVALID_MATRIX_TEMPLATE_PARAMETERS) #endif }; @@ -370,19 +377,19 @@ typedef Quaternion Quaterniond; ***************************************************************************/ namespace internal { - template - struct traits, _Options> > : traits > + template + struct traits, Options_> > : traits > { - typedef Map, _Options> Coefficients; + typedef Map, Options_> Coefficients; }; } namespace internal { - template - struct traits, _Options> > : traits > + template + struct traits, Options_> > : traits > { - typedef Map, _Options> Coefficients; - typedef traits > TraitsBase; + typedef Map, Options_> Coefficients; + typedef traits > TraitsBase; enum { Flags = TraitsBase::Flags & ~LvalueBit }; @@ -392,22 +399,22 @@ namespace internal { /** \ingroup Geometry_Module * \brief Quaternion expression mapping a constant memory buffer * - * \tparam _Scalar the type of the Quaternion coefficients - * \tparam _Options see class Map + * \tparam Scalar_ the type of the Quaternion coefficients + * \tparam Options_ see class Map * * This is a specialization of class Map for Quaternion. This class allows to view * a 4 scalar memory buffer as an Eigen's Quaternion object. * * \sa class Map, class Quaternion, class QuaternionBase */ -template -class Map, _Options > - : public QuaternionBase, _Options> > +template +class Map, Options_ > + : public QuaternionBase, Options_> > { public: - typedef QuaternionBase, _Options> > Base; + typedef QuaternionBase, Options_> > Base; - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef typename internal::traits::Coefficients Coefficients; EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Map) using Base::operator*=; @@ -417,7 +424,7 @@ class Map, _Options > * The pointer \a coeffs must reference the four coefficients of Quaternion in the following order: * \code *coeffs == {x, y, z, w} \endcode * - * If the template parameter _Options is set to #Aligned, then the pointer coeffs must be aligned. */ + * If the template parameter Options_ is set to #Aligned, then the pointer coeffs must be aligned. */ EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE Map(const Scalar* coeffs) : m_coeffs(coeffs) {} EIGEN_DEVICE_FUNC inline const Coefficients& coeffs() const { return m_coeffs;} @@ -429,22 +436,22 @@ class Map, _Options > /** \ingroup Geometry_Module * \brief Expression of a quaternion from a memory buffer * - * \tparam _Scalar the type of the Quaternion coefficients - * \tparam _Options see class Map + * \tparam Scalar_ the type of the Quaternion coefficients + * \tparam Options_ see class Map * * This is a specialization of class Map for Quaternion. This class allows to view * a 4 scalar memory buffer as an Eigen's Quaternion object. * * \sa class Map, class Quaternion, class QuaternionBase */ -template -class Map, _Options > - : public QuaternionBase, _Options> > +template +class Map, Options_ > + : public QuaternionBase, Options_> > { public: - typedef QuaternionBase, _Options> > Base; + typedef QuaternionBase, Options_> > Base; - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef typename internal::traits::Coefficients Coefficients; EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Map) using Base::operator*=; @@ -454,7 +461,7 @@ class Map, _Options > * The pointer \a coeffs must reference the four coefficients of Quaternion in the following order: * \code *coeffs == {x, y, z, w} \endcode * - * If the template parameter _Options is set to #Aligned, then the pointer coeffs must be aligned. */ + * If the template parameter Options_ is set to #Aligned, then the pointer coeffs must be aligned. */ EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE Map(Scalar* coeffs) : m_coeffs(coeffs) {} EIGEN_DEVICE_FUNC inline Coefficients& coeffs() { return m_coeffs; } @@ -654,7 +661,7 @@ EIGEN_DEVICE_FUNC inline Derived& QuaternionBase::setFromTwoVectors(con { c = numext::maxi(c,Scalar(-1)); Matrix m; m << v0.transpose(), v1.transpose(); - JacobiSVD > svd(m, ComputeFullV); + JacobiSVD, ComputeFullV> svd(m); Vector3 axis = svd.matrixV().col(2); Scalar w2 = (Scalar(1)+c)*Scalar(0.5); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Rotation2D.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Rotation2D.h index d0bd57569f2..32af255044e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Rotation2D.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Rotation2D.h @@ -10,6 +10,9 @@ #ifndef EIGEN_ROTATION2D_H #define EIGEN_ROTATION2D_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \geometry_module \ingroup Geometry_Module @@ -18,7 +21,7 @@ namespace Eigen { * * \brief Represents a rotation/orientation in a 2 dimensional space. * - * \tparam _Scalar the scalar type, i.e., the type of the coefficients + * \tparam Scalar_ the scalar type, i.e., the type of the coefficients * * This class is equivalent to a single scalar representing a counter clock wise rotation * as a single angle in radian. It provides some additional features such as the automatic @@ -31,16 +34,16 @@ namespace Eigen { namespace internal { -template struct traits > +template struct traits > { - typedef _Scalar Scalar; + typedef Scalar_ Scalar; }; } // end namespace internal -template -class Rotation2D : public RotationBase,2> +template +class Rotation2D : public RotationBase,2> { - typedef RotationBase,2> Base; + typedef RotationBase,2> Base; public: @@ -48,7 +51,7 @@ class Rotation2D : public RotationBase,2> enum { Dim = 2 }; /** the scalar type of the coefficients */ - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef Matrix Vector2; typedef Matrix Matrix2; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/RotationBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/RotationBase.h index f0ee0bd03c5..d685bc7f225 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/RotationBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/RotationBase.h @@ -10,6 +10,9 @@ #ifndef EIGEN_ROTATIONBASE_H #define EIGEN_ROTATIONBASE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { // forward declaration @@ -23,13 +26,13 @@ struct rotation_base_generic_product_selector; * \brief Common base class for compact rotation representations * * \tparam Derived is the derived type, i.e., a rotation type - * \tparam _Dim the dimension of the space + * \tparam Dim_ the dimension of the space */ -template +template class RotationBase { public: - enum { Dim = _Dim }; + enum { Dim = Dim_ }; /** the scalar type of the coefficients */ typedef typename internal::traits::Scalar Scalar; @@ -135,9 +138,9 @@ struct rotation_base_generic_product_selector +template template -EIGEN_DEVICE_FUNC Matrix<_Scalar, _Rows, _Cols, _Storage, _MaxRows, _MaxCols> +EIGEN_DEVICE_FUNC Matrix ::Matrix(const RotationBase& r) { EIGEN_STATIC_ASSERT_MATRIX_SPECIFIC_SIZE(Matrix,int(OtherDerived::Dim),int(OtherDerived::Dim)) @@ -148,10 +151,10 @@ ::Matrix(const RotationBase& r) * * \brief Set a Dim x Dim rotation matrix from the rotation \a r */ -template +template template -EIGEN_DEVICE_FUNC Matrix<_Scalar, _Rows, _Cols, _Storage, _MaxRows, _MaxCols>& -Matrix<_Scalar, _Rows, _Cols, _Storage, _MaxRows, _MaxCols> +EIGEN_DEVICE_FUNC Matrix& +Matrix ::operator=(const RotationBase& r) { EIGEN_STATIC_ASSERT_MATRIX_SPECIFIC_SIZE(Matrix,int(OtherDerived::Dim),int(OtherDerived::Dim)) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Scaling.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Scaling.h index d352f1f2b8a..7da2dbee908 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Scaling.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Scaling.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SCALING_H #define EIGEN_SCALING_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \geometry_module \ingroup Geometry_Module @@ -18,7 +21,7 @@ namespace Eigen { * * \brief Represents a generic uniform scaling transformation * - * \tparam _Scalar the scalar type, i.e., the type of the coefficients. + * \tparam Scalar_ the scalar type, i.e., the type of the coefficients. * * This class represent a uniform scaling transformation. It is the return * type of Scaling(Scalar), and most of the time this is the only way it @@ -45,12 +48,12 @@ namespace internal }; } -template +template class UniformScaling { public: /** the scalar type of the coefficients */ - typedef _Scalar Scalar; + typedef Scalar_ Scalar; protected: @@ -160,6 +163,11 @@ template inline const DiagonalWrapper Scaling(const MatrixBase& coeffs) { return coeffs.asDiagonal(); } +/** Constructs an axis aligned scaling expression from vector \a coeffs when passed as an rvalue reference */ +template +inline typename DiagonalWrapper::PlainObject Scaling(MatrixBase&& coeffs) +{ return typename DiagonalWrapper::PlainObject(std::move(coeffs.derived())); } + /** \deprecated */ typedef DiagonalMatrix AlignedScaling2f; /** \deprecated */ diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Transform.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Transform.h index 52b8c2a4eb8..1cfff1b7bdb 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Transform.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Transform.h @@ -12,6 +12,9 @@ #ifndef EIGEN_TRANSFORM_H #define EIGEN_TRANSFORM_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -63,15 +66,15 @@ struct transform_construct_from_matrix; template struct transform_take_affine_part; -template -struct traits > +template +struct traits > { - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef Eigen::Index StorageIndex; typedef Dense StorageKind; enum { - Dim1 = _Dim==Dynamic ? _Dim : _Dim + 1, - RowsAtCompileTime = _Mode==Projective ? Dim1 : _Dim, + Dim1 = Dim_==Dynamic ? Dim_ : Dim_ + 1, + RowsAtCompileTime = Mode_==Projective ? Dim1 : Dim_, ColsAtCompileTime = Dim1, MaxRowsAtCompileTime = RowsAtCompileTime, MaxColsAtCompileTime = ColsAtCompileTime, @@ -89,9 +92,9 @@ template struct transform_make_affine; * * \brief Represents an homogeneous transformation in a N dimensional space * - * \tparam _Scalar the scalar type, i.e., the type of the coefficients - * \tparam _Dim the dimension of the space - * \tparam _Mode the type of the transformation. Can be: + * \tparam Scalar_ the scalar type, i.e., the type of the coefficients + * \tparam Dim_ the dimension of the space + * \tparam Mode_ the type of the transformation. Can be: * - #Affine: the transformation is stored as a (Dim+1)^2 matrix, * where the last row is assumed to be [0 ... 0 1]. * - #AffineCompact: the transformation is stored as a (Dim)x(Dim+1) matrix. @@ -100,7 +103,7 @@ template struct transform_make_affine; * - #Isometry: same as #Affine with the additional assumption that * the linear part represents a rotation. This assumption is exploited * to speed up some functions such as inverse() and rotation(). - * \tparam _Options has the same meaning as in class Matrix. It allows to specify DontAlign and/or RowMajor. + * \tparam Options_ has the same meaning as in class Matrix. It allows to specify DontAlign and/or RowMajor. * These Options are passed directly to the underlying matrix type. * * The homography is internally represented and stored by a matrix which @@ -200,20 +203,20 @@ template struct transform_make_affine; * * \sa class Matrix, class Quaternion */ -template +template class Transform { public: - EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(_Scalar,_Dim==Dynamic ? Dynamic : (_Dim+1)*(_Dim+1)) + EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar_,Dim_==Dynamic ? Dynamic : (Dim_+1)*(Dim_+1)) enum { - Mode = _Mode, - Options = _Options, - Dim = _Dim, ///< space dimension in which the transformation holds - HDim = _Dim+1, ///< size of a respective homogeneous vector + Mode = Mode_, + Options = Options_, + Dim = Dim_, ///< space dimension in which the transformation holds + HDim = Dim_+1, ///< size of a respective homogeneous vector Rows = int(Mode)==(AffineCompact) ? Dim : HDim }; /** the scalar type of the coefficients */ - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef Eigen::Index StorageIndex; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 /** type of the matrix used to represent the transformation */ @@ -227,13 +230,13 @@ class Transform /** type of read reference to the linear part of the transformation */ typedef const Block ConstLinearPart; /** type of read/write reference to the affine part of the transformation */ - typedef typename internal::conditional >::type AffinePart; + Block > AffinePart; /** type of read reference to the affine part of the transformation */ - typedef typename internal::conditional >::type ConstAffinePart; + const Block > ConstAffinePart; /** type of a vector */ typedef Matrix VectorType; /** type of a read/write reference to the translation part of the rotation */ @@ -317,12 +320,12 @@ class Transform check_template_params(); // prevent conversions as: // Affine | AffineCompact | Isometry = Projective - EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(OtherMode==int(Projective), Mode==int(Projective)), + EIGEN_STATIC_ASSERT(internal::check_implication(OtherMode==int(Projective), Mode==int(Projective)), YOU_PERFORMED_AN_INVALID_TRANSFORMATION_CONVERSION) // prevent conversions as: // Isometry = Affine | AffineCompact - EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(OtherMode==int(Affine)||OtherMode==int(AffineCompact), Mode!=int(Isometry)), + EIGEN_STATIC_ASSERT(internal::check_implication(OtherMode==int(Affine)||OtherMode==int(AffineCompact), Mode!=int(Isometry)), YOU_PERFORMED_AN_INVALID_TRANSFORMATION_CONVERSION) enum { ModeIsAffineCompact = Mode == int(AffineCompact), @@ -367,9 +370,11 @@ class Transform } #ifdef EIGEN_QT_SUPPORT + #if (QT_VERSION < QT_VERSION_CHECK(6, 0, 0)) inline Transform(const QMatrix& other); inline Transform& operator=(const QMatrix& other); inline QMatrix toQMatrix(void) const; + #endif inline Transform(const QTransform& other); inline Transform& operator=(const QTransform& other); inline QTransform toQTransform(void) const; @@ -443,7 +448,7 @@ class Transform * \li a general transformation matrix of size Dim+1 x Dim+1. */ template friend - EIGEN_DEVICE_FUNC inline const typename internal::transform_left_product_impl::ResultType + EIGEN_DEVICE_FUNC inline const typename internal::transform_left_product_impl::ResultType operator * (const EigenBase &a, const Transform &b) { return internal::transform_left_product_impl::run(a.derived(),b); } @@ -596,7 +601,7 @@ class Transform template EIGEN_DEVICE_FUNC inline Transform operator*(const RotationBase& r) const; - typedef typename internal::conditional::type RotationReturnType; + typedef std::conditional_t RotationReturnType; EIGEN_DEVICE_FUNC RotationReturnType rotation() const; template @@ -732,6 +737,8 @@ typedef Transform Projective3d; **************************/ #ifdef EIGEN_QT_SUPPORT + +#if (QT_VERSION < QT_VERSION_CHECK(6, 0, 0)) /** Initializes \c *this from a QMatrix assuming the dimension is 2. * * This function is available only if the token EIGEN_QT_SUPPORT is defined. @@ -776,6 +783,7 @@ QMatrix Transform::toQMatrix(void) const m_matrix.coeff(0,1), m_matrix.coeff(1,1), m_matrix.coeff(0,2), m_matrix.coeff(1,2)); } +#endif /** Initializes \c *this from a QTransform assuming the dimension is 2. * @@ -1098,7 +1106,7 @@ template EIGEN_DEVICE_FUNC void Transform::computeRotationScaling(RotationMatrixType *rotation, ScalingMatrixType *scaling) const { // Note that JacobiSVD is faster than BDCSVD for small matrices. - JacobiSVD svd(linear(), ComputeFullU | ComputeFullV); + JacobiSVD svd(linear()); Scalar x = (svd.matrixU() * svd.matrixV().adjoint()).determinant() < Scalar(0) ? Scalar(-1) : Scalar(1); // so x has absolute value 1 VectorType sv(svd.singularValues()); @@ -1128,7 +1136,7 @@ template EIGEN_DEVICE_FUNC void Transform::computeScalingRotation(ScalingMatrixType *scaling, RotationMatrixType *rotation) const { // Note that JacobiSVD is faster than BDCSVD for small matrices. - JacobiSVD svd(linear(), ComputeFullU | ComputeFullV); + JacobiSVD svd(linear()); Scalar x = (svd.matrixU() * svd.matrixV().adjoint()).determinant() < Scalar(0) ? Scalar(-1) : Scalar(1); // so x has absolute value 1 VectorType sv(svd.singularValues()); @@ -1259,17 +1267,17 @@ template struct transform_take_affine_part { typedef typename TransformType::MatrixType MatrixType; typedef typename TransformType::AffinePart AffinePart; typedef typename TransformType::ConstAffinePart ConstAffinePart; - static inline AffinePart run(MatrixType& m) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AffinePart run(MatrixType& m) { return m.template block(0,0); } - static inline ConstAffinePart run(const MatrixType& m) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ConstAffinePart run(const MatrixType& m) { return m.template block(0,0); } }; template struct transform_take_affine_part > { typedef typename Transform::MatrixType MatrixType; - static inline MatrixType& run(MatrixType& m) { return m; } - static inline const MatrixType& run(const MatrixType& m) { return m; } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE MatrixType& run(MatrixType& m) { return m; } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const MatrixType& run(const MatrixType& m) { return m; } }; /***************************************************** @@ -1279,7 +1287,7 @@ struct transform_take_affine_part > template struct transform_construct_from_matrix { - static inline void run(Transform *transform, const Other& other) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Transform *transform, const Other& other) { transform->linear() = other; transform->translation().setZero(); @@ -1290,7 +1298,7 @@ struct transform_construct_from_matrix template struct transform_construct_from_matrix { - static inline void run(Transform *transform, const Other& other) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Transform *transform, const Other& other) { transform->affine() = other; transform->makeAffine(); @@ -1300,14 +1308,14 @@ struct transform_construct_from_matrix template struct transform_construct_from_matrix { - static inline void run(Transform *transform, const Other& other) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Transform *transform, const Other& other) { transform->matrix() = other; } }; template struct transform_construct_from_matrix { - static inline void run(Transform *transform, const Other& other) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Transform *transform, const Other& other) { transform->matrix() = other.template block(0,0); } }; @@ -1397,7 +1405,7 @@ struct transform_right_product_impl< TransformType, MatrixType, 2, 1> // rhs is Dim = TransformType::Dim, HDim = TransformType::HDim, OtherRows = MatrixType::RowsAtCompileTime, - WorkingRows = EIGEN_PLAIN_ENUM_MIN(TransformMatrix::RowsAtCompileTime,HDim) + WorkingRows = plain_enum_min(TransformMatrix::RowsAtCompileTime, HDim) }; typedef typename MatrixType::PlainObject ResultType; @@ -1424,7 +1432,7 @@ struct transform_left_product_impl typedef Transform TransformType; typedef typename TransformType::MatrixType MatrixType; typedef Transform ResultType; - static ResultType run(const Other& other,const TransformType& tr) + static EIGEN_DEVICE_FUNC ResultType run(const Other& other,const TransformType& tr) { return ResultType(other * tr.matrix()); } }; @@ -1435,7 +1443,7 @@ struct transform_left_product_impl TransformType; typedef typename TransformType::MatrixType MatrixType; typedef Transform ResultType; - static ResultType run(const Other& other,const TransformType& tr) + static EIGEN_DEVICE_FUNC ResultType run(const Other& other,const TransformType& tr) { ResultType res; res.matrix().noalias() = other.template block(0,0) * tr.matrix(); @@ -1451,7 +1459,7 @@ struct transform_left_product_impl typedef Transform TransformType; typedef typename TransformType::MatrixType MatrixType; typedef TransformType ResultType; - static ResultType run(const Other& other,const TransformType& tr) + static EIGEN_DEVICE_FUNC ResultType run(const Other& other,const TransformType& tr) { ResultType res; res.affine().noalias() = other * tr.matrix(); @@ -1467,7 +1475,7 @@ struct transform_left_product_impl TransformType; typedef typename TransformType::MatrixType MatrixType; typedef TransformType ResultType; - static ResultType run(const Other& other,const TransformType& tr) + static EIGEN_DEVICE_FUNC ResultType run(const Other& other,const TransformType& tr) { ResultType res; res.matrix().noalias() = other.template block(0,0) * tr.matrix(); @@ -1483,7 +1491,7 @@ struct transform_left_product_impl typedef Transform TransformType; typedef typename TransformType::MatrixType MatrixType; typedef TransformType ResultType; - static ResultType run(const Other& other, const TransformType& tr) + static EIGEN_DEVICE_FUNC ResultType run(const Other& other, const TransformType& tr) { TransformType res; if(Mode!=int(AffineCompact)) @@ -1505,7 +1513,7 @@ struct transform_transform_product_impl typedef Transform Lhs; typedef Transform Rhs; typedef Transform ResultType; - static ResultType run(const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC ResultType run(const Lhs& lhs, const Rhs& rhs) { ResultType res; res.linear() = lhs.linear() * rhs.linear(); @@ -1521,7 +1529,7 @@ struct transform_transform_product_impl typedef Transform Lhs; typedef Transform Rhs; typedef Transform ResultType; - static ResultType run(const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC ResultType run(const Lhs& lhs, const Rhs& rhs) { return ResultType( lhs.matrix() * rhs.matrix() ); } @@ -1533,7 +1541,7 @@ struct transform_transform_product_impl Lhs; typedef Transform Rhs; typedef Transform ResultType; - static ResultType run(const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC ResultType run(const Lhs& lhs, const Rhs& rhs) { ResultType res; res.matrix().template topRows() = lhs.matrix() * rhs.matrix(); @@ -1548,7 +1556,7 @@ struct transform_transform_product_impl Lhs; typedef Transform Rhs; typedef Transform ResultType; - static ResultType run(const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC ResultType run(const Lhs& lhs, const Rhs& rhs) { ResultType res(lhs.matrix().template leftCols() * rhs.matrix()); res.matrix().col(Dim) += lhs.matrix().col(Dim); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Translation.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Translation.h index 8c229012190..25435d71a90 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Translation.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Translation.h @@ -10,6 +10,9 @@ #ifndef EIGEN_TRANSLATION_H #define EIGEN_TRANSLATION_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \geometry_module \ingroup Geometry_Module @@ -18,23 +21,23 @@ namespace Eigen { * * \brief Represents a translation transformation * - * \tparam _Scalar the scalar type, i.e., the type of the coefficients. - * \tparam _Dim the dimension of the space, can be a compile time value or Dynamic + * \tparam Scalar_ the scalar type, i.e., the type of the coefficients. + * \tparam Dim_ the dimension of the space, can be a compile time value or Dynamic * * \note This class is not aimed to be used to store a translation transformation, * but rather to make easier the constructions and updates of Transform objects. * * \sa class Scaling, class Transform */ -template +template class Translation { public: - EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(_Scalar,_Dim) + EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar_,Dim_) /** dimension of the space */ - enum { Dim = _Dim }; + enum { Dim = Dim_ }; /** the scalar type of the coefficients */ - typedef _Scalar Scalar; + typedef Scalar_ Scalar; /** corresponding vector type */ typedef Matrix VectorType; /** corresponding linear transformation matrix type */ @@ -131,7 +134,7 @@ class Translation /** Applies translation to vector */ template - inline typename internal::enable_if::type + inline std::enable_if_t operator* (const MatrixBase& vec) const { return m_coeffs + vec.derived(); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Umeyama.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Umeyama.h index 6b755008fdc..87c89d6a4c6 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Umeyama.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/Umeyama.h @@ -16,6 +16,9 @@ // * Eigen/SVD // * Eigen/Array +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { #ifndef EIGEN_PARSED_BY_DOXYGEN @@ -32,10 +35,10 @@ template struct umeyama_transform_matrix_type { enum { - MinRowsAtCompileTime = EIGEN_SIZE_MIN_PREFER_DYNAMIC(MatrixType::RowsAtCompileTime, OtherMatrixType::RowsAtCompileTime), + MinRowsAtCompileTime = internal::min_size_prefer_dynamic(MatrixType::RowsAtCompileTime, OtherMatrixType::RowsAtCompileTime), // When possible we want to choose some small fixed size value since the result - // is likely to fit on the stack. So here, EIGEN_SIZE_MIN_PREFER_DYNAMIC is not what we want. + // is likely to fit on the stack. So here, min_size_prefer_dynamic is not what we want. HomogeneousDimension = int(MinRowsAtCompileTime) == Dynamic ? Dynamic : int(MinRowsAtCompileTime)+1 }; @@ -102,7 +105,7 @@ umeyama(const MatrixBase& src, const MatrixBase& dst, boo EIGEN_STATIC_ASSERT((internal::is_same::Scalar>::value), YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) - enum { Dimension = EIGEN_SIZE_MIN_PREFER_DYNAMIC(Derived::RowsAtCompileTime, OtherDerived::RowsAtCompileTime) }; + enum { Dimension = internal::min_size_prefer_dynamic(Derived::RowsAtCompileTime, OtherDerived::RowsAtCompileTime) }; typedef Matrix VectorType; typedef Matrix MatrixType; @@ -122,13 +125,10 @@ umeyama(const MatrixBase& src, const MatrixBase& dst, boo const RowMajorMatrixType src_demean = src.colwise() - src_mean; const RowMajorMatrixType dst_demean = dst.colwise() - dst_mean; - // Eq. (36)-(37) - const Scalar src_var = src_demean.rowwise().squaredNorm().sum() * one_over_n; - // Eq. (38) const MatrixType sigma = one_over_n * dst_demean * src_demean.transpose(); - JacobiSVD svd(sigma, ComputeFullU | ComputeFullV); + JacobiSVD svd(sigma); // Initialize the resulting transformation with an identity matrix... TransformationMatrixType Rt = TransformationMatrixType::Identity(m+1,m+1); @@ -136,14 +136,19 @@ umeyama(const MatrixBase& src, const MatrixBase& dst, boo // Eq. (39) VectorType S = VectorType::Ones(m); - if ( svd.matrixU().determinant() * svd.matrixV().determinant() < 0 ) - S(m-1) = -1; + if ( svd.matrixU().determinant() * svd.matrixV().determinant() < 0 ) { + Index tmp = m - 1; + S(tmp) = -1; + } // Eq. (40) and (43) Rt.block(0,0,m,m).noalias() = svd.matrixU() * S.asDiagonal() * svd.matrixV().transpose(); if (with_scaling) { + // Eq. (36)-(37) + const Scalar src_var = src_demean.rowwise().squaredNorm().sum() * one_over_n; + // Eq. (42) const Scalar c = Scalar(1)/src_var * svd.singularValues().dot(S); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/arch/Geometry_SIMD.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/arch/Geometry_SIMD.h index 9af6a9af720..d280fbfccc3 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/arch/Geometry_SIMD.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Geometry/arch/Geometry_SIMD.h @@ -11,6 +11,9 @@ #ifndef EIGEN_GEOMETRY_SIMD_H #define EIGEN_GEOMETRY_SIMD_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/BlockHouseholder.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/BlockHouseholder.h index 39ce1c2a0ef..5896f876878 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/BlockHouseholder.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/BlockHouseholder.h @@ -13,6 +13,9 @@ // This file contains some helper function to deal with block householder reflectors +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -85,7 +88,7 @@ void make_block_householder_triangular_factor(TriangularFactorType& triFactor, c template void apply_block_householder_on_the_left(MatrixType& mat, const VectorsType& vectors, const CoeffsType& hCoeffs, bool forward) { - enum { TFactorSize = MatrixType::ColsAtCompileTime }; + enum { TFactorSize = VectorsType::ColsAtCompileTime }; Index nbVecs = vectors.cols(); Matrix T(nbVecs,nbVecs); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/Householder.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/Householder.h index 5bc037f00d1..0010d1f23be 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/Householder.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/Householder.h @@ -11,6 +11,9 @@ #ifndef EIGEN_HOUSEHOLDER_H #define EIGEN_HOUSEHOLDER_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -69,7 +72,7 @@ void MatrixBase::makeHouseholder( Scalar& tau, RealScalar& beta) const { - using std::sqrt; + using numext::sqrt; using numext::conj; EIGEN_STATIC_ASSERT_VECTOR_ONLY(EssentialPart) @@ -122,7 +125,7 @@ void MatrixBase::applyHouseholderOnTheLeft( { *this *= Scalar(1)-tau; } - else if(tau!=Scalar(0)) + else if(!numext::is_exactly_zero(tau)) { Map::type> tmp(workspace,cols()); Block bottom(derived(), 1, 0, rows()-1, cols()); @@ -160,7 +163,7 @@ void MatrixBase::applyHouseholderOnTheRight( { *this *= Scalar(1)-tau; } - else if(tau!=Scalar(0)) + else if(!numext::is_exactly_zero(tau)) { Map::type> tmp(workspace,rows()); Block right(derived(), 0, 1, rows(), cols()-1); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/HouseholderSequence.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/HouseholderSequence.h index 022f6c3dba8..132d0496271 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/HouseholderSequence.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/HouseholderSequence.h @@ -11,6 +11,9 @@ #ifndef EIGEN_HOUSEHOLDER_SEQUENCE_H #define EIGEN_HOUSEHOLDER_SEQUENCE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \ingroup Householder_Module @@ -131,34 +134,34 @@ template class HouseholderS typedef typename internal::traits::Scalar Scalar; typedef HouseholderSequence< - typename internal::conditional::IsComplex, - typename internal::remove_all::type, - VectorsType>::type, - typename internal::conditional::IsComplex, - typename internal::remove_all::type, - CoeffsType>::type, + std::conditional_t::IsComplex, + internal::remove_all_t, + VectorsType>, + std::conditional_t::IsComplex, + internal::remove_all_t, + CoeffsType>, Side > ConjugateReturnType; typedef HouseholderSequence< VectorsType, - typename internal::conditional::IsComplex, - typename internal::remove_all::type, - CoeffsType>::type, + std::conditional_t::IsComplex, + internal::remove_all_t, + CoeffsType>, Side > AdjointReturnType; typedef HouseholderSequence< - typename internal::conditional::IsComplex, - typename internal::remove_all::type, - VectorsType>::type, + std::conditional_t::IsComplex, + internal::remove_all_t, + VectorsType>, CoeffsType, Side > TransposeReturnType; typedef HouseholderSequence< - typename internal::add_const::type, - typename internal::add_const::type, + std::add_const_t, + std::add_const_t, Side > ConstHouseholderSequence; @@ -255,10 +258,10 @@ template class HouseholderS */ template EIGEN_DEVICE_FUNC - inline typename internal::conditional::type + inline std::conditional_t conjugateIf() const { - typedef typename internal::conditional::type ReturnType; + typedef std::conditional_t ReturnType; return ReturnType(m_vectors.template conjugateIf(), m_coeffs.template conjugateIf()); } @@ -382,21 +385,25 @@ template class HouseholderS Index bs = end-k; Index start = k + m_shift; - typedef Block::type,Dynamic,Dynamic> SubVectorsType; + typedef Block,Dynamic,Dynamic> SubVectorsType; SubVectorsType sub_vecs1(m_vectors.const_cast_derived(), Side==OnTheRight ? k : start, Side==OnTheRight ? start : k, Side==OnTheRight ? bs : m_vectors.rows()-start, Side==OnTheRight ? m_vectors.cols()-start : bs); - typename internal::conditional, SubVectorsType&>::type sub_vecs(sub_vecs1); + std::conditional_t, SubVectorsType&> sub_vecs(sub_vecs1); - Index dstStart = dst.rows()-rows()+m_shift+k; Index dstRows = rows()-m_shift-k; - Block sub_dst(dst, - dstStart, - inputIsIdentity ? dstStart : 0, - dstRows, - inputIsIdentity ? dstRows : dst.cols()); - apply_block_householder_on_the_left(sub_dst, sub_vecs, m_coeffs.segment(k, bs), !m_reverse); + + if (inputIsIdentity) + { + Block sub_dst = dst.bottomRightCorner(dstRows, dstRows); + apply_block_householder_on_the_left(sub_dst, sub_vecs, m_coeffs.segment(k, bs), !m_reverse); + } + else + { + auto sub_dst = dst.bottomRows(dstRows); + apply_block_householder_on_the_left(sub_dst, sub_vecs, m_coeffs.segment(k, bs), !m_reverse); + } } } else @@ -405,9 +412,18 @@ template class HouseholderS for(Index k = 0; k < m_length; ++k) { Index actual_k = m_reverse ? k : m_length-k-1; - Index dstStart = rows()-m_shift-actual_k; - dst.bottomRightCorner(dstStart, inputIsIdentity ? dstStart : dst.cols()) - .applyHouseholderOnTheLeft(essentialVector(actual_k), m_coeffs.coeff(actual_k), workspace.data()); + Index dstRows = rows()-m_shift-actual_k; + + if (inputIsIdentity) + { + Block sub_dst = dst.bottomRightCorner(dstRows, dstRows); + sub_dst.applyHouseholderOnTheLeft(essentialVector(actual_k), m_coeffs.coeff(actual_k), workspace.data()); + } + else + { + auto sub_dst = dst.bottomRows(dstRows); + sub_dst.applyHouseholderOnTheLeft(essentialVector(actual_k), m_coeffs.coeff(actual_k), workspace.data()); + } } } } @@ -428,7 +444,7 @@ template class HouseholderS return res; } - template friend struct internal::hseq_side_dependent_impl; + template friend struct internal::hseq_side_dependent_impl; /** \brief Sets the length of the Householder sequence. * \param [in] length New value for the length. diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/InternalHeaderCheck.h new file mode 100644 index 00000000000..70de89bf2dd --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Householder/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_HOUSEHOLDER_MODULE_H +#error "Please include Eigen/Householder instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h index a117fc15519..6e2122f4ee2 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h @@ -10,6 +10,9 @@ #ifndef EIGEN_BASIC_PRECONDITIONERS_H #define EIGEN_BASIC_PRECONDITIONERS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \ingroup IterativeLinearSolvers_Module @@ -21,7 +24,7 @@ namespace Eigen { A.diagonal().asDiagonal() . x = b \endcode * - * \tparam _Scalar the type of the scalar. + * \tparam Scalar_ the type of the scalar. * * \implsparsesolverconcept * @@ -32,10 +35,10 @@ namespace Eigen { * * \sa class LeastSquareDiagonalPreconditioner, class ConjugateGradient */ -template +template class DiagonalPreconditioner { - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef Matrix Vector; public: typedef typename Vector::StorageIndex StorageIndex; @@ -116,7 +119,7 @@ class DiagonalPreconditioner (A.adjoint() * A).diagonal().asDiagonal() * x = b \endcode * - * \tparam _Scalar the type of the scalar. + * \tparam Scalar_ the type of the scalar. * * \implsparsesolverconcept * @@ -124,12 +127,12 @@ class DiagonalPreconditioner * * \sa class LeastSquaresConjugateGradient, class DiagonalPreconditioner */ -template -class LeastSquareDiagonalPreconditioner : public DiagonalPreconditioner<_Scalar> +template +class LeastSquareDiagonalPreconditioner : public DiagonalPreconditioner { - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef typename NumTraits::Real RealScalar; - typedef DiagonalPreconditioner<_Scalar> Base; + typedef DiagonalPreconditioner Base; using Base::m_invdiag; public: diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/BiCGSTAB.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/BiCGSTAB.h index 153acef65ba..467d4aaa761 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/BiCGSTAB.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/BiCGSTAB.h @@ -11,6 +11,9 @@ #ifndef EIGEN_BICGSTAB_H #define EIGEN_BICGSTAB_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -49,9 +52,9 @@ bool bicgstab(const MatrixType& mat, const Rhs& rhs, Dest& x, x.setZero(); return true; } - Scalar rho = 1; - Scalar alpha = 1; - Scalar w = 1; + Scalar rho (1); + Scalar alpha (1); + Scalar w (1); VectorType v = VectorType::Zero(n), p = VectorType::Zero(n); VectorType y(n), z(n); @@ -108,17 +111,17 @@ bool bicgstab(const MatrixType& mat, const Rhs& rhs, Dest& x, } -template< typename _MatrixType, - typename _Preconditioner = DiagonalPreconditioner > +template< typename MatrixType_, + typename Preconditioner_ = DiagonalPreconditioner > class BiCGSTAB; namespace internal { -template< typename _MatrixType, typename _Preconditioner> -struct traits > +template< typename MatrixType_, typename Preconditioner_> +struct traits > { - typedef _MatrixType MatrixType; - typedef _Preconditioner Preconditioner; + typedef MatrixType_ MatrixType; + typedef Preconditioner_ Preconditioner; }; } @@ -129,8 +132,8 @@ struct traits > * This class allows to solve for A.x = b sparse linear problems using a bi conjugate gradient * stabilized algorithm. The vectors x and b can be either dense or sparse. * - * \tparam _MatrixType the type of the sparse matrix A, can be a dense or a sparse matrix. - * \tparam _Preconditioner the type of the preconditioner. Default is DiagonalPreconditioner + * \tparam MatrixType_ the type of the sparse matrix A, can be a dense or a sparse matrix. + * \tparam Preconditioner_ the type of the preconditioner. Default is DiagonalPreconditioner * * \implsparsesolverconcept * @@ -154,8 +157,8 @@ struct traits > * * \sa class SimplicialCholesky, DiagonalPreconditioner, IdentityPreconditioner */ -template< typename _MatrixType, typename _Preconditioner> -class BiCGSTAB : public IterativeSolverBase > +template< typename MatrixType_, typename Preconditioner_> +class BiCGSTAB : public IterativeSolverBase > { typedef IterativeSolverBase Base; using Base::matrix; @@ -164,10 +167,10 @@ class BiCGSTAB : public IterativeSolverBase VectorType; @@ -56,7 +57,7 @@ void conjugate_gradient(const MatrixType& mat, const Rhs& rhs, Dest& x, if (residualNorm2 < threshold) { iters = 0; - tol_error = sqrt(residualNorm2 / rhsNorm2); + tol_error = numext::sqrt(residualNorm2 / rhsNorm2); return; } @@ -86,23 +87,23 @@ void conjugate_gradient(const MatrixType& mat, const Rhs& rhs, Dest& x, p = z + beta * p; // update search direction i++; } - tol_error = sqrt(residualNorm2 / rhsNorm2); + tol_error = numext::sqrt(residualNorm2 / rhsNorm2); iters = i; } } -template< typename _MatrixType, int _UpLo=Lower, - typename _Preconditioner = DiagonalPreconditioner > +template< typename MatrixType_, int UpLo_=Lower, + typename Preconditioner_ = DiagonalPreconditioner > class ConjugateGradient; namespace internal { -template< typename _MatrixType, int _UpLo, typename _Preconditioner> -struct traits > +template< typename MatrixType_, int UpLo_, typename Preconditioner_> +struct traits > { - typedef _MatrixType MatrixType; - typedef _Preconditioner Preconditioner; + typedef MatrixType_ MatrixType; + typedef Preconditioner_ Preconditioner; }; } @@ -113,11 +114,11 @@ struct traits > * This class allows to solve for A.x = b linear problems using an iterative conjugate gradient algorithm. * The matrix A must be selfadjoint. The matrix A and the vectors x and b can be either dense or sparse. * - * \tparam _MatrixType the type of the matrix A, can be a dense or a sparse matrix. - * \tparam _UpLo the triangular part that will be used for the computations. It can be Lower, + * \tparam MatrixType_ the type of the matrix A, can be a dense or a sparse matrix. + * \tparam UpLo_ the triangular part that will be used for the computations. It can be Lower, * \c Upper, or \c Lower|Upper in which the full matrix entries will be considered. * Default is \c Lower, best performance is \c Lower|Upper. - * \tparam _Preconditioner the type of the preconditioner. Default is DiagonalPreconditioner + * \tparam Preconditioner_ the type of the preconditioner. Default is DiagonalPreconditioner * * \implsparsesolverconcept * @@ -127,8 +128,8 @@ struct traits > * * The tolerance corresponds to the relative residual error: |Ax-b|/|b| * - * \b Performance: Even though the default value of \c _UpLo is \c Lower, significantly higher performance is - * achieved when using a complete matrix and \b Lower|Upper as the \a _UpLo template parameter. Moreover, in this + * \b Performance: Even though the default value of \c UpLo_ is \c Lower, significantly higher performance is + * achieved when using a complete matrix and \b Lower|Upper as the \a UpLo_ template parameter. Moreover, in this * case multi-threading can be exploited if the user code is compiled with OpenMP enabled. * See \ref TopicMultiThreading for details. * @@ -154,8 +155,8 @@ struct traits > * * \sa class LeastSquaresConjugateGradient, class SimplicialCholesky, DiagonalPreconditioner, IdentityPreconditioner */ -template< typename _MatrixType, int _UpLo, typename _Preconditioner> -class ConjugateGradient : public IterativeSolverBase > +template< typename MatrixType_, int UpLo_, typename Preconditioner_> +class ConjugateGradient : public IterativeSolverBase > { typedef IterativeSolverBase Base; using Base::matrix; @@ -164,13 +165,13 @@ class ConjugateGradient : public IterativeSolverBase::IsComplex) }; - typedef typename internal::conditional, ActualMatrixType const&>::type RowMajorWrapper; - EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(MatrixWrapper::MatrixFree,UpLo==(Lower|Upper)),MATRIX_FREE_CONJUGATE_GRADIENT_IS_COMPATIBLE_WITH_UPPER_UNION_LOWER_MODE_ONLY); - typedef typename internal::conditional::Type - >::type SelfAdjointWrapper; + typedef std::conditional_t, ActualMatrixType const&> RowMajorWrapper; + EIGEN_STATIC_ASSERT(internal::check_implication(MatrixWrapper::MatrixFree,UpLo==(Lower|Upper)),MATRIX_FREE_CONJUGATE_GRADIENT_IS_COMPATIBLE_WITH_UPPER_UNION_LOWER_MODE_ONLY); + typedef std::conditional_t::Type + > SelfAdjointWrapper; m_iterations = Base::maxIterations(); m_error = Base::m_tolerance; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IncompleteCholesky.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IncompleteCholesky.h index 7803fd8170f..1e645049d45 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IncompleteCholesky.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IncompleteCholesky.h @@ -14,6 +14,9 @@ #include #include +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** * \brief Modified Incomplete Cholesky with dual threshold @@ -22,10 +25,9 @@ namespace Eigen { * Limited memory, SIAM J. Sci. Comput. 21(1), pp. 24-45, 1999 * * \tparam Scalar the scalar type of the input matrices - * \tparam _UpLo The triangular part that will be used for the computations. It can be Lower + * \tparam UpLo_ The triangular part that will be used for the computations. It can be Lower * or Upper. Default is Lower. - * \tparam _OrderingType The ordering method to use, either AMDOrdering<> or NaturalOrdering<>. Default is AMDOrdering, - * unless EIGEN_MPL2_ONLY is defined, in which case the default is NaturalOrdering. + * \tparam OrderingType_ The ordering method to use, either AMDOrdering<> or NaturalOrdering<>. Default is AMDOrdering. * * \implsparsesolverconcept * @@ -41,15 +43,15 @@ namespace Eigen { * the info() method, then you can either increase the initial shift, or better use another preconditioning technique. * */ -template > -class IncompleteCholesky : public SparseSolverBase > +template > +class IncompleteCholesky : public SparseSolverBase > { protected: - typedef SparseSolverBase > Base; + typedef SparseSolverBase > Base; using Base::m_isInitialized; public: typedef typename NumTraits::Real RealScalar; - typedef _OrderingType OrderingType; + typedef OrderingType_ OrderingType; typedef typename OrderingType::PermutationType PermutationType; typedef typename PermutationType::StorageIndex StorageIndex; typedef SparseMatrix FactorType; @@ -57,7 +59,7 @@ class IncompleteCholesky : public SparseSolverBase VectorRx; typedef Matrix VectorIx; typedef std::vector > VectorList; - enum { UpLo = _UpLo }; + enum { UpLo = UpLo_ }; enum { ColsAtCompileTime = Dynamic, MaxColsAtCompileTime = Dynamic @@ -160,13 +162,13 @@ class IncompleteCholesky : public SparseSolverBase -template -void IncompleteCholesky::factorize(const _MatrixType& mat) +template +template +void IncompleteCholesky::factorize(const MatrixType_& mat) { using std::sqrt; eigen_assert(m_analysisIsOk && "analyzePattern() should be called first"); @@ -199,12 +201,12 @@ void IncompleteCholesky::factorize(const _MatrixType { // The temporary is needed to make sure that the diagonal entry is properly sorted FactorType tmp(mat.rows(), mat.cols()); - tmp = mat.template selfadjointView<_UpLo>().twistedBy(m_perm); + tmp = mat.template selfadjointView().twistedBy(m_perm); m_L.template selfadjointView() = tmp.template selfadjointView(); } else { - m_L.template selfadjointView() = mat.template selfadjointView<_UpLo>(); + m_L.template selfadjointView() = mat.template selfadjointView(); } Index n = m_L.cols(); @@ -369,8 +371,8 @@ void IncompleteCholesky::factorize(const _MatrixType } while(m_info!=Success); } -template -inline void IncompleteCholesky::updateList(Ref colPtr, Ref rowIdx, Ref vals, const Index& col, const Index& jk, VectorIx& firstElt, VectorList& listCol) +template +inline void IncompleteCholesky::updateList(Ref colPtr, Ref rowIdx, Ref vals, const Index& col, const Index& jk, VectorIx& firstElt, VectorList& listCol) { if (jk < colPtr(col+1) ) { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IncompleteLUT.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IncompleteLUT.h index cdcf709eb63..6b2a113bc2a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IncompleteLUT.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IncompleteLUT.h @@ -12,6 +12,9 @@ #define EIGEN_INCOMPLETE_LUT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -95,15 +98,15 @@ Index QuickSplit(VectorV &row, VectorI &ind, Index ncut) * alternatively, on GMANE: * http://comments.gmane.org/gmane.comp.lib.eigen/3302 */ -template -class IncompleteLUT : public SparseSolverBase > +template +class IncompleteLUT : public SparseSolverBase > { protected: typedef SparseSolverBase Base; using Base::m_isInitialized; public: - typedef _Scalar Scalar; - typedef _StorageIndex StorageIndex; + typedef Scalar_ Scalar; + typedef StorageIndex_ StorageIndex; typedef typename NumTraits::Real RealScalar; typedef Matrix Vector; typedef Matrix VectorI; @@ -219,8 +222,8 @@ void IncompleteLUT::setFillfactor(int fillfactor) } template -template -void IncompleteLUT::analyzePattern(const _MatrixType& amat) +template +void IncompleteLUT::analyzePattern(const MatrixType_& amat) { // Compute the Fill-reducing permutation // Since ILUT does not perform any numerical pivoting, @@ -240,8 +243,8 @@ void IncompleteLUT::analyzePattern(const _MatrixType& amat) } template -template -void IncompleteLUT::factorize(const _MatrixType& amat) +template +void IncompleteLUT::factorize(const MatrixType_& amat) { using std::sqrt; using std::swap; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/InternalHeaderCheck.h new file mode 100644 index 00000000000..b657e84de26 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_ITERATIVELINEARSOLVERS_MODULE_H +#error "Please include Eigen/IterativeLinearSolvers instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h index 28a0c5109e9..4287e36532d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h @@ -10,6 +10,9 @@ #ifndef EIGEN_ITERATIVE_SOLVER_BASE_H #define EIGEN_ITERATIVE_SOLVER_BASE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -40,7 +43,7 @@ struct is_ref_compatible_impl template struct is_ref_compatible { - enum { value = is_ref_compatible_impl::type>::value }; + enum { value = is_ref_compatible_impl>::value }; }; template::value> @@ -77,16 +80,16 @@ class generic_matrix_wrapper template void grab(const EigenBase &mat) { - m_matrix.~Ref(); - ::new (&m_matrix) Ref(mat.derived()); + internal::destroy_at(&m_matrix); + internal::construct_at(&m_matrix, mat.derived()); } void grab(const Ref &mat) { if(&(mat.derived()) != &m_matrix) { - m_matrix.~Ref(); - ::new (&m_matrix) Ref(mat); + internal::destroy_at(&m_matrix); + internal::construct_at(&m_matrix, mat); } } @@ -186,6 +189,9 @@ class IterativeSolverBase : public SparseSolverBase compute(matrix()); } + + IterativeSolverBase(IterativeSolverBase&&) = default; + ~IterativeSolverBase() {} /** Initializes the iterative solver for the sparsity pattern of the matrix \a A for further solving \c Ax=b problems. @@ -295,7 +301,7 @@ class IterativeSolverBase : public SparseSolverBase /** \returns the number of iterations performed during the last solve */ Index iterations() const { - eigen_assert(m_isInitialized && "ConjugateGradient is not initialized."); + eigen_assert(m_isInitialized && "IterativeSolverBase is not initialized."); return m_iterations; } @@ -304,7 +310,7 @@ class IterativeSolverBase : public SparseSolverBase */ RealScalar error() const { - eigen_assert(m_isInitialized && "ConjugateGradient is not initialized."); + eigen_assert(m_isInitialized && "IterativeSolverBase is not initialized."); return m_error; } @@ -364,7 +370,7 @@ class IterativeSolverBase : public SparseSolverBase } template - typename internal::enable_if::type + std::enable_if_t _solve_with_guess_impl(const Rhs& b, MatrixBase &aDest) const { eigen_assert(rows()==b.rows()); @@ -389,7 +395,7 @@ class IterativeSolverBase : public SparseSolverBase } template - typename internal::enable_if::type + std::enable_if_t _solve_with_guess_impl(const Rhs& b, MatrixBase &dest) const { derived()._solve_vector_with_guess_impl(b,dest.derived()); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/LeastSquareConjugateGradient.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/LeastSquareConjugateGradient.h index 203fd0ec63f..db87a428029 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/LeastSquareConjugateGradient.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/IterativeLinearSolvers/LeastSquareConjugateGradient.h @@ -10,6 +10,9 @@ #ifndef EIGEN_LEAST_SQUARE_CONJUGATE_GRADIENT_H #define EIGEN_LEAST_SQUARE_CONJUGATE_GRADIENT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -73,7 +76,7 @@ void least_square_conjugate_gradient(const MatrixType& mat, const Rhs& rhs, Dest Scalar alpha = absNew / tmp.squaredNorm(); // the amount we travel on dir x += alpha * p; // update solution residual -= alpha * tmp; // update residual - normal_residual = mat.adjoint() * residual; // update residual of the normal equation + normal_residual.noalias() = mat.adjoint() * residual; // update residual of the normal equation residualNorm2 = normal_residual.squaredNorm(); if(residualNorm2 < threshold) @@ -93,17 +96,17 @@ void least_square_conjugate_gradient(const MatrixType& mat, const Rhs& rhs, Dest } -template< typename _MatrixType, - typename _Preconditioner = LeastSquareDiagonalPreconditioner > +template< typename MatrixType_, + typename Preconditioner_ = LeastSquareDiagonalPreconditioner > class LeastSquaresConjugateGradient; namespace internal { -template< typename _MatrixType, typename _Preconditioner> -struct traits > +template< typename MatrixType_, typename Preconditioner_> +struct traits > { - typedef _MatrixType MatrixType; - typedef _Preconditioner Preconditioner; + typedef MatrixType_ MatrixType; + typedef Preconditioner_ Preconditioner; }; } @@ -111,13 +114,13 @@ struct traits > /** \ingroup IterativeLinearSolvers_Module * \brief A conjugate gradient solver for sparse (or dense) least-square problems * - * This class allows to solve for A x = b linear problems using an iterative conjugate gradient algorithm. + * This class solves for the least-squares solution to A x = b using an iterative conjugate gradient algorithm. * The matrix A can be non symmetric and rectangular, but the matrix A' A should be positive-definite to guaranty stability. * Otherwise, the SparseLU or SparseQR classes might be preferable. * The matrix A and the vectors x and b can be either dense or sparse. * - * \tparam _MatrixType the type of the matrix A, can be a dense or a sparse matrix. - * \tparam _Preconditioner the type of the preconditioner. Default is LeastSquareDiagonalPreconditioner + * \tparam MatrixType_ the type of the matrix A, can be a dense or a sparse matrix. + * \tparam Preconditioner_ the type of the preconditioner. Default is LeastSquareDiagonalPreconditioner * * \implsparsesolverconcept * @@ -145,8 +148,8 @@ struct traits > * * \sa class ConjugateGradient, SparseLU, SparseQR */ -template< typename _MatrixType, typename _Preconditioner> -class LeastSquaresConjugateGradient : public IterativeSolverBase > +template< typename MatrixType_, typename Preconditioner_> +class LeastSquaresConjugateGradient : public IterativeSolverBase > { typedef IterativeSolverBase Base; using Base::matrix; @@ -155,10 +158,10 @@ class LeastSquaresConjugateGradient : public IterativeSolverBase class SolveWithGuess; @@ -83,7 +86,7 @@ struct evaluator > evaluator(const SolveType& solve) : m_result(solve.rows(), solve.cols()) { - ::new (static_cast(this)) Base(m_result); + internal::construct_at(this, m_result); m_result = solve.guess(); solve.dec()._solve_with_guess_impl(solve.rhs(), m_result); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Jacobi/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Jacobi/InternalHeaderCheck.h new file mode 100644 index 00000000000..b17b1f27bb4 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Jacobi/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_JACOBI_MODULE_H +#error "Please include Eigen/Jacobi instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Jacobi/Jacobi.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Jacobi/Jacobi.h index 76668a574ae..efa57eba626 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Jacobi/Jacobi.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/Jacobi/Jacobi.h @@ -11,6 +11,9 @@ #ifndef EIGEN_JACOBI_H #define EIGEN_JACOBI_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \ingroup Jacobi_Module @@ -161,7 +164,7 @@ template EIGEN_DEVICE_FUNC void JacobiRotation::makeGivens(const Scalar& p, const Scalar& q, Scalar* r) { - makeGivens(p, q, r, typename internal::conditional::IsComplex, internal::true_type, internal::false_type>::type()); + makeGivens(p, q, r, std::conditional_t::IsComplex, internal::true_type, internal::false_type>()); } @@ -232,13 +235,13 @@ void JacobiRotation::makeGivens(const Scalar& p, const Scalar& q, Scalar { using std::sqrt; using std::abs; - if(q==Scalar(0)) + if(numext::is_exactly_zero(q)) { m_c = p::size, - OtherPacketSize = packet_traits::size - }; typedef typename packet_traits::type Packet; typedef typename packet_traits::type OtherPacket; + constexpr int RequiredAlignment = + (std::max)(unpacket_traits::alignment, unpacket_traits::alignment); + constexpr Index PacketSize = packet_traits::size; + /*** dynamic-size vectorized paths ***/ - if(SizeAtCompileTime == Dynamic && ((incrx==1 && incry==1) || PacketSize == 1)) + if(size >= 2 * PacketSize && SizeAtCompileTime == Dynamic && ((incrx == 1 && incry == 1) || PacketSize == 1)) { // both vectors are sequentially stored in memory => vectorization - enum { Peeling = 2 }; + constexpr Index Peeling = 2; Index alignedStart = internal::first_default_aligned(y, size); Index alignedEnd = alignedStart + ((size-alignedStart)/PacketSize)*PacketSize; @@ -421,11 +424,11 @@ struct apply_rotation_in_the_plane_selector0) // FIXME should be compared to the required alignment + else if(SizeAtCompileTime != Dynamic && MinAlignment >= RequiredAlignment) { const OtherPacket pc = pset1(c); const OtherPacket ps = pset1(s); - conj_helper::IsComplex,false> pcj; + conj_helper::IsComplex,false> pcj; conj_helper pm; Scalar* EIGEN_RESTRICT px = x; Scalar* EIGEN_RESTRICT py = y; @@ -450,11 +453,11 @@ struct apply_rotation_in_the_plane_selector EIGEN_DEVICE_FUNC -void /*EIGEN_DONT_INLINE*/ apply_rotation_in_the_plane(DenseBase& xpr_x, DenseBase& xpr_y, const JacobiRotation& j) +void inline apply_rotation_in_the_plane(DenseBase& xpr_x, DenseBase& xpr_y, const JacobiRotation& j) { typedef typename VectorX::Scalar Scalar; - const bool Vectorizable = (int(VectorX::Flags) & int(VectorY::Flags) & PacketAccessBit) - && (int(packet_traits::size) == int(packet_traits::size)); + constexpr bool Vectorizable = (int(evaluator::Flags) & int(evaluator::Flags) & PacketAccessBit) && + (int(packet_traits::size) == int(packet_traits::size)); eigen_assert(xpr_x.size() == xpr_y.size()); Index size = xpr_x.size(); @@ -466,14 +469,12 @@ void /*EIGEN_DONT_INLINE*/ apply_rotation_in_the_plane(DenseBase& xpr_x OtherScalar c = j.c(); OtherScalar s = j.s(); - if (c==OtherScalar(1) && s==OtherScalar(0)) + if (numext::is_exactly_one(c) && numext::is_exactly_zero(s)) return; - apply_rotation_in_the_plane_selector< - Scalar,OtherScalar, - VectorX::SizeAtCompileTime, - EIGEN_PLAIN_ENUM_MIN(evaluator::Alignment, evaluator::Alignment), - Vectorizable>::run(x,incrx,y,incry,size,c,s); + constexpr int Alignment = (std::min)(int(evaluator::Alignment), int(evaluator::Alignment)); + apply_rotation_in_the_plane_selector::run( + x, incrx, y, incry, size, c, s); } } // end namespace internal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/KLUSupport/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/KLUSupport/InternalHeaderCheck.h new file mode 100644 index 00000000000..eb1d67159f2 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/KLUSupport/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_KLUSUPPORT_MODULE_H +#error "Please include Eigen/KLUSupport instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/KLUSupport/KLUSupport.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/KLUSupport/KLUSupport.h index 215db35b03f..0c6c240ab63 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/KLUSupport/KLUSupport.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/KLUSupport/KLUSupport.h @@ -10,6 +10,9 @@ #ifndef EIGEN_KLUSUPPORT_H #define EIGEN_KLUSUPPORT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /* TODO extract L, extract U, compute det, etc... */ @@ -23,7 +26,7 @@ namespace Eigen { * * \warning The input matrix A should be in a \b compressed and \b column-major form. * Otherwise an expensive copy will be made. You can call the inexpensive makeCompressed() to get a compressed matrix. - * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> * * \implsparsesolverconcept * @@ -56,15 +59,15 @@ inline klu_numeric* klu_factor(int Ap[], int Ai[], std::complex Ax[], kl } -template -class KLU : public SparseSolverBase > +template +class KLU : public SparseSolverBase > { protected: - typedef SparseSolverBase > Base; + typedef SparseSolverBase > Base; using Base::m_isInitialized; public: using Base::_solve_impl; - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef typename MatrixType::StorageIndex StorageIndex; @@ -263,16 +266,16 @@ class KLU : public SparseSolverBase > template void grab(const EigenBase &A) { - mp_matrix.~KLUMatrixRef(); - ::new (&mp_matrix) KLUMatrixRef(A.derived()); + internal::destroy_at(&mp_matrix); + internal::construct_at(&mp_matrix, A.derived()); } void grab(const KLUMatrixRef &A) { if(&(A.derived()) != &mp_matrix) { - mp_matrix.~KLUMatrixRef(); - ::new (&mp_matrix) KLUMatrixRef(A); + internal::destroy_at(&mp_matrix); + internal::construct_at(&mp_matrix, A); } } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/Determinant.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/Determinant.h index 3a41e6fcba3..3d26afac9cc 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/Determinant.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/Determinant.h @@ -10,6 +10,9 @@ #ifndef EIGEN_DETERMINANT_H #define EIGEN_DETERMINANT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -80,8 +83,8 @@ template struct determinant_impl Scalar d3_1 = det3(m, 0,d2_23, 2,d2_03, 3,d2_02); Scalar d3_2 = det3(m, 0,d2_13, 1,d2_03, 3,d2_01); Scalar d3_3 = det3(m, 0,d2_12, 1,d2_02, 2,d2_01); - return internal::pmadd(-m(0,3),d3_0, m(1,3)*d3_1) + - internal::pmadd(-m(2,3),d3_2, m(3,3)*d3_3); + return internal::pmadd(static_cast(-m(0,3)),d3_0, static_cast(m(1,3)*d3_1)) + + internal::pmadd(static_cast(-m(2,3)),d3_2, static_cast(m(3,3)*d3_3)); } protected: static EIGEN_DEVICE_FUNC @@ -93,7 +96,7 @@ template struct determinant_impl static EIGEN_DEVICE_FUNC Scalar det3(const Derived& m, Index i0, const Scalar& d0, Index i1, const Scalar& d1, Index i2, const Scalar& d2) { - return internal::pmadd(m(i0,2), d0, internal::pmadd(-m(i1,2), d1, m(i2,2)*d2)); + return internal::pmadd(m(i0,2), d0, internal::pmadd(static_cast(-m(i1,2)), d1, static_cast(m(i2,2)*d2))); } }; @@ -109,7 +112,7 @@ inline typename internal::traits::Scalar MatrixBase::determina { eigen_assert(rows() == cols()); typedef typename internal::nested_eval::type Nested; - return internal::determinant_impl::type>::run(derived()); + return internal::determinant_impl>::run(derived()); } } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/FullPivLU.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/FullPivLU.h index ba1749fa69c..e3cd8cd83a4 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/FullPivLU.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/FullPivLU.h @@ -10,15 +10,18 @@ #ifndef EIGEN_LU_H #define EIGEN_LU_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template struct traits > - : traits<_MatrixType> +template struct traits > + : traits { typedef MatrixXpr XprKind; typedef SolverStorage StorageKind; - typedef int StorageIndex; + typedef PermutationIndex_ StorageIndex; enum { Flags = 0 }; }; @@ -30,7 +33,7 @@ template struct traits > * * \brief LU decomposition of a matrix with complete pivoting, and related features * - * \tparam _MatrixType the type of the matrix of which we are computing the LU decomposition + * \tparam MatrixType_ the type of the matrix of which we are computing the LU decomposition * * This class represents a LU decomposition of any matrix, with complete pivoting: the matrix A is * decomposed as \f$ A = P^{-1} L U Q^{-1} \f$ where L is unit-lower-triangular, U is @@ -57,11 +60,11 @@ template struct traits > * * \sa MatrixBase::fullPivLu(), MatrixBase::determinant(), MatrixBase::inverse() */ -template class FullPivLU - : public SolverBase > +template class FullPivLU + : public SolverBase > { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef SolverBase Base; friend class SolverBase; @@ -70,10 +73,11 @@ template class FullPivLU MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; - typedef typename internal::plain_row_type::type IntRowVectorType; - typedef typename internal::plain_col_type::type IntColVectorType; - typedef PermutationMatrix PermutationQType; - typedef PermutationMatrix PermutationPType; + using PermutationIndex = PermutationIndex_; + typedef typename internal::plain_row_type::type IntRowVectorType; + typedef typename internal::plain_col_type::type IntColVectorType; + typedef PermutationMatrix PermutationQType; + typedef PermutationMatrix PermutationPType; typedef typename MatrixType::PlainObject PlainObject; /** @@ -419,10 +423,7 @@ template class FullPivLU protected: - static void check_template_parameters() - { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - } + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) void computeInPlace(); @@ -438,14 +439,14 @@ template class FullPivLU bool m_isInitialized, m_usePrescribedThreshold; }; -template -FullPivLU::FullPivLU() +template +FullPivLU::FullPivLU() : m_isInitialized(false), m_usePrescribedThreshold(false) { } -template -FullPivLU::FullPivLU(Index rows, Index cols) +template +FullPivLU::FullPivLU(Index rows, Index cols) : m_lu(rows, cols), m_p(rows), m_q(cols), @@ -456,9 +457,9 @@ FullPivLU::FullPivLU(Index rows, Index cols) { } -template +template template -FullPivLU::FullPivLU(const EigenBase& matrix) +FullPivLU::FullPivLU(const EigenBase& matrix) : m_lu(matrix.rows(), matrix.cols()), m_p(matrix.rows()), m_q(matrix.cols()), @@ -470,9 +471,9 @@ FullPivLU::FullPivLU(const EigenBase& matrix) compute(matrix.derived()); } -template +template template -FullPivLU::FullPivLU(EigenBase& matrix) +FullPivLU::FullPivLU(EigenBase& matrix) : m_lu(matrix.derived()), m_p(matrix.rows()), m_q(matrix.cols()), @@ -484,13 +485,10 @@ FullPivLU::FullPivLU(EigenBase& matrix) computeInPlace(); } -template -void FullPivLU::computeInPlace() +template +void FullPivLU::computeInPlace() { - check_template_parameters(); - - // the permutations are stored as int indices, so just to be sure: - eigen_assert(m_lu.rows()<=NumTraits::highest() && m_lu.cols()<=NumTraits::highest()); + eigen_assert(m_lu.rows()<=NumTraits::highest() && m_lu.cols()<=NumTraits::highest()); m_l1_norm = m_lu.cwiseAbs().colwise().sum().maxCoeff(); @@ -522,7 +520,7 @@ void FullPivLU::computeInPlace() row_of_biggest_in_corner += k; // correct the values! since they were computed in the corner, col_of_biggest_in_corner += k; // need to add k to them. - if(biggest_in_corner==Score(0)) + if(numext::is_exactly_zero(biggest_in_corner)) { // before exiting, make sure to initialize the still uninitialized transpositions // in a sane state without destroying what we already have. @@ -577,8 +575,8 @@ void FullPivLU::computeInPlace() m_isInitialized = true; } -template -typename internal::traits::Scalar FullPivLU::determinant() const +template +typename internal::traits::Scalar FullPivLU::determinant() const { eigen_assert(m_isInitialized && "LU is not initialized."); eigen_assert(m_lu.rows() == m_lu.cols() && "You can't take the determinant of a non-square matrix!"); @@ -588,8 +586,8 @@ typename internal::traits::Scalar FullPivLU::determinant /** \returns the matrix represented by the decomposition, * i.e., it returns the product: \f$ P^{-1} L U Q^{-1} \f$. * This function is provided for debug purposes. */ -template -MatrixType FullPivLU::reconstructedMatrix() const +template +MatrixType FullPivLU::reconstructedMatrix() const { eigen_assert(m_isInitialized && "LU is not initialized."); const Index smalldim = (std::min)(m_lu.rows(), m_lu.cols()); @@ -613,15 +611,16 @@ MatrixType FullPivLU::reconstructedMatrix() const /********* Implementation of kernel() **************************************************/ namespace internal { -template -struct kernel_retval > - : kernel_retval_base > +template +struct kernel_retval > + : kernel_retval_base > { - EIGEN_MAKE_KERNEL_HELPERS(FullPivLU<_MatrixType>) + using DecompositionType = FullPivLU; + EIGEN_MAKE_KERNEL_HELPERS(DecompositionType) - enum { MaxSmallDimAtCompileTime = EIGEN_SIZE_MIN_PREFER_FIXED( - MatrixType::MaxColsAtCompileTime, - MatrixType::MaxRowsAtCompileTime) + enum { MaxSmallDimAtCompileTime = min_size_prefer_fixed( + MatrixType::MaxColsAtCompileTime, + MatrixType::MaxRowsAtCompileTime) }; template void evalTo(Dest& dst) const @@ -699,15 +698,16 @@ struct kernel_retval > /***** Implementation of image() *****************************************************/ -template -struct image_retval > - : image_retval_base > +template +struct image_retval > + : image_retval_base > { - EIGEN_MAKE_IMAGE_HELPERS(FullPivLU<_MatrixType>) + using DecompositionType = FullPivLU; + EIGEN_MAKE_IMAGE_HELPERS(DecompositionType) - enum { MaxSmallDimAtCompileTime = EIGEN_SIZE_MIN_PREFER_FIXED( - MatrixType::MaxColsAtCompileTime, - MatrixType::MaxRowsAtCompileTime) + enum { MaxSmallDimAtCompileTime = min_size_prefer_fixed( + MatrixType::MaxColsAtCompileTime, + MatrixType::MaxRowsAtCompileTime) }; template void evalTo(Dest& dst) const @@ -740,9 +740,9 @@ struct image_retval > } // end namespace internal #ifndef EIGEN_PARSED_BY_DOXYGEN -template +template template -void FullPivLU<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const +void FullPivLU::_solve_impl(const RhsType &rhs, DstType &dst) const { /* The decomposition PAQ = LU can be rewritten as A = P^{-1} L U Q^{-1}. * So we proceed as follows: @@ -787,9 +787,9 @@ void FullPivLU<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const dst.row(permutationQ().indices().coeff(i)).setZero(); } -template +template template -void FullPivLU<_MatrixType>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +void FullPivLU::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const { /* The decomposition PAQ = LU can be rewritten as A = P^{-1} L U Q^{-1}, * and since permutations are real and unitary, we can write this @@ -845,10 +845,10 @@ namespace internal { /***** Implementation of inverse() *****************************************************/ -template -struct Assignment >, internal::assign_op::Scalar>, Dense2Dense> +template +struct Assignment >, internal::assign_op::Scalar>, Dense2Dense> { - typedef FullPivLU LuType; + typedef FullPivLU LuType; typedef Inverse SrcXprType; static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op &) { @@ -866,10 +866,11 @@ struct Assignment >, internal::assign_ * \sa class FullPivLU */ template -inline const FullPivLU::PlainObject> +template +inline const FullPivLU::PlainObject, PermutationIndex> MatrixBase::fullPivLu() const { - return FullPivLU(eval()); + return FullPivLU(eval()); } } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/InternalHeaderCheck.h new file mode 100644 index 00000000000..f346b17957c --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_LU_MODULE_H +#error "Please include Eigen/LU instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/InverseImpl.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/InverseImpl.h index a40cefa9e7d..19d540cc97c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/InverseImpl.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/InverseImpl.h @@ -11,6 +11,9 @@ #ifndef EIGEN_INVERSE_IMPL_H #define EIGEN_INVERSE_IMPL_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -309,13 +312,13 @@ struct Assignment, internal::assign_op4) || (extract_data(src.nestedExpression())!=extract_data(dst))) && "Aliasing problem detected in inverse(), you need to do inverse().eval() here."); typedef typename internal::nested_eval::type ActualXprType; - typedef typename internal::remove_all::type ActualXprTypeCleanded; + typedef internal::remove_all_t ActualXprTypeCleanded; ActualXprType actual_xpr(src.nestedExpression()); @@ -385,11 +388,11 @@ inline void MatrixBase::computeInverseAndDetWithCheck( eigen_assert(rows() == cols()); // for 2x2, it's worth giving a chance to avoid evaluating. // for larger sizes, evaluating has negligible cost and limits code size. - typedef typename internal::conditional< + typedef std::conditional_t< RowsAtCompileTime == 2, - typename internal::remove_all::type>::type, + internal::remove_all_t::type>, PlainObject - >::type MatrixType; + > MatrixType; internal::compute_inverse_and_det_with_check::run (derived(), absDeterminantThreshold, inverse, determinant, invertible); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/PartialPivLU.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/PartialPivLU.h index 34aed72494d..2301ebdc9a9 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/PartialPivLU.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/PartialPivLU.h @@ -11,16 +11,19 @@ #ifndef EIGEN_PARTIALLU_H #define EIGEN_PARTIALLU_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template struct traits > - : traits<_MatrixType> +template struct traits > + : traits { typedef MatrixXpr XprKind; typedef SolverStorage StorageKind; - typedef int StorageIndex; - typedef traits<_MatrixType> BaseTraits; + typedef PermutationIndex_ StorageIndex; + typedef traits BaseTraits; enum { Flags = BaseTraits::Flags & RowMajorBit, CoeffReadCost = Dynamic @@ -46,7 +49,7 @@ struct enable_if_ref,Derived> { * * \brief LU decomposition of a matrix with partial pivoting, and related features * - * \tparam _MatrixType the type of the matrix of which we are computing the LU decomposition + * \tparam MatrixType_ the type of the matrix of which we are computing the LU decomposition * * This class represents a LU decomposition of a \b square \b invertible matrix, with partial pivoting: the matrix A * is decomposed as A = PLU where L is unit-lower-triangular, U is upper-triangular, and P @@ -73,12 +76,12 @@ struct enable_if_ref,Derived> { * * \sa MatrixBase::partialPivLu(), MatrixBase::determinant(), MatrixBase::inverse(), MatrixBase::computeInverse(), class FullPivLU */ -template class PartialPivLU - : public SolverBase > +template class PartialPivLU + : public SolverBase > { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef SolverBase Base; friend class SolverBase; @@ -87,8 +90,9 @@ template class PartialPivLU MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; - typedef PermutationMatrix PermutationType; - typedef Transpositions TranspositionType; + using PermutationIndex = PermutationIndex_; + typedef PermutationMatrix PermutationType; + typedef Transpositions TranspositionType; typedef typename MatrixType::PlainObject PlainObject; /** @@ -265,10 +269,7 @@ template class PartialPivLU protected: - static void check_template_parameters() - { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - } + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) void compute(); @@ -280,8 +281,8 @@ template class PartialPivLU bool m_isInitialized; }; -template -PartialPivLU::PartialPivLU() +template +PartialPivLU::PartialPivLU() : m_lu(), m_p(), m_rowsTranspositions(), @@ -291,8 +292,8 @@ PartialPivLU::PartialPivLU() { } -template -PartialPivLU::PartialPivLU(Index size) +template +PartialPivLU::PartialPivLU(Index size) : m_lu(size, size), m_p(size), m_rowsTranspositions(size), @@ -302,9 +303,9 @@ PartialPivLU::PartialPivLU(Index size) { } -template +template template -PartialPivLU::PartialPivLU(const EigenBase& matrix) +PartialPivLU::PartialPivLU(const EigenBase& matrix) : m_lu(matrix.rows(),matrix.cols()), m_p(matrix.rows()), m_rowsTranspositions(matrix.rows()), @@ -315,9 +316,9 @@ PartialPivLU::PartialPivLU(const EigenBase& matrix) compute(matrix.derived()); } -template +template template -PartialPivLU::PartialPivLU(EigenBase& matrix) +PartialPivLU::PartialPivLU(EigenBase& matrix) : m_lu(matrix.derived()), m_p(matrix.rows()), m_rowsTranspositions(matrix.rows()), @@ -334,12 +335,12 @@ namespace internal { template struct partial_lu_impl { - static const int UnBlockedBound = 16; - static const bool UnBlockedAtCompileTime = SizeAtCompileTime!=Dynamic && SizeAtCompileTime<=UnBlockedBound; - static const int ActualSizeAtCompileTime = UnBlockedAtCompileTime ? SizeAtCompileTime : Dynamic; + static constexpr int UnBlockedBound = 16; + static constexpr bool UnBlockedAtCompileTime = SizeAtCompileTime!=Dynamic && SizeAtCompileTime<=UnBlockedBound; + static constexpr int ActualSizeAtCompileTime = UnBlockedAtCompileTime ? SizeAtCompileTime : Dynamic; // Remaining rows and columns at compile-time: - static const int RRows = SizeAtCompileTime==2 ? 1 : Dynamic; - static const int RCols = SizeAtCompileTime==2 ? 1 : Dynamic; + static constexpr int RRows = SizeAtCompileTime==2 ? 1 : Dynamic; + static constexpr int RCols = SizeAtCompileTime==2 ? 1 : Dynamic; typedef Matrix MatrixType; typedef Ref MatrixTypeRef; typedef Ref > BlockType; @@ -379,7 +380,7 @@ struct partial_lu_impl row_transpositions[k] = PivIndex(row_of_biggest_in_col); - if(biggest_in_corner != Score(0)) + if(!numext::is_exactly_zero(biggest_in_corner)) { if(k != row_of_biggest_in_col) { @@ -405,7 +406,7 @@ struct partial_lu_impl { Index k = endk; row_transpositions[k] = PivIndex(k); - if (Scoring()(lu(k, k)) == Score(0) && first_zero_pivot == -1) + if (numext::is_exactly_zero(Scoring()(lu(k, k))) && first_zero_pivot == -1) first_zero_pivot = k; } @@ -515,19 +516,16 @@ void partial_lu_inplace(MatrixType& lu, TranspositionType& row_transpositions, t partial_lu_impl < typename MatrixType::Scalar, MatrixType::Flags&RowMajorBit?RowMajor:ColMajor, typename TranspositionType::StorageIndex, - EIGEN_SIZE_MIN_PREFER_FIXED(MatrixType::RowsAtCompileTime,MatrixType::ColsAtCompileTime)> + internal::min_size_prefer_fixed(MatrixType::RowsAtCompileTime, MatrixType::ColsAtCompileTime)> ::blocked_lu(lu.rows(), lu.cols(), &lu.coeffRef(0,0), lu.outerStride(), &row_transpositions.coeffRef(0), nb_transpositions); } } // end namespace internal -template -void PartialPivLU::compute() +template +void PartialPivLU::compute() { - check_template_parameters(); - - // the row permutation is stored as int indices, so just to be sure: - eigen_assert(m_lu.rows()::highest()); + eigen_assert(m_lu.rows()::highest()); if(m_lu.cols()>0) m_l1_norm = m_lu.cwiseAbs().colwise().sum().maxCoeff(); @@ -548,8 +546,8 @@ void PartialPivLU::compute() m_isInitialized = true; } -template -typename PartialPivLU::Scalar PartialPivLU::determinant() const +template +typename PartialPivLU::Scalar PartialPivLU::determinant() const { eigen_assert(m_isInitialized && "PartialPivLU is not initialized."); return Scalar(m_det_p) * m_lu.diagonal().prod(); @@ -558,8 +556,8 @@ typename PartialPivLU::Scalar PartialPivLU::determinant( /** \returns the matrix represented by the decomposition, * i.e., it returns the product: P^{-1} L U. * This function is provided for debug purpose. */ -template -MatrixType PartialPivLU::reconstructedMatrix() const +template +MatrixType PartialPivLU::reconstructedMatrix() const { eigen_assert(m_isInitialized && "LU is not initialized."); // LU @@ -577,10 +575,10 @@ MatrixType PartialPivLU::reconstructedMatrix() const namespace internal { /***** Implementation of inverse() *****************************************************/ -template -struct Assignment >, internal::assign_op::Scalar>, Dense2Dense> +template +struct Assignment >, internal::assign_op::Scalar>, Dense2Dense> { - typedef PartialPivLU LuType; + typedef PartialPivLU LuType; typedef Inverse SrcXprType; static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op &) { @@ -598,10 +596,11 @@ struct Assignment >, internal::assi * \sa class PartialPivLU */ template -inline const PartialPivLU::PlainObject> +template +inline const PartialPivLU::PlainObject, PermutationIndex> MatrixBase::partialPivLu() const { - return PartialPivLU(eval()); + return PartialPivLU(eval()); } /** \lu_module @@ -613,10 +612,11 @@ MatrixBase::partialPivLu() const * \sa class PartialPivLU */ template -inline const PartialPivLU::PlainObject> +template +inline const PartialPivLU::PlainObject, PermutationIndex> MatrixBase::lu() const { - return PartialPivLU(eval()); + return PartialPivLU(eval()); } } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/PartialPivLU_LAPACKE.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/PartialPivLU_LAPACKE.h index 755168a9460..8501b614075 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/PartialPivLU_LAPACKE.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/PartialPivLU_LAPACKE.h @@ -33,48 +33,62 @@ #ifndef EIGEN_PARTIALLU_LAPACK_H #define EIGEN_PARTIALLU_LAPACK_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -/** \internal Specialization for the data types supported by LAPACKe */ - -#define EIGEN_LAPACKE_LU_PARTPIV(EIGTYPE, LAPACKE_TYPE, LAPACKE_PREFIX) \ -template \ -struct partial_lu_impl \ -{ \ - /* \internal performs the LU decomposition in-place of the matrix represented */ \ - static lapack_int blocked_lu(Index rows, Index cols, EIGTYPE* lu_data, Index luStride, lapack_int* row_transpositions, lapack_int& nb_transpositions, lapack_int maxBlockSize=256) \ - { \ - EIGEN_UNUSED_VARIABLE(maxBlockSize);\ - lapack_int matrix_order, first_zero_pivot; \ - lapack_int m, n, lda, *ipiv, info; \ - EIGTYPE* a; \ -/* Set up parameters for ?getrf */ \ - matrix_order = StorageOrder==RowMajor ? LAPACK_ROW_MAJOR : LAPACK_COL_MAJOR; \ - lda = convert_index(luStride); \ - a = lu_data; \ - ipiv = row_transpositions; \ - m = convert_index(rows); \ - n = convert_index(cols); \ - nb_transpositions = 0; \ -\ - info = LAPACKE_##LAPACKE_PREFIX##getrf( matrix_order, m, n, (LAPACKE_TYPE*)a, lda, ipiv ); \ -\ - for(int i=0;i= 0); \ -/* something should be done with nb_transpositions */ \ -\ - first_zero_pivot = info; \ - return first_zero_pivot; \ - } \ +namespace lapacke_helpers { +// ------------------------------------------------------------------------------------------------------------------- +// Generic lapacke partial lu implementation that converts arguments and dispatches to the function above +// ------------------------------------------------------------------------------------------------------------------- + +template +struct lapacke_partial_lu { + /** \internal performs the LU decomposition in-place of the matrix represented */ + static lapack_int blocked_lu(Index rows, Index cols, Scalar* lu_data, Index luStride, lapack_int* row_transpositions, + lapack_int& nb_transpositions, lapack_int maxBlockSize=256) + { + EIGEN_UNUSED_VARIABLE(maxBlockSize); + // Set up parameters for getrf + lapack_int matrix_order = StorageOrder==RowMajor ? LAPACK_ROW_MAJOR : LAPACK_COL_MAJOR; + lapack_int lda = to_lapack(luStride); + Scalar* a = lu_data; + lapack_int* ipiv = row_transpositions; + lapack_int m = to_lapack(rows); + lapack_int n = to_lapack(cols); + nb_transpositions = 0; + + lapack_int info = getrf(matrix_order, m, n, to_lapack(a), lda, ipiv ); + eigen_assert(info >= 0); + + for(int i=0; i \ +struct partial_lu_impl : public lapacke_helpers::lapacke_partial_lu {}; + +EIGEN_LAPACKE_PARTIAL_LU(double) +EIGEN_LAPACKE_PARTIAL_LU(float) +EIGEN_LAPACKE_PARTIAL_LU(std::complex) +EIGEN_LAPACKE_PARTIAL_LU(std::complex) -EIGEN_LAPACKE_LU_PARTPIV(double, double, d) -EIGEN_LAPACKE_LU_PARTPIV(float, float, s) -EIGEN_LAPACKE_LU_PARTPIV(dcomplex, lapack_complex_double, z) -EIGEN_LAPACKE_LU_PARTPIV(scomplex, lapack_complex_float, c) +#undef EIGEN_LAPACKE_PARTIAL_LU } // end namespace internal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/arch/InverseSize4.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/arch/InverseSize4.h index a232ffc0aad..2cd92b5c8b3 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/arch/InverseSize4.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/LU/arch/InverseSize4.h @@ -35,6 +35,16 @@ #ifndef EIGEN_INVERSE_SIZE_4_H #define EIGEN_INVERSE_SIZE_4_H +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + +#if EIGEN_COMP_GNUC_STRICT +// These routines requires bit manipulation of the sign, which is not compatible +// with fastmath. +#pragma GCC push_options +#pragma GCC optimize ("no-fast-math") +#endif + namespace Eigen { namespace internal @@ -48,7 +58,7 @@ struct compute_inverse_size4::Alignment, StorageOrdersMatch = (MatrixType::Flags & RowMajorBit) == (ResultType::Flags & RowMajorBit) }; - typedef typename conditional<(MatrixType::Flags & LinearAccessBit), MatrixType const &, typename MatrixType::PlainObject>::type ActualMatrixType; + typedef std::conditional_t<(MatrixType::Flags & LinearAccessBit), MatrixType const &, typename MatrixType::PlainObject> ActualMatrixType; static void run(const MatrixType &mat, ResultType &result) { @@ -56,10 +66,10 @@ struct compute_inverse_size4(data); - Packet4f _L2 = ploadt(data + stride*4); - Packet4f _L3 = ploadt(data + stride*8); - Packet4f _L4 = ploadt(data + stride*12); + Packet4f L1 = ploadt(data); + Packet4f L2 = ploadt(data + stride*4); + Packet4f L3 = ploadt(data + stride*8); + Packet4f L4 = ploadt(data + stride*12); // Four 2x2 sub-matrices of the input matrix // input = [[A, B], @@ -68,17 +78,17 @@ struct compute_inverse_size4(1.0f), det); + Packet4f rd = preciprocal(det); // Four sub-matrices of the inverse Packet4f iA, iB, iC, iD; @@ -143,8 +153,8 @@ struct compute_inverse_size4(0x80000000u), numext::bit_cast(0x80000000u), 0.0f}; - const Packet4f p4f_sign_PNNP = ploadu(sign_mask); + EIGEN_ALIGN_MAX const float sign_mask[4] = {0.0f, -0.0f, -0.0f, 0.0f}; + const Packet4f p4f_sign_PNNP = pload(sign_mask); rd = pxor(rd, p4f_sign_PNNP); iA = pmul(iA, rd); iB = pmul(iB, rd); @@ -173,9 +183,9 @@ struct compute_inverse_size4::Alignment, StorageOrdersMatch = (MatrixType::Flags & RowMajorBit) == (ResultType::Flags & RowMajorBit) }; - typedef typename conditional<(MatrixType::Flags & LinearAccessBit), - MatrixType const &, - typename MatrixType::PlainObject>::type + typedef std::conditional_t<(MatrixType::Flags & LinearAccessBit), + MatrixType const &, + typename MatrixType::PlainObject> ActualMatrixType; static void run(const MatrixType &mat, ResultType &result) @@ -326,10 +336,10 @@ struct compute_inverse_size4(0x8000000000000000ull)}; - const double sign_mask2[2] = {numext::bit_cast(0x8000000000000000ull), 0.0}; - const Packet2d sign_PN = ploadu(sign_mask1); - const Packet2d sign_NP = ploadu(sign_mask2); + EIGEN_ALIGN_MAX const double sign_mask1[2] = {0.0, -0.0}; + EIGEN_ALIGN_MAX const double sign_mask2[2] = {-0.0, 0.0}; + const Packet2d sign_PN = pload(sign_mask1); + const Packet2d sign_NP = pload(sign_mask2); d1 = pxor(rd, sign_PN); d2 = pxor(rd, sign_NP); @@ -348,4 +358,9 @@ struct compute_inverse_size4 class PastixLU; -template class PastixLLT; -template class PastixLDLT; +template class PastixLU; +template class PastixLLT; +template class PastixLDLT; namespace internal { template struct pastix_traits; - template - struct pastix_traits< PastixLU<_MatrixType> > + template + struct pastix_traits< PastixLU > { - typedef _MatrixType MatrixType; - typedef typename _MatrixType::Scalar Scalar; - typedef typename _MatrixType::RealScalar RealScalar; - typedef typename _MatrixType::StorageIndex StorageIndex; + typedef MatrixType_ MatrixType; + typedef typename MatrixType_::Scalar Scalar; + typedef typename MatrixType_::RealScalar RealScalar; + typedef typename MatrixType_::StorageIndex StorageIndex; }; - template - struct pastix_traits< PastixLLT<_MatrixType,Options> > + template + struct pastix_traits< PastixLLT > { - typedef _MatrixType MatrixType; - typedef typename _MatrixType::Scalar Scalar; - typedef typename _MatrixType::RealScalar RealScalar; - typedef typename _MatrixType::StorageIndex StorageIndex; + typedef MatrixType_ MatrixType; + typedef typename MatrixType_::Scalar Scalar; + typedef typename MatrixType_::RealScalar RealScalar; + typedef typename MatrixType_::StorageIndex StorageIndex; }; - template - struct pastix_traits< PastixLDLT<_MatrixType,Options> > + template + struct pastix_traits< PastixLDLT > { - typedef _MatrixType MatrixType; - typedef typename _MatrixType::Scalar Scalar; - typedef typename _MatrixType::RealScalar RealScalar; - typedef typename _MatrixType::StorageIndex StorageIndex; + typedef MatrixType_ MatrixType; + typedef typename MatrixType_::Scalar Scalar; + typedef typename MatrixType_::RealScalar RealScalar; + typedef typename MatrixType_::StorageIndex StorageIndex; }; inline void eigen_pastix(pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, float *vals, int *perm, int * invp, float *x, int nbrhs, int *iparm, double *dparm) @@ -134,8 +137,8 @@ class PastixBase : public SparseSolverBase public: using Base::_solve_impl; - typedef typename internal::pastix_traits::MatrixType _MatrixType; - typedef _MatrixType MatrixType; + typedef typename internal::pastix_traits::MatrixType MatrixType_; + typedef MatrixType_ MatrixType; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef typename MatrixType::StorageIndex StorageIndex; @@ -397,7 +400,7 @@ bool PastixBase::_solve_impl(const MatrixBase &b, MatrixBase &x * This interface can symmetrize the input matrix otherwise. * The vectors or matrices X and B can be either dense or sparse. * - * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> * \tparam IsStrSym Indicates if the input matrix has a symmetric pattern, default is false * NOTE : Note that if the analysis and factorization phase are called separately, * the input matrix will be symmetrized at each call, hence it is advised to @@ -408,11 +411,11 @@ bool PastixBase::_solve_impl(const MatrixBase &b, MatrixBase &x * \sa \ref TutorialSparseSolverConcept, class SparseLU * */ -template -class PastixLU : public PastixBase< PastixLU<_MatrixType> > +template +class PastixLU : public PastixBase< PastixLU > { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef PastixBase > Base; typedef typename Base::ColSpMatrix ColSpMatrix; typedef typename MatrixType::StorageIndex StorageIndex; @@ -520,16 +523,16 @@ class PastixLU : public PastixBase< PastixLU<_MatrixType> > * * \sa \ref TutorialSparseSolverConcept, class SimplicialLLT */ -template -class PastixLLT : public PastixBase< PastixLLT<_MatrixType, _UpLo> > +template +class PastixLLT : public PastixBase< PastixLLT > { public: - typedef _MatrixType MatrixType; - typedef PastixBase > Base; + typedef MatrixType_ MatrixType; + typedef PastixBase > Base; typedef typename Base::ColSpMatrix ColSpMatrix; public: - enum { UpLo = _UpLo }; + enum { UpLo = UpLo_ }; PastixLLT() : Base() { init(); @@ -604,16 +607,16 @@ class PastixLLT : public PastixBase< PastixLLT<_MatrixType, _UpLo> > * * \sa \ref TutorialSparseSolverConcept, class SimplicialLDLT */ -template -class PastixLDLT : public PastixBase< PastixLDLT<_MatrixType, _UpLo> > +template +class PastixLDLT : public PastixBase< PastixLDLT > { public: - typedef _MatrixType MatrixType; - typedef PastixBase > Base; + typedef MatrixType_ MatrixType; + typedef PastixBase > Base; typedef typename Base::ColSpMatrix ColSpMatrix; public: - enum { UpLo = _UpLo }; + enum { UpLo = UpLo_ }; PastixLDLT():Base() { init(); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/PardisoSupport/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/PardisoSupport/InternalHeaderCheck.h new file mode 100644 index 00000000000..8ef33f051ca --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/PardisoSupport/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_PARDISOSUPPORT_MODULE_H +#error "Please include Eigen/PardisoSupport instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/PardisoSupport/PardisoSupport.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/PardisoSupport/PardisoSupport.h index f89b79bd55b..f9f79796f55 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/PardisoSupport/PardisoSupport.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/PardisoSupport/PardisoSupport.h @@ -32,11 +32,14 @@ #ifndef EIGEN_PARDISOSUPPORT_H #define EIGEN_PARDISOSUPPORT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { -template class PardisoLU; -template class PardisoLLT; -template class PardisoLDLT; +template class PardisoLU; +template class PardisoLLT; +template class PardisoLDLT; namespace internal { @@ -66,31 +69,31 @@ namespace internal template struct pardiso_traits; - template - struct pardiso_traits< PardisoLU<_MatrixType> > + template + struct pardiso_traits< PardisoLU > { - typedef _MatrixType MatrixType; - typedef typename _MatrixType::Scalar Scalar; - typedef typename _MatrixType::RealScalar RealScalar; - typedef typename _MatrixType::StorageIndex StorageIndex; + typedef MatrixType_ MatrixType; + typedef typename MatrixType_::Scalar Scalar; + typedef typename MatrixType_::RealScalar RealScalar; + typedef typename MatrixType_::StorageIndex StorageIndex; }; - template - struct pardiso_traits< PardisoLLT<_MatrixType, Options> > + template + struct pardiso_traits< PardisoLLT > { - typedef _MatrixType MatrixType; - typedef typename _MatrixType::Scalar Scalar; - typedef typename _MatrixType::RealScalar RealScalar; - typedef typename _MatrixType::StorageIndex StorageIndex; + typedef MatrixType_ MatrixType; + typedef typename MatrixType_::Scalar Scalar; + typedef typename MatrixType_::RealScalar RealScalar; + typedef typename MatrixType_::StorageIndex StorageIndex; }; - template - struct pardiso_traits< PardisoLDLT<_MatrixType, Options> > + template + struct pardiso_traits< PardisoLDLT > { - typedef _MatrixType MatrixType; - typedef typename _MatrixType::Scalar Scalar; - typedef typename _MatrixType::RealScalar RealScalar; - typedef typename _MatrixType::StorageIndex StorageIndex; + typedef MatrixType_ MatrixType; + typedef typename MatrixType_::Scalar Scalar; + typedef typename MatrixType_::RealScalar RealScalar; + typedef typename MatrixType_::StorageIndex StorageIndex; }; } // end namespace internal @@ -271,8 +274,8 @@ Derived& PardisoImpl::compute(const MatrixType& a) m_matrix.valuePtr(), m_matrix.outerIndexPtr(), m_matrix.innerIndexPtr(), m_perm.data(), 0, m_iparm.data(), m_msglvl, NULL, NULL); manageErrorCode(error); - m_analysisIsOk = true; - m_factorizationIsOk = true; + m_analysisIsOk = m_info == Eigen::Success; + m_factorizationIsOk = m_info == Eigen::Success; m_isInitialized = true; return derived(); } @@ -293,7 +296,7 @@ Derived& PardisoImpl::analyzePattern(const MatrixType& a) m_perm.data(), 0, m_iparm.data(), m_msglvl, NULL, NULL); manageErrorCode(error); - m_analysisIsOk = true; + m_analysisIsOk = m_info == Eigen::Success; m_factorizationIsOk = false; m_isInitialized = true; return derived(); @@ -313,7 +316,7 @@ Derived& PardisoImpl::factorize(const MatrixType& a) m_perm.data(), 0, m_iparm.data(), m_msglvl, NULL, NULL); manageErrorCode(error); - m_factorizationIsOk = true; + m_factorizationIsOk = m_info == Eigen::Success; return derived(); } @@ -375,7 +378,7 @@ void PardisoImpl::_solve_impl(const MatrixBase &b, MatrixBase * By default, it runs in in-core mode. To enable PARDISO's out-of-core feature, set: * \code solver.pardisoParameterArray()[59] = 1; \endcode * - * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> * * \implsparsesolverconcept * @@ -437,21 +440,21 @@ class PardisoLU : public PardisoImpl< PardisoLU > * * \sa \ref TutorialSparseSolverConcept, class SimplicialLLT */ -template -class PardisoLLT : public PardisoImpl< PardisoLLT > +template +class PardisoLLT : public PardisoImpl< PardisoLLT > { protected: - typedef PardisoImpl< PardisoLLT > Base; + typedef PardisoImpl< PardisoLLT > Base; using Base::pardisoInit; using Base::m_matrix; - friend class PardisoImpl< PardisoLLT >; + friend class PardisoImpl< PardisoLLT >; public: typedef typename Base::Scalar Scalar; typedef typename Base::RealScalar RealScalar; typedef typename Base::StorageIndex StorageIndex; - enum { UpLo = _UpLo }; + enum { UpLo = UpLo_ }; using Base::compute; PardisoLLT() diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/ColPivHouseholderQR.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/ColPivHouseholderQR.h index 9b677e9bf45..0f48683791c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/ColPivHouseholderQR.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/ColPivHouseholderQR.h @@ -11,15 +11,18 @@ #ifndef EIGEN_COLPIVOTINGHOUSEHOLDERQR_H #define EIGEN_COLPIVOTINGHOUSEHOLDERQR_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template struct traits > - : traits<_MatrixType> +template struct traits > + : traits { typedef MatrixXpr XprKind; typedef SolverStorage StorageKind; - typedef int StorageIndex; + typedef PermutationIndex_ PermutationIndex; enum { Flags = 0 }; }; @@ -31,7 +34,7 @@ template struct traits > * * \brief Householder rank-revealing QR decomposition of a matrix with column-pivoting * - * \tparam _MatrixType the type of the matrix of which we are computing the QR decomposition + * \tparam MatrixType_ the type of the matrix of which we are computing the QR decomposition * * This class performs a rank-revealing QR decomposition of a matrix \b A into matrices \b P, \b Q and \b R * such that @@ -48,31 +51,41 @@ template struct traits > * * \sa MatrixBase::colPivHouseholderQr() */ -template class ColPivHouseholderQR - : public SolverBase > +template class ColPivHouseholderQR + : public SolverBase > { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef SolverBase Base; friend class SolverBase; - + typedef PermutationIndex_ PermutationIndex; EIGEN_GENERIC_PUBLIC_INTERFACE(ColPivHouseholderQR) + enum { MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; typedef typename internal::plain_diag_type::type HCoeffsType; - typedef PermutationMatrix PermutationType; - typedef typename internal::plain_row_type::type IntRowVectorType; + typedef PermutationMatrix PermutationType; + typedef typename internal::plain_row_type::type IntRowVectorType; typedef typename internal::plain_row_type::type RowVectorType; typedef typename internal::plain_row_type::type RealRowVectorType; - typedef HouseholderSequence::type> HouseholderSequenceType; + typedef HouseholderSequence> HouseholderSequenceType; typedef typename MatrixType::PlainObject PlainObject; - private: - - typedef typename PermutationType::StorageIndex PermIndexType; +private: + void init(Index rows, Index cols) { + Index diag = numext::mini(rows, cols); + m_hCoeffs.resize(diag); + m_colsPermutation.resize(cols); + m_colsTranspositions.resize(cols); + m_temp.resize(cols); + m_colNormsUpdated.resize(cols); + m_colNormsDirect.resize(cols); + m_isInitialized = false; + m_usePrescribedThreshold = false; + } public: @@ -99,16 +112,7 @@ template class ColPivHouseholderQR * according to the specified problem \a size. * \sa ColPivHouseholderQR() */ - ColPivHouseholderQR(Index rows, Index cols) - : m_qr(rows, cols), - m_hCoeffs((std::min)(rows,cols)), - m_colsPermutation(PermIndexType(cols)), - m_colsTranspositions(cols), - m_temp(cols), - m_colNormsUpdated(cols), - m_colNormsDirect(cols), - m_isInitialized(false), - m_usePrescribedThreshold(false) {} + ColPivHouseholderQR(Index rows, Index cols) : m_qr(rows, cols) { init(rows, cols); } /** \brief Constructs a QR factorization from a given matrix * @@ -122,18 +126,9 @@ template class ColPivHouseholderQR * * \sa compute() */ - template - explicit ColPivHouseholderQR(const EigenBase& matrix) - : m_qr(matrix.rows(), matrix.cols()), - m_hCoeffs((std::min)(matrix.rows(),matrix.cols())), - m_colsPermutation(PermIndexType(matrix.cols())), - m_colsTranspositions(matrix.cols()), - m_temp(matrix.cols()), - m_colNormsUpdated(matrix.cols()), - m_colNormsDirect(matrix.cols()), - m_isInitialized(false), - m_usePrescribedThreshold(false) - { + template + explicit ColPivHouseholderQR(const EigenBase& matrix) : m_qr(matrix.rows(), matrix.cols()) { + init(matrix.rows(), matrix.cols()); compute(matrix.derived()); } @@ -143,18 +138,9 @@ template class ColPivHouseholderQR * * \sa ColPivHouseholderQR(const EigenBase&) */ - template - explicit ColPivHouseholderQR(EigenBase& matrix) - : m_qr(matrix.derived()), - m_hCoeffs((std::min)(matrix.rows(),matrix.cols())), - m_colsPermutation(PermIndexType(matrix.cols())), - m_colsTranspositions(matrix.cols()), - m_temp(matrix.cols()), - m_colNormsUpdated(matrix.cols()), - m_colNormsDirect(matrix.cols()), - m_isInitialized(false), - m_usePrescribedThreshold(false) - { + template + explicit ColPivHouseholderQR(EigenBase& matrix) : m_qr(matrix.derived()) { + init(matrix.rows(), matrix.cols()); computeInPlace(); } @@ -217,6 +203,21 @@ template class ColPivHouseholderQR return m_colsPermutation; } + /** \returns the determinant of the matrix of which + * *this is the QR decomposition. It has only linear complexity + * (that is, O(n) where n is the dimension of the square matrix) + * as the QR decomposition has already been computed. + * + * \note This is only for square matrices. + * + * \warning a determinant can be very big or small, so for matrices + * of large enough dimension, there is a risk of overflow/underflow. + * One way to work around that is to use logAbsDeterminant() instead. + * + * \sa absDeterminant(), logAbsDeterminant(), MatrixBase::determinant() + */ + typename MatrixType::Scalar determinant() const; + /** \returns the absolute value of the determinant of the matrix of which * *this is the QR decomposition. It has only linear complexity * (that is, O(n) where n is the dimension of the square matrix) @@ -228,7 +229,7 @@ template class ColPivHouseholderQR * of large enough dimension, there is a risk of overflow/underflow. * One way to work around that is to use logAbsDeterminant() instead. * - * \sa logAbsDeterminant(), MatrixBase::determinant() + * \sa determinant(), logAbsDeterminant(), MatrixBase::determinant() */ typename MatrixType::RealScalar absDeterminant() const; @@ -242,7 +243,7 @@ template class ColPivHouseholderQR * \note This method is useful to work around the risk of overflow/underflow that's inherent * to determinant computation. * - * \sa absDeterminant(), MatrixBase::determinant() + * \sa determinant(), absDeterminant(), MatrixBase::determinant() */ typename MatrixType::RealScalar logAbsDeterminant() const; @@ -424,12 +425,9 @@ template class ColPivHouseholderQR protected: - friend class CompleteOrthogonalDecomposition; + friend class CompleteOrthogonalDecomposition; - static void check_template_parameters() - { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - } + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) void computeInPlace(); @@ -443,11 +441,21 @@ template class ColPivHouseholderQR bool m_isInitialized, m_usePrescribedThreshold; RealScalar m_prescribedThreshold, m_maxpivot; Index m_nonzero_pivots; - Index m_det_pq; + Index m_det_p; }; -template -typename MatrixType::RealScalar ColPivHouseholderQR::absDeterminant() const +template +typename MatrixType::Scalar ColPivHouseholderQR::determinant() const +{ + eigen_assert(m_isInitialized && "HouseholderQR is not initialized."); + eigen_assert(m_qr.rows() == m_qr.cols() && "You can't take the determinant of a non-square matrix!"); + Scalar detQ; + internal::householder_determinant::IsComplex>::run(m_hCoeffs, detQ); + return m_qr.diagonal().prod() * detQ * Scalar(m_det_p); +} + +template +typename MatrixType::RealScalar ColPivHouseholderQR::absDeterminant() const { using std::abs; eigen_assert(m_isInitialized && "ColPivHouseholderQR is not initialized."); @@ -455,8 +463,8 @@ typename MatrixType::RealScalar ColPivHouseholderQR::absDeterminant( return abs(m_qr.diagonal().prod()); } -template -typename MatrixType::RealScalar ColPivHouseholderQR::logAbsDeterminant() const +template +typename MatrixType::RealScalar ColPivHouseholderQR::logAbsDeterminant() const { eigen_assert(m_isInitialized && "ColPivHouseholderQR is not initialized."); eigen_assert(m_qr.rows() == m_qr.cols() && "You can't take the determinant of a non-square matrix!"); @@ -469,22 +477,20 @@ typename MatrixType::RealScalar ColPivHouseholderQR::logAbsDetermina * * \sa class ColPivHouseholderQR, ColPivHouseholderQR(const MatrixType&) */ -template +template template -ColPivHouseholderQR& ColPivHouseholderQR::compute(const EigenBase& matrix) +ColPivHouseholderQR& ColPivHouseholderQR::compute(const EigenBase& matrix) { m_qr = matrix.derived(); computeInPlace(); return *this; } -template -void ColPivHouseholderQR::computeInPlace() +template +void ColPivHouseholderQR::computeInPlace() { - check_template_parameters(); - // the column permutation is stored as int indices, so just to be sure: - eigen_assert(m_qr.cols()<=NumTraits::highest()); + eigen_assert(m_qr.cols()<=NumTraits::highest()); using std::abs; @@ -527,7 +533,7 @@ void ColPivHouseholderQR::computeInPlace() m_nonzero_pivots = k; // apply the transposition to the columns - m_colsTranspositions.coeffRef(k) = biggest_col_index; + m_colsTranspositions.coeffRef(k) = static_cast(biggest_col_index); if(k != biggest_col_index) { m_qr.col(k).swap(m_qr.col(biggest_col_index)); std::swap(m_colNormsUpdated.coeffRef(k), m_colNormsUpdated.coeffRef(biggest_col_index)); @@ -555,7 +561,7 @@ void ColPivHouseholderQR::computeInPlace() // http://www.netlib.org/lapack/lawnspdf/lawn176.pdf // and used in LAPACK routines xGEQPF and xGEQP3. // See lines 278-297 in http://www.netlib.org/lapack/explore-html/dc/df4/sgeqpf_8f_source.html - if (m_colNormsUpdated.coeffRef(j) != RealScalar(0)) { + if (!numext::is_exactly_zero(m_colNormsUpdated.coeffRef(j))) { RealScalar temp = abs(m_qr.coeffRef(k, j)) / m_colNormsUpdated.coeffRef(j); temp = (RealScalar(1) + temp) * (RealScalar(1) - temp); temp = temp < RealScalar(0) ? RealScalar(0) : temp; @@ -573,18 +579,18 @@ void ColPivHouseholderQR::computeInPlace() } } - m_colsPermutation.setIdentity(PermIndexType(cols)); - for(PermIndexType k = 0; k < size/*m_nonzero_pivots*/; ++k) - m_colsPermutation.applyTranspositionOnTheRight(k, PermIndexType(m_colsTranspositions.coeff(k))); + m_colsPermutation.setIdentity(cols); + for(Index k = 0; k < size/*m_nonzero_pivots*/; ++k) + m_colsPermutation.applyTranspositionOnTheRight(k, static_cast(m_colsTranspositions.coeff(k))); - m_det_pq = (number_of_transpositions%2) ? -1 : 1; + m_det_p = (number_of_transpositions%2) ? -1 : 1; m_isInitialized = true; } #ifndef EIGEN_PARSED_BY_DOXYGEN -template +template template -void ColPivHouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const +void ColPivHouseholderQR::_solve_impl(const RhsType &rhs, DstType &dst) const { const Index nonzero_pivots = nonzeroPivots(); @@ -606,9 +612,9 @@ void ColPivHouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType & for(Index i = nonzero_pivots; i < cols(); ++i) dst.row(m_colsPermutation.indices().coeff(i)).setZero(); } -template +template template -void ColPivHouseholderQR<_MatrixType>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +void ColPivHouseholderQR::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const { const Index nonzero_pivots = nonzeroPivots(); @@ -634,10 +640,10 @@ void ColPivHouseholderQR<_MatrixType>::_solve_impl_transposed(const RhsType &rhs namespace internal { -template -struct Assignment >, internal::assign_op::Scalar>, Dense2Dense> +template +struct Assignment >, internal::assign_op::Scalar>, Dense2Dense> { - typedef ColPivHouseholderQR QrType; + typedef ColPivHouseholderQR QrType; typedef Inverse SrcXprType; static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op &) { @@ -650,8 +656,8 @@ struct Assignment >, interna /** \returns the matrix Q as a sequence of householder transformations. * You can extract the meaningful part only by using: * \code qr.householderQ().setLength(qr.nonzeroPivots()) \endcode*/ -template -typename ColPivHouseholderQR::HouseholderSequenceType ColPivHouseholderQR +template +typename ColPivHouseholderQR::HouseholderSequenceType ColPivHouseholderQR ::householderQ() const { eigen_assert(m_isInitialized && "ColPivHouseholderQR is not initialized."); @@ -663,10 +669,11 @@ typename ColPivHouseholderQR::HouseholderSequenceType ColPivHousehol * \sa class ColPivHouseholderQR */ template -const ColPivHouseholderQR::PlainObject> +template +const ColPivHouseholderQR::PlainObject, PermutationIndexType> MatrixBase::colPivHouseholderQr() const { - return ColPivHouseholderQR(eval()); + return ColPivHouseholderQR(eval()); } } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/ColPivHouseholderQR_LAPACKE.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/ColPivHouseholderQR_LAPACKE.h index 4e9651f83d5..8745ae47bdd 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/ColPivHouseholderQR_LAPACKE.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/ColPivHouseholderQR_LAPACKE.h @@ -34,64 +34,117 @@ #ifndef EIGEN_COLPIVOTINGHOUSEHOLDERQR_LAPACKE_H #define EIGEN_COLPIVOTINGHOUSEHOLDERQR_LAPACKE_H -namespace Eigen { - -/** \internal Specialization for the data types supported by LAPACKe */ - -#define EIGEN_LAPACKE_QR_COLPIV(EIGTYPE, LAPACKE_TYPE, LAPACKE_PREFIX, EIGCOLROW, LAPACKE_COLROW) \ -template<> template inline \ -ColPivHouseholderQR >& \ -ColPivHouseholderQR >::compute( \ - const EigenBase& matrix) \ -\ -{ \ - using std::abs; \ - typedef Matrix MatrixType; \ - typedef MatrixType::RealScalar RealScalar; \ - Index rows = matrix.rows();\ - Index cols = matrix.cols();\ -\ - m_qr = matrix;\ - Index size = m_qr.diagonalSize();\ - m_hCoeffs.resize(size);\ -\ - m_colsTranspositions.resize(cols);\ - /*Index number_of_transpositions = 0;*/ \ -\ - m_nonzero_pivots = 0; \ - m_maxpivot = RealScalar(0);\ - m_colsPermutation.resize(cols); \ - m_colsPermutation.indices().setZero(); \ -\ - lapack_int lda = internal::convert_index(m_qr.outerStride()); \ - lapack_int matrix_order = LAPACKE_COLROW; \ - LAPACKE_##LAPACKE_PREFIX##geqp3( matrix_order, internal::convert_index(rows), internal::convert_index(cols), \ - (LAPACKE_TYPE*)m_qr.data(), lda, (lapack_int*)m_colsPermutation.indices().data(), (LAPACKE_TYPE*)m_hCoeffs.data()); \ - m_isInitialized = true; \ - m_maxpivot=m_qr.diagonal().cwiseAbs().maxCoeff(); \ - m_hCoeffs.adjointInPlace(); \ - RealScalar premultiplied_threshold = abs(m_maxpivot) * threshold(); \ - lapack_int *perm = m_colsPermutation.indices().data(); \ - for(Index i=0;i premultiplied_threshold);\ - } \ - for(Index i=0;i + inline lapack_int call_geqp3(int matrix_layout, lapack_int m, lapack_int n, Scalar* a, lapack_int lda, lapack_int* jpvt, Scalar* tau); + template<> + inline lapack_int call_geqp3(int matrix_layout, lapack_int m, lapack_int n, float* a, lapack_int lda, lapack_int* jpvt, float* tau) + { return LAPACKE_sgeqp3(matrix_layout, m, n, a, lda, jpvt, tau); } + template<> + inline lapack_int call_geqp3(int matrix_layout, lapack_int m, lapack_int n, double* a, lapack_int lda, lapack_int* jpvt, double* tau) + { return LAPACKE_dgeqp3(matrix_layout, m, n, a, lda, jpvt, tau); } + template<> + inline lapack_int call_geqp3(int matrix_layout, lapack_int m, lapack_int n, lapack_complex_float* a, lapack_int lda, lapack_int* jpvt, lapack_complex_float* tau) + { return LAPACKE_cgeqp3(matrix_layout, m, n, a, lda, jpvt, tau); } + template<> + inline lapack_int call_geqp3(int matrix_layout, lapack_int m, lapack_int n, lapack_complex_double* a, lapack_int lda, lapack_int* jpvt, lapack_complex_double* tau) + { return LAPACKE_zgeqp3(matrix_layout, m, n, a, lda, jpvt, tau); } + + template + struct ColPivHouseholderQR_LAPACKE_impl { + typedef typename MatrixType::Scalar Scalar; + typedef typename MatrixType::RealScalar RealScalar; + typedef typename internal::lapacke_helpers::translate_type_imp::type LapackeType; + static constexpr int LapackeStorage = MatrixType::IsRowMajor ? (LAPACK_ROW_MAJOR) : (LAPACK_COL_MAJOR); + + typedef typename internal::plain_diag_type::type HCoeffsType; + typedef PermutationMatrix PermutationType; + + static void run(MatrixType& qr, HCoeffsType& hCoeffs, PermutationType& colsPermutation, Index& nonzero_pivots, + RealScalar& maxpivot, bool usePrescribedThreshold, RealScalar prescribedThreshold, Index& det_p, + bool& isInitialized) { + + isInitialized = false; + hCoeffs.resize(qr.diagonalSize()); + nonzero_pivots = 0; + maxpivot = RealScalar(0); + colsPermutation.resize(qr.cols()); + colsPermutation.indices().setZero(); + + lapack_int rows = internal::lapacke_helpers::to_lapack(qr.rows()); + lapack_int cols = internal::lapacke_helpers::to_lapack(qr.cols()); + LapackeType* qr_data = (LapackeType*)(qr.data()); + lapack_int lda = internal::lapacke_helpers::to_lapack(qr.outerStride()); + lapack_int* perm_data = colsPermutation.indices().data(); + LapackeType* hCoeffs_data = (LapackeType*)(hCoeffs.data()); + + lapack_int info = call_geqp3(LapackeStorage, rows, cols, qr_data, lda, perm_data, hCoeffs_data); + if (info != 0) return; + + maxpivot = qr.diagonal().cwiseAbs().maxCoeff(); + hCoeffs.adjointInPlace(); + RealScalar defaultThreshold = NumTraits::epsilon() * RealScalar(qr.diagonalSize()); + RealScalar threshold = usePrescribedThreshold ? prescribedThreshold : defaultThreshold; + RealScalar premultiplied_threshold = maxpivot * threshold; + nonzero_pivots = (qr.diagonal().cwiseAbs().array() > premultiplied_threshold).count(); + colsPermutation.indices().array() -= 1; + det_p = colsPermutation.determinant(); + isInitialized = true; + }; + + static void init(Index rows, Index cols, HCoeffsType& hCoeffs, PermutationType& colsPermutation, + bool& usePrescribedThreshold, bool& isInitialized) { + + Index diag = numext::mini(rows, cols); + hCoeffs.resize(diag); + colsPermutation.resize(cols); + usePrescribedThreshold = false; + isInitialized = false; + } + }; + + #define COLPIVQR_LAPACKE_COMPUTEINPLACE(EIGTYPE) \ + template <> inline void ColPivHouseholderQR::computeInPlace() { \ + ColPivHouseholderQR_LAPACKE_impl::run(m_qr, m_hCoeffs, m_colsPermutation, m_nonzero_pivots, \ + m_maxpivot, m_usePrescribedThreshold, m_prescribedThreshold, \ + m_det_p, m_isInitialized); } \ + + #define COLPIVQR_LAPACKE_INIT(EIGTYPE) \ + template <> inline void ColPivHouseholderQR::init(Index rows, Index cols) { \ + ColPivHouseholderQR_LAPACKE_impl::init(rows, cols, m_hCoeffs, m_colsPermutation, m_isInitialized, \ + m_usePrescribedThreshold); } \ + + #define COLPIVQR_LAPACKE(EIGTYPE) \ + COLPIVQR_LAPACKE_COMPUTEINPLACE(EIGTYPE) \ + COLPIVQR_LAPACKE_INIT(EIGTYPE) \ + COLPIVQR_LAPACKE_COMPUTEINPLACE(Ref) \ + COLPIVQR_LAPACKE_INIT(Ref) \ + + typedef Matrix MatrixXfC; + typedef Matrix MatrixXdC; + typedef Matrix, Dynamic, Dynamic, ColMajor> MatrixXcfC; + typedef Matrix, Dynamic, Dynamic, ColMajor> MatrixXcdC; + typedef Matrix MatrixXfR; + typedef Matrix MatrixXdR; + typedef Matrix, Dynamic, Dynamic, RowMajor> MatrixXcfR; + typedef Matrix, Dynamic, Dynamic, RowMajor> MatrixXcdR; + + COLPIVQR_LAPACKE(MatrixXfC) + COLPIVQR_LAPACKE(MatrixXdC) + COLPIVQR_LAPACKE(MatrixXcfC) + COLPIVQR_LAPACKE(MatrixXcdC) + COLPIVQR_LAPACKE(MatrixXfR) + COLPIVQR_LAPACKE(MatrixXdR) + COLPIVQR_LAPACKE(MatrixXcfR) + COLPIVQR_LAPACKE(MatrixXcdR) + +#endif +} // end namespace Eigen + +#endif // EIGEN_COLPIVOTINGHOUSEHOLDERQR_LAPACKE_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/CompleteOrthogonalDecomposition.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/CompleteOrthogonalDecomposition.h index 486d3373af5..305328653f1 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/CompleteOrthogonalDecomposition.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/CompleteOrthogonalDecomposition.h @@ -10,15 +10,18 @@ #ifndef EIGEN_COMPLETEORTHOGONALDECOMPOSITION_H #define EIGEN_COMPLETEORTHOGONALDECOMPOSITION_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template -struct traits > - : traits<_MatrixType> { +template +struct traits > + : traits { typedef MatrixXpr XprKind; typedef SolverStorage StorageKind; - typedef int StorageIndex; + typedef PermutationIndex_ PermutationIndex; enum { Flags = 0 }; }; @@ -30,7 +33,7 @@ struct traits > * * \brief Complete orthogonal decomposition (COD) of a matrix. * - * \param MatrixType the type of the matrix of which we are computing the COD. + * \tparam MatrixType_ the type of the matrix of which we are computing the COD. * * This class performs a rank-revealing complete orthogonal decomposition of a * matrix \b A into matrices \b P, \b Q, \b T, and \b Z such that @@ -47,23 +50,23 @@ struct traits > * * \sa MatrixBase::completeOrthogonalDecomposition() */ -template class CompleteOrthogonalDecomposition - : public SolverBase > +template class CompleteOrthogonalDecomposition + : public SolverBase > { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef SolverBase Base; template friend struct internal::solve_assertion; - + typedef PermutationIndex_ PermutationIndex; EIGEN_GENERIC_PUBLIC_INTERFACE(CompleteOrthogonalDecomposition) enum { MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; typedef typename internal::plain_diag_type::type HCoeffsType; - typedef PermutationMatrix + typedef PermutationMatrix PermutationType; typedef typename internal::plain_row_type::type IntRowVectorType; @@ -71,14 +74,11 @@ template class CompleteOrthogonalDecomposition typedef typename internal::plain_row_type::type RealRowVectorType; typedef HouseholderSequence< - MatrixType, typename internal::remove_all< - typename HCoeffsType::ConjugateReturnType>::type> + MatrixType, internal::remove_all_t< + typename HCoeffsType::ConjugateReturnType>> HouseholderSequenceType; typedef typename MatrixType::PlainObject PlainObject; - private: - typedef typename PermutationType::Index PermIndexType; - public: /** * \brief Default Constructor. @@ -177,7 +177,7 @@ template class CompleteOrthogonalDecomposition * \code matrixT().template triangularView() \endcode * For rank-deficient matrices, use * \code - * matrixR().topLeftCorner(rank(), rank()).template triangularView() + * matrixT().topLeftCorner(rank(), rank()).template triangularView() * \endcode */ const MatrixType& matrixT() const { return m_cpqr.matrixQR(); } @@ -195,6 +195,21 @@ template class CompleteOrthogonalDecomposition return m_cpqr.colsPermutation(); } + /** \returns the determinant of the matrix of which + * *this is the complete orthogonal decomposition. It has only linear + * complexity (that is, O(n) where n is the dimension of the square matrix) + * as the complete orthogonal decomposition has already been computed. + * + * \note This is only for square matrices. + * + * \warning a determinant can be very big or small, so for matrices + * of large enough dimension, there is a risk of overflow/underflow. + * One way to work around that is to use logAbsDeterminant() instead. + * + * \sa absDeterminant(), logAbsDeterminant(), MatrixBase::determinant() + */ + typename MatrixType::Scalar determinant() const; + /** \returns the absolute value of the determinant of the matrix of which * *this is the complete orthogonal decomposition. It has only linear * complexity (that is, O(n) where n is the dimension of the square matrix) @@ -206,7 +221,7 @@ template class CompleteOrthogonalDecomposition * of large enough dimension, there is a risk of overflow/underflow. * One way to work around that is to use logAbsDeterminant() instead. * - * \sa logAbsDeterminant(), MatrixBase::determinant() + * \sa determinant(), logAbsDeterminant(), MatrixBase::determinant() */ typename MatrixType::RealScalar absDeterminant() const; @@ -221,7 +236,7 @@ template class CompleteOrthogonalDecomposition * \note This method is useful to work around the risk of overflow/underflow * that's inherent to determinant computation. * - * \sa absDeterminant(), MatrixBase::determinant() + * \sa determinant(), absDeterminant(), MatrixBase::determinant() */ typename MatrixType::RealScalar logAbsDeterminant() const; @@ -377,9 +392,7 @@ template class CompleteOrthogonalDecomposition #endif protected: - static void check_template_parameters() { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - } + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) template void _check_solve_assertion(const Rhs& b) const { @@ -402,20 +415,26 @@ template class CompleteOrthogonalDecomposition template void applyZAdjointOnTheLeftInPlace(Rhs& rhs) const; - ColPivHouseholderQR m_cpqr; + ColPivHouseholderQR m_cpqr; HCoeffsType m_zCoeffs; RowVectorType m_temp; }; -template +template +typename MatrixType::Scalar +CompleteOrthogonalDecomposition::determinant() const { + return m_cpqr.determinant(); +} + +template typename MatrixType::RealScalar -CompleteOrthogonalDecomposition::absDeterminant() const { +CompleteOrthogonalDecomposition::absDeterminant() const { return m_cpqr.absDeterminant(); } -template +template typename MatrixType::RealScalar -CompleteOrthogonalDecomposition::logAbsDeterminant() const { +CompleteOrthogonalDecomposition::logAbsDeterminant() const { return m_cpqr.logAbsDeterminant(); } @@ -426,13 +445,10 @@ CompleteOrthogonalDecomposition::logAbsDeterminant() const { * \sa class CompleteOrthogonalDecomposition, * CompleteOrthogonalDecomposition(const MatrixType&) */ -template -void CompleteOrthogonalDecomposition::computeInPlace() +template +void CompleteOrthogonalDecomposition::computeInPlace() { - check_template_parameters(); - - // the column permutation is stored as int indices, so just to be sure: - eigen_assert(m_cpqr.cols() <= NumTraits::highest()); + eigen_assert(m_cpqr.cols() <= NumTraits::highest()); const Index rank = m_cpqr.rank(); const Index cols = m_cpqr.cols(); @@ -484,9 +500,9 @@ void CompleteOrthogonalDecomposition::computeInPlace() } } -template +template template -void CompleteOrthogonalDecomposition::applyZOnTheLeftInPlace( +void CompleteOrthogonalDecomposition::applyZOnTheLeftInPlace( Rhs& rhs) const { const Index cols = this->cols(); const Index nrhs = rhs.cols(); @@ -506,9 +522,9 @@ void CompleteOrthogonalDecomposition::applyZOnTheLeftInPlace( } } -template +template template -void CompleteOrthogonalDecomposition::applyZAdjointOnTheLeftInPlace( +void CompleteOrthogonalDecomposition::applyZAdjointOnTheLeftInPlace( Rhs& rhs) const { const Index cols = this->cols(); const Index nrhs = rhs.cols(); @@ -529,9 +545,9 @@ void CompleteOrthogonalDecomposition::applyZAdjointOnTheLeftInPlace( } #ifndef EIGEN_PARSED_BY_DOXYGEN -template +template template -void CompleteOrthogonalDecomposition<_MatrixType>::_solve_impl( +void CompleteOrthogonalDecomposition::_solve_impl( const RhsType& rhs, DstType& dst) const { const Index rank = this->rank(); if (rank == 0) { @@ -561,9 +577,9 @@ void CompleteOrthogonalDecomposition<_MatrixType>::_solve_impl( dst = colsPermutation() * dst; } -template +template template -void CompleteOrthogonalDecomposition<_MatrixType>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +void CompleteOrthogonalDecomposition::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const { const Index rank = this->rank(); @@ -592,17 +608,17 @@ void CompleteOrthogonalDecomposition<_MatrixType>::_solve_impl_transposed(const namespace internal { -template -struct traits > > +template +struct traits > > : traits::PlainObject> { enum { Flags = 0 }; }; -template -struct Assignment >, internal::assign_op::Scalar>, Dense2Dense> +template +struct Assignment >, internal::assign_op::Scalar>, Dense2Dense> { - typedef CompleteOrthogonalDecomposition CodType; + typedef CompleteOrthogonalDecomposition CodType; typedef Inverse SrcXprType; static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op &) { @@ -614,9 +630,9 @@ struct Assignment -typename CompleteOrthogonalDecomposition::HouseholderSequenceType -CompleteOrthogonalDecomposition::householderQ() const { +template +typename CompleteOrthogonalDecomposition::HouseholderSequenceType +CompleteOrthogonalDecomposition::householderQ() const { return m_cpqr.householderQ(); } @@ -625,7 +641,8 @@ CompleteOrthogonalDecomposition::householderQ() const { * \sa class CompleteOrthogonalDecomposition */ template -const CompleteOrthogonalDecomposition::PlainObject> +template +const CompleteOrthogonalDecomposition::PlainObject, PermutationIndex> MatrixBase::completeOrthogonalDecomposition() const { return CompleteOrthogonalDecomposition(eval()); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/FullPivHouseholderQR.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/FullPivHouseholderQR.h index d0664a1d8c7..89e546c3ea7 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/FullPivHouseholderQR.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/FullPivHouseholderQR.h @@ -11,23 +11,26 @@ #ifndef EIGEN_FULLPIVOTINGHOUSEHOLDERQR_H #define EIGEN_FULLPIVOTINGHOUSEHOLDERQR_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template struct traits > - : traits<_MatrixType> +template struct traits > + : traits { typedef MatrixXpr XprKind; typedef SolverStorage StorageKind; - typedef int StorageIndex; + typedef PermutationIndex_ PermutationIndex; enum { Flags = 0 }; }; -template struct FullPivHouseholderQRMatrixQReturnType; +template struct FullPivHouseholderQRMatrixQReturnType; -template -struct traits > +template +struct traits > { typedef typename MatrixType::PlainObject ReturnType; }; @@ -40,7 +43,7 @@ struct traits > * * \brief Householder rank-revealing QR decomposition of a matrix with full pivoting * - * \tparam _MatrixType the type of the matrix of which we are computing the QR decomposition + * \tparam MatrixType_ the type of the matrix of which we are computing the QR decomposition * * This class performs a rank-revealing QR decomposition of a matrix \b A into matrices \b P, \b P', \b Q and \b R * such that @@ -57,26 +60,27 @@ struct traits > * * \sa MatrixBase::fullPivHouseholderQr() */ -template class FullPivHouseholderQR - : public SolverBase > +template class FullPivHouseholderQR + : public SolverBase > { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef SolverBase Base; friend class SolverBase; - + typedef PermutationIndex_ PermutationIndex; EIGEN_GENERIC_PUBLIC_INTERFACE(FullPivHouseholderQR) + enum { MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; - typedef internal::FullPivHouseholderQRMatrixQReturnType MatrixQReturnType; + typedef internal::FullPivHouseholderQRMatrixQReturnType MatrixQReturnType; typedef typename internal::plain_diag_type::type HCoeffsType; - typedef Matrix IntDiagSizeVectorType; - typedef PermutationMatrix PermutationType; + typedef Matrix IntDiagSizeVectorType; + typedef PermutationMatrix PermutationType; typedef typename internal::plain_row_type::type RowVectorType; typedef typename internal::plain_col_type::type ColVectorType; typedef typename MatrixType::PlainObject PlainObject; @@ -208,6 +212,21 @@ template class FullPivHouseholderQR return m_rows_transpositions; } + /** \returns the determinant of the matrix of which + * *this is the QR decomposition. It has only linear complexity + * (that is, O(n) where n is the dimension of the square matrix) + * as the QR decomposition has already been computed. + * + * \note This is only for square matrices. + * + * \warning a determinant can be very big or small, so for matrices + * of large enough dimension, there is a risk of overflow/underflow. + * One way to work around that is to use logAbsDeterminant() instead. + * + * \sa absDeterminant(), logAbsDeterminant(), MatrixBase::determinant() + */ + typename MatrixType::Scalar determinant() const; + /** \returns the absolute value of the determinant of the matrix of which * *this is the QR decomposition. It has only linear complexity * (that is, O(n) where n is the dimension of the square matrix) @@ -219,7 +238,7 @@ template class FullPivHouseholderQR * of large enough dimension, there is a risk of overflow/underflow. * One way to work around that is to use logAbsDeterminant() instead. * - * \sa logAbsDeterminant(), MatrixBase::determinant() + * \sa determinant(), logAbsDeterminant(), MatrixBase::determinant() */ typename MatrixType::RealScalar absDeterminant() const; @@ -233,7 +252,7 @@ template class FullPivHouseholderQR * \note This method is useful to work around the risk of overflow/underflow that's inherent * to determinant computation. * - * \sa absDeterminant(), MatrixBase::determinant() + * \sa determinant(), absDeterminant(), MatrixBase::determinant() */ typename MatrixType::RealScalar logAbsDeterminant() const; @@ -403,10 +422,7 @@ template class FullPivHouseholderQR protected: - static void check_template_parameters() - { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - } + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) void computeInPlace(); @@ -420,11 +436,21 @@ template class FullPivHouseholderQR RealScalar m_prescribedThreshold, m_maxpivot; Index m_nonzero_pivots; RealScalar m_precision; - Index m_det_pq; + Index m_det_p; }; -template -typename MatrixType::RealScalar FullPivHouseholderQR::absDeterminant() const +template +typename MatrixType::Scalar FullPivHouseholderQR::determinant() const +{ + eigen_assert(m_isInitialized && "HouseholderQR is not initialized."); + eigen_assert(m_qr.rows() == m_qr.cols() && "You can't take the determinant of a non-square matrix!"); + Scalar detQ; + internal::householder_determinant::IsComplex>::run(m_hCoeffs, detQ); + return m_qr.diagonal().prod() * detQ * Scalar(m_det_p); +} + +template +typename MatrixType::RealScalar FullPivHouseholderQR::absDeterminant() const { using std::abs; eigen_assert(m_isInitialized && "FullPivHouseholderQR is not initialized."); @@ -432,8 +458,8 @@ typename MatrixType::RealScalar FullPivHouseholderQR::absDeterminant return abs(m_qr.diagonal().prod()); } -template -typename MatrixType::RealScalar FullPivHouseholderQR::logAbsDeterminant() const +template +typename MatrixType::RealScalar FullPivHouseholderQR::logAbsDeterminant() const { eigen_assert(m_isInitialized && "FullPivHouseholderQR is not initialized."); eigen_assert(m_qr.rows() == m_qr.cols() && "You can't take the determinant of a non-square matrix!"); @@ -446,20 +472,19 @@ typename MatrixType::RealScalar FullPivHouseholderQR::logAbsDetermin * * \sa class FullPivHouseholderQR, FullPivHouseholderQR(const MatrixType&) */ -template +template template -FullPivHouseholderQR& FullPivHouseholderQR::compute(const EigenBase& matrix) +FullPivHouseholderQR& FullPivHouseholderQR::compute(const EigenBase& matrix) { m_qr = matrix.derived(); computeInPlace(); return *this; } -template -void FullPivHouseholderQR::computeInPlace() +template +void FullPivHouseholderQR::computeInPlace() { - check_template_parameters(); - + eigen_assert(m_qr.cols() <= NumTraits::highest()); using std::abs; Index rows = m_qr.rows(); Index cols = m_qr.cols(); @@ -501,15 +526,15 @@ void FullPivHouseholderQR::computeInPlace() m_nonzero_pivots = k; for(Index i = k; i < size; i++) { - m_rows_transpositions.coeffRef(i) = internal::convert_index(i); - m_cols_transpositions.coeffRef(i) = internal::convert_index(i); + m_rows_transpositions.coeffRef(i) = internal::convert_index(i); + m_cols_transpositions.coeffRef(i) = internal::convert_index(i); m_hCoeffs.coeffRef(i) = Scalar(0); } break; } - m_rows_transpositions.coeffRef(k) = internal::convert_index(row_of_biggest_in_corner); - m_cols_transpositions.coeffRef(k) = internal::convert_index(col_of_biggest_in_corner); + m_rows_transpositions.coeffRef(k) = internal::convert_index(row_of_biggest_in_corner); + m_cols_transpositions.coeffRef(k) = internal::convert_index(col_of_biggest_in_corner); if(k != row_of_biggest_in_corner) { m_qr.row(k).tail(cols-k).swap(m_qr.row(row_of_biggest_in_corner).tail(cols-k)); ++number_of_transpositions; @@ -534,14 +559,14 @@ void FullPivHouseholderQR::computeInPlace() for(Index k = 0; k < size; ++k) m_cols_permutation.applyTranspositionOnTheRight(k, m_cols_transpositions.coeff(k)); - m_det_pq = (number_of_transpositions%2) ? -1 : 1; + m_det_p = (number_of_transpositions%2) ? -1 : 1; m_isInitialized = true; } #ifndef EIGEN_PARSED_BY_DOXYGEN -template +template template -void FullPivHouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const +void FullPivHouseholderQR::_solve_impl(const RhsType &rhs, DstType &dst) const { const Index l_rank = rank(); @@ -573,9 +598,9 @@ void FullPivHouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType for(Index i = l_rank; i < cols(); ++i) dst.row(m_cols_permutation.indices().coeff(i)).setZero(); } -template +template template -void FullPivHouseholderQR<_MatrixType>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +void FullPivHouseholderQR::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const { const Index l_rank = rank(); @@ -612,10 +637,10 @@ void FullPivHouseholderQR<_MatrixType>::_solve_impl_transposed(const RhsType &rh namespace internal { -template -struct Assignment >, internal::assign_op::Scalar>, Dense2Dense> +template +struct Assignment >, internal::assign_op::Scalar>, Dense2Dense> { - typedef FullPivHouseholderQR QrType; + typedef FullPivHouseholderQR QrType; typedef Inverse SrcXprType; static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op &) { @@ -629,11 +654,11 @@ struct Assignment >, intern * * \tparam MatrixType type of underlying dense matrix */ -template struct FullPivHouseholderQRMatrixQReturnType - : public ReturnByValue > +template struct FullPivHouseholderQRMatrixQReturnType + : public ReturnByValue > { public: - typedef typename FullPivHouseholderQR::IntDiagSizeVectorType IntDiagSizeVectorType; + typedef typename FullPivHouseholderQR::IntDiagSizeVectorType IntDiagSizeVectorType; typedef typename internal::plain_diag_type::type HCoeffsType; typedef Matrix WorkVectorType; @@ -690,8 +715,8 @@ template struct FullPivHouseholderQRMatrixQReturnType } // end namespace internal -template -inline typename FullPivHouseholderQR::MatrixQReturnType FullPivHouseholderQR::matrixQ() const +template +inline typename FullPivHouseholderQR::MatrixQReturnType FullPivHouseholderQR::matrixQ() const { eigen_assert(m_isInitialized && "FullPivHouseholderQR is not initialized."); return MatrixQReturnType(m_qr, m_hCoeffs, m_rows_transpositions); @@ -702,10 +727,11 @@ inline typename FullPivHouseholderQR::MatrixQReturnType FullPivHouse * \sa class FullPivHouseholderQR */ template -const FullPivHouseholderQR::PlainObject> +template +const FullPivHouseholderQR::PlainObject, PermutationIndex> MatrixBase::fullPivHouseholderQr() const { - return FullPivHouseholderQR(eval()); + return FullPivHouseholderQR(eval()); } } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/HouseholderQR.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/HouseholderQR.h index 801739fbd81..68c69da3ce6 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/HouseholderQR.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/HouseholderQR.h @@ -12,11 +12,14 @@ #ifndef EIGEN_QR_H #define EIGEN_QR_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -template struct traits > - : traits<_MatrixType> +template struct traits > + : traits { typedef MatrixXpr XprKind; typedef SolverStorage StorageKind; @@ -33,7 +36,7 @@ template struct traits > * * \brief Householder QR decomposition of a matrix * - * \tparam _MatrixType the type of the matrix of which we are computing the QR decomposition + * \tparam MatrixType_ the type of the matrix of which we are computing the QR decomposition * * This class performs a QR decomposition of a matrix \b A into matrices \b Q and \b R * such that @@ -53,12 +56,12 @@ template struct traits > * * \sa MatrixBase::householderQr() */ -template class HouseholderQR - : public SolverBase > +template class HouseholderQR + : public SolverBase > { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef SolverBase Base; friend class SolverBase; @@ -70,7 +73,7 @@ template class HouseholderQR typedef Matrix MatrixQType; typedef typename internal::plain_diag_type::type HCoeffsType; typedef typename internal::plain_row_type::type RowVectorType; - typedef HouseholderSequence::type> HouseholderSequenceType; + typedef HouseholderSequence> HouseholderSequenceType; /** * \brief Default Constructor. @@ -182,6 +185,21 @@ template class HouseholderQR return *this; } + /** \returns the determinant of the matrix of which + * *this is the QR decomposition. It has only linear complexity + * (that is, O(n) where n is the dimension of the square matrix) + * as the QR decomposition has already been computed. + * + * \note This is only for square matrices. + * + * \warning a determinant can be very big or small, so for matrices + * of large enough dimension, there is a risk of overflow/underflow. + * One way to work around that is to use logAbsDeterminant() instead. + * + * \sa absDeterminant(), logAbsDeterminant(), MatrixBase::determinant() + */ + typename MatrixType::Scalar determinant() const; + /** \returns the absolute value of the determinant of the matrix of which * *this is the QR decomposition. It has only linear complexity * (that is, O(n) where n is the dimension of the square matrix) @@ -193,7 +211,7 @@ template class HouseholderQR * of large enough dimension, there is a risk of overflow/underflow. * One way to work around that is to use logAbsDeterminant() instead. * - * \sa logAbsDeterminant(), MatrixBase::determinant() + * \sa determinant(), logAbsDeterminant(), MatrixBase::determinant() */ typename MatrixType::RealScalar absDeterminant() const; @@ -207,7 +225,7 @@ template class HouseholderQR * \note This method is useful to work around the risk of overflow/underflow that's inherent * to determinant computation. * - * \sa absDeterminant(), MatrixBase::determinant() + * \sa determinant(), absDeterminant(), MatrixBase::determinant() */ typename MatrixType::RealScalar logAbsDeterminant() const; @@ -230,10 +248,7 @@ template class HouseholderQR protected: - static void check_template_parameters() - { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - } + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) void computeInPlace(); @@ -243,6 +258,57 @@ template class HouseholderQR bool m_isInitialized; }; +namespace internal { + +/** \internal */ +template +struct householder_determinant +{ + static void run(const HCoeffs& hCoeffs, Scalar& out_det) + { + out_det = Scalar(1); + Index size = hCoeffs.rows(); + for (Index i = 0; i < size; i ++) + { + // For each valid reflection Q_n, + // det(Q_n) = - conj(h_n) / h_n + // where h_n is the Householder coefficient. + if (hCoeffs(i) != Scalar(0)) + out_det *= - numext::conj(hCoeffs(i)) / hCoeffs(i); + } + } +}; + +/** \internal */ +template +struct householder_determinant +{ + static void run(const HCoeffs& hCoeffs, Scalar& out_det) + { + bool negated = false; + Index size = hCoeffs.rows(); + for (Index i = 0; i < size; i ++) + { + // Each valid reflection negates the determinant. + if (hCoeffs(i) != Scalar(0)) + negated ^= true; + } + out_det = negated ? Scalar(-1) : Scalar(1); + } +}; + +} // end namespace internal + +template +typename MatrixType::Scalar HouseholderQR::determinant() const +{ + eigen_assert(m_isInitialized && "HouseholderQR is not initialized."); + eigen_assert(m_qr.rows() == m_qr.cols() && "You can't take the determinant of a non-square matrix!"); + Scalar detQ; + internal::householder_determinant::IsComplex>::run(m_hCoeffs, detQ); + return m_qr.diagonal().prod() * detQ; +} + template typename MatrixType::RealScalar HouseholderQR::absDeterminant() const { @@ -297,6 +363,43 @@ void householder_qr_inplace_unblocked(MatrixQR& mat, HCoeffs& hCoeffs, typename } } +// TODO: add a corresponding public API for updating a QR factorization +/** \internal + * Basically a modified copy of @c Eigen::internal::householder_qr_inplace_unblocked that + * performs a rank-1 update of the QR matrix in compact storage. This function assumes, that + * the first @c k-1 columns of the matrix @c mat contain the QR decomposition of \f$A^N\f$ up to + * column k-1. Then the QR decomposition of the k-th column (given by @c newColumn) is computed by + * applying the k-1 Householder projectors on it and finally compute the projector \f$H_k\f$ of + * it. On exit the matrix @c mat and the vector @c hCoeffs contain the QR decomposition of the + * first k columns of \f$A^N\f$. The \a tempData argument must point to at least mat.cols() scalars. */ +template +void householder_qr_inplace_update(MatrixQR& mat, HCoeffs& hCoeffs, const VectorQR& newColumn, + typename MatrixQR::Index k, typename MatrixQR::Scalar* tempData) { + typedef typename MatrixQR::Index Index; + typedef typename MatrixQR::RealScalar RealScalar; + Index rows = mat.rows(); + + eigen_assert(k < mat.cols()); + eigen_assert(k < rows); + eigen_assert(hCoeffs.size() == mat.cols()); + eigen_assert(newColumn.size() == rows); + eigen_assert(tempData); + + // Store new column in mat at column k + mat.col(k) = newColumn; + // Apply H = H_1...H_{k-1} on newColumn (skip if k=0) + for (Index i = 0; i < k; ++i) { + Index remainingRows = rows - i; + mat.col(k) + .tail(remainingRows) + .applyHouseholderOnTheLeft(mat.col(i).tail(remainingRows - 1), hCoeffs.coeffRef(i), tempData + i + 1); + } + // Construct Householder projector in-place in column k + RealScalar beta; + mat.col(k).tail(rows - k).makeHouseholderInPlace(hCoeffs.coeffRef(k), beta); + mat.coeffRef(k, k) = beta; +} + /** \internal */ template +template template -void HouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const +void HouseholderQR::_solve_impl(const RhsType &rhs, DstType &dst) const { const Index rank = (std::min)(rows(), cols()); @@ -374,9 +477,9 @@ void HouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) c dst.bottomRows(cols()-rank).setZero(); } -template +template template -void HouseholderQR<_MatrixType>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +void HouseholderQR::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const { const Index rank = (std::min)(rows(), cols()); @@ -403,8 +506,6 @@ void HouseholderQR<_MatrixType>::_solve_impl_transposed(const RhsType &rhs, DstT template void HouseholderQR::computeInPlace() { - check_template_parameters(); - Index rows = m_qr.rows(); Index cols = m_qr.cols(); Index size = (std::min)(rows,cols); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/HouseholderQR_LAPACKE.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/HouseholderQR_LAPACKE.h index 1dc7d5363f1..0bd6a2b31ba 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/HouseholderQR_LAPACKE.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/HouseholderQR_LAPACKE.h @@ -34,32 +34,42 @@ #ifndef EIGEN_QR_LAPACKE_H #define EIGEN_QR_LAPACKE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { -/** \internal Specialization for the data types supported by LAPACKe */ +namespace lapacke_helpers { -#define EIGEN_LAPACKE_QR_NOPIV(EIGTYPE, LAPACKE_TYPE, LAPACKE_PREFIX) \ -template \ -struct householder_qr_inplace_blocked \ -{ \ - static void run(MatrixQR& mat, HCoeffs& hCoeffs, Index = 32, \ - typename MatrixQR::Scalar* = 0) \ - { \ - lapack_int m = (lapack_int) mat.rows(); \ - lapack_int n = (lapack_int) mat.cols(); \ - lapack_int lda = (lapack_int) mat.outerStride(); \ - lapack_int matrix_order = (MatrixQR::IsRowMajor) ? LAPACK_ROW_MAJOR : LAPACK_COL_MAJOR; \ - LAPACKE_##LAPACKE_PREFIX##geqrf( matrix_order, m, n, (LAPACKE_TYPE*)mat.data(), lda, (LAPACKE_TYPE*)hCoeffs.data()); \ - hCoeffs.adjointInPlace(); \ - } \ +template +struct lapacke_hqr +{ + static void run(MatrixQR& mat, HCoeffs& hCoeffs, Index = 32, typename MatrixQR::Scalar* = 0) + { + lapack_int m = to_lapack(mat.rows()); + lapack_int n = to_lapack(mat.cols()); + lapack_int lda = to_lapack(mat.outerStride()); + lapack_int matrix_order = lapack_storage_of(mat); + geqrf(matrix_order, m, n, to_lapack(mat.data()), lda, to_lapack(hCoeffs.data())); + hCoeffs.adjointInPlace(); + } }; -EIGEN_LAPACKE_QR_NOPIV(double, double, d) -EIGEN_LAPACKE_QR_NOPIV(float, float, s) -EIGEN_LAPACKE_QR_NOPIV(dcomplex, lapack_complex_double, z) -EIGEN_LAPACKE_QR_NOPIV(scomplex, lapack_complex_float, c) +} + +/** \internal Specialization for the data types supported by LAPACKe */ +#define EIGEN_LAPACKE_HH_QR(EIGTYPE) \ +template \ +struct householder_qr_inplace_blocked : public lapacke_helpers::lapacke_hqr {}; + +EIGEN_LAPACKE_HH_QR(double) +EIGEN_LAPACKE_HH_QR(float) +EIGEN_LAPACKE_HH_QR(std::complex) +EIGEN_LAPACKE_HH_QR(std::complex) + +#undef EIGEN_LAPACKE_HH_QR } // end namespace internal diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/InternalHeaderCheck.h new file mode 100644 index 00000000000..bf8df01c398 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/QR/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_QR_MODULE_H +#error "Please include Eigen/QR instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SPQRSupport/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SPQRSupport/InternalHeaderCheck.h new file mode 100644 index 00000000000..8d94ba4b609 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SPQRSupport/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_SPQRSUPPORT_MODULE_H +#error "Please include Eigen/SPQRSupport instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SPQRSupport/SuiteSparseQRSupport.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SPQRSupport/SuiteSparseQRSupport.h index 013c7ae7a9c..7256393d698 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SPQRSupport/SuiteSparseQRSupport.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SPQRSupport/SuiteSparseQRSupport.h @@ -11,6 +11,9 @@ #ifndef EIGEN_SUITESPARSEQRSUPPORT_H #define EIGEN_SUITESPARSEQRSUPPORT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template class SPQR; @@ -50,21 +53,21 @@ namespace Eigen { * R is the sparse triangular factor. Use matrixQR() to get it as SparseMatrix. * NOTE : The Index type of R is always SuiteSparse_long. You can get it with SPQR::Index * - * \tparam _MatrixType The type of the sparse matrix A, must be a column-major SparseMatrix<> + * \tparam MatrixType_ The type of the sparse matrix A, must be a column-major SparseMatrix<> * * \implsparsesolverconcept * * */ -template -class SPQR : public SparseSolverBase > +template +class SPQR : public SparseSolverBase > { protected: - typedef SparseSolverBase > Base; + typedef SparseSolverBase > Base; using Base::m_isInitialized; public: - typedef typename _MatrixType::Scalar Scalar; - typedef typename _MatrixType::RealScalar RealScalar; + typedef typename MatrixType_::Scalar Scalar; + typedef typename MatrixType_::RealScalar RealScalar; typedef SuiteSparse_long StorageIndex ; typedef SparseMatrix MatrixType; typedef Map > PermutationType; @@ -90,7 +93,7 @@ class SPQR : public SparseSolverBase > cholmod_l_start(&m_cc); } - explicit SPQR(const _MatrixType& matrix) + explicit SPQR(const MatrixType_& matrix) : m_analysisIsOk(false), m_factorizationIsOk(false), m_isRUpToDate(false), @@ -122,7 +125,7 @@ class SPQR : public SparseSolverBase > std::free(m_HPinv); } - void compute(const _MatrixType& matrix) + void compute(const MatrixType_& matrix) { if(m_isInitialized) SPQR_free(); @@ -137,7 +140,7 @@ class SPQR : public SparseSolverBase > { RealScalar max2Norm = 0.0; for (int j = 0; j < mat.cols(); j++) max2Norm = numext::maxi(max2Norm, mat.col(j).norm()); - if(max2Norm==RealScalar(0)) + if(numext::is_exactly_zero(max2Norm)) max2Norm = RealScalar(1); pivotThreshold = 20 * (mat.rows() + mat.cols()) * max2Norm * NumTraits::epsilon(); } @@ -258,12 +261,12 @@ class SPQR : public SparseSolverBase > int m_ordering; // Ordering method to use, see SPQR's manual int m_allow_tol; // Allow to use some tolerance during numerical factorization. RealScalar m_tolerance; // treat columns with 2-norm below this tolerance as zero - mutable cholmod_sparse *m_cR; // The sparse R factor in cholmod format + mutable cholmod_sparse *m_cR = nullptr; // The sparse R factor in cholmod format mutable MatrixType m_R; // The sparse matrix R in Eigen format - mutable StorageIndex *m_E; // The permutation applied to columns - mutable cholmod_sparse *m_H; //The householder vectors - mutable StorageIndex *m_HPinv; // The row permutation of H - mutable cholmod_dense *m_HTau; // The Householder coefficients + mutable StorageIndex *m_E = nullptr; // The permutation applied to columns + mutable cholmod_sparse *m_H = nullptr; //The householder vectors + mutable StorageIndex *m_HPinv = nullptr; // The row permutation of H + mutable cholmod_dense *m_HTau = nullptr; // The Householder coefficients mutable Index m_rank; // The rank of the matrix mutable cholmod_common m_cc; // Workspace and parameters bool m_useDefaultThreshold; // Use default threshold diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/BDCSVD.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/BDCSVD.h index a76a8dd04e8..a4ba8a7a480 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/BDCSVD.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/BDCSVD.h @@ -1,9 +1,9 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. -// +// // We used the "A Divide-And-Conquer Algorithm for the Bidiagonal SVD" // research report written by Ming Gu and Stanley C.Eisenstat -// The code variable names correspond to the names they used in their +// The code variable names correspond to the names they used in their // report // // Copyright (C) 2013 Gauthier Brun @@ -27,6 +27,9 @@ #define eigen_internal_assert(X) assert(X); #endif +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE #include #endif @@ -36,21 +39,39 @@ namespace Eigen { #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE IOFormat bdcsvdfmt(8, 0, ", ", "\n", " [", "]"); #endif - -template class BDCSVD; + +template +class BDCSVD; namespace internal { -template -struct traits > - : traits<_MatrixType> -{ - typedef _MatrixType MatrixType; -}; +template +struct traits > : svd_traits { + typedef MatrixType_ MatrixType; +}; + +template +struct allocate_small_svd { + static void run(JacobiSVD& smallSvd, Index rows, Index cols, unsigned int computationOptions) { + (void)computationOptions; + smallSvd = JacobiSVD(rows, cols); + } +}; + +EIGEN_DIAGNOSTICS(push) +EIGEN_DISABLE_DEPRECATED_WARNING + +template +struct allocate_small_svd { + static void run(JacobiSVD& smallSvd, Index rows, Index cols, unsigned int computationOptions) { + smallSvd = JacobiSVD(rows, cols, computationOptions); + } +}; + +EIGEN_DIAGNOSTICS(pop) } // end namespace internal - - + /** \ingroup SVD_Module * * @@ -58,7 +79,14 @@ struct traits > * * \brief class Bidiagonal Divide and Conquer SVD * - * \tparam _MatrixType the type of the matrix of which we are computing the SVD decomposition + * \tparam MatrixType_ the type of the matrix of which we are computing the SVD decomposition + * + * \tparam Options_ this optional parameter allows one to specify options for computing unitaries \a U and \a V. + * Possible values are #ComputeThinU, #ComputeThinV, #ComputeFullU, #ComputeFullV, and + * #DisableQRDecomposition. It is not possible to request both the thin and full version of \a U or + * \a V. By default, unitaries are not computed. BDCSVD uses R-Bidiagonalization to improve + * performance on tall and wide matrices. For backwards compatility, the option + * #DisableQRDecomposition can be used to disable this optimization. * * This class first reduces the input matrix to bi-diagonal form using class UpperBidiagonalization, * and then performs a divide-and-conquer diagonalization. Small blocks are diagonalized using class JacobiSVD. @@ -73,35 +101,39 @@ struct traits > * * \sa class JacobiSVD */ -template -class BDCSVD : public SVDBase > -{ +template +class BDCSVD : public SVDBase > { typedef SVDBase Base; - + public: using Base::rows; using Base::cols; + using Base::diagSize; using Base::computeU; using Base::computeV; - - typedef _MatrixType MatrixType; - typedef typename MatrixType::Scalar Scalar; - typedef typename NumTraits::Real RealScalar; + + typedef MatrixType_ MatrixType; + typedef typename Base::Scalar Scalar; + typedef typename Base::RealScalar RealScalar; typedef typename NumTraits::Literal Literal; + typedef typename Base::Index Index; enum { - RowsAtCompileTime = MatrixType::RowsAtCompileTime, - ColsAtCompileTime = MatrixType::ColsAtCompileTime, - DiagSizeAtCompileTime = EIGEN_SIZE_MIN_PREFER_DYNAMIC(RowsAtCompileTime, ColsAtCompileTime), - MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, - MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, - MaxDiagSizeAtCompileTime = EIGEN_SIZE_MIN_PREFER_FIXED(MaxRowsAtCompileTime, MaxColsAtCompileTime), - MatrixOptions = MatrixType::Options + Options = Options_, + QRDecomposition = Options & internal::QRPreconditionerBits, + ComputationOptions = Options & internal::ComputationOptionsBits, + RowsAtCompileTime = Base::RowsAtCompileTime, + ColsAtCompileTime = Base::ColsAtCompileTime, + DiagSizeAtCompileTime = Base::DiagSizeAtCompileTime, + MaxRowsAtCompileTime = Base::MaxRowsAtCompileTime, + MaxColsAtCompileTime = Base::MaxColsAtCompileTime, + MaxDiagSizeAtCompileTime = Base::MaxDiagSizeAtCompileTime, + MatrixOptions = Base::MatrixOptions }; typedef typename Base::MatrixUType MatrixUType; typedef typename Base::MatrixVType MatrixVType; typedef typename Base::SingularValuesType SingularValuesType; - + typedef Matrix MatrixX; typedef Matrix MatrixXr; typedef Matrix VectorType; @@ -118,70 +150,95 @@ class BDCSVD : public SVDBase > BDCSVD() : m_algoswap(16), m_isTranspose(false), m_compU(false), m_compV(false), m_numIters(0) {} + /** \brief Default Constructor with memory preallocation + * + * Like the default constructor but with preallocation of the internal data + * according to the specified problem size and \a Options template parameter. + * \sa BDCSVD() + */ + BDCSVD(Index rows, Index cols) : m_algoswap(16), m_numIters(0) { + allocate(rows, cols, internal::get_computation_options(Options)); + } /** \brief Default Constructor with memory preallocation * * Like the default constructor but with preallocation of the internal data - * according to the specified problem size. + * according to the specified problem size and the \a computationOptions. + * + * One \b cannot request unitiaries using both the \a Options template parameter + * and the constructor. If possible, prefer using the \a Options template parameter. + * + * \param computationOptions specifification for computing Thin/Full unitaries U/V * \sa BDCSVD() + * + * \deprecated Will be removed in the next major Eigen version. Options should + * be specified in the \a Options template parameter. */ - BDCSVD(Index rows, Index cols, unsigned int computationOptions = 0) - : m_algoswap(16), m_numIters(0) - { + EIGEN_DEPRECATED + BDCSVD(Index rows, Index cols, unsigned int computationOptions) : m_algoswap(16), m_numIters(0) { + internal::check_svd_options_assertions(computationOptions, rows, cols); allocate(rows, cols, computationOptions); } - /** \brief Constructor performing the decomposition of given matrix. + /** \brief Constructor performing the decomposition of given matrix, using the custom options specified + * with the \a Options template paramter. * * \param matrix the matrix to decompose - * \param computationOptions optional parameter allowing to specify if you want full or thin U or V unitaries to be computed. - * By default, none is computed. This is a bit - field, the possible bits are #ComputeFullU, #ComputeThinU, - * #ComputeFullV, #ComputeThinV. - * - * Thin unitaries are only available if your matrix type has a Dynamic number of columns (for example MatrixXf). They also are not - * available with the (non - default) FullPivHouseholderQR preconditioner. */ - BDCSVD(const MatrixType& matrix, unsigned int computationOptions = 0) - : m_algoswap(16), m_numIters(0) - { - compute(matrix, computationOptions); + BDCSVD(const MatrixType& matrix) : m_algoswap(16), m_numIters(0) { + compute_impl(matrix, internal::get_computation_options(Options)); } - ~BDCSVD() - { - } - - /** \brief Method performing the decomposition of given matrix using custom options. + /** \brief Constructor performing the decomposition of given matrix using specified options + * for computing unitaries. + * + * One \b cannot request unitiaries using both the \a Options template parameter + * and the constructor. If possible, prefer using the \a Options template parameter. * * \param matrix the matrix to decompose - * \param computationOptions optional parameter allowing to specify if you want full or thin U or V unitaries to be computed. - * By default, none is computed. This is a bit - field, the possible bits are #ComputeFullU, #ComputeThinU, - * #ComputeFullV, #ComputeThinV. + * \param computationOptions specifification for computing Thin/Full unitaries U/V * - * Thin unitaries are only available if your matrix type has a Dynamic number of columns (for example MatrixXf). They also are not - * available with the (non - default) FullPivHouseholderQR preconditioner. + * \deprecated Will be removed in the next major Eigen version. Options should + * be specified in the \a Options template parameter. */ - BDCSVD& compute(const MatrixType& matrix, unsigned int computationOptions); + EIGEN_DEPRECATED + BDCSVD(const MatrixType& matrix, unsigned int computationOptions) : m_algoswap(16), m_numIters(0) { + internal::check_svd_options_assertions(computationOptions, matrix.rows(), matrix.cols()); + compute_impl(matrix, computationOptions); + } + + ~BDCSVD() {} - /** \brief Method performing the decomposition of given matrix using current options. + /** \brief Method performing the decomposition of given matrix. Computes Thin/Full unitaries U/V if specified + * using the \a Options template parameter or the class constructor. * * \param matrix the matrix to decompose + */ + BDCSVD& compute(const MatrixType& matrix) { return compute_impl(matrix, m_computationOptions); } + + /** \brief Method performing the decomposition of given matrix, as specified by + * the `computationOptions` parameter. * - * This method uses the current \a computationOptions, as already passed to the constructor or to compute(const MatrixType&, unsigned int). + * \param matrix the matrix to decompose + * \param computationOptions specify whether to compute Thin/Full unitaries U/V + * + * \deprecated Will be removed in the next major Eigen version. Options should + * be specified in the \a Options template parameter. */ - BDCSVD& compute(const MatrixType& matrix) - { - return compute(matrix, this->m_computationOptions); + EIGEN_DEPRECATED + BDCSVD& compute(const MatrixType& matrix, unsigned int computationOptions) { + internal::check_svd_options_assertions(computationOptions, matrix.rows(), matrix.cols()); + return compute_impl(matrix, computationOptions); } - void setSwitchSize(int s) + void setSwitchSize(int s) { eigen_assert(s>=3 && "BDCSVD the size of the algo switch has to be at least 3."); m_algoswap = s; } - + private: - void allocate(Index rows, Index cols, unsigned int computationOptions); + BDCSVD& compute_impl(const MatrixType& matrix, unsigned int computationOptions); void divide(Index firstCol, Index lastCol, Index firstRowW, Index firstColW, Index shift); void computeSVDofM(Index firstCol, Index n, MatrixXr& U, VectorType& singVals, MatrixXr& V); void computeSingVals(const ArrayRef& col0, const ArrayRef& diag, const IndicesRef& perm, VectorType& singVals, ArrayRef shifts, ArrayRef mus); @@ -194,84 +251,106 @@ class BDCSVD : public SVDBase > void copyUV(const HouseholderU &householderU, const HouseholderV &householderV, const NaiveU &naiveU, const NaiveV &naivev); void structured_update(Block A, const MatrixXr &B, Index n1); static RealScalar secularEq(RealScalar x, const ArrayRef& col0, const ArrayRef& diag, const IndicesRef &perm, const ArrayRef& diagShifted, RealScalar shift); + template + void computeBaseCase(SVDType& svd, Index n, Index firstCol, Index firstRowW, Index firstColW, Index shift); -protected: + protected: + void allocate(Index rows, Index cols, unsigned int computationOptions); MatrixXr m_naiveU, m_naiveV; MatrixXr m_computed; Index m_nRec; ArrayXr m_workspace; ArrayXi m_workspaceI; int m_algoswap; - bool m_isTranspose, m_compU, m_compV; - - using Base::m_singularValues; - using Base::m_diagSize; - using Base::m_computeFullU; - using Base::m_computeFullV; + bool m_isTranspose, m_compU, m_compV, m_useQrDecomp; + JacobiSVD smallSvd; + HouseholderQR qrDecomp; + internal::UpperBidiagonalization bid; + MatrixX copyWorkspace; + MatrixX reducedTriangle; + + using Base::m_computationOptions; using Base::m_computeThinU; using Base::m_computeThinV; - using Base::m_matrixU; - using Base::m_matrixV; using Base::m_info; using Base::m_isInitialized; + using Base::m_matrixU; + using Base::m_matrixV; using Base::m_nonzeroSingularValues; + using Base::m_singularValues; -public: + public: int m_numIters; -}; //end class BDCSVD - +}; // end class BDCSVD // Method to allocate and initialize matrix and attributes -template -void BDCSVD::allocate(Eigen::Index rows, Eigen::Index cols, unsigned int computationOptions) -{ - m_isTranspose = (cols > rows); - +template +void BDCSVD::allocate(Index rows, Index cols, unsigned int computationOptions) { if (Base::allocate(rows, cols, computationOptions)) return; - - m_computed = MatrixXr::Zero(m_diagSize + 1, m_diagSize ); + + if (cols < m_algoswap) + internal::allocate_small_svd::run(smallSvd, rows, cols, computationOptions); + + m_computed = MatrixXr::Zero(diagSize() + 1, diagSize() ); m_compU = computeV(); m_compV = computeU(); + m_isTranspose = (cols > rows); if (m_isTranspose) std::swap(m_compU, m_compV); - - if (m_compU) m_naiveU = MatrixXr::Zero(m_diagSize + 1, m_diagSize + 1 ); - else m_naiveU = MatrixXr::Zero(2, m_diagSize + 1 ); - - if (m_compV) m_naiveV = MatrixXr::Zero(m_diagSize, m_diagSize); - - m_workspace.resize((m_diagSize+1)*(m_diagSize+1)*3); - m_workspaceI.resize(3*m_diagSize); -}// end allocate -template -BDCSVD& BDCSVD::compute(const MatrixType& matrix, unsigned int computationOptions) -{ + // kMinAspectRatio is the crossover point that determines if we perform R-Bidiagonalization + // or bidiagonalize the input matrix directly. + // It is based off of LAPACK's dgesdd routine, which uses 11.0/6.0 + // we use a larger scalar to prevent a regression for relatively square matrices. + constexpr Index kMinAspectRatio = 4; + constexpr bool disableQrDecomp = static_cast(QRDecomposition) == static_cast(DisableQRDecomposition); + m_useQrDecomp = !disableQrDecomp && ((rows / kMinAspectRatio > cols) || (cols / kMinAspectRatio > rows)); + if (m_useQrDecomp) { + qrDecomp = HouseholderQR((std::max)(rows, cols), (std::min)(rows, cols)); + reducedTriangle = MatrixX(diagSize(), diagSize()); + } + + copyWorkspace = MatrixX(m_isTranspose ? cols : rows, m_isTranspose ? rows : cols); + bid = internal::UpperBidiagonalization(m_useQrDecomp ? diagSize() : copyWorkspace.rows(), + m_useQrDecomp ? diagSize() : copyWorkspace.cols()); + + if (m_compU) m_naiveU = MatrixXr::Zero(diagSize() + 1, diagSize() + 1 ); + else m_naiveU = MatrixXr::Zero(2, diagSize() + 1 ); + + if (m_compV) m_naiveV = MatrixXr::Zero(diagSize(), diagSize()); + + m_workspace.resize((diagSize()+1)*(diagSize()+1)*3); + m_workspaceI.resize(3*diagSize()); +} // end allocate + +template +BDCSVD& BDCSVD::compute_impl(const MatrixType& matrix, + unsigned int computationOptions) { #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE std::cout << "\n\n\n======================================================================================================================\n\n\n"; #endif - allocate(matrix.rows(), matrix.cols(), computationOptions); using std::abs; + allocate(matrix.rows(), matrix.cols(), computationOptions); + const RealScalar considerZero = (std::numeric_limits::min)(); - + //**** step -1 - If the problem is too small, directly falls back to JacobiSVD and return if(matrix.cols() < m_algoswap) { - // FIXME this line involves temporaries - JacobiSVD jsvd(matrix,computationOptions); + smallSvd.compute(matrix); m_isInitialized = true; - m_info = jsvd.info(); + m_info = smallSvd.info(); if (m_info == Success || m_info == NoConvergence) { - if(computeU()) m_matrixU = jsvd.matrixU(); - if(computeV()) m_matrixV = jsvd.matrixV(); - m_singularValues = jsvd.singularValues(); - m_nonzeroSingularValues = jsvd.nonzeroSingularValues(); + if (computeU()) m_matrixU = smallSvd.matrixU(); + if (computeV()) m_matrixV = smallSvd.matrixV(); + m_singularValues = smallSvd.singularValues(); + m_nonzeroSingularValues = smallSvd.nonzeroSingularValues(); } return *this; } - + //**** step 0 - Copy the input matrix and apply scaling to reduce over/under-flows RealScalar scale = matrix.cwiseAbs().template maxCoeff(); if (!(numext::isfinite)(scale)) { @@ -280,75 +359,89 @@ BDCSVD& BDCSVD::compute(const MatrixType& matrix, unsign return *this; } - if(scale==Literal(0)) scale = Literal(1); - MatrixX copy; - if (m_isTranspose) copy = matrix.adjoint()/scale; - else copy = matrix/scale; - - //**** step 1 - Bidiagonalization - // FIXME this line involves temporaries - internal::UpperBidiagonalization bid(copy); + if(numext::is_exactly_zero(scale)) scale = Literal(1); + + if (m_isTranspose) copyWorkspace = matrix.adjoint() / scale; + else copyWorkspace = matrix / scale; + + //**** step 1 - Bidiagonalization. + // If the problem is sufficiently rectangular, we perform R-Bidiagonalization: compute A = Q(R/0) + // and then bidiagonalize R. Otherwise, if the problem is relatively square, we + // bidiagonalize the input matrix directly. + if (m_useQrDecomp) { + qrDecomp.compute(copyWorkspace); + reducedTriangle = qrDecomp.matrixQR().topRows(diagSize()); + reducedTriangle.template triangularView().setZero(); + bid.compute(reducedTriangle); + } else { + bid.compute(copyWorkspace); + } //**** step 2 - Divide & Conquer m_naiveU.setZero(); m_naiveV.setZero(); // FIXME this line involves a temporary matrix - m_computed.topRows(m_diagSize) = bid.bidiagonal().toDenseMatrix().transpose(); + m_computed.topRows(diagSize()) = bid.bidiagonal().toDenseMatrix().transpose(); m_computed.template bottomRows<1>().setZero(); - divide(0, m_diagSize - 1, 0, 0, 0); + divide(0, diagSize() - 1, 0, 0, 0); if (m_info != Success && m_info != NoConvergence) { m_isInitialized = true; return *this; } - + //**** step 3 - Copy singular values and vectors - for (int i=0; i -template -void BDCSVD::copyUV(const HouseholderU &householderU, const HouseholderV &householderV, const NaiveU &naiveU, const NaiveV &naiveV) -{ +template +template +void BDCSVD::copyUV(const HouseholderU& householderU, const HouseholderV& householderV, + const NaiveU& naiveU, const NaiveV& naiveV) { // Note exchange of U and V: m_matrixU is set from m_naiveV and vice versa if (computeU()) { - Index Ucols = m_computeThinU ? m_diagSize : householderU.cols(); - m_matrixU = MatrixX::Identity(householderU.cols(), Ucols); - m_matrixU.topLeftCorner(m_diagSize, m_diagSize) = naiveV.template cast().topLeftCorner(m_diagSize, m_diagSize); - householderU.applyThisOnTheLeft(m_matrixU); // FIXME this line involves a temporary buffer + Index Ucols = m_computeThinU ? diagSize() : rows(); + m_matrixU = MatrixX::Identity(rows(), Ucols); + m_matrixU.topLeftCorner(diagSize(), diagSize()) = naiveV.template cast().topLeftCorner(diagSize(), diagSize()); + // FIXME the following conditionals involve temporary buffers + if (m_useQrDecomp) m_matrixU.topLeftCorner(householderU.cols(), diagSize()).applyOnTheLeft(householderU); + else m_matrixU.applyOnTheLeft(householderU); } if (computeV()) { - Index Vcols = m_computeThinV ? m_diagSize : householderV.cols(); - m_matrixV = MatrixX::Identity(householderV.cols(), Vcols); - m_matrixV.topLeftCorner(m_diagSize, m_diagSize) = naiveU.template cast().topLeftCorner(m_diagSize, m_diagSize); - householderV.applyThisOnTheLeft(m_matrixV); // FIXME this line involves a temporary buffer + Index Vcols = m_computeThinV ? diagSize() : cols(); + m_matrixV = MatrixX::Identity(cols(), Vcols); + m_matrixV.topLeftCorner(diagSize(), diagSize()) = naiveU.template cast().topLeftCorner(diagSize(), diagSize()); + // FIXME the following conditionals involve temporary buffers + if (m_useQrDecomp) m_matrixV.topLeftCorner(householderV.cols(), diagSize()).applyOnTheLeft(householderV); + else m_matrixV.applyOnTheLeft(householderV); } } @@ -360,9 +453,8 @@ void BDCSVD::copyUV(const HouseholderU &householderU, const Househol * We can thus pack them prior to the the matrix product. However, this is only worth the effort if the matrix is large * enough. */ -template -void BDCSVD::structured_update(Block A, const MatrixXr &B, Index n1) -{ +template +void BDCSVD::structured_update(Block A, const MatrixXr& B, Index n1) { Index n = A.rows(); if(n>100) { @@ -389,7 +481,7 @@ void BDCSVD::structured_update(Block A, co ++k2; } } - + A.topRows(n1).noalias() = A1.leftCols(k1) * B1.topRows(k1); A.bottomRows(n2).noalias() = A2.leftCols(k2) * B2.topRows(k2); } @@ -401,19 +493,37 @@ void BDCSVD::structured_update(Block A, co } } -// The divide algorithm is done "in place", we are always working on subsets of the same matrix. The divide methods takes as argument the -// place of the submatrix we are currently working on. +template +template +void BDCSVD::computeBaseCase(SVDType& svd, Index n, Index firstCol, Index firstRowW, + Index firstColW, Index shift) { + svd.compute(m_computed.block(firstCol, firstCol, n + 1, n)); + m_info = svd.info(); + if (m_info != Success && m_info != NoConvergence) return; + if (m_compU) + m_naiveU.block(firstCol, firstCol, n + 1, n + 1).real() = svd.matrixU(); + else { + m_naiveU.row(0).segment(firstCol, n + 1).real() = svd.matrixU().row(0); + m_naiveU.row(1).segment(firstCol, n + 1).real() = svd.matrixU().row(n); + } + if (m_compV) m_naiveV.block(firstRowW, firstColW, n, n).real() = svd.matrixV(); + m_computed.block(firstCol + shift, firstCol + shift, n + 1, n).setZero(); + m_computed.diagonal().segment(firstCol + shift, n) = svd.singularValues().head(n); +} + +// The divide algorithm is done "in place", we are always working on subsets of the same matrix. The divide methods +// takes as argument the place of the submatrix we are currently working on. //@param firstCol : The Index of the first column of the submatrix of m_computed and for m_naiveU; -//@param lastCol : The Index of the last column of the submatrix of m_computed and for m_naiveU; +//@param lastCol : The Index of the last column of the submatrix of m_computed and for m_naiveU; // lastCol + 1 - firstCol is the size of the submatrix. //@param firstRowW : The Index of the first row of the matrix W that we are to change. (see the reference paper section 1 for more information on W) //@param firstColW : Same as firstRowW with the column. -//@param shift : Each time one takes the left submatrix, one must add 1 to the shift. Why? Because! We actually want the last column of the U submatrix +//@param shift : Each time one takes the left submatrix, one must add 1 to the shift. Why? Because! We actually want the last column of the U submatrix // to become the first column (*coeff) and to shift all the other columns to the right. There are more details on the reference paper. -template -void BDCSVD::divide(Eigen::Index firstCol, Eigen::Index lastCol, Eigen::Index firstRowW, Eigen::Index firstColW, Eigen::Index shift) -{ +template +void BDCSVD::divide(Index firstCol, Index lastCol, Index firstRowW, + Index firstColW, Index shift) { // requires rows = cols + 1; using std::pow; using std::sqrt; @@ -422,36 +532,30 @@ void BDCSVD::divide(Eigen::Index firstCol, Eigen::Index lastCol, Eig const Index k = n/2; const RealScalar considerZero = (std::numeric_limits::min)(); RealScalar alphaK; - RealScalar betaK; - RealScalar r0; + RealScalar betaK; + RealScalar r0; RealScalar lambda, phi, c0, s0; VectorType l, f; - // We use the other algorithm which is more efficient for small + // We use the other algorithm which is more efficient for small // matrices. if (n < m_algoswap) { - // FIXME this line involves temporaries - JacobiSVD b(m_computed.block(firstCol, firstCol, n + 1, n), ComputeFullU | (m_compV ? ComputeFullV : 0)); - m_info = b.info(); - if (m_info != Success && m_info != NoConvergence) return; - if (m_compU) - m_naiveU.block(firstCol, firstCol, n + 1, n + 1).real() = b.matrixU(); - else - { - m_naiveU.row(0).segment(firstCol, n + 1).real() = b.matrixU().row(0); - m_naiveU.row(1).segment(firstCol, n + 1).real() = b.matrixU().row(n); + // FIXME this block involves temporaries + if (m_compV) { + JacobiSVD baseSvd; + computeBaseCase(baseSvd, n, firstCol, firstRowW, firstColW, shift); + } else { + JacobiSVD baseSvd; + computeBaseCase(baseSvd, n, firstCol, firstRowW, firstColW, shift); } - if (m_compV) m_naiveV.block(firstRowW, firstColW, n, n).real() = b.matrixV(); - m_computed.block(firstCol + shift, firstCol + shift, n + 1, n).setZero(); - m_computed.diagonal().segment(firstCol + shift, n) = b.singularValues().head(n); return; } // We use the divide and conquer algorithm alphaK = m_computed(firstCol + k, firstCol + k); betaK = m_computed(firstCol + k + 1, firstCol + k); // The divide must be done in that order in order to have good results. Divide change the data inside the submatrices - // and the divide of the right submatrice reads one column of the left submatrice. That's why we need to treat the - // right submatrix before the left one. + // and the divide of the right submatrice reads one column of the left submatrice. That's why we need to treat the + // right submatrix before the left one. divide(k + 1 + firstCol, lastCol, k + 1 + firstRowW, k + 1 + firstColW, shift); if (m_info != Success && m_info != NoConvergence) return; divide(firstCol, k - 1 + firstCol, firstRowW, firstColW + 1, shift + 1); @@ -461,8 +565,8 @@ void BDCSVD::divide(Eigen::Index firstCol, Eigen::Index lastCol, Eig { lambda = m_naiveU(firstCol + k, firstCol + k); phi = m_naiveU(firstCol + k + 1, lastCol + 1); - } - else + } + else { lambda = m_naiveU(1, firstCol + k); phi = m_naiveU(0, lastCol + 1); @@ -472,8 +576,8 @@ void BDCSVD::divide(Eigen::Index firstCol, Eigen::Index lastCol, Eig { l = m_naiveU.row(firstCol + k).segment(firstCol, k); f = m_naiveU.row(firstCol + k + 1).segment(firstCol + k + 1, n - k - 1); - } - else + } + else { l = m_naiveU.row(1).segment(firstCol, k); f = m_naiveU.row(0).segment(firstCol + k + 1, n - k - 1); @@ -489,52 +593,52 @@ void BDCSVD::divide(Eigen::Index firstCol, Eigen::Index lastCol, Eig c0 = alphaK * lambda / r0; s0 = betaK * phi / r0; } - + #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(m_naiveU.allFinite()); - assert(m_naiveV.allFinite()); - assert(m_computed.allFinite()); + eigen_internal_assert(m_naiveU.allFinite()); + eigen_internal_assert(m_naiveV.allFinite()); + eigen_internal_assert(m_computed.allFinite()); #endif - + if (m_compU) { - MatrixXr q1 (m_naiveU.col(firstCol + k).segment(firstCol, k + 1)); + MatrixXr q1 (m_naiveU.col(firstCol + k).segment(firstCol, k + 1)); // we shiftW Q1 to the right - for (Index i = firstCol + k - 1; i >= firstCol; i--) + for (Index i = firstCol + k - 1; i >= firstCol; i--) m_naiveU.col(i + 1).segment(firstCol, k + 1) = m_naiveU.col(i).segment(firstCol, k + 1); // we shift q1 at the left with a factor c0 m_naiveU.col(firstCol).segment( firstCol, k + 1) = (q1 * c0); // last column = q1 * - s0 m_naiveU.col(lastCol + 1).segment(firstCol, k + 1) = (q1 * ( - s0)); // first column = q2 * s0 - m_naiveU.col(firstCol).segment(firstCol + k + 1, n - k) = m_naiveU.col(lastCol + 1).segment(firstCol + k + 1, n - k) * s0; + m_naiveU.col(firstCol).segment(firstCol + k + 1, n - k) = m_naiveU.col(lastCol + 1).segment(firstCol + k + 1, n - k) * s0; // q2 *= c0 m_naiveU.col(lastCol + 1).segment(firstCol + k + 1, n - k) *= c0; - } - else + } + else { RealScalar q1 = m_naiveU(0, firstCol + k); // we shift Q1 to the right - for (Index i = firstCol + k - 1; i >= firstCol; i--) + for (Index i = firstCol + k - 1; i >= firstCol; i--) m_naiveU(0, i + 1) = m_naiveU(0, i); // we shift q1 at the left with a factor c0 m_naiveU(0, firstCol) = (q1 * c0); // last column = q1 * - s0 m_naiveU(0, lastCol + 1) = (q1 * ( - s0)); // first column = q2 * s0 - m_naiveU(1, firstCol) = m_naiveU(1, lastCol + 1) *s0; + m_naiveU(1, firstCol) = m_naiveU(1, lastCol + 1) *s0; // q2 *= c0 m_naiveU(1, lastCol + 1) *= c0; m_naiveU.row(1).segment(firstCol + 1, k).setZero(); m_naiveU.row(0).segment(firstCol + k + 1, n - k - 1).setZero(); } - + #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(m_naiveU.allFinite()); - assert(m_naiveV.allFinite()); - assert(m_computed.allFinite()); + eigen_internal_assert(m_naiveU.allFinite()); + eigen_internal_assert(m_naiveV.allFinite()); + eigen_internal_assert(m_computed.allFinite()); #endif - + m_computed(firstCol + shift, firstCol + shift) = r0; m_computed.col(firstCol + shift).segment(firstCol + shift + 1, k) = alphaK * l.transpose().real(); m_computed.col(firstCol + shift).segment(firstCol + shift + k + 1, n - k - 1) = betaK * f.transpose().real(); @@ -551,21 +655,21 @@ void BDCSVD::divide(Eigen::Index firstCol, Eigen::Index lastCol, Eig std::cout << "err: " << ((tmp1-tmp2).abs()>1e-12*tmp2.abs()).transpose() << "\n"; static int count = 0; std::cout << "# " << ++count << "\n\n"; - assert((tmp1-tmp2).matrix().norm() < 1e-14*tmp2.matrix().norm()); -// assert(count<681); -// assert(((tmp1-tmp2).abs()<1e-13*tmp2.abs()).all()); + eigen_internal_assert((tmp1-tmp2).matrix().norm() < 1e-14*tmp2.matrix().norm()); +// eigen_internal_assert(count<681); +// eigen_internal_assert(((tmp1-tmp2).abs()<1e-13*tmp2.abs()).all()); #endif - + // Third part: compute SVD of combined matrix MatrixXr UofSVD, VofSVD; VectorType singVals; computeSVDofM(firstCol + shift, n, UofSVD, singVals, VofSVD); - + #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(UofSVD.allFinite()); - assert(VofSVD.allFinite()); + eigen_internal_assert(UofSVD.allFinite()); + eigen_internal_assert(VofSVD.allFinite()); #endif - + if (m_compU) structured_update(m_naiveU.block(firstCol, firstCol, n + 1, n + 1), UofSVD, (n+2)/2); else @@ -574,18 +678,18 @@ void BDCSVD::divide(Eigen::Index firstCol, Eigen::Index lastCol, Eig tmp.noalias() = m_naiveU.middleCols(firstCol, n+1) * UofSVD; m_naiveU.middleCols(firstCol, n + 1) = tmp; } - + if (m_compV) structured_update(m_naiveV.block(firstRowW, firstColW, n, n), VofSVD, (n+1)/2); - + #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(m_naiveU.allFinite()); - assert(m_naiveV.allFinite()); - assert(m_computed.allFinite()); + eigen_internal_assert(m_naiveU.allFinite()); + eigen_internal_assert(m_naiveV.allFinite()); + eigen_internal_assert(m_computed.allFinite()); #endif - + m_computed.block(firstCol + shift, firstCol + shift, n, n).setZero(); m_computed.block(firstCol + shift, firstCol + shift, n, n).diagonal() = singVals; -}// end divide +} // end divide // Compute SVD of m_computed.block(firstCol, firstCol, n + 1, n); this block only has non-zeros in // the first column and on the diagonal and has undergone deflation, so diagonal is in increasing @@ -595,9 +699,9 @@ void BDCSVD::divide(Eigen::Index firstCol, Eigen::Index lastCol, Eig // TODO Opportunities for optimization: better root finding algo, better stopping criterion, better // handling of round-off errors, be consistent in ordering // For instance, to solve the secular equation using FMM, see http://www.stat.uchicago.edu/~lekheng/courses/302/classics/greengard-rokhlin.pdf -template -void BDCSVD::computeSVDofM(Eigen::Index firstCol, Eigen::Index n, MatrixXr& U, VectorType& singVals, MatrixXr& V) -{ +template +void BDCSVD::computeSVDofM(Index firstCol, Index n, MatrixXr& U, + VectorType& singVals, MatrixXr& V) { const RealScalar considerZero = (std::numeric_limits::min)(); using std::abs; ArrayRef col0 = m_computed.col(firstCol).segment(firstCol, n); @@ -614,18 +718,21 @@ void BDCSVD::computeSVDofM(Eigen::Index firstCol, Eigen::Index n, Ma if (col0.hasNaN() || diag.hasNaN()) std::cout << "\n\nHAS NAN\n\n"; #endif - + // Many singular values might have been deflated, the zero ones have been moved to the end, // but others are interleaved and we must ignore them at this stage. // To this end, let's compute a permutation skipping them: Index actual_n = n; - while(actual_n>1 && diag(actual_n-1)==Literal(0)) {--actual_n; eigen_internal_assert(col0(actual_n)==Literal(0)); } + while(actual_n>1 && numext::is_exactly_zero(diag(actual_n - 1))) { + --actual_n; + eigen_internal_assert(numext::is_exactly_zero(col0(actual_n))); + } Index m = 0; // size of the deflated problem for(Index k=0;kconsiderZero) m_workspaceI(m++) = k; Map perm(m_workspaceI.data(),m); - + Map shifts(m_workspace.data()+1*n, n); Map mus(m_workspace.data()+2*n, n); Map zhat(m_workspace.data()+3*n, n); @@ -635,58 +742,58 @@ void BDCSVD::computeSVDofM(Eigen::Index firstCol, Eigen::Index n, Ma std::cout << " z: " << col0.transpose() << "\n"; std::cout << " d: " << diag.transpose() << "\n"; #endif - + // Compute singVals, shifts, and mus computeSingVals(col0, diag, perm, singVals, shifts, mus); - + #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE std::cout << " j: " << (m_computed.block(firstCol, firstCol, n, n)).jacobiSvd().singularValues().transpose().reverse() << "\n\n"; std::cout << " sing-val: " << singVals.transpose() << "\n"; std::cout << " mu: " << mus.transpose() << "\n"; std::cout << " shift: " << shifts.transpose() << "\n"; - + { std::cout << "\n\n mus: " << mus.head(actual_n).transpose() << "\n\n"; std::cout << " check1 (expect0) : " << ((singVals.array()-(shifts+mus)) / singVals.array()).head(actual_n).transpose() << "\n\n"; - assert((((singVals.array()-(shifts+mus)) / singVals.array()).head(actual_n) >= 0).all()); + eigen_internal_assert((((singVals.array()-(shifts+mus)) / singVals.array()).head(actual_n) >= 0).all()); std::cout << " check2 (>0) : " << ((singVals.array()-diag) / singVals.array()).head(actual_n).transpose() << "\n\n"; - assert((((singVals.array()-diag) / singVals.array()).head(actual_n) >= 0).all()); + eigen_internal_assert((((singVals.array()-diag) / singVals.array()).head(actual_n) >= 0).all()); } #endif - + #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(singVals.allFinite()); - assert(mus.allFinite()); - assert(shifts.allFinite()); + eigen_internal_assert(singVals.allFinite()); + eigen_internal_assert(mus.allFinite()); + eigen_internal_assert(shifts.allFinite()); #endif - + // Compute zhat perturbCol0(col0, diag, perm, singVals, shifts, mus, zhat); #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE std::cout << " zhat: " << zhat.transpose() << "\n"; #endif - + #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(zhat.allFinite()); + eigen_internal_assert(zhat.allFinite()); #endif - + computeSingVecs(zhat, diag, perm, singVals, shifts, mus, U, V); - + #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE std::cout << "U^T U: " << (U.transpose() * U - MatrixXr(MatrixXr::Identity(U.cols(),U.cols()))).norm() << "\n"; std::cout << "V^T V: " << (V.transpose() * V - MatrixXr(MatrixXr::Identity(V.cols(),V.cols()))).norm() << "\n"; #endif - + #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(m_naiveU.allFinite()); - assert(m_naiveV.allFinite()); - assert(m_computed.allFinite()); - assert(U.allFinite()); - assert(V.allFinite()); -// assert((U.transpose() * U - MatrixXr(MatrixXr::Identity(U.cols(),U.cols()))).norm() < 100*NumTraits::epsilon() * n); -// assert((V.transpose() * V - MatrixXr(MatrixXr::Identity(V.cols(),V.cols()))).norm() < 100*NumTraits::epsilon() * n); + eigen_internal_assert(m_naiveU.allFinite()); + eigen_internal_assert(m_naiveV.allFinite()); + eigen_internal_assert(m_computed.allFinite()); + eigen_internal_assert(U.allFinite()); + eigen_internal_assert(V.allFinite()); +// eigen_internal_assert((U.transpose() * U - MatrixXr(MatrixXr::Identity(U.cols(),U.cols()))).norm() < 100*NumTraits::epsilon() * n); +// eigen_internal_assert((V.transpose() * V - MatrixXr(MatrixXr::Identity(V.cols(),V.cols()))).norm() < 100*NumTraits::epsilon() * n); #endif - + // Because of deflation, the singular values might not be completely sorted. // Fortunately, reordering them is a O(n) problem for(Index i=0; i::computeSVDofM(Eigen::Index firstCol, Eigen::Index n, Ma bool singular_values_sorted = (((singVals.segment(1,actual_n-1)-singVals.head(actual_n-1))).array() >= 0).all(); if(!singular_values_sorted) std::cout << "Singular values are not sorted: " << singVals.segment(1,actual_n).transpose() << "\n"; - assert(singular_values_sorted); + eigen_internal_assert(singular_values_sorted); } #endif - + // Reverse order so that singular values in increased order // Because of deflation, the zeros singular-values are already at the end singVals.head(actual_n).reverseInPlace(); U.leftCols(actual_n).rowwise().reverseInPlace(); if (m_compV) V.leftCols(actual_n).rowwise().reverseInPlace(); - + #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE JacobiSVD jsvd(m_computed.block(firstCol, firstCol, n, n) ); std::cout << " * j: " << jsvd.singularValues().transpose() << "\n\n"; @@ -723,9 +830,10 @@ void BDCSVD::computeSVDofM(Eigen::Index firstCol, Eigen::Index n, Ma #endif } -template -typename BDCSVD::RealScalar BDCSVD::secularEq(RealScalar mu, const ArrayRef& col0, const ArrayRef& diag, const IndicesRef &perm, const ArrayRef& diagShifted, RealScalar shift) -{ +template +typename BDCSVD::RealScalar BDCSVD::secularEq( + RealScalar mu, const ArrayRef& col0, const ArrayRef& diag, const IndicesRef& perm, const ArrayRef& diagShifted, + RealScalar shift) { Index m = perm.size(); RealScalar res = Literal(1); for(Index i=0; i::RealScalar BDCSVD::secularEq(RealScalar res += (col0(j) / (diagShifted(j) - mu)) * (col0(j) / (diag(j) + shift + mu)); } return res; - } -template -void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& diag, const IndicesRef &perm, - VectorType& singVals, ArrayRef shifts, ArrayRef mus) -{ +template +void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& diag, const IndicesRef& perm, + VectorType& singVals, ArrayRef shifts, ArrayRef mus) { using std::abs; using std::swap; using std::sqrt; @@ -751,11 +857,11 @@ void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& d Index actual_n = n; // Note that here actual_n is computed based on col0(i)==0 instead of diag(i)==0 as above // because 1) we have diag(i)==0 => col0(i)==0 and 2) if col0(i)==0, then diag(i) is already a singular value. - while(actual_n>1 && col0(actual_n-1)==Literal(0)) --actual_n; + while(actual_n>1 && numext::is_exactly_zero(col0(actual_n - 1))) --actual_n; for (Index k = 0; k < n; ++k) { - if (col0(k) == Literal(0) || actual_n==1) + if (numext::is_exactly_zero(col0(k)) || actual_n == 1) { // if col0(k) == 0, then entry is deflated, so singular value is on diagonal // if actual_n==1, then the deflated problem is already diagonalized @@ -763,7 +869,7 @@ void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& d mus(k) = Literal(0); shifts(k) = k==0 ? col0(0) : diag(k); continue; - } + } // otherwise, use secular equation to find singular value RealScalar left = diag(k); @@ -776,7 +882,7 @@ void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& d // recall that at this stage we assume that z[j]!=0 and all entries for which z[j]==0 have been put aside. // This should be equivalent to using perm[] Index l = k+1; - while(col0(l)==Literal(0)) { ++l; eigen_internal_assert(l::computeSingVals(const ArrayRef& col0, const ArrayRef& d << " " << secularEq(left+RealScalar(0.999999)*(right-left), col0, diag, perm, diag, 0) << "\n"; #endif RealScalar shift = (k == actual_n-1 || fMid > Literal(0)) ? left : right; - + // measure everything relative to shift Map diagShifted(m_workspace.data()+4*n, n); diagShifted = diag - shift; @@ -811,7 +917,8 @@ void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& d { // check that after the shift, f(mid) is still negative: RealScalar midShifted = (right - left) / RealScalar(2); - if(shift==right) + // we can test exact equality here, because shift comes from `... ? left : right` + if(numext::equal_strict(shift, right)) midShifted = -midShifted; RealScalar fMidShifted = secularEq(midShifted, col0, diag, perm, diagShifted, shift); if(fMidShifted>0) @@ -821,10 +928,11 @@ void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& d diagShifted = diag - shift; } } - + // initial guess RealScalar muPrev, muCur; - if (shift == left) + // we can test exact equality here, because shift comes from `... ? left : right` + if (numext::equal_strict(shift, left)) { muPrev = (right - left) * RealScalar(0.1); if (k == actual_n-1) muCur = right - left; @@ -847,7 +955,7 @@ void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& d // rational interpolation: fit a function of the form a / mu + b through the two previous // iterates and use its zero to compute the next iterate bool useBisection = fPrev*fCur>Literal(0); - while (fCur!=Literal(0) && abs(muCur - muPrev) > Literal(8) * NumTraits::epsilon() * numext::maxi(abs(muCur), abs(muPrev)) && abs(fCur - fPrev)>NumTraits::epsilon() && !useBisection) + while (!numext::is_exactly_zero(fCur) && abs(muCur - muPrev) > Literal(8) * NumTraits::epsilon() * numext::maxi(abs(muCur), abs(muPrev)) && abs(fCur - fPrev) > NumTraits::epsilon() && !useBisection) { ++m_numIters; @@ -859,16 +967,17 @@ void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& d RealScalar fZero = secularEq(muZero, col0, diag, perm, diagShifted, shift); #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert((numext::isfinite)(fZero)); + eigen_internal_assert((numext::isfinite)(fZero)); #endif - + muPrev = muCur; fPrev = fCur; muCur = muZero; fCur = fZero; - - if (shift == left && (muCur < Literal(0) || muCur > right - left)) useBisection = true; - if (shift == right && (muCur < -(right - left) || muCur > Literal(0))) useBisection = true; + + // we can test exact equality here, because shift comes from `... ? left : right` + if (numext::equal_strict(shift, left) && (muCur < Literal(0) || muCur > right - left)) useBisection = true; + if (numext::equal_strict(shift, right) && (muCur < -(right - left) || muCur > Literal(0))) useBisection = true; if (abs(fCur)>abs(fPrev)) useBisection = true; } @@ -879,7 +988,8 @@ void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& d std::cout << "useBisection for k = " << k << ", actual_n = " << actual_n << "\n"; #endif RealScalar leftShifted, rightShifted; - if (shift == left) + // we can test exact equality here, because shift comes from `... ? left : right` + if (numext::equal_strict(shift, left)) { // to avoid overflow, we must have mu > max(real_min, |z(k)|/sqrt(real_max)), // the factor 2 is to be more conservative @@ -910,13 +1020,13 @@ void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& d #ifdef EIGEN_BDCSVD_SANITY_CHECKS if(!(numext::isfinite)(fLeft)) std::cout << "f(" << leftShifted << ") =" << fLeft << " ; " << left << " " << shift << " " << right << "\n"; - assert((numext::isfinite)(fLeft)); + eigen_internal_assert((numext::isfinite)(fLeft)); if(!(numext::isfinite)(fRight)) std::cout << "f(" << rightShifted << ") =" << fRight << " ; " << left << " " << shift << " " << right << "\n"; - // assert((numext::isfinite)(fRight)); + // eigen_internal_assert((numext::isfinite)(fRight)); #endif - + #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE if(!(fLeft * fRight<0)) { @@ -950,18 +1060,19 @@ void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& d } muCur = (leftShifted + rightShifted) / Literal(2); } - else + else { // We have a problem as shifting on the left or right give either a positive or negative value // at the middle of [left,right]... // Instead fo abbording or entering an infinite loop, // let's just use the middle as the estimated zero-crossing: muCur = (right - left) * RealScalar(0.5); - if(shift == right) + // we can test exact equality here, because shift comes from `... ? left : right` + if(numext::equal_strict(shift, right)) muCur = -muCur; } } - + singVals[k] = shift + muCur; shifts[k] = shift; mus[k] = muCur; @@ -971,25 +1082,23 @@ void BDCSVD::computeSingVals(const ArrayRef& col0, const ArrayRef& d std::cout << "found " << singVals[k] << " == " << shift << " + " << muCur << " from " << diag(k) << " .. " << diag(k+1) << "\n"; #endif #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(k==0 || singVals[k]>=singVals[k-1]); - assert(singVals[k]>=diag(k)); + eigen_internal_assert(k==0 || singVals[k]>=singVals[k-1]); + eigen_internal_assert(singVals[k]>=diag(k)); #endif // perturb singular value slightly if it equals diagonal entry to avoid division by zero later // (deflation is supposed to avoid this from happening) // - this does no seem to be necessary anymore - -// if (singVals[k] == left) singVals[k] *= 1 + NumTraits::epsilon(); -// if (singVals[k] == right) singVals[k] *= 1 - NumTraits::epsilon(); + // if (singVals[k] == left) singVals[k] *= 1 + NumTraits::epsilon(); + // if (singVals[k] == right) singVals[k] *= 1 - NumTraits::epsilon(); } } - // zhat is perturbation of col0 for which singular vectors can be computed stably (see Section 3.1) -template -void BDCSVD::perturbCol0 - (const ArrayRef& col0, const ArrayRef& diag, const IndicesRef &perm, const VectorType& singVals, - const ArrayRef& shifts, const ArrayRef& mus, ArrayRef zhat) -{ +template +void BDCSVD::perturbCol0(const ArrayRef& col0, const ArrayRef& diag, const IndicesRef& perm, + const VectorType& singVals, const ArrayRef& shifts, const ArrayRef& mus, + ArrayRef zhat) { using std::sqrt; Index n = col0.size(); Index m = perm.size(); @@ -1002,7 +1111,7 @@ void BDCSVD::perturbCol0 // The offset permits to skip deflated entries while computing zhat for (Index k = 0; k < n; ++k) { - if (col0(k) == Literal(0)) // deflated + if (numext::is_exactly_zero(col0(k))) // deflated zhat(k) = Literal(0); else { @@ -1015,7 +1124,7 @@ void BDCSVD::perturbCol0 std::cout << "prod = " << "(" << singVals(lastIdx) << " + " << dk << ") * (" << mus(lastIdx) << " + (" << shifts(lastIdx) << " - " << dk << "))" << "\n"; std::cout << " = " << singVals(lastIdx) + dk << " * " << mus(lastIdx) + (shifts(lastIdx) - dk) << "\n"; } - assert(prod>=0); + eigen_internal_assert(prod>=0); #endif for(Index l = 0; l::perturbCol0 std::cout << " " << "j=" << j << "\n"; } #endif - Index j = i= k && l == 0) { + m_info = NumericalIssue; + prod = 0; + break; + } + Index j = i 0 ? perm(l-1) : i; #ifdef EIGEN_BDCSVD_SANITY_CHECKS if(!(dk!=Literal(0) || diag(i)!=Literal(0))) { std::cout << "k=" << k << ", i=" << i << ", l=" << l << ", perm.size()=" << perm.size() << "\n"; } - assert(dk!=Literal(0) || diag(i)!=Literal(0)); + eigen_internal_assert(dk!=Literal(0) || diag(i)!=Literal(0)); #endif prod *= ((singVals(j)+dk) / ((diag(i)+dk))) * ((mus(j)+(shifts(j)-dk)) / ((diag(i)-dk))); #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(prod>=0); + eigen_internal_assert(prod>=0); #endif #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE if(i!=k && numext::abs(((singVals(j)+dk)*(mus(j)+(shifts(j)-dk)))/((diag(i)+dk)*(diag(i)-dk)) - 1) > 0.9 ) @@ -1057,7 +1173,7 @@ void BDCSVD::perturbCol0 #endif RealScalar tmp = sqrt(prod); #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert((numext::isfinite)(tmp)); + eigen_internal_assert((numext::isfinite)(tmp)); #endif zhat(k) = col0(k) > Literal(0) ? RealScalar(tmp) : RealScalar(-tmp); } @@ -1065,17 +1181,16 @@ void BDCSVD::perturbCol0 } // compute singular vectors -template -void BDCSVD::computeSingVecs - (const ArrayRef& zhat, const ArrayRef& diag, const IndicesRef &perm, const VectorType& singVals, - const ArrayRef& shifts, const ArrayRef& mus, MatrixXr& U, MatrixXr& V) -{ +template +void BDCSVD::computeSingVecs(const ArrayRef& zhat, const ArrayRef& diag, const IndicesRef& perm, + const VectorType& singVals, const ArrayRef& shifts, + const ArrayRef& mus, MatrixXr& U, MatrixXr& V) { Index n = zhat.size(); Index m = perm.size(); - + for (Index k = 0; k < n; ++k) { - if (zhat(k) == Literal(0)) + if (numext::is_exactly_zero(zhat(k))) { U.col(k) = VectorType::Unit(n+1, k); if (m_compV) V.col(k) = VectorType::Unit(n, k); @@ -1090,7 +1205,7 @@ void BDCSVD::computeSingVecs } U(n,k) = Literal(0); U.col(k).normalize(); - + if (m_compV) { V.col(k).setZero(); @@ -1107,13 +1222,12 @@ void BDCSVD::computeSingVecs U.col(n) = VectorType::Unit(n+1, n); } - // page 12_13 // i >= 1, di almost null and zi non null. // We use a rotation to zero out zi applied to the left of M -template -void BDCSVD::deflation43(Eigen::Index firstCol, Eigen::Index shift, Eigen::Index i, Eigen::Index size) -{ +template +void BDCSVD::deflation43(Index firstCol, Index shift, Index i, + Index size) { using std::abs; using std::sqrt; using std::pow; @@ -1121,28 +1235,28 @@ void BDCSVD::deflation43(Eigen::Index firstCol, Eigen::Index shift, RealScalar c = m_computed(start, start); RealScalar s = m_computed(start+i, start); RealScalar r = numext::hypot(c,s); - if (r == Literal(0)) + if (numext::is_exactly_zero(r)) { m_computed(start+i, start+i) = Literal(0); return; } - m_computed(start,start) = r; + m_computed(start,start) = r; m_computed(start+i, start) = Literal(0); m_computed(start+i, start+i) = Literal(0); - + JacobiRotation J(c/r,-s/r); if (m_compU) m_naiveU.middleRows(firstCol, size+1).applyOnTheRight(firstCol, firstCol+i, J); else m_naiveU.applyOnTheRight(firstCol, firstCol+i, J); -}// end deflation 43 - +} // end deflation 43 // page 13 // i,j >= 1, i!=j and |di - dj| < epsilon * norm2(M) // We apply two rotations to have zj = 0; // TODO deflation44 is still broken and not properly tested -template -void BDCSVD::deflation44(Eigen::Index firstColu , Eigen::Index firstColm, Eigen::Index firstRowW, Eigen::Index firstColW, Eigen::Index i, Eigen::Index j, Eigen::Index size) -{ +template +void BDCSVD::deflation44(Index firstColu, Index firstColm, Index firstRowW, + Index firstColW, Index i, Index j, + Index size) { using std::abs; using std::sqrt; using std::conj; @@ -1161,7 +1275,7 @@ void BDCSVD::deflation44(Eigen::Index firstColu , Eigen::Index first << m_computed(firstColm + i+1, firstColm+i+1) << " " << m_computed(firstColm + i+2, firstColm+i+2) << "\n"; #endif - if (r==Literal(0)) + if (numext::is_exactly_zero(r)) { m_computed(firstColm + i, firstColm + i) = m_computed(firstColm + j, firstColm + j); return; @@ -1176,39 +1290,38 @@ void BDCSVD::deflation44(Eigen::Index firstColu , Eigen::Index first if (m_compU) m_naiveU.middleRows(firstColu, size+1).applyOnTheRight(firstColu + i, firstColu + j, J); else m_naiveU.applyOnTheRight(firstColu+i, firstColu+j, J); if (m_compV) m_naiveV.middleRows(firstRowW, size).applyOnTheRight(firstColW + i, firstColW + j, J); -}// end deflation 44 - +} // end deflation 44 // acts on block from (firstCol+shift, firstCol+shift) to (lastCol+shift, lastCol+shift) [inclusive] -template -void BDCSVD::deflation(Eigen::Index firstCol, Eigen::Index lastCol, Eigen::Index k, Eigen::Index firstRowW, Eigen::Index firstColW, Eigen::Index shift) -{ +template +void BDCSVD::deflation(Index firstCol, Index lastCol, Index k, + Index firstRowW, Index firstColW, Index shift) { using std::sqrt; using std::abs; const Index length = lastCol + 1 - firstCol; - + Block col0(m_computed, firstCol+shift, firstCol+shift, length, 1); Diagonal fulldiag(m_computed); VectorBlock,Dynamic> diag(fulldiag, firstCol+shift, length); - + const RealScalar considerZero = (std::numeric_limits::min)(); RealScalar maxDiag = diag.tail((std::max)(Index(1),length-1)).cwiseAbs().maxCoeff(); RealScalar epsilon_strict = numext::maxi(considerZero,NumTraits::epsilon() * maxDiag); RealScalar epsilon_coarse = Literal(8) * NumTraits::epsilon() * numext::maxi(col0.cwiseAbs().maxCoeff(), maxDiag); - + #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(m_naiveU.allFinite()); - assert(m_naiveV.allFinite()); - assert(m_computed.allFinite()); + eigen_internal_assert(m_naiveU.allFinite()); + eigen_internal_assert(m_naiveV.allFinite()); + eigen_internal_assert(m_computed.allFinite()); #endif -#ifdef EIGEN_BDCSVD_DEBUG_VERBOSE +#ifdef EIGEN_BDCSVD_DEBUG_VERBOSE std::cout << "\ndeflate:" << diag.head(k+1).transpose() << " | " << diag.segment(k+1,length-k-1).transpose() << "\n"; #endif - + //condition 4.1 if (diag(0) < epsilon_coarse) - { + { #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE std::cout << "deflation 4.1, because " << diag(0) << " < " << epsilon_coarse << "\n"; #endif @@ -1236,31 +1349,31 @@ void BDCSVD::deflation(Eigen::Index firstCol, Eigen::Index lastCol, } #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(m_naiveU.allFinite()); - assert(m_naiveV.allFinite()); - assert(m_computed.allFinite()); + eigen_internal_assert(m_naiveU.allFinite()); + eigen_internal_assert(m_naiveV.allFinite()); + eigen_internal_assert(m_computed.allFinite()); #endif #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE std::cout << "to be sorted: " << diag.transpose() << "\n\n"; std::cout << " : " << col0.transpose() << "\n\n"; #endif { - // Check for total deflation + // Check for total deflation: // If we have a total deflation, then we have to consider col0(0)==diag(0) as a singular value during sorting. const bool total_deflation = (col0.tail(length-1).array().abs()::deflation(Eigen::Index firstCol, Eigen::Index lastCol, else permutation[p] = i++; } } - + // If we have a total deflation, then we have to insert diag(0) at the right place if(total_deflation) { @@ -1286,22 +1399,22 @@ void BDCSVD::deflation(Eigen::Index firstCol, Eigen::Index lastCol, } } } - + // Current index of each col, and current column of each index Index *realInd = m_workspaceI.data()+length; Index *realCol = m_workspaceI.data()+2*length; - + for(int pos = 0; pos< length; pos++) { realCol[pos] = pos; realInd[pos] = pos; } - + for(Index i = total_deflation?0:1; i < length; i++) { const Index pi = permutation[length - (total_deflation ? i+1 : i)]; const Index J = realCol[pi]; - + using std::swap; // swap diagonal and first column entries: swap(diag(i), diag(J)); @@ -1324,7 +1437,7 @@ void BDCSVD::deflation(Eigen::Index firstCol, Eigen::Index lastCol, std::cout << "sorted: " << diag.transpose().format(bdcsvdfmt) << "\n"; std::cout << " : " << col0.transpose() << "\n\n"; #endif - + //condition 4.4 { Index i = length-1; @@ -1339,18 +1452,18 @@ void BDCSVD::deflation(Eigen::Index firstCol, Eigen::Index lastCol, deflation44(firstCol, firstCol + shift, firstRowW, firstColW, i-1, i, length); } } - + #ifdef EIGEN_BDCSVD_SANITY_CHECKS for(Index j=2;j::deflation(Eigen::Index firstCol, Eigen::Index lastCol, * * \sa class BDCSVD */ -template -BDCSVD::PlainObject> -MatrixBase::bdcSvd(unsigned int computationOptions) const -{ - return BDCSVD(*this, computationOptions); +template +template +BDCSVD::PlainObject, Options> MatrixBase::bdcSvd() const { + return BDCSVD(*this); +} + +/** \svd_module + * + * \return the singular value decomposition of \c *this computed by Divide & Conquer algorithm + * + * \sa class BDCSVD + */ +template +template +BDCSVD::PlainObject, Options> MatrixBase::bdcSvd( + unsigned int computationOptions) const { + return BDCSVD(*this, computationOptions); } } // end namespace Eigen diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/BDCSVD_LAPACKE.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/BDCSVD_LAPACKE.h new file mode 100644 index 00000000000..9a1e8437d68 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/BDCSVD_LAPACKE.h @@ -0,0 +1,163 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2022 Melven Roehrig-Zoellner +// Copyright (c) 2011, Intel Corporation. All rights reserved. +// +// This file is based on the JacobiSVD_LAPACKE.h originally from Intel - +// see license notice below: +/* + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + ******************************************************************************** + * Content : Eigen bindings to LAPACKe + * Singular Value Decomposition - SVD (divide and conquer variant) + ******************************************************************************** +*/ +#ifndef EIGEN_BDCSVD_LAPACKE_H +#define EIGEN_BDCSVD_LAPACKE_H + +namespace Eigen { + +namespace internal { + +namespace lapacke_helpers { + +/** \internal Specialization for the data types supported by LAPACKe */ + +// defining a derived class to allow access to protected members +template +class BDCSVD_LAPACKE : public BDCSVD { + typedef BDCSVD SVD; + typedef typename SVD::MatrixType MatrixType; + typedef typename SVD::Scalar Scalar; + typedef typename SVD::RealScalar RealScalar; + +public: + // construct this by moving from a parent object + BDCSVD_LAPACKE(SVD&& svd) : SVD(std::move(svd)) {} + + void compute_impl_lapacke(const MatrixType& matrix, unsigned int computationOptions) { + + SVD::allocate(matrix.rows(), matrix.cols(), computationOptions); + + SVD::m_nonzeroSingularValues = SVD::m_diagSize; + + // prepare arguments to ?gesdd + const lapack_int matrix_order = lapack_storage_of(matrix); + const char jobz = (SVD::m_computeFullU || SVD::m_computeFullV) ? 'A' : (SVD::m_computeThinU || SVD::m_computeThinV) ? 'S' : 'N'; + const lapack_int u_cols = (jobz == 'A') ? to_lapack(SVD::rows()) : (jobz == 'S') ? to_lapack(SVD::diagSize()) : 1; + const lapack_int vt_rows = (jobz == 'A') ? to_lapack(SVD::cols()) : (jobz == 'S') ? to_lapack(SVD::diagSize()) : 1; + lapack_int ldu, ldvt; + Scalar *u, *vt, dummy; + MatrixType localU; + if (SVD::computeU() && !(SVD::m_computeThinU && SVD::m_computeFullV) ) { + ldu = to_lapack(SVD::m_matrixU.outerStride()); + u = SVD::m_matrixU.data(); + } else if (SVD::computeV()) { + localU.resize(SVD::rows(), u_cols); + ldu = to_lapack(localU.outerStride()); + u = localU.data(); + } else { ldu=1; u=&dummy; } + MatrixType localV; + if (SVD::computeU() || SVD::computeV()) { + localV.resize(vt_rows, SVD::cols()); + ldvt = to_lapack(localV.outerStride()); + vt = localV.data(); + } else { ldvt=1; vt=&dummy; } + MatrixType temp; temp = matrix; + + // actual call to ?gesdd + lapack_int info = gesdd( matrix_order, jobz, to_lapack(SVD::rows()), to_lapack(SVD::cols()), + to_lapack(temp.data()), to_lapack(temp.outerStride()), (RealScalar*)SVD::m_singularValues.data(), + to_lapack(u), ldu, to_lapack(vt), ldvt); + + // Check the result of the LAPACK call + if (info < 0 || !SVD::m_singularValues.allFinite()) { + // this includes info == -4 => NaN entry in A + SVD::m_info = InvalidInput; + } else if (info > 0 ) { + SVD::m_info = NoConvergence; + } else { + SVD::m_info = Success; + if (SVD::m_computeThinU && SVD::m_computeFullV) { + SVD::m_matrixU = localU.leftCols(SVD::m_matrixU.cols()); + } + if (SVD::computeV()) { + SVD::m_matrixV = localV.adjoint().leftCols(SVD::m_matrixV.cols()); + } + } + SVD::m_isInitialized = true; + } +}; + +template +BDCSVD& BDCSVD_wrapper(BDCSVD& svd, const MatrixType_& matrix, int computationOptions) +{ + // we need to move to the wrapper type and back + BDCSVD_LAPACKE tmpSvd(std::move(svd)); + tmpSvd.compute_impl_lapacke(matrix, computationOptions); + svd = std::move(tmpSvd); + return svd; +} + +} // end namespace lapacke_helpers + +} // end namespace internal + +#define EIGEN_LAPACKE_SDD(EIGTYPE, EIGCOLROW, OPTIONS) \ +template<> inline \ +BDCSVD, OPTIONS>& \ +BDCSVD, OPTIONS>::compute_impl(const Matrix& matrix, unsigned int computationOptions) {\ + return internal::lapacke_helpers::BDCSVD_wrapper(*this, matrix, computationOptions); \ +} + +#define EIGEN_LAPACK_SDD_OPTIONS(OPTIONS) \ + EIGEN_LAPACKE_SDD(double, ColMajor, OPTIONS) \ + EIGEN_LAPACKE_SDD(float, ColMajor, OPTIONS) \ + EIGEN_LAPACKE_SDD(dcomplex, ColMajor, OPTIONS) \ + EIGEN_LAPACKE_SDD(scomplex, ColMajor, OPTIONS) \ +\ + EIGEN_LAPACKE_SDD(double, RowMajor, OPTIONS) \ + EIGEN_LAPACKE_SDD(float, RowMajor, OPTIONS) \ + EIGEN_LAPACKE_SDD(dcomplex, RowMajor, OPTIONS) \ + EIGEN_LAPACKE_SDD(scomplex, RowMajor, OPTIONS) + +EIGEN_LAPACK_SDD_OPTIONS(0) +EIGEN_LAPACK_SDD_OPTIONS(ComputeThinU) +EIGEN_LAPACK_SDD_OPTIONS(ComputeThinV) +EIGEN_LAPACK_SDD_OPTIONS(ComputeFullU) +EIGEN_LAPACK_SDD_OPTIONS(ComputeFullV) +EIGEN_LAPACK_SDD_OPTIONS(ComputeThinU | ComputeThinV) +EIGEN_LAPACK_SDD_OPTIONS(ComputeFullU | ComputeFullV) +EIGEN_LAPACK_SDD_OPTIONS(ComputeThinU | ComputeFullV) +EIGEN_LAPACK_SDD_OPTIONS(ComputeFullU | ComputeThinV) + +#undef EIGEN_LAPACK_SDD_OPTIONS + +#undef EIGEN_LAPACKE_SDD + +} // end namespace Eigen + +#endif // EIGEN_BDCSVD_LAPACKE_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/InternalHeaderCheck.h new file mode 100644 index 00000000000..fa67b96acd4 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_SVD_MODULE_H +#error "Please include Eigen/SVD instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/JacobiSVD.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/JacobiSVD.h index 9d95acdf678..6b77d28a4fa 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/JacobiSVD.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/JacobiSVD.h @@ -11,13 +11,16 @@ #ifndef EIGEN_JACOBISVD_H #define EIGEN_JACOBISVD_H -namespace Eigen { +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { namespace internal { + // forward declaration (needed by ICC) // the empty body is required by MSVC -template::IsComplex> +template ::IsComplex> struct svd_precondition_2x2_block_to_be_real {}; /*** QR preconditioners (R-SVD) @@ -44,47 +47,40 @@ struct qr_preconditioner_should_do_anything }; }; -template::ret -> struct qr_preconditioner_impl {}; +template ::ret> +struct qr_preconditioner_impl {}; -template -class qr_preconditioner_impl -{ -public: - void allocate(const JacobiSVD&) {} - bool run(JacobiSVD&, const MatrixType&) - { - return false; - } +template +class qr_preconditioner_impl { + public: + void allocate(const JacobiSVD&) {} + bool run(JacobiSVD&, const MatrixType&) { return false; } }; /*** preconditioner using FullPivHouseholderQR ***/ -template -class qr_preconditioner_impl -{ -public: +template +class qr_preconditioner_impl { + public: typedef typename MatrixType::Scalar Scalar; - enum - { - RowsAtCompileTime = MatrixType::RowsAtCompileTime, - MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime - }; - typedef Matrix WorkspaceType; + typedef JacobiSVD SVDType; - void allocate(const JacobiSVD& svd) - { + enum { WorkspaceSize = MatrixType::RowsAtCompileTime, MaxWorkspaceSize = MatrixType::MaxRowsAtCompileTime }; + + typedef Matrix WorkspaceType; + + void allocate(const SVDType& svd) { if (svd.rows() != m_qr.rows() || svd.cols() != m_qr.cols()) { - m_qr.~QRType(); - ::new (&m_qr) QRType(svd.rows(), svd.cols()); + internal::destroy_at(&m_qr); + internal::construct_at(&m_qr, svd.rows(), svd.cols()); } if (svd.m_computeFullU) m_workspace.resize(svd.rows()); } - bool run(JacobiSVD& svd, const MatrixType& matrix) - { + bool run(SVDType& svd, const MatrixType& matrix) { if(matrix.rows() > matrix.cols()) { m_qr.compute(matrix); @@ -95,43 +91,43 @@ class qr_preconditioner_impl QRType; QRType m_qr; WorkspaceType m_workspace; }; -template -class qr_preconditioner_impl -{ -public: +template +class qr_preconditioner_impl { + public: typedef typename MatrixType::Scalar Scalar; - enum - { + typedef JacobiSVD SVDType; + + enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, - Options = MatrixType::Options + MatrixOptions = MatrixType::Options }; - typedef typename internal::make_proper_matrix_type< - Scalar, ColsAtCompileTime, RowsAtCompileTime, Options, MaxColsAtCompileTime, MaxRowsAtCompileTime - >::type TransposeTypeWithSameStorageOrder; + typedef typename internal::make_proper_matrix_type::type + TransposeTypeWithSameStorageOrder; - void allocate(const JacobiSVD& svd) - { + void allocate(const SVDType& svd) { if (svd.cols() != m_qr.rows() || svd.rows() != m_qr.cols()) { - m_qr.~QRType(); - ::new (&m_qr) QRType(svd.cols(), svd.rows()); + internal::destroy_at(&m_qr); + internal::construct_at(&m_qr, svd.cols(), svd.rows()); } m_adjoint.resize(svd.cols(), svd.rows()); if (svd.m_computeFullV) m_workspace.resize(svd.cols()); } - bool run(JacobiSVD& svd, const MatrixType& matrix) - { + bool run(SVDType& svd, const MatrixType& matrix) { if(matrix.cols() > matrix.rows()) { m_adjoint = matrix.adjoint(); @@ -143,32 +139,41 @@ class qr_preconditioner_impl QRType; QRType m_qr; TransposeTypeWithSameStorageOrder m_adjoint; - typename internal::plain_row_type::type m_workspace; + typename plain_row_type::type m_workspace; }; /*** preconditioner using ColPivHouseholderQR ***/ -template -class qr_preconditioner_impl -{ -public: - void allocate(const JacobiSVD& svd) - { +template +class qr_preconditioner_impl { + public: + typedef typename MatrixType::Scalar Scalar; + typedef JacobiSVD SVDType; + + enum { + WorkspaceSize = internal::traits::MatrixUColsAtCompileTime, + MaxWorkspaceSize = internal::traits::MatrixUMaxColsAtCompileTime + }; + + typedef Matrix WorkspaceType; + + void allocate(const SVDType& svd) { if (svd.rows() != m_qr.rows() || svd.cols() != m_qr.cols()) { - m_qr.~QRType(); - ::new (&m_qr) QRType(svd.rows(), svd.cols()); + internal::destroy_at(&m_qr); + internal::construct_at(&m_qr, svd.rows(), svd.cols()); } if (svd.m_computeFullU) m_workspace.resize(svd.rows()); else if (svd.m_computeThinU) m_workspace.resize(svd.cols()); } - bool run(JacobiSVD& svd, const MatrixType& matrix) - { + bool run(SVDType& svd, const MatrixType& matrix) { if(matrix.rows() > matrix.cols()) { m_qr.compute(matrix); @@ -188,41 +193,44 @@ class qr_preconditioner_impl QRType; QRType m_qr; - typename internal::plain_col_type::type m_workspace; + WorkspaceType m_workspace; }; -template -class qr_preconditioner_impl -{ -public: +template +class qr_preconditioner_impl { + public: typedef typename MatrixType::Scalar Scalar; - enum - { + typedef JacobiSVD SVDType; + + enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, - Options = MatrixType::Options + MatrixOptions = MatrixType::Options, + WorkspaceSize = internal::traits::MatrixVColsAtCompileTime, + MaxWorkspaceSize = internal::traits::MatrixVMaxColsAtCompileTime }; - typedef typename internal::make_proper_matrix_type< - Scalar, ColsAtCompileTime, RowsAtCompileTime, Options, MaxColsAtCompileTime, MaxRowsAtCompileTime - >::type TransposeTypeWithSameStorageOrder; + typedef Matrix WorkspaceType; - void allocate(const JacobiSVD& svd) - { + typedef typename internal::make_proper_matrix_type::type + TransposeTypeWithSameStorageOrder; + + void allocate(const SVDType& svd) { if (svd.cols() != m_qr.rows() || svd.rows() != m_qr.cols()) { - m_qr.~QRType(); - ::new (&m_qr) QRType(svd.cols(), svd.rows()); + internal::destroy_at(&m_qr); + internal::construct_at(&m_qr, svd.cols(), svd.rows()); } if (svd.m_computeFullV) m_workspace.resize(svd.cols()); else if (svd.m_computeThinV) m_workspace.resize(svd.rows()); m_adjoint.resize(svd.cols(), svd.rows()); } - bool run(JacobiSVD& svd, const MatrixType& matrix) - { + bool run(SVDType& svd, const MatrixType& matrix) { if(matrix.cols() > matrix.rows()) { m_adjoint = matrix.adjoint(); @@ -245,28 +253,35 @@ class qr_preconditioner_impl QRType; QRType m_qr; TransposeTypeWithSameStorageOrder m_adjoint; - typename internal::plain_row_type::type m_workspace; + WorkspaceType m_workspace; }; /*** preconditioner using HouseholderQR ***/ -template -class qr_preconditioner_impl -{ -public: - void allocate(const JacobiSVD& svd) - { +template +class qr_preconditioner_impl { + public: + typedef typename MatrixType::Scalar Scalar; + typedef JacobiSVD SVDType; + + enum { + WorkspaceSize = internal::traits::MatrixUColsAtCompileTime, + MaxWorkspaceSize = internal::traits::MatrixUMaxColsAtCompileTime + }; + + typedef Matrix WorkspaceType; + + void allocate(const SVDType& svd) { if (svd.rows() != m_qr.rows() || svd.cols() != m_qr.cols()) { - m_qr.~QRType(); - ::new (&m_qr) QRType(svd.rows(), svd.cols()); + internal::destroy_at(&m_qr); + internal::construct_at(&m_qr, svd.rows(), svd.cols()); } if (svd.m_computeFullU) m_workspace.resize(svd.rows()); else if (svd.m_computeThinU) m_workspace.resize(svd.cols()); } - bool run(JacobiSVD& svd, const MatrixType& matrix) - { + bool run(SVDType& svd, const MatrixType& matrix) { if(matrix.rows() > matrix.cols()) { m_qr.compute(matrix); @@ -282,44 +297,47 @@ class qr_preconditioner_impl QRType; QRType m_qr; - typename internal::plain_col_type::type m_workspace; + WorkspaceType m_workspace; }; -template -class qr_preconditioner_impl -{ -public: +template +class qr_preconditioner_impl { + public: typedef typename MatrixType::Scalar Scalar; - enum - { + typedef JacobiSVD SVDType; + + enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, - Options = MatrixType::Options + MatrixOptions = MatrixType::Options, + WorkspaceSize = internal::traits::MatrixVColsAtCompileTime, + MaxWorkspaceSize = internal::traits::MatrixVMaxColsAtCompileTime }; - typedef typename internal::make_proper_matrix_type< - Scalar, ColsAtCompileTime, RowsAtCompileTime, Options, MaxColsAtCompileTime, MaxRowsAtCompileTime - >::type TransposeTypeWithSameStorageOrder; + typedef Matrix WorkspaceType; - void allocate(const JacobiSVD& svd) - { + typedef typename internal::make_proper_matrix_type::type + TransposeTypeWithSameStorageOrder; + + void allocate(const SVDType& svd) { if (svd.cols() != m_qr.rows() || svd.rows() != m_qr.cols()) { - m_qr.~QRType(); - ::new (&m_qr) QRType(svd.cols(), svd.rows()); + internal::destroy_at(&m_qr); + internal::construct_at(&m_qr, svd.cols(), svd.rows()); } if (svd.m_computeFullV) m_workspace.resize(svd.cols()); else if (svd.m_computeThinV) m_workspace.resize(svd.rows()); m_adjoint.resize(svd.cols(), svd.rows()); } - bool run(JacobiSVD& svd, const MatrixType& matrix) - { + bool run(SVDType& svd, const MatrixType& matrix) { if(matrix.cols() > matrix.rows()) { m_adjoint = matrix.adjoint(); @@ -342,7 +360,7 @@ class qr_preconditioner_impl QRType; QRType m_qr; TransposeTypeWithSameStorageOrder m_adjoint; - typename internal::plain_row_type::type m_workspace; + WorkspaceType m_workspace; }; /*** 2x2 SVD implementation @@ -350,18 +368,16 @@ class qr_preconditioner_impl -struct svd_precondition_2x2_block_to_be_real -{ - typedef JacobiSVD SVD; +template +struct svd_precondition_2x2_block_to_be_real { + typedef JacobiSVD SVD; typedef typename MatrixType::RealScalar RealScalar; static bool run(typename SVD::WorkMatrixType&, SVD&, Index, Index, RealScalar&) { return true; } }; -template -struct svd_precondition_2x2_block_to_be_real -{ - typedef JacobiSVD SVD; +template +struct svd_precondition_2x2_block_to_be_real { + typedef JacobiSVD SVD; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; static bool run(typename SVD::WorkMatrixType& work_matrix, SVD& svd, Index p, Index q, RealScalar& maxDiagEntry) @@ -375,7 +391,7 @@ struct svd_precondition_2x2_block_to_be_real const RealScalar considerAsZero = (std::numeric_limits::min)(); const RealScalar precision = NumTraits::epsilon(); - if(n==0) + if(numext::is_exactly_zero(n)) { // make sure first column is zero work_matrix.coeffRef(p,p) = work_matrix.coeffRef(q,p) = Scalar(0); @@ -423,249 +439,256 @@ struct svd_precondition_2x2_block_to_be_real } }; -template -struct traits > - : traits<_MatrixType> -{ - typedef _MatrixType MatrixType; +template +struct traits > : svd_traits { + typedef MatrixType_ MatrixType; }; } // end namespace internal /** \ingroup SVD_Module - * - * - * \class JacobiSVD - * - * \brief Two-sided Jacobi SVD decomposition of a rectangular matrix - * - * \tparam _MatrixType the type of the matrix of which we are computing the SVD decomposition - * \tparam QRPreconditioner this optional parameter allows to specify the type of QR decomposition that will be used internally - * for the R-SVD step for non-square matrices. See discussion of possible values below. - * - * SVD decomposition consists in decomposing any n-by-p matrix \a A as a product - * \f[ A = U S V^* \f] - * where \a U is a n-by-n unitary, \a V is a p-by-p unitary, and \a S is a n-by-p real positive matrix which is zero outside of its main diagonal; - * the diagonal entries of S are known as the \em singular \em values of \a A and the columns of \a U and \a V are known as the left - * and right \em singular \em vectors of \a A respectively. - * - * Singular values are always sorted in decreasing order. - * - * This JacobiSVD decomposition computes only the singular values by default. If you want \a U or \a V, you need to ask for them explicitly. - * - * You can ask for only \em thin \a U or \a V to be computed, meaning the following. In case of a rectangular n-by-p matrix, letting \a m be the - * smaller value among \a n and \a p, there are only \a m singular vectors; the remaining columns of \a U and \a V do not correspond to actual - * singular vectors. Asking for \em thin \a U or \a V means asking for only their \a m first columns to be formed. So \a U is then a n-by-m matrix, - * and \a V is then a p-by-m matrix. Notice that thin \a U and \a V are all you need for (least squares) solving. - * - * Here's an example demonstrating basic usage: - * \include JacobiSVD_basic.cpp - * Output: \verbinclude JacobiSVD_basic.out - * - * This JacobiSVD class is a two-sided Jacobi R-SVD decomposition, ensuring optimal reliability and accuracy. The downside is that it's slower than - * bidiagonalizing SVD algorithms for large square matrices; however its complexity is still \f$ O(n^2p) \f$ where \a n is the smaller dimension and - * \a p is the greater dimension, meaning that it is still of the same order of complexity as the faster bidiagonalizing R-SVD algorithms. - * In particular, like any R-SVD, it takes advantage of non-squareness in that its complexity is only linear in the greater dimension. - * - * If the input matrix has inf or nan coefficients, the result of the computation is undefined, but the computation is guaranteed to - * terminate in finite (and reasonable) time. - * - * The possible values for QRPreconditioner are: - * \li ColPivHouseholderQRPreconditioner is the default. In practice it's very safe. It uses column-pivoting QR. - * \li FullPivHouseholderQRPreconditioner, is the safest and slowest. It uses full-pivoting QR. - * Contrary to other QRs, it doesn't allow computing thin unitaries. - * \li HouseholderQRPreconditioner is the fastest, and less safe and accurate than the pivoting variants. It uses non-pivoting QR. - * This is very similar in safety and accuracy to the bidiagonalization process used by bidiagonalizing SVD algorithms (since bidiagonalization - * is inherently non-pivoting). However the resulting SVD is still more reliable than bidiagonalizing SVDs because the Jacobi-based iterarive - * process is more reliable than the optimized bidiagonal SVD iterations. - * \li NoQRPreconditioner allows not to use a QR preconditioner at all. This is useful if you know that you will only be computing - * JacobiSVD decompositions of square matrices. Non-square matrices require a QR preconditioner. Using this option will result in - * faster compilation and smaller executable code. It won't significantly speed up computation, since JacobiSVD is always checking - * if QR preconditioning is needed before applying it anyway. - * - * \sa MatrixBase::jacobiSvd() - */ -template class JacobiSVD - : public SVDBase > -{ - typedef SVDBase Base; - public: - - typedef _MatrixType MatrixType; - typedef typename MatrixType::Scalar Scalar; - typedef typename NumTraits::Real RealScalar; - enum { - RowsAtCompileTime = MatrixType::RowsAtCompileTime, - ColsAtCompileTime = MatrixType::ColsAtCompileTime, - DiagSizeAtCompileTime = EIGEN_SIZE_MIN_PREFER_DYNAMIC(RowsAtCompileTime,ColsAtCompileTime), - MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, - MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, - MaxDiagSizeAtCompileTime = EIGEN_SIZE_MIN_PREFER_FIXED(MaxRowsAtCompileTime,MaxColsAtCompileTime), - MatrixOptions = MatrixType::Options - }; - - typedef typename Base::MatrixUType MatrixUType; - typedef typename Base::MatrixVType MatrixVType; - typedef typename Base::SingularValuesType SingularValuesType; - - typedef typename internal::plain_row_type::type RowType; - typedef typename internal::plain_col_type::type ColType; - typedef Matrix - WorkMatrixType; - - /** \brief Default Constructor. - * - * The default constructor is useful in cases in which the user intends to - * perform decompositions via JacobiSVD::compute(const MatrixType&). - */ - JacobiSVD() - {} - - - /** \brief Default Constructor with memory preallocation - * - * Like the default constructor but with preallocation of the internal data - * according to the specified problem size. - * \sa JacobiSVD() - */ - JacobiSVD(Index rows, Index cols, unsigned int computationOptions = 0) - { - allocate(rows, cols, computationOptions); - } + * + * + * \class JacobiSVD + * + * \brief Two-sided Jacobi SVD decomposition of a rectangular matrix + * + * \tparam MatrixType_ the type of the matrix of which we are computing the SVD decomposition + * \tparam Options this optional parameter allows one to specify the type of QR decomposition that will be used + * internally for the R-SVD step for non-square matrices. Additionally, it allows one to specify whether to compute thin + * or full unitaries \a U and \a V. See discussion of possible values below. + * + * SVD decomposition consists in decomposing any n-by-p matrix \a A as a product + * \f[ A = U S V^* \f] + * where \a U is a n-by-n unitary, \a V is a p-by-p unitary, and \a S is a n-by-p real positive matrix which is zero + * outside of its main diagonal; the diagonal entries of S are known as the \em singular \em values of \a A and the + * columns of \a U and \a V are known as the left and right \em singular \em vectors of \a A respectively. + * + * Singular values are always sorted in decreasing order. + * + * This JacobiSVD decomposition computes only the singular values by default. If you want \a U or \a V, you need to ask + * for them explicitly. + * + * You can ask for only \em thin \a U or \a V to be computed, meaning the following. In case of a rectangular n-by-p + * matrix, letting \a m be the smaller value among \a n and \a p, there are only \a m singular vectors; the remaining + * columns of \a U and \a V do not correspond to actual singular vectors. Asking for \em thin \a U or \a V means asking + * for only their \a m first columns to be formed. So \a U is then a n-by-m matrix, and \a V is then a p-by-m matrix. + * Notice that thin \a U and \a V are all you need for (least squares) solving. + * + * Here's an example demonstrating basic usage: + * \include JacobiSVD_basic.cpp + * Output: \verbinclude JacobiSVD_basic.out + * + * This JacobiSVD class is a two-sided Jacobi R-SVD decomposition, ensuring optimal reliability and accuracy. The + * downside is that it's slower than bidiagonalizing SVD algorithms for large square matrices; however its complexity is + * still \f$ O(n^2p) \f$ where \a n is the smaller dimension and \a p is the greater dimension, meaning that it is still + * of the same order of complexity as the faster bidiagonalizing R-SVD algorithms. In particular, like any R-SVD, it + * takes advantage of non-squareness in that its complexity is only linear in the greater dimension. + * + * If the input matrix has inf or nan coefficients, the result of the computation is undefined, but the computation is + * guaranteed to terminate in finite (and reasonable) time. + * + * The possible QR preconditioners that can be set with Options template parameter are: + * \li ColPivHouseholderQRPreconditioner is the default. In practice it's very safe. It uses column-pivoting QR. + * \li FullPivHouseholderQRPreconditioner, is the safest and slowest. It uses full-pivoting QR. + * Contrary to other QRs, it doesn't allow computing thin unitaries. + * \li HouseholderQRPreconditioner is the fastest, and less safe and accurate than the pivoting variants. It uses + * non-pivoting QR. This is very similar in safety and accuracy to the bidiagonalization process used by bidiagonalizing + * SVD algorithms (since bidiagonalization is inherently non-pivoting). However the resulting SVD is still more reliable + * than bidiagonalizing SVDs because the Jacobi-based iterarive process is more reliable than the optimized bidiagonal + * SVD iterations. \li NoQRPreconditioner allows not to use a QR preconditioner at all. This is useful if you know that + * you will only be computing JacobiSVD decompositions of square matrices. Non-square matrices require a QR + * preconditioner. Using this option will result in faster compilation and smaller executable code. It won't + * significantly speed up computation, since JacobiSVD is always checking if QR preconditioning is needed before + * applying it anyway. + * + * One may also use the Options template parameter to specify how the unitaries should be computed. The options are + * #ComputeThinU, #ComputeThinV, #ComputeFullU, #ComputeFullV. It is not possible to request both the thin and full + * versions of a unitary. By default, unitaries will not be computed. + * + * You can set the QRPreconditioner and unitary options together: JacobiSVD + * + * \sa MatrixBase::jacobiSvd() + */ +template +class JacobiSVD : public SVDBase > { + typedef SVDBase Base; + + public: + typedef MatrixType_ MatrixType; + typedef typename Base::Scalar Scalar; + typedef typename Base::RealScalar RealScalar; + typedef typename Base::Index Index; + enum : int { + Options = Options_, + QRPreconditioner = internal::get_qr_preconditioner(Options), + RowsAtCompileTime = Base::RowsAtCompileTime, + ColsAtCompileTime = Base::ColsAtCompileTime, + DiagSizeAtCompileTime = Base::DiagSizeAtCompileTime, + MaxRowsAtCompileTime = Base::MaxRowsAtCompileTime, + MaxColsAtCompileTime = Base::MaxColsAtCompileTime, + MaxDiagSizeAtCompileTime = Base::MaxDiagSizeAtCompileTime, + MatrixOptions = Base::MatrixOptions + }; - /** \brief Constructor performing the decomposition of given matrix. - * - * \param matrix the matrix to decompose - * \param computationOptions optional parameter allowing to specify if you want full or thin U or V unitaries to be computed. - * By default, none is computed. This is a bit-field, the possible bits are #ComputeFullU, #ComputeThinU, - * #ComputeFullV, #ComputeThinV. - * - * Thin unitaries are only available if your matrix type has a Dynamic number of columns (for example MatrixXf). They also are not - * available with the (non-default) FullPivHouseholderQR preconditioner. - */ - explicit JacobiSVD(const MatrixType& matrix, unsigned int computationOptions = 0) - { - compute(matrix, computationOptions); - } + typedef typename Base::MatrixUType MatrixUType; + typedef typename Base::MatrixVType MatrixVType; + typedef typename Base::SingularValuesType SingularValuesType; + typedef Matrix + WorkMatrixType; + + /** \brief Default Constructor. + * + * The default constructor is useful in cases in which the user intends to + * perform decompositions via JacobiSVD::compute(const MatrixType&). + */ + JacobiSVD() {} + + /** \brief Default Constructor with memory preallocation + * + * Like the default constructor but with preallocation of the internal data + * according to the specified problem size and \a Options template parameter. + * + * \sa JacobiSVD() + */ + JacobiSVD(Index rows, Index cols) { allocate(rows, cols, internal::get_computation_options(Options)); } + + /** \brief Default Constructor with memory preallocation + * + * Like the default constructor but with preallocation of the internal data + * according to the specified problem size. + * + * One \b cannot request unitaries using both the \a Options template parameter + * and the constructor. If possible, prefer using the \a Options template parameter. + * + * \param computationOptions specify whether to compute Thin/Full unitaries U/V + * \sa JacobiSVD() + * + * \deprecated Will be removed in the next major Eigen version. Options should + * be specified in the \a Options template parameter. + */ + EIGEN_DEPRECATED + JacobiSVD(Index rows, Index cols, unsigned int computationOptions) { + internal::check_svd_options_assertions(computationOptions, rows, cols); + allocate(rows, cols, computationOptions); + } - /** \brief Method performing the decomposition of given matrix using custom options. - * - * \param matrix the matrix to decompose - * \param computationOptions optional parameter allowing to specify if you want full or thin U or V unitaries to be computed. - * By default, none is computed. This is a bit-field, the possible bits are #ComputeFullU, #ComputeThinU, - * #ComputeFullV, #ComputeThinV. - * - * Thin unitaries are only available if your matrix type has a Dynamic number of columns (for example MatrixXf). They also are not - * available with the (non-default) FullPivHouseholderQR preconditioner. - */ - JacobiSVD& compute(const MatrixType& matrix, unsigned int computationOptions); - - /** \brief Method performing the decomposition of given matrix using current options. - * - * \param matrix the matrix to decompose - * - * This method uses the current \a computationOptions, as already passed to the constructor or to compute(const MatrixType&, unsigned int). - */ - JacobiSVD& compute(const MatrixType& matrix) - { - return compute(matrix, m_computationOptions); - } + /** \brief Constructor performing the decomposition of given matrix, using the custom options specified + * with the \a Options template paramter. + * + * \param matrix the matrix to decompose + */ + explicit JacobiSVD(const MatrixType& matrix) { compute_impl(matrix, internal::get_computation_options(Options)); } + + /** \brief Constructor performing the decomposition of given matrix using specified options + * for computing unitaries. + * + * One \b cannot request unitiaries using both the \a Options template parameter + * and the constructor. If possible, prefer using the \a Options template parameter. + * + * \param matrix the matrix to decompose + * \param computationOptions specify whether to compute Thin/Full unitaries U/V + * + * \deprecated Will be removed in the next major Eigen version. Options should + * be specified in the \a Options template parameter. + */ + // EIGEN_DEPRECATED // TODO(cantonios): re-enable after fixing a few 3p libraries that error on deprecation warnings. + JacobiSVD(const MatrixType& matrix, unsigned int computationOptions) { + internal::check_svd_options_assertions(computationOptions, matrix.rows(), matrix.cols()); + compute_impl(matrix, computationOptions); + } + + /** \brief Method performing the decomposition of given matrix. Computes Thin/Full unitaries U/V if specified + * using the \a Options template parameter or the class constructor. + * + * \param matrix the matrix to decompose + */ + JacobiSVD& compute(const MatrixType& matrix) { return compute_impl(matrix, m_computationOptions); } + + /** \brief Method performing the decomposition of given matrix, as specified by + * the `computationOptions` parameter. + * + * \param matrix the matrix to decompose + * \param computationOptions specify whether to compute Thin/Full unitaries U/V + * + * \deprecated Will be removed in the next major Eigen version. Options should + * be specified in the \a Options template parameter. + */ + EIGEN_DEPRECATED + JacobiSVD& compute(const MatrixType& matrix, unsigned int computationOptions) { + internal::check_svd_options_assertions(m_computationOptions, matrix.rows(), matrix.cols()); + return compute_impl(matrix, computationOptions); + } - using Base::computeU; - using Base::computeV; - using Base::rows; - using Base::cols; - using Base::rank; - - private: - void allocate(Index rows, Index cols, unsigned int computationOptions); - - protected: - using Base::m_matrixU; - using Base::m_matrixV; - using Base::m_singularValues; - using Base::m_info; - using Base::m_isInitialized; - using Base::m_isAllocated; - using Base::m_usePrescribedThreshold; - using Base::m_computeFullU; - using Base::m_computeThinU; - using Base::m_computeFullV; - using Base::m_computeThinV; - using Base::m_computationOptions; - using Base::m_nonzeroSingularValues; - using Base::m_rows; - using Base::m_cols; - using Base::m_diagSize; - using Base::m_prescribedThreshold; - WorkMatrixType m_workMatrix; - - template - friend struct internal::svd_precondition_2x2_block_to_be_real; - template - friend struct internal::qr_preconditioner_impl; - - internal::qr_preconditioner_impl m_qr_precond_morecols; - internal::qr_preconditioner_impl m_qr_precond_morerows; - MatrixType m_scaledMatrix; + using Base::computeU; + using Base::computeV; + using Base::rows; + using Base::cols; + using Base::diagSize; + using Base::rank; + + private: + void allocate(Index rows, Index cols, unsigned int computationOptions); + JacobiSVD& compute_impl(const MatrixType& matrix, unsigned int computationOptions); + + protected: + using Base::m_computationOptions; + using Base::m_computeFullU; + using Base::m_computeFullV; + using Base::m_computeThinU; + using Base::m_computeThinV; + using Base::m_info; + using Base::m_isAllocated; + using Base::m_isInitialized; + using Base::m_matrixU; + using Base::m_matrixV; + using Base::m_nonzeroSingularValues; + using Base::m_prescribedThreshold; + using Base::m_singularValues; + using Base::m_usePrescribedThreshold; + using Base::ShouldComputeThinU; + using Base::ShouldComputeThinV; + + EIGEN_STATIC_ASSERT(!(ShouldComputeThinU && int(QRPreconditioner) == int(FullPivHouseholderQRPreconditioner)) && + !(ShouldComputeThinU && int(QRPreconditioner) == int(FullPivHouseholderQRPreconditioner)), + "JacobiSVD: can't compute thin U or thin V with the FullPivHouseholderQR preconditioner. " + "Use the ColPivHouseholderQR preconditioner instead.") + + template + friend struct internal::svd_precondition_2x2_block_to_be_real; + template + friend struct internal::qr_preconditioner_impl; + + internal::qr_preconditioner_impl + m_qr_precond_morecols; + internal::qr_preconditioner_impl + m_qr_precond_morerows; + WorkMatrixType m_workMatrix; + MatrixType m_scaledMatrix; }; -template -void JacobiSVD::allocate(Eigen::Index rows, Eigen::Index cols, unsigned int computationOptions) -{ - eigen_assert(rows >= 0 && cols >= 0); +template +void JacobiSVD::allocate(Index rows_, Index cols_, unsigned int computationOptions_) { + if (Base::allocate(rows_, cols_, computationOptions_)) return; - if (m_isAllocated && - rows == m_rows && - cols == m_cols && - computationOptions == m_computationOptions) - { - return; - } + eigen_assert(!(ShouldComputeThinU && int(QRPreconditioner) == int(FullPivHouseholderQRPreconditioner)) && + !(ShouldComputeThinU && int(QRPreconditioner) == int(FullPivHouseholderQRPreconditioner)) && + "JacobiSVD: can't compute thin U or thin V with the FullPivHouseholderQR preconditioner. " + "Use the ColPivHouseholderQR preconditioner instead."); - m_rows = rows; - m_cols = cols; - m_info = Success; - m_isInitialized = false; - m_isAllocated = true; - m_computationOptions = computationOptions; - m_computeFullU = (computationOptions & ComputeFullU) != 0; - m_computeThinU = (computationOptions & ComputeThinU) != 0; - m_computeFullV = (computationOptions & ComputeFullV) != 0; - m_computeThinV = (computationOptions & ComputeThinV) != 0; - eigen_assert(!(m_computeFullU && m_computeThinU) && "JacobiSVD: you can't ask for both full and thin U"); - eigen_assert(!(m_computeFullV && m_computeThinV) && "JacobiSVD: you can't ask for both full and thin V"); - eigen_assert(EIGEN_IMPLIES(m_computeThinU || m_computeThinV, MatrixType::ColsAtCompileTime==Dynamic) && - "JacobiSVD: thin U and V are only available when your matrix has a dynamic number of columns."); - if (QRPreconditioner == FullPivHouseholderQRPreconditioner) - { - eigen_assert(!(m_computeThinU || m_computeThinV) && - "JacobiSVD: can't compute thin U or thin V with the FullPivHouseholderQR preconditioner. " - "Use the ColPivHouseholderQR preconditioner instead."); - } - m_diagSize = (std::min)(m_rows, m_cols); - m_singularValues.resize(m_diagSize); - if(RowsAtCompileTime==Dynamic) - m_matrixU.resize(m_rows, m_computeFullU ? m_rows - : m_computeThinU ? m_diagSize - : 0); - if(ColsAtCompileTime==Dynamic) - m_matrixV.resize(m_cols, m_computeFullV ? m_cols - : m_computeThinV ? m_diagSize - : 0); - m_workMatrix.resize(m_diagSize, m_diagSize); - - if(m_cols>m_rows) m_qr_precond_morecols.allocate(*this); - if(m_rows>m_cols) m_qr_precond_morerows.allocate(*this); - if(m_rows!=m_cols) m_scaledMatrix.resize(rows,cols); + m_workMatrix.resize(diagSize(), diagSize()); + if(cols()>rows()) m_qr_precond_morecols.allocate(*this); + if(rows()>cols()) m_qr_precond_morerows.allocate(*this); + if(rows()!=cols()) m_scaledMatrix.resize(rows(),cols()); } -template -JacobiSVD& -JacobiSVD::compute(const MatrixType& matrix, unsigned int computationOptions) -{ +template +JacobiSVD& JacobiSVD::compute_impl(const MatrixType& matrix, + unsigned int computationOptions) { using std::abs; + allocate(matrix.rows(), matrix.cols(), computationOptions); // currently we stop when we reach precision 2*epsilon as the last bit of precision can require an unreasonable number of iterations, @@ -680,13 +703,14 @@ JacobiSVD::compute(const MatrixType& matrix, unsig if (!(numext::isfinite)(scale)) { m_isInitialized = true; m_info = InvalidInput; + m_nonzeroSingularValues = 0; return *this; } - if(scale==RealScalar(0)) scale = RealScalar(1); + if(numext::is_exactly_zero(scale)) scale = RealScalar(1); /*** step 1. The R-SVD step: we use a QR decomposition to reduce to the case of a square matrix */ - if(m_rows!=m_cols) + if(rows() != cols()) { m_scaledMatrix = matrix / scale; m_qr_precond_morecols.run(*this, m_scaledMatrix); @@ -694,11 +718,11 @@ JacobiSVD::compute(const MatrixType& matrix, unsig } else { - m_workMatrix = matrix.block(0,0,m_diagSize,m_diagSize) / scale; - if(m_computeFullU) m_matrixU.setIdentity(m_rows,m_rows); - if(m_computeThinU) m_matrixU.setIdentity(m_rows,m_diagSize); - if(m_computeFullV) m_matrixV.setIdentity(m_cols,m_cols); - if(m_computeThinV) m_matrixV.setIdentity(m_cols, m_diagSize); + m_workMatrix = matrix.template topLeftCorner(diagSize(),diagSize()) / scale; + if(m_computeFullU) m_matrixU.setIdentity(rows(),rows()); + if(m_computeThinU) m_matrixU.setIdentity(rows(),diagSize()); + if(m_computeFullV) m_matrixV.setIdentity(cols(),cols()); + if(m_computeThinV) m_matrixV.setIdentity(cols(),diagSize()); } /*** step 2. The main Jacobi SVD iteration. ***/ @@ -711,7 +735,7 @@ JacobiSVD::compute(const MatrixType& matrix, unsig // do a sweep: for all index pairs (p,q), perform SVD of the corresponding 2x2 sub-matrix - for(Index p = 1; p < m_diagSize; ++p) + for(Index p = 1; p < diagSize(); ++p) { for(Index q = 0; q < p; ++q) { @@ -724,8 +748,8 @@ JacobiSVD::compute(const MatrixType& matrix, unsig finished = false; // perform SVD decomposition of 2x2 sub-matrix corresponding to indices p,q to make it diagonal // the complex to real operation returns true if the updated 2x2 block is not already diagonal - if(internal::svd_precondition_2x2_block_to_be_real::run(m_workMatrix, *this, p, q, maxDiagEntry)) - { + if (internal::svd_precondition_2x2_block_to_be_real::run(m_workMatrix, *this, p, q, + maxDiagEntry)) { JacobiRotation j_left, j_right; internal::real_2x2_jacobi_svd(m_workMatrix, p, q, &j_left, &j_right); @@ -746,7 +770,7 @@ JacobiSVD::compute(const MatrixType& matrix, unsig /*** step 3. The work matrix is now diagonal, so ensure it's positive so its diagonal entries are the singular values ***/ - for(Index i = 0; i < m_diagSize; ++i) + for(Index i = 0; i < diagSize(); ++i) { // For a complex matrix, some diagonal coefficients might note have been // treated by svd_precondition_2x2_block_to_be_real, and the imaginary part @@ -770,12 +794,12 @@ JacobiSVD::compute(const MatrixType& matrix, unsig /*** step 4. Sort singular values in descending order and compute the number of nonzero singular values ***/ - m_nonzeroSingularValues = m_diagSize; - for(Index i = 0; i < m_diagSize; i++) + m_nonzeroSingularValues = diagSize(); + for(Index i = 0; i < diagSize(); i++) { Index pos; - RealScalar maxRemainingSingularValue = m_singularValues.tail(m_diagSize-i).maxCoeff(&pos); - if(maxRemainingSingularValue == RealScalar(0)) + RealScalar maxRemainingSingularValue = m_singularValues.tail(diagSize()-i).maxCoeff(&pos); + if(numext::is_exactly_zero(maxRemainingSingularValue)) { m_nonzeroSingularValues = i; break; @@ -800,13 +824,19 @@ JacobiSVD::compute(const MatrixType& matrix, unsig * * \sa class JacobiSVD */ -template -JacobiSVD::PlainObject> -MatrixBase::jacobiSvd(unsigned int computationOptions) const -{ - return JacobiSVD(*this, computationOptions); +template +template +JacobiSVD::PlainObject, Options> MatrixBase::jacobiSvd() const { + return JacobiSVD(*this); +} + +template +template +JacobiSVD::PlainObject, Options> MatrixBase::jacobiSvd( + unsigned int computationOptions) const { + return JacobiSVD(*this, computationOptions); } -} // end namespace Eigen +} // end namespace Eigen #endif // EIGEN_JACOBISVD_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/JacobiSVD_LAPACKE.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/JacobiSVD_LAPACKE.h index ff0516f6116..12c86143a21 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/JacobiSVD_LAPACKE.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/JacobiSVD_LAPACKE.h @@ -33,14 +33,18 @@ #ifndef EIGEN_JACOBISVD_LAPACKE_H #define EIGEN_JACOBISVD_LAPACKE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \internal Specialization for the data types supported by LAPACKe */ -#define EIGEN_LAPACKE_SVD(EIGTYPE, LAPACKE_TYPE, LAPACKE_RTYPE, LAPACKE_PREFIX, EIGCOLROW, LAPACKE_COLROW) \ +#define EIGEN_LAPACKE_SVD(EIGTYPE, LAPACKE_TYPE, LAPACKE_RTYPE, LAPACKE_PREFIX, EIGCOLROW, LAPACKE_COLROW, OPTIONS) \ template<> inline \ -JacobiSVD, ColPivHouseholderQRPreconditioner>& \ -JacobiSVD, ColPivHouseholderQRPreconditioner>::compute(const Matrix& matrix, unsigned int computationOptions) \ +JacobiSVD, OPTIONS>& \ +JacobiSVD, OPTIONS>::compute_impl(const Matrix& matrix, \ + unsigned int computationOptions) \ { \ typedef Matrix MatrixType; \ /*typedef MatrixType::Scalar Scalar;*/ \ @@ -48,7 +52,7 @@ JacobiSVD, ColPiv allocate(matrix.rows(), matrix.cols(), computationOptions); \ \ /*const RealScalar precision = RealScalar(2) * NumTraits::epsilon();*/ \ - m_nonzeroSingularValues = m_diagSize; \ + m_nonzeroSingularValues = diagSize(); \ \ lapack_int lda = internal::convert_index(matrix.outerStride()), ldu, ldvt; \ lapack_int matrix_order = LAPACKE_COLROW; \ @@ -61,30 +65,49 @@ JacobiSVD, ColPiv u = (LAPACKE_TYPE*)m_matrixU.data(); \ } else { ldu=1; u=&dummy; }\ MatrixType localV; \ - lapack_int vt_rows = (m_computeFullV) ? internal::convert_index(m_cols) : (m_computeThinV) ? internal::convert_index(m_diagSize) : 1; \ + lapack_int vt_rows = (m_computeFullV) ? internal::convert_index(cols()) : (m_computeThinV) ? internal::convert_index(diagSize()) : 1; \ if (computeV()) { \ - localV.resize(vt_rows, m_cols); \ + localV.resize(vt_rows, cols()); \ ldvt = internal::convert_index(localV.outerStride()); \ vt = (LAPACKE_TYPE*)localV.data(); \ } else { ldvt=1; vt=&dummy; }\ - Matrix superb; superb.resize(m_diagSize, 1); \ + Matrix superb; superb.resize(diagSize(), 1); \ MatrixType m_temp; m_temp = matrix; \ - LAPACKE_##LAPACKE_PREFIX##gesvd( matrix_order, jobu, jobvt, internal::convert_index(m_rows), internal::convert_index(m_cols), (LAPACKE_TYPE*)m_temp.data(), lda, (LAPACKE_RTYPE*)m_singularValues.data(), u, ldu, vt, ldvt, superb.data()); \ - if (computeV()) m_matrixV = localV.adjoint(); \ - /* for(int i=0;i(rows()), internal::convert_index(cols()), (LAPACKE_TYPE*)m_temp.data(), lda, (LAPACKE_RTYPE*)m_singularValues.data(), u, ldu, vt, ldvt, superb.data()); \ + /* Check the result of the LAPACK call */ \ + if (info < 0 || !m_singularValues.allFinite()) { \ + m_info = InvalidInput; \ + } else if (info > 0 ) { \ + m_info = NoConvergence; \ + } else { \ + m_info = Success; \ + if (computeV()) m_matrixV = localV.adjoint(); \ + } \ + /* for(int i=0;i +void check_svd_options_assertions(unsigned int computationOptions, Index rows, Index cols) { + EIGEN_STATIC_ASSERT((Options & ComputationOptionsBits) == 0, + "SVDBase: Cannot request U or V using both static and runtime options, even if they match. " + "Requesting unitaries at runtime is DEPRECATED: " + "Prefer requesting unitaries statically, using the Options template parameter."); + eigen_assert(!(should_svd_compute_thin_u(computationOptions) && cols < rows && MatrixType::RowsAtCompileTime != Dynamic) && + !(should_svd_compute_thin_v(computationOptions) && rows < cols && MatrixType::ColsAtCompileTime != Dynamic) && + "SVDBase: If thin U is requested at runtime, your matrix must have more rows than columns or a dynamic number of rows." + "Similarly, if thin V is requested at runtime, you matrix must have more columns than rows or a dynamic number of columns."); + (void)computationOptions; + (void)rows; + (void)cols; +} + template struct traits > : traits { @@ -27,6 +61,29 @@ template struct traits > typedef int StorageIndex; enum { Flags = 0 }; }; + +template +struct svd_traits : traits { + static constexpr int Options = Options_; + static constexpr bool ShouldComputeFullU = internal::should_svd_compute_full_u(Options); + static constexpr bool ShouldComputeThinU = internal::should_svd_compute_thin_u(Options); + static constexpr bool ShouldComputeFullV = internal::should_svd_compute_full_v(Options); + static constexpr bool ShouldComputeThinV = internal::should_svd_compute_thin_v(Options); + enum { + DiagSizeAtCompileTime = + internal::min_size_prefer_dynamic(MatrixType::RowsAtCompileTime, MatrixType::ColsAtCompileTime), + MaxDiagSizeAtCompileTime = + internal::min_size_prefer_dynamic(MatrixType::MaxRowsAtCompileTime, MatrixType::MaxColsAtCompileTime), + MatrixUColsAtCompileTime = ShouldComputeThinU ? DiagSizeAtCompileTime + : MatrixType::RowsAtCompileTime, + MatrixVColsAtCompileTime = ShouldComputeThinV ? DiagSizeAtCompileTime + : MatrixType::ColsAtCompileTime, + MatrixUMaxColsAtCompileTime = ShouldComputeThinU ? MaxDiagSizeAtCompileTime + : MatrixType::MaxRowsAtCompileTime, + MatrixVMaxColsAtCompileTime = ShouldComputeThinV ? MaxDiagSizeAtCompileTime + : MatrixType::MaxColsAtCompileTime + }; +}; } /** \ingroup SVD_Module @@ -52,7 +109,7 @@ template struct traits > * singular vectors. Asking for \em thin \a U or \a V means asking for only their \a m first columns to be formed. So \a U is then a n-by-m matrix, * and \a V is then a p-by-m matrix. Notice that thin \a U and \a V are all you need for (least squares) solving. * - * The status of the computation can be retrived using the \a info() method. Unless \a info() returns \a Success, the results should be not + * The status of the computation can be retrieved using the \a info() method. Unless \a info() returns \a Success, the results should be not * considered well defined. * * If the input matrix has inf or nan coefficients, the result of the computation is undefined, and \a info() will return \a InvalidInput, but the computation is guaranteed to @@ -72,20 +129,38 @@ template class SVDBase typedef typename NumTraits::Real RealScalar; typedef typename Eigen::internal::traits::StorageIndex StorageIndex; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 + + static constexpr bool ShouldComputeFullU = internal::traits::ShouldComputeFullU; + static constexpr bool ShouldComputeThinU = internal::traits::ShouldComputeThinU; + static constexpr bool ShouldComputeFullV = internal::traits::ShouldComputeFullV; + static constexpr bool ShouldComputeThinV = internal::traits::ShouldComputeThinV; + enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, - DiagSizeAtCompileTime = EIGEN_SIZE_MIN_PREFER_DYNAMIC(RowsAtCompileTime,ColsAtCompileTime), + DiagSizeAtCompileTime = internal::min_size_prefer_dynamic(RowsAtCompileTime, ColsAtCompileTime), MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, - MaxDiagSizeAtCompileTime = EIGEN_SIZE_MIN_PREFER_FIXED(MaxRowsAtCompileTime,MaxColsAtCompileTime), - MatrixOptions = MatrixType::Options + MaxDiagSizeAtCompileTime = internal::min_size_prefer_fixed(MaxRowsAtCompileTime, MaxColsAtCompileTime), + MatrixOptions = MatrixType::Options, + MatrixUColsAtCompileTime = internal::traits::MatrixUColsAtCompileTime, + MatrixVColsAtCompileTime = internal::traits::MatrixVColsAtCompileTime, + MatrixUMaxColsAtCompileTime = internal::traits::MatrixUMaxColsAtCompileTime, + MatrixVMaxColsAtCompileTime = internal::traits::MatrixVMaxColsAtCompileTime }; - typedef Matrix MatrixUType; - typedef Matrix MatrixVType; + EIGEN_STATIC_ASSERT(!(ShouldComputeFullU && ShouldComputeThinU), "SVDBase: Cannot request both full and thin U") + EIGEN_STATIC_ASSERT(!(ShouldComputeFullV && ShouldComputeThinV), "SVDBase: Cannot request both full and thin V") + + typedef + typename internal::make_proper_matrix_type::type MatrixUType; + typedef + typename internal::make_proper_matrix_type::type MatrixVType; + typedef typename internal::plain_diag_type::type SingularValuesType; - + Derived& derived() { return *static_cast(this); } const Derived& derived() const { return *static_cast(this); } @@ -209,8 +284,9 @@ template class SVDBase /** \returns true if \a V (full or thin) is asked for in this SVD decomposition */ inline bool computeV() const { return m_computeFullV || m_computeThinV; } - inline Index rows() const { return m_rows; } - inline Index cols() const { return m_cols; } + inline Index rows() const { return m_rows.value(); } + inline Index cols() const { return m_cols.value(); } + inline Index diagSize() const { return m_diagSize.value(); } #ifdef EIGEN_PARSED_BY_DOXYGEN /** \returns a (least squares) solution of \f$ A x = b \f$ using the current SVD decomposition of A. @@ -249,10 +325,7 @@ template class SVDBase protected: - static void check_template_parameters() - { - EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); - } + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) void _check_compute_assertions() const { eigen_assert(m_isInitialized && "SVD is not initialized."); @@ -267,7 +340,7 @@ template class SVDBase } // return true if already allocated - bool allocate(Index rows, Index cols, unsigned int computationOptions) ; + bool allocate(Index rows, Index cols, unsigned int computationOptions); MatrixUType m_matrixU; MatrixVType m_matrixV; @@ -277,7 +350,10 @@ template class SVDBase bool m_computeFullU, m_computeThinU; bool m_computeFullV, m_computeThinV; unsigned int m_computationOptions; - Index m_nonzeroSingularValues, m_rows, m_cols, m_diagSize; + Index m_nonzeroSingularValues; + internal::variable_if_dynamic m_rows; + internal::variable_if_dynamic m_cols; + internal::variable_if_dynamic m_diagSize; RealScalar m_prescribedThreshold; /** \brief Default Constructor. @@ -285,21 +361,23 @@ template class SVDBase * Default constructor of SVDBase */ SVDBase() - : m_info(Success), - m_isInitialized(false), - m_isAllocated(false), - m_usePrescribedThreshold(false), - m_computeFullU(false), - m_computeThinU(false), - m_computeFullV(false), - m_computeThinV(false), - m_computationOptions(0), - m_rows(-1), m_cols(-1), m_diagSize(0) - { - check_template_parameters(); - } - - + : m_matrixU(MatrixUType()), + m_matrixV(MatrixVType()), + m_singularValues(SingularValuesType()), + m_info(Success), + m_isInitialized(false), + m_isAllocated(false), + m_usePrescribedThreshold(false), + m_computeFullU(false), + m_computeThinU(false), + m_computeFullV(false), + m_computeThinV(false), + m_computationOptions(0), + m_nonzeroSingularValues(0), + m_rows(RowsAtCompileTime), + m_cols(ColsAtCompileTime), + m_diagSize(DiagSizeAtCompileTime), + m_prescribedThreshold(0) {} }; #ifndef EIGEN_PARSED_BY_DOXYGEN @@ -333,40 +411,38 @@ void SVDBase::_solve_impl_transposed(const RhsType &rhs, DstType &dst) } #endif -template -bool SVDBase::allocate(Index rows, Index cols, unsigned int computationOptions) -{ +template +bool SVDBase::allocate(Index rows, Index cols, unsigned int computationOptions) { eigen_assert(rows >= 0 && cols >= 0); if (m_isAllocated && - rows == m_rows && - cols == m_cols && + rows == m_rows.value() && + cols == m_cols.value() && computationOptions == m_computationOptions) { return true; } - m_rows = rows; - m_cols = cols; + m_rows.setValue(rows); + m_cols.setValue(cols); m_info = Success; m_isInitialized = false; m_isAllocated = true; m_computationOptions = computationOptions; - m_computeFullU = (computationOptions & ComputeFullU) != 0; - m_computeThinU = (computationOptions & ComputeThinU) != 0; - m_computeFullV = (computationOptions & ComputeFullV) != 0; - m_computeThinV = (computationOptions & ComputeThinV) != 0; + m_computeFullU = ShouldComputeFullU || internal::should_svd_compute_full_u(computationOptions); + m_computeThinU = ShouldComputeThinU || internal::should_svd_compute_thin_u(computationOptions); + m_computeFullV = ShouldComputeFullV || internal::should_svd_compute_full_v(computationOptions); + m_computeThinV = ShouldComputeThinV || internal::should_svd_compute_thin_v(computationOptions); + eigen_assert(!(m_computeFullU && m_computeThinU) && "SVDBase: you can't ask for both full and thin U"); eigen_assert(!(m_computeFullV && m_computeThinV) && "SVDBase: you can't ask for both full and thin V"); - eigen_assert(EIGEN_IMPLIES(m_computeThinU || m_computeThinV, MatrixType::ColsAtCompileTime==Dynamic) && - "SVDBase: thin U and V are only available when your matrix has a dynamic number of columns."); - m_diagSize = (std::min)(m_rows, m_cols); - m_singularValues.resize(m_diagSize); + m_diagSize.setValue(numext::mini(m_rows.value(), m_cols.value())); + m_singularValues.resize(m_diagSize.value()); if(RowsAtCompileTime==Dynamic) - m_matrixU.resize(m_rows, m_computeFullU ? m_rows : m_computeThinU ? m_diagSize : 0); + m_matrixU.resize(m_rows.value(), m_computeFullU ? m_rows.value() : m_computeThinU ? m_diagSize.value() : 0); if(ColsAtCompileTime==Dynamic) - m_matrixV.resize(m_cols, m_computeFullV ? m_cols : m_computeThinV ? m_diagSize : 0); + m_matrixV.resize(m_cols.value(), m_computeFullV ? m_cols.value() : m_computeThinV ? m_diagSize.value() : 0); return false; } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/UpperBidiagonalization.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/UpperBidiagonalization.h index 997defc4744..ec2fc536908 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/UpperBidiagonalization.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SVD/UpperBidiagonalization.h @@ -11,17 +11,20 @@ #ifndef EIGEN_BIDIAGONALIZATION_H #define EIGEN_BIDIAGONALIZATION_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { // UpperBidiagonalization will probably be replaced by a Bidiagonalization class, don't want to make it stable API. // At the same time, it's useful to keep for now as it's about the only thing that is testing the BandMatrix class. -template class UpperBidiagonalization +template class UpperBidiagonalization { public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, @@ -37,10 +40,10 @@ template class UpperBidiagonalization typedef Matrix SuperDiagVectorType; typedef HouseholderSequence< const MatrixType, - const typename internal::remove_all::ConjugateReturnType>::type + const internal::remove_all_t::ConjugateReturnType> > HouseholderUSequenceType; typedef HouseholderSequence< - const typename internal::remove_all::type, + const internal::remove_all_t, Diagonal, OnTheRight > HouseholderVSequenceType; @@ -51,7 +54,7 @@ template class UpperBidiagonalization * The default constructor is useful in cases in which the user intends to * perform decompositions via Bidiagonalization::compute(const MatrixType&). */ - UpperBidiagonalization() : m_householder(), m_bidiagonal(), m_isInitialized(false) {} + UpperBidiagonalization() : m_householder(), m_bidiagonal(0, 0), m_isInitialized(false) {} explicit UpperBidiagonalization(const MatrixType& matrix) : m_householder(matrix.rows(), matrix.cols()), @@ -60,7 +63,13 @@ template class UpperBidiagonalization { compute(matrix); } - + + UpperBidiagonalization(Index rows, Index cols) + : m_householder(rows, cols), + m_bidiagonal(cols, cols), + m_isInitialized(false) + {} + UpperBidiagonalization& compute(const MatrixType& matrix); UpperBidiagonalization& computeUnblocked(const MatrixType& matrix); @@ -161,13 +170,13 @@ void upperbidiagonalization_blocked_helper(MatrixType& A, typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef typename NumTraits::Literal Literal; - enum { StorageOrder = traits::Flags & RowMajorBit }; - typedef InnerStride ColInnerStride; - typedef InnerStride RowInnerStride; + static constexpr int StorageOrder = (traits::Flags & RowMajorBit) ? RowMajor : ColMajor; + typedef InnerStride ColInnerStride; + typedef InnerStride RowInnerStride; typedef Ref, 0, ColInnerStride> SubColumnType; typedef Ref, 0, RowInnerStride> SubRowType; typedef Ref > SubMatType; - + Index brows = A.rows(); Index bcols = A.cols(); @@ -293,7 +302,7 @@ void upperbidiagonalization_inplace_blocked(MatrixType& A, BidiagType& bidiagona Index size = (std::min)(rows, cols); // X and Y are work space - enum { StorageOrder = traits::Flags & RowMajorBit }; + static constexpr int StorageOrder = (traits::Flags & RowMajorBit) ? RowMajor : ColMajor; Matrix(); + typename MatrixType::RealScalar* upper_diagonal_ptr = upper_diagonal.size() > 0 ? &upper_diagonal.coeffRef(k) : nullptr; + if(k+bs==cols || bcols<48) // somewhat arbitrary threshold { upperbidiagonalization_inplace_unblocked(B, &(bidiagonal.template diagonal<0>().coeffRef(k)), - &(bidiagonal.template diagonal<1>().coeffRef(k)), + upper_diagonal_ptr, X.data() ); break; // We're done @@ -346,7 +359,7 @@ void upperbidiagonalization_inplace_blocked(MatrixType& A, BidiagType& bidiagona { upperbidiagonalization_blocked_helper( B, &(bidiagonal.template diagonal<0>().coeffRef(k)), - &(bidiagonal.template diagonal<1>().coeffRef(k)), + upper_diagonal_ptr, bs, X.topLeftCorner(brows,bs), Y.topLeftCorner(bcols,bs) @@ -355,8 +368,8 @@ void upperbidiagonalization_inplace_blocked(MatrixType& A, BidiagType& bidiagona } } -template -UpperBidiagonalization<_MatrixType>& UpperBidiagonalization<_MatrixType>::computeUnblocked(const _MatrixType& matrix) +template +UpperBidiagonalization& UpperBidiagonalization::computeUnblocked(const MatrixType_& matrix) { Index rows = matrix.rows(); Index cols = matrix.cols(); @@ -377,8 +390,8 @@ UpperBidiagonalization<_MatrixType>& UpperBidiagonalization<_MatrixType>::comput return *this; } -template -UpperBidiagonalization<_MatrixType>& UpperBidiagonalization<_MatrixType>::compute(const _MatrixType& matrix) +template +UpperBidiagonalization& UpperBidiagonalization::compute(const MatrixType_& matrix) { Index rows = matrix.rows(); Index cols = matrix.cols(); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCholesky/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCholesky/InternalHeaderCheck.h new file mode 100644 index 00000000000..f8d876286fe --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCholesky/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_SPARSECHOLESKY_MODULE_H +#error "Please include Eigen/SparseCholesky instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCholesky/SimplicialCholesky.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCholesky/SimplicialCholesky.h index 9f93e3255da..059066cfb01 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCholesky/SimplicialCholesky.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCholesky/SimplicialCholesky.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SIMPLICIAL_CHOLESKY_H #define EIGEN_SIMPLICIAL_CHOLESKY_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { enum SimplicialCholeskyMode { @@ -271,17 +274,17 @@ class SimplicialCholeskyBase : public SparseSolverBase RealScalar m_shiftScale; }; -template > class SimplicialLLT; -template > class SimplicialLDLT; -template > class SimplicialCholesky; +template > class SimplicialLLT; +template > class SimplicialLDLT; +template > class SimplicialCholesky; namespace internal { -template struct traits > +template struct traits > { - typedef _MatrixType MatrixType; - typedef _Ordering OrderingType; - enum { UpLo = _UpLo }; + typedef MatrixType_ MatrixType; + typedef Ordering_ OrderingType; + enum { UpLo = UpLo_ }; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::StorageIndex StorageIndex; typedef SparseMatrix CholMatrixType; @@ -291,11 +294,11 @@ template struct traits struct traits > +template struct traits > { - typedef _MatrixType MatrixType; - typedef _Ordering OrderingType; - enum { UpLo = _UpLo }; + typedef MatrixType_ MatrixType; + typedef Ordering_ OrderingType; + enum { UpLo = UpLo_ }; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::StorageIndex StorageIndex; typedef SparseMatrix CholMatrixType; @@ -305,11 +308,11 @@ template struct traits struct traits > +template struct traits > { - typedef _MatrixType MatrixType; - typedef _Ordering OrderingType; - enum { UpLo = _UpLo }; + typedef MatrixType_ MatrixType; + typedef Ordering_ OrderingType; + enum { UpLo = UpLo_ }; }; } @@ -325,21 +328,21 @@ template struct traits - * \tparam _UpLo the triangular part that will be used for the computations. It can be Lower + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ the triangular part that will be used for the computations. It can be Lower * or Upper. Default is Lower. - * \tparam _Ordering The ordering method to use, either AMDOrdering<> or NaturalOrdering<>. Default is AMDOrdering<> + * \tparam Ordering_ The ordering method to use, either AMDOrdering<> or NaturalOrdering<>. Default is AMDOrdering<> * * \implsparsesolverconcept * * \sa class SimplicialLDLT, class AMDOrdering, class NaturalOrdering */ -template - class SimplicialLLT : public SimplicialCholeskyBase > +template + class SimplicialLLT : public SimplicialCholeskyBase > { public: - typedef _MatrixType MatrixType; - enum { UpLo = _UpLo }; + typedef MatrixType_ MatrixType; + enum { UpLo = UpLo_ }; typedef SimplicialCholeskyBase Base; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; @@ -416,21 +419,21 @@ template * In order to reduce the fill-in, a symmetric permutation P is applied prior to the factorization * such that the factorized matrix is P A P^-1. * - * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> - * \tparam _UpLo the triangular part that will be used for the computations. It can be Lower + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ the triangular part that will be used for the computations. It can be Lower * or Upper. Default is Lower. - * \tparam _Ordering The ordering method to use, either AMDOrdering<> or NaturalOrdering<>. Default is AMDOrdering<> + * \tparam Ordering_ The ordering method to use, either AMDOrdering<> or NaturalOrdering<>. Default is AMDOrdering<> * * \implsparsesolverconcept * * \sa class SimplicialLLT, class AMDOrdering, class NaturalOrdering */ -template - class SimplicialLDLT : public SimplicialCholeskyBase > +template + class SimplicialLDLT : public SimplicialCholeskyBase > { public: - typedef _MatrixType MatrixType; - enum { UpLo = _UpLo }; + typedef MatrixType_ MatrixType; + enum { UpLo = UpLo_ }; typedef SimplicialCholeskyBase Base; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; @@ -507,12 +510,12 @@ template * * \sa class SimplicialLDLT, class SimplicialLLT */ -template - class SimplicialCholesky : public SimplicialCholeskyBase > +template + class SimplicialCholesky : public SimplicialCholeskyBase > { public: - typedef _MatrixType MatrixType; - enum { UpLo = _UpLo }; + typedef MatrixType_ MatrixType; + enum { UpLo = UpLo_ }; typedef SimplicialCholeskyBase Base; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h index 72e1740c193..086e0f7ee78 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h @@ -20,6 +20,9 @@ the Mozilla Public License v. 2.0, as stated at the top of this file. #ifndef EIGEN_SIMPLICIAL_CHOLESKY_IMPL_H #define EIGEN_SIMPLICIAL_CHOLESKY_IMPL_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/AmbiVector.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/AmbiVector.h index 2cb7747cc99..a663ff6f42b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/AmbiVector.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/AmbiVector.h @@ -10,6 +10,9 @@ #ifndef EIGEN_AMBIVECTOR_H #define EIGEN_AMBIVECTOR_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -19,12 +22,12 @@ namespace internal { * * See BasicSparseLLT and SparseProduct for usage examples. */ -template +template class AmbiVector { public: - typedef _Scalar Scalar; - typedef _StorageIndex StorageIndex; + typedef Scalar_ Scalar; + typedef StorageIndex_ StorageIndex; typedef typename NumTraits::Real RealScalar; explicit AmbiVector(Index size) @@ -125,8 +128,8 @@ class AmbiVector }; /** \returns the number of non zeros in the current sub vector */ -template -Index AmbiVector<_Scalar,_StorageIndex>::nonZeros() const +template +Index AmbiVector::nonZeros() const { if (m_mode==IsSparse) return m_llSize; @@ -134,8 +137,8 @@ Index AmbiVector<_Scalar,_StorageIndex>::nonZeros() const return m_end - m_start; } -template -void AmbiVector<_Scalar,_StorageIndex>::init(double estimatedDensity) +template +void AmbiVector::init(double estimatedDensity) { if (estimatedDensity>0.1) init(IsDense); @@ -143,8 +146,8 @@ void AmbiVector<_Scalar,_StorageIndex>::init(double estimatedDensity) init(IsSparse); } -template -void AmbiVector<_Scalar,_StorageIndex>::init(int mode) +template +void AmbiVector::init(int mode) { m_mode = mode; // This is only necessary in sparse mode, but we set these unconditionally to avoid some maybe-uninitialized warnings @@ -160,15 +163,15 @@ void AmbiVector<_Scalar,_StorageIndex>::init(int mode) * * Don't worry, this function is extremely cheap. */ -template -void AmbiVector<_Scalar,_StorageIndex>::restart() +template +void AmbiVector::restart() { m_llCurrent = m_llStart; } /** Set all coefficients of current subvector to zero */ -template -void AmbiVector<_Scalar,_StorageIndex>::setZero() +template +void AmbiVector::setZero() { if (m_mode==IsDense) { @@ -183,8 +186,8 @@ void AmbiVector<_Scalar,_StorageIndex>::setZero() } } -template -_Scalar& AmbiVector<_Scalar,_StorageIndex>::coeffRef(Index i) +template +Scalar_& AmbiVector::coeffRef(Index i) { if (m_mode==IsDense) return m_buffer[i]; @@ -252,8 +255,8 @@ _Scalar& AmbiVector<_Scalar,_StorageIndex>::coeffRef(Index i) } } -template -_Scalar& AmbiVector<_Scalar,_StorageIndex>::coeff(Index i) +template +Scalar_& AmbiVector::coeff(Index i) { if (m_mode==IsDense) return m_buffer[i]; @@ -280,11 +283,11 @@ _Scalar& AmbiVector<_Scalar,_StorageIndex>::coeff(Index i) } /** Iterator over the nonzero coefficients */ -template -class AmbiVector<_Scalar,_StorageIndex>::Iterator +template +class AmbiVector::Iterator { public: - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef typename NumTraits::Real RealScalar; /** Default constructor diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/CompressedStorage.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/CompressedStorage.h index acd986fab5c..f9412999bbf 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/CompressedStorage.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/CompressedStorage.h @@ -10,6 +10,9 @@ #ifndef EIGEN_COMPRESSED_STORAGE_H #define EIGEN_COMPRESSED_STORAGE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -18,13 +21,13 @@ namespace internal { * Stores a sparse set of values as a list of values and a list of indices. * */ -template +template class CompressedStorage { public: - typedef _Scalar Scalar; - typedef _StorageIndex StorageIndex; + typedef Scalar_ Scalar; + typedef StorageIndex_ StorageIndex; protected: @@ -69,8 +72,8 @@ class CompressedStorage ~CompressedStorage() { - delete[] m_values; - delete[] m_indices; + conditional_aligned_delete_auto(m_values, m_allocatedSize); + conditional_aligned_delete_auto(m_indices, m_allocatedSize); } void reserve(Index size) @@ -130,15 +133,7 @@ class CompressedStorage /** \returns the largest \c k in [start,end) such that for all \c j in [start,k) index[\c j]\<\a key */ inline Index searchLowerIndex(Index start, Index end, Index key) const { - while(end>start) - { - Index mid = (end+start)>>1; - if (m_indices[mid](std::distance(m_indices, std::lower_bound(m_indices + start, m_indices + end, key))); } /** \returns the stored value at index \a key @@ -178,24 +173,13 @@ class CompressedStorage { if (m_allocatedSize newValues(m_allocatedSize); - internal::scoped_array newIndices(m_allocatedSize); - - // copy first chunk - internal::smart_copy(m_values, m_values +id, newValues.ptr()); - internal::smart_copy(m_indices, m_indices+id, newIndices.ptr()); - - // copy the rest - if(m_size>id) - { - internal::smart_copy(m_values +id, m_values +m_size, newValues.ptr() +id+1); - internal::smart_copy(m_indices+id, m_indices+m_size, newIndices.ptr()+id+1); - } - std::swap(m_values,newValues.ptr()); - std::swap(m_indices,newIndices.ptr()); + Index newAllocatedSize = 2 * (m_size + 1); + m_values = conditional_aligned_realloc_new_auto(m_values, newAllocatedSize, m_allocatedSize); + m_indices = + conditional_aligned_realloc_new_auto(m_indices, newAllocatedSize, m_allocatedSize); + m_allocatedSize = newAllocatedSize; } - else if(m_size>id) + if(m_size>id) { internal::smart_memmove(m_values +id, m_values +m_size, m_values +id+1); internal::smart_memmove(m_indices+id, m_indices+m_size, m_indices+id+1); @@ -207,36 +191,11 @@ class CompressedStorage return m_values[id]; } - void moveChunk(Index from, Index to, Index chunkSize) + inline void moveChunk(Index from, Index to, Index chunkSize) { - eigen_internal_assert(to+chunkSize <= m_size); - if(to>from && from+chunkSize>to) - { - // move backward - internal::smart_memmove(m_values+from, m_values+from+chunkSize, m_values+to); - internal::smart_memmove(m_indices+from, m_indices+from+chunkSize, m_indices+to); - } - else - { - internal::smart_copy(m_values+from, m_values+from+chunkSize, m_values+to); - internal::smart_copy(m_indices+from, m_indices+from+chunkSize, m_indices+to); - } - } - - void prune(const Scalar& reference, const RealScalar& epsilon = NumTraits::dummy_precision()) - { - Index k = 0; - Index n = size(); - for (Index i=0; i= 0 && to+chunkSize <= m_size); + internal::smart_memmove(m_values + from, m_values + from + chunkSize, m_values + to); + internal::smart_memmove(m_indices + from, m_indices + from + chunkSize, m_indices + to); } protected: @@ -247,15 +206,8 @@ class CompressedStorage EIGEN_SPARSE_COMPRESSED_STORAGE_REALLOCATE_PLUGIN #endif eigen_internal_assert(size!=m_allocatedSize); - internal::scoped_array newValues(size); - internal::scoped_array newIndices(size); - Index copySize = (std::min)(size, m_size); - if (copySize>0) { - internal::smart_copy(m_values, m_values+copySize, newValues.ptr()); - internal::smart_copy(m_indices, m_indices+copySize, newIndices.ptr()); - } - std::swap(m_values,newValues.ptr()); - std::swap(m_indices,newIndices.ptr()); + m_values = conditional_aligned_realloc_new_auto(m_values, size, m_allocatedSize); + m_indices = conditional_aligned_realloc_new_auto(m_indices, size, m_allocatedSize); m_allocatedSize = size; } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h index 948650253b5..44721acbcb6 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h @@ -10,6 +10,9 @@ #ifndef EIGEN_CONSERVATIVESPARSESPARSEPRODUCT_H #define EIGEN_CONSERVATIVESPARSESPARSEPRODUCT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -17,9 +20,9 @@ namespace internal { template static void conservative_sparse_sparse_product_impl(const Lhs& lhs, const Rhs& rhs, ResultType& res, bool sortedInsertion = false) { - typedef typename remove_all::type::Scalar LhsScalar; - typedef typename remove_all::type::Scalar RhsScalar; - typedef typename remove_all::type::Scalar ResScalar; + typedef typename remove_all_t::Scalar LhsScalar; + typedef typename remove_all_t::Scalar RhsScalar; + typedef typename remove_all_t::Scalar ResScalar; // make sure to call innerSize/outerSize since we fake the storage order. Index rows = lhs.innerSize(); @@ -124,6 +127,11 @@ static void conservative_sparse_sparse_product_impl(const Lhs& lhs, const Rhs& r namespace internal { + +// Helper template to generate new sparse matrix types +template +using WithStorageOrder = SparseMatrix; + template::Flags&RowMajorBit) ? RowMajor : ColMajor, int RhsStorageOrder = (traits::Flags&RowMajorBit) ? RowMajor : ColMajor, @@ -133,20 +141,20 @@ struct conservative_sparse_sparse_product_selector; template struct conservative_sparse_sparse_product_selector { - typedef typename remove_all::type LhsCleaned; + typedef remove_all_t LhsCleaned; typedef typename LhsCleaned::Scalar Scalar; static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res) { - typedef SparseMatrix RowMajorMatrix; - typedef SparseMatrix ColMajorMatrixAux; - typedef typename sparse_eval::type ColMajorMatrix; + using RowMajorMatrix = WithStorageOrder; + using ColMajorMatrixAux = WithStorageOrder; // If the result is tall and thin (in the extreme case a column vector) // then it is faster to sort the coefficients inplace instead of transposing twice. // FIXME, the following heuristic is probably not very good. if(lhs.rows()>rhs.cols()) { + using ColMajorMatrix = typename sparse_eval::type; ColMajorMatrix resCol(lhs.rows(),rhs.cols()); // perform sorted insertion internal::conservative_sparse_sparse_product_impl(lhs, rhs, resCol, true); @@ -168,8 +176,8 @@ struct conservative_sparse_sparse_product_selector RowMajorRhs; - typedef SparseMatrix RowMajorRes; + using RowMajorRhs = WithStorageOrder; + using RowMajorRes = WithStorageOrder; RowMajorRhs rhsRow = rhs; RowMajorRes resRow(lhs.rows(), rhs.cols()); internal::conservative_sparse_sparse_product_impl(rhsRow, lhs, resRow); @@ -182,8 +190,8 @@ struct conservative_sparse_sparse_product_selector RowMajorLhs; - typedef SparseMatrix RowMajorRes; + using RowMajorLhs = WithStorageOrder; + using RowMajorRes = WithStorageOrder; RowMajorLhs lhsRow = lhs; RowMajorRes resRow(lhs.rows(), rhs.cols()); internal::conservative_sparse_sparse_product_impl(rhs, lhsRow, resRow); @@ -196,9 +204,9 @@ struct conservative_sparse_sparse_product_selector RowMajorMatrix; - RowMajorMatrix resRow(lhs.rows(), rhs.cols()); - internal::conservative_sparse_sparse_product_impl(rhs, lhs, resRow); + using RowMajorRes = WithStorageOrder; + RowMajorRes resRow(lhs.rows(), rhs.cols()); + internal::conservative_sparse_sparse_product_impl(rhs, lhs, resRow); res = resRow; } }; @@ -207,13 +215,13 @@ struct conservative_sparse_sparse_product_selector struct conservative_sparse_sparse_product_selector { - typedef typename traits::type>::Scalar Scalar; + typedef typename traits>::Scalar Scalar; static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res) { - typedef SparseMatrix ColMajorMatrix; - ColMajorMatrix resCol(lhs.rows(), rhs.cols()); - internal::conservative_sparse_sparse_product_impl(lhs, rhs, resCol); + using ColMajorRes = WithStorageOrder; + ColMajorRes resCol(lhs.rows(), rhs.cols()); + internal::conservative_sparse_sparse_product_impl(lhs, rhs, resCol); res = resCol; } }; @@ -223,8 +231,8 @@ struct conservative_sparse_sparse_product_selector ColMajorLhs; - typedef SparseMatrix ColMajorRes; + using ColMajorLhs = WithStorageOrder; + using ColMajorRes = WithStorageOrder; ColMajorLhs lhsCol = lhs; ColMajorRes resCol(lhs.rows(), rhs.cols()); internal::conservative_sparse_sparse_product_impl(lhsCol, rhs, resCol); @@ -237,8 +245,8 @@ struct conservative_sparse_sparse_product_selector ColMajorRhs; - typedef SparseMatrix ColMajorRes; + using ColMajorRhs = WithStorageOrder; + using ColMajorRes = WithStorageOrder; ColMajorRhs rhsCol = rhs; ColMajorRes resCol(lhs.rows(), rhs.cols()); internal::conservative_sparse_sparse_product_impl(lhs, rhsCol, resCol); @@ -251,12 +259,12 @@ struct conservative_sparse_sparse_product_selector RowMajorMatrix; - typedef SparseMatrix ColMajorMatrix; - RowMajorMatrix resRow(lhs.rows(),rhs.cols()); - internal::conservative_sparse_sparse_product_impl(rhs, lhs, resRow); + using ColMajorRes = WithStorageOrder; + using RowMajorRes = WithStorageOrder; + RowMajorRes resRow(lhs.rows(),rhs.cols()); + internal::conservative_sparse_sparse_product_impl(rhs, lhs, resRow); // sort the non zeros: - ColMajorMatrix resCol(resRow); + ColMajorRes resCol(resRow); res = resCol; } }; @@ -269,8 +277,8 @@ namespace internal { template static void sparse_sparse_to_dense_product_impl(const Lhs& lhs, const Rhs& rhs, ResultType& res) { - typedef typename remove_all::type::Scalar LhsScalar; - typedef typename remove_all::type::Scalar RhsScalar; + typedef typename remove_all_t::Scalar LhsScalar; + typedef typename remove_all_t::Scalar RhsScalar; Index cols = rhs.outerSize(); eigen_assert(lhs.outerSize() == rhs.innerSize()); @@ -317,7 +325,7 @@ struct sparse_sparse_to_dense_product_selector ColMajorLhs; + using ColMajorLhs = WithStorageOrder; ColMajorLhs lhsCol(lhs); internal::sparse_sparse_to_dense_product_impl(lhsCol, rhs, res); } @@ -328,7 +336,7 @@ struct sparse_sparse_to_dense_product_selector ColMajorRhs; + using ColMajorRhs = WithStorageOrder; ColMajorRhs rhsCol(rhs); internal::sparse_sparse_to_dense_product_impl(lhs, rhsCol, res); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/InternalHeaderCheck.h new file mode 100644 index 00000000000..9de59365704 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_SPARSECORE_MODULE_H +#error "Please include Eigen/SparseCore instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/MappedSparseMatrix.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/MappedSparseMatrix.h deleted file mode 100644 index 67718c85be9..00000000000 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/MappedSparseMatrix.h +++ /dev/null @@ -1,67 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2008-2014 Gael Guennebaud -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_MAPPED_SPARSEMATRIX_H -#define EIGEN_MAPPED_SPARSEMATRIX_H - -namespace Eigen { - -/** \deprecated Use Map > - * \class MappedSparseMatrix - * - * \brief Sparse matrix - * - * \param _Scalar the scalar type, i.e. the type of the coefficients - * - * See http://www.netlib.org/linalg/html_templates/node91.html for details on the storage scheme. - * - */ -namespace internal { -template -struct traits > : traits > -{}; -} // end namespace internal - -template -class MappedSparseMatrix - : public Map > -{ - typedef Map > Base; - - public: - - typedef typename Base::StorageIndex StorageIndex; - typedef typename Base::Scalar Scalar; - - inline MappedSparseMatrix(Index rows, Index cols, Index nnz, StorageIndex* outerIndexPtr, StorageIndex* innerIndexPtr, Scalar* valuePtr, StorageIndex* innerNonZeroPtr = 0) - : Base(rows, cols, nnz, outerIndexPtr, innerIndexPtr, valuePtr, innerNonZeroPtr) - {} - - /** Empty destructor */ - inline ~MappedSparseMatrix() {} -}; - -namespace internal { - -template -struct evaluator > - : evaluator > > -{ - typedef MappedSparseMatrix<_Scalar,_Options,_StorageIndex> XprType; - typedef evaluator > Base; - - evaluator() : Base() {} - explicit evaluator(const XprType &mat) : Base(mat) {} -}; - -} - -} // end namespace Eigen - -#endif // EIGEN_MAPPED_SPARSEMATRIX_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseAssign.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseAssign.h index 905485c88ed..e6fd714f6d2 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseAssign.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseAssign.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSEASSIGN_H #define EIGEN_SPARSEASSIGN_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template @@ -76,14 +79,20 @@ void assign_sparse_to_sparse(DstXprType &dst, const SrcXprType &src) SrcEvaluatorType srcEvaluator(src); - const bool transpose = (DstEvaluatorType::Flags & RowMajorBit) != (SrcEvaluatorType::Flags & RowMajorBit); + constexpr bool transpose = (DstEvaluatorType::Flags & RowMajorBit) != (SrcEvaluatorType::Flags & RowMajorBit); const Index outerEvaluationSize = (SrcEvaluatorType::Flags&RowMajorBit) ? src.rows() : src.cols(); + + Index reserveSize = 0; + for (Index j = 0; j < outerEvaluationSize; ++j) + for (typename SrcEvaluatorType::InnerIterator it(srcEvaluator, j); it; ++it) + reserveSize++; + if ((!transpose) && src.isRValue()) { // eval without temporary dst.resize(src.rows(), src.cols()); dst.setZero(); - dst.reserve((std::min)(src.rows()*src.cols(), (std::max)(src.rows(),src.cols())*2)); + dst.reserve(reserveSize); for (Index j=0; j dense1 = dense2; dense1 += sparse; template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - typename internal::enable_if::Shape,DenseShape>::value>::type + std::enable_if_t::Shape,DenseShape>::value> run(DstXprType &dst, const CwiseBinaryOp, const Lhs, const Rhs> &src, const internal::assign_op& /*func*/) { @@ -188,7 +197,7 @@ struct assignment_from_dense_op_sparse // Specialization for dense1 = sparse - dense2; -> dense1 = -dense2; dense1 += sparse; template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - typename internal::enable_if::Shape,DenseShape>::value>::type + std::enable_if_t::Shape,DenseShape>::value> run(DstXprType &dst, const CwiseBinaryOp, const Lhs, const Rhs> &src, const internal::assign_op& /*func*/) { @@ -206,8 +215,8 @@ struct assignment_from_dense_op_sparse template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> \ struct Assignment, const Lhs, const Rhs>, internal::ASSIGN_OP, \ Sparse2Dense, \ - typename internal::enable_if< internal::is_same::Shape,DenseShape>::value \ - || internal::is_same::Shape,DenseShape>::value>::type> \ + std::enable_if_t< internal::is_same::Shape,DenseShape>::value \ + || internal::is_same::Shape,DenseShape>::value>> \ : assignment_from_dense_op_sparse, internal::ASSIGN_OP2 > \ {} diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseBlock.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseBlock.h index 5b4f6cc9f3f..d1458048b06 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseBlock.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseBlock.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSE_BLOCK_H #define EIGEN_SPARSE_BLOCK_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { // Subset of columns or rows @@ -17,7 +20,7 @@ template class BlockImpl : public SparseMatrixBase > { - typedef typename internal::remove_all::type _MatrixTypeNested; + typedef internal::remove_all_t MatrixTypeNested_; typedef Block BlockType; public: enum { IsRowMajor = internal::traits::IsRowMajor }; @@ -96,7 +99,7 @@ template class sparse_matrix_block_impl : public SparseCompressedBase > { - typedef typename internal::remove_all::type _MatrixTypeNested; + typedef internal::remove_all_t MatrixTypeNested_; typedef Block BlockType; typedef SparseCompressedBase > Base; using Base::convert_index; @@ -119,8 +122,8 @@ class sparse_matrix_block_impl template inline BlockType& operator=(const SparseMatrixBase& other) { - typedef typename internal::remove_all::type _NestedMatrixType; - _NestedMatrixType& matrix = m_matrix; + typedef internal::remove_all_t NestedMatrixType_; + NestedMatrixType_& matrix = m_matrix; // This assignment is slow if this vector set is not empty // and/or it is not at the end of the nonzeros of the underlying matrix. @@ -283,13 +286,13 @@ class sparse_matrix_block_impl } // namespace internal -template -class BlockImpl,BlockRows,BlockCols,true,Sparse> - : public internal::sparse_matrix_block_impl,BlockRows,BlockCols> +template +class BlockImpl,BlockRows,BlockCols,true,Sparse> + : public internal::sparse_matrix_block_impl,BlockRows,BlockCols> { public: - typedef _StorageIndex StorageIndex; - typedef SparseMatrix<_Scalar, _Options, _StorageIndex> SparseMatrixType; + typedef StorageIndex_ StorageIndex; + typedef SparseMatrix SparseMatrixType; typedef internal::sparse_matrix_block_impl Base; inline BlockImpl(SparseMatrixType& xpr, Index i) : Base(xpr, i) @@ -302,13 +305,13 @@ class BlockImpl,BlockRows,BlockCo using Base::operator=; }; -template -class BlockImpl,BlockRows,BlockCols,true,Sparse> - : public internal::sparse_matrix_block_impl,BlockRows,BlockCols> +template +class BlockImpl,BlockRows,BlockCols,true,Sparse> + : public internal::sparse_matrix_block_impl,BlockRows,BlockCols> { public: - typedef _StorageIndex StorageIndex; - typedef const SparseMatrix<_Scalar, _Options, _StorageIndex> SparseMatrixType; + typedef StorageIndex_ StorageIndex; + typedef const SparseMatrix SparseMatrixType; typedef internal::sparse_matrix_block_impl Base; inline BlockImpl(SparseMatrixType& xpr, Index i) : Base(xpr, i) @@ -340,7 +343,7 @@ class BlockImpl enum { IsRowMajor = internal::traits::IsRowMajor }; EIGEN_SPARSE_PUBLIC_INTERFACE(BlockType) - typedef typename internal::remove_all::type _MatrixTypeNested; + typedef internal::remove_all_t MatrixTypeNested_; /** Column or Row constructor */ @@ -429,17 +432,12 @@ struct unary_evaluator, IteratorBa enum { IsRowMajor = XprType::IsRowMajor, - - OuterVector = (BlockCols==1 && ArgType::IsRowMajor) - | // FIXME | instead of || to please GCC 4.4.0 stupid warning "suggest parentheses around &&". - // revert to || as soon as not needed anymore. - (BlockRows==1 && !ArgType::IsRowMajor), - + OuterVector = (BlockCols == 1 && ArgType::IsRowMajor) || (BlockRows == 1 && !ArgType::IsRowMajor), CoeffReadCost = evaluator::CoeffReadCost, Flags = XprType::Flags }; - typedef typename internal::conditional::type InnerIterator; + typedef std::conditional_t InnerIterator; explicit unary_evaluator(const XprType& op) : m_argImpl(op.nestedExpression()), m_block(op) @@ -467,7 +465,7 @@ template class unary_evaluator, IteratorBased>::InnerVectorInnerIterator : public EvalIterator { - // NOTE MSVC fails to compile if we don't explicitely "import" IsRowMajor from unary_evaluator + // NOTE MSVC fails to compile if we don't explicitly "import" IsRowMajor from unary_evaluator // because the base class EvalIterator has a private IsRowMajor enum too. (bug #1786) // NOTE We cannot call it IsRowMajor because it would shadow unary_evaluator::IsRowMajor enum { XprIsRowMajor = unary_evaluator::IsRowMajor }; @@ -533,8 +531,8 @@ class unary_evaluator, IteratorBas while(++m_outerPos, IteratorBas inline operator bool() const { return m_outerPos < m_end; } }; -template -struct unary_evaluator,BlockRows,BlockCols,true>, IteratorBased> - : evaluator,BlockRows,BlockCols,true> > > +template +struct unary_evaluator,BlockRows,BlockCols,true>, IteratorBased> + : evaluator,BlockRows,BlockCols,true> > > { - typedef Block,BlockRows,BlockCols,true> XprType; + typedef Block,BlockRows,BlockCols,true> XprType; typedef evaluator > Base; explicit unary_evaluator(const XprType &xpr) : Base(xpr) {} }; -template -struct unary_evaluator,BlockRows,BlockCols,true>, IteratorBased> - : evaluator,BlockRows,BlockCols,true> > > +template +struct unary_evaluator,BlockRows,BlockCols,true>, IteratorBased> + : evaluator,BlockRows,BlockCols,true> > > { - typedef Block,BlockRows,BlockCols,true> XprType; + typedef Block,BlockRows,BlockCols,true> XprType; typedef evaluator > Base; explicit unary_evaluator(const XprType &xpr) : Base(xpr) {} }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseColEtree.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseColEtree.h index ebe02d1ab02..7cead043d90 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseColEtree.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseColEtree.h @@ -31,6 +31,9 @@ #ifndef SPARSE_COLETREE_H #define SPARSE_COLETREE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCompressedBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCompressedBase.h index 6a2c7a8ce63..c28cc951803 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCompressedBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCompressedBase.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSE_COMPRESSED_BASE_H #define EIGEN_SPARSE_COMPRESSED_BASE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template class SparseCompressedBase; @@ -20,6 +23,9 @@ template struct traits > : traits {}; +template +struct inner_sort_impl; + } // end namespace internal /** \ingroup SparseCore_Module @@ -55,14 +61,14 @@ class SparseCompressedBase /** \returns the number of non zero coefficients */ inline Index nonZeros() const { - if(Derived::IsVectorAtCompileTime && outerIndexPtr()==0) - return derived().nonZeros(); - else if(isCompressed()) - return outerIndexPtr()[derived().outerSize()]-outerIndexPtr()[0]; - else if(derived().outerSize()==0) - return 0; - else - return innerNonZeros().sum(); + if (Derived::IsVectorAtCompileTime && outerIndexPtr() == 0) + return derived().nonZeros(); + else if (derived().outerSize() == 0) + return 0; + else if (isCompressed()) + return outerIndexPtr()[derived().outerSize()] - outerIndexPtr()[0]; + else + return innerNonZeros().sum(); } /** \returns a const pointer to the array of values. @@ -124,6 +130,40 @@ class SparseCompressedBase * * \sa valuePtr(), isCompressed() */ Map > coeffs() { eigen_assert(isCompressed()); return Array::Map(valuePtr(),nonZeros()); } + + /** sorts the inner vectors in the range [begin,end) with respect to `Comp` + * \sa innerIndicesAreSorted() */ + template > + inline void sortInnerIndices(Index begin, Index end) { + eigen_assert(begin >= 0 && end <= derived().outerSize() && end >= begin); + internal::inner_sort_impl::run(*this, begin, end); + } + + /** \returns the index of the first inner vector in the range [begin,end) that is not sorted with respect to `Comp`, or `end` if the range is fully sorted + * \sa sortInnerIndices() */ + template > + inline Index innerIndicesAreSorted(Index begin, Index end) const { + eigen_assert(begin >= 0 && end <= derived().outerSize() && end >= begin); + return internal::inner_sort_impl::check(*this, begin, end); + } + + /** sorts the inner vectors in the range [0,outerSize) with respect to `Comp` + * \sa innerIndicesAreSorted() */ + template > + inline void sortInnerIndices() { + Index begin = 0; + Index end = derived().outerSize(); + internal::inner_sort_impl::run(*this, begin, end); + } + + /** \returns the index of the first inner vector in the range [0,outerSize) that is not sorted with respect to `Comp`, or `outerSize` if the range is fully sorted + * \sa sortInnerIndices() */ + template> + inline Index innerIndicesAreSorted() const { + Index begin = 0; + Index end = derived().outerSize(); + return internal::inner_sort_impl::check(*this, begin, end); + } protected: /** Default constructor. Do nothing. */ @@ -194,8 +234,7 @@ class SparseCompressedBase::InnerIterator } } - explicit InnerIterator(const SparseCompressedBase& mat) - : m_values(mat.valuePtr()), m_indices(mat.innerIndexPtr()), m_outer(0), m_id(0), m_end(mat.nonZeros()) + explicit InnerIterator(const SparseCompressedBase& mat) : InnerIterator(mat, Index(0)) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived); } @@ -305,6 +344,180 @@ class SparseCompressedBase::ReverseInnerIterator namespace internal { +// modified from https://artificial-mind.net/blog/2020/11/28/std-sort-multiple-ranges + +template +class StorageVal; +template +class StorageRef; +template +class CompressedStorageIterator; + +// class to hold an index/value pair +template +class StorageVal +{ +public: + + StorageVal(const StorageIndex& innerIndex, const Scalar& value) : m_innerIndex(innerIndex), m_value(value) {} + StorageVal(const StorageVal& other) : m_innerIndex(other.m_innerIndex), m_value(other.m_value) {} + StorageVal(StorageVal&& other) = default; + + inline const StorageIndex& key() const { return m_innerIndex; } + inline StorageIndex& key() { return m_innerIndex; } + inline const Scalar& value() const { return m_value; } + inline Scalar& value() { return m_value; } + + // enables StorageVal to be compared with respect to any type that is convertible to StorageIndex + inline operator StorageIndex() const { return m_innerIndex; } + +protected: + StorageIndex m_innerIndex; + Scalar m_value; +private: + StorageVal() = delete; +}; +// class to hold an index/value iterator pair +// used to define assignment, swap, and comparison operators for CompressedStorageIterator +template +class StorageRef +{ +public: + using value_type = StorageVal; + + // StorageRef Needs to be move-able for sort on macos. + StorageRef(StorageRef&& other) = default; + + inline StorageRef& operator=(const StorageRef& other) { + key() = other.key(); + value() = other.value(); + return *this; + } + inline StorageRef& operator=(const value_type& other) { + key() = other.key(); + value() = other.value(); + return *this; + } + inline operator value_type() const { return value_type(key(), value()); } + inline friend void swap(const StorageRef& a, const StorageRef& b) { + std::iter_swap(a.keyPtr(), b.keyPtr()); + std::iter_swap(a.valuePtr(), b.valuePtr()); + } + + inline const StorageIndex& key() const { return *m_innerIndexIterator; } + inline StorageIndex& key() { return *m_innerIndexIterator; } + inline const Scalar& value() const { return *m_valueIterator; } + inline Scalar& value() { return *m_valueIterator; } + inline StorageIndex* keyPtr() const { return m_innerIndexIterator; } + inline Scalar* valuePtr() const { return m_valueIterator; } + + // enables StorageRef to be compared with respect to any type that is convertible to StorageIndex + inline operator StorageIndex() const { return *m_innerIndexIterator; } + +protected: + StorageIndex* m_innerIndexIterator; + Scalar* m_valueIterator; +private: + StorageRef() = delete; + // these constructors are called by the CompressedStorageIterator constructors for convenience only + StorageRef(StorageIndex* innerIndexIterator, Scalar* valueIterator) : m_innerIndexIterator(innerIndexIterator), m_valueIterator(valueIterator) {} + StorageRef(const StorageRef& other) : m_innerIndexIterator(other.m_innerIndexIterator), m_valueIterator(other.m_valueIterator) {} + + friend class CompressedStorageIterator; +}; + +// STL-compatible iterator class that operates on inner indices and values +template +class CompressedStorageIterator +{ +public: + using iterator_category = std::random_access_iterator_tag; + using reference = StorageRef; + using difference_type = Index; + using value_type = typename reference::value_type; + using pointer = value_type*; + + CompressedStorageIterator() = delete; + CompressedStorageIterator(difference_type index, StorageIndex* innerIndexPtr, Scalar* valuePtr) : m_index(index), m_data(innerIndexPtr, valuePtr) {} + CompressedStorageIterator(difference_type index, reference data) : m_index(index), m_data(data) {} + CompressedStorageIterator(const CompressedStorageIterator& other) : m_index(other.m_index), m_data(other.m_data) {} + CompressedStorageIterator(CompressedStorageIterator&& other) = default; + inline CompressedStorageIterator& operator=(const CompressedStorageIterator& other) { + m_index = other.m_index; + m_data = other.m_data; + return *this; + } + + inline CompressedStorageIterator operator+(difference_type offset) const { return CompressedStorageIterator(m_index + offset, m_data); } + inline CompressedStorageIterator operator-(difference_type offset) const { return CompressedStorageIterator(m_index - offset, m_data); } + inline difference_type operator-(const CompressedStorageIterator& other) const { return m_index - other.m_index; } + inline CompressedStorageIterator& operator++() { ++m_index; return *this; } + inline CompressedStorageIterator& operator--() { --m_index; return *this; } + inline CompressedStorageIterator& operator+=(difference_type offset) { m_index += offset; return *this; } + inline CompressedStorageIterator& operator-=(difference_type offset) { m_index -= offset; return *this; } + inline reference operator*() const { return reference(m_data.keyPtr() + m_index, m_data.valuePtr() + m_index); } + + #define MAKE_COMP(OP) inline bool operator OP(const CompressedStorageIterator& other) const { return m_index OP other.m_index; } + MAKE_COMP(<) + MAKE_COMP(>) + MAKE_COMP(>=) + MAKE_COMP(<=) + MAKE_COMP(!=) + MAKE_COMP(==) + #undef MAKE_COMP + +protected: + difference_type m_index; + reference m_data; +}; + +template +struct inner_sort_impl { + typedef typename Derived::Scalar Scalar; + typedef typename Derived::StorageIndex StorageIndex; + static inline void run(SparseCompressedBase& obj, Index begin, Index end) { + const bool is_compressed = obj.isCompressed(); + for (Index outer = begin; outer < end; outer++) { + Index begin_offset = obj.outerIndexPtr()[outer]; + Index end_offset = is_compressed ? obj.outerIndexPtr()[outer + 1] : (begin_offset + obj.innerNonZeroPtr()[outer]); + CompressedStorageIterator begin_it(begin_offset, obj.innerIndexPtr(), obj.valuePtr()); + CompressedStorageIterator end_it(end_offset, obj.innerIndexPtr(), obj.valuePtr()); + std::sort(begin_it, end_it, Comp()); + } + } + static inline Index check(const SparseCompressedBase& obj, Index begin, Index end) { + const bool is_compressed = obj.isCompressed(); + for (Index outer = begin; outer < end; outer++) { + Index begin_offset = obj.outerIndexPtr()[outer]; + Index end_offset = is_compressed ? obj.outerIndexPtr()[outer + 1] : (begin_offset + obj.innerNonZeroPtr()[outer]); + const StorageIndex* begin_it = obj.innerIndexPtr() + begin_offset; + const StorageIndex* end_it = obj.innerIndexPtr() + end_offset; + bool is_sorted = std::is_sorted(begin_it, end_it, Comp()); + if (!is_sorted) return outer; + } + return end; + } +}; +template +struct inner_sort_impl { + typedef typename Derived::Scalar Scalar; + typedef typename Derived::StorageIndex StorageIndex; + static inline void run(SparseCompressedBase& obj, Index, Index) { + Index begin_offset = 0; + Index end_offset = obj.nonZeros(); + CompressedStorageIterator begin_it(begin_offset, obj.innerIndexPtr(), obj.valuePtr()); + CompressedStorageIterator end_it(end_offset, obj.innerIndexPtr(), obj.valuePtr()); + std::sort(begin_it, end_it, Comp()); + } + static inline Index check(const SparseCompressedBase& obj, Index, Index) { + Index begin_offset = 0; + Index end_offset = obj.nonZeros(); + const StorageIndex* begin_it = obj.innerIndexPtr() + begin_offset; + const StorageIndex* end_it = obj.innerIndexPtr() + end_offset; + return std::is_sorted(begin_it, end_it, Comp()) ? 1 : 0; + } +}; + template struct evaluator > : evaluator_base diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCwiseBinaryOp.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCwiseBinaryOp.h index 9b0d3f98dcd..92be6d2720a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCwiseBinaryOp.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCwiseBinaryOp.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSE_CWISE_BINARY_OP_H #define EIGEN_SPARSE_CWISE_BINARY_OP_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { // Here we have to handle 3 cases: @@ -40,18 +43,21 @@ class CwiseBinaryOpImpl typedef CwiseBinaryOp Derived; typedef SparseMatrixBase Base; EIGEN_SPARSE_PUBLIC_INTERFACE(Derived) - CwiseBinaryOpImpl() - { - EIGEN_STATIC_ASSERT(( - (!internal::is_same::StorageKind, - typename internal::traits::StorageKind>::value) - || ((internal::evaluator::Flags&RowMajorBit) == (internal::evaluator::Flags&RowMajorBit))), - THE_STORAGE_ORDER_OF_BOTH_SIDES_MUST_MATCH); - } + EIGEN_STATIC_ASSERT(( + (!internal::is_same::StorageKind, + typename internal::traits::StorageKind>::value) + || ((internal::evaluator::Flags&RowMajorBit) == (internal::evaluator::Flags&RowMajorBit))), + THE_STORAGE_ORDER_OF_BOTH_SIDES_MUST_MATCH) }; namespace internal { +// The default evaluator performs an "arithmetic" operation on two input arrays. +// Given input arrays 'lhs' and 'rhs' and binary functor 'func', +// the sparse destination array 'dst' is evaluated as follows: +// if lhs(i,j) and rhs(i,j) are present, dst(i,j) = func(lhs(i,j), rhs(i,j)) +// if lhs(i,j) is present and rhs(i,j) is null, dst(i,j) = func(lhs(i,j), 0) +// if lhs(i,j) is null and rhs(i,j) is present, dst(i,j) = func(0, rhs(i,j)) // Generic "sparse OP sparse" template struct binary_sparse_evaluator; @@ -73,7 +79,7 @@ struct binary_evaluator, IteratorBased, Iterat public: EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer) - : m_lhsIter(aEval.m_lhsImpl,outer), m_rhsIter(aEval.m_rhsImpl,outer), m_functor(aEval.m_functor) + : m_lhsIter(aEval.m_lhsImpl,outer), m_rhsIter(aEval.m_rhsImpl,outer), m_functor(aEval.m_functor), m_value(Scalar(0)) { this->operator++(); } @@ -101,7 +107,6 @@ struct binary_evaluator, IteratorBased, Iterat } else { - m_value = Scalar(0); // this is to avoid a compilation warning m_id = -1; } return *this; @@ -369,32 +374,39 @@ struct binary_evaluator, Lhs, Rhs>, Iter // "sparse && sparse" template -struct binary_evaluator, IteratorBased, IteratorBased> - : sparse_conjunction_evaluator > +struct binary_evaluator, Lhs, Rhs>, IteratorBased, IteratorBased> + : sparse_conjunction_evaluator, Lhs, Rhs> > { - typedef CwiseBinaryOp XprType; + typedef CwiseBinaryOp, Lhs, Rhs> XprType; typedef sparse_conjunction_evaluator Base; explicit binary_evaluator(const XprType& xpr) : Base(xpr) {} }; // "dense && sparse" template -struct binary_evaluator, IndexBased, IteratorBased> - : sparse_conjunction_evaluator > +struct binary_evaluator, Lhs, Rhs>, IndexBased, IteratorBased> + : sparse_conjunction_evaluator, Lhs, Rhs> > { - typedef CwiseBinaryOp XprType; + typedef CwiseBinaryOp, Lhs, Rhs> XprType; typedef sparse_conjunction_evaluator Base; explicit binary_evaluator(const XprType& xpr) : Base(xpr) {} }; // "sparse && dense" template -struct binary_evaluator, IteratorBased, IndexBased> - : sparse_conjunction_evaluator > +struct binary_evaluator, Lhs, Rhs>, IteratorBased, IndexBased> + : sparse_conjunction_evaluator, Lhs, Rhs> > { - typedef CwiseBinaryOp XprType; + typedef CwiseBinaryOp, Lhs, Rhs> XprType; typedef sparse_conjunction_evaluator Base; explicit binary_evaluator(const XprType& xpr) : Base(xpr) {} }; +// The conjunction "^" evaluator performs a logical "and" or set "intersection" operation on two input arrays. +// Given input arrays 'lhs' and 'rhs' and binary functor 'func', +// the sparse destination array 'dst' is evaluated as follows: +// if lhs(i,j) and rhs(i,j) are present, dst(i,j) = func(lhs(i,j), rhs(i,j)) +// if lhs(i,j) is present and rhs(i,j) is null, dst(i,j) is null +// if lhs(i,j) is null and rhs(i,j) is present, dst(i,j) is null + // "sparse ^ sparse" template struct sparse_conjunction_evaluator @@ -627,6 +639,273 @@ struct sparse_conjunction_evaluator evaluator m_rhsImpl; }; +template::Kind, + typename RhsKind = typename evaluator_traits::Kind, + typename LhsScalar = typename traits::Scalar, + typename RhsScalar = typename traits::Scalar> struct sparse_disjunction_evaluator; + +// The disjunction "v" evaluator performs a logical "or" or set "union" operation on two input arrays. +// Given input arrays 'lhs' and 'rhs' and binary functor 'func', +// the sparse destination array 'dst' is evaluated as follows: +// if lhs(i,j) and rhs(i,j) are present, dst(i,j) = func(lhs(i,j), rhs(i,j)) +// if lhs(i,j) is present and rhs(i,j) is null, dst(i,j) = lhs(i,j) +// if lhs(i,j) is null and rhs(i,j) is present, dst(i,j) = rhs(i,j) + +// "sparse v sparse" +template +struct sparse_disjunction_evaluator : evaluator_base { + protected: + typedef typename XprType::Functor BinaryOp; + typedef typename XprType::Lhs LhsArg; + typedef typename XprType::Rhs RhsArg; + typedef typename evaluator::InnerIterator LhsIterator; + typedef typename evaluator::InnerIterator RhsIterator; + typedef typename XprType::StorageIndex StorageIndex; + typedef typename traits::Scalar Scalar; + + public: + class InnerIterator { + public: + EIGEN_STRONG_INLINE InnerIterator(const sparse_disjunction_evaluator& aEval, Index outer) + : m_lhsIter(aEval.m_lhsImpl, outer), + m_rhsIter(aEval.m_rhsImpl, outer), + m_functor(aEval.m_functor), + m_value(Scalar(0)) { + this->operator++(); + } + + EIGEN_STRONG_INLINE InnerIterator& operator++() { + if (m_lhsIter && m_rhsIter && (m_lhsIter.index() == m_rhsIter.index())) { + m_id = m_lhsIter.index(); + m_value = m_functor(m_lhsIter.value(), m_rhsIter.value()); + ++m_lhsIter; + ++m_rhsIter; + } else if (m_lhsIter && (!m_rhsIter || (m_lhsIter.index() < m_rhsIter.index()))) { + m_id = m_lhsIter.index(); + m_value = m_lhsIter.value(); + ++m_lhsIter; + } else if (m_rhsIter && (!m_lhsIter || (m_lhsIter.index() > m_rhsIter.index()))) { + m_id = m_rhsIter.index(); + m_value = m_rhsIter.value(); + ++m_rhsIter; + } else { + m_id = -1; + } + return *this; + } + + EIGEN_STRONG_INLINE Scalar value() const { return m_value; } + + EIGEN_STRONG_INLINE StorageIndex index() const { return m_id; } + EIGEN_STRONG_INLINE Index outer() const { return m_lhsIter.outer(); } + EIGEN_STRONG_INLINE Index row() const { return LhsArg::IsRowMajor ? m_lhsIter.row() : index(); } + EIGEN_STRONG_INLINE Index col() const { return LhsArg::IsRowMajor ? index() : m_lhsIter.col(); } + + EIGEN_STRONG_INLINE operator bool() const { return m_id >= 0; } + + protected: + LhsIterator m_lhsIter; + RhsIterator m_rhsIter; + const BinaryOp& m_functor; + Scalar m_value; + StorageIndex m_id; + }; + + enum { + CoeffReadCost = int(evaluator::CoeffReadCost) + int(evaluator::CoeffReadCost) + + int(functor_traits::Cost), + Flags = XprType::Flags + }; + + explicit sparse_disjunction_evaluator(const XprType& xpr) + : m_functor(xpr.functor()), m_lhsImpl(xpr.lhs()), m_rhsImpl(xpr.rhs()) { + EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits::Cost); + EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); + } + + inline Index nonZerosEstimate() const { return m_lhsImpl.nonZerosEstimate() + m_rhsImpl.nonZerosEstimate(); } + + protected: + const BinaryOp m_functor; + evaluator m_lhsImpl; + evaluator m_rhsImpl; +}; + +// "dense v sparse" +template +struct sparse_disjunction_evaluator : evaluator_base { + protected: + typedef typename XprType::Functor BinaryOp; + typedef typename XprType::Lhs LhsArg; + typedef typename XprType::Rhs RhsArg; + typedef evaluator LhsEvaluator; + typedef typename evaluator::InnerIterator RhsIterator; + typedef typename XprType::StorageIndex StorageIndex; + typedef typename traits::Scalar Scalar; + + public: + class InnerIterator { + enum { IsRowMajor = (int(RhsArg::Flags) & RowMajorBit) == RowMajorBit }; + + public: + EIGEN_STRONG_INLINE InnerIterator(const sparse_disjunction_evaluator& aEval, Index outer) + : m_lhsEval(aEval.m_lhsImpl), + m_rhsIter(aEval.m_rhsImpl, outer), + m_functor(aEval.m_functor), + m_value(0), + m_id(-1), + m_innerSize(aEval.m_expr.rhs().innerSize()) { + this->operator++(); + } + + EIGEN_STRONG_INLINE InnerIterator& operator++() { + ++m_id; + if (m_id < m_innerSize) { + Scalar lhsVal = m_lhsEval.coeff(IsRowMajor ? m_rhsIter.outer() : m_id, IsRowMajor ? m_id : m_rhsIter.outer()); + if (m_rhsIter && m_rhsIter.index() == m_id) { + m_value = m_functor(lhsVal, m_rhsIter.value()); + ++m_rhsIter; + } else + m_value = lhsVal; + } + + return *this; + } + + EIGEN_STRONG_INLINE Scalar value() const { + eigen_internal_assert(m_id < m_innerSize); + return m_value; + } + + EIGEN_STRONG_INLINE StorageIndex index() const { return m_id; } + EIGEN_STRONG_INLINE Index outer() const { return m_rhsIter.outer(); } + EIGEN_STRONG_INLINE Index row() const { return IsRowMajor ? m_rhsIter.outer() : m_id; } + EIGEN_STRONG_INLINE Index col() const { return IsRowMajor ? m_id : m_rhsIter.outer(); } + + EIGEN_STRONG_INLINE operator bool() const { return m_id < m_innerSize; } + + protected: + const evaluator& m_lhsEval; + RhsIterator m_rhsIter; + const BinaryOp& m_functor; + Scalar m_value; + StorageIndex m_id; + StorageIndex m_innerSize; + }; + + enum { + CoeffReadCost = int(evaluator::CoeffReadCost) + int(evaluator::CoeffReadCost) + + int(functor_traits::Cost), + Flags = XprType::Flags + }; + + explicit sparse_disjunction_evaluator(const XprType& xpr) + : m_functor(xpr.functor()), m_lhsImpl(xpr.lhs()), m_rhsImpl(xpr.rhs()), m_expr(xpr) { + EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits::Cost); + EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); + } + + inline Index nonZerosEstimate() const { return m_expr.size(); } + + protected: + const BinaryOp m_functor; + evaluator m_lhsImpl; + evaluator m_rhsImpl; + const XprType& m_expr; +}; + +// "sparse v dense" +template +struct sparse_disjunction_evaluator : evaluator_base { + protected: + typedef typename XprType::Functor BinaryOp; + typedef typename XprType::Lhs LhsArg; + typedef typename XprType::Rhs RhsArg; + typedef typename evaluator::InnerIterator LhsIterator; + typedef evaluator RhsEvaluator; + typedef typename XprType::StorageIndex StorageIndex; + typedef typename traits::Scalar Scalar; + + public: + class InnerIterator { + enum { IsRowMajor = (int(LhsArg::Flags) & RowMajorBit) == RowMajorBit }; + + public: + EIGEN_STRONG_INLINE InnerIterator(const sparse_disjunction_evaluator& aEval, Index outer) + : m_lhsIter(aEval.m_lhsImpl, outer), + m_rhsEval(aEval.m_rhsImpl), + m_functor(aEval.m_functor), + m_value(0), + m_id(-1), + m_innerSize(aEval.m_expr.lhs().innerSize()) { + this->operator++(); + } + + EIGEN_STRONG_INLINE InnerIterator& operator++() { + ++m_id; + if (m_id < m_innerSize) { + Scalar rhsVal = m_rhsEval.coeff(IsRowMajor ? m_lhsIter.outer() : m_id, IsRowMajor ? m_id : m_lhsIter.outer()); + if (m_lhsIter && m_lhsIter.index() == m_id) { + m_value = m_functor(m_lhsIter.value(), rhsVal); + ++m_lhsIter; + } else + m_value = rhsVal; + } + + return *this; + } + + EIGEN_STRONG_INLINE Scalar value() const { + eigen_internal_assert(m_id < m_innerSize); + return m_value; + } + + EIGEN_STRONG_INLINE StorageIndex index() const { return m_id; } + EIGEN_STRONG_INLINE Index outer() const { return m_lhsIter.outer(); } + EIGEN_STRONG_INLINE Index row() const { return IsRowMajor ? m_lhsIter.outer() : m_id; } + EIGEN_STRONG_INLINE Index col() const { return IsRowMajor ? m_id : m_lhsIter.outer(); } + + EIGEN_STRONG_INLINE operator bool() const { return m_id < m_innerSize; } + + protected: + LhsIterator m_lhsIter; + const evaluator& m_rhsEval; + const BinaryOp& m_functor; + Scalar m_value; + StorageIndex m_id; + StorageIndex m_innerSize; + }; + + enum { + CoeffReadCost = int(evaluator::CoeffReadCost) + int(evaluator::CoeffReadCost) + + int(functor_traits::Cost), + Flags = XprType::Flags + }; + + explicit sparse_disjunction_evaluator(const XprType& xpr) + : m_functor(xpr.functor()), m_lhsImpl(xpr.lhs()), m_rhsImpl(xpr.rhs()), m_expr(xpr) { + EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits::Cost); + EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); + } + + inline Index nonZerosEstimate() const { return m_expr.size(); } + + protected: + const BinaryOp m_functor; + evaluator m_lhsImpl; + evaluator m_rhsImpl; + const XprType& m_expr; +}; + +// when DupFunc is wrapped with scalar_dup_op, use disjunction evaulator +template +struct binary_evaluator, Lhs, Rhs>, IteratorBased, IteratorBased> + : sparse_disjunction_evaluator, Lhs, Rhs> > { + typedef CwiseBinaryOp, Lhs, Rhs> XprType; + typedef sparse_disjunction_evaluator Base; + explicit binary_evaluator(const XprType& xpr) : Base(xpr) {} +}; } /*************************************************************************** diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCwiseUnaryOp.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCwiseUnaryOp.h index 32dac0f7863..974b22f440e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCwiseUnaryOp.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseCwiseUnaryOp.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSE_CWISE_UNARY_OP_H #define EIGEN_SPARSE_CWISE_UNARY_OP_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDenseProduct.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDenseProduct.h index f005a18a18e..6c3cd29025b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDenseProduct.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDenseProduct.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSEDENSEPRODUCT_H #define EIGEN_SPARSEDENSEPRODUCT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -26,9 +29,9 @@ struct sparse_time_dense_product_impl; template struct sparse_time_dense_product_impl { - typedef typename internal::remove_all::type Lhs; - typedef typename internal::remove_all::type Rhs; - typedef typename internal::remove_all::type Res; + typedef internal::remove_all_t Lhs; + typedef internal::remove_all_t Rhs; + typedef internal::remove_all_t Res; typedef typename evaluator::InnerIterator LhsInnerIterator; typedef evaluator LhsEval; static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const typename Res::Scalar& alpha) @@ -63,18 +66,26 @@ struct sparse_time_dense_product_impl let's disable it for now as it is conflicting with generic scalar*matrix and matrix*scalar operators -// template -// struct ScalarBinaryOpTraits > +// template +// struct ScalarBinaryOpTraits > // { // enum { // Defined = 1 @@ -85,9 +96,9 @@ struct sparse_time_dense_product_impl struct sparse_time_dense_product_impl { - typedef typename internal::remove_all::type Lhs; - typedef typename internal::remove_all::type Rhs; - typedef typename internal::remove_all::type Res; + typedef internal::remove_all_t Lhs; + typedef internal::remove_all_t Rhs; + typedef internal::remove_all_t Res; typedef evaluator LhsEval; typedef typename LhsEval::InnerIterator LhsInnerIterator; static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const AlphaType& alpha) @@ -109,9 +120,9 @@ struct sparse_time_dense_product_impl struct sparse_time_dense_product_impl { - typedef typename internal::remove_all::type Lhs; - typedef typename internal::remove_all::type Rhs; - typedef typename internal::remove_all::type Res; + typedef internal::remove_all_t Lhs; + typedef internal::remove_all_t Rhs; + typedef internal::remove_all_t Res; typedef evaluator LhsEval; typedef typename LhsEval::InnerIterator LhsInnerIterator; static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const typename Res::Scalar& alpha) @@ -149,9 +160,9 @@ struct sparse_time_dense_product_impl struct sparse_time_dense_product_impl { - typedef typename internal::remove_all::type Lhs; - typedef typename internal::remove_all::type Rhs; - typedef typename internal::remove_all::type Res; + typedef internal::remove_all_t Lhs; + typedef internal::remove_all_t Rhs; + typedef internal::remove_all_t Res; typedef typename evaluator::InnerIterator LhsInnerIterator; static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const typename Res::Scalar& alpha) { @@ -226,16 +237,16 @@ template struct sparse_dense_outer_product_evaluator { protected: - typedef typename conditional::type Lhs1; - typedef typename conditional::type ActualRhs; + typedef std::conditional_t Lhs1; + typedef std::conditional_t ActualRhs; typedef Product ProdXprType; // if the actual left-hand side is a dense vector, // then build a sparse-view so that we can seamlessly iterate over it. - typedef typename conditional::StorageKind,Sparse>::value, - Lhs1, SparseView >::type ActualLhs; - typedef typename conditional::StorageKind,Sparse>::value, - Lhs1 const&, SparseView >::type LhsArg; + typedef std::conditional_t::StorageKind,Sparse>::value, + Lhs1, SparseView > ActualLhs; + typedef std::conditional_t::StorageKind,Sparse>::value, + Lhs1 const&, SparseView > LhsArg; typedef evaluator LhsEval; typedef evaluator RhsEval; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDiagonalProduct.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDiagonalProduct.h index 941c03be3de..49bc851272d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDiagonalProduct.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDiagonalProduct.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSE_DIAGONAL_PRODUCT_H #define EIGEN_SPARSE_DIAGONAL_PRODUCT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { // The product of a diagonal matrix with a sparse matrix can be easily diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDot.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDot.h index 38bc4aa9ea0..826c09b954f 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDot.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseDot.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSE_DOT_H #define EIGEN_SPARSE_DOT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseFuzzy.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseFuzzy.h index 7d47eb94d2f..7ca42432064 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseFuzzy.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseFuzzy.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSE_FUZZY_H #define EIGEN_SPARSE_FUZZY_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template @@ -17,9 +20,9 @@ template bool SparseMatrixBase::isApprox(const SparseMatrixBase& other, const RealScalar &prec) const { const typename internal::nested_eval::type actualA(derived()); - typename internal::conditional::type, - const PlainObject>::type actualB(other.derived()); + const PlainObject> actualB(other.derived()); return (actualA - actualB).squaredNorm() <= prec * prec * numext::mini(actualA.squaredNorm(), actualB.squaredNorm()); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMap.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMap.h index f99be3379db..f2923a3f01d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMap.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMap.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSE_MAP_H #define EIGEN_SPARSE_MAP_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -58,12 +61,12 @@ class SparseMapBase using Base::operator=; protected: - typedef typename internal::conditional< - bool(internal::is_lvalue::value), - Scalar *, const Scalar *>::type ScalarPointer; - typedef typename internal::conditional< - bool(internal::is_lvalue::value), - StorageIndex *, const StorageIndex *>::type IndexPointer; + typedef std::conditional_t< + bool(internal::is_lvalue::value), + Scalar *, const Scalar *> ScalarPointer; + typedef std::conditional_t< + bool(internal::is_lvalue::value), + StorageIndex *, const StorageIndex *> IndexPointer; Index m_outerSize; Index m_innerSize; @@ -237,6 +240,7 @@ class Map /** Constructs a read-write Map to a sparse matrix of size \a rows x \a cols, containing \a nnz non-zero coefficients, * stored as a sparse format as defined by the pointers \a outerIndexPtr, \a innerIndexPtr, and \a valuePtr. * If the optional parameter \a innerNonZerosPtr is the null pointer, then a standard compressed format is assumed. + * The inner indices must be sorted appropriately. * * This constructor is available only if \c SparseMatrixType is non-const. * diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMatrix.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMatrix.h index 616b4a0c24e..8537c798b27 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMatrix.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMatrix.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSEMATRIX_H #define EIGEN_SPARSEMATRIX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \ingroup SparseCore_Module @@ -29,10 +32,10 @@ namespace Eigen { * * More details on this storage sceheme are given in the \ref TutorialSparse "manual pages". * - * \tparam _Scalar the scalar type, i.e. the type of the coefficients - * \tparam _Options Union of bit flags controlling the storage scheme. Currently the only possibility + * \tparam Scalar_ the scalar type, i.e. the type of the coefficients + * \tparam Options_ Union of bit flags controlling the storage scheme. Currently the only possibility * is ColMajor or RowMajor. The default is 0 which means column-major. - * \tparam _StorageIndex the type of the indices. It has to be a \b signed type (e.g., short, int, std::ptrdiff_t). Default is \c int. + * \tparam StorageIndex_ the type of the indices. It has to be a \b signed type (e.g., short, int, std::ptrdiff_t). Default is \c int. * * \warning In %Eigen 3.2, the undocumented type \c SparseMatrix::Index was improperly defined as the storage index type (e.g., int), * whereas it is now (starting from %Eigen 3.3) deprecated and always defined as Eigen::Index. @@ -43,11 +46,11 @@ namespace Eigen { */ namespace internal { -template -struct traits > +template +struct traits > { - typedef _Scalar Scalar; - typedef _StorageIndex StorageIndex; + typedef Scalar_ Scalar; + typedef StorageIndex_ StorageIndex; typedef Sparse StorageKind; typedef MatrixXpr XprKind; enum { @@ -55,21 +58,21 @@ struct traits > ColsAtCompileTime = Dynamic, MaxRowsAtCompileTime = Dynamic, MaxColsAtCompileTime = Dynamic, - Flags = _Options | NestByRefBit | LvalueBit | CompressedAccessBit, + Flags = Options_ | NestByRefBit | LvalueBit | CompressedAccessBit, SupportedAccessPatterns = InnerRandomAccessPattern }; }; -template -struct traits, DiagIndex> > +template +struct traits, DiagIndex> > { - typedef SparseMatrix<_Scalar, _Options, _StorageIndex> MatrixType; + typedef SparseMatrix MatrixType; typedef typename ref_selector::type MatrixTypeNested; - typedef typename remove_reference::type _MatrixTypeNested; + typedef std::remove_reference_t MatrixTypeNested_; - typedef _Scalar Scalar; + typedef Scalar_ Scalar; typedef Dense StorageKind; - typedef _StorageIndex StorageIndex; + typedef StorageIndex_ StorageIndex; typedef MatrixXpr XprKind; enum { @@ -81,24 +84,49 @@ struct traits, DiagIndex }; }; -template -struct traits, DiagIndex> > - : public traits, DiagIndex> > +template +struct traits, DiagIndex> > + : public traits, DiagIndex> > { enum { Flags = 0 }; }; +template +struct sparse_reserve_op { + EIGEN_DEVICE_FUNC sparse_reserve_op(Index begin, Index end, Index size) { + Index range = numext::mini(end - begin, size); + m_begin = begin; + m_end = begin + range; + m_val = StorageIndex(size / range); + m_remainder = StorageIndex(size % range); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageIndex operator()(IndexType i) const { + if ((i >= m_begin) && (i < m_end)) + return m_val + ((i - m_begin) < m_remainder ? 1 : 0); + else + return 0; + } + StorageIndex m_val, m_remainder; + Index m_begin, m_end; +}; + +template +struct functor_traits> { + enum { Cost = 1, PacketAccess = false, IsRepeatable = true }; +}; + } // end namespace internal -template +template class SparseMatrix - : public SparseCompressedBase > + : public SparseCompressedBase > { typedef SparseCompressedBase Base; using Base::convert_index; - friend class SparseVector<_Scalar,0,_StorageIndex>; + friend class SparseVector; template friend struct internal::Assignment; public: @@ -108,7 +136,7 @@ class SparseMatrix using Base::operator+=; using Base::operator-=; - typedef MappedSparseMatrix Map; + typedef Eigen::Map> Map; typedef Diagonal DiagonalReturnType; typedef Diagonal ConstDiagonalReturnType; typedef typename Base::InnerIterator InnerIterator; @@ -118,13 +146,13 @@ class SparseMatrix using Base::IsRowMajor; typedef internal::CompressedStorage Storage; enum { - Options = _Options + Options = Options_ }; typedef typename Base::IndexVector IndexVector; typedef typename Base::ScalarVector ScalarVector; protected: - typedef SparseMatrix TransposedSparseMatrix; + typedef SparseMatrix TransposedSparseMatrix; Index m_outerSize; Index m_innerSize; @@ -194,7 +222,7 @@ class SparseMatrix const Index outer = IsRowMajor ? row : col; const Index inner = IsRowMajor ? col : row; Index end = m_innerNonZeros ? m_outerIndex[outer] + m_innerNonZeros[outer] : m_outerIndex[outer+1]; - return m_data.atInRange(m_outerIndex[outer], end, StorageIndex(inner)); + return m_data.atInRange(m_outerIndex[outer], end, inner); } /** \returns a non-const reference to the value of the matrix at position \a i, \a j @@ -208,20 +236,28 @@ class SparseMatrix inline Scalar& coeffRef(Index row, Index col) { eigen_assert(row>=0 && row=0 && col=start && "you probably called coeffRef on a non finalized matrix"); - if(end<=start) - return insert(row,col); - const Index p = m_data.searchLowerIndex(start,end-1,StorageIndex(inner)); - if((p= start && "you probably called coeffRef on a non finalized matrix"); + Index dst = start == end ? end : m_data.searchLowerIndex(start, end, inner); + if (dst == end) { + Index capacity = m_outerIndex[outer + 1] - end; + if (capacity > 0) { + // implies uncompressed: push to back of vector + m_innerNonZeros[outer]++; + m_data.index(end) = StorageIndex(inner); + m_data.value(end) = Scalar(0); + return m_data.value(end); + } + } + if ((dst < end) && (m_data.index(dst) == inner)) + // this coefficient exists, return a refernece to it + return m_data.value(dst); else - return insert(row,col); + // insertion will require reconfiguring the buffer + return insertAtByOuterInner(outer, inner, dst); } /** \returns a reference to a novel non zero coefficient with coordinates \a row x \a col. @@ -239,7 +275,7 @@ class SparseMatrix * if the elements of each inner vector are inserted in increasing inner index order, and in O(nnz_j) for a random insertion. * */ - Scalar& insert(Index row, Index col); + inline Scalar& insert(Index row, Index col); public: @@ -253,9 +289,10 @@ class SparseMatrix inline void setZero() { m_data.clear(); - memset(m_outerIndex, 0, (m_outerSize+1)*sizeof(StorageIndex)); - if(m_innerNonZeros) - memset(m_innerNonZeros, 0, (m_outerSize)*sizeof(StorageIndex)); + std::fill_n(m_outerIndex, m_outerSize + 1, StorageIndex(0)); + if(m_innerNonZeros) { + std::fill_n(m_innerNonZeros, m_outerSize, StorageIndex(0)); + } } /** Preallocates \a reserveSize non zeros. @@ -285,10 +322,7 @@ class SparseMatrix #else template inline void reserve(const SizesType& reserveSizes, const typename SizesType::value_type& enableif = - #if (!EIGEN_COMP_MSVC) || (EIGEN_COMP_MSVC>=1500) // MSVC 2005 fails to compile with this typename - typename - #endif - SizesType::value_type()) + typename SizesType::value_type()) { EIGEN_UNUSED_VARIABLE(enableif); reserveInnerVectors(reserveSizes); @@ -301,71 +335,70 @@ class SparseMatrix if(isCompressed()) { Index totalReserveSize = 0; + for (Index j = 0; j < m_outerSize; ++j) totalReserveSize += internal::convert_index(reserveSizes[j]); + + // if reserveSizes is empty, don't do anything! + if (totalReserveSize == 0) return; + // turn the matrix into non-compressed mode - m_innerNonZeros = static_cast(std::malloc(m_outerSize * sizeof(StorageIndex))); - if (!m_innerNonZeros) internal::throw_std_bad_alloc(); + m_innerNonZeros = internal::conditional_aligned_new_auto(m_outerSize); // temporarily use m_innerSizes to hold the new starting points. StorageIndex* newOuterIndex = m_innerNonZeros; - StorageIndex count = 0; + Index count = 0; for(Index j=0; j(count); + Index reserveSize = internal::convert_index(reserveSizes[j]); + count += reserveSize + internal::convert_index(m_outerIndex[j+1]-m_outerIndex[j]); } + m_data.reserve(totalReserveSize); StorageIndex previousOuterIndex = m_outerIndex[m_outerSize]; for(Index j=m_outerSize-1; j>=0; --j) { StorageIndex innerNNZ = previousOuterIndex - m_outerIndex[j]; - for(Index i=innerNNZ-1; i>=0; --i) - { - m_data.index(newOuterIndex[j]+i) = m_data.index(m_outerIndex[j]+i); - m_data.value(newOuterIndex[j]+i) = m_data.value(m_outerIndex[j]+i); - } + StorageIndex begin = m_outerIndex[j]; + StorageIndex end = begin + innerNNZ; + StorageIndex target = newOuterIndex[j]; + internal::smart_memmove(innerIndexPtr() + begin, innerIndexPtr() + end, innerIndexPtr() + target); + internal::smart_memmove(valuePtr() + begin, valuePtr() + end, valuePtr() + target); previousOuterIndex = m_outerIndex[j]; m_outerIndex[j] = newOuterIndex[j]; m_innerNonZeros[j] = innerNNZ; } if(m_outerSize>0) - m_outerIndex[m_outerSize] = m_outerIndex[m_outerSize-1] + m_innerNonZeros[m_outerSize-1] + reserveSizes[m_outerSize-1]; + m_outerIndex[m_outerSize] = m_outerIndex[m_outerSize-1] + m_innerNonZeros[m_outerSize-1] + internal::convert_index(reserveSizes[m_outerSize-1]); m_data.resize(m_outerIndex[m_outerSize]); } else { - StorageIndex* newOuterIndex = static_cast(std::malloc((m_outerSize+1)*sizeof(StorageIndex))); - if (!newOuterIndex) internal::throw_std_bad_alloc(); + StorageIndex* newOuterIndex = internal::conditional_aligned_new_auto(m_outerSize + 1); - StorageIndex count = 0; + Index count = 0; for(Index j=0; j(reserveSizes[j], alreadyReserved); - count += toReserve + m_innerNonZeros[j]; + newOuterIndex[j] = internal::convert_index(count); + Index alreadyReserved = internal::convert_index(m_outerIndex[j+1] - m_outerIndex[j] - m_innerNonZeros[j]); + Index reserveSize = internal::convert_index(reserveSizes[j]); + Index toReserve = numext::maxi(reserveSize, alreadyReserved); + count += toReserve + internal::convert_index(m_innerNonZeros[j]); } - newOuterIndex[m_outerSize] = count; - + newOuterIndex[m_outerSize] = internal::convert_index(count); + m_data.resize(count); for(Index j=m_outerSize-1; j>=0; --j) { - Index offset = newOuterIndex[j] - m_outerIndex[j]; - if(offset>0) - { - StorageIndex innerNNZ = m_innerNonZeros[j]; - for(Index i=innerNNZ-1; i>=0; --i) - { - m_data.index(newOuterIndex[j]+i) = m_data.index(m_outerIndex[j]+i); - m_data.value(newOuterIndex[j]+i) = m_data.value(m_outerIndex[j]+i); - } - } + StorageIndex innerNNZ = m_innerNonZeros[j]; + StorageIndex begin = m_outerIndex[j]; + StorageIndex target = newOuterIndex[j]; + m_data.moveChunk(begin, target, innerNNZ); } std::swap(m_outerIndex, newOuterIndex); - std::free(newOuterIndex); + internal::conditional_aligned_delete_auto(newOuterIndex, m_outerSize + 1); } } @@ -394,7 +427,7 @@ class SparseMatrix { eigen_assert(Index(m_outerIndex[outer+1]) == m_data.size() && "Invalid ordered insertion (invalid outer index)"); eigen_assert( (m_outerIndex[outer+1]-m_outerIndex[outer]==0 || m_data.index(m_data.size()-1)= 0 && j >= 0 && j + num <= m_outerSize && "Invalid parameters"); + + const Index newRows = IsRowMajor ? m_outerSize - num : rows(); + const Index newCols = IsRowMajor ? cols() : m_outerSize - num; + + const Index begin = j + num; + const Index end = m_outerSize; + const Index target = j; + + // if the removed vectors are not empty, uncompress the matrix + if (m_outerIndex[j + num] > m_outerIndex[j]) uncompress(); + + // shift m_outerIndex and m_innerNonZeros [num] to the left + internal::smart_memmove(m_outerIndex + begin, m_outerIndex + end + 1, m_outerIndex + target); + if (!isCompressed()) + internal::smart_memmove(m_innerNonZeros + begin, m_innerNonZeros + end, m_innerNonZeros + target); + + // if m_outerIndex[0] > 0, shift the data within the first vector while it is easy to do so + if (m_outerIndex[0] > StorageIndex(0)) { + uncompress(); + const Index from = internal::convert_index(m_outerIndex[0]); + const Index to = Index(0); + const Index chunkSize = internal::convert_index(m_innerNonZeros[0]); + m_data.moveChunk(from, to, chunkSize); + m_outerIndex[0] = StorageIndex(0); + } + + // truncate the matrix to the smaller size + conservativeResize(newRows, newCols); + } + + // insert empty outer vectors at indices j, j+1 ... j+num-1 and resize the matrix + void insertEmptyOuterVectors(Index j, Index num = 1) { + EIGEN_USING_STD(fill_n); + eigen_assert(num >= 0 && j >= 0 && j < m_outerSize && "Invalid parameters"); + + const Index newRows = IsRowMajor ? m_outerSize + num : rows(); + const Index newCols = IsRowMajor ? cols() : m_outerSize + num; + + const Index begin = j; + const Index end = m_outerSize; + const Index target = j + num; + + // expand the matrix to the larger size + conservativeResize(newRows, newCols); + + // shift m_outerIndex and m_innerNonZeros [num] to the right + internal::smart_memmove(m_outerIndex + begin, m_outerIndex + end + 1, m_outerIndex + target); + // m_outerIndex[begin] == m_outerIndex[target], set all indices in this range to same value + fill_n(m_outerIndex + begin, num, m_outerIndex[begin]); + + if (!isCompressed()) { + internal::smart_memmove(m_innerNonZeros + begin, m_innerNonZeros + end, m_innerNonZeros + target); + // set the nonzeros of the newly inserted vectors to 0 + fill_n(m_innerNonZeros + begin, num, StorageIndex(0)); + } + } template void setFromTriplets(const InputIterators& begin, const InputIterators& end); @@ -448,10 +539,26 @@ class SparseMatrix template void setFromTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func); - void sumupDuplicates() { collapseDuplicates(internal::scalar_sum_op()); } + template + void collapseDuplicates(DenseBase& wi, DupFunctor dup_func = DupFunctor()); + + template + void setFromSortedTriplets(const InputIterators& begin, const InputIterators& end); + + template + void setFromSortedTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func); + + template + void insertFromTriplets(const InputIterators& begin, const InputIterators& end); + + template + void insertFromTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func); + + template + void insertFromSortedTriplets(const InputIterators& begin, const InputIterators& end); - template - void collapseDuplicates(DupFunctor dup_func = DupFunctor()); + template + void insertFromSortedTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func); //--- @@ -459,51 +566,70 @@ class SparseMatrix * same as insert(Index,Index) except that the indices are given relative to the storage order */ Scalar& insertByOuterInner(Index j, Index i) { - return insert(IsRowMajor ? j : i, IsRowMajor ? i : j); + Index start = m_outerIndex[j]; + Index end = isCompressed() ? m_outerIndex[j + 1] : start + m_innerNonZeros[j]; + Index dst = start == end ? end : m_data.searchLowerIndex(start, end, i); + if (dst == end) { + Index capacity = m_outerIndex[j + 1] - end; + if (capacity > 0) { + // implies uncompressed: push to back of vector + m_innerNonZeros[j]++; + m_data.index(end) = StorageIndex(i); + m_data.value(end) = Scalar(0); + return m_data.value(end); + } + } + eigen_assert((dst == end || m_data.index(dst) != i) && + "you cannot insert an element that already exists, you must call coeffRef to this end"); + return insertAtByOuterInner(j, i, dst); } /** Turns the matrix into the \em compressed format. */ void makeCompressed() { - if(isCompressed()) - return; + if (isCompressed()) return; eigen_internal_assert(m_outerIndex!=0 && m_outerSize>0); - - Index oldStart = m_outerIndex[1]; + + StorageIndex start = m_outerIndex[1]; m_outerIndex[1] = m_innerNonZeros[0]; - for(Index j=1; j0) + StorageIndex end = start + m_innerNonZeros[j]; + StorageIndex nextStart = m_outerIndex[j + 1]; + // dont forget to move the last chunk! + bool breakUpCopy = (end != nextStart) || (j == m_outerSize - 1); + if (breakUpCopy) { - for(Index k=0; k 0) m_data.moveChunk(copyStart, copyTarget, chunkSize); + copyStart = nextStart; + copyTarget += chunkSize; } - m_outerIndex[j+1] = m_outerIndex[j] + m_innerNonZeros[j]; - oldStart = nextOldStart; + start = nextStart; + m_outerIndex[j + 1] = m_outerIndex[j] + m_innerNonZeros[j]; } - std::free(m_innerNonZeros); - m_innerNonZeros = 0; m_data.resize(m_outerIndex[m_outerSize]); + + // release as much memory as possible + internal::conditional_aligned_delete_auto(m_innerNonZeros, m_outerSize); + m_innerNonZeros = 0; m_data.squeeze(); } /** Turns the matrix into the uncompressed mode */ void uncompress() { - if(m_innerNonZeros != 0) - return; - m_innerNonZeros = static_cast(std::malloc(m_outerSize * sizeof(StorageIndex))); - for (Index i = 0; i < m_outerSize; i++) - { - m_innerNonZeros[i] = m_outerIndex[i+1] - m_outerIndex[i]; - } + if (!isCompressed()) return; + m_innerNonZeros = internal::conditional_aligned_new_auto(m_outerSize); + if (m_outerIndex[m_outerSize] == 0) + std::fill_n(m_innerNonZeros, m_outerSize, StorageIndex(0)); + else + for (Index j = 0; j < m_outerSize; j++) m_innerNonZeros[j] = m_outerIndex[j + 1] - m_outerIndex[j]; } /** Suppresses all nonzeros which are \b much \b smaller \b than \a reference under the tolerance \a epsilon */ @@ -522,27 +648,32 @@ class SparseMatrix template void prune(const KeepFunc& keep = KeepFunc()) { - // TODO optimize the uncompressed mode to avoid moving and allocating the data twice - makeCompressed(); - StorageIndex k = 0; for(Index j=0; jrows() == rows && this->cols() == cols) return; - + void conservativeResize(Index rows, Index cols) { + // If one dimension is null, then there is nothing to be preserved - if(rows==0 || cols==0) return resize(rows,cols); + if (rows == 0 || cols == 0) return resize(rows, cols); - Index innerChange = IsRowMajor ? cols - this->cols() : rows - this->rows(); - Index outerChange = IsRowMajor ? rows - this->rows() : cols - this->cols(); - StorageIndex newInnerSize = convert_index(IsRowMajor ? cols : rows); + Index newOuterSize = IsRowMajor ? rows : cols; + Index newInnerSize = IsRowMajor ? cols : rows; - // Deals with inner non zeros - if (m_innerNonZeros) - { - // Resize m_innerNonZeros - StorageIndex *newInnerNonZeros = static_cast(std::realloc(m_innerNonZeros, (m_outerSize + outerChange) * sizeof(StorageIndex))); - if (!newInnerNonZeros) internal::throw_std_bad_alloc(); - m_innerNonZeros = newInnerNonZeros; - - for(Index i=m_outerSize; i(std::malloc((m_outerSize + outerChange) * sizeof(StorageIndex))); - if (!m_innerNonZeros) internal::throw_std_bad_alloc(); - for(Index i = 0; i < m_outerSize + (std::min)(outerChange, Index(0)); i++) - m_innerNonZeros[i] = m_outerIndex[i+1] - m_outerIndex[i]; - for(Index i = m_outerSize; i < m_outerSize + outerChange; i++) - m_innerNonZeros[i] = 0; + Index innerChange = newInnerSize - m_innerSize; + Index outerChange = newOuterSize - m_outerSize; + + if (outerChange != 0) { + m_outerIndex = internal::conditional_aligned_realloc_new_auto( + m_outerIndex, newOuterSize + 1, m_outerSize + 1); + + if (!isCompressed()) + m_innerNonZeros = internal::conditional_aligned_realloc_new_auto( + m_innerNonZeros, newOuterSize, m_outerSize); + + if (outerChange > 0) { + StorageIndex lastIdx = m_outerSize == 0 ? StorageIndex(0) : m_outerIndex[m_outerSize]; + std::fill_n(m_outerIndex + m_outerSize, outerChange + 1, lastIdx); + + if (!isCompressed()) std::fill_n(m_innerNonZeros + m_outerSize, outerChange, StorageIndex(0)); + } } - - // Change the m_innerNonZeros in case of a decrease of inner size - if (m_innerNonZeros && innerChange < 0) - { - for(Index i = 0; i < m_outerSize + (std::min)(outerChange, Index(0)); i++) - { - StorageIndex &n = m_innerNonZeros[i]; - StorageIndex start = m_outerIndex[i]; - while (n > 0 && m_data.index(start+n-1) >= newInnerSize) --n; + m_outerSize = newOuterSize; + + if (innerChange < 0) { + for (Index j = 0; j < m_outerSize; j++) { + Index start = m_outerIndex[j]; + Index end = isCompressed() ? m_outerIndex[j + 1] : start + m_innerNonZeros[j]; + Index lb = m_data.searchLowerIndex(start, end, newInnerSize); + if (lb != end) { + uncompress(); + m_innerNonZeros[j] = StorageIndex(lb - start); + } } } - m_innerSize = newInnerSize; - // Re-allocate outer index structure if necessary - if (outerChange == 0) - return; - - StorageIndex *newOuterIndex = static_cast(std::realloc(m_outerIndex, (m_outerSize + outerChange + 1) * sizeof(StorageIndex))); - if (!newOuterIndex) internal::throw_std_bad_alloc(); - m_outerIndex = newOuterIndex; - if (outerChange > 0) - { - StorageIndex lastIdx = m_outerSize == 0 ? 0 : m_outerIndex[m_outerSize]; - for(Index i=m_outerSize; i(std::malloc((outerSize + 1) * sizeof(StorageIndex))); - if (!m_outerIndex) internal::throw_std_bad_alloc(); - + + if ((m_outerIndex == 0) || (m_outerSize != outerSize)) { + m_outerIndex = internal::conditional_aligned_realloc_new_auto(m_outerIndex, outerSize + 1, m_outerSize + 1); m_outerSize = outerSize; } - if(m_innerNonZeros) - { - std::free(m_innerNonZeros); - m_innerNonZeros = 0; - } - memset(m_outerIndex, 0, (m_outerSize+1)*sizeof(StorageIndex)); + + internal::conditional_aligned_delete_auto(m_innerNonZeros, m_outerSize); + m_innerNonZeros = 0; + + std::fill_n(m_outerIndex, m_outerSize + 1, StorageIndex(0)); } /** \internal @@ -662,9 +772,8 @@ class SparseMatrix /** Default constructor yielding an empty \c 0 \c x \c 0 matrix */ inline SparseMatrix() - : m_outerSize(-1), m_innerSize(0), m_outerIndex(0), m_innerNonZeros(0) + : m_outerSize(0), m_innerSize(0), m_outerIndex(0), m_innerNonZeros(0) { - check_template_parameters(); resize(0, 0); } @@ -672,7 +781,6 @@ class SparseMatrix inline SparseMatrix(Index rows, Index cols) : m_outerSize(0), m_innerSize(0), m_outerIndex(0), m_innerNonZeros(0) { - check_template_parameters(); resize(rows, cols); } @@ -683,7 +791,6 @@ class SparseMatrix { EIGEN_STATIC_ASSERT((internal::is_same::value), YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) - check_template_parameters(); const bool needToTranspose = (Flags & RowMajorBit) != (internal::evaluator::Flags & RowMajorBit); if (needToTranspose) *this = other.derived(); @@ -695,21 +802,24 @@ class SparseMatrix internal::call_assignment_no_alias(*this, other.derived()); } } - + /** Constructs a sparse matrix from the sparse selfadjoint view \a other */ template inline SparseMatrix(const SparseSelfAdjointView& other) : m_outerSize(0), m_innerSize(0), m_outerIndex(0), m_innerNonZeros(0) { - check_template_parameters(); Base::operator=(other); } + inline SparseMatrix(SparseMatrix&& other) : Base(), m_outerSize(0), m_innerSize(0), m_outerIndex(0), m_innerNonZeros(0) + { + *this = other.derived().markAsRValue(); + } + /** Copy constructor (it performs a deep copy) */ inline SparseMatrix(const SparseMatrix& other) : Base(), m_outerSize(0), m_innerSize(0), m_outerIndex(0), m_innerNonZeros(0) { - check_template_parameters(); *this = other.derived(); } @@ -718,17 +828,15 @@ class SparseMatrix SparseMatrix(const ReturnByValue& other) : Base(), m_outerSize(0), m_innerSize(0), m_outerIndex(0), m_innerNonZeros(0) { - check_template_parameters(); initAssignment(other); other.evalTo(*this); } - + /** \brief Copy constructor with in-place evaluation */ template explicit SparseMatrix(const DiagonalBase& other) : Base(), m_outerSize(0), m_innerSize(0), m_outerIndex(0), m_innerNonZeros(0) { - check_template_parameters(); *this = other.derived(); } @@ -748,14 +856,17 @@ class SparseMatrix * This function also turns the matrix into compressed mode, and drop any reserved memory. */ inline void setIdentity() { - eigen_assert(rows() == cols() && "ONLY FOR SQUARED MATRICES"); - this->m_data.resize(rows()); - Eigen::Map(this->m_data.indexPtr(), rows()).setLinSpaced(0, StorageIndex(rows()-1)); - Eigen::Map(this->m_data.valuePtr(), rows()).setOnes(); - Eigen::Map(this->m_outerIndex, rows()+1).setLinSpaced(0, StorageIndex(rows())); - std::free(m_innerNonZeros); + eigen_assert(m_outerSize == m_innerSize && "ONLY FOR SQUARED MATRICES"); + internal::conditional_aligned_delete_auto(m_innerNonZeros, m_outerSize); m_innerNonZeros = 0; + m_data.resize(m_outerSize); + // is it necessary to squeeze? + m_data.squeeze(); + std::iota(m_outerIndex, m_outerIndex + m_outerSize + 1, StorageIndex(0)); + std::iota(innerIndexPtr(), innerIndexPtr() + m_outerSize, StorageIndex(0)); + std::fill_n(valuePtr(), m_outerSize, Scalar(1)); } + inline SparseMatrix& operator=(const SparseMatrix& other) { if (other.isRValue()) @@ -781,6 +892,10 @@ class SparseMatrix return *this; } + inline SparseMatrix& operator=(SparseMatrix&& other) { + return *this = other.derived().markAsRValue(); + } + #ifndef EIGEN_PARSED_BY_DOXYGEN template inline SparseMatrix& operator=(const EigenBase& other) @@ -793,6 +908,7 @@ class SparseMatrix template EIGEN_DONT_INLINE SparseMatrix& operator=(const SparseMatrixBase& other); +#ifndef EIGEN_NO_IO friend std::ostream & operator << (std::ostream & s, const SparseMatrix& m) { EIGEN_DBG_SPARSE( @@ -837,12 +953,13 @@ class SparseMatrix s << static_cast&>(m); return s; } +#endif /** Destructor */ inline ~SparseMatrix() { - std::free(m_outerIndex); - std::free(m_innerNonZeros); + internal::conditional_aligned_delete_auto(m_outerIndex, m_outerSize + 1); + internal::conditional_aligned_delete_auto(m_innerNonZeros, m_outerSize); } /** Overloaded for performance */ @@ -858,16 +975,13 @@ class SparseMatrix void initAssignment(const Other& other) { resize(other.rows(), other.cols()); - if(m_innerNonZeros) - { - std::free(m_innerNonZeros); - m_innerNonZeros = 0; - } + internal::conditional_aligned_delete_auto(m_innerNonZeros, m_outerSize); + m_innerNonZeros = 0; } /** \internal * \sa insert(Index,Index) */ - EIGEN_DONT_INLINE Scalar& insertCompressed(Index row, Index col); + EIGEN_DEPRECATED EIGEN_DONT_INLINE Scalar& insertCompressed(Index row, Index col); /** \internal * A vector object that is equal to 0 everywhere but v at the position i */ @@ -886,7 +1000,7 @@ class SparseMatrix /** \internal * \sa insert(Index,Index) */ - EIGEN_DONT_INLINE Scalar& insertUncompressed(Index row, Index col); + EIGEN_DEPRECATED EIGEN_DONT_INLINE Scalar& insertUncompressed(Index row, Index col); public: /** \internal @@ -900,8 +1014,9 @@ class SparseMatrix eigen_assert(m_innerNonZeros[outer]<=(m_outerIndex[outer+1] - m_outerIndex[outer])); Index p = m_outerIndex[outer] + m_innerNonZeros[outer]++; - m_data.index(p) = convert_index(inner); - return (m_data.value(p) = Scalar(0)); + m_data.index(p) = StorageIndex(inner); + m_data.value(p) = Scalar(0); + return m_data.value(p); } protected: struct IndexPosPair { @@ -915,108 +1030,123 @@ class SparseMatrix * 1 - if *this is overwritten (Func==assign_op) or *this is empty, then we can work treat *this as a dense vector expression. * 2 - otherwise, for each diagonal coeff, * 2.a - if it already exists, then we update it, - * 2.b - otherwise, if *this is uncompressed and that the current inner-vector has empty room for at least 1 element, then we perform an in-place insertion. - * 2.c - otherwise, we'll have to reallocate and copy everything, so instead of doing so for each new element, it is recorded in a std::vector. - * 3 - at the end, if some entries failed to be inserted in-place, then we alloc a new buffer, copy each chunk at the right position, and insert the new elements. - * - * TODO: some piece of code could be isolated and reused for a general in-place update strategy. - * TODO: if we start to defer the insertion of some elements (i.e., case 2.c executed once), - * then it *might* be better to disable case 2.b since they will have to be copied anyway. + * 2.b - if the correct position is at the end of the vector, and there is capacity, push to back + * 2.b - otherwise, the insertion requires a data move, record insertion locations and handle in a second pass + * 3 - at the end, if some entries failed to be updated in-place, then we alloc a new buffer, copy each chunk at the right position, and insert the new elements. */ template void assignDiagonal(const DiagXpr diagXpr, const Func& assignFunc) { + + constexpr StorageIndex kEmptyIndexVal(-1); + typedef typename ScalarVector::AlignedMapType ValueMap; + Index n = diagXpr.size(); const bool overwrite = internal::is_same >::value; if(overwrite) { - if((this->rows()!=n) || (this->cols()!=n)) - this->resize(n, n); + if((m_outerSize != n) || (m_innerSize != n)) + resize(n, n); } if(m_data.size()==0 || overwrite) { - typedef Array ArrayXI; - this->makeCompressed(); - this->resizeNonZeros(n); - Eigen::Map(this->innerIndexPtr(), n).setLinSpaced(0,StorageIndex(n)-1); - Eigen::Map(this->outerIndexPtr(), n+1).setLinSpaced(0,StorageIndex(n)); - Eigen::Map > values = this->coeffs(); - values.setZero(); - internal::call_assignment_no_alias(values, diagXpr, assignFunc); + internal::conditional_aligned_delete_auto(m_innerNonZeros, m_outerSize); + m_innerNonZeros = 0; + resizeNonZeros(n); + ValueMap valueMap(valuePtr(), n); + std::iota(m_outerIndex, m_outerIndex + n + 1, StorageIndex(0)); + std::iota(innerIndexPtr(), innerIndexPtr() + n, StorageIndex(0)); + valueMap.setZero(); + internal::call_assignment_no_alias(valueMap, diagXpr, assignFunc); } else { - bool isComp = isCompressed(); - internal::evaluator diaEval(diagXpr); - std::vector newEntries; - - // 1 - try in-place update and record insertion failures - for(Index i = 0; ilower_bound(i,i); - Index p = lb.value; - if(lb.found) - { - // the coeff already exists - assignFunc.assignCoeff(m_data.value(p), diaEval.coeff(i)); - } - else if((!isComp) && m_innerNonZeros[i] < (m_outerIndex[i+1]-m_outerIndex[i])) - { - // non compressed mode with local room for inserting one element - m_data.moveChunk(p, p+1, m_outerIndex[i]+m_innerNonZeros[i]-p); - m_innerNonZeros[i]++; - m_data.value(p) = Scalar(0); - m_data.index(p) = StorageIndex(i); - assignFunc.assignCoeff(m_data.value(p), diaEval.coeff(i)); - } - else - { - // defer insertion - newEntries.push_back(IndexPosPair(i,p)); - } - } - // 2 - insert deferred entries - Index n_entries = Index(newEntries.size()); - if(n_entries>0) - { - Storage newData(m_data.size()+n_entries); - Index prev_p = 0; - Index prev_i = 0; - for(Index k=0; k diaEval(diagXpr); + + ei_declare_aligned_stack_constructed_variable(StorageIndex, tmp, n, 0); + typename IndexVector::AlignedMapType insertionLocations(tmp, n); + insertionLocations.setConstant(kEmptyIndexVal); + + Index deferredInsertions = 0; + Index shift = 0; + + for (Index j = 0; j < n; j++) { + Index begin = m_outerIndex[j]; + Index end = isCompressed() ? m_outerIndex[j + 1] : begin + m_innerNonZeros[j]; + Index capacity = m_outerIndex[j + 1] - end; + Index dst = m_data.searchLowerIndex(begin, end, j); + // the entry exists: update it now + if (dst != end && m_data.index(dst) == StorageIndex(j)) assignFunc.assignCoeff(m_data.value(dst), diaEval.coeff(j)); + // the entry belongs at the back of the vector: push to back + else if (dst == end && capacity > 0) + assignFunc.assignCoeff(insertBackUncompressed(j, j), diaEval.coeff(j)); + // the insertion requires a data move, record insertion location and handle in second pass + else { + insertionLocations.coeffRef(j) = StorageIndex(dst); + deferredInsertions++; + // if there is no capacity, all vectors to the right of this are shifted + if (capacity == 0) shift++; + } } - m_data.swap(newData); - } + + if (deferredInsertions > 0) { + + m_data.resize(m_data.size() + shift); + Index copyEnd = isCompressed() ? m_outerIndex[m_outerSize] + : m_outerIndex[m_outerSize - 1] + m_innerNonZeros[m_outerSize - 1]; + for (Index j = m_outerSize - 1; deferredInsertions > 0; j--) { + Index begin = m_outerIndex[j]; + Index end = isCompressed() ? m_outerIndex[j + 1] : begin + m_innerNonZeros[j]; + Index capacity = m_outerIndex[j + 1] - end; + + bool doInsertion = insertionLocations(j) >= 0; + bool breakUpCopy = doInsertion && (capacity > 0); + // break up copy for sorted insertion into inactive nonzeros + // optionally, add another criterium, i.e. 'breakUpCopy || (capacity > threhsold)' + // where `threshold >= 0` to skip inactive nonzeros in each vector + // this reduces the total number of copied elements, but requires more moveChunk calls + if (breakUpCopy) { + Index copyBegin = m_outerIndex[j + 1]; + Index to = copyBegin + shift; + Index chunkSize = copyEnd - copyBegin; + m_data.moveChunk(copyBegin, to, chunkSize); + copyEnd = end; + } + + m_outerIndex[j + 1] += shift; + + if (doInsertion) { + // if there is capacity, shift into the inactive nonzeros + if (capacity > 0) shift++; + Index copyBegin = insertionLocations(j); + Index to = copyBegin + shift; + Index chunkSize = copyEnd - copyBegin; + m_data.moveChunk(copyBegin, to, chunkSize); + Index dst = to - 1; + m_data.index(dst) = StorageIndex(j); + m_data.value(dst) = Scalar(0); + assignFunc.assignCoeff(m_data.value(dst), diaEval.coeff(j)); + if (!isCompressed()) m_innerNonZeros[j]++; + shift--; + deferredInsertions--; + copyEnd = copyBegin; + } + } + } + eigen_assert((shift == 0) && (deferredInsertions == 0)); } } + /* These functions are used to avoid a redundant binary search operation in functions such as coeffRef() and assume `dst` is the appropriate sorted insertion point */ + EIGEN_STRONG_INLINE Scalar& insertAtByOuterInner(Index outer, Index inner, Index dst); + Scalar& insertCompressedAtByOuterInner(Index outer, Index inner, Index dst); + Scalar& insertUncompressedAtByOuterInner(Index outer, Index inner, Index dst); + private: - static void check_template_parameters() - { - EIGEN_STATIC_ASSERT(NumTraits::IsSigned,THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE); - EIGEN_STATIC_ASSERT((Options&(ColMajor|RowMajor))==Options,INVALID_MATRIX_TEMPLATE_PARAMETERS); - } + EIGEN_STATIC_ASSERT(NumTraits::IsSigned,THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE) + EIGEN_STATIC_ASSERT((Options&(ColMajor|RowMajor))==Options,INVALID_MATRIX_TEMPLATE_PARAMETERS) struct default_prunning_func { default_prunning_func(const Scalar& ref, const RealScalar& eps) : reference(ref), epsilon(eps) {} @@ -1031,56 +1161,181 @@ class SparseMatrix namespace internal { -template -void set_from_triplets(const InputIterator& begin, const InputIterator& end, SparseMatrixType& mat, DupFunctor dup_func) -{ - enum { IsRowMajor = SparseMatrixType::IsRowMajor }; - typedef typename SparseMatrixType::Scalar Scalar; - typedef typename SparseMatrixType::StorageIndex StorageIndex; - SparseMatrix trMat(mat.rows(),mat.cols()); +// Creates a compressed sparse matrix from a range of unsorted triplets +// Requires temporary storage to handle duplicate entries +template +void set_from_triplets(const InputIterator& begin, const InputIterator& end, SparseMatrixType& mat, + DupFunctor dup_func) { + constexpr bool IsRowMajor = SparseMatrixType::IsRowMajor; + using StorageIndex = typename SparseMatrixType::StorageIndex; + using IndexMap = typename VectorX::AlignedMapType; + using TransposedSparseMatrix = SparseMatrix; + + if (begin == end) return; + + // There are two strategies to consider for constructing a matrix from unordered triplets: + // A) construct the 'mat' in its native storage order and sort in-place (less memory); or, + // B) construct the transposed matrix and use an implicit sort upon assignment to `mat` (less time). + // This routine uses B) for faster execution time. + TransposedSparseMatrix trmat(mat.rows(), mat.cols()); + + // scan triplets to determine allocation size before constructing matrix + Index nonZeros = 0; + for (InputIterator it(begin); it != end; ++it) { + eigen_assert(it->row() >= 0 && it->row() < mat.rows() && it->col() >= 0 && it->col() < mat.cols()); + StorageIndex j = convert_index(IsRowMajor ? it->col() : it->row()); + if (nonZeros == NumTraits::highest()) internal::throw_std_bad_alloc(); + trmat.outerIndexPtr()[j + 1]++; + nonZeros++; + } - if(begin!=end) - { - // pass 1: count the nnz per inner-vector - typename SparseMatrixType::IndexVector wi(trMat.outerSize()); - wi.setZero(); - for(InputIterator it(begin); it!=end; ++it) - { - eigen_assert(it->row()>=0 && it->row()col()>=0 && it->col()col() : it->row())++; - } + std::partial_sum(trmat.outerIndexPtr(), trmat.outerIndexPtr() + trmat.outerSize() + 1, trmat.outerIndexPtr()); + eigen_assert(nonZeros == trmat.outerIndexPtr()[trmat.outerSize()]); + trmat.resizeNonZeros(nonZeros); + + // construct temporary array to track insertions (outersize) and collapse duplicates (innersize) + ei_declare_aligned_stack_constructed_variable(StorageIndex, tmp, numext::maxi(mat.innerSize(), mat.outerSize()), 0); + smart_copy(trmat.outerIndexPtr(), trmat.outerIndexPtr() + trmat.outerSize(), tmp); + + // push triplets to back of each vector + for (InputIterator it(begin); it != end; ++it) { + StorageIndex j = convert_index(IsRowMajor ? it->col() : it->row()); + StorageIndex i = convert_index(IsRowMajor ? it->row() : it->col()); + StorageIndex k = tmp[j]; + trmat.data().index(k) = i; + trmat.data().value(k) = it->value(); + tmp[j]++; + } - // pass 2: insert all the elements into trMat - trMat.reserve(wi); - for(InputIterator it(begin); it!=end; ++it) - trMat.insertBackUncompressed(it->row(),it->col()) = it->value(); + IndexMap wi(tmp, trmat.innerSize()); + trmat.collapseDuplicates(wi, dup_func); + // implicit sorting + mat = trmat; +} - // pass 3: - trMat.collapseDuplicates(dup_func); +// Creates a compressed sparse matrix from a sorted range of triplets +template +void set_from_triplets_sorted(const InputIterator& begin, const InputIterator& end, SparseMatrixType& mat, + DupFunctor dup_func) { + constexpr bool IsRowMajor = SparseMatrixType::IsRowMajor; + using StorageIndex = typename SparseMatrixType::StorageIndex; + + if (begin == end) return; + + constexpr StorageIndex kEmptyIndexValue(-1); + // deallocate inner nonzeros if present and zero outerIndexPtr + mat.resize(mat.rows(), mat.cols()); + // use outer indices to count non zero entries (excluding duplicate entries) + StorageIndex previous_j = kEmptyIndexValue; + StorageIndex previous_i = kEmptyIndexValue; + // scan triplets to determine allocation size before constructing matrix + Index nonZeros = 0; + for (InputIterator it(begin); it != end; ++it) { + eigen_assert(it->row() >= 0 && it->row() < mat.rows() && it->col() >= 0 && it->col() < mat.cols()); + StorageIndex j = convert_index(IsRowMajor ? it->row() : it->col()); + StorageIndex i = convert_index(IsRowMajor ? it->col() : it->row()); + eigen_assert(j > previous_j || (j == previous_j && i >= previous_i)); + // identify duplicates by examining previous location + bool duplicate = (previous_j == j) && (previous_i == i); + if (!duplicate) { + if (nonZeros == NumTraits::highest()) internal::throw_std_bad_alloc(); + nonZeros++; + mat.outerIndexPtr()[j + 1]++; + previous_j = j; + previous_i = i; + } } - // pass 4: transposed copy -> implicit sorting - mat = trMat; + // finalize outer indices and allocate memory + std::partial_sum(mat.outerIndexPtr(), mat.outerIndexPtr() + mat.outerSize() + 1, mat.outerIndexPtr()); + eigen_assert(nonZeros == mat.outerIndexPtr()[mat.outerSize()]); + mat.resizeNonZeros(nonZeros); + + previous_i = kEmptyIndexValue; + previous_j = kEmptyIndexValue; + Index back = 0; + for (InputIterator it(begin); it != end; ++it) { + StorageIndex j = convert_index(IsRowMajor ? it->row() : it->col()); + StorageIndex i = convert_index(IsRowMajor ? it->col() : it->row()); + bool duplicate = (previous_j == j) && (previous_i == i); + if (duplicate) { + mat.data().value(back - 1) = dup_func(mat.data().value(back - 1), it->value()); + } else { + // push triplets to back + mat.data().index(back) = i; + mat.data().value(back) = it->value(); + previous_j = j; + previous_i = i; + back++; + } + } + eigen_assert(back == nonZeros); + // matrix is finalized } +// thin wrapper around a generic binary functor to use the sparse disjunction evaulator instead of the default "arithmetic" evaulator +template +struct scalar_disjunction_op +{ + using result_type = typename result_of::type; + scalar_disjunction_op(const DupFunctor& op) : m_functor(op) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return m_functor(a, b); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const DupFunctor& functor() const { return m_functor; } + const DupFunctor& m_functor; +}; + +template +struct functor_traits> : public functor_traits {}; + +// Creates a compressed sparse matrix from its existing entries and those from an unsorted range of triplets +template +void insert_from_triplets(const InputIterator& begin, const InputIterator& end, SparseMatrixType& mat, + DupFunctor dup_func) { + using Scalar = typename SparseMatrixType::Scalar; + using SrcXprType = CwiseBinaryOp, const SparseMatrixType, const SparseMatrixType>; + + // set_from_triplets is necessary to sort the inner indices and remove the duplicate entries + SparseMatrixType trips(mat.rows(), mat.cols()); + set_from_triplets(begin, end, trips, dup_func); + + SrcXprType src = mat.binaryExpr(trips, scalar_disjunction_op(dup_func)); + // the sparse assignment procedure creates a temporary matrix and swaps the final result + assign_sparse_to_sparse(mat, src); +} + +// Creates a compressed sparse matrix from its existing entries and those from an sorted range of triplets +template +void insert_from_triplets_sorted(const InputIterator& begin, const InputIterator& end, SparseMatrixType& mat, + DupFunctor dup_func) { + using Scalar = typename SparseMatrixType::Scalar; + using SrcXprType = CwiseBinaryOp, const SparseMatrixType, const SparseMatrixType>; + + // TODO: process triplets without making a copy + SparseMatrixType trips(mat.rows(), mat.cols()); + set_from_triplets_sorted(begin, end, trips, dup_func); + + SrcXprType src = mat.binaryExpr(trips, scalar_disjunction_op(dup_func)); + // the sparse assignment procedure creates a temporary matrix and swaps the final result + assign_sparse_to_sparse(mat, src); } +} // namespace internal -/** Fill the matrix \c *this with the list of \em triplets defined by the iterator range \a begin - \a end. +/** Fill the matrix \c *this with the list of \em triplets defined in the half-open range from \a begin to \a end. * * A \em triplet is a tuple (i,j,value) defining a non-zero element. - * The input list of triplets does not have to be sorted, and can contains duplicated elements. + * The input list of triplets does not have to be sorted, and may contain duplicated elements. * In any case, the result is a \b sorted and \b compressed sparse matrix where the duplicates have been summed up. * This is a \em O(n) operation, with \em n the number of triplet elements. - * The initial contents of \c *this is destroyed. + * The initial contents of \c *this are destroyed. * The matrix \c *this must be properly resized beforehand using the SparseMatrix(Index,Index) constructor, * or the resize(Index,Index) method. The sizes are not extracted from the triplet list. * * The \a InputIterators value_type must provide the following interface: * \code * Scalar value() const; // the value - * Scalar row() const; // the row index i - * Scalar col() const; // the column index j + * IndexType row() const; // the row index i + * IndexType col() const; // the column index j * \endcode * See for instance the Eigen::Triplet template class. * @@ -1103,73 +1358,194 @@ void set_from_triplets(const InputIterator& begin, const InputIterator& end, Spa * an abstract iterator over a complex data-structure that would be expensive to evaluate. The triplets should rather * be explicitly stored into a std::vector for instance. */ -template +template template -void SparseMatrix::setFromTriplets(const InputIterators& begin, const InputIterators& end) +void SparseMatrix::setFromTriplets(const InputIterators& begin, const InputIterators& end) { - internal::set_from_triplets >(begin, end, *this, internal::scalar_sum_op()); + internal::set_from_triplets >(begin, end, *this, internal::scalar_sum_op()); } /** The same as setFromTriplets but when duplicates are met the functor \a dup_func is applied: * \code * value = dup_func(OldValue, NewValue) - * \endcode + * \endcode * Here is a C++11 example keeping the latest entry only: * \code * mat.setFromTriplets(triplets.begin(), triplets.end(), [] (const Scalar&,const Scalar &b) { return b; }); * \endcode */ -template -template -void SparseMatrix::setFromTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func) +template +template +void SparseMatrix::setFromTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func) { - internal::set_from_triplets, DupFunctor>(begin, end, *this, dup_func); + internal::set_from_triplets, DupFunctor>(begin, end, *this, dup_func); } -/** \internal */ -template -template -void SparseMatrix::collapseDuplicates(DupFunctor dup_func) +/** The same as setFromTriplets but triplets are assumed to be pre-sorted. This is faster and requires less temporary storage. + * Two triplets `a` and `b` are appropriately ordered if: + * \code + * ColMajor: ((a.col() != b.col()) ? (a.col() < b.col()) : (a.row() < b.row()) + * RowMajor: ((a.row() != b.row()) ? (a.row() < b.row()) : (a.col() < b.col()) + * \endcode + */ +template +template +void SparseMatrix::setFromSortedTriplets(const InputIterators& begin, const InputIterators& end) { - eigen_assert(!isCompressed()); - // TODO, in practice we should be able to use m_innerNonZeros for that task - IndexVector wi(innerSize()); - wi.fill(-1); + internal::set_from_triplets_sorted >(begin, end, *this, internal::scalar_sum_op()); +} + +/** The same as setFromSortedTriplets but when duplicates are met the functor \a dup_func is applied: + * \code + * value = dup_func(OldValue, NewValue) + * \endcode + * Here is a C++11 example keeping the latest entry only: + * \code + * mat.setFromSortedTriplets(triplets.begin(), triplets.end(), [] (const Scalar&,const Scalar &b) { return b; }); + * \endcode + */ +template +template +void SparseMatrix::setFromSortedTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func) +{ + internal::set_from_triplets_sorted, DupFunctor>(begin, end, *this, dup_func); +} + +/** Insert a batch of elements into the matrix \c *this with the list of \em triplets defined in the half-open range from \a begin to \a end. + * + * A \em triplet is a tuple (i,j,value) defining a non-zero element. + * The input list of triplets does not have to be sorted, and may contain duplicated elements. + * In any case, the result is a \b sorted and \b compressed sparse matrix where the duplicates have been summed up. + * This is a \em O(n) operation, with \em n the number of triplet elements. + * The initial contents of \c *this are preserved (except for the summation of duplicate elements). + * The matrix \c *this must be properly sized beforehand. The sizes are not extracted from the triplet list. + * + * The \a InputIterators value_type must provide the following interface: + * \code + * Scalar value() const; // the value + * IndexType row() const; // the row index i + * IndexType col() const; // the column index j + * \endcode + * See for instance the Eigen::Triplet template class. + * + * Here is a typical usage example: + * \code + SparseMatrixType m(rows,cols); // m contains nonzero entries + typedef Triplet T; + std::vector tripletList; + tripletList.reserve(estimation_of_entries); + for(...) + { + // ... + tripletList.push_back(T(i,j,v_ij)); + } + + m.insertFromTriplets(tripletList.begin(), tripletList.end()); + // m is ready to go! + * \endcode + * + * \warning The list of triplets is read multiple times (at least twice). Therefore, it is not recommended to define + * an abstract iterator over a complex data-structure that would be expensive to evaluate. The triplets should rather + * be explicitly stored into a std::vector for instance. + */ +template +template +void SparseMatrix::insertFromTriplets(const InputIterators& begin, const InputIterators& end) +{ + internal::insert_from_triplets >(begin, end, *this, internal::scalar_sum_op()); +} + +/** The same as insertFromTriplets but when duplicates are met the functor \a dup_func is applied: + * \code + * value = dup_func(OldValue, NewValue) + * \endcode + * Here is a C++11 example keeping the latest entry only: + * \code + * mat.insertFromTriplets(triplets.begin(), triplets.end(), [] (const Scalar&,const Scalar &b) { return b; }); + * \endcode + */ +template +template +void SparseMatrix::insertFromTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func) +{ + internal::insert_from_triplets, DupFunctor>(begin, end, *this, dup_func); +} + +/** The same as insertFromTriplets but triplets are assumed to be pre-sorted. This is faster and requires less temporary storage. + * Two triplets `a` and `b` are appropriately ordered if: + * \code + * ColMajor: ((a.col() != b.col()) ? (a.col() < b.col()) : (a.row() < b.row()) + * RowMajor: ((a.row() != b.row()) ? (a.row() < b.row()) : (a.col() < b.col()) + * \endcode + */ +template +template +void SparseMatrix::insertFromSortedTriplets(const InputIterators& begin, const InputIterators& end) +{ + internal::insert_from_triplets_sorted >(begin, end, *this, internal::scalar_sum_op()); +} + +/** The same as insertFromSortedTriplets but when duplicates are met the functor \a dup_func is applied: + * \code + * value = dup_func(OldValue, NewValue) + * \endcode + * Here is a C++11 example keeping the latest entry only: + * \code + * mat.insertFromSortedTriplets(triplets.begin(), triplets.end(), [] (const Scalar&,const Scalar &b) { return b; }); + * \endcode + */ +template +template +void SparseMatrix::insertFromSortedTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func) +{ + internal::insert_from_triplets_sorted, DupFunctor>(begin, end, *this, dup_func); +} + +/** \internal */ +template +template +void SparseMatrix::collapseDuplicates(DenseBase& wi, DupFunctor dup_func) { + // removes duplicate entries and compresses the matrix + // the excess allocated memory is not released + // the inner indices do not need to be sorted, nor is the matrix returned in a sorted state + eigen_assert(wi.size() == m_innerSize); + constexpr StorageIndex kEmptyIndexValue(-1); + wi.setConstant(kEmptyIndexValue); StorageIndex count = 0; + const bool is_compressed = isCompressed(); // for each inner-vector, wi[inner_index] will hold the position of first element into the index/value buffers - for(Index j=0; j=start) - { - // we already meet this entry => accumulate it + for (Index j = 0; j < m_outerSize; ++j) { + const StorageIndex newBegin = count; + const StorageIndex end = is_compressed ? m_outerIndex[j + 1] : m_outerIndex[j] + m_innerNonZeros[j]; + for (StorageIndex k = m_outerIndex[j]; k < end; ++k) { + StorageIndex i = m_data.index(k); + if (wi(i) >= newBegin) { + // entry at k is a duplicate + // accumulate it into the primary entry located at wi(i) m_data.value(wi(i)) = dup_func(m_data.value(wi(i)), m_data.value(k)); - } - else - { + } else { + // k is the primary entry in j with inner index i + // shift it to the left and record its location at wi(i) + m_data.index(count) = i; m_data.value(count) = m_data.value(k); - m_data.index(count) = m_data.index(k); wi(i) = count; ++count; } } - m_outerIndex[j] = start; + m_outerIndex[j] = newBegin; } m_outerIndex[m_outerSize] = count; + m_data.resize(count); - // turn the matrix into compressed form - std::free(m_innerNonZeros); + // turn the matrix into compressed form (if it is not already) + internal::conditional_aligned_delete_auto(m_innerNonZeros, m_outerSize); m_innerNonZeros = 0; - m_data.resize(m_outerIndex[m_outerSize]); } -template +/** \internal */ +template template -EIGEN_DONT_INLINE SparseMatrix& SparseMatrix::operator=(const SparseMatrixBase& other) +EIGEN_DONT_INLINE SparseMatrix& SparseMatrix::operator=(const SparseMatrixBase& other) { EIGEN_STATIC_ASSERT((internal::is_same::value), YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) @@ -1189,8 +1565,8 @@ EIGEN_DONT_INLINE SparseMatrix& SparseMatrix::type >::type OtherCopy; - typedef typename internal::remove_all::type _OtherCopy; - typedef internal::evaluator<_OtherCopy> OtherCopyEval; + typedef internal::remove_all_t OtherCopy_; + typedef internal::evaluator OtherCopyEval; OtherCopy otherCopy(other.derived()); OtherCopyEval otherCopyEval(otherCopy); @@ -1240,279 +1616,298 @@ EIGEN_DONT_INLINE SparseMatrix& SparseMatrix -typename SparseMatrix<_Scalar,_Options,_StorageIndex>::Scalar& SparseMatrix<_Scalar,_Options,_StorageIndex>::insert(Index row, Index col) -{ - eigen_assert(row>=0 && row=0 && col(std::malloc(m_outerSize * sizeof(StorageIndex))); - if(!m_innerNonZeros) internal::throw_std_bad_alloc(); - - memset(m_innerNonZeros, 0, (m_outerSize)*sizeof(StorageIndex)); - - // pack all inner-vectors to the end of the pre-allocated space - // and allocate the entire free-space to the first inner-vector - StorageIndex end = convert_index(m_data.allocatedSize()); - for(Index j=1; j<=m_outerSize; ++j) - m_outerIndex[j] = end; - } - else - { - // turn the matrix into non-compressed mode - m_innerNonZeros = static_cast(std::malloc(m_outerSize * sizeof(StorageIndex))); - if(!m_innerNonZeros) internal::throw_std_bad_alloc(); - for(Index j=0; j=0 && m_innerNonZeros[j]==0) - m_outerIndex[j--] = p; - - // push back the new element - ++m_innerNonZeros[outer]; - m_data.append(Scalar(0), inner); - - // check for reallocation - if(data_end != m_data.allocatedSize()) - { - // m_data has been reallocated - // -> move remaining inner-vectors back to the end of the free-space - // so that the entire free-space is allocated to the current inner-vector. - eigen_internal_assert(data_end < m_data.allocatedSize()); - StorageIndex new_end = convert_index(m_data.allocatedSize()); - for(Index k=outer+1; k<=m_outerSize; ++k) - if(m_outerIndex[k]==data_end) - m_outerIndex[k] = new_end; - } - return m_data.value(p); - } - - // Second case: the next inner-vector is packed to the end - // and the current inner-vector end match the used-space. - if(m_outerIndex[outer+1]==data_end && m_outerIndex[outer]+m_innerNonZeros[outer]==m_data.size()) - { - eigen_internal_assert(outer+1==m_outerSize || m_innerNonZeros[outer+1]==0); - - // add space for the new element - ++m_innerNonZeros[outer]; - m_data.resize(m_data.size()+1); - - // check for reallocation - if(data_end != m_data.allocatedSize()) - { - // m_data has been reallocated - // -> move remaining inner-vectors back to the end of the free-space - // so that the entire free-space is allocated to the current inner-vector. - eigen_internal_assert(data_end < m_data.allocatedSize()); - StorageIndex new_end = convert_index(m_data.allocatedSize()); - for(Index k=outer+1; k<=m_outerSize; ++k) - if(m_outerIndex[k]==data_end) - m_outerIndex[k] = new_end; - } - - // and insert it at the right position (sorted insertion) - Index startId = m_outerIndex[outer]; - Index p = m_outerIndex[outer]+m_innerNonZeros[outer]-1; - while ( (p > startId) && (m_data.index(p-1) > inner) ) - { - m_data.index(p) = m_data.index(p-1); - m_data.value(p) = m_data.value(p-1); - --p; +template +inline typename SparseMatrix::Scalar& +SparseMatrix::insert(Index row, Index col) { + return insertByOuterInner(IsRowMajor ? row : col, IsRowMajor ? col : row); +} + +template +EIGEN_STRONG_INLINE typename SparseMatrix::Scalar& +SparseMatrix::insertAtByOuterInner(Index outer, Index inner, Index dst) { + // random insertion into compressed matrix is very slow + uncompress(); + return insertUncompressedAtByOuterInner(outer, inner, dst); +} + +template +EIGEN_DEPRECATED EIGEN_DONT_INLINE typename SparseMatrix::Scalar& +SparseMatrix::insertUncompressed(Index row, Index col) { + eigen_assert(!isCompressed()); + Index outer = IsRowMajor ? row : col; + Index inner = IsRowMajor ? col : row; + Index start = m_outerIndex[outer]; + Index end = start + m_innerNonZeros[outer]; + Index dst = start == end ? end : m_data.searchLowerIndex(start, end, inner); + if (dst == end) { + Index capacity = m_outerIndex[outer + 1] - end; + if (capacity > 0) { + // implies uncompressed: push to back of vector + m_innerNonZeros[outer]++; + m_data.index(end) = StorageIndex(inner); + m_data.value(end) = Scalar(0); + return m_data.value(end); } - - m_data.index(p) = convert_index(inner); - return (m_data.value(p) = Scalar(0)); - } - - if(m_data.size() != m_data.allocatedSize()) - { - // make sure the matrix is compatible to random un-compressed insertion: - m_data.resize(m_data.allocatedSize()); - this->reserveInnerVectors(Array::Constant(m_outerSize, 2)); } - - return insertUncompressed(row,col); + eigen_assert((dst == end || m_data.index(dst) != inner) && + "you cannot insert an element that already exists, you must call coeffRef to this end"); + return insertUncompressedAtByOuterInner(outer, inner, dst); } - -template -EIGEN_DONT_INLINE typename SparseMatrix<_Scalar,_Options,_StorageIndex>::Scalar& SparseMatrix<_Scalar,_Options,_StorageIndex>::insertUncompressed(Index row, Index col) -{ - eigen_assert(!isCompressed()); - const Index outer = IsRowMajor ? row : col; - const StorageIndex inner = convert_index(IsRowMajor ? col : row); +template +EIGEN_DEPRECATED EIGEN_DONT_INLINE typename SparseMatrix::Scalar& +SparseMatrix::insertCompressed(Index row, Index col) { + eigen_assert(isCompressed()); + Index outer = IsRowMajor ? row : col; + Index inner = IsRowMajor ? col : row; + Index start = m_outerIndex[outer]; + Index end = m_outerIndex[outer + 1]; + Index dst = start == end ? end : m_data.searchLowerIndex(start, end, inner); + eigen_assert((dst == end || m_data.index(dst) != inner) && + "you cannot insert an element that already exists, you must call coeffRef to this end"); + return insertCompressedAtByOuterInner(outer, inner, dst); +} - Index room = m_outerIndex[outer+1] - m_outerIndex[outer]; - StorageIndex innerNNZ = m_innerNonZeros[outer]; - if(innerNNZ>=room) - { - // this inner vector is full, we need to reallocate the whole buffer :( - reserve(SingletonVector(outer,std::max(2,innerNNZ))); +template +typename SparseMatrix::Scalar& +SparseMatrix::insertCompressedAtByOuterInner(Index outer, Index inner, Index dst) { + eigen_assert(isCompressed()); + // compressed insertion always requires expanding the buffer + // first, check if there is adequate allocated memory + if (m_data.allocatedSize() <= m_data.size()) { + // if there is no capacity for a single insertion, double the capacity + // increase capacity by a mininum of 32 + Index minReserve = 32; + Index reserveSize = numext::maxi(minReserve, m_data.allocatedSize()); + m_data.reserve(reserveSize); } + m_data.resize(m_data.size() + 1); + Index chunkSize = m_outerIndex[m_outerSize] - dst; + // shift the existing data to the right if necessary + m_data.moveChunk(dst, dst + 1, chunkSize); + // update nonzero counts + // potentially O(outerSize) bottleneck! + for (Index j = outer; j < m_outerSize; j++) m_outerIndex[j + 1]++; + // initialize the coefficient + m_data.index(dst) = StorageIndex(inner); + m_data.value(dst) = Scalar(0); + // return a reference to the coefficient + return m_data.value(dst); +} - Index startId = m_outerIndex[outer]; - Index p = startId + m_innerNonZeros[outer]; - while ( (p > startId) && (m_data.index(p-1) > inner) ) - { - m_data.index(p) = m_data.index(p-1); - m_data.value(p) = m_data.value(p-1); - --p; +template +typename SparseMatrix::Scalar& +SparseMatrix::insertUncompressedAtByOuterInner(Index outer, Index inner, Index dst) { + eigen_assert(!isCompressed()); + // find a vector with capacity, starting at `outer` and searching to the left and right + for (Index leftTarget = outer - 1, rightTarget = outer; (leftTarget >= 0) || (rightTarget < m_outerSize);) { + if (rightTarget < m_outerSize) { + Index start = m_outerIndex[rightTarget]; + Index end = start + m_innerNonZeros[rightTarget]; + Index nextStart = m_outerIndex[rightTarget + 1]; + Index capacity = nextStart - end; + if (capacity > 0) { + // move [dst, end) to dst+1 and insert at dst + Index chunkSize = end - dst; + if (chunkSize > 0) m_data.moveChunk(dst, dst + 1, chunkSize); + m_innerNonZeros[outer]++; + for (Index j = outer; j < rightTarget; j++) m_outerIndex[j + 1]++; + m_data.index(dst) = StorageIndex(inner); + m_data.value(dst) = Scalar(0); + return m_data.value(dst); + } + rightTarget++; + } + if (leftTarget >= 0) { + Index start = m_outerIndex[leftTarget]; + Index end = start + m_innerNonZeros[leftTarget]; + Index nextStart = m_outerIndex[leftTarget + 1]; + Index capacity = nextStart - end; + if (capacity > 0) { + // tricky: dst is a lower bound, so we must insert at dst-1 when shifting left + // move [nextStart, dst) to nextStart-1 and insert at dst-1 + Index chunkSize = dst - nextStart; + if (chunkSize > 0) m_data.moveChunk(nextStart, nextStart - 1, chunkSize); + m_innerNonZeros[outer]++; + for (Index j = leftTarget; j < outer; j++) m_outerIndex[j + 1]--; + m_data.index(dst - 1) = StorageIndex(inner); + m_data.value(dst - 1) = Scalar(0); + return m_data.value(dst - 1); + } + leftTarget--; + } } - eigen_assert((p<=startId || m_data.index(p-1)!=inner) && "you cannot insert an element that already exists, you must call coeffRef to this end"); + // no room for interior insertion + // nonZeros() == m_data.size() + // record offset as outerIndxPtr will change + Index dst_offset = dst - m_outerIndex[outer]; + // allocate space for random insertion + if (m_data.allocatedSize() == 0) { + // fast method to allocate space for one element per vector in empty matrix + m_data.resize(m_outerSize); + std::iota(m_outerIndex, m_outerIndex + m_outerSize + 1, StorageIndex(0)); + } else { + // check for integer overflow: if maxReserveSize == 0, insertion is not possible + Index maxReserveSize = static_cast(NumTraits::highest()) - m_data.allocatedSize(); + eigen_assert(maxReserveSize > 0); + if (m_outerSize <= maxReserveSize) { + // allocate space for one additional element per vector + reserveInnerVectors(IndexVector::Constant(m_outerSize, 1)); + } else { + // handle the edge case where StorageIndex is insufficient to reserve outerSize additional elements + // allocate space for one additional element in the interval [outer,maxReserveSize) + typedef internal::sparse_reserve_op ReserveSizesOp; + typedef CwiseNullaryOp ReserveSizesXpr; + ReserveSizesXpr reserveSizesXpr(m_outerSize, 1, ReserveSizesOp(outer, m_outerSize, maxReserveSize)); + reserveInnerVectors(reserveSizesXpr); + } + } + // insert element at `dst` with new outer indices + Index start = m_outerIndex[outer]; + Index end = start + m_innerNonZeros[outer]; + Index new_dst = start + dst_offset; + Index chunkSize = end - new_dst; + if (chunkSize > 0) m_data.moveChunk(new_dst, new_dst + 1, chunkSize); m_innerNonZeros[outer]++; + m_data.index(new_dst) = StorageIndex(inner); + m_data.value(new_dst) = Scalar(0); + return m_data.value(new_dst); +} + +namespace internal { + + template + struct evaluator> + : evaluator>> { + typedef evaluator>> Base; + typedef SparseMatrix SparseMatrixType; + evaluator() : Base() {} + explicit evaluator(const SparseMatrixType& mat) : Base(mat) {} + }; - m_data.index(p) = inner; - return (m_data.value(p) = Scalar(0)); } -template -EIGEN_DONT_INLINE typename SparseMatrix<_Scalar,_Options,_StorageIndex>::Scalar& SparseMatrix<_Scalar,_Options,_StorageIndex>::insertCompressed(Index row, Index col) -{ - eigen_assert(isCompressed()); +// Specialization for SparseMatrix. +// Serializes [rows, cols, isCompressed, outerSize, innerBufferSize, +// innerNonZeros, outerIndices, innerIndices, values]. +template +class Serializer, void> { + public: + typedef SparseMatrix SparseMat; + + struct Header { + typename SparseMat::Index rows; + typename SparseMat::Index cols; + bool compressed; + Index outer_size; + Index inner_buffer_size; + }; - const Index outer = IsRowMajor ? row : col; - const Index inner = IsRowMajor ? col : row; + EIGEN_DEVICE_FUNC size_t size(const SparseMat& value) const { + // innerNonZeros. + std::size_t num_storage_indices = value.isCompressed() ? 0 : value.outerSize(); + // Outer indices. + num_storage_indices += value.outerSize() + 1; + // Inner indices. + const StorageIndex inner_buffer_size = value.outerIndexPtr()[value.outerSize()]; + num_storage_indices += inner_buffer_size; + // Values. + std::size_t num_values = inner_buffer_size; + return sizeof(Header) + sizeof(Scalar) * num_values + + sizeof(StorageIndex) * num_storage_indices; + } - Index previousOuter = outer; - if (m_outerIndex[outer+1]==0) - { - // we start a new inner vector - while (previousOuter>=0 && m_outerIndex[previousOuter]==0) - { - m_outerIndex[previousOuter] = convert_index(m_data.size()); - --previousOuter; + EIGEN_DEVICE_FUNC uint8_t* serialize(uint8_t* dest, uint8_t* end, + const SparseMat& value) { + if (EIGEN_PREDICT_FALSE(dest == nullptr)) return nullptr; + if (EIGEN_PREDICT_FALSE(dest + size(value) > end)) return nullptr; + + const size_t header_bytes = sizeof(Header); + Header header = {value.rows(), value.cols(), value.isCompressed(), + value.outerSize(), value.outerIndexPtr()[value.outerSize()]}; + EIGEN_USING_STD(memcpy) + memcpy(dest, &header, header_bytes); + dest += header_bytes; + + // innerNonZeros. + if (!header.compressed) { + std::size_t data_bytes = sizeof(StorageIndex) * header.outer_size; + memcpy(dest, value.innerNonZeroPtr(), data_bytes); + dest += data_bytes; } - m_outerIndex[outer+1] = m_outerIndex[outer]; - } - // here we have to handle the tricky case where the outerIndex array - // starts with: [ 0 0 0 0 0 1 ...] and we are inserted in, e.g., - // the 2nd inner vector... - bool isLastVec = (!(previousOuter==-1 && m_data.size()!=0)) - && (std::size_t(m_outerIndex[outer+1]) == m_data.size()); + // Outer indices. + std::size_t data_bytes = sizeof(StorageIndex) * (header.outer_size + 1); + memcpy(dest, value.outerIndexPtr(), data_bytes); + dest += data_bytes; - std::size_t startId = m_outerIndex[outer]; - // FIXME let's make sure sizeof(long int) == sizeof(std::size_t) - std::size_t p = m_outerIndex[outer+1]; - ++m_outerIndex[outer+1]; + // Inner indices. + data_bytes = sizeof(StorageIndex) * header.inner_buffer_size; + memcpy(dest, value.innerIndexPtr(), data_bytes); + dest += data_bytes; - double reallocRatio = 1; - if (m_data.allocatedSize()<=m_data.size()) - { - // if there is no preallocated memory, let's reserve a minimum of 32 elements - if (m_data.size()==0) - { - m_data.reserve(32); - } - else - { - // we need to reallocate the data, to reduce multiple reallocations - // we use a smart resize algorithm based on the current filling ratio - // in addition, we use double to avoid integers overflows - double nnzEstimate = double(m_outerIndex[outer])*double(m_outerSize)/double(outer+1); - reallocRatio = (nnzEstimate-double(m_data.size()))/double(m_data.size()); - // furthermore we bound the realloc ratio to: - // 1) reduce multiple minor realloc when the matrix is almost filled - // 2) avoid to allocate too much memory when the matrix is almost empty - reallocRatio = (std::min)((std::max)(reallocRatio,1.5),8.); - } + // Values. + data_bytes = sizeof(Scalar) * header.inner_buffer_size; + memcpy(dest, value.valuePtr(), data_bytes); + dest += data_bytes; + + return dest; } - m_data.resize(m_data.size()+1,reallocRatio); - if (!isLastVec) - { - if (previousOuter==-1) - { - // oops wrong guess. - // let's correct the outer offsets - for (Index k=0; k<=(outer+1); ++k) - m_outerIndex[k] = 0; - Index k=outer+1; - while(m_outerIndex[k]==0) - m_outerIndex[k++] = 1; - while (k<=m_outerSize && m_outerIndex[k]!=0) - m_outerIndex[k++]++; - p = 0; - --k; - k = m_outerIndex[k]-1; - while (k>0) - { - m_data.index(k) = m_data.index(k-1); - m_data.value(k) = m_data.value(k-1); - k--; - } + EIGEN_DEVICE_FUNC const uint8_t* deserialize(const uint8_t* src, + const uint8_t* end, + SparseMat& value) const { + if (EIGEN_PREDICT_FALSE(src == nullptr)) return nullptr; + if (EIGEN_PREDICT_FALSE(src + sizeof(Header) > end)) return nullptr; + + const size_t header_bytes = sizeof(Header); + Header header; + EIGEN_USING_STD(memcpy) + memcpy(&header, src, header_bytes); + src += header_bytes; + + value.setZero(); + value.resize(header.rows, header.cols); + if (header.compressed) { + value.makeCompressed(); + } else { + value.uncompress(); } - else - { - // we are not inserting into the last inner vec - // update outer indices: - Index j = outer+2; - while (j<=m_outerSize && m_outerIndex[j]!=0) - m_outerIndex[j++]++; - --j; - // shift data of last vecs: - Index k = m_outerIndex[j]-1; - while (k>=Index(p)) - { - m_data.index(k) = m_data.index(k-1); - m_data.value(k) = m_data.value(k-1); - k--; - } + + // Adjust value ptr size. + value.data().resize(header.inner_buffer_size); + + // Initialize compressed state and inner non-zeros. + if (!header.compressed) { + // Inner non-zero counts. + std::size_t data_bytes = sizeof(StorageIndex) * header.outer_size; + if (EIGEN_PREDICT_FALSE(src + data_bytes > end)) return nullptr; + memcpy(value.innerNonZeroPtr(), src, data_bytes); + src += data_bytes; } - } - while ( (p > startId) && (m_data.index(p-1) > inner) ) - { - m_data.index(p) = m_data.index(p-1); - m_data.value(p) = m_data.value(p-1); - --p; + // Outer indices. + std::size_t data_bytes = sizeof(StorageIndex) * (header.outer_size + 1); + if (EIGEN_PREDICT_FALSE(src + data_bytes > end)) return nullptr; + memcpy(value.outerIndexPtr(), src, data_bytes); + src += data_bytes; + + // Inner indices. + data_bytes = sizeof(StorageIndex) * header.inner_buffer_size; + if (EIGEN_PREDICT_FALSE(src + data_bytes > end)) return nullptr; + memcpy(value.innerIndexPtr(), src, data_bytes); + src += data_bytes; + + // Values. + data_bytes = sizeof(Scalar) * header.inner_buffer_size; + if (EIGEN_PREDICT_FALSE(src + data_bytes > end)) return nullptr; + memcpy(value.valuePtr(), src, data_bytes); + src += data_bytes; + return src; } - - m_data.index(p) = inner; - return (m_data.value(p) = Scalar(0)); -} - -namespace internal { - -template -struct evaluator > - : evaluator > > -{ - typedef evaluator > > Base; - typedef SparseMatrix<_Scalar,_Options,_StorageIndex> SparseMatrixType; - evaluator() : Base() {} - explicit evaluator(const SparseMatrixType &mat) : Base(mat) {} }; -} - } // end namespace Eigen #endif // EIGEN_SPARSEMATRIX_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMatrixBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMatrixBase.h index 229449f0227..f96d2a7889d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMatrixBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseMatrixBase.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSEMATRIXBASE_H #define EIGEN_SPARSEMATRIXBASE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \ingroup SparseCore_Module @@ -69,8 +72,7 @@ template class SparseMatrixBase * \sa MatrixBase::rows(), MatrixBase::cols(), RowsAtCompileTime, SizeAtCompileTime */ - SizeAtCompileTime = (internal::size_at_compile_time::RowsAtCompileTime, - internal::traits::ColsAtCompileTime>::ret), + SizeAtCompileTime = (internal::size_of_xpr_at_compile_time::ret), /**< This is equal to the number of coefficients, i.e. the number of * rows times the number of columns, or to \a Dynamic if this is not * known at compile-time. \sa RowsAtCompileTime, ColsAtCompileTime */ @@ -78,8 +80,7 @@ template class SparseMatrixBase MaxRowsAtCompileTime = RowsAtCompileTime, MaxColsAtCompileTime = ColsAtCompileTime, - MaxSizeAtCompileTime = (internal::size_at_compile_time::ret), + MaxSizeAtCompileTime = internal::size_at_compile_time(MaxRowsAtCompileTime, MaxColsAtCompileTime), IsVectorAtCompileTime = RowsAtCompileTime == 1 || ColsAtCompileTime == 1, /**< This is set to true if either the number of rows or the number of @@ -103,17 +104,17 @@ template class SparseMatrixBase : int(IsRowMajor) ? int(ColsAtCompileTime) : int(RowsAtCompileTime), #ifndef EIGEN_PARSED_BY_DOXYGEN - _HasDirectAccess = (int(Flags)&DirectAccessBit) ? 1 : 0 // workaround sunCC + HasDirectAccess_ = (int(Flags)&DirectAccessBit) ? 1 : 0 // workaround sunCC #endif }; /** \internal the return type of MatrixBase::adjoint() */ - typedef typename internal::conditional::IsComplex, + typedef std::conditional_t::IsComplex, CwiseUnaryOp, Eigen::Transpose >, Transpose - >::type AdjointReturnType; + > AdjointReturnType; typedef Transpose TransposeReturnType; - typedef typename internal::add_const >::type ConstTransposeReturnType; + typedef Transpose ConstTransposeReturnType; // FIXME storage order do not match evaluator storage order typedef SparseMatrix PlainObject; @@ -129,7 +130,7 @@ template class SparseMatrixBase /** \internal the return type of coeff() */ - typedef typename internal::conditional<_HasDirectAccess, const Scalar&, Scalar>::type CoeffReturnType; + typedef std::conditional_t CoeffReturnType; /** \internal Represents a matrix with all coefficients equal to one another*/ typedef CwiseNullaryOp,Matrix > ConstantReturnType; @@ -137,8 +138,8 @@ template class SparseMatrixBase /** type of the equivalent dense matrix */ typedef Matrix DenseMatrixType; /** type of the equivalent square matrix */ - typedef Matrix SquareMatrixType; + typedef Matrix SquareMatrixType; inline const Derived& derived() const { return *static_cast(this); } inline Derived& derived() { return *static_cast(this); } @@ -159,11 +160,11 @@ template class SparseMatrixBase #define EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL #define EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(COND) #endif -# include "../plugins/CommonCwiseUnaryOps.h" -# include "../plugins/CommonCwiseBinaryOps.h" -# include "../plugins/MatrixCwiseUnaryOps.h" -# include "../plugins/MatrixCwiseBinaryOps.h" -# include "../plugins/BlockMethods.h" +# include "../plugins/CommonCwiseUnaryOps.inc" +# include "../plugins/CommonCwiseBinaryOps.inc" +# include "../plugins/MatrixCwiseUnaryOps.inc" +# include "../plugins/MatrixCwiseBinaryOps.inc" +# include "../plugins/BlockMethods.inc" # ifdef EIGEN_SPARSEMATRIXBASE_PLUGIN # include EIGEN_SPARSEMATRIXBASE_PLUGIN # endif @@ -214,11 +215,11 @@ template class SparseMatrixBase inline void assignGeneric(const OtherDerived& other); public: - +#ifndef EIGEN_NO_IO friend std::ostream & operator << (std::ostream & s, const SparseMatrixBase& m) { typedef typename Derived::Nested Nested; - typedef typename internal::remove_all::type NestedCleaned; + typedef internal::remove_all_t NestedCleaned; if (Flags&RowMajorBit) { @@ -263,6 +264,7 @@ template class SparseMatrixBase } return s; } +#endif template Derived& operator+=(const SparseMatrixBase& other); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparsePermutation.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparsePermutation.h index ef38357aefd..2ae9c15fbf9 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparsePermutation.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparsePermutation.h @@ -12,73 +12,149 @@ // This file implements sparse * permutation products +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { +template::value> +struct XprHelper +{ + XprHelper(const ExpressionType& xpr) : m_xpr(xpr) {} + inline const PlainObjectType& xpr() const { return m_xpr; } + // this is a new PlainObjectType initialized by xpr + const PlainObjectType m_xpr; +}; +template +struct XprHelper +{ + XprHelper(const ExpressionType& xpr) : m_xpr(xpr) {} + inline const PlainObjectType& xpr() const { return m_xpr; } + // this is a reference to xpr + const PlainObjectType& m_xpr; +}; + +template +struct PermHelper +{ + using IndicesType = typename PermDerived::IndicesType; + using PermutationIndex = typename IndicesType::Scalar; + using type = PermutationMatrix; + PermHelper(const PermDerived& perm) : m_perm(perm.inverse()) {} + inline const type& perm() const { return m_perm; } + // this is a new PermutationMatrix initialized by perm.inverse() + const type m_perm; +}; +template +struct PermHelper +{ + using type = PermDerived; + PermHelper(const PermDerived& perm) : m_perm(perm) {} + inline const type& perm() const { return m_perm; } + // this is a reference to perm + const type& m_perm; +}; + template struct permutation_matrix_product { - typedef typename nested_eval::type MatrixType; - typedef typename remove_all::type MatrixTypeCleaned; - - typedef typename MatrixTypeCleaned::Scalar Scalar; - typedef typename MatrixTypeCleaned::StorageIndex StorageIndex; - - enum { - SrcStorageOrder = MatrixTypeCleaned::Flags&RowMajorBit ? RowMajor : ColMajor, - MoveOuter = SrcStorageOrder==RowMajor ? Side==OnTheLeft : Side==OnTheRight - }; - - typedef typename internal::conditional, - SparseMatrix >::type ReturnType; - - template - static inline void run(Dest& dst, const PermutationType& perm, const ExpressionType& xpr) - { - MatrixType mat(xpr); - if(MoveOuter) - { - SparseMatrix tmp(mat.rows(), mat.cols()); - Matrix sizes(mat.outerSize()); - for(Index j=0; j::type; + using MatrixTypeCleaned = remove_all_t; + + using Scalar = typename MatrixTypeCleaned::Scalar; + using StorageIndex = typename MatrixTypeCleaned::StorageIndex; + + // the actual "return type" is `Dest`. this is a temporary type + using ReturnType = SparseMatrix; + using TmpHelper = XprHelper; + + static constexpr bool NeedOuterPermutation = ExpressionType::IsRowMajor ? Side == OnTheLeft : Side == OnTheRight; + static constexpr bool NeedInversePermutation = Transposed ? Side == OnTheLeft : Side == OnTheRight; + + template + static inline void permute_outer(Dest& dst, const PermutationType& perm, const ExpressionType& xpr) { + + // if ExpressionType is not ReturnType, evaluate `xpr` (allocation) + // otherwise, just reference `xpr` + // TODO: handle trivial expressions such as CwiseBinaryOp without temporary + const TmpHelper tmpHelper(xpr); + const ReturnType& tmp = tmpHelper.xpr(); + + ReturnType result(tmp.rows(), tmp.cols()); + + for (Index j = 0; j < tmp.outerSize(); j++) { Index jp = perm.indices().coeff(j); - sizes[((Side==OnTheLeft) ^ Transposed) ? jp : j] = StorageIndex(mat.innerVector(((Side==OnTheRight) ^ Transposed) ? jp : j).nonZeros()); + Index jsrc = NeedInversePermutation ? jp : j; + Index jdst = NeedInversePermutation ? j : jp; + Index begin = tmp.outerIndexPtr()[jsrc]; + Index end = tmp.isCompressed() ? tmp.outerIndexPtr()[jsrc + 1] : begin + tmp.innerNonZeroPtr()[jsrc]; + result.outerIndexPtr()[jdst + 1] += end - begin; } - tmp.reserve(sizes); - for(Index j=0; j tmp(mat.rows(), mat.cols()); - Matrix sizes(tmp.outerSize()); - sizes.setZero(); - PermutationMatrix perm_cpy; - if((Side==OnTheLeft) ^ Transposed) - perm_cpy = perm; - else - perm_cpy = perm.transpose(); - - for(Index j=0; j + static inline void permute_inner(Dest& dst, const PermutationType& perm, const ExpressionType& xpr) { + using InnerPermHelper = PermHelper; + using InnerPermType = typename InnerPermHelper::type; + + // if ExpressionType is not ReturnType, evaluate `xpr` (allocation) + // otherwise, just reference `xpr` + // TODO: handle trivial expressions such as CwiseBinaryOp without temporary + const TmpHelper tmpHelper(xpr); + const ReturnType& tmp = tmpHelper.xpr(); + + // if inverse permutation of inner indices is requested, calculate perm.inverse() (allocation) + // otherwise, just reference `perm` + const InnerPermHelper permHelper(perm); + const InnerPermType& innerPerm = permHelper.perm(); + + ReturnType result(tmp.rows(), tmp.cols()); + + for (Index j = 0; j < tmp.outerSize(); j++) { + Index begin = tmp.outerIndexPtr()[j]; + Index end = tmp.isCompressed() ? tmp.outerIndexPtr()[j + 1] : begin + tmp.innerNonZeroPtr()[j]; + result.outerIndexPtr()[j + 1] += end - begin; + } + + std::partial_sum(result.outerIndexPtr(), result.outerIndexPtr() + result.outerSize() + 1, result.outerIndexPtr()); + result.resizeNonZeros(result.nonZeros()); + + for (Index j = 0; j < tmp.outerSize(); j++) { + Index begin = tmp.outerIndexPtr()[j]; + Index end = tmp.isCompressed() ? tmp.outerIndexPtr()[j + 1] : begin + tmp.innerNonZeroPtr()[j]; + Index target = result.outerIndexPtr()[j]; + std::transform(tmp.innerIndexPtr() + begin, tmp.innerIndexPtr() + end, result.innerIndexPtr() + target, + [&innerPerm](StorageIndex i) { return innerPerm.indices().coeff(i); }); + smart_copy(tmp.valuePtr() + begin, tmp.valuePtr() + end, result.valuePtr() + target); + } + // the inner indices were permuted, and must be sorted + result.sortInnerIndices(); + dst = std::move(result); + } + + template = 0> + static inline void run(Dest& dst, const PermutationType& perm, const ExpressionType& xpr) { permute_outer(dst, perm, xpr); } + + template = 0> + static inline void run(Dest& dst, const PermutationType& perm, const ExpressionType& xpr) { permute_inner(dst, perm, xpr); } }; } @@ -107,7 +183,7 @@ struct product_evaluator, ProductTag, Permut explicit product_evaluator(const XprType& xpr) : m_result(xpr.rows(), xpr.cols()) { - ::new (static_cast(this)) Base(m_result); + internal::construct_at(this, m_result); generic_product_impl::evalTo(m_result, xpr.lhs(), xpr.rhs()); } @@ -141,35 +217,34 @@ struct product_evaluator, ProductTag, Sparse } // end namespace internal /** \returns the matrix with the permutation applied to the columns - */ -template -inline const Product -operator*(const SparseMatrixBase& matrix, const PermutationBase& perm) -{ return Product(matrix.derived(), perm.derived()); } + */ +template +inline const Product operator*( + const SparseMatrixBase& matrix, const PermutationBase& perm) { + return Product(matrix.derived(), perm.derived()); +} /** \returns the matrix with the permutation applied to the rows - */ -template -inline const Product -operator*( const PermutationBase& perm, const SparseMatrixBase& matrix) -{ return Product(perm.derived(), matrix.derived()); } - + */ +template +inline const Product operator*( + const PermutationBase& perm, const SparseMatrixBase& matrix) { + return Product(perm.derived(), matrix.derived()); +} /** \returns the matrix with the inverse permutation applied to the columns. - */ -template -inline const Product, AliasFreeProduct> -operator*(const SparseMatrixBase& matrix, const InverseImpl& tperm) -{ + */ +template +inline const Product, AliasFreeProduct> operator*( + const SparseMatrixBase& matrix, const InverseImpl& tperm) { return Product, AliasFreeProduct>(matrix.derived(), tperm.derived()); } /** \returns the matrix with the inverse permutation applied to the rows. - */ -template -inline const Product, SparseDerived, AliasFreeProduct> -operator*(const InverseImpl& tperm, const SparseMatrixBase& matrix) -{ + */ +template +inline const Product, SparseDerived, AliasFreeProduct> operator*( + const InverseImpl& tperm, const SparseMatrixBase& matrix) { return Product, SparseDerived, AliasFreeProduct>(tperm.derived(), matrix.derived()); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseProduct.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseProduct.h index af8a7744ddb..64a63a38f3a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseProduct.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseProduct.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSEPRODUCT_H #define EIGEN_SPARSEPRODUCT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \returns an expression of the product of two sparse matrices. @@ -45,19 +48,19 @@ struct generic_product_impl // dense += sparse * sparse template - static void addTo(Dest& dst, const ActualLhs& lhs, const Rhs& rhs, typename enable_if::Shape,DenseShape>::value,int*>::type* = 0) + static void addTo(Dest& dst, const ActualLhs& lhs, const Rhs& rhs, std::enable_if_t::Shape,DenseShape>::value,int*>* = 0) { typedef typename nested_eval::type LhsNested; typedef typename nested_eval::type RhsNested; LhsNested lhsNested(lhs); RhsNested rhsNested(rhs); - internal::sparse_sparse_to_dense_product_selector::type, - typename remove_all::type, Dest>::run(lhsNested,rhsNested,dst); + internal::sparse_sparse_to_dense_product_selector, + remove_all_t, Dest>::run(lhsNested,rhsNested,dst); } // dense -= sparse * sparse template - static void subTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, typename enable_if::Shape,DenseShape>::value,int*>::type* = 0) + static void subTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, std::enable_if_t::Shape,DenseShape>::value,int*>* = 0) { addTo(dst, -lhs, rhs); } @@ -72,8 +75,8 @@ struct generic_product_impl typedef typename nested_eval::type RhsNested; LhsNested lhsNested(lhs); RhsNested rhsNested(rhs); - internal::conservative_sparse_sparse_product_selector::type, - typename remove_all::type, Dest>::run(lhsNested,rhsNested,dst); + internal::conservative_sparse_sparse_product_selector, + remove_all_t, Dest>::run(lhsNested,rhsNested,dst); } // dense = sparse * sparse @@ -147,14 +150,14 @@ struct unary_evaluator >, IteratorBased> : m_result(xpr.rows(), xpr.cols()) { using std::abs; - ::new (static_cast(this)) Base(m_result); + internal::construct_at(this, m_result); typedef typename nested_eval::type LhsNested; typedef typename nested_eval::type RhsNested; LhsNested lhsNested(xpr.nestedExpression().lhs()); RhsNested rhsNested(xpr.nestedExpression().rhs()); - internal::sparse_sparse_product_with_pruning_selector::type, - typename remove_all::type, PlainObject>::run(lhsNested,rhsNested,m_result, + internal::sparse_sparse_product_with_pruning_selector, + remove_all_t, PlainObject>::run(lhsNested,rhsNested,m_result, abs(xpr.reference())*xpr.epsilon()); } @@ -165,9 +168,9 @@ struct unary_evaluator >, IteratorBased> } // end namespace internal // sparse matrix = sparse-product (can be sparse*sparse, sparse*perm, etc.) -template +template template -SparseMatrix& SparseMatrix::operator=(const Product& src) +SparseMatrix& SparseMatrix::operator=(const Product& src) { // std::cout << "in Assignment : " << DstOptions << "\n"; SparseMatrix dst(src.rows(),src.cols()); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseRedux.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseRedux.h index 45877496270..e39a944d5f2 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseRedux.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseRedux.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSEREDUX_H #define EIGEN_SPARSEREDUX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template @@ -25,9 +28,9 @@ SparseMatrixBase::sum() const return res; } -template -typename internal::traits >::Scalar -SparseMatrix<_Scalar,_Options,_Index>::sum() const +template +typename internal::traits >::Scalar +SparseMatrix::sum() const { eigen_assert(rows()>0 && cols()>0 && "you are using a non initialized matrix"); if(this->isCompressed()) @@ -36,9 +39,9 @@ SparseMatrix<_Scalar,_Options,_Index>::sum() const return Base::sum(); } -template -typename internal::traits >::Scalar -SparseVector<_Scalar,_Options,_Index>::sum() const +template +typename internal::traits >::Scalar +SparseVector::sum() const { eigen_assert(rows()>0 && cols()>0 && "you are using a non initialized matrix"); return Matrix::Map(m_data.valuePtr(), m_data.size()).sum(); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseRef.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseRef.h index 748f87d6263..a69ad017689 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseRef.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseRef.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSE_REF_H #define EIGEN_SPARSE_REF_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { enum { @@ -20,13 +23,13 @@ namespace internal { template class SparseRefBase; -template -struct traits, _Options, _StrideType> > +template +struct traits, Options_, StrideType_> > : public traits > { typedef SparseMatrix PlainObjectType; enum { - Options = _Options, + Options = Options_, Flags = traits::Flags | CompressedAccessBit | NestByRefBit }; @@ -35,27 +38,27 @@ struct traits, _Options, _Stride StorageOrderMatch = PlainObjectType::IsVectorAtCompileTime || Derived::IsVectorAtCompileTime || ((PlainObjectType::Flags&RowMajorBit)==(Derived::Flags&RowMajorBit)), MatchAtCompileTime = (Derived::Flags&CompressedAccessBit) && StorageOrderMatch }; - typedef typename internal::conditional::type type; + typedef std::conditional_t type; }; }; -template -struct traits, _Options, _StrideType> > - : public traits, _Options, _StrideType> > +template +struct traits, Options_, StrideType_> > + : public traits, Options_, StrideType_> > { enum { Flags = (traits >::Flags | CompressedAccessBit | NestByRefBit) & ~LvalueBit }; }; -template -struct traits, _Options, _StrideType> > +template +struct traits, Options_, StrideType_> > : public traits > { typedef SparseVector PlainObjectType; enum { - Options = _Options, + Options = Options_, Flags = traits::Flags | CompressedAccessBit | NestByRefBit }; @@ -63,14 +66,14 @@ struct traits, _Options, _Stride enum { MatchAtCompileTime = (Derived::Flags&CompressedAccessBit) && Derived::IsVectorAtCompileTime }; - typedef typename internal::conditional::type type; + typedef std::conditional_t type; }; }; -template -struct traits, _Options, _StrideType> > - : public traits, _Options, _StrideType> > +template +struct traits, Options_, StrideType_> > + : public traits, Options_, StrideType_> > { enum { Flags = (traits >::Flags | CompressedAccessBit | NestByRefBit) & ~LvalueBit @@ -98,9 +101,9 @@ template class SparseRefBase void construct(Expression& expr) { if(expr.outerIndexPtr()==0) - ::new (static_cast(this)) Base(expr.size(), expr.nonZeros(), expr.innerIndexPtr(), expr.valuePtr()); + internal::construct_at(this, expr.size(), expr.nonZeros(), expr.innerIndexPtr(), expr.valuePtr()); else - ::new (static_cast(this)) Base(expr.rows(), expr.cols(), expr.nonZeros(), expr.outerIndexPtr(), expr.innerIndexPtr(), expr.valuePtr(), expr.innerNonZeroPtr()); + internal::construct_at(this, expr.rows(), expr.cols(), expr.nonZeros(), expr.outerIndexPtr(), expr.innerIndexPtr(), expr.valuePtr(), expr.innerNonZeroPtr()); } }; @@ -133,7 +136,7 @@ class Ref template inline Ref(const SparseMatrix& expr); template - inline Ref(const MappedSparseMatrix& expr); + inline Ref(const Map>& expr); public: typedef internal::SparseRefBase Base; @@ -148,15 +151,15 @@ class Ref eigen_assert( ((Options & int(StandardCompressedFormat))==0) || (expr.isCompressed()) ); Base::construct(expr.derived()); } - + template - inline Ref(MappedSparseMatrix& expr) + inline Ref(Map >& expr) { EIGEN_STATIC_ASSERT(bool(Traits::template match >::MatchAtCompileTime), STORAGE_LAYOUT_DOES_NOT_MATCH); eigen_assert( ((Options & int(StandardCompressedFormat))==0) || (expr.isCompressed()) ); Base::construct(expr.derived()); } - + template inline Ref(const SparseCompressedBase& expr) #else @@ -201,8 +204,7 @@ class Ref, Options, StrideType ~Ref() { if(m_hasCopy) { - TPlainObjectType* obj = reinterpret_cast(&m_storage); - obj->~TPlainObjectType(); + internal::destroy_at(reinterpret_cast(&m_storage)); } } @@ -213,8 +215,7 @@ class Ref, Options, StrideType { if((Options & int(StandardCompressedFormat)) && (!expr.isCompressed())) { - TPlainObjectType* obj = reinterpret_cast(&m_storage); - ::new (obj) TPlainObjectType(expr); + TPlainObjectType* obj = internal::construct_at(reinterpret_cast(&m_storage), expr); m_hasCopy = true; Base::construct(*obj); } @@ -227,8 +228,7 @@ class Ref, Options, StrideType template void construct(const Expression& expr, internal::false_type) { - TPlainObjectType* obj = reinterpret_cast(&m_storage); - ::new (obj) TPlainObjectType(expr); + TPlainObjectType* obj = internal::construct_at(reinterpret_cast(&m_storage), expr); m_hasCopy = true; Base::construct(*obj); } @@ -319,8 +319,7 @@ class Ref, Options, StrideType ~Ref() { if(m_hasCopy) { - TPlainObjectType* obj = reinterpret_cast(&m_storage); - obj->~TPlainObjectType(); + internal::destroy_at(reinterpret_cast(&m_storage)); } } @@ -335,8 +334,7 @@ class Ref, Options, StrideType template void construct(const Expression& expr, internal::false_type) { - TPlainObjectType* obj = reinterpret_cast(&m_storage); - ::new (obj) TPlainObjectType(expr); + TPlainObjectType* obj = internal::construct_at(reinterpret_cast(&m_storage), expr); m_hasCopy = true; Base::construct(*obj); } @@ -355,7 +353,7 @@ struct evaluator, Options, Strid : evaluator, Options, StrideType> > > { typedef evaluator, Options, StrideType> > > Base; - typedef Ref, Options, StrideType> XprType; + typedef Ref, Options, StrideType> XprType; evaluator() : Base() {} explicit evaluator(const XprType &mat) : Base(mat) {} }; @@ -365,7 +363,7 @@ struct evaluator, Options, : evaluator, Options, StrideType> > > { typedef evaluator, Options, StrideType> > > Base; - typedef Ref, Options, StrideType> XprType; + typedef Ref, Options, StrideType> XprType; evaluator() : Base() {} explicit evaluator(const XprType &mat) : Base(mat) {} }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSelfAdjointView.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSelfAdjointView.h index 85b00e10e95..d775a3f4d88 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSelfAdjointView.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSelfAdjointView.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSE_SELFADJOINTVIEW_H #define EIGEN_SPARSE_SELFADJOINTVIEW_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \ingroup SparseCore_Module @@ -40,13 +43,13 @@ void permute_symm_to_fullsymm(const MatrixType& mat, SparseMatrix class SparseSelfAdjointView - : public EigenBase > +template class SparseSelfAdjointView + : public EigenBase > { public: enum { - Mode = _Mode, + Mode = Mode_, TransposeMode = ((Mode & Upper) ? Lower : 0) | ((Mode & Lower) ? Upper : 0), RowsAtCompileTime = internal::traits::RowsAtCompileTime, ColsAtCompileTime = internal::traits::ColsAtCompileTime @@ -57,7 +60,7 @@ template class SparseSelfAdjointView typedef typename MatrixType::StorageIndex StorageIndex; typedef Matrix VectorI; typedef typename internal::ref_selector::non_const_type MatrixTypeNested; - typedef typename internal::remove_all::type _MatrixTypeNested; + typedef internal::remove_all_t MatrixTypeNested_; explicit inline SparseSelfAdjointView(MatrixType& matrix) : m_matrix(matrix) { @@ -68,8 +71,8 @@ template class SparseSelfAdjointView inline Index cols() const { return m_matrix.cols(); } /** \internal \returns a reference to the nested matrix */ - const _MatrixTypeNested& matrix() const { return m_matrix; } - typename internal::remove_reference::type& matrix() { return m_matrix; } + const MatrixTypeNested_& matrix() const { return m_matrix; } + std::remove_reference_t& matrix() { return m_matrix; } /** \returns an expression of the matrix product between a sparse self-adjoint matrix \c *this and a sparse matrix \a rhs. * @@ -124,9 +127,9 @@ template class SparseSelfAdjointView /** \returns an expression of P H P^-1 */ // TODO implement twists in a more evaluator friendly fashion - SparseSymmetricPermutationProduct<_MatrixTypeNested,Mode> twistedBy(const PermutationMatrix& perm) const + SparseSymmetricPermutationProduct twistedBy(const PermutationMatrix& perm) const { - return SparseSymmetricPermutationProduct<_MatrixTypeNested,Mode>(m_matrix, perm); + return SparseSymmetricPermutationProduct(m_matrix, perm); } template @@ -260,15 +263,6 @@ struct Assignment run(tmp, src, AssignOpType()); dst -= tmp; } - - template - static void run(DynamicSparseMatrix& dst, const SrcXprType &src, const AssignOpType&/*func*/) - { - // TODO directly evaluate into dst; - SparseMatrix tmp(dst.rows(),dst.cols()); - internal::permute_symm_to_fullsymm(src.matrix(), tmp); - dst = tmp; - } }; } // end namespace internal @@ -285,7 +279,7 @@ inline void sparse_selfadjoint_time_dense_product(const SparseLhsType& lhs, cons EIGEN_ONLY_USED_FOR_DEBUG(alpha); typedef typename internal::nested_eval::type SparseLhsTypeNested; - typedef typename internal::remove_all::type SparseLhsTypeNestedCleaned; + typedef internal::remove_all_t SparseLhsTypeNestedCleaned; typedef evaluator LhsEval; typedef typename LhsEval::InnerIterator LhsIterator; typedef typename SparseLhsType::Scalar LhsScalar; @@ -347,7 +341,7 @@ struct generic_product_impl static void scaleAndAddTo(Dest& dst, const LhsView& lhsView, const Rhs& rhs, const typename Dest::Scalar& alpha) { - typedef typename LhsView::_MatrixTypeNested Lhs; + typedef typename LhsView::MatrixTypeNested_ Lhs; typedef typename nested_eval::type LhsNested; typedef typename nested_eval::type RhsNested; LhsNested lhsNested(lhsView.matrix()); @@ -364,7 +358,7 @@ struct generic_product_impl static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const RhsView& rhsView, const typename Dest::Scalar& alpha) { - typedef typename RhsView::_MatrixTypeNested Rhs; + typedef typename RhsView::MatrixTypeNested_ Rhs; typedef typename nested_eval::type LhsNested; typedef typename nested_eval::type RhsNested; LhsNested lhsNested(lhs); @@ -390,7 +384,7 @@ struct product_evaluator, ProductTag, Spar product_evaluator(const XprType& xpr) : m_lhs(xpr.lhs()), m_result(xpr.rows(), xpr.cols()) { - ::new (static_cast(this)) Base(m_result); + internal::construct_at(this, m_result); generic_product_impl::evalTo(m_result, m_lhs, xpr.rhs()); } @@ -516,7 +510,7 @@ void permute_symm_to_fullsymm(const MatrixType& mat, SparseMatrix +template void permute_symm_to_symm(const MatrixType& mat, SparseMatrix& _dest, const typename MatrixType::StorageIndex* perm) { typedef typename MatrixType::StorageIndex StorageIndex; @@ -529,8 +523,8 @@ void permute_symm_to_symm(const MatrixType& mat, SparseMatrix VectorI; typedef typename MatrixType::Nested MatrixTypeNested; - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; SparseSymmetricPermutationProduct(const MatrixType& mat, const Perm& perm) : m_matrix(mat), m_perm(perm) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSolverBase.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSolverBase.h index b4c9a422f05..c2926f40ca0 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSolverBase.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSolverBase.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSESOLVERBASE_H #define EIGEN_SPARSESOLVERBASE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -19,7 +22,7 @@ namespace internal { * The rhs is decomposed into small vertical panels which are solved through dense temporaries. */ template -typename enable_if::type +std::enable_if_t solve_sparse_through_dense_panels(const Decomposition &dec, const Rhs& rhs, Dest &dest) { EIGEN_STATIC_ASSERT((Dest::Flags&RowMajorBit)==0,THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES); @@ -43,7 +46,7 @@ solve_sparse_through_dense_panels(const Decomposition &dec, const Rhs& rhs, Dest // Overload for vector as rhs template -typename enable_if::type +std::enable_if_t solve_sparse_through_dense_panels(const Decomposition &dec, const Rhs& rhs, Dest &dest) { typedef typename Dest::Scalar DestScalar; @@ -73,6 +76,8 @@ class SparseSolverBase : internal::noncopyable : m_isInitialized(false) {} + SparseSolverBase(SparseSolverBase&&other ) : internal::noncopyable{}, m_isInitialized{other.m_isInitialized} {} + ~SparseSolverBase() {} diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSparseProductWithPruning.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSparseProductWithPruning.h index 88820a48f36..9ee9292ebc1 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSparseProductWithPruning.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseSparseProductWithPruning.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSESPARSEPRODUCTWITHPRUNING_H #define EIGEN_SPARSESPARSEPRODUCTWITHPRUNING_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -21,9 +24,9 @@ static void sparse_sparse_product_with_pruning_impl(const Lhs& lhs, const Rhs& r { // return sparse_sparse_product_with_pruning_impl2(lhs,rhs,res); - typedef typename remove_all::type::Scalar RhsScalar; - typedef typename remove_all::type::Scalar ResScalar; - typedef typename remove_all::type::StorageIndex StorageIndex; + typedef typename remove_all_t::Scalar RhsScalar; + typedef typename remove_all_t::Scalar ResScalar; + typedef typename remove_all_t::StorageIndex StorageIndex; // make sure to call innerSize/outerSize since we fake the storage order. Index rows = lhs.innerSize(); @@ -90,9 +93,9 @@ struct sparse_sparse_product_with_pruning_selector::type _res(res.rows(), res.cols()); - internal::sparse_sparse_product_with_pruning_impl(lhs, rhs, _res, tolerance); - res.swap(_res); + remove_all_t res_(res.rows(), res.cols()); + internal::sparse_sparse_product_with_pruning_impl(lhs, rhs, res_, tolerance); + res.swap(res_); } }; @@ -104,9 +107,9 @@ struct sparse_sparse_product_with_pruning_selector SparseTemporaryType; - SparseTemporaryType _res(res.rows(), res.cols()); - internal::sparse_sparse_product_with_pruning_impl(lhs, rhs, _res, tolerance); - res = _res; + SparseTemporaryType res_(res.rows(), res.cols()); + internal::sparse_sparse_product_with_pruning_impl(lhs, rhs, res_, tolerance); + res = res_; } }; @@ -117,9 +120,9 @@ struct sparse_sparse_product_with_pruning_selector::type _res(res.rows(), res.cols()); - internal::sparse_sparse_product_with_pruning_impl(rhs, lhs, _res, tolerance); - res.swap(_res); + remove_all_t res_(res.rows(), res.cols()); + internal::sparse_sparse_product_with_pruning_impl(rhs, lhs, res_, tolerance); + res.swap(res_); } }; @@ -137,9 +140,9 @@ struct sparse_sparse_product_with_pruning_selector SparseTemporaryType; -// SparseTemporaryType _res(res.cols(), res.rows()); -// sparse_sparse_product_with_pruning_impl(rhs, lhs, _res); -// res = _res.transpose(); +// SparseTemporaryType res_(res.cols(), res.rows()); +// sparse_sparse_product_with_pruning_impl(rhs, lhs, res_); +// res = res_.transpose(); } }; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseTranspose.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseTranspose.h index 3757d4c6b07..b774cafa3ca 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseTranspose.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseTranspose.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSETRANSPOSE_H #define EIGEN_SPARSETRANSPOSE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseTriangularView.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseTriangularView.h index 9ac120266a8..2e53114c250 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseTriangularView.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseTriangularView.h @@ -11,6 +11,9 @@ #ifndef EIGEN_SPARSE_TRIANGULARVIEW_H #define EIGEN_SPARSE_TRIANGULARVIEW_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \ingroup SparseCore_Module @@ -44,8 +47,8 @@ template class TriangularViewImpl::type MatrixTypeNestedNonRef; - typedef typename internal::remove_all::type MatrixTypeNestedCleaned; + typedef std::remove_reference_t MatrixTypeNestedNonRef; + typedef internal::remove_all_t MatrixTypeNestedCleaned; template EIGEN_DEVICE_FUNC @@ -149,8 +152,8 @@ struct unary_evaluator, IteratorBased> } } -// inline Index row() const { return (ArgType::Flags&RowMajorBit ? Base::outer() : this->index()); } -// inline Index col() const { return (ArgType::Flags&RowMajorBit ? this->index() : Base::outer()); } + inline Index row() const { return (ArgType::Flags&RowMajorBit ? Base::outer() : this->index()); } + inline Index col() const { return (ArgType::Flags&RowMajorBit ? this->index() : Base::outer()); } inline StorageIndex index() const { if(HasUnitDiag && m_returnOne) return internal::convert_index(Base::outer()); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseUtil.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseUtil.h index ceb9368879f..c9b5e8334a7 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseUtil.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseUtil.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSEUTIL_H #define EIGEN_SPARSEUTIL_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { #ifdef NDEBUG @@ -49,10 +52,8 @@ const int InnerRandomAccessPattern = 0x2 | CoherentAccessPattern; const int OuterRandomAccessPattern = 0x4 | CoherentAccessPattern; const int RandomAccessPattern = 0x8 | OuterRandomAccessPattern | InnerRandomAccessPattern; -template class SparseMatrix; -template class DynamicSparseMatrix; -template class SparseVector; -template class MappedSparseMatrix; +template class SparseMatrix; +template class SparseVector; template class SparseSelfAdjointView; template class SparseDiagonalProduct; @@ -65,10 +66,10 @@ template class SparseDenseOuterProdu template struct SparseSparseProductReturnType; template::ColsAtCompileTime,internal::traits::RowsAtCompileTime)> struct DenseSparseProductReturnType; + int InnerSize = internal::min_size_prefer_fixed(internal::traits::ColsAtCompileTime, internal::traits::RowsAtCompileTime)> struct DenseSparseProductReturnType; template::ColsAtCompileTime,internal::traits::RowsAtCompileTime)> struct SparseDenseProductReturnType; + int InnerSize = internal::min_size_prefer_fixed(internal::traits::ColsAtCompileTime, internal::traits::RowsAtCompileTime)> struct SparseDenseProductReturnType; template class SparseSymmetricPermutationProduct; namespace internal { @@ -80,41 +81,41 @@ template struct eval {}; template struct sparse_eval { - typedef typename traits::Scalar _Scalar; - typedef typename traits::StorageIndex _StorageIndex; + typedef typename traits::Scalar Scalar_; + typedef typename traits::StorageIndex StorageIndex_; public: - typedef SparseVector<_Scalar, RowMajor, _StorageIndex> type; + typedef SparseVector type; }; template struct sparse_eval { - typedef typename traits::Scalar _Scalar; - typedef typename traits::StorageIndex _StorageIndex; + typedef typename traits::Scalar Scalar_; + typedef typename traits::StorageIndex StorageIndex_; public: - typedef SparseVector<_Scalar, ColMajor, _StorageIndex> type; + typedef SparseVector type; }; // TODO this seems almost identical to plain_matrix_type template struct sparse_eval { - typedef typename traits::Scalar _Scalar; - typedef typename traits::StorageIndex _StorageIndex; - enum { _Options = ((Flags&RowMajorBit)==RowMajorBit) ? RowMajor : ColMajor }; + typedef typename traits::Scalar Scalar_; + typedef typename traits::StorageIndex StorageIndex_; + enum { Options_ = ((Flags&RowMajorBit)==RowMajorBit) ? RowMajor : ColMajor }; public: - typedef SparseMatrix<_Scalar, _Options, _StorageIndex> type; + typedef SparseMatrix type; }; template struct sparse_eval { - typedef typename traits::Scalar _Scalar; + typedef typename traits::Scalar Scalar_; public: - typedef Matrix<_Scalar, 1, 1> type; + typedef Matrix type; }; template struct plain_matrix_type { - typedef typename traits::Scalar _Scalar; - typedef typename traits::StorageIndex _StorageIndex; - enum { _Options = ((evaluator::Flags&RowMajorBit)==RowMajorBit) ? RowMajor : ColMajor }; + typedef typename traits::Scalar Scalar_; + typedef typename traits::StorageIndex StorageIndex_; + enum { Options_ = ((evaluator::Flags&RowMajorBit)==RowMajorBit) ? RowMajor : ColMajor }; public: - typedef SparseMatrix<_Scalar, _Options, _StorageIndex> type; + typedef SparseMatrix type; }; template diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseVector.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseVector.h index 05779be685b..fd5eb19ba42 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseVector.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseVector.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSEVECTOR_H #define EIGEN_SPARSEVECTOR_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /** \ingroup SparseCore_Module @@ -17,7 +20,7 @@ namespace Eigen { * * \brief a sparse vector class * - * \tparam _Scalar the scalar type, i.e. the type of the coefficients + * \tparam Scalar_ the scalar type, i.e. the type of the coefficients * * See http://www.netlib.org/linalg/html_templates/node91.html for details on the storage scheme. * @@ -26,21 +29,21 @@ namespace Eigen { */ namespace internal { -template -struct traits > +template +struct traits > { - typedef _Scalar Scalar; - typedef _StorageIndex StorageIndex; + typedef Scalar_ Scalar; + typedef StorageIndex_ StorageIndex; typedef Sparse StorageKind; typedef MatrixXpr XprKind; enum { - IsColVector = (_Options & RowMajorBit) ? 0 : 1, + IsColVector = (Options_ & RowMajorBit) ? 0 : 1, RowsAtCompileTime = IsColVector ? Dynamic : 1, ColsAtCompileTime = IsColVector ? 1 : Dynamic, MaxRowsAtCompileTime = RowsAtCompileTime, MaxColsAtCompileTime = ColsAtCompileTime, - Flags = _Options | NestByRefBit | LvalueBit | (IsColVector ? 0 : RowMajorBit) | CompressedAccessBit, + Flags = Options_ | NestByRefBit | LvalueBit | (IsColVector ? 0 : RowMajorBit) | CompressedAccessBit, SupportedAccessPatterns = InnerRandomAccessPattern }; }; @@ -60,9 +63,9 @@ struct sparse_vector_assign_selector; } -template +template class SparseVector - : public SparseCompressedBase > + : public SparseCompressedBase > { typedef SparseCompressedBase Base; using Base::convert_index; @@ -75,7 +78,7 @@ class SparseVector enum { IsColVector = internal::traits::IsColVector }; enum { - Options = _Options + Options = Options_ }; EIGEN_STRONG_INLINE Index rows() const { return IsColVector ? m_size : 1; } @@ -207,9 +210,33 @@ class SparseVector inline void finalize() {} /** \copydoc SparseMatrix::prune(const Scalar&,const RealScalar&) */ - void prune(const Scalar& reference, const RealScalar& epsilon = NumTraits::dummy_precision()) + Index prune(const Scalar& reference, const RealScalar& epsilon = NumTraits::dummy_precision()) { + return prune([&](const Scalar& val){ return !internal::isMuchSmallerThan(val, reference, epsilon); }); + } + + /** + * \brief Prunes the entries of the vector based on a `predicate` + * \tparam F Type of the predicate. + * \param keep_predicate The predicate that is used to test whether a value should be kept. A callable that + * gets passed om a `Scalar` value and returns a boolean. If the predicate returns true, the value is kept. + * \return The new number of structural non-zeros. + */ + template + Index prune(F&& keep_predicate) { - m_data.prune(reference,epsilon); + Index k = 0; + Index n = m_data.size(); + for (Index i = 0; i < n; ++i) + { + if (keep_predicate(m_data.value(i))) + { + m_data.value(k) = std::move(m_data.value(i)); + m_data.index(k) = m_data.index(i); + ++k; + } + } + m_data.resize(k); + return k; } /** Resizes the sparse vector to \a rows x \a cols @@ -256,11 +283,11 @@ class SparseVector void resizeNonZeros(Index size) { m_data.resize(size); } - inline SparseVector() : m_size(0) { check_template_parameters(); resize(0); } + inline SparseVector() : m_size(0) { resize(0); } - explicit inline SparseVector(Index size) : m_size(0) { check_template_parameters(); resize(size); } + explicit inline SparseVector(Index size) : m_size(0) { resize(size); } - inline SparseVector(Index rows, Index cols) : m_size(0) { check_template_parameters(); resize(rows,cols); } + inline SparseVector(Index rows, Index cols) : m_size(0) { resize(rows,cols); } template inline SparseVector(const SparseMatrixBase& other) @@ -269,14 +296,12 @@ class SparseVector #ifdef EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN #endif - check_template_parameters(); *this = other.derived(); } inline SparseVector(const SparseVector& other) : Base(other), m_size(0) { - check_template_parameters(); *this = other.derived(); } @@ -329,6 +354,7 @@ class SparseVector } #endif +#ifndef EIGEN_NO_IO friend std::ostream & operator << (std::ostream & s, const SparseVector& m) { for (Index i=0; i::IsSigned,THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE); - EIGEN_STATIC_ASSERT((_Options&(ColMajor|RowMajor))==Options,INVALID_MATRIX_TEMPLATE_PARAMETERS); - } - + EIGEN_STATIC_ASSERT(NumTraits::IsSigned,THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE) + EIGEN_STATIC_ASSERT((Options_&(ColMajor|RowMajor))==Options,INVALID_MATRIX_TEMPLATE_PARAMETERS) + Storage m_data; Index m_size; }; namespace internal { -template -struct evaluator > - : evaluator_base > +template +struct evaluator > + : evaluator_base > { - typedef SparseVector<_Scalar,_Options,_Index> SparseVectorType; + typedef SparseVector SparseVectorType; typedef evaluator_base Base; typedef typename SparseVectorType::InnerIterator InnerIterator; typedef typename SparseVectorType::ReverseInnerIterator ReverseInnerIterator; enum { - CoeffReadCost = NumTraits<_Scalar>::ReadCost, + CoeffReadCost = NumTraits::ReadCost, Flags = SparseVectorType::Flags }; @@ -473,6 +496,78 @@ struct sparse_vector_assign_selector { } +// Specialization for SparseVector. +// Serializes [size, numNonZeros, innerIndices, values]. +template +class Serializer, void> { + public: + typedef SparseVector SparseMat; + + struct Header { + typename SparseMat::Index size; + Index num_non_zeros; + }; + + EIGEN_DEVICE_FUNC size_t size(const SparseMat& value) const { + return sizeof(Header) + + (sizeof(Scalar) + sizeof(StorageIndex)) * value.nonZeros(); + } + + EIGEN_DEVICE_FUNC uint8_t* serialize(uint8_t* dest, uint8_t* end, + const SparseMat& value) { + if (EIGEN_PREDICT_FALSE(dest == nullptr)) return nullptr; + if (EIGEN_PREDICT_FALSE(dest + size(value) > end)) return nullptr; + + const size_t header_bytes = sizeof(Header); + Header header = {value.innerSize(), value.nonZeros()}; + EIGEN_USING_STD(memcpy) + memcpy(dest, &header, header_bytes); + dest += header_bytes; + + // Inner indices. + std::size_t data_bytes = sizeof(StorageIndex) * header.num_non_zeros; + memcpy(dest, value.innerIndexPtr(), data_bytes); + dest += data_bytes; + + // Values. + data_bytes = sizeof(Scalar) * header.num_non_zeros; + memcpy(dest, value.valuePtr(), data_bytes); + dest += data_bytes; + + return dest; + } + + EIGEN_DEVICE_FUNC const uint8_t* deserialize(const uint8_t* src, + const uint8_t* end, + SparseMat& value) const { + if (EIGEN_PREDICT_FALSE(src == nullptr)) return nullptr; + if (EIGEN_PREDICT_FALSE(src + sizeof(Header) > end)) return nullptr; + + const size_t header_bytes = sizeof(Header); + Header header; + EIGEN_USING_STD(memcpy) + memcpy(&header, src, header_bytes); + src += header_bytes; + + value.setZero(); + value.resize(header.size); + value.resizeNonZeros(header.num_non_zeros); + + // Inner indices. + std::size_t data_bytes = sizeof(StorageIndex) * header.num_non_zeros; + if (EIGEN_PREDICT_FALSE(src + data_bytes > end)) return nullptr; + memcpy(value.innerIndexPtr(), src, data_bytes); + src += data_bytes; + + // Values. + data_bytes = sizeof(Scalar) * header.num_non_zeros; + if (EIGEN_PREDICT_FALSE(src + data_bytes > end)) return nullptr; + memcpy(value.valuePtr(), src, data_bytes); + src += data_bytes; + return src; + } +}; + } // end namespace Eigen #endif // EIGEN_SPARSEVECTOR_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseView.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseView.h index 92b3d1f7bad..a1cdcf87534 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseView.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/SparseView.h @@ -11,6 +11,9 @@ #ifndef EIGEN_SPARSEVIEW_H #define EIGEN_SPARSEVIEW_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -45,11 +48,11 @@ template class SparseView : public SparseMatrixBase > { typedef typename MatrixType::Nested MatrixTypeNested; - typedef typename internal::remove_all::type _MatrixTypeNested; + typedef internal::remove_all_t MatrixTypeNested_; typedef SparseMatrixBase Base; public: EIGEN_SPARSE_PUBLIC_INTERFACE(SparseView) - typedef typename internal::remove_all::type NestedExpression; + typedef internal::remove_all_t NestedExpression; explicit SparseView(const MatrixType& mat, const Scalar& reference = Scalar(0), const RealScalar &epsilon = NumTraits::dummy_precision()) @@ -62,7 +65,7 @@ class SparseView : public SparseMatrixBase > inline Index outerSize() const { return m_matrix.outerSize(); } /** \returns the nested expression */ - const typename internal::remove_all::type& + const internal::remove_all_t& nestedExpression() const { return m_matrix; } Scalar reference() const { return m_reference; } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/TriangularSolver.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/TriangularSolver.h index f9c56ba7980..39fa7c7dd56 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/TriangularSolver.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseCore/TriangularSolver.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SPARSETRIANGULARSOLVER_H #define EIGEN_SPARSETRIANGULARSOLVER_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -114,7 +117,7 @@ struct sparse_solve_triangular_selector for(Index i=0; i for(Index i=lhs.cols()-1; i>=0; --i) { Scalar& tmp = other.coeffRef(i,col); - if (tmp!=Scalar(0)) // optimization when other is actually sparse + if (!numext::is_exactly_zero(tmp)) // optimization when other is actually sparse { if(!(Mode & UnitDiag)) { @@ -182,11 +185,11 @@ void TriangularViewImpl::solveInPlace(MatrixBase::Flags & RowMajorBit }; - typedef typename internal::conditional::type, OtherDerived&>::type OtherCopy; + typedef std::conditional_t::type, OtherDerived&> OtherCopy; OtherCopy otherCopy(other.derived()); - internal::sparse_solve_triangular_selector::type, Mode>::run(derived().nestedExpression(), otherCopy); + internal::sparse_solve_triangular_selector, Mode>::run(derived().nestedExpression(), otherCopy); if (copy) other = otherCopy; @@ -239,7 +242,7 @@ struct sparse_solve_triangular_sparse_selector { tempVector.restart(); Scalar& ci = tempVector.coeffRef(i); - if (ci!=Scalar(0)) + if (!numext::is_exactly_zero(ci)) { // find typename Lhs::InnerIterator it(lhs, i); @@ -270,11 +273,11 @@ struct sparse_solve_triangular_sparse_selector } - Index count = 0; +// Index count = 0; // FIXME compute a reference value to filter zeros for (typename AmbiVector::Iterator it(tempVector/*,1e-12*/); it; ++it) { - ++ count; +// ++ count; // std::cerr << "fill " << it.index() << ", " << col << "\n"; // std::cout << it.value() << " "; // FIXME use insertBack @@ -299,8 +302,8 @@ void TriangularViewImpl::solveInPlace(SparseMatrixBa // enum { copy = internal::traits::Flags & RowMajorBit }; -// typedef typename internal::conditional::type, OtherDerived&>::type OtherCopy; +// typedef std::conditional_t::type, OtherDerived&> OtherCopy; // OtherCopy otherCopy(other.derived()); internal::sparse_solve_triangular_sparse_selector::run(derived().nestedExpression(), other.derived()); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/InternalHeaderCheck.h new file mode 100644 index 00000000000..78ebfcc045c --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_SPARSELU_MODULE_H +#error "Please include Eigen/SparseLU instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU.h index 0c8d8939be2..4edd26793be 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU.h @@ -12,9 +12,12 @@ #ifndef EIGEN_SPARSE_LU_H #define EIGEN_SPARSE_LU_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { -template > class SparseLU; +template > class SparseLU; template struct SparseLUMatrixLReturnType; template struct SparseLUMatrixUReturnType; @@ -35,9 +38,10 @@ class SparseLUTransposeView : public SparseSolverBasem_sparseLU = view.m_sparseLU; + this->m_isInitialized = view.m_isInitialized; } void setIsInitialized(const bool isInitialized) {this->m_isInitialized = isInitialized;} void setSparseLU(SparseLUType* sparseLU) {m_sparseLU = sparseLU;} @@ -119,25 +123,25 @@ class SparseLUTransposeView : public SparseSolverBase - * \tparam _OrderingType The ordering method to use, either AMD, COLAMD or METIS. Default is COLMAD + * \tparam MatrixType_ The type of the sparse matrix. It must be a column-major SparseMatrix<> + * \tparam OrderingType_ The ordering method to use, either AMD, COLAMD or METIS. Default is COLMAD * * \implsparsesolverconcept * * \sa \ref TutorialSparseSolverConcept * \sa \ref OrderingMethods_Module */ -template -class SparseLU : public SparseSolverBase >, public internal::SparseLUImpl +template +class SparseLU : public SparseSolverBase >, public internal::SparseLUImpl { protected: - typedef SparseSolverBase > APIBase; + typedef SparseSolverBase > APIBase; using APIBase::m_isInitialized; public: using APIBase::_solve_impl; - typedef _MatrixType MatrixType; - typedef _OrderingType OrderingType; + typedef MatrixType_ MatrixType; + typedef OrderingType_ OrderingType; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef typename MatrixType::StorageIndex StorageIndex; @@ -197,9 +201,9 @@ class SparseLU : public SparseSolverBase >, * * \sa adjoint(), solve() */ - const SparseLUTransposeView > transpose() + const SparseLUTransposeView > transpose() { - SparseLUTransposeView > transposeView; + SparseLUTransposeView > transposeView; transposeView.setSparseLU(this); transposeView.setIsInitialized(this->m_isInitialized); return transposeView; @@ -218,9 +222,9 @@ class SparseLU : public SparseSolverBase >, * * \sa transpose(), solve() */ - const SparseLUTransposeView > adjoint() + const SparseLUTransposeView > adjoint() { - SparseLUTransposeView > adjointView; + SparseLUTransposeView > adjointView; adjointView.setSparseLU(this); adjointView.setIsInitialized(this->m_isInitialized); return adjointView; @@ -250,9 +254,9 @@ class SparseLU : public SparseSolverBase >, * y = b; matrixU().solveInPlace(y); * \endcode */ - SparseLUMatrixUReturnType > matrixU() const + SparseLUMatrixUReturnType > > matrixU() const { - return SparseLUMatrixUReturnType >(m_Lstore, m_Ustore); + return SparseLUMatrixUReturnType > >(m_Lstore, m_Ustore); } /** @@ -452,8 +456,8 @@ class SparseLU : public SparseSolverBase >, return (m_detPermR * m_detPermC) > 0 ? det : -det; } - Index nnzL() const { return m_nnzL; }; - Index nnzU() const { return m_nnzU; }; + Index nnzL() const { return m_nnzL; } + Index nnzU() const { return m_nnzU; } protected: // Functions @@ -474,7 +478,7 @@ class SparseLU : public SparseSolverBase >, std::string m_lastError; NCMatrix m_mat; // The input (permuted ) matrix SCMatrix m_Lstore; // The lower triangular matrix (supernodal) - MappedSparseMatrix m_Ustore; // The upper triangular matrix + Map> m_Ustore; // The upper triangular matrix PermutationType m_perm_c; // Column permutation PermutationType m_perm_r ; // Row permutation IndexVector m_etree; // Column elimination tree @@ -752,10 +756,13 @@ void SparseLU::factorize(const MatrixType& matrix) info = Base::pivotL(jj, m_diagpivotthresh, m_perm_r.indices(), iperm_c.indices(), pivrow, m_glu); if ( info ) { - m_lastError = "THE MATRIX IS STRUCTURALLY SINGULAR ... ZERO COLUMN AT "; + m_lastError = "THE MATRIX IS STRUCTURALLY SINGULAR"; +#ifndef EIGEN_NO_IO std::ostringstream returnInfo; - returnInfo << info; + returnInfo << " ... ZERO COLUMN AT "; + returnInfo << info; m_lastError += returnInfo.str(); +#endif m_info = NumericalIssue; m_factorizationIsOk = false; return; @@ -789,7 +796,7 @@ void SparseLU::factorize(const MatrixType& matrix) // Create supernode matrix L m_Lstore.setInfos(m, n, m_glu.lusup, m_glu.xlusup, m_glu.lsub, m_glu.xlsub, m_glu.supno, m_glu.xsup); // Create the column major upper sparse matrix U; - new (&m_Ustore) MappedSparseMatrix ( m, n, m_nnzU, m_glu.xusub.data(), m_glu.usub.data(), m_glu.ucol.data() ); + new (&m_Ustore) Map> ( m, n, m_nnzU, m_glu.xusub.data(), m_glu.usub.data(), m_glu.ucol.data() ); m_info = Success; m_factorizationIsOk = true; @@ -814,6 +821,31 @@ struct SparseLUMatrixLReturnType : internal::no_assignment_operator m_mapL.template solveTransposedInPlace(X); } + SparseMatrix toSparse() const { + ArrayXi colCount = ArrayXi::Ones(cols()); + for (Index i = 0; i < cols(); i++) { + typename MappedSupernodalType::InnerIterator iter(m_mapL, i); + for (; iter; ++iter) { + if (iter.row() > iter.col()) { + colCount(iter.col())++; + } + } + } + SparseMatrix sL(rows(), cols()); + sL.reserve(colCount); + for (Index i = 0; i < cols(); i++) { + sL.insert(i, i) = 1.0; + typename MappedSupernodalType::InnerIterator iter(m_mapL, i); + for (; iter; ++iter) { + if (iter.row() > iter.col()) { + sL.insert(iter.row(), iter.col()) = iter.value(); + } + } + } + sL.makeCompressed(); + return sL; + } + const MappedSupernodalType& m_mapL; }; @@ -830,7 +862,6 @@ struct SparseLUMatrixUReturnType : internal::no_assignment_operator template void solveInPlace(MatrixBase &X) const { Index nrhs = X.cols(); - Index n = X.rows(); // Backward solve with U for (Index k = m_mapL.nsuper(); k >= 0; k--) { @@ -850,7 +881,7 @@ struct SparseLUMatrixUReturnType : internal::no_assignment_operator { // FIXME: the following lines should use Block expressions and not Map! Map, 0, OuterStride<> > A( &(m_mapL.valuePtr()[luptr]), nsupc, nsupc, OuterStride<>(lda) ); - Map< Matrix, 0, OuterStride<> > U (&(X.coeffRef(fsupc,0)), nsupc, nrhs, OuterStride<>(n) ); + typename Dest::RowsBlockXpr U = X.derived().middleRows(fsupc, nsupc); U = A.template triangularView().solve(U); } @@ -873,7 +904,6 @@ struct SparseLUMatrixUReturnType : internal::no_assignment_operator { using numext::conj; Index nrhs = X.cols(); - Index n = X.rows(); // Forward solve with U for (Index k = 0; k <= m_mapL.nsuper(); k++) { @@ -904,7 +934,7 @@ struct SparseLUMatrixUReturnType : internal::no_assignment_operator else { Map, 0, OuterStride<> > A( &(m_mapL.valuePtr()[luptr]), nsupc, nsupc, OuterStride<>(lda) ); - Map< Matrix, 0, OuterStride<> > U (&(X(fsupc,0)), nsupc, nrhs, OuterStride<>(n) ); + typename Dest::RowsBlockXpr U = X.derived().middleRows(fsupc, nsupc); if(Conjugate) U = A.adjoint().template triangularView().solve(U); else @@ -913,6 +943,32 @@ struct SparseLUMatrixUReturnType : internal::no_assignment_operator }// End For U-solve } + SparseMatrix toSparse() { + ArrayXi rowCount = ArrayXi::Zero(rows()); + for (Index i = 0; i < cols(); i++) { + typename MatrixLType::InnerIterator iter(m_mapL, i); + for (; iter; ++iter) { + if (iter.row() <= iter.col()) { + rowCount(iter.row())++; + } + } + } + + SparseMatrix sU(rows(), cols()); + sU.reserve(rowCount); + for (Index i = 0; i < cols(); i++) { + typename MatrixLType::InnerIterator iter(m_mapL, i); + for (; iter; ++iter) { + if (iter.row() <= iter.col()) { + sU.insert(iter.row(), iter.col()) = iter.value(); + } + } + } + sU.makeCompressed(); + const SparseMatrix u = m_mapU; // convert to RowMajor + sU += u; + return sU; + } const MatrixLType& m_mapL; const MatrixUType& m_mapU; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLUImpl.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLUImpl.h index fc0cfc4de1a..ff2b111d473 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLUImpl.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLUImpl.h @@ -9,6 +9,9 @@ #ifndef SPARSELU_IMPL_H #define SPARSELU_IMPL_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Memory.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Memory.h index 349bfd585bd..cc26256e276 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Memory.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Memory.h @@ -31,6 +31,9 @@ #ifndef EIGEN_SPARSELU_MEMORY #define EIGEN_SPARSELU_MEMORY +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Structs.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Structs.h index cf5ec449bec..f1f109a2f8c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Structs.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Structs.h @@ -68,10 +68,13 @@ #ifndef EIGEN_LU_STRUCTS #define EIGEN_LU_STRUCTS +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { - -typedef enum {LUSUP, UCOL, LSUB, USUB, LLVL, ULVL} MemType; + +enum MemType {LUSUP, UCOL, LSUB, USUB, LLVL, ULVL}; template struct LU_GlobalLU_t { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_SupernodalMatrix.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_SupernodalMatrix.h index 0be293d17fa..627e55d90b9 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_SupernodalMatrix.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_SupernodalMatrix.h @@ -11,6 +11,9 @@ #ifndef EIGEN_SPARSELU_SUPERNODAL_MATRIX_H #define EIGEN_SPARSELU_SUPERNODAL_MATRIX_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -29,12 +32,12 @@ namespace internal { * SuperInnerIterator to iterate through all supernodes * Function for triangular solve */ -template +template class MappedSuperNodalMatrix { public: - typedef _Scalar Scalar; - typedef _StorageIndex StorageIndex; + typedef Scalar_ Scalar; + typedef StorageIndex_ StorageIndex; typedef Matrix IndexVector; typedef Matrix ScalarVector; public: @@ -274,9 +277,8 @@ void MappedSuperNodalMatrix::solveInPlace( MatrixBase&X) co // Triangular solve Map, 0, OuterStride<> > A( &(Lval[luptr]), nsupc, nsupc, OuterStride<>(lda) ); - Map< Matrix, 0, OuterStride<> > U (&(X(fsupc,0)), nsupc, nrhs, OuterStride<>(n) ); - U = A.template triangularView().solve(U); - + typename Dest::RowsBlockXpr U = X.derived().middleRows(fsupc, nsupc); + U = A.template triangularView().solve(U); // Matrix-vector product new (&A) Map, 0, OuterStride<> > ( &(Lval[luptr+nsupc]), nrow, nsupc, OuterStride<>(lda) ); work.topRows(nrow).noalias() = A * U; @@ -349,7 +351,7 @@ void MappedSuperNodalMatrix::solveTransposedInPlace( MatrixBase, 0, OuterStride<> > A( &(Lval[luptr+nsupc]), nrow, nsupc, OuterStride<>(lda) ); - Map< Matrix, 0, OuterStride<> > U (&(X(fsupc,0)), nsupc, nrhs, OuterStride<>(n) ); + typename Dest::RowsBlockXpr U = X.derived().middleRows(fsupc, nsupc); if(Conjugate) U = U - A.adjoint() * work.topRows(nrow); else diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Utils.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Utils.h index 9e3dab44d99..79544e69f7b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Utils.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_Utils.h @@ -11,6 +11,9 @@ #ifndef EIGEN_SPARSELU_UTILS_H #define EIGEN_SPARSELU_UTILS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_column_bmod.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_column_bmod.h index b57f06802e2..04166076c3a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_column_bmod.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_column_bmod.h @@ -31,6 +31,9 @@ #ifndef SPARSELU_COLUMN_BMOD_H #define SPARSELU_COLUMN_BMOD_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_column_dfs.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_column_dfs.h index 5a2c941b4ab..ddead00d72f 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_column_dfs.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_column_dfs.h @@ -31,6 +31,9 @@ #define SPARSELU_COLUMN_DFS_H template class SparseLUImpl; +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_copy_to_ucol.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_copy_to_ucol.h index c32d8d8b14b..b9e7c627721 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_copy_to_ucol.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_copy_to_ucol.h @@ -29,6 +29,9 @@ #ifndef SPARSELU_COPY_TO_UCOL_H #define SPARSELU_COPY_TO_UCOL_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_gemm_kernel.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_gemm_kernel.h deleted file mode 100644 index e37c2fe0d02..00000000000 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_gemm_kernel.h +++ /dev/null @@ -1,280 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2012 Gael Guennebaud -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_SPARSELU_GEMM_KERNEL_H -#define EIGEN_SPARSELU_GEMM_KERNEL_H - -namespace Eigen { - -namespace internal { - - -/** \internal - * A general matrix-matrix product kernel optimized for the SparseLU factorization. - * - A, B, and C must be column major - * - lda and ldc must be multiples of the respective packet size - * - C must have the same alignment as A - */ -template -EIGEN_DONT_INLINE -void sparselu_gemm(Index m, Index n, Index d, const Scalar* A, Index lda, const Scalar* B, Index ldb, Scalar* C, Index ldc) -{ - using namespace Eigen::internal; - - typedef typename packet_traits::type Packet; - enum { - NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS, - PacketSize = packet_traits::size, - PM = 8, // peeling in M - RN = 2, // register blocking - RK = NumberOfRegisters>=16 ? 4 : 2, // register blocking - BM = 4096/sizeof(Scalar), // number of rows of A-C per chunk - SM = PM*PacketSize // step along M - }; - Index d_end = (d/RK)*RK; // number of columns of A (rows of B) suitable for full register blocking - Index n_end = (n/RN)*RN; // number of columns of B-C suitable for processing RN columns at once - Index i0 = internal::first_default_aligned(A,m); - - eigen_internal_assert(((lda%PacketSize)==0) && ((ldc%PacketSize)==0) && (i0==internal::first_default_aligned(C,m))); - - // handle the non aligned rows of A and C without any optimization: - for(Index i=0; i(BM, m-ib); // actual number of rows - Index actual_b_end1 = (actual_b/SM)*SM; // actual number of rows suitable for peeling - Index actual_b_end2 = (actual_b/PacketSize)*PacketSize; // actual number of rows suitable for vectorization - - // Let's process two columns of B-C at once - for(Index j=0; j(Bc0[0]); } - { b10 = pset1(Bc0[1]); } - if(RK==4) { b20 = pset1(Bc0[2]); } - if(RK==4) { b30 = pset1(Bc0[3]); } - { b01 = pset1(Bc1[0]); } - { b11 = pset1(Bc1[1]); } - if(RK==4) { b21 = pset1(Bc1[2]); } - if(RK==4) { b31 = pset1(Bc1[3]); } - - Packet a0, a1, a2, a3, c0, c1, t0, t1; - - const Scalar* A0 = A+ib+(k+0)*lda; - const Scalar* A1 = A+ib+(k+1)*lda; - const Scalar* A2 = A+ib+(k+2)*lda; - const Scalar* A3 = A+ib+(k+3)*lda; - - Scalar* C0 = C+ib+(j+0)*ldc; - Scalar* C1 = C+ib+(j+1)*ldc; - - a0 = pload(A0); - a1 = pload(A1); - if(RK==4) - { - a2 = pload(A2); - a3 = pload(A3); - } - else - { - // workaround "may be used uninitialized in this function" warning - a2 = a3 = a0; - } - -#define KMADD(c, a, b, tmp) {tmp = b; tmp = pmul(a,tmp); c = padd(c,tmp);} -#define WORK(I) \ - c0 = pload(C0+i+(I)*PacketSize); \ - c1 = pload(C1+i+(I)*PacketSize); \ - KMADD(c0, a0, b00, t0) \ - KMADD(c1, a0, b01, t1) \ - a0 = pload(A0+i+(I+1)*PacketSize); \ - KMADD(c0, a1, b10, t0) \ - KMADD(c1, a1, b11, t1) \ - a1 = pload(A1+i+(I+1)*PacketSize); \ - if(RK==4){ KMADD(c0, a2, b20, t0) }\ - if(RK==4){ KMADD(c1, a2, b21, t1) }\ - if(RK==4){ a2 = pload(A2+i+(I+1)*PacketSize); }\ - if(RK==4){ KMADD(c0, a3, b30, t0) }\ - if(RK==4){ KMADD(c1, a3, b31, t1) }\ - if(RK==4){ a3 = pload(A3+i+(I+1)*PacketSize); }\ - pstore(C0+i+(I)*PacketSize, c0); \ - pstore(C1+i+(I)*PacketSize, c1) - - // process rows of A' - C' with aggressive vectorization and peeling - for(Index i=0; i0) - { - const Scalar* Bc0 = B+(n-1)*ldb; - - for(Index k=0; k(Bc0[0]); - b10 = pset1(Bc0[1]); - if(RK==4) b20 = pset1(Bc0[2]); - if(RK==4) b30 = pset1(Bc0[3]); - - Packet a0, a1, a2, a3, c0, t0/*, t1*/; - - const Scalar* A0 = A+ib+(k+0)*lda; - const Scalar* A1 = A+ib+(k+1)*lda; - const Scalar* A2 = A+ib+(k+2)*lda; - const Scalar* A3 = A+ib+(k+3)*lda; - - Scalar* C0 = C+ib+(n_end)*ldc; - - a0 = pload(A0); - a1 = pload(A1); - if(RK==4) - { - a2 = pload(A2); - a3 = pload(A3); - } - else - { - // workaround "may be used uninitialized in this function" warning - a2 = a3 = a0; - } - -#define WORK(I) \ - c0 = pload(C0+i+(I)*PacketSize); \ - KMADD(c0, a0, b00, t0) \ - a0 = pload(A0+i+(I+1)*PacketSize); \ - KMADD(c0, a1, b10, t0) \ - a1 = pload(A1+i+(I+1)*PacketSize); \ - if(RK==4){ KMADD(c0, a2, b20, t0) }\ - if(RK==4){ a2 = pload(A2+i+(I+1)*PacketSize); }\ - if(RK==4){ KMADD(c0, a3, b30, t0) }\ - if(RK==4){ a3 = pload(A3+i+(I+1)*PacketSize); }\ - pstore(C0+i+(I)*PacketSize, c0); - - // aggressive vectorization and peeling - for(Index i=0; i0) - { - for(Index j=0; j1 ? Aligned : 0 - }; - typedef Map, Alignment > MapVector; - typedef Map, Alignment > ConstMapVector; - if(rd==1) MapVector(C+j*ldc+ib,actual_b) += B[0+d_end+j*ldb] * ConstMapVector(A+(d_end+0)*lda+ib, actual_b); - - else if(rd==2) MapVector(C+j*ldc+ib,actual_b) += B[0+d_end+j*ldb] * ConstMapVector(A+(d_end+0)*lda+ib, actual_b) - + B[1+d_end+j*ldb] * ConstMapVector(A+(d_end+1)*lda+ib, actual_b); - - else MapVector(C+j*ldc+ib,actual_b) += B[0+d_end+j*ldb] * ConstMapVector(A+(d_end+0)*lda+ib, actual_b) - + B[1+d_end+j*ldb] * ConstMapVector(A+(d_end+1)*lda+ib, actual_b) - + B[2+d_end+j*ldb] * ConstMapVector(A+(d_end+2)*lda+ib, actual_b); - } - } - - } // blocking on the rows of A and C -} -#undef KMADD - -} // namespace internal - -} // namespace Eigen - -#endif // EIGEN_SPARSELU_GEMM_KERNEL_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_heap_relax_snode.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_heap_relax_snode.h index 6f75d500e5f..4df0dcd2853 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_heap_relax_snode.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_heap_relax_snode.h @@ -28,6 +28,9 @@ #ifndef SPARSELU_HEAP_RELAX_SNODE_H #define SPARSELU_HEAP_RELAX_SNODE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -75,8 +78,6 @@ void SparseLUImpl::heap_relax_snode (const Index n, IndexVe // Identify the relaxed supernodes by postorder traversal of the etree Index snode_start; // beginning of a snode StorageIndex k; - Index nsuper_et_post = 0; // Number of relaxed snodes in postordered etree - Index nsuper_et = 0; // Number of relaxed snodes in the original etree StorageIndex l; for (j = 0; j < n; ) { @@ -88,7 +89,6 @@ void SparseLUImpl::heap_relax_snode (const Index n, IndexVe parent = et(j); } // Found a supernode in postordered etree, j is the last column - ++nsuper_et_post; k = StorageIndex(n); for (Index i = snode_start; i <= j; ++i) k = (std::min)(k, inv_post(i)); @@ -97,7 +97,6 @@ void SparseLUImpl::heap_relax_snode (const Index n, IndexVe { // This is also a supernode in the original etree relax_end(k) = l; // Record last column - ++nsuper_et; } else { @@ -107,7 +106,6 @@ void SparseLUImpl::heap_relax_snode (const Index n, IndexVe if (descendants(i) == 0) { relax_end(l) = l; - ++nsuper_et; } } } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_kernel_bmod.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_kernel_bmod.h index 8c1b3e8bc67..da3204157f2 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_kernel_bmod.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_kernel_bmod.h @@ -11,6 +11,9 @@ #ifndef SPARSELU_KERNEL_BMOD_H #define SPARSELU_KERNEL_BMOD_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -69,8 +72,7 @@ EIGEN_DONT_INLINE void LU_kernel_bmod::run(const Index seg Index aligned_with_B_offset = (PacketSize-internal::first_default_aligned(B.data(), PacketSize))%PacketSize; Map, 0, OuterStride<> > l(tempv.data()+segsize+aligned_offset+aligned_with_B_offset, nrow, OuterStride<>(ldl) ); - l.setZero(); - internal::sparselu_gemm(l.rows(), l.cols(), B.cols(), B.data(), B.outerStride(), u.data(), u.outerStride(), l.data(), l.outerStride()); + l.noalias() = B * u; // Scatter tempv[] into SPA dense[] as a temporary storage isub = lptr + no_zeros; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_panel_bmod.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_panel_bmod.h index f052001c8f8..d12d0e2027a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_panel_bmod.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_panel_bmod.h @@ -31,6 +31,9 @@ #ifndef SPARSELU_PANEL_BMOD_H #define SPARSELU_PANEL_BMOD_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -148,8 +151,7 @@ void SparseLUImpl::panel_bmod(const Index m, const Index w, Index offset = (PacketSize-internal::first_default_aligned(B.data(), PacketSize)) % PacketSize; MappedMatrixBlock L(tempv.data()+w*ldu+offset, nrow, u_cols, OuterStride<>(ldl)); - L.setZero(); - internal::sparselu_gemm(L.rows(), L.cols(), B.cols(), B.data(), B.outerStride(), U.data(), U.outerStride(), L.data(), L.outerStride()); + L.noalias() = B * U; // scatter U and L u_col = 0; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_panel_dfs.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_panel_dfs.h index 155df733687..5a3fd10b83d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_panel_dfs.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_panel_dfs.h @@ -30,6 +30,9 @@ #ifndef SPARSELU_PANEL_DFS_H #define SPARSELU_PANEL_DFS_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_pivotL.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_pivotL.h index a86dac93fa9..79510bc998c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_pivotL.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_pivotL.h @@ -30,6 +30,9 @@ #ifndef SPARSELU_PIVOTL_H #define SPARSELU_PIVOTL_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_pruneL.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_pruneL.h index ad32fed5e6b..c184274224c 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_pruneL.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_pruneL.h @@ -30,6 +30,9 @@ #ifndef SPARSELU_PRUNEL_H #define SPARSELU_PRUNEL_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_relax_snode.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_relax_snode.h index c408d01b406..61531499218 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_relax_snode.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseLU/SparseLU_relax_snode.h @@ -28,6 +28,9 @@ #ifndef SPARSELU_RELAX_SNODE_H #define SPARSELU_RELAX_SNODE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseQR/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseQR/InternalHeaderCheck.h new file mode 100644 index 00000000000..0564e932aeb --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseQR/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_SPARSEQR_MODULE_H +#error "Please include Eigen/SparseQR instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseQR/SparseQR.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseQR/SparseQR.h index d1fb96f5cba..a027a2d7813 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseQR/SparseQR.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SparseQR/SparseQR.h @@ -11,6 +11,9 @@ #ifndef EIGEN_SPARSE_QR_H #define EIGEN_SPARSE_QR_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { template class SparseQR; @@ -59,8 +62,8 @@ namespace internal { * R is the sparse triangular or trapezoidal matrix. The later occurs when A is rank-deficient. * matrixR().topLeftCorner(rank(), rank()) always returns a triangular factor of full rank. * - * \tparam _MatrixType The type of the sparse matrix A, must be a column-major SparseMatrix<> - * \tparam _OrderingType The fill-reducing ordering method. See the \link OrderingMethods_Module + * \tparam MatrixType_ The type of the sparse matrix A, must be a column-major SparseMatrix<> + * \tparam OrderingType_ The fill-reducing ordering method. See the \link OrderingMethods_Module * OrderingMethods \endlink module for the list of built-in and external ordering methods. * * \implsparsesolverconcept @@ -80,16 +83,16 @@ namespace internal { * \warning For complex matrices matrixQ().transpose() will actually return the adjoint matrix. * */ -template -class SparseQR : public SparseSolverBase > +template +class SparseQR : public SparseSolverBase > { protected: - typedef SparseSolverBase > Base; + typedef SparseSolverBase > Base; using Base::m_isInitialized; public: using Base::_solve_impl; - typedef _MatrixType MatrixType; - typedef _OrderingType OrderingType; + typedef MatrixType_ MatrixType; + typedef OrderingType_ OrderingType; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef typename MatrixType::StorageIndex StorageIndex; @@ -321,7 +324,7 @@ void SparseQR::analyzePattern(const MatrixType& mat) { eigen_assert(mat.isCompressed() && "SparseQR requires a sparse matrix in compressed mode. Call .makeCompressed() before passing it to SparseQR"); // Copy to a column major matrix if the input is rowmajor - typename internal::conditional::type matCpy(mat); + std::conditional_t matCpy(mat); // Compute the column fill reducing ordering OrderingType ord; ord(matCpy, m_perm_c); diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdDeque.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdDeque.h index 6d47e757220..1e951821d64 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdDeque.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdDeque.h @@ -11,6 +11,10 @@ #ifndef EIGEN_STDDEQUE_H #define EIGEN_STDDEQUE_H +#ifndef EIGEN_STDDEQUE_MODULE_H +#error "Please include Eigen/StdDeque instead of including this file directly." +#endif + #include "details.h" /** @@ -44,73 +48,4 @@ namespace std \ }; \ } -// check whether we really need the std::deque specialization -#if !EIGEN_HAS_CXX11_CONTAINERS && !(defined(_GLIBCXX_DEQUE) && (!EIGEN_GNUC_AT_LEAST(4,1))) /* Note that before gcc-4.1 we already have: std::deque::resize(size_type,const T&). */ - -namespace std { - -#define EIGEN_STD_DEQUE_SPECIALIZATION_BODY \ - public: \ - typedef T value_type; \ - typedef typename deque_base::allocator_type allocator_type; \ - typedef typename deque_base::size_type size_type; \ - typedef typename deque_base::iterator iterator; \ - typedef typename deque_base::const_iterator const_iterator; \ - explicit deque(const allocator_type& a = allocator_type()) : deque_base(a) {} \ - template \ - deque(InputIterator first, InputIterator last, const allocator_type& a = allocator_type()) \ - : deque_base(first, last, a) {} \ - deque(const deque& c) : deque_base(c) {} \ - explicit deque(size_type num, const value_type& val = value_type()) : deque_base(num, val) {} \ - deque(iterator start_, iterator end_) : deque_base(start_, end_) {} \ - deque& operator=(const deque& x) { \ - deque_base::operator=(x); \ - return *this; \ - } - - template - class deque > - : public deque > -{ - typedef deque > deque_base; - EIGEN_STD_DEQUE_SPECIALIZATION_BODY - - void resize(size_type new_size) - { resize(new_size, T()); } - -#if defined(_DEQUE_) - // workaround MSVC std::deque implementation - void resize(size_type new_size, const value_type& x) - { - if (deque_base::size() < new_size) - deque_base::_Insert_n(deque_base::end(), new_size - deque_base::size(), x); - else if (new_size < deque_base::size()) - deque_base::erase(deque_base::begin() + new_size, deque_base::end()); - } - void push_back(const value_type& x) - { deque_base::push_back(x); } - void push_front(const value_type& x) - { deque_base::push_front(x); } - using deque_base::insert; - iterator insert(const_iterator position, const value_type& x) - { return deque_base::insert(position,x); } - void insert(const_iterator position, size_type new_size, const value_type& x) - { deque_base::insert(position, new_size, x); } -#else - // default implementation which should always work. - void resize(size_type new_size, const value_type& x) - { - if (new_size < deque_base::size()) - deque_base::erase(deque_base::begin() + new_size, deque_base::end()); - else if (new_size > deque_base::size()) - deque_base::insert(deque_base::end(), new_size - deque_base::size(), x); - } -#endif - }; -} - -#endif // check whether specialization is actually required - #endif // EIGEN_STDDEQUE_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdList.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdList.h index 8ba3fada0a5..da36677a225 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdList.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdList.h @@ -10,6 +10,10 @@ #ifndef EIGEN_STDLIST_H #define EIGEN_STDLIST_H +#ifndef EIGEN_STDLIST_MODULE_H +#error "Please include Eigen/StdList instead of including this file directly." +#endif + #include "details.h" /** @@ -43,64 +47,4 @@ namespace std \ }; \ } -// check whether we really need the std::list specialization -#if !EIGEN_HAS_CXX11_CONTAINERS && !(defined(_GLIBCXX_LIST) && (!EIGEN_GNUC_AT_LEAST(4,1))) /* Note that before gcc-4.1 we already have: std::list::resize(size_type,const T&). */ - -namespace std -{ - -#define EIGEN_STD_LIST_SPECIALIZATION_BODY \ - public: \ - typedef T value_type; \ - typedef typename list_base::allocator_type allocator_type; \ - typedef typename list_base::size_type size_type; \ - typedef typename list_base::iterator iterator; \ - typedef typename list_base::const_iterator const_iterator; \ - explicit list(const allocator_type& a = allocator_type()) : list_base(a) {} \ - template \ - list(InputIterator first, InputIterator last, const allocator_type& a = allocator_type()) \ - : list_base(first, last, a) {} \ - list(const list& c) : list_base(c) {} \ - explicit list(size_type num, const value_type& val = value_type()) : list_base(num, val) {} \ - list(iterator start_, iterator end_) : list_base(start_, end_) {} \ - list& operator=(const list& x) { \ - list_base::operator=(x); \ - return *this; \ - } - - template - class list > - : public list > - { - typedef list > list_base; - EIGEN_STD_LIST_SPECIALIZATION_BODY - - void resize(size_type new_size) - { resize(new_size, T()); } - - void resize(size_type new_size, const value_type& x) - { - if (list_base::size() < new_size) - list_base::insert(list_base::end(), new_size - list_base::size(), x); - else - while (new_size < list_base::size()) list_base::pop_back(); - } - -#if defined(_LIST_) - // workaround MSVC std::list implementation - void push_back(const value_type& x) - { list_base::push_back(x); } - using list_base::insert; - iterator insert(const_iterator position, const value_type& x) - { return list_base::insert(position,x); } - void insert(const_iterator position, size_type new_size, const value_type& x) - { list_base::insert(position, new_size, x); } -#endif - }; -} - -#endif // check whether specialization is actually required - #endif // EIGEN_STDLIST_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdVector.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdVector.h index 9fcf19bce85..02dfb391865 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdVector.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/StdVector.h @@ -11,6 +11,10 @@ #ifndef EIGEN_STDVECTOR_H #define EIGEN_STDVECTOR_H +#ifndef EIGEN_STDVECTOR_MODULE_H +#error "Please include Eigen/StdVector instead of including this file directly." +#endif + #include "details.h" /** @@ -44,88 +48,4 @@ namespace std \ }; \ } -// Don't specialize if containers are implemented according to C++11 -#if !EIGEN_HAS_CXX11_CONTAINERS - -namespace std { - -#define EIGEN_STD_VECTOR_SPECIALIZATION_BODY \ - public: \ - typedef T value_type; \ - typedef typename vector_base::allocator_type allocator_type; \ - typedef typename vector_base::size_type size_type; \ - typedef typename vector_base::iterator iterator; \ - typedef typename vector_base::const_iterator const_iterator; \ - explicit vector(const allocator_type& a = allocator_type()) : vector_base(a) {} \ - template \ - vector(InputIterator first, InputIterator last, const allocator_type& a = allocator_type()) \ - : vector_base(first, last, a) {} \ - vector(const vector& c) : vector_base(c) {} \ - explicit vector(size_type num, const value_type& val = value_type()) : vector_base(num, val) {} \ - vector(iterator start_, iterator end_) : vector_base(start_, end_) {} \ - vector& operator=(const vector& x) { \ - vector_base::operator=(x); \ - return *this; \ - } - - template - class vector > - : public vector > -{ - typedef vector > vector_base; - EIGEN_STD_VECTOR_SPECIALIZATION_BODY - - void resize(size_type new_size) - { resize(new_size, T()); } - -#if defined(_VECTOR_) - // workaround MSVC std::vector implementation - void resize(size_type new_size, const value_type& x) - { - if (vector_base::size() < new_size) - vector_base::_Insert_n(vector_base::end(), new_size - vector_base::size(), x); - else if (new_size < vector_base::size()) - vector_base::erase(vector_base::begin() + new_size, vector_base::end()); - } - void push_back(const value_type& x) - { vector_base::push_back(x); } - using vector_base::insert; - iterator insert(const_iterator position, const value_type& x) - { return vector_base::insert(position,x); } - void insert(const_iterator position, size_type new_size, const value_type& x) - { vector_base::insert(position, new_size, x); } -#elif defined(_GLIBCXX_VECTOR) && (!(EIGEN_GNUC_AT_LEAST(4,1))) - /* Note that before gcc-4.1 we already have: std::vector::resize(size_type,const T&). - * However, this specialization is still needed to make the above EIGEN_DEFINE_STL_VECTOR_SPECIALIZATION trick to work. */ - void resize(size_type new_size, const value_type& x) - { - vector_base::resize(new_size,x); - } -#elif defined(_GLIBCXX_VECTOR) && EIGEN_GNUC_AT_LEAST(4,2) - // workaround GCC std::vector implementation - void resize(size_type new_size, const value_type& x) - { - if (new_size < vector_base::size()) - vector_base::_M_erase_at_end(this->_M_impl._M_start + new_size); - else - vector_base::insert(vector_base::end(), new_size - vector_base::size(), x); - } -#else - // either GCC 4.1 or non-GCC - // default implementation which should always work. - void resize(size_type new_size, const value_type& x) - { - if (new_size < vector_base::size()) - vector_base::erase(vector_base::begin() + new_size, vector_base::end()); - else if (new_size > vector_base::size()) - vector_base::insert(vector_base::end(), new_size - vector_base::size(), x); - } -#endif - }; -} -#endif // !EIGEN_HAS_CXX11_CONTAINERS - - #endif // EIGEN_STDVECTOR_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/details.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/details.h index 2cfd13e03aa..29fd8714d89 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/details.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/StlSupport/details.h @@ -52,11 +52,7 @@ namespace Eigen { // in std::vector::resize(size_t s,T x) won't be aligned and generate an error // even if this function is never called. Whence this little wrapper. #define EIGEN_WORKAROUND_MSVC_STL_SUPPORT(T) \ - typename Eigen::internal::conditional< \ - Eigen::internal::is_arithmetic::value, \ - T, \ - Eigen::internal::workaround_msvc_stl_support \ - >::type + std::conditional_t::value, T, Eigen::internal::workaround_msvc_stl_support > namespace internal { template struct workaround_msvc_stl_support : public T diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SuperLUSupport/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SuperLUSupport/InternalHeaderCheck.h new file mode 100644 index 00000000000..94a62b55066 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SuperLUSupport/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_SUPERLUSUPPORT_MODULE_H +#error "Please include Eigen/SuperLUSupport instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SuperLUSupport/SuperLUSupport.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SuperLUSupport/SuperLUSupport.h index d1d3ad7f192..419d7d8e94e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SuperLUSupport/SuperLUSupport.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/SuperLUSupport/SuperLUSupport.h @@ -10,6 +10,9 @@ #ifndef EIGEN_SUPERLUSUPPORT_H #define EIGEN_SUPERLUSUPPORT_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { #if defined(SUPERLU_MAJOR_VERSION) && (SUPERLU_MAJOR_VERSION >= 5) @@ -295,14 +298,14 @@ SluMatrix asSluMatrix(MatrixType& mat) /** View a Super LU matrix as an Eigen expression */ template -MappedSparseMatrix map_superlu(SluMatrix& sluMat) +Map > map_superlu(SluMatrix& sluMat) { eigen_assert(((Flags&RowMajor)==RowMajor && sluMat.Stype == SLU_NR) || ((Flags&ColMajor)==ColMajor && sluMat.Stype == SLU_NC)); Index outerSize = (Flags&RowMajor)==RowMajor ? sluMat.ncol : sluMat.nrow; - return MappedSparseMatrix( + return Map >( sluMat.nrow, sluMat.ncol, sluMat.storage.outerInd[outerSize], sluMat.storage.outerInd, sluMat.storage.innerInd, reinterpret_cast(sluMat.storage.values) ); } @@ -313,7 +316,7 @@ MappedSparseMatrix map_superlu(SluMatrix& sluMat) * \class SuperLUBase * \brief The base class for the direct and incomplete LU factorization of SuperLU */ -template +template class SuperLUBase : public SparseSolverBase { protected: @@ -321,7 +324,7 @@ class SuperLUBase : public SparseSolverBase using Base::derived; using Base::m_isInitialized; public: - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef typename MatrixType::StorageIndex StorageIndex; @@ -476,7 +479,7 @@ class SuperLUBase : public SparseSolverBase * using the SuperLU library. The sparse matrix A must be squared and invertible. The vectors or matrices * X and B can be either dense or sparse. * - * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> * * \warning This class is only for the 4.x versions of SuperLU. The 3.x and 5.x versions are not supported. * @@ -484,12 +487,12 @@ class SuperLUBase : public SparseSolverBase * * \sa \ref TutorialSparseSolverConcept, class SparseLU */ -template -class SuperLU : public SuperLUBase<_MatrixType,SuperLU<_MatrixType> > +template +class SuperLU : public SuperLUBase > { public: - typedef SuperLUBase<_MatrixType,SuperLU> Base; - typedef _MatrixType MatrixType; + typedef SuperLUBase Base; + typedef MatrixType_ MatrixType; typedef typename Base::Scalar Scalar; typedef typename Base::RealScalar RealScalar; typedef typename Base::StorageIndex StorageIndex; @@ -830,19 +833,19 @@ typename SuperLU::Scalar SuperLU::determinant() const * * \warning This class is only for the 4.x versions of SuperLU. The 3.x and 5.x versions are not supported. * - * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> * * \implsparsesolverconcept * * \sa \ref TutorialSparseSolverConcept, class IncompleteLUT, class ConjugateGradient, class BiCGSTAB */ -template -class SuperILU : public SuperLUBase<_MatrixType,SuperILU<_MatrixType> > +template +class SuperILU : public SuperLUBase > { public: - typedef SuperLUBase<_MatrixType,SuperILU> Base; - typedef _MatrixType MatrixType; + typedef SuperLUBase Base; + typedef MatrixType_ MatrixType; typedef typename Base::Scalar Scalar; typedef typename Base::RealScalar RealScalar; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/Barrier.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/Barrier.h new file mode 100644 index 00000000000..8b2f8da7167 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/Barrier.h @@ -0,0 +1,70 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Rasmus Munk Larsen +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Barrier is an object that allows one or more threads to wait until +// Notify has been called a specified number of times. + +#ifndef EIGEN_CXX11_THREADPOOL_BARRIER_H +#define EIGEN_CXX11_THREADPOOL_BARRIER_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +class Barrier { + public: + Barrier(unsigned int count) : state_(count << 1), notified_(false) { + eigen_plain_assert(((count << 1) >> 1) == count); + } + ~Barrier() { eigen_plain_assert((state_ >> 1) == 0); } + + void Notify() { + unsigned int v = state_.fetch_sub(2, std::memory_order_acq_rel) - 2; + if (v != 1) { + // Clear the lowest bit (waiter flag) and check that the original state + // value was not zero. If it was zero, it means that notify was called + // more times than the original count. + eigen_plain_assert(((v + 2) & ~1) != 0); + return; // either count has not dropped to 0, or waiter is not waiting + } + EIGEN_MUTEX_LOCK l(mu_); + eigen_plain_assert(!notified_); + notified_ = true; + cv_.notify_all(); + } + + void Wait() { + unsigned int v = state_.fetch_or(1, std::memory_order_acq_rel); + if ((v >> 1) == 0) return; + EIGEN_MUTEX_LOCK l(mu_); + while (!notified_) { + cv_.wait(l); + } + } + + private: + EIGEN_MUTEX mu_; + EIGEN_CONDVAR cv_; + std::atomic state_; // low bit is waiter flag + bool notified_; +}; + +// Notification is an object that allows a user to to wait for another +// thread to signal a notification that an event has occurred. +// +// Multiple threads can wait on the same Notification object, +// but only one caller must call Notify() on the object. +struct Notification : Barrier { + Notification() : Barrier(1){}; +}; + +} // namespace Eigen + +#endif // EIGEN_CXX11_THREADPOOL_BARRIER_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/EventCount.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/EventCount.h new file mode 100644 index 00000000000..1e8aa8a81a7 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/EventCount.h @@ -0,0 +1,252 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Dmitry Vyukov +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_THREADPOOL_EVENTCOUNT_H +#define EIGEN_CXX11_THREADPOOL_EVENTCOUNT_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +// EventCount allows to wait for arbitrary predicates in non-blocking +// algorithms. Think of condition variable, but wait predicate does not need to +// be protected by a mutex. Usage: +// Waiting thread does: +// +// if (predicate) +// return act(); +// EventCount::Waiter& w = waiters[my_index]; +// ec.Prewait(&w); +// if (predicate) { +// ec.CancelWait(&w); +// return act(); +// } +// ec.CommitWait(&w); +// +// Notifying thread does: +// +// predicate = true; +// ec.Notify(true); +// +// Notify is cheap if there are no waiting threads. Prewait/CommitWait are not +// cheap, but they are executed only if the preceding predicate check has +// failed. +// +// Algorithm outline: +// There are two main variables: predicate (managed by user) and state_. +// Operation closely resembles Dekker mutual algorithm: +// https://en.wikipedia.org/wiki/Dekker%27s_algorithm +// Waiting thread sets state_ then checks predicate, Notifying thread sets +// predicate then checks state_. Due to seq_cst fences in between these +// operations it is guaranteed than either waiter will see predicate change +// and won't block, or notifying thread will see state_ change and will unblock +// the waiter, or both. But it can't happen that both threads don't see each +// other changes, which would lead to deadlock. +class EventCount { + public: + class Waiter; + + EventCount(MaxSizeVector& waiters) + : state_(kStackMask), waiters_(waiters) { + eigen_plain_assert(waiters.size() < (1 << kWaiterBits) - 1); + } + + ~EventCount() { + // Ensure there are no waiters. + eigen_plain_assert(state_.load() == kStackMask); + } + + // Prewait prepares for waiting. + // After calling Prewait, the thread must re-check the wait predicate + // and then call either CancelWait or CommitWait. + void Prewait() { + uint64_t state = state_.load(std::memory_order_relaxed); + for (;;) { + CheckState(state); + uint64_t newstate = state + kWaiterInc; + CheckState(newstate); + if (state_.compare_exchange_weak(state, newstate, + std::memory_order_seq_cst)) + return; + } + } + + // CommitWait commits waiting after Prewait. + void CommitWait(Waiter* w) { + eigen_plain_assert((w->epoch & ~kEpochMask) == 0); + w->state = Waiter::kNotSignaled; + const uint64_t me = (w - &waiters_[0]) | w->epoch; + uint64_t state = state_.load(std::memory_order_seq_cst); + for (;;) { + CheckState(state, true); + uint64_t newstate; + if ((state & kSignalMask) != 0) { + // Consume the signal and return immediately. + newstate = state - kWaiterInc - kSignalInc; + } else { + // Remove this thread from pre-wait counter and add to the waiter stack. + newstate = ((state & kWaiterMask) - kWaiterInc) | me; + w->next.store(state & (kStackMask | kEpochMask), + std::memory_order_relaxed); + } + CheckState(newstate); + if (state_.compare_exchange_weak(state, newstate, + std::memory_order_acq_rel)) { + if ((state & kSignalMask) == 0) { + w->epoch += kEpochInc; + Park(w); + } + return; + } + } + } + + // CancelWait cancels effects of the previous Prewait call. + void CancelWait() { + uint64_t state = state_.load(std::memory_order_relaxed); + for (;;) { + CheckState(state, true); + uint64_t newstate = state - kWaiterInc; + // We don't know if the thread was also notified or not, + // so we should not consume a signal unconditionally. + // Only if number of waiters is equal to number of signals, + // we know that the thread was notified and we must take away the signal. + if (((state & kWaiterMask) >> kWaiterShift) == + ((state & kSignalMask) >> kSignalShift)) + newstate -= kSignalInc; + CheckState(newstate); + if (state_.compare_exchange_weak(state, newstate, + std::memory_order_acq_rel)) + return; + } + } + + // Notify wakes one or all waiting threads. + // Must be called after changing the associated wait predicate. + void Notify(bool notifyAll) { + std::atomic_thread_fence(std::memory_order_seq_cst); + uint64_t state = state_.load(std::memory_order_acquire); + for (;;) { + CheckState(state); + const uint64_t waiters = (state & kWaiterMask) >> kWaiterShift; + const uint64_t signals = (state & kSignalMask) >> kSignalShift; + // Easy case: no waiters. + if ((state & kStackMask) == kStackMask && waiters == signals) return; + uint64_t newstate; + if (notifyAll) { + // Empty wait stack and set signal to number of pre-wait threads. + newstate = + (state & kWaiterMask) | (waiters << kSignalShift) | kStackMask; + } else if (signals < waiters) { + // There is a thread in pre-wait state, unblock it. + newstate = state + kSignalInc; + } else { + // Pop a waiter from list and unpark it. + Waiter* w = &waiters_[state & kStackMask]; + uint64_t next = w->next.load(std::memory_order_relaxed); + newstate = (state & (kWaiterMask | kSignalMask)) | next; + } + CheckState(newstate); + if (state_.compare_exchange_weak(state, newstate, + std::memory_order_acq_rel)) { + if (!notifyAll && (signals < waiters)) + return; // unblocked pre-wait thread + if ((state & kStackMask) == kStackMask) return; + Waiter* w = &waiters_[state & kStackMask]; + if (!notifyAll) w->next.store(kStackMask, std::memory_order_relaxed); + Unpark(w); + return; + } + } + } + + class Waiter { + friend class EventCount; + // Align to 128 byte boundary to prevent false sharing with other Waiter + // objects in the same vector. + EIGEN_ALIGN_TO_BOUNDARY(128) std::atomic next; + EIGEN_MUTEX mu; + EIGEN_CONDVAR cv; + uint64_t epoch = 0; + unsigned state = kNotSignaled; + enum { + kNotSignaled, + kWaiting, + kSignaled, + }; + }; + + private: + // State_ layout: + // - low kWaiterBits is a stack of waiters committed wait + // (indexes in waiters_ array are used as stack elements, + // kStackMask means empty stack). + // - next kWaiterBits is count of waiters in prewait state. + // - next kWaiterBits is count of pending signals. + // - remaining bits are ABA counter for the stack. + // (stored in Waiter node and incremented on push). + static const uint64_t kWaiterBits = 14; + static const uint64_t kStackMask = (1ull << kWaiterBits) - 1; + static const uint64_t kWaiterShift = kWaiterBits; + static const uint64_t kWaiterMask = ((1ull << kWaiterBits) - 1) + << kWaiterShift; + static const uint64_t kWaiterInc = 1ull << kWaiterShift; + static const uint64_t kSignalShift = 2 * kWaiterBits; + static const uint64_t kSignalMask = ((1ull << kWaiterBits) - 1) + << kSignalShift; + static const uint64_t kSignalInc = 1ull << kSignalShift; + static const uint64_t kEpochShift = 3 * kWaiterBits; + static const uint64_t kEpochBits = 64 - kEpochShift; + static const uint64_t kEpochMask = ((1ull << kEpochBits) - 1) << kEpochShift; + static const uint64_t kEpochInc = 1ull << kEpochShift; + std::atomic state_; + MaxSizeVector& waiters_; + + static void CheckState(uint64_t state, bool waiter = false) { + static_assert(kEpochBits >= 20, "not enough bits to prevent ABA problem"); + const uint64_t waiters = (state & kWaiterMask) >> kWaiterShift; + const uint64_t signals = (state & kSignalMask) >> kSignalShift; + eigen_plain_assert(waiters >= signals); + eigen_plain_assert(waiters < (1 << kWaiterBits) - 1); + eigen_plain_assert(!waiter || waiters > 0); + (void)waiters; + (void)signals; + } + + void Park(Waiter* w) { + EIGEN_MUTEX_LOCK lock(w->mu); + while (w->state != Waiter::kSignaled) { + w->state = Waiter::kWaiting; + w->cv.wait(lock); + } + } + + void Unpark(Waiter* w) { + for (Waiter* next; w; w = next) { + uint64_t wnext = w->next.load(std::memory_order_relaxed) & kStackMask; + next = wnext == kStackMask ? nullptr : &waiters_[internal::convert_index(wnext)]; + unsigned state; + { + EIGEN_MUTEX_LOCK lock(w->mu); + state = w->state; + w->state = Waiter::kSignaled; + } + // Avoid notifying if it wasn't waiting. + if (state == Waiter::kWaiting) w->cv.notify_one(); + } + } + + EventCount(const EventCount&) = delete; + void operator=(const EventCount&) = delete; +}; + +} // namespace Eigen + +#endif // EIGEN_CXX11_THREADPOOL_EVENTCOUNT_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/InternalHeaderCheck.h new file mode 100644 index 00000000000..44c0fca2051 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_THREADPOOL_MODULE_H +#error "Please include unsupported/Eigen/CXX11/ThreadPool instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/NonBlockingThreadPool.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/NonBlockingThreadPool.h new file mode 100644 index 00000000000..6d0c3ada180 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/NonBlockingThreadPool.h @@ -0,0 +1,489 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Dmitry Vyukov +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_THREADPOOL_NONBLOCKING_THREAD_POOL_H +#define EIGEN_CXX11_THREADPOOL_NONBLOCKING_THREAD_POOL_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +template +class ThreadPoolTempl : public Eigen::ThreadPoolInterface { + public: + typedef typename Environment::Task Task; + typedef RunQueue Queue; + + ThreadPoolTempl(int num_threads, Environment env = Environment()) + : ThreadPoolTempl(num_threads, true, env) {} + + ThreadPoolTempl(int num_threads, bool allow_spinning, + Environment env = Environment()) + : env_(env), + num_threads_(num_threads), + allow_spinning_(allow_spinning), + thread_data_(num_threads), + all_coprimes_(num_threads), + waiters_(num_threads), + global_steal_partition_(EncodePartition(0, num_threads_)), + blocked_(0), + spinning_(0), + done_(false), + cancelled_(false), + ec_(waiters_) { + waiters_.resize(num_threads_); + // Calculate coprimes of all numbers [1, num_threads]. + // Coprimes are used for random walks over all threads in Steal + // and NonEmptyQueueIndex. Iteration is based on the fact that if we take + // a random starting thread index t and calculate num_threads - 1 subsequent + // indices as (t + coprime) % num_threads, we will cover all threads without + // repetitions (effectively getting a presudo-random permutation of thread + // indices). + eigen_plain_assert(num_threads_ < kMaxThreads); + for (int i = 1; i <= num_threads_; ++i) { + all_coprimes_.emplace_back(i); + ComputeCoprimes(i, &all_coprimes_.back()); + } +#ifndef EIGEN_THREAD_LOCAL + init_barrier_.reset(new Barrier(num_threads_)); +#endif + thread_data_.resize(num_threads_); + for (int i = 0; i < num_threads_; i++) { + SetStealPartition(i, EncodePartition(0, num_threads_)); + thread_data_[i].thread.reset( + env_.CreateThread([this, i]() { WorkerLoop(i); })); + } +#ifndef EIGEN_THREAD_LOCAL + // Wait for workers to initialize per_thread_map_. Otherwise we might race + // with them in Schedule or CurrentThreadId. + init_barrier_->Wait(); +#endif + } + + ~ThreadPoolTempl() { + done_ = true; + + // Now if all threads block without work, they will start exiting. + // But note that threads can continue to work arbitrary long, + // block, submit new work, unblock and otherwise live full life. + if (!cancelled_) { + ec_.Notify(true); + } else { + // Since we were cancelled, there might be entries in the queues. + // Empty them to prevent their destructor from asserting. + for (size_t i = 0; i < thread_data_.size(); i++) { + thread_data_[i].queue.Flush(); + } + } + // Join threads explicitly (by destroying) to avoid destruction order within + // this class. + for (size_t i = 0; i < thread_data_.size(); ++i) + thread_data_[i].thread.reset(); + } + + void SetStealPartitions(const std::vector>& partitions) { + eigen_plain_assert(partitions.size() == static_cast(num_threads_)); + + // Pass this information to each thread queue. + for (int i = 0; i < num_threads_; i++) { + const auto& pair = partitions[i]; + unsigned start = pair.first, end = pair.second; + AssertBounds(start, end); + unsigned val = EncodePartition(start, end); + SetStealPartition(i, val); + } + } + + void Schedule(std::function fn) EIGEN_OVERRIDE { + ScheduleWithHint(std::move(fn), 0, num_threads_); + } + + void ScheduleWithHint(std::function fn, int start, + int limit) override { + Task t = env_.CreateTask(std::move(fn)); + PerThread* pt = GetPerThread(); + if (pt->pool == this) { + // Worker thread of this pool, push onto the thread's queue. + Queue& q = thread_data_[pt->thread_id].queue; + t = q.PushFront(std::move(t)); + } else { + // A free-standing thread (or worker of another pool), push onto a random + // queue. + eigen_plain_assert(start < limit); + eigen_plain_assert(limit <= num_threads_); + int num_queues = limit - start; + int rnd = Rand(&pt->rand) % num_queues; + eigen_plain_assert(start + rnd < limit); + Queue& q = thread_data_[start + rnd].queue; + t = q.PushBack(std::move(t)); + } + // Note: below we touch this after making w available to worker threads. + // Strictly speaking, this can lead to a racy-use-after-free. Consider that + // Schedule is called from a thread that is neither main thread nor a worker + // thread of this pool. Then, execution of w directly or indirectly + // completes overall computations, which in turn leads to destruction of + // this. We expect that such scenario is prevented by program, that is, + // this is kept alive while any threads can potentially be in Schedule. + if (!t.f) { + ec_.Notify(false); + } else { + env_.ExecuteTask(t); // Push failed, execute directly. + } + } + + void Cancel() EIGEN_OVERRIDE { + cancelled_ = true; + done_ = true; + + // Let each thread know it's been cancelled. +#ifdef EIGEN_THREAD_ENV_SUPPORTS_CANCELLATION + for (size_t i = 0; i < thread_data_.size(); i++) { + thread_data_[i].thread->OnCancel(); + } +#endif + + // Wake up the threads without work to let them exit on their own. + ec_.Notify(true); + } + + int NumThreads() const EIGEN_FINAL { return num_threads_; } + + int CurrentThreadId() const EIGEN_FINAL { + const PerThread* pt = const_cast(this)->GetPerThread(); + if (pt->pool == this) { + return pt->thread_id; + } else { + return -1; + } + } + + private: + // Create a single atomic that encodes start and limit information for + // each thread. + // We expect num_threads_ < 65536, so we can store them in a single + // std::atomic. + // Exposed publicly as static functions so that external callers can reuse + // this encode/decode logic for maintaining their own thread-safe copies of + // scheduling and steal domain(s). + static const int kMaxPartitionBits = 16; + static const int kMaxThreads = 1 << kMaxPartitionBits; + + inline unsigned EncodePartition(unsigned start, unsigned limit) { + return (start << kMaxPartitionBits) | limit; + } + + inline void DecodePartition(unsigned val, unsigned* start, unsigned* limit) { + *limit = val & (kMaxThreads - 1); + val >>= kMaxPartitionBits; + *start = val; + } + + void AssertBounds(int start, int end) { + eigen_plain_assert(start >= 0); + eigen_plain_assert(start < end); // non-zero sized partition + eigen_plain_assert(end <= num_threads_); + } + + inline void SetStealPartition(size_t i, unsigned val) { + thread_data_[i].steal_partition.store(val, std::memory_order_relaxed); + } + + inline unsigned GetStealPartition(int i) { + return thread_data_[i].steal_partition.load(std::memory_order_relaxed); + } + + void ComputeCoprimes(int N, MaxSizeVector* coprimes) { + for (int i = 1; i <= N; i++) { + unsigned a = i; + unsigned b = N; + // If GCD(a, b) == 1, then a and b are coprimes. + while (b != 0) { + unsigned tmp = a; + a = b; + b = tmp % b; + } + if (a == 1) { + coprimes->push_back(i); + } + } + } + + typedef typename Environment::EnvThread Thread; + + struct PerThread { + constexpr PerThread() : pool(NULL), rand(0), thread_id(-1) {} + ThreadPoolTempl* pool; // Parent pool, or null for normal threads. + uint64_t rand; // Random generator state. + int thread_id; // Worker thread index in pool. +#ifndef EIGEN_THREAD_LOCAL + // Prevent false sharing. + char pad_[128]; +#endif + }; + + struct ThreadData { + constexpr ThreadData() : thread(), steal_partition(0), queue() {} + std::unique_ptr thread; + std::atomic steal_partition; + Queue queue; + }; + + Environment env_; + const int num_threads_; + const bool allow_spinning_; + MaxSizeVector thread_data_; + MaxSizeVector> all_coprimes_; + MaxSizeVector waiters_; + unsigned global_steal_partition_; + std::atomic blocked_; + std::atomic spinning_; + std::atomic done_; + std::atomic cancelled_; + EventCount ec_; +#ifndef EIGEN_THREAD_LOCAL + std::unique_ptr init_barrier_; + EIGEN_MUTEX per_thread_map_mutex_; // Protects per_thread_map_. + std::unordered_map> per_thread_map_; +#endif + + // Main worker thread loop. + void WorkerLoop(int thread_id) { +#ifndef EIGEN_THREAD_LOCAL + std::unique_ptr new_pt(new PerThread()); + per_thread_map_mutex_.lock(); + bool insertOK = per_thread_map_.emplace(GlobalThreadIdHash(), std::move(new_pt)).second; + eigen_plain_assert(insertOK); + EIGEN_UNUSED_VARIABLE(insertOK); + per_thread_map_mutex_.unlock(); + init_barrier_->Notify(); + init_barrier_->Wait(); +#endif + PerThread* pt = GetPerThread(); + pt->pool = this; + pt->rand = GlobalThreadIdHash(); + pt->thread_id = thread_id; + Queue& q = thread_data_[thread_id].queue; + EventCount::Waiter* waiter = &waiters_[thread_id]; + // TODO(dvyukov,rmlarsen): The time spent in NonEmptyQueueIndex() is + // proportional to num_threads_ and we assume that new work is scheduled at + // a constant rate, so we set spin_count to 5000 / num_threads_. The + // constant was picked based on a fair dice roll, tune it. + const int spin_count = + allow_spinning_ && num_threads_ > 0 ? 5000 / num_threads_ : 0; + if (num_threads_ == 1) { + // For num_threads_ == 1 there is no point in going through the expensive + // steal loop. Moreover, since NonEmptyQueueIndex() calls PopBack() on the + // victim queues it might reverse the order in which ops are executed + // compared to the order in which they are scheduled, which tends to be + // counter-productive for the types of I/O workloads the single thread + // pools tend to be used for. + while (!cancelled_) { + Task t = q.PopFront(); + for (int i = 0; i < spin_count && !t.f; i++) { + if (!cancelled_.load(std::memory_order_relaxed)) { + t = q.PopFront(); + } + } + if (!t.f) { + if (!WaitForWork(waiter, &t)) { + return; + } + } + if (t.f) { + env_.ExecuteTask(t); + } + } + } else { + while (!cancelled_) { + Task t = q.PopFront(); + if (!t.f) { + t = LocalSteal(); + if (!t.f) { + t = GlobalSteal(); + if (!t.f) { + // Leave one thread spinning. This reduces latency. + if (allow_spinning_ && !spinning_ && !spinning_.exchange(true)) { + for (int i = 0; i < spin_count && !t.f; i++) { + if (!cancelled_.load(std::memory_order_relaxed)) { + t = GlobalSteal(); + } else { + return; + } + } + spinning_ = false; + } + if (!t.f) { + if (!WaitForWork(waiter, &t)) { + return; + } + } + } + } + } + if (t.f) { + env_.ExecuteTask(t); + } + } + } + } + + // Steal tries to steal work from other worker threads in the range [start, + // limit) in best-effort manner. + Task Steal(unsigned start, unsigned limit) { + PerThread* pt = GetPerThread(); + const size_t size = limit - start; + unsigned r = Rand(&pt->rand); + // Reduce r into [0, size) range, this utilizes trick from + // https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + eigen_plain_assert(all_coprimes_[size - 1].size() < (1<<30)); + unsigned victim = ((uint64_t)r * (uint64_t)size) >> 32; + unsigned index = ((uint64_t) all_coprimes_[size - 1].size() * (uint64_t)r) >> 32; + unsigned inc = all_coprimes_[size - 1][index]; + + for (unsigned i = 0; i < size; i++) { + eigen_plain_assert(start + victim < limit); + Task t = thread_data_[start + victim].queue.PopBack(); + if (t.f) { + return t; + } + victim += inc; + if (victim >= size) { + victim -= size; + } + } + return Task(); + } + + // Steals work within threads belonging to the partition. + Task LocalSteal() { + PerThread* pt = GetPerThread(); + unsigned partition = GetStealPartition(pt->thread_id); + // If thread steal partition is the same as global partition, there is no + // need to go through the steal loop twice. + if (global_steal_partition_ == partition) return Task(); + unsigned start, limit; + DecodePartition(partition, &start, &limit); + AssertBounds(start, limit); + + return Steal(start, limit); + } + + // Steals work from any other thread in the pool. + Task GlobalSteal() { + return Steal(0, num_threads_); + } + + + // WaitForWork blocks until new work is available (returns true), or if it is + // time to exit (returns false). Can optionally return a task to execute in t + // (in such case t.f != nullptr on return). + bool WaitForWork(EventCount::Waiter* waiter, Task* t) { + eigen_plain_assert(!t->f); + // We already did best-effort emptiness check in Steal, so prepare for + // blocking. + ec_.Prewait(); + // Now do a reliable emptiness check. + int victim = NonEmptyQueueIndex(); + if (victim != -1) { + ec_.CancelWait(); + if (cancelled_) { + return false; + } else { + *t = thread_data_[victim].queue.PopBack(); + return true; + } + } + // Number of blocked threads is used as termination condition. + // If we are shutting down and all worker threads blocked without work, + // that's we are done. + blocked_++; + // TODO is blocked_ required to be unsigned? + if (done_ && blocked_ == static_cast(num_threads_)) { + ec_.CancelWait(); + // Almost done, but need to re-check queues. + // Consider that all queues are empty and all worker threads are preempted + // right after incrementing blocked_ above. Now a free-standing thread + // submits work and calls destructor (which sets done_). If we don't + // re-check queues, we will exit leaving the work unexecuted. + if (NonEmptyQueueIndex() != -1) { + // Note: we must not pop from queues before we decrement blocked_, + // otherwise the following scenario is possible. Consider that instead + // of checking for emptiness we popped the only element from queues. + // Now other worker threads can start exiting, which is bad if the + // work item submits other work. So we just check emptiness here, + // which ensures that all worker threads exit at the same time. + blocked_--; + return true; + } + // Reached stable termination state. + ec_.Notify(true); + return false; + } + ec_.CommitWait(waiter); + blocked_--; + return true; + } + + int NonEmptyQueueIndex() { + PerThread* pt = GetPerThread(); + // We intentionally design NonEmptyQueueIndex to steal work from + // anywhere in the queue so threads don't block in WaitForWork() forever + // when all threads in their partition go to sleep. Steal is still local. + const size_t size = thread_data_.size(); + unsigned r = Rand(&pt->rand); + unsigned inc = all_coprimes_[size - 1][r % all_coprimes_[size - 1].size()]; + unsigned victim = r % size; + for (unsigned i = 0; i < size; i++) { + if (!thread_data_[victim].queue.Empty()) { + return victim; + } + victim += inc; + if (victim >= size) { + victim -= size; + } + } + return -1; + } + + static EIGEN_STRONG_INLINE uint64_t GlobalThreadIdHash() { + return std::hash()(std::this_thread::get_id()); + } + + EIGEN_STRONG_INLINE PerThread* GetPerThread() { +#ifndef EIGEN_THREAD_LOCAL + static PerThread dummy; + auto it = per_thread_map_.find(GlobalThreadIdHash()); + if (it == per_thread_map_.end()) { + return &dummy; + } else { + return it->second.get(); + } +#else + EIGEN_THREAD_LOCAL PerThread per_thread_; + PerThread* pt = &per_thread_; + return pt; +#endif + } + + static EIGEN_STRONG_INLINE unsigned Rand(uint64_t* state) { + uint64_t current = *state; + // Update the internal state + *state = current * 6364136223846793005ULL + 0xda3e39cb94b95bdbULL; + // Generate the random output (using the PCG-XSH-RS scheme) + return static_cast((current ^ (current >> 22)) >> + (22 + (current >> 61))); + } +}; + +typedef ThreadPoolTempl ThreadPool; + +} // namespace Eigen + +#endif // EIGEN_CXX11_THREADPOOL_NONBLOCKING_THREAD_POOL_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/RunQueue.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/RunQueue.h new file mode 100644 index 00000000000..44b668d96f3 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/RunQueue.h @@ -0,0 +1,239 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Dmitry Vyukov +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_THREADPOOL_RUNQUEUE_H +#define EIGEN_CXX11_THREADPOOL_RUNQUEUE_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +// RunQueue is a fixed-size, partially non-blocking deque or Work items. +// Operations on front of the queue must be done by a single thread (owner), +// operations on back of the queue can be done by multiple threads concurrently. +// +// Algorithm outline: +// All remote threads operating on the queue back are serialized by a mutex. +// This ensures that at most two threads access state: owner and one remote +// thread (Size aside). The algorithm ensures that the occupied region of the +// underlying array is logically continuous (can wraparound, but no stray +// occupied elements). Owner operates on one end of this region, remote thread +// operates on the other end. Synchronization between these threads +// (potential consumption of the last element and take up of the last empty +// element) happens by means of state variable in each element. States are: +// empty, busy (in process of insertion of removal) and ready. Threads claim +// elements (empty->busy and ready->busy transitions) by means of a CAS +// operation. The finishing transition (busy->empty and busy->ready) are done +// with plain store as the element is exclusively owned by the current thread. +// +// Note: we could permit only pointers as elements, then we would not need +// separate state variable as null/non-null pointer value would serve as state, +// but that would require malloc/free per operation for large, complex values +// (and this is designed to store std::function<()>). +template +class RunQueue { + public: + RunQueue() : front_(0), back_(0) { + // require power-of-two for fast masking + eigen_plain_assert((kSize & (kSize - 1)) == 0); + eigen_plain_assert(kSize > 2); // why would you do this? + eigen_plain_assert(kSize <= (64 << 10)); // leave enough space for counter + for (unsigned i = 0; i < kSize; i++) + array_[i].state.store(kEmpty, std::memory_order_relaxed); + } + + ~RunQueue() { eigen_plain_assert(Size() == 0); } + + // PushFront inserts w at the beginning of the queue. + // If queue is full returns w, otherwise returns default-constructed Work. + Work PushFront(Work w) { + unsigned front = front_.load(std::memory_order_relaxed); + Elem* e = &array_[front & kMask]; + uint8_t s = e->state.load(std::memory_order_relaxed); + if (s != kEmpty || + !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) + return w; + front_.store(front + 1 + (kSize << 1), std::memory_order_relaxed); + e->w = std::move(w); + e->state.store(kReady, std::memory_order_release); + return Work(); + } + + // PopFront removes and returns the first element in the queue. + // If the queue was empty returns default-constructed Work. + Work PopFront() { + unsigned front = front_.load(std::memory_order_relaxed); + Elem* e = &array_[(front - 1) & kMask]; + uint8_t s = e->state.load(std::memory_order_relaxed); + if (s != kReady || + !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) + return Work(); + Work w = std::move(e->w); + e->state.store(kEmpty, std::memory_order_release); + front = ((front - 1) & kMask2) | (front & ~kMask2); + front_.store(front, std::memory_order_relaxed); + return w; + } + + // PushBack adds w at the end of the queue. + // If queue is full returns w, otherwise returns default-constructed Work. + Work PushBack(Work w) { + EIGEN_MUTEX_LOCK lock(mutex_); + unsigned back = back_.load(std::memory_order_relaxed); + Elem* e = &array_[(back - 1) & kMask]; + uint8_t s = e->state.load(std::memory_order_relaxed); + if (s != kEmpty || + !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) + return w; + back = ((back - 1) & kMask2) | (back & ~kMask2); + back_.store(back, std::memory_order_relaxed); + e->w = std::move(w); + e->state.store(kReady, std::memory_order_release); + return Work(); + } + + // PopBack removes and returns the last elements in the queue. + Work PopBack() { + if (Empty()) return Work(); + EIGEN_MUTEX_LOCK lock(mutex_); + unsigned back = back_.load(std::memory_order_relaxed); + Elem* e = &array_[back & kMask]; + uint8_t s = e->state.load(std::memory_order_relaxed); + if (s != kReady || + !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) + return Work(); + Work w = std::move(e->w); + e->state.store(kEmpty, std::memory_order_release); + back_.store(back + 1 + (kSize << 1), std::memory_order_relaxed); + return w; + } + + // PopBackHalf removes and returns half last elements in the queue. + // Returns number of elements removed. + unsigned PopBackHalf(std::vector* result) { + if (Empty()) return 0; + EIGEN_MUTEX_LOCK lock(mutex_); + unsigned back = back_.load(std::memory_order_relaxed); + unsigned size = Size(); + unsigned mid = back; + if (size > 1) mid = back + (size - 1) / 2; + unsigned n = 0; + unsigned start = 0; + for (; static_cast(mid - back) >= 0; mid--) { + Elem* e = &array_[mid & kMask]; + uint8_t s = e->state.load(std::memory_order_relaxed); + if (n == 0) { + if (s != kReady || !e->state.compare_exchange_strong( + s, kBusy, std::memory_order_acquire)) + continue; + start = mid; + } else { + // Note: no need to store temporal kBusy, we exclusively own these + // elements. + eigen_plain_assert(s == kReady); + } + result->push_back(std::move(e->w)); + e->state.store(kEmpty, std::memory_order_release); + n++; + } + if (n != 0) + back_.store(start + 1 + (kSize << 1), std::memory_order_relaxed); + return n; + } + + // Size returns current queue size. + // Can be called by any thread at any time. + unsigned Size() const { return SizeOrNotEmpty(); } + + // Empty tests whether container is empty. + // Can be called by any thread at any time. + bool Empty() const { return SizeOrNotEmpty() == 0; } + + // Delete all the elements from the queue. + void Flush() { + while (!Empty()) { + PopFront(); + } + } + + private: + static const unsigned kMask = kSize - 1; + static const unsigned kMask2 = (kSize << 1) - 1; + struct Elem { + std::atomic state; + Work w; + }; + enum { + kEmpty, + kBusy, + kReady, + }; + EIGEN_MUTEX mutex_; + // Low log(kSize) + 1 bits in front_ and back_ contain rolling index of + // front/back, respectively. The remaining bits contain modification counters + // that are incremented on Push operations. This allows us to (1) distinguish + // between empty and full conditions (if we would use log(kSize) bits for + // position, these conditions would be indistinguishable); (2) obtain + // consistent snapshot of front_/back_ for Size operation using the + // modification counters. + std::atomic front_; + std::atomic back_; + Elem array_[kSize]; + + // SizeOrNotEmpty returns current queue size; if NeedSizeEstimate is false, + // only whether the size is 0 is guaranteed to be correct. + // Can be called by any thread at any time. + template + unsigned SizeOrNotEmpty() const { + // Emptiness plays critical role in thread pool blocking. So we go to great + // effort to not produce false positives (claim non-empty queue as empty). + unsigned front = front_.load(std::memory_order_acquire); + for (;;) { + // Capture a consistent snapshot of front/tail. + unsigned back = back_.load(std::memory_order_acquire); + unsigned front1 = front_.load(std::memory_order_relaxed); + if (front != front1) { + front = front1; + std::atomic_thread_fence(std::memory_order_acquire); + continue; + } + if (NeedSizeEstimate) { + return CalculateSize(front, back); + } else { + // This value will be 0 if the queue is empty, and undefined otherwise. + unsigned maybe_zero = ((front ^ back) & kMask2); + // Queue size estimate must agree with maybe zero check on the queue + // empty/non-empty state. + eigen_assert((CalculateSize(front, back) == 0) == (maybe_zero == 0)); + return maybe_zero; + } + } + } + + EIGEN_ALWAYS_INLINE + unsigned CalculateSize(unsigned front, unsigned back) const { + int size = (front & kMask2) - (back & kMask2); + // Fix overflow. + if (size < 0) size += 2 * kSize; + // Order of modification in push/pop is crafted to make the queue look + // larger than it is during concurrent modifications. E.g. push can + // increment size before the corresponding pop has decremented it. + // So the computed size can be up to kSize + 1, fix it. + if (size > static_cast(kSize)) size = kSize; + return static_cast(size); + } + + RunQueue(const RunQueue&) = delete; + void operator=(const RunQueue&) = delete; +}; + +} // namespace Eigen + +#endif // EIGEN_CXX11_THREADPOOL_RUNQUEUE_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadCancel.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadCancel.h new file mode 100644 index 00000000000..a05685f11d8 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadCancel.h @@ -0,0 +1,23 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_THREADPOOL_THREAD_CANCEL_H +#define EIGEN_CXX11_THREADPOOL_THREAD_CANCEL_H + +// Try to come up with a portable way to cancel a thread +#if EIGEN_OS_GNULINUX + #define EIGEN_THREAD_CANCEL(t) \ + pthread_cancel(t.native_handle()); + #define EIGEN_SUPPORTS_THREAD_CANCELLATION 1 +#else +#define EIGEN_THREAD_CANCEL(t) +#endif + + +#endif // EIGEN_CXX11_THREADPOOL_THREAD_CANCEL_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadEnvironment.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadEnvironment.h new file mode 100644 index 00000000000..0e3a49b128a --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadEnvironment.h @@ -0,0 +1,43 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_THREADPOOL_THREAD_ENVIRONMENT_H +#define EIGEN_CXX11_THREADPOOL_THREAD_ENVIRONMENT_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +struct StlThreadEnvironment { + struct Task { + std::function f; + }; + + // EnvThread constructor must start the thread, + // destructor must join the thread. + class EnvThread { + public: + EnvThread(std::function f) : thr_(std::move(f)) {} + ~EnvThread() { thr_.join(); } + // This function is called when the threadpool is cancelled. + void OnCancel() { } + + private: + std::thread thr_; + }; + + EnvThread* CreateThread(std::function f) { return new EnvThread(std::move(f)); } + Task CreateTask(std::function f) { return Task{std::move(f)}; } + void ExecuteTask(const Task& t) { t.f(); } +}; + +} // namespace Eigen + +#endif // EIGEN_CXX11_THREADPOOL_THREAD_ENVIRONMENT_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadLocal.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadLocal.h new file mode 100644 index 00000000000..ef9c61e61e5 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadLocal.h @@ -0,0 +1,300 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_THREADPOOL_THREAD_LOCAL_H +#define EIGEN_CXX11_THREADPOOL_THREAD_LOCAL_H + +#ifdef EIGEN_AVOID_THREAD_LOCAL + +#ifdef EIGEN_THREAD_LOCAL +#undef EIGEN_THREAD_LOCAL +#endif + +#else + +#if ((EIGEN_COMP_GNUC) || __has_feature(cxx_thread_local) || EIGEN_COMP_MSVC ) +#define EIGEN_THREAD_LOCAL static thread_local +#endif + +// Disable TLS for Apple and Android builds with older toolchains. +#if defined(__APPLE__) +// Included for TARGET_OS_IPHONE, __IPHONE_OS_VERSION_MIN_REQUIRED, +// __IPHONE_8_0. +#include +#include +#endif +// Checks whether C++11's `thread_local` storage duration specifier is +// supported. +#if EIGEN_COMP_CLANGAPPLE && ((EIGEN_COMP_CLANGAPPLE < 8000042) || \ + (TARGET_OS_IPHONE && __IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_9_0)) +// Notes: Xcode's clang did not support `thread_local` until version +// 8, and even then not for all iOS < 9.0. +#undef EIGEN_THREAD_LOCAL + +#elif defined(__ANDROID__) && EIGEN_COMP_CLANG +// There are platforms for which TLS should not be used even though the compiler +// makes it seem like it's supported (Android NDK < r12b for example). +// This is primarily because of linker problems and toolchain misconfiguration: +// TLS isn't supported until NDK r12b per +// https://developer.android.com/ndk/downloads/revision_history.html +// Since NDK r16, `__NDK_MAJOR__` and `__NDK_MINOR__` are defined in +// . For NDK < r16, users should define these macros, +// e.g. `-D__NDK_MAJOR__=11 -D__NKD_MINOR__=0` for NDK r11. +#if __has_include() +#include +#endif // __has_include() +#if defined(__ANDROID__) && defined(__clang__) && defined(__NDK_MAJOR__) && \ + defined(__NDK_MINOR__) && \ + ((__NDK_MAJOR__ < 12) || ((__NDK_MAJOR__ == 12) && (__NDK_MINOR__ < 1))) +#undef EIGEN_THREAD_LOCAL +#endif +#endif // defined(__ANDROID__) && defined(__clang__) + +#endif // EIGEN_AVOID_THREAD_LOCAL + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { +template +struct ThreadLocalNoOpInitialize { + void operator()(T&) const {} +}; + +template +struct ThreadLocalNoOpRelease { + void operator()(T&) const {} +}; + +} // namespace internal + +// Thread local container for elements of type T, that does not use thread local +// storage. As long as the number of unique threads accessing this storage +// is smaller than `capacity_`, it is lock-free and wait-free. Otherwise it will +// use a mutex for synchronization. +// +// Type `T` has to be default constructible, and by default each thread will get +// a default constructed value. It is possible to specify custom `initialize` +// callable, that will be called lazily from each thread accessing this object, +// and will be passed a default initialized object of type `T`. Also it's +// possible to pass a custom `release` callable, that will be invoked before +// calling ~T(). +// +// Example: +// +// struct Counter { +// int value = 0; +// } +// +// Eigen::ThreadLocal counter(10); +// +// // Each thread will have access to it's own counter object. +// Counter& cnt = counter.local(); +// cnt++; +// +// WARNING: Eigen::ThreadLocal uses the OS-specific value returned by +// std::this_thread::get_id() to identify threads. This value is not guaranteed +// to be unique except for the life of the thread. A newly created thread may +// get an OS-specific ID equal to that of an already destroyed thread. +// +// Somewhat similar to TBB thread local storage, with similar restrictions: +// https://www.threadingbuildingblocks.org/docs/help/reference/thread_local_storage/enumerable_thread_specific_cls.html +// +template , + typename Release = internal::ThreadLocalNoOpRelease> +class ThreadLocal { + // We preallocate default constructed elements in MaxSizedVector. + static_assert(std::is_default_constructible::value, + "ThreadLocal data type must be default constructible"); + + public: + explicit ThreadLocal(int capacity) + : ThreadLocal(capacity, internal::ThreadLocalNoOpInitialize(), + internal::ThreadLocalNoOpRelease()) {} + + ThreadLocal(int capacity, Initialize initialize) + : ThreadLocal(capacity, std::move(initialize), + internal::ThreadLocalNoOpRelease()) {} + + ThreadLocal(int capacity, Initialize initialize, Release release) + : initialize_(std::move(initialize)), + release_(std::move(release)), + capacity_(capacity), + data_(capacity_), + ptr_(capacity_), + filled_records_(0) { + eigen_assert(capacity_ >= 0); + data_.resize(capacity_); + for (int i = 0; i < capacity_; ++i) { + ptr_.emplace_back(nullptr); + } + } + + T& local() { + std::thread::id this_thread = std::this_thread::get_id(); + if (capacity_ == 0) return SpilledLocal(this_thread); + + std::size_t h = std::hash()(this_thread); + const int start_idx = h % capacity_; + + // NOTE: From the definition of `std::this_thread::get_id()` it is + // guaranteed that we never can have concurrent insertions with the same key + // to our hash-map like data structure. If we didn't find an element during + // the initial traversal, it's guaranteed that no one else could have + // inserted it while we are in this function. This allows to massively + // simplify out lock-free insert-only hash map. + + // Check if we already have an element for `this_thread`. + int idx = start_idx; + while (ptr_[idx].load() != nullptr) { + ThreadIdAndValue& record = *(ptr_[idx].load()); + if (record.thread_id == this_thread) return record.value; + + idx += 1; + if (idx >= capacity_) idx -= capacity_; + if (idx == start_idx) break; + } + + // If we are here, it means that we found an insertion point in lookup + // table at `idx`, or we did a full traversal and table is full. + + // If lock-free storage is full, fallback on mutex. + if (filled_records_.load() >= capacity_) return SpilledLocal(this_thread); + + // We double check that we still have space to insert an element into a lock + // free storage. If old value in `filled_records_` is larger than the + // records capacity, it means that some other thread added an element while + // we were traversing lookup table. + int insertion_index = + filled_records_.fetch_add(1, std::memory_order_relaxed); + if (insertion_index >= capacity_) return SpilledLocal(this_thread); + + // At this point it's guaranteed that we can access to + // data_[insertion_index_] without a data race. + data_[insertion_index].thread_id = this_thread; + initialize_(data_[insertion_index].value); + + // That's the pointer we'll put into the lookup table. + ThreadIdAndValue* inserted = &data_[insertion_index]; + + // We'll use nullptr pointer to ThreadIdAndValue in a compare-and-swap loop. + ThreadIdAndValue* empty = nullptr; + + // Now we have to find an insertion point into the lookup table. We start + // from the `idx` that was identified as an insertion point above, it's + // guaranteed that we will have an empty record somewhere in a lookup table + // (because we created a record in the `data_`). + const int insertion_idx = idx; + + do { + // Always start search from the original insertion candidate. + idx = insertion_idx; + while (ptr_[idx].load() != nullptr) { + idx += 1; + if (idx >= capacity_) idx -= capacity_; + // If we did a full loop, it means that we don't have any free entries + // in the lookup table, and this means that something is terribly wrong. + eigen_assert(idx != insertion_idx); + } + // Atomic CAS of the pointer guarantees that any other thread, that will + // follow this pointer will see all the mutations in the `data_`. + } while (!ptr_[idx].compare_exchange_weak(empty, inserted)); + + return inserted->value; + } + + // WARN: It's not thread safe to call it concurrently with `local()`. + void ForEach(std::function f) { + // Reading directly from `data_` is unsafe, because only CAS to the + // record in `ptr_` makes all changes visible to other threads. + for (auto& ptr : ptr_) { + ThreadIdAndValue* record = ptr.load(); + if (record == nullptr) continue; + f(record->thread_id, record->value); + } + + // We did not spill into the map based storage. + if (filled_records_.load(std::memory_order_relaxed) < capacity_) return; + + // Adds a happens before edge from the last call to SpilledLocal(). + EIGEN_MUTEX_LOCK lock(mu_); + for (auto& kv : per_thread_map_) { + f(kv.first, kv.second); + } + } + + // WARN: It's not thread safe to call it concurrently with `local()`. + ~ThreadLocal() { + // Reading directly from `data_` is unsafe, because only CAS to the record + // in `ptr_` makes all changes visible to other threads. + for (auto& ptr : ptr_) { + ThreadIdAndValue* record = ptr.load(); + if (record == nullptr) continue; + release_(record->value); + } + + // We did not spill into the map based storage. + if (filled_records_.load(std::memory_order_relaxed) < capacity_) return; + + // Adds a happens before edge from the last call to SpilledLocal(). + EIGEN_MUTEX_LOCK lock(mu_); + for (auto& kv : per_thread_map_) { + release_(kv.second); + } + } + + private: + struct ThreadIdAndValue { + std::thread::id thread_id; + T value; + }; + + // Use unordered map guarded by a mutex when lock free storage is full. + T& SpilledLocal(std::thread::id this_thread) { + EIGEN_MUTEX_LOCK lock(mu_); + + auto it = per_thread_map_.find(this_thread); + if (it == per_thread_map_.end()) { + auto result = per_thread_map_.emplace(this_thread, T()); + eigen_assert(result.second); + initialize_((*result.first).second); + return (*result.first).second; + } else { + return it->second; + } + } + + Initialize initialize_; + Release release_; + const int capacity_; + + // Storage that backs lock-free lookup table `ptr_`. Records stored in this + // storage contiguously starting from index 0. + MaxSizeVector data_; + + // Atomic pointers to the data stored in `data_`. Used as a lookup table for + // linear probing hash map (https://en.wikipedia.org/wiki/Linear_probing). + MaxSizeVector> ptr_; + + // Number of records stored in the `data_`. + std::atomic filled_records_; + + // We fallback on per thread map if lock-free storage is full. In practice + // this should never happen, if `capacity_` is a reasonable estimate of the + // number of threads running in a system. + EIGEN_MUTEX mu_; // Protects per_thread_map_. + std::unordered_map per_thread_map_; +}; + +} // namespace Eigen + +#endif // EIGEN_CXX11_THREADPOOL_THREAD_LOCAL_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadPoolInterface.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadPoolInterface.h new file mode 100644 index 00000000000..9b01c875b19 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadPoolInterface.h @@ -0,0 +1,51 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_THREADPOOL_THREAD_POOL_INTERFACE_H +#define EIGEN_CXX11_THREADPOOL_THREAD_POOL_INTERFACE_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +// This defines an interface that ThreadPoolDevice can take to use +// custom thread pools underneath. +class ThreadPoolInterface { + public: + // Submits a closure to be run by a thread in the pool. + virtual void Schedule(std::function fn) = 0; + + // Submits a closure to be run by threads in the range [start, end) in the + // pool. + virtual void ScheduleWithHint(std::function fn, int /*start*/, + int /*end*/) { + // Just defer to Schedule in case sub-classes aren't interested in + // overriding this functionality. + Schedule(fn); + } + + // If implemented, stop processing the closures that have been enqueued. + // Currently running closures may still be processed. + // If not implemented, does nothing. + virtual void Cancel() {} + + // Returns the number of threads in the pool. + virtual int NumThreads() const = 0; + + // Returns a logical thread index between 0 and NumThreads() - 1 if called + // from one of the threads in the pool. Returns -1 otherwise. + virtual int CurrentThreadId() const = 0; + + virtual ~ThreadPoolInterface() {} +}; + +} // namespace Eigen + +#endif // EIGEN_CXX11_THREADPOOL_THREAD_POOL_INTERFACE_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadYield.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadYield.h new file mode 100644 index 00000000000..f556ff632d5 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/ThreadPool/ThreadYield.h @@ -0,0 +1,16 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_THREADPOOL_THREAD_YIELD_H +#define EIGEN_CXX11_THREADPOOL_THREAD_YIELD_H + +// Try to come up with a portable way to yield +#define EIGEN_THREAD_YIELD() std::this_thread::yield() + +#endif // EIGEN_CXX11_THREADPOOL_THREAD_YIELD_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/UmfPackSupport/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/UmfPackSupport/InternalHeaderCheck.h new file mode 100644 index 00000000000..64112f1a4ca --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/UmfPackSupport/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_UMFPACKSUPPORT_MODULE_H +#error "Please include Eigen/UmfPackSupport instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/UmfPackSupport/UmfPackSupport.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/UmfPackSupport/UmfPackSupport.h index e3a333f80f0..da631d3b758 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/UmfPackSupport/UmfPackSupport.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/UmfPackSupport/UmfPackSupport.h @@ -20,6 +20,9 @@ #endif #endif +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { /* TODO extract L, extract U, compute det, etc... */ @@ -278,21 +281,21 @@ inline SuiteSparse_long umfpack_get_determinant(std::complex *Mx, double * * \warning The input matrix A should be in a \b compressed and \b column-major form. * Otherwise an expensive copy will be made. You can call the inexpensive makeCompressed() to get a compressed matrix. - * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> * * \implsparsesolverconcept * * \sa \ref TutorialSparseSolverConcept, class SparseLU */ -template -class UmfPackLU : public SparseSolverBase > +template +class UmfPackLU : public SparseSolverBase > { protected: - typedef SparseSolverBase > Base; + typedef SparseSolverBase > Base; using Base::m_isInitialized; public: using Base::_solve_impl; - typedef _MatrixType MatrixType; + typedef MatrixType_ MatrixType; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef typename MatrixType::StorageIndex StorageIndex; @@ -529,16 +532,16 @@ class UmfPackLU : public SparseSolverBase > template void grab(const EigenBase &A) { - mp_matrix.~UmfpackMatrixRef(); - ::new (&mp_matrix) UmfpackMatrixRef(A.derived()); + internal::destroy_at(&mp_matrix); + internal::construct_at(&mp_matrix, A.derived()); } void grab(const UmfpackMatrixRef &A) { if(&(A.derived()) != &mp_matrix) { - mp_matrix.~UmfpackMatrixRef(); - ::new (&mp_matrix) UmfpackMatrixRef(A); + internal::destroy_at(&mp_matrix); + internal::construct_at(&mp_matrix, A); } } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/Image.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/Image.h index b8b8a045529..d7c99f0697e 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/Image.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/Image.h @@ -10,6 +10,9 @@ #ifndef EIGEN_MISC_IMAGE_H #define EIGEN_MISC_IMAGE_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -32,10 +35,10 @@ struct traits > > ReturnType; }; -template struct image_retval_base - : public ReturnByValue > +template struct image_retval_base + : public ReturnByValue > { - typedef _DecompositionType DecompositionType; + typedef DecompositionType_ DecompositionType; typedef typename DecompositionType::MatrixType MatrixType; typedef ReturnByValue Base; diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/InternalHeaderCheck.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/InternalHeaderCheck.h new file mode 100644 index 00000000000..1cea572dcf9 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_CORE_MODULE_H +#error "Please include Eigen/Core instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/Kernel.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/Kernel.h index bef5d6ff583..676f0caa565 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/Kernel.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/Kernel.h @@ -10,6 +10,9 @@ #ifndef EIGEN_MISC_KERNEL_H #define EIGEN_MISC_KERNEL_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { @@ -34,10 +37,10 @@ struct traits > > ReturnType; }; -template struct kernel_retval_base - : public ReturnByValue > +template struct kernel_retval_base + : public ReturnByValue > { - typedef _DecompositionType DecompositionType; + typedef DecompositionType_ DecompositionType; typedef ReturnByValue Base; explicit kernel_retval_base(const DecompositionType& dec) diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/RealSvd2x2.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/RealSvd2x2.h index abb4d3c2fc7..084bd358aaf 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/RealSvd2x2.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/RealSvd2x2.h @@ -11,6 +11,9 @@ #ifndef EIGEN_REALSVD2X2_H #define EIGEN_REALSVD2X2_H +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + namespace Eigen { namespace internal { diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/blas.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/blas.h index 25215b15e8a..0170eef4de2 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/blas.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/blas.h @@ -1,440 +1,66 @@ -#ifndef BLAS_H -#define BLAS_H +#ifndef EIGEN_MISC_BLAS_H +#define EIGEN_MISC_BLAS_H -#ifdef __cplusplus -extern "C" -{ -#endif +extern "C" { #define BLASFUNC(FUNC) FUNC##_ -#ifdef __WIN64__ -typedef long long BLASLONG; -typedef unsigned long long BLASULONG; -#else -typedef long BLASLONG; -typedef unsigned long BLASULONG; -#endif - -int BLASFUNC(xerbla)(const char *, int *info, int); - -float BLASFUNC(sdot) (int *, float *, int *, float *, int *); -float BLASFUNC(sdsdot)(int *, float *, float *, int *, float *, int *); - -double BLASFUNC(dsdot) (int *, float *, int *, float *, int *); -double BLASFUNC(ddot) (int *, double *, int *, double *, int *); -double BLASFUNC(qdot) (int *, double *, int *, double *, int *); - -int BLASFUNC(cdotuw) (int *, float *, int *, float *, int *, float*); -int BLASFUNC(cdotcw) (int *, float *, int *, float *, int *, float*); -int BLASFUNC(zdotuw) (int *, double *, int *, double *, int *, double*); -int BLASFUNC(zdotcw) (int *, double *, int *, double *, int *, double*); - -int BLASFUNC(saxpy) (const int *, const float *, const float *, const int *, float *, const int *); -int BLASFUNC(daxpy) (const int *, const double *, const double *, const int *, double *, const int *); -int BLASFUNC(qaxpy) (const int *, const double *, const double *, const int *, double *, const int *); -int BLASFUNC(caxpy) (const int *, const float *, const float *, const int *, float *, const int *); -int BLASFUNC(zaxpy) (const int *, const double *, const double *, const int *, double *, const int *); -int BLASFUNC(xaxpy) (const int *, const double *, const double *, const int *, double *, const int *); -int BLASFUNC(caxpyc)(const int *, const float *, const float *, const int *, float *, const int *); -int BLASFUNC(zaxpyc)(const int *, const double *, const double *, const int *, double *, const int *); -int BLASFUNC(xaxpyc)(const int *, const double *, const double *, const int *, double *, const int *); - -int BLASFUNC(scopy) (int *, float *, int *, float *, int *); -int BLASFUNC(dcopy) (int *, double *, int *, double *, int *); -int BLASFUNC(qcopy) (int *, double *, int *, double *, int *); -int BLASFUNC(ccopy) (int *, float *, int *, float *, int *); -int BLASFUNC(zcopy) (int *, double *, int *, double *, int *); -int BLASFUNC(xcopy) (int *, double *, int *, double *, int *); - -int BLASFUNC(sswap) (int *, float *, int *, float *, int *); -int BLASFUNC(dswap) (int *, double *, int *, double *, int *); -int BLASFUNC(qswap) (int *, double *, int *, double *, int *); -int BLASFUNC(cswap) (int *, float *, int *, float *, int *); -int BLASFUNC(zswap) (int *, double *, int *, double *, int *); -int BLASFUNC(xswap) (int *, double *, int *, double *, int *); - -float BLASFUNC(sasum) (int *, float *, int *); -float BLASFUNC(scasum)(int *, float *, int *); -double BLASFUNC(dasum) (int *, double *, int *); -double BLASFUNC(qasum) (int *, double *, int *); -double BLASFUNC(dzasum)(int *, double *, int *); -double BLASFUNC(qxasum)(int *, double *, int *); - -int BLASFUNC(isamax)(int *, float *, int *); -int BLASFUNC(idamax)(int *, double *, int *); -int BLASFUNC(iqamax)(int *, double *, int *); -int BLASFUNC(icamax)(int *, float *, int *); -int BLASFUNC(izamax)(int *, double *, int *); -int BLASFUNC(ixamax)(int *, double *, int *); - -int BLASFUNC(ismax) (int *, float *, int *); -int BLASFUNC(idmax) (int *, double *, int *); -int BLASFUNC(iqmax) (int *, double *, int *); -int BLASFUNC(icmax) (int *, float *, int *); -int BLASFUNC(izmax) (int *, double *, int *); -int BLASFUNC(ixmax) (int *, double *, int *); - -int BLASFUNC(isamin)(int *, float *, int *); -int BLASFUNC(idamin)(int *, double *, int *); -int BLASFUNC(iqamin)(int *, double *, int *); -int BLASFUNC(icamin)(int *, float *, int *); -int BLASFUNC(izamin)(int *, double *, int *); -int BLASFUNC(ixamin)(int *, double *, int *); +/* Level 1 routines */ -int BLASFUNC(ismin)(int *, float *, int *); -int BLASFUNC(idmin)(int *, double *, int *); -int BLASFUNC(iqmin)(int *, double *, int *); -int BLASFUNC(icmin)(int *, float *, int *); -int BLASFUNC(izmin)(int *, double *, int *); -int BLASFUNC(ixmin)(int *, double *, int *); - -float BLASFUNC(samax) (int *, float *, int *); -double BLASFUNC(damax) (int *, double *, int *); -double BLASFUNC(qamax) (int *, double *, int *); -float BLASFUNC(scamax)(int *, float *, int *); -double BLASFUNC(dzamax)(int *, double *, int *); -double BLASFUNC(qxamax)(int *, double *, int *); - -float BLASFUNC(samin) (int *, float *, int *); -double BLASFUNC(damin) (int *, double *, int *); -double BLASFUNC(qamin) (int *, double *, int *); -float BLASFUNC(scamin)(int *, float *, int *); -double BLASFUNC(dzamin)(int *, double *, int *); -double BLASFUNC(qxamin)(int *, double *, int *); - -float BLASFUNC(smax) (int *, float *, int *); -double BLASFUNC(dmax) (int *, double *, int *); -double BLASFUNC(qmax) (int *, double *, int *); -float BLASFUNC(scmax) (int *, float *, int *); -double BLASFUNC(dzmax) (int *, double *, int *); -double BLASFUNC(qxmax) (int *, double *, int *); - -float BLASFUNC(smin) (int *, float *, int *); -double BLASFUNC(dmin) (int *, double *, int *); -double BLASFUNC(qmin) (int *, double *, int *); -float BLASFUNC(scmin) (int *, float *, int *); -double BLASFUNC(dzmin) (int *, double *, int *); -double BLASFUNC(qxmin) (int *, double *, int *); - -int BLASFUNC(sscal) (int *, float *, float *, int *); -int BLASFUNC(dscal) (int *, double *, double *, int *); -int BLASFUNC(qscal) (int *, double *, double *, int *); -int BLASFUNC(cscal) (int *, float *, float *, int *); -int BLASFUNC(zscal) (int *, double *, double *, int *); -int BLASFUNC(xscal) (int *, double *, double *, int *); -int BLASFUNC(csscal)(int *, float *, float *, int *); -int BLASFUNC(zdscal)(int *, double *, double *, int *); -int BLASFUNC(xqscal)(int *, double *, double *, int *); - -float BLASFUNC(snrm2) (int *, float *, int *); -float BLASFUNC(scnrm2)(int *, float *, int *); - -double BLASFUNC(dnrm2) (int *, double *, int *); -double BLASFUNC(qnrm2) (int *, double *, int *); -double BLASFUNC(dznrm2)(int *, double *, int *); -double BLASFUNC(qxnrm2)(int *, double *, int *); - -int BLASFUNC(srot) (int *, float *, int *, float *, int *, float *, float *); -int BLASFUNC(drot) (int *, double *, int *, double *, int *, double *, double *); -int BLASFUNC(qrot) (int *, double *, int *, double *, int *, double *, double *); -int BLASFUNC(csrot) (int *, float *, int *, float *, int *, float *, float *); -int BLASFUNC(zdrot) (int *, double *, int *, double *, int *, double *, double *); -int BLASFUNC(xqrot) (int *, double *, int *, double *, int *, double *, double *); - -int BLASFUNC(srotg) (float *, float *, float *, float *); -int BLASFUNC(drotg) (double *, double *, double *, double *); -int BLASFUNC(qrotg) (double *, double *, double *, double *); -int BLASFUNC(crotg) (float *, float *, float *, float *); -int BLASFUNC(zrotg) (double *, double *, double *, double *); -int BLASFUNC(xrotg) (double *, double *, double *, double *); - -int BLASFUNC(srotmg)(float *, float *, float *, float *, float *); -int BLASFUNC(drotmg)(double *, double *, double *, double *, double *); - -int BLASFUNC(srotm) (int *, float *, int *, float *, int *, float *); -int BLASFUNC(drotm) (int *, double *, int *, double *, int *, double *); -int BLASFUNC(qrotm) (int *, double *, int *, double *, int *, double *); +int BLASFUNC(saxpy)(const int *, const float *, const float *, const int *, float *, const int *); +int BLASFUNC(daxpy)(const int *, const double *, const double *, const int *, double *, const int *); +int BLASFUNC(caxpy)(const int *, const float *, const float *, const int *, float *, const int *); +int BLASFUNC(zaxpy)(const int *, const double *, const double *, const int *, double *, const int *); /* Level 2 routines */ -int BLASFUNC(sger)(int *, int *, float *, float *, int *, - float *, int *, float *, int *); -int BLASFUNC(dger)(int *, int *, double *, double *, int *, - double *, int *, double *, int *); -int BLASFUNC(qger)(int *, int *, double *, double *, int *, - double *, int *, double *, int *); -int BLASFUNC(cgeru)(int *, int *, float *, float *, int *, - float *, int *, float *, int *); -int BLASFUNC(cgerc)(int *, int *, float *, float *, int *, - float *, int *, float *, int *); -int BLASFUNC(zgeru)(int *, int *, double *, double *, int *, - double *, int *, double *, int *); -int BLASFUNC(zgerc)(int *, int *, double *, double *, int *, - double *, int *, double *, int *); -int BLASFUNC(xgeru)(int *, int *, double *, double *, int *, - double *, int *, double *, int *); -int BLASFUNC(xgerc)(int *, int *, double *, double *, int *, - double *, int *, double *, int *); - int BLASFUNC(sgemv)(const char *, const int *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); int BLASFUNC(dgemv)(const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(qgemv)(const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); int BLASFUNC(cgemv)(const char *, const int *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); int BLASFUNC(zgemv)(const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(xgemv)(const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); - -int BLASFUNC(strsv) (const char *, const char *, const char *, const int *, const float *, const int *, float *, const int *); -int BLASFUNC(dtrsv) (const char *, const char *, const char *, const int *, const double *, const int *, double *, const int *); -int BLASFUNC(qtrsv) (const char *, const char *, const char *, const int *, const double *, const int *, double *, const int *); -int BLASFUNC(ctrsv) (const char *, const char *, const char *, const int *, const float *, const int *, float *, const int *); -int BLASFUNC(ztrsv) (const char *, const char *, const char *, const int *, const double *, const int *, double *, const int *); -int BLASFUNC(xtrsv) (const char *, const char *, const char *, const int *, const double *, const int *, double *, const int *); -int BLASFUNC(stpsv) (char *, char *, char *, int *, float *, float *, int *); -int BLASFUNC(dtpsv) (char *, char *, char *, int *, double *, double *, int *); -int BLASFUNC(qtpsv) (char *, char *, char *, int *, double *, double *, int *); -int BLASFUNC(ctpsv) (char *, char *, char *, int *, float *, float *, int *); -int BLASFUNC(ztpsv) (char *, char *, char *, int *, double *, double *, int *); -int BLASFUNC(xtpsv) (char *, char *, char *, int *, double *, double *, int *); +int BLASFUNC(strmv)(const char *, const char *, const char *, const int *, const float *, const int *, float *, const int *); +int BLASFUNC(dtrmv)(const char *, const char *, const char *, const int *, const double *, const int *, double *, const int *); +int BLASFUNC(ctrmv)(const char *, const char *, const char *, const int *, const float *, const int *, float *, const int *); +int BLASFUNC(ztrmv)(const char *, const char *, const char *, const int *, const double *, const int *, double *, const int *); -int BLASFUNC(strmv) (const char *, const char *, const char *, const int *, const float *, const int *, float *, const int *); -int BLASFUNC(dtrmv) (const char *, const char *, const char *, const int *, const double *, const int *, double *, const int *); -int BLASFUNC(qtrmv) (const char *, const char *, const char *, const int *, const double *, const int *, double *, const int *); -int BLASFUNC(ctrmv) (const char *, const char *, const char *, const int *, const float *, const int *, float *, const int *); -int BLASFUNC(ztrmv) (const char *, const char *, const char *, const int *, const double *, const int *, double *, const int *); -int BLASFUNC(xtrmv) (const char *, const char *, const char *, const int *, const double *, const int *, double *, const int *); +int BLASFUNC(ssymv)(const char *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); +int BLASFUNC(dsymv)(const char *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(stpmv) (char *, char *, char *, int *, float *, float *, int *); -int BLASFUNC(dtpmv) (char *, char *, char *, int *, double *, double *, int *); -int BLASFUNC(qtpmv) (char *, char *, char *, int *, double *, double *, int *); -int BLASFUNC(ctpmv) (char *, char *, char *, int *, float *, float *, int *); -int BLASFUNC(ztpmv) (char *, char *, char *, int *, double *, double *, int *); -int BLASFUNC(xtpmv) (char *, char *, char *, int *, double *, double *, int *); - -int BLASFUNC(stbmv) (char *, char *, char *, int *, int *, float *, int *, float *, int *); -int BLASFUNC(dtbmv) (char *, char *, char *, int *, int *, double *, int *, double *, int *); -int BLASFUNC(qtbmv) (char *, char *, char *, int *, int *, double *, int *, double *, int *); -int BLASFUNC(ctbmv) (char *, char *, char *, int *, int *, float *, int *, float *, int *); -int BLASFUNC(ztbmv) (char *, char *, char *, int *, int *, double *, int *, double *, int *); -int BLASFUNC(xtbmv) (char *, char *, char *, int *, int *, double *, int *, double *, int *); - -int BLASFUNC(stbsv) (char *, char *, char *, int *, int *, float *, int *, float *, int *); -int BLASFUNC(dtbsv) (char *, char *, char *, int *, int *, double *, int *, double *, int *); -int BLASFUNC(qtbsv) (char *, char *, char *, int *, int *, double *, int *, double *, int *); -int BLASFUNC(ctbsv) (char *, char *, char *, int *, int *, float *, int *, float *, int *); -int BLASFUNC(ztbsv) (char *, char *, char *, int *, int *, double *, int *, double *, int *); -int BLASFUNC(xtbsv) (char *, char *, char *, int *, int *, double *, int *, double *, int *); - -int BLASFUNC(ssymv) (const char *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); -int BLASFUNC(dsymv) (const char *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(qsymv) (const char *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); - -int BLASFUNC(sspmv) (char *, int *, float *, float *, - float *, int *, float *, float *, int *); -int BLASFUNC(dspmv) (char *, int *, double *, double *, - double *, int *, double *, double *, int *); -int BLASFUNC(qspmv) (char *, int *, double *, double *, - double *, int *, double *, double *, int *); - -int BLASFUNC(ssyr) (const char *, const int *, const float *, const float *, const int *, float *, const int *); -int BLASFUNC(dsyr) (const char *, const int *, const double *, const double *, const int *, double *, const int *); -int BLASFUNC(qsyr) (const char *, const int *, const double *, const double *, const int *, double *, const int *); - -int BLASFUNC(ssyr2) (const char *, const int *, const float *, const float *, const int *, const float *, const int *, float *, const int *); -int BLASFUNC(dsyr2) (const char *, const int *, const double *, const double *, const int *, const double *, const int *, double *, const int *); -int BLASFUNC(qsyr2) (const char *, const int *, const double *, const double *, const int *, const double *, const int *, double *, const int *); -int BLASFUNC(csyr2) (const char *, const int *, const float *, const float *, const int *, const float *, const int *, float *, const int *); -int BLASFUNC(zsyr2) (const char *, const int *, const double *, const double *, const int *, const double *, const int *, double *, const int *); -int BLASFUNC(xsyr2) (const char *, const int *, const double *, const double *, const int *, const double *, const int *, double *, const int *); - -int BLASFUNC(sspr) (char *, int *, float *, float *, int *, - float *); -int BLASFUNC(dspr) (char *, int *, double *, double *, int *, - double *); -int BLASFUNC(qspr) (char *, int *, double *, double *, int *, - double *); - -int BLASFUNC(sspr2) (char *, int *, float *, - float *, int *, float *, int *, float *); -int BLASFUNC(dspr2) (char *, int *, double *, - double *, int *, double *, int *, double *); -int BLASFUNC(qspr2) (char *, int *, double *, - double *, int *, double *, int *, double *); -int BLASFUNC(cspr2) (char *, int *, float *, - float *, int *, float *, int *, float *); -int BLASFUNC(zspr2) (char *, int *, double *, - double *, int *, double *, int *, double *); -int BLASFUNC(xspr2) (char *, int *, double *, - double *, int *, double *, int *, double *); - -int BLASFUNC(cher) (char *, int *, float *, float *, int *, - float *, int *); -int BLASFUNC(zher) (char *, int *, double *, double *, int *, - double *, int *); -int BLASFUNC(xher) (char *, int *, double *, double *, int *, - double *, int *); - -int BLASFUNC(chpr) (char *, int *, float *, float *, int *, float *); -int BLASFUNC(zhpr) (char *, int *, double *, double *, int *, double *); -int BLASFUNC(xhpr) (char *, int *, double *, double *, int *, double *); - -int BLASFUNC(cher2) (char *, int *, float *, - float *, int *, float *, int *, float *, int *); -int BLASFUNC(zher2) (char *, int *, double *, - double *, int *, double *, int *, double *, int *); -int BLASFUNC(xher2) (char *, int *, double *, - double *, int *, double *, int *, double *, int *); - -int BLASFUNC(chpr2) (char *, int *, float *, - float *, int *, float *, int *, float *); -int BLASFUNC(zhpr2) (char *, int *, double *, - double *, int *, double *, int *, double *); -int BLASFUNC(xhpr2) (char *, int *, double *, - double *, int *, double *, int *, double *); - -int BLASFUNC(chemv) (const char *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); -int BLASFUNC(zhemv) (const char *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(xhemv) (const char *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); - -int BLASFUNC(chpmv) (char *, int *, float *, float *, - float *, int *, float *, float *, int *); -int BLASFUNC(zhpmv) (char *, int *, double *, double *, - double *, int *, double *, double *, int *); -int BLASFUNC(xhpmv) (char *, int *, double *, double *, - double *, int *, double *, double *, int *); - -int BLASFUNC(snorm)(char *, int *, int *, float *, int *); -int BLASFUNC(dnorm)(char *, int *, int *, double *, int *); -int BLASFUNC(cnorm)(char *, int *, int *, float *, int *); -int BLASFUNC(znorm)(char *, int *, int *, double *, int *); - -int BLASFUNC(sgbmv)(char *, int *, int *, int *, int *, float *, float *, int *, - float *, int *, float *, float *, int *); -int BLASFUNC(dgbmv)(char *, int *, int *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); -int BLASFUNC(qgbmv)(char *, int *, int *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); -int BLASFUNC(cgbmv)(char *, int *, int *, int *, int *, float *, float *, int *, - float *, int *, float *, float *, int *); -int BLASFUNC(zgbmv)(char *, int *, int *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); -int BLASFUNC(xgbmv)(char *, int *, int *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); - -int BLASFUNC(ssbmv)(char *, int *, int *, float *, float *, int *, - float *, int *, float *, float *, int *); -int BLASFUNC(dsbmv)(char *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); -int BLASFUNC(qsbmv)(char *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); -int BLASFUNC(csbmv)(char *, int *, int *, float *, float *, int *, - float *, int *, float *, float *, int *); -int BLASFUNC(zsbmv)(char *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); -int BLASFUNC(xsbmv)(char *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); - -int BLASFUNC(chbmv)(char *, int *, int *, float *, float *, int *, - float *, int *, float *, float *, int *); -int BLASFUNC(zhbmv)(char *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); -int BLASFUNC(xhbmv)(char *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); +int BLASFUNC(chemv)(const char *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); +int BLASFUNC(zhemv)(const char *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); /* Level 3 routines */ int BLASFUNC(sgemm)(const char *, const char *, const int *, const int *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); int BLASFUNC(dgemm)(const char *, const char *, const int *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(qgemm)(const char *, const char *, const int *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); int BLASFUNC(cgemm)(const char *, const char *, const int *, const int *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); int BLASFUNC(zgemm)(const char *, const char *, const int *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(xgemm)(const char *, const char *, const int *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); - -int BLASFUNC(cgemm3m)(char *, char *, int *, int *, int *, float *, - float *, int *, float *, int *, float *, float *, int *); -int BLASFUNC(zgemm3m)(char *, char *, int *, int *, int *, double *, - double *, int *, double *, int *, double *, double *, int *); -int BLASFUNC(xgemm3m)(char *, char *, int *, int *, int *, double *, - double *, int *, double *, int *, double *, double *, int *); - -int BLASFUNC(sge2mm)(char *, char *, char *, int *, int *, - float *, float *, int *, float *, int *, - float *, float *, int *); -int BLASFUNC(dge2mm)(char *, char *, char *, int *, int *, - double *, double *, int *, double *, int *, - double *, double *, int *); -int BLASFUNC(cge2mm)(char *, char *, char *, int *, int *, - float *, float *, int *, float *, int *, - float *, float *, int *); -int BLASFUNC(zge2mm)(char *, char *, char *, int *, int *, - double *, double *, int *, double *, int *, - double *, double *, int *); int BLASFUNC(strsm)(const char *, const char *, const char *, const char *, const int *, const int *, const float *, const float *, const int *, float *, const int *); int BLASFUNC(dtrsm)(const char *, const char *, const char *, const char *, const int *, const int *, const double *, const double *, const int *, double *, const int *); -int BLASFUNC(qtrsm)(const char *, const char *, const char *, const char *, const int *, const int *, const double *, const double *, const int *, double *, const int *); int BLASFUNC(ctrsm)(const char *, const char *, const char *, const char *, const int *, const int *, const float *, const float *, const int *, float *, const int *); int BLASFUNC(ztrsm)(const char *, const char *, const char *, const char *, const int *, const int *, const double *, const double *, const int *, double *, const int *); -int BLASFUNC(xtrsm)(const char *, const char *, const char *, const char *, const int *, const int *, const double *, const double *, const int *, double *, const int *); int BLASFUNC(strmm)(const char *, const char *, const char *, const char *, const int *, const int *, const float *, const float *, const int *, float *, const int *); int BLASFUNC(dtrmm)(const char *, const char *, const char *, const char *, const int *, const int *, const double *, const double *, const int *, double *, const int *); -int BLASFUNC(qtrmm)(const char *, const char *, const char *, const char *, const int *, const int *, const double *, const double *, const int *, double *, const int *); int BLASFUNC(ctrmm)(const char *, const char *, const char *, const char *, const int *, const int *, const float *, const float *, const int *, float *, const int *); int BLASFUNC(ztrmm)(const char *, const char *, const char *, const char *, const int *, const int *, const double *, const double *, const int *, double *, const int *); -int BLASFUNC(xtrmm)(const char *, const char *, const char *, const char *, const int *, const int *, const double *, const double *, const int *, double *, const int *); int BLASFUNC(ssymm)(const char *, const char *, const int *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); int BLASFUNC(dsymm)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(qsymm)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(csymm)(const char *, const char *, const int *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); -int BLASFUNC(zsymm)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(xsymm)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); - -int BLASFUNC(csymm3m)(char *, char *, int *, int *, float *, float *, int *, float *, int *, float *, float *, int *); -int BLASFUNC(zsymm3m)(char *, char *, int *, int *, double *, double *, int *, double *, int *, double *, double *, int *); -int BLASFUNC(xsymm3m)(char *, char *, int *, int *, double *, double *, int *, double *, int *, double *, double *, int *); int BLASFUNC(ssyrk)(const char *, const char *, const int *, const int *, const float *, const float *, const int *, const float *, float *, const int *); int BLASFUNC(dsyrk)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(qsyrk)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(csyrk)(const char *, const char *, const int *, const int *, const float *, const float *, const int *, const float *, float *, const int *); -int BLASFUNC(zsyrk)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(xsyrk)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, double *, const int *); - -int BLASFUNC(ssyr2k)(const char *, const char *, const int *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); -int BLASFUNC(dsyr2k)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double*, const int *, const double *, double *, const int *); -int BLASFUNC(qsyr2k)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double*, const int *, const double *, double *, const int *); -int BLASFUNC(csyr2k)(const char *, const char *, const int *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); -int BLASFUNC(zsyr2k)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double*, const int *, const double *, double *, const int *); -int BLASFUNC(xsyr2k)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double*, const int *, const double *, double *, const int *); int BLASFUNC(chemm)(const char *, const char *, const int *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); int BLASFUNC(zhemm)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(xhemm)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); - -int BLASFUNC(chemm3m)(char *, char *, int *, int *, float *, float *, int *, - float *, int *, float *, float *, int *); -int BLASFUNC(zhemm3m)(char *, char *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); -int BLASFUNC(xhemm3m)(char *, char *, int *, int *, double *, double *, int *, - double *, int *, double *, double *, int *); int BLASFUNC(cherk)(const char *, const char *, const int *, const int *, const float *, const float *, const int *, const float *, float *, const int *); int BLASFUNC(zherk)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(xherk)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, double *, const int *); - -int BLASFUNC(cher2k)(const char *, const char *, const int *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); -int BLASFUNC(zher2k)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(xher2k)(const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(cher2m)(const char *, const char *, const char *, const int *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); -int BLASFUNC(zher2m)(const char *, const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double*, const int *, const double *, double *, const int *); -int BLASFUNC(xher2m)(const char *, const char *, const char *, const int *, const int *, const double *, const double *, const int *, const double*, const int *, const double *, double *, const int *); +#undef BLASFUNC -#ifdef __cplusplus } -#endif #endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/lapack.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/lapack.h deleted file mode 100644 index 249f3575c6d..00000000000 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/lapack.h +++ /dev/null @@ -1,152 +0,0 @@ -#ifndef LAPACK_H -#define LAPACK_H - -#include "blas.h" - -#ifdef __cplusplus -extern "C" -{ -#endif - -int BLASFUNC(csymv) (const char *, const int *, const float *, const float *, const int *, const float *, const int *, const float *, float *, const int *); -int BLASFUNC(zsymv) (const char *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); -int BLASFUNC(xsymv) (const char *, const int *, const double *, const double *, const int *, const double *, const int *, const double *, double *, const int *); - - -int BLASFUNC(cspmv) (char *, int *, float *, float *, - float *, int *, float *, float *, int *); -int BLASFUNC(zspmv) (char *, int *, double *, double *, - double *, int *, double *, double *, int *); -int BLASFUNC(xspmv) (char *, int *, double *, double *, - double *, int *, double *, double *, int *); - -int BLASFUNC(csyr) (char *, int *, float *, float *, int *, - float *, int *); -int BLASFUNC(zsyr) (char *, int *, double *, double *, int *, - double *, int *); -int BLASFUNC(xsyr) (char *, int *, double *, double *, int *, - double *, int *); - -int BLASFUNC(cspr) (char *, int *, float *, float *, int *, - float *); -int BLASFUNC(zspr) (char *, int *, double *, double *, int *, - double *); -int BLASFUNC(xspr) (char *, int *, double *, double *, int *, - double *); - -int BLASFUNC(sgemt)(char *, int *, int *, float *, float *, int *, - float *, int *); -int BLASFUNC(dgemt)(char *, int *, int *, double *, double *, int *, - double *, int *); -int BLASFUNC(cgemt)(char *, int *, int *, float *, float *, int *, - float *, int *); -int BLASFUNC(zgemt)(char *, int *, int *, double *, double *, int *, - double *, int *); - -int BLASFUNC(sgema)(char *, char *, int *, int *, float *, - float *, int *, float *, float *, int *, float *, int *); -int BLASFUNC(dgema)(char *, char *, int *, int *, double *, - double *, int *, double*, double *, int *, double*, int *); -int BLASFUNC(cgema)(char *, char *, int *, int *, float *, - float *, int *, float *, float *, int *, float *, int *); -int BLASFUNC(zgema)(char *, char *, int *, int *, double *, - double *, int *, double*, double *, int *, double*, int *); - -int BLASFUNC(sgems)(char *, char *, int *, int *, float *, - float *, int *, float *, float *, int *, float *, int *); -int BLASFUNC(dgems)(char *, char *, int *, int *, double *, - double *, int *, double*, double *, int *, double*, int *); -int BLASFUNC(cgems)(char *, char *, int *, int *, float *, - float *, int *, float *, float *, int *, float *, int *); -int BLASFUNC(zgems)(char *, char *, int *, int *, double *, - double *, int *, double*, double *, int *, double*, int *); - -int BLASFUNC(sgetf2)(int *, int *, float *, int *, int *, int *); -int BLASFUNC(dgetf2)(int *, int *, double *, int *, int *, int *); -int BLASFUNC(qgetf2)(int *, int *, double *, int *, int *, int *); -int BLASFUNC(cgetf2)(int *, int *, float *, int *, int *, int *); -int BLASFUNC(zgetf2)(int *, int *, double *, int *, int *, int *); -int BLASFUNC(xgetf2)(int *, int *, double *, int *, int *, int *); - -int BLASFUNC(sgetrf)(int *, int *, float *, int *, int *, int *); -int BLASFUNC(dgetrf)(int *, int *, double *, int *, int *, int *); -int BLASFUNC(qgetrf)(int *, int *, double *, int *, int *, int *); -int BLASFUNC(cgetrf)(int *, int *, float *, int *, int *, int *); -int BLASFUNC(zgetrf)(int *, int *, double *, int *, int *, int *); -int BLASFUNC(xgetrf)(int *, int *, double *, int *, int *, int *); - -int BLASFUNC(slaswp)(int *, float *, int *, int *, int *, int *, int *); -int BLASFUNC(dlaswp)(int *, double *, int *, int *, int *, int *, int *); -int BLASFUNC(qlaswp)(int *, double *, int *, int *, int *, int *, int *); -int BLASFUNC(claswp)(int *, float *, int *, int *, int *, int *, int *); -int BLASFUNC(zlaswp)(int *, double *, int *, int *, int *, int *, int *); -int BLASFUNC(xlaswp)(int *, double *, int *, int *, int *, int *, int *); - -int BLASFUNC(sgetrs)(char *, int *, int *, float *, int *, int *, float *, int *, int *); -int BLASFUNC(dgetrs)(char *, int *, int *, double *, int *, int *, double *, int *, int *); -int BLASFUNC(qgetrs)(char *, int *, int *, double *, int *, int *, double *, int *, int *); -int BLASFUNC(cgetrs)(char *, int *, int *, float *, int *, int *, float *, int *, int *); -int BLASFUNC(zgetrs)(char *, int *, int *, double *, int *, int *, double *, int *, int *); -int BLASFUNC(xgetrs)(char *, int *, int *, double *, int *, int *, double *, int *, int *); - -int BLASFUNC(sgesv)(int *, int *, float *, int *, int *, float *, int *, int *); -int BLASFUNC(dgesv)(int *, int *, double *, int *, int *, double*, int *, int *); -int BLASFUNC(qgesv)(int *, int *, double *, int *, int *, double*, int *, int *); -int BLASFUNC(cgesv)(int *, int *, float *, int *, int *, float *, int *, int *); -int BLASFUNC(zgesv)(int *, int *, double *, int *, int *, double*, int *, int *); -int BLASFUNC(xgesv)(int *, int *, double *, int *, int *, double*, int *, int *); - -int BLASFUNC(spotf2)(char *, int *, float *, int *, int *); -int BLASFUNC(dpotf2)(char *, int *, double *, int *, int *); -int BLASFUNC(qpotf2)(char *, int *, double *, int *, int *); -int BLASFUNC(cpotf2)(char *, int *, float *, int *, int *); -int BLASFUNC(zpotf2)(char *, int *, double *, int *, int *); -int BLASFUNC(xpotf2)(char *, int *, double *, int *, int *); - -int BLASFUNC(spotrf)(char *, int *, float *, int *, int *); -int BLASFUNC(dpotrf)(char *, int *, double *, int *, int *); -int BLASFUNC(qpotrf)(char *, int *, double *, int *, int *); -int BLASFUNC(cpotrf)(char *, int *, float *, int *, int *); -int BLASFUNC(zpotrf)(char *, int *, double *, int *, int *); -int BLASFUNC(xpotrf)(char *, int *, double *, int *, int *); - -int BLASFUNC(slauu2)(char *, int *, float *, int *, int *); -int BLASFUNC(dlauu2)(char *, int *, double *, int *, int *); -int BLASFUNC(qlauu2)(char *, int *, double *, int *, int *); -int BLASFUNC(clauu2)(char *, int *, float *, int *, int *); -int BLASFUNC(zlauu2)(char *, int *, double *, int *, int *); -int BLASFUNC(xlauu2)(char *, int *, double *, int *, int *); - -int BLASFUNC(slauum)(char *, int *, float *, int *, int *); -int BLASFUNC(dlauum)(char *, int *, double *, int *, int *); -int BLASFUNC(qlauum)(char *, int *, double *, int *, int *); -int BLASFUNC(clauum)(char *, int *, float *, int *, int *); -int BLASFUNC(zlauum)(char *, int *, double *, int *, int *); -int BLASFUNC(xlauum)(char *, int *, double *, int *, int *); - -int BLASFUNC(strti2)(char *, char *, int *, float *, int *, int *); -int BLASFUNC(dtrti2)(char *, char *, int *, double *, int *, int *); -int BLASFUNC(qtrti2)(char *, char *, int *, double *, int *, int *); -int BLASFUNC(ctrti2)(char *, char *, int *, float *, int *, int *); -int BLASFUNC(ztrti2)(char *, char *, int *, double *, int *, int *); -int BLASFUNC(xtrti2)(char *, char *, int *, double *, int *, int *); - -int BLASFUNC(strtri)(char *, char *, int *, float *, int *, int *); -int BLASFUNC(dtrtri)(char *, char *, int *, double *, int *, int *); -int BLASFUNC(qtrtri)(char *, char *, int *, double *, int *, int *); -int BLASFUNC(ctrtri)(char *, char *, int *, float *, int *, int *); -int BLASFUNC(ztrtri)(char *, char *, int *, double *, int *, int *); -int BLASFUNC(xtrtri)(char *, char *, int *, double *, int *, int *); - -int BLASFUNC(spotri)(char *, int *, float *, int *, int *); -int BLASFUNC(dpotri)(char *, int *, double *, int *, int *); -int BLASFUNC(qpotri)(char *, int *, double *, int *, int *); -int BLASFUNC(cpotri)(char *, int *, float *, int *, int *); -int BLASFUNC(zpotri)(char *, int *, double *, int *, int *); -int BLASFUNC(xpotri)(char *, int *, double *, int *, int *); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/lapacke.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/lapacke.h index 3d8e24f5a2e..c20204cb655 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/lapacke.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/lapacke.h @@ -46,7 +46,11 @@ #include #ifndef lapack_int -#define lapack_int int + #ifdef LAPACK_ILP64 + #define lapack_int int64_t + #else + #define lapack_int int + #endif #endif #ifndef lapack_logical @@ -72,8 +76,7 @@ /* Complex type (single precision) */ #ifndef lapack_complex_float -#include -#define lapack_complex_float float _Complex +#define lapack_complex_float std::complex #endif #ifndef lapack_complex_float_real @@ -88,8 +91,7 @@ lapack_complex_float lapack_make_complex_float( float re, float im ); /* Complex type (double precision) */ #ifndef lapack_complex_double -#include -#define lapack_complex_double double _Complex +#define lapack_complex_double std::complex #endif #ifndef lapack_complex_double_real diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/lapacke_helpers.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/lapacke_helpers.h new file mode 100644 index 00000000000..dc4a04e669b --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/misc/lapacke_helpers.h @@ -0,0 +1,155 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2021 Erik Schultheis +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_LAPACKE_HELPERS_H +#define EIGEN_LAPACKE_HELPERS_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +#ifdef EIGEN_USE_MKL +#include "mkl_lapacke.h" +#else +#include "lapacke.h" +#endif + +namespace Eigen { +namespace internal { +/** + * \internal + * \brief Implementation details and helper functions for the lapacke glue code. + */ +namespace lapacke_helpers { + +// --------------------------------------------------------------------------------------------------------------------- +// Translation from Eigen to Lapacke for types and constants +// --------------------------------------------------------------------------------------------------------------------- + +// For complex numbers, the types in Eigen and Lapacke are different, but layout compatible. +template +struct translate_type_imp; +template<> +struct translate_type_imp { + using type = float; +}; +template<> +struct translate_type_imp { + using type = double; +}; +template<> +struct translate_type_imp> { + using type = lapack_complex_double; +}; +template<> +struct translate_type_imp> { + using type = lapack_complex_float; +}; + +/// Given an Eigen types, this is defined to be the corresponding, layout-compatible lapack type +template +using translated_type = typename translate_type_imp::type; + +/// These functions convert their arguments from Eigen to Lapack types +/// This function performs conversion for any of the translations defined above. +template> +EIGEN_ALWAYS_INLINE auto to_lapack(Source value) { return static_cast(value); } + +/// This function performs conversions for pointer types corresponding to the translations abovce. +/// This is valid because the translations are between layout-compatible types. +template> +EIGEN_ALWAYS_INLINE auto to_lapack(Source *value) { return reinterpret_cast(value); } + +/// This function converts the Eigen Index to a lapack index, with possible range checks +/// \sa internal::convert_index +EIGEN_ALWAYS_INLINE lapack_int to_lapack(Index index) { + return convert_index(index); +} + +/// translates storage order of the given Eigen object to the corresponding lapack constant +template +EIGEN_ALWAYS_INLINE EIGEN_CONSTEXPR lapack_int lapack_storage_of(const EigenBase &) { + return Derived::IsRowMajor ? LAPACK_ROW_MAJOR : LAPACK_COL_MAJOR; +} + +// --------------------------------------------------------------------------------------------------------------------- +// Automatic generation of low-level wrappers +// --------------------------------------------------------------------------------------------------------------------- + +/*! + * \internal + * \brief Helper type to facilitate the wrapping of raw LAPACKE functions for different types into a single, overloaded C++ function. + * This is achieved in combination with \r EIGEN_MAKE_LAPACKE_WRAPPER + * \details This implementation works by providing an overloaded call function that just forwards its arguments to the + * underlying lapack function. Each of these overloads is enabled only if the call is actually well formed. + * Because these lapack functions take pointers to the underlying scalar type as arguments, even though the actual Scalars + * would be implicitly convertible, the pointers are not and therefore only a single overload can be valid at the same time. + * Thus, despite all functions taking fully generic `Args&&... args` as arguments, there is never any ambiguity. + */ +template +struct WrappingHelper { + // The naming of double, single, double complex and single complex is purely for readability + // and doesn't actually affect the workings of this class. In principle, the arguments can + // be supplied in any permuted order. + DoubleFn double_; SingleFn single_; DoubleCpxFn double_cpx_; SingleCpxFn single_cpx_; + + template + auto call(Args&&... args) -> decltype(double_(std::forward(args)...)) { + return double_(std::forward(args)...); + } + + template + auto call(Args&&... args) -> decltype(single_(std::forward(args)...)){ + return single_(std::forward(args)...); + } + + template + auto call(Args&&... args) -> decltype(double_cpx_(std::forward(args)...)){ + return double_cpx_(std::forward(args)...); + } + + template + auto call(Args&&... args) -> decltype(single_cpx_(std::forward(args)...)){ + return single_cpx_(std::forward(args)...); + } +}; + +/** \internal Helper function that generates a `WrappingHelper` object with the given function pointers and + * invokes its `call` method, thus selecting one of the overloads. + * \sa EIGEN_MAKE_LAPACKE_WRAPPER + */ +template +EIGEN_ALWAYS_INLINE auto call_wrapper(DoubleFn df, SingleFn sf, DoubleCpxFn dcf, SingleCpxFn scf, Args&&... args) { + WrappingHelper helper{df, sf, dcf, scf}; + return helper.call(std::forward(args)...); +} + +/** + * \internal + * Generates a new function `Function` that dispatches to the corresponding LAPACKE_? prefixed functions. + * \sa WrappingHelper + */ +#define EIGEN_MAKE_LAPACKE_WRAPPER(FUNCTION) \ +template \ +EIGEN_ALWAYS_INLINE auto FUNCTION(Args&&... args) { return call_wrapper(LAPACKE_d##FUNCTION, LAPACKE_s##FUNCTION, LAPACKE_z##FUNCTION, LAPACKE_c##FUNCTION, std::forward(args)...); } + +// Now with this macro and the helper wrappers, we can generate the dispatch for all the lapacke functions that are +// used in Eigen. +// We define these here instead of in the files where they are used because this allows us to #undef the macro again +// right here +EIGEN_MAKE_LAPACKE_WRAPPER(potrf) +EIGEN_MAKE_LAPACKE_WRAPPER(getrf) +EIGEN_MAKE_LAPACKE_WRAPPER(geqrf) +EIGEN_MAKE_LAPACKE_WRAPPER(gesdd) + +#undef EIGEN_MAKE_LAPACKE_WRAPPER +} +} +} + +#endif // EIGEN_LAPACKE_HELPERS_H diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseBinaryOps.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseBinaryOps.inc similarity index 80% rename from Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseBinaryOps.h rename to Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseBinaryOps.inc index 1b422e2015a..35461da8bf8 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseBinaryOps.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseBinaryOps.inc @@ -30,7 +30,7 @@ operator/(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const * * \sa max() */ -template +template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> #ifdef EIGEN_PARSED_BY_DOXYGEN @@ -43,24 +43,11 @@ min return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), other.derived()); } -template -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> -#ifdef EIGEN_PARSED_BY_DOXYGEN -min -#else -(min) -#endif -(const OtherDerived &other) const -{ - return (min)(other); -} - /** \returns an expression of the coefficient-wise min of \c *this and scalar \a other * * \sa max() */ -template +template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const CwiseNullaryOp, PlainObject> > @@ -74,19 +61,6 @@ min return (min)(Derived::PlainObject::Constant(rows(), cols(), other)); } -EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, - const CwiseNullaryOp, PlainObject> > -#ifdef EIGEN_PARSED_BY_DOXYGEN -min -#else -(min) -#endif -(const Scalar &other) const -{ - return (min)(Derived::PlainObject::Constant(rows(), cols(), other)); -} - /** \returns an expression of the coefficient-wise max of \c *this and \a other * * Example: \include Cwise_max.cpp @@ -94,7 +68,7 @@ min * * \sa min() */ -template +template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> #ifdef EIGEN_PARSED_BY_DOXYGEN @@ -107,26 +81,13 @@ max return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), other.derived()); } -template -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> -#ifdef EIGEN_PARSED_BY_DOXYGEN -max -#else -(max) -#endif -(const OtherDerived &other) const -{ - return (max)(other); -} - /** \returns an expression of the coefficient-wise max of \c *this and scalar \a other * * \sa min() */ -template +template EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, + EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const CwiseNullaryOp, PlainObject> > #ifdef EIGEN_PARSED_BY_DOXYGEN max @@ -138,19 +99,6 @@ max return (max)(Derived::PlainObject::Constant(rows(), cols(), other)); } -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, - const CwiseNullaryOp, PlainObject> > -#ifdef EIGEN_PARSED_BY_DOXYGEN -max -#else -(max) -#endif -(const Scalar &other) const -{ - return (max)(Derived::PlainObject::Constant(rows(), cols(), other)); -} - /** \returns an expression of the coefficient-wise absdiff of \c *this and \a other * * Example: \include Cwise_absolute_difference.cpp @@ -186,24 +134,12 @@ absolute_difference */ EIGEN_MAKE_CWISE_BINARY_OP(pow,pow) -#ifndef EIGEN_PARSED_BY_DOXYGEN -EIGEN_MAKE_SCALAR_BINARY_OP_ONTHERIGHT(pow,pow) -#else -/** \returns an expression of the coefficients of \c *this rasied to the constant power \a exponent +/** \returns an expression of the coefficient-wise atan2(\c *this, \a y), where \a y is the given array argument. * - * \tparam T is the scalar type of \a exponent. It must be compatible with the scalar type of the given expression. + * This function computes the coefficient-wise atan2. * - * This function computes the coefficient-wise power. The function MatrixBase::pow() in the - * unsupported module MatrixFunctions computes the matrix power. - * - * Example: \include Cwise_pow.cpp - * Output: \verbinclude Cwise_pow.out - * - * \sa ArrayBase::pow(ArrayBase), square(), cube(), exp(), log() */ -template -const CwiseBinaryOp,Derived,Constant > pow(const T& exponent) const; -#endif +EIGEN_MAKE_CWISE_BINARY_OP(atan2,atan2) // TODO code generating macros could be moved to Macros.h and could include generation of documentation @@ -371,25 +307,6 @@ const CwiseBinaryOp,Constant,Derived operator/(const T& s,const StorageBaseType& a); #endif -/** \returns an expression of the coefficient-wise ^ operator of *this and \a other - * - * \warning this operator is for expression of bool only. - * - * Example: \include Cwise_boolean_xor.cpp - * Output: \verbinclude Cwise_boolean_xor.out - * - * \sa operator&&(), select() - */ -template -EIGEN_DEVICE_FUNC -inline const CwiseBinaryOp -operator^(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const -{ - EIGEN_STATIC_ASSERT((internal::is_same::value && internal::is_same::value), - THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL); - return CwiseBinaryOp(derived(),other.derived()); -} - // NOTE disabled until we agree on argument order #if 0 /** \cpp11 \returns an expression of the coefficient-wise polygamma function. diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseUnaryOps.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseUnaryOps.inc similarity index 90% rename from Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseUnaryOps.h rename to Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseUnaryOps.inc index 13c55f4b115..b0123261d0b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseUnaryOps.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ArrayCwiseUnaryOps.inc @@ -2,12 +2,15 @@ typedef CwiseUnaryOp, const Derived> AbsReturnType; typedef CwiseUnaryOp, const Derived> ArgReturnType; +typedef CwiseUnaryOp, const Derived> CArgReturnType; typedef CwiseUnaryOp, const Derived> Abs2ReturnType; typedef CwiseUnaryOp, const Derived> SqrtReturnType; +typedef CwiseUnaryOp, const Derived> CbrtReturnType; typedef CwiseUnaryOp, const Derived> RsqrtReturnType; typedef CwiseUnaryOp, const Derived> SignReturnType; typedef CwiseUnaryOp, const Derived> InverseReturnType; typedef CwiseUnaryOp, const Derived> BooleanNotReturnType; +typedef CwiseUnaryOp, const Derived> BitwiseNotReturnType; typedef CwiseUnaryOp, const Derived> ExpReturnType; typedef CwiseUnaryOp, const Derived> Expm1ReturnType; @@ -24,11 +27,9 @@ typedef CwiseUnaryOp, const Derived> AtanReturn typedef CwiseUnaryOp, const Derived> TanhReturnType; typedef CwiseUnaryOp, const Derived> LogisticReturnType; typedef CwiseUnaryOp, const Derived> SinhReturnType; -#if EIGEN_HAS_CXX11_MATH typedef CwiseUnaryOp, const Derived> AtanhReturnType; typedef CwiseUnaryOp, const Derived> AsinhReturnType; typedef CwiseUnaryOp, const Derived> AcoshReturnType; -#endif typedef CwiseUnaryOp, const Derived> CoshReturnType; typedef CwiseUnaryOp, const Derived> SquareReturnType; typedef CwiseUnaryOp, const Derived> CubeReturnType; @@ -68,6 +69,10 @@ arg() const return ArgReturnType(derived()); } +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE const CArgReturnType +carg() const { return CArgReturnType(derived()); } + /** \returns an expression of the coefficient-wise squared absolute value of \c *this * * Example: \include Cwise_abs2.cpp @@ -180,7 +185,7 @@ log2() const * Example: \include Cwise_sqrt.cpp * Output: \verbinclude Cwise_sqrt.out * - * \sa Math functions, pow(), square() + * \sa Math functions, pow(), square(), cbrt() */ EIGEN_DEVICE_FUNC inline const SqrtReturnType @@ -189,6 +194,22 @@ sqrt() const return SqrtReturnType(derived()); } +/** \returns an expression of the coefficient-wise cube root of *this. + * + * This function computes the coefficient-wise cube root. + * + * Example: \include Cwise_cbrt.cpp + * Output: \verbinclude Cwise_cbrt.out + * + * \sa Math functions, sqrt(), pow(), square() + */ +EIGEN_DEVICE_FUNC +inline const CbrtReturnType +cbrt() const +{ + return CbrtReturnType(derived()); +} + /** \returns an expression of the coefficient-wise inverse square root of *this. * * This function computes the coefficient-wise inverse square root. @@ -355,7 +376,6 @@ cosh() const return CoshReturnType(derived()); } -#if EIGEN_HAS_CXX11_MATH /** \returns an expression of the coefficient-wise inverse hyperbolic tan of *this. * * \sa Math functions, atanh(), asinh(), acosh() @@ -388,7 +408,6 @@ acosh() const { return AcoshReturnType(derived()); } -#endif /** \returns an expression of the coefficient-wise logistic of *this. */ @@ -579,8 +598,6 @@ isFinite() const } /** \returns an expression of the coefficient-wise ! operator of *this - * - * \warning this operator is for expression of bool only. * * Example: \include Cwise_boolean_not.cpp * Output: \verbinclude Cwise_boolean_not.out @@ -591,11 +608,18 @@ EIGEN_DEVICE_FUNC inline const BooleanNotReturnType operator!() const { - EIGEN_STATIC_ASSERT((internal::is_same::value), - THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL); return BooleanNotReturnType(derived()); } +/** \returns an expression of the bitwise ~ operator of *this + */ +EIGEN_DEVICE_FUNC +inline const BitwiseNotReturnType +operator~() const +{ + return BitwiseNotReturnType(derived()); +} + // --- SpecialFunctions module --- @@ -694,3 +718,32 @@ ndtri() const { return NdtriReturnType(derived()); } + +template +using UnaryPowReturnType = + std::enable_if_t::Real>::value, + CwiseUnaryOp, const Derived>>; + +#ifndef EIGEN_PARSED_BY_DOXYGEN +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const UnaryPowReturnType pow( + const ScalarExponent& exponent) const { + return UnaryPowReturnType(derived(), internal::scalar_unary_pow_op(exponent)); +#else +/** \returns an expression of the coefficients of \c *this rasied to the constant power \a exponent + * + * \tparam T is the scalar type of \a exponent. It must be compatible with the scalar type of the given expression. + * + * This function computes the coefficient-wise power. The function MatrixBase::pow() in the + * unsupported module MatrixFunctions computes the matrix power. + * + * Example: \include Cwise_pow.cpp + * Output: \verbinclude Cwise_pow.out + * + * \sa ArrayBase::pow(ArrayBase), square(), cube(), exp(), log() + */ +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const UnaryPowReturnType pow( + const ScalarExponent& exponent) const; +#endif +} diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/BlockMethods.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/BlockMethods.inc similarity index 99% rename from Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/BlockMethods.h rename to Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/BlockMethods.inc index 63a52a6ffaa..68b94130b59 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/BlockMethods.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/BlockMethods.inc @@ -1418,19 +1418,19 @@ innerVectors(Index outerStart, Index outerSize) const */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -typename internal::conditional::type +std::conditional_t subVector(Index i) { - return typename internal::conditional::type(derived(),i); + return std::conditional_t(derived(),i); } /** This is the const version of subVector(Index) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -typename internal::conditional::type +std::conditional_t subVector(Index i) const { - return typename internal::conditional::type(derived(),i); + return std::conditional_t(derived(),i); } /** \returns the number of subvectors (rows or columns) in the direction \c Direction diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseBinaryOps.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseBinaryOps.inc similarity index 51% rename from Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseBinaryOps.h rename to Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseBinaryOps.inc index 8b6730ede02..964913be7c8 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseBinaryOps.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseBinaryOps.inc @@ -12,7 +12,7 @@ /** \returns an expression of the difference of \c *this and \a other * - * \note If you want to substract a given scalar from all coefficients, see Cwise::operator-(). + * \note If you want to subtract a given scalar from all coefficients, see Cwise::operator-(). * * \sa class CwiseBinaryOp, operator-=() */ @@ -77,39 +77,61 @@ const CwiseBinaryOp,Derived,Constant > #endif /** \returns an expression of the coefficient-wise boolean \b and operator of \c *this and \a other - * - * \warning this operator is for expression of bool only. - * - * Example: \include Cwise_boolean_and.cpp - * Output: \verbinclude Cwise_boolean_and.out - * - * \sa operator||(), select() - */ -template -EIGEN_DEVICE_FUNC -inline const CwiseBinaryOp -operator&&(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const -{ - EIGEN_STATIC_ASSERT((internal::is_same::value && internal::is_same::value), - THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL); - return CwiseBinaryOp(derived(),other.derived()); + * + * Example: \include Cwise_boolean_and.cpp + * Output: \verbinclude Cwise_boolean_and.out + * + * \sa operator||(), select() + */ +template +EIGEN_DEVICE_FUNC inline const CwiseBinaryOp, const Derived, const OtherDerived> +operator&&(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const { + return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), + other.derived()); } /** \returns an expression of the coefficient-wise boolean \b or operator of \c *this and \a other - * - * \warning this operator is for expression of bool only. - * - * Example: \include Cwise_boolean_or.cpp - * Output: \verbinclude Cwise_boolean_or.out - * - * \sa operator&&(), select() - */ -template -EIGEN_DEVICE_FUNC -inline const CwiseBinaryOp -operator||(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const -{ - EIGEN_STATIC_ASSERT((internal::is_same::value && internal::is_same::value), - THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL); - return CwiseBinaryOp(derived(),other.derived()); + * + * Example: \include Cwise_boolean_or.cpp + * Output: \verbinclude Cwise_boolean_or.out + * + * \sa operator&&(), select() + */ +template +EIGEN_DEVICE_FUNC inline const CwiseBinaryOp, const Derived, const OtherDerived> +operator||(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const { + return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), + other.derived()); +} + +/** \returns an expression of the bitwise \b and operator of \c *this and \a other + * + * \sa operator|(), operator^() + */ +template +EIGEN_DEVICE_FUNC inline const CwiseBinaryOp, const Derived, const OtherDerived> +operator&(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const { + return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), + other.derived()); +} + +/** \returns an expression of the bitwise boolean \b or operator of \c *this and \a other + * + * \sa operator&(), operator^() + */ +template +EIGEN_DEVICE_FUNC inline const CwiseBinaryOp, const Derived, const OtherDerived> +operator|(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const { + return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), + other.derived()); +} + +/** \returns an expression of the bitwise xor operator of *this and \a other + * \sa operator&(), operator|() + */ +template +EIGEN_DEVICE_FUNC inline const CwiseBinaryOp, const Derived, const OtherDerived> +operator^(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const { + return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), + other.derived()); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseUnaryOps.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseUnaryOps.inc similarity index 83% rename from Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseUnaryOps.h rename to Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseUnaryOps.inc index 5418dc4154f..1c6b2845104 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseUnaryOps.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/CommonCwiseUnaryOps.inc @@ -13,20 +13,20 @@ #ifndef EIGEN_PARSED_BY_DOXYGEN /** \internal the return type of conjugate() */ -typedef typename internal::conditional::IsComplex, - const CwiseUnaryOp, const Derived>, - const Derived& - >::type ConjugateReturnType; +typedef std::conditional_t::IsComplex, + const CwiseUnaryOp, const Derived>, + const Derived& + > ConjugateReturnType; /** \internal the return type of real() const */ -typedef typename internal::conditional::IsComplex, - const CwiseUnaryOp, const Derived>, - const Derived& - >::type RealReturnType; +typedef std::conditional_t::IsComplex, + const CwiseUnaryOp, const Derived>, + const Derived& + > RealReturnType; /** \internal the return type of real() */ -typedef typename internal::conditional::IsComplex, - CwiseUnaryView, Derived>, - Derived& - >::type NonConstRealReturnType; +typedef std::conditional_t::IsComplex, + CwiseUnaryView, Derived>, + Derived& + > NonConstRealReturnType; /** \internal the return type of imag() const */ typedef CwiseUnaryOp, const Derived> ImagReturnType; /** \internal the return type of imag() */ @@ -45,7 +45,7 @@ inline const NegativeReturnType operator-() const { return NegativeReturnType(derived()); } -template struct CastXpr { typedef typename internal::cast_return_type, const Derived> >::type Type; }; +template struct CastXpr { typedef typename internal::cast_return_type, const Derived> >::type Type; }; /// \returns an expression of \c *this with the \a Scalar type casted to /// \a NewScalar. @@ -83,10 +83,10 @@ EIGEN_DOC_UNARY_ADDONS(conjugate,complex conjugate) /// \sa conjugate() template EIGEN_DEVICE_FUNC -inline typename internal::conditional::type +inline std::conditional_t conjugateIf() const { - typedef typename internal::conditional::type ReturnType; + typedef std::conditional_t ReturnType; return ReturnType(derived()); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/IndexedViewMethods.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/IndexedViewMethods.h deleted file mode 100644 index 5bfb19ac6cd..00000000000 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/IndexedViewMethods.h +++ /dev/null @@ -1,262 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2017 Gael Guennebaud -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#if !defined(EIGEN_PARSED_BY_DOXYGEN) - -// This file is automatically included twice to generate const and non-const versions - -#ifndef EIGEN_INDEXED_VIEW_METHOD_2ND_PASS -#define EIGEN_INDEXED_VIEW_METHOD_CONST const -#define EIGEN_INDEXED_VIEW_METHOD_TYPE ConstIndexedViewType -#else -#define EIGEN_INDEXED_VIEW_METHOD_CONST -#define EIGEN_INDEXED_VIEW_METHOD_TYPE IndexedViewType -#endif - -#ifndef EIGEN_INDEXED_VIEW_METHOD_2ND_PASS -protected: - -// define some aliases to ease readability - -template -struct IvcRowType : public internal::IndexedViewCompatibleType {}; - -template -struct IvcColType : public internal::IndexedViewCompatibleType {}; - -template -struct IvcType : public internal::IndexedViewCompatibleType {}; - -typedef typename internal::IndexedViewCompatibleType::type IvcIndex; - -template -typename IvcRowType::type -ivcRow(const Indices& indices) const { - return internal::makeIndexedViewCompatible(indices, internal::variable_if_dynamic(derived().rows()),Specialized); -} - -template -typename IvcColType::type -ivcCol(const Indices& indices) const { - return internal::makeIndexedViewCompatible(indices, internal::variable_if_dynamic(derived().cols()),Specialized); -} - -template -typename IvcColType::type -ivcSize(const Indices& indices) const { - return internal::makeIndexedViewCompatible(indices, internal::variable_if_dynamic(derived().size()),Specialized); -} - -public: - -#endif - -template -struct EIGEN_INDEXED_VIEW_METHOD_TYPE { - typedef IndexedView::type, - typename IvcColType::type> type; -}; - -// This is the generic version - -template -typename internal::enable_if::value - && internal::traits::type>::ReturnAsIndexedView, - typename EIGEN_INDEXED_VIEW_METHOD_TYPE::type >::type -operator()(const RowIndices& rowIndices, const ColIndices& colIndices) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - return typename EIGEN_INDEXED_VIEW_METHOD_TYPE::type - (derived(), ivcRow(rowIndices), ivcCol(colIndices)); -} - -// The following overload returns a Block<> object - -template -typename internal::enable_if::value - && internal::traits::type>::ReturnAsBlock, - typename internal::traits::type>::BlockType>::type -operator()(const RowIndices& rowIndices, const ColIndices& colIndices) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - typedef typename internal::traits::type>::BlockType BlockType; - typename IvcRowType::type actualRowIndices = ivcRow(rowIndices); - typename IvcColType::type actualColIndices = ivcCol(colIndices); - return BlockType(derived(), - internal::first(actualRowIndices), - internal::first(actualColIndices), - internal::size(actualRowIndices), - internal::size(actualColIndices)); -} - -// The following overload returns a Scalar - -template -typename internal::enable_if::value - && internal::traits::type>::ReturnAsScalar, - CoeffReturnType >::type -operator()(const RowIndices& rowIndices, const ColIndices& colIndices) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - return Base::operator()(internal::eval_expr_given_size(rowIndices,rows()),internal::eval_expr_given_size(colIndices,cols())); -} - -#if EIGEN_HAS_STATIC_ARRAY_TEMPLATE - -// The following three overloads are needed to handle raw Index[N] arrays. - -template -IndexedView::type> -operator()(const RowIndicesT (&rowIndices)[RowIndicesN], const ColIndices& colIndices) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - return IndexedView::type> - (derived(), rowIndices, ivcCol(colIndices)); -} - -template -IndexedView::type, const ColIndicesT (&)[ColIndicesN]> -operator()(const RowIndices& rowIndices, const ColIndicesT (&colIndices)[ColIndicesN]) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - return IndexedView::type,const ColIndicesT (&)[ColIndicesN]> - (derived(), ivcRow(rowIndices), colIndices); -} - -template -IndexedView -operator()(const RowIndicesT (&rowIndices)[RowIndicesN], const ColIndicesT (&colIndices)[ColIndicesN]) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - return IndexedView - (derived(), rowIndices, colIndices); -} - -#endif // EIGEN_HAS_STATIC_ARRAY_TEMPLATE - -// Overloads for 1D vectors/arrays - -template -typename internal::enable_if< - IsRowMajor && (!(internal::get_compile_time_incr::type>::value==1 || internal::is_valid_index_type::value)), - IndexedView::type> >::type -operator()(const Indices& indices) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return IndexedView::type> - (derived(), IvcIndex(0), ivcCol(indices)); -} - -template -typename internal::enable_if< - (!IsRowMajor) && (!(internal::get_compile_time_incr::type>::value==1 || internal::is_valid_index_type::value)), - IndexedView::type,IvcIndex> >::type -operator()(const Indices& indices) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return IndexedView::type,IvcIndex> - (derived(), ivcRow(indices), IvcIndex(0)); -} - -template -typename internal::enable_if< - (internal::get_compile_time_incr::type>::value==1) && (!internal::is_valid_index_type::value) && (!symbolic::is_symbolic::value), - VectorBlock::value> >::type -operator()(const Indices& indices) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - typename IvcType::type actualIndices = ivcSize(indices); - return VectorBlock::value> - (derived(), internal::first(actualIndices), internal::size(actualIndices)); -} - -template -typename internal::enable_if::value, CoeffReturnType >::type -operator()(const IndexType& id) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - return Base::operator()(internal::eval_expr_given_size(id,size())); -} - -#if EIGEN_HAS_STATIC_ARRAY_TEMPLATE - -template -typename internal::enable_if >::type -operator()(const IndicesT (&indices)[IndicesN]) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return IndexedView - (derived(), IvcIndex(0), indices); -} - -template -typename internal::enable_if >::type -operator()(const IndicesT (&indices)[IndicesN]) EIGEN_INDEXED_VIEW_METHOD_CONST -{ - EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return IndexedView - (derived(), indices, IvcIndex(0)); -} - -#endif // EIGEN_HAS_STATIC_ARRAY_TEMPLATE - -#undef EIGEN_INDEXED_VIEW_METHOD_CONST -#undef EIGEN_INDEXED_VIEW_METHOD_TYPE - -#ifndef EIGEN_INDEXED_VIEW_METHOD_2ND_PASS -#define EIGEN_INDEXED_VIEW_METHOD_2ND_PASS -#include "IndexedViewMethods.h" -#undef EIGEN_INDEXED_VIEW_METHOD_2ND_PASS -#endif - -#else // EIGEN_PARSED_BY_DOXYGEN - -/** - * \returns a generic submatrix view defined by the rows and columns indexed \a rowIndices and \a colIndices respectively. - * - * Each parameter must either be: - * - An integer indexing a single row or column - * - Eigen::all indexing the full set of respective rows or columns in increasing order - * - An ArithmeticSequence as returned by the Eigen::seq and Eigen::seqN functions - * - Any %Eigen's vector/array of integers or expressions - * - Plain C arrays: \c int[N] - * - And more generally any type exposing the following two member functions: - * \code - * operator[]() const; - * size() const; - * \endcode - * where \c stands for any integer type compatible with Eigen::Index (i.e. \c std::ptrdiff_t). - * - * The last statement implies compatibility with \c std::vector, \c std::valarray, \c std::array, many of the Range-v3's ranges, etc. - * - * If the submatrix can be represented using a starting position \c (i,j) and positive sizes \c (rows,columns), then this - * method will returns a Block object after extraction of the relevant information from the passed arguments. This is the case - * when all arguments are either: - * - An integer - * - Eigen::all - * - An ArithmeticSequence with compile-time increment strictly equal to 1, as returned by Eigen::seq(a,b), and Eigen::seqN(a,N). - * - * Otherwise a more general IndexedView object will be returned, after conversion of the inputs - * to more suitable types \c RowIndices' and \c ColIndices'. - * - * For 1D vectors and arrays, you better use the operator()(const Indices&) overload, which behave the same way but taking a single parameter. - * - * See also this question and its answer for an example of how to duplicate coefficients. - * - * \sa operator()(const Indices&), class Block, class IndexedView, DenseBase::block(Index,Index,Index,Index) - */ -template -IndexedView_or_Block -operator()(const RowIndices& rowIndices, const ColIndices& colIndices); - -/** This is an overload of operator()(const RowIndices&, const ColIndices&) for 1D vectors or arrays - * - * \only_for_vectors - */ -template -IndexedView_or_VectorBlock -operator()(const Indices& indices); - -#endif // EIGEN_PARSED_BY_DOXYGEN diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/IndexedViewMethods.inc b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/IndexedViewMethods.inc new file mode 100644 index 00000000000..e5432eacce7 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/IndexedViewMethods.inc @@ -0,0 +1,350 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +#if !defined(EIGEN_PARSED_BY_DOXYGEN) + +protected: +// define some aliases to ease readability + +template +using IvcRowType = typename internal::IndexedViewCompatibleType::type; + +template +using IvcColType = typename internal::IndexedViewCompatibleType::type; + +template +using IvcType = typename internal::IndexedViewCompatibleType::type; + +typedef typename internal::IndexedViewCompatibleType::type IvcIndex; + +template +inline IvcRowType ivcRow(const Indices& indices) const { + return internal::makeIndexedViewCompatible( + indices, internal::variable_if_dynamic(derived().rows()), Specialized); +} + +template +inline IvcColType ivcCol(const Indices& indices) const { + return internal::makeIndexedViewCompatible( + indices, internal::variable_if_dynamic(derived().cols()), Specialized); +} + +template +inline IvcType ivcSize(const Indices& indices) const { + return internal::makeIndexedViewCompatible( + indices, internal::variable_if_dynamic(derived().size()), Specialized); +} + +// this helper class assumes internal::valid_indexed_view_overload::value == true +template , IvcColType>>::ReturnAsScalar, + bool UseBlock = internal::traits, IvcColType>>::ReturnAsBlock, + bool UseGeneric = internal::traits, IvcColType>>::ReturnAsIndexedView> +struct IndexedViewSelector; + +// Generic +template +struct IndexedViewSelector { + using ReturnType = IndexedView, IvcColType>; + using ConstReturnType = IndexedView, IvcColType>; + + static inline ReturnType run(Derived& derived, const RowIndices& rowIndices, const ColIndices& colIndices) { + return ReturnType(derived, derived.ivcRow(rowIndices), derived.ivcCol(colIndices)); + } + static inline ConstReturnType run(const Derived& derived, const RowIndices& rowIndices, + const ColIndices& colIndices) { + return ConstReturnType(derived, derived.ivcRow(rowIndices), derived.ivcCol(colIndices)); + } +}; + +// Block +template +struct IndexedViewSelector { + using IndexedViewType = IndexedView, IvcColType>; + using ConstIndexedViewType = IndexedView, IvcColType>; + using ReturnType = typename internal::traits::BlockType; + using ConstReturnType = typename internal::traits::BlockType; + + static inline ReturnType run(Derived& derived, const RowIndices& rowIndices, const ColIndices& colIndices) { + IvcRowType actualRowIndices = derived.ivcRow(rowIndices); + IvcColType actualColIndices = derived.ivcCol(colIndices); + return ReturnType(derived, internal::first(actualRowIndices), internal::first(actualColIndices), + internal::index_list_size(actualRowIndices), internal::index_list_size(actualColIndices)); + } + static inline ConstReturnType run(const Derived& derived, const RowIndices& rowIndices, + const ColIndices& colIndices) { + IvcRowType actualRowIndices = derived.ivcRow(rowIndices); + IvcColType actualColIndices = derived.ivcCol(colIndices); + return ConstReturnType(derived, internal::first(actualRowIndices), internal::first(actualColIndices), + internal::index_list_size(actualRowIndices), internal::index_list_size(actualColIndices)); + } +}; + +// Symbolic +template +struct IndexedViewSelector { + using ReturnType = typename DenseBase::Scalar&; + using ConstReturnType = typename DenseBase::CoeffReturnType; + + static inline ReturnType run(Derived& derived, const RowIndices& rowIndices, const ColIndices& colIndices) { + return derived(internal::eval_expr_given_size(rowIndices, derived.rows()), + internal::eval_expr_given_size(colIndices, derived.cols())); + } + static inline ConstReturnType run(const Derived& derived, const RowIndices& rowIndices, + const ColIndices& colIndices) { + return derived(internal::eval_expr_given_size(rowIndices, derived.rows()), + internal::eval_expr_given_size(colIndices, derived.cols())); + } +}; + +// this helper class assumes internal::is_valid_index_type::value == false +template ::value, + bool UseBlock = !UseSymbolic && internal::get_compile_time_incr>::value == 1, + bool UseGeneric = !UseSymbolic && !UseBlock> +struct VectorIndexedViewSelector; + +// Generic +template +struct VectorIndexedViewSelector { + + static constexpr bool IsRowMajor = DenseBase::IsRowMajor; + + using RowMajorReturnType = IndexedView>; + using ConstRowMajorReturnType = IndexedView>; + + using ColMajorReturnType = IndexedView, IvcIndex>; + using ConstColMajorReturnType = IndexedView, IvcIndex>; + + using ReturnType = typename internal::conditional::type; + using ConstReturnType = + typename internal::conditional::type; + + template = true> + static inline RowMajorReturnType run(Derived& derived, const Indices& indices) { + return RowMajorReturnType(derived, IvcIndex(0), derived.ivcCol(indices)); + } + template = true> + static inline ConstRowMajorReturnType run(const Derived& derived, const Indices& indices) { + return ConstRowMajorReturnType(derived, IvcIndex(0), derived.ivcCol(indices)); + } + template = true> + static inline ColMajorReturnType run(Derived& derived, const Indices& indices) { + return ColMajorReturnType(derived, derived.ivcRow(indices), IvcIndex(0)); + } + template = true> + static inline ConstColMajorReturnType run(const Derived& derived, const Indices& indices) { + return ConstColMajorReturnType(derived, derived.ivcRow(indices), IvcIndex(0)); + } +}; + +// Block +template +struct VectorIndexedViewSelector { + + using ReturnType = VectorBlock::value>; + using ConstReturnType = VectorBlock::value>; + + static inline ReturnType run(Derived& derived, const Indices& indices) { + IvcType actualIndices = derived.ivcSize(indices); + return ReturnType(derived, internal::first(actualIndices), internal::index_list_size(actualIndices)); + } + static inline ConstReturnType run(const Derived& derived, const Indices& indices) { + IvcType actualIndices = derived.ivcSize(indices); + return ConstReturnType(derived, internal::first(actualIndices), internal::index_list_size(actualIndices)); + } +}; + +// Symbolic +template +struct VectorIndexedViewSelector { + + using ReturnType = typename DenseBase::Scalar&; + using ConstReturnType = typename DenseBase::CoeffReturnType; + + static inline ReturnType run(Derived& derived, const Indices& id) { + return derived(internal::eval_expr_given_size(id, derived.size())); + } + static inline ConstReturnType run(const Derived& derived, const Indices& id) { + return derived(internal::eval_expr_given_size(id, derived.size())); + } +}; + +// SFINAE dummy types + +template +using EnableOverload = std::enable_if_t< + internal::valid_indexed_view_overload::value && internal::is_lvalue::value, bool>; + +template +using EnableConstOverload = + std::enable_if_t::value, bool>; + +template +using EnableVectorOverload = + std::enable_if_t::value && internal::is_lvalue::value, bool>; + +template +using EnableConstVectorOverload = std::enable_if_t::value, bool>; + +public: + +// Public API for 2D matrices/arrays + +// non-const versions + +template +using IndexedViewType = typename IndexedViewSelector::ReturnType; + +template = true> +IndexedViewType operator()(const RowIndices& rowIndices, const ColIndices& colIndices) { + return IndexedViewSelector::run(derived(), rowIndices, colIndices); +} + +template , + EnableOverload = true> +IndexedViewType operator()(const RowType (&rowIndices)[RowSize], const ColIndices& colIndices) { + return IndexedViewSelector::run(derived(), RowIndices{rowIndices}, colIndices); +} + +template , + EnableOverload = true> +IndexedViewType operator()(const RowIndices& rowIndices, const ColType (&colIndices)[ColSize]) { + return IndexedViewSelector::run(derived(), rowIndices, ColIndices{colIndices}); +} + +template , typename ColIndices = Array, + EnableOverload = true> +IndexedViewType operator()(const RowType (&rowIndices)[RowSize], + const ColType (&colIndices)[ColSize]) { + return IndexedViewSelector::run(derived(), RowIndices{rowIndices}, ColIndices{colIndices}); +} + +// const versions + +template +using ConstIndexedViewType = typename IndexedViewSelector::ConstReturnType; + +template = true> +ConstIndexedViewType operator()(const RowIndices& rowIndices, + const ColIndices& colIndices) const { + return IndexedViewSelector::run(derived(), rowIndices, colIndices); +} + +template , + EnableConstOverload = true> +ConstIndexedViewType operator()(const RowType (&rowIndices)[RowSize], + const ColIndices& colIndices) const { + return IndexedViewSelector::run(derived(), RowIndices{rowIndices}, colIndices); +} + +template , + EnableConstOverload = true> +ConstIndexedViewType operator()(const RowIndices& rowIndices, + const ColType (&colIndices)[ColSize]) const { + return IndexedViewSelector::run(derived(), rowIndices, ColIndices{colIndices}); +} + +template , typename ColIndices = Array, + EnableConstOverload = true> +ConstIndexedViewType operator()(const RowType (&rowIndices)[RowSize], + const ColType (&colIndices)[ColSize]) const { + return IndexedViewSelector::run(derived(), RowIndices{rowIndices}, ColIndices{colIndices}); +} + +// Public API for 1D vectors/arrays + +// non-const versions + +template +using VectorIndexedViewType = typename VectorIndexedViewSelector::ReturnType; + +template = true> +VectorIndexedViewType operator()(const Indices& indices) { + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return VectorIndexedViewSelector::run(derived(), indices); +} + +template , + EnableVectorOverload = true> +VectorIndexedViewType operator()(const IndexType (&indices)[Size]) { + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return VectorIndexedViewSelector::run(derived(), Indices{indices}); +} + +// const versions + +template +using ConstVectorIndexedViewType = typename VectorIndexedViewSelector::ConstReturnType; + +template = true> +ConstVectorIndexedViewType operator()(const Indices& indices) const { + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return VectorIndexedViewSelector::run(derived(), indices); +} + +template , + EnableConstVectorOverload = true> +ConstVectorIndexedViewType operator()(const IndexType (&indices)[Size]) const { + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return VectorIndexedViewSelector::run(derived(), Indices{indices}); +} + +#else // EIGEN_PARSED_BY_DOXYGEN + +/** + * \returns a generic submatrix view defined by the rows and columns indexed \a rowIndices and \a colIndices respectively. + * + * Each parameter must either be: + * - An integer indexing a single row or column + * - Eigen::placeholders::all indexing the full set of respective rows or columns in increasing order + * - An ArithmeticSequence as returned by the Eigen::seq and Eigen::seqN functions + * - Any %Eigen's vector/array of integers or expressions + * - Plain C arrays: \c int[N] + * - And more generally any type exposing the following two member functions: + * \code + * operator[]() const; + * size() const; + * \endcode + * where \c stands for any integer type compatible with Eigen::Index (i.e. \c std::ptrdiff_t). + * + * The last statement implies compatibility with \c std::vector, \c std::valarray, \c std::array, many of the Range-v3's ranges, etc. + * + * If the submatrix can be represented using a starting position \c (i,j) and positive sizes \c (rows,columns), then this + * method will returns a Block object after extraction of the relevant information from the passed arguments. This is the case + * when all arguments are either: + * - An integer + * - Eigen::placeholders::all + * - An ArithmeticSequence with compile-time increment strictly equal to 1, as returned by Eigen::seq(a,b), and Eigen::seqN(a,N). + * + * Otherwise a more general IndexedView object will be returned, after conversion of the inputs + * to more suitable types \c RowIndices' and \c ColIndices'. + * + * For 1D vectors and arrays, you better use the operator()(const Indices&) overload, which behave the same way but taking a single parameter. + * + * See also this question and its answer for an example of how to duplicate coefficients. + * + * \sa operator()(const Indices&), class Block, class IndexedView, DenseBase::block(Index,Index,Index,Index) + */ +template +IndexedView_or_Block +operator()(const RowIndices& rowIndices, const ColIndices& colIndices); + +/** This is an overload of operator()(const RowIndices&, const ColIndices&) for 1D vectors or arrays + * + * \only_for_vectors + */ +template +IndexedView_or_VectorBlock +operator()(const Indices& indices); + +#endif // EIGEN_PARSED_BY_DOXYGEN diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/InternalHeaderCheck.inc b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/InternalHeaderCheck.inc new file mode 100644 index 00000000000..ac6821d1864 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/InternalHeaderCheck.inc @@ -0,0 +1,3 @@ +#ifndef EIGEN_CORE_MODULE_H +#error "Please include Eigen/plugins instead of including headers inside the src directory directly." +#endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseBinaryOps.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseBinaryOps.h deleted file mode 100644 index 514d83a71de..00000000000 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseBinaryOps.h +++ /dev/null @@ -1,184 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2008-2009 Gael Guennebaud -// Copyright (C) 2006-2008 Benoit Jacob -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -// This file is a base class plugin containing matrix specifics coefficient wise functions. - -/** \returns an expression of the Schur product (coefficient wise product) of *this and \a other - * - * Example: \include MatrixBase_cwiseProduct.cpp - * Output: \verbinclude MatrixBase_cwiseProduct.out - * - * \sa class CwiseBinaryOp, cwiseAbs2 - */ -template -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,product) -cwiseProduct(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const -{ - return EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,product)(derived(), other.derived()); -} - -/** \returns an expression of the coefficient-wise == operator of *this and \a other - * - * \warning this performs an exact comparison, which is generally a bad idea with floating-point types. - * In order to check for equality between two vectors or matrices with floating-point coefficients, it is - * generally a far better idea to use a fuzzy comparison as provided by isApprox() and - * isMuchSmallerThan(). - * - * Example: \include MatrixBase_cwiseEqual.cpp - * Output: \verbinclude MatrixBase_cwiseEqual.out - * - * \sa cwiseNotEqual(), isApprox(), isMuchSmallerThan() - */ -template -EIGEN_DEVICE_FUNC -inline const CwiseBinaryOp, const Derived, const OtherDerived> -cwiseEqual(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const -{ - return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), other.derived()); -} - -/** \returns an expression of the coefficient-wise != operator of *this and \a other - * - * \warning this performs an exact comparison, which is generally a bad idea with floating-point types. - * In order to check for equality between two vectors or matrices with floating-point coefficients, it is - * generally a far better idea to use a fuzzy comparison as provided by isApprox() and - * isMuchSmallerThan(). - * - * Example: \include MatrixBase_cwiseNotEqual.cpp - * Output: \verbinclude MatrixBase_cwiseNotEqual.out - * - * \sa cwiseEqual(), isApprox(), isMuchSmallerThan() - */ -template -EIGEN_DEVICE_FUNC -inline const CwiseBinaryOp, const Derived, const OtherDerived> -cwiseNotEqual(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const -{ - return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), other.derived()); -} - -/** \returns an expression of the coefficient-wise min of *this and \a other - * - * Example: \include MatrixBase_cwiseMin.cpp - * Output: \verbinclude MatrixBase_cwiseMin.out - * - * \sa class CwiseBinaryOp, max() - */ -template -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> -cwiseMin(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const -{ - return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), other.derived()); -} - -template -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> -cwiseMin(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const -{ - return cwiseMin(other); -} - -/** \returns an expression of the coefficient-wise min of *this and scalar \a other - * - * \sa class CwiseBinaryOp, min() - */ -template -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const ConstantReturnType> -cwiseMin(const Scalar &other) const -{ - return cwiseMin(Derived::Constant(rows(), cols(), other)); -} - -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const ConstantReturnType> -cwiseMin(const Scalar &other) const -{ - return cwiseMin(Derived::Constant(rows(), cols(), other)); -} - -/** \returns an expression of the coefficient-wise max of *this and \a other - * - * Example: \include MatrixBase_cwiseMax.cpp - * Output: \verbinclude MatrixBase_cwiseMax.out - * - * \sa class CwiseBinaryOp, min() - */ -template -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> -cwiseMax(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const -{ - return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), other.derived()); -} - -template -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> -cwiseMax(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const -{ - return cwiseMax(other); -} - -/** \returns an expression of the coefficient-wise max of *this and scalar \a other - * - * \sa class CwiseBinaryOp, min() - */ -template -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const ConstantReturnType> -cwiseMax(const Scalar &other) const -{ - return cwiseMax(Derived::Constant(rows(), cols(), other)); -} - -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const ConstantReturnType> -cwiseMax(const Scalar &other) const -{ - return cwiseMax(Derived::Constant(rows(), cols(), other)); -} - - -/** \returns an expression of the coefficient-wise quotient of *this and \a other - * - * Example: \include MatrixBase_cwiseQuotient.cpp - * Output: \verbinclude MatrixBase_cwiseQuotient.out - * - * \sa class CwiseBinaryOp, cwiseProduct(), cwiseInverse() - */ -template -EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> -cwiseQuotient(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const -{ - return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), other.derived()); -} - -typedef CwiseBinaryOp, const Derived, const ConstantReturnType> CwiseScalarEqualReturnType; - -/** \returns an expression of the coefficient-wise == operator of \c *this and a scalar \a s - * - * \warning this performs an exact comparison, which is generally a bad idea with floating-point types. - * In order to check for equality between two vectors or matrices with floating-point coefficients, it is - * generally a far better idea to use a fuzzy comparison as provided by isApprox() and - * isMuchSmallerThan(). - * - * \sa cwiseEqual(const MatrixBase &) const - */ -EIGEN_DEVICE_FUNC -inline const CwiseScalarEqualReturnType -cwiseEqual(const Scalar& s) const -{ - return CwiseScalarEqualReturnType(derived(), Derived::Constant(rows(), cols(), s), internal::scalar_cmp_op()); -} diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseBinaryOps.inc b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseBinaryOps.inc new file mode 100644 index 00000000000..fd5941452d8 --- /dev/null +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseBinaryOps.inc @@ -0,0 +1,307 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2008-2009 Gael Guennebaud +// Copyright (C) 2006-2008 Benoit Jacob +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// This file is a base class plugin containing matrix specifics coefficient wise functions. + +/** \returns an expression of the Schur product (coefficient wise product) of *this and \a other + * + * Example: \include MatrixBase_cwiseProduct.cpp + * Output: \verbinclude MatrixBase_cwiseProduct.out + * + * \sa class CwiseBinaryOp, cwiseAbs2 + */ +template +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE const EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,product) +cwiseProduct(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const +{ + return EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,product)(derived(), other.derived()); +} + +template using CwiseBinaryEqualReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; +template using CwiseBinaryNotEqualReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; +template using CwiseBinaryLessReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; +template using CwiseBinaryGreaterReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; +template using CwiseBinaryLessOrEqualReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; +template using CwiseBinaryGreaterOrEqualReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; + +/** \returns an expression of the coefficient-wise == operator of *this and \a other + * + * \warning this performs an exact comparison, which is generally a bad idea with floating-point types. + * In order to check for equality between two vectors or matrices with floating-point coefficients, it is + * generally a far better idea to use a fuzzy comparison as provided by isApprox() and + * isMuchSmallerThan(). + * + * Example: \include MatrixBase_cwiseEqual.cpp + * Output: \verbinclude MatrixBase_cwiseEqual.out + * + * \sa cwiseNotEqual(), isApprox(), isMuchSmallerThan() + */ +template +EIGEN_DEVICE_FUNC +inline const CwiseBinaryEqualReturnType +cwiseEqual(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const +{ + return CwiseBinaryEqualReturnType(derived(), other.derived()); +} + +/** \returns an expression of the coefficient-wise != operator of *this and \a other + * + * \warning this performs an exact comparison, which is generally a bad idea with floating-point types. + * In order to check for equality between two vectors or matrices with floating-point coefficients, it is + * generally a far better idea to use a fuzzy comparison as provided by isApprox() and + * isMuchSmallerThan(). + * + * Example: \include MatrixBase_cwiseNotEqual.cpp + * Output: \verbinclude MatrixBase_cwiseNotEqual.out + * + * \sa cwiseEqual(), isApprox(), isMuchSmallerThan() + */ +template +EIGEN_DEVICE_FUNC +inline const CwiseBinaryNotEqualReturnType +cwiseNotEqual(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const +{ + return CwiseBinaryNotEqualReturnType(derived(), other.derived()); +} + +/** \returns an expression of the coefficient-wise < operator of *this and \a other */ +template +EIGEN_DEVICE_FUNC +inline const CwiseBinaryLessReturnType +cwiseLess(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const +{ + return CwiseBinaryLessReturnType(derived(), other.derived()); +} + +/** \returns an expression of the coefficient-wise > operator of *this and \a other */ +template +EIGEN_DEVICE_FUNC +inline const CwiseBinaryGreaterReturnType +cwiseGreater(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const +{ + return CwiseBinaryGreaterReturnType(derived(), other.derived()); +} + +/** \returns an expression of the coefficient-wise <= operator of *this and \a other */ +template +EIGEN_DEVICE_FUNC +inline const CwiseBinaryLessOrEqualReturnType +cwiseLessOrEqual(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const +{ + return CwiseBinaryLessOrEqualReturnType(derived(), other.derived()); +} + +/** \returns an expression of the coefficient-wise >= operator of *this and \a other */ +template +EIGEN_DEVICE_FUNC +inline const CwiseBinaryGreaterOrEqualReturnType +cwiseGreaterOrEqual(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const +{ + return CwiseBinaryGreaterOrEqualReturnType(derived(), other.derived()); +} + +/** \returns an expression of the coefficient-wise min of *this and \a other + * + * Example: \include MatrixBase_cwiseMin.cpp + * Output: \verbinclude MatrixBase_cwiseMin.out + * + * \sa class CwiseBinaryOp, max() + */ +template +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> +cwiseMin(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const +{ + return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), other.derived()); +} + +/** \returns an expression of the coefficient-wise min of *this and scalar \a other + * + * \sa class CwiseBinaryOp, min() + */ +template +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const ConstantReturnType> +cwiseMin(const Scalar &other) const +{ + return cwiseMin(Derived::Constant(rows(), cols(), other)); +} + +/** \returns an expression of the coefficient-wise max of *this and \a other + * + * Example: \include MatrixBase_cwiseMax.cpp + * Output: \verbinclude MatrixBase_cwiseMax.out + * + * \sa class CwiseBinaryOp, min() + */ +template +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> +cwiseMax(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const +{ + return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), other.derived()); +} + +/** \returns an expression of the coefficient-wise max of *this and scalar \a other + * + * \sa class CwiseBinaryOp, min() + */ +template +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const ConstantReturnType> +cwiseMax(const Scalar &other) const +{ + return cwiseMax(Derived::Constant(rows(), cols(), other)); +} + + +/** \returns an expression of the coefficient-wise quotient of *this and \a other + * + * Example: \include MatrixBase_cwiseQuotient.cpp + * Output: \verbinclude MatrixBase_cwiseQuotient.out + * + * \sa class CwiseBinaryOp, cwiseProduct(), cwiseInverse() + */ +template +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE const CwiseBinaryOp, const Derived, const OtherDerived> +cwiseQuotient(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const +{ + return CwiseBinaryOp, const Derived, const OtherDerived>(derived(), other.derived()); +} + +using CwiseScalarEqualReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; +using CwiseScalarNotEqualReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; +using CwiseScalarLessReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; +using CwiseScalarGreaterReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; +using CwiseScalarLessOrEqualReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; +using CwiseScalarGreaterOrEqualReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; + +/** \returns an expression of the coefficient-wise == operator of \c *this and a scalar \a s + * + * \warning this performs an exact comparison, which is generally a bad idea with floating-point types. + * In order to check for equality between two vectors or matrices with floating-point coefficients, it is + * generally a far better idea to use a fuzzy comparison as provided by isApprox() and + * isMuchSmallerThan(). + * + * \sa cwiseEqual(const MatrixBase &) const + */ +EIGEN_DEVICE_FUNC +inline const CwiseScalarEqualReturnType +cwiseEqual(const Scalar& s) const +{ + return CwiseScalarEqualReturnType(derived(), Derived::Constant(rows(), cols(), s)); +} + + +/** \returns an expression of the coefficient-wise == operator of \c *this and a scalar \a s + * + * \warning this performs an exact comparison, which is generally a bad idea with floating-point types. + * In order to check for equality between two vectors or matrices with floating-point coefficients, it is + * generally a far better idea to use a fuzzy comparison as provided by isApprox() and + * isMuchSmallerThan(). + * + * \sa cwiseEqual(const MatrixBase &) const + */ +EIGEN_DEVICE_FUNC +inline const CwiseScalarNotEqualReturnType +cwiseNotEqual(const Scalar& s) const +{ + return CwiseScalarNotEqualReturnType(derived(), Derived::Constant(rows(), cols(), s)); +} + +/** \returns an expression of the coefficient-wise < operator of \c *this and a scalar \a s */ +EIGEN_DEVICE_FUNC +inline const CwiseScalarLessReturnType +cwiseLess(const Scalar& s) const +{ + return CwiseScalarLessReturnType(derived(), Derived::Constant(rows(), cols(), s)); +} + +/** \returns an expression of the coefficient-wise > operator of \c *this and a scalar \a s */ +EIGEN_DEVICE_FUNC +inline const CwiseScalarGreaterReturnType +cwiseGreater(const Scalar& s) const +{ + return CwiseScalarGreaterReturnType(derived(), Derived::Constant(rows(), cols(), s)); +} + +/** \returns an expression of the coefficient-wise <= operator of \c *this and a scalar \a s */ +EIGEN_DEVICE_FUNC +inline const CwiseScalarLessOrEqualReturnType +cwiseLessOrEqual(const Scalar& s) const +{ + return CwiseScalarLessOrEqualReturnType(derived(), Derived::Constant(rows(), cols(), s)); +} + +/** \returns an expression of the coefficient-wise >= operator of \c *this and a scalar \a s */ +EIGEN_DEVICE_FUNC +inline const CwiseScalarGreaterOrEqualReturnType +cwiseGreaterOrEqual(const Scalar& s) const +{ + return CwiseScalarGreaterOrEqualReturnType(derived(), Derived::Constant(rows(), cols(), s)); +} + +template using CwiseBinaryTypedEqualReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; +template using CwiseBinaryTypedNotEqualReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; +template using CwiseBinaryTypedLessReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; +template using CwiseBinaryTypedGreaterReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; +template using CwiseBinaryTypedLessOrEqualReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; +template using CwiseBinaryTypedGreaterOrEqualReturnType = CwiseBinaryOp, const Derived, const OtherDerived>; + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryTypedEqualReturnType +cwiseTypedEqual(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const { return CwiseBinaryTypedEqualReturnType(derived(), other.derived()); } + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryTypedNotEqualReturnType +cwiseTypedNotEqual(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const { return CwiseBinaryTypedNotEqualReturnType(derived(), other.derived()); } + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryTypedLessReturnType +cwiseTypedLess(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const { return CwiseBinaryTypedLessReturnType(derived(), other.derived()); } + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryTypedGreaterReturnType +cwiseTypedGreater(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const { return CwiseBinaryTypedGreaterReturnType(derived(), other.derived()); } + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryTypedLessOrEqualReturnType +cwiseTypedLessOrEqual(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const { return CwiseBinaryTypedLessOrEqualReturnType(derived(), other.derived()); } + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryTypedGreaterOrEqualReturnType +cwiseTypedGreaterOrEqual(const EIGEN_CURRENT_STORAGE_BASE_CLASS& other) const { return CwiseBinaryTypedGreaterOrEqualReturnType(derived(), other.derived()); } + +using CwiseScalarTypedEqualReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; +using CwiseScalarTypedNotEqualReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; +using CwiseScalarTypedLessReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; +using CwiseScalarTypedGreaterReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; +using CwiseScalarTypedLessOrEqualReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; +using CwiseScalarTypedGreaterOrEqualReturnType = CwiseBinaryOp, const Derived, const ConstantReturnType>; + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseScalarTypedEqualReturnType +cwiseTypedEqual(const Scalar& s) const { return CwiseScalarTypedEqualReturnType(derived(), ConstantReturnType(rows(), cols(), s)); } + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseScalarTypedNotEqualReturnType +cwiseTypedNotEqual(const Scalar& s) const { return CwiseScalarTypedNotEqualReturnType(derived(), ConstantReturnType(rows(), cols(), s)); } + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseScalarTypedLessReturnType +cwiseTypedLess(const Scalar& s) const { return CwiseScalarTypedLessReturnType(derived(), ConstantReturnType(rows(), cols(), s)); } + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseScalarTypedGreaterReturnType +cwiseTypedGreater(const Scalar& s) const { return CwiseScalarTypedGreaterReturnType(derived(), ConstantReturnType(rows(), cols(), s)); } + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseScalarTypedLessOrEqualReturnType +cwiseTypedLessOrEqual(const Scalar& s) const { return CwiseScalarTypedLessOrEqualReturnType(derived(), ConstantReturnType(rows(), cols(), s)); } + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseScalarTypedGreaterOrEqualReturnType +cwiseTypedGreaterOrEqual(const Scalar& s) const { return CwiseScalarTypedGreaterOrEqualReturnType(derived(), ConstantReturnType(rows(), cols(), s)); } diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseUnaryOps.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseUnaryOps.inc similarity index 72% rename from Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseUnaryOps.h rename to Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseUnaryOps.inc index 0514d8f78b5..cb65e171f2d 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseUnaryOps.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/MatrixCwiseUnaryOps.inc @@ -15,7 +15,9 @@ typedef CwiseUnaryOp, const Derived> CwiseAbsReturnType; typedef CwiseUnaryOp, const Derived> CwiseAbs2ReturnType; typedef CwiseUnaryOp, const Derived> CwiseArgReturnType; +typedef CwiseUnaryOp, const Derived> CwiseCArgReturnType; typedef CwiseUnaryOp, const Derived> CwiseSqrtReturnType; +typedef CwiseUnaryOp, const Derived> CwiseCbrtReturnType; typedef CwiseUnaryOp, const Derived> CwiseSignReturnType; typedef CwiseUnaryOp, const Derived> CwiseInverseReturnType; @@ -52,12 +54,25 @@ cwiseAbs2() const { return CwiseAbs2ReturnType(derived()); } /// EIGEN_DOC_UNARY_ADDONS(cwiseSqrt,square-root) /// -/// \sa cwisePow(), cwiseSquare() +/// \sa cwisePow(), cwiseSquare(), cwiseCbrt() /// EIGEN_DEVICE_FUNC inline const CwiseSqrtReturnType cwiseSqrt() const { return CwiseSqrtReturnType(derived()); } +/// \returns an expression of the coefficient-wise cube root of *this. +/// +/// Example: \include MatrixBase_cwiseCbrt.cpp +/// Output: \verbinclude MatrixBase_cwiseCbrt.out +/// +EIGEN_DOC_UNARY_ADDONS(cwiseCbrt,cube-root) +/// +/// \sa cwiseSqrt(), cwiseSquare(), cwisePow() +/// +EIGEN_DEVICE_FUNC +inline const CwiseCbrtReturnType +cwiseCbrt() const { return CwiseSCbrtReturnType(derived()); } + /// \returns an expression of the coefficient-wise signum of *this. /// /// Example: \include MatrixBase_cwiseSign.cpp @@ -93,3 +108,17 @@ EIGEN_DOC_UNARY_ADDONS(cwiseArg,arg) EIGEN_DEVICE_FUNC inline const CwiseArgReturnType cwiseArg() const { return CwiseArgReturnType(derived()); } + +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE const CwiseCArgReturnType +cwiseCArg() const { return CwiseCArgReturnType(derived()); } + +template +using CwisePowReturnType = + std::enable_if_t::Real>::value, + CwiseUnaryOp, const Derived>>; + +template +EIGEN_DEVICE_FUNC inline const CwisePowReturnType cwisePow(const ScalarExponent& exponent) const { + return CwisePowReturnType(derived(), internal::scalar_unary_pow_op(exponent)); +} diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ReshapedMethods.h b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ReshapedMethods.inc similarity index 95% rename from Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ReshapedMethods.h rename to Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ReshapedMethods.inc index 482a6b045ef..db2b395bf8a 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ReshapedMethods.h +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/Eigen/src/plugins/ReshapedMethods.inc @@ -105,13 +105,13 @@ EIGEN_DEVICE_FUNC inline Reshaped::value, internal::get_compiletime_reshape_size::value, - internal::get_compiletime_reshape_order::value> + internal::get_compiletime_reshape_order(Flags, Order)> reshaped(NRowsType nRows, NColsType nCols) EIGEN_RESHAPED_METHOD_CONST { return Reshaped::value, internal::get_compiletime_reshape_size::value, - internal::get_compiletime_reshape_order::value> + internal::get_compiletime_reshape_order(Flags, Order)> (derived(), internal::get_runtime_reshape_size(nRows,internal::get_runtime_value(nCols),size()), internal::get_runtime_reshape_size(nCols,internal::get_runtime_value(nRows),size())); @@ -129,12 +129,12 @@ reshaped() EIGEN_RESHAPED_METHOD_CONST template EIGEN_DEVICE_FUNC inline Reshaped::value> + internal::get_compiletime_reshape_order(Flags, Order)> reshaped() EIGEN_RESHAPED_METHOD_CONST { EIGEN_STATIC_ASSERT(Order==RowMajor || Order==ColMajor || Order==AutoOrder, INVALID_TEMPLATE_PARAMETER); return Reshaped::value> + internal::get_compiletime_reshape_order(Flags, Order)> (derived(), size(), 1); } @@ -142,7 +142,7 @@ reshaped() EIGEN_RESHAPED_METHOD_CONST #ifndef EIGEN_RESHAPED_METHOD_2ND_PASS #define EIGEN_RESHAPED_METHOD_2ND_PASS -#include "ReshapedMethods.h" +#include "ReshapedMethods.inc" #undef EIGEN_RESHAPED_METHOD_2ND_PASS #endif diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/README.kitware.md b/Modules/ThirdParty/Eigen3/src/itkeigen/README.kitware.md index e6c5def96c9..f8bf67c960b 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/README.kitware.md +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/README.kitware.md @@ -7,6 +7,5 @@ This branch contains changes required to embed Eigen into ITK. This process. * Add `.gitattributes` to avoid failures in ITK's content checks. -* Remove executable permissions from include files. * Integrate the CMake build with ITK's module system. diff --git a/Modules/ThirdParty/Eigen3/src/itkeigen/cmake/Eigen3Config.cmake.in b/Modules/ThirdParty/Eigen3/src/itkeigen/cmake/Eigen3Config.cmake.in index b2a35fe3f1f..20acbdfcf12 100644 --- a/Modules/ThirdParty/Eigen3/src/itkeigen/cmake/Eigen3Config.cmake.in +++ b/Modules/ThirdParty/Eigen3/src/itkeigen/cmake/Eigen3Config.cmake.in @@ -6,18 +6,3 @@ if (NOT TARGET eigen) include ("${CMAKE_CURRENT_LIST_DIR}/@EIGEN3_TARGETS_FILE@") endif () - -# Legacy variables, do *not* use. May be removed in the future. - -# set (EIGEN3_FOUND 1) -# set (EIGEN3_USE_FILE "${CMAKE_CURRENT_LIST_DIR}/UseEigen3.cmake") -# -# set (EIGEN3_DEFINITIONS "@EIGEN_DEFINITIONS@") -# set (EIGEN3_INCLUDE_DIR "@PACKAGE_EIGEN_INCLUDE_DIR@") -# set (EIGEN3_INCLUDE_DIRS "@PACKAGE_EIGEN_INCLUDE_DIR@") -# set (EIGEN3_ROOT_DIR "@PACKAGE_EIGEN_ROOT_DIR@") -# -# set (EIGEN3_VERSION_STRING "@EIGEN_VERSION_STRING@") -# set (EIGEN3_VERSION_MAJOR "@EIGEN_VERSION_MAJOR@") -# set (EIGEN3_VERSION_MINOR "@EIGEN_VERSION_MINOR@") -# set (EIGEN3_VERSION_PATCH "@EIGEN_VERSION_PATCH@")