Some basic fixes to build system, clang IR generation, suppress endogenous LLVM warnings with MSVC 2017

Lots of visibility fixes. Add CountTrailingZeros compatibility bit-util function

More Windows fixes, date/time compatibility

More fixes, make sure gandiva-precompiled tests are part of gandiva-tests label

More date/time fixes

Linting fixes

Fix date_time_test.cc

Restore accidental change to arrow/CMakeLists.txt

Fix pragma

Remove debugging output

Add missing include for date.h on non Windows

It's needed on all platforms.

Compile again after rebasing. Make DecimalTypeUtil internal

Code review comments

IR compilation fixes since Arrow symbols are being compiled to bitcode

Don't use static linking in any gandiva unit tests

Linux fixes

Change-Id: Id40cf6a034bffd20433445857245b96f9ed916f0

Move ARROW_BUILD_GANDIVA to global scope in appveyor.yml

Change-Id: I678c0933912d8695b264b361070f29a992b1f255

Fixes after rebase

ARROW-3846: minor fixes (#5)

* ARROW-3846: minor fixes

* ARROW-3846: comment out new test

Author: wesm

appveyor.yml

@@ -47,6 +47,7 @@ matrix:
 environment:
   global:
     USE_CLCACHE: true
+    ARROW_BUILD_GANDIVA: "OFF"
     PYTHON: "3.6"
     ARCH: "64"
 

ci/appveyor-cpp-build.bat

@@ -104,6 +104,13 @@ conda create -n arrow -q -y -c conda-forge ^
 
 call activate arrow
 
+set ARROW_LLVM_VERSION=6.0.1
+
+if "%ARROW_BUILD_GANDIVA%" == "ON" (
+  @rem Install llvmdev in the toolchain if building gandiva.dll
+  conda install -q -y llvmdev=%ARROW_LLVM_VERSION% || exit /B
+)
+
 @rem Use Boost from Anaconda
 set BOOST_ROOT=%CONDA_PREFIX%\Library
 set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib

ci/cpp-msvc-build-main.bat

@@ -55,6 +55,7 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^
       -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^
       -DARROW_CXXFLAGS="%ARROW_CXXFLAGS%" ^
       -DCMAKE_CXX_FLAGS_RELEASE="/MD %CMAKE_CXX_FLAGS_RELEASE%" ^
+      -DARROW_GANDIVA=%ARROW_BUILD_GANDIVA% ^
       -DARROW_PARQUET=ON ^
       -DARROW_PYTHON=ON ^
       ..  || exit /B

cpp/CMakeLists.txt

@@ -309,8 +309,12 @@ Note that this requires linking Boost statically"
 
     set(BROTLI_MSVC_STATIC_LIB_SUFFIX "-static" CACHE STRING
       "Brotli static lib suffix used on Windows with MSVC (default -static)")
+    set(PROTOBUF_MSVC_STATIC_LIB_SUFFIX "" CACHE STRING
+      "Protobuf static lib suffix used on Windows with MSVC (default is empty string)")
+    set(RE2_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
+      "re2 static lib suffix used on Windows with MSVC (default is _static)")
     set(SNAPPY_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
-      "Snappy static lib suffix used on Windows with MSVC (default is empty string)")
+      "Snappy static lib suffix used on Windows with MSVC (default is _static)")
     set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
       "Lz4 static lib suffix used on Windows with MSVC (default _static)")
     set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING

cpp/cmake_modules/FindProtobuf.cmake

@@ -44,12 +44,12 @@ if (EXISTS "${_protobuf_path}/lib/${CMAKE_LIBRARY_ARCHITECTURE}")
   set (lib_dirs "lib/${CMAKE_LIBRARY_ARCHITECTURE}" ${lib_dirs})
 endif ()
 
-find_library (PROTOBUF_LIBRARY NAMES protobuf PATHS
+find_library (PROTOBUF_LIBRARY NAMES protobuf libprotobuf PATHS
   ${_protobuf_path}
   NO_DEFAULT_PATH
   PATH_SUFFIXES ${lib_dirs})
 
-find_library (PROTOC_LIBRARY NAMES protoc PATHS
+find_library (PROTOC_LIBRARY NAMES protoc libprotoc PATHS
   ${_protobuf_path}
   NO_DEFAULT_PATH
   PATH_SUFFIXES ${lib_dirs})
@@ -66,7 +66,7 @@ if (PROTOBUF_INCLUDE_DIR AND PROTOBUF_LIBRARY AND PROTOC_LIBRARY AND PROTOBUF_EX
   get_filename_component (PROTOBUF_LIBS ${PROTOBUF_LIBRARY} PATH)
   set (PROTOBUF_LIB_NAME protobuf)
   set (PROTOC_LIB_NAME protoc)
-  set (PROTOBUF_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOBUF_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set (PROTOBUF_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOBUF_LIB_NAME}${PROTOBUF_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
   set (PROTOC_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
 else ()
   set (PROTOBUF_FOUND FALSE)
@@ -81,7 +81,7 @@ if (PROTOBUF_FOUND)
   message (STATUS "Found the Protoc executable: ${PROTOBUF_EXECUTABLE}")
 else()
   if (_protobuf_path)
-    set (PROTOBUF_ERR_MSG "Could not find Protobuf. Looked in ${_protobuf_path}.")
+    set (PROTOBUF_ERR_MSG "Could not find Protobuf. Looked in ${_protobuf_path}")
   else ()
     set (PROTOBUF_ERR_MSG "Could not find Protobuf in system search paths.")
   endif()
@@ -100,4 +100,3 @@ mark_as_advanced (
   PROTOBUF_STATIC_LIB
   PROTOC_STATIC_LIB
 )
-

cpp/cmake_modules/FindRE2.cmake

@@ -45,14 +45,18 @@ if (EXISTS "${_re2_path}/lib/${CMAKE_LIBRARY_ARCHITECTURE}")
   set (lib_dirs "lib/${CMAKE_LIBRARY_ARCHITECTURE}" ${lib_dirs})
 endif ()
 
-find_library(RE2_STATIC_LIB NAMES libre2${CMAKE_STATIC_LIBRARY_SUFFIX}
+set(RE2_LIB_NAME re2)
+set(RE2_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}${RE2_LIB_NAME}${RE2_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+set(RE2_SHARED_LIB_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${RE2_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+
+find_library(RE2_STATIC_LIB NAMES ${RE2_STATIC_LIB_NAME}
   PATHS ${_re2_path}
         NO_DEFAULT_PATH
   PATH_SUFFIXES ${lib_dirs}
   DOC   "Google's re2 regex static library"
 )
 
-find_library(RE2_SHARED_LIB NAMES libre2${CMAKE_SHARED_LIBRARY_SUFFIX}
+find_library(RE2_SHARED_LIB NAMES ${RE2_SHARED_LIB_NAME}
   PATHS ${_re2_path}
         NO_DEFAULT_PATH
   PATH_SUFFIXES ${lib_dirs}

cpp/src/arrow/util/bit-util-test.cc

@@ -788,6 +788,30 @@ TEST(BitUtil, CountLeadingZeros) {
   EXPECT_EQ(BitUtil::CountLeadingZeros(U64(ULLONG_MAX)), 0);
 }
 
+TEST(BitUtil, CountTrailingZeros) {
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U32(0)), 32);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U32(1) << 31), 31);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U32(1) << 30), 30);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U32(1) << 29), 29);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U32(1) << 28), 28);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U32(8)), 3);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U32(4)), 2);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U32(2)), 1);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U32(1)), 0);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U32(ULONG_MAX)), 0);
+
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U64(0)), 64);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U64(1) << 63), 63);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U64(1) << 62), 62);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U64(1) << 61), 61);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U64(1) << 60), 60);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U64(8)), 3);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U64(4)), 2);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U64(2)), 1);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U64(1)), 0);
+  EXPECT_EQ(BitUtil::CountTrailingZeros(U64(ULLONG_MAX)), 0);
+}
+
 #undef U32
 #undef U64
 

cpp/src/arrow/util/bit-util.h

@@ -45,6 +45,7 @@
 #if defined(_MSC_VER)
 #include <intrin.h>
 #pragma intrinsic(_BitScanReverse)
+#pragma intrinsic(_BitScanForward)
 #define ARROW_BYTE_SWAP64 _byteswap_uint64
 #define ARROW_BYTE_SWAP32 _byteswap_ulong
 #else
@@ -182,6 +183,56 @@ static inline int CountLeadingZeros(uint64_t value) {
 #endif
 }
 
+static inline int CountTrailingZeros(uint32_t value) {
+#if defined(__clang__) || defined(__GNUC__)
+  if (value == 0) return 32;
+  return static_cast<int>(__builtin_ctzl(value));
+#elif defined(_MSC_VER)
+  unsigned long index;  // NOLINT
+  if (_BitScanForward(&index, value)) {
+    return static_cast<int>(index);
+  } else {
+    return 32;
+  }
+#else
+  int bitpos = 0;
+  if (value) {
+    while (value & 1 == 0) {
+      value >>= 1;
+      ++bitpos;
+    }
+  } else {
+    bitpos = 32;
+  }
+  return bitpos;
+#endif
+}
+
+static inline int CountTrailingZeros(uint64_t value) {
+#if defined(__clang__) || defined(__GNUC__)
+  if (value == 0) return 64;
+  return static_cast<int>(__builtin_ctzll(value));
+#elif defined(_MSC_VER)
+  unsigned long index;  // NOLINT
+  if (_BitScanForward64(&index, value)) {
+    return static_cast<int>(index);
+  } else {
+    return 64;
+  }
+#else
+  int bitpos = 0;
+  if (value) {
+    while (value & 1 == 0) {
+      value >>= 1;
+      ++bitpos;
+    }
+  } else {
+    bitpos = 64;
+  }
+  return bitpos;
+#endif
+}
+
 // Returns the minimum number of bits needed to represent an unsigned value
 static inline int NumRequiredBits(uint64_t x) { return 64 - CountLeadingZeros(x); }
 

cpp/src/gandiva/CMakeLists.txt

@@ -32,8 +32,8 @@ set(GANDIVA_BC_INSTALL_DIR
   ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/gandiva)
 
 set(GANDIVA_BC_FILE_NAME irhelpers.bc)
-set(GANDIVA_BC_INSTALL_PATH ${GANDIVA_BC_INSTALL_DIR}/${GANDIVA_BC_FILE_NAME})
-set(GANDIVA_BC_OUTPUT_PATH ${CMAKE_CURRENT_BINARY_DIR}/${GANDIVA_BC_FILE_NAME})
+set(GANDIVA_BC_INSTALL_PATH "${GANDIVA_BC_INSTALL_DIR}/${GANDIVA_BC_FILE_NAME}")
+set(GANDIVA_BC_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${GANDIVA_BC_FILE_NAME}")
 install(FILES
   ${GANDIVA_BC_OUTPUT_PATH}
   DESTINATION ${GANDIVA_BC_INSTALL_DIR})
@@ -95,16 +95,35 @@ if (ARROW_GANDIVA_STATIC_LIBSTDCPP
     -static-libgcc)
 endif()
 
+# if (MSVC)
+#   # Symbols that need to be made public in gandiva.dll for LLVM IR
+#   # compilation
+#   set(MSVC_SYMBOL_EXPORTS _Init_thread_header)
+#   foreach(SYMBOL ${MSVC_SYMBOL_EXPORTS})
+#     set(GANDIVA_SHARED_LINK_FLAGS "${GANDIVA_SHARED_LINK_FLAGS} /EXPORT:${SYMBOL}")
+#   endforeach()
+# endif()
+
 ADD_ARROW_LIB(gandiva
   SOURCES ${SRC_FILES}
   OUTPUTS GANDIVA_LIBRARIES
   DEPENDENCIES arrow_dependencies precompiled
   EXTRA_INCLUDES
   $<TARGET_PROPERTY:LLVM::LLVM_INTERFACE,INTERFACE_INCLUDE_DIRECTORIES>
+  SHARED_LINK_FLAGS ${GANDIVA_SHARED_LINK_FLAGS}
   SHARED_LINK_LIBS arrow_shared
   SHARED_PRIVATE_LINK_LIBS ${GANDIVA_SHARED_PRIVATE_LINK_LIBS}
   STATIC_LINK_LIBS ${GANDIVA_STATIC_LINK_LIBS})
 
+foreach(LIB_TARGET ${GANDIVA_LIBRARIES})
+  target_compile_definitions(${LIB_TARGET}
+    PRIVATE GANDIVA_EXPORTING)
+endforeach()
+
+if (ARROW_BUILD_STATIC AND WIN32)
+  target_compile_definitions(gandiva_static PUBLIC GANDIVA_STATIC)
+endif()
+
 add_dependencies(gandiva ${GANDIVA_LIBRARIES})
 
 # install for gandiva

cpp/src/gandiva/annotator.h

@@ -27,12 +27,13 @@
 #include "gandiva/eval_batch.h"
 #include "gandiva/gandiva_aliases.h"
 #include "gandiva/logging.h"
+#include "gandiva/visibility.h"
 
 namespace gandiva {
 
 /// \brief annotate the arrow fields in an expression, and use that
 /// to convert the incoming arrow-format row batch to an EvalBatch.
-class Annotator {
+class GANDIVA_EXPORT Annotator {
  public:
   Annotator() : buffer_count_(0), local_bitmap_count_(0) {}