[DO-NOT-MERGE][SPARK-23710] Upgrade built-in Hive to 2.3.4(Without hive-thriftserver)

core/pom.xml

@@ -391,16 +391,42 @@
      placed in the "provided" scope, rather than the "compile" scope, and NoClassDefFoundError exceptions are handled
      when the user has not explicitly compiled with the Hive module.
     -->
+    <dependency>
+      <groupId>${hive.group}</groupId>
+      <artifactId>hive-common</artifactId>
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>${hive.group}</groupId>
       <artifactId>hive-exec</artifactId>
+      <classifier>core</classifier>
       <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>${hive.group}</groupId>
       <artifactId>hive-metastore</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>${hive.group}</groupId>
+      <artifactId>hive-serde</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-storage-api</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive.shims</groupId>
+      <artifactId>hive-shims-common</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive.shims</groupId>
+      <artifactId>hive-shims-0.23</artifactId>
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.thrift</groupId>
       <artifactId>libthrift</artifactId>

dev/deps/spark-deps-hadoop-2.7

@@ -1,14 +1,11 @@
-JavaEWAH-0.3.2.jar
 RoaringBitmap-0.5.11.jar
 ST4-4.0.4.jar
 activation-1.1.1.jar
 aircompressor-0.10.jar
-antlr-2.7.7.jar
-antlr-runtime-3.4.jar
+antlr-runtime-3.5.2.jar
 antlr4-runtime-4.7.1.jar
 aopalliance-1.0.jar
 aopalliance-repackaged-2.4.0-b34.jar
-apache-log4j-extras-1.2.17.jar
 apacheds-i18n-2.0.0-M15.jar
 apacheds-kerberos-codec-2.0.0-M15.jar
 api-asn1-api-1.0.0-M20.jar
@@ -18,15 +15,16 @@ arrow-format-0.10.0.jar
 arrow-memory-0.10.0.jar
 arrow-vector-0.10.0.jar
 automaton-1.11-8.jar
+avatica-core-1.13.0.jar
 avro-1.8.2.jar
 avro-ipc-1.8.2.jar
 avro-mapred-1.8.2-hadoop2.jar
 bonecp-0.8.0.RELEASE.jar
 breeze-macros_2.12-0.13.2.jar
 breeze_2.12-0.13.2.jar
-calcite-avatica-1.2.0-incubating.jar
-calcite-core-1.2.0-incubating.jar
-calcite-linq4j-1.2.0-incubating.jar
+calcite-core-1.10.0.jar
+calcite-druid-1.10.0.jar
+calcite-linq4j-1.10.0.jar
 chill-java-0.9.3.jar
 chill_2.12-0.9.3.jar
 commons-beanutils-1.7.0.jar
@@ -44,7 +42,7 @@ commons-httpclient-3.1.jar
 commons-io-2.4.jar
 commons-lang-2.6.jar
 commons-lang3-3.8.1.jar
-commons-logging-1.1.3.jar
+commons-logging-1.2.jar
 commons-math3-3.4.1.jar
 commons-net-3.1.jar
 commons-pool-1.5.4.jar
@@ -53,9 +51,9 @@ core-1.1.2.jar
 curator-client-2.7.1.jar
 curator-framework-2.7.1.jar
 curator-recipes-2.7.1.jar
-datanucleus-api-jdo-3.2.6.jar
-datanucleus-core-3.2.10.jar
-datanucleus-rdbms-3.2.9.jar
+datanucleus-api-jdo-4.2.4.jar
+datanucleus-core-4.1.17.jar
+datanucleus-rdbms-4.1.19.jar
 derby-10.12.1.1.jar
 eigenbase-properties-1.1.5.jar
 flatbuffers-1.2.0-3f79e055.jar
@@ -79,6 +77,19 @@ hadoop-yarn-client-2.7.4.jar
 hadoop-yarn-common-2.7.4.jar
 hadoop-yarn-server-common-2.7.4.jar
 hadoop-yarn-server-web-proxy-2.7.4.jar
+hive-beeline-2.3.4.jar
+hive-cli-2.3.4.jar
+hive-common-2.3.4.jar
+hive-exec-2.3.4-core.jar
+hive-jdbc-2.3.4.jar
+hive-llap-client-2.3.4.jar
+hive-llap-common-2.3.4.jar
+hive-metastore-2.3.4.jar
+hive-serde-2.3.4.jar
+hive-service-rpc-2.3.4.jar
+hive-shims-0.23-2.3.4.jar
+hive-shims-common-2.3.4.jar
+hive-storage-api-2.7.0.jar
 hk2-api-2.4.0-b34.jar
 hk2-locator-2.4.0-b34.jar
 hk2-utils-2.4.0-b34.jar
@@ -103,6 +114,7 @@ javassist-3.18.1-GA.jar
 javax.annotation-api-1.2.jar
 javax.inject-1.jar
 javax.inject-2.4.0-b34.jar
+javax.jdo-3.2.0-m3.jar
 javax.servlet-api-3.1.0.jar
 javax.ws.rs-api-2.0.1.jar
 javolution-5.5.1.jar
@@ -123,6 +135,7 @@ jline-2.14.6.jar
 joda-time-2.9.3.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
+json-1.8.jar
 json4s-ast_2.12-3.5.3.jar
 json4s-core_2.12-3.5.3.jar
 json4s-jackson_2.12-3.5.3.jar
@@ -155,8 +168,8 @@ objenesis-2.5.1.jar
 okhttp-3.8.1.jar
 okio-1.13.0.jar
 opencsv-2.3.jar
-orc-core-1.5.4-nohive.jar
-orc-mapreduce-1.5.4-nohive.jar
+orc-core-1.5.4.jar
+orc-mapreduce-1.5.4.jar
 orc-shims-1.5.4.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar
@@ -166,7 +179,7 @@ parquet-common-1.10.0.jar
 parquet-encoding-1.10.0.jar
 parquet-format-2.4.0.jar
 parquet-hadoop-1.10.0.jar
-parquet-hadoop-bundle-1.6.0.jar
+parquet-hadoop-bundle-1.10.0.jar
 parquet-jackson-1.10.0.jar
 protobuf-java-2.5.0.jar
 py4j-0.10.8.1.jar
@@ -180,15 +193,14 @@ shapeless_2.12-2.3.2.jar
 slf4j-api-1.7.16.jar
 slf4j-log4j12-1.7.16.jar
 snakeyaml-1.18.jar
-snappy-0.2.jar
 snappy-java-1.1.7.1.jar
 spire-macros_2.12-0.13.0.jar
 spire_2.12-0.13.0.jar
 stax-api-1.0-2.jar
 stax-api-1.0.1.jar
 stream-2.7.0.jar
-stringtemplate-3.2.1.jar
 super-csv-2.2.0.jar
+transaction-api-1.1.jar
 univocity-parsers-2.7.3.jar
 validation-api-1.1.0.Final.jar
 xbean-asm7-shaded-4.12.jar

dev/deps/spark-deps-hadoop-3.1

@@ -1,30 +1,28 @@
 HikariCP-java7-2.4.12.jar
-JavaEWAH-0.3.2.jar
 RoaringBitmap-0.5.11.jar
 ST4-4.0.4.jar
 accessors-smart-1.2.jar
 activation-1.1.1.jar
 aircompressor-0.10.jar
-antlr-2.7.7.jar
-antlr-runtime-3.4.jar
+antlr-runtime-3.5.2.jar
 antlr4-runtime-4.7.1.jar
 aopalliance-1.0.jar
 aopalliance-repackaged-2.4.0-b34.jar
-apache-log4j-extras-1.2.17.jar
 arpack_combined_all-0.1.jar
 arrow-format-0.10.0.jar
 arrow-memory-0.10.0.jar
 arrow-vector-0.10.0.jar
 automaton-1.11-8.jar
+avatica-core-1.13.0.jar
 avro-1.8.2.jar
 avro-ipc-1.8.2.jar
 avro-mapred-1.8.2-hadoop2.jar
 bonecp-0.8.0.RELEASE.jar
 breeze-macros_2.12-0.13.2.jar
 breeze_2.12-0.13.2.jar
-calcite-avatica-1.2.0-incubating.jar
-calcite-core-1.2.0-incubating.jar
-calcite-linq4j-1.2.0-incubating.jar
+calcite-core-1.10.0.jar
+calcite-druid-1.10.0.jar
+calcite-linq4j-1.10.0.jar
 chill-java-0.9.3.jar
 chill_2.12-0.9.3.jar
 commons-beanutils-1.9.3.jar
@@ -41,7 +39,7 @@ commons-httpclient-3.1.jar
 commons-io-2.4.jar
 commons-lang-2.6.jar
 commons-lang3-3.8.1.jar
-commons-logging-1.1.3.jar
+commons-logging-1.2.jar
 commons-math3-3.4.1.jar
 commons-net-3.1.jar
 commons-pool-1.5.4.jar
@@ -50,9 +48,9 @@ core-1.1.2.jar
 curator-client-2.12.0.jar
 curator-framework-2.12.0.jar
 curator-recipes-2.12.0.jar
-datanucleus-api-jdo-3.2.6.jar
-datanucleus-core-3.2.10.jar
-datanucleus-rdbms-3.2.9.jar
+datanucleus-api-jdo-4.2.4.jar
+datanucleus-core-4.1.17.jar
+datanucleus-rdbms-4.1.19.jar
 derby-10.12.1.1.jar
 dnsjava-2.1.7.jar
 ehcache-3.3.1.jar
@@ -78,6 +76,19 @@ hadoop-yarn-common-3.1.0.jar
 hadoop-yarn-registry-3.1.0.jar
 hadoop-yarn-server-common-3.1.0.jar
 hadoop-yarn-server-web-proxy-3.1.0.jar
+hive-beeline-2.3.4.jar
+hive-cli-2.3.4.jar
+hive-common-2.3.4.jar
+hive-exec-2.3.4-core.jar
+hive-jdbc-2.3.4.jar
+hive-llap-client-2.3.4.jar
+hive-llap-common-2.3.4.jar
+hive-metastore-2.3.4.jar
+hive-serde-2.3.4.jar
+hive-service-rpc-2.3.4.jar
+hive-shims-0.23-2.3.4.jar
+hive-shims-common-2.3.4.jar
+hive-storage-api-2.7.0.jar
 hk2-api-2.4.0-b34.jar
 hk2-locator-2.4.0-b34.jar
 hk2-utils-2.4.0-b34.jar
@@ -102,6 +113,7 @@ javassist-3.18.1-GA.jar
 javax.annotation-api-1.2.jar
 javax.inject-1.jar
 javax.inject-2.4.0-b34.jar
+javax.jdo-3.2.0-m3.jar
 javax.servlet-api-3.1.0.jar
 javax.ws.rs-api-2.0.1.jar
 javolution-5.5.1.jar
@@ -122,6 +134,7 @@ jline-2.14.6.jar
 joda-time-2.9.3.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
+json-1.8.jar
 json-smart-2.3.jar
 json4s-ast_2.12-3.5.3.jar
 json4s-core_2.12-3.5.3.jar
@@ -172,8 +185,8 @@ okhttp-2.7.5.jar
 okhttp-3.8.1.jar
 okio-1.13.0.jar
 opencsv-2.3.jar
-orc-core-1.5.4-nohive.jar
-orc-mapreduce-1.5.4-nohive.jar
+orc-core-1.5.4.jar
+orc-mapreduce-1.5.4.jar
 orc-shims-1.5.4.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar
@@ -183,7 +196,7 @@ parquet-common-1.10.0.jar
 parquet-encoding-1.10.0.jar
 parquet-format-2.4.0.jar
 parquet-hadoop-1.10.0.jar
-parquet-hadoop-bundle-1.6.0.jar
+parquet-hadoop-bundle-1.10.0.jar
 parquet-jackson-1.10.0.jar
 protobuf-java-2.5.0.jar
 py4j-0.10.8.1.jar
@@ -198,16 +211,15 @@ shapeless_2.12-2.3.2.jar
 slf4j-api-1.7.16.jar
 slf4j-log4j12-1.7.16.jar
 snakeyaml-1.18.jar
-snappy-0.2.jar
 snappy-java-1.1.7.1.jar
 spire-macros_2.12-0.13.0.jar
 spire_2.12-0.13.0.jar
 stax-api-1.0.1.jar
 stax2-api-3.1.4.jar
 stream-2.7.0.jar
-stringtemplate-3.2.1.jar
 super-csv-2.2.0.jar
 token-provider-1.0.1.jar
+transaction-api-1.1.jar
 univocity-parsers-2.7.3.jar
 validation-api-1.1.0.Final.jar
 woodstox-core-5.0.3.jar

examples/pom.xml

@@ -110,7 +110,7 @@
       <version>3.7.0</version>
     </dependency>
     <dependency>
-      <groupId>com.twitter</groupId>
+      <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-hadoop-bundle</artifactId>
       <scope>provided</scope>
     </dependency>