From 4265afb7bb3262105a28194e3f16d7eec5b4ae27 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 15 Jan 2016 15:52:05 -0800 Subject: [PATCH 01/15] SPARK-12807 WiP shading patch + integration tests (after shading) to verify shading is there. currently they fail --- network/yarn/pom.xml | 63 ++++++++++++++++++ .../spark/network/yarn/ShadedJacksonIT.java | 65 +++++++++++++++++++ pom.xml | 5 ++ 3 files changed, 133 insertions(+) create mode 100644 network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index a28785b16e1e6..d6ab6811978b3 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -49,6 +49,27 @@ spark-test-tags_${scala.binary.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + com.fasterxml.jackson.core + jackson-annotations + + + + com.fasterxml.jackson.core + jackson-core + + + + com.fasterxml.jackson.module + jackson-module-scala_${scala.binary.version} + + org.apache.hadoop @@ -74,6 +95,10 @@ *:* + com.fasterxml.jackson.core:jackson-databind + com.fasterxml.jackson.core:jackson-annotations + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.module:jackson-module-scala_${scala.binary.version} @@ -86,6 +111,12 @@ + + + com.fasterxml.jackson + org.spark-project.com.fasterxml.jackson + + @@ -96,6 +127,38 @@ + + + org.apache.maven.plugins + maven-failsafe-plugin + + + run-integration-tests + + integration-test + verify + + + + + true + false + + **/*IT.java + + + + + + + org.apache.maven.surefire + surefire-junit47 + 2.16 + + + + + diff --git a/network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java b/network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java new file mode 100644 index 0000000000000..9a7d75646992e --- /dev/null +++ b/network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.yarn; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URL; + +/** + * This test suite is to be run as an integration test, after packaging. + * It verifies that the shaded jackson artifacts can be loaded + */ +public class ShadedJacksonIT { + + private static final Logger log = + LoggerFactory.getLogger(ShadedJacksonIT.class); + + private static final String SHADED_JACKSON_PACKAGE = "org.spark-project.com.fasterxml.jackson."; + + @Test + public void testJacksonShadedAnnotation() throws Throwable { + loadClassAsResource(SHADED_JACKSON_PACKAGE + "annotation.JacksonAnnotation"); + } + + @Test + public void testJacksonShadedCore() throws Throwable { + loadClassAsResource(SHADED_JACKSON_PACKAGE + "core.JsonParser"); + } + + @Test + public void testJacksonShadedDatabind() throws Throwable { + loadClassAsResource(SHADED_JACKSON_PACKAGE + "databind.JsonSerializer"); + } + + @Test + public void testJacksonShadedScala() throws Throwable { + loadClassAsResource(SHADED_JACKSON_PACKAGE + "scala.SetModule"); + } + + private void loadClassAsResource(String classname) throws Throwable { + String fullClassName = classname + ".class"; + URL url = this.getClass().getClassLoader().getResource(fullClassName); + Assert.assertNotNull("Failed to find " + fullClassName, null); + log.info("Loaded {} at {}", classname, url); + } +} diff --git a/pom.xml b/pom.xml index d0387aca66d0d..96c4ea06ede83 100644 --- a/pom.xml +++ b/pom.xml @@ -615,6 +615,11 @@ jackson-annotations ${fasterxml.jackson.version} + + com.fasterxml.jackson.core + jackson-core + ${fasterxml.jackson.version} + From 2a876fe0bacc5a5de0c09dd0afc1d0a23b5203e6 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 15 Jan 2016 15:58:55 -0800 Subject: [PATCH 02/15] SPARK-12807 add log4j.properties test resource --- .../yarn/src/test/resources/log4j.properties | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 network/yarn/src/test/resources/log4j.properties diff --git a/network/yarn/src/test/resources/log4j.properties b/network/yarn/src/test/resources/log4j.properties new file mode 100644 index 0000000000000..40ef9fb97f27d --- /dev/null +++ b/network/yarn/src/test/resources/log4j.properties @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +log4j.rootCategory=DEBUG, file +log4j.appender.file=org.apache.log4j.FileAppender +log4j.appender.file.append=true +log4j.appender.file.file=target/integration-tests.log +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n + +# Ignore messages below warning level from a few verbose libraries. +log4j.logger.com.sun.jersey=WARN +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.eclipse.jetty=WARN +log4j.logger.org.mortbay=WARN +log4j.logger.org.spark-project.jetty=WARN From 4221e733914b1c015804fed64140a0a93e1a01f2 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 15 Jan 2016 17:02:12 -0800 Subject: [PATCH 03/15] SPARK-12807 IT test to use / paths. Moot, as tests classpaths aren't right --- .../apache/spark/network/yarn/ShadedJacksonIT.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java b/network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java index 9a7d75646992e..e03b2eca58d53 100644 --- a/network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java +++ b/network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java @@ -34,26 +34,30 @@ public class ShadedJacksonIT { private static final Logger log = LoggerFactory.getLogger(ShadedJacksonIT.class); - private static final String SHADED_JACKSON_PACKAGE = "org.spark-project.com.fasterxml.jackson."; + private static final String SHADED_JACKSON_PACKAGE = "/org/spark-project/com/fasterxml/jackson/"; + @Test + public void logClasspath() throws Throwable { + log.info("classpath={}", System.getProperty("java.class.path")); + } @Test public void testJacksonShadedAnnotation() throws Throwable { - loadClassAsResource(SHADED_JACKSON_PACKAGE + "annotation.JacksonAnnotation"); + loadClassAsResource(SHADED_JACKSON_PACKAGE + "annotation/JacksonAnnotation"); } @Test public void testJacksonShadedCore() throws Throwable { - loadClassAsResource(SHADED_JACKSON_PACKAGE + "core.JsonParser"); + loadClassAsResource(SHADED_JACKSON_PACKAGE + "core/JsonParser"); } @Test public void testJacksonShadedDatabind() throws Throwable { - loadClassAsResource(SHADED_JACKSON_PACKAGE + "databind.JsonSerializer"); + loadClassAsResource(SHADED_JACKSON_PACKAGE + "databind/JsonSerializer"); } @Test public void testJacksonShadedScala() throws Throwable { - loadClassAsResource(SHADED_JACKSON_PACKAGE + "scala.SetModule"); + loadClassAsResource(SHADED_JACKSON_PACKAGE + "scala/SetModule"); } private void loadClassAsResource(String classname) throws Throwable { From 35d478dfd9eb0125e1b88794995c61ecbb7de205 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 15 Jan 2016 17:03:03 -0800 Subject: [PATCH 04/15] SPARK-12807 Switch to using Ant to scan internals of zipfile. Guaranteed to be asserting state of JAR --- network/yarn/pom.xml | 64 ++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index d6ab6811978b3..d30c35825bc1e 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -35,6 +35,8 @@ network-yarn provided + ${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar + org/spark-project/com/fasterxml/jackson @@ -91,14 +93,14 @@ maven-shade-plugin false - ${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar + ${shuffle.jar} - *:* com.fasterxml.jackson.core:jackson-databind com.fasterxml.jackson.core:jackson-annotations com.fasterxml.jackson.core:jackson-core com.fasterxml.jackson.module:jackson-module-scala_${scala.binary.version} + *:* @@ -115,6 +117,9 @@ com.fasterxml.jackson org.spark-project.com.fasterxml.jackson + + com.fasterxml.jackson.** + @@ -130,32 +135,51 @@ org.apache.maven.plugins - maven-failsafe-plugin + maven-antrun-plugin - run-integration-tests + verify - integration-test - verify + run - true - false - - **/*IT.java - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Verified shading - - - - org.apache.maven.surefire - surefire-junit47 - 2.16 - - + + From a57853b2a1c8efa06fba0c4de9ca6ce980b04596 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 15 Jan 2016 17:18:27 -0800 Subject: [PATCH 05/15] SPARK-12807 class check taking place in verify phase; works on clean builds --- network/yarn/pom.xml | 71 ++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index d30c35825bc1e..666d2ef3d713f 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -142,44 +142,43 @@ run + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Verified shading of Jackson ${fasterxml.jackson.version} + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verified shading - - - From 25144ecb312d52a1db03f086b1fd617aa69238a5 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 15 Jan 2016 17:23:34 -0800 Subject: [PATCH 06/15] SPARK-12807 add probe for scala module --- network/yarn/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index 666d2ef3d713f..3dad346c7756c 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -174,6 +174,16 @@ + + + + + + + + + Verified shading of Jackson ${fasterxml.jackson.version} From 586fc797886c06d5402dfacef8b70e52448b6f8d Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 15 Jan 2016 17:23:56 -0800 Subject: [PATCH 07/15] SPARK-12807 cut explicit references for jackson --- network/yarn/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index 3dad346c7756c..a3da63ad1344c 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -96,10 +96,6 @@ ${shuffle.jar} - com.fasterxml.jackson.core:jackson-databind - com.fasterxml.jackson.core:jackson-annotations - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.module:jackson-module-scala_${scala.binary.version} *:* From 32980eab464e67ceeab742a41262bbb4aefc1b35 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 15 Jan 2016 17:37:45 -0800 Subject: [PATCH 08/15] SPARK-12807 tightened build: reduced diff to master, leave out jackson-module-scala --- network/yarn/pom.xml | 33 --------------------------------- pom.xml | 5 ----- 2 files changed, 38 deletions(-) diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index a3da63ad1344c..cd2fa7090fe07 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -51,27 +51,6 @@ spark-test-tags_${scala.binary.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.core - jackson-annotations - - - - com.fasterxml.jackson.core - jackson-core - - - - com.fasterxml.jackson.module - jackson-module-scala_${scala.binary.version} - - org.apache.hadoop @@ -170,16 +149,6 @@ - - - - - - - - - Verified shading of Jackson ${fasterxml.jackson.version} @@ -187,7 +156,5 @@ - - diff --git a/pom.xml b/pom.xml index 96c4ea06ede83..d0387aca66d0d 100644 --- a/pom.xml +++ b/pom.xml @@ -615,11 +615,6 @@ jackson-annotations ${fasterxml.jackson.version} - - com.fasterxml.jackson.core - jackson-core - ${fasterxml.jackson.version} - From 2e694aaf419c7d9f4f2fdd52f0b1dd7c354f9e0f Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 19 Jan 2016 11:11:13 -0800 Subject: [PATCH 09/15] SPARK-12807 macrodefing the build process, for scalability of checks --- network/yarn/pom.xml | 76 +++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index cd2fa7090fe07..d9705832121e1 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -36,7 +36,7 @@ provided ${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar - org/spark-project/com/fasterxml/jackson + org/spark-project/ @@ -108,6 +108,7 @@ + org.apache.maven.plugins maven-antrun-plugin @@ -119,37 +120,39 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verified shading of Jackson ${fasterxml.jackson.version} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Verified dependency shading @@ -157,4 +160,13 @@ + + + + leveldb-provided + + + + + From f16cf135ccf3e21a4340f25d2184e1141b9acc8c Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 19 Jan 2016 11:13:29 -0800 Subject: [PATCH 10/15] SPARK-12807 extends , which can be used for probing unshaded artifacts --- network/yarn/pom.xml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index d9705832121e1..c6848bb42e8fa 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -128,7 +128,7 @@ + name="@{resource}"/> @@ -138,15 +138,7 @@ - - - - - - - - - + From 7489f89dcb85d14fc8431a9424f0c7eeae398d86 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 19 Jan 2016 11:33:21 -0800 Subject: [PATCH 11/15] SPARK-12807 add leveldb-provided profile, which excludes leveldb from the artifact. the build fails, as the verify stage is still probing for it --- network/yarn/pom.xml | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index c6848bb42e8fa..50362cdaf0361 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -55,6 +55,12 @@ org.apache.hadoop hadoop-client + + + org.fusesource.leveldbjni + leveldbjni-all + + org.slf4j @@ -96,6 +102,20 @@ com.fasterxml.jackson.** + + io.netty + org.spark-project.io.netty + + io.netty.** + + + + javax.annotation + org.spark-project.javax.annotation + + javax.annotation.** + + @@ -123,12 +143,11 @@ - + - + @@ -141,10 +160,13 @@ + Verifying dependency shading - Verified dependency shading + + + @@ -157,7 +179,17 @@ leveldb-provided - + + org.apache.spark + spark-network-shuffle_${scala.binary.version} + ${project.version} + + + org.fusesource.leveldbjni + leveldbjni-all + + + From f39275d9b8b4fa784aad4de83934f93c0bb4dece Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 19 Jan 2016 11:37:13 -0800 Subject: [PATCH 12/15] SPARK-12807 cut leveldb check --- network/yarn/pom.xml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index 50362cdaf0361..037152f7b708d 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -140,33 +140,26 @@ - + - + - + - - - - - - Verifying dependency shading - From 81f73046dd29dfc5726310eb32341b31a28adfb5 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 19 Jan 2016 16:43:42 -0800 Subject: [PATCH 13/15] SPARK-12807 cull (broken) test methods obsoleted by build file checks --- .../spark/network/yarn/ShadedJacksonIT.java | 69 ------------------- .../yarn/src/test/resources/log4j.properties | 31 --------- 2 files changed, 100 deletions(-) delete mode 100644 network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java delete mode 100644 network/yarn/src/test/resources/log4j.properties diff --git a/network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java b/network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java deleted file mode 100644 index e03b2eca58d53..0000000000000 --- a/network/yarn/src/test/java/org/apache/spark/network/yarn/ShadedJacksonIT.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.network.yarn; - -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.URL; - -/** - * This test suite is to be run as an integration test, after packaging. - * It verifies that the shaded jackson artifacts can be loaded - */ -public class ShadedJacksonIT { - - private static final Logger log = - LoggerFactory.getLogger(ShadedJacksonIT.class); - - private static final String SHADED_JACKSON_PACKAGE = "/org/spark-project/com/fasterxml/jackson/"; - - @Test - public void logClasspath() throws Throwable { - log.info("classpath={}", System.getProperty("java.class.path")); - } - @Test - public void testJacksonShadedAnnotation() throws Throwable { - loadClassAsResource(SHADED_JACKSON_PACKAGE + "annotation/JacksonAnnotation"); - } - - @Test - public void testJacksonShadedCore() throws Throwable { - loadClassAsResource(SHADED_JACKSON_PACKAGE + "core/JsonParser"); - } - - @Test - public void testJacksonShadedDatabind() throws Throwable { - loadClassAsResource(SHADED_JACKSON_PACKAGE + "databind/JsonSerializer"); - } - - @Test - public void testJacksonShadedScala() throws Throwable { - loadClassAsResource(SHADED_JACKSON_PACKAGE + "scala/SetModule"); - } - - private void loadClassAsResource(String classname) throws Throwable { - String fullClassName = classname + ".class"; - URL url = this.getClass().getClassLoader().getResource(fullClassName); - Assert.assertNotNull("Failed to find " + fullClassName, null); - log.info("Loaded {} at {}", classname, url); - } -} diff --git a/network/yarn/src/test/resources/log4j.properties b/network/yarn/src/test/resources/log4j.properties deleted file mode 100644 index 40ef9fb97f27d..0000000000000 --- a/network/yarn/src/test/resources/log4j.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Set everything to be logged to the file target/unit-tests.log -log4j.rootCategory=DEBUG, file -log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=true -log4j.appender.file.file=target/integration-tests.log -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n - -# Ignore messages below warning level from a few verbose libraries. -log4j.logger.com.sun.jersey=WARN -log4j.logger.org.apache.hadoop=WARN -log4j.logger.org.eclipse.jetty=WARN -log4j.logger.org.mortbay=WARN -log4j.logger.org.spark-project.jetty=WARN From e8d4b3ac72e94222489ae3fc9d0a1fe5f0c44c5e Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 5 Feb 2016 12:44:00 +0000 Subject: [PATCH 14/15] [SPARK-12807] remove netty & javax.annotation from relocation --- network/yarn/pom.xml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index 037152f7b708d..a817cd4baa0b5 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -102,20 +102,6 @@ com.fasterxml.jackson.** - - io.netty - org.spark-project.io.netty - - io.netty.** - - - - javax.annotation - org.spark-project.javax.annotation - - javax.annotation.** - - @@ -158,8 +144,6 @@ - - From 947a798894010d93b5b0ac64f6307202b0b919bf Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 8 Feb 2016 12:08:18 +0000 Subject: [PATCH 15/15] [SPARK-12807] remove all leveldb related work; rely on hadoop and shuffle versions being in perfect sync, so duplication is harmless --- network/yarn/pom.xml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index a817cd4baa0b5..3cb44324f25f2 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -55,12 +55,6 @@ org.apache.hadoop hadoop-client - - - org.fusesource.leveldbjni - leveldbjni-all - - org.slf4j @@ -151,23 +145,4 @@ - - - - leveldb-provided - - - org.apache.spark - spark-network-shuffle_${scala.binary.version} - ${project.version} - - - org.fusesource.leveldbjni - leveldbjni-all - - - - - -