From 424092a3fa32228a5ed6a8003562d89cc9f6ae1a Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 19 May 2020 14:11:24 +0900 Subject: [PATCH 1/8] Add RealBrowserUISeleniumSuite.scala --- .../spark/ui/RealBrowserUISeleniumSuite.scala | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala diff --git a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala new file mode 100644 index 0000000000000..232a36f4c9ea6 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.ui + +import org.openqa.selenium.{By, WebDriver} +import org.scalatest._ +import org.scalatest.concurrent.Eventually._ +import org.scalatest.time.SpanSugar._ +import org.scalatestplus.selenium.WebBrowser + +import org.apache.spark._ +import org.apache.spark.LocalSparkContext.withSpark +import org.apache.spark.internal.config.MEMORY_OFFHEAP_SIZE +import org.apache.spark.internal.config.UI.{UI_ENABLED, UI_KILL_ENABLED, UI_PORT} +import org.apache.spark.util.CallSite + +/** + * Selenium tests for the Spark Web UI with real web browsers. + */ +abstract class RealBrowserUISeleniumSuite(val driverProp: String) + extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll { + + implicit var webDriver: WebDriver + private val driverPropPrefix = "spark.test." + + assume( + sys.props(driverPropPrefix + driverProp) !== null, + "System property " + driverPropPrefix + driverProp + + " should be set to the corresponding driver path.") + + sys.props(driverProp) = sys.props(driverPropPrefix + driverProp) + + test("SPARK-31534: text for tooltip should be escaped") { + withSpark(newSparkContext()) { sc => + sc.setLocalProperty(CallSite.LONG_FORM, "collect at :25") + sc.setLocalProperty(CallSite.SHORT_FORM, "collect at :25") + sc.parallelize(1 to 10).collect + + eventually(timeout(10.seconds), interval(50.milliseconds)) { + goToUi(sc, "/jobs") + + val jobDesc = + webDriver.findElement(By.cssSelector("div[class='application-timeline-content']")) + jobDesc.getAttribute("data-title") should include ("collect at <console>:25") + + goToUi(sc, "/jobs/job/?id=0") + webDriver.get(sc.ui.get.webUrl.stripSuffix("/") + "/jobs/job/?id=0") + val stageDesc = webDriver.findElement(By.cssSelector("div[class='job-timeline-content']")) + stageDesc.getAttribute("data-title") should include ("collect at <console>:25") + + // Open DAG Viz. + webDriver.findElement(By.id("job-dag-viz")).click() + val nodeDesc = webDriver.findElement(By.cssSelector("g[class='node_0 node']")) + nodeDesc.getAttribute("name") should include ("collect at <console>:25") + } + } + } + + /** + * Create a test SparkContext with the SparkUI enabled. + * It is safe to `get` the SparkUI directly from the SparkContext returned here. + */ + private def newSparkContext( + killEnabled: Boolean = true, + master: String = "local", + additionalConfs: Map[String, String] = Map.empty): SparkContext = { + val conf = new SparkConf() + .setMaster(master) + .setAppName("test") + .set(UI_ENABLED, true) + .set(UI_PORT, 0) + .set(UI_KILL_ENABLED, killEnabled) + .set(MEMORY_OFFHEAP_SIZE.key, "64m") + additionalConfs.foreach { case (k, v) => conf.set(k, v) } + val sc = new SparkContext(conf) + assert(sc.ui.isDefined) + sc + } + + def goToUi(sc: SparkContext, path: String): Unit = { + goToUi(sc.ui.get, path) + } + + def goToUi(ui: SparkUI, path: String): Unit = { + go to (ui.webUrl.stripSuffix("/") + path) + } +} From 4fe3e80f9d1bc908dbf18248af8ea71a27f0201a Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 19 May 2020 14:12:11 +0900 Subject: [PATCH 2/8] Add ChromeUITest.java --- .../org/apache/spark/tags/ChromeUITest.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java diff --git a/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java b/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java new file mode 100644 index 0000000000000..e3fed3d656d20 --- /dev/null +++ b/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.tags; + +import java.lang.annotation.*; + +import org.scalatest.TagAnnotation; + +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface ChromeUITest { } From 760eb0a30e24ca2801415a2bc64649805e29f1c3 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 19 May 2020 14:12:32 +0900 Subject: [PATCH 3/8] Add ChromeUISeeniumSuite.scala --- .../spark/ui/ChromeUISeleniumSuite.scala | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala diff --git a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala new file mode 100644 index 0000000000000..9ba705c4abd75 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.ui + +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions} + +import org.apache.spark.tags.ChromeUITest + +/** + * Selenium tests for the Spark Web UI with Chrome. + */ +@ChromeUITest +class ChromeUISeleniumSuite extends RealBrowserUISeleniumSuite("webdriver.chrome.driver") { + + override var webDriver: WebDriver = _ + + override def beforeAll(): Unit = { + super.beforeAll() + val chromeOptions = new ChromeOptions + chromeOptions.addArguments("--headless", "--disable-gpu") + webDriver = new ChromeDriver(chromeOptions) + } + + override def afterAll(): Unit = { + try { + if (webDriver != null) { + webDriver.quit() + } + } finally { + super.afterAll() + } + } +} From c2f6a6d8d65f440135584aee392b6c214a58eeb5 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 19 May 2020 14:13:56 +0900 Subject: [PATCH 4/8] Remove JavaScript related tests from UISeleniumSuite.scala --- .../org/apache/spark/ui/UISeleniumSuite.scala | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index 3ec9385116408..909056eab8c5a 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -773,33 +773,6 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B } } - test("SPARK-31534: text for tooltip should be escaped") { - withSpark(newSparkContext()) { sc => - sc.setLocalProperty(CallSite.LONG_FORM, "collect at :25") - sc.setLocalProperty(CallSite.SHORT_FORM, "collect at :25") - sc.parallelize(1 to 10).collect - - val driver = webDriver.asInstanceOf[HtmlUnitDriver] - driver.setJavascriptEnabled(true) - - eventually(timeout(10.seconds), interval(50.milliseconds)) { - goToUi(sc, "/jobs") - val jobDesc = - driver.findElement(By.cssSelector("div[class='application-timeline-content']")) - jobDesc.getAttribute("data-title") should include ("collect at <console>:25") - - goToUi(sc, "/jobs/job/?id=0") - val stageDesc = driver.findElement(By.cssSelector("div[class='job-timeline-content']")) - stageDesc.getAttribute("data-title") should include ("collect at <console>:25") - - // Open DAG Viz. - driver.findElement(By.id("job-dag-viz")).click() - val nodeDesc = driver.findElement(By.cssSelector("g[class='node_0 node']")) - nodeDesc.getAttribute("name") should include ("collect at <console>:25") - } - } - } - def goToUi(sc: SparkContext, path: String): Unit = { goToUi(sc.ui.get, path) } From e701dc99048e63a4a8472d16eb70b2a41223ca45 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 19 May 2020 14:16:17 +0900 Subject: [PATCH 5/8] Disable tests taged as 'ChromeUITest' when tests are run through run-tests.py --- dev/run-tests.py | 5 +++++ pom.xml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/dev/run-tests.py b/dev/run-tests.py index 5255a77ec2081..5f9cd6beb332a 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -33,6 +33,9 @@ from sparktestsupport.toposort import toposort_flatten import sparktestsupport.modules as modules +always_excluded_tags = [ + "org.apache.spark.tags.ChromeUITest" +] # ------------------------------------------------------------------------------------------------- # Functions for traversing module dependency graph @@ -606,6 +609,8 @@ def main(): print("[info] Found the following changed modules:", ", ".join(x.name for x in changed_modules)) + excluded_tags.extend(always_excluded_tags) + # setup environment variables # note - the 'root' module doesn't collect environment variables for all modules. Because the # environment variables should not be set if a module is not changed, even if running the 'root' diff --git a/pom.xml b/pom.xml index fd4cebcd37319..6620673a7e5fc 100644 --- a/pom.xml +++ b/pom.xml @@ -243,6 +243,7 @@ things breaking. --> ${session.executionRootDirectory} + 1g @@ -2512,6 +2513,7 @@ false false true + ${spark.test.webdriver.chrome.driver} __not_used__ From 6ccbc391067995205127425ef59c4a9f1fe40520 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Wed, 20 May 2020 09:00:38 +0900 Subject: [PATCH 6/8] Fixed RealBrowserUISeleniumSuite. --- .../spark/ui/RealBrowserUISeleniumSuite.scala | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala index 232a36f4c9ea6..6793540c7e43b 100644 --- a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala @@ -38,12 +38,19 @@ abstract class RealBrowserUISeleniumSuite(val driverProp: String) implicit var webDriver: WebDriver private val driverPropPrefix = "spark.test." - assume( - sys.props(driverPropPrefix + driverProp) !== null, - "System property " + driverPropPrefix + driverProp + - " should be set to the corresponding driver path.") + override def beforeAll() { + super.beforeAll() + assume( + sys.props(driverPropPrefix + driverProp) !== null, + "System property " + driverPropPrefix + driverProp + + " should be set to the corresponding driver path.") + sys.props(driverProp) = sys.props(driverPropPrefix + driverProp) + } - sys.props(driverProp) = sys.props(driverPropPrefix + driverProp) + override def afterAll(): Unit = { + sys.props.remove(driverProp) + super.afterAll() + } test("SPARK-31534: text for tooltip should be escaped") { withSpark(newSparkContext()) { sc => From 6b47fe8cccca0460ec7c0db0e16c29535f1d01b8 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Sun, 24 May 2020 13:38:52 +0900 Subject: [PATCH 7/8] Disabled newly added suite by default. --- .../org/apache/spark/ui/RealBrowserUISeleniumSuite.scala | 2 +- pom.xml | 2 +- project/SparkBuild.scala | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala index 6793540c7e43b..84f888267857e 100644 --- a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala @@ -38,7 +38,7 @@ abstract class RealBrowserUISeleniumSuite(val driverProp: String) implicit var webDriver: WebDriver private val driverPropPrefix = "spark.test." - override def beforeAll() { + override def beforeAll(): Unit = { super.beforeAll() assume( sys.props(driverPropPrefix + driverProp) !== null, diff --git a/pom.xml b/pom.xml index 6620673a7e5fc..2df3f69ad3a72 100644 --- a/pom.xml +++ b/pom.xml @@ -204,7 +204,7 @@ org.fusesource.leveldbjni ${java.home} - + org.apache.spark.tags.ChromeUITest diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 65937ad3cefe3..1c3356ef921bf 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -967,6 +967,9 @@ object TestSettings { "2.12" } */ + + private val defaultExcludedTagsForScalaTest = Seq("org.apache.spark.tags.ChromeUITest") + lazy val settings = Seq ( // Fork new JVMs for tests and set Java options for those fork := true, @@ -1001,9 +1004,9 @@ object TestSettings { javaOptions += "-Xmx3g", // Exclude tags defined in a system property testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest, - sys.props.get("test.exclude.tags").map { tags => - tags.split(",").flatMap { tag => Seq("-l", tag) }.toSeq - }.getOrElse(Nil): _*), + sys.props.get("test.exclude.tags").map(tag => tag.split(",").toSeq) + .getOrElse(defaultExcludedTagsForScalaTest).filter(!_.trim.isEmpty) + .flatMap(tag => Seq("-l", tag)): _*), testOptions in Test += Tests.Argument(TestFrameworks.JUnit, sys.props.get("test.exclude.tags").map { tags => Seq("--exclude-categories=" + tags) From 70d07a3785799e5f44e21cb9c541faaa650ad6c0 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Mon, 25 May 2020 12:24:06 +0900 Subject: [PATCH 8/8] Introduced test.default.exclude.tags to set excluded tests by default. --- dev/run-tests.py | 5 ----- pom.xml | 7 +++++-- project/SparkBuild.scala | 10 +++++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index 5f9cd6beb332a..5255a77ec2081 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -33,9 +33,6 @@ from sparktestsupport.toposort import toposort_flatten import sparktestsupport.modules as modules -always_excluded_tags = [ - "org.apache.spark.tags.ChromeUITest" -] # ------------------------------------------------------------------------------------------------- # Functions for traversing module dependency graph @@ -609,8 +606,6 @@ def main(): print("[info] Found the following changed modules:", ", ".join(x.name for x in changed_modules)) - excluded_tags.extend(always_excluded_tags) - # setup environment variables # note - the 'root' module doesn't collect environment variables for all modules. Because the # environment variables should not be set if a module is not changed, even if running the 'root' diff --git a/pom.xml b/pom.xml index 2df3f69ad3a72..3d70767028a87 100644 --- a/pom.xml +++ b/pom.xml @@ -204,7 +204,10 @@ org.fusesource.leveldbjni ${java.home} - org.apache.spark.tags.ChromeUITest + + + org.apache.spark.tags.ChromeUITest + @@ -2517,7 +2520,7 @@ __not_used__ - ${test.exclude.tags} + ${test.exclude.tags},${test.default.exclude.tags} ${test.include.tags} diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 1c3356ef921bf..884ff4c63a8ec 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -968,7 +968,7 @@ object TestSettings { } */ - private val defaultExcludedTagsForScalaTest = Seq("org.apache.spark.tags.ChromeUITest") + private val defaultExcludedTags = Seq("org.apache.spark.tags.ChromeUITest") lazy val settings = Seq ( // Fork new JVMs for tests and set Java options for those @@ -1004,8 +1004,12 @@ object TestSettings { javaOptions += "-Xmx3g", // Exclude tags defined in a system property testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest, - sys.props.get("test.exclude.tags").map(tag => tag.split(",").toSeq) - .getOrElse(defaultExcludedTagsForScalaTest).filter(!_.trim.isEmpty) + sys.props.get("test.exclude.tags").map { tags => + tags.split(",").flatMap { tag => Seq("-l", tag) }.toSeq + }.getOrElse(Nil): _*), + testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest, + sys.props.get("test.default.exclude.tags").map(tags => tags.split(",").toSeq) + .map(tags => tags.filter(!_.trim.isEmpty)).getOrElse(defaultExcludedTags) .flatMap(tag => Seq("-l", tag)): _*), testOptions in Test += Tests.Argument(TestFrameworks.JUnit, sys.props.get("test.exclude.tags").map { tags =>