From c855c2ce1239650edbaf86a53372adbfd4b3278b Mon Sep 17 00:00:00 2001 From: Brendan Dwyer Date: Sat, 10 Dec 2016 13:00:45 -0800 Subject: [PATCH 1/6] set default location of spark.sql.warehouse.dir --- R/pkg/R/sparkR.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R index c57cc8f285613..a46fe9b2cc43c 100644 --- a/R/pkg/R/sparkR.R +++ b/R/pkg/R/sparkR.R @@ -362,6 +362,10 @@ sparkR.session <- function( enableHiveSupport = TRUE, ...) { + if(length(sparkConfig[["spark.sql.warehouse.dir"]]) == 0) { + sparkConfig[["spark.sql.warehouse.dir"]] <- tempdir() + } + sparkConfigMap <- convertNamedListToEnv(sparkConfig) namedParams <- list(...) if (length(namedParams) > 0) { From 79caf1c3cf94b0e97ab09106aca5bab7d9fbf1c9 Mon Sep 17 00:00:00 2001 From: Brendan Dwyer Date: Sat, 10 Dec 2016 13:58:11 -0800 Subject: [PATCH 2/6] fixed R style --- R/pkg/R/sparkR.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R index a46fe9b2cc43c..56d70b2b436c8 100644 --- a/R/pkg/R/sparkR.R +++ b/R/pkg/R/sparkR.R @@ -362,10 +362,10 @@ sparkR.session <- function( enableHiveSupport = TRUE, ...) { - if(length(sparkConfig[["spark.sql.warehouse.dir"]]) == 0) { + if (length(sparkConfig[["spark.sql.warehouse.dir"]]) == 0) { sparkConfig[["spark.sql.warehouse.dir"]] <- tempdir() } - + sparkConfigMap <- convertNamedListToEnv(sparkConfig) namedParams <- list(...) if (length(namedParams) > 0) { From cee39446899387fa40c3e853cd53d3c0d043c4bd Mon Sep 17 00:00:00 2001 From: Brendan Dwyer Date: Mon, 12 Dec 2016 12:28:06 -0800 Subject: [PATCH 3/6] added test case --- R/pkg/inst/tests/testthat/test_sparkR.R | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/R/pkg/inst/tests/testthat/test_sparkR.R b/R/pkg/inst/tests/testthat/test_sparkR.R index f73fc6baeccef..15e2c7ca85d28 100644 --- a/R/pkg/inst/tests/testthat/test_sparkR.R +++ b/R/pkg/inst/tests/testthat/test_sparkR.R @@ -44,3 +44,11 @@ test_that("sparkCheckInstall", { deployMode <- "client" expect_error(sparkCheckInstall(sparkHome, master, deployMode)) }) + +test_that("sparkR.session", { + # nothing should be written outside the tempdir() without explicit user premission + inital_working_directory_files <- list.files() + sparkR.session() + expect_equal(inital_working_directory_files, list.files()) + +}) From 17b7bffcc8dbbe3c0d91b1ee9cccff56d038e8ba Mon Sep 17 00:00:00 2001 From: Brendan Dwyer Date: Mon, 12 Dec 2016 14:26:27 -0800 Subject: [PATCH 4/6] added SQL query to test --- R/pkg/inst/tests/testthat/test_sparkR.R | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/R/pkg/inst/tests/testthat/test_sparkR.R b/R/pkg/inst/tests/testthat/test_sparkR.R index 15e2c7ca85d28..5364c59bce164 100644 --- a/R/pkg/inst/tests/testthat/test_sparkR.R +++ b/R/pkg/inst/tests/testthat/test_sparkR.R @@ -49,6 +49,12 @@ test_that("sparkR.session", { # nothing should be written outside the tempdir() without explicit user premission inital_working_directory_files <- list.files() sparkR.session() + df <- data.frame("col1" = c(1, 2, 3, 4, 5, 6), + "col2" = c(1, 0, 0, 1, 1, 0), + "col3" = c(1, 0, 0, 2, 6, 2)) + df <- as.DataFrame(df) + createOrReplaceTempView(df, "table") + result <- sql("SELECT * FROM `table`") + sparkR.session.stop() expect_equal(inital_working_directory_files, list.files()) - }) From 84a110e138601173a3d4b42cedcdb8db597fa07d Mon Sep 17 00:00:00 2001 From: Brendan Dwyer Date: Mon, 12 Dec 2016 16:34:23 -0800 Subject: [PATCH 5/6] attempt to fix failing unit test --- R/pkg/inst/tests/testthat/test_sparkR.R | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/R/pkg/inst/tests/testthat/test_sparkR.R b/R/pkg/inst/tests/testthat/test_sparkR.R index 5364c59bce164..353887a8c6b66 100644 --- a/R/pkg/inst/tests/testthat/test_sparkR.R +++ b/R/pkg/inst/tests/testthat/test_sparkR.R @@ -46,15 +46,10 @@ test_that("sparkCheckInstall", { }) test_that("sparkR.session", { - # nothing should be written outside the tempdir() without explicit user premission + # nothing should be written outside tempdir() without explicit user permission inital_working_directory_files <- list.files() sparkR.session() - df <- data.frame("col1" = c(1, 2, 3, 4, 5, 6), - "col2" = c(1, 0, 0, 1, 1, 0), - "col3" = c(1, 0, 0, 2, 6, 2)) - df <- as.DataFrame(df) - createOrReplaceTempView(df, "table") - result <- sql("SELECT * FROM `table`") + result <- sql("SELECT 1") sparkR.session.stop() expect_equal(inital_working_directory_files, list.files()) }) From 9142397b39ff5886cf51a4255591251241d103b8 Mon Sep 17 00:00:00 2001 From: Brendan Dwyer Date: Tue, 13 Dec 2016 11:17:19 -0800 Subject: [PATCH 6/6] remove hive support from unit test --- R/pkg/inst/tests/testthat/test_sparkR.R | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/R/pkg/inst/tests/testthat/test_sparkR.R b/R/pkg/inst/tests/testthat/test_sparkR.R index 353887a8c6b66..f4e36c9c55ca0 100644 --- a/R/pkg/inst/tests/testthat/test_sparkR.R +++ b/R/pkg/inst/tests/testthat/test_sparkR.R @@ -48,8 +48,13 @@ test_that("sparkCheckInstall", { test_that("sparkR.session", { # nothing should be written outside tempdir() without explicit user permission inital_working_directory_files <- list.files() - sparkR.session() - result <- sql("SELECT 1") + sparkR.session(enableHiveSupport = FALSE) + df <- data.frame("col1" = c(1, 2, 3, 4, 5, 6), + "col2" = c(1, 0, 0, 1, 1, 0), + "col3" = c(1, 0, 0, 2, 6, 2)) + df <- as.DataFrame(df) + createOrReplaceTempView(df, "table") + result <- sql("SELECT * FROM `table`") sparkR.session.stop() expect_equal(inital_working_directory_files, list.files()) })