From 62872f1269b944f1fdebe6c48612278e21dc0049 Mon Sep 17 00:00:00 2001 From: Narine Kokhlikyan Date: Mon, 4 Jan 2016 13:19:47 -0800 Subject: [PATCH 1/5] Some fixes for saveAsTable method --- R/pkg/R/DataFrame.R | 15 ++++++++++----- R/pkg/R/generics.R | 2 +- R/pkg/inst/tests/test_sparkSQL.R | 12 ++++++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R index 81b4e6b91d8a2..eabb88b2a5720 100644 --- a/R/pkg/R/DataFrame.R +++ b/R/pkg/R/DataFrame.R @@ -1956,13 +1956,18 @@ setMethod("saveDF", #' saveAsTable(df, "myfile") #' } setMethod("saveAsTable", - signature(df = "DataFrame", tableName = "character", source = "character", - mode = "character"), + signature(df = "DataFrame", tableName = "character"), function(df, tableName, source = NULL, mode="append", ...){ if (is.null(source)) { - sqlContext <- get(".sparkRSQLsc", envir = .sparkREnv) - source <- callJMethod(sqlContext, "getConf", "spark.sql.sources.default", - "org.apache.spark.sql.parquet") + if (exists(".sparkRSQLsc", envir = .sparkREnv)) { + sqlContext <- get(".sparkRSQLsc", envir = .sparkREnv) + } else if (exists(".sparkRHivesc", envir = .sparkREnv)) { + sqlContext <- get(".sparkRHivesc", envir = .sparkREnv) + } else { + stop("sparkRHive or sparkRSQL context has to be specified") + } + source <- callJMethod(sqlContext, "getConf", "spark.sql.sources.default", + "org.apache.spark.sql.parquet") } allModes <- c("append", "overwrite", "error", "ignore") # nolint start diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R index 711ce38f9e104..dbb7a93be8c62 100644 --- a/R/pkg/R/generics.R +++ b/R/pkg/R/generics.R @@ -521,7 +521,7 @@ setGeneric("saveAsParquetFile", function(x, path) { standardGeneric("saveAsParqu #' @rdname saveAsTable #' @export -setGeneric("saveAsTable", function(df, tableName, source, mode, ...) { +setGeneric("saveAsTable", function(df, tableName, ...) { standardGeneric("saveAsTable") }) diff --git a/R/pkg/inst/tests/test_sparkSQL.R b/R/pkg/inst/tests/test_sparkSQL.R index 1e7cb54099703..40f68dd8dd09f 100644 --- a/R/pkg/inst/tests/test_sparkSQL.R +++ b/R/pkg/inst/tests/test_sparkSQL.R @@ -867,6 +867,18 @@ test_that("test HiveContext", { df3 <- sql(hiveCtx, "select * from json") expect_is(df3, "DataFrame") expect_equal(count(df3), 6) + + hivetestDataPath <- tempfile(pattern="sparkr-test", fileext=".tmp") + saveAsTable(df, "hivetestbl", path = hivetestDataPath) + df4 <- sql(hiveCtx, "select * from hivetestbl") + expect_is(df4, "DataFrame") + expect_equal(count(df4), 6) + + parquetDataPath <- tempfile(pattern="sparkr-test", fileext=".tmp") + saveAsTable(df, "parquetest", "parquet", path=parquetDataPath) + df5 <- sql(hiveCtx, "select * from parquetest") + expect_is(df5, "DataFrame") + expect_equal(count(df5), 6) }) test_that("column operators", { From 86c1432655f4b16ba3eb8769eabe9228831c094b Mon Sep 17 00:00:00 2001 From: Narine Kokhlikyan Date: Wed, 6 Jan 2016 10:12:52 -0800 Subject: [PATCH 2/5] change the signature --- R/pkg/R/generics.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R index 586c4ac71e083..1816852ed2c13 100644 --- a/R/pkg/R/generics.R +++ b/R/pkg/R/generics.R @@ -521,7 +521,7 @@ setGeneric("sampleBy", function(x, col, fractions, seed) { standardGeneric("samp #' @rdname saveAsTable #' @export -setGeneric("saveAsTable", function(df, tableName, ...) { +setGeneric("saveAsTable", function(df, tableName, source = NULL, mode = "error", ...) { standardGeneric("saveAsTable") }) From faa4971571b761ddc88ae7f819c61c8a65741441 Mon Sep 17 00:00:00 2001 From: Narine Kokhlikyan Date: Thu, 7 Jan 2016 09:53:38 -0800 Subject: [PATCH 3/5] fixed style check errors --- R/pkg/inst/tests/testthat/test_sparkSQL.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R index aa16e94dd6d38..e59f98518314c 100644 --- a/R/pkg/inst/tests/testthat/test_sparkSQL.R +++ b/R/pkg/inst/tests/testthat/test_sparkSQL.R @@ -914,7 +914,7 @@ test_that("test HiveContext", { expect_is(df4, "DataFrame") expect_equal(count(df4), 3) unlink(hivetestDataPath) - + parquetDataPath <- tempfile(pattern="sparkr-test", fileext=".tmp") invisible(saveAsTable(df, "parquetest", "parquet", mode="overwrite", path=parquetDataPath)) df5 <- sql(hiveCtx, "select * from parquetest") From fdd38c59158b5e56d72b0c4ee241592dda24c9ff Mon Sep 17 00:00:00 2001 From: Narine Kokhlikyan Date: Wed, 20 Jan 2016 15:19:52 -0800 Subject: [PATCH 4/5] Added the changes to write.df and saveDF too --- R/pkg/R/DataFrame.R | 8 +++++++- R/pkg/R/generics.R | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R index 261eabd4f6771..ba3c2f218c72b 100644 --- a/R/pkg/R/DataFrame.R +++ b/R/pkg/R/DataFrame.R @@ -1975,7 +1975,13 @@ setMethod("write.df", signature(df = "DataFrame", path = "character"), function(df, path, source = NULL, mode = "error", ...){ if (is.null(source)) { - sqlContext <- get(".sparkRSQLsc", envir = .sparkREnv) + if (exists(".sparkRSQLsc", envir = .sparkREnv)) { + sqlContext <- get(".sparkRSQLsc", envir = .sparkREnv) + } else if (exists(".sparkRHivesc", envir = .sparkREnv)) { + sqlContext <- get(".sparkRHivesc", envir = .sparkREnv) + } else { + stop("sparkRHive or sparkRSQL context has to be specified") + } source <- callJMethod(sqlContext, "getConf", "spark.sql.sources.default", "org.apache.spark.sql.parquet") } diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R index dfb5b1f3b578c..59fee1d1d2b02 100644 --- a/R/pkg/R/generics.R +++ b/R/pkg/R/generics.R @@ -531,11 +531,11 @@ setGeneric("transform", function(`_data`, ...) {standardGeneric("transform") }) #' @rdname write.df #' @export -setGeneric("write.df", function(df, path, ...) { standardGeneric("write.df") }) +setGeneric("write.df", function(df, path, source = NULL, mode = "error", ...) { standardGeneric("write.df") }) #' @rdname write.df #' @export -setGeneric("saveDF", function(df, path, ...) { standardGeneric("saveDF") }) +setGeneric("saveDF", function(df, path, source = NULL, mode = "error", ...) { standardGeneric("saveDF") }) #' @rdname write.json #' @export From 3c2431402b0b859bbc08487012528fd8e3f4e40d Mon Sep 17 00:00:00 2001 From: Narine Kokhlikyan Date: Thu, 21 Jan 2016 11:37:01 -0800 Subject: [PATCH 5/5] fixed stylecheck error --- R/pkg/R/generics.R | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R index 8c14aaf7b770a..04784d51566cb 100644 --- a/R/pkg/R/generics.R +++ b/R/pkg/R/generics.R @@ -552,11 +552,15 @@ setGeneric("transform", function(`_data`, ...) {standardGeneric("transform") }) #' @rdname write.df #' @export -setGeneric("write.df", function(df, path, source = NULL, mode = "error", ...) { standardGeneric("write.df") }) +setGeneric("write.df", function(df, path, source = NULL, mode = "error", ...) { + standardGeneric("write.df") +}) #' @rdname write.df #' @export -setGeneric("saveDF", function(df, path, source = NULL, mode = "error", ...) { standardGeneric("saveDF") }) +setGeneric("saveDF", function(df, path, source = NULL, mode = "error", ...) { + standardGeneric("saveDF") +}) #' @rdname write.json #' @export