From 03274568eef0a42d2e70bcec895d9a01ed111ecf Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 12 May 2021 13:31:33 +0900 Subject: [PATCH 1/3] Fix lambda variable name issues in nested DataFrame functions in R APIs --- R/pkg/R/functions.R | 7 ++++++- R/pkg/tests/fulltests/test_sparkSQL.R | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R index ff20c9696099c..3ee1cd5b50b48 100644 --- a/R/pkg/R/functions.R +++ b/R/pkg/R/functions.R @@ -3670,7 +3670,12 @@ unresolved_named_lambda_var <- function(...) { "org.apache.spark.sql.Column", newJObject( "org.apache.spark.sql.catalyst.expressions.UnresolvedNamedLambdaVariable", - list(...) + lapply(list(...), function(x) { + handledCallJStatic( + "org.apache.spark.sql.catalyst.expressions.UnresolvedNamedLambdaVariable", + "freshVarName", + x) + }) ) ) column(jc) diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R index 4a4b7ac66ef4b..2eefad19d676e 100644 --- a/R/pkg/tests/fulltests/test_sparkSQL.R +++ b/R/pkg/tests/fulltests/test_sparkSQL.R @@ -2161,6 +2161,20 @@ test_that("higher order functions", { expect_error(array_transform("xs", function(...) 42)) }) +test("SPARK-34794: lambda variables must be resolved properly in nested higher order functions") { + df <- sql("SELECT array(1, 2, 3) as numbers, array('a', 'b', 'c') as letters") + ret <- first(select( + df, + array_transform("numbers", function(number) { + array_transform("letters", function(latter) { + struct(alias(number, "n"), alias(latter, "l")) + }) + }) + )) + + expect_equal(1, ret[[1]][[1]][[1]][[1]]$n) +} + test_that("group by, agg functions", { df <- read.json(jsonPath) df1 <- agg(df, name = "max", age = "sum") From f17b074c8c5d1e22f87e4b92523f1ceff37b7bff Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 12 May 2021 13:41:37 +0900 Subject: [PATCH 2/3] nit --- R/pkg/tests/fulltests/test_sparkSQL.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R index 2eefad19d676e..6c1a34db7e731 100644 --- a/R/pkg/tests/fulltests/test_sparkSQL.R +++ b/R/pkg/tests/fulltests/test_sparkSQL.R @@ -2161,7 +2161,7 @@ test_that("higher order functions", { expect_error(array_transform("xs", function(...) 42)) }) -test("SPARK-34794: lambda variables must be resolved properly in nested higher order functions") { +test("SPARK-34794: lambda variables must be resolved properly in nested higher order functions", { df <- sql("SELECT array(1, 2, 3) as numbers, array('a', 'b', 'c') as letters") ret <- first(select( df, @@ -2173,7 +2173,7 @@ test("SPARK-34794: lambda variables must be resolved properly in nested higher o )) expect_equal(1, ret[[1]][[1]][[1]][[1]]$n) -} +}) test_that("group by, agg functions", { df <- read.json(jsonPath) From 67e4b98e3a05eb9ca8d2313fa49e0213b57566a7 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 12 May 2021 15:15:15 +0900 Subject: [PATCH 3/3] oops --- R/pkg/tests/fulltests/test_sparkSQL.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R index 6c1a34db7e731..30daa2064355d 100644 --- a/R/pkg/tests/fulltests/test_sparkSQL.R +++ b/R/pkg/tests/fulltests/test_sparkSQL.R @@ -2161,7 +2161,7 @@ test_that("higher order functions", { expect_error(array_transform("xs", function(...) 42)) }) -test("SPARK-34794: lambda variables must be resolved properly in nested higher order functions", { +test_that("SPARK-34794: lambda vars must be resolved properly in nested higher order functions", { df <- sql("SELECT array(1, 2, 3) as numbers, array('a', 'b', 'c') as letters") ret <- first(select( df,