From 7de8ccc05158f5b60af96c8cf22a2b9e20675817 Mon Sep 17 00:00:00 2001 From: Andrew Ray Date: Tue, 1 Aug 2017 14:12:00 -0500 Subject: [PATCH 1/2] the fix --- .../datasources/jdbc/JDBCRelation.scala | 3 ++- .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala index 658d13768a976..09b46c516c499 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala @@ -64,7 +64,8 @@ private[sql] object JDBCRelation extends Logging { s"bound. Lower bound: $lowerBound; Upper bound: $upperBound") val numPartitions = - if ((upperBound - lowerBound) >= partitioning.numPartitions) { + if ((upperBound - lowerBound) >= partitioning.numPartitions || + (upperBound - lowerBound) < 0) { partitioning.numPartitions } else { logWarning("The number of partitions is reduced because the specified number of " + diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index d1daf860fdfff..dfa5c45bd8a0e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -96,6 +96,15 @@ class JDBCSuite extends SparkFunSuite | partitionColumn 'THEID', lowerBound '1', upperBound '4', numPartitions '3') """.stripMargin.replaceAll("\n", " ")) + sql( + s""" + |CREATE OR REPLACE TEMPORARY VIEW partsoverflow + |USING org.apache.spark.sql.jdbc + |OPTIONS (url '$url', dbtable 'TEST.PEOPLE', user 'testUser', password 'testPass', + | partitionColumn 'THEID', lowerBound '-9223372036854775808', + | upperBound '9223372036854775807', numPartitions '3') + """.stripMargin.replaceAll("\n", " ")) + conn.prepareStatement("create table test.inttypes (a INT, b BOOLEAN, c TINYINT, " + "d SMALLINT, e BIGINT)").executeUpdate() conn.prepareStatement("insert into test.inttypes values (1, false, 3, 4, 1234567890123)" @@ -376,6 +385,13 @@ class JDBCSuite extends SparkFunSuite assert(ids(2) === 3) } + test("overflow of partition bound difference does not give negative stride") { + val df = sql("SELECT * FROM partsoverflow") + checkNumPartitions(df, expectedNumPartitions = 3) + assert(df.collect().length == 3) + + } + test("Register JDBC query with renamed fields") { // Regression test for bug SPARK-7345 sql( From 9587bf1fc99125735583883eba30f6526744ad53 Mon Sep 17 00:00:00 2001 From: Andrew Ray Date: Tue, 1 Aug 2017 16:03:59 -0500 Subject: [PATCH 2/2] address pr comments --- .../spark/sql/execution/datasources/jdbc/JDBCRelation.scala | 4 ++-- .../src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala index 09b46c516c499..68ff53c612188 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala @@ -64,8 +64,8 @@ private[sql] object JDBCRelation extends Logging { s"bound. Lower bound: $lowerBound; Upper bound: $upperBound") val numPartitions = - if ((upperBound - lowerBound) >= partitioning.numPartitions || - (upperBound - lowerBound) < 0) { + if ((upperBound - lowerBound) >= partitioning.numPartitions || /* check for overflow */ + (upperBound - lowerBound) < 0) { partitioning.numPartitions } else { logWarning("The number of partitions is reduced because the specified number of " + diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index dfa5c45bd8a0e..24f46a6a057d9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -389,7 +389,6 @@ class JDBCSuite extends SparkFunSuite val df = sql("SELECT * FROM partsoverflow") checkNumPartitions(df, expectedNumPartitions = 3) assert(df.collect().length == 3) - } test("Register JDBC query with renamed fields") {