Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package org.apache.spark.sql
import java.math.MathContext
import java.sql.{Date, Timestamp}
import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period, ZoneId}
import java.time.temporal.ChronoUnit

import scala.collection.mutable
import scala.util.{Random, Try}
Expand Down Expand Up @@ -272,17 +273,8 @@ object RandomDataGenerator {
val ns = rand.nextLong()
new CalendarInterval(months, days, ns)
})
case DayTimeIntervalType => Some(() => {
val maxSeconds = Duration.ofDays(106751991).getSeconds
val seconds = rand.nextLong() % maxSeconds
val nanoAdjustment = rand.nextLong() % 999999000
Duration.ofSeconds(seconds, nanoAdjustment)
})
case YearMonthIntervalType => Some(() => {
val years = rand.nextInt() % 178956970
val months = rand.nextInt() % 12
Period.of(years, months, 0)
})
case DayTimeIntervalType => Some(() => Duration.of(rand.nextLong(), ChronoUnit.MICROS))
case YearMonthIntervalType => Some(() => Period.ofMonths(rand.nextInt()).normalized())
case DecimalType.Fixed(precision, scale) => Some(
() => BigDecimal.apply(
rand.nextLong() % math.pow(10, precision).toLong,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,4 +143,18 @@ class RandomDataGeneratorSuite extends SparkFunSuite with SQLHelper {
assert(!Arrays.equals(array1, arrayExpected))
assert(Arrays.equals(array2, arrayExpected))
}

test("SPARK-35116: The generated data fits the precision of DayTimeIntervalType in spark") {
Seq(DayTimeIntervalType, YearMonthIntervalType).foreach { dt =>
for (seed <- 1 to 1000) {
val generator = RandomDataGenerator.forType(dt, false, new Random(seed)).get
val toCatalyst = CatalystTypeConverters.createToCatalystConverter(dt)
val toScala = CatalystTypeConverters.createToScalaConverter(dt)
val data = generator.apply()
val catalyst = toCatalyst(data)
val convertedBack = toScala(catalyst)
assert(data == convertedBack)
}
}
}
}