From 778ae86d444abc04aa303acd70b36e49a2d8e711 Mon Sep 17 00:00:00 2001 From: Octavian Sima Date: Tue, 2 Feb 2021 01:12:03 +0000 Subject: [PATCH 1/5] string to stringtype error --- .../edu/berkeley/cs/rise/opaque/Utils.scala | 40 ++++++++++++++----- .../berkeley/cs/rise/opaque/TPCHTests.scala | 2 +- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala b/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala index 46c5325a8b..2c3a1e1138 100644 --- a/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala +++ b/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala @@ -109,6 +109,7 @@ import edu.berkeley.cs.rise.opaque.expressions.VectorMultiply import edu.berkeley.cs.rise.opaque.expressions.VectorSum import edu.berkeley.cs.rise.opaque.logical.ConvertToOpaqueOperators import edu.berkeley.cs.rise.opaque.logical.EncryptLocalRelation +import org.apache.spark.sql.catalyst.expressions.PromotePrecision object Utils extends Logging { private val perf: Boolean = System.getenv("SGX_PERF") == "1" @@ -350,11 +351,22 @@ object Utils extends Logging { rdd.foreach(x => {}) } - + def castValue(value: Any, dataType: DataType): (Any, DataType) = { + var newVal = value + var newDataType = dataType + dataType match { + case _: DecimalType => { + newVal = value.toString().toFloat + newDataType = FloatType + } + case _ => + } + (newVal, newDataType) + } def flatbuffersCreateField( builder: FlatBufferBuilder, value: Any, dataType: DataType, isNull: Boolean): Int = { - (value, dataType) match { + (castValue(value, dataType)) match { case (b: Boolean, BooleanType) => tuix.Field.createField( builder, @@ -779,6 +791,17 @@ object Utils extends Logging { op(fromChildren, tree) } + def getColType(dataType: DataType) = { + dataType match { + case IntegerType => tuix.ColType.IntegerType + case LongType => tuix.ColType.LongType + case FloatType => tuix.ColType.FloatType + case DoubleType => tuix.ColType.DoubleType + case StringType => tuix.ColType.StringType + case DecimalType() => tuix.ColType.StringType + } + } + /** Serialize an Expression into a tuix.Expr. Returns the offset of the written tuix.Expr. */ def flatbuffersSerializeExpression( builder: FlatBufferBuilder, expr: Expression, input: Seq[Attribute]): Int = { @@ -811,14 +834,7 @@ object Utils extends Logging { tuix.Cast.createCast( builder, childOffset, - dataType match { - case IntegerType => tuix.ColType.IntegerType - case LongType => tuix.ColType.LongType - case FloatType => tuix.ColType.FloatType - case DoubleType => tuix.ColType.DoubleType - case StringType => tuix.ColType.StringType - })) - + getColType(dataType))) // Arithmetic case (Add(left, right), Seq(leftOffset, rightOffset)) => tuix.Expr.createExpr( @@ -1081,6 +1097,10 @@ object Utils extends Logging { tuix.ExprUnion.ClosestPoint, tuix.ClosestPoint.createClosestPoint( builder, leftOffset, rightOffset)) + case (_, Seq(childOffset)) => + // This case is used to match against CheckOverflow and promote_precision + // which are used in decimal operations. When decimals are supported, it should be deleted + childOffset } } } diff --git a/src/test/scala/edu/berkeley/cs/rise/opaque/TPCHTests.scala b/src/test/scala/edu/berkeley/cs/rise/opaque/TPCHTests.scala index d003c835f3..d014011af4 100644 --- a/src/test/scala/edu/berkeley/cs/rise/opaque/TPCHTests.scala +++ b/src/test/scala/edu/berkeley/cs/rise/opaque/TPCHTests.scala @@ -104,7 +104,7 @@ trait TPCHTests extends OpaqueTestsBase { self => tpch.query(19, securityLevel, spark.sqlContext, numPartitions).collect.toSet } - testAgainstSpark("TPC-H 20", ignore) { securityLevel => + testAgainstSpark("TPC-H 20") { securityLevel => tpch.query(20, securityLevel, spark.sqlContext, numPartitions).collect.toSet } From a0a117c85d5aa19b1e78f66b8c5ba77b9b0c9fa4 Mon Sep 17 00:00:00 2001 From: Octavian Sima Date: Tue, 2 Feb 2021 01:42:04 +0000 Subject: [PATCH 2/5] tpch 20 passes --- src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala b/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala index 2c3a1e1138..812ec287a1 100644 --- a/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala +++ b/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala @@ -798,7 +798,7 @@ object Utils extends Logging { case FloatType => tuix.ColType.FloatType case DoubleType => tuix.ColType.DoubleType case StringType => tuix.ColType.StringType - case DecimalType() => tuix.ColType.StringType + case DecimalType() => tuix.ColType.FloatType } } From e54648c596ff320854693121d0e04b2b44341dd8 Mon Sep 17 00:00:00 2001 From: Octavian Sima Date: Tue, 2 Feb 2021 01:50:15 +0000 Subject: [PATCH 3/5] cleanup --- src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala b/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala index 812ec287a1..3626d0daa2 100644 --- a/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala +++ b/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala @@ -109,7 +109,6 @@ import edu.berkeley.cs.rise.opaque.expressions.VectorMultiply import edu.berkeley.cs.rise.opaque.expressions.VectorSum import edu.berkeley.cs.rise.opaque.logical.ConvertToOpaqueOperators import edu.berkeley.cs.rise.opaque.logical.EncryptLocalRelation -import org.apache.spark.sql.catalyst.expressions.PromotePrecision object Utils extends Logging { private val perf: Boolean = System.getenv("SGX_PERF") == "1" @@ -351,7 +350,7 @@ object Utils extends Logging { rdd.foreach(x => {}) } - def castValue(value: Any, dataType: DataType): (Any, DataType) = { + def castToSupportedValue(value: Any, dataType: DataType): (Any, DataType) = { var newVal = value var newDataType = dataType dataType match { @@ -366,7 +365,7 @@ object Utils extends Logging { def flatbuffersCreateField( builder: FlatBufferBuilder, value: Any, dataType: DataType, isNull: Boolean): Int = { - (castValue(value, dataType)) match { + (castToSupportedValue(value, dataType)) match { case (b: Boolean, BooleanType) => tuix.Field.createField( builder, @@ -409,7 +408,7 @@ object Utils extends Logging { tuix.FieldUnion.FloatField, tuix.FloatField.createFloatField(builder, x), isNull) - case (null, FloatType) => + case (null, FloatType) | (null, DecimalType()) => tuix.Field.createField( builder, tuix.FieldUnion.FloatField, From c668e02a9f307aa074427faf7a7ebfb461658467 Mon Sep 17 00:00:00 2001 From: Octavian Sima Date: Wed, 3 Feb 2021 17:33:16 +0000 Subject: [PATCH 4/5] implemented changes --- .../edu/berkeley/cs/rise/opaque/Utils.scala | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala b/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala index 3626d0daa2..518ba2de18 100644 --- a/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala +++ b/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala @@ -73,7 +73,6 @@ import org.apache.spark.sql.catalyst.expressions.SortOrder import org.apache.spark.sql.catalyst.expressions.StartsWith import org.apache.spark.sql.catalyst.expressions.Substring import org.apache.spark.sql.catalyst.expressions.Subtract -import org.apache.spark.sql.catalyst.expressions.TimeAdd import org.apache.spark.sql.catalyst.expressions.UnaryMinus import org.apache.spark.sql.catalyst.expressions.Upper import org.apache.spark.sql.catalyst.expressions.Year @@ -109,6 +108,8 @@ import edu.berkeley.cs.rise.opaque.expressions.VectorMultiply import edu.berkeley.cs.rise.opaque.expressions.VectorSum import edu.berkeley.cs.rise.opaque.logical.ConvertToOpaqueOperators import edu.berkeley.cs.rise.opaque.logical.EncryptLocalRelation +import org.apache.spark.sql.catalyst.expressions.PromotePrecision +import org.apache.spark.sql.catalyst.expressions.CheckOverflow object Utils extends Logging { private val perf: Boolean = System.getenv("SGX_PERF") == "1" @@ -350,22 +351,9 @@ object Utils extends Logging { rdd.foreach(x => {}) } - def castToSupportedValue(value: Any, dataType: DataType): (Any, DataType) = { - var newVal = value - var newDataType = dataType - dataType match { - case _: DecimalType => { - newVal = value.toString().toFloat - newDataType = FloatType - } - case _ => - } - (newVal, newDataType) - } - def flatbuffersCreateField( builder: FlatBufferBuilder, value: Any, dataType: DataType, isNull: Boolean): Int = { - (castToSupportedValue(value, dataType)) match { + (value, dataType) match { case (b: Boolean, BooleanType) => tuix.Field.createField( builder, @@ -408,7 +396,19 @@ object Utils extends Logging { tuix.FieldUnion.FloatField, tuix.FloatField.createFloatField(builder, x), isNull) - case (null, FloatType) | (null, DecimalType()) => + case (null, FloatType) => + tuix.Field.createField( + builder, + tuix.FieldUnion.FloatField, + tuix.FloatField.createFloatField(builder, 0), + isNull) + case (x: Decimal, DecimalType()) => + tuix.Field.createField( + builder, + tuix.FieldUnion.FloatField, + tuix.FloatField.createFloatField(builder, x.toString().toFloat), + isNull) + case (null, DecimalType()) => tuix.Field.createField( builder, tuix.FieldUnion.FloatField, @@ -795,9 +795,10 @@ object Utils extends Logging { case IntegerType => tuix.ColType.IntegerType case LongType => tuix.ColType.LongType case FloatType => tuix.ColType.FloatType + case DecimalType() => tuix.ColType.FloatType case DoubleType => tuix.ColType.DoubleType case StringType => tuix.ColType.StringType - case DecimalType() => tuix.ColType.FloatType + case _ => throw new OpaqueException("Type not supported: " + dataType.toString()) } } @@ -1096,10 +1097,17 @@ object Utils extends Logging { tuix.ExprUnion.ClosestPoint, tuix.ClosestPoint.createClosestPoint( builder, leftOffset, rightOffset)) - case (_, Seq(childOffset)) => - // This case is used to match against CheckOverflow and promote_precision - // which are used in decimal operations. When decimals are supported, it should be deleted + + case (PromotePrecision(child), Seq(childOffset)) => + // TODO: Implement decimal serialization, followed by PromotePrecision + childOffset + + case (CheckOverflow(child, dataType, _), Seq(childOffset)) => + // TODO: Implement decimal serialization, followed by CheckOverflow childOffset + + case (_, Seq(childOffset)) => + throw new OpaqueException("Expression not supported: " + expr.toString()) } } } From 10715d49ca01696be77cae62658c9269c1f1c6ed Mon Sep 17 00:00:00 2001 From: Octavian Sima Date: Mon, 8 Feb 2021 19:16:19 +0000 Subject: [PATCH 5/5] decimal.tofloat --- src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala b/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala index 518ba2de18..6f92877b44 100644 --- a/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala +++ b/src/main/scala/edu/berkeley/cs/rise/opaque/Utils.scala @@ -406,7 +406,7 @@ object Utils extends Logging { tuix.Field.createField( builder, tuix.FieldUnion.FloatField, - tuix.FloatField.createFloatField(builder, x.toString().toFloat), + tuix.FloatField.createFloatField(builder, x.toFloat), isNull) case (null, DecimalType()) => tuix.Field.createField(