From 612885a6fd6dfbe0f696ac1a5f6f2c2835b3bd84 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Wed, 5 Aug 2020 12:51:07 -0700 Subject: [PATCH 01/21] add lineage to index records --- .../hyperspace/actions/CreateActionBase.scala | 29 ++++++++++++++++--- .../hyperspace/index/IndexConstants.scala | 2 ++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala b/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala index dc3a17a1e..3dee75096 100644 --- a/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala +++ b/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala @@ -16,9 +16,11 @@ package com.microsoft.hyperspace.actions +import org.apache.commons.io.FilenameUtils import org.apache.hadoop.fs.Path import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, PartitioningAwareFileIndex} +import org.apache.spark.sql.functions.{input_file_name, udf} import org.apache.spark.sql.sources.DataSourceRegister import com.microsoft.hyperspace.HyperspaceException @@ -53,8 +55,11 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) // Resolve the passed column names with existing column names from the dataframe. val (resolvedIndexedColumns, resolvedIncludedColumns) = resolveConfig(df, indexConfig) val schema = { - val allColumns = resolvedIndexedColumns ++ resolvedIncludedColumns - df.select(allColumns.head, allColumns.tail: _*).schema + val indexConfigColumns = resolvedIndexedColumns ++ resolvedIncludedColumns + val allColumns = indexConfigColumns ++ internalColumns(df, indexConfigColumns) + df.select(allColumns.head, allColumns.tail: _*) + .withColumn(IndexConstants.DATA_FILE_NAME_COLUMN, fileName(input_file_name())) + .schema } signatureProvider.signature(df.queryExecution.optimizedPlan) match { @@ -126,8 +131,11 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) .toInt val (resolvedIndexedColumns, resolvedIncludedColumns) = resolveConfig(df, indexConfig) - val selectedColumns = resolvedIndexedColumns ++ resolvedIncludedColumns - val indexDataFrame = df.select(selectedColumns.head, selectedColumns.tail: _*) + val indexConfigColumns = resolvedIndexedColumns ++ resolvedIncludedColumns + val selectedColumns = indexConfigColumns ++ internalColumns(df, indexConfigColumns) + val indexDataFrame = df + .select(selectedColumns.head, selectedColumns.tail: _*) + .withColumn(IndexConstants.DATA_FILE_NAME_COLUMN, fileName(input_file_name())) // run job val repartitionedIndexDataFrame = @@ -163,4 +171,17 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) s"from available source columns '${dfColumnNames.mkString(",")}'") } } + + private def internalColumns(df: DataFrame, indexConfigColumns: Seq[String]): Seq[String] = { + val partitionSchema = df.queryExecution.optimizedPlan.collect { + case LogicalRelation(HadoopFsRelation(_, pSchema, _, _, _, _), _, _, _) => pSchema + } + + val spark = df.sparkSession + partitionSchema.head + .map(_.name) + .filter(ResolverUtils.resolve(spark, _, indexConfigColumns).isEmpty) + } + + private val fileName = udf((fullFilePath: String) => FilenameUtils.getBaseName(fullFilePath)) } diff --git a/src/main/scala/com/microsoft/hyperspace/index/IndexConstants.scala b/src/main/scala/com/microsoft/hyperspace/index/IndexConstants.scala index b8554c78c..ca9592d2d 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/IndexConstants.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/IndexConstants.scala @@ -47,4 +47,6 @@ object IndexConstants { val PLAIN_TEXT = "plaintext" val HTML = "html" } + + val DATA_FILE_NAME_COLUMN = "_data_file_name" } From 8f7c5869b83bf84f008d5afc1c34b216f000455a Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Wed, 5 Aug 2020 16:09:11 -0700 Subject: [PATCH 02/21] port benchmark code --- .../hyperspace/perf/BenchmarkConfig.scala | 95 + .../hyperspace/perf/BenchmarkConstants.scala | 33 + .../hyperspace/perf/BenchmarkRunner.scala | 429 ++ .../hyperspace/perf/TPCDSBenchmark.scala | 4657 +++++++++++++++++ .../hyperspace/perf/TPCHBenchmark.scala | 833 +++ 5 files changed, 6047 insertions(+) create mode 100644 src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConfig.scala create mode 100644 src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConstants.scala create mode 100644 src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala create mode 100644 src/main/scala/com/microsoft/hyperspace/perf/TPCDSBenchmark.scala create mode 100644 src/main/scala/com/microsoft/hyperspace/perf/TPCHBenchmark.scala diff --git a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConfig.scala b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConfig.scala new file mode 100644 index 000000000..296d90a65 --- /dev/null +++ b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConfig.scala @@ -0,0 +1,95 @@ +/* + * Copyright (2020) The Hyperspace Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.microsoft.hyperspace.perf + +import java.util.Locale + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import scala.io.Source + +class BenchmarkConfig(configFile: String) { + + private val configsMap = readBenchmarkConfigs(configFile) + + def benchmarkName: String = { + val benchmark = configsMap(BenchmarkConstants.BENCHMARK_NAME) + .asInstanceOf[String] + .trim() + .toLowerCase() + + if (benchmark.equals("tpch") || benchmark.equals("tpcds")) { + benchmark + } else { + throw new IllegalArgumentException(s"Unknown benchmark $benchmark.") + } + } + + def iterations: Int = configsMap.getOrElse(BenchmarkConstants.ITERATIONS, 1).asInstanceOf[Int] + + def dataPath: String = + configsMap(BenchmarkConstants.DATA_DIRECTORY).asInstanceOf[String] + + def systemPath: String = + configsMap(BenchmarkConstants.SYSTEM_DIRECTORY).asInstanceOf[String] + + def enableHyperspace: Boolean = + configsMap.getOrElse(BenchmarkConstants.ENABLE_HYPERSPACE, false).asInstanceOf[Boolean] + + def createIndex: Boolean = + configsMap.getOrElse(BenchmarkConstants.CREATE_INDEX, false).asInstanceOf[Boolean] + + def runWorkload: Boolean = + configsMap.getOrElse(BenchmarkConstants.RUN_WORKLOAD, false).asInstanceOf[Boolean] + + def workload: Seq[String] = configsMap(BenchmarkConstants.WORKLOAD).asInstanceOf[Seq[String]] + + def format: String = { + val format = + configsMap + .getOrElse(BenchmarkConstants.FORMAT, "") + .asInstanceOf[String] + .trim + .toLowerCase(Locale.ROOT) + if (format.equals("")) { + "parquet" + } else if (format.equals("parquet") || format.equals("csv")) { + format + } else { + throw new IllegalArgumentException(s"Unknown format $format.") + } + } + + def buckets: String = + configsMap + .getOrElse(BenchmarkConstants.INDEX_BUCKETS, BenchmarkConstants.DEFAULT_BUCKETS) + .asInstanceOf[String] + + private def readBenchmarkConfigs(filePath: String) = { + val src = Source.fromFile(filePath) + val configs = src.mkString + val mapper = new ObjectMapper + mapper.registerModule(DefaultScalaModule) + val configsMap = mapper.readValue(configs, classOf[Map[String, Any]]) + src.close() + configsMap + } +} + +object BenchmarkConfig { + def apply(configFile: String): BenchmarkConfig = new BenchmarkConfig(configFile) +} diff --git a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConstants.scala b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConstants.scala new file mode 100644 index 000000000..0b8ac4455 --- /dev/null +++ b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConstants.scala @@ -0,0 +1,33 @@ +/* + * Copyright (2020) The Hyperspace Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.microsoft.hyperspace.perf + +object BenchmarkConstants { + val BENCHMARK_NAME = "benchmark" + val ITERATIONS = "iterations" + val DATA_DIRECTORY = "data" + val SYSTEM_DIRECTORY = "system" + val ENABLE_HYPERSPACE = "enableHyperspace" + val CREATE_INDEX = "createIndex" + val RUN_WORKLOAD = "runWorkload" + val WORKLOAD = "workload" + val FORMAT = "format" + val INDEX_BUCKETS = "buckets" + + val CHUNKS_UNDEFINED = -1 + val DEFAULT_BUCKETS = "200" +} diff --git a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala new file mode 100644 index 000000000..6925252f1 --- /dev/null +++ b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala @@ -0,0 +1,429 @@ +/* + * Copyright (2020) The Hyperspace Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.microsoft.hyperspace.perf + +import org.apache.hadoop.fs.Path +import org.apache.spark.SparkConf +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.sql.functions.{current_timestamp, date_format} +import org.apache.spark.sql.types.StructType + +import com.microsoft.hyperspace._ +import com.microsoft.hyperspace.index.{IndexConfig, IndexConstants} +import com.microsoft.hyperspace.util.FileUtils + +object BenchmarkRunner { + // scalastyle:off + + def main(args: Array[String]): Unit = { + if (args.length < 1) { + throw new IllegalArgumentException(s"Path to benchmark configuration file not specified.") + } + + val configs = BenchmarkConfig(args(0)) + + val benchmarkName = configs.benchmarkName + val dataDir = configs.dataPath + val systemDir = configs.systemPath + val createIx = configs.createIndex + val runWorkload = configs.runWorkload + val enableHyperspace = configs.enableHyperspace + val format = configs.format + val buckets = configs.buckets + + val spark = SparkSession + .builder() + .getOrCreate() + + // Change log level to clean up output + spark.sparkContext.setLogLevel("ERROR") + + // Config overrides + spark.conf.set("spark.sql.join.preferSortMergeJoin", "true") + + val appStartTime = getAppStartTime(spark) + spark.conf.set(IndexConstants.INDEX_SYSTEM_PATH, systemDir) + + if (createIx) { + println(s"Creating indexes with $buckets buckets on data files in $format format ...") + println(s"\tdata directory\t$dataDir") + println(s"\tsystem directory\t$systemDir") + + val totalCreateIXStats = setupIndexes(spark, benchmarkName, dataDir, buckets, format) + + saveCreateIndexResults( + spark, + dataDir, + systemDir, + appStartTime, + benchmarkName, + format, + buckets, + totalCreateIXStats._2, + totalCreateIXStats._1) + + // Get Index Sizes + println("Collecting index sizes and printing sample records from each.") + val indexNames = + if (benchmarkName.equals("tpch")) TPCHBenchmark.recommendedIndexes.map(x => parseIndexConf(x).config.indexName) + else TPCDSBenchmark.recommendedIndexes.map(x => parseIndexConf(x).config.indexName) + + var indexesSizeStats = Seq[Tuple2[String, Long]]() + indexNames.foreach { x => + val indexDirPath = s"$systemDir/$x/v__=0" + val size = FileUtils.getDirectorySize(new Path(indexDirPath)) + indexesSizeStats :+= (x, size) + + println(s" Sample record from index $x (total index size is $size):") + spark.read.parquet(indexDirPath).show(2) + } + + println("Index Size Summary:\n") + indexesSizeStats.sortBy(_._1).foreach(xs => println(s"${xs._1}, ${xs._2}")) + } + + if (runWorkload) { + val iterations = configs.iterations + val workload = configs.workload + + // Validate workload + validateWorkload(benchmarkName, workload) + + println(s"\nRunning queries for $iterations iterations from workload $workload.") + println(s"data directory\t$dataDir") + println(s"system directory\t$systemDir") + + // Setup tables + setupBaseTables(spark, benchmarkName, dataDir, format) + + // Run benchmark + if (enableHyperspace) spark.enableHyperspace() + val benchmarkQueries = + if (benchmarkName.equals("tpch")) TPCHBenchmark.queries else TPCDSBenchmark.queries + + var stats = Seq[Tuple3[String, Long, Long]]() + for (q <- workload) { + val query = benchmarkQueries.filter(x => x.id == q).head + for (i <- 1 to iterations) { + val stat = runQuery(spark, query) + stats :+= stat + + println( + s"${stat._1} (Iteration $i, EnableHyperspace $enableHyperspace)\tExecution: ${stat._2} ms, Explain: ${stat._3} ms.") + } + } + + println("\nWorkload execution finished.") + + // show results on output + prettyPrint( + spark, + appStartTime, + benchmarkName, + format, + enableHyperspace, + buckets, + dataDir, + systemDir, + workload, + stats) + + // save results in csv format + saveRunBenchmarkResults( + spark, + dataDir, + systemDir, + appStartTime, + benchmarkName, + format, + enableHyperspace, + buckets, + workload, + stats) + } + + // Stop session + spark.stop() + } + + private def runQuery(spark: SparkSession, query: Query): Tuple3[String, Long, Long] = { + println(s"\n------- Running Query ${query.id} -------") + try { + val df = spark.sql(query.text) + + val explStartTime = System.currentTimeMillis() + df.explain(true) + val explEndTime = System.currentTimeMillis() + + val duration = { + val execStartTime = System.currentTimeMillis() + df.show + val execEndTime = System.currentTimeMillis() + execEndTime - execStartTime + } + + (query.id, duration, explEndTime - explStartTime) + } catch { + case e: Exception => + println(s"Unexpected exception running query ${query.id}: ${e.getMessage}") + (query.id, -1, -1) + } + } + + private def setupBaseTables( + spark: SparkSession, + benchmark: String, + dataDir: String, + format: String): Unit = { + println(s"Creating base tables from format $format ...") + val tables = if (benchmark.equals("tpch")) TPCHBenchmark.tables else TPCDSBenchmark.tables + tables.foreach { table => + val baseTableData = s"$dataDir/${table._1}" + val df = readBaseTableData(spark, baseTableData, format) + df.createOrReplaceTempView(table._1) + println(s"\tTable ${table._1} created from data under $baseTableData.") + } + } + + private def setupIndexes( + spark: SparkSession, + benchmark: String, + dataDir: String, + buckets: String, + format: String): Tuple2[Long, Int] = { + val hyperspace = Hyperspace() + require(!spark.conf.get(IndexConstants.INDEX_SYSTEM_PATH).isEmpty) + spark.conf.set(IndexConstants.INDEX_NUM_BUCKETS, buckets) + + val indexes = + if (benchmark.equals("tpch")) TPCHBenchmark.recommendedIndexes + else TPCDSBenchmark.recommendedIndexes + val tables = if (benchmark.equals("tpch")) TPCHBenchmark.tables else TPCDSBenchmark.tables + + val indexCount = indexes.length + val indexInfo = indexes.map(x => parseIndexConf(x)) + + // Validate index configs + indexInfo.foreach { x => + if (!validateIndexConfig(tables, x)) { + throw new IllegalArgumentException(s"Invalid indexConfig ${x.config.indexName}.") + } + } + + // Validate index buckets config + require(spark.conf.get(IndexConstants.INDEX_NUM_BUCKETS).equals(buckets)) + + // Create indexes + var totalTime = 0L + indexInfo.foreach { x => + val baseTableData = s"$dataDir/${x.tableName}" + totalTime += createIndex(spark, hyperspace, baseTableData, x, format) + } + + hyperspace.indexes.show(100) + (totalTime, indexCount) + } + + private def validateIndexConfig( + tables: Array[(String, StructType)], + indexInfo: CoveringIndex): Boolean = { + val baseTable = tables.filter(e => e._1 == indexInfo.tableName) + if (baseTable.length != 1) { return false } + + val tableColumns = baseTable(0)._2.fieldNames + val indexedColumns = indexInfo.config.indexedColumns + val includedColumns = indexInfo.config.includedColumns + indexedColumns.forall(tableColumns.contains) && includedColumns.forall(tableColumns.contains) + } + + private def validateWorkload(benchmark: String, workload: Seq[String]): Unit = { + val benchmarkQueries = + if (benchmark.equals("tpch")) TPCHBenchmark.queries else TPCDSBenchmark.queries + require(workload != null && benchmarkQueries.length > 0) + + workload.foreach { qid => + if (benchmarkQueries.filter(x => x.id == qid).isEmpty) { + throw new IllegalArgumentException(s"Unknown query $qid in the workload.") + } + } + } + + private def createIndex( + spark: SparkSession, + hyperspace: Hyperspace, + baseTableData: String, + indexInfo: CoveringIndex, + format: String): Long = { + val df = readBaseTableData(spark, baseTableData, format) + println( + s"\tCreating index ${indexInfo.config.indexName} on data files under $baseTableData ...") + + val startTime = System.currentTimeMillis() + hyperspace.createIndex(df, indexInfo.config) + val endTime = System.currentTimeMillis() + + endTime - startTime + } + + private def parseIndexConf(content: String): CoveringIndex = { + val tokens = content.split(";") + require(tokens.size == 3 || tokens.size == 4) + val indexedColumns = tokens(2).split(",") + val includedColumns = if (tokens.size == 4) { tokens(3).split(",") } else { Array[String]() } + CoveringIndex(tokens(1), IndexConfig(tokens(0), indexedColumns.toSeq, includedColumns.toSeq)) + } + + private def readBaseTableData( + spark: SparkSession, + baseTableData: String, + format: String): DataFrame = { + if (format.equals("parquet")) { + spark.read.parquet(baseTableData) + } else if (format.equals("csv")) { + spark.read.option("header", "true").csv(baseTableData) + } else { + throw new IllegalArgumentException(s"Unknown format $format.") + } + } + + private def prettyPrint( + spark: SparkSession, + startTime: String, + benchmarkName: String, + format: String, + enableHyperspace: Boolean, + buckets: String, + dataDir: String, + systemDir: String, + workload: Seq[String], + stats: Seq[Tuple3[String, Long, Long]]): Unit = { + val appId = spark.sparkContext.applicationId + val appName = spark.sparkContext.appName + val execNum = spark.conf.get("spark.executor.instances", "Not Found") + val cores = spark.conf.get("spark.executor.cores", "Not Found") + val memory = spark.conf.get("spark.executor.memory", "Not Found") + val driverMemory = spark.conf.get("spark.driver.memory", "Not Found") + val driverCores = spark.conf.get("spark.driver.cores", "Not Found") + + println(s"\n\n----- Summary -----") + println(s"***** $appName *****\n") + + println(s"***** Query Execution Times *****\n") + for (x <- workload) { + print(s"$x\t") + stats.filter(r => r._1 == x).foreach(t => print(s"\t${t._2}")) + println() + } + + println(s"\n\n***** Query Explain Times *****\n") + for (x <- workload) { + print(s"$x\t") + stats.filter(r => r._1 == x).foreach(t => print(s"\t${t._3}")) + println() + } + + println(s"\napplication start time:\t$startTime") + println(s"applicationId\t$appId") + println(s"dataset\t$benchmarkName") + println(s"format\t$format") + println(s"enableHyperspace\t$enableHyperspace") + println(s"buckets\t$buckets") + println(s"requested executor number\t$execNum") + println(s"requested executor cores\t$cores") + println(s"requested executor memory\t$memory") + println(s"driver memory\t$driverMemory") + println(s"driver cores\t$driverCores") + println(s"data directory\t$dataDir") + println(s"system directory\t$systemDir") + println(s"-------------------") + } + + private def saveRunBenchmarkResults( + spark: SparkSession, + dataPath: String, + systemPath: String, + startTime: String, + benchmarkName: String, + format: String, + enableHyperspace: Boolean, + buckets: String, + workload: Seq[String], + stats: Seq[Tuple3[String, Long, Long]]): Unit = { + val appId = spark.sparkContext.applicationId + val execNum = spark.conf.get("spark.executor.instances", "Not Found") + val cores = spark.conf.get("spark.executor.cores", "Not Found") + val memory = spark.conf.get("spark.executor.memory", "Not Found") + + val prefix = + s"$startTime,$appId,$dataPath,$systemPath,$benchmarkName,$format,$execNum,$cores,$memory,$enableHyperspace,$buckets" + + val header = + "appTS,appId,dataPath,systemPath,dataset,format,execNum,execCore,execMem,enableHyperspace,indexBuckets,queryId,iteration,qExecTime,qExplainTime" + var rows = List[String]() + for (x <- workload) { + val queryStats = stats.filter(r => r._1 == x) + for (i <- queryStats.indices) { + rows :+= s"$prefix,$x,$i,${queryStats(i)._2},${queryStats(i)._3}" + } + } + + println(s"Workload execution stats in the csv format:") + println(s"\n$header") + rows.foreach(println(_)) + } + + private def saveCreateIndexResults( + spark: SparkSession, + dataPath: String, + systemPath: String, + startTime: String, + benchmarkName: String, + format: String, + buckets: String, + ixCount: Int, + totalTime: Long): Unit = { + val appId = spark.sparkContext.applicationId + val execNum = spark.conf.get("spark.executor.instances", "Not Found") + val cores = spark.conf.get("spark.executor.cores", "Not Found") + val memory = spark.conf.get("spark.executor.memory", "Not Found") + + val statToSave = + s"$startTime,$appId,$dataPath,$systemPath,$benchmarkName,$format,$execNum,$cores,$memory,$buckets,$ixCount,$totalTime" + + val header = + "appTS,appId,dataPath,systemPath,dataset,format,execNum,execCore,execMem,indexBuckets,indexCount,totalCreateIXTime" + + println(s"Create indexes stats in the csv format:") + println(s"\n$header") + println(statToSave) + } + + private def getAppStartTime(spark: SparkSession): String = { + val dateFormat = "yyyy-MM-dd HH:mm:ss" + spark + .range(1) + .select(date_format(current_timestamp, dateFormat)) + .first() + .mkString + } + + // scalastyle:on +} + +case class CoveringIndex(tableName: String, config: IndexConfig) + +case class Query(id: String, text: String) diff --git a/src/main/scala/com/microsoft/hyperspace/perf/TPCDSBenchmark.scala b/src/main/scala/com/microsoft/hyperspace/perf/TPCDSBenchmark.scala new file mode 100644 index 000000000..4f02cea6b --- /dev/null +++ b/src/main/scala/com/microsoft/hyperspace/perf/TPCDSBenchmark.scala @@ -0,0 +1,4657 @@ +/* + * Copyright (2020) The Hyperspace Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.microsoft.hyperspace.perf + +object TPCDSBenchmark { + // scalastyle:off + + import org.apache.spark.sql.types._ + + val call_center = StructType( + Array( + StructField("cc_call_center_sk", LongType), + StructField("cc_call_center_id", StringType), + StructField("cc_rec_start_date", StringType), + StructField("cc_rec_end_date", StringType), + StructField("cc_closed_date_sk", LongType), + StructField("cc_open_date_sk", LongType), + StructField("cc_name", StringType), + StructField("cc_class", StringType), + StructField("cc_employees", IntegerType), + StructField("cc_sq_ft", IntegerType), + StructField("cc_hours", StringType), + StructField("cc_manager", StringType), + StructField("cc_mkt_id", IntegerType), + StructField("cc_mkt_class", StringType), + StructField("cc_mkt_desc", StringType), + StructField("cc_market_manager", StringType), + StructField("cc_division", IntegerType), + StructField("cc_division_name", StringType), + StructField("cc_company", IntegerType), + StructField("cc_company_name", StringType), + StructField("cc_street_number", StringType), + StructField("cc_street_name", StringType), + StructField("cc_street_type", StringType), + StructField("cc_suite_number", StringType), + StructField("cc_city", StringType), + StructField("cc_county", StringType), + StructField("cc_state", StringType), + StructField("cc_zip", StringType), + StructField("cc_country", StringType), + StructField("cc_gmt_offset", DoubleType), + StructField("cc_tax_percentage", DoubleType))) + + val catalog_page = StructType( + Array( + StructField("cp_catalog_page_sk", LongType), + StructField("cp_catalog_page_id", StringType), + StructField("cp_start_date_sk", LongType), + StructField("cp_end_date_sk", LongType), + StructField("cp_department", StringType), + StructField("cp_catalog_number", IntegerType), + StructField("cp_catalog_page_number", IntegerType), + StructField("cp_description", StringType), + StructField("cp_type", StringType))) + + val catalog_returns = StructType( + Array( + StructField("cr_returned_date_sk", LongType), + StructField("cr_returned_time_sk", LongType), + StructField("cr_item_sk", LongType), + StructField("cr_refunded_customer_sk", LongType), + StructField("cr_refunded_cdemo_sk", LongType), + StructField("cr_refunded_hdemo_sk", LongType), + StructField("cr_refunded_addr_sk", LongType), + StructField("cr_returning_customer_sk", LongType), + StructField("cr_returning_cdemo_sk", LongType), + StructField("cr_returning_hdemo_sk", LongType), + StructField("cr_returning_addr_sk", LongType), + StructField("cr_call_center_sk", LongType), + StructField("cr_catalog_page_sk", LongType), + StructField("cr_ship_mode_sk", LongType), + StructField("cr_warehouse_sk", LongType), + StructField("cr_reason_sk", LongType), + StructField("cr_order_number", LongType), + StructField("cr_return_quantity", IntegerType), + StructField("cr_return_amount", DoubleType), + StructField("cr_return_tax", DoubleType), + StructField("cr_return_amt_inc_tax", DoubleType), + StructField("cr_fee", DoubleType), + StructField("cr_return_ship_cost", DoubleType), + StructField("cr_refunded_cash", DoubleType), + StructField("cr_reversed_charge", DoubleType), + StructField("cr_store_credit", DoubleType), + StructField("cr_net_loss", DoubleType))) + + val catalog_sales = StructType( + Array( + StructField("cs_sold_date_sk", LongType), + StructField("cs_sold_time_sk", LongType), + StructField("cs_ship_date_sk", LongType), + StructField("cs_bill_customer_sk", LongType), + StructField("cs_bill_cdemo_sk", LongType), + StructField("cs_bill_hdemo_sk", LongType), + StructField("cs_bill_addr_sk", LongType), + StructField("cs_ship_customer_sk", LongType), + StructField("cs_ship_cdemo_sk", LongType), + StructField("cs_ship_hdemo_sk", LongType), + StructField("cs_ship_addr_sk", LongType), + StructField("cs_call_center_sk", LongType), + StructField("cs_catalog_page_sk", LongType), + StructField("cs_ship_mode_sk", LongType), + StructField("cs_warehouse_sk", LongType), + StructField("cs_item_sk", LongType), + StructField("cs_promo_sk", LongType), + StructField("cs_order_number", LongType), + StructField("cs_quantity", IntegerType), + StructField("cs_wholesale_cost", DoubleType), + StructField("cs_list_price", DoubleType), + StructField("cs_sales_price", DoubleType), + StructField("cs_ext_discount_amt", DoubleType), + StructField("cs_ext_sales_price", DoubleType), + StructField("cs_ext_wholesale_cost", DoubleType), + StructField("cs_ext_list_price", DoubleType), + StructField("cs_ext_tax", DoubleType), + StructField("cs_coupon_amt", DoubleType), + StructField("cs_ext_ship_cost", DoubleType), + StructField("cs_net_paid", DoubleType), + StructField("cs_net_paid_inc_tax", DoubleType), + StructField("cs_net_paid_inc_ship", DoubleType), + StructField("cs_net_paid_inc_ship_tax", DoubleType), + StructField("cs_net_profit", DoubleType))) + + val customer_address = StructType( + Array( + StructField("ca_address_sk", LongType), + StructField("ca_address_id", StringType), + StructField("ca_street_number", StringType), + StructField("ca_street_name", StringType), + StructField("ca_street_type", StringType), + StructField("ca_suite_number", StringType), + StructField("ca_city", StringType), + StructField("ca_county", StringType), + StructField("ca_state", StringType), + StructField("ca_zip", StringType), + StructField("ca_country", StringType), + StructField("ca_gmt_offset", DoubleType), + StructField("ca_location_type", StringType))) + + val customer_demographics = StructType( + Array( + StructField("cd_demo_sk", LongType), + StructField("cd_gender", StringType), + StructField("cd_marital_status", StringType), + StructField("cd_education_status", StringType), + StructField("cd_purchase_estimate", IntegerType), + StructField("cd_credit_rating", StringType), + StructField("cd_dep_count", IntegerType), + StructField("cd_dep_employed_count", IntegerType), + StructField("cd_dep_college_count", IntegerType))) + + val customer = StructType( + Array( + StructField("c_customer_sk", LongType), + StructField("c_customer_id", StringType), + StructField("c_current_cdemo_sk", LongType), + StructField("c_current_hdemo_sk", LongType), + StructField("c_current_addr_sk", LongType), + StructField("c_first_shipto_date_sk", LongType), + StructField("c_first_sales_date_sk", LongType), + StructField("c_salutation", StringType), + StructField("c_first_name", StringType), + StructField("c_last_name", StringType), + StructField("c_preferred_cust_flag", StringType), + StructField("c_birth_day", IntegerType), + StructField("c_birth_month", IntegerType), + StructField("c_birth_year", IntegerType), + StructField("c_birth_country", StringType), + StructField("c_login", StringType), + StructField("c_email_address", StringType), + StructField("c_last_review_date", StringType))) + + val date_dim = StructType( + Array( + StructField("d_date_sk", LongType), + StructField("d_date_id", StringType), + StructField("d_date", StringType), + StructField("d_month_seq", IntegerType), + StructField("d_week_seq", IntegerType), + StructField("d_quarter_seq", IntegerType), + StructField("d_year", IntegerType), + StructField("d_dow", IntegerType), + StructField("d_moy", IntegerType), + StructField("d_dom", IntegerType), + StructField("d_qoy", IntegerType), + StructField("d_fy_year", IntegerType), + StructField("d_fy_quarter_seq", IntegerType), + StructField("d_fy_week_seq", IntegerType), + StructField("d_day_name", StringType), + StructField("d_quarter_name", StringType), + StructField("d_holiday", StringType), + StructField("d_weekend", StringType), + StructField("d_following_holiday", StringType), + StructField("d_first_dom", IntegerType), + StructField("d_last_dom", IntegerType), + StructField("d_same_day_ly", IntegerType), + StructField("d_same_day_lq", IntegerType), + StructField("d_current_day", StringType), + StructField("d_current_week", StringType), + StructField("d_current_month", StringType), + StructField("d_current_quarter", StringType), + StructField("d_current_year", StringType))) + + val household_demographics = StructType( + Array( + StructField("hd_demo_sk", LongType), + StructField("hd_income_band_sk", LongType), + StructField("hd_buy_potential", StringType), + StructField("hd_dep_count", IntegerType), + StructField("hd_vehicle_count", IntegerType))) + + val inventory = StructType( + Array( + StructField("inv_date_sk", LongType), + StructField("inv_item_sk", LongType), + StructField("inv_warehouse_sk", LongType), + StructField("inv_quantity_on_hand", IntegerType))) + + val income_band = StructType( + Array( + StructField("ib_income_band_sk", LongType), + StructField("ib_lower_bound", IntegerType), + StructField("ib_upper_bound", IntegerType))) + + val item = StructType( + Array( + StructField("i_item_sk", LongType), + StructField("i_item_id", StringType), + StructField("i_rec_start_date", StringType), + StructField("i_rec_end_date", StringType), + StructField("i_item_desc", StringType), + StructField("i_current_price", DoubleType), + StructField("i_wholesale_cost", DoubleType), + StructField("i_brand_id", IntegerType), + StructField("i_brand", StringType), + StructField("i_class_id", IntegerType), + StructField("i_class", StringType), + StructField("i_category_id", IntegerType), + StructField("i_category", StringType), + StructField("i_manufact_id", IntegerType), + StructField("i_manufact", StringType), + StructField("i_size", StringType), + StructField("i_formulation", StringType), + StructField("i_color", StringType), + StructField("i_units", StringType), + StructField("i_container", StringType), + StructField("i_manager_id", IntegerType), + StructField("i_product_name", StringType))) + + val promotion = StructType( + Array( + StructField("p_promo_sk", LongType), + StructField("p_promo_id", StringType), + StructField("p_start_date_sk", LongType), + StructField("p_end_date_sk", LongType), + StructField("p_item_sk", LongType), + StructField("p_cost", DoubleType), + StructField("p_response_target", IntegerType), + StructField("p_promo_name", StringType), + StructField("p_channel_dmail", StringType), + StructField("p_channel_email", StringType), + StructField("p_channel_catalog", StringType), + StructField("p_channel_tv", StringType), + StructField("p_channel_radio", StringType), + StructField("p_channel_press", StringType), + StructField("p_channel_event", StringType), + StructField("p_channel_demo", StringType), + StructField("p_channel_details", StringType), + StructField("p_purpose", StringType), + StructField("p_discount_active", StringType))) + + val reason = StructType( + Array( + StructField("r_reason_sk", LongType), + StructField("r_reason_id", StringType), + StructField("r_reason_desc", StringType))) + + val ship_mode = StructType( + Array( + StructField("sm_ship_mode_sk", LongType), + StructField("sm_ship_mode_id", StringType), + StructField("sm_type", StringType), + StructField("sm_code", StringType), + StructField("sm_carrier", StringType), + StructField("sm_contract", StringType))) + + val store_returns = StructType( + Array( + StructField("sr_returned_date_sk", LongType), + StructField("sr_return_time_sk", LongType), + StructField("sr_item_sk", LongType), + StructField("sr_customer_sk", LongType), + StructField("sr_cdemo_sk", LongType), + StructField("sr_hdemo_sk", LongType), + StructField("sr_addr_sk", LongType), + StructField("sr_store_sk", LongType), + StructField("sr_reason_sk", LongType), + StructField("sr_ticket_number", LongType), + StructField("sr_return_quantity", IntegerType), + StructField("sr_return_amt", DoubleType), + StructField("sr_return_tax", DoubleType), + StructField("sr_return_amt_inc_tax", DoubleType), + StructField("sr_fee", DoubleType), + StructField("sr_return_ship_cost", DoubleType), + StructField("sr_refunded_cash", DoubleType), + StructField("sr_reversed_charge", DoubleType), + StructField("sr_store_credit", DoubleType), + StructField("sr_net_loss", DoubleType))) + + val store_sales = StructType( + Array( + StructField("ss_sold_date_sk", LongType), + StructField("ss_sold_time_sk", LongType), + StructField("ss_item_sk", LongType), + StructField("ss_customer_sk", LongType), + StructField("ss_cdemo_sk", LongType), + StructField("ss_hdemo_sk", LongType), + StructField("ss_addr_sk", LongType), + StructField("ss_store_sk", LongType), + StructField("ss_promo_sk", LongType), + StructField("ss_ticket_number", LongType), + StructField("ss_quantity", IntegerType), + StructField("ss_wholesale_cost", DoubleType), + StructField("ss_list_price", DoubleType), + StructField("ss_sales_price", DoubleType), + StructField("ss_ext_discount_amt", DoubleType), + StructField("ss_ext_sales_price", DoubleType), + StructField("ss_ext_wholesale_cost", DoubleType), + StructField("ss_ext_list_price", DoubleType), + StructField("ss_ext_tax", DoubleType), + StructField("ss_coupon_amt", DoubleType), + StructField("ss_net_paid", DoubleType), + StructField("ss_net_paid_inc_tax", DoubleType), + StructField("ss_net_profit", DoubleType))) + + val store = StructType( + Array( + StructField("s_store_sk", LongType), + StructField("s_store_id", StringType), + StructField("s_rec_start_date", StringType), + StructField("s_rec_end_date", StringType), + StructField("s_closed_date_sk", LongType), + StructField("s_store_name", StringType), + StructField("s_number_employees", IntegerType), + StructField("s_floor_space", IntegerType), + StructField("s_hours", StringType), + StructField("s_manager", StringType), + StructField("s_market_id", IntegerType), + StructField("s_geography_class", StringType), + StructField("s_market_desc", StringType), + StructField("s_market_manager", StringType), + StructField("s_division_id", IntegerType), + StructField("s_division_name", StringType), + StructField("s_company_id", IntegerType), + StructField("s_company_name", StringType), + StructField("s_street_number", StringType), + StructField("s_street_name", StringType), + StructField("s_street_type", StringType), + StructField("s_suite_number", StringType), + StructField("s_city", StringType), + StructField("s_county", StringType), + StructField("s_state", StringType), + StructField("s_zip", StringType), + StructField("s_country", StringType), + StructField("s_gmt_offset", DoubleType), + StructField("s_tax_precentage", DoubleType))) + + val time_dim = StructType( + Array( + StructField("t_time_sk", LongType), + StructField("t_time_id", StringType), + StructField("t_time", IntegerType), + StructField("t_hour", IntegerType), + StructField("t_minute", IntegerType), + StructField("t_second", IntegerType), + StructField("t_am_pm", StringType), + StructField("t_shift", StringType), + StructField("t_sub_shift", StringType), + StructField("t_meal_time", StringType))) + + val warehouse = StructType( + Array( + StructField("w_warehouse_sk", LongType), + StructField("w_warehouse_id", StringType), + StructField("w_warehouse_name", StringType), + StructField("w_warehouse_sq_ft", IntegerType), + StructField("w_street_number", StringType), + StructField("w_street_name", StringType), + StructField("w_street_type", StringType), + StructField("w_suite_number", StringType), + StructField("w_city", StringType), + StructField("w_county", StringType), + StructField("w_state", StringType), + StructField("w_zip", StringType), + StructField("w_country", StringType), + StructField("w_gmt_offset", DoubleType))) + + val web_page = StructType( + Array( + StructField("wp_web_page_sk", LongType), + StructField("wp_web_page_id", StringType), + StructField("wp_rec_start_date", StringType), + StructField("wp_rec_end_date", StringType), + StructField("wp_creation_date_sk", LongType), + StructField("wp_access_date_sk", LongType), + StructField("wp_autogen_flag", StringType), + StructField("wp_customer_sk", LongType), + StructField("wp_url", StringType), + StructField("wp_type", StringType), + StructField("wp_char_count", IntegerType), + StructField("wp_link_count", IntegerType), + StructField("wp_image_count", IntegerType), + StructField("wp_max_ad_count", IntegerType))) + + val web_returns = StructType( + Array( + StructField("wr_returned_date_sk", LongType), + StructField("wr_returned_time_sk", LongType), + StructField("wr_item_sk", LongType), + StructField("wr_refunded_customer_sk", LongType), + StructField("wr_refunded_cdemo_sk", LongType), + StructField("wr_refunded_hdemo_sk", LongType), + StructField("wr_refunded_addr_sk", LongType), + StructField("wr_returning_customer_sk", LongType), + StructField("wr_returning_cdemo_sk", LongType), + StructField("wr_returning_hdemo_sk", LongType), + StructField("wr_returning_addr_sk", LongType), + StructField("wr_web_page_sk", LongType), + StructField("wr_reason_sk", LongType), + StructField("wr_order_number", LongType), + StructField("wr_return_quantity", IntegerType), + StructField("wr_return_amt", DoubleType), + StructField("wr_return_tax", DoubleType), + StructField("wr_return_amt_inc_tax", DoubleType), + StructField("wr_fee", DoubleType), + StructField("wr_return_ship_cost", DoubleType), + StructField("wr_refunded_cash", DoubleType), + StructField("wr_reversed_charge", DoubleType), + StructField("wr_account_credit", DoubleType), + StructField("wr_net_loss", DoubleType))) + + val web_sales = StructType( + Array( + StructField("ws_sold_date_sk", LongType), + StructField("ws_sold_time_sk", LongType), + StructField("ws_ship_date_sk", LongType), + StructField("ws_item_sk", LongType), + StructField("ws_bill_customer_sk", LongType), + StructField("ws_bill_cdemo_sk", LongType), + StructField("ws_bill_hdemo_sk", LongType), + StructField("ws_bill_addr_sk", LongType), + StructField("ws_ship_customer_sk", LongType), + StructField("ws_ship_cdemo_sk", LongType), + StructField("ws_ship_hdemo_sk", LongType), + StructField("ws_ship_addr_sk", LongType), + StructField("ws_web_page_sk", LongType), + StructField("ws_web_site_sk", LongType), + StructField("ws_ship_mode_sk", LongType), + StructField("ws_warehouse_sk", LongType), + StructField("ws_promo_sk", LongType), + StructField("ws_order_number", LongType), + StructField("ws_quantity", IntegerType), + StructField("ws_wholesale_cost", DoubleType), + StructField("ws_list_price", DoubleType), + StructField("ws_sales_price", DoubleType), + StructField("ws_ext_discount_amt", DoubleType), + StructField("ws_ext_sales_price", DoubleType), + StructField("ws_ext_wholesale_cost", DoubleType), + StructField("ws_ext_list_price", DoubleType), + StructField("ws_ext_tax", DoubleType), + StructField("ws_coupon_amt", DoubleType), + StructField("ws_ext_ship_cost", DoubleType), + StructField("ws_net_paid", DoubleType), + StructField("ws_net_paid_inc_tax", DoubleType), + StructField("ws_net_paid_inc_ship", DoubleType), + StructField("ws_net_paid_inc_ship_tax", DoubleType), + StructField("ws_net_profit", DoubleType))) + + val web_site = StructType( + Array( + StructField("web_site_sk", LongType), + StructField("web_site_id", StringType), + StructField("web_rec_start_date", StringType), + StructField("web_rec_end_date", StringType), + StructField("web_name", StringType), + StructField("web_open_date_sk", LongType), + StructField("web_close_date_sk", LongType), + StructField("web_class", StringType), + StructField("web_manager", StringType), + StructField("web_mkt_id", IntegerType), + StructField("web_mkt_class", StringType), + StructField("web_mkt_desc", StringType), + StructField("web_market_manager", StringType), + StructField("web_company_id", IntegerType), + StructField("web_company_name", StringType), + StructField("web_street_number", StringType), + StructField("web_street_name", StringType), + StructField("web_street_type", StringType), + StructField("web_suite_number", StringType), + StructField("web_city", StringType), + StructField("web_county", StringType), + StructField("web_state", StringType), + StructField("web_zip", StringType), + StructField("web_country", StringType), + StructField("web_gmt_offset", DoubleType), + StructField("web_tax_percentage", DoubleType))) + + val tables = Array( + ("call_center", call_center), + ("catalog_page", catalog_page), + ("catalog_returns", catalog_returns), + ("catalog_sales", catalog_sales), + ("customer_address", customer_address), + ("customer_demographics", customer_demographics), + ("customer", customer), + ("date_dim", date_dim), + ("household_demographics", household_demographics), + ("income_band", income_band), + ("inventory", inventory), + ("item", item), + ("promotion", promotion), + ("reason", reason), + ("ship_mode", ship_mode), + ("store_returns", store_returns), + ("store_sales", store_sales), + ("store", store), + ("time_dim", time_dim), + ("warehouse", warehouse), + ("web_page", web_page), + ("web_returns", web_returns), + ("web_sales", web_sales), + ("web_site", web_site)) + + // should be random generated based on scale + // RC=ulist(random(1, rowcount("store_sales")/5,uniform),5); + val rc = Array(1000000, 1000000, 1000000, 1000000, 1000000) + + val queries = Array( + Query( + "q1", + """ + | WITH customer_total_return AS + | (SELECT sr_customer_sk AS ctr_customer_sk, sr_store_sk AS ctr_store_sk, + | sum(sr_return_amt) AS ctr_total_return + | FROM store_returns, date_dim + | WHERE sr_returned_date_sk = d_date_sk AND d_year = 2000 + | GROUP BY sr_customer_sk, sr_store_sk) + | SELECT c_customer_id + | FROM customer_total_return ctr1, store, customer + | WHERE ctr1.ctr_total_return > + | (SELECT avg(ctr_total_return)*1.2 + | FROM customer_total_return ctr2 + | WHERE ctr1.ctr_store_sk = ctr2.ctr_store_sk) + | AND s_store_sk = ctr1.ctr_store_sk + | AND s_state = 'TN' + | AND ctr1.ctr_customer_sk = c_customer_sk + | ORDER BY c_customer_id LIMIT 100 + """.stripMargin), + Query( + "q2", + """ + | WITH wscs as + | (SELECT sold_date_sk, sales_price + | FROM (SELECT ws_sold_date_sk sold_date_sk, ws_ext_sales_price sales_price + | FROM web_sales) x + | UNION ALL + | (SELECT cs_sold_date_sk sold_date_sk, cs_ext_sales_price sales_price + | FROM catalog_sales)), + | wswscs AS + | (SELECT d_week_seq, + | sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, + | sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, + | sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, + | sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, + | sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, + | sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, + | sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales + | FROM wscs, date_dim + | WHERE d_date_sk = sold_date_sk + | GROUP BY d_week_seq) + | SELECT d_week_seq1 + | ,round(sun_sales1/sun_sales2,2) + | ,round(mon_sales1/mon_sales2,2) + | ,round(tue_sales1/tue_sales2,2) + | ,round(wed_sales1/wed_sales2,2) + | ,round(thu_sales1/thu_sales2,2) + | ,round(fri_sales1/fri_sales2,2) + | ,round(sat_sales1/sat_sales2,2) + | FROM + | (SELECT wswscs.d_week_seq d_week_seq1 + | ,sun_sales sun_sales1 + | ,mon_sales mon_sales1 + | ,tue_sales tue_sales1 + | ,wed_sales wed_sales1 + | ,thu_sales thu_sales1 + | ,fri_sales fri_sales1 + | ,sat_sales sat_sales1 + | FROM wswscs,date_dim + | WHERE date_dim.d_week_seq = wswscs.d_week_seq AND d_year = 2001) y, + | (SELECT wswscs.d_week_seq d_week_seq2 + | ,sun_sales sun_sales2 + | ,mon_sales mon_sales2 + | ,tue_sales tue_sales2 + | ,wed_sales wed_sales2 + | ,thu_sales thu_sales2 + | ,fri_sales fri_sales2 + | ,sat_sales sat_sales2 + | FROM wswscs, date_dim + | WHERE date_dim.d_week_seq = wswscs.d_week_seq AND d_year = 2001 + 1) z + | WHERE d_week_seq1=d_week_seq2-53 + | ORDER BY d_week_seq1 + """.stripMargin), + Query( + "q3", + """ + | SELECT dt.d_year, item.i_brand_id brand_id, item.i_brand brand,SUM(ss_ext_sales_price) sum_agg + | FROM date_dim dt, store_sales, item + | WHERE dt.d_date_sk = store_sales.ss_sold_date_sk + | AND store_sales.ss_item_sk = item.i_item_sk + | AND item.i_manufact_id = 128 + | AND dt.d_moy=11 + | GROUP BY dt.d_year, item.i_brand, item.i_brand_id + | ORDER BY dt.d_year, sum_agg desc, brand_id + | LIMIT 100 + """.stripMargin), + Query( + "q4", + """ + |WITH year_total AS ( + | SELECT c_customer_id customer_id, + | c_first_name customer_first_name, + | c_last_name customer_last_name, + | c_preferred_cust_flag customer_preferred_cust_flag, + | c_birth_country customer_birth_country, + | c_login customer_login, + | c_email_address customer_email_address, + | d_year dyear, + | sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total, + | 's' sale_type + | FROM customer, store_sales, date_dim + | WHERE c_customer_sk = ss_customer_sk AND ss_sold_date_sk = d_date_sk + | GROUP BY c_customer_id, + | c_first_name, + | c_last_name, + | c_preferred_cust_flag, + | c_birth_country, + | c_login, + | c_email_address, + | d_year + | UNION ALL + | SELECT c_customer_id customer_id, + | c_first_name customer_first_name, + | c_last_name customer_last_name, + | c_preferred_cust_flag customer_preferred_cust_flag, + | c_birth_country customer_birth_country, + | c_login customer_login, + | c_email_address customer_email_address, + | d_year dyear, + | sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total, + | 'c' sale_type + | FROM customer, catalog_sales, date_dim + | WHERE c_customer_sk = cs_bill_customer_sk AND cs_sold_date_sk = d_date_sk + | GROUP BY c_customer_id, + | c_first_name, + | c_last_name, + | c_preferred_cust_flag, + | c_birth_country, + | c_login, + | c_email_address, + | d_year + | UNION ALL + | SELECT c_customer_id customer_id + | ,c_first_name customer_first_name + | ,c_last_name customer_last_name + | ,c_preferred_cust_flag customer_preferred_cust_flag + | ,c_birth_country customer_birth_country + | ,c_login customer_login + | ,c_email_address customer_email_address + | ,d_year dyear + | ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total + | ,'w' sale_type + | FROM customer, web_sales, date_dim + | WHERE c_customer_sk = ws_bill_customer_sk AND ws_sold_date_sk = d_date_sk + | GROUP BY c_customer_id, + | c_first_name, + | c_last_name, + | c_preferred_cust_flag, + | c_birth_country, + | c_login, + | c_email_address, + | d_year) + | SELECT + | t_s_secyear.customer_id, + | t_s_secyear.customer_first_name, + | t_s_secyear.customer_last_name, + | t_s_secyear.customer_preferred_cust_flag, + | t_s_secyear.customer_birth_country, + | t_s_secyear.customer_login, + | t_s_secyear.customer_email_address + | FROM year_total t_s_firstyear, year_total t_s_secyear, year_total t_c_firstyear, + | year_total t_c_secyear, year_total t_w_firstyear, year_total t_w_secyear + | WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id + | and t_s_firstyear.customer_id = t_c_secyear.customer_id + | and t_s_firstyear.customer_id = t_c_firstyear.customer_id + | and t_s_firstyear.customer_id = t_w_firstyear.customer_id + | and t_s_firstyear.customer_id = t_w_secyear.customer_id + | and t_s_firstyear.sale_type = 's' + | and t_c_firstyear.sale_type = 'c' + | and t_w_firstyear.sale_type = 'w' + | and t_s_secyear.sale_type = 's' + | and t_c_secyear.sale_type = 'c' + | and t_w_secyear.sale_type = 'w' + | and t_s_firstyear.dyear = 2001 + | and t_s_secyear.dyear = 2001+1 + | and t_c_firstyear.dyear = 2001 + | and t_c_secyear.dyear = 2001+1 + | and t_w_firstyear.dyear = 2001 + | and t_w_secyear.dyear = 2001+1 + | and t_s_firstyear.year_total > 0 + | and t_c_firstyear.year_total > 0 + | and t_w_firstyear.year_total > 0 + | and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end + | > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end + | and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end + | > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end + | ORDER BY + | t_s_secyear.customer_id, + | t_s_secyear.customer_first_name, + | t_s_secyear.customer_last_name, + | t_s_secyear.customer_preferred_cust_flag, + | t_s_secyear.customer_birth_country, + | t_s_secyear.customer_login, + | t_s_secyear.customer_email_address + | LIMIT 100 + """.stripMargin), + // Modifications: "+ days" -> date_add + // Modifications: "||" -> concat + Query( + "q5", + """ + | WITH ssr AS + | (SELECT s_store_id, + | sum(sales_price) as sales, + | sum(profit) as profit, + | sum(return_amt) as returns, + | sum(net_loss) as profit_loss + | FROM + | (SELECT ss_store_sk as store_sk, + | ss_sold_date_sk as date_sk, + | ss_ext_sales_price as sales_price, + | ss_net_profit as profit, + | cast(0 as decimal(7,2)) as return_amt, + | cast(0 as decimal(7,2)) as net_loss + | FROM store_sales + | UNION ALL + | SELECT sr_store_sk as store_sk, + | sr_returned_date_sk as date_sk, + | cast(0 as decimal(7,2)) as sales_price, + | cast(0 as decimal(7,2)) as profit, + | sr_return_amt as return_amt, + | sr_net_loss as net_loss + | FROM store_returns) + | salesreturns, date_dim, store + | WHERE date_sk = d_date_sk + | and d_date between cast('2000-08-23' as date) + | and ((cast('2000-08-23' as date) + interval 14 days)) + | and store_sk = s_store_sk + | GROUP BY s_store_id), + | csr AS + | (SELECT cp_catalog_page_id, + | sum(sales_price) as sales, + | sum(profit) as profit, + | sum(return_amt) as returns, + | sum(net_loss) as profit_loss + | FROM + | (SELECT cs_catalog_page_sk as page_sk, + | cs_sold_date_sk as date_sk, + | cs_ext_sales_price as sales_price, + | cs_net_profit as profit, + | cast(0 as decimal(7,2)) as return_amt, + | cast(0 as decimal(7,2)) as net_loss + | FROM catalog_sales + | UNION ALL + | SELECT cr_catalog_page_sk as page_sk, + | cr_returned_date_sk as date_sk, + | cast(0 as decimal(7,2)) as sales_price, + | cast(0 as decimal(7,2)) as profit, + | cr_return_amount as return_amt, + | cr_net_loss as net_loss + | from catalog_returns + | ) salesreturns, date_dim, catalog_page + | WHERE date_sk = d_date_sk + | and d_date between cast('2000-08-23' as date) + | and ((cast('2000-08-23' as date) + interval 14 days)) + | and page_sk = cp_catalog_page_sk + | GROUP BY cp_catalog_page_id) + | , + | wsr AS + | (SELECT web_site_id, + | sum(sales_price) as sales, + | sum(profit) as profit, + | sum(return_amt) as returns, + | sum(net_loss) as profit_loss + | from + | (select ws_web_site_sk as wsr_web_site_sk, + | ws_sold_date_sk as date_sk, + | ws_ext_sales_price as sales_price, + | ws_net_profit as profit, + | cast(0 as decimal(7,2)) as return_amt, + | cast(0 as decimal(7,2)) as net_loss + | from web_sales + | union all + | select ws_web_site_sk as wsr_web_site_sk, + | wr_returned_date_sk as date_sk, + | cast(0 as decimal(7,2)) as sales_price, + | cast(0 as decimal(7,2)) as profit, + | wr_return_amt as return_amt, + | wr_net_loss as net_loss + | FROM web_returns LEFT OUTER JOIN web_sales on + | ( wr_item_sk = ws_item_sk + | and wr_order_number = ws_order_number) + | ) salesreturns, date_dim, web_site + | WHERE date_sk = d_date_sk + | and d_date between cast('2000-08-23' as date) + | and ((cast('2000-08-23' as date) + interval 14 days)) + | and wsr_web_site_sk = web_site_sk + | GROUP BY web_site_id) + | SELECT channel, + | id, + | sum(sales) as sales, + | sum(returns) as returns, + | sum(profit) as profit + | from + | (select 'store channel' as channel, + | concat('store', s_store_id) as id, + | sales, + | returns, + | (profit - profit_loss) as profit + | FROM ssr + | UNION ALL + | select 'catalog channel' as channel, + | concat('catalog_page', cp_catalog_page_id) as id, + | sales, + | returns, + | (profit - profit_loss) as profit + | FROM csr + | UNION ALL + | SELECT 'web channel' as channel, + | concat('web_site', web_site_id) as id, + | sales, + | returns, + | (profit - profit_loss) as profit + | FROM wsr + | ) x + | GROUP BY ROLLUP (channel, id) + | ORDER BY channel, id + | LIMIT 100 + """.stripMargin), + Query( + "q6", + """ + | SELECT a.ca_state state, count(*) cnt + | FROM + | customer_address a, customer c, store_sales s, date_dim d, item i + | WHERE a.ca_address_sk = c.c_current_addr_sk + | AND c.c_customer_sk = s.ss_customer_sk + | AND s.ss_sold_date_sk = d.d_date_sk + | AND s.ss_item_sk = i.i_item_sk + | AND d.d_month_seq = + | (SELECT distinct (d_month_seq) FROM date_dim + | WHERE d_year = 2000 AND d_moy = 1) + | AND i.i_current_price > 1.2 * + | (SELECT avg(j.i_current_price) FROM item j + | WHERE j.i_category = i.i_category) + | GROUP BY a.ca_state + | HAVING count(*) >= 10 + | ORDER BY cnt LIMIT 100 + """.stripMargin), + Query( + "q7", + """ + | SELECT i_item_id, + | avg(ss_quantity) agg1, + | avg(ss_list_price) agg2, + | avg(ss_coupon_amt) agg3, + | avg(ss_sales_price) agg4 + | FROM store_sales, customer_demographics, date_dim, item, promotion + | WHERE ss_sold_date_sk = d_date_sk AND + | ss_item_sk = i_item_sk AND + | ss_cdemo_sk = cd_demo_sk AND + | ss_promo_sk = p_promo_sk AND + | cd_gender = 'M' AND + | cd_marital_status = 'S' AND + | cd_education_status = 'College' AND + | (p_channel_email = 'N' or p_channel_event = 'N') AND + | d_year = 2000 + | GROUP BY i_item_id + | ORDER BY i_item_id LIMIT 100 + """.stripMargin), + Query( + "q8", + """ + | select s_store_name, sum(ss_net_profit) + | from store_sales, date_dim, store, + | (SELECT ca_zip + | from ( + | (SELECT substr(ca_zip,1,5) ca_zip FROM customer_address + | WHERE substr(ca_zip,1,5) IN ( + | '24128','76232','65084','87816','83926','77556','20548', + | '26231','43848','15126','91137','61265','98294','25782', + | '17920','18426','98235','40081','84093','28577','55565', + | '17183','54601','67897','22752','86284','18376','38607', + | '45200','21756','29741','96765','23932','89360','29839', + | '25989','28898','91068','72550','10390','18845','47770', + | '82636','41367','76638','86198','81312','37126','39192', + | '88424','72175','81426','53672','10445','42666','66864', + | '66708','41248','48583','82276','18842','78890','49448', + | '14089','38122','34425','79077','19849','43285','39861', + | '66162','77610','13695','99543','83444','83041','12305', + | '57665','68341','25003','57834','62878','49130','81096', + | '18840','27700','23470','50412','21195','16021','76107', + | '71954','68309','18119','98359','64544','10336','86379', + | '27068','39736','98569','28915','24206','56529','57647', + | '54917','42961','91110','63981','14922','36420','23006', + | '67467','32754','30903','20260','31671','51798','72325', + | '85816','68621','13955','36446','41766','68806','16725', + | '15146','22744','35850','88086','51649','18270','52867', + | '39972','96976','63792','11376','94898','13595','10516', + | '90225','58943','39371','94945','28587','96576','57855', + | '28488','26105','83933','25858','34322','44438','73171', + | '30122','34102','22685','71256','78451','54364','13354', + | '45375','40558','56458','28286','45266','47305','69399', + | '83921','26233','11101','15371','69913','35942','15882', + | '25631','24610','44165','99076','33786','70738','26653', + | '14328','72305','62496','22152','10144','64147','48425', + | '14663','21076','18799','30450','63089','81019','68893', + | '24996','51200','51211','45692','92712','70466','79994', + | '22437','25280','38935','71791','73134','56571','14060', + | '19505','72425','56575','74351','68786','51650','20004', + | '18383','76614','11634','18906','15765','41368','73241', + | '76698','78567','97189','28545','76231','75691','22246', + | '51061','90578','56691','68014','51103','94167','57047', + | '14867','73520','15734','63435','25733','35474','24676', + | '94627','53535','17879','15559','53268','59166','11928', + | '59402','33282','45721','43933','68101','33515','36634', + | '71286','19736','58058','55253','67473','41918','19515', + | '36495','19430','22351','77191','91393','49156','50298', + | '87501','18652','53179','18767','63193','23968','65164', + | '68880','21286','72823','58470','67301','13394','31016', + | '70372','67030','40604','24317','45748','39127','26065', + | '77721','31029','31880','60576','24671','45549','13376', + | '50016','33123','19769','22927','97789','46081','72151', + | '15723','46136','51949','68100','96888','64528','14171', + | '79777','28709','11489','25103','32213','78668','22245', + | '15798','27156','37930','62971','21337','51622','67853', + | '10567','38415','15455','58263','42029','60279','37125', + | '56240','88190','50308','26859','64457','89091','82136', + | '62377','36233','63837','58078','17043','30010','60099', + | '28810','98025','29178','87343','73273','30469','64034', + | '39516','86057','21309','90257','67875','40162','11356', + | '73650','61810','72013','30431','22461','19512','13375', + | '55307','30625','83849','68908','26689','96451','38193', + | '46820','88885','84935','69035','83144','47537','56616', + | '94983','48033','69952','25486','61547','27385','61860', + | '58048','56910','16807','17871','35258','31387','35458', + | '35576')) + | INTERSECT + | (select ca_zip + | FROM + | (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt + | FROM customer_address, customer + | WHERE ca_address_sk = c_current_addr_sk and + | c_preferred_cust_flag='Y' + | group by ca_zip + | having count(*) > 10) A1) + | ) A2 + | ) V1 + | where ss_store_sk = s_store_sk + | and ss_sold_date_sk = d_date_sk + | and d_qoy = 2 and d_year = 1998 + | and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) + | group by s_store_name + | order by s_store_name LIMIT 100 + """.stripMargin), + Query( + "q9", + s""" + |select case when (select count(*) from store_sales + | where ss_quantity between 1 and 20) > ${rc(0)} + | then (select avg(ss_ext_discount_amt) from store_sales + | where ss_quantity between 1 and 20) + | else (select avg(ss_net_paid) from store_sales + | where ss_quantity between 1 and 20) end bucket1 , + | case when (select count(*) from store_sales + | where ss_quantity between 21 and 40) > ${rc(1)} + | then (select avg(ss_ext_discount_amt) from store_sales + | where ss_quantity between 21 and 40) + | else (select avg(ss_net_paid) from store_sales + | where ss_quantity between 21 and 40) end bucket2, + | case when (select count(*) from store_sales + | where ss_quantity between 41 and 60) > ${rc(2)} + | then (select avg(ss_ext_discount_amt) from store_sales + | where ss_quantity between 41 and 60) + | else (select avg(ss_net_paid) from store_sales + | where ss_quantity between 41 and 60) end bucket3, + | case when (select count(*) from store_sales + | where ss_quantity between 61 and 80) > ${rc(3)} + | then (select avg(ss_ext_discount_amt) from store_sales + | where ss_quantity between 61 and 80) + | else (select avg(ss_net_paid) from store_sales + | where ss_quantity between 61 and 80) end bucket4, + | case when (select count(*) from store_sales + | where ss_quantity between 81 and 100) > ${rc(4)} + | then (select avg(ss_ext_discount_amt) from store_sales + | where ss_quantity between 81 and 100) + | else (select avg(ss_net_paid) from store_sales + | where ss_quantity between 81 and 100) end bucket5 + |from reason + |where r_reason_sk = 1 + """.stripMargin), + Query( + "q10", + """ + | select + | cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, + | cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3, + | cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5, + | cd_dep_college_count, count(*) cnt6 + | from + | customer c, customer_address ca, customer_demographics + | where + | c.c_current_addr_sk = ca.ca_address_sk and + | ca_county in ('Rush County','Toole County','Jefferson County', + | 'Dona Ana County','La Porte County') and + | cd_demo_sk = c.c_current_cdemo_sk AND + | exists (select * from store_sales, date_dim + | where c.c_customer_sk = ss_customer_sk AND + | ss_sold_date_sk = d_date_sk AND + | d_year = 2002 AND + | d_moy between 1 AND 1+3) AND + | (exists (select * from web_sales, date_dim + | where c.c_customer_sk = ws_bill_customer_sk AND + | ws_sold_date_sk = d_date_sk AND + | d_year = 2002 AND + | d_moy between 1 AND 1+3) or + | exists (select * from catalog_sales, date_dim + | where c.c_customer_sk = cs_ship_customer_sk AND + | cs_sold_date_sk = d_date_sk AND + | d_year = 2002 AND + | d_moy between 1 AND 1+3)) + | group by cd_gender, + | cd_marital_status, + | cd_education_status, + | cd_purchase_estimate, + | cd_credit_rating, + | cd_dep_count, + | cd_dep_employed_count, + | cd_dep_college_count + | order by cd_gender, + | cd_marital_status, + | cd_education_status, + | cd_purchase_estimate, + | cd_credit_rating, + | cd_dep_count, + | cd_dep_employed_count, + | cd_dep_college_count + |LIMIT 100 + """.stripMargin), + Query( + "q11", + """ + | with year_total as ( + | select c_customer_id customer_id + | ,c_first_name customer_first_name + | ,c_last_name customer_last_name + | ,c_preferred_cust_flag customer_preferred_cust_flag + | ,c_birth_country customer_birth_country + | ,c_login customer_login + | ,c_email_address customer_email_address + | ,d_year dyear + | ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total + | ,'s' sale_type + | from customer, store_sales, date_dim + | where c_customer_sk = ss_customer_sk + | and ss_sold_date_sk = d_date_sk + | group by c_customer_id + | ,c_first_name + | ,c_last_name + | ,d_year + | ,c_preferred_cust_flag + | ,c_birth_country + | ,c_login + | ,c_email_address + | ,d_year + | union all + | select c_customer_id customer_id + | ,c_first_name customer_first_name + | ,c_last_name customer_last_name + | ,c_preferred_cust_flag customer_preferred_cust_flag + | ,c_birth_country customer_birth_country + | ,c_login customer_login + | ,c_email_address customer_email_address + | ,d_year dyear + | ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total + | ,'w' sale_type + | from customer, web_sales, date_dim + | where c_customer_sk = ws_bill_customer_sk + | and ws_sold_date_sk = d_date_sk + | group by + | c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, + | c_login, c_email_address, d_year) + | select + | t_s_secyear.customer_preferred_cust_flag + | from year_total t_s_firstyear + | ,year_total t_s_secyear + | ,year_total t_w_firstyear + | ,year_total t_w_secyear + | where t_s_secyear.customer_id = t_s_firstyear.customer_id + | and t_s_firstyear.customer_id = t_w_secyear.customer_id + | and t_s_firstyear.customer_id = t_w_firstyear.customer_id + | and t_s_firstyear.sale_type = 's' + | and t_w_firstyear.sale_type = 'w' + | and t_s_secyear.sale_type = 's' + | and t_w_secyear.sale_type = 'w' + | and t_s_firstyear.dyear = 2001 + | and t_s_secyear.dyear = 2001+1 + | and t_w_firstyear.dyear = 2001 + | and t_w_secyear.dyear = 2001+1 + | and t_s_firstyear.year_total > 0 + | and t_w_firstyear.year_total > 0 + | and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end + | > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end + | order by t_s_secyear.customer_preferred_cust_flag + | LIMIT 100 + """.stripMargin), + // Modifications: "+ days" -> date_add + Query( + "q12", + """ + | select + | i_item_desc, i_category, i_class, i_current_price, + | sum(ws_ext_sales_price) as itemrevenue, + | sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over + | (partition by i_class) as revenueratio + | from + | web_sales, item, date_dim + | where + | ws_item_sk = i_item_sk + | and i_category in ('Sports', 'Books', 'Home') + | and ws_sold_date_sk = d_date_sk + | and d_date between cast('1999-02-22' as date) + | and (cast('1999-02-22' as date) + interval 30 days) + | group by + | i_item_id, i_item_desc, i_category, i_class, i_current_price + | order by + | i_category, i_class, i_item_id, i_item_desc, revenueratio + | LIMIT 100 + """.stripMargin), + Query( + "q13", + """ + | select avg(ss_quantity) + | ,avg(ss_ext_sales_price) + | ,avg(ss_ext_wholesale_cost) + | ,sum(ss_ext_wholesale_cost) + | from store_sales + | ,store + | ,customer_demographics + | ,household_demographics + | ,customer_address + | ,date_dim + | where s_store_sk = ss_store_sk + | and ss_sold_date_sk = d_date_sk and d_year = 2001 + | and((ss_hdemo_sk=hd_demo_sk + | and cd_demo_sk = ss_cdemo_sk + | and cd_marital_status = 'M' + | and cd_education_status = 'Advanced Degree' + | and ss_sales_price between 100.00 and 150.00 + | and hd_dep_count = 3 + | )or + | (ss_hdemo_sk=hd_demo_sk + | and cd_demo_sk = ss_cdemo_sk + | and cd_marital_status = 'S' + | and cd_education_status = 'College' + | and ss_sales_price between 50.00 and 100.00 + | and hd_dep_count = 1 + | ) or + | (ss_hdemo_sk=hd_demo_sk + | and cd_demo_sk = ss_cdemo_sk + | and cd_marital_status = 'W' + | and cd_education_status = '2 yr Degree' + | and ss_sales_price between 150.00 and 200.00 + | and hd_dep_count = 1 + | )) + | and((ss_addr_sk = ca_address_sk + | and ca_country = 'United States' + | and ca_state in ('TX', 'OH', 'TX') + | and ss_net_profit between 100 and 200 + | ) or + | (ss_addr_sk = ca_address_sk + | and ca_country = 'United States' + | and ca_state in ('OR', 'NM', 'KY') + | and ss_net_profit between 150 and 300 + | ) or + | (ss_addr_sk = ca_address_sk + | and ca_country = 'United States' + | and ca_state in ('VA', 'TX', 'MS') + | and ss_net_profit between 50 and 250 + | )) + """.stripMargin), + Query( + "q14a", + """ + |with cross_items as + | (select i_item_sk ss_item_sk + | from item, + | (select iss.i_brand_id brand_id, iss.i_class_id class_id, iss.i_category_id category_id + | from store_sales, item iss, date_dim d1 + | where ss_item_sk = iss.i_item_sk + and ss_sold_date_sk = d1.d_date_sk + | and d1.d_year between 1999 AND 1999 + 2 + | intersect + | select ics.i_brand_id, ics.i_class_id, ics.i_category_id + | from catalog_sales, item ics, date_dim d2 + | where cs_item_sk = ics.i_item_sk + | and cs_sold_date_sk = d2.d_date_sk + | and d2.d_year between 1999 AND 1999 + 2 + | intersect + | select iws.i_brand_id, iws.i_class_id, iws.i_category_id + | from web_sales, item iws, date_dim d3 + | where ws_item_sk = iws.i_item_sk + | and ws_sold_date_sk = d3.d_date_sk + | and d3.d_year between 1999 AND 1999 + 2) x + | where i_brand_id = brand_id + | and i_class_id = class_id + | and i_category_id = category_id + |), + | avg_sales as + | (select avg(quantity*list_price) average_sales + | from ( + | select ss_quantity quantity, ss_list_price list_price + | from store_sales, date_dim + | where ss_sold_date_sk = d_date_sk + | and d_year between 1999 and 2001 + | union all + | select cs_quantity quantity, cs_list_price list_price + | from catalog_sales, date_dim + | where cs_sold_date_sk = d_date_sk + | and d_year between 1999 and 1999 + 2 + | union all + | select ws_quantity quantity, ws_list_price list_price + | from web_sales, date_dim + | where ws_sold_date_sk = d_date_sk + | and d_year between 1999 and 1999 + 2) x) + | select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) + | from( + | select 'store' channel, i_brand_id,i_class_id + | ,i_category_id,sum(ss_quantity*ss_list_price) sales + | , count(*) number_sales + | from store_sales, item, date_dim + | where ss_item_sk in (select ss_item_sk from cross_items) + | and ss_item_sk = i_item_sk + | and ss_sold_date_sk = d_date_sk + | and d_year = 1999+2 + | and d_moy = 11 + | group by i_brand_id,i_class_id,i_category_id + | having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) + | union all + | select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales + | from catalog_sales, item, date_dim + | where cs_item_sk in (select ss_item_sk from cross_items) + | and cs_item_sk = i_item_sk + | and cs_sold_date_sk = d_date_sk + | and d_year = 1999+2 + | and d_moy = 11 + | group by i_brand_id,i_class_id,i_category_id + | having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) + | union all + | select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales + | from web_sales, item, date_dim + | where ws_item_sk in (select ss_item_sk from cross_items) + | and ws_item_sk = i_item_sk + | and ws_sold_date_sk = d_date_sk + | and d_year = 1999+2 + | and d_moy = 11 + | group by i_brand_id,i_class_id,i_category_id + | having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) + | ) y + | group by rollup (channel, i_brand_id,i_class_id,i_category_id) + | order by channel,i_brand_id,i_class_id,i_category_id + | limit 100 + """.stripMargin), + Query( + "q14b", + """ + | with cross_items as + | (select i_item_sk ss_item_sk + | from item, + | (select iss.i_brand_id brand_id, iss.i_class_id class_id, iss.i_category_id category_id + | from store_sales, item iss, date_dim d1 + | where ss_item_sk = iss.i_item_sk + | and ss_sold_date_sk = d1.d_date_sk + | and d1.d_year between 1999 AND 1999 + 2 + | intersect + | select ics.i_brand_id, ics.i_class_id, ics.i_category_id + | from catalog_sales, item ics, date_dim d2 + | where cs_item_sk = ics.i_item_sk + | and cs_sold_date_sk = d2.d_date_sk + | and d2.d_year between 1999 AND 1999 + 2 + | intersect + | select iws.i_brand_id, iws.i_class_id, iws.i_category_id + | from web_sales, item iws, date_dim d3 + | where ws_item_sk = iws.i_item_sk + | and ws_sold_date_sk = d3.d_date_sk + | and d3.d_year between 1999 AND 1999 + 2) x + | where i_brand_id = brand_id + | and i_class_id = class_id + | and i_category_id = category_id + | ), + | avg_sales as + | (select avg(quantity*list_price) average_sales + | from (select ss_quantity quantity, ss_list_price list_price + | from store_sales, date_dim + | where ss_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 + | union all + | select cs_quantity quantity, cs_list_price list_price + | from catalog_sales, date_dim + | where cs_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 + | union all + | select ws_quantity quantity, ws_list_price list_price + | from web_sales, date_dim + | where ws_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2) x) + | select * from + | (select 'store' channel, i_brand_id,i_class_id,i_category_id + | ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales + | from store_sales, item, date_dim + | where ss_item_sk in (select ss_item_sk from cross_items) + | and ss_item_sk = i_item_sk + | and ss_sold_date_sk = d_date_sk + | and d_week_seq = (select d_week_seq from date_dim + | where d_year = 1999 + 1 and d_moy = 12 and d_dom = 11) + | group by i_brand_id,i_class_id,i_category_id + | having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, + | (select 'store' channel, i_brand_id,i_class_id + | ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales + | from store_sales, item, date_dim + | where ss_item_sk in (select ss_item_sk from cross_items) + | and ss_item_sk = i_item_sk + | and ss_sold_date_sk = d_date_sk + | and d_week_seq = (select d_week_seq from date_dim + | where d_year = 1999 and d_moy = 12 and d_dom = 11) + | group by i_brand_id,i_class_id,i_category_id + | having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year + | where this_year.i_brand_id= last_year.i_brand_id + | and this_year.i_class_id = last_year.i_class_id + | and this_year.i_category_id = last_year.i_category_id + | order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id + | limit 100 + """.stripMargin), + Query( + "q15", + """ + | select ca_zip, sum(cs_sales_price) + | from catalog_sales, customer, customer_address, date_dim + | where cs_bill_customer_sk = c_customer_sk + | and c_current_addr_sk = ca_address_sk + | and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', + | '85392', '85460', '80348', '81792') + | or ca_state in ('CA','WA','GA') + | or cs_sales_price > 500) + | and cs_sold_date_sk = d_date_sk + | and d_qoy = 2 and d_year = 2001 + | group by ca_zip + | order by ca_zip + | limit 100 + """.stripMargin), + // Modifications: " -> ` + Query( + "q16", + """ + | select + | count(distinct cs_order_number) as `order count`, + | sum(cs_ext_ship_cost) as `total shipping cost`, + | sum(cs_net_profit) as `total net profit` + | from + | catalog_sales cs1, date_dim, customer_address, call_center + | where + | d_date between '2002-02-01' and (cast('2002-02-01' as date) + interval 60 days) + | and cs1.cs_ship_date_sk = d_date_sk + | and cs1.cs_ship_addr_sk = ca_address_sk + | and ca_state = 'GA' + | and cs1.cs_call_center_sk = cc_call_center_sk + | and cc_county in ('Williamson County','Williamson County','Williamson County','Williamson County', 'Williamson County') + | and exists (select * + | from catalog_sales cs2 + | where cs1.cs_order_number = cs2.cs_order_number + | and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) + | and not exists(select * + | from catalog_returns cr1 + | where cs1.cs_order_number = cr1.cr_order_number) + | order by count(distinct cs_order_number) + | limit 100 + """.stripMargin), + Query( + "q17", + """ + | select i_item_id + | ,i_item_desc + | ,s_state + | ,count(ss_quantity) as store_sales_quantitycount + | ,avg(ss_quantity) as store_sales_quantityave + | ,stddev_samp(ss_quantity) as store_sales_quantitystdev + | ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov + | ,count(sr_return_quantity) as_store_returns_quantitycount + | ,avg(sr_return_quantity) as_store_returns_quantityave + | ,stddev_samp(sr_return_quantity) as_store_returns_quantitystdev + | ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov + | ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave + | ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitystdev + | ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov + | from store_sales, store_returns, catalog_sales, date_dim d1, date_dim d2, date_dim d3, store, item + | where d1.d_quarter_name = '2001Q1' + | and d1.d_date_sk = ss_sold_date_sk + | and i_item_sk = ss_item_sk + | and s_store_sk = ss_store_sk + | and ss_customer_sk = sr_customer_sk + | and ss_item_sk = sr_item_sk + | and ss_ticket_number = sr_ticket_number + | and sr_returned_date_sk = d2.d_date_sk + | and d2.d_quarter_name in ('2001Q1','2001Q2','2001Q3') + | and sr_customer_sk = cs_bill_customer_sk + | and sr_item_sk = cs_item_sk + | and cs_sold_date_sk = d3.d_date_sk + | and d3.d_quarter_name in ('2001Q1','2001Q2','2001Q3') + | group by i_item_id, i_item_desc, s_state + | order by i_item_id, i_item_desc, s_state + | limit 100 + """.stripMargin), + // Modifications: "numeric" -> "decimal" + Query( + "q18", + """ + | select i_item_id, + | ca_country, + | ca_state, + | ca_county, + | avg( cast(cs_quantity as decimal(12,2))) agg1, + | avg( cast(cs_list_price as decimal(12,2))) agg2, + | avg( cast(cs_coupon_amt as decimal(12,2))) agg3, + | avg( cast(cs_sales_price as decimal(12,2))) agg4, + | avg( cast(cs_net_profit as decimal(12,2))) agg5, + | avg( cast(c_birth_year as decimal(12,2))) agg6, + | avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 + | from catalog_sales, customer_demographics cd1, + | customer_demographics cd2, customer, customer_address, date_dim, item + | where cs_sold_date_sk = d_date_sk and + | cs_item_sk = i_item_sk and + | cs_bill_cdemo_sk = cd1.cd_demo_sk and + | cs_bill_customer_sk = c_customer_sk and + | cd1.cd_gender = 'F' and + | cd1.cd_education_status = 'Unknown' and + | c_current_cdemo_sk = cd2.cd_demo_sk and + | c_current_addr_sk = ca_address_sk and + | c_birth_month in (1,6,8,9,12,2) and + | d_year = 1998 and + | ca_state in ('MS','IN','ND','OK','NM','VA','MS') + | group by rollup (i_item_id, ca_country, ca_state, ca_county) + | order by ca_country, ca_state, ca_county, i_item_id + | LIMIT 100 + """.stripMargin), + Query( + "q19", + """ + | select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, + | sum(ss_ext_sales_price) ext_price + | from date_dim, store_sales, item,customer,customer_address,store + | where d_date_sk = ss_sold_date_sk + | and ss_item_sk = i_item_sk + | and i_manager_id = 8 + | and d_moy = 11 + | and d_year = 1998 + | and ss_customer_sk = c_customer_sk + | and c_current_addr_sk = ca_address_sk + | and substr(ca_zip,1,5) <> substr(s_zip,1,5) + | and ss_store_sk = s_store_sk + | group by i_brand, i_brand_id, i_manufact_id, i_manufact + | order by ext_price desc, brand, brand_id, i_manufact_id, i_manufact + | limit 100 + """.stripMargin), + Query( + "q20", + """ + |select i_item_desc + | ,i_category + | ,i_class + | ,i_current_price + | ,sum(cs_ext_sales_price) as itemrevenue + | ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over + | (partition by i_class) as revenueratio + | from catalog_sales, item, date_dim + | where cs_item_sk = i_item_sk + | and i_category in ('Sports', 'Books', 'Home') + | and cs_sold_date_sk = d_date_sk + | and d_date between cast('1999-02-22' as date) + | and (cast('1999-02-22' as date) + interval 30 days) + | group by i_item_id, i_item_desc, i_category, i_class, i_current_price + | order by i_category, i_class, i_item_id, i_item_desc, revenueratio + | limit 100 + """.stripMargin), + // Modifications: "+ days" -> date_add + Query( + "q21", + """ + | select * from( + | select w_warehouse_name, i_item_id, + | sum(case when (cast(d_date as date) < cast ('2000-03-11' as date)) + | then inv_quantity_on_hand + | else 0 end) as inv_before, + | sum(case when (cast(d_date as date) >= cast ('2000-03-11' as date)) + | then inv_quantity_on_hand + | else 0 end) as inv_after + | from inventory, warehouse, item, date_dim + | where i_current_price between 0.99 and 1.49 + | and i_item_sk = inv_item_sk + | and inv_warehouse_sk = w_warehouse_sk + | and inv_date_sk = d_date_sk + | and d_date between (cast('2000-03-11' as date) - interval 30 days) + | and (cast('2000-03-11' as date) + interval 30 days) + | group by w_warehouse_name, i_item_id) x + | where (case when inv_before > 0 + | then inv_after / inv_before + | else null + | end) between 2.0/3.0 and 3.0/2.0 + | order by w_warehouse_name, i_item_id + | limit 100 + """.stripMargin), + Query( + "q22", + """ + | select i_product_name, i_brand, i_class, i_category, avg(inv_quantity_on_hand) qoh + | from inventory, date_dim, item, warehouse + | where inv_date_sk=d_date_sk + | and inv_item_sk=i_item_sk + | and inv_warehouse_sk = w_warehouse_sk + | and d_month_seq between 1200 and 1200 + 11 + | group by rollup(i_product_name, i_brand, i_class, i_category) + | order by qoh, i_product_name, i_brand, i_class, i_category + | limit 100 + """.stripMargin), + Query( + "q23a", + """ + | with frequent_ss_items as + | (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt + | from store_sales, date_dim, item + | where ss_sold_date_sk = d_date_sk + | and ss_item_sk = i_item_sk + | and d_year in (2000, 2000+1, 2000+2,2000+3) + | group by substr(i_item_desc,1,30),i_item_sk,d_date + | having count(*) >4), + | max_store_sales as + | (select max(csales) tpcds_cmax + | from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales + | from store_sales, customer, date_dim + | where ss_customer_sk = c_customer_sk + | and ss_sold_date_sk = d_date_sk + | and d_year in (2000, 2000+1, 2000+2,2000+3) + | group by c_customer_sk) x), + | best_ss_customer as + | (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales + | from store_sales, customer + | where ss_customer_sk = c_customer_sk + | group by c_customer_sk + | having sum(ss_quantity*ss_sales_price) > (50/100.0) * + | (select * from max_store_sales)) + | select sum(sales) + | from ((select cs_quantity*cs_list_price sales + | from catalog_sales, date_dim + | where d_year = 2000 + | and d_moy = 2 + | and cs_sold_date_sk = d_date_sk + | and cs_item_sk in (select item_sk from frequent_ss_items) + | and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer)) + | union all + | (select ws_quantity*ws_list_price sales + | from web_sales, date_dim + | where d_year = 2000 + | and d_moy = 2 + | and ws_sold_date_sk = d_date_sk + | and ws_item_sk in (select item_sk from frequent_ss_items) + | and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer))) y + | limit 100 + """.stripMargin), + Query( + "q23b", + """ + | + | with frequent_ss_items as + | (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt + | from store_sales, date_dim, item + | where ss_sold_date_sk = d_date_sk + | and ss_item_sk = i_item_sk + | and d_year in (2000, 2000+1, 2000+2,2000+3) + | group by substr(i_item_desc,1,30),i_item_sk,d_date + | having count(*) > 4), + | max_store_sales as + | (select max(csales) tpcds_cmax + | from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales + | from store_sales, customer, date_dim + | where ss_customer_sk = c_customer_sk + | and ss_sold_date_sk = d_date_sk + | and d_year in (2000, 2000+1, 2000+2,2000+3) + | group by c_customer_sk) x), + | best_ss_customer as + | (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales + | from store_sales + | ,customer + | where ss_customer_sk = c_customer_sk + | group by c_customer_sk + | having sum(ss_quantity*ss_sales_price) > (50/100.0) * + | (select * from max_store_sales)) + | select c_last_name,c_first_name,sales + | from ((select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales + | from catalog_sales, customer, date_dim + | where d_year = 2000 + | and d_moy = 2 + | and cs_sold_date_sk = d_date_sk + | and cs_item_sk in (select item_sk from frequent_ss_items) + | and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) + | and cs_bill_customer_sk = c_customer_sk + | group by c_last_name,c_first_name) + | union all + | (select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales + | from web_sales, customer, date_dim + | where d_year = 2000 + | and d_moy = 2 + | and ws_sold_date_sk = d_date_sk + | and ws_item_sk in (select item_sk from frequent_ss_items) + | and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) + | and ws_bill_customer_sk = c_customer_sk + | group by c_last_name,c_first_name)) y + | order by c_last_name,c_first_name,sales + | limit 100 + """.stripMargin), + Query( + "q24a", + """ + | with ssales as + | (select c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, + | i_current_price, i_manager_id, i_units, i_size, sum(ss_net_paid) netpaid + | from store_sales, store_returns, store, item, customer, customer_address + | where ss_ticket_number = sr_ticket_number + | and ss_item_sk = sr_item_sk + | and ss_customer_sk = c_customer_sk + | and ss_item_sk = i_item_sk + | and ss_store_sk = s_store_sk + | and c_birth_country = upper(ca_country) + | and s_zip = ca_zip + | and s_market_id = 8 + | group by c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, + | i_current_price, i_manager_id, i_units, i_size) + | select c_last_name, c_first_name, s_store_name, sum(netpaid) paid + | from ssales + | where i_color = 'pale' + | group by c_last_name, c_first_name, s_store_name + | having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) + """.stripMargin), + Query( + "q24b", + """ + | with ssales as + | (select c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, + | i_current_price, i_manager_id, i_units, i_size, sum(ss_net_paid) netpaid + | from store_sales, store_returns, store, item, customer, customer_address + | where ss_ticket_number = sr_ticket_number + | and ss_item_sk = sr_item_sk + | and ss_customer_sk = c_customer_sk + | and ss_item_sk = i_item_sk + | and ss_store_sk = s_store_sk + | and c_birth_country = upper(ca_country) + | and s_zip = ca_zip + | and s_market_id = 8 + | group by c_last_name, c_first_name, s_store_name, ca_state, s_state, + | i_color, i_current_price, i_manager_id, i_units, i_size) + | select c_last_name, c_first_name, s_store_name, sum(netpaid) paid + | from ssales + | where i_color = 'chiffon' + | group by c_last_name, c_first_name, s_store_name + | having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) + """.stripMargin), + Query( + "q25", + """ + | select i_item_id, i_item_desc, s_store_id, s_store_name, + | sum(ss_net_profit) as store_sales_profit, + | sum(sr_net_loss) as store_returns_loss, + | sum(cs_net_profit) as catalog_sales_profit + | from + | store_sales, store_returns, catalog_sales, date_dim d1, date_dim d2, date_dim d3, + | store, item + | where + | d1.d_moy = 4 + | and d1.d_year = 2001 + | and d1.d_date_sk = ss_sold_date_sk + | and i_item_sk = ss_item_sk + | and s_store_sk = ss_store_sk + | and ss_customer_sk = sr_customer_sk + | and ss_item_sk = sr_item_sk + | and ss_ticket_number = sr_ticket_number + | and sr_returned_date_sk = d2.d_date_sk + | and d2.d_moy between 4 and 10 + | and d2.d_year = 2001 + | and sr_customer_sk = cs_bill_customer_sk + | and sr_item_sk = cs_item_sk + | and cs_sold_date_sk = d3.d_date_sk + | and d3.d_moy between 4 and 10 + | and d3.d_year = 2001 + | group by + | i_item_id, i_item_desc, s_store_id, s_store_name + | order by + | i_item_id, i_item_desc, s_store_id, s_store_name + | limit 100 + """.stripMargin), + Query( + "q26", + """ + | select i_item_id, + | avg(cs_quantity) agg1, + | avg(cs_list_price) agg2, + | avg(cs_coupon_amt) agg3, + | avg(cs_sales_price) agg4 + | from catalog_sales, customer_demographics, date_dim, item, promotion + | where cs_sold_date_sk = d_date_sk and + | cs_item_sk = i_item_sk and + | cs_bill_cdemo_sk = cd_demo_sk and + | cs_promo_sk = p_promo_sk and + | cd_gender = 'M' and + | cd_marital_status = 'S' and + | cd_education_status = 'College' and + | (p_channel_email = 'N' or p_channel_event = 'N') and + | d_year = 2000 + | group by i_item_id + | order by i_item_id + | limit 100 + """.stripMargin), + Query( + "q27", + """ + | select i_item_id, + | s_state, grouping(s_state) g_state, + | avg(ss_quantity) agg1, + | avg(ss_list_price) agg2, + | avg(ss_coupon_amt) agg3, + | avg(ss_sales_price) agg4 + | from store_sales, customer_demographics, date_dim, store, item + | where ss_sold_date_sk = d_date_sk and + | ss_item_sk = i_item_sk and + | ss_store_sk = s_store_sk and + | ss_cdemo_sk = cd_demo_sk and + | cd_gender = 'M' and + | cd_marital_status = 'S' and + | cd_education_status = 'College' and + | d_year = 2002 and + | s_state in ('TN','TN', 'TN', 'TN', 'TN', 'TN') + | group by rollup (i_item_id, s_state) + | order by i_item_id, s_state + | limit 100 + """.stripMargin), + Query( + "q28", + """ + | select * + | from (select avg(ss_list_price) B1_LP + | ,count(ss_list_price) B1_CNT + | ,count(distinct ss_list_price) B1_CNTD + | from store_sales + | where ss_quantity between 0 and 5 + | and (ss_list_price between 8 and 8+10 + | or ss_coupon_amt between 459 and 459+1000 + | or ss_wholesale_cost between 57 and 57+20)) B1, + | (select avg(ss_list_price) B2_LP + | ,count(ss_list_price) B2_CNT + | ,count(distinct ss_list_price) B2_CNTD + | from store_sales + | where ss_quantity between 6 and 10 + | and (ss_list_price between 90 and 90+10 + | or ss_coupon_amt between 2323 and 2323+1000 + | or ss_wholesale_cost between 31 and 31+20)) B2, + | (select avg(ss_list_price) B3_LP + | ,count(ss_list_price) B3_CNT + | ,count(distinct ss_list_price) B3_CNTD + | from store_sales + | where ss_quantity between 11 and 15 + | and (ss_list_price between 142 and 142+10 + | or ss_coupon_amt between 12214 and 12214+1000 + | or ss_wholesale_cost between 79 and 79+20)) B3, + | (select avg(ss_list_price) B4_LP + | ,count(ss_list_price) B4_CNT + | ,count(distinct ss_list_price) B4_CNTD + | from store_sales + | where ss_quantity between 16 and 20 + | and (ss_list_price between 135 and 135+10 + | or ss_coupon_amt between 6071 and 6071+1000 + | or ss_wholesale_cost between 38 and 38+20)) B4, + | (select avg(ss_list_price) B5_LP + | ,count(ss_list_price) B5_CNT + | ,count(distinct ss_list_price) B5_CNTD + | from store_sales + | where ss_quantity between 21 and 25 + | and (ss_list_price between 122 and 122+10 + | or ss_coupon_amt between 836 and 836+1000 + | or ss_wholesale_cost between 17 and 17+20)) B5, + | (select avg(ss_list_price) B6_LP + | ,count(ss_list_price) B6_CNT + | ,count(distinct ss_list_price) B6_CNTD + | from store_sales + | where ss_quantity between 26 and 30 + | and (ss_list_price between 154 and 154+10 + | or ss_coupon_amt between 7326 and 7326+1000 + | or ss_wholesale_cost between 7 and 7+20)) B6 + | limit 100 + """.stripMargin), + Query( + "q29", + """ + | select + | i_item_id + | ,i_item_desc + | ,s_store_id + | ,s_store_name + | ,sum(ss_quantity) as store_sales_quantity + | ,sum(sr_return_quantity) as store_returns_quantity + | ,sum(cs_quantity) as catalog_sales_quantity + | from + | store_sales, store_returns, catalog_sales, date_dim d1, date_dim d2, + | date_dim d3, store, item + | where + | d1.d_moy = 9 + | and d1.d_year = 1999 + | and d1.d_date_sk = ss_sold_date_sk + | and i_item_sk = ss_item_sk + | and s_store_sk = ss_store_sk + | and ss_customer_sk = sr_customer_sk + | and ss_item_sk = sr_item_sk + | and ss_ticket_number = sr_ticket_number + | and sr_returned_date_sk = d2.d_date_sk + | and d2.d_moy between 9 and 9 + 3 + | and d2.d_year = 1999 + | and sr_customer_sk = cs_bill_customer_sk + | and sr_item_sk = cs_item_sk + | and cs_sold_date_sk = d3.d_date_sk + | and d3.d_year in (1999,1999+1,1999+2) + | group by + | i_item_id, i_item_desc, s_store_id, s_store_name + | order by + | i_item_id, i_item_desc, s_store_id, s_store_name + | limit 100 + """.stripMargin), + Query( + "q30", + """ + | with customer_total_return as + | (select wr_returning_customer_sk as ctr_customer_sk + | ,ca_state as ctr_state, + | sum(wr_return_amt) as ctr_total_return + | from web_returns, date_dim, customer_address + | where wr_returned_date_sk = d_date_sk + | and d_year = 2002 + | and wr_returning_addr_sk = ca_address_sk + | group by wr_returning_customer_sk,ca_state) + | select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag + | ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address + | ,c_last_review_date,ctr_total_return + | from customer_total_return ctr1, customer_address, customer + | where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 + | from customer_total_return ctr2 + | where ctr1.ctr_state = ctr2.ctr_state) + | and ca_address_sk = c_current_addr_sk + | and ca_state = 'GA' + | and ctr1.ctr_customer_sk = c_customer_sk + | order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag + | ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address + | ,c_last_review_date,ctr_total_return + | limit 100 + """.stripMargin), + Query( + "q31", + """ + | with ss as + | (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales + | from store_sales,date_dim,customer_address + | where ss_sold_date_sk = d_date_sk + | and ss_addr_sk=ca_address_sk + | group by ca_county,d_qoy, d_year), + | ws as + | (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales + | from web_sales,date_dim,customer_address + | where ws_sold_date_sk = d_date_sk + | and ws_bill_addr_sk=ca_address_sk + | group by ca_county,d_qoy, d_year) + | select + | ss1.ca_county + | ,ss1.d_year + | ,ws2.web_sales/ws1.web_sales web_q1_q2_increase + | ,ss2.store_sales/ss1.store_sales store_q1_q2_increase + | ,ws3.web_sales/ws2.web_sales web_q2_q3_increase + | ,ss3.store_sales/ss2.store_sales store_q2_q3_increase + | from + | ss ss1, ss ss2, ss ss3, ws ws1, ws ws2, ws ws3 + | where + | ss1.d_qoy = 1 + | and ss1.d_year = 2000 + | and ss1.ca_county = ss2.ca_county + | and ss2.d_qoy = 2 + | and ss2.d_year = 2000 + | and ss2.ca_county = ss3.ca_county + | and ss3.d_qoy = 3 + | and ss3.d_year = 2000 + | and ss1.ca_county = ws1.ca_county + | and ws1.d_qoy = 1 + | and ws1.d_year = 2000 + | and ws1.ca_county = ws2.ca_county + | and ws2.d_qoy = 2 + | and ws2.d_year = 2000 + | and ws1.ca_county = ws3.ca_county + | and ws3.d_qoy = 3 + | and ws3.d_year = 2000 + | and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end + | > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end + | and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end + | > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end + | order by ss1.ca_county + """.stripMargin), + // Modifications: " -> ` + Query( + "q32", + """ + | select sum(cs_ext_discount_amt) as `excess discount amount` + | from + | catalog_sales, item, date_dim + | where + | i_manufact_id = 977 + | and i_item_sk = cs_item_sk + | and d_date between '2000-01-27' and (cast('2000-01-27' as date) + interval 90 days) + | and d_date_sk = cs_sold_date_sk + | and cs_ext_discount_amt > ( + | select 1.3 * avg(cs_ext_discount_amt) + | from catalog_sales, date_dim + | where cs_item_sk = i_item_sk + | and d_date between '2000-01-27]' and (cast('2000-01-27' as date) + interval 90 days) + | and d_date_sk = cs_sold_date_sk) + |limit 100 + """.stripMargin), + Query( + "q33", + """ + | with ss as ( + | select + | i_manufact_id,sum(ss_ext_sales_price) total_sales + | from + | store_sales, date_dim, customer_address, item + | where + | i_manufact_id in (select i_manufact_id + | from item + | where i_category in ('Electronics')) + | and ss_item_sk = i_item_sk + | and ss_sold_date_sk = d_date_sk + | and d_year = 1998 + | and d_moy = 5 + | and ss_addr_sk = ca_address_sk + | and ca_gmt_offset = -5 + | group by i_manufact_id), cs as + | (select i_manufact_id, sum(cs_ext_sales_price) total_sales + | from catalog_sales, date_dim, customer_address, item + | where + | i_manufact_id in ( + | select i_manufact_id from item + | where + | i_category in ('Electronics')) + | and cs_item_sk = i_item_sk + | and cs_sold_date_sk = d_date_sk + | and d_year = 1998 + | and d_moy = 5 + | and cs_bill_addr_sk = ca_address_sk + | and ca_gmt_offset = -5 + | group by i_manufact_id), + | ws as ( + | select i_manufact_id,sum(ws_ext_sales_price) total_sales + | from + | web_sales, date_dim, customer_address, item + | where + | i_manufact_id in (select i_manufact_id from item + | where i_category in ('Electronics')) + | and ws_item_sk = i_item_sk + | and ws_sold_date_sk = d_date_sk + | and d_year = 1998 + | and d_moy = 5 + | and ws_bill_addr_sk = ca_address_sk + | and ca_gmt_offset = -5 + | group by i_manufact_id) + | select i_manufact_id ,sum(total_sales) total_sales + | from (select * from ss + | union all + | select * from cs + | union all + | select * from ws) tmp1 + | group by i_manufact_id + | order by total_sales + |limit 100 + """.stripMargin), + Query( + "q34", + """ + | select c_last_name, c_first_name, c_salutation, c_preferred_cust_flag, ss_ticket_number, + | cnt + | FROM + | (select ss_ticket_number, ss_customer_sk, count(*) cnt + | from store_sales,date_dim,store,household_demographics + | where store_sales.ss_sold_date_sk = date_dim.d_date_sk + | and store_sales.ss_store_sk = store.s_store_sk + | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + | and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) + | and (household_demographics.hd_buy_potential = '>10000' or + | household_demographics.hd_buy_potential = 'unknown') + | and household_demographics.hd_vehicle_count > 0 + | and (case when household_demographics.hd_vehicle_count > 0 + | then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count + | else null + | end) > 1.2 + | and date_dim.d_year in (1999, 1999+1, 1999+2) + | and store.s_county in ('Williamson County','Williamson County','Williamson County','Williamson County', + | 'Williamson County','Williamson County','Williamson County','Williamson County') + | group by ss_ticket_number,ss_customer_sk) dn,customer + | where ss_customer_sk = c_customer_sk + | and cnt between 15 and 20 + | order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc + """.stripMargin), + Query( + "q35", + """ + | select + | ca_state, + | cd_gender, + | cd_marital_status, + | count(*) cnt1, + | min(cd_dep_count), + | max(cd_dep_count), + | avg(cd_dep_count), + | cd_dep_employed_count, + | count(*) cnt2, + | min(cd_dep_employed_count), + | max(cd_dep_employed_count), + | avg(cd_dep_employed_count), + | cd_dep_college_count, + | count(*) cnt3, + | min(cd_dep_college_count), + | max(cd_dep_college_count), + | avg(cd_dep_college_count) + | from + | customer c,customer_address ca,customer_demographics + | where + | c.c_current_addr_sk = ca.ca_address_sk and + | cd_demo_sk = c.c_current_cdemo_sk and + | exists (select * from store_sales, date_dim + | where c.c_customer_sk = ss_customer_sk and + | ss_sold_date_sk = d_date_sk and + | d_year = 2002 and + | d_qoy < 4) and + | (exists (select * from web_sales, date_dim + | where c.c_customer_sk = ws_bill_customer_sk and + | ws_sold_date_sk = d_date_sk and + | d_year = 2002 and + | d_qoy < 4) or + | exists (select * from catalog_sales, date_dim + | where c.c_customer_sk = cs_ship_customer_sk and + | cs_sold_date_sk = d_date_sk and + | d_year = 2002 and + | d_qoy < 4)) + | group by ca_state, cd_gender, cd_marital_status, cd_dep_count, + | cd_dep_employed_count, cd_dep_college_count + | order by ca_state, cd_gender, cd_marital_status, cd_dep_count, + | cd_dep_employed_count, cd_dep_college_count + | limit 100 + """.stripMargin), + Query( + "q36", + """ + | select + | sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin + | ,i_category + | ,i_class + | ,grouping(i_category)+grouping(i_class) as lochierarchy + | ,rank() over ( + | partition by grouping(i_category)+grouping(i_class), + | case when grouping(i_class) = 0 then i_category end + | order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent + | from + | store_sales, date_dim d1, item, store + | where + | d1.d_year = 2001 + | and d1.d_date_sk = ss_sold_date_sk + | and i_item_sk = ss_item_sk + | and s_store_sk = ss_store_sk + | and s_state in ('TN','TN','TN','TN','TN','TN','TN','TN') + | group by rollup(i_category,i_class) + | order by + | lochierarchy desc + | ,case when lochierarchy = 0 then i_category end + | ,rank_within_parent + | limit 100 + """.stripMargin), + // Modifications: "+ days" -> date_add + Query( + "q37", + """ + | select i_item_id, i_item_desc, i_current_price + | from item, inventory, date_dim, catalog_sales + | where i_current_price between 68 and 68 + 30 + | and inv_item_sk = i_item_sk + | and d_date_sk=inv_date_sk + | and d_date between cast('2000-02-01' as date) and (cast('2000-02-01' as date) + interval 60 days) + | and i_manufact_id in (677,940,694,808) + | and inv_quantity_on_hand between 100 and 500 + | and cs_item_sk = i_item_sk + | group by i_item_id,i_item_desc,i_current_price + | order by i_item_id + | limit 100 + """.stripMargin), + Query( + "q38", + """ + | select count(*) from ( + | select distinct c_last_name, c_first_name, d_date + | from store_sales, date_dim, customer + | where store_sales.ss_sold_date_sk = date_dim.d_date_sk + | and store_sales.ss_customer_sk = customer.c_customer_sk + | and d_month_seq between 1200 and 1200 + 11 + | intersect + | select distinct c_last_name, c_first_name, d_date + | from catalog_sales, date_dim, customer + | where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk + | and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk + | and d_month_seq between 1200 and 1200 + 11 + | intersect + | select distinct c_last_name, c_first_name, d_date + | from web_sales, date_dim, customer + | where web_sales.ws_sold_date_sk = date_dim.d_date_sk + | and web_sales.ws_bill_customer_sk = customer.c_customer_sk + | and d_month_seq between 1200 and 1200 + 11 + | ) hot_cust + | limit 100 + """.stripMargin), + Query( + "q39a", + """ + | with inv as + | (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy + | ,stdev,mean, case mean when 0 then null else stdev/mean end cov + | from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy + | ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean + | from inventory, item, warehouse, date_dim + | where inv_item_sk = i_item_sk + | and inv_warehouse_sk = w_warehouse_sk + | and inv_date_sk = d_date_sk + | and d_year = 2001 + | group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo + | where case mean when 0 then 0 else stdev/mean end > 1) + | select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov + | ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov + | from inv inv1,inv inv2 + | where inv1.i_item_sk = inv2.i_item_sk + | and inv1.w_warehouse_sk = inv2.w_warehouse_sk + | and inv1.d_moy=1 + | and inv2.d_moy=1+1 + | order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov + | ,inv2.d_moy,inv2.mean, inv2.cov + """.stripMargin), + Query( + "q39b", + """ + | with inv as + | (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy + | ,stdev,mean, case mean when 0 then null else stdev/mean end cov + | from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy + | ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean + | from inventory, item, warehouse, date_dim + | where inv_item_sk = i_item_sk + | and inv_warehouse_sk = w_warehouse_sk + | and inv_date_sk = d_date_sk + | and d_year = 2001 + | group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo + | where case mean when 0 then 0 else stdev/mean end > 1) + | select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov + | ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov + | from inv inv1,inv inv2 + | where inv1.i_item_sk = inv2.i_item_sk + | and inv1.w_warehouse_sk = inv2.w_warehouse_sk + | and inv1.d_moy=1 + | and inv2.d_moy=1+1 + | and inv1.cov > 1.5 + | order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov + | ,inv2.d_moy,inv2.mean, inv2.cov + """.stripMargin), + // Modifications: "+ days" -> date_add + Query( + "q40", + """ + | select + | w_state + | ,i_item_id + | ,sum(case when (cast(d_date as date) < cast('2000-03-11' as date)) + | then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before + | ,sum(case when (cast(d_date as date) >= cast('2000-03-11' as date)) + | then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after + | from + | catalog_sales left outer join catalog_returns on + | (cs_order_number = cr_order_number + | and cs_item_sk = cr_item_sk) + | ,warehouse, item, date_dim + | where + | i_current_price between 0.99 and 1.49 + | and i_item_sk = cs_item_sk + | and cs_warehouse_sk = w_warehouse_sk + | and cs_sold_date_sk = d_date_sk + | and d_date between (cast('2000-03-11' as date) - interval 30 days) + | and (cast('2000-03-11' as date) + interval 30 days) + | group by w_state,i_item_id + | order by w_state,i_item_id + | limit 100 + """.stripMargin), + Query( + "q41", + """ + | select distinct(i_product_name) + | from item i1 + | where i_manufact_id between 738 and 738+40 + | and (select count(*) as item_cnt + | from item + | where (i_manufact = i1.i_manufact and + | ((i_category = 'Women' and + | (i_color = 'powder' or i_color = 'khaki') and + | (i_units = 'Ounce' or i_units = 'Oz') and + | (i_size = 'medium' or i_size = 'extra large') + | ) or + | (i_category = 'Women' and + | (i_color = 'brown' or i_color = 'honeydew') and + | (i_units = 'Bunch' or i_units = 'Ton') and + | (i_size = 'N/A' or i_size = 'small') + | ) or + | (i_category = 'Men' and + | (i_color = 'floral' or i_color = 'deep') and + | (i_units = 'N/A' or i_units = 'Dozen') and + | (i_size = 'petite' or i_size = 'large') + | ) or + | (i_category = 'Men' and + | (i_color = 'light' or i_color = 'cornflower') and + | (i_units = 'Box' or i_units = 'Pound') and + | (i_size = 'medium' or i_size = 'extra large') + | ))) or + | (i_manufact = i1.i_manufact and + | ((i_category = 'Women' and + | (i_color = 'midnight' or i_color = 'snow') and + | (i_units = 'Pallet' or i_units = 'Gross') and + | (i_size = 'medium' or i_size = 'extra large') + | ) or + | (i_category = 'Women' and + | (i_color = 'cyan' or i_color = 'papaya') and + | (i_units = 'Cup' or i_units = 'Dram') and + | (i_size = 'N/A' or i_size = 'small') + | ) or + | (i_category = 'Men' and + | (i_color = 'orange' or i_color = 'frosted') and + | (i_units = 'Each' or i_units = 'Tbl') and + | (i_size = 'petite' or i_size = 'large') + | ) or + | (i_category = 'Men' and + | (i_color = 'forest' or i_color = 'ghost') and + | (i_units = 'Lb' or i_units = 'Bundle') and + | (i_size = 'medium' or i_size = 'extra large') + | )))) > 0 + | order by i_product_name + | limit 100 + """.stripMargin), + Query( + "q42", + """ + | select dt.d_year, item.i_category_id, item.i_category, sum(ss_ext_sales_price) + | from date_dim dt, store_sales, item + | where dt.d_date_sk = store_sales.ss_sold_date_sk + | and store_sales.ss_item_sk = item.i_item_sk + | and item.i_manager_id = 1 + | and dt.d_moy=11 + | and dt.d_year=2000 + | group by dt.d_year + | ,item.i_category_id + | ,item.i_category + | order by sum(ss_ext_sales_price) desc,dt.d_year + | ,item.i_category_id + | ,item.i_category + | limit 100 + """.stripMargin), + Query( + "q43", + """ + | select s_store_name, s_store_id, + | sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, + | sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, + | sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, + | sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, + | sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, + | sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, + | sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales + | from date_dim, store_sales, store + | where d_date_sk = ss_sold_date_sk and + | s_store_sk = ss_store_sk and + | s_gmt_offset = -5 and + | d_year = 2000 + | group by s_store_name, s_store_id + | order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales, + | thu_sales,fri_sales,sat_sales + | limit 100 + """.stripMargin), + Query( + "q44", + """ + | select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing + | from(select * + | from (select item_sk,rank() over (order by rank_col asc) rnk + | from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col + | from store_sales ss1 + | where ss_store_sk = 4 + | group by ss_item_sk + | having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col + | from store_sales + | where ss_store_sk = 4 + | and ss_addr_sk is null + | group by ss_store_sk))V1)V11 + | where rnk < 11) asceding, + | (select * + | from (select item_sk,rank() over (order by rank_col desc) rnk + | from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col + | from store_sales ss1 + | where ss_store_sk = 4 + | group by ss_item_sk + | having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col + | from store_sales + | where ss_store_sk = 4 + | and ss_addr_sk is null + | group by ss_store_sk))V2)V21 + | where rnk < 11) descending, + | item i1, item i2 + | where asceding.rnk = descending.rnk + | and i1.i_item_sk=asceding.item_sk + | and i2.i_item_sk=descending.item_sk + | order by asceding.rnk + | limit 100 + """.stripMargin), + Query( + "q45", + """ + | select ca_zip, ca_city, sum(ws_sales_price) + | from web_sales, customer, customer_address, date_dim, item + | where ws_bill_customer_sk = c_customer_sk + | and c_current_addr_sk = ca_address_sk + | and ws_item_sk = i_item_sk + | and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') + | or + | i_item_id in (select i_item_id + | from item + | where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) + | ) + | ) + | and ws_sold_date_sk = d_date_sk + | and d_qoy = 2 and d_year = 2001 + | group by ca_zip, ca_city + | order by ca_zip, ca_city + | limit 100 + """.stripMargin), + Query( + "q46", + """ + | select c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number, amt,profit + | from + | (select ss_ticket_number + | ,ss_customer_sk + | ,ca_city bought_city + | ,sum(ss_coupon_amt) amt + | ,sum(ss_net_profit) profit + | from store_sales, date_dim, store, household_demographics, customer_address + | where store_sales.ss_sold_date_sk = date_dim.d_date_sk + | and store_sales.ss_store_sk = store.s_store_sk + | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + | and store_sales.ss_addr_sk = customer_address.ca_address_sk + | and (household_demographics.hd_dep_count = 4 or + | household_demographics.hd_vehicle_count= 3) + | and date_dim.d_dow in (6,0) + | and date_dim.d_year in (1999,1999+1,1999+2) + | and store.s_city in ('Fairview','Midway','Fairview','Fairview','Fairview') + | group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr + | where ss_customer_sk = c_customer_sk + | and customer.c_current_addr_sk = current_addr.ca_address_sk + | and current_addr.ca_city <> bought_city + | order by c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number + | limit 100 + """.stripMargin), + Query( + "q47", + """ + | with v1 as( + | select i_category, i_brand, + | s_store_name, s_company_name, + | d_year, d_moy, + | sum(ss_sales_price) sum_sales, + | avg(sum(ss_sales_price)) over + | (partition by i_category, i_brand, + | s_store_name, s_company_name, d_year) + | avg_monthly_sales, + | rank() over + | (partition by i_category, i_brand, + | s_store_name, s_company_name + | order by d_year, d_moy) rn + | from item, store_sales, date_dim, store + | where ss_item_sk = i_item_sk and + | ss_sold_date_sk = d_date_sk and + | ss_store_sk = s_store_sk and + | ( + | d_year = 1999 or + | ( d_year = 1999-1 and d_moy =12) or + | ( d_year = 1999+1 and d_moy =1) + | ) + | group by i_category, i_brand, + | s_store_name, s_company_name, + | d_year, d_moy), + | v2 as( + | select v1.i_category, v1.i_brand, v1.s_store_name, v1.s_company_name, v1.d_year, + v1.d_moy, v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, + v1_lead.sum_sales nsum + | from v1, v1 v1_lag, v1 v1_lead + | where v1.i_category = v1_lag.i_category and + | v1.i_category = v1_lead.i_category and + | v1.i_brand = v1_lag.i_brand and + | v1.i_brand = v1_lead.i_brand and + | v1.s_store_name = v1_lag.s_store_name and + | v1.s_store_name = v1_lead.s_store_name and + | v1.s_company_name = v1_lag.s_company_name and + | v1.s_company_name = v1_lead.s_company_name and + | v1.rn = v1_lag.rn + 1 and + | v1.rn = v1_lead.rn - 1) + | select * from v2 + | where d_year = 1999 and + | avg_monthly_sales > 0 and + | case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 + | order by sum_sales - avg_monthly_sales, 3 + | limit 100 + """.stripMargin), + Query( + "q48", + """ + | select sum (ss_quantity) + | from store_sales, store, customer_demographics, customer_address, date_dim + | where s_store_sk = ss_store_sk + | and ss_sold_date_sk = d_date_sk and d_year = 2001 + | and + | ( + | ( + | cd_demo_sk = ss_cdemo_sk + | and + | cd_marital_status = 'M' + | and + | cd_education_status = '4 yr Degree' + | and + | ss_sales_price between 100.00 and 150.00 + | ) + | or + | ( + | cd_demo_sk = ss_cdemo_sk + | and + | cd_marital_status = 'D' + | and + | cd_education_status = '2 yr Degree' + | and + | ss_sales_price between 50.00 and 100.00 + | ) + | or + | ( + | cd_demo_sk = ss_cdemo_sk + | and + | cd_marital_status = 'S' + | and + | cd_education_status = 'College' + | and + | ss_sales_price between 150.00 and 200.00 + | ) + | ) + | and + | ( + | ( + | ss_addr_sk = ca_address_sk + | and + | ca_country = 'United States' + | and + | ca_state in ('CO', 'OH', 'TX') + | and ss_net_profit between 0 and 2000 + | ) + | or + | (ss_addr_sk = ca_address_sk + | and + | ca_country = 'United States' + | and + | ca_state in ('OR', 'MN', 'KY') + | and ss_net_profit between 150 and 3000 + | ) + | or + | (ss_addr_sk = ca_address_sk + | and + | ca_country = 'United States' + | and + | ca_state in ('VA', 'CA', 'MS') + | and ss_net_profit between 50 and 25000 + | ) + | ) + """.stripMargin), + // Modifications: "dec" -> "decimal" + Query( + "q49", + """ + | select 'web' as channel, web.item, web.return_ratio, web.return_rank, web.currency_rank + | from ( + | select + | item, return_ratio, currency_ratio, + | rank() over (order by return_ratio) as return_rank, + | rank() over (order by currency_ratio) as currency_rank + | from + | ( select ws.ws_item_sk as item + | ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ + | cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio + | ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ + | cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio + | from + | web_sales ws left outer join web_returns wr + | on (ws.ws_order_number = wr.wr_order_number and + | ws.ws_item_sk = wr.wr_item_sk) + | ,date_dim + | where + | wr.wr_return_amt > 10000 + | and ws.ws_net_profit > 1 + | and ws.ws_net_paid > 0 + | and ws.ws_quantity > 0 + | and ws_sold_date_sk = d_date_sk + | and d_year = 2001 + | and d_moy = 12 + | group by ws.ws_item_sk + | ) in_web + | ) web + | where (web.return_rank <= 10 or web.currency_rank <= 10) + | union + | select + | 'catalog' as channel, catalog.item, catalog.return_ratio, + | catalog.return_rank, catalog.currency_rank + | from ( + | select + | item, return_ratio, currency_ratio, + | rank() over (order by return_ratio) as return_rank, + | rank() over (order by currency_ratio) as currency_rank + | from + | ( select + | cs.cs_item_sk as item + | ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ + | cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio + | ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ + | cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio + | from + | catalog_sales cs left outer join catalog_returns cr + | on (cs.cs_order_number = cr.cr_order_number and + | cs.cs_item_sk = cr.cr_item_sk) + | ,date_dim + | where + | cr.cr_return_amount > 10000 + | and cs.cs_net_profit > 1 + | and cs.cs_net_paid > 0 + | and cs.cs_quantity > 0 + | and cs_sold_date_sk = d_date_sk + | and d_year = 2001 + | and d_moy = 12 + | group by cs.cs_item_sk + | ) in_cat + | ) catalog + | where (catalog.return_rank <= 10 or catalog.currency_rank <=10) + | union + | select + | 'store' as channel, store.item, store.return_ratio, + | store.return_rank, store.currency_rank + | from ( + | select + | item, return_ratio, currency_ratio, + | rank() over (order by return_ratio) as return_rank, + | rank() over (order by currency_ratio) as currency_rank + | from + | ( select sts.ss_item_sk as item + | ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/ + | cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio + | ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/ + | cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio + | from + | store_sales sts left outer join store_returns sr + | on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) + | ,date_dim + | where + | sr.sr_return_amt > 10000 + | and sts.ss_net_profit > 1 + | and sts.ss_net_paid > 0 + | and sts.ss_quantity > 0 + | and ss_sold_date_sk = d_date_sk + | and d_year = 2001 + | and d_moy = 12 + | group by sts.ss_item_sk + | ) in_store + | ) store + | where (store.return_rank <= 10 or store.currency_rank <= 10) + | order by 1,4,5 + | limit 100 + """.stripMargin), + // Modifications: " -> ` + Query( + "q50", + """ + | select + | s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, + | s_suite_number, s_city, s_county, s_state, s_zip + | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` + | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and + | (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` + | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and + | (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` + | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and + | (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` + | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` + | from + | store_sales, store_returns, store, date_dim d1, date_dim d2 + | where + | d2.d_year = 2001 + | and d2.d_moy = 8 + | and ss_ticket_number = sr_ticket_number + | and ss_item_sk = sr_item_sk + | and ss_sold_date_sk = d1.d_date_sk + | and sr_returned_date_sk = d2.d_date_sk + | and ss_customer_sk = sr_customer_sk + | and ss_store_sk = s_store_sk + | group by + | s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, + | s_suite_number, s_city, s_county, s_state, s_zip + | order by + | s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, + | s_suite_number, s_city, s_county, s_state, s_zip + | limit 100 + """.stripMargin), + Query( + "q51", + """ + | WITH web_v1 as ( + | select + | ws_item_sk item_sk, d_date, + | sum(sum(ws_sales_price)) + | over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales + | from web_sales, date_dim + | where ws_sold_date_sk=d_date_sk + | and d_month_seq between 1200 and 1200+11 + | and ws_item_sk is not NULL + | group by ws_item_sk, d_date), + | store_v1 as ( + | select + | ss_item_sk item_sk, d_date, + | sum(sum(ss_sales_price)) + | over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales + | from store_sales, date_dim + | where ss_sold_date_sk=d_date_sk + | and d_month_seq between 1200 and 1200+11 + | and ss_item_sk is not NULL + | group by ss_item_sk, d_date) + | select * + | from (select item_sk, d_date, web_sales, store_sales + | ,max(web_sales) + | over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative + | ,max(store_sales) + | over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative + | from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk + | ,case when web.d_date is not null then web.d_date else store.d_date end d_date + | ,web.cume_sales web_sales + | ,store.cume_sales store_sales + | from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk + | and web.d_date = store.d_date) + | )x )y + | where web_cumulative > store_cumulative + | order by item_sk, d_date + | limit 100 + """.stripMargin), + Query( + "q52", + """ + | select dt.d_year + | ,item.i_brand_id brand_id + | ,item.i_brand brand + | ,sum(ss_ext_sales_price) ext_price + | from date_dim dt, store_sales, item + | where dt.d_date_sk = store_sales.ss_sold_date_sk + | and store_sales.ss_item_sk = item.i_item_sk + | and item.i_manager_id = 1 + | and dt.d_moy=11 + | and dt.d_year=2000 + | group by dt.d_year, item.i_brand, item.i_brand_id + | order by dt.d_year, ext_price desc, brand_id + |limit 100 + """.stripMargin), + Query( + "q53", + """ + | select * from + | (select i_manufact_id, + | sum(ss_sales_price) sum_sales, + | avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales + | from item, store_sales, date_dim, store + | where ss_item_sk = i_item_sk and + | ss_sold_date_sk = d_date_sk and + | ss_store_sk = s_store_sk and + | d_month_seq in (1200,1200+1,1200+2,1200+3,1200+4,1200+5,1200+6, + | 1200+7,1200+8,1200+9,1200+10,1200+11) and + | ((i_category in ('Books','Children','Electronics') and + | i_class in ('personal','portable','reference','self-help') and + | i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', + | 'exportiunivamalg #9','scholaramalgamalg #9')) + | or + | (i_category in ('Women','Music','Men') and + | i_class in ('accessories','classical','fragrances','pants') and + | i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', + | 'importoamalg #1'))) + | group by i_manufact_id, d_qoy ) tmp1 + | where case when avg_quarterly_sales > 0 + | then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales + | else null end > 0.1 + | order by avg_quarterly_sales, + | sum_sales, + | i_manufact_id + | limit 100 + """.stripMargin), + Query( + "q54", + """ + | with my_customers as ( + | select distinct c_customer_sk + | , c_current_addr_sk + | from + | ( select cs_sold_date_sk sold_date_sk, + | cs_bill_customer_sk customer_sk, + | cs_item_sk item_sk + | from catalog_sales + | union all + | select ws_sold_date_sk sold_date_sk, + | ws_bill_customer_sk customer_sk, + | ws_item_sk item_sk + | from web_sales + | ) cs_or_ws_sales, + | item, + | date_dim, + | customer + | where sold_date_sk = d_date_sk + | and item_sk = i_item_sk + | and i_category = 'Women' + | and i_class = 'maternity' + | and c_customer_sk = cs_or_ws_sales.customer_sk + | and d_moy = 12 + | and d_year = 1998 + | ) + | , my_revenue as ( + | select c_customer_sk, + | sum(ss_ext_sales_price) as revenue + | from my_customers, + | store_sales, + | customer_address, + | store, + | date_dim + | where c_current_addr_sk = ca_address_sk + | and ca_county = s_county + | and ca_state = s_state + | and ss_sold_date_sk = d_date_sk + | and c_customer_sk = ss_customer_sk + | and d_month_seq between (select distinct d_month_seq+1 + | from date_dim where d_year = 1998 and d_moy = 12) + | and (select distinct d_month_seq+3 + | from date_dim where d_year = 1998 and d_moy = 12) + | group by c_customer_sk + | ) + | , segments as + | (select cast((revenue/50) as int) as segment from my_revenue) + | select segment, count(*) as num_customers, segment*50 as segment_base + | from segments + | group by segment + | order by segment, num_customers + | limit 100 + """.stripMargin), + Query( + "q55", + """ + |select i_brand_id brand_id, i_brand brand, + | sum(ss_ext_sales_price) ext_price + | from date_dim, store_sales, item + | where d_date_sk = ss_sold_date_sk + | and ss_item_sk = i_item_sk + | and i_manager_id=28 + | and d_moy=11 + | and d_year=1999 + | group by i_brand, i_brand_id + | order by ext_price desc, brand_id + | limit 100 + """.stripMargin), + Query( + "q56", + """ + | with ss as ( + | select i_item_id,sum(ss_ext_sales_price) total_sales + | from + | store_sales, date_dim, customer_address, item + | where + | i_item_id in (select i_item_id from item where i_color in ('slate','blanched','burnished')) + | and ss_item_sk = i_item_sk + | and ss_sold_date_sk = d_date_sk + | and d_year = 2001 + | and d_moy = 2 + | and ss_addr_sk = ca_address_sk + | and ca_gmt_offset = -5 + | group by i_item_id), + | cs as ( + | select i_item_id,sum(cs_ext_sales_price) total_sales + | from + | catalog_sales, date_dim, customer_address, item + | where + | i_item_id in (select i_item_id from item where i_color in ('slate','blanched','burnished')) + | and cs_item_sk = i_item_sk + | and cs_sold_date_sk = d_date_sk + | and d_year = 2001 + | and d_moy = 2 + | and cs_bill_addr_sk = ca_address_sk + | and ca_gmt_offset = -5 + | group by i_item_id), + | ws as ( + | select i_item_id,sum(ws_ext_sales_price) total_sales + | from + | web_sales, date_dim, customer_address, item + | where + | i_item_id in (select i_item_id from item where i_color in ('slate','blanched','burnished')) + | and ws_item_sk = i_item_sk + | and ws_sold_date_sk = d_date_sk + | and d_year = 2001 + | and d_moy = 2 + | and ws_bill_addr_sk = ca_address_sk + | and ca_gmt_offset = -5 + | group by i_item_id) + | select i_item_id ,sum(total_sales) total_sales + | from (select * from ss + | union all + | select * from cs + | union all + | select * from ws) tmp1 + | group by i_item_id + | order by total_sales + | limit 100 + """.stripMargin), + Query( + "q57", + """ + | with v1 as( + | select i_category, i_brand, + | cc_name, + | d_year, d_moy, + | sum(cs_sales_price) sum_sales, + | avg(sum(cs_sales_price)) over + | (partition by i_category, i_brand, cc_name, d_year) + | avg_monthly_sales, + | rank() over + | (partition by i_category, i_brand, cc_name + | order by d_year, d_moy) rn + | from item, catalog_sales, date_dim, call_center + | where cs_item_sk = i_item_sk and + | cs_sold_date_sk = d_date_sk and + | cc_call_center_sk= cs_call_center_sk and + | ( + | d_year = 1999 or + | ( d_year = 1999-1 and d_moy =12) or + | ( d_year = 1999+1 and d_moy =1) + | ) + | group by i_category, i_brand, + | cc_name , d_year, d_moy), + | v2 as( + | select v1.i_category, v1.i_brand, v1.cc_name, v1.d_year, v1.d_moy + | ,v1.avg_monthly_sales + | ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum + | from v1, v1 v1_lag, v1 v1_lead + | where v1.i_category = v1_lag.i_category and + | v1.i_category = v1_lead.i_category and + | v1.i_brand = v1_lag.i_brand and + | v1.i_brand = v1_lead.i_brand and + | v1. cc_name = v1_lag. cc_name and + | v1. cc_name = v1_lead. cc_name and + | v1.rn = v1_lag.rn + 1 and + | v1.rn = v1_lead.rn - 1) + | select * from v2 + | where d_year = 1999 and + | avg_monthly_sales > 0 and + | case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 + | order by sum_sales - avg_monthly_sales, 3 + | limit 100 + """.stripMargin), + Query( + "q58", + """ + | with ss_items as + | (select i_item_id item_id, sum(ss_ext_sales_price) ss_item_rev + | from store_sales, item, date_dim + | where ss_item_sk = i_item_sk + | and d_date in (select d_date + | from date_dim + | where d_week_seq = (select d_week_seq + | from date_dim + | where d_date = '2000-01-03')) + | and ss_sold_date_sk = d_date_sk + | group by i_item_id), + | cs_items as + | (select i_item_id item_id + | ,sum(cs_ext_sales_price) cs_item_rev + | from catalog_sales, item, date_dim + | where cs_item_sk = i_item_sk + | and d_date in (select d_date + | from date_dim + | where d_week_seq = (select d_week_seq + | from date_dim + | where d_date = '2000-01-03')) + | and cs_sold_date_sk = d_date_sk + | group by i_item_id), + | ws_items as + | (select i_item_id item_id, sum(ws_ext_sales_price) ws_item_rev + | from web_sales, item, date_dim + | where ws_item_sk = i_item_sk + | and d_date in (select d_date + | from date_dim + | where d_week_seq =(select d_week_seq + | from date_dim + | where d_date = '2000-01-03')) + | and ws_sold_date_sk = d_date_sk + | group by i_item_id) + | select ss_items.item_id + | ,ss_item_rev + | ,ss_item_rev/(ss_item_rev+cs_item_rev+ws_item_rev)/3 * 100 ss_dev + | ,cs_item_rev + | ,cs_item_rev/(ss_item_rev+cs_item_rev+ws_item_rev)/3 * 100 cs_dev + | ,ws_item_rev + | ,ws_item_rev/(ss_item_rev+cs_item_rev+ws_item_rev)/3 * 100 ws_dev + | ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average + | from ss_items,cs_items,ws_items + | where ss_items.item_id=cs_items.item_id + | and ss_items.item_id=ws_items.item_id + | and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev + | and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev + | and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev + | and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev + | and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev + | and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev + | order by item_id, ss_item_rev + | limit 100 + """.stripMargin), + Query( + "q59", + """ + | with wss as + | (select d_week_seq, + | ss_store_sk, + | sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, + | sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, + | sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, + | sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, + | sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, + | sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, + | sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales + | from store_sales,date_dim + | where d_date_sk = ss_sold_date_sk + | group by d_week_seq,ss_store_sk + | ) + | select s_store_name1,s_store_id1,d_week_seq1 + | ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 + | ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 + | ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 + | from + | (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 + | ,s_store_id s_store_id1,sun_sales sun_sales1 + | ,mon_sales mon_sales1,tue_sales tue_sales1 + | ,wed_sales wed_sales1,thu_sales thu_sales1 + | ,fri_sales fri_sales1,sat_sales sat_sales1 + | from wss,store,date_dim d + | where d.d_week_seq = wss.d_week_seq and + | ss_store_sk = s_store_sk and + | d_month_seq between 1212 and 1212 + 11) y, + | (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 + | ,s_store_id s_store_id2,sun_sales sun_sales2 + | ,mon_sales mon_sales2,tue_sales tue_sales2 + | ,wed_sales wed_sales2,thu_sales thu_sales2 + | ,fri_sales fri_sales2,sat_sales sat_sales2 + | from wss,store,date_dim d + | where d.d_week_seq = wss.d_week_seq and + | ss_store_sk = s_store_sk and + | d_month_seq between 1212+ 12 and 1212 + 23) x + | where s_store_id1=s_store_id2 + | and d_week_seq1=d_week_seq2-52 + | order by s_store_name1,s_store_id1,d_week_seq1 + | limit 100 + """.stripMargin), + Query( + "q60", + """ + | with ss as ( + | select i_item_id,sum(ss_ext_sales_price) total_sales + | from store_sales, date_dim, customer_address, item + | where + | i_item_id in (select i_item_id from item where i_category in ('Music')) + | and ss_item_sk = i_item_sk + | and ss_sold_date_sk = d_date_sk + | and d_year = 1998 + | and d_moy = 9 + | and ss_addr_sk = ca_address_sk + | and ca_gmt_offset = -5 + | group by i_item_id), + | cs as ( + | select i_item_id,sum(cs_ext_sales_price) total_sales + | from catalog_sales, date_dim, customer_address, item + | where + | i_item_id in (select i_item_id from item where i_category in ('Music')) + | and cs_item_sk = i_item_sk + | and cs_sold_date_sk = d_date_sk + | and d_year = 1998 + | and d_moy = 9 + | and cs_bill_addr_sk = ca_address_sk + | and ca_gmt_offset = -5 + | group by i_item_id), + | ws as ( + | select i_item_id,sum(ws_ext_sales_price) total_sales + | from web_sales, date_dim, customer_address, item + | where + | i_item_id in (select i_item_id from item where i_category in ('Music')) + | and ws_item_sk = i_item_sk + | and ws_sold_date_sk = d_date_sk + | and d_year = 1998 + | and d_moy = 9 + | and ws_bill_addr_sk = ca_address_sk + | and ca_gmt_offset = -5 + | group by i_item_id) + | select i_item_id, sum(total_sales) total_sales + | from (select * from ss + | union all + | select * from cs + | union all + | select * from ws) tmp1 + | group by i_item_id + | order by i_item_id, total_sales + | limit 100 + """.stripMargin), + Query( + "q61", + s""" + | select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 + | from + | (select sum(ss_ext_sales_price) promotions + | from store_sales, store, promotion, date_dim, customer, customer_address, item + | where ss_sold_date_sk = d_date_sk + | and ss_store_sk = s_store_sk + | and ss_promo_sk = p_promo_sk + | and ss_customer_sk= c_customer_sk + | and ca_address_sk = c_current_addr_sk + | and ss_item_sk = i_item_sk + | and ca_gmt_offset = -5 + | and i_category = 'Jewelry' + | and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') + | and s_gmt_offset = -5 + | and d_year = 1998 + | and d_moy = 11) promotional_sales, + | (select sum(ss_ext_sales_price) total + | from store_sales, store, date_dim, customer, customer_address, item + | where ss_sold_date_sk = d_date_sk + | and ss_store_sk = s_store_sk + | and ss_customer_sk= c_customer_sk + | and ca_address_sk = c_current_addr_sk + | and ss_item_sk = i_item_sk + | and ca_gmt_offset = -5 + | and i_category = 'Jewelry' + | and s_gmt_offset = -5 + | and d_year = 1998 + | and d_moy = 11) all_sales + | order by promotions, total + | limit 100 + """.stripMargin), + // Modifications: " -> ` + Query( + "q62", + """ + | select + | substr(w_warehouse_name,1,20) + | ,sm_type + | ,web_name + | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` + | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and + | (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` + | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and + | (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` + | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and + | (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` + | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` + | from + | web_sales, warehouse, ship_mode, web_site, date_dim + | where + | d_month_seq between 1200 and 1200 + 11 + | and ws_ship_date_sk = d_date_sk + | and ws_warehouse_sk = w_warehouse_sk + | and ws_ship_mode_sk = sm_ship_mode_sk + | and ws_web_site_sk = web_site_sk + | group by + | substr(w_warehouse_name,1,20), sm_type, web_name + | order by + | substr(w_warehouse_name,1,20), sm_type, web_name + | limit 100 + """.stripMargin), + Query( + "q63", + """ + | select * + | from (select i_manager_id + | ,sum(ss_sales_price) sum_sales + | ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales + | from item + | ,store_sales + | ,date_dim + | ,store + | where ss_item_sk = i_item_sk + | and ss_sold_date_sk = d_date_sk + | and ss_store_sk = s_store_sk + | and d_month_seq in (1200,1200+1,1200+2,1200+3,1200+4,1200+5,1200+6,1200+7, + | 1200+8,1200+9,1200+10,1200+11) + | and (( i_category in ('Books','Children','Electronics') + | and i_class in ('personal','portable','refernece','self-help') + | and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', + | 'exportiunivamalg #9','scholaramalgamalg #9')) + | or( i_category in ('Women','Music','Men') + | and i_class in ('accessories','classical','fragrances','pants') + | and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', + | 'importoamalg #1'))) + | group by i_manager_id, d_moy) tmp1 + | where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 + | order by i_manager_id + | ,avg_monthly_sales + | ,sum_sales + | limit 100 + """.stripMargin), + Query( + "q64", + """ + | with cs_ui as + | (select cs_item_sk + | ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund + | from catalog_sales + | ,catalog_returns + | where cs_item_sk = cr_item_sk + | and cs_order_number = cr_order_number + | group by cs_item_sk + | having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), + | cross_sales as + | (select i_product_name product_name, i_item_sk item_sk, s_store_name store_name, s_zip store_zip, + | ad1.ca_street_number b_street_number, ad1.ca_street_name b_streen_name, ad1.ca_city b_city, + | ad1.ca_zip b_zip, ad2.ca_street_number c_street_number, ad2.ca_street_name c_street_name, + | ad2.ca_city c_city, ad2.ca_zip c_zip, d1.d_year as syear, d2.d_year as fsyear, d3.d_year s2year, + | count(*) cnt, sum(ss_wholesale_cost) s1, sum(ss_list_price) s2, sum(ss_coupon_amt) s3 + | FROM store_sales, store_returns, cs_ui, date_dim d1, date_dim d2, date_dim d3, + | store, customer, customer_demographics cd1, customer_demographics cd2, + | promotion, household_demographics hd1, household_demographics hd2, + | customer_address ad1, customer_address ad2, income_band ib1, income_band ib2, item + | WHERE ss_store_sk = s_store_sk AND + | ss_sold_date_sk = d1.d_date_sk AND + | ss_customer_sk = c_customer_sk AND + | ss_cdemo_sk= cd1.cd_demo_sk AND + | ss_hdemo_sk = hd1.hd_demo_sk AND + | ss_addr_sk = ad1.ca_address_sk and + | ss_item_sk = i_item_sk and + | ss_item_sk = sr_item_sk and + | ss_ticket_number = sr_ticket_number and + | ss_item_sk = cs_ui.cs_item_sk and + | c_current_cdemo_sk = cd2.cd_demo_sk AND + | c_current_hdemo_sk = hd2.hd_demo_sk AND + | c_current_addr_sk = ad2.ca_address_sk and + | c_first_sales_date_sk = d2.d_date_sk and + | c_first_shipto_date_sk = d3.d_date_sk and + | ss_promo_sk = p_promo_sk and + | hd1.hd_income_band_sk = ib1.ib_income_band_sk and + | hd2.hd_income_band_sk = ib2.ib_income_band_sk and + | cd1.cd_marital_status <> cd2.cd_marital_status and + | i_color in ('purple','burlywood','indian','spring','floral','medium') and + | i_current_price between 64 and 64 + 10 and + | i_current_price between 64 + 1 and 64 + 15 + | group by i_product_name, i_item_sk, s_store_name, s_zip, ad1.ca_street_number, + | ad1.ca_street_name, ad1.ca_city, ad1.ca_zip, ad2.ca_street_number, + | ad2.ca_street_name, ad2.ca_city, ad2.ca_zip, d1.d_year, d2.d_year, d3.d_year + | ) + | select cs1.product_name, cs1.store_name, cs1.store_zip, cs1.b_street_number, + | cs1.b_streen_name, cs1.b_city, cs1.b_zip, cs1.c_street_number, cs1.c_street_name, + | cs1.c_city, cs1.c_zip, cs1.syear, cs1.cnt, cs1.s1, cs1.s2, cs1.s3, cs2.s1, + | cs2.s2, cs2.s3, cs2.syear, cs2.cnt + | from cross_sales cs1,cross_sales cs2 + | where cs1.item_sk=cs2.item_sk and + | cs1.syear = 1999 and + | cs2.syear = 1999 + 1 and + | cs2.cnt <= cs1.cnt and + | cs1.store_name = cs2.store_name and + | cs1.store_zip = cs2.store_zip + | order by cs1.product_name, cs1.store_name, cs2.cnt + """.stripMargin), + Query( + "q65", + """ + | select + | s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand + | from store, item, + | (select ss_store_sk, avg(revenue) as ave + | from + | (select ss_store_sk, ss_item_sk, + | sum(ss_sales_price) as revenue + | from store_sales, date_dim + | where ss_sold_date_sk = d_date_sk and d_month_seq between 1176 and 1176+11 + | group by ss_store_sk, ss_item_sk) sa + | group by ss_store_sk) sb, + | (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue + | from store_sales, date_dim + | where ss_sold_date_sk = d_date_sk and d_month_seq between 1176 and 1176+11 + | group by ss_store_sk, ss_item_sk) sc + | where sb.ss_store_sk = sc.ss_store_sk and + | sc.revenue <= 0.1 * sb.ave and + | s_store_sk = sc.ss_store_sk and + | i_item_sk = sc.ss_item_sk + | order by s_store_name, i_item_desc + | limit 100 + """.stripMargin), + // Modifications: "||" -> concat + Query( + "q66", + """ + | select w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, + | ship_carriers, year + | ,sum(jan_sales) as jan_sales + | ,sum(feb_sales) as feb_sales + | ,sum(mar_sales) as mar_sales + | ,sum(apr_sales) as apr_sales + | ,sum(may_sales) as may_sales + | ,sum(jun_sales) as jun_sales + | ,sum(jul_sales) as jul_sales + | ,sum(aug_sales) as aug_sales + | ,sum(sep_sales) as sep_sales + | ,sum(oct_sales) as oct_sales + | ,sum(nov_sales) as nov_sales + | ,sum(dec_sales) as dec_sales + | ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot + | ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot + | ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot + | ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot + | ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot + | ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot + | ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot + | ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot + | ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot + | ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot + | ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot + | ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot + | ,sum(jan_net) as jan_net + | ,sum(feb_net) as feb_net + | ,sum(mar_net) as mar_net + | ,sum(apr_net) as apr_net + | ,sum(may_net) as may_net + | ,sum(jun_net) as jun_net + | ,sum(jul_net) as jul_net + | ,sum(aug_net) as aug_net + | ,sum(sep_net) as sep_net + | ,sum(oct_net) as oct_net + | ,sum(nov_net) as nov_net + | ,sum(dec_net) as dec_net + | from ( + | (select + | w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country + | ,concat('DHL', ',', 'BARIAN') as ship_carriers + | ,d_year as year + | ,sum(case when d_moy = 1 then ws_ext_sales_price * ws_quantity else 0 end) as jan_sales + | ,sum(case when d_moy = 2 then ws_ext_sales_price * ws_quantity else 0 end) as feb_sales + | ,sum(case when d_moy = 3 then ws_ext_sales_price * ws_quantity else 0 end) as mar_sales + | ,sum(case when d_moy = 4 then ws_ext_sales_price * ws_quantity else 0 end) as apr_sales + | ,sum(case when d_moy = 5 then ws_ext_sales_price * ws_quantity else 0 end) as may_sales + | ,sum(case when d_moy = 6 then ws_ext_sales_price * ws_quantity else 0 end) as jun_sales + | ,sum(case when d_moy = 7 then ws_ext_sales_price * ws_quantity else 0 end) as jul_sales + | ,sum(case when d_moy = 8 then ws_ext_sales_price * ws_quantity else 0 end) as aug_sales + | ,sum(case when d_moy = 9 then ws_ext_sales_price * ws_quantity else 0 end) as sep_sales + | ,sum(case when d_moy = 10 then ws_ext_sales_price * ws_quantity else 0 end) as oct_sales + | ,sum(case when d_moy = 11 then ws_ext_sales_price * ws_quantity else 0 end) as nov_sales + | ,sum(case when d_moy = 12 then ws_ext_sales_price * ws_quantity else 0 end) as dec_sales + | ,sum(case when d_moy = 1 then ws_net_paid * ws_quantity else 0 end) as jan_net + | ,sum(case when d_moy = 2 then ws_net_paid * ws_quantity else 0 end) as feb_net + | ,sum(case when d_moy = 3 then ws_net_paid * ws_quantity else 0 end) as mar_net + | ,sum(case when d_moy = 4 then ws_net_paid * ws_quantity else 0 end) as apr_net + | ,sum(case when d_moy = 5 then ws_net_paid * ws_quantity else 0 end) as may_net + | ,sum(case when d_moy = 6 then ws_net_paid * ws_quantity else 0 end) as jun_net + | ,sum(case when d_moy = 7 then ws_net_paid * ws_quantity else 0 end) as jul_net + | ,sum(case when d_moy = 8 then ws_net_paid * ws_quantity else 0 end) as aug_net + | ,sum(case when d_moy = 9 then ws_net_paid * ws_quantity else 0 end) as sep_net + | ,sum(case when d_moy = 10 then ws_net_paid * ws_quantity else 0 end) as oct_net + | ,sum(case when d_moy = 11 then ws_net_paid * ws_quantity else 0 end) as nov_net + | ,sum(case when d_moy = 12 then ws_net_paid * ws_quantity else 0 end) as dec_net + | from + | web_sales, warehouse, date_dim, time_dim, ship_mode + | where + | ws_warehouse_sk = w_warehouse_sk + | and ws_sold_date_sk = d_date_sk + | and ws_sold_time_sk = t_time_sk + | and ws_ship_mode_sk = sm_ship_mode_sk + | and d_year = 2001 + | and t_time between 30838 and 30838+28800 + | and sm_carrier in ('DHL','BARIAN') + | group by + | w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, d_year) + | union all + | (select w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country + | ,concat('DHL', ',', 'BARIAN') as ship_carriers + | ,d_year as year + | ,sum(case when d_moy = 1 then cs_sales_price * cs_quantity else 0 end) as jan_sales + | ,sum(case when d_moy = 2 then cs_sales_price * cs_quantity else 0 end) as feb_sales + | ,sum(case when d_moy = 3 then cs_sales_price * cs_quantity else 0 end) as mar_sales + | ,sum(case when d_moy = 4 then cs_sales_price * cs_quantity else 0 end) as apr_sales + | ,sum(case when d_moy = 5 then cs_sales_price * cs_quantity else 0 end) as may_sales + | ,sum(case when d_moy = 6 then cs_sales_price * cs_quantity else 0 end) as jun_sales + | ,sum(case when d_moy = 7 then cs_sales_price * cs_quantity else 0 end) as jul_sales + | ,sum(case when d_moy = 8 then cs_sales_price * cs_quantity else 0 end) as aug_sales + | ,sum(case when d_moy = 9 then cs_sales_price * cs_quantity else 0 end) as sep_sales + | ,sum(case when d_moy = 10 then cs_sales_price * cs_quantity else 0 end) as oct_sales + | ,sum(case when d_moy = 11 then cs_sales_price * cs_quantity else 0 end) as nov_sales + | ,sum(case when d_moy = 12 then cs_sales_price * cs_quantity else 0 end) as dec_sales + | ,sum(case when d_moy = 1 then cs_net_paid_inc_tax * cs_quantity else 0 end) as jan_net + | ,sum(case when d_moy = 2 then cs_net_paid_inc_tax * cs_quantity else 0 end) as feb_net + | ,sum(case when d_moy = 3 then cs_net_paid_inc_tax * cs_quantity else 0 end) as mar_net + | ,sum(case when d_moy = 4 then cs_net_paid_inc_tax * cs_quantity else 0 end) as apr_net + | ,sum(case when d_moy = 5 then cs_net_paid_inc_tax * cs_quantity else 0 end) as may_net + | ,sum(case when d_moy = 6 then cs_net_paid_inc_tax * cs_quantity else 0 end) as jun_net + | ,sum(case when d_moy = 7 then cs_net_paid_inc_tax * cs_quantity else 0 end) as jul_net + | ,sum(case when d_moy = 8 then cs_net_paid_inc_tax * cs_quantity else 0 end) as aug_net + | ,sum(case when d_moy = 9 then cs_net_paid_inc_tax * cs_quantity else 0 end) as sep_net + | ,sum(case when d_moy = 10 then cs_net_paid_inc_tax * cs_quantity else 0 end) as oct_net + | ,sum(case when d_moy = 11 then cs_net_paid_inc_tax * cs_quantity else 0 end) as nov_net + | ,sum(case when d_moy = 12 then cs_net_paid_inc_tax * cs_quantity else 0 end) as dec_net + | from + | catalog_sales, warehouse, date_dim, time_dim, ship_mode + | where + | cs_warehouse_sk = w_warehouse_sk + | and cs_sold_date_sk = d_date_sk + | and cs_sold_time_sk = t_time_sk + | and cs_ship_mode_sk = sm_ship_mode_sk + | and d_year = 2001 + | and t_time between 30838 AND 30838+28800 + | and sm_carrier in ('DHL','BARIAN') + | group by + | w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, d_year + | ) + | ) x + | group by + | w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, + | ship_carriers, year + | order by w_warehouse_name + | limit 100 + """.stripMargin), + Query( + "q67", + """ + | select * from + | (select i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy, s_store_id, + | sumsales, rank() over (partition by i_category order by sumsales desc) rk + | from + | (select i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy, + | s_store_id, sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales + | from store_sales, date_dim, store, item + | where ss_sold_date_sk=d_date_sk + | and ss_item_sk=i_item_sk + | and ss_store_sk = s_store_sk + | and d_month_seq between 1200 and 1200+11 + | group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, + | d_moy,s_store_id))dw1) dw2 + | where rk <= 100 + | order by + | i_category, i_class, i_brand, i_product_name, d_year, + | d_qoy, d_moy, s_store_id, sumsales, rk + | limit 100 + """.stripMargin), + Query( + "q68", + """ + | select + | c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number, extended_price, + | extended_tax, list_price + | from (select + | ss_ticket_number, ss_customer_sk, ca_city bought_city, + | sum(ss_ext_sales_price) extended_price, + | sum(ss_ext_list_price) list_price, + | sum(ss_ext_tax) extended_tax + | from store_sales, date_dim, store, household_demographics, customer_address + | where store_sales.ss_sold_date_sk = date_dim.d_date_sk + | and store_sales.ss_store_sk = store.s_store_sk + | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + | and store_sales.ss_addr_sk = customer_address.ca_address_sk + | and date_dim.d_dom between 1 and 2 + | and (household_demographics.hd_dep_count = 4 or + | household_demographics.hd_vehicle_count = 3) + | and date_dim.d_year in (1999,1999+1,1999+2) + | and store.s_city in ('Midway','Fairview') + | group by ss_ticket_number, ss_customer_sk, ss_addr_sk,ca_city) dn, + | customer, + | customer_address current_addr + | where ss_customer_sk = c_customer_sk + | and customer.c_current_addr_sk = current_addr.ca_address_sk + | and current_addr.ca_city <> bought_city + | order by c_last_name, ss_ticket_number + | limit 100 + """.stripMargin), + Query( + "q69", + """ + | select + | cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, + | cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 + | from + | customer c,customer_address ca,customer_demographics + | where + | c.c_current_addr_sk = ca.ca_address_sk and + | ca_state in ('KY', 'GA', 'NM') and + | cd_demo_sk = c.c_current_cdemo_sk and + | exists (select * from store_sales, date_dim + | where c.c_customer_sk = ss_customer_sk and + | ss_sold_date_sk = d_date_sk and + | d_year = 2001 and + | d_moy between 4 and 4+2) and + | (not exists (select * from web_sales, date_dim + | where c.c_customer_sk = ws_bill_customer_sk and + | ws_sold_date_sk = d_date_sk and + | d_year = 2001 and + | d_moy between 4 and 4+2) and + | not exists (select * from catalog_sales, date_dim + | where c.c_customer_sk = cs_ship_customer_sk and + | cs_sold_date_sk = d_date_sk and + | d_year = 2001 and + | d_moy between 4 and 4+2)) + | group by cd_gender, cd_marital_status, cd_education_status, + | cd_purchase_estimate, cd_credit_rating + | order by cd_gender, cd_marital_status, cd_education_status, + | cd_purchase_estimate, cd_credit_rating + | limit 100 + """.stripMargin), + Query( + "q70", + """ + | select + | sum(ss_net_profit) as total_sum, s_state, s_county + | ,grouping(s_state)+grouping(s_county) as lochierarchy + | ,rank() over ( + | partition by grouping(s_state)+grouping(s_county), + | case when grouping(s_county) = 0 then s_state end + | order by sum(ss_net_profit) desc) as rank_within_parent + | from + | store_sales, date_dim d1, store + | where + | d1.d_month_seq between 1200 and 1200+11 + | and d1.d_date_sk = ss_sold_date_sk + | and s_store_sk = ss_store_sk + | and s_state in + | (select s_state from + | (select s_state as s_state, + | rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking + | from store_sales, store, date_dim + | where d_month_seq between 1200 and 1200+11 + | and d_date_sk = ss_sold_date_sk + | and s_store_sk = ss_store_sk + | group by s_state) tmp1 + | where ranking <= 5) + | group by rollup(s_state,s_county) + | order by + | lochierarchy desc + | ,case when lochierarchy = 0 then s_state end + | ,rank_within_parent + | limit 100 + """.stripMargin), + Query( + "q71", + """ + | select i_brand_id brand_id, i_brand brand,t_hour,t_minute, + | sum(ext_price) ext_price + | from item, + | (select + | ws_ext_sales_price as ext_price, + | ws_sold_date_sk as sold_date_sk, + | ws_item_sk as sold_item_sk, + | ws_sold_time_sk as time_sk + | from web_sales, date_dim + | where d_date_sk = ws_sold_date_sk + | and d_moy=11 + | and d_year=1999 + | union all + | select + | cs_ext_sales_price as ext_price, + | cs_sold_date_sk as sold_date_sk, + | cs_item_sk as sold_item_sk, + | cs_sold_time_sk as time_sk + | from catalog_sales, date_dim + | where d_date_sk = cs_sold_date_sk + | and d_moy=11 + | and d_year=1999 + | union all + | select + | ss_ext_sales_price as ext_price, + | ss_sold_date_sk as sold_date_sk, + | ss_item_sk as sold_item_sk, + | ss_sold_time_sk as time_sk + | from store_sales,date_dim + | where d_date_sk = ss_sold_date_sk + | and d_moy=11 + | and d_year=1999 + | ) as tmp, time_dim + | where + | sold_item_sk = i_item_sk + | and i_manager_id=1 + | and time_sk = t_time_sk + | and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') + | group by i_brand, i_brand_id,t_hour,t_minute + | order by ext_price desc, brand_id + """.stripMargin), + // Modifications: "+ days" -> date_add + Query( + "q72", + """ + | select i_item_desc + | ,w_warehouse_name + | ,d1.d_week_seq + | ,count(case when p_promo_sk is null then 1 else 0 end) no_promo + | ,count(case when p_promo_sk is not null then 1 else 0 end) promo + | ,count(*) total_cnt + | from catalog_sales + | join inventory on (cs_item_sk = inv_item_sk) + | join warehouse on (w_warehouse_sk=inv_warehouse_sk) + | join item on (i_item_sk = cs_item_sk) + | join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) + | join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) + | join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) + | join date_dim d2 on (inv_date_sk = d2.d_date_sk) + | join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) + | left outer join promotion on (cs_promo_sk=p_promo_sk) + | left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) + | where d1.d_week_seq = d2.d_week_seq + | and inv_quantity_on_hand < cs_quantity + | and d3.d_date > (cast(d1.d_date AS DATE) + interval 5 days) + | and hd_buy_potential = '>10000' + | and d1.d_year = 1999 + | and hd_buy_potential = '>10000' + | and cd_marital_status = 'D' + | and d1.d_year = 1999 + | group by i_item_desc,w_warehouse_name,d1.d_week_seq + | order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq + | limit 100 + """.stripMargin), + Query( + "q72b", + """ + | select i_item_desc + | ,w_warehouse_name + | ,d1.d_week_seq + | ,count(case when p_promo_sk is null then 1 else 0 end) no_promo + | ,count(case when p_promo_sk is not null then 1 else 0 end) promo + | ,count(*) total_cnt + | from catalog_sales + | join item on (i_item_sk = cs_item_sk) + | join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) + | join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) + | join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) + | join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) + | join inventory on (cs_item_sk = inv_item_sk) + | join warehouse on (w_warehouse_sk=inv_warehouse_sk) + | join date_dim d2 on (inv_date_sk = d2.d_date_sk) + | left outer join promotion on (cs_promo_sk=p_promo_sk) + | left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) + | where d1.d_week_seq = d2.d_week_seq + | and inv_quantity_on_hand < cs_quantity + | and d3.d_date > (cast(d1.d_date AS DATE) + interval 5 days) + | and hd_buy_potential = '>10000' + | and d1.d_year = 1999 + | and hd_buy_potential = '>10000' + | and cd_marital_status = 'D' + | and d1.d_year = 1999 + | group by i_item_desc,w_warehouse_name,d1.d_week_seq + | order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq + | limit 100 + """.stripMargin), + Query( + "q73", + """ + | select + | c_last_name, c_first_name, c_salutation, c_preferred_cust_flag, + | ss_ticket_number, cnt from + | (select ss_ticket_number, ss_customer_sk, count(*) cnt + | from store_sales,date_dim,store,household_demographics + | where store_sales.ss_sold_date_sk = date_dim.d_date_sk + | and store_sales.ss_store_sk = store.s_store_sk + | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + | and date_dim.d_dom between 1 and 2 + | and (household_demographics.hd_buy_potential = '>10000' or + | household_demographics.hd_buy_potential = 'unknown') + | and household_demographics.hd_vehicle_count > 0 + | and case when household_demographics.hd_vehicle_count > 0 then + | household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 + | and date_dim.d_year in (1999,1999+1,1999+2) + | and store.s_county in ('Williamson County','Franklin Parish','Bronx County','Orange County') + | group by ss_ticket_number,ss_customer_sk) dj,customer + | where ss_customer_sk = c_customer_sk + | and cnt between 1 and 5 + | order by cnt desc + """.stripMargin), + Query( + "q74", + """ + | with year_total as ( + | select + | c_customer_id customer_id, c_first_name customer_first_name, + | c_last_name customer_last_name, d_year as year, + | sum(ss_net_paid) year_total, 's' sale_type + | from + | customer, store_sales, date_dim + | where c_customer_sk = ss_customer_sk + | and ss_sold_date_sk = d_date_sk + | and d_year in (2001,2001+1) + | group by + | c_customer_id, c_first_name, c_last_name, d_year + | union all + | select + | c_customer_id customer_id, c_first_name customer_first_name, + | c_last_name customer_last_name, d_year as year, + | sum(ws_net_paid) year_total, 'w' sale_type + | from + | customer, web_sales, date_dim + | where c_customer_sk = ws_bill_customer_sk + | and ws_sold_date_sk = d_date_sk + | and d_year in (2001,2001+1) + | group by + | c_customer_id, c_first_name, c_last_name, d_year) + | select + | t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name + | from + | year_total t_s_firstyear, year_total t_s_secyear, + | year_total t_w_firstyear, year_total t_w_secyear + | where t_s_secyear.customer_id = t_s_firstyear.customer_id + | and t_s_firstyear.customer_id = t_w_secyear.customer_id + | and t_s_firstyear.customer_id = t_w_firstyear.customer_id + | and t_s_firstyear.sale_type = 's' + | and t_w_firstyear.sale_type = 'w' + | and t_s_secyear.sale_type = 's' + | and t_w_secyear.sale_type = 'w' + | and t_s_firstyear.year = 2001 + | and t_s_secyear.year = 2001+1 + | and t_w_firstyear.year = 2001 + | and t_w_secyear.year = 2001+1 + | and t_s_firstyear.year_total > 0 + | and t_w_firstyear.year_total > 0 + | and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end + | > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end + | order by 1, 1, 1 + | limit 100 + """.stripMargin), + Query( + "q75", + """ + | WITH all_sales AS ( + | SELECT + | d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, + | SUM(sales_cnt) AS sales_cnt, SUM(sales_amt) AS sales_amt + | FROM ( + | SELECT + | d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, + | cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt, + | cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt + | FROM catalog_sales + | JOIN item ON i_item_sk=cs_item_sk + | JOIN date_dim ON d_date_sk=cs_sold_date_sk + | LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number + | AND cs_item_sk=cr_item_sk) + | WHERE i_category='Books' + | UNION + | SELECT + | d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, + | ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt, + | ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt + | FROM store_sales + | JOIN item ON i_item_sk=ss_item_sk + | JOIN date_dim ON d_date_sk=ss_sold_date_sk + | LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number + | AND ss_item_sk=sr_item_sk) + | WHERE i_category='Books' + | UNION + | SELECT + | d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, + | ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt, + | ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt + | FROM web_sales + | JOIN item ON i_item_sk=ws_item_sk + | JOIN date_dim ON d_date_sk=ws_sold_date_sk + | LEFT JOIN web_returns ON (ws_order_number=wr_order_number + | AND ws_item_sk=wr_item_sk) + | WHERE i_category='Books') sales_detail + | GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) + | SELECT + | prev_yr.d_year AS prev_year, curr_yr.d_year AS year, curr_yr.i_brand_id, + | curr_yr.i_class_id, curr_yr.i_category_id, curr_yr.i_manufact_id, + | prev_yr.sales_cnt AS prev_yr_cnt, curr_yr.sales_cnt AS curr_yr_cnt, + | curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff, + | curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff + | FROM all_sales curr_yr, all_sales prev_yr + | WHERE curr_yr.i_brand_id=prev_yr.i_brand_id + | AND curr_yr.i_class_id=prev_yr.i_class_id + | AND curr_yr.i_category_id=prev_yr.i_category_id + | AND curr_yr.i_manufact_id=prev_yr.i_manufact_id + | AND curr_yr.d_year=2002 + | AND prev_yr.d_year=2002-1 + | AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 + | ORDER BY sales_cnt_diff + | LIMIT 100 + """.stripMargin), + Query( + "q76", + """ + | SELECT + | channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, + | SUM(ext_sales_price) sales_amt + | FROM( + | SELECT + | 'store' as channel, ss_store_sk col_name, d_year, d_qoy, i_category, + | ss_ext_sales_price ext_sales_price + | FROM store_sales, item, date_dim + | WHERE ss_store_sk IS NULL + | AND ss_sold_date_sk=d_date_sk + | AND ss_item_sk=i_item_sk + | UNION ALL + | SELECT + | 'web' as channel, ws_ship_customer_sk col_name, d_year, d_qoy, i_category, + | ws_ext_sales_price ext_sales_price + | FROM web_sales, item, date_dim + | WHERE ws_ship_customer_sk IS NULL + | AND ws_sold_date_sk=d_date_sk + | AND ws_item_sk=i_item_sk + | UNION ALL + | SELECT + | 'catalog' as channel, cs_ship_addr_sk col_name, d_year, d_qoy, i_category, + | cs_ext_sales_price ext_sales_price + | FROM catalog_sales, item, date_dim + | WHERE cs_ship_addr_sk IS NULL + | AND cs_sold_date_sk=d_date_sk + | AND cs_item_sk=i_item_sk) foo + | GROUP BY channel, col_name, d_year, d_qoy, i_category + | ORDER BY channel, col_name, d_year, d_qoy, i_category + | limit 100 + """.stripMargin), + // Modifications: "+ days" -> date_add + Query( + "q77", + """ + | with ss as + | (select s_store_sk, sum(ss_ext_sales_price) as sales, sum(ss_net_profit) as profit + | from store_sales, date_dim, store + | where ss_sold_date_sk = d_date_sk + | and d_date between cast('2000-08-03' as date) and + | (cast('2000-08-03' as date) + interval 30 days) + | and ss_store_sk = s_store_sk + | group by s_store_sk), + | sr as + | (select s_store_sk, sum(sr_return_amt) as returns, sum(sr_net_loss) as profit_loss + | from store_returns, date_dim, store + | where sr_returned_date_sk = d_date_sk + | and d_date between cast('2000-08-03' as date) and + | (cast('2000-08-03' as date) + interval 30 days) + | and sr_store_sk = s_store_sk + | group by s_store_sk), + | cs as + | (select cs_call_center_sk, sum(cs_ext_sales_price) as sales, sum(cs_net_profit) as profit + | from catalog_sales, date_dim + | where cs_sold_date_sk = d_date_sk + | and d_date between cast('2000-08-03' as date) and + | (cast('2000-08-03' as date) + interval 30 days) + | group by cs_call_center_sk), + | cr as + | (select sum(cr_return_amount) as returns, sum(cr_net_loss) as profit_loss + | from catalog_returns, date_dim + | where cr_returned_date_sk = d_date_sk + | and d_date between cast('2000-08-03]' as date) and + | (cast('2000-08-03' as date) + interval 30 day)), + | ws as + | (select wp_web_page_sk, sum(ws_ext_sales_price) as sales, sum(ws_net_profit) as profit + | from web_sales, date_dim, web_page + | where ws_sold_date_sk = d_date_sk + | and d_date between cast('2000-08-03' as date) and + | (cast('2000-08-03' as date) + interval 30 days) + | and ws_web_page_sk = wp_web_page_sk + | group by wp_web_page_sk), + | wr as + | (select wp_web_page_sk, sum(wr_return_amt) as returns, sum(wr_net_loss) as profit_loss + | from web_returns, date_dim, web_page + | where wr_returned_date_sk = d_date_sk + | and d_date between cast('2000-08-03' as date) and + | (cast('2000-08-03' as date) + interval 30 days) + | and wr_web_page_sk = wp_web_page_sk + | group by wp_web_page_sk) + | select channel, id, sum(sales) as sales, sum(returns) as returns, sum(profit) as profit + | from + | (select + | 'store channel' as channel, ss.s_store_sk as id, sales, + | coalesce(returns, 0) as returns, (profit - coalesce(profit_loss,0)) as profit + | from ss left join sr + | on ss.s_store_sk = sr.s_store_sk + | union all + | select + | 'catalog channel' as channel, cs_call_center_sk as id, sales, + | returns, (profit - profit_loss) as profit + | from cs, cr + | union all + | select + | 'web channel' as channel, ws.wp_web_page_sk as id, sales, + | coalesce(returns, 0) returns, (profit - coalesce(profit_loss,0)) as profit + | from ws left join wr + | on ws.wp_web_page_sk = wr.wp_web_page_sk + | ) x + | group by rollup(channel, id) + | order by channel, id + | limit 100 + """.stripMargin), + Query( + "q78", + """ + | with ws as + | (select d_year AS ws_sold_year, ws_item_sk, + | ws_bill_customer_sk ws_customer_sk, + | sum(ws_quantity) ws_qty, + | sum(ws_wholesale_cost) ws_wc, + | sum(ws_sales_price) ws_sp + | from web_sales + | left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk + | join date_dim on ws_sold_date_sk = d_date_sk + | where wr_order_number is null + | group by d_year, ws_item_sk, ws_bill_customer_sk + | ), + | cs as + | (select d_year AS cs_sold_year, cs_item_sk, + | cs_bill_customer_sk cs_customer_sk, + | sum(cs_quantity) cs_qty, + | sum(cs_wholesale_cost) cs_wc, + | sum(cs_sales_price) cs_sp + | from catalog_sales + | left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk + | join date_dim on cs_sold_date_sk = d_date_sk + | where cr_order_number is null + | group by d_year, cs_item_sk, cs_bill_customer_sk + | ), + | ss as + | (select d_year AS ss_sold_year, ss_item_sk, + | ss_customer_sk, + | sum(ss_quantity) ss_qty, + | sum(ss_wholesale_cost) ss_wc, + | sum(ss_sales_price) ss_sp + | from store_sales + | left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk + | join date_dim on ss_sold_date_sk = d_date_sk + | where sr_ticket_number is null + | group by d_year, ss_item_sk, ss_customer_sk + | ) + | select + | round(ss_qty/(coalesce(ws_qty+cs_qty,1)),2) ratio, + | ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, + | coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, + | coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, + | coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price + | from ss + | left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) + | left join cs on (cs_sold_year=ss_sold_year and ss_item_sk=cs_item_sk and cs_customer_sk=ss_customer_sk) + | where coalesce(ws_qty,0)>0 and coalesce(cs_qty, 0)>0 and ss_sold_year=2000 + | order by + | ratio, + | ss_qty desc, ss_wc desc, ss_sp desc, + | other_chan_qty, + | other_chan_wholesale_cost, + | other_chan_sales_price, + | round(ss_qty/(coalesce(ws_qty+cs_qty,1)),2) + | limit 100 + """.stripMargin), + Query( + "q79", + """ + | select + | c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit + | from + | (select ss_ticket_number + | ,ss_customer_sk + | ,store.s_city + | ,sum(ss_coupon_amt) amt + | ,sum(ss_net_profit) profit + | from store_sales,date_dim,store,household_demographics + | where store_sales.ss_sold_date_sk = date_dim.d_date_sk + | and store_sales.ss_store_sk = store.s_store_sk + | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + | and (household_demographics.hd_dep_count = 6 or + | household_demographics.hd_vehicle_count > 2) + | and date_dim.d_dow = 1 + | and date_dim.d_year in (1999,1999+1,1999+2) + | and store.s_number_employees between 200 and 295 + | group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer + | where ss_customer_sk = c_customer_sk + | order by c_last_name,c_first_name,substr(s_city,1,30), profit + | limit 100 + """.stripMargin), + // Modifications: "+ days" -> date_add + // Modifications: "||" -> "concat" + Query( + "q80", + """ + | with ssr as + | (select s_store_id as store_id, + | sum(ss_ext_sales_price) as sales, + | sum(coalesce(sr_return_amt, 0)) as returns, + | sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit + | from store_sales left outer join store_returns on + | (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), + | date_dim, store, item, promotion + | where ss_sold_date_sk = d_date_sk + | and d_date between cast('2000-08-23' as date) + | and (cast('2000-08-23' as date) + interval 30 days) + | and ss_store_sk = s_store_sk + | and ss_item_sk = i_item_sk + | and i_current_price > 50 + | and ss_promo_sk = p_promo_sk + | and p_channel_tv = 'N' + | group by s_store_id), + | csr as + | (select cp_catalog_page_id as catalog_page_id, + | sum(cs_ext_sales_price) as sales, + | sum(coalesce(cr_return_amount, 0)) as returns, + | sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit + | from catalog_sales left outer join catalog_returns on + | (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), + | date_dim, catalog_page, item, promotion + | where cs_sold_date_sk = d_date_sk + | and d_date between cast('2000-08-23' as date) + | and (cast('2000-08-23' as date) + interval 30 days) + | and cs_catalog_page_sk = cp_catalog_page_sk + | and cs_item_sk = i_item_sk + | and i_current_price > 50 + | and cs_promo_sk = p_promo_sk + | and p_channel_tv = 'N' + | group by cp_catalog_page_id), + | wsr as + | (select web_site_id, + | sum(ws_ext_sales_price) as sales, + | sum(coalesce(wr_return_amt, 0)) as returns, + | sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit + | from web_sales left outer join web_returns on + | (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), + | date_dim, web_site, item, promotion + | where ws_sold_date_sk = d_date_sk + | and d_date between cast('2000-08-23' as date) + | and (cast('2000-08-23' as date) + interval 30 days) + | and ws_web_site_sk = web_site_sk + | and ws_item_sk = i_item_sk + | and i_current_price > 50 + | and ws_promo_sk = p_promo_sk + | and p_channel_tv = 'N' + | group by web_site_id) + | select channel, id, sum(sales) as sales, sum(returns) as returns, sum(profit) as profit + | from (select + | 'store channel' as channel, concat('store', store_id) as id, sales, returns, profit + | from ssr + | union all + | select + | 'catalog channel' as channel, concat('catalog_page', catalog_page_id) as id, + | sales, returns, profit + | from csr + | union all + | select + | 'web channel' as channel, concat('web_site', web_site_id) as id, sales, returns, profit + | from wsr) x + | group by rollup (channel, id) + | order by channel, id + | limit 100 + """.stripMargin), + Query( + "q81", + """ + | with customer_total_return as + | (select + | cr_returning_customer_sk as ctr_customer_sk, ca_state as ctr_state, + | sum(cr_return_amt_inc_tax) as ctr_total_return + | from catalog_returns, date_dim, customer_address + | where cr_returned_date_sk = d_date_sk + | and d_year = 2000 + | and cr_returning_addr_sk = ca_address_sk + | group by cr_returning_customer_sk, ca_state ) + | select + | c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name, + | ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country, + | ca_gmt_offset,ca_location_type,ctr_total_return + | from customer_total_return ctr1, customer_address, customer + | where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 + | from customer_total_return ctr2 + | where ctr1.ctr_state = ctr2.ctr_state) + | and ca_address_sk = c_current_addr_sk + | and ca_state = 'GA' + | and ctr1.ctr_customer_sk = c_customer_sk + | order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name + | ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset + | ,ca_location_type,ctr_total_return + | limit 100 + """.stripMargin), + Query( + "q82", + """ + | select i_item_id, i_item_desc, i_current_price + | from item, inventory, date_dim, store_sales + | where i_current_price between 62 and 62+30 + | and inv_item_sk = i_item_sk + | and d_date_sk=inv_date_sk + | and d_date between cast('2000-05-25' as date) and (cast('2000-05-25' as date) + interval 60 days) + | and i_manufact_id in (129, 270, 821, 423) + | and inv_quantity_on_hand between 100 and 500 + | and ss_item_sk = i_item_sk + | group by i_item_id,i_item_desc,i_current_price + | order by i_item_id + | limit 100 + """.stripMargin), + Query( + "q83", + """ + | with sr_items as + | (select i_item_id item_id, sum(sr_return_quantity) sr_item_qty + | from store_returns, item, date_dim + | where sr_item_sk = i_item_sk + | and d_date in (select d_date from date_dim where d_week_seq in + | (select d_week_seq from date_dim where d_date in ('2000-06-30','2000-09-27','2000-11-17'))) + | and sr_returned_date_sk = d_date_sk + | group by i_item_id), + | cr_items as + | (select i_item_id item_id, sum(cr_return_quantity) cr_item_qty + | from catalog_returns, item, date_dim + | where cr_item_sk = i_item_sk + | and d_date in (select d_date from date_dim where d_week_seq in + | (select d_week_seq from date_dim where d_date in ('2000-06-30','2000-09-27','2000-11-17'))) + | and cr_returned_date_sk = d_date_sk + | group by i_item_id), + | wr_items as + | (select i_item_id item_id, sum(wr_return_quantity) wr_item_qty + | from web_returns, item, date_dim + | where wr_item_sk = i_item_sk and d_date in + | (select d_date from date_dim where d_week_seq in + | (select d_week_seq from date_dim where d_date in ('2000-06-30','2000-09-27','2000-11-17'))) + | and wr_returned_date_sk = d_date_sk + | group by i_item_id) + | select sr_items.item_id + | ,sr_item_qty + | ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev + | ,cr_item_qty + | ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev + | ,wr_item_qty + | ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev + | ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average + | from sr_items, cr_items, wr_items + | where sr_items.item_id=cr_items.item_id + | and sr_items.item_id=wr_items.item_id + | order by sr_items.item_id, sr_item_qty + | limit 100 + """.stripMargin), + // Modifications: "||" -> concat + Query( + "q84", + """ + | select c_customer_id as customer_id + | ,concat(c_last_name, ', ', c_first_name) as customername + | from customer + | ,customer_address + | ,customer_demographics + | ,household_demographics + | ,income_band + | ,store_returns + | where ca_city = 'Edgewood' + | and c_current_addr_sk = ca_address_sk + | and ib_lower_bound >= 38128 + | and ib_upper_bound <= 38128 + 50000 + | and ib_income_band_sk = hd_income_band_sk + | and cd_demo_sk = c_current_cdemo_sk + | and hd_demo_sk = c_current_hdemo_sk + | and sr_cdemo_sk = cd_demo_sk + | order by c_customer_id + | limit 100 + """.stripMargin), + Query( + "q85", + """ + | select + | substr(r_reason_desc,1,20), avg(ws_quantity), avg(wr_refunded_cash), avg(wr_fee) + | from web_sales, web_returns, web_page, customer_demographics cd1, + | customer_demographics cd2, customer_address, date_dim, reason + | where ws_web_page_sk = wp_web_page_sk + | and ws_item_sk = wr_item_sk + | and ws_order_number = wr_order_number + | and ws_sold_date_sk = d_date_sk and d_year = 2000 + | and cd1.cd_demo_sk = wr_refunded_cdemo_sk + | and cd2.cd_demo_sk = wr_returning_cdemo_sk + | and ca_address_sk = wr_refunded_addr_sk + | and r_reason_sk = wr_reason_sk + | and + | ( + | ( + | cd1.cd_marital_status = 'M' + | and + | cd1.cd_marital_status = cd2.cd_marital_status + | and + | cd1.cd_education_status = 'Advanced Degree' + | and + | cd1.cd_education_status = cd2.cd_education_status + | and + | ws_sales_price between 100.00 and 150.00 + | ) + | or + | ( + | cd1.cd_marital_status = 'S' + | and + | cd1.cd_marital_status = cd2.cd_marital_status + | and + | cd1.cd_education_status = 'College' + | and + | cd1.cd_education_status = cd2.cd_education_status + | and + | ws_sales_price between 50.00 and 100.00 + | ) + | or + | ( + | cd1.cd_marital_status = 'W' + | and + | cd1.cd_marital_status = cd2.cd_marital_status + | and + | cd1.cd_education_status = '2 yr Degree' + | and + | cd1.cd_education_status = cd2.cd_education_status + | and + | ws_sales_price between 150.00 and 200.00 + | ) + | ) + | and + | ( + | ( + | ca_country = 'United States' + | and + | ca_state in ('IN', 'OH', 'NJ') + | and ws_net_profit between 100 and 200 + | ) + | or + | ( + | ca_country = 'United States' + | and + | ca_state in ('WI', 'CT', 'KY') + | and ws_net_profit between 150 and 300 + | ) + | or + | ( + | ca_country = 'United States' + | and + | ca_state in ('LA', 'IA', 'AR') + | and ws_net_profit between 50 and 250 + | ) + | ) + | group by r_reason_desc + | order by substr(r_reason_desc,1,20) + | ,avg(ws_quantity) + | ,avg(wr_refunded_cash) + | ,avg(wr_fee) + | limit 100 + """.stripMargin), + Query( + "q86", + """ + | select sum(ws_net_paid) as total_sum, i_category, i_class, + | grouping(i_category)+grouping(i_class) as lochierarchy, + | rank() over ( + | partition by grouping(i_category)+grouping(i_class), + | case when grouping(i_class) = 0 then i_category end + | order by sum(ws_net_paid) desc) as rank_within_parent + | from + | web_sales, date_dim d1, item + | where + | d1.d_month_seq between 1200 and 1200+11 + | and d1.d_date_sk = ws_sold_date_sk + | and i_item_sk = ws_item_sk + | group by rollup(i_category,i_class) + | order by + | lochierarchy desc, + | case when lochierarchy = 0 then i_category end, + | rank_within_parent + | limit 100 + """.stripMargin), + Query( + "q87", + """ + | select count(*) + | from ((select distinct c_last_name, c_first_name, d_date + | from store_sales, date_dim, customer + | where store_sales.ss_sold_date_sk = date_dim.d_date_sk + | and store_sales.ss_customer_sk = customer.c_customer_sk + | and d_month_seq between 1200 and 1200+11) + | except + | (select distinct c_last_name, c_first_name, d_date + | from catalog_sales, date_dim, customer + | where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk + | and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk + | and d_month_seq between 1200 and 1200+11) + | except + | (select distinct c_last_name, c_first_name, d_date + | from web_sales, date_dim, customer + | where web_sales.ws_sold_date_sk = date_dim.d_date_sk + | and web_sales.ws_bill_customer_sk = customer.c_customer_sk + | and d_month_seq between 1200 and 1200+11) + |) cool_cust + """.stripMargin), + Query( + "q88", + """ + | select * + | from + | (select count(*) h8_30_to_9 + | from store_sales, household_demographics , time_dim, store + | where ss_sold_time_sk = time_dim.t_time_sk + | and ss_hdemo_sk = household_demographics.hd_demo_sk + | and ss_store_sk = s_store_sk + | and time_dim.t_hour = 8 + | and time_dim.t_minute >= 30 + | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or + | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or + | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) + | and store.s_store_name = 'ese') s1, + | (select count(*) h9_to_9_30 + | from store_sales, household_demographics , time_dim, store + | where ss_sold_time_sk = time_dim.t_time_sk + | and ss_hdemo_sk = household_demographics.hd_demo_sk + | and ss_store_sk = s_store_sk + | and time_dim.t_hour = 9 + | and time_dim.t_minute < 30 + | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or + | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or + | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) + | and store.s_store_name = 'ese') s2, + | (select count(*) h9_30_to_10 + | from store_sales, household_demographics , time_dim, store + | where ss_sold_time_sk = time_dim.t_time_sk + | and ss_hdemo_sk = household_demographics.hd_demo_sk + | and ss_store_sk = s_store_sk + | and time_dim.t_hour = 9 + | and time_dim.t_minute >= 30 + | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or + | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or + | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) + | and store.s_store_name = 'ese') s3, + | (select count(*) h10_to_10_30 + | from store_sales, household_demographics , time_dim, store + | where ss_sold_time_sk = time_dim.t_time_sk + | and ss_hdemo_sk = household_demographics.hd_demo_sk + | and ss_store_sk = s_store_sk + | and time_dim.t_hour = 10 + | and time_dim.t_minute < 30 + | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or + | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or + | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) + | and store.s_store_name = 'ese') s4, + | (select count(*) h10_30_to_11 + | from store_sales, household_demographics , time_dim, store + | where ss_sold_time_sk = time_dim.t_time_sk + | and ss_hdemo_sk = household_demographics.hd_demo_sk + | and ss_store_sk = s_store_sk + | and time_dim.t_hour = 10 + | and time_dim.t_minute >= 30 + | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or + | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or + | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) + | and store.s_store_name = 'ese') s5, + | (select count(*) h11_to_11_30 + | from store_sales, household_demographics , time_dim, store + | where ss_sold_time_sk = time_dim.t_time_sk + | and ss_hdemo_sk = household_demographics.hd_demo_sk + | and ss_store_sk = s_store_sk + | and time_dim.t_hour = 11 + | and time_dim.t_minute < 30 + | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or + | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or + | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) + | and store.s_store_name = 'ese') s6, + | (select count(*) h11_30_to_12 + | from store_sales, household_demographics , time_dim, store + | where ss_sold_time_sk = time_dim.t_time_sk + | and ss_hdemo_sk = household_demographics.hd_demo_sk + | and ss_store_sk = s_store_sk + | and time_dim.t_hour = 11 + | and time_dim.t_minute >= 30 + | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or + | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or + | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) + | and store.s_store_name = 'ese') s7, + | (select count(*) h12_to_12_30 + | from store_sales, household_demographics , time_dim, store + | where ss_sold_time_sk = time_dim.t_time_sk + | and ss_hdemo_sk = household_demographics.hd_demo_sk + | and ss_store_sk = s_store_sk + | and time_dim.t_hour = 12 + | and time_dim.t_minute < 30 + | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or + | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or + | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) + | and store.s_store_name = 'ese') s8 + """.stripMargin), + Query( + "q89", + """ + | select * + | from( + | select i_category, i_class, i_brand, + | s_store_name, s_company_name, + | d_moy, + | sum(ss_sales_price) sum_sales, + | avg(sum(ss_sales_price)) over + | (partition by i_category, i_brand, s_store_name, s_company_name) + | avg_monthly_sales + | from item, store_sales, date_dim, store + | where ss_item_sk = i_item_sk and + | ss_sold_date_sk = d_date_sk and + | ss_store_sk = s_store_sk and + | d_year in (1999) and + | ((i_category in ('Books','Electronics','Sports') and + | i_class in ('computers','stereo','football')) + | or (i_category in ('Men','Jewelry','Women') and + | i_class in ('shirts','birdal','dresses'))) + | group by i_category, i_class, i_brand, + | s_store_name, s_company_name, d_moy) tmp1 + | where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 + | order by sum_sales - avg_monthly_sales, s_store_name + | limit 100 + """.stripMargin), + Query( + "q90", + """ + | select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio + | from ( select count(*) amc + | from web_sales, household_demographics , time_dim, web_page + | where ws_sold_time_sk = time_dim.t_time_sk + | and ws_ship_hdemo_sk = household_demographics.hd_demo_sk + | and ws_web_page_sk = web_page.wp_web_page_sk + | and time_dim.t_hour between 8 and 8+1 + | and household_demographics.hd_dep_count = 6 + | and web_page.wp_char_count between 5000 and 5200) at, + | ( select count(*) pmc + | from web_sales, household_demographics , time_dim, web_page + | where ws_sold_time_sk = time_dim.t_time_sk + | and ws_ship_hdemo_sk = household_demographics.hd_demo_sk + | and ws_web_page_sk = web_page.wp_web_page_sk + | and time_dim.t_hour between 19 and 19+1 + | and household_demographics.hd_dep_count = 6 + | and web_page.wp_char_count between 5000 and 5200) pt + | order by am_pm_ratio + | limit 100 + """.stripMargin), + Query( + "q91", + """ + | select + | cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, + | sum(cr_net_loss) Returns_Loss + | from + | call_center, catalog_returns, date_dim, customer, customer_address, + | customer_demographics, household_demographics + | where + | cr_call_center_sk = cc_call_center_sk + | and cr_returned_date_sk = d_date_sk + | and cr_returning_customer_sk = c_customer_sk + | and cd_demo_sk = c_current_cdemo_sk + | and hd_demo_sk = c_current_hdemo_sk + | and ca_address_sk = c_current_addr_sk + | and d_year = 1998 + | and d_moy = 11 + | and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') + | or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) + | and hd_buy_potential like 'Unknown%' + | and ca_gmt_offset = -7 + | group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status + | order by sum(cr_net_loss) desc + """.stripMargin), + // Modifications: "+ days" -> date_add + // Modifications: " -> ` + Query( + "q92", + """ + | select sum(ws_ext_discount_amt) as `Excess Discount Amount` + | from web_sales, item, date_dim + | where i_manufact_id = 350 + | and i_item_sk = ws_item_sk + | and d_date between '2000-01-27' and (cast('2000-01-27' as date) + interval 90 days) + | and d_date_sk = ws_sold_date_sk + | and ws_ext_discount_amt > + | ( + | SELECT 1.3 * avg(ws_ext_discount_amt) + | FROM web_sales, date_dim + | WHERE ws_item_sk = i_item_sk + | and d_date between '2000-01-27' and (cast('2000-01-27' as date) + interval 90 days) + | and d_date_sk = ws_sold_date_sk + | ) + | order by sum(ws_ext_discount_amt) + | limit 100 + """.stripMargin), + Query( + "q93", + """ + | select ss_customer_sk, sum(act_sales) sumsales + | from (select + | ss_item_sk, ss_ticket_number, ss_customer_sk, + | case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price + | else (ss_quantity*ss_sales_price) end act_sales + | from store_sales + | left outer join store_returns + | on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number), + | reason + | where sr_reason_sk = r_reason_sk and r_reason_desc = 'reason 28') t + | group by ss_customer_sk + | order by sumsales, ss_customer_sk + | limit 100 + """.stripMargin), + // Modifications: "+ days" -> date_add + // Modifications: " -> ` + Query( + "q94", + """ + | select + | count(distinct ws_order_number) as `order count` + | ,sum(ws_ext_ship_cost) as `total shipping cost` + | ,sum(ws_net_profit) as `total net profit` + | from + | web_sales ws1, date_dim, customer_address, web_site + | where + | d_date between '1999-02-01' and + | (cast('1999-02-01' as date) + interval 60 days) + | and ws1.ws_ship_date_sk = d_date_sk + | and ws1.ws_ship_addr_sk = ca_address_sk + | and ca_state = 'IL' + | and ws1.ws_web_site_sk = web_site_sk + | and web_company_name = 'pri' + | and exists (select * + | from web_sales ws2 + | where ws1.ws_order_number = ws2.ws_order_number + | and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) + | and not exists(select * + | from web_returns wr1 + | where ws1.ws_order_number = wr1.wr_order_number) + | order by count(distinct ws_order_number) + | limit 100 + """.stripMargin), + // Modifications: "+ days" -> date_add + Query( + "q95", + """ + | with ws_wh as + | (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 + | from web_sales ws1,web_sales ws2 + | where ws1.ws_order_number = ws2.ws_order_number + | and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) + | select + | count(distinct ws_order_number) as `order count` + | ,sum(ws_ext_ship_cost) as `total shipping cost` + | ,sum(ws_net_profit) as `total net profit` + | from + | web_sales ws1, date_dim, customer_address, web_site + | where + | d_date between '1999-02-01' and + | (cast('1999-02-01' as date) + interval 60 days) + | and ws1.ws_ship_date_sk = d_date_sk + | and ws1.ws_ship_addr_sk = ca_address_sk + | and ca_state = 'IL' + | and ws1.ws_web_site_sk = web_site_sk + | and web_company_name = 'pri' + | and ws1.ws_order_number in (select ws_order_number + | from ws_wh) + | and ws1.ws_order_number in (select wr_order_number + | from web_returns,ws_wh + | where wr_order_number = ws_wh.ws_order_number) + | order by count(distinct ws_order_number) + | limit 100 + """.stripMargin), + Query( + "q96", + """ + | select count(*) + | from store_sales, household_demographics, time_dim, store + | where ss_sold_time_sk = time_dim.t_time_sk + | and ss_hdemo_sk = household_demographics.hd_demo_sk + | and ss_store_sk = s_store_sk + | and time_dim.t_hour = 20 + | and time_dim.t_minute >= 30 + | and household_demographics.hd_dep_count = 7 + | and store.s_store_name = 'ese' + | order by count(*) + | limit 100 + """.stripMargin), + Query( + "q97", + """ + | with ssci as ( + | select ss_customer_sk customer_sk, ss_item_sk item_sk + | from store_sales,date_dim + | where ss_sold_date_sk = d_date_sk + | and d_month_seq between 1200 and 1200 + 11 + | group by ss_customer_sk, ss_item_sk), + | csci as( + | select cs_bill_customer_sk customer_sk, cs_item_sk item_sk + | from catalog_sales,date_dim + | where cs_sold_date_sk = d_date_sk + | and d_month_seq between 1200 and 1200 + 11 + | group by cs_bill_customer_sk, cs_item_sk) + | select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only + | ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only + | ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog + | from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk + | and ssci.item_sk = csci.item_sk) + | limit 100 + """.stripMargin), + // Modifications: "+ days" -> date_add + Query( + "q98", + """ + |select i_item_desc, i_category, i_class, i_current_price + | ,sum(ss_ext_sales_price) as itemrevenue + | ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over + | (partition by i_class) as revenueratio + |from + | store_sales, item, date_dim + |where + | ss_item_sk = i_item_sk + | and i_category in ('Sports', 'Books', 'Home') + | and ss_sold_date_sk = d_date_sk + | and d_date between cast('1999-02-22' as date) + | and (cast('1999-02-22' as date) + interval 30 days) + |group by + | i_item_id, i_item_desc, i_category, i_class, i_current_price + |order by + | i_category, i_class, i_item_id, i_item_desc, revenueratio + """.stripMargin), + // Modifications: " -> ` + Query( + "q99", + """ + | select + | substr(w_warehouse_name,1,20), sm_type, cc_name + | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` + | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and + | (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` + | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and + | (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` + | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and + | (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` + | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` + | from + | catalog_sales, warehouse, ship_mode, call_center, date_dim + | where + | d_month_seq between 1200 and 1200 + 11 + | and cs_ship_date_sk = d_date_sk + | and cs_warehouse_sk = w_warehouse_sk + | and cs_ship_mode_sk = sm_ship_mode_sk + | and cs_call_center_sk = cc_call_center_sk + | group by + | substr(w_warehouse_name,1,20), sm_type, cc_name + | order by substr(w_warehouse_name,1,20), sm_type, cc_name + | limit 100 + """.stripMargin), + Query( + "qSsMax", + """ + |select + | count(*) as total, + | count(ss_sold_date_sk) as not_null_total, + | count(distinct ss_sold_date_sk) as unique_days, + | max(ss_sold_date_sk) as max_ss_sold_date_sk, + | max(ss_sold_time_sk) as max_ss_sold_time_sk, + | max(ss_item_sk) as max_ss_item_sk, + | max(ss_customer_sk) as max_ss_customer_sk, + | max(ss_cdemo_sk) as max_ss_cdemo_sk, + | max(ss_hdemo_sk) as max_ss_hdemo_sk, + | max(ss_addr_sk) as max_ss_addr_sk, + | max(ss_store_sk) as max_ss_store_sk, + | max(ss_promo_sk) as max_ss_promo_sk + |from store_sales + """.stripMargin)) + + val recommendedIndexes = Array( + "JoinIndex00-index-33-inv_item_sk-3;inventory;inv_item_sk;inv_date_sk,inv_warehouse_sk,inv_quantity_on_hand", + "JoinIndex01-index-34-inv_date_sk-3;inventory;inv_date_sk;inv_warehouse_sk,inv_quantity_on_hand,inv_item_sk", + "JoinIndex02-index-42-inv_warehouse_sk-3;inventory;inv_warehouse_sk;inv_date_sk,inv_quantity_on_hand,inv_item_sk", + "JoinIndex03-index-0-d_date_sk-10;date_dim;d_date_sk;d_moy,d_year,d_date,d_month_seq,d_qoy,d_dom,d_week_seq,d_dow,d_day_name,d_quarter_name", + "FilterIndex04-index-11-d_year-5;date_dim;d_year;d_date_sk,d_date,d_week_seq,d_moy,d_day_name", + "FilterIndex06-index-14-d_date-2;date_dim;d_date;d_date_sk,d_week_seq", + "FilterIndex07-index-17-d_month_seq-7;date_dim;d_month_seq;d_date_sk,d_date,d_day_name,d_week_seq,d_moy,d_qoy,d_year", + "JoinIndex08-index-2-i_item_sk-17;item;i_item_sk;i_item_id,i_item_desc,i_manager_id,i_brand,i_brand_id,i_category,i_class,i_current_price,i_class_id,i_category_id,i_product_name,i_size,i_color,i_units,i_manufact_id,i_manufact,i_wholesale_cost", + "FilterIndex10-index-39-i_category-9;item;i_category;i_item_id,i_item_desc,i_class,i_item_sk,i_current_price,i_manufact_id,i_class_id,i_brand_id,i_category_id", + "JoinIndex12-index-1-ss_sold_date_sk-21;store_sales;ss_sold_date_sk;ss_ext_sales_price,ss_item_sk,ss_sales_price,ss_store_sk,ss_addr_sk,ss_hdemo_sk,ss_ticket_number,ss_customer_sk,ss_quantity,ss_list_price,ss_net_profit,ss_coupon_amt,ss_ext_list_price,ss_ext_discount_amt,ss_wholesale_cost,ss_cdemo_sk,ss_promo_sk,ss_ext_wholesale_cost,ss_ext_tax,ss_net_paid,ss_sold_time_sk", + "JoinIndex13-index-3-ss_item_sk-16;store_sales;ss_item_sk;ss_ext_sales_price,ss_sold_date_sk,ss_sales_price,ss_store_sk,ss_addr_sk,ss_quantity,ss_list_price,ss_net_paid,ss_ticket_number,ss_customer_sk,ss_net_profit,ss_promo_sk,ss_wholesale_cost,ss_coupon_amt,ss_cdemo_sk,ss_hdemo_sk", + "JoinIndex14-index-5-ss_store_sk-20;store_sales;ss_store_sk;ss_sales_price,ss_item_sk,ss_sold_date_sk,ss_net_profit,ss_hdemo_sk,ss_ticket_number,ss_customer_sk,ss_coupon_amt,ss_addr_sk,ss_net_paid,ss_sold_time_sk,ss_quantity,ss_list_price,ss_cdemo_sk,ss_ext_sales_price,ss_promo_sk,ss_ext_wholesale_cost,ss_wholesale_cost,ss_ext_list_price,ss_ext_tax", + "JoinIndex15-index-12-ss_customer_sk-20;store_sales;ss_customer_sk;ss_hdemo_sk,ss_ticket_number,ss_sold_date_sk,ss_store_sk,ss_quantity,ss_item_sk,ss_addr_sk,ss_net_profit,ss_coupon_amt,ss_net_paid,ss_promo_sk,ss_list_price,ss_cdemo_sk,ss_wholesale_cost,ss_ext_list_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_tax,ss_sales_price", + "JoinIndex17-index-27-ss_hdemo_sk-19;store_sales;ss_hdemo_sk;ss_addr_sk,ss_net_profit,ss_coupon_amt,ss_ticket_number,ss_sold_date_sk,ss_customer_sk,ss_store_sk,ss_sold_time_sk,ss_promo_sk,ss_item_sk,ss_list_price,ss_cdemo_sk,ss_wholesale_cost,ss_ext_list_price,ss_ext_tax,ss_ext_sales_price,ss_sales_price,ss_quantity,ss_ext_wholesale_cost", + "JoinIndex19-index-47-ss_cdemo_sk-16;store_sales;ss_cdemo_sk;ss_store_sk,ss_addr_sk,ss_promo_sk,ss_item_sk,ss_hdemo_sk,ss_list_price,ss_coupon_amt,ss_ticket_number,ss_sold_date_sk,ss_customer_sk,ss_wholesale_cost,ss_sales_price,ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit", + "JoinIndex20-index-43-t_time_sk-4;time_dim;t_time_sk;t_hour,t_minute,t_time,t_meal_time", + "JoinIndex21-index-20-sr_item_sk-7;store_returns;sr_item_sk;sr_ticket_number,sr_return_amt,sr_return_quantity,sr_returned_date_sk,sr_customer_sk,sr_reason_sk,sr_net_loss", + "JoinIndex23-index-37-sr_returned_date_sk-7;store_returns;sr_returned_date_sk;sr_item_sk,sr_return_quantity,sr_customer_sk,sr_ticket_number,sr_return_amt,sr_store_sk,sr_net_loss", + "JoinIndex24-index-4-s_store_sk-15;store;s_store_sk;s_state,s_store_name,s_store_id,s_market_id,s_zip,s_city,s_company_name,s_county,s_company_id,s_street_name,s_street_number,s_suite_number,s_street_type,s_gmt_offset,s_number_employees", + "JoinIndex25-index-18-cd_demo_sk-8;customer_demographics;cd_demo_sk;cd_education_status,cd_marital_status,cd_gender,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_college_count,cd_dep_employed_count", + "JoinIndex26-index-6-cs_sold_date_sk-26;catalog_sales;cs_sold_date_sk;cs_bill_addr_sk,cs_ext_sales_price,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_list_price,cs_order_number,cs_bill_hdemo_sk,cs_promo_sk,cs_ship_date_sk,cs_bill_cdemo_sk,cs_net_profit,cs_catalog_page_sk,cs_call_center_sk,cs_sales_price,cs_coupon_amt,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_list_price,cs_warehouse_sk,cs_ship_addr_sk,cs_wholesale_cost,cs_sold_time_sk,cs_net_paid,cs_ship_mode_sk,cs_net_paid_inc_tax", + "JoinIndex27-index-10-cs_item_sk-22;catalog_sales;cs_item_sk;cs_sold_date_sk,cs_bill_addr_sk,cs_ext_sales_price,cs_quantity,cs_bill_customer_sk,cs_list_price,cs_order_number,cs_bill_hdemo_sk,cs_promo_sk,cs_ship_date_sk,cs_bill_cdemo_sk,cs_ship_addr_sk,cs_net_profit,cs_net_paid,cs_ext_discount_amt,cs_catalog_page_sk,cs_coupon_amt,cs_sales_price,cs_ext_list_price,cs_warehouse_sk,cs_wholesale_cost,cs_call_center_sk", + "JoinIndex28-index-23-cs_bill_customer_sk-12;catalog_sales;cs_bill_customer_sk;cs_sold_date_sk,cs_quantity,cs_item_sk,cs_list_price,cs_sales_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_list_price,cs_ext_sales_price,cs_net_profit,cs_coupon_amt,cs_bill_cdemo_sk", + "JoinIndex31-index-7-ca_address_sk-11;customer_address;ca_address_sk;ca_state,ca_gmt_offset,ca_country,ca_city,ca_county,ca_zip,ca_street_name,ca_street_number,ca_street_type,ca_suite_number,ca_location_type", + "JoinIndex35-index-24-w_warehouse_sk-6;warehouse;w_warehouse_sk;w_warehouse_name,w_state,w_country,w_county,w_city,w_warehouse_sq_ft", + "JoinIndex36-index-46-web_site_sk-3;web_site;web_site_sk;web_company_name,web_site_id,web_name", + "JoinIndex37-index-9-ws_sold_date_sk-21;web_sales;ws_sold_date_sk;ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_quantity,ws_bill_customer_sk,ws_list_price,ws_net_paid,ws_order_number,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_time_sk,ws_ext_discount_amt,ws_net_profit,ws_web_page_sk,ws_sales_price,ws_ext_list_price,ws_ext_wholesale_cost,ws_promo_sk,ws_web_site_sk,ws_wholesale_cost,ws_ship_customer_sk", + "JoinIndex38-index-15-ws_item_sk-16;web_sales;ws_item_sk;ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,ws_quantity,ws_bill_customer_sk,ws_list_price,ws_ship_customer_sk,ws_order_number,ws_sales_price,ws_net_profit,ws_web_page_sk,ws_web_site_sk,ws_net_paid,ws_ext_discount_amt,ws_wholesale_cost,ws_promo_sk", + "JoinIndex39-index-30-ws_bill_customer_sk-10;web_sales;ws_bill_customer_sk;ws_item_sk,ws_quantity,ws_sold_date_sk,ws_list_price,ws_ext_list_price,ws_ext_discount_amt,ws_net_paid,ws_ext_sales_price,ws_ext_wholesale_cost,ws_sales_price", + "JoinIndex40-index-31-ws_order_number-16;web_sales;ws_order_number;ws_warehouse_sk,ws_ship_addr_sk,ws_net_profit,ws_ship_date_sk,ws_ext_ship_cost,ws_web_site_sk,ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,ws_web_page_sk,ws_sales_price,ws_ext_sales_price,ws_promo_sk,ws_wholesale_cost,ws_bill_customer_sk", + "JoinIndex41-index-36-wr_item_sk-11;web_returns;wr_item_sk;wr_return_quantity,wr_order_number,wr_return_amt,wr_returned_date_sk,wr_net_loss,wr_refunded_cdemo_sk,wr_reason_sk,wr_refunded_cash,wr_fee,wr_refunded_addr_sk,wr_returning_cdemo_sk", + "JoinIndex42-index-41-wr_order_number-11;web_returns;wr_order_number;wr_return_quantity,wr_item_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,wr_refunded_cdemo_sk,wr_reason_sk,wr_refunded_cash,wr_fee,wr_refunded_addr_sk,wr_returning_cdemo_sk", + "JoinIndex43-index-8-c_customer_sk-17;customer;c_customer_sk;c_current_addr_sk,c_first_name,c_last_name,c_current_cdemo_sk,c_preferred_cust_flag,c_salutation,c_birth_country,c_login,c_customer_id,c_email_address,c_birth_year,c_birth_month,c_current_hdemo_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_last_review_date,c_birth_day", + "JoinIndex44-index-16-c_current_addr_sk-17;customer;c_current_addr_sk;c_customer_sk,c_current_cdemo_sk,c_first_name,c_last_name,c_customer_id,c_last_review_date,c_email_address,c_salutation,c_login,c_birth_year,c_birth_month,c_birth_country,c_birth_day,c_preferred_cust_flag,c_current_hdemo_sk,c_first_sales_date_sk,c_first_shipto_date_sk", + "JoinIndex46-index-19-hd_demo_sk-4;household_demographics;hd_demo_sk;hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_income_band_sk", + "JoinIndex47-index-26-cr_item_sk-8;catalog_returns;cr_item_sk;cr_order_number,cr_return_amount,cr_return_quantity,cr_returned_date_sk,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_net_loss", + "JoinIndex49-index-40-p_promo_sk-4;promotion;p_promo_sk;p_channel_event,p_channel_email,p_channel_dmail,p_channel_tv") + + // scalastyle:on +} diff --git a/src/main/scala/com/microsoft/hyperspace/perf/TPCHBenchmark.scala b/src/main/scala/com/microsoft/hyperspace/perf/TPCHBenchmark.scala new file mode 100644 index 000000000..db61f0a8f --- /dev/null +++ b/src/main/scala/com/microsoft/hyperspace/perf/TPCHBenchmark.scala @@ -0,0 +1,833 @@ +/* + * Copyright (2020) The Hyperspace Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.microsoft.hyperspace.perf + +object TPCHBenchmark { + // scalastyle:off + + import org.apache.spark.sql.types._ + + val partSchema = StructType( + Seq( + StructField("p_partkey", LongType), + StructField("p_name", StringType), + StructField("p_mfgr", StringType), + StructField("p_brand", StringType), + StructField("p_type", StringType), + StructField("p_size", IntegerType), + StructField("p_container", StringType), + StructField("p_retailprice", DoubleType), + StructField("p_comment", StringType))) + + val partsuppSchema = StructType( + Seq( + StructField("ps_partkey", LongType), + StructField("ps_suppkey", LongType), + StructField("ps_availqty", IntegerType), + StructField("ps_supplycost", DoubleType), + StructField("ps_comment", StringType))) + + val supplierSchema = StructType( + Seq( + StructField("s_suppkey", LongType), + StructField("s_name", StringType), + StructField("s_address", StringType), + StructField("s_nationkey", IntegerType), + StructField("s_phone", StringType), + StructField("s_acctbal", DoubleType), + StructField("s_comment", StringType))) + + val customerSchema = StructType( + Seq( + StructField("c_custkey", LongType), + StructField("c_name", StringType), + StructField("c_address", StringType), + StructField("c_nationkey", IntegerType), + StructField("c_phone", StringType), + StructField("c_acctbal", DoubleType), + StructField("c_mktsegment", StringType), + StructField("c_comment", StringType))) + + val ordersSchema = StructType( + Seq( + StructField("o_orderkey", LongType), + StructField("o_custkey", LongType), + StructField("o_orderstatus", StringType), + StructField("o_totalprice", DoubleType), + StructField("o_orderdate", StringType), + StructField("o_orderpriority", StringType), + StructField("o_clerk", StringType), + StructField("o_shippriority", IntegerType), + StructField("o_comment", StringType))) + + val lineitemSchema = StructType( + Seq( + StructField("l_orderkey", LongType), + StructField("l_partkey", LongType), + StructField("l_suppkey", LongType), + StructField("l_linenumber", IntegerType), + StructField("l_quantity", DoubleType), + StructField("l_extendedprice", DoubleType), + StructField("l_discount", DoubleType), + StructField("l_tax", DoubleType), + StructField("l_returnflag", StringType), + StructField("l_linestatus", StringType), + StructField("l_shipdate", StringType), + StructField("l_commitdate", StringType), + StructField("l_receiptdate", StringType), + StructField("l_shipinstruct", StringType), + StructField("l_shipmode", StringType), + StructField("l_comment", StringType))) + + val regionSchema = StructType( + Seq( + StructField("r_regionkey", IntegerType), + StructField("r_name", StringType), + StructField("r_comment", StringType))) + + val nationSchema = StructType( + Seq( + StructField("n_nationkey", IntegerType), + StructField("n_name", StringType), + StructField("n_regionkey", IntegerType), + StructField("n_comment", StringType))) + + val tables = Array( + ("customer", customerSchema), + ("lineitem", lineitemSchema), + ("nation", nationSchema), + ("orders", ordersSchema), + ("part", partSchema), + ("partsupp", partsuppSchema), + ("region", regionSchema), + ("supplier", supplierSchema)) + + val queries = Array( + Query( + "q1", + """select + | l_returnflag, + | l_linestatus, + | sum(l_quantity) as sum_qty, + | sum(l_extendedprice) as sum_base_price, + | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, + | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, + | avg(l_quantity) as avg_qty, + | avg(l_extendedprice) as avg_price, + | avg(l_discount) as avg_disc, + | count(*) as count_order + |from + | lineitem + |where + | l_shipdate <= date '1998-12-01' - interval '90' day + |group by + | l_returnflag, + | l_linestatus + |order by + | l_returnflag, + | l_linestatus + """.stripMargin), + Query( + "q2", + """select + | s_acctbal, + | s_name, + | n_name, + | p_partkey, + | p_mfgr, + | s_address, + | s_phone, + | s_comment + |from + | part, + | supplier, + | partsupp, + | nation, + | region + |where + | p_partkey = ps_partkey + | and s_suppkey = ps_suppkey + | and p_size = 15 + | and p_type like '%BRASS' + | and s_nationkey = n_nationkey + | and n_regionkey = r_regionkey + | and r_name = 'EUROPE' + | and ps_supplycost = ( + | select + | min(ps_supplycost) + | from + | partsupp, + | supplier, + | nation, + | region + | where + | p_partkey = ps_partkey + | and s_suppkey = ps_suppkey + | and s_nationkey = n_nationkey + | and n_regionkey = r_regionkey + | and r_name = 'EUROPE' + | ) + |order by + | s_acctbal desc, + | n_name, + | s_name, + | p_partkey + """.stripMargin), + Query( + "q3", + """select + | l_orderkey, + | sum(l_extendedprice * (1 - l_discount)) as revenue, + | o_orderdate, + | o_shippriority + |from + | customer, + | orders, + | lineitem + |where + | c_mktsegment = 'BUILDING' + | and c_custkey = o_custkey + | and l_orderkey = o_orderkey + | and o_orderdate < date '1995-03-15' + | and l_shipdate > date '1995-03-15' + |group by + | l_orderkey, + | o_orderdate, + | o_shippriority + |order by + | revenue desc, + | o_orderdate + """.stripMargin), + Query( + "q4", + """select + | o_orderpriority, + | count(*) as order_count + |from + | orders + |where + | o_orderdate >= date '1993-07-01' + | and o_orderdate < date '1993-07-01' + interval '3' month + | and exists ( + | select + | * + | from + | lineitem + | where + | l_orderkey = o_orderkey + | and l_commitdate < l_receiptdate + | ) + |group by + | o_orderpriority + |order by + | o_orderpriority + """.stripMargin), + Query( + "q5", + """select + | n_name, + | sum(l_extendedprice * (1 - l_discount)) as revenue + |from + | customer, + | orders, + | lineitem, + | supplier, + | nation, + | region + |where + | c_custkey = o_custkey + | and l_orderkey = o_orderkey + | and l_suppkey = s_suppkey + | and c_nationkey = s_nationkey + | and s_nationkey = n_nationkey + | and n_regionkey = r_regionkey + | and r_name = 'ASIA' + | and o_orderdate >= date '1994-01-01' + | and o_orderdate < date '1994-01-01' + interval '1' year + |group by + | n_name + |order by + | revenue desc + """.stripMargin), + Query( + "q6", + """select + | sum(l_extendedprice * l_discount) as revenue + |from + | lineitem + |where + | l_shipdate >= date '1994-01-01' + | and l_shipdate < date '1994-01-01' + interval '1' year + | and l_discount between .06 - 0.01 and .06 + 0.01 + | and l_quantity < 24 + """.stripMargin), + Query( + "q7", + """select + | supp_nation, + | cust_nation, + | l_year, + | sum(volume) as revenue + |from + | ( + | select + | n1.n_name as supp_nation, + | n2.n_name as cust_nation, + | year(l_shipdate) as l_year, + | l_extendedprice * (1 - l_discount) as volume + | from + | supplier, + | lineitem, + | orders, + | customer, + | nation n1, + | nation n2 + | where + | s_suppkey = l_suppkey + | and o_orderkey = l_orderkey + | and c_custkey = o_custkey + | and s_nationkey = n1.n_nationkey + | and c_nationkey = n2.n_nationkey + | and ( + | (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY') + | or (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE') + | ) + | and l_shipdate between date '1995-01-01' and date '1996-12-31' + | ) as shipping + |group by + | supp_nation, + | cust_nation, + | l_year + |order by + | supp_nation, + | cust_nation, + | l_year""".stripMargin), + Query( + "q8", + """select + | o_year, + | sum(case + | when nation = 'BRAZIL' then volume + | else 0 + | end) / sum(volume) as mkt_share + |from + | ( + | select + | year(o_orderdate) as o_year, + | l_extendedprice * (1 - l_discount) as volume, + | n2.n_name as nation + | from + | part, + | supplier, + | lineitem, + | orders, + | customer, + | nation n1, + | nation n2, + | region + | where + | p_partkey = l_partkey + | and s_suppkey = l_suppkey + | and l_orderkey = o_orderkey + | and o_custkey = c_custkey + | and c_nationkey = n1.n_nationkey + | and n1.n_regionkey = r_regionkey + | and r_name = 'AMERICA' + | and s_nationkey = n2.n_nationkey + | and o_orderdate between date '1995-01-01' and date '1996-12-31' + | and p_type = 'ECONOMY ANODIZED STEEL' + | ) as all_nations + |group by + | o_year + |order by + | o_year""".stripMargin), + Query( + "q9", + """select + | nation, + | o_year, + | sum(amount) as sum_profit + |from + | ( + | select + | n_name as nation, + | year(o_orderdate) as o_year, + | l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount + | from + | part, + | supplier, + | lineitem, + | partsupp, + | orders, + | nation + | where + | s_suppkey = l_suppkey + | and ps_suppkey = l_suppkey + | and ps_partkey = l_partkey + | and p_partkey = l_partkey + | and o_orderkey = l_orderkey + | and s_nationkey = n_nationkey + | and p_name like '%green%' + | ) as profit + |group by + | nation, + | o_year + |order by + | nation, + | o_year desc""".stripMargin), + Query( + "q10", + """select + | c_custkey, + | c_name, + | sum(l_extendedprice * (1 - l_discount)) as revenue, + | c_acctbal, + | n_name, + | c_address, + | c_phone, + | c_comment + |from + | customer, + | orders, + | lineitem, + | nation + |where + | c_custkey = o_custkey + | and l_orderkey = o_orderkey + | and o_orderdate >= date '1993-10-01' + | and o_orderdate < date '1993-10-01' + interval '3' month + | and l_returnflag = 'R' + | and c_nationkey = n_nationkey + |group by + | c_custkey, + | c_name, + | c_acctbal, + | c_phone, + | n_name, + | c_address, + | c_comment + |order by + | revenue desc + """.stripMargin), + Query( + "q11", + """select + | ps_partkey, + | sum(ps_supplycost * ps_availqty) as value + |from + | partsupp, + | supplier, + | nation + |where + | ps_suppkey = s_suppkey + | and s_nationkey = n_nationkey + | and n_name = 'GERMANY' + |group by + | ps_partkey having + | sum(ps_supplycost * ps_availqty) > ( + | select + | sum(ps_supplycost * ps_availqty) * 0.0000001000000 + | from + | partsupp, + | supplier, + | nation + | where + | ps_suppkey = s_suppkey + | and s_nationkey = n_nationkey + | and n_name = 'GERMANY' + | ) + |order by + | value desc + """.stripMargin), + Query( + "q12", + """select + | l_shipmode, + | sum(case + | when o_orderpriority = '1-URGENT' + | or o_orderpriority = '2-HIGH' + | then 1 + | else 0 + | end) as high_line_count, + | sum(case + | when o_orderpriority <> '1-URGENT' + | and o_orderpriority <> '2-HIGH' + | then 1 + | else 0 + | end) as low_line_count + |from + | orders, + | lineitem + |where + | o_orderkey = l_orderkey + | and l_shipmode in ('MAIL', 'SHIP') + | and l_commitdate < l_receiptdate + | and l_shipdate < l_commitdate + | and l_receiptdate >= date '1994-01-01' + | and l_receiptdate < date '1994-01-01' + interval '1' year + |group by + | l_shipmode + |order by + | l_shipmode + """.stripMargin), + Query( + "q13", + """select + | c_count, + | count(*) as custdist + |from + | ( + | select + | c_custkey, + | count(o_orderkey) as c_count + | from + | customer left outer join orders on + | c_custkey = o_custkey + | and o_comment not like '%special%requests%' + | group by + | c_custkey + | ) as c_orders + |group by + | c_count + |order by + | custdist desc, + | c_count desc""".stripMargin), + Query( + "q14", + """ + |select + | 100.00 * sum(case + | when p_type like 'PROMO%' + | then l_extendedprice * (1 - l_discount) + | else 0 + | end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue + |from + | lineitem, + | part + |where + | l_partkey = p_partkey + | and l_shipdate >= date '1995-09-01' + | and l_shipdate < date '1995-09-01' + interval '1' month + """.stripMargin), + Query( + "q15", + """with revenue0 as + | (select + | l_suppkey as supplier_no, + | cast(sum(l_extendedprice * (1 - l_discount)) as int) as total_revenue + | from + | lineitem + | where + | l_shipdate >= date '1996-01-01' + | and l_shipdate < date '1996-01-01' + interval '3' month + | group by + | l_suppkey) + | + |select + | s_suppkey, + | s_name, + | s_address, + | s_phone, + | total_revenue + |from + | supplier, + | revenue0 + |where + | s_suppkey = supplier_no + | and total_revenue = ( + | select + | cast(max(total_revenue) as int) + | from + | revenue0 + | ) + |order by + | s_suppkey + |""".stripMargin), + Query( + "q16", + """ + |select + | p_brand, + | p_type, + | p_size, + | count(distinct ps_suppkey) as supplier_cnt + |from + | partsupp, + | part + |where + | p_partkey = ps_partkey + | and p_brand <> 'Brand#45' + | and p_type not like 'MEDIUM POLISHED%' + | and p_size in (49, 14, 23, 45, 19, 3, 36, 9) + | and ps_suppkey not in ( + | select + | s_suppkey + | from + | supplier + | where + | s_comment like '%Customer%Complaints%' + | ) + |group by + | p_brand, + | p_type, + | p_size + |order by + | supplier_cnt desc, + | p_brand, + | p_type, + | p_size + """.stripMargin), + Query( + "q17", + """ + |select + | sum(l_extendedprice) / 7.0 as avg_yearly + |from + | lineitem, + | part + |where + | p_partkey = l_partkey + | and p_brand = 'Brand#23' + | and p_container = 'MED BOX' + | and l_quantity < ( + | select + | 0.2 * avg(l_quantity) + | from + | lineitem + | where + | l_partkey = p_partkey + | ) + """.stripMargin), + Query( + "q18", + """ + |select + | c_name, + | c_custkey, + | o_orderkey, + | o_orderdate, + | o_totalprice, + | sum(l_quantity) + |from + | customer, + | orders, + | lineitem + |where + | o_orderkey in ( + | select + | l_orderkey + | from + | lineitem + | group by + | l_orderkey having + | sum(l_quantity) > 300 + | ) + | and c_custkey = o_custkey + | and o_orderkey = l_orderkey + |group by + | c_name, + | c_custkey, + | o_orderkey, + | o_orderdate, + | o_totalprice + |order by + | o_totalprice desc, + | o_orderdate + """.stripMargin), + Query( + "q19", + """ + |select + | sum(l_extendedprice* (1 - l_discount)) as revenue + |from + | lineitem, + | part + |where + | ( + | p_partkey = l_partkey + | and p_brand = 'Brand#12' + | and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') + | and l_quantity >= 1 and l_quantity <= 1 + 10 + | and p_size between 1 and 5 + | and l_shipmode in ('AIR', 'AIR REG') + | and l_shipinstruct = 'DELIVER IN PERSON' + | ) + | or + | ( + | p_partkey = l_partkey + | and p_brand = 'Brand#23' + | and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') + | and l_quantity >= 10 and l_quantity <= 10 + 10 + | and p_size between 1 and 10 + | and l_shipmode in ('AIR', 'AIR REG') + | and l_shipinstruct = 'DELIVER IN PERSON' + | ) + | or + | ( + | p_partkey = l_partkey + | and p_brand = 'Brand#34' + | and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') + | and l_quantity >= 20 and l_quantity <= 20 + 10 + | and p_size between 1 and 15 + | and l_shipmode in ('AIR', 'AIR REG') + | and l_shipinstruct = 'DELIVER IN PERSON' + | ) + """.stripMargin), + Query( + "q20", + """ + |select + | s_name, + | s_address + |from + | supplier, + | nation + |where + | s_suppkey in ( + | select + | ps_suppkey + | from + | partsupp + | where + | ps_partkey in ( + | select + | p_partkey + | from + | part + | where + | p_name like 'forest%' + | ) + | and ps_availqty > ( + | select + | 0.5 * sum(l_quantity) + | from + | lineitem + | where + | l_partkey = ps_partkey + | and l_suppkey = ps_suppkey + | and l_shipdate >= date '1994-01-01' + | and l_shipdate < date '1994-01-01' + interval '1' year + | ) + | ) + | and s_nationkey = n_nationkey + | and n_name = 'CANADA' + |order by + | s_name + """.stripMargin), + Query( + "q21", + """ + |select + | s_name, + | count(*) as numwait + |from + | supplier, + | lineitem l1, + | orders, + | nation + |where + | s_suppkey = l1.l_suppkey + | and o_orderkey = l1.l_orderkey + | and o_orderstatus = 'F' + | and l1.l_receiptdate > l1.l_commitdate + | and exists ( + | select + | * + | from + | lineitem l2 + | where + | l2.l_orderkey = l1.l_orderkey + | and l2.l_suppkey <> l1.l_suppkey + | ) + | and not exists ( + | select + | * + | from + | lineitem l3 + | where + | l3.l_orderkey = l1.l_orderkey + | and l3.l_suppkey <> l1.l_suppkey + | and l3.l_receiptdate > l3.l_commitdate + | ) + | and s_nationkey = n_nationkey + | and n_name = 'SAUDI ARABIA' + |group by + | s_name + |order by + | numwait desc, + | s_name + """.stripMargin), + Query( + "q22", + """select + | cntrycode, + | count(*) as numcust, + | sum(c_acctbal) as totacctbal + |from + | ( + | select + | substring(c_phone, 1, 2) as cntrycode, + | c_acctbal + | from + | customer + | where + | substring(c_phone, 1, 2) in + | ('13', '31', '23', '29', '30', '18', '17') + | and c_acctbal > ( + | select + | avg(c_acctbal) + | from + | customer + | where + | c_acctbal > 0.00 + | and substring(c_phone, 1, 2) in + | ('13', '31', '23', '29', '30', '18', '17') + | ) + | and not exists ( + | select + | * + | from + | orders + | where + | o_custkey = c_custkey + | ) + | ) as custsale + |group by + | cntrycode + |order by + | cntrycode""".stripMargin)) + + val recommendedIndexes = Array( + "IX1_supplier;supplier;s_suppkey;s_nationkey,s_name,s_address,s_comment,s_phone,s_acctbal", + "IX2_supplier;supplier;s_nationkey;s_suppkey,s_name,s_acctbal,s_address,s_comment,s_phone", + "IX3_region;region;r_regionkey;r_name", + "IX5_part;part;p_partkey;p_type,p_name,p_brand,p_size,p_container,p_mfgr", + "IX11_lineitem;lineitem;l_orderkey;l_quantity,l_suppkey,l_extendedprice,l_partkey,l_discount,l_receiptdate,l_commitdate,l_shipdate,l_shipmode,l_returnflag", + "IX12_lineitem;lineitem;l_partkey;l_suppkey,l_quantity,l_shipdate,l_extendedprice,l_discount,l_orderkey,l_shipinstruct,l_shipmode", + "IX13_lineitem;lineitem;l_suppkey;l_extendedprice,l_shipdate,l_discount,l_orderkey,l_receiptdate,l_commitdate,l_quantity,l_partkey", + "IX14_lineitem;lineitem;l_shipdate;l_linestatus,l_quantity,l_extendedprice,l_tax,l_returnflag,l_discount,l_orderkey,l_suppkey,l_partkey", + "IX15_lineitem;lineitem;l_commitdate,l_receiptdate;l_suppkey,l_orderkey", + "IX20_partsupp;partsupp;ps_suppkey;ps_supplycost,ps_partkey,ps_availqty", + "IX21_partsupp;partsupp;ps_partkey;ps_supplycost,ps_suppkey,ps_availqty", + "IX23_orders;orders;o_orderkey;o_custkey,o_orderdate,o_totalprice,o_shippriority,o_orderstatus,o_orderpriority", + "IX24_orders;orders;o_custkey;o_orderkey,o_orderdate,o_comment,o_totalprice,o_shippriority", + "IX27_nation;nation;n_nationkey;n_name,n_regionkey", + "IX30_customer;customer;c_custkey;c_nationkey,c_name,c_phone,c_address,c_comment,c_acctbal,c_mktsegment", + "IX32_customer;customer;c_acctbal;c_phone,c_custkey") + + // scalastyle:on +} From 4fdb8cca6ac670cfbe38528b7eb07d1ec48c43e6 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Wed, 5 Aug 2020 16:10:20 -0700 Subject: [PATCH 03/21] misc cleanup in benchmark runner code --- .../scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala index 6925252f1..ffae937bb 100644 --- a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala +++ b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala @@ -48,7 +48,7 @@ object BenchmarkRunner { val spark = SparkSession .builder() .getOrCreate() - + // Change log level to clean up output spark.sparkContext.setLogLevel("ERROR") From 211a3286c1e237e1774c83ea309633d38b97c629 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Wed, 5 Aug 2020 18:31:52 -0700 Subject: [PATCH 04/21] remove truncate from index sample record show --- .../scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala index ffae937bb..3fb425291 100644 --- a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala +++ b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala @@ -89,7 +89,7 @@ object BenchmarkRunner { indexesSizeStats :+= (x, size) println(s" Sample record from index $x (total index size is $size):") - spark.read.parquet(indexDirPath).show(2) + spark.read.parquet(indexDirPath).show(2, false) } println("Index Size Summary:\n") From f8ed0d0f313d3b1c4eccb4f02ca992b6a6acb478 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Wed, 5 Aug 2020 20:07:52 -0700 Subject: [PATCH 05/21] add misc to benchmark runner --- .../hyperspace/perf/BenchmarkRunner.scala | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala index 3fb425291..634282900 100644 --- a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala +++ b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala @@ -48,6 +48,10 @@ object BenchmarkRunner { val spark = SparkSession .builder() .getOrCreate() + // ------------------ +// val sparkConf = new SparkConf().setMaster("local[*]") +// val spark = SparkSession.builder.config(sparkConf).getOrCreate() + // ------------------ // Change log level to clean up output spark.sparkContext.setLogLevel("ERROR") @@ -79,7 +83,8 @@ object BenchmarkRunner { // Get Index Sizes println("Collecting index sizes and printing sample records from each.") val indexNames = - if (benchmarkName.equals("tpch")) TPCHBenchmark.recommendedIndexes.map(x => parseIndexConf(x).config.indexName) + if (benchmarkName.equals("tpch")) + TPCHBenchmark.recommendedIndexes.map(x => parseIndexConf(x).config.indexName) else TPCDSBenchmark.recommendedIndexes.map(x => parseIndexConf(x).config.indexName) var indexesSizeStats = Seq[Tuple2[String, Long]]() @@ -209,6 +214,11 @@ object BenchmarkRunner { require(!spark.conf.get(IndexConstants.INDEX_SYSTEM_PATH).isEmpty) spark.conf.set(IndexConstants.INDEX_NUM_BUCKETS, buckets) + // ---------- pouriap delete me --------------- + println(s"Printing indexes ") + hyperspace.indexes.show(100) + // ------------------------------------ + val indexes = if (benchmark.equals("tpch")) TPCHBenchmark.recommendedIndexes else TPCDSBenchmark.recommendedIndexes @@ -272,6 +282,11 @@ object BenchmarkRunner { println( s"\tCreating index ${indexInfo.config.indexName} on data files under $baseTableData ...") + // ----- pouriap delete me ----- + println(s"Sample records from base table ") + df.show(2, false) + //------------------------------ + val startTime = System.currentTimeMillis() hyperspace.createIndex(df, indexInfo.config) val endTime = System.currentTimeMillis() From edbc612fd04f98ede9287cc18c37f5b0bc2fb9a9 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Thu, 6 Aug 2020 09:34:00 -0700 Subject: [PATCH 06/21] add more details to index stats report --- .../hyperspace/perf/BenchmarkRunner.scala | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala index 634282900..61008be84 100644 --- a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala +++ b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala @@ -81,7 +81,7 @@ object BenchmarkRunner { totalCreateIXStats._1) // Get Index Sizes - println("Collecting index sizes and printing sample records from each.") + println("Collecting index sizes.") val indexNames = if (benchmarkName.equals("tpch")) TPCHBenchmark.recommendedIndexes.map(x => parseIndexConf(x).config.indexName) @@ -92,13 +92,14 @@ object BenchmarkRunner { val indexDirPath = s"$systemDir/$x/v__=0" val size = FileUtils.getDirectorySize(new Path(indexDirPath)) indexesSizeStats :+= (x, size) - - println(s" Sample record from index $x (total index size is $size):") - spark.read.parquet(indexDirPath).show(2, false) +// println(s" Sample record from index $x (total index size is $size):") +// spark.read.parquet(indexDirPath).show(2, false) } println("Index Size Summary:\n") - indexesSizeStats.sortBy(_._1).foreach(xs => println(s"${xs._1}, ${xs._2}")) + indexesSizeStats.sortBy(_._1).foreach{xs => + val readableSize = getReadableSize(xs._2) + println(s"${xs._1}, ${xs._2}, ${readableSize._1}${readableSize._2}")} } if (runWorkload) { @@ -212,12 +213,11 @@ object BenchmarkRunner { format: String): Tuple2[Long, Int] = { val hyperspace = Hyperspace() require(!spark.conf.get(IndexConstants.INDEX_SYSTEM_PATH).isEmpty) + val systemPath = spark.conf.get(IndexConstants.INDEX_SYSTEM_PATH) spark.conf.set(IndexConstants.INDEX_NUM_BUCKETS, buckets) - // ---------- pouriap delete me --------------- - println(s"Printing indexes ") - hyperspace.indexes.show(100) - // ------------------------------------ + println(s"Printing existing indexes (if any) before index creation (under systemPath: $systemPath ).") + hyperspace.indexes.show(100, false) val indexes = if (benchmark.equals("tpch")) TPCHBenchmark.recommendedIndexes @@ -242,9 +242,13 @@ object BenchmarkRunner { indexInfo.foreach { x => val baseTableData = s"$dataDir/${x.tableName}" totalTime += createIndex(spark, hyperspace, baseTableData, x, format) + + println(s"Print sample records from index ${x.config.indexName}") + val indexFilesPath = s"$systemPath/${x.config.indexName}/v__=0" + spark.read.parquet(indexFilesPath).show(2, false) } - hyperspace.indexes.show(100) + hyperspace.indexes.show(100, false) (totalTime, indexCount) } @@ -282,11 +286,6 @@ object BenchmarkRunner { println( s"\tCreating index ${indexInfo.config.indexName} on data files under $baseTableData ...") - // ----- pouriap delete me ----- - println(s"Sample records from base table ") - df.show(2, false) - //------------------------------ - val startTime = System.currentTimeMillis() hyperspace.createIndex(df, indexInfo.config) val endTime = System.currentTimeMillis() @@ -436,6 +435,18 @@ object BenchmarkRunner { .mkString } + private def getReadableSize(size: Long): Tuple2[Long, String] = { + if (size < 1024) { + return (size, "Bytes") + } else if (size < 1024 * 1024) { + return (size / 1024, "KB") + } else if (size < 1024 * 1024 * 1024) { + return (size / (1024 * 1024), "MB") + } + + (size / (1024 * 1024 * 1024), "GB") + } + // scalastyle:on } From 74f1ed7cf9f9a2cfdc5a7fb3cf71eb3ab2f66395 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Thu, 6 Aug 2020 09:52:59 -0700 Subject: [PATCH 07/21] Revert "Merge branch 'pouriap/benchmarkCode' into pouriap/lineage" This reverts commit f0e5383ab9ebcdbf26c381d92ce93a3f4103876a, reversing changes made to 495decfd564518499d0562c76e8da96ded95cbeb. --- .../hyperspace/perf/BenchmarkConfig.scala | 95 - .../hyperspace/perf/BenchmarkConstants.scala | 33 - .../hyperspace/perf/BenchmarkRunner.scala | 455 -- .../hyperspace/perf/TPCDSBenchmark.scala | 4657 ----------------- .../hyperspace/perf/TPCHBenchmark.scala | 833 --- 5 files changed, 6073 deletions(-) delete mode 100644 src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConfig.scala delete mode 100644 src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConstants.scala delete mode 100644 src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala delete mode 100644 src/main/scala/com/microsoft/hyperspace/perf/TPCDSBenchmark.scala delete mode 100644 src/main/scala/com/microsoft/hyperspace/perf/TPCHBenchmark.scala diff --git a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConfig.scala b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConfig.scala deleted file mode 100644 index 296d90a65..000000000 --- a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConfig.scala +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (2020) The Hyperspace Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.microsoft.hyperspace.perf - -import java.util.Locale - -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.DefaultScalaModule -import scala.io.Source - -class BenchmarkConfig(configFile: String) { - - private val configsMap = readBenchmarkConfigs(configFile) - - def benchmarkName: String = { - val benchmark = configsMap(BenchmarkConstants.BENCHMARK_NAME) - .asInstanceOf[String] - .trim() - .toLowerCase() - - if (benchmark.equals("tpch") || benchmark.equals("tpcds")) { - benchmark - } else { - throw new IllegalArgumentException(s"Unknown benchmark $benchmark.") - } - } - - def iterations: Int = configsMap.getOrElse(BenchmarkConstants.ITERATIONS, 1).asInstanceOf[Int] - - def dataPath: String = - configsMap(BenchmarkConstants.DATA_DIRECTORY).asInstanceOf[String] - - def systemPath: String = - configsMap(BenchmarkConstants.SYSTEM_DIRECTORY).asInstanceOf[String] - - def enableHyperspace: Boolean = - configsMap.getOrElse(BenchmarkConstants.ENABLE_HYPERSPACE, false).asInstanceOf[Boolean] - - def createIndex: Boolean = - configsMap.getOrElse(BenchmarkConstants.CREATE_INDEX, false).asInstanceOf[Boolean] - - def runWorkload: Boolean = - configsMap.getOrElse(BenchmarkConstants.RUN_WORKLOAD, false).asInstanceOf[Boolean] - - def workload: Seq[String] = configsMap(BenchmarkConstants.WORKLOAD).asInstanceOf[Seq[String]] - - def format: String = { - val format = - configsMap - .getOrElse(BenchmarkConstants.FORMAT, "") - .asInstanceOf[String] - .trim - .toLowerCase(Locale.ROOT) - if (format.equals("")) { - "parquet" - } else if (format.equals("parquet") || format.equals("csv")) { - format - } else { - throw new IllegalArgumentException(s"Unknown format $format.") - } - } - - def buckets: String = - configsMap - .getOrElse(BenchmarkConstants.INDEX_BUCKETS, BenchmarkConstants.DEFAULT_BUCKETS) - .asInstanceOf[String] - - private def readBenchmarkConfigs(filePath: String) = { - val src = Source.fromFile(filePath) - val configs = src.mkString - val mapper = new ObjectMapper - mapper.registerModule(DefaultScalaModule) - val configsMap = mapper.readValue(configs, classOf[Map[String, Any]]) - src.close() - configsMap - } -} - -object BenchmarkConfig { - def apply(configFile: String): BenchmarkConfig = new BenchmarkConfig(configFile) -} diff --git a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConstants.scala b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConstants.scala deleted file mode 100644 index 0b8ac4455..000000000 --- a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkConstants.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (2020) The Hyperspace Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.microsoft.hyperspace.perf - -object BenchmarkConstants { - val BENCHMARK_NAME = "benchmark" - val ITERATIONS = "iterations" - val DATA_DIRECTORY = "data" - val SYSTEM_DIRECTORY = "system" - val ENABLE_HYPERSPACE = "enableHyperspace" - val CREATE_INDEX = "createIndex" - val RUN_WORKLOAD = "runWorkload" - val WORKLOAD = "workload" - val FORMAT = "format" - val INDEX_BUCKETS = "buckets" - - val CHUNKS_UNDEFINED = -1 - val DEFAULT_BUCKETS = "200" -} diff --git a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala b/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala deleted file mode 100644 index 61008be84..000000000 --- a/src/main/scala/com/microsoft/hyperspace/perf/BenchmarkRunner.scala +++ /dev/null @@ -1,455 +0,0 @@ -/* - * Copyright (2020) The Hyperspace Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.microsoft.hyperspace.perf - -import org.apache.hadoop.fs.Path -import org.apache.spark.SparkConf -import org.apache.spark.sql.{DataFrame, SparkSession} -import org.apache.spark.sql.functions.{current_timestamp, date_format} -import org.apache.spark.sql.types.StructType - -import com.microsoft.hyperspace._ -import com.microsoft.hyperspace.index.{IndexConfig, IndexConstants} -import com.microsoft.hyperspace.util.FileUtils - -object BenchmarkRunner { - // scalastyle:off - - def main(args: Array[String]): Unit = { - if (args.length < 1) { - throw new IllegalArgumentException(s"Path to benchmark configuration file not specified.") - } - - val configs = BenchmarkConfig(args(0)) - - val benchmarkName = configs.benchmarkName - val dataDir = configs.dataPath - val systemDir = configs.systemPath - val createIx = configs.createIndex - val runWorkload = configs.runWorkload - val enableHyperspace = configs.enableHyperspace - val format = configs.format - val buckets = configs.buckets - - val spark = SparkSession - .builder() - .getOrCreate() - // ------------------ -// val sparkConf = new SparkConf().setMaster("local[*]") -// val spark = SparkSession.builder.config(sparkConf).getOrCreate() - // ------------------ - - // Change log level to clean up output - spark.sparkContext.setLogLevel("ERROR") - - // Config overrides - spark.conf.set("spark.sql.join.preferSortMergeJoin", "true") - - val appStartTime = getAppStartTime(spark) - spark.conf.set(IndexConstants.INDEX_SYSTEM_PATH, systemDir) - - if (createIx) { - println(s"Creating indexes with $buckets buckets on data files in $format format ...") - println(s"\tdata directory\t$dataDir") - println(s"\tsystem directory\t$systemDir") - - val totalCreateIXStats = setupIndexes(spark, benchmarkName, dataDir, buckets, format) - - saveCreateIndexResults( - spark, - dataDir, - systemDir, - appStartTime, - benchmarkName, - format, - buckets, - totalCreateIXStats._2, - totalCreateIXStats._1) - - // Get Index Sizes - println("Collecting index sizes.") - val indexNames = - if (benchmarkName.equals("tpch")) - TPCHBenchmark.recommendedIndexes.map(x => parseIndexConf(x).config.indexName) - else TPCDSBenchmark.recommendedIndexes.map(x => parseIndexConf(x).config.indexName) - - var indexesSizeStats = Seq[Tuple2[String, Long]]() - indexNames.foreach { x => - val indexDirPath = s"$systemDir/$x/v__=0" - val size = FileUtils.getDirectorySize(new Path(indexDirPath)) - indexesSizeStats :+= (x, size) -// println(s" Sample record from index $x (total index size is $size):") -// spark.read.parquet(indexDirPath).show(2, false) - } - - println("Index Size Summary:\n") - indexesSizeStats.sortBy(_._1).foreach{xs => - val readableSize = getReadableSize(xs._2) - println(s"${xs._1}, ${xs._2}, ${readableSize._1}${readableSize._2}")} - } - - if (runWorkload) { - val iterations = configs.iterations - val workload = configs.workload - - // Validate workload - validateWorkload(benchmarkName, workload) - - println(s"\nRunning queries for $iterations iterations from workload $workload.") - println(s"data directory\t$dataDir") - println(s"system directory\t$systemDir") - - // Setup tables - setupBaseTables(spark, benchmarkName, dataDir, format) - - // Run benchmark - if (enableHyperspace) spark.enableHyperspace() - val benchmarkQueries = - if (benchmarkName.equals("tpch")) TPCHBenchmark.queries else TPCDSBenchmark.queries - - var stats = Seq[Tuple3[String, Long, Long]]() - for (q <- workload) { - val query = benchmarkQueries.filter(x => x.id == q).head - for (i <- 1 to iterations) { - val stat = runQuery(spark, query) - stats :+= stat - - println( - s"${stat._1} (Iteration $i, EnableHyperspace $enableHyperspace)\tExecution: ${stat._2} ms, Explain: ${stat._3} ms.") - } - } - - println("\nWorkload execution finished.") - - // show results on output - prettyPrint( - spark, - appStartTime, - benchmarkName, - format, - enableHyperspace, - buckets, - dataDir, - systemDir, - workload, - stats) - - // save results in csv format - saveRunBenchmarkResults( - spark, - dataDir, - systemDir, - appStartTime, - benchmarkName, - format, - enableHyperspace, - buckets, - workload, - stats) - } - - // Stop session - spark.stop() - } - - private def runQuery(spark: SparkSession, query: Query): Tuple3[String, Long, Long] = { - println(s"\n------- Running Query ${query.id} -------") - try { - val df = spark.sql(query.text) - - val explStartTime = System.currentTimeMillis() - df.explain(true) - val explEndTime = System.currentTimeMillis() - - val duration = { - val execStartTime = System.currentTimeMillis() - df.show - val execEndTime = System.currentTimeMillis() - execEndTime - execStartTime - } - - (query.id, duration, explEndTime - explStartTime) - } catch { - case e: Exception => - println(s"Unexpected exception running query ${query.id}: ${e.getMessage}") - (query.id, -1, -1) - } - } - - private def setupBaseTables( - spark: SparkSession, - benchmark: String, - dataDir: String, - format: String): Unit = { - println(s"Creating base tables from format $format ...") - val tables = if (benchmark.equals("tpch")) TPCHBenchmark.tables else TPCDSBenchmark.tables - tables.foreach { table => - val baseTableData = s"$dataDir/${table._1}" - val df = readBaseTableData(spark, baseTableData, format) - df.createOrReplaceTempView(table._1) - println(s"\tTable ${table._1} created from data under $baseTableData.") - } - } - - private def setupIndexes( - spark: SparkSession, - benchmark: String, - dataDir: String, - buckets: String, - format: String): Tuple2[Long, Int] = { - val hyperspace = Hyperspace() - require(!spark.conf.get(IndexConstants.INDEX_SYSTEM_PATH).isEmpty) - val systemPath = spark.conf.get(IndexConstants.INDEX_SYSTEM_PATH) - spark.conf.set(IndexConstants.INDEX_NUM_BUCKETS, buckets) - - println(s"Printing existing indexes (if any) before index creation (under systemPath: $systemPath ).") - hyperspace.indexes.show(100, false) - - val indexes = - if (benchmark.equals("tpch")) TPCHBenchmark.recommendedIndexes - else TPCDSBenchmark.recommendedIndexes - val tables = if (benchmark.equals("tpch")) TPCHBenchmark.tables else TPCDSBenchmark.tables - - val indexCount = indexes.length - val indexInfo = indexes.map(x => parseIndexConf(x)) - - // Validate index configs - indexInfo.foreach { x => - if (!validateIndexConfig(tables, x)) { - throw new IllegalArgumentException(s"Invalid indexConfig ${x.config.indexName}.") - } - } - - // Validate index buckets config - require(spark.conf.get(IndexConstants.INDEX_NUM_BUCKETS).equals(buckets)) - - // Create indexes - var totalTime = 0L - indexInfo.foreach { x => - val baseTableData = s"$dataDir/${x.tableName}" - totalTime += createIndex(spark, hyperspace, baseTableData, x, format) - - println(s"Print sample records from index ${x.config.indexName}") - val indexFilesPath = s"$systemPath/${x.config.indexName}/v__=0" - spark.read.parquet(indexFilesPath).show(2, false) - } - - hyperspace.indexes.show(100, false) - (totalTime, indexCount) - } - - private def validateIndexConfig( - tables: Array[(String, StructType)], - indexInfo: CoveringIndex): Boolean = { - val baseTable = tables.filter(e => e._1 == indexInfo.tableName) - if (baseTable.length != 1) { return false } - - val tableColumns = baseTable(0)._2.fieldNames - val indexedColumns = indexInfo.config.indexedColumns - val includedColumns = indexInfo.config.includedColumns - indexedColumns.forall(tableColumns.contains) && includedColumns.forall(tableColumns.contains) - } - - private def validateWorkload(benchmark: String, workload: Seq[String]): Unit = { - val benchmarkQueries = - if (benchmark.equals("tpch")) TPCHBenchmark.queries else TPCDSBenchmark.queries - require(workload != null && benchmarkQueries.length > 0) - - workload.foreach { qid => - if (benchmarkQueries.filter(x => x.id == qid).isEmpty) { - throw new IllegalArgumentException(s"Unknown query $qid in the workload.") - } - } - } - - private def createIndex( - spark: SparkSession, - hyperspace: Hyperspace, - baseTableData: String, - indexInfo: CoveringIndex, - format: String): Long = { - val df = readBaseTableData(spark, baseTableData, format) - println( - s"\tCreating index ${indexInfo.config.indexName} on data files under $baseTableData ...") - - val startTime = System.currentTimeMillis() - hyperspace.createIndex(df, indexInfo.config) - val endTime = System.currentTimeMillis() - - endTime - startTime - } - - private def parseIndexConf(content: String): CoveringIndex = { - val tokens = content.split(";") - require(tokens.size == 3 || tokens.size == 4) - val indexedColumns = tokens(2).split(",") - val includedColumns = if (tokens.size == 4) { tokens(3).split(",") } else { Array[String]() } - CoveringIndex(tokens(1), IndexConfig(tokens(0), indexedColumns.toSeq, includedColumns.toSeq)) - } - - private def readBaseTableData( - spark: SparkSession, - baseTableData: String, - format: String): DataFrame = { - if (format.equals("parquet")) { - spark.read.parquet(baseTableData) - } else if (format.equals("csv")) { - spark.read.option("header", "true").csv(baseTableData) - } else { - throw new IllegalArgumentException(s"Unknown format $format.") - } - } - - private def prettyPrint( - spark: SparkSession, - startTime: String, - benchmarkName: String, - format: String, - enableHyperspace: Boolean, - buckets: String, - dataDir: String, - systemDir: String, - workload: Seq[String], - stats: Seq[Tuple3[String, Long, Long]]): Unit = { - val appId = spark.sparkContext.applicationId - val appName = spark.sparkContext.appName - val execNum = spark.conf.get("spark.executor.instances", "Not Found") - val cores = spark.conf.get("spark.executor.cores", "Not Found") - val memory = spark.conf.get("spark.executor.memory", "Not Found") - val driverMemory = spark.conf.get("spark.driver.memory", "Not Found") - val driverCores = spark.conf.get("spark.driver.cores", "Not Found") - - println(s"\n\n----- Summary -----") - println(s"***** $appName *****\n") - - println(s"***** Query Execution Times *****\n") - for (x <- workload) { - print(s"$x\t") - stats.filter(r => r._1 == x).foreach(t => print(s"\t${t._2}")) - println() - } - - println(s"\n\n***** Query Explain Times *****\n") - for (x <- workload) { - print(s"$x\t") - stats.filter(r => r._1 == x).foreach(t => print(s"\t${t._3}")) - println() - } - - println(s"\napplication start time:\t$startTime") - println(s"applicationId\t$appId") - println(s"dataset\t$benchmarkName") - println(s"format\t$format") - println(s"enableHyperspace\t$enableHyperspace") - println(s"buckets\t$buckets") - println(s"requested executor number\t$execNum") - println(s"requested executor cores\t$cores") - println(s"requested executor memory\t$memory") - println(s"driver memory\t$driverMemory") - println(s"driver cores\t$driverCores") - println(s"data directory\t$dataDir") - println(s"system directory\t$systemDir") - println(s"-------------------") - } - - private def saveRunBenchmarkResults( - spark: SparkSession, - dataPath: String, - systemPath: String, - startTime: String, - benchmarkName: String, - format: String, - enableHyperspace: Boolean, - buckets: String, - workload: Seq[String], - stats: Seq[Tuple3[String, Long, Long]]): Unit = { - val appId = spark.sparkContext.applicationId - val execNum = spark.conf.get("spark.executor.instances", "Not Found") - val cores = spark.conf.get("spark.executor.cores", "Not Found") - val memory = spark.conf.get("spark.executor.memory", "Not Found") - - val prefix = - s"$startTime,$appId,$dataPath,$systemPath,$benchmarkName,$format,$execNum,$cores,$memory,$enableHyperspace,$buckets" - - val header = - "appTS,appId,dataPath,systemPath,dataset,format,execNum,execCore,execMem,enableHyperspace,indexBuckets,queryId,iteration,qExecTime,qExplainTime" - var rows = List[String]() - for (x <- workload) { - val queryStats = stats.filter(r => r._1 == x) - for (i <- queryStats.indices) { - rows :+= s"$prefix,$x,$i,${queryStats(i)._2},${queryStats(i)._3}" - } - } - - println(s"Workload execution stats in the csv format:") - println(s"\n$header") - rows.foreach(println(_)) - } - - private def saveCreateIndexResults( - spark: SparkSession, - dataPath: String, - systemPath: String, - startTime: String, - benchmarkName: String, - format: String, - buckets: String, - ixCount: Int, - totalTime: Long): Unit = { - val appId = spark.sparkContext.applicationId - val execNum = spark.conf.get("spark.executor.instances", "Not Found") - val cores = spark.conf.get("spark.executor.cores", "Not Found") - val memory = spark.conf.get("spark.executor.memory", "Not Found") - - val statToSave = - s"$startTime,$appId,$dataPath,$systemPath,$benchmarkName,$format,$execNum,$cores,$memory,$buckets,$ixCount,$totalTime" - - val header = - "appTS,appId,dataPath,systemPath,dataset,format,execNum,execCore,execMem,indexBuckets,indexCount,totalCreateIXTime" - - println(s"Create indexes stats in the csv format:") - println(s"\n$header") - println(statToSave) - } - - private def getAppStartTime(spark: SparkSession): String = { - val dateFormat = "yyyy-MM-dd HH:mm:ss" - spark - .range(1) - .select(date_format(current_timestamp, dateFormat)) - .first() - .mkString - } - - private def getReadableSize(size: Long): Tuple2[Long, String] = { - if (size < 1024) { - return (size, "Bytes") - } else if (size < 1024 * 1024) { - return (size / 1024, "KB") - } else if (size < 1024 * 1024 * 1024) { - return (size / (1024 * 1024), "MB") - } - - (size / (1024 * 1024 * 1024), "GB") - } - - // scalastyle:on -} - -case class CoveringIndex(tableName: String, config: IndexConfig) - -case class Query(id: String, text: String) diff --git a/src/main/scala/com/microsoft/hyperspace/perf/TPCDSBenchmark.scala b/src/main/scala/com/microsoft/hyperspace/perf/TPCDSBenchmark.scala deleted file mode 100644 index 4f02cea6b..000000000 --- a/src/main/scala/com/microsoft/hyperspace/perf/TPCDSBenchmark.scala +++ /dev/null @@ -1,4657 +0,0 @@ -/* - * Copyright (2020) The Hyperspace Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.microsoft.hyperspace.perf - -object TPCDSBenchmark { - // scalastyle:off - - import org.apache.spark.sql.types._ - - val call_center = StructType( - Array( - StructField("cc_call_center_sk", LongType), - StructField("cc_call_center_id", StringType), - StructField("cc_rec_start_date", StringType), - StructField("cc_rec_end_date", StringType), - StructField("cc_closed_date_sk", LongType), - StructField("cc_open_date_sk", LongType), - StructField("cc_name", StringType), - StructField("cc_class", StringType), - StructField("cc_employees", IntegerType), - StructField("cc_sq_ft", IntegerType), - StructField("cc_hours", StringType), - StructField("cc_manager", StringType), - StructField("cc_mkt_id", IntegerType), - StructField("cc_mkt_class", StringType), - StructField("cc_mkt_desc", StringType), - StructField("cc_market_manager", StringType), - StructField("cc_division", IntegerType), - StructField("cc_division_name", StringType), - StructField("cc_company", IntegerType), - StructField("cc_company_name", StringType), - StructField("cc_street_number", StringType), - StructField("cc_street_name", StringType), - StructField("cc_street_type", StringType), - StructField("cc_suite_number", StringType), - StructField("cc_city", StringType), - StructField("cc_county", StringType), - StructField("cc_state", StringType), - StructField("cc_zip", StringType), - StructField("cc_country", StringType), - StructField("cc_gmt_offset", DoubleType), - StructField("cc_tax_percentage", DoubleType))) - - val catalog_page = StructType( - Array( - StructField("cp_catalog_page_sk", LongType), - StructField("cp_catalog_page_id", StringType), - StructField("cp_start_date_sk", LongType), - StructField("cp_end_date_sk", LongType), - StructField("cp_department", StringType), - StructField("cp_catalog_number", IntegerType), - StructField("cp_catalog_page_number", IntegerType), - StructField("cp_description", StringType), - StructField("cp_type", StringType))) - - val catalog_returns = StructType( - Array( - StructField("cr_returned_date_sk", LongType), - StructField("cr_returned_time_sk", LongType), - StructField("cr_item_sk", LongType), - StructField("cr_refunded_customer_sk", LongType), - StructField("cr_refunded_cdemo_sk", LongType), - StructField("cr_refunded_hdemo_sk", LongType), - StructField("cr_refunded_addr_sk", LongType), - StructField("cr_returning_customer_sk", LongType), - StructField("cr_returning_cdemo_sk", LongType), - StructField("cr_returning_hdemo_sk", LongType), - StructField("cr_returning_addr_sk", LongType), - StructField("cr_call_center_sk", LongType), - StructField("cr_catalog_page_sk", LongType), - StructField("cr_ship_mode_sk", LongType), - StructField("cr_warehouse_sk", LongType), - StructField("cr_reason_sk", LongType), - StructField("cr_order_number", LongType), - StructField("cr_return_quantity", IntegerType), - StructField("cr_return_amount", DoubleType), - StructField("cr_return_tax", DoubleType), - StructField("cr_return_amt_inc_tax", DoubleType), - StructField("cr_fee", DoubleType), - StructField("cr_return_ship_cost", DoubleType), - StructField("cr_refunded_cash", DoubleType), - StructField("cr_reversed_charge", DoubleType), - StructField("cr_store_credit", DoubleType), - StructField("cr_net_loss", DoubleType))) - - val catalog_sales = StructType( - Array( - StructField("cs_sold_date_sk", LongType), - StructField("cs_sold_time_sk", LongType), - StructField("cs_ship_date_sk", LongType), - StructField("cs_bill_customer_sk", LongType), - StructField("cs_bill_cdemo_sk", LongType), - StructField("cs_bill_hdemo_sk", LongType), - StructField("cs_bill_addr_sk", LongType), - StructField("cs_ship_customer_sk", LongType), - StructField("cs_ship_cdemo_sk", LongType), - StructField("cs_ship_hdemo_sk", LongType), - StructField("cs_ship_addr_sk", LongType), - StructField("cs_call_center_sk", LongType), - StructField("cs_catalog_page_sk", LongType), - StructField("cs_ship_mode_sk", LongType), - StructField("cs_warehouse_sk", LongType), - StructField("cs_item_sk", LongType), - StructField("cs_promo_sk", LongType), - StructField("cs_order_number", LongType), - StructField("cs_quantity", IntegerType), - StructField("cs_wholesale_cost", DoubleType), - StructField("cs_list_price", DoubleType), - StructField("cs_sales_price", DoubleType), - StructField("cs_ext_discount_amt", DoubleType), - StructField("cs_ext_sales_price", DoubleType), - StructField("cs_ext_wholesale_cost", DoubleType), - StructField("cs_ext_list_price", DoubleType), - StructField("cs_ext_tax", DoubleType), - StructField("cs_coupon_amt", DoubleType), - StructField("cs_ext_ship_cost", DoubleType), - StructField("cs_net_paid", DoubleType), - StructField("cs_net_paid_inc_tax", DoubleType), - StructField("cs_net_paid_inc_ship", DoubleType), - StructField("cs_net_paid_inc_ship_tax", DoubleType), - StructField("cs_net_profit", DoubleType))) - - val customer_address = StructType( - Array( - StructField("ca_address_sk", LongType), - StructField("ca_address_id", StringType), - StructField("ca_street_number", StringType), - StructField("ca_street_name", StringType), - StructField("ca_street_type", StringType), - StructField("ca_suite_number", StringType), - StructField("ca_city", StringType), - StructField("ca_county", StringType), - StructField("ca_state", StringType), - StructField("ca_zip", StringType), - StructField("ca_country", StringType), - StructField("ca_gmt_offset", DoubleType), - StructField("ca_location_type", StringType))) - - val customer_demographics = StructType( - Array( - StructField("cd_demo_sk", LongType), - StructField("cd_gender", StringType), - StructField("cd_marital_status", StringType), - StructField("cd_education_status", StringType), - StructField("cd_purchase_estimate", IntegerType), - StructField("cd_credit_rating", StringType), - StructField("cd_dep_count", IntegerType), - StructField("cd_dep_employed_count", IntegerType), - StructField("cd_dep_college_count", IntegerType))) - - val customer = StructType( - Array( - StructField("c_customer_sk", LongType), - StructField("c_customer_id", StringType), - StructField("c_current_cdemo_sk", LongType), - StructField("c_current_hdemo_sk", LongType), - StructField("c_current_addr_sk", LongType), - StructField("c_first_shipto_date_sk", LongType), - StructField("c_first_sales_date_sk", LongType), - StructField("c_salutation", StringType), - StructField("c_first_name", StringType), - StructField("c_last_name", StringType), - StructField("c_preferred_cust_flag", StringType), - StructField("c_birth_day", IntegerType), - StructField("c_birth_month", IntegerType), - StructField("c_birth_year", IntegerType), - StructField("c_birth_country", StringType), - StructField("c_login", StringType), - StructField("c_email_address", StringType), - StructField("c_last_review_date", StringType))) - - val date_dim = StructType( - Array( - StructField("d_date_sk", LongType), - StructField("d_date_id", StringType), - StructField("d_date", StringType), - StructField("d_month_seq", IntegerType), - StructField("d_week_seq", IntegerType), - StructField("d_quarter_seq", IntegerType), - StructField("d_year", IntegerType), - StructField("d_dow", IntegerType), - StructField("d_moy", IntegerType), - StructField("d_dom", IntegerType), - StructField("d_qoy", IntegerType), - StructField("d_fy_year", IntegerType), - StructField("d_fy_quarter_seq", IntegerType), - StructField("d_fy_week_seq", IntegerType), - StructField("d_day_name", StringType), - StructField("d_quarter_name", StringType), - StructField("d_holiday", StringType), - StructField("d_weekend", StringType), - StructField("d_following_holiday", StringType), - StructField("d_first_dom", IntegerType), - StructField("d_last_dom", IntegerType), - StructField("d_same_day_ly", IntegerType), - StructField("d_same_day_lq", IntegerType), - StructField("d_current_day", StringType), - StructField("d_current_week", StringType), - StructField("d_current_month", StringType), - StructField("d_current_quarter", StringType), - StructField("d_current_year", StringType))) - - val household_demographics = StructType( - Array( - StructField("hd_demo_sk", LongType), - StructField("hd_income_band_sk", LongType), - StructField("hd_buy_potential", StringType), - StructField("hd_dep_count", IntegerType), - StructField("hd_vehicle_count", IntegerType))) - - val inventory = StructType( - Array( - StructField("inv_date_sk", LongType), - StructField("inv_item_sk", LongType), - StructField("inv_warehouse_sk", LongType), - StructField("inv_quantity_on_hand", IntegerType))) - - val income_band = StructType( - Array( - StructField("ib_income_band_sk", LongType), - StructField("ib_lower_bound", IntegerType), - StructField("ib_upper_bound", IntegerType))) - - val item = StructType( - Array( - StructField("i_item_sk", LongType), - StructField("i_item_id", StringType), - StructField("i_rec_start_date", StringType), - StructField("i_rec_end_date", StringType), - StructField("i_item_desc", StringType), - StructField("i_current_price", DoubleType), - StructField("i_wholesale_cost", DoubleType), - StructField("i_brand_id", IntegerType), - StructField("i_brand", StringType), - StructField("i_class_id", IntegerType), - StructField("i_class", StringType), - StructField("i_category_id", IntegerType), - StructField("i_category", StringType), - StructField("i_manufact_id", IntegerType), - StructField("i_manufact", StringType), - StructField("i_size", StringType), - StructField("i_formulation", StringType), - StructField("i_color", StringType), - StructField("i_units", StringType), - StructField("i_container", StringType), - StructField("i_manager_id", IntegerType), - StructField("i_product_name", StringType))) - - val promotion = StructType( - Array( - StructField("p_promo_sk", LongType), - StructField("p_promo_id", StringType), - StructField("p_start_date_sk", LongType), - StructField("p_end_date_sk", LongType), - StructField("p_item_sk", LongType), - StructField("p_cost", DoubleType), - StructField("p_response_target", IntegerType), - StructField("p_promo_name", StringType), - StructField("p_channel_dmail", StringType), - StructField("p_channel_email", StringType), - StructField("p_channel_catalog", StringType), - StructField("p_channel_tv", StringType), - StructField("p_channel_radio", StringType), - StructField("p_channel_press", StringType), - StructField("p_channel_event", StringType), - StructField("p_channel_demo", StringType), - StructField("p_channel_details", StringType), - StructField("p_purpose", StringType), - StructField("p_discount_active", StringType))) - - val reason = StructType( - Array( - StructField("r_reason_sk", LongType), - StructField("r_reason_id", StringType), - StructField("r_reason_desc", StringType))) - - val ship_mode = StructType( - Array( - StructField("sm_ship_mode_sk", LongType), - StructField("sm_ship_mode_id", StringType), - StructField("sm_type", StringType), - StructField("sm_code", StringType), - StructField("sm_carrier", StringType), - StructField("sm_contract", StringType))) - - val store_returns = StructType( - Array( - StructField("sr_returned_date_sk", LongType), - StructField("sr_return_time_sk", LongType), - StructField("sr_item_sk", LongType), - StructField("sr_customer_sk", LongType), - StructField("sr_cdemo_sk", LongType), - StructField("sr_hdemo_sk", LongType), - StructField("sr_addr_sk", LongType), - StructField("sr_store_sk", LongType), - StructField("sr_reason_sk", LongType), - StructField("sr_ticket_number", LongType), - StructField("sr_return_quantity", IntegerType), - StructField("sr_return_amt", DoubleType), - StructField("sr_return_tax", DoubleType), - StructField("sr_return_amt_inc_tax", DoubleType), - StructField("sr_fee", DoubleType), - StructField("sr_return_ship_cost", DoubleType), - StructField("sr_refunded_cash", DoubleType), - StructField("sr_reversed_charge", DoubleType), - StructField("sr_store_credit", DoubleType), - StructField("sr_net_loss", DoubleType))) - - val store_sales = StructType( - Array( - StructField("ss_sold_date_sk", LongType), - StructField("ss_sold_time_sk", LongType), - StructField("ss_item_sk", LongType), - StructField("ss_customer_sk", LongType), - StructField("ss_cdemo_sk", LongType), - StructField("ss_hdemo_sk", LongType), - StructField("ss_addr_sk", LongType), - StructField("ss_store_sk", LongType), - StructField("ss_promo_sk", LongType), - StructField("ss_ticket_number", LongType), - StructField("ss_quantity", IntegerType), - StructField("ss_wholesale_cost", DoubleType), - StructField("ss_list_price", DoubleType), - StructField("ss_sales_price", DoubleType), - StructField("ss_ext_discount_amt", DoubleType), - StructField("ss_ext_sales_price", DoubleType), - StructField("ss_ext_wholesale_cost", DoubleType), - StructField("ss_ext_list_price", DoubleType), - StructField("ss_ext_tax", DoubleType), - StructField("ss_coupon_amt", DoubleType), - StructField("ss_net_paid", DoubleType), - StructField("ss_net_paid_inc_tax", DoubleType), - StructField("ss_net_profit", DoubleType))) - - val store = StructType( - Array( - StructField("s_store_sk", LongType), - StructField("s_store_id", StringType), - StructField("s_rec_start_date", StringType), - StructField("s_rec_end_date", StringType), - StructField("s_closed_date_sk", LongType), - StructField("s_store_name", StringType), - StructField("s_number_employees", IntegerType), - StructField("s_floor_space", IntegerType), - StructField("s_hours", StringType), - StructField("s_manager", StringType), - StructField("s_market_id", IntegerType), - StructField("s_geography_class", StringType), - StructField("s_market_desc", StringType), - StructField("s_market_manager", StringType), - StructField("s_division_id", IntegerType), - StructField("s_division_name", StringType), - StructField("s_company_id", IntegerType), - StructField("s_company_name", StringType), - StructField("s_street_number", StringType), - StructField("s_street_name", StringType), - StructField("s_street_type", StringType), - StructField("s_suite_number", StringType), - StructField("s_city", StringType), - StructField("s_county", StringType), - StructField("s_state", StringType), - StructField("s_zip", StringType), - StructField("s_country", StringType), - StructField("s_gmt_offset", DoubleType), - StructField("s_tax_precentage", DoubleType))) - - val time_dim = StructType( - Array( - StructField("t_time_sk", LongType), - StructField("t_time_id", StringType), - StructField("t_time", IntegerType), - StructField("t_hour", IntegerType), - StructField("t_minute", IntegerType), - StructField("t_second", IntegerType), - StructField("t_am_pm", StringType), - StructField("t_shift", StringType), - StructField("t_sub_shift", StringType), - StructField("t_meal_time", StringType))) - - val warehouse = StructType( - Array( - StructField("w_warehouse_sk", LongType), - StructField("w_warehouse_id", StringType), - StructField("w_warehouse_name", StringType), - StructField("w_warehouse_sq_ft", IntegerType), - StructField("w_street_number", StringType), - StructField("w_street_name", StringType), - StructField("w_street_type", StringType), - StructField("w_suite_number", StringType), - StructField("w_city", StringType), - StructField("w_county", StringType), - StructField("w_state", StringType), - StructField("w_zip", StringType), - StructField("w_country", StringType), - StructField("w_gmt_offset", DoubleType))) - - val web_page = StructType( - Array( - StructField("wp_web_page_sk", LongType), - StructField("wp_web_page_id", StringType), - StructField("wp_rec_start_date", StringType), - StructField("wp_rec_end_date", StringType), - StructField("wp_creation_date_sk", LongType), - StructField("wp_access_date_sk", LongType), - StructField("wp_autogen_flag", StringType), - StructField("wp_customer_sk", LongType), - StructField("wp_url", StringType), - StructField("wp_type", StringType), - StructField("wp_char_count", IntegerType), - StructField("wp_link_count", IntegerType), - StructField("wp_image_count", IntegerType), - StructField("wp_max_ad_count", IntegerType))) - - val web_returns = StructType( - Array( - StructField("wr_returned_date_sk", LongType), - StructField("wr_returned_time_sk", LongType), - StructField("wr_item_sk", LongType), - StructField("wr_refunded_customer_sk", LongType), - StructField("wr_refunded_cdemo_sk", LongType), - StructField("wr_refunded_hdemo_sk", LongType), - StructField("wr_refunded_addr_sk", LongType), - StructField("wr_returning_customer_sk", LongType), - StructField("wr_returning_cdemo_sk", LongType), - StructField("wr_returning_hdemo_sk", LongType), - StructField("wr_returning_addr_sk", LongType), - StructField("wr_web_page_sk", LongType), - StructField("wr_reason_sk", LongType), - StructField("wr_order_number", LongType), - StructField("wr_return_quantity", IntegerType), - StructField("wr_return_amt", DoubleType), - StructField("wr_return_tax", DoubleType), - StructField("wr_return_amt_inc_tax", DoubleType), - StructField("wr_fee", DoubleType), - StructField("wr_return_ship_cost", DoubleType), - StructField("wr_refunded_cash", DoubleType), - StructField("wr_reversed_charge", DoubleType), - StructField("wr_account_credit", DoubleType), - StructField("wr_net_loss", DoubleType))) - - val web_sales = StructType( - Array( - StructField("ws_sold_date_sk", LongType), - StructField("ws_sold_time_sk", LongType), - StructField("ws_ship_date_sk", LongType), - StructField("ws_item_sk", LongType), - StructField("ws_bill_customer_sk", LongType), - StructField("ws_bill_cdemo_sk", LongType), - StructField("ws_bill_hdemo_sk", LongType), - StructField("ws_bill_addr_sk", LongType), - StructField("ws_ship_customer_sk", LongType), - StructField("ws_ship_cdemo_sk", LongType), - StructField("ws_ship_hdemo_sk", LongType), - StructField("ws_ship_addr_sk", LongType), - StructField("ws_web_page_sk", LongType), - StructField("ws_web_site_sk", LongType), - StructField("ws_ship_mode_sk", LongType), - StructField("ws_warehouse_sk", LongType), - StructField("ws_promo_sk", LongType), - StructField("ws_order_number", LongType), - StructField("ws_quantity", IntegerType), - StructField("ws_wholesale_cost", DoubleType), - StructField("ws_list_price", DoubleType), - StructField("ws_sales_price", DoubleType), - StructField("ws_ext_discount_amt", DoubleType), - StructField("ws_ext_sales_price", DoubleType), - StructField("ws_ext_wholesale_cost", DoubleType), - StructField("ws_ext_list_price", DoubleType), - StructField("ws_ext_tax", DoubleType), - StructField("ws_coupon_amt", DoubleType), - StructField("ws_ext_ship_cost", DoubleType), - StructField("ws_net_paid", DoubleType), - StructField("ws_net_paid_inc_tax", DoubleType), - StructField("ws_net_paid_inc_ship", DoubleType), - StructField("ws_net_paid_inc_ship_tax", DoubleType), - StructField("ws_net_profit", DoubleType))) - - val web_site = StructType( - Array( - StructField("web_site_sk", LongType), - StructField("web_site_id", StringType), - StructField("web_rec_start_date", StringType), - StructField("web_rec_end_date", StringType), - StructField("web_name", StringType), - StructField("web_open_date_sk", LongType), - StructField("web_close_date_sk", LongType), - StructField("web_class", StringType), - StructField("web_manager", StringType), - StructField("web_mkt_id", IntegerType), - StructField("web_mkt_class", StringType), - StructField("web_mkt_desc", StringType), - StructField("web_market_manager", StringType), - StructField("web_company_id", IntegerType), - StructField("web_company_name", StringType), - StructField("web_street_number", StringType), - StructField("web_street_name", StringType), - StructField("web_street_type", StringType), - StructField("web_suite_number", StringType), - StructField("web_city", StringType), - StructField("web_county", StringType), - StructField("web_state", StringType), - StructField("web_zip", StringType), - StructField("web_country", StringType), - StructField("web_gmt_offset", DoubleType), - StructField("web_tax_percentage", DoubleType))) - - val tables = Array( - ("call_center", call_center), - ("catalog_page", catalog_page), - ("catalog_returns", catalog_returns), - ("catalog_sales", catalog_sales), - ("customer_address", customer_address), - ("customer_demographics", customer_demographics), - ("customer", customer), - ("date_dim", date_dim), - ("household_demographics", household_demographics), - ("income_band", income_band), - ("inventory", inventory), - ("item", item), - ("promotion", promotion), - ("reason", reason), - ("ship_mode", ship_mode), - ("store_returns", store_returns), - ("store_sales", store_sales), - ("store", store), - ("time_dim", time_dim), - ("warehouse", warehouse), - ("web_page", web_page), - ("web_returns", web_returns), - ("web_sales", web_sales), - ("web_site", web_site)) - - // should be random generated based on scale - // RC=ulist(random(1, rowcount("store_sales")/5,uniform),5); - val rc = Array(1000000, 1000000, 1000000, 1000000, 1000000) - - val queries = Array( - Query( - "q1", - """ - | WITH customer_total_return AS - | (SELECT sr_customer_sk AS ctr_customer_sk, sr_store_sk AS ctr_store_sk, - | sum(sr_return_amt) AS ctr_total_return - | FROM store_returns, date_dim - | WHERE sr_returned_date_sk = d_date_sk AND d_year = 2000 - | GROUP BY sr_customer_sk, sr_store_sk) - | SELECT c_customer_id - | FROM customer_total_return ctr1, store, customer - | WHERE ctr1.ctr_total_return > - | (SELECT avg(ctr_total_return)*1.2 - | FROM customer_total_return ctr2 - | WHERE ctr1.ctr_store_sk = ctr2.ctr_store_sk) - | AND s_store_sk = ctr1.ctr_store_sk - | AND s_state = 'TN' - | AND ctr1.ctr_customer_sk = c_customer_sk - | ORDER BY c_customer_id LIMIT 100 - """.stripMargin), - Query( - "q2", - """ - | WITH wscs as - | (SELECT sold_date_sk, sales_price - | FROM (SELECT ws_sold_date_sk sold_date_sk, ws_ext_sales_price sales_price - | FROM web_sales) x - | UNION ALL - | (SELECT cs_sold_date_sk sold_date_sk, cs_ext_sales_price sales_price - | FROM catalog_sales)), - | wswscs AS - | (SELECT d_week_seq, - | sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, - | sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, - | sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, - | sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, - | sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, - | sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, - | sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales - | FROM wscs, date_dim - | WHERE d_date_sk = sold_date_sk - | GROUP BY d_week_seq) - | SELECT d_week_seq1 - | ,round(sun_sales1/sun_sales2,2) - | ,round(mon_sales1/mon_sales2,2) - | ,round(tue_sales1/tue_sales2,2) - | ,round(wed_sales1/wed_sales2,2) - | ,round(thu_sales1/thu_sales2,2) - | ,round(fri_sales1/fri_sales2,2) - | ,round(sat_sales1/sat_sales2,2) - | FROM - | (SELECT wswscs.d_week_seq d_week_seq1 - | ,sun_sales sun_sales1 - | ,mon_sales mon_sales1 - | ,tue_sales tue_sales1 - | ,wed_sales wed_sales1 - | ,thu_sales thu_sales1 - | ,fri_sales fri_sales1 - | ,sat_sales sat_sales1 - | FROM wswscs,date_dim - | WHERE date_dim.d_week_seq = wswscs.d_week_seq AND d_year = 2001) y, - | (SELECT wswscs.d_week_seq d_week_seq2 - | ,sun_sales sun_sales2 - | ,mon_sales mon_sales2 - | ,tue_sales tue_sales2 - | ,wed_sales wed_sales2 - | ,thu_sales thu_sales2 - | ,fri_sales fri_sales2 - | ,sat_sales sat_sales2 - | FROM wswscs, date_dim - | WHERE date_dim.d_week_seq = wswscs.d_week_seq AND d_year = 2001 + 1) z - | WHERE d_week_seq1=d_week_seq2-53 - | ORDER BY d_week_seq1 - """.stripMargin), - Query( - "q3", - """ - | SELECT dt.d_year, item.i_brand_id brand_id, item.i_brand brand,SUM(ss_ext_sales_price) sum_agg - | FROM date_dim dt, store_sales, item - | WHERE dt.d_date_sk = store_sales.ss_sold_date_sk - | AND store_sales.ss_item_sk = item.i_item_sk - | AND item.i_manufact_id = 128 - | AND dt.d_moy=11 - | GROUP BY dt.d_year, item.i_brand, item.i_brand_id - | ORDER BY dt.d_year, sum_agg desc, brand_id - | LIMIT 100 - """.stripMargin), - Query( - "q4", - """ - |WITH year_total AS ( - | SELECT c_customer_id customer_id, - | c_first_name customer_first_name, - | c_last_name customer_last_name, - | c_preferred_cust_flag customer_preferred_cust_flag, - | c_birth_country customer_birth_country, - | c_login customer_login, - | c_email_address customer_email_address, - | d_year dyear, - | sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total, - | 's' sale_type - | FROM customer, store_sales, date_dim - | WHERE c_customer_sk = ss_customer_sk AND ss_sold_date_sk = d_date_sk - | GROUP BY c_customer_id, - | c_first_name, - | c_last_name, - | c_preferred_cust_flag, - | c_birth_country, - | c_login, - | c_email_address, - | d_year - | UNION ALL - | SELECT c_customer_id customer_id, - | c_first_name customer_first_name, - | c_last_name customer_last_name, - | c_preferred_cust_flag customer_preferred_cust_flag, - | c_birth_country customer_birth_country, - | c_login customer_login, - | c_email_address customer_email_address, - | d_year dyear, - | sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total, - | 'c' sale_type - | FROM customer, catalog_sales, date_dim - | WHERE c_customer_sk = cs_bill_customer_sk AND cs_sold_date_sk = d_date_sk - | GROUP BY c_customer_id, - | c_first_name, - | c_last_name, - | c_preferred_cust_flag, - | c_birth_country, - | c_login, - | c_email_address, - | d_year - | UNION ALL - | SELECT c_customer_id customer_id - | ,c_first_name customer_first_name - | ,c_last_name customer_last_name - | ,c_preferred_cust_flag customer_preferred_cust_flag - | ,c_birth_country customer_birth_country - | ,c_login customer_login - | ,c_email_address customer_email_address - | ,d_year dyear - | ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total - | ,'w' sale_type - | FROM customer, web_sales, date_dim - | WHERE c_customer_sk = ws_bill_customer_sk AND ws_sold_date_sk = d_date_sk - | GROUP BY c_customer_id, - | c_first_name, - | c_last_name, - | c_preferred_cust_flag, - | c_birth_country, - | c_login, - | c_email_address, - | d_year) - | SELECT - | t_s_secyear.customer_id, - | t_s_secyear.customer_first_name, - | t_s_secyear.customer_last_name, - | t_s_secyear.customer_preferred_cust_flag, - | t_s_secyear.customer_birth_country, - | t_s_secyear.customer_login, - | t_s_secyear.customer_email_address - | FROM year_total t_s_firstyear, year_total t_s_secyear, year_total t_c_firstyear, - | year_total t_c_secyear, year_total t_w_firstyear, year_total t_w_secyear - | WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id - | and t_s_firstyear.customer_id = t_c_secyear.customer_id - | and t_s_firstyear.customer_id = t_c_firstyear.customer_id - | and t_s_firstyear.customer_id = t_w_firstyear.customer_id - | and t_s_firstyear.customer_id = t_w_secyear.customer_id - | and t_s_firstyear.sale_type = 's' - | and t_c_firstyear.sale_type = 'c' - | and t_w_firstyear.sale_type = 'w' - | and t_s_secyear.sale_type = 's' - | and t_c_secyear.sale_type = 'c' - | and t_w_secyear.sale_type = 'w' - | and t_s_firstyear.dyear = 2001 - | and t_s_secyear.dyear = 2001+1 - | and t_c_firstyear.dyear = 2001 - | and t_c_secyear.dyear = 2001+1 - | and t_w_firstyear.dyear = 2001 - | and t_w_secyear.dyear = 2001+1 - | and t_s_firstyear.year_total > 0 - | and t_c_firstyear.year_total > 0 - | and t_w_firstyear.year_total > 0 - | and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end - | > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end - | and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end - | > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end - | ORDER BY - | t_s_secyear.customer_id, - | t_s_secyear.customer_first_name, - | t_s_secyear.customer_last_name, - | t_s_secyear.customer_preferred_cust_flag, - | t_s_secyear.customer_birth_country, - | t_s_secyear.customer_login, - | t_s_secyear.customer_email_address - | LIMIT 100 - """.stripMargin), - // Modifications: "+ days" -> date_add - // Modifications: "||" -> concat - Query( - "q5", - """ - | WITH ssr AS - | (SELECT s_store_id, - | sum(sales_price) as sales, - | sum(profit) as profit, - | sum(return_amt) as returns, - | sum(net_loss) as profit_loss - | FROM - | (SELECT ss_store_sk as store_sk, - | ss_sold_date_sk as date_sk, - | ss_ext_sales_price as sales_price, - | ss_net_profit as profit, - | cast(0 as decimal(7,2)) as return_amt, - | cast(0 as decimal(7,2)) as net_loss - | FROM store_sales - | UNION ALL - | SELECT sr_store_sk as store_sk, - | sr_returned_date_sk as date_sk, - | cast(0 as decimal(7,2)) as sales_price, - | cast(0 as decimal(7,2)) as profit, - | sr_return_amt as return_amt, - | sr_net_loss as net_loss - | FROM store_returns) - | salesreturns, date_dim, store - | WHERE date_sk = d_date_sk - | and d_date between cast('2000-08-23' as date) - | and ((cast('2000-08-23' as date) + interval 14 days)) - | and store_sk = s_store_sk - | GROUP BY s_store_id), - | csr AS - | (SELECT cp_catalog_page_id, - | sum(sales_price) as sales, - | sum(profit) as profit, - | sum(return_amt) as returns, - | sum(net_loss) as profit_loss - | FROM - | (SELECT cs_catalog_page_sk as page_sk, - | cs_sold_date_sk as date_sk, - | cs_ext_sales_price as sales_price, - | cs_net_profit as profit, - | cast(0 as decimal(7,2)) as return_amt, - | cast(0 as decimal(7,2)) as net_loss - | FROM catalog_sales - | UNION ALL - | SELECT cr_catalog_page_sk as page_sk, - | cr_returned_date_sk as date_sk, - | cast(0 as decimal(7,2)) as sales_price, - | cast(0 as decimal(7,2)) as profit, - | cr_return_amount as return_amt, - | cr_net_loss as net_loss - | from catalog_returns - | ) salesreturns, date_dim, catalog_page - | WHERE date_sk = d_date_sk - | and d_date between cast('2000-08-23' as date) - | and ((cast('2000-08-23' as date) + interval 14 days)) - | and page_sk = cp_catalog_page_sk - | GROUP BY cp_catalog_page_id) - | , - | wsr AS - | (SELECT web_site_id, - | sum(sales_price) as sales, - | sum(profit) as profit, - | sum(return_amt) as returns, - | sum(net_loss) as profit_loss - | from - | (select ws_web_site_sk as wsr_web_site_sk, - | ws_sold_date_sk as date_sk, - | ws_ext_sales_price as sales_price, - | ws_net_profit as profit, - | cast(0 as decimal(7,2)) as return_amt, - | cast(0 as decimal(7,2)) as net_loss - | from web_sales - | union all - | select ws_web_site_sk as wsr_web_site_sk, - | wr_returned_date_sk as date_sk, - | cast(0 as decimal(7,2)) as sales_price, - | cast(0 as decimal(7,2)) as profit, - | wr_return_amt as return_amt, - | wr_net_loss as net_loss - | FROM web_returns LEFT OUTER JOIN web_sales on - | ( wr_item_sk = ws_item_sk - | and wr_order_number = ws_order_number) - | ) salesreturns, date_dim, web_site - | WHERE date_sk = d_date_sk - | and d_date between cast('2000-08-23' as date) - | and ((cast('2000-08-23' as date) + interval 14 days)) - | and wsr_web_site_sk = web_site_sk - | GROUP BY web_site_id) - | SELECT channel, - | id, - | sum(sales) as sales, - | sum(returns) as returns, - | sum(profit) as profit - | from - | (select 'store channel' as channel, - | concat('store', s_store_id) as id, - | sales, - | returns, - | (profit - profit_loss) as profit - | FROM ssr - | UNION ALL - | select 'catalog channel' as channel, - | concat('catalog_page', cp_catalog_page_id) as id, - | sales, - | returns, - | (profit - profit_loss) as profit - | FROM csr - | UNION ALL - | SELECT 'web channel' as channel, - | concat('web_site', web_site_id) as id, - | sales, - | returns, - | (profit - profit_loss) as profit - | FROM wsr - | ) x - | GROUP BY ROLLUP (channel, id) - | ORDER BY channel, id - | LIMIT 100 - """.stripMargin), - Query( - "q6", - """ - | SELECT a.ca_state state, count(*) cnt - | FROM - | customer_address a, customer c, store_sales s, date_dim d, item i - | WHERE a.ca_address_sk = c.c_current_addr_sk - | AND c.c_customer_sk = s.ss_customer_sk - | AND s.ss_sold_date_sk = d.d_date_sk - | AND s.ss_item_sk = i.i_item_sk - | AND d.d_month_seq = - | (SELECT distinct (d_month_seq) FROM date_dim - | WHERE d_year = 2000 AND d_moy = 1) - | AND i.i_current_price > 1.2 * - | (SELECT avg(j.i_current_price) FROM item j - | WHERE j.i_category = i.i_category) - | GROUP BY a.ca_state - | HAVING count(*) >= 10 - | ORDER BY cnt LIMIT 100 - """.stripMargin), - Query( - "q7", - """ - | SELECT i_item_id, - | avg(ss_quantity) agg1, - | avg(ss_list_price) agg2, - | avg(ss_coupon_amt) agg3, - | avg(ss_sales_price) agg4 - | FROM store_sales, customer_demographics, date_dim, item, promotion - | WHERE ss_sold_date_sk = d_date_sk AND - | ss_item_sk = i_item_sk AND - | ss_cdemo_sk = cd_demo_sk AND - | ss_promo_sk = p_promo_sk AND - | cd_gender = 'M' AND - | cd_marital_status = 'S' AND - | cd_education_status = 'College' AND - | (p_channel_email = 'N' or p_channel_event = 'N') AND - | d_year = 2000 - | GROUP BY i_item_id - | ORDER BY i_item_id LIMIT 100 - """.stripMargin), - Query( - "q8", - """ - | select s_store_name, sum(ss_net_profit) - | from store_sales, date_dim, store, - | (SELECT ca_zip - | from ( - | (SELECT substr(ca_zip,1,5) ca_zip FROM customer_address - | WHERE substr(ca_zip,1,5) IN ( - | '24128','76232','65084','87816','83926','77556','20548', - | '26231','43848','15126','91137','61265','98294','25782', - | '17920','18426','98235','40081','84093','28577','55565', - | '17183','54601','67897','22752','86284','18376','38607', - | '45200','21756','29741','96765','23932','89360','29839', - | '25989','28898','91068','72550','10390','18845','47770', - | '82636','41367','76638','86198','81312','37126','39192', - | '88424','72175','81426','53672','10445','42666','66864', - | '66708','41248','48583','82276','18842','78890','49448', - | '14089','38122','34425','79077','19849','43285','39861', - | '66162','77610','13695','99543','83444','83041','12305', - | '57665','68341','25003','57834','62878','49130','81096', - | '18840','27700','23470','50412','21195','16021','76107', - | '71954','68309','18119','98359','64544','10336','86379', - | '27068','39736','98569','28915','24206','56529','57647', - | '54917','42961','91110','63981','14922','36420','23006', - | '67467','32754','30903','20260','31671','51798','72325', - | '85816','68621','13955','36446','41766','68806','16725', - | '15146','22744','35850','88086','51649','18270','52867', - | '39972','96976','63792','11376','94898','13595','10516', - | '90225','58943','39371','94945','28587','96576','57855', - | '28488','26105','83933','25858','34322','44438','73171', - | '30122','34102','22685','71256','78451','54364','13354', - | '45375','40558','56458','28286','45266','47305','69399', - | '83921','26233','11101','15371','69913','35942','15882', - | '25631','24610','44165','99076','33786','70738','26653', - | '14328','72305','62496','22152','10144','64147','48425', - | '14663','21076','18799','30450','63089','81019','68893', - | '24996','51200','51211','45692','92712','70466','79994', - | '22437','25280','38935','71791','73134','56571','14060', - | '19505','72425','56575','74351','68786','51650','20004', - | '18383','76614','11634','18906','15765','41368','73241', - | '76698','78567','97189','28545','76231','75691','22246', - | '51061','90578','56691','68014','51103','94167','57047', - | '14867','73520','15734','63435','25733','35474','24676', - | '94627','53535','17879','15559','53268','59166','11928', - | '59402','33282','45721','43933','68101','33515','36634', - | '71286','19736','58058','55253','67473','41918','19515', - | '36495','19430','22351','77191','91393','49156','50298', - | '87501','18652','53179','18767','63193','23968','65164', - | '68880','21286','72823','58470','67301','13394','31016', - | '70372','67030','40604','24317','45748','39127','26065', - | '77721','31029','31880','60576','24671','45549','13376', - | '50016','33123','19769','22927','97789','46081','72151', - | '15723','46136','51949','68100','96888','64528','14171', - | '79777','28709','11489','25103','32213','78668','22245', - | '15798','27156','37930','62971','21337','51622','67853', - | '10567','38415','15455','58263','42029','60279','37125', - | '56240','88190','50308','26859','64457','89091','82136', - | '62377','36233','63837','58078','17043','30010','60099', - | '28810','98025','29178','87343','73273','30469','64034', - | '39516','86057','21309','90257','67875','40162','11356', - | '73650','61810','72013','30431','22461','19512','13375', - | '55307','30625','83849','68908','26689','96451','38193', - | '46820','88885','84935','69035','83144','47537','56616', - | '94983','48033','69952','25486','61547','27385','61860', - | '58048','56910','16807','17871','35258','31387','35458', - | '35576')) - | INTERSECT - | (select ca_zip - | FROM - | (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt - | FROM customer_address, customer - | WHERE ca_address_sk = c_current_addr_sk and - | c_preferred_cust_flag='Y' - | group by ca_zip - | having count(*) > 10) A1) - | ) A2 - | ) V1 - | where ss_store_sk = s_store_sk - | and ss_sold_date_sk = d_date_sk - | and d_qoy = 2 and d_year = 1998 - | and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) - | group by s_store_name - | order by s_store_name LIMIT 100 - """.stripMargin), - Query( - "q9", - s""" - |select case when (select count(*) from store_sales - | where ss_quantity between 1 and 20) > ${rc(0)} - | then (select avg(ss_ext_discount_amt) from store_sales - | where ss_quantity between 1 and 20) - | else (select avg(ss_net_paid) from store_sales - | where ss_quantity between 1 and 20) end bucket1 , - | case when (select count(*) from store_sales - | where ss_quantity between 21 and 40) > ${rc(1)} - | then (select avg(ss_ext_discount_amt) from store_sales - | where ss_quantity between 21 and 40) - | else (select avg(ss_net_paid) from store_sales - | where ss_quantity between 21 and 40) end bucket2, - | case when (select count(*) from store_sales - | where ss_quantity between 41 and 60) > ${rc(2)} - | then (select avg(ss_ext_discount_amt) from store_sales - | where ss_quantity between 41 and 60) - | else (select avg(ss_net_paid) from store_sales - | where ss_quantity between 41 and 60) end bucket3, - | case when (select count(*) from store_sales - | where ss_quantity between 61 and 80) > ${rc(3)} - | then (select avg(ss_ext_discount_amt) from store_sales - | where ss_quantity between 61 and 80) - | else (select avg(ss_net_paid) from store_sales - | where ss_quantity between 61 and 80) end bucket4, - | case when (select count(*) from store_sales - | where ss_quantity between 81 and 100) > ${rc(4)} - | then (select avg(ss_ext_discount_amt) from store_sales - | where ss_quantity between 81 and 100) - | else (select avg(ss_net_paid) from store_sales - | where ss_quantity between 81 and 100) end bucket5 - |from reason - |where r_reason_sk = 1 - """.stripMargin), - Query( - "q10", - """ - | select - | cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, - | cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3, - | cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5, - | cd_dep_college_count, count(*) cnt6 - | from - | customer c, customer_address ca, customer_demographics - | where - | c.c_current_addr_sk = ca.ca_address_sk and - | ca_county in ('Rush County','Toole County','Jefferson County', - | 'Dona Ana County','La Porte County') and - | cd_demo_sk = c.c_current_cdemo_sk AND - | exists (select * from store_sales, date_dim - | where c.c_customer_sk = ss_customer_sk AND - | ss_sold_date_sk = d_date_sk AND - | d_year = 2002 AND - | d_moy between 1 AND 1+3) AND - | (exists (select * from web_sales, date_dim - | where c.c_customer_sk = ws_bill_customer_sk AND - | ws_sold_date_sk = d_date_sk AND - | d_year = 2002 AND - | d_moy between 1 AND 1+3) or - | exists (select * from catalog_sales, date_dim - | where c.c_customer_sk = cs_ship_customer_sk AND - | cs_sold_date_sk = d_date_sk AND - | d_year = 2002 AND - | d_moy between 1 AND 1+3)) - | group by cd_gender, - | cd_marital_status, - | cd_education_status, - | cd_purchase_estimate, - | cd_credit_rating, - | cd_dep_count, - | cd_dep_employed_count, - | cd_dep_college_count - | order by cd_gender, - | cd_marital_status, - | cd_education_status, - | cd_purchase_estimate, - | cd_credit_rating, - | cd_dep_count, - | cd_dep_employed_count, - | cd_dep_college_count - |LIMIT 100 - """.stripMargin), - Query( - "q11", - """ - | with year_total as ( - | select c_customer_id customer_id - | ,c_first_name customer_first_name - | ,c_last_name customer_last_name - | ,c_preferred_cust_flag customer_preferred_cust_flag - | ,c_birth_country customer_birth_country - | ,c_login customer_login - | ,c_email_address customer_email_address - | ,d_year dyear - | ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total - | ,'s' sale_type - | from customer, store_sales, date_dim - | where c_customer_sk = ss_customer_sk - | and ss_sold_date_sk = d_date_sk - | group by c_customer_id - | ,c_first_name - | ,c_last_name - | ,d_year - | ,c_preferred_cust_flag - | ,c_birth_country - | ,c_login - | ,c_email_address - | ,d_year - | union all - | select c_customer_id customer_id - | ,c_first_name customer_first_name - | ,c_last_name customer_last_name - | ,c_preferred_cust_flag customer_preferred_cust_flag - | ,c_birth_country customer_birth_country - | ,c_login customer_login - | ,c_email_address customer_email_address - | ,d_year dyear - | ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total - | ,'w' sale_type - | from customer, web_sales, date_dim - | where c_customer_sk = ws_bill_customer_sk - | and ws_sold_date_sk = d_date_sk - | group by - | c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, - | c_login, c_email_address, d_year) - | select - | t_s_secyear.customer_preferred_cust_flag - | from year_total t_s_firstyear - | ,year_total t_s_secyear - | ,year_total t_w_firstyear - | ,year_total t_w_secyear - | where t_s_secyear.customer_id = t_s_firstyear.customer_id - | and t_s_firstyear.customer_id = t_w_secyear.customer_id - | and t_s_firstyear.customer_id = t_w_firstyear.customer_id - | and t_s_firstyear.sale_type = 's' - | and t_w_firstyear.sale_type = 'w' - | and t_s_secyear.sale_type = 's' - | and t_w_secyear.sale_type = 'w' - | and t_s_firstyear.dyear = 2001 - | and t_s_secyear.dyear = 2001+1 - | and t_w_firstyear.dyear = 2001 - | and t_w_secyear.dyear = 2001+1 - | and t_s_firstyear.year_total > 0 - | and t_w_firstyear.year_total > 0 - | and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end - | > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end - | order by t_s_secyear.customer_preferred_cust_flag - | LIMIT 100 - """.stripMargin), - // Modifications: "+ days" -> date_add - Query( - "q12", - """ - | select - | i_item_desc, i_category, i_class, i_current_price, - | sum(ws_ext_sales_price) as itemrevenue, - | sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over - | (partition by i_class) as revenueratio - | from - | web_sales, item, date_dim - | where - | ws_item_sk = i_item_sk - | and i_category in ('Sports', 'Books', 'Home') - | and ws_sold_date_sk = d_date_sk - | and d_date between cast('1999-02-22' as date) - | and (cast('1999-02-22' as date) + interval 30 days) - | group by - | i_item_id, i_item_desc, i_category, i_class, i_current_price - | order by - | i_category, i_class, i_item_id, i_item_desc, revenueratio - | LIMIT 100 - """.stripMargin), - Query( - "q13", - """ - | select avg(ss_quantity) - | ,avg(ss_ext_sales_price) - | ,avg(ss_ext_wholesale_cost) - | ,sum(ss_ext_wholesale_cost) - | from store_sales - | ,store - | ,customer_demographics - | ,household_demographics - | ,customer_address - | ,date_dim - | where s_store_sk = ss_store_sk - | and ss_sold_date_sk = d_date_sk and d_year = 2001 - | and((ss_hdemo_sk=hd_demo_sk - | and cd_demo_sk = ss_cdemo_sk - | and cd_marital_status = 'M' - | and cd_education_status = 'Advanced Degree' - | and ss_sales_price between 100.00 and 150.00 - | and hd_dep_count = 3 - | )or - | (ss_hdemo_sk=hd_demo_sk - | and cd_demo_sk = ss_cdemo_sk - | and cd_marital_status = 'S' - | and cd_education_status = 'College' - | and ss_sales_price between 50.00 and 100.00 - | and hd_dep_count = 1 - | ) or - | (ss_hdemo_sk=hd_demo_sk - | and cd_demo_sk = ss_cdemo_sk - | and cd_marital_status = 'W' - | and cd_education_status = '2 yr Degree' - | and ss_sales_price between 150.00 and 200.00 - | and hd_dep_count = 1 - | )) - | and((ss_addr_sk = ca_address_sk - | and ca_country = 'United States' - | and ca_state in ('TX', 'OH', 'TX') - | and ss_net_profit between 100 and 200 - | ) or - | (ss_addr_sk = ca_address_sk - | and ca_country = 'United States' - | and ca_state in ('OR', 'NM', 'KY') - | and ss_net_profit between 150 and 300 - | ) or - | (ss_addr_sk = ca_address_sk - | and ca_country = 'United States' - | and ca_state in ('VA', 'TX', 'MS') - | and ss_net_profit between 50 and 250 - | )) - """.stripMargin), - Query( - "q14a", - """ - |with cross_items as - | (select i_item_sk ss_item_sk - | from item, - | (select iss.i_brand_id brand_id, iss.i_class_id class_id, iss.i_category_id category_id - | from store_sales, item iss, date_dim d1 - | where ss_item_sk = iss.i_item_sk - and ss_sold_date_sk = d1.d_date_sk - | and d1.d_year between 1999 AND 1999 + 2 - | intersect - | select ics.i_brand_id, ics.i_class_id, ics.i_category_id - | from catalog_sales, item ics, date_dim d2 - | where cs_item_sk = ics.i_item_sk - | and cs_sold_date_sk = d2.d_date_sk - | and d2.d_year between 1999 AND 1999 + 2 - | intersect - | select iws.i_brand_id, iws.i_class_id, iws.i_category_id - | from web_sales, item iws, date_dim d3 - | where ws_item_sk = iws.i_item_sk - | and ws_sold_date_sk = d3.d_date_sk - | and d3.d_year between 1999 AND 1999 + 2) x - | where i_brand_id = brand_id - | and i_class_id = class_id - | and i_category_id = category_id - |), - | avg_sales as - | (select avg(quantity*list_price) average_sales - | from ( - | select ss_quantity quantity, ss_list_price list_price - | from store_sales, date_dim - | where ss_sold_date_sk = d_date_sk - | and d_year between 1999 and 2001 - | union all - | select cs_quantity quantity, cs_list_price list_price - | from catalog_sales, date_dim - | where cs_sold_date_sk = d_date_sk - | and d_year between 1999 and 1999 + 2 - | union all - | select ws_quantity quantity, ws_list_price list_price - | from web_sales, date_dim - | where ws_sold_date_sk = d_date_sk - | and d_year between 1999 and 1999 + 2) x) - | select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) - | from( - | select 'store' channel, i_brand_id,i_class_id - | ,i_category_id,sum(ss_quantity*ss_list_price) sales - | , count(*) number_sales - | from store_sales, item, date_dim - | where ss_item_sk in (select ss_item_sk from cross_items) - | and ss_item_sk = i_item_sk - | and ss_sold_date_sk = d_date_sk - | and d_year = 1999+2 - | and d_moy = 11 - | group by i_brand_id,i_class_id,i_category_id - | having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) - | union all - | select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales - | from catalog_sales, item, date_dim - | where cs_item_sk in (select ss_item_sk from cross_items) - | and cs_item_sk = i_item_sk - | and cs_sold_date_sk = d_date_sk - | and d_year = 1999+2 - | and d_moy = 11 - | group by i_brand_id,i_class_id,i_category_id - | having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) - | union all - | select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales - | from web_sales, item, date_dim - | where ws_item_sk in (select ss_item_sk from cross_items) - | and ws_item_sk = i_item_sk - | and ws_sold_date_sk = d_date_sk - | and d_year = 1999+2 - | and d_moy = 11 - | group by i_brand_id,i_class_id,i_category_id - | having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) - | ) y - | group by rollup (channel, i_brand_id,i_class_id,i_category_id) - | order by channel,i_brand_id,i_class_id,i_category_id - | limit 100 - """.stripMargin), - Query( - "q14b", - """ - | with cross_items as - | (select i_item_sk ss_item_sk - | from item, - | (select iss.i_brand_id brand_id, iss.i_class_id class_id, iss.i_category_id category_id - | from store_sales, item iss, date_dim d1 - | where ss_item_sk = iss.i_item_sk - | and ss_sold_date_sk = d1.d_date_sk - | and d1.d_year between 1999 AND 1999 + 2 - | intersect - | select ics.i_brand_id, ics.i_class_id, ics.i_category_id - | from catalog_sales, item ics, date_dim d2 - | where cs_item_sk = ics.i_item_sk - | and cs_sold_date_sk = d2.d_date_sk - | and d2.d_year between 1999 AND 1999 + 2 - | intersect - | select iws.i_brand_id, iws.i_class_id, iws.i_category_id - | from web_sales, item iws, date_dim d3 - | where ws_item_sk = iws.i_item_sk - | and ws_sold_date_sk = d3.d_date_sk - | and d3.d_year between 1999 AND 1999 + 2) x - | where i_brand_id = brand_id - | and i_class_id = class_id - | and i_category_id = category_id - | ), - | avg_sales as - | (select avg(quantity*list_price) average_sales - | from (select ss_quantity quantity, ss_list_price list_price - | from store_sales, date_dim - | where ss_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 - | union all - | select cs_quantity quantity, cs_list_price list_price - | from catalog_sales, date_dim - | where cs_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 - | union all - | select ws_quantity quantity, ws_list_price list_price - | from web_sales, date_dim - | where ws_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2) x) - | select * from - | (select 'store' channel, i_brand_id,i_class_id,i_category_id - | ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales - | from store_sales, item, date_dim - | where ss_item_sk in (select ss_item_sk from cross_items) - | and ss_item_sk = i_item_sk - | and ss_sold_date_sk = d_date_sk - | and d_week_seq = (select d_week_seq from date_dim - | where d_year = 1999 + 1 and d_moy = 12 and d_dom = 11) - | group by i_brand_id,i_class_id,i_category_id - | having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, - | (select 'store' channel, i_brand_id,i_class_id - | ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales - | from store_sales, item, date_dim - | where ss_item_sk in (select ss_item_sk from cross_items) - | and ss_item_sk = i_item_sk - | and ss_sold_date_sk = d_date_sk - | and d_week_seq = (select d_week_seq from date_dim - | where d_year = 1999 and d_moy = 12 and d_dom = 11) - | group by i_brand_id,i_class_id,i_category_id - | having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year - | where this_year.i_brand_id= last_year.i_brand_id - | and this_year.i_class_id = last_year.i_class_id - | and this_year.i_category_id = last_year.i_category_id - | order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id - | limit 100 - """.stripMargin), - Query( - "q15", - """ - | select ca_zip, sum(cs_sales_price) - | from catalog_sales, customer, customer_address, date_dim - | where cs_bill_customer_sk = c_customer_sk - | and c_current_addr_sk = ca_address_sk - | and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', - | '85392', '85460', '80348', '81792') - | or ca_state in ('CA','WA','GA') - | or cs_sales_price > 500) - | and cs_sold_date_sk = d_date_sk - | and d_qoy = 2 and d_year = 2001 - | group by ca_zip - | order by ca_zip - | limit 100 - """.stripMargin), - // Modifications: " -> ` - Query( - "q16", - """ - | select - | count(distinct cs_order_number) as `order count`, - | sum(cs_ext_ship_cost) as `total shipping cost`, - | sum(cs_net_profit) as `total net profit` - | from - | catalog_sales cs1, date_dim, customer_address, call_center - | where - | d_date between '2002-02-01' and (cast('2002-02-01' as date) + interval 60 days) - | and cs1.cs_ship_date_sk = d_date_sk - | and cs1.cs_ship_addr_sk = ca_address_sk - | and ca_state = 'GA' - | and cs1.cs_call_center_sk = cc_call_center_sk - | and cc_county in ('Williamson County','Williamson County','Williamson County','Williamson County', 'Williamson County') - | and exists (select * - | from catalog_sales cs2 - | where cs1.cs_order_number = cs2.cs_order_number - | and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) - | and not exists(select * - | from catalog_returns cr1 - | where cs1.cs_order_number = cr1.cr_order_number) - | order by count(distinct cs_order_number) - | limit 100 - """.stripMargin), - Query( - "q17", - """ - | select i_item_id - | ,i_item_desc - | ,s_state - | ,count(ss_quantity) as store_sales_quantitycount - | ,avg(ss_quantity) as store_sales_quantityave - | ,stddev_samp(ss_quantity) as store_sales_quantitystdev - | ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov - | ,count(sr_return_quantity) as_store_returns_quantitycount - | ,avg(sr_return_quantity) as_store_returns_quantityave - | ,stddev_samp(sr_return_quantity) as_store_returns_quantitystdev - | ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov - | ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave - | ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitystdev - | ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov - | from store_sales, store_returns, catalog_sales, date_dim d1, date_dim d2, date_dim d3, store, item - | where d1.d_quarter_name = '2001Q1' - | and d1.d_date_sk = ss_sold_date_sk - | and i_item_sk = ss_item_sk - | and s_store_sk = ss_store_sk - | and ss_customer_sk = sr_customer_sk - | and ss_item_sk = sr_item_sk - | and ss_ticket_number = sr_ticket_number - | and sr_returned_date_sk = d2.d_date_sk - | and d2.d_quarter_name in ('2001Q1','2001Q2','2001Q3') - | and sr_customer_sk = cs_bill_customer_sk - | and sr_item_sk = cs_item_sk - | and cs_sold_date_sk = d3.d_date_sk - | and d3.d_quarter_name in ('2001Q1','2001Q2','2001Q3') - | group by i_item_id, i_item_desc, s_state - | order by i_item_id, i_item_desc, s_state - | limit 100 - """.stripMargin), - // Modifications: "numeric" -> "decimal" - Query( - "q18", - """ - | select i_item_id, - | ca_country, - | ca_state, - | ca_county, - | avg( cast(cs_quantity as decimal(12,2))) agg1, - | avg( cast(cs_list_price as decimal(12,2))) agg2, - | avg( cast(cs_coupon_amt as decimal(12,2))) agg3, - | avg( cast(cs_sales_price as decimal(12,2))) agg4, - | avg( cast(cs_net_profit as decimal(12,2))) agg5, - | avg( cast(c_birth_year as decimal(12,2))) agg6, - | avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 - | from catalog_sales, customer_demographics cd1, - | customer_demographics cd2, customer, customer_address, date_dim, item - | where cs_sold_date_sk = d_date_sk and - | cs_item_sk = i_item_sk and - | cs_bill_cdemo_sk = cd1.cd_demo_sk and - | cs_bill_customer_sk = c_customer_sk and - | cd1.cd_gender = 'F' and - | cd1.cd_education_status = 'Unknown' and - | c_current_cdemo_sk = cd2.cd_demo_sk and - | c_current_addr_sk = ca_address_sk and - | c_birth_month in (1,6,8,9,12,2) and - | d_year = 1998 and - | ca_state in ('MS','IN','ND','OK','NM','VA','MS') - | group by rollup (i_item_id, ca_country, ca_state, ca_county) - | order by ca_country, ca_state, ca_county, i_item_id - | LIMIT 100 - """.stripMargin), - Query( - "q19", - """ - | select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, - | sum(ss_ext_sales_price) ext_price - | from date_dim, store_sales, item,customer,customer_address,store - | where d_date_sk = ss_sold_date_sk - | and ss_item_sk = i_item_sk - | and i_manager_id = 8 - | and d_moy = 11 - | and d_year = 1998 - | and ss_customer_sk = c_customer_sk - | and c_current_addr_sk = ca_address_sk - | and substr(ca_zip,1,5) <> substr(s_zip,1,5) - | and ss_store_sk = s_store_sk - | group by i_brand, i_brand_id, i_manufact_id, i_manufact - | order by ext_price desc, brand, brand_id, i_manufact_id, i_manufact - | limit 100 - """.stripMargin), - Query( - "q20", - """ - |select i_item_desc - | ,i_category - | ,i_class - | ,i_current_price - | ,sum(cs_ext_sales_price) as itemrevenue - | ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over - | (partition by i_class) as revenueratio - | from catalog_sales, item, date_dim - | where cs_item_sk = i_item_sk - | and i_category in ('Sports', 'Books', 'Home') - | and cs_sold_date_sk = d_date_sk - | and d_date between cast('1999-02-22' as date) - | and (cast('1999-02-22' as date) + interval 30 days) - | group by i_item_id, i_item_desc, i_category, i_class, i_current_price - | order by i_category, i_class, i_item_id, i_item_desc, revenueratio - | limit 100 - """.stripMargin), - // Modifications: "+ days" -> date_add - Query( - "q21", - """ - | select * from( - | select w_warehouse_name, i_item_id, - | sum(case when (cast(d_date as date) < cast ('2000-03-11' as date)) - | then inv_quantity_on_hand - | else 0 end) as inv_before, - | sum(case when (cast(d_date as date) >= cast ('2000-03-11' as date)) - | then inv_quantity_on_hand - | else 0 end) as inv_after - | from inventory, warehouse, item, date_dim - | where i_current_price between 0.99 and 1.49 - | and i_item_sk = inv_item_sk - | and inv_warehouse_sk = w_warehouse_sk - | and inv_date_sk = d_date_sk - | and d_date between (cast('2000-03-11' as date) - interval 30 days) - | and (cast('2000-03-11' as date) + interval 30 days) - | group by w_warehouse_name, i_item_id) x - | where (case when inv_before > 0 - | then inv_after / inv_before - | else null - | end) between 2.0/3.0 and 3.0/2.0 - | order by w_warehouse_name, i_item_id - | limit 100 - """.stripMargin), - Query( - "q22", - """ - | select i_product_name, i_brand, i_class, i_category, avg(inv_quantity_on_hand) qoh - | from inventory, date_dim, item, warehouse - | where inv_date_sk=d_date_sk - | and inv_item_sk=i_item_sk - | and inv_warehouse_sk = w_warehouse_sk - | and d_month_seq between 1200 and 1200 + 11 - | group by rollup(i_product_name, i_brand, i_class, i_category) - | order by qoh, i_product_name, i_brand, i_class, i_category - | limit 100 - """.stripMargin), - Query( - "q23a", - """ - | with frequent_ss_items as - | (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt - | from store_sales, date_dim, item - | where ss_sold_date_sk = d_date_sk - | and ss_item_sk = i_item_sk - | and d_year in (2000, 2000+1, 2000+2,2000+3) - | group by substr(i_item_desc,1,30),i_item_sk,d_date - | having count(*) >4), - | max_store_sales as - | (select max(csales) tpcds_cmax - | from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales - | from store_sales, customer, date_dim - | where ss_customer_sk = c_customer_sk - | and ss_sold_date_sk = d_date_sk - | and d_year in (2000, 2000+1, 2000+2,2000+3) - | group by c_customer_sk) x), - | best_ss_customer as - | (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales - | from store_sales, customer - | where ss_customer_sk = c_customer_sk - | group by c_customer_sk - | having sum(ss_quantity*ss_sales_price) > (50/100.0) * - | (select * from max_store_sales)) - | select sum(sales) - | from ((select cs_quantity*cs_list_price sales - | from catalog_sales, date_dim - | where d_year = 2000 - | and d_moy = 2 - | and cs_sold_date_sk = d_date_sk - | and cs_item_sk in (select item_sk from frequent_ss_items) - | and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer)) - | union all - | (select ws_quantity*ws_list_price sales - | from web_sales, date_dim - | where d_year = 2000 - | and d_moy = 2 - | and ws_sold_date_sk = d_date_sk - | and ws_item_sk in (select item_sk from frequent_ss_items) - | and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer))) y - | limit 100 - """.stripMargin), - Query( - "q23b", - """ - | - | with frequent_ss_items as - | (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt - | from store_sales, date_dim, item - | where ss_sold_date_sk = d_date_sk - | and ss_item_sk = i_item_sk - | and d_year in (2000, 2000+1, 2000+2,2000+3) - | group by substr(i_item_desc,1,30),i_item_sk,d_date - | having count(*) > 4), - | max_store_sales as - | (select max(csales) tpcds_cmax - | from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales - | from store_sales, customer, date_dim - | where ss_customer_sk = c_customer_sk - | and ss_sold_date_sk = d_date_sk - | and d_year in (2000, 2000+1, 2000+2,2000+3) - | group by c_customer_sk) x), - | best_ss_customer as - | (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales - | from store_sales - | ,customer - | where ss_customer_sk = c_customer_sk - | group by c_customer_sk - | having sum(ss_quantity*ss_sales_price) > (50/100.0) * - | (select * from max_store_sales)) - | select c_last_name,c_first_name,sales - | from ((select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales - | from catalog_sales, customer, date_dim - | where d_year = 2000 - | and d_moy = 2 - | and cs_sold_date_sk = d_date_sk - | and cs_item_sk in (select item_sk from frequent_ss_items) - | and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) - | and cs_bill_customer_sk = c_customer_sk - | group by c_last_name,c_first_name) - | union all - | (select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales - | from web_sales, customer, date_dim - | where d_year = 2000 - | and d_moy = 2 - | and ws_sold_date_sk = d_date_sk - | and ws_item_sk in (select item_sk from frequent_ss_items) - | and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) - | and ws_bill_customer_sk = c_customer_sk - | group by c_last_name,c_first_name)) y - | order by c_last_name,c_first_name,sales - | limit 100 - """.stripMargin), - Query( - "q24a", - """ - | with ssales as - | (select c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, - | i_current_price, i_manager_id, i_units, i_size, sum(ss_net_paid) netpaid - | from store_sales, store_returns, store, item, customer, customer_address - | where ss_ticket_number = sr_ticket_number - | and ss_item_sk = sr_item_sk - | and ss_customer_sk = c_customer_sk - | and ss_item_sk = i_item_sk - | and ss_store_sk = s_store_sk - | and c_birth_country = upper(ca_country) - | and s_zip = ca_zip - | and s_market_id = 8 - | group by c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, - | i_current_price, i_manager_id, i_units, i_size) - | select c_last_name, c_first_name, s_store_name, sum(netpaid) paid - | from ssales - | where i_color = 'pale' - | group by c_last_name, c_first_name, s_store_name - | having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) - """.stripMargin), - Query( - "q24b", - """ - | with ssales as - | (select c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, - | i_current_price, i_manager_id, i_units, i_size, sum(ss_net_paid) netpaid - | from store_sales, store_returns, store, item, customer, customer_address - | where ss_ticket_number = sr_ticket_number - | and ss_item_sk = sr_item_sk - | and ss_customer_sk = c_customer_sk - | and ss_item_sk = i_item_sk - | and ss_store_sk = s_store_sk - | and c_birth_country = upper(ca_country) - | and s_zip = ca_zip - | and s_market_id = 8 - | group by c_last_name, c_first_name, s_store_name, ca_state, s_state, - | i_color, i_current_price, i_manager_id, i_units, i_size) - | select c_last_name, c_first_name, s_store_name, sum(netpaid) paid - | from ssales - | where i_color = 'chiffon' - | group by c_last_name, c_first_name, s_store_name - | having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) - """.stripMargin), - Query( - "q25", - """ - | select i_item_id, i_item_desc, s_store_id, s_store_name, - | sum(ss_net_profit) as store_sales_profit, - | sum(sr_net_loss) as store_returns_loss, - | sum(cs_net_profit) as catalog_sales_profit - | from - | store_sales, store_returns, catalog_sales, date_dim d1, date_dim d2, date_dim d3, - | store, item - | where - | d1.d_moy = 4 - | and d1.d_year = 2001 - | and d1.d_date_sk = ss_sold_date_sk - | and i_item_sk = ss_item_sk - | and s_store_sk = ss_store_sk - | and ss_customer_sk = sr_customer_sk - | and ss_item_sk = sr_item_sk - | and ss_ticket_number = sr_ticket_number - | and sr_returned_date_sk = d2.d_date_sk - | and d2.d_moy between 4 and 10 - | and d2.d_year = 2001 - | and sr_customer_sk = cs_bill_customer_sk - | and sr_item_sk = cs_item_sk - | and cs_sold_date_sk = d3.d_date_sk - | and d3.d_moy between 4 and 10 - | and d3.d_year = 2001 - | group by - | i_item_id, i_item_desc, s_store_id, s_store_name - | order by - | i_item_id, i_item_desc, s_store_id, s_store_name - | limit 100 - """.stripMargin), - Query( - "q26", - """ - | select i_item_id, - | avg(cs_quantity) agg1, - | avg(cs_list_price) agg2, - | avg(cs_coupon_amt) agg3, - | avg(cs_sales_price) agg4 - | from catalog_sales, customer_demographics, date_dim, item, promotion - | where cs_sold_date_sk = d_date_sk and - | cs_item_sk = i_item_sk and - | cs_bill_cdemo_sk = cd_demo_sk and - | cs_promo_sk = p_promo_sk and - | cd_gender = 'M' and - | cd_marital_status = 'S' and - | cd_education_status = 'College' and - | (p_channel_email = 'N' or p_channel_event = 'N') and - | d_year = 2000 - | group by i_item_id - | order by i_item_id - | limit 100 - """.stripMargin), - Query( - "q27", - """ - | select i_item_id, - | s_state, grouping(s_state) g_state, - | avg(ss_quantity) agg1, - | avg(ss_list_price) agg2, - | avg(ss_coupon_amt) agg3, - | avg(ss_sales_price) agg4 - | from store_sales, customer_demographics, date_dim, store, item - | where ss_sold_date_sk = d_date_sk and - | ss_item_sk = i_item_sk and - | ss_store_sk = s_store_sk and - | ss_cdemo_sk = cd_demo_sk and - | cd_gender = 'M' and - | cd_marital_status = 'S' and - | cd_education_status = 'College' and - | d_year = 2002 and - | s_state in ('TN','TN', 'TN', 'TN', 'TN', 'TN') - | group by rollup (i_item_id, s_state) - | order by i_item_id, s_state - | limit 100 - """.stripMargin), - Query( - "q28", - """ - | select * - | from (select avg(ss_list_price) B1_LP - | ,count(ss_list_price) B1_CNT - | ,count(distinct ss_list_price) B1_CNTD - | from store_sales - | where ss_quantity between 0 and 5 - | and (ss_list_price between 8 and 8+10 - | or ss_coupon_amt between 459 and 459+1000 - | or ss_wholesale_cost between 57 and 57+20)) B1, - | (select avg(ss_list_price) B2_LP - | ,count(ss_list_price) B2_CNT - | ,count(distinct ss_list_price) B2_CNTD - | from store_sales - | where ss_quantity between 6 and 10 - | and (ss_list_price between 90 and 90+10 - | or ss_coupon_amt between 2323 and 2323+1000 - | or ss_wholesale_cost between 31 and 31+20)) B2, - | (select avg(ss_list_price) B3_LP - | ,count(ss_list_price) B3_CNT - | ,count(distinct ss_list_price) B3_CNTD - | from store_sales - | where ss_quantity between 11 and 15 - | and (ss_list_price between 142 and 142+10 - | or ss_coupon_amt between 12214 and 12214+1000 - | or ss_wholesale_cost between 79 and 79+20)) B3, - | (select avg(ss_list_price) B4_LP - | ,count(ss_list_price) B4_CNT - | ,count(distinct ss_list_price) B4_CNTD - | from store_sales - | where ss_quantity between 16 and 20 - | and (ss_list_price between 135 and 135+10 - | or ss_coupon_amt between 6071 and 6071+1000 - | or ss_wholesale_cost between 38 and 38+20)) B4, - | (select avg(ss_list_price) B5_LP - | ,count(ss_list_price) B5_CNT - | ,count(distinct ss_list_price) B5_CNTD - | from store_sales - | where ss_quantity between 21 and 25 - | and (ss_list_price between 122 and 122+10 - | or ss_coupon_amt between 836 and 836+1000 - | or ss_wholesale_cost between 17 and 17+20)) B5, - | (select avg(ss_list_price) B6_LP - | ,count(ss_list_price) B6_CNT - | ,count(distinct ss_list_price) B6_CNTD - | from store_sales - | where ss_quantity between 26 and 30 - | and (ss_list_price between 154 and 154+10 - | or ss_coupon_amt between 7326 and 7326+1000 - | or ss_wholesale_cost between 7 and 7+20)) B6 - | limit 100 - """.stripMargin), - Query( - "q29", - """ - | select - | i_item_id - | ,i_item_desc - | ,s_store_id - | ,s_store_name - | ,sum(ss_quantity) as store_sales_quantity - | ,sum(sr_return_quantity) as store_returns_quantity - | ,sum(cs_quantity) as catalog_sales_quantity - | from - | store_sales, store_returns, catalog_sales, date_dim d1, date_dim d2, - | date_dim d3, store, item - | where - | d1.d_moy = 9 - | and d1.d_year = 1999 - | and d1.d_date_sk = ss_sold_date_sk - | and i_item_sk = ss_item_sk - | and s_store_sk = ss_store_sk - | and ss_customer_sk = sr_customer_sk - | and ss_item_sk = sr_item_sk - | and ss_ticket_number = sr_ticket_number - | and sr_returned_date_sk = d2.d_date_sk - | and d2.d_moy between 9 and 9 + 3 - | and d2.d_year = 1999 - | and sr_customer_sk = cs_bill_customer_sk - | and sr_item_sk = cs_item_sk - | and cs_sold_date_sk = d3.d_date_sk - | and d3.d_year in (1999,1999+1,1999+2) - | group by - | i_item_id, i_item_desc, s_store_id, s_store_name - | order by - | i_item_id, i_item_desc, s_store_id, s_store_name - | limit 100 - """.stripMargin), - Query( - "q30", - """ - | with customer_total_return as - | (select wr_returning_customer_sk as ctr_customer_sk - | ,ca_state as ctr_state, - | sum(wr_return_amt) as ctr_total_return - | from web_returns, date_dim, customer_address - | where wr_returned_date_sk = d_date_sk - | and d_year = 2002 - | and wr_returning_addr_sk = ca_address_sk - | group by wr_returning_customer_sk,ca_state) - | select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag - | ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address - | ,c_last_review_date,ctr_total_return - | from customer_total_return ctr1, customer_address, customer - | where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 - | from customer_total_return ctr2 - | where ctr1.ctr_state = ctr2.ctr_state) - | and ca_address_sk = c_current_addr_sk - | and ca_state = 'GA' - | and ctr1.ctr_customer_sk = c_customer_sk - | order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag - | ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address - | ,c_last_review_date,ctr_total_return - | limit 100 - """.stripMargin), - Query( - "q31", - """ - | with ss as - | (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales - | from store_sales,date_dim,customer_address - | where ss_sold_date_sk = d_date_sk - | and ss_addr_sk=ca_address_sk - | group by ca_county,d_qoy, d_year), - | ws as - | (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales - | from web_sales,date_dim,customer_address - | where ws_sold_date_sk = d_date_sk - | and ws_bill_addr_sk=ca_address_sk - | group by ca_county,d_qoy, d_year) - | select - | ss1.ca_county - | ,ss1.d_year - | ,ws2.web_sales/ws1.web_sales web_q1_q2_increase - | ,ss2.store_sales/ss1.store_sales store_q1_q2_increase - | ,ws3.web_sales/ws2.web_sales web_q2_q3_increase - | ,ss3.store_sales/ss2.store_sales store_q2_q3_increase - | from - | ss ss1, ss ss2, ss ss3, ws ws1, ws ws2, ws ws3 - | where - | ss1.d_qoy = 1 - | and ss1.d_year = 2000 - | and ss1.ca_county = ss2.ca_county - | and ss2.d_qoy = 2 - | and ss2.d_year = 2000 - | and ss2.ca_county = ss3.ca_county - | and ss3.d_qoy = 3 - | and ss3.d_year = 2000 - | and ss1.ca_county = ws1.ca_county - | and ws1.d_qoy = 1 - | and ws1.d_year = 2000 - | and ws1.ca_county = ws2.ca_county - | and ws2.d_qoy = 2 - | and ws2.d_year = 2000 - | and ws1.ca_county = ws3.ca_county - | and ws3.d_qoy = 3 - | and ws3.d_year = 2000 - | and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end - | > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end - | and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end - | > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end - | order by ss1.ca_county - """.stripMargin), - // Modifications: " -> ` - Query( - "q32", - """ - | select sum(cs_ext_discount_amt) as `excess discount amount` - | from - | catalog_sales, item, date_dim - | where - | i_manufact_id = 977 - | and i_item_sk = cs_item_sk - | and d_date between '2000-01-27' and (cast('2000-01-27' as date) + interval 90 days) - | and d_date_sk = cs_sold_date_sk - | and cs_ext_discount_amt > ( - | select 1.3 * avg(cs_ext_discount_amt) - | from catalog_sales, date_dim - | where cs_item_sk = i_item_sk - | and d_date between '2000-01-27]' and (cast('2000-01-27' as date) + interval 90 days) - | and d_date_sk = cs_sold_date_sk) - |limit 100 - """.stripMargin), - Query( - "q33", - """ - | with ss as ( - | select - | i_manufact_id,sum(ss_ext_sales_price) total_sales - | from - | store_sales, date_dim, customer_address, item - | where - | i_manufact_id in (select i_manufact_id - | from item - | where i_category in ('Electronics')) - | and ss_item_sk = i_item_sk - | and ss_sold_date_sk = d_date_sk - | and d_year = 1998 - | and d_moy = 5 - | and ss_addr_sk = ca_address_sk - | and ca_gmt_offset = -5 - | group by i_manufact_id), cs as - | (select i_manufact_id, sum(cs_ext_sales_price) total_sales - | from catalog_sales, date_dim, customer_address, item - | where - | i_manufact_id in ( - | select i_manufact_id from item - | where - | i_category in ('Electronics')) - | and cs_item_sk = i_item_sk - | and cs_sold_date_sk = d_date_sk - | and d_year = 1998 - | and d_moy = 5 - | and cs_bill_addr_sk = ca_address_sk - | and ca_gmt_offset = -5 - | group by i_manufact_id), - | ws as ( - | select i_manufact_id,sum(ws_ext_sales_price) total_sales - | from - | web_sales, date_dim, customer_address, item - | where - | i_manufact_id in (select i_manufact_id from item - | where i_category in ('Electronics')) - | and ws_item_sk = i_item_sk - | and ws_sold_date_sk = d_date_sk - | and d_year = 1998 - | and d_moy = 5 - | and ws_bill_addr_sk = ca_address_sk - | and ca_gmt_offset = -5 - | group by i_manufact_id) - | select i_manufact_id ,sum(total_sales) total_sales - | from (select * from ss - | union all - | select * from cs - | union all - | select * from ws) tmp1 - | group by i_manufact_id - | order by total_sales - |limit 100 - """.stripMargin), - Query( - "q34", - """ - | select c_last_name, c_first_name, c_salutation, c_preferred_cust_flag, ss_ticket_number, - | cnt - | FROM - | (select ss_ticket_number, ss_customer_sk, count(*) cnt - | from store_sales,date_dim,store,household_demographics - | where store_sales.ss_sold_date_sk = date_dim.d_date_sk - | and store_sales.ss_store_sk = store.s_store_sk - | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - | and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) - | and (household_demographics.hd_buy_potential = '>10000' or - | household_demographics.hd_buy_potential = 'unknown') - | and household_demographics.hd_vehicle_count > 0 - | and (case when household_demographics.hd_vehicle_count > 0 - | then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count - | else null - | end) > 1.2 - | and date_dim.d_year in (1999, 1999+1, 1999+2) - | and store.s_county in ('Williamson County','Williamson County','Williamson County','Williamson County', - | 'Williamson County','Williamson County','Williamson County','Williamson County') - | group by ss_ticket_number,ss_customer_sk) dn,customer - | where ss_customer_sk = c_customer_sk - | and cnt between 15 and 20 - | order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc - """.stripMargin), - Query( - "q35", - """ - | select - | ca_state, - | cd_gender, - | cd_marital_status, - | count(*) cnt1, - | min(cd_dep_count), - | max(cd_dep_count), - | avg(cd_dep_count), - | cd_dep_employed_count, - | count(*) cnt2, - | min(cd_dep_employed_count), - | max(cd_dep_employed_count), - | avg(cd_dep_employed_count), - | cd_dep_college_count, - | count(*) cnt3, - | min(cd_dep_college_count), - | max(cd_dep_college_count), - | avg(cd_dep_college_count) - | from - | customer c,customer_address ca,customer_demographics - | where - | c.c_current_addr_sk = ca.ca_address_sk and - | cd_demo_sk = c.c_current_cdemo_sk and - | exists (select * from store_sales, date_dim - | where c.c_customer_sk = ss_customer_sk and - | ss_sold_date_sk = d_date_sk and - | d_year = 2002 and - | d_qoy < 4) and - | (exists (select * from web_sales, date_dim - | where c.c_customer_sk = ws_bill_customer_sk and - | ws_sold_date_sk = d_date_sk and - | d_year = 2002 and - | d_qoy < 4) or - | exists (select * from catalog_sales, date_dim - | where c.c_customer_sk = cs_ship_customer_sk and - | cs_sold_date_sk = d_date_sk and - | d_year = 2002 and - | d_qoy < 4)) - | group by ca_state, cd_gender, cd_marital_status, cd_dep_count, - | cd_dep_employed_count, cd_dep_college_count - | order by ca_state, cd_gender, cd_marital_status, cd_dep_count, - | cd_dep_employed_count, cd_dep_college_count - | limit 100 - """.stripMargin), - Query( - "q36", - """ - | select - | sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin - | ,i_category - | ,i_class - | ,grouping(i_category)+grouping(i_class) as lochierarchy - | ,rank() over ( - | partition by grouping(i_category)+grouping(i_class), - | case when grouping(i_class) = 0 then i_category end - | order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent - | from - | store_sales, date_dim d1, item, store - | where - | d1.d_year = 2001 - | and d1.d_date_sk = ss_sold_date_sk - | and i_item_sk = ss_item_sk - | and s_store_sk = ss_store_sk - | and s_state in ('TN','TN','TN','TN','TN','TN','TN','TN') - | group by rollup(i_category,i_class) - | order by - | lochierarchy desc - | ,case when lochierarchy = 0 then i_category end - | ,rank_within_parent - | limit 100 - """.stripMargin), - // Modifications: "+ days" -> date_add - Query( - "q37", - """ - | select i_item_id, i_item_desc, i_current_price - | from item, inventory, date_dim, catalog_sales - | where i_current_price between 68 and 68 + 30 - | and inv_item_sk = i_item_sk - | and d_date_sk=inv_date_sk - | and d_date between cast('2000-02-01' as date) and (cast('2000-02-01' as date) + interval 60 days) - | and i_manufact_id in (677,940,694,808) - | and inv_quantity_on_hand between 100 and 500 - | and cs_item_sk = i_item_sk - | group by i_item_id,i_item_desc,i_current_price - | order by i_item_id - | limit 100 - """.stripMargin), - Query( - "q38", - """ - | select count(*) from ( - | select distinct c_last_name, c_first_name, d_date - | from store_sales, date_dim, customer - | where store_sales.ss_sold_date_sk = date_dim.d_date_sk - | and store_sales.ss_customer_sk = customer.c_customer_sk - | and d_month_seq between 1200 and 1200 + 11 - | intersect - | select distinct c_last_name, c_first_name, d_date - | from catalog_sales, date_dim, customer - | where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk - | and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk - | and d_month_seq between 1200 and 1200 + 11 - | intersect - | select distinct c_last_name, c_first_name, d_date - | from web_sales, date_dim, customer - | where web_sales.ws_sold_date_sk = date_dim.d_date_sk - | and web_sales.ws_bill_customer_sk = customer.c_customer_sk - | and d_month_seq between 1200 and 1200 + 11 - | ) hot_cust - | limit 100 - """.stripMargin), - Query( - "q39a", - """ - | with inv as - | (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy - | ,stdev,mean, case mean when 0 then null else stdev/mean end cov - | from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy - | ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean - | from inventory, item, warehouse, date_dim - | where inv_item_sk = i_item_sk - | and inv_warehouse_sk = w_warehouse_sk - | and inv_date_sk = d_date_sk - | and d_year = 2001 - | group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo - | where case mean when 0 then 0 else stdev/mean end > 1) - | select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov - | ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov - | from inv inv1,inv inv2 - | where inv1.i_item_sk = inv2.i_item_sk - | and inv1.w_warehouse_sk = inv2.w_warehouse_sk - | and inv1.d_moy=1 - | and inv2.d_moy=1+1 - | order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov - | ,inv2.d_moy,inv2.mean, inv2.cov - """.stripMargin), - Query( - "q39b", - """ - | with inv as - | (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy - | ,stdev,mean, case mean when 0 then null else stdev/mean end cov - | from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy - | ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean - | from inventory, item, warehouse, date_dim - | where inv_item_sk = i_item_sk - | and inv_warehouse_sk = w_warehouse_sk - | and inv_date_sk = d_date_sk - | and d_year = 2001 - | group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo - | where case mean when 0 then 0 else stdev/mean end > 1) - | select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov - | ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov - | from inv inv1,inv inv2 - | where inv1.i_item_sk = inv2.i_item_sk - | and inv1.w_warehouse_sk = inv2.w_warehouse_sk - | and inv1.d_moy=1 - | and inv2.d_moy=1+1 - | and inv1.cov > 1.5 - | order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov - | ,inv2.d_moy,inv2.mean, inv2.cov - """.stripMargin), - // Modifications: "+ days" -> date_add - Query( - "q40", - """ - | select - | w_state - | ,i_item_id - | ,sum(case when (cast(d_date as date) < cast('2000-03-11' as date)) - | then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before - | ,sum(case when (cast(d_date as date) >= cast('2000-03-11' as date)) - | then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after - | from - | catalog_sales left outer join catalog_returns on - | (cs_order_number = cr_order_number - | and cs_item_sk = cr_item_sk) - | ,warehouse, item, date_dim - | where - | i_current_price between 0.99 and 1.49 - | and i_item_sk = cs_item_sk - | and cs_warehouse_sk = w_warehouse_sk - | and cs_sold_date_sk = d_date_sk - | and d_date between (cast('2000-03-11' as date) - interval 30 days) - | and (cast('2000-03-11' as date) + interval 30 days) - | group by w_state,i_item_id - | order by w_state,i_item_id - | limit 100 - """.stripMargin), - Query( - "q41", - """ - | select distinct(i_product_name) - | from item i1 - | where i_manufact_id between 738 and 738+40 - | and (select count(*) as item_cnt - | from item - | where (i_manufact = i1.i_manufact and - | ((i_category = 'Women' and - | (i_color = 'powder' or i_color = 'khaki') and - | (i_units = 'Ounce' or i_units = 'Oz') and - | (i_size = 'medium' or i_size = 'extra large') - | ) or - | (i_category = 'Women' and - | (i_color = 'brown' or i_color = 'honeydew') and - | (i_units = 'Bunch' or i_units = 'Ton') and - | (i_size = 'N/A' or i_size = 'small') - | ) or - | (i_category = 'Men' and - | (i_color = 'floral' or i_color = 'deep') and - | (i_units = 'N/A' or i_units = 'Dozen') and - | (i_size = 'petite' or i_size = 'large') - | ) or - | (i_category = 'Men' and - | (i_color = 'light' or i_color = 'cornflower') and - | (i_units = 'Box' or i_units = 'Pound') and - | (i_size = 'medium' or i_size = 'extra large') - | ))) or - | (i_manufact = i1.i_manufact and - | ((i_category = 'Women' and - | (i_color = 'midnight' or i_color = 'snow') and - | (i_units = 'Pallet' or i_units = 'Gross') and - | (i_size = 'medium' or i_size = 'extra large') - | ) or - | (i_category = 'Women' and - | (i_color = 'cyan' or i_color = 'papaya') and - | (i_units = 'Cup' or i_units = 'Dram') and - | (i_size = 'N/A' or i_size = 'small') - | ) or - | (i_category = 'Men' and - | (i_color = 'orange' or i_color = 'frosted') and - | (i_units = 'Each' or i_units = 'Tbl') and - | (i_size = 'petite' or i_size = 'large') - | ) or - | (i_category = 'Men' and - | (i_color = 'forest' or i_color = 'ghost') and - | (i_units = 'Lb' or i_units = 'Bundle') and - | (i_size = 'medium' or i_size = 'extra large') - | )))) > 0 - | order by i_product_name - | limit 100 - """.stripMargin), - Query( - "q42", - """ - | select dt.d_year, item.i_category_id, item.i_category, sum(ss_ext_sales_price) - | from date_dim dt, store_sales, item - | where dt.d_date_sk = store_sales.ss_sold_date_sk - | and store_sales.ss_item_sk = item.i_item_sk - | and item.i_manager_id = 1 - | and dt.d_moy=11 - | and dt.d_year=2000 - | group by dt.d_year - | ,item.i_category_id - | ,item.i_category - | order by sum(ss_ext_sales_price) desc,dt.d_year - | ,item.i_category_id - | ,item.i_category - | limit 100 - """.stripMargin), - Query( - "q43", - """ - | select s_store_name, s_store_id, - | sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, - | sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, - | sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, - | sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, - | sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, - | sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, - | sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales - | from date_dim, store_sales, store - | where d_date_sk = ss_sold_date_sk and - | s_store_sk = ss_store_sk and - | s_gmt_offset = -5 and - | d_year = 2000 - | group by s_store_name, s_store_id - | order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales, - | thu_sales,fri_sales,sat_sales - | limit 100 - """.stripMargin), - Query( - "q44", - """ - | select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing - | from(select * - | from (select item_sk,rank() over (order by rank_col asc) rnk - | from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col - | from store_sales ss1 - | where ss_store_sk = 4 - | group by ss_item_sk - | having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col - | from store_sales - | where ss_store_sk = 4 - | and ss_addr_sk is null - | group by ss_store_sk))V1)V11 - | where rnk < 11) asceding, - | (select * - | from (select item_sk,rank() over (order by rank_col desc) rnk - | from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col - | from store_sales ss1 - | where ss_store_sk = 4 - | group by ss_item_sk - | having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col - | from store_sales - | where ss_store_sk = 4 - | and ss_addr_sk is null - | group by ss_store_sk))V2)V21 - | where rnk < 11) descending, - | item i1, item i2 - | where asceding.rnk = descending.rnk - | and i1.i_item_sk=asceding.item_sk - | and i2.i_item_sk=descending.item_sk - | order by asceding.rnk - | limit 100 - """.stripMargin), - Query( - "q45", - """ - | select ca_zip, ca_city, sum(ws_sales_price) - | from web_sales, customer, customer_address, date_dim, item - | where ws_bill_customer_sk = c_customer_sk - | and c_current_addr_sk = ca_address_sk - | and ws_item_sk = i_item_sk - | and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') - | or - | i_item_id in (select i_item_id - | from item - | where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) - | ) - | ) - | and ws_sold_date_sk = d_date_sk - | and d_qoy = 2 and d_year = 2001 - | group by ca_zip, ca_city - | order by ca_zip, ca_city - | limit 100 - """.stripMargin), - Query( - "q46", - """ - | select c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number, amt,profit - | from - | (select ss_ticket_number - | ,ss_customer_sk - | ,ca_city bought_city - | ,sum(ss_coupon_amt) amt - | ,sum(ss_net_profit) profit - | from store_sales, date_dim, store, household_demographics, customer_address - | where store_sales.ss_sold_date_sk = date_dim.d_date_sk - | and store_sales.ss_store_sk = store.s_store_sk - | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - | and store_sales.ss_addr_sk = customer_address.ca_address_sk - | and (household_demographics.hd_dep_count = 4 or - | household_demographics.hd_vehicle_count= 3) - | and date_dim.d_dow in (6,0) - | and date_dim.d_year in (1999,1999+1,1999+2) - | and store.s_city in ('Fairview','Midway','Fairview','Fairview','Fairview') - | group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr - | where ss_customer_sk = c_customer_sk - | and customer.c_current_addr_sk = current_addr.ca_address_sk - | and current_addr.ca_city <> bought_city - | order by c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number - | limit 100 - """.stripMargin), - Query( - "q47", - """ - | with v1 as( - | select i_category, i_brand, - | s_store_name, s_company_name, - | d_year, d_moy, - | sum(ss_sales_price) sum_sales, - | avg(sum(ss_sales_price)) over - | (partition by i_category, i_brand, - | s_store_name, s_company_name, d_year) - | avg_monthly_sales, - | rank() over - | (partition by i_category, i_brand, - | s_store_name, s_company_name - | order by d_year, d_moy) rn - | from item, store_sales, date_dim, store - | where ss_item_sk = i_item_sk and - | ss_sold_date_sk = d_date_sk and - | ss_store_sk = s_store_sk and - | ( - | d_year = 1999 or - | ( d_year = 1999-1 and d_moy =12) or - | ( d_year = 1999+1 and d_moy =1) - | ) - | group by i_category, i_brand, - | s_store_name, s_company_name, - | d_year, d_moy), - | v2 as( - | select v1.i_category, v1.i_brand, v1.s_store_name, v1.s_company_name, v1.d_year, - v1.d_moy, v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, - v1_lead.sum_sales nsum - | from v1, v1 v1_lag, v1 v1_lead - | where v1.i_category = v1_lag.i_category and - | v1.i_category = v1_lead.i_category and - | v1.i_brand = v1_lag.i_brand and - | v1.i_brand = v1_lead.i_brand and - | v1.s_store_name = v1_lag.s_store_name and - | v1.s_store_name = v1_lead.s_store_name and - | v1.s_company_name = v1_lag.s_company_name and - | v1.s_company_name = v1_lead.s_company_name and - | v1.rn = v1_lag.rn + 1 and - | v1.rn = v1_lead.rn - 1) - | select * from v2 - | where d_year = 1999 and - | avg_monthly_sales > 0 and - | case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 - | order by sum_sales - avg_monthly_sales, 3 - | limit 100 - """.stripMargin), - Query( - "q48", - """ - | select sum (ss_quantity) - | from store_sales, store, customer_demographics, customer_address, date_dim - | where s_store_sk = ss_store_sk - | and ss_sold_date_sk = d_date_sk and d_year = 2001 - | and - | ( - | ( - | cd_demo_sk = ss_cdemo_sk - | and - | cd_marital_status = 'M' - | and - | cd_education_status = '4 yr Degree' - | and - | ss_sales_price between 100.00 and 150.00 - | ) - | or - | ( - | cd_demo_sk = ss_cdemo_sk - | and - | cd_marital_status = 'D' - | and - | cd_education_status = '2 yr Degree' - | and - | ss_sales_price between 50.00 and 100.00 - | ) - | or - | ( - | cd_demo_sk = ss_cdemo_sk - | and - | cd_marital_status = 'S' - | and - | cd_education_status = 'College' - | and - | ss_sales_price between 150.00 and 200.00 - | ) - | ) - | and - | ( - | ( - | ss_addr_sk = ca_address_sk - | and - | ca_country = 'United States' - | and - | ca_state in ('CO', 'OH', 'TX') - | and ss_net_profit between 0 and 2000 - | ) - | or - | (ss_addr_sk = ca_address_sk - | and - | ca_country = 'United States' - | and - | ca_state in ('OR', 'MN', 'KY') - | and ss_net_profit between 150 and 3000 - | ) - | or - | (ss_addr_sk = ca_address_sk - | and - | ca_country = 'United States' - | and - | ca_state in ('VA', 'CA', 'MS') - | and ss_net_profit between 50 and 25000 - | ) - | ) - """.stripMargin), - // Modifications: "dec" -> "decimal" - Query( - "q49", - """ - | select 'web' as channel, web.item, web.return_ratio, web.return_rank, web.currency_rank - | from ( - | select - | item, return_ratio, currency_ratio, - | rank() over (order by return_ratio) as return_rank, - | rank() over (order by currency_ratio) as currency_rank - | from - | ( select ws.ws_item_sk as item - | ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ - | cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio - | ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ - | cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio - | from - | web_sales ws left outer join web_returns wr - | on (ws.ws_order_number = wr.wr_order_number and - | ws.ws_item_sk = wr.wr_item_sk) - | ,date_dim - | where - | wr.wr_return_amt > 10000 - | and ws.ws_net_profit > 1 - | and ws.ws_net_paid > 0 - | and ws.ws_quantity > 0 - | and ws_sold_date_sk = d_date_sk - | and d_year = 2001 - | and d_moy = 12 - | group by ws.ws_item_sk - | ) in_web - | ) web - | where (web.return_rank <= 10 or web.currency_rank <= 10) - | union - | select - | 'catalog' as channel, catalog.item, catalog.return_ratio, - | catalog.return_rank, catalog.currency_rank - | from ( - | select - | item, return_ratio, currency_ratio, - | rank() over (order by return_ratio) as return_rank, - | rank() over (order by currency_ratio) as currency_rank - | from - | ( select - | cs.cs_item_sk as item - | ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ - | cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio - | ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ - | cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio - | from - | catalog_sales cs left outer join catalog_returns cr - | on (cs.cs_order_number = cr.cr_order_number and - | cs.cs_item_sk = cr.cr_item_sk) - | ,date_dim - | where - | cr.cr_return_amount > 10000 - | and cs.cs_net_profit > 1 - | and cs.cs_net_paid > 0 - | and cs.cs_quantity > 0 - | and cs_sold_date_sk = d_date_sk - | and d_year = 2001 - | and d_moy = 12 - | group by cs.cs_item_sk - | ) in_cat - | ) catalog - | where (catalog.return_rank <= 10 or catalog.currency_rank <=10) - | union - | select - | 'store' as channel, store.item, store.return_ratio, - | store.return_rank, store.currency_rank - | from ( - | select - | item, return_ratio, currency_ratio, - | rank() over (order by return_ratio) as return_rank, - | rank() over (order by currency_ratio) as currency_rank - | from - | ( select sts.ss_item_sk as item - | ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/ - | cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio - | ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/ - | cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio - | from - | store_sales sts left outer join store_returns sr - | on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) - | ,date_dim - | where - | sr.sr_return_amt > 10000 - | and sts.ss_net_profit > 1 - | and sts.ss_net_paid > 0 - | and sts.ss_quantity > 0 - | and ss_sold_date_sk = d_date_sk - | and d_year = 2001 - | and d_moy = 12 - | group by sts.ss_item_sk - | ) in_store - | ) store - | where (store.return_rank <= 10 or store.currency_rank <= 10) - | order by 1,4,5 - | limit 100 - """.stripMargin), - // Modifications: " -> ` - Query( - "q50", - """ - | select - | s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, - | s_suite_number, s_city, s_county, s_state, s_zip - | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` - | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and - | (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` - | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and - | (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` - | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and - | (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` - | ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` - | from - | store_sales, store_returns, store, date_dim d1, date_dim d2 - | where - | d2.d_year = 2001 - | and d2.d_moy = 8 - | and ss_ticket_number = sr_ticket_number - | and ss_item_sk = sr_item_sk - | and ss_sold_date_sk = d1.d_date_sk - | and sr_returned_date_sk = d2.d_date_sk - | and ss_customer_sk = sr_customer_sk - | and ss_store_sk = s_store_sk - | group by - | s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, - | s_suite_number, s_city, s_county, s_state, s_zip - | order by - | s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, - | s_suite_number, s_city, s_county, s_state, s_zip - | limit 100 - """.stripMargin), - Query( - "q51", - """ - | WITH web_v1 as ( - | select - | ws_item_sk item_sk, d_date, - | sum(sum(ws_sales_price)) - | over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales - | from web_sales, date_dim - | where ws_sold_date_sk=d_date_sk - | and d_month_seq between 1200 and 1200+11 - | and ws_item_sk is not NULL - | group by ws_item_sk, d_date), - | store_v1 as ( - | select - | ss_item_sk item_sk, d_date, - | sum(sum(ss_sales_price)) - | over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales - | from store_sales, date_dim - | where ss_sold_date_sk=d_date_sk - | and d_month_seq between 1200 and 1200+11 - | and ss_item_sk is not NULL - | group by ss_item_sk, d_date) - | select * - | from (select item_sk, d_date, web_sales, store_sales - | ,max(web_sales) - | over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative - | ,max(store_sales) - | over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative - | from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk - | ,case when web.d_date is not null then web.d_date else store.d_date end d_date - | ,web.cume_sales web_sales - | ,store.cume_sales store_sales - | from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk - | and web.d_date = store.d_date) - | )x )y - | where web_cumulative > store_cumulative - | order by item_sk, d_date - | limit 100 - """.stripMargin), - Query( - "q52", - """ - | select dt.d_year - | ,item.i_brand_id brand_id - | ,item.i_brand brand - | ,sum(ss_ext_sales_price) ext_price - | from date_dim dt, store_sales, item - | where dt.d_date_sk = store_sales.ss_sold_date_sk - | and store_sales.ss_item_sk = item.i_item_sk - | and item.i_manager_id = 1 - | and dt.d_moy=11 - | and dt.d_year=2000 - | group by dt.d_year, item.i_brand, item.i_brand_id - | order by dt.d_year, ext_price desc, brand_id - |limit 100 - """.stripMargin), - Query( - "q53", - """ - | select * from - | (select i_manufact_id, - | sum(ss_sales_price) sum_sales, - | avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales - | from item, store_sales, date_dim, store - | where ss_item_sk = i_item_sk and - | ss_sold_date_sk = d_date_sk and - | ss_store_sk = s_store_sk and - | d_month_seq in (1200,1200+1,1200+2,1200+3,1200+4,1200+5,1200+6, - | 1200+7,1200+8,1200+9,1200+10,1200+11) and - | ((i_category in ('Books','Children','Electronics') and - | i_class in ('personal','portable','reference','self-help') and - | i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', - | 'exportiunivamalg #9','scholaramalgamalg #9')) - | or - | (i_category in ('Women','Music','Men') and - | i_class in ('accessories','classical','fragrances','pants') and - | i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', - | 'importoamalg #1'))) - | group by i_manufact_id, d_qoy ) tmp1 - | where case when avg_quarterly_sales > 0 - | then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales - | else null end > 0.1 - | order by avg_quarterly_sales, - | sum_sales, - | i_manufact_id - | limit 100 - """.stripMargin), - Query( - "q54", - """ - | with my_customers as ( - | select distinct c_customer_sk - | , c_current_addr_sk - | from - | ( select cs_sold_date_sk sold_date_sk, - | cs_bill_customer_sk customer_sk, - | cs_item_sk item_sk - | from catalog_sales - | union all - | select ws_sold_date_sk sold_date_sk, - | ws_bill_customer_sk customer_sk, - | ws_item_sk item_sk - | from web_sales - | ) cs_or_ws_sales, - | item, - | date_dim, - | customer - | where sold_date_sk = d_date_sk - | and item_sk = i_item_sk - | and i_category = 'Women' - | and i_class = 'maternity' - | and c_customer_sk = cs_or_ws_sales.customer_sk - | and d_moy = 12 - | and d_year = 1998 - | ) - | , my_revenue as ( - | select c_customer_sk, - | sum(ss_ext_sales_price) as revenue - | from my_customers, - | store_sales, - | customer_address, - | store, - | date_dim - | where c_current_addr_sk = ca_address_sk - | and ca_county = s_county - | and ca_state = s_state - | and ss_sold_date_sk = d_date_sk - | and c_customer_sk = ss_customer_sk - | and d_month_seq between (select distinct d_month_seq+1 - | from date_dim where d_year = 1998 and d_moy = 12) - | and (select distinct d_month_seq+3 - | from date_dim where d_year = 1998 and d_moy = 12) - | group by c_customer_sk - | ) - | , segments as - | (select cast((revenue/50) as int) as segment from my_revenue) - | select segment, count(*) as num_customers, segment*50 as segment_base - | from segments - | group by segment - | order by segment, num_customers - | limit 100 - """.stripMargin), - Query( - "q55", - """ - |select i_brand_id brand_id, i_brand brand, - | sum(ss_ext_sales_price) ext_price - | from date_dim, store_sales, item - | where d_date_sk = ss_sold_date_sk - | and ss_item_sk = i_item_sk - | and i_manager_id=28 - | and d_moy=11 - | and d_year=1999 - | group by i_brand, i_brand_id - | order by ext_price desc, brand_id - | limit 100 - """.stripMargin), - Query( - "q56", - """ - | with ss as ( - | select i_item_id,sum(ss_ext_sales_price) total_sales - | from - | store_sales, date_dim, customer_address, item - | where - | i_item_id in (select i_item_id from item where i_color in ('slate','blanched','burnished')) - | and ss_item_sk = i_item_sk - | and ss_sold_date_sk = d_date_sk - | and d_year = 2001 - | and d_moy = 2 - | and ss_addr_sk = ca_address_sk - | and ca_gmt_offset = -5 - | group by i_item_id), - | cs as ( - | select i_item_id,sum(cs_ext_sales_price) total_sales - | from - | catalog_sales, date_dim, customer_address, item - | where - | i_item_id in (select i_item_id from item where i_color in ('slate','blanched','burnished')) - | and cs_item_sk = i_item_sk - | and cs_sold_date_sk = d_date_sk - | and d_year = 2001 - | and d_moy = 2 - | and cs_bill_addr_sk = ca_address_sk - | and ca_gmt_offset = -5 - | group by i_item_id), - | ws as ( - | select i_item_id,sum(ws_ext_sales_price) total_sales - | from - | web_sales, date_dim, customer_address, item - | where - | i_item_id in (select i_item_id from item where i_color in ('slate','blanched','burnished')) - | and ws_item_sk = i_item_sk - | and ws_sold_date_sk = d_date_sk - | and d_year = 2001 - | and d_moy = 2 - | and ws_bill_addr_sk = ca_address_sk - | and ca_gmt_offset = -5 - | group by i_item_id) - | select i_item_id ,sum(total_sales) total_sales - | from (select * from ss - | union all - | select * from cs - | union all - | select * from ws) tmp1 - | group by i_item_id - | order by total_sales - | limit 100 - """.stripMargin), - Query( - "q57", - """ - | with v1 as( - | select i_category, i_brand, - | cc_name, - | d_year, d_moy, - | sum(cs_sales_price) sum_sales, - | avg(sum(cs_sales_price)) over - | (partition by i_category, i_brand, cc_name, d_year) - | avg_monthly_sales, - | rank() over - | (partition by i_category, i_brand, cc_name - | order by d_year, d_moy) rn - | from item, catalog_sales, date_dim, call_center - | where cs_item_sk = i_item_sk and - | cs_sold_date_sk = d_date_sk and - | cc_call_center_sk= cs_call_center_sk and - | ( - | d_year = 1999 or - | ( d_year = 1999-1 and d_moy =12) or - | ( d_year = 1999+1 and d_moy =1) - | ) - | group by i_category, i_brand, - | cc_name , d_year, d_moy), - | v2 as( - | select v1.i_category, v1.i_brand, v1.cc_name, v1.d_year, v1.d_moy - | ,v1.avg_monthly_sales - | ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum - | from v1, v1 v1_lag, v1 v1_lead - | where v1.i_category = v1_lag.i_category and - | v1.i_category = v1_lead.i_category and - | v1.i_brand = v1_lag.i_brand and - | v1.i_brand = v1_lead.i_brand and - | v1. cc_name = v1_lag. cc_name and - | v1. cc_name = v1_lead. cc_name and - | v1.rn = v1_lag.rn + 1 and - | v1.rn = v1_lead.rn - 1) - | select * from v2 - | where d_year = 1999 and - | avg_monthly_sales > 0 and - | case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 - | order by sum_sales - avg_monthly_sales, 3 - | limit 100 - """.stripMargin), - Query( - "q58", - """ - | with ss_items as - | (select i_item_id item_id, sum(ss_ext_sales_price) ss_item_rev - | from store_sales, item, date_dim - | where ss_item_sk = i_item_sk - | and d_date in (select d_date - | from date_dim - | where d_week_seq = (select d_week_seq - | from date_dim - | where d_date = '2000-01-03')) - | and ss_sold_date_sk = d_date_sk - | group by i_item_id), - | cs_items as - | (select i_item_id item_id - | ,sum(cs_ext_sales_price) cs_item_rev - | from catalog_sales, item, date_dim - | where cs_item_sk = i_item_sk - | and d_date in (select d_date - | from date_dim - | where d_week_seq = (select d_week_seq - | from date_dim - | where d_date = '2000-01-03')) - | and cs_sold_date_sk = d_date_sk - | group by i_item_id), - | ws_items as - | (select i_item_id item_id, sum(ws_ext_sales_price) ws_item_rev - | from web_sales, item, date_dim - | where ws_item_sk = i_item_sk - | and d_date in (select d_date - | from date_dim - | where d_week_seq =(select d_week_seq - | from date_dim - | where d_date = '2000-01-03')) - | and ws_sold_date_sk = d_date_sk - | group by i_item_id) - | select ss_items.item_id - | ,ss_item_rev - | ,ss_item_rev/(ss_item_rev+cs_item_rev+ws_item_rev)/3 * 100 ss_dev - | ,cs_item_rev - | ,cs_item_rev/(ss_item_rev+cs_item_rev+ws_item_rev)/3 * 100 cs_dev - | ,ws_item_rev - | ,ws_item_rev/(ss_item_rev+cs_item_rev+ws_item_rev)/3 * 100 ws_dev - | ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average - | from ss_items,cs_items,ws_items - | where ss_items.item_id=cs_items.item_id - | and ss_items.item_id=ws_items.item_id - | and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev - | and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev - | and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev - | and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev - | and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev - | and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev - | order by item_id, ss_item_rev - | limit 100 - """.stripMargin), - Query( - "q59", - """ - | with wss as - | (select d_week_seq, - | ss_store_sk, - | sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, - | sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, - | sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, - | sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, - | sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, - | sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, - | sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales - | from store_sales,date_dim - | where d_date_sk = ss_sold_date_sk - | group by d_week_seq,ss_store_sk - | ) - | select s_store_name1,s_store_id1,d_week_seq1 - | ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 - | ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 - | ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 - | from - | (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 - | ,s_store_id s_store_id1,sun_sales sun_sales1 - | ,mon_sales mon_sales1,tue_sales tue_sales1 - | ,wed_sales wed_sales1,thu_sales thu_sales1 - | ,fri_sales fri_sales1,sat_sales sat_sales1 - | from wss,store,date_dim d - | where d.d_week_seq = wss.d_week_seq and - | ss_store_sk = s_store_sk and - | d_month_seq between 1212 and 1212 + 11) y, - | (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 - | ,s_store_id s_store_id2,sun_sales sun_sales2 - | ,mon_sales mon_sales2,tue_sales tue_sales2 - | ,wed_sales wed_sales2,thu_sales thu_sales2 - | ,fri_sales fri_sales2,sat_sales sat_sales2 - | from wss,store,date_dim d - | where d.d_week_seq = wss.d_week_seq and - | ss_store_sk = s_store_sk and - | d_month_seq between 1212+ 12 and 1212 + 23) x - | where s_store_id1=s_store_id2 - | and d_week_seq1=d_week_seq2-52 - | order by s_store_name1,s_store_id1,d_week_seq1 - | limit 100 - """.stripMargin), - Query( - "q60", - """ - | with ss as ( - | select i_item_id,sum(ss_ext_sales_price) total_sales - | from store_sales, date_dim, customer_address, item - | where - | i_item_id in (select i_item_id from item where i_category in ('Music')) - | and ss_item_sk = i_item_sk - | and ss_sold_date_sk = d_date_sk - | and d_year = 1998 - | and d_moy = 9 - | and ss_addr_sk = ca_address_sk - | and ca_gmt_offset = -5 - | group by i_item_id), - | cs as ( - | select i_item_id,sum(cs_ext_sales_price) total_sales - | from catalog_sales, date_dim, customer_address, item - | where - | i_item_id in (select i_item_id from item where i_category in ('Music')) - | and cs_item_sk = i_item_sk - | and cs_sold_date_sk = d_date_sk - | and d_year = 1998 - | and d_moy = 9 - | and cs_bill_addr_sk = ca_address_sk - | and ca_gmt_offset = -5 - | group by i_item_id), - | ws as ( - | select i_item_id,sum(ws_ext_sales_price) total_sales - | from web_sales, date_dim, customer_address, item - | where - | i_item_id in (select i_item_id from item where i_category in ('Music')) - | and ws_item_sk = i_item_sk - | and ws_sold_date_sk = d_date_sk - | and d_year = 1998 - | and d_moy = 9 - | and ws_bill_addr_sk = ca_address_sk - | and ca_gmt_offset = -5 - | group by i_item_id) - | select i_item_id, sum(total_sales) total_sales - | from (select * from ss - | union all - | select * from cs - | union all - | select * from ws) tmp1 - | group by i_item_id - | order by i_item_id, total_sales - | limit 100 - """.stripMargin), - Query( - "q61", - s""" - | select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 - | from - | (select sum(ss_ext_sales_price) promotions - | from store_sales, store, promotion, date_dim, customer, customer_address, item - | where ss_sold_date_sk = d_date_sk - | and ss_store_sk = s_store_sk - | and ss_promo_sk = p_promo_sk - | and ss_customer_sk= c_customer_sk - | and ca_address_sk = c_current_addr_sk - | and ss_item_sk = i_item_sk - | and ca_gmt_offset = -5 - | and i_category = 'Jewelry' - | and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') - | and s_gmt_offset = -5 - | and d_year = 1998 - | and d_moy = 11) promotional_sales, - | (select sum(ss_ext_sales_price) total - | from store_sales, store, date_dim, customer, customer_address, item - | where ss_sold_date_sk = d_date_sk - | and ss_store_sk = s_store_sk - | and ss_customer_sk= c_customer_sk - | and ca_address_sk = c_current_addr_sk - | and ss_item_sk = i_item_sk - | and ca_gmt_offset = -5 - | and i_category = 'Jewelry' - | and s_gmt_offset = -5 - | and d_year = 1998 - | and d_moy = 11) all_sales - | order by promotions, total - | limit 100 - """.stripMargin), - // Modifications: " -> ` - Query( - "q62", - """ - | select - | substr(w_warehouse_name,1,20) - | ,sm_type - | ,web_name - | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` - | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and - | (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` - | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and - | (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` - | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and - | (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` - | ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` - | from - | web_sales, warehouse, ship_mode, web_site, date_dim - | where - | d_month_seq between 1200 and 1200 + 11 - | and ws_ship_date_sk = d_date_sk - | and ws_warehouse_sk = w_warehouse_sk - | and ws_ship_mode_sk = sm_ship_mode_sk - | and ws_web_site_sk = web_site_sk - | group by - | substr(w_warehouse_name,1,20), sm_type, web_name - | order by - | substr(w_warehouse_name,1,20), sm_type, web_name - | limit 100 - """.stripMargin), - Query( - "q63", - """ - | select * - | from (select i_manager_id - | ,sum(ss_sales_price) sum_sales - | ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales - | from item - | ,store_sales - | ,date_dim - | ,store - | where ss_item_sk = i_item_sk - | and ss_sold_date_sk = d_date_sk - | and ss_store_sk = s_store_sk - | and d_month_seq in (1200,1200+1,1200+2,1200+3,1200+4,1200+5,1200+6,1200+7, - | 1200+8,1200+9,1200+10,1200+11) - | and (( i_category in ('Books','Children','Electronics') - | and i_class in ('personal','portable','refernece','self-help') - | and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', - | 'exportiunivamalg #9','scholaramalgamalg #9')) - | or( i_category in ('Women','Music','Men') - | and i_class in ('accessories','classical','fragrances','pants') - | and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', - | 'importoamalg #1'))) - | group by i_manager_id, d_moy) tmp1 - | where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 - | order by i_manager_id - | ,avg_monthly_sales - | ,sum_sales - | limit 100 - """.stripMargin), - Query( - "q64", - """ - | with cs_ui as - | (select cs_item_sk - | ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund - | from catalog_sales - | ,catalog_returns - | where cs_item_sk = cr_item_sk - | and cs_order_number = cr_order_number - | group by cs_item_sk - | having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), - | cross_sales as - | (select i_product_name product_name, i_item_sk item_sk, s_store_name store_name, s_zip store_zip, - | ad1.ca_street_number b_street_number, ad1.ca_street_name b_streen_name, ad1.ca_city b_city, - | ad1.ca_zip b_zip, ad2.ca_street_number c_street_number, ad2.ca_street_name c_street_name, - | ad2.ca_city c_city, ad2.ca_zip c_zip, d1.d_year as syear, d2.d_year as fsyear, d3.d_year s2year, - | count(*) cnt, sum(ss_wholesale_cost) s1, sum(ss_list_price) s2, sum(ss_coupon_amt) s3 - | FROM store_sales, store_returns, cs_ui, date_dim d1, date_dim d2, date_dim d3, - | store, customer, customer_demographics cd1, customer_demographics cd2, - | promotion, household_demographics hd1, household_demographics hd2, - | customer_address ad1, customer_address ad2, income_band ib1, income_band ib2, item - | WHERE ss_store_sk = s_store_sk AND - | ss_sold_date_sk = d1.d_date_sk AND - | ss_customer_sk = c_customer_sk AND - | ss_cdemo_sk= cd1.cd_demo_sk AND - | ss_hdemo_sk = hd1.hd_demo_sk AND - | ss_addr_sk = ad1.ca_address_sk and - | ss_item_sk = i_item_sk and - | ss_item_sk = sr_item_sk and - | ss_ticket_number = sr_ticket_number and - | ss_item_sk = cs_ui.cs_item_sk and - | c_current_cdemo_sk = cd2.cd_demo_sk AND - | c_current_hdemo_sk = hd2.hd_demo_sk AND - | c_current_addr_sk = ad2.ca_address_sk and - | c_first_sales_date_sk = d2.d_date_sk and - | c_first_shipto_date_sk = d3.d_date_sk and - | ss_promo_sk = p_promo_sk and - | hd1.hd_income_band_sk = ib1.ib_income_band_sk and - | hd2.hd_income_band_sk = ib2.ib_income_band_sk and - | cd1.cd_marital_status <> cd2.cd_marital_status and - | i_color in ('purple','burlywood','indian','spring','floral','medium') and - | i_current_price between 64 and 64 + 10 and - | i_current_price between 64 + 1 and 64 + 15 - | group by i_product_name, i_item_sk, s_store_name, s_zip, ad1.ca_street_number, - | ad1.ca_street_name, ad1.ca_city, ad1.ca_zip, ad2.ca_street_number, - | ad2.ca_street_name, ad2.ca_city, ad2.ca_zip, d1.d_year, d2.d_year, d3.d_year - | ) - | select cs1.product_name, cs1.store_name, cs1.store_zip, cs1.b_street_number, - | cs1.b_streen_name, cs1.b_city, cs1.b_zip, cs1.c_street_number, cs1.c_street_name, - | cs1.c_city, cs1.c_zip, cs1.syear, cs1.cnt, cs1.s1, cs1.s2, cs1.s3, cs2.s1, - | cs2.s2, cs2.s3, cs2.syear, cs2.cnt - | from cross_sales cs1,cross_sales cs2 - | where cs1.item_sk=cs2.item_sk and - | cs1.syear = 1999 and - | cs2.syear = 1999 + 1 and - | cs2.cnt <= cs1.cnt and - | cs1.store_name = cs2.store_name and - | cs1.store_zip = cs2.store_zip - | order by cs1.product_name, cs1.store_name, cs2.cnt - """.stripMargin), - Query( - "q65", - """ - | select - | s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand - | from store, item, - | (select ss_store_sk, avg(revenue) as ave - | from - | (select ss_store_sk, ss_item_sk, - | sum(ss_sales_price) as revenue - | from store_sales, date_dim - | where ss_sold_date_sk = d_date_sk and d_month_seq between 1176 and 1176+11 - | group by ss_store_sk, ss_item_sk) sa - | group by ss_store_sk) sb, - | (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue - | from store_sales, date_dim - | where ss_sold_date_sk = d_date_sk and d_month_seq between 1176 and 1176+11 - | group by ss_store_sk, ss_item_sk) sc - | where sb.ss_store_sk = sc.ss_store_sk and - | sc.revenue <= 0.1 * sb.ave and - | s_store_sk = sc.ss_store_sk and - | i_item_sk = sc.ss_item_sk - | order by s_store_name, i_item_desc - | limit 100 - """.stripMargin), - // Modifications: "||" -> concat - Query( - "q66", - """ - | select w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, - | ship_carriers, year - | ,sum(jan_sales) as jan_sales - | ,sum(feb_sales) as feb_sales - | ,sum(mar_sales) as mar_sales - | ,sum(apr_sales) as apr_sales - | ,sum(may_sales) as may_sales - | ,sum(jun_sales) as jun_sales - | ,sum(jul_sales) as jul_sales - | ,sum(aug_sales) as aug_sales - | ,sum(sep_sales) as sep_sales - | ,sum(oct_sales) as oct_sales - | ,sum(nov_sales) as nov_sales - | ,sum(dec_sales) as dec_sales - | ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot - | ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot - | ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot - | ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot - | ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot - | ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot - | ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot - | ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot - | ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot - | ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot - | ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot - | ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot - | ,sum(jan_net) as jan_net - | ,sum(feb_net) as feb_net - | ,sum(mar_net) as mar_net - | ,sum(apr_net) as apr_net - | ,sum(may_net) as may_net - | ,sum(jun_net) as jun_net - | ,sum(jul_net) as jul_net - | ,sum(aug_net) as aug_net - | ,sum(sep_net) as sep_net - | ,sum(oct_net) as oct_net - | ,sum(nov_net) as nov_net - | ,sum(dec_net) as dec_net - | from ( - | (select - | w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country - | ,concat('DHL', ',', 'BARIAN') as ship_carriers - | ,d_year as year - | ,sum(case when d_moy = 1 then ws_ext_sales_price * ws_quantity else 0 end) as jan_sales - | ,sum(case when d_moy = 2 then ws_ext_sales_price * ws_quantity else 0 end) as feb_sales - | ,sum(case when d_moy = 3 then ws_ext_sales_price * ws_quantity else 0 end) as mar_sales - | ,sum(case when d_moy = 4 then ws_ext_sales_price * ws_quantity else 0 end) as apr_sales - | ,sum(case when d_moy = 5 then ws_ext_sales_price * ws_quantity else 0 end) as may_sales - | ,sum(case when d_moy = 6 then ws_ext_sales_price * ws_quantity else 0 end) as jun_sales - | ,sum(case when d_moy = 7 then ws_ext_sales_price * ws_quantity else 0 end) as jul_sales - | ,sum(case when d_moy = 8 then ws_ext_sales_price * ws_quantity else 0 end) as aug_sales - | ,sum(case when d_moy = 9 then ws_ext_sales_price * ws_quantity else 0 end) as sep_sales - | ,sum(case when d_moy = 10 then ws_ext_sales_price * ws_quantity else 0 end) as oct_sales - | ,sum(case when d_moy = 11 then ws_ext_sales_price * ws_quantity else 0 end) as nov_sales - | ,sum(case when d_moy = 12 then ws_ext_sales_price * ws_quantity else 0 end) as dec_sales - | ,sum(case when d_moy = 1 then ws_net_paid * ws_quantity else 0 end) as jan_net - | ,sum(case when d_moy = 2 then ws_net_paid * ws_quantity else 0 end) as feb_net - | ,sum(case when d_moy = 3 then ws_net_paid * ws_quantity else 0 end) as mar_net - | ,sum(case when d_moy = 4 then ws_net_paid * ws_quantity else 0 end) as apr_net - | ,sum(case when d_moy = 5 then ws_net_paid * ws_quantity else 0 end) as may_net - | ,sum(case when d_moy = 6 then ws_net_paid * ws_quantity else 0 end) as jun_net - | ,sum(case when d_moy = 7 then ws_net_paid * ws_quantity else 0 end) as jul_net - | ,sum(case when d_moy = 8 then ws_net_paid * ws_quantity else 0 end) as aug_net - | ,sum(case when d_moy = 9 then ws_net_paid * ws_quantity else 0 end) as sep_net - | ,sum(case when d_moy = 10 then ws_net_paid * ws_quantity else 0 end) as oct_net - | ,sum(case when d_moy = 11 then ws_net_paid * ws_quantity else 0 end) as nov_net - | ,sum(case when d_moy = 12 then ws_net_paid * ws_quantity else 0 end) as dec_net - | from - | web_sales, warehouse, date_dim, time_dim, ship_mode - | where - | ws_warehouse_sk = w_warehouse_sk - | and ws_sold_date_sk = d_date_sk - | and ws_sold_time_sk = t_time_sk - | and ws_ship_mode_sk = sm_ship_mode_sk - | and d_year = 2001 - | and t_time between 30838 and 30838+28800 - | and sm_carrier in ('DHL','BARIAN') - | group by - | w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, d_year) - | union all - | (select w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country - | ,concat('DHL', ',', 'BARIAN') as ship_carriers - | ,d_year as year - | ,sum(case when d_moy = 1 then cs_sales_price * cs_quantity else 0 end) as jan_sales - | ,sum(case when d_moy = 2 then cs_sales_price * cs_quantity else 0 end) as feb_sales - | ,sum(case when d_moy = 3 then cs_sales_price * cs_quantity else 0 end) as mar_sales - | ,sum(case when d_moy = 4 then cs_sales_price * cs_quantity else 0 end) as apr_sales - | ,sum(case when d_moy = 5 then cs_sales_price * cs_quantity else 0 end) as may_sales - | ,sum(case when d_moy = 6 then cs_sales_price * cs_quantity else 0 end) as jun_sales - | ,sum(case when d_moy = 7 then cs_sales_price * cs_quantity else 0 end) as jul_sales - | ,sum(case when d_moy = 8 then cs_sales_price * cs_quantity else 0 end) as aug_sales - | ,sum(case when d_moy = 9 then cs_sales_price * cs_quantity else 0 end) as sep_sales - | ,sum(case when d_moy = 10 then cs_sales_price * cs_quantity else 0 end) as oct_sales - | ,sum(case when d_moy = 11 then cs_sales_price * cs_quantity else 0 end) as nov_sales - | ,sum(case when d_moy = 12 then cs_sales_price * cs_quantity else 0 end) as dec_sales - | ,sum(case when d_moy = 1 then cs_net_paid_inc_tax * cs_quantity else 0 end) as jan_net - | ,sum(case when d_moy = 2 then cs_net_paid_inc_tax * cs_quantity else 0 end) as feb_net - | ,sum(case when d_moy = 3 then cs_net_paid_inc_tax * cs_quantity else 0 end) as mar_net - | ,sum(case when d_moy = 4 then cs_net_paid_inc_tax * cs_quantity else 0 end) as apr_net - | ,sum(case when d_moy = 5 then cs_net_paid_inc_tax * cs_quantity else 0 end) as may_net - | ,sum(case when d_moy = 6 then cs_net_paid_inc_tax * cs_quantity else 0 end) as jun_net - | ,sum(case when d_moy = 7 then cs_net_paid_inc_tax * cs_quantity else 0 end) as jul_net - | ,sum(case when d_moy = 8 then cs_net_paid_inc_tax * cs_quantity else 0 end) as aug_net - | ,sum(case when d_moy = 9 then cs_net_paid_inc_tax * cs_quantity else 0 end) as sep_net - | ,sum(case when d_moy = 10 then cs_net_paid_inc_tax * cs_quantity else 0 end) as oct_net - | ,sum(case when d_moy = 11 then cs_net_paid_inc_tax * cs_quantity else 0 end) as nov_net - | ,sum(case when d_moy = 12 then cs_net_paid_inc_tax * cs_quantity else 0 end) as dec_net - | from - | catalog_sales, warehouse, date_dim, time_dim, ship_mode - | where - | cs_warehouse_sk = w_warehouse_sk - | and cs_sold_date_sk = d_date_sk - | and cs_sold_time_sk = t_time_sk - | and cs_ship_mode_sk = sm_ship_mode_sk - | and d_year = 2001 - | and t_time between 30838 AND 30838+28800 - | and sm_carrier in ('DHL','BARIAN') - | group by - | w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, d_year - | ) - | ) x - | group by - | w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, - | ship_carriers, year - | order by w_warehouse_name - | limit 100 - """.stripMargin), - Query( - "q67", - """ - | select * from - | (select i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy, s_store_id, - | sumsales, rank() over (partition by i_category order by sumsales desc) rk - | from - | (select i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy, - | s_store_id, sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales - | from store_sales, date_dim, store, item - | where ss_sold_date_sk=d_date_sk - | and ss_item_sk=i_item_sk - | and ss_store_sk = s_store_sk - | and d_month_seq between 1200 and 1200+11 - | group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, - | d_moy,s_store_id))dw1) dw2 - | where rk <= 100 - | order by - | i_category, i_class, i_brand, i_product_name, d_year, - | d_qoy, d_moy, s_store_id, sumsales, rk - | limit 100 - """.stripMargin), - Query( - "q68", - """ - | select - | c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number, extended_price, - | extended_tax, list_price - | from (select - | ss_ticket_number, ss_customer_sk, ca_city bought_city, - | sum(ss_ext_sales_price) extended_price, - | sum(ss_ext_list_price) list_price, - | sum(ss_ext_tax) extended_tax - | from store_sales, date_dim, store, household_demographics, customer_address - | where store_sales.ss_sold_date_sk = date_dim.d_date_sk - | and store_sales.ss_store_sk = store.s_store_sk - | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - | and store_sales.ss_addr_sk = customer_address.ca_address_sk - | and date_dim.d_dom between 1 and 2 - | and (household_demographics.hd_dep_count = 4 or - | household_demographics.hd_vehicle_count = 3) - | and date_dim.d_year in (1999,1999+1,1999+2) - | and store.s_city in ('Midway','Fairview') - | group by ss_ticket_number, ss_customer_sk, ss_addr_sk,ca_city) dn, - | customer, - | customer_address current_addr - | where ss_customer_sk = c_customer_sk - | and customer.c_current_addr_sk = current_addr.ca_address_sk - | and current_addr.ca_city <> bought_city - | order by c_last_name, ss_ticket_number - | limit 100 - """.stripMargin), - Query( - "q69", - """ - | select - | cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, - | cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 - | from - | customer c,customer_address ca,customer_demographics - | where - | c.c_current_addr_sk = ca.ca_address_sk and - | ca_state in ('KY', 'GA', 'NM') and - | cd_demo_sk = c.c_current_cdemo_sk and - | exists (select * from store_sales, date_dim - | where c.c_customer_sk = ss_customer_sk and - | ss_sold_date_sk = d_date_sk and - | d_year = 2001 and - | d_moy between 4 and 4+2) and - | (not exists (select * from web_sales, date_dim - | where c.c_customer_sk = ws_bill_customer_sk and - | ws_sold_date_sk = d_date_sk and - | d_year = 2001 and - | d_moy between 4 and 4+2) and - | not exists (select * from catalog_sales, date_dim - | where c.c_customer_sk = cs_ship_customer_sk and - | cs_sold_date_sk = d_date_sk and - | d_year = 2001 and - | d_moy between 4 and 4+2)) - | group by cd_gender, cd_marital_status, cd_education_status, - | cd_purchase_estimate, cd_credit_rating - | order by cd_gender, cd_marital_status, cd_education_status, - | cd_purchase_estimate, cd_credit_rating - | limit 100 - """.stripMargin), - Query( - "q70", - """ - | select - | sum(ss_net_profit) as total_sum, s_state, s_county - | ,grouping(s_state)+grouping(s_county) as lochierarchy - | ,rank() over ( - | partition by grouping(s_state)+grouping(s_county), - | case when grouping(s_county) = 0 then s_state end - | order by sum(ss_net_profit) desc) as rank_within_parent - | from - | store_sales, date_dim d1, store - | where - | d1.d_month_seq between 1200 and 1200+11 - | and d1.d_date_sk = ss_sold_date_sk - | and s_store_sk = ss_store_sk - | and s_state in - | (select s_state from - | (select s_state as s_state, - | rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking - | from store_sales, store, date_dim - | where d_month_seq between 1200 and 1200+11 - | and d_date_sk = ss_sold_date_sk - | and s_store_sk = ss_store_sk - | group by s_state) tmp1 - | where ranking <= 5) - | group by rollup(s_state,s_county) - | order by - | lochierarchy desc - | ,case when lochierarchy = 0 then s_state end - | ,rank_within_parent - | limit 100 - """.stripMargin), - Query( - "q71", - """ - | select i_brand_id brand_id, i_brand brand,t_hour,t_minute, - | sum(ext_price) ext_price - | from item, - | (select - | ws_ext_sales_price as ext_price, - | ws_sold_date_sk as sold_date_sk, - | ws_item_sk as sold_item_sk, - | ws_sold_time_sk as time_sk - | from web_sales, date_dim - | where d_date_sk = ws_sold_date_sk - | and d_moy=11 - | and d_year=1999 - | union all - | select - | cs_ext_sales_price as ext_price, - | cs_sold_date_sk as sold_date_sk, - | cs_item_sk as sold_item_sk, - | cs_sold_time_sk as time_sk - | from catalog_sales, date_dim - | where d_date_sk = cs_sold_date_sk - | and d_moy=11 - | and d_year=1999 - | union all - | select - | ss_ext_sales_price as ext_price, - | ss_sold_date_sk as sold_date_sk, - | ss_item_sk as sold_item_sk, - | ss_sold_time_sk as time_sk - | from store_sales,date_dim - | where d_date_sk = ss_sold_date_sk - | and d_moy=11 - | and d_year=1999 - | ) as tmp, time_dim - | where - | sold_item_sk = i_item_sk - | and i_manager_id=1 - | and time_sk = t_time_sk - | and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') - | group by i_brand, i_brand_id,t_hour,t_minute - | order by ext_price desc, brand_id - """.stripMargin), - // Modifications: "+ days" -> date_add - Query( - "q72", - """ - | select i_item_desc - | ,w_warehouse_name - | ,d1.d_week_seq - | ,count(case when p_promo_sk is null then 1 else 0 end) no_promo - | ,count(case when p_promo_sk is not null then 1 else 0 end) promo - | ,count(*) total_cnt - | from catalog_sales - | join inventory on (cs_item_sk = inv_item_sk) - | join warehouse on (w_warehouse_sk=inv_warehouse_sk) - | join item on (i_item_sk = cs_item_sk) - | join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) - | join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) - | join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) - | join date_dim d2 on (inv_date_sk = d2.d_date_sk) - | join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) - | left outer join promotion on (cs_promo_sk=p_promo_sk) - | left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) - | where d1.d_week_seq = d2.d_week_seq - | and inv_quantity_on_hand < cs_quantity - | and d3.d_date > (cast(d1.d_date AS DATE) + interval 5 days) - | and hd_buy_potential = '>10000' - | and d1.d_year = 1999 - | and hd_buy_potential = '>10000' - | and cd_marital_status = 'D' - | and d1.d_year = 1999 - | group by i_item_desc,w_warehouse_name,d1.d_week_seq - | order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq - | limit 100 - """.stripMargin), - Query( - "q72b", - """ - | select i_item_desc - | ,w_warehouse_name - | ,d1.d_week_seq - | ,count(case when p_promo_sk is null then 1 else 0 end) no_promo - | ,count(case when p_promo_sk is not null then 1 else 0 end) promo - | ,count(*) total_cnt - | from catalog_sales - | join item on (i_item_sk = cs_item_sk) - | join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) - | join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) - | join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) - | join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) - | join inventory on (cs_item_sk = inv_item_sk) - | join warehouse on (w_warehouse_sk=inv_warehouse_sk) - | join date_dim d2 on (inv_date_sk = d2.d_date_sk) - | left outer join promotion on (cs_promo_sk=p_promo_sk) - | left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) - | where d1.d_week_seq = d2.d_week_seq - | and inv_quantity_on_hand < cs_quantity - | and d3.d_date > (cast(d1.d_date AS DATE) + interval 5 days) - | and hd_buy_potential = '>10000' - | and d1.d_year = 1999 - | and hd_buy_potential = '>10000' - | and cd_marital_status = 'D' - | and d1.d_year = 1999 - | group by i_item_desc,w_warehouse_name,d1.d_week_seq - | order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq - | limit 100 - """.stripMargin), - Query( - "q73", - """ - | select - | c_last_name, c_first_name, c_salutation, c_preferred_cust_flag, - | ss_ticket_number, cnt from - | (select ss_ticket_number, ss_customer_sk, count(*) cnt - | from store_sales,date_dim,store,household_demographics - | where store_sales.ss_sold_date_sk = date_dim.d_date_sk - | and store_sales.ss_store_sk = store.s_store_sk - | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - | and date_dim.d_dom between 1 and 2 - | and (household_demographics.hd_buy_potential = '>10000' or - | household_demographics.hd_buy_potential = 'unknown') - | and household_demographics.hd_vehicle_count > 0 - | and case when household_demographics.hd_vehicle_count > 0 then - | household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 - | and date_dim.d_year in (1999,1999+1,1999+2) - | and store.s_county in ('Williamson County','Franklin Parish','Bronx County','Orange County') - | group by ss_ticket_number,ss_customer_sk) dj,customer - | where ss_customer_sk = c_customer_sk - | and cnt between 1 and 5 - | order by cnt desc - """.stripMargin), - Query( - "q74", - """ - | with year_total as ( - | select - | c_customer_id customer_id, c_first_name customer_first_name, - | c_last_name customer_last_name, d_year as year, - | sum(ss_net_paid) year_total, 's' sale_type - | from - | customer, store_sales, date_dim - | where c_customer_sk = ss_customer_sk - | and ss_sold_date_sk = d_date_sk - | and d_year in (2001,2001+1) - | group by - | c_customer_id, c_first_name, c_last_name, d_year - | union all - | select - | c_customer_id customer_id, c_first_name customer_first_name, - | c_last_name customer_last_name, d_year as year, - | sum(ws_net_paid) year_total, 'w' sale_type - | from - | customer, web_sales, date_dim - | where c_customer_sk = ws_bill_customer_sk - | and ws_sold_date_sk = d_date_sk - | and d_year in (2001,2001+1) - | group by - | c_customer_id, c_first_name, c_last_name, d_year) - | select - | t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name - | from - | year_total t_s_firstyear, year_total t_s_secyear, - | year_total t_w_firstyear, year_total t_w_secyear - | where t_s_secyear.customer_id = t_s_firstyear.customer_id - | and t_s_firstyear.customer_id = t_w_secyear.customer_id - | and t_s_firstyear.customer_id = t_w_firstyear.customer_id - | and t_s_firstyear.sale_type = 's' - | and t_w_firstyear.sale_type = 'w' - | and t_s_secyear.sale_type = 's' - | and t_w_secyear.sale_type = 'w' - | and t_s_firstyear.year = 2001 - | and t_s_secyear.year = 2001+1 - | and t_w_firstyear.year = 2001 - | and t_w_secyear.year = 2001+1 - | and t_s_firstyear.year_total > 0 - | and t_w_firstyear.year_total > 0 - | and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end - | > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end - | order by 1, 1, 1 - | limit 100 - """.stripMargin), - Query( - "q75", - """ - | WITH all_sales AS ( - | SELECT - | d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, - | SUM(sales_cnt) AS sales_cnt, SUM(sales_amt) AS sales_amt - | FROM ( - | SELECT - | d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, - | cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt, - | cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt - | FROM catalog_sales - | JOIN item ON i_item_sk=cs_item_sk - | JOIN date_dim ON d_date_sk=cs_sold_date_sk - | LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number - | AND cs_item_sk=cr_item_sk) - | WHERE i_category='Books' - | UNION - | SELECT - | d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, - | ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt, - | ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt - | FROM store_sales - | JOIN item ON i_item_sk=ss_item_sk - | JOIN date_dim ON d_date_sk=ss_sold_date_sk - | LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number - | AND ss_item_sk=sr_item_sk) - | WHERE i_category='Books' - | UNION - | SELECT - | d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id, - | ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt, - | ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt - | FROM web_sales - | JOIN item ON i_item_sk=ws_item_sk - | JOIN date_dim ON d_date_sk=ws_sold_date_sk - | LEFT JOIN web_returns ON (ws_order_number=wr_order_number - | AND ws_item_sk=wr_item_sk) - | WHERE i_category='Books') sales_detail - | GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) - | SELECT - | prev_yr.d_year AS prev_year, curr_yr.d_year AS year, curr_yr.i_brand_id, - | curr_yr.i_class_id, curr_yr.i_category_id, curr_yr.i_manufact_id, - | prev_yr.sales_cnt AS prev_yr_cnt, curr_yr.sales_cnt AS curr_yr_cnt, - | curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff, - | curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff - | FROM all_sales curr_yr, all_sales prev_yr - | WHERE curr_yr.i_brand_id=prev_yr.i_brand_id - | AND curr_yr.i_class_id=prev_yr.i_class_id - | AND curr_yr.i_category_id=prev_yr.i_category_id - | AND curr_yr.i_manufact_id=prev_yr.i_manufact_id - | AND curr_yr.d_year=2002 - | AND prev_yr.d_year=2002-1 - | AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 - | ORDER BY sales_cnt_diff - | LIMIT 100 - """.stripMargin), - Query( - "q76", - """ - | SELECT - | channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, - | SUM(ext_sales_price) sales_amt - | FROM( - | SELECT - | 'store' as channel, ss_store_sk col_name, d_year, d_qoy, i_category, - | ss_ext_sales_price ext_sales_price - | FROM store_sales, item, date_dim - | WHERE ss_store_sk IS NULL - | AND ss_sold_date_sk=d_date_sk - | AND ss_item_sk=i_item_sk - | UNION ALL - | SELECT - | 'web' as channel, ws_ship_customer_sk col_name, d_year, d_qoy, i_category, - | ws_ext_sales_price ext_sales_price - | FROM web_sales, item, date_dim - | WHERE ws_ship_customer_sk IS NULL - | AND ws_sold_date_sk=d_date_sk - | AND ws_item_sk=i_item_sk - | UNION ALL - | SELECT - | 'catalog' as channel, cs_ship_addr_sk col_name, d_year, d_qoy, i_category, - | cs_ext_sales_price ext_sales_price - | FROM catalog_sales, item, date_dim - | WHERE cs_ship_addr_sk IS NULL - | AND cs_sold_date_sk=d_date_sk - | AND cs_item_sk=i_item_sk) foo - | GROUP BY channel, col_name, d_year, d_qoy, i_category - | ORDER BY channel, col_name, d_year, d_qoy, i_category - | limit 100 - """.stripMargin), - // Modifications: "+ days" -> date_add - Query( - "q77", - """ - | with ss as - | (select s_store_sk, sum(ss_ext_sales_price) as sales, sum(ss_net_profit) as profit - | from store_sales, date_dim, store - | where ss_sold_date_sk = d_date_sk - | and d_date between cast('2000-08-03' as date) and - | (cast('2000-08-03' as date) + interval 30 days) - | and ss_store_sk = s_store_sk - | group by s_store_sk), - | sr as - | (select s_store_sk, sum(sr_return_amt) as returns, sum(sr_net_loss) as profit_loss - | from store_returns, date_dim, store - | where sr_returned_date_sk = d_date_sk - | and d_date between cast('2000-08-03' as date) and - | (cast('2000-08-03' as date) + interval 30 days) - | and sr_store_sk = s_store_sk - | group by s_store_sk), - | cs as - | (select cs_call_center_sk, sum(cs_ext_sales_price) as sales, sum(cs_net_profit) as profit - | from catalog_sales, date_dim - | where cs_sold_date_sk = d_date_sk - | and d_date between cast('2000-08-03' as date) and - | (cast('2000-08-03' as date) + interval 30 days) - | group by cs_call_center_sk), - | cr as - | (select sum(cr_return_amount) as returns, sum(cr_net_loss) as profit_loss - | from catalog_returns, date_dim - | where cr_returned_date_sk = d_date_sk - | and d_date between cast('2000-08-03]' as date) and - | (cast('2000-08-03' as date) + interval 30 day)), - | ws as - | (select wp_web_page_sk, sum(ws_ext_sales_price) as sales, sum(ws_net_profit) as profit - | from web_sales, date_dim, web_page - | where ws_sold_date_sk = d_date_sk - | and d_date between cast('2000-08-03' as date) and - | (cast('2000-08-03' as date) + interval 30 days) - | and ws_web_page_sk = wp_web_page_sk - | group by wp_web_page_sk), - | wr as - | (select wp_web_page_sk, sum(wr_return_amt) as returns, sum(wr_net_loss) as profit_loss - | from web_returns, date_dim, web_page - | where wr_returned_date_sk = d_date_sk - | and d_date between cast('2000-08-03' as date) and - | (cast('2000-08-03' as date) + interval 30 days) - | and wr_web_page_sk = wp_web_page_sk - | group by wp_web_page_sk) - | select channel, id, sum(sales) as sales, sum(returns) as returns, sum(profit) as profit - | from - | (select - | 'store channel' as channel, ss.s_store_sk as id, sales, - | coalesce(returns, 0) as returns, (profit - coalesce(profit_loss,0)) as profit - | from ss left join sr - | on ss.s_store_sk = sr.s_store_sk - | union all - | select - | 'catalog channel' as channel, cs_call_center_sk as id, sales, - | returns, (profit - profit_loss) as profit - | from cs, cr - | union all - | select - | 'web channel' as channel, ws.wp_web_page_sk as id, sales, - | coalesce(returns, 0) returns, (profit - coalesce(profit_loss,0)) as profit - | from ws left join wr - | on ws.wp_web_page_sk = wr.wp_web_page_sk - | ) x - | group by rollup(channel, id) - | order by channel, id - | limit 100 - """.stripMargin), - Query( - "q78", - """ - | with ws as - | (select d_year AS ws_sold_year, ws_item_sk, - | ws_bill_customer_sk ws_customer_sk, - | sum(ws_quantity) ws_qty, - | sum(ws_wholesale_cost) ws_wc, - | sum(ws_sales_price) ws_sp - | from web_sales - | left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk - | join date_dim on ws_sold_date_sk = d_date_sk - | where wr_order_number is null - | group by d_year, ws_item_sk, ws_bill_customer_sk - | ), - | cs as - | (select d_year AS cs_sold_year, cs_item_sk, - | cs_bill_customer_sk cs_customer_sk, - | sum(cs_quantity) cs_qty, - | sum(cs_wholesale_cost) cs_wc, - | sum(cs_sales_price) cs_sp - | from catalog_sales - | left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk - | join date_dim on cs_sold_date_sk = d_date_sk - | where cr_order_number is null - | group by d_year, cs_item_sk, cs_bill_customer_sk - | ), - | ss as - | (select d_year AS ss_sold_year, ss_item_sk, - | ss_customer_sk, - | sum(ss_quantity) ss_qty, - | sum(ss_wholesale_cost) ss_wc, - | sum(ss_sales_price) ss_sp - | from store_sales - | left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk - | join date_dim on ss_sold_date_sk = d_date_sk - | where sr_ticket_number is null - | group by d_year, ss_item_sk, ss_customer_sk - | ) - | select - | round(ss_qty/(coalesce(ws_qty+cs_qty,1)),2) ratio, - | ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, - | coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, - | coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, - | coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price - | from ss - | left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) - | left join cs on (cs_sold_year=ss_sold_year and ss_item_sk=cs_item_sk and cs_customer_sk=ss_customer_sk) - | where coalesce(ws_qty,0)>0 and coalesce(cs_qty, 0)>0 and ss_sold_year=2000 - | order by - | ratio, - | ss_qty desc, ss_wc desc, ss_sp desc, - | other_chan_qty, - | other_chan_wholesale_cost, - | other_chan_sales_price, - | round(ss_qty/(coalesce(ws_qty+cs_qty,1)),2) - | limit 100 - """.stripMargin), - Query( - "q79", - """ - | select - | c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit - | from - | (select ss_ticket_number - | ,ss_customer_sk - | ,store.s_city - | ,sum(ss_coupon_amt) amt - | ,sum(ss_net_profit) profit - | from store_sales,date_dim,store,household_demographics - | where store_sales.ss_sold_date_sk = date_dim.d_date_sk - | and store_sales.ss_store_sk = store.s_store_sk - | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - | and (household_demographics.hd_dep_count = 6 or - | household_demographics.hd_vehicle_count > 2) - | and date_dim.d_dow = 1 - | and date_dim.d_year in (1999,1999+1,1999+2) - | and store.s_number_employees between 200 and 295 - | group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer - | where ss_customer_sk = c_customer_sk - | order by c_last_name,c_first_name,substr(s_city,1,30), profit - | limit 100 - """.stripMargin), - // Modifications: "+ days" -> date_add - // Modifications: "||" -> "concat" - Query( - "q80", - """ - | with ssr as - | (select s_store_id as store_id, - | sum(ss_ext_sales_price) as sales, - | sum(coalesce(sr_return_amt, 0)) as returns, - | sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit - | from store_sales left outer join store_returns on - | (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), - | date_dim, store, item, promotion - | where ss_sold_date_sk = d_date_sk - | and d_date between cast('2000-08-23' as date) - | and (cast('2000-08-23' as date) + interval 30 days) - | and ss_store_sk = s_store_sk - | and ss_item_sk = i_item_sk - | and i_current_price > 50 - | and ss_promo_sk = p_promo_sk - | and p_channel_tv = 'N' - | group by s_store_id), - | csr as - | (select cp_catalog_page_id as catalog_page_id, - | sum(cs_ext_sales_price) as sales, - | sum(coalesce(cr_return_amount, 0)) as returns, - | sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit - | from catalog_sales left outer join catalog_returns on - | (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), - | date_dim, catalog_page, item, promotion - | where cs_sold_date_sk = d_date_sk - | and d_date between cast('2000-08-23' as date) - | and (cast('2000-08-23' as date) + interval 30 days) - | and cs_catalog_page_sk = cp_catalog_page_sk - | and cs_item_sk = i_item_sk - | and i_current_price > 50 - | and cs_promo_sk = p_promo_sk - | and p_channel_tv = 'N' - | group by cp_catalog_page_id), - | wsr as - | (select web_site_id, - | sum(ws_ext_sales_price) as sales, - | sum(coalesce(wr_return_amt, 0)) as returns, - | sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit - | from web_sales left outer join web_returns on - | (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), - | date_dim, web_site, item, promotion - | where ws_sold_date_sk = d_date_sk - | and d_date between cast('2000-08-23' as date) - | and (cast('2000-08-23' as date) + interval 30 days) - | and ws_web_site_sk = web_site_sk - | and ws_item_sk = i_item_sk - | and i_current_price > 50 - | and ws_promo_sk = p_promo_sk - | and p_channel_tv = 'N' - | group by web_site_id) - | select channel, id, sum(sales) as sales, sum(returns) as returns, sum(profit) as profit - | from (select - | 'store channel' as channel, concat('store', store_id) as id, sales, returns, profit - | from ssr - | union all - | select - | 'catalog channel' as channel, concat('catalog_page', catalog_page_id) as id, - | sales, returns, profit - | from csr - | union all - | select - | 'web channel' as channel, concat('web_site', web_site_id) as id, sales, returns, profit - | from wsr) x - | group by rollup (channel, id) - | order by channel, id - | limit 100 - """.stripMargin), - Query( - "q81", - """ - | with customer_total_return as - | (select - | cr_returning_customer_sk as ctr_customer_sk, ca_state as ctr_state, - | sum(cr_return_amt_inc_tax) as ctr_total_return - | from catalog_returns, date_dim, customer_address - | where cr_returned_date_sk = d_date_sk - | and d_year = 2000 - | and cr_returning_addr_sk = ca_address_sk - | group by cr_returning_customer_sk, ca_state ) - | select - | c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name, - | ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country, - | ca_gmt_offset,ca_location_type,ctr_total_return - | from customer_total_return ctr1, customer_address, customer - | where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 - | from customer_total_return ctr2 - | where ctr1.ctr_state = ctr2.ctr_state) - | and ca_address_sk = c_current_addr_sk - | and ca_state = 'GA' - | and ctr1.ctr_customer_sk = c_customer_sk - | order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name - | ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset - | ,ca_location_type,ctr_total_return - | limit 100 - """.stripMargin), - Query( - "q82", - """ - | select i_item_id, i_item_desc, i_current_price - | from item, inventory, date_dim, store_sales - | where i_current_price between 62 and 62+30 - | and inv_item_sk = i_item_sk - | and d_date_sk=inv_date_sk - | and d_date between cast('2000-05-25' as date) and (cast('2000-05-25' as date) + interval 60 days) - | and i_manufact_id in (129, 270, 821, 423) - | and inv_quantity_on_hand between 100 and 500 - | and ss_item_sk = i_item_sk - | group by i_item_id,i_item_desc,i_current_price - | order by i_item_id - | limit 100 - """.stripMargin), - Query( - "q83", - """ - | with sr_items as - | (select i_item_id item_id, sum(sr_return_quantity) sr_item_qty - | from store_returns, item, date_dim - | where sr_item_sk = i_item_sk - | and d_date in (select d_date from date_dim where d_week_seq in - | (select d_week_seq from date_dim where d_date in ('2000-06-30','2000-09-27','2000-11-17'))) - | and sr_returned_date_sk = d_date_sk - | group by i_item_id), - | cr_items as - | (select i_item_id item_id, sum(cr_return_quantity) cr_item_qty - | from catalog_returns, item, date_dim - | where cr_item_sk = i_item_sk - | and d_date in (select d_date from date_dim where d_week_seq in - | (select d_week_seq from date_dim where d_date in ('2000-06-30','2000-09-27','2000-11-17'))) - | and cr_returned_date_sk = d_date_sk - | group by i_item_id), - | wr_items as - | (select i_item_id item_id, sum(wr_return_quantity) wr_item_qty - | from web_returns, item, date_dim - | where wr_item_sk = i_item_sk and d_date in - | (select d_date from date_dim where d_week_seq in - | (select d_week_seq from date_dim where d_date in ('2000-06-30','2000-09-27','2000-11-17'))) - | and wr_returned_date_sk = d_date_sk - | group by i_item_id) - | select sr_items.item_id - | ,sr_item_qty - | ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev - | ,cr_item_qty - | ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev - | ,wr_item_qty - | ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev - | ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average - | from sr_items, cr_items, wr_items - | where sr_items.item_id=cr_items.item_id - | and sr_items.item_id=wr_items.item_id - | order by sr_items.item_id, sr_item_qty - | limit 100 - """.stripMargin), - // Modifications: "||" -> concat - Query( - "q84", - """ - | select c_customer_id as customer_id - | ,concat(c_last_name, ', ', c_first_name) as customername - | from customer - | ,customer_address - | ,customer_demographics - | ,household_demographics - | ,income_band - | ,store_returns - | where ca_city = 'Edgewood' - | and c_current_addr_sk = ca_address_sk - | and ib_lower_bound >= 38128 - | and ib_upper_bound <= 38128 + 50000 - | and ib_income_band_sk = hd_income_band_sk - | and cd_demo_sk = c_current_cdemo_sk - | and hd_demo_sk = c_current_hdemo_sk - | and sr_cdemo_sk = cd_demo_sk - | order by c_customer_id - | limit 100 - """.stripMargin), - Query( - "q85", - """ - | select - | substr(r_reason_desc,1,20), avg(ws_quantity), avg(wr_refunded_cash), avg(wr_fee) - | from web_sales, web_returns, web_page, customer_demographics cd1, - | customer_demographics cd2, customer_address, date_dim, reason - | where ws_web_page_sk = wp_web_page_sk - | and ws_item_sk = wr_item_sk - | and ws_order_number = wr_order_number - | and ws_sold_date_sk = d_date_sk and d_year = 2000 - | and cd1.cd_demo_sk = wr_refunded_cdemo_sk - | and cd2.cd_demo_sk = wr_returning_cdemo_sk - | and ca_address_sk = wr_refunded_addr_sk - | and r_reason_sk = wr_reason_sk - | and - | ( - | ( - | cd1.cd_marital_status = 'M' - | and - | cd1.cd_marital_status = cd2.cd_marital_status - | and - | cd1.cd_education_status = 'Advanced Degree' - | and - | cd1.cd_education_status = cd2.cd_education_status - | and - | ws_sales_price between 100.00 and 150.00 - | ) - | or - | ( - | cd1.cd_marital_status = 'S' - | and - | cd1.cd_marital_status = cd2.cd_marital_status - | and - | cd1.cd_education_status = 'College' - | and - | cd1.cd_education_status = cd2.cd_education_status - | and - | ws_sales_price between 50.00 and 100.00 - | ) - | or - | ( - | cd1.cd_marital_status = 'W' - | and - | cd1.cd_marital_status = cd2.cd_marital_status - | and - | cd1.cd_education_status = '2 yr Degree' - | and - | cd1.cd_education_status = cd2.cd_education_status - | and - | ws_sales_price between 150.00 and 200.00 - | ) - | ) - | and - | ( - | ( - | ca_country = 'United States' - | and - | ca_state in ('IN', 'OH', 'NJ') - | and ws_net_profit between 100 and 200 - | ) - | or - | ( - | ca_country = 'United States' - | and - | ca_state in ('WI', 'CT', 'KY') - | and ws_net_profit between 150 and 300 - | ) - | or - | ( - | ca_country = 'United States' - | and - | ca_state in ('LA', 'IA', 'AR') - | and ws_net_profit between 50 and 250 - | ) - | ) - | group by r_reason_desc - | order by substr(r_reason_desc,1,20) - | ,avg(ws_quantity) - | ,avg(wr_refunded_cash) - | ,avg(wr_fee) - | limit 100 - """.stripMargin), - Query( - "q86", - """ - | select sum(ws_net_paid) as total_sum, i_category, i_class, - | grouping(i_category)+grouping(i_class) as lochierarchy, - | rank() over ( - | partition by grouping(i_category)+grouping(i_class), - | case when grouping(i_class) = 0 then i_category end - | order by sum(ws_net_paid) desc) as rank_within_parent - | from - | web_sales, date_dim d1, item - | where - | d1.d_month_seq between 1200 and 1200+11 - | and d1.d_date_sk = ws_sold_date_sk - | and i_item_sk = ws_item_sk - | group by rollup(i_category,i_class) - | order by - | lochierarchy desc, - | case when lochierarchy = 0 then i_category end, - | rank_within_parent - | limit 100 - """.stripMargin), - Query( - "q87", - """ - | select count(*) - | from ((select distinct c_last_name, c_first_name, d_date - | from store_sales, date_dim, customer - | where store_sales.ss_sold_date_sk = date_dim.d_date_sk - | and store_sales.ss_customer_sk = customer.c_customer_sk - | and d_month_seq between 1200 and 1200+11) - | except - | (select distinct c_last_name, c_first_name, d_date - | from catalog_sales, date_dim, customer - | where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk - | and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk - | and d_month_seq between 1200 and 1200+11) - | except - | (select distinct c_last_name, c_first_name, d_date - | from web_sales, date_dim, customer - | where web_sales.ws_sold_date_sk = date_dim.d_date_sk - | and web_sales.ws_bill_customer_sk = customer.c_customer_sk - | and d_month_seq between 1200 and 1200+11) - |) cool_cust - """.stripMargin), - Query( - "q88", - """ - | select * - | from - | (select count(*) h8_30_to_9 - | from store_sales, household_demographics , time_dim, store - | where ss_sold_time_sk = time_dim.t_time_sk - | and ss_hdemo_sk = household_demographics.hd_demo_sk - | and ss_store_sk = s_store_sk - | and time_dim.t_hour = 8 - | and time_dim.t_minute >= 30 - | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or - | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or - | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) - | and store.s_store_name = 'ese') s1, - | (select count(*) h9_to_9_30 - | from store_sales, household_demographics , time_dim, store - | where ss_sold_time_sk = time_dim.t_time_sk - | and ss_hdemo_sk = household_demographics.hd_demo_sk - | and ss_store_sk = s_store_sk - | and time_dim.t_hour = 9 - | and time_dim.t_minute < 30 - | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or - | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or - | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) - | and store.s_store_name = 'ese') s2, - | (select count(*) h9_30_to_10 - | from store_sales, household_demographics , time_dim, store - | where ss_sold_time_sk = time_dim.t_time_sk - | and ss_hdemo_sk = household_demographics.hd_demo_sk - | and ss_store_sk = s_store_sk - | and time_dim.t_hour = 9 - | and time_dim.t_minute >= 30 - | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or - | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or - | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) - | and store.s_store_name = 'ese') s3, - | (select count(*) h10_to_10_30 - | from store_sales, household_demographics , time_dim, store - | where ss_sold_time_sk = time_dim.t_time_sk - | and ss_hdemo_sk = household_demographics.hd_demo_sk - | and ss_store_sk = s_store_sk - | and time_dim.t_hour = 10 - | and time_dim.t_minute < 30 - | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or - | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or - | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) - | and store.s_store_name = 'ese') s4, - | (select count(*) h10_30_to_11 - | from store_sales, household_demographics , time_dim, store - | where ss_sold_time_sk = time_dim.t_time_sk - | and ss_hdemo_sk = household_demographics.hd_demo_sk - | and ss_store_sk = s_store_sk - | and time_dim.t_hour = 10 - | and time_dim.t_minute >= 30 - | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or - | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or - | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) - | and store.s_store_name = 'ese') s5, - | (select count(*) h11_to_11_30 - | from store_sales, household_demographics , time_dim, store - | where ss_sold_time_sk = time_dim.t_time_sk - | and ss_hdemo_sk = household_demographics.hd_demo_sk - | and ss_store_sk = s_store_sk - | and time_dim.t_hour = 11 - | and time_dim.t_minute < 30 - | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or - | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or - | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) - | and store.s_store_name = 'ese') s6, - | (select count(*) h11_30_to_12 - | from store_sales, household_demographics , time_dim, store - | where ss_sold_time_sk = time_dim.t_time_sk - | and ss_hdemo_sk = household_demographics.hd_demo_sk - | and ss_store_sk = s_store_sk - | and time_dim.t_hour = 11 - | and time_dim.t_minute >= 30 - | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or - | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or - | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) - | and store.s_store_name = 'ese') s7, - | (select count(*) h12_to_12_30 - | from store_sales, household_demographics , time_dim, store - | where ss_sold_time_sk = time_dim.t_time_sk - | and ss_hdemo_sk = household_demographics.hd_demo_sk - | and ss_store_sk = s_store_sk - | and time_dim.t_hour = 12 - | and time_dim.t_minute < 30 - | and ((household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or - | (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or - | (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) - | and store.s_store_name = 'ese') s8 - """.stripMargin), - Query( - "q89", - """ - | select * - | from( - | select i_category, i_class, i_brand, - | s_store_name, s_company_name, - | d_moy, - | sum(ss_sales_price) sum_sales, - | avg(sum(ss_sales_price)) over - | (partition by i_category, i_brand, s_store_name, s_company_name) - | avg_monthly_sales - | from item, store_sales, date_dim, store - | where ss_item_sk = i_item_sk and - | ss_sold_date_sk = d_date_sk and - | ss_store_sk = s_store_sk and - | d_year in (1999) and - | ((i_category in ('Books','Electronics','Sports') and - | i_class in ('computers','stereo','football')) - | or (i_category in ('Men','Jewelry','Women') and - | i_class in ('shirts','birdal','dresses'))) - | group by i_category, i_class, i_brand, - | s_store_name, s_company_name, d_moy) tmp1 - | where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 - | order by sum_sales - avg_monthly_sales, s_store_name - | limit 100 - """.stripMargin), - Query( - "q90", - """ - | select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio - | from ( select count(*) amc - | from web_sales, household_demographics , time_dim, web_page - | where ws_sold_time_sk = time_dim.t_time_sk - | and ws_ship_hdemo_sk = household_demographics.hd_demo_sk - | and ws_web_page_sk = web_page.wp_web_page_sk - | and time_dim.t_hour between 8 and 8+1 - | and household_demographics.hd_dep_count = 6 - | and web_page.wp_char_count between 5000 and 5200) at, - | ( select count(*) pmc - | from web_sales, household_demographics , time_dim, web_page - | where ws_sold_time_sk = time_dim.t_time_sk - | and ws_ship_hdemo_sk = household_demographics.hd_demo_sk - | and ws_web_page_sk = web_page.wp_web_page_sk - | and time_dim.t_hour between 19 and 19+1 - | and household_demographics.hd_dep_count = 6 - | and web_page.wp_char_count between 5000 and 5200) pt - | order by am_pm_ratio - | limit 100 - """.stripMargin), - Query( - "q91", - """ - | select - | cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, - | sum(cr_net_loss) Returns_Loss - | from - | call_center, catalog_returns, date_dim, customer, customer_address, - | customer_demographics, household_demographics - | where - | cr_call_center_sk = cc_call_center_sk - | and cr_returned_date_sk = d_date_sk - | and cr_returning_customer_sk = c_customer_sk - | and cd_demo_sk = c_current_cdemo_sk - | and hd_demo_sk = c_current_hdemo_sk - | and ca_address_sk = c_current_addr_sk - | and d_year = 1998 - | and d_moy = 11 - | and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') - | or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) - | and hd_buy_potential like 'Unknown%' - | and ca_gmt_offset = -7 - | group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status - | order by sum(cr_net_loss) desc - """.stripMargin), - // Modifications: "+ days" -> date_add - // Modifications: " -> ` - Query( - "q92", - """ - | select sum(ws_ext_discount_amt) as `Excess Discount Amount` - | from web_sales, item, date_dim - | where i_manufact_id = 350 - | and i_item_sk = ws_item_sk - | and d_date between '2000-01-27' and (cast('2000-01-27' as date) + interval 90 days) - | and d_date_sk = ws_sold_date_sk - | and ws_ext_discount_amt > - | ( - | SELECT 1.3 * avg(ws_ext_discount_amt) - | FROM web_sales, date_dim - | WHERE ws_item_sk = i_item_sk - | and d_date between '2000-01-27' and (cast('2000-01-27' as date) + interval 90 days) - | and d_date_sk = ws_sold_date_sk - | ) - | order by sum(ws_ext_discount_amt) - | limit 100 - """.stripMargin), - Query( - "q93", - """ - | select ss_customer_sk, sum(act_sales) sumsales - | from (select - | ss_item_sk, ss_ticket_number, ss_customer_sk, - | case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price - | else (ss_quantity*ss_sales_price) end act_sales - | from store_sales - | left outer join store_returns - | on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number), - | reason - | where sr_reason_sk = r_reason_sk and r_reason_desc = 'reason 28') t - | group by ss_customer_sk - | order by sumsales, ss_customer_sk - | limit 100 - """.stripMargin), - // Modifications: "+ days" -> date_add - // Modifications: " -> ` - Query( - "q94", - """ - | select - | count(distinct ws_order_number) as `order count` - | ,sum(ws_ext_ship_cost) as `total shipping cost` - | ,sum(ws_net_profit) as `total net profit` - | from - | web_sales ws1, date_dim, customer_address, web_site - | where - | d_date between '1999-02-01' and - | (cast('1999-02-01' as date) + interval 60 days) - | and ws1.ws_ship_date_sk = d_date_sk - | and ws1.ws_ship_addr_sk = ca_address_sk - | and ca_state = 'IL' - | and ws1.ws_web_site_sk = web_site_sk - | and web_company_name = 'pri' - | and exists (select * - | from web_sales ws2 - | where ws1.ws_order_number = ws2.ws_order_number - | and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) - | and not exists(select * - | from web_returns wr1 - | where ws1.ws_order_number = wr1.wr_order_number) - | order by count(distinct ws_order_number) - | limit 100 - """.stripMargin), - // Modifications: "+ days" -> date_add - Query( - "q95", - """ - | with ws_wh as - | (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 - | from web_sales ws1,web_sales ws2 - | where ws1.ws_order_number = ws2.ws_order_number - | and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) - | select - | count(distinct ws_order_number) as `order count` - | ,sum(ws_ext_ship_cost) as `total shipping cost` - | ,sum(ws_net_profit) as `total net profit` - | from - | web_sales ws1, date_dim, customer_address, web_site - | where - | d_date between '1999-02-01' and - | (cast('1999-02-01' as date) + interval 60 days) - | and ws1.ws_ship_date_sk = d_date_sk - | and ws1.ws_ship_addr_sk = ca_address_sk - | and ca_state = 'IL' - | and ws1.ws_web_site_sk = web_site_sk - | and web_company_name = 'pri' - | and ws1.ws_order_number in (select ws_order_number - | from ws_wh) - | and ws1.ws_order_number in (select wr_order_number - | from web_returns,ws_wh - | where wr_order_number = ws_wh.ws_order_number) - | order by count(distinct ws_order_number) - | limit 100 - """.stripMargin), - Query( - "q96", - """ - | select count(*) - | from store_sales, household_demographics, time_dim, store - | where ss_sold_time_sk = time_dim.t_time_sk - | and ss_hdemo_sk = household_demographics.hd_demo_sk - | and ss_store_sk = s_store_sk - | and time_dim.t_hour = 20 - | and time_dim.t_minute >= 30 - | and household_demographics.hd_dep_count = 7 - | and store.s_store_name = 'ese' - | order by count(*) - | limit 100 - """.stripMargin), - Query( - "q97", - """ - | with ssci as ( - | select ss_customer_sk customer_sk, ss_item_sk item_sk - | from store_sales,date_dim - | where ss_sold_date_sk = d_date_sk - | and d_month_seq between 1200 and 1200 + 11 - | group by ss_customer_sk, ss_item_sk), - | csci as( - | select cs_bill_customer_sk customer_sk, cs_item_sk item_sk - | from catalog_sales,date_dim - | where cs_sold_date_sk = d_date_sk - | and d_month_seq between 1200 and 1200 + 11 - | group by cs_bill_customer_sk, cs_item_sk) - | select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only - | ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only - | ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog - | from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk - | and ssci.item_sk = csci.item_sk) - | limit 100 - """.stripMargin), - // Modifications: "+ days" -> date_add - Query( - "q98", - """ - |select i_item_desc, i_category, i_class, i_current_price - | ,sum(ss_ext_sales_price) as itemrevenue - | ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over - | (partition by i_class) as revenueratio - |from - | store_sales, item, date_dim - |where - | ss_item_sk = i_item_sk - | and i_category in ('Sports', 'Books', 'Home') - | and ss_sold_date_sk = d_date_sk - | and d_date between cast('1999-02-22' as date) - | and (cast('1999-02-22' as date) + interval 30 days) - |group by - | i_item_id, i_item_desc, i_category, i_class, i_current_price - |order by - | i_category, i_class, i_item_id, i_item_desc, revenueratio - """.stripMargin), - // Modifications: " -> ` - Query( - "q99", - """ - | select - | substr(w_warehouse_name,1,20), sm_type, cc_name - | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` - | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and - | (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` - | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and - | (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` - | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and - | (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` - | ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` - | from - | catalog_sales, warehouse, ship_mode, call_center, date_dim - | where - | d_month_seq between 1200 and 1200 + 11 - | and cs_ship_date_sk = d_date_sk - | and cs_warehouse_sk = w_warehouse_sk - | and cs_ship_mode_sk = sm_ship_mode_sk - | and cs_call_center_sk = cc_call_center_sk - | group by - | substr(w_warehouse_name,1,20), sm_type, cc_name - | order by substr(w_warehouse_name,1,20), sm_type, cc_name - | limit 100 - """.stripMargin), - Query( - "qSsMax", - """ - |select - | count(*) as total, - | count(ss_sold_date_sk) as not_null_total, - | count(distinct ss_sold_date_sk) as unique_days, - | max(ss_sold_date_sk) as max_ss_sold_date_sk, - | max(ss_sold_time_sk) as max_ss_sold_time_sk, - | max(ss_item_sk) as max_ss_item_sk, - | max(ss_customer_sk) as max_ss_customer_sk, - | max(ss_cdemo_sk) as max_ss_cdemo_sk, - | max(ss_hdemo_sk) as max_ss_hdemo_sk, - | max(ss_addr_sk) as max_ss_addr_sk, - | max(ss_store_sk) as max_ss_store_sk, - | max(ss_promo_sk) as max_ss_promo_sk - |from store_sales - """.stripMargin)) - - val recommendedIndexes = Array( - "JoinIndex00-index-33-inv_item_sk-3;inventory;inv_item_sk;inv_date_sk,inv_warehouse_sk,inv_quantity_on_hand", - "JoinIndex01-index-34-inv_date_sk-3;inventory;inv_date_sk;inv_warehouse_sk,inv_quantity_on_hand,inv_item_sk", - "JoinIndex02-index-42-inv_warehouse_sk-3;inventory;inv_warehouse_sk;inv_date_sk,inv_quantity_on_hand,inv_item_sk", - "JoinIndex03-index-0-d_date_sk-10;date_dim;d_date_sk;d_moy,d_year,d_date,d_month_seq,d_qoy,d_dom,d_week_seq,d_dow,d_day_name,d_quarter_name", - "FilterIndex04-index-11-d_year-5;date_dim;d_year;d_date_sk,d_date,d_week_seq,d_moy,d_day_name", - "FilterIndex06-index-14-d_date-2;date_dim;d_date;d_date_sk,d_week_seq", - "FilterIndex07-index-17-d_month_seq-7;date_dim;d_month_seq;d_date_sk,d_date,d_day_name,d_week_seq,d_moy,d_qoy,d_year", - "JoinIndex08-index-2-i_item_sk-17;item;i_item_sk;i_item_id,i_item_desc,i_manager_id,i_brand,i_brand_id,i_category,i_class,i_current_price,i_class_id,i_category_id,i_product_name,i_size,i_color,i_units,i_manufact_id,i_manufact,i_wholesale_cost", - "FilterIndex10-index-39-i_category-9;item;i_category;i_item_id,i_item_desc,i_class,i_item_sk,i_current_price,i_manufact_id,i_class_id,i_brand_id,i_category_id", - "JoinIndex12-index-1-ss_sold_date_sk-21;store_sales;ss_sold_date_sk;ss_ext_sales_price,ss_item_sk,ss_sales_price,ss_store_sk,ss_addr_sk,ss_hdemo_sk,ss_ticket_number,ss_customer_sk,ss_quantity,ss_list_price,ss_net_profit,ss_coupon_amt,ss_ext_list_price,ss_ext_discount_amt,ss_wholesale_cost,ss_cdemo_sk,ss_promo_sk,ss_ext_wholesale_cost,ss_ext_tax,ss_net_paid,ss_sold_time_sk", - "JoinIndex13-index-3-ss_item_sk-16;store_sales;ss_item_sk;ss_ext_sales_price,ss_sold_date_sk,ss_sales_price,ss_store_sk,ss_addr_sk,ss_quantity,ss_list_price,ss_net_paid,ss_ticket_number,ss_customer_sk,ss_net_profit,ss_promo_sk,ss_wholesale_cost,ss_coupon_amt,ss_cdemo_sk,ss_hdemo_sk", - "JoinIndex14-index-5-ss_store_sk-20;store_sales;ss_store_sk;ss_sales_price,ss_item_sk,ss_sold_date_sk,ss_net_profit,ss_hdemo_sk,ss_ticket_number,ss_customer_sk,ss_coupon_amt,ss_addr_sk,ss_net_paid,ss_sold_time_sk,ss_quantity,ss_list_price,ss_cdemo_sk,ss_ext_sales_price,ss_promo_sk,ss_ext_wholesale_cost,ss_wholesale_cost,ss_ext_list_price,ss_ext_tax", - "JoinIndex15-index-12-ss_customer_sk-20;store_sales;ss_customer_sk;ss_hdemo_sk,ss_ticket_number,ss_sold_date_sk,ss_store_sk,ss_quantity,ss_item_sk,ss_addr_sk,ss_net_profit,ss_coupon_amt,ss_net_paid,ss_promo_sk,ss_list_price,ss_cdemo_sk,ss_wholesale_cost,ss_ext_list_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_tax,ss_sales_price", - "JoinIndex17-index-27-ss_hdemo_sk-19;store_sales;ss_hdemo_sk;ss_addr_sk,ss_net_profit,ss_coupon_amt,ss_ticket_number,ss_sold_date_sk,ss_customer_sk,ss_store_sk,ss_sold_time_sk,ss_promo_sk,ss_item_sk,ss_list_price,ss_cdemo_sk,ss_wholesale_cost,ss_ext_list_price,ss_ext_tax,ss_ext_sales_price,ss_sales_price,ss_quantity,ss_ext_wholesale_cost", - "JoinIndex19-index-47-ss_cdemo_sk-16;store_sales;ss_cdemo_sk;ss_store_sk,ss_addr_sk,ss_promo_sk,ss_item_sk,ss_hdemo_sk,ss_list_price,ss_coupon_amt,ss_ticket_number,ss_sold_date_sk,ss_customer_sk,ss_wholesale_cost,ss_sales_price,ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit", - "JoinIndex20-index-43-t_time_sk-4;time_dim;t_time_sk;t_hour,t_minute,t_time,t_meal_time", - "JoinIndex21-index-20-sr_item_sk-7;store_returns;sr_item_sk;sr_ticket_number,sr_return_amt,sr_return_quantity,sr_returned_date_sk,sr_customer_sk,sr_reason_sk,sr_net_loss", - "JoinIndex23-index-37-sr_returned_date_sk-7;store_returns;sr_returned_date_sk;sr_item_sk,sr_return_quantity,sr_customer_sk,sr_ticket_number,sr_return_amt,sr_store_sk,sr_net_loss", - "JoinIndex24-index-4-s_store_sk-15;store;s_store_sk;s_state,s_store_name,s_store_id,s_market_id,s_zip,s_city,s_company_name,s_county,s_company_id,s_street_name,s_street_number,s_suite_number,s_street_type,s_gmt_offset,s_number_employees", - "JoinIndex25-index-18-cd_demo_sk-8;customer_demographics;cd_demo_sk;cd_education_status,cd_marital_status,cd_gender,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_college_count,cd_dep_employed_count", - "JoinIndex26-index-6-cs_sold_date_sk-26;catalog_sales;cs_sold_date_sk;cs_bill_addr_sk,cs_ext_sales_price,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_list_price,cs_order_number,cs_bill_hdemo_sk,cs_promo_sk,cs_ship_date_sk,cs_bill_cdemo_sk,cs_net_profit,cs_catalog_page_sk,cs_call_center_sk,cs_sales_price,cs_coupon_amt,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_list_price,cs_warehouse_sk,cs_ship_addr_sk,cs_wholesale_cost,cs_sold_time_sk,cs_net_paid,cs_ship_mode_sk,cs_net_paid_inc_tax", - "JoinIndex27-index-10-cs_item_sk-22;catalog_sales;cs_item_sk;cs_sold_date_sk,cs_bill_addr_sk,cs_ext_sales_price,cs_quantity,cs_bill_customer_sk,cs_list_price,cs_order_number,cs_bill_hdemo_sk,cs_promo_sk,cs_ship_date_sk,cs_bill_cdemo_sk,cs_ship_addr_sk,cs_net_profit,cs_net_paid,cs_ext_discount_amt,cs_catalog_page_sk,cs_coupon_amt,cs_sales_price,cs_ext_list_price,cs_warehouse_sk,cs_wholesale_cost,cs_call_center_sk", - "JoinIndex28-index-23-cs_bill_customer_sk-12;catalog_sales;cs_bill_customer_sk;cs_sold_date_sk,cs_quantity,cs_item_sk,cs_list_price,cs_sales_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_list_price,cs_ext_sales_price,cs_net_profit,cs_coupon_amt,cs_bill_cdemo_sk", - "JoinIndex31-index-7-ca_address_sk-11;customer_address;ca_address_sk;ca_state,ca_gmt_offset,ca_country,ca_city,ca_county,ca_zip,ca_street_name,ca_street_number,ca_street_type,ca_suite_number,ca_location_type", - "JoinIndex35-index-24-w_warehouse_sk-6;warehouse;w_warehouse_sk;w_warehouse_name,w_state,w_country,w_county,w_city,w_warehouse_sq_ft", - "JoinIndex36-index-46-web_site_sk-3;web_site;web_site_sk;web_company_name,web_site_id,web_name", - "JoinIndex37-index-9-ws_sold_date_sk-21;web_sales;ws_sold_date_sk;ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_quantity,ws_bill_customer_sk,ws_list_price,ws_net_paid,ws_order_number,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_time_sk,ws_ext_discount_amt,ws_net_profit,ws_web_page_sk,ws_sales_price,ws_ext_list_price,ws_ext_wholesale_cost,ws_promo_sk,ws_web_site_sk,ws_wholesale_cost,ws_ship_customer_sk", - "JoinIndex38-index-15-ws_item_sk-16;web_sales;ws_item_sk;ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,ws_quantity,ws_bill_customer_sk,ws_list_price,ws_ship_customer_sk,ws_order_number,ws_sales_price,ws_net_profit,ws_web_page_sk,ws_web_site_sk,ws_net_paid,ws_ext_discount_amt,ws_wholesale_cost,ws_promo_sk", - "JoinIndex39-index-30-ws_bill_customer_sk-10;web_sales;ws_bill_customer_sk;ws_item_sk,ws_quantity,ws_sold_date_sk,ws_list_price,ws_ext_list_price,ws_ext_discount_amt,ws_net_paid,ws_ext_sales_price,ws_ext_wholesale_cost,ws_sales_price", - "JoinIndex40-index-31-ws_order_number-16;web_sales;ws_order_number;ws_warehouse_sk,ws_ship_addr_sk,ws_net_profit,ws_ship_date_sk,ws_ext_ship_cost,ws_web_site_sk,ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,ws_web_page_sk,ws_sales_price,ws_ext_sales_price,ws_promo_sk,ws_wholesale_cost,ws_bill_customer_sk", - "JoinIndex41-index-36-wr_item_sk-11;web_returns;wr_item_sk;wr_return_quantity,wr_order_number,wr_return_amt,wr_returned_date_sk,wr_net_loss,wr_refunded_cdemo_sk,wr_reason_sk,wr_refunded_cash,wr_fee,wr_refunded_addr_sk,wr_returning_cdemo_sk", - "JoinIndex42-index-41-wr_order_number-11;web_returns;wr_order_number;wr_return_quantity,wr_item_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,wr_refunded_cdemo_sk,wr_reason_sk,wr_refunded_cash,wr_fee,wr_refunded_addr_sk,wr_returning_cdemo_sk", - "JoinIndex43-index-8-c_customer_sk-17;customer;c_customer_sk;c_current_addr_sk,c_first_name,c_last_name,c_current_cdemo_sk,c_preferred_cust_flag,c_salutation,c_birth_country,c_login,c_customer_id,c_email_address,c_birth_year,c_birth_month,c_current_hdemo_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_last_review_date,c_birth_day", - "JoinIndex44-index-16-c_current_addr_sk-17;customer;c_current_addr_sk;c_customer_sk,c_current_cdemo_sk,c_first_name,c_last_name,c_customer_id,c_last_review_date,c_email_address,c_salutation,c_login,c_birth_year,c_birth_month,c_birth_country,c_birth_day,c_preferred_cust_flag,c_current_hdemo_sk,c_first_sales_date_sk,c_first_shipto_date_sk", - "JoinIndex46-index-19-hd_demo_sk-4;household_demographics;hd_demo_sk;hd_vehicle_count,hd_dep_count,hd_buy_potential,hd_income_band_sk", - "JoinIndex47-index-26-cr_item_sk-8;catalog_returns;cr_item_sk;cr_order_number,cr_return_amount,cr_return_quantity,cr_returned_date_sk,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_net_loss", - "JoinIndex49-index-40-p_promo_sk-4;promotion;p_promo_sk;p_channel_event,p_channel_email,p_channel_dmail,p_channel_tv") - - // scalastyle:on -} diff --git a/src/main/scala/com/microsoft/hyperspace/perf/TPCHBenchmark.scala b/src/main/scala/com/microsoft/hyperspace/perf/TPCHBenchmark.scala deleted file mode 100644 index db61f0a8f..000000000 --- a/src/main/scala/com/microsoft/hyperspace/perf/TPCHBenchmark.scala +++ /dev/null @@ -1,833 +0,0 @@ -/* - * Copyright (2020) The Hyperspace Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.microsoft.hyperspace.perf - -object TPCHBenchmark { - // scalastyle:off - - import org.apache.spark.sql.types._ - - val partSchema = StructType( - Seq( - StructField("p_partkey", LongType), - StructField("p_name", StringType), - StructField("p_mfgr", StringType), - StructField("p_brand", StringType), - StructField("p_type", StringType), - StructField("p_size", IntegerType), - StructField("p_container", StringType), - StructField("p_retailprice", DoubleType), - StructField("p_comment", StringType))) - - val partsuppSchema = StructType( - Seq( - StructField("ps_partkey", LongType), - StructField("ps_suppkey", LongType), - StructField("ps_availqty", IntegerType), - StructField("ps_supplycost", DoubleType), - StructField("ps_comment", StringType))) - - val supplierSchema = StructType( - Seq( - StructField("s_suppkey", LongType), - StructField("s_name", StringType), - StructField("s_address", StringType), - StructField("s_nationkey", IntegerType), - StructField("s_phone", StringType), - StructField("s_acctbal", DoubleType), - StructField("s_comment", StringType))) - - val customerSchema = StructType( - Seq( - StructField("c_custkey", LongType), - StructField("c_name", StringType), - StructField("c_address", StringType), - StructField("c_nationkey", IntegerType), - StructField("c_phone", StringType), - StructField("c_acctbal", DoubleType), - StructField("c_mktsegment", StringType), - StructField("c_comment", StringType))) - - val ordersSchema = StructType( - Seq( - StructField("o_orderkey", LongType), - StructField("o_custkey", LongType), - StructField("o_orderstatus", StringType), - StructField("o_totalprice", DoubleType), - StructField("o_orderdate", StringType), - StructField("o_orderpriority", StringType), - StructField("o_clerk", StringType), - StructField("o_shippriority", IntegerType), - StructField("o_comment", StringType))) - - val lineitemSchema = StructType( - Seq( - StructField("l_orderkey", LongType), - StructField("l_partkey", LongType), - StructField("l_suppkey", LongType), - StructField("l_linenumber", IntegerType), - StructField("l_quantity", DoubleType), - StructField("l_extendedprice", DoubleType), - StructField("l_discount", DoubleType), - StructField("l_tax", DoubleType), - StructField("l_returnflag", StringType), - StructField("l_linestatus", StringType), - StructField("l_shipdate", StringType), - StructField("l_commitdate", StringType), - StructField("l_receiptdate", StringType), - StructField("l_shipinstruct", StringType), - StructField("l_shipmode", StringType), - StructField("l_comment", StringType))) - - val regionSchema = StructType( - Seq( - StructField("r_regionkey", IntegerType), - StructField("r_name", StringType), - StructField("r_comment", StringType))) - - val nationSchema = StructType( - Seq( - StructField("n_nationkey", IntegerType), - StructField("n_name", StringType), - StructField("n_regionkey", IntegerType), - StructField("n_comment", StringType))) - - val tables = Array( - ("customer", customerSchema), - ("lineitem", lineitemSchema), - ("nation", nationSchema), - ("orders", ordersSchema), - ("part", partSchema), - ("partsupp", partsuppSchema), - ("region", regionSchema), - ("supplier", supplierSchema)) - - val queries = Array( - Query( - "q1", - """select - | l_returnflag, - | l_linestatus, - | sum(l_quantity) as sum_qty, - | sum(l_extendedprice) as sum_base_price, - | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, - | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, - | avg(l_quantity) as avg_qty, - | avg(l_extendedprice) as avg_price, - | avg(l_discount) as avg_disc, - | count(*) as count_order - |from - | lineitem - |where - | l_shipdate <= date '1998-12-01' - interval '90' day - |group by - | l_returnflag, - | l_linestatus - |order by - | l_returnflag, - | l_linestatus - """.stripMargin), - Query( - "q2", - """select - | s_acctbal, - | s_name, - | n_name, - | p_partkey, - | p_mfgr, - | s_address, - | s_phone, - | s_comment - |from - | part, - | supplier, - | partsupp, - | nation, - | region - |where - | p_partkey = ps_partkey - | and s_suppkey = ps_suppkey - | and p_size = 15 - | and p_type like '%BRASS' - | and s_nationkey = n_nationkey - | and n_regionkey = r_regionkey - | and r_name = 'EUROPE' - | and ps_supplycost = ( - | select - | min(ps_supplycost) - | from - | partsupp, - | supplier, - | nation, - | region - | where - | p_partkey = ps_partkey - | and s_suppkey = ps_suppkey - | and s_nationkey = n_nationkey - | and n_regionkey = r_regionkey - | and r_name = 'EUROPE' - | ) - |order by - | s_acctbal desc, - | n_name, - | s_name, - | p_partkey - """.stripMargin), - Query( - "q3", - """select - | l_orderkey, - | sum(l_extendedprice * (1 - l_discount)) as revenue, - | o_orderdate, - | o_shippriority - |from - | customer, - | orders, - | lineitem - |where - | c_mktsegment = 'BUILDING' - | and c_custkey = o_custkey - | and l_orderkey = o_orderkey - | and o_orderdate < date '1995-03-15' - | and l_shipdate > date '1995-03-15' - |group by - | l_orderkey, - | o_orderdate, - | o_shippriority - |order by - | revenue desc, - | o_orderdate - """.stripMargin), - Query( - "q4", - """select - | o_orderpriority, - | count(*) as order_count - |from - | orders - |where - | o_orderdate >= date '1993-07-01' - | and o_orderdate < date '1993-07-01' + interval '3' month - | and exists ( - | select - | * - | from - | lineitem - | where - | l_orderkey = o_orderkey - | and l_commitdate < l_receiptdate - | ) - |group by - | o_orderpriority - |order by - | o_orderpriority - """.stripMargin), - Query( - "q5", - """select - | n_name, - | sum(l_extendedprice * (1 - l_discount)) as revenue - |from - | customer, - | orders, - | lineitem, - | supplier, - | nation, - | region - |where - | c_custkey = o_custkey - | and l_orderkey = o_orderkey - | and l_suppkey = s_suppkey - | and c_nationkey = s_nationkey - | and s_nationkey = n_nationkey - | and n_regionkey = r_regionkey - | and r_name = 'ASIA' - | and o_orderdate >= date '1994-01-01' - | and o_orderdate < date '1994-01-01' + interval '1' year - |group by - | n_name - |order by - | revenue desc - """.stripMargin), - Query( - "q6", - """select - | sum(l_extendedprice * l_discount) as revenue - |from - | lineitem - |where - | l_shipdate >= date '1994-01-01' - | and l_shipdate < date '1994-01-01' + interval '1' year - | and l_discount between .06 - 0.01 and .06 + 0.01 - | and l_quantity < 24 - """.stripMargin), - Query( - "q7", - """select - | supp_nation, - | cust_nation, - | l_year, - | sum(volume) as revenue - |from - | ( - | select - | n1.n_name as supp_nation, - | n2.n_name as cust_nation, - | year(l_shipdate) as l_year, - | l_extendedprice * (1 - l_discount) as volume - | from - | supplier, - | lineitem, - | orders, - | customer, - | nation n1, - | nation n2 - | where - | s_suppkey = l_suppkey - | and o_orderkey = l_orderkey - | and c_custkey = o_custkey - | and s_nationkey = n1.n_nationkey - | and c_nationkey = n2.n_nationkey - | and ( - | (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY') - | or (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE') - | ) - | and l_shipdate between date '1995-01-01' and date '1996-12-31' - | ) as shipping - |group by - | supp_nation, - | cust_nation, - | l_year - |order by - | supp_nation, - | cust_nation, - | l_year""".stripMargin), - Query( - "q8", - """select - | o_year, - | sum(case - | when nation = 'BRAZIL' then volume - | else 0 - | end) / sum(volume) as mkt_share - |from - | ( - | select - | year(o_orderdate) as o_year, - | l_extendedprice * (1 - l_discount) as volume, - | n2.n_name as nation - | from - | part, - | supplier, - | lineitem, - | orders, - | customer, - | nation n1, - | nation n2, - | region - | where - | p_partkey = l_partkey - | and s_suppkey = l_suppkey - | and l_orderkey = o_orderkey - | and o_custkey = c_custkey - | and c_nationkey = n1.n_nationkey - | and n1.n_regionkey = r_regionkey - | and r_name = 'AMERICA' - | and s_nationkey = n2.n_nationkey - | and o_orderdate between date '1995-01-01' and date '1996-12-31' - | and p_type = 'ECONOMY ANODIZED STEEL' - | ) as all_nations - |group by - | o_year - |order by - | o_year""".stripMargin), - Query( - "q9", - """select - | nation, - | o_year, - | sum(amount) as sum_profit - |from - | ( - | select - | n_name as nation, - | year(o_orderdate) as o_year, - | l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount - | from - | part, - | supplier, - | lineitem, - | partsupp, - | orders, - | nation - | where - | s_suppkey = l_suppkey - | and ps_suppkey = l_suppkey - | and ps_partkey = l_partkey - | and p_partkey = l_partkey - | and o_orderkey = l_orderkey - | and s_nationkey = n_nationkey - | and p_name like '%green%' - | ) as profit - |group by - | nation, - | o_year - |order by - | nation, - | o_year desc""".stripMargin), - Query( - "q10", - """select - | c_custkey, - | c_name, - | sum(l_extendedprice * (1 - l_discount)) as revenue, - | c_acctbal, - | n_name, - | c_address, - | c_phone, - | c_comment - |from - | customer, - | orders, - | lineitem, - | nation - |where - | c_custkey = o_custkey - | and l_orderkey = o_orderkey - | and o_orderdate >= date '1993-10-01' - | and o_orderdate < date '1993-10-01' + interval '3' month - | and l_returnflag = 'R' - | and c_nationkey = n_nationkey - |group by - | c_custkey, - | c_name, - | c_acctbal, - | c_phone, - | n_name, - | c_address, - | c_comment - |order by - | revenue desc - """.stripMargin), - Query( - "q11", - """select - | ps_partkey, - | sum(ps_supplycost * ps_availqty) as value - |from - | partsupp, - | supplier, - | nation - |where - | ps_suppkey = s_suppkey - | and s_nationkey = n_nationkey - | and n_name = 'GERMANY' - |group by - | ps_partkey having - | sum(ps_supplycost * ps_availqty) > ( - | select - | sum(ps_supplycost * ps_availqty) * 0.0000001000000 - | from - | partsupp, - | supplier, - | nation - | where - | ps_suppkey = s_suppkey - | and s_nationkey = n_nationkey - | and n_name = 'GERMANY' - | ) - |order by - | value desc - """.stripMargin), - Query( - "q12", - """select - | l_shipmode, - | sum(case - | when o_orderpriority = '1-URGENT' - | or o_orderpriority = '2-HIGH' - | then 1 - | else 0 - | end) as high_line_count, - | sum(case - | when o_orderpriority <> '1-URGENT' - | and o_orderpriority <> '2-HIGH' - | then 1 - | else 0 - | end) as low_line_count - |from - | orders, - | lineitem - |where - | o_orderkey = l_orderkey - | and l_shipmode in ('MAIL', 'SHIP') - | and l_commitdate < l_receiptdate - | and l_shipdate < l_commitdate - | and l_receiptdate >= date '1994-01-01' - | and l_receiptdate < date '1994-01-01' + interval '1' year - |group by - | l_shipmode - |order by - | l_shipmode - """.stripMargin), - Query( - "q13", - """select - | c_count, - | count(*) as custdist - |from - | ( - | select - | c_custkey, - | count(o_orderkey) as c_count - | from - | customer left outer join orders on - | c_custkey = o_custkey - | and o_comment not like '%special%requests%' - | group by - | c_custkey - | ) as c_orders - |group by - | c_count - |order by - | custdist desc, - | c_count desc""".stripMargin), - Query( - "q14", - """ - |select - | 100.00 * sum(case - | when p_type like 'PROMO%' - | then l_extendedprice * (1 - l_discount) - | else 0 - | end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue - |from - | lineitem, - | part - |where - | l_partkey = p_partkey - | and l_shipdate >= date '1995-09-01' - | and l_shipdate < date '1995-09-01' + interval '1' month - """.stripMargin), - Query( - "q15", - """with revenue0 as - | (select - | l_suppkey as supplier_no, - | cast(sum(l_extendedprice * (1 - l_discount)) as int) as total_revenue - | from - | lineitem - | where - | l_shipdate >= date '1996-01-01' - | and l_shipdate < date '1996-01-01' + interval '3' month - | group by - | l_suppkey) - | - |select - | s_suppkey, - | s_name, - | s_address, - | s_phone, - | total_revenue - |from - | supplier, - | revenue0 - |where - | s_suppkey = supplier_no - | and total_revenue = ( - | select - | cast(max(total_revenue) as int) - | from - | revenue0 - | ) - |order by - | s_suppkey - |""".stripMargin), - Query( - "q16", - """ - |select - | p_brand, - | p_type, - | p_size, - | count(distinct ps_suppkey) as supplier_cnt - |from - | partsupp, - | part - |where - | p_partkey = ps_partkey - | and p_brand <> 'Brand#45' - | and p_type not like 'MEDIUM POLISHED%' - | and p_size in (49, 14, 23, 45, 19, 3, 36, 9) - | and ps_suppkey not in ( - | select - | s_suppkey - | from - | supplier - | where - | s_comment like '%Customer%Complaints%' - | ) - |group by - | p_brand, - | p_type, - | p_size - |order by - | supplier_cnt desc, - | p_brand, - | p_type, - | p_size - """.stripMargin), - Query( - "q17", - """ - |select - | sum(l_extendedprice) / 7.0 as avg_yearly - |from - | lineitem, - | part - |where - | p_partkey = l_partkey - | and p_brand = 'Brand#23' - | and p_container = 'MED BOX' - | and l_quantity < ( - | select - | 0.2 * avg(l_quantity) - | from - | lineitem - | where - | l_partkey = p_partkey - | ) - """.stripMargin), - Query( - "q18", - """ - |select - | c_name, - | c_custkey, - | o_orderkey, - | o_orderdate, - | o_totalprice, - | sum(l_quantity) - |from - | customer, - | orders, - | lineitem - |where - | o_orderkey in ( - | select - | l_orderkey - | from - | lineitem - | group by - | l_orderkey having - | sum(l_quantity) > 300 - | ) - | and c_custkey = o_custkey - | and o_orderkey = l_orderkey - |group by - | c_name, - | c_custkey, - | o_orderkey, - | o_orderdate, - | o_totalprice - |order by - | o_totalprice desc, - | o_orderdate - """.stripMargin), - Query( - "q19", - """ - |select - | sum(l_extendedprice* (1 - l_discount)) as revenue - |from - | lineitem, - | part - |where - | ( - | p_partkey = l_partkey - | and p_brand = 'Brand#12' - | and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') - | and l_quantity >= 1 and l_quantity <= 1 + 10 - | and p_size between 1 and 5 - | and l_shipmode in ('AIR', 'AIR REG') - | and l_shipinstruct = 'DELIVER IN PERSON' - | ) - | or - | ( - | p_partkey = l_partkey - | and p_brand = 'Brand#23' - | and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') - | and l_quantity >= 10 and l_quantity <= 10 + 10 - | and p_size between 1 and 10 - | and l_shipmode in ('AIR', 'AIR REG') - | and l_shipinstruct = 'DELIVER IN PERSON' - | ) - | or - | ( - | p_partkey = l_partkey - | and p_brand = 'Brand#34' - | and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') - | and l_quantity >= 20 and l_quantity <= 20 + 10 - | and p_size between 1 and 15 - | and l_shipmode in ('AIR', 'AIR REG') - | and l_shipinstruct = 'DELIVER IN PERSON' - | ) - """.stripMargin), - Query( - "q20", - """ - |select - | s_name, - | s_address - |from - | supplier, - | nation - |where - | s_suppkey in ( - | select - | ps_suppkey - | from - | partsupp - | where - | ps_partkey in ( - | select - | p_partkey - | from - | part - | where - | p_name like 'forest%' - | ) - | and ps_availqty > ( - | select - | 0.5 * sum(l_quantity) - | from - | lineitem - | where - | l_partkey = ps_partkey - | and l_suppkey = ps_suppkey - | and l_shipdate >= date '1994-01-01' - | and l_shipdate < date '1994-01-01' + interval '1' year - | ) - | ) - | and s_nationkey = n_nationkey - | and n_name = 'CANADA' - |order by - | s_name - """.stripMargin), - Query( - "q21", - """ - |select - | s_name, - | count(*) as numwait - |from - | supplier, - | lineitem l1, - | orders, - | nation - |where - | s_suppkey = l1.l_suppkey - | and o_orderkey = l1.l_orderkey - | and o_orderstatus = 'F' - | and l1.l_receiptdate > l1.l_commitdate - | and exists ( - | select - | * - | from - | lineitem l2 - | where - | l2.l_orderkey = l1.l_orderkey - | and l2.l_suppkey <> l1.l_suppkey - | ) - | and not exists ( - | select - | * - | from - | lineitem l3 - | where - | l3.l_orderkey = l1.l_orderkey - | and l3.l_suppkey <> l1.l_suppkey - | and l3.l_receiptdate > l3.l_commitdate - | ) - | and s_nationkey = n_nationkey - | and n_name = 'SAUDI ARABIA' - |group by - | s_name - |order by - | numwait desc, - | s_name - """.stripMargin), - Query( - "q22", - """select - | cntrycode, - | count(*) as numcust, - | sum(c_acctbal) as totacctbal - |from - | ( - | select - | substring(c_phone, 1, 2) as cntrycode, - | c_acctbal - | from - | customer - | where - | substring(c_phone, 1, 2) in - | ('13', '31', '23', '29', '30', '18', '17') - | and c_acctbal > ( - | select - | avg(c_acctbal) - | from - | customer - | where - | c_acctbal > 0.00 - | and substring(c_phone, 1, 2) in - | ('13', '31', '23', '29', '30', '18', '17') - | ) - | and not exists ( - | select - | * - | from - | orders - | where - | o_custkey = c_custkey - | ) - | ) as custsale - |group by - | cntrycode - |order by - | cntrycode""".stripMargin)) - - val recommendedIndexes = Array( - "IX1_supplier;supplier;s_suppkey;s_nationkey,s_name,s_address,s_comment,s_phone,s_acctbal", - "IX2_supplier;supplier;s_nationkey;s_suppkey,s_name,s_acctbal,s_address,s_comment,s_phone", - "IX3_region;region;r_regionkey;r_name", - "IX5_part;part;p_partkey;p_type,p_name,p_brand,p_size,p_container,p_mfgr", - "IX11_lineitem;lineitem;l_orderkey;l_quantity,l_suppkey,l_extendedprice,l_partkey,l_discount,l_receiptdate,l_commitdate,l_shipdate,l_shipmode,l_returnflag", - "IX12_lineitem;lineitem;l_partkey;l_suppkey,l_quantity,l_shipdate,l_extendedprice,l_discount,l_orderkey,l_shipinstruct,l_shipmode", - "IX13_lineitem;lineitem;l_suppkey;l_extendedprice,l_shipdate,l_discount,l_orderkey,l_receiptdate,l_commitdate,l_quantity,l_partkey", - "IX14_lineitem;lineitem;l_shipdate;l_linestatus,l_quantity,l_extendedprice,l_tax,l_returnflag,l_discount,l_orderkey,l_suppkey,l_partkey", - "IX15_lineitem;lineitem;l_commitdate,l_receiptdate;l_suppkey,l_orderkey", - "IX20_partsupp;partsupp;ps_suppkey;ps_supplycost,ps_partkey,ps_availqty", - "IX21_partsupp;partsupp;ps_partkey;ps_supplycost,ps_suppkey,ps_availqty", - "IX23_orders;orders;o_orderkey;o_custkey,o_orderdate,o_totalprice,o_shippriority,o_orderstatus,o_orderpriority", - "IX24_orders;orders;o_custkey;o_orderkey,o_orderdate,o_comment,o_totalprice,o_shippriority", - "IX27_nation;nation;n_nationkey;n_name,n_regionkey", - "IX30_customer;customer;c_custkey;c_nationkey,c_name,c_phone,c_address,c_comment,c_acctbal,c_mktsegment", - "IX32_customer;customer;c_acctbal;c_phone,c_custkey") - - // scalastyle:on -} From 5d8c118b09fd08cf884f8d450b89a62114928170 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Fri, 7 Aug 2020 10:57:05 -0700 Subject: [PATCH 08/21] add record lineage with tests - part1 --- .../com/microsoft/hyperspace/SampleData.scala | 12 +++--- .../hyperspace/index/CreateIndexTests.scala | 41 ++++++++++++++++++- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/test/scala/com/microsoft/hyperspace/SampleData.scala b/src/test/scala/com/microsoft/hyperspace/SampleData.scala index 245e6b059..290eb4f46 100644 --- a/src/test/scala/com/microsoft/hyperspace/SampleData.scala +++ b/src/test/scala/com/microsoft/hyperspace/SampleData.scala @@ -25,10 +25,10 @@ object SampleData { ("2017-09-03", "fd093f8a05604515957083e70cb3dceb", "facebook", 1, 3000), ("2017-09-03", "af3ed6a197a8447cba8bc8ea21fad208", "facebook", 1, 3000), ("2017-09-03", "975134eca06c4711a0406d0464cbe7d6", "facebook", 1, 4000), - ("2017-09-03", "e90a6028e15b4f4593eef557daf5166d", "zanahoria", 2, 3000), - ("2017-09-03", "576ed96b0d5340aa98a47de15c9f87ce", "facebook", 2, 3000), - ("2017-09-03", "50d690516ca641438166049a6303650c", "ibraco", 2, 1000), - ("2017-09-03", "380786e6495d4cd8a5dd4cc8d3d12917", "facebook", 2, 3000), - ("2017-09-03", "ff60e4838b92421eafc3e6ee59a9e9f1", "mi perro", 2, 2000), - ("2017-09-03", "187696fe0a6a40cc9516bc6e47c70bc1", "multicines", 4, 3000)) + ("2018-09-03", "e90a6028e15b4f4593eef557daf5166d", "ibraco", 2, 3000), + ("2018-09-03", "576ed96b0d5340aa98a47de15c9f87ce", "facebook", 2, 3000), + ("2018-09-03", "50d690516ca641438166049a6303650c", "ibraco", 2, 1000), + ("2019-10-03", "380786e6495d4cd8a5dd4cc8d3d12917", "facebook", 2, 3000), + ("2019-10-03", "ff60e4838b92421eafc3e6ee59a9e9f1", "mi perro", 2, 2000), + ("2019-10-03", "187696fe0a6a40cc9516bc6e47c70bc1", "facebook", 4, 3000)) } diff --git a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala index 1a40186ed..bb1600d8d 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala @@ -19,7 +19,7 @@ package com.microsoft.hyperspace.index import scala.collection.mutable.WrappedArray import org.apache.hadoop.fs.Path -import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.catalyst.plans.SQLHelper import com.microsoft.hyperspace.{Hyperspace, HyperspaceException, SampleData} @@ -29,8 +29,11 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { override val systemPath = new Path("src/test/resources/indexLocation") private val sampleData = SampleData.testData private val sampleParquetDataLocation = "src/test/resources/sampleparquet" + private val samplePartitionedParquetDataLocation = "src/test/resources/samplepartitionedparquet" private val indexConfig1 = IndexConfig("index1", Seq("RGUID"), Seq("Date")) private val indexConfig2 = IndexConfig("index2", Seq("Query"), Seq("imprs")) + private val indexConfig3 = IndexConfig("index3", Seq("imprs"), Seq("clicks")) + private val indexConfig4 = IndexConfig("index3", Seq("Date", "Query"), Seq("clicks")) private var df: DataFrame = _ private var hyperspace: Hyperspace = _ @@ -41,15 +44,34 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { import sparkSession.implicits._ hyperspace = new Hyperspace(sparkSession) FileUtils.delete(new Path(sampleParquetDataLocation)) + FileUtils.delete(new Path(samplePartitionedParquetDataLocation)) + // save test non-partitioned. val dfFromSample = sampleData.toDF("Date", "RGUID", "Query", "imprs", "clicks") dfFromSample.write.parquet(sampleParquetDataLocation) + // save test data partitioned. + // `Date` is the first partition key and `Query` is the second partition key. + dfFromSample.select("Date").distinct().collectAsList().forEach { d => + val date = d.get(0) + dfFromSample.filter($"date" === date).select("Query").distinct().collectAsList().forEach { + q => + val query = q.get(0) + val partitionPath = s"$samplePartitionedParquetDataLocation/Date=$date/Query=$query" + dfFromSample + .filter($"date" === date && $"Query" === query) + .select("RGUID", "imprs", "clicks") + .write + .parquet(partitionPath) + } + } + df = spark.read.parquet(sampleParquetDataLocation) } override def afterAll(): Unit = { FileUtils.delete(new Path(sampleParquetDataLocation)) + FileUtils.delete(new Path(samplePartitionedParquetDataLocation)) super.afterAll() } @@ -138,4 +160,21 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { exception.getMessage.contains( "Only creating index over HDFS file based scan nodes is supported.")) } + + test("Check lineage in index records for non-partitioned data") {} + + test( + "Check lineage in index records for partitioned data when partition key is not in config") { + + } + + test( + "Check lineage in index records for partitioned data when partition key is in config") { + + } + + test( + "Check lineage in index records for partitioned data when partition key is load path") { + + } } From 3cdb22f9f85e2904d579f1189a9d01a6f615f67f Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Mon, 10 Aug 2020 15:45:41 -0700 Subject: [PATCH 09/21] remove lineage column from relation schema during transformation --- .../microsoft/hyperspace/index/rules/FilterIndexRule.scala | 4 ++-- .../microsoft/hyperspace/index/rules/JoinIndexRule.scala | 2 +- .../com/microsoft/hyperspace/index/IndexManagerTests.scala | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala b/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala index 2dc7067a0..057fd7b84 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.types.StructType import com.microsoft.hyperspace.{ActiveSparkSession, Hyperspace} -import com.microsoft.hyperspace.index.IndexLogEntry +import com.microsoft.hyperspace.index.{IndexConstants, IndexLogEntry} import com.microsoft.hyperspace.telemetry.{AppInfo, HyperspaceEventLogging, HyperspaceIndexUsageEvent} import com.microsoft.hyperspace.util.ResolverUtils @@ -109,7 +109,7 @@ object FilterIndexRule val newRelation = HadoopFsRelation( newLocation, new StructType(), - index.schema, + StructType(index.schema.filter(!_.name.equals(IndexConstants.DATA_FILE_NAME_COLUMN))), None, // Do not set BucketSpec to avoid limiting Spark's degree of parallelism new ParquetFileFormat, Map())(spark) diff --git a/src/main/scala/com/microsoft/hyperspace/index/rules/JoinIndexRule.scala b/src/main/scala/com/microsoft/hyperspace/index/rules/JoinIndexRule.scala index 026ba484d..f1ea327ba 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/rules/JoinIndexRule.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/rules/JoinIndexRule.scala @@ -145,7 +145,7 @@ object JoinIndexRule val relation = HadoopFsRelation( location, new StructType(), - index.schema, + StructType(index.schema.filter(!_.name.equals(IndexConstants.DATA_FILE_NAME_COLUMN))), Some(bucketSpec), new ParquetFileFormat, Map())(spark) diff --git a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala index add2dcf69..d6df1c14b 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala @@ -70,7 +70,11 @@ class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite { indexConfig1.indexedColumns, indexConfig1.includedColumns, 200, - StructType(Seq(StructField("RGUID", StringType), StructField("Date", StringType))).json, + StructType( + Seq( + StructField("RGUID", StringType), + StructField("Date", StringType), + StructField(IndexConstants.DATA_FILE_NAME_COLUMN, StringType))).json, s"$indexStorageLocation/index1/v__=0", Constants.States.ACTIVE) assert(actual.equals(expected)) From 11a0d1c198365c32ed0d09cde1d7f368413fa249 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Mon, 10 Aug 2020 16:57:32 -0700 Subject: [PATCH 10/21] Modify rules to exclude lineage columns and add lineage tests --- .../index/rules/FilterIndexRule.scala | 2 +- .../index/rules/JoinIndexRule.scala | 28 ++--- .../hyperspace/index/CreateIndexTests.scala | 118 +++++++++++++----- 3 files changed, 99 insertions(+), 49 deletions(-) diff --git a/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala b/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala index 057fd7b84..07431b163 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala @@ -109,7 +109,7 @@ object FilterIndexRule val newRelation = HadoopFsRelation( newLocation, new StructType(), - StructType(index.schema.filter(!_.name.equals(IndexConstants.DATA_FILE_NAME_COLUMN))), + StructType(index.schema.filter(fsRelation.schema.contains(_))), None, // Do not set BucketSpec to avoid limiting Spark's degree of parallelism new ParquetFileFormat, Map())(spark) diff --git a/src/main/scala/com/microsoft/hyperspace/index/rules/JoinIndexRule.scala b/src/main/scala/com/microsoft/hyperspace/index/rules/JoinIndexRule.scala index f1ea327ba..6f5f03901 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/rules/JoinIndexRule.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/rules/JoinIndexRule.scala @@ -136,26 +136,26 @@ object JoinIndexRule * @return replacement plan */ private def getReplacementPlan(index: IndexLogEntry, plan: LogicalPlan): LogicalPlan = { - val bucketSpec = BucketSpec( - numBuckets = index.numBuckets, - bucketColumnNames = index.indexedColumns, - sortColumnNames = index.indexedColumns) - - val location = new InMemoryFileIndex(spark, Seq(new Path(index.content.root)), Map(), None) - val relation = HadoopFsRelation( - location, - new StructType(), - StructType(index.schema.filter(!_.name.equals(IndexConstants.DATA_FILE_NAME_COLUMN))), - Some(bucketSpec), - new ParquetFileFormat, - Map())(spark) - // Here we can't replace the plan completely with the index. This will create problems. // For e.g. if Project(A,B) -> Filter(C = 10) -> Scan (A,B,C,D,E) // if we replace this plan with index Scan (A,B,C), we lose the Filter(C=10) and it will // lead to wrong results. So we only replace the base relation. plan transform { case baseRelation @ LogicalRelation(_: HadoopFsRelation, baseOutput, _, _) => + val bucketSpec = BucketSpec( + numBuckets = index.numBuckets, + bucketColumnNames = index.indexedColumns, + sortColumnNames = index.indexedColumns) + + val location = new InMemoryFileIndex(spark, Seq(new Path(index.content.root)), Map(), None) + val relation = HadoopFsRelation( + location, + new StructType(), + StructType(index.schema.filter(baseRelation.schema.contains(_))), + Some(bucketSpec), + new ParquetFileFormat, + Map())(spark) + val updatedOutput = baseOutput.filter(attr => relation.schema.fieldNames.contains(attr.name)) baseRelation.copy(relation = relation, output = updatedOutput) diff --git a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala index bb1600d8d..96976d0e7 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala @@ -19,7 +19,7 @@ package com.microsoft.hyperspace.index import scala.collection.mutable.WrappedArray import org.apache.hadoop.fs.Path -import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.plans.SQLHelper import com.microsoft.hyperspace.{Hyperspace, HyperspaceException, SampleData} @@ -28,13 +28,16 @@ import com.microsoft.hyperspace.util.FileUtils class CreateIndexTests extends HyperspaceSuite with SQLHelper { override val systemPath = new Path("src/test/resources/indexLocation") private val sampleData = SampleData.testData - private val sampleParquetDataLocation = "src/test/resources/sampleparquet" + private val sampleNonPartitionedParquetDataLocation = "src/test/resources/sampleparquet" private val samplePartitionedParquetDataLocation = "src/test/resources/samplepartitionedparquet" + private val partitionKey1 = "Date" + private val partitionKey2 = "Query" private val indexConfig1 = IndexConfig("index1", Seq("RGUID"), Seq("Date")) private val indexConfig2 = IndexConfig("index2", Seq("Query"), Seq("imprs")) private val indexConfig3 = IndexConfig("index3", Seq("imprs"), Seq("clicks")) - private val indexConfig4 = IndexConfig("index3", Seq("Date", "Query"), Seq("clicks")) - private var df: DataFrame = _ + private val indexConfig4 = IndexConfig("index4", Seq("Date", "Query"), Seq("clicks")) + private var nonPartitionedDataDF: DataFrame = _ + private var partitionedDataDF: DataFrame = _ private var hyperspace: Hyperspace = _ override def beforeAll(): Unit = { @@ -43,34 +46,39 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { val sparkSession = spark import sparkSession.implicits._ hyperspace = new Hyperspace(sparkSession) - FileUtils.delete(new Path(sampleParquetDataLocation)) + FileUtils.delete(new Path(sampleNonPartitionedParquetDataLocation)) FileUtils.delete(new Path(samplePartitionedParquetDataLocation)) // save test non-partitioned. val dfFromSample = sampleData.toDF("Date", "RGUID", "Query", "imprs", "clicks") - dfFromSample.write.parquet(sampleParquetDataLocation) + dfFromSample.write.parquet(sampleNonPartitionedParquetDataLocation) + nonPartitionedDataDF = spark.read.parquet(sampleNonPartitionedParquetDataLocation) // save test data partitioned. // `Date` is the first partition key and `Query` is the second partition key. - dfFromSample.select("Date").distinct().collectAsList().forEach { d => + dfFromSample.select(partitionKey1).distinct().collectAsList().forEach { d => val date = d.get(0) - dfFromSample.filter($"date" === date).select("Query").distinct().collectAsList().forEach { - q => + dfFromSample + .filter($"date" === date) + .select(partitionKey2) + .distinct() + .collectAsList() + .forEach { q => val query = q.get(0) - val partitionPath = s"$samplePartitionedParquetDataLocation/Date=$date/Query=$query" + val partitionPath = + s"$samplePartitionedParquetDataLocation/$partitionKey1=$date/$partitionKey2=$query" dfFromSample .filter($"date" === date && $"Query" === query) .select("RGUID", "imprs", "clicks") .write .parquet(partitionPath) - } + } } - - df = spark.read.parquet(sampleParquetDataLocation) + partitionedDataDF = spark.read.parquet(samplePartitionedParquetDataLocation) } override def afterAll(): Unit = { - FileUtils.delete(new Path(sampleParquetDataLocation)) + FileUtils.delete(new Path(sampleNonPartitionedParquetDataLocation)) FileUtils.delete(new Path(samplePartitionedParquetDataLocation)) super.afterAll() } @@ -80,36 +88,40 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { } test("Creating one index.") { - hyperspace.createIndex(df, indexConfig1) + hyperspace.createIndex(nonPartitionedDataDF, indexConfig1) val count = hyperspace.indexes.where(s"name = '${indexConfig1.indexName}' ").count assert(count == 1) } test("Creating index with existing index name fails.") { - hyperspace.createIndex(df, indexConfig1) + hyperspace.createIndex(nonPartitionedDataDF, indexConfig1) val exception = intercept[HyperspaceException] { - hyperspace.createIndex(df, indexConfig2.copy(indexName = "index1")) + hyperspace.createIndex(nonPartitionedDataDF, indexConfig2.copy(indexName = "index1")) } assert(exception.getMessage.contains("Another Index with name index1 already exists")) } test("Creating index with existing index name (case-insensitive) fails.") { - hyperspace.createIndex(df, indexConfig1) + hyperspace.createIndex(nonPartitionedDataDF, indexConfig1) val exception = intercept[HyperspaceException] { - hyperspace.createIndex(df, indexConfig1.copy(indexName = "INDEX1")) + hyperspace.createIndex(nonPartitionedDataDF, indexConfig1.copy(indexName = "INDEX1")) } assert(exception.getMessage.contains("Another Index with name INDEX1 already exists")) } test("Index creation fails since indexConfig does not satisfy the table schema.") { val exception = intercept[HyperspaceException] { - hyperspace.createIndex(df, IndexConfig("index1", Seq("IllegalColA"), Seq("IllegalColB"))) + hyperspace.createIndex( + nonPartitionedDataDF, + IndexConfig("index1", Seq("IllegalColA"), Seq("IllegalColB"))) } assert(exception.getMessage.contains("Index config is not applicable to dataframe schema")) } test("Index creation passes with columns of different case if case-sensitivity is false.") { - hyperspace.createIndex(df, IndexConfig("index1", Seq("qUeRy"), Seq("ImpRS"))) + hyperspace.createIndex( + nonPartitionedDataDF, + IndexConfig("index1", Seq("qUeRy"), Seq("ImpRS"))) val indexes = hyperspace.indexes.where(s"name = '${indexConfig1.indexName}' ") assert(indexes.count == 1) assert( @@ -123,14 +135,16 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { test("Index creation fails with columns of different case if case-sensitivity is true.") { withSQLConf("spark.sql.caseSensitive" -> "true") { val exception = intercept[HyperspaceException] { - hyperspace.createIndex(df, IndexConfig("index1", Seq("qUeRy"), Seq("ImpRS"))) + hyperspace.createIndex( + nonPartitionedDataDF, + IndexConfig("index1", Seq("qUeRy"), Seq("ImpRS"))) } assert(exception.getMessage.contains("Index config is not applicable to dataframe schema.")) } } test("Index creation fails since the dataframe has a filter node.") { - val dfWithFilter = df.filter("Query='facebook'") + val dfWithFilter = nonPartitionedDataDF.filter("Query='facebook'") val exception = intercept[HyperspaceException] { hyperspace.createIndex(dfWithFilter, indexConfig1) } @@ -140,7 +154,7 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { } test("Index creation fails since the dataframe has a projection node.") { - val dfWithSelect = df.select("Query") + val dfWithSelect = nonPartitionedDataDF.select("Query") val exception = intercept[HyperspaceException] { hyperspace.createIndex(dfWithSelect, indexConfig1) } @@ -150,9 +164,12 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { } test("Index creation fails since the dataframe has a join node.") { - val dfJoin = df - .join(df, df("Query") === df("Query")) - .select(df("RGUID"), df("Query"), df("imprs")) + val dfJoin = nonPartitionedDataDF + .join(nonPartitionedDataDF, nonPartitionedDataDF("Query") === nonPartitionedDataDF("Query")) + .select( + nonPartitionedDataDF("RGUID"), + nonPartitionedDataDF("Query"), + nonPartitionedDataDF("imprs")) val exception = intercept[HyperspaceException] { hyperspace.createIndex(dfJoin, indexConfig1) } @@ -161,20 +178,53 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { "Only creating index over HDFS file based scan nodes is supported.")) } - test("Check lineage in index records for non-partitioned data") {} + test("Check lineage in index records for non-partitioned data.") { + hyperspace.createIndex(nonPartitionedDataDF, indexConfig1) + val indexRecordsDF = spark.read.parquet( + s"$systemPath/${indexConfig1.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") + + // For non-partitioned data, only file name lineage column should be added to index schema. + indexRecordsDF.schema.fields.corresponds( + indexConfig1.indexedColumns ++ indexConfig1.includedColumns ++ + Seq(IndexConstants.DATA_FILE_NAME_COLUMN))(_.name.equals(_)) + } - test( - "Check lineage in index records for partitioned data when partition key is not in config") { + test("Check lineage in index records for partitioned data when partition key is not in config.") { + hyperspace.createIndex(partitionedDataDF, indexConfig3) + val indexRecordsDF = spark.read.parquet( + s"$systemPath/${indexConfig3.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") + // For partitioned data, beside file name lineage column all partition keys columns + // should be added to index schema if they are not already among index config columns. + indexRecordsDF.schema.fields.corresponds( + indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ + Seq(IndexConstants.DATA_FILE_NAME_COLUMN, partitionKey1, partitionKey2))(_.name.equals(_)) } - test( - "Check lineage in index records for partitioned data when partition key is in config") { + test("Check lineage in index records for partitioned data when partition key is in config.") { + hyperspace.createIndex(partitionedDataDF, indexConfig4) + val indexRecordsDF = spark.read.parquet( + s"$systemPath/${indexConfig4.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") + // For partitioned data, if partition keys are already in index config columns, + // there should be no duplicates due to adding lineage. + indexRecordsDF.schema.fields.corresponds( + indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ + Seq(IndexConstants.DATA_FILE_NAME_COLUMN))(_.name.equals(_)) } - test( - "Check lineage in index records for partitioned data when partition key is load path") { + test("Check lineage in index records for partitioned data when partition key is in load path.") { + val dataDF = + spark.read.parquet(s"$samplePartitionedParquetDataLocation/$partitionKey1=2017-09-03") + hyperspace.createIndex(dataDF, indexConfig3) + val indexRecordsDF = spark.read.parquet( + s"$systemPath/${indexConfig3.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") + + // As data load path includes first partition key, index schema should only contain + // file name column and second partition key column as lineage columns. + indexRecordsDF.schema.fields.corresponds( + indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ + Seq(IndexConstants.DATA_FILE_NAME_COLUMN, partitionKey2))(_.name.equals(_)) } } From 6522e0a2447b925b16db953baf3380d2731712fb Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Mon, 10 Aug 2020 17:01:28 -0700 Subject: [PATCH 11/21] remove redundant import --- .../com/microsoft/hyperspace/index/rules/FilterIndexRule.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala b/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala index 07431b163..8b7260fc0 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.types.StructType import com.microsoft.hyperspace.{ActiveSparkSession, Hyperspace} -import com.microsoft.hyperspace.index.{IndexConstants, IndexLogEntry} +import com.microsoft.hyperspace.index.IndexLogEntry import com.microsoft.hyperspace.telemetry.{AppInfo, HyperspaceEventLogging, HyperspaceIndexUsageEvent} import com.microsoft.hyperspace.util.ResolverUtils From b55c8841b6ed2ae2a475ec0fcb61400da7047728 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Tue, 11 Aug 2020 11:35:57 -0700 Subject: [PATCH 12/21] nit fix in test code --- .../scala/com/microsoft/hyperspace/index/CreateIndexTests.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala index 96976d0e7..e81faa203 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala @@ -225,6 +225,5 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { indexRecordsDF.schema.fields.corresponds( indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ Seq(IndexConstants.DATA_FILE_NAME_COLUMN, partitionKey2))(_.name.equals(_)) - } } From a5ec6d9079d001dd6bc3f914e85b3f1c5dc03ae8 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Tue, 11 Aug 2020 14:55:16 -0700 Subject: [PATCH 13/21] Change in CreateIndex tests --- .../com/microsoft/hyperspace/index/CreateIndexTests.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala index e81faa203..49d4235d6 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala @@ -56,14 +56,14 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { // save test data partitioned. // `Date` is the first partition key and `Query` is the second partition key. - dfFromSample.select(partitionKey1).distinct().collectAsList().forEach { d => + dfFromSample.select(partitionKey1).distinct().collect().foreach { d => val date = d.get(0) dfFromSample .filter($"date" === date) .select(partitionKey2) .distinct() - .collectAsList() - .forEach { q => + .collect() + .foreach { q => val query = q.get(0) val partitionPath = s"$samplePartitionedParquetDataLocation/$partitionKey1=$date/$partitionKey2=$query" From a9ed33733ff7139e829a61154ba619fe4959118d Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Mon, 17 Aug 2020 19:16:30 -0700 Subject: [PATCH 14/21] add lineage to index records with tests --- .../hyperspace/actions/CreateActionBase.scala | 59 +++++--- .../hyperspace/index/IndexConstants.scala | 2 + .../index/rules/FilterIndexRule.scala | 2 +- .../com/microsoft/hyperspace/SampleData.scala | 35 +++++ .../hyperspace/index/CreateIndexTests.scala | 79 +++++----- .../index/E2EHyperspaceRulesTests.scala | 137 ++++++++++++++---- .../hyperspace/index/IndexManagerTests.scala | 47 +++--- 7 files changed, 253 insertions(+), 108 deletions(-) diff --git a/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala b/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala index 3dee75096..023503080 100644 --- a/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala +++ b/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala @@ -20,6 +20,7 @@ import org.apache.commons.io.FilenameUtils import org.apache.hadoop.fs.Path import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, PartitioningAwareFileIndex} +import org.apache.spark.sql.expressions.UserDefinedFunction import org.apache.spark.sql.functions.{input_file_name, udf} import org.apache.spark.sql.sources.DataSourceRegister @@ -53,14 +54,8 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) val signatureProvider = LogicalPlanSignatureProvider.create() // Resolve the passed column names with existing column names from the dataframe. - val (resolvedIndexedColumns, resolvedIncludedColumns) = resolveConfig(df, indexConfig) - val schema = { - val indexConfigColumns = resolvedIndexedColumns ++ resolvedIncludedColumns - val allColumns = indexConfigColumns ++ internalColumns(df, indexConfigColumns) - df.select(allColumns.head, allColumns.tail: _*) - .withColumn(IndexConstants.DATA_FILE_NAME_COLUMN, fileName(input_file_name())) - .schema - } + val (indexDataFrame, resolvedIndexedColumns, resolvedIncludedColumns) = + getIndexDataFrame(spark, df, indexConfig) signatureProvider.signature(df.queryExecution.optimizedPlan) match { case Some(s) => @@ -81,7 +76,7 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) CoveringIndex.Properties( CoveringIndex.Properties .Columns(resolvedIndexedColumns, resolvedIncludedColumns), - IndexLogEntry.schemaString(schema), + IndexLogEntry.schemaString(indexDataFrame.schema), numBuckets)), Content(path.toString, Seq()), Source(SparkPlan(sourcePlanProperties)), @@ -130,12 +125,7 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) IndexConstants.INDEX_NUM_BUCKETS_DEFAULT.toString) .toInt - val (resolvedIndexedColumns, resolvedIncludedColumns) = resolveConfig(df, indexConfig) - val indexConfigColumns = resolvedIndexedColumns ++ resolvedIncludedColumns - val selectedColumns = indexConfigColumns ++ internalColumns(df, indexConfigColumns) - val indexDataFrame = df - .select(selectedColumns.head, selectedColumns.tail: _*) - .withColumn(IndexConstants.DATA_FILE_NAME_COLUMN, fileName(input_file_name())) + val (indexDataFrame, resolvedIndexedColumns, _) = getIndexDataFrame(spark, df, indexConfig) // run job val repartitionedIndexDataFrame = @@ -172,16 +162,43 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) } } - private def internalColumns(df: DataFrame, indexConfigColumns: Seq[String]): Seq[String] = { + private def getIndexDataFrame( + spark: SparkSession, + df: DataFrame, + indexConfig: IndexConfig): (DataFrame, Seq[String], Seq[String]) = { + val (resolvedIndexedColumns, resolvedIncludedColumns) = resolveConfig(df, indexConfig) + val indexConfigColumns = resolvedIndexedColumns ++ resolvedIncludedColumns + val addLineage = spark.sessionState.conf + .getConfString( + IndexConstants.INDEX_LINEAGE_ENABLED, + IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) + .toBoolean + + val indexDF = if (addLineage) { + // Lineage is captured using two sets of columns: + // 1. DATA_FILE_NAME_COLUMN column contains source data filename for each index record. + // 2. If source data is partitioned, all partitioning key(s) are added to index schema. + val additionalPartitioningColumns = getPartitioningColumns(df).filter( + ResolverUtils.resolve(spark, _, indexConfigColumns).isEmpty) + val dataFrameColumns = indexConfigColumns ++ additionalPartitioningColumns + df.select(dataFrameColumns.head, dataFrameColumns.tail: _*) + .withColumn(IndexConstants.DATA_FILE_NAME_COLUMN, getFileName(input_file_name())) + } else { + df.select(indexConfigColumns.head, indexConfigColumns.tail: _*) + } + + (indexDF, resolvedIndexedColumns, resolvedIncludedColumns) + } + + private def getPartitioningColumns(df: DataFrame): Seq[String] = { + // Extract partitioning keys, if original data is partitioned. val partitionSchema = df.queryExecution.optimizedPlan.collect { case LogicalRelation(HadoopFsRelation(_, pSchema, _, _, _, _), _, _, _) => pSchema } - val spark = df.sparkSession - partitionSchema.head - .map(_.name) - .filter(ResolverUtils.resolve(spark, _, indexConfigColumns).isEmpty) + partitionSchema.head.map(_.name) } - private val fileName = udf((fullFilePath: String) => FilenameUtils.getBaseName(fullFilePath)) + private val getFileName: UserDefinedFunction = udf( + (fullFilePath: String) => FilenameUtils.getBaseName(fullFilePath)) } diff --git a/src/main/scala/com/microsoft/hyperspace/index/IndexConstants.scala b/src/main/scala/com/microsoft/hyperspace/index/IndexConstants.scala index ca9592d2d..60d1fd987 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/IndexConstants.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/IndexConstants.scala @@ -49,4 +49,6 @@ object IndexConstants { } val DATA_FILE_NAME_COLUMN = "_data_file_name" + val INDEX_LINEAGE_ENABLED = "spark.hyperspace.index.lineage.enabled" + val INDEX_LINEAGE_ENABLED_DEFAULT = "false" } diff --git a/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala b/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala index 8b7260fc0..b2d82fe15 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/rules/FilterIndexRule.scala @@ -109,7 +109,7 @@ object FilterIndexRule val newRelation = HadoopFsRelation( newLocation, new StructType(), - StructType(index.schema.filter(fsRelation.schema.contains(_))), + StructType(index.schema.filter(fsRelation.schema.contains)), None, // Do not set BucketSpec to avoid limiting Spark's degree of parallelism new ParquetFileFormat, Map())(spark) diff --git a/src/test/scala/com/microsoft/hyperspace/SampleData.scala b/src/test/scala/com/microsoft/hyperspace/SampleData.scala index 290eb4f46..a7afe8cc8 100644 --- a/src/test/scala/com/microsoft/hyperspace/SampleData.scala +++ b/src/test/scala/com/microsoft/hyperspace/SampleData.scala @@ -16,6 +16,8 @@ package com.microsoft.hyperspace +import org.apache.spark.sql.SparkSession + /** * Sample data for testing. */ @@ -31,4 +33,37 @@ object SampleData { ("2019-10-03", "380786e6495d4cd8a5dd4cc8d3d12917", "facebook", 2, 3000), ("2019-10-03", "ff60e4838b92421eafc3e6ee59a9e9f1", "mi perro", 2, 2000), ("2019-10-03", "187696fe0a6a40cc9516bc6e47c70bc1", "facebook", 4, 3000)) + + // column names and partition keys for partitioned case + val colnames = Seq("Date", "RGUID", "Query", "imprs", "clicks") + val partitionKey1 = "Date" + val partitionKey2 = "Query" + + def saveTestDataNonPartitioned(spark: SparkSession, path: String, colNames: String*): Unit = { + import spark.implicits._ + assert(colNames.length == 5) + testData.toDF(colNames: _*).write.parquet(path) + } + + def saveTestDataPartitioned(spark: SparkSession, path: String): Unit = { + // `Date` is the first partition key and `Query` is the second partition key. + import spark.implicits._ + val df = testData.toDF(colnames: _*) + df.select(partitionKey1).distinct().collect().foreach { d => + val date = d.get(0) + df.filter($"Date" === date) + .select(partitionKey2) + .distinct() + .collect() + .foreach { q => + val query = q.get(0) + val partitionPath = + s"$path/$partitionKey1=$date/$partitionKey2=$query" + df.filter($"Date" === date && $"Query" === query) + .select("RGUID", "imprs", "clicks") + .write + .parquet(partitionPath) + } + } + } } diff --git a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala index 49d4235d6..abd93c585 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala @@ -27,11 +27,8 @@ import com.microsoft.hyperspace.util.FileUtils class CreateIndexTests extends HyperspaceSuite with SQLHelper { override val systemPath = new Path("src/test/resources/indexLocation") - private val sampleData = SampleData.testData private val sampleNonPartitionedParquetDataLocation = "src/test/resources/sampleparquet" private val samplePartitionedParquetDataLocation = "src/test/resources/samplepartitionedparquet" - private val partitionKey1 = "Date" - private val partitionKey2 = "Query" private val indexConfig1 = IndexConfig("index1", Seq("RGUID"), Seq("Date")) private val indexConfig2 = IndexConfig("index2", Seq("Query"), Seq("imprs")) private val indexConfig3 = IndexConfig("index3", Seq("imprs"), Seq("clicks")) @@ -44,36 +41,23 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { super.beforeAll() val sparkSession = spark - import sparkSession.implicits._ hyperspace = new Hyperspace(sparkSession) FileUtils.delete(new Path(sampleNonPartitionedParquetDataLocation)) FileUtils.delete(new Path(samplePartitionedParquetDataLocation)) // save test non-partitioned. - val dfFromSample = sampleData.toDF("Date", "RGUID", "Query", "imprs", "clicks") - dfFromSample.write.parquet(sampleNonPartitionedParquetDataLocation) + SampleData.saveTestDataNonPartitioned( + spark, + sampleNonPartitionedParquetDataLocation, + "Date", + "RGUID", + "Query", + "imprs", + "clicks") nonPartitionedDataDF = spark.read.parquet(sampleNonPartitionedParquetDataLocation) // save test data partitioned. - // `Date` is the first partition key and `Query` is the second partition key. - dfFromSample.select(partitionKey1).distinct().collect().foreach { d => - val date = d.get(0) - dfFromSample - .filter($"date" === date) - .select(partitionKey2) - .distinct() - .collect() - .foreach { q => - val query = q.get(0) - val partitionPath = - s"$samplePartitionedParquetDataLocation/$partitionKey1=$date/$partitionKey2=$query" - dfFromSample - .filter($"date" === date && $"Query" === query) - .select("RGUID", "imprs", "clicks") - .write - .parquet(partitionPath) - } - } + SampleData.saveTestDataPartitioned(spark, samplePartitionedParquetDataLocation) partitionedDataDF = spark.read.parquet(samplePartitionedParquetDataLocation) } @@ -179,51 +163,70 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { } test("Check lineage in index records for non-partitioned data.") { + spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") hyperspace.createIndex(nonPartitionedDataDF, indexConfig1) val indexRecordsDF = spark.read.parquet( s"$systemPath/${indexConfig1.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") // For non-partitioned data, only file name lineage column should be added to index schema. - indexRecordsDF.schema.fields.corresponds( - indexConfig1.indexedColumns ++ indexConfig1.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN))(_.name.equals(_)) + assert( + indexRecordsDF.schema.fieldNames.sorted.corresponds( + (indexConfig1.indexedColumns ++ indexConfig1.includedColumns ++ + Seq(IndexConstants.DATA_FILE_NAME_COLUMN)).sorted)(_.equals(_))) + spark.conf + .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) } test("Check lineage in index records for partitioned data when partition key is not in config.") { + spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") hyperspace.createIndex(partitionedDataDF, indexConfig3) val indexRecordsDF = spark.read.parquet( s"$systemPath/${indexConfig3.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") // For partitioned data, beside file name lineage column all partition keys columns // should be added to index schema if they are not already among index config columns. - indexRecordsDF.schema.fields.corresponds( - indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN, partitionKey1, partitionKey2))(_.name.equals(_)) + assert( + indexRecordsDF.schema.fieldNames.sorted.corresponds( + (indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ + Seq( + IndexConstants.DATA_FILE_NAME_COLUMN, + SampleData.partitionKey1, + SampleData.partitionKey2)).sorted)(_.equals(_))) + spark.conf + .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) } test("Check lineage in index records for partitioned data when partition key is in config.") { + spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") hyperspace.createIndex(partitionedDataDF, indexConfig4) val indexRecordsDF = spark.read.parquet( s"$systemPath/${indexConfig4.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") // For partitioned data, if partition keys are already in index config columns, // there should be no duplicates due to adding lineage. - indexRecordsDF.schema.fields.corresponds( - indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN))(_.name.equals(_)) + assert( + indexRecordsDF.schema.fieldNames.sorted.corresponds( + (indexConfig4.indexedColumns ++ indexConfig4.includedColumns ++ + Seq(IndexConstants.DATA_FILE_NAME_COLUMN)).sorted)(_.equals(_))) + spark.conf + .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) } test("Check lineage in index records for partitioned data when partition key is in load path.") { + spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") val dataDF = - spark.read.parquet(s"$samplePartitionedParquetDataLocation/$partitionKey1=2017-09-03") + spark.read.parquet( + s"$samplePartitionedParquetDataLocation/${SampleData.partitionKey1}=2017-09-03") hyperspace.createIndex(dataDF, indexConfig3) val indexRecordsDF = spark.read.parquet( s"$systemPath/${indexConfig3.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") // As data load path includes first partition key, index schema should only contain // file name column and second partition key column as lineage columns. - indexRecordsDF.schema.fields.corresponds( - indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN, partitionKey2))(_.name.equals(_)) + assert(indexRecordsDF.schema.fieldNames.sorted.corresponds( + (indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ + Seq(IndexConstants.DATA_FILE_NAME_COLUMN, SampleData.partitionKey2)).sorted)(_.equals(_))) + spark.conf + .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) } } diff --git a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala index 3b8f09ab7..557813d7b 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala @@ -27,25 +27,34 @@ import com.microsoft.hyperspace.{Hyperspace, Implicits, SampleData} import com.microsoft.hyperspace.index.rules.{FilterIndexRule, JoinIndexRule} class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { - private val sampleData = SampleData.testData private val testDir = "src/test/resources/e2eTests/" - private val sampleParquetDataLocation = testDir + "sampleparquet" + private val sampleNonPartitionedParquetDataLocation = testDir + "sampleparquet" + private val samplePartitionedParquetDataLocation = testDir + "samplepartitionedparquet" override val systemPath = new Path(testDir + "indexLocation") - private val fileSystem = new Path(sampleParquetDataLocation).getFileSystem(new Configuration) + private val fileSystem = + new Path(sampleNonPartitionedParquetDataLocation).getFileSystem(new Configuration) private var hyperspace: Hyperspace = _ override def beforeAll(): Unit = { super.beforeAll() spark.conf.set("spark.sql.autoBroadcastJoinThreshold", -1) - - import spark.implicits._ hyperspace = new Hyperspace(spark) - - fileSystem.delete(new Path(sampleParquetDataLocation), true) - - val dfFromSample = sampleData.toDF("c1", "c2", "c3", "c4", "c5") - dfFromSample.write.parquet(sampleParquetDataLocation) + fileSystem.delete(new Path(sampleNonPartitionedParquetDataLocation), true) + fileSystem.delete(new Path(samplePartitionedParquetDataLocation), true) + + // save test non-partitioned. + SampleData.saveTestDataNonPartitioned( + spark, + sampleNonPartitionedParquetDataLocation, + "c1", + "c2", + "c3", + "c4", + "c5") + + // save test partitioned. + SampleData.saveTestDataPartitioned(spark, samplePartitionedParquetDataLocation) } before { @@ -81,8 +90,8 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { .containsSlice(expectedOptimizationRuleBatch)) } - test("E2E test for filter query.") { - val df = spark.read.parquet(sampleParquetDataLocation) + test("E2E test for filter query on non-partitioned data.") { + val df = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val indexConfig = IndexConfig("filterIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(df, indexConfig) @@ -92,8 +101,24 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) } + test("E2E test for filter query on partitioned data with lineage.") { + spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") + + val df = spark.read.parquet(samplePartitionedParquetDataLocation) + val indexConfig = IndexConfig("filterIndex", Seq("Query"), Seq("Date")) + + hyperspace.createIndex(df, indexConfig) + + def query(): DataFrame = df.filter("Query == 'facebook'").select("Query", "Date") + + verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) + + spark.conf + .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) + } + test("E2E test for case insensitive filter query utilizing indexes.") { - val df = spark.read.parquet(sampleParquetDataLocation) + val df = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val indexConfig = IndexConfig("filterIndex", Seq("C3"), Seq("C1")) hyperspace.createIndex(df, indexConfig) @@ -105,7 +130,7 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } test("E2E test for case sensitive filter query where changing conf changes behavior.") { - val df = spark.read.parquet(sampleParquetDataLocation) + val df = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val indexConfig = IndexConfig("filterIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(df, indexConfig) @@ -122,8 +147,8 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } } - test("E2E test for filter query when all columns are selected.") { - val df = spark.read.parquet(sampleParquetDataLocation) + test("E2E test for filter query when all columns are selected on non-partitioned data.") { + val df = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val indexConfig = IndexConfig("filterIndex", Seq("c4", "c3"), Seq("c1", "c2", "c5")) hyperspace.createIndex(df, indexConfig) @@ -137,13 +162,35 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) } - test("E2E test for join query.") { - val leftDf = spark.read.parquet(sampleParquetDataLocation) + test( + "E2E test for filter query when all columns are selected on partitioned data with lineage.") { + spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") + + val df = spark.read.parquet(samplePartitionedParquetDataLocation) + val indexConfig = + IndexConfig("filterIndex", Seq("imprs", "Query"), Seq("Date", "RGUID", "clicks")) + + hyperspace.createIndex(df, indexConfig) + df.createOrReplaceTempView("t") + + def query(): DataFrame = spark.sql("SELECT * from t where imprs = 1") + + // Verify no Project node is present in the query plan, as a result of using SELECT * + assert(query().queryExecution.optimizedPlan.collect { case p: Project => p }.isEmpty) + + verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) + + spark.conf + .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) + } + + test("E2E test for join query on non-partitioned data.") { + val leftDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val leftDfIndexConfig = IndexConfig("leftIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(leftDf, leftDfIndexConfig) - val rightDf = spark.read.parquet(sampleParquetDataLocation) + val rightDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("c4")) hyperspace.createIndex(rightDf, rightDfIndexConfig) @@ -158,12 +205,40 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { getIndexFilesPath(rightDfIndexConfig.indexName))) } + test("E2E test for join query on partitioned data with lineage.") { + spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") + + val leftDf = spark.read.parquet(samplePartitionedParquetDataLocation) + val leftDfIndexConfig = IndexConfig("leftIndex", Seq("Query"), Seq("Date")) + + hyperspace.createIndex(leftDf, leftDfIndexConfig) + + val rightDf = spark.read.parquet(samplePartitionedParquetDataLocation) + val rightDfIndexConfig = IndexConfig("rightIndex", Seq("Query"), Seq("imprs")) + hyperspace.createIndex(rightDf, rightDfIndexConfig) + + def query(): DataFrame = { + leftDf + .join(rightDf, leftDf("Query") === rightDf("Query")) + .select(leftDf("Date"), rightDf("imprs")) + } + + verifyIndexUsage( + query, + Seq( + getIndexFilesPath(leftDfIndexConfig.indexName), + getIndexFilesPath(rightDfIndexConfig.indexName))) + + spark.conf + .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) + } + test("E2E test for join query with case-insensitive column names.") { - val leftDf = spark.read.parquet(sampleParquetDataLocation) + val leftDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val leftDfIndexConfig = IndexConfig("leftIndex", Seq("C3"), Seq("c1")) hyperspace.createIndex(leftDf, leftDfIndexConfig) - val rightDf = spark.read.parquet(sampleParquetDataLocation) + val rightDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("C4")) hyperspace.createIndex(rightDf, rightDfIndexConfig) @@ -187,11 +262,11 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } withView("t1", "t2") { - val leftDf = spark.read.parquet(sampleParquetDataLocation) + val leftDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val leftDfIndexConfig = IndexConfig("leftIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(leftDf, leftDfIndexConfig) - val rightDf = spark.read.parquet(sampleParquetDataLocation) + val rightDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("c4")) hyperspace.createIndex(rightDf, rightDfIndexConfig) @@ -214,11 +289,11 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { test("E2E test for join query on catalog temp tables/views") { withView("t1", "t2") { - val leftDf = spark.read.parquet(sampleParquetDataLocation) + val leftDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val leftDfIndexConfig = IndexConfig("leftIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(leftDf, leftDfIndexConfig) - val rightDf = spark.read.parquet(sampleParquetDataLocation) + val rightDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("c4")) hyperspace.createIndex(rightDf, rightDfIndexConfig) @@ -244,13 +319,13 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { |CREATE EXTERNAL TABLE t1 |(c1 string, c3 string) |STORED AS PARQUET - |LOCATION '$sampleParquetDataLocation' + |LOCATION '$sampleNonPartitionedParquetDataLocation' """.stripMargin) spark.sql(s""" |CREATE EXTERNAL TABLE t2 |(c3 string, c4 int) |STORED AS PARQUET - |LOCATION '$sampleParquetDataLocation' + |LOCATION '$sampleNonPartitionedParquetDataLocation' """.stripMargin) val leftDf = spark.table("t1") @@ -276,7 +351,7 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { withTable("t1", "t2") { val table1Location = testDir + "tables/t1" val table2Location = testDir + "tables/t2" - val originalDf = spark.read.parquet(sampleParquetDataLocation) + val originalDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) originalDf.select("c1", "c3").write.option("path", table1Location).saveAsTable("t1") originalDf.select("c3", "c4").write.option("path", table2Location).saveAsTable("t2") @@ -298,14 +373,14 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } test("E2E test for join query with two child sub-query as both filter query.") { - val leftDf = spark.read.parquet(sampleParquetDataLocation) + val leftDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val leftDfJoinIndexConfig = IndexConfig("leftJoinIndex", Seq("c3"), Seq("c4")) hyperspace.createIndex(leftDf, leftDfJoinIndexConfig) val leftDfFilterIndexConfig = IndexConfig("leftDfFilterIndex", Seq("c4"), Seq("c3")) hyperspace.createIndex(leftDf, leftDfFilterIndexConfig) - val rightDf = spark.read.parquet(sampleParquetDataLocation) + val rightDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val rightDfJoinIndexConfig = IndexConfig("rightDfJoinIndex", Seq("c3"), Seq("c5")) hyperspace.createIndex(rightDf, rightDfJoinIndexConfig) @@ -330,7 +405,7 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } test("E2E test for first enableHyperspace() followed by disableHyperspace().") { - val df = spark.read.parquet(sampleParquetDataLocation) + val df = spark.read.parquet(sampleNonPartitionedParquetDataLocation) val indexConfig = IndexConfig("filterIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(df, indexConfig) diff --git a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala index d6df1c14b..3ffcd6b15 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala @@ -61,23 +61,12 @@ class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite { super.afterAll() } - test("Verify that indexes() returns the correct dataframe.") { - import spark.implicits._ - hyperspace.createIndex(df, indexConfig1) - val actual = hyperspace.indexes.as[IndexSummary].collect()(0) - val expected = new IndexSummary( - indexConfig1.indexName, - indexConfig1.indexedColumns, - indexConfig1.includedColumns, - 200, - StructType( - Seq( - StructField("RGUID", StringType), - StructField("Date", StringType), - StructField(IndexConstants.DATA_FILE_NAME_COLUMN, StringType))).json, - s"$indexStorageLocation/index1/v__=0", - Constants.States.ACTIVE) - assert(actual.equals(expected)) + test("Verify that indexes() returns the correct dataframe without lineage.") { + verifyIndexesOutput(false) + } + + test("Verify that indexes() returns the correct dataframe with lineage.") { + verifyIndexesOutput(true) } test("Verify getIndexes()") { @@ -301,4 +290,28 @@ class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite { private def verifyIndexes(expectedIndexes: Seq[IndexLogEntry]): Unit = { assert(IndexCollectionManager(spark).getIndexes().toSet == expectedIndexes.toSet) } + + private def verifyIndexesOutput(enableLineage: Boolean): Unit = { + spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, if (enableLineage) "true" else "false") + import spark.implicits._ + hyperspace.createIndex(df, indexConfig1) + val actual = hyperspace.indexes.as[IndexSummary].collect()(0) + var expectedSchema = StructType( + Seq(StructField("RGUID", StringType), StructField("Date", StringType))) + if (enableLineage) { + expectedSchema = + expectedSchema.add(StructField(IndexConstants.DATA_FILE_NAME_COLUMN, StringType)) + } + val expected = new IndexSummary( + indexConfig1.indexName, + indexConfig1.indexedColumns, + indexConfig1.includedColumns, + 200, + expectedSchema.json, + s"$indexStorageLocation/index1/v__=0", + Constants.States.ACTIVE) + assert(actual.equals(expected)) + spark.conf + .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) + } } From 208357157ad22165f50be1d3a59c41004105468e Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Thu, 20 Aug 2020 12:40:51 -0700 Subject: [PATCH 15/21] add lineage to index records --- .../hyperspace/actions/CreateActionBase.scala | 31 +++-- .../com/microsoft/hyperspace/SampleData.scala | 33 ++--- .../hyperspace/index/CreateIndexTests.scala | 126 +++++++++--------- .../index/E2EHyperspaceRulesTests.scala | 93 ++++++------- .../hyperspace/index/IndexManagerTests.scala | 57 ++++---- 5 files changed, 158 insertions(+), 182 deletions(-) diff --git a/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala b/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala index e5f201aea..411a6c0ed 100644 --- a/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala +++ b/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala @@ -61,7 +61,7 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) // Resolve the passed column names with existing column names from the dataframe. val (indexDataFrame, resolvedIndexedColumns, resolvedIncludedColumns) = - getIndexDataFrame(spark, df, indexConfig) + prepareIndexDataFrame(spark, df, indexConfig) signatureProvider.signature(df.queryExecution.optimizedPlan) match { case Some(s) => @@ -133,7 +133,7 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) IndexConstants.INDEX_NUM_BUCKETS_DEFAULT.toString) .toInt - val (indexDataFrame, resolvedIndexedColumns, _) = getIndexDataFrame(spark, df, indexConfig) + val (indexDataFrame, resolvedIndexedColumns, _) = prepareIndexDataFrame(spark, df, indexConfig) // run job val repartitionedIndexDataFrame = @@ -186,12 +186,12 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) } } - private def getIndexDataFrame( + private def prepareIndexDataFrame( spark: SparkSession, df: DataFrame, indexConfig: IndexConfig): (DataFrame, Seq[String], Seq[String]) = { val (resolvedIndexedColumns, resolvedIncludedColumns) = resolveConfig(df, indexConfig) - val indexConfigColumns = resolvedIndexedColumns ++ resolvedIncludedColumns + val columnsFromIndexConfig = resolvedIndexedColumns ++ resolvedIncludedColumns val addLineage = spark.sessionState.conf .getConfString( IndexConstants.INDEX_LINEAGE_ENABLED, @@ -201,26 +201,29 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) val indexDF = if (addLineage) { // Lineage is captured using two sets of columns: // 1. DATA_FILE_NAME_COLUMN column contains source data filename for each index record. - // 2. If source data is partitioned, all partitioning key(s) are added to index schema. - val additionalPartitioningColumns = getPartitioningColumns(df).filter( - ResolverUtils.resolve(spark, _, indexConfigColumns).isEmpty) - val dataFrameColumns = indexConfigColumns ++ additionalPartitioningColumns - df.select(dataFrameColumns.head, dataFrameColumns.tail: _*) + // 2. If source data is partitioned, all partitioning key(s) are added to index schema + // as columns if they are not already part of the schema. + val missingPartitionColumns = getPartitionColumns(df).filter( + ResolverUtils.resolve(spark, _, columnsFromIndexConfig).isEmpty) + val allIndexColumns = columnsFromIndexConfig ++ missingPartitionColumns + df.select(allIndexColumns.head, allIndexColumns.tail: _*) .withColumn(IndexConstants.DATA_FILE_NAME_COLUMN, getFileName(input_file_name())) } else { - df.select(indexConfigColumns.head, indexConfigColumns.tail: _*) + df.select(columnsFromIndexConfig.head, columnsFromIndexConfig.tail: _*) } (indexDF, resolvedIndexedColumns, resolvedIncludedColumns) } - private def getPartitioningColumns(df: DataFrame): Seq[String] = { - // Extract partitioning keys, if original data is partitioned. - val partitionSchema = df.queryExecution.optimizedPlan.collect { + private def getPartitionColumns(df: DataFrame): Seq[String] = { + // Extract partition keys, if original data is partitioned. + val partitionSchemas = df.queryExecution.optimizedPlan.collect { case LogicalRelation(HadoopFsRelation(_, pSchema, _, _, _, _), _, _, _) => pSchema } - partitionSchema.head.map(_.name) + // Currently we only support creating an index on a single LogicalRelation. + assert(partitionSchemas.length == 1) + partitionSchemas.head.map(_.name) } private val getFileName: UserDefinedFunction = udf( diff --git a/src/test/scala/com/microsoft/hyperspace/SampleData.scala b/src/test/scala/com/microsoft/hyperspace/SampleData.scala index a7afe8cc8..cff7fbeac 100644 --- a/src/test/scala/com/microsoft/hyperspace/SampleData.scala +++ b/src/test/scala/com/microsoft/hyperspace/SampleData.scala @@ -34,36 +34,19 @@ object SampleData { ("2019-10-03", "ff60e4838b92421eafc3e6ee59a9e9f1", "mi perro", 2, 2000), ("2019-10-03", "187696fe0a6a40cc9516bc6e47c70bc1", "facebook", 4, 3000)) - // column names and partition keys for partitioned case - val colnames = Seq("Date", "RGUID", "Query", "imprs", "clicks") - val partitionKey1 = "Date" - val partitionKey2 = "Query" - def saveTestDataNonPartitioned(spark: SparkSession, path: String, colNames: String*): Unit = { import spark.implicits._ - assert(colNames.length == 5) testData.toDF(colNames: _*).write.parquet(path) } - def saveTestDataPartitioned(spark: SparkSession, path: String): Unit = { - // `Date` is the first partition key and `Query` is the second partition key. + def saveTestDataPartitioned( + spark: SparkSession, + path: String, + partitionKey1: String, + partitionKey2: String, + colNames: String*): Unit = { import spark.implicits._ - val df = testData.toDF(colnames: _*) - df.select(partitionKey1).distinct().collect().foreach { d => - val date = d.get(0) - df.filter($"Date" === date) - .select(partitionKey2) - .distinct() - .collect() - .foreach { q => - val query = q.get(0) - val partitionPath = - s"$path/$partitionKey1=$date/$partitionKey2=$query" - df.filter($"Date" === date && $"Query" === query) - .select("RGUID", "imprs", "clicks") - .write - .parquet(partitionPath) - } - } + val df = testData.toDF(colNames: _*) + df.write.partitionBy(partitionKey1, partitionKey2).parquet(path) } } diff --git a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala index abd93c585..9109eec9b 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala @@ -29,6 +29,8 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { override val systemPath = new Path("src/test/resources/indexLocation") private val sampleNonPartitionedParquetDataLocation = "src/test/resources/sampleparquet" private val samplePartitionedParquetDataLocation = "src/test/resources/samplepartitionedparquet" + private val partitionKey1 = "Date" + private val partitionKey2 = "Query" private val indexConfig1 = IndexConfig("index1", Seq("RGUID"), Seq("Date")) private val indexConfig2 = IndexConfig("index2", Seq("Query"), Seq("imprs")) private val indexConfig3 = IndexConfig("index3", Seq("imprs"), Seq("clicks")) @@ -45,19 +47,21 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { FileUtils.delete(new Path(sampleNonPartitionedParquetDataLocation)) FileUtils.delete(new Path(samplePartitionedParquetDataLocation)) + val dataColumns = Seq("Date", "RGUID", "Query", "imprs", "clicks") // save test non-partitioned. SampleData.saveTestDataNonPartitioned( spark, sampleNonPartitionedParquetDataLocation, - "Date", - "RGUID", - "Query", - "imprs", - "clicks") + dataColumns: _*) nonPartitionedDataDF = spark.read.parquet(sampleNonPartitionedParquetDataLocation) // save test data partitioned. - SampleData.saveTestDataPartitioned(spark, samplePartitionedParquetDataLocation) + SampleData.saveTestDataPartitioned( + spark, + samplePartitionedParquetDataLocation, + partitionKey1, + partitionKey2, + dataColumns: _*) partitionedDataDF = spark.read.parquet(samplePartitionedParquetDataLocation) } @@ -163,70 +167,68 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { } test("Check lineage in index records for non-partitioned data.") { - spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") - hyperspace.createIndex(nonPartitionedDataDF, indexConfig1) - val indexRecordsDF = spark.read.parquet( - s"$systemPath/${indexConfig1.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") - - // For non-partitioned data, only file name lineage column should be added to index schema. - assert( - indexRecordsDF.schema.fieldNames.sorted.corresponds( - (indexConfig1.indexedColumns ++ indexConfig1.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN)).sorted)(_.equals(_))) - spark.conf - .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { + hyperspace.createIndex(nonPartitionedDataDF, indexConfig1) + val indexRecordsDF = spark.read.parquet( + s"$systemPath/${indexConfig1.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") + + // For non-partitioned data, only file name lineage column should be added to index schema. + assert( + indexRecordsDF.schema.fieldNames.sorted.corresponds( + (indexConfig1.indexedColumns ++ indexConfig1.includedColumns ++ + Seq(IndexConstants.DATA_FILE_NAME_COLUMN)).sorted)(_.equals(_))) + } } test("Check lineage in index records for partitioned data when partition key is not in config.") { - spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") - hyperspace.createIndex(partitionedDataDF, indexConfig3) - val indexRecordsDF = spark.read.parquet( - s"$systemPath/${indexConfig3.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") - - // For partitioned data, beside file name lineage column all partition keys columns - // should be added to index schema if they are not already among index config columns. - assert( - indexRecordsDF.schema.fieldNames.sorted.corresponds( - (indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ - Seq( - IndexConstants.DATA_FILE_NAME_COLUMN, - SampleData.partitionKey1, - SampleData.partitionKey2)).sorted)(_.equals(_))) - spark.conf - .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { + hyperspace.createIndex(partitionedDataDF, indexConfig3) + val indexRecordsDF = spark.read.parquet( + s"$systemPath/${indexConfig3.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") + + // For partitioned data, beside file name lineage column all partition keys columns + // should be added to index schema if they are not already among index config columns. + assert( + indexRecordsDF.schema.fieldNames.sorted.corresponds( + (indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ + Seq( + IndexConstants.DATA_FILE_NAME_COLUMN, + partitionKey1, + partitionKey2)).sorted)(_.equals(_))) + } } test("Check lineage in index records for partitioned data when partition key is in config.") { - spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") - hyperspace.createIndex(partitionedDataDF, indexConfig4) - val indexRecordsDF = spark.read.parquet( - s"$systemPath/${indexConfig4.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") - - // For partitioned data, if partition keys are already in index config columns, - // there should be no duplicates due to adding lineage. - assert( - indexRecordsDF.schema.fieldNames.sorted.corresponds( - (indexConfig4.indexedColumns ++ indexConfig4.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN)).sorted)(_.equals(_))) - spark.conf - .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { + hyperspace.createIndex(partitionedDataDF, indexConfig4) + val indexRecordsDF = spark.read.parquet( + s"$systemPath/${indexConfig4.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") + + // For partitioned data, if partition keys are already in index config columns, + // there should be no duplicates due to adding lineage. + assert( + indexRecordsDF.schema.fieldNames.sorted.corresponds( + (indexConfig4.indexedColumns ++ indexConfig4.includedColumns ++ + Seq(IndexConstants.DATA_FILE_NAME_COLUMN)).sorted)(_.equals(_))) + } } test("Check lineage in index records for partitioned data when partition key is in load path.") { - spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") - val dataDF = - spark.read.parquet( - s"$samplePartitionedParquetDataLocation/${SampleData.partitionKey1}=2017-09-03") - hyperspace.createIndex(dataDF, indexConfig3) - val indexRecordsDF = spark.read.parquet( - s"$systemPath/${indexConfig3.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") - - // As data load path includes first partition key, index schema should only contain - // file name column and second partition key column as lineage columns. - assert(indexRecordsDF.schema.fieldNames.sorted.corresponds( - (indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN, SampleData.partitionKey2)).sorted)(_.equals(_))) - spark.conf - .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { + val dataDF = + spark.read.parquet( + s"$samplePartitionedParquetDataLocation/$partitionKey1=2017-09-03") + hyperspace.createIndex(dataDF, indexConfig3) + val indexRecordsDF = spark.read.parquet( + s"$systemPath/${indexConfig3.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") + + // As data load path includes first partition key, index schema should only contain + // file name column and second partition key column as lineage columns. + assert( + indexRecordsDF.schema.fieldNames.sorted.corresponds( + (indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ + Seq(IndexConstants.DATA_FILE_NAME_COLUMN, partitionKey2)).sorted)( + _.equals(_))) + } } } diff --git a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala index 099f1754b..f1dd4b796 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala @@ -44,18 +44,20 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { fileSystem.delete(new Path(sampleNonPartitionedParquetDataLocation), true) fileSystem.delete(new Path(samplePartitionedParquetDataLocation), true) + val dataColumns = Seq("c1", "c2", "c3", "c4", "c5") // save test non-partitioned. SampleData.saveTestDataNonPartitioned( spark, sampleNonPartitionedParquetDataLocation, - "c1", - "c2", - "c3", - "c4", - "c5") + dataColumns: _*) // save test partitioned. - SampleData.saveTestDataPartitioned(spark, samplePartitionedParquetDataLocation) + SampleData.saveTestDataPartitioned( + spark, + samplePartitionedParquetDataLocation, + "c1", + "c3", + dataColumns: _*) } before { @@ -103,19 +105,16 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } test("E2E test for filter query on partitioned data with lineage.") { - spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { + val df = spark.read.parquet(samplePartitionedParquetDataLocation) + val indexConfig = IndexConfig("filterIndex", Seq("c3"), Seq("c1")) - val df = spark.read.parquet(samplePartitionedParquetDataLocation) - val indexConfig = IndexConfig("filterIndex", Seq("Query"), Seq("Date")) + hyperspace.createIndex(df, indexConfig) - hyperspace.createIndex(df, indexConfig) - - def query(): DataFrame = df.filter("Query == 'facebook'").select("Query", "Date") + def query(): DataFrame = df.filter("c3 == 'facebook'").select("c3", "c1") - verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) - - spark.conf - .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) + verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) + } } test("E2E test for case insensitive filter query utilizing indexes.") { @@ -165,24 +164,21 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { test( "E2E test for filter query when all columns are selected on partitioned data with lineage.") { - spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") - - val df = spark.read.parquet(samplePartitionedParquetDataLocation) - val indexConfig = - IndexConfig("filterIndex", Seq("imprs", "Query"), Seq("Date", "RGUID", "clicks")) - - hyperspace.createIndex(df, indexConfig) - df.createOrReplaceTempView("t") + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { + val df = spark.read.parquet(samplePartitionedParquetDataLocation) + val indexConfig = + IndexConfig("filterIndex", Seq("c4", "c3"), Seq("c1", "c2", "c5")) - def query(): DataFrame = spark.sql("SELECT * from t where imprs = 1") + hyperspace.createIndex(df, indexConfig) + df.createOrReplaceTempView("t") - // Verify no Project node is present in the query plan, as a result of using SELECT * - assert(query().queryExecution.optimizedPlan.collect { case p: Project => p }.isEmpty) + def query(): DataFrame = spark.sql("SELECT * from t where c4 = 1") - verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) + // Verify no Project node is present in the query plan, as a result of using SELECT * + assert(query().queryExecution.optimizedPlan.collect { case p: Project => p }.isEmpty) - spark.conf - .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) + verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) + } } test("E2E test for join query on non-partitioned data.") { @@ -207,31 +203,28 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } test("E2E test for join query on partitioned data with lineage.") { - spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, "true") + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { + val leftDf = spark.read.parquet(samplePartitionedParquetDataLocation) + val leftDfIndexConfig = IndexConfig("leftIndex", Seq("c3"), Seq("c1")) - val leftDf = spark.read.parquet(samplePartitionedParquetDataLocation) - val leftDfIndexConfig = IndexConfig("leftIndex", Seq("Query"), Seq("Date")) + hyperspace.createIndex(leftDf, leftDfIndexConfig) - hyperspace.createIndex(leftDf, leftDfIndexConfig) + val rightDf = spark.read.parquet(samplePartitionedParquetDataLocation) + val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("c4")) + hyperspace.createIndex(rightDf, rightDfIndexConfig) - val rightDf = spark.read.parquet(samplePartitionedParquetDataLocation) - val rightDfIndexConfig = IndexConfig("rightIndex", Seq("Query"), Seq("imprs")) - hyperspace.createIndex(rightDf, rightDfIndexConfig) + def query(): DataFrame = { + leftDf + .join(rightDf, leftDf("c3") === rightDf("c3")) + .select(leftDf("c1"), rightDf("c4")) + } - def query(): DataFrame = { - leftDf - .join(rightDf, leftDf("Query") === rightDf("Query")) - .select(leftDf("Date"), rightDf("imprs")) + verifyIndexUsage( + query, + Seq( + getIndexFilesPath(leftDfIndexConfig.indexName), + getIndexFilesPath(rightDfIndexConfig.indexName))) } - - verifyIndexUsage( - query, - Seq( - getIndexFilesPath(leftDfIndexConfig.indexName), - getIndexFilesPath(rightDfIndexConfig.indexName))) - - spark.conf - .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) } test("E2E test for join query with case-insensitive column names.") { diff --git a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala index 316b6809e..2507f5077 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala @@ -19,6 +19,7 @@ package com.microsoft.hyperspace.index import org.apache.hadoop.fs.Path import org.apache.spark.SparkFunSuite import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, PartitioningAwareFileIndex} import org.apache.spark.sql.sources.DataSourceRegister import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} @@ -29,7 +30,7 @@ import com.microsoft.hyperspace.actions.Constants import com.microsoft.hyperspace.index.Content.Directory.FileInfo import com.microsoft.hyperspace.util.{FileUtils, PathUtils} -class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite { +class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite with SQLHelper { private val sampleParquetDataLocation = "src/test/resources/sampleparquet" private val indexStorageLocation = PathUtils.makeAbsolute("src/test/resources/indexLocation").toString @@ -63,12 +64,30 @@ class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite { super.afterAll() } - test("Verify that indexes() returns the correct dataframe without lineage.") { - verifyIndexesOutput(false) - } - - test("Verify that indexes() returns the correct dataframe with lineage.") { - verifyIndexesOutput(true) + test("Verify that indexes() returns the correct dataframe with and without lineage.") { + Seq(true, false).foreach { enableLineage => + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { + import spark.implicits._ + hyperspace.createIndex(df, indexConfig1) + val actual = hyperspace.indexes.as[IndexSummary].collect()(0) + var expectedSchema = + StructType(Seq(StructField("RGUID", StringType), StructField("Date", StringType))) + if (enableLineage) { + expectedSchema = + expectedSchema.add(StructField(IndexConstants.DATA_FILE_NAME_COLUMN, StringType)) + } + val expected = new IndexSummary( + indexConfig1.indexName, + indexConfig1.indexedColumns, + indexConfig1.includedColumns, + 200, + expectedSchema.json, + s"$indexStorageLocation/index1/v__=0", + Constants.States.ACTIVE) + assert(actual.equals(expected)) + } + FileUtils.delete(new Path(indexStorageLocation)) + } } test("Verify getIndexes()") { @@ -292,28 +311,4 @@ class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite { private def verifyIndexes(expectedIndexes: Seq[IndexLogEntry]): Unit = { assert(IndexCollectionManager(spark).getIndexes().toSet == expectedIndexes.toSet) } - - private def verifyIndexesOutput(enableLineage: Boolean): Unit = { - spark.conf.set(IndexConstants.INDEX_LINEAGE_ENABLED, if (enableLineage) "true" else "false") - import spark.implicits._ - hyperspace.createIndex(df, indexConfig1) - val actual = hyperspace.indexes.as[IndexSummary].collect()(0) - var expectedSchema = StructType( - Seq(StructField("RGUID", StringType), StructField("Date", StringType))) - if (enableLineage) { - expectedSchema = - expectedSchema.add(StructField(IndexConstants.DATA_FILE_NAME_COLUMN, StringType)) - } - val expected = new IndexSummary( - indexConfig1.indexName, - indexConfig1.indexedColumns, - indexConfig1.includedColumns, - 200, - expectedSchema.json, - s"$indexStorageLocation/index1/v__=0", - Constants.States.ACTIVE) - assert(actual.equals(expected)) - spark.conf - .set(IndexConstants.INDEX_LINEAGE_ENABLED, IndexConstants.INDEX_LINEAGE_ENABLED_DEFAULT) - } } From c8b80a3924032289122edea5c6a26e896c786bc9 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Thu, 20 Aug 2020 15:47:07 -0700 Subject: [PATCH 16/21] add lineage to index records --- .../hyperspace/actions/CreateActionBase.scala | 2 +- .../com/microsoft/hyperspace/SampleData.scala | 21 ++++++------ .../hyperspace/index/CreateIndexTests.scala | 34 ++++++++----------- .../index/E2EHyperspaceRulesTests.scala | 14 ++++---- 4 files changed, 31 insertions(+), 40 deletions(-) diff --git a/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala b/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala index 411a6c0ed..225dd17b4 100644 --- a/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala +++ b/src/main/scala/com/microsoft/hyperspace/actions/CreateActionBase.scala @@ -227,5 +227,5 @@ private[actions] abstract class CreateActionBase(dataManager: IndexDataManager) } private val getFileName: UserDefinedFunction = udf( - (fullFilePath: String) => FilenameUtils.getBaseName(fullFilePath)) + (fullFilePath: String) => FilenameUtils.getName(fullFilePath)) } diff --git a/src/test/scala/com/microsoft/hyperspace/SampleData.scala b/src/test/scala/com/microsoft/hyperspace/SampleData.scala index cff7fbeac..9b82e4561 100644 --- a/src/test/scala/com/microsoft/hyperspace/SampleData.scala +++ b/src/test/scala/com/microsoft/hyperspace/SampleData.scala @@ -34,19 +34,18 @@ object SampleData { ("2019-10-03", "ff60e4838b92421eafc3e6ee59a9e9f1", "mi perro", 2, 2000), ("2019-10-03", "187696fe0a6a40cc9516bc6e47c70bc1", "facebook", 4, 3000)) - def saveTestDataNonPartitioned(spark: SparkSession, path: String, colNames: String*): Unit = { - import spark.implicits._ - testData.toDF(colNames: _*).write.parquet(path) - } - - def saveTestDataPartitioned( + def save( spark: SparkSession, path: String, - partitionKey1: String, - partitionKey2: String, - colNames: String*): Unit = { + columns: Seq[String], + partitionColumns: Option[Seq[String]] = None): Unit = { import spark.implicits._ - val df = testData.toDF(colNames: _*) - df.write.partitionBy(partitionKey1, partitionKey2).parquet(path) + val df = testData.toDF(columns: _*) + partitionColumns match { + case Some(pcs) => + df.write.partitionBy(pcs: _*).parquet(path) + case None => + df.write.parquet(path) + } } } diff --git a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala index 9109eec9b..615a55dc2 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala @@ -27,10 +27,10 @@ import com.microsoft.hyperspace.util.FileUtils class CreateIndexTests extends HyperspaceSuite with SQLHelper { override val systemPath = new Path("src/test/resources/indexLocation") - private val sampleNonPartitionedParquetDataLocation = "src/test/resources/sampleparquet" - private val samplePartitionedParquetDataLocation = "src/test/resources/samplepartitionedparquet" - private val partitionKey1 = "Date" - private val partitionKey2 = "Query" + private val testDir = "src/test/resources/createIndexTests/" + private val sampleNonPartitionedParquetDataLocation = testDir + "sampleparquet" + private val samplePartitionedParquetDataLocation = testDir + "samplepartitionedparquet" + private val partitionKeys = Seq("Date", "Query") private val indexConfig1 = IndexConfig("index1", Seq("RGUID"), Seq("Date")) private val indexConfig2 = IndexConfig("index2", Seq("Query"), Seq("imprs")) private val indexConfig3 = IndexConfig("index3", Seq("imprs"), Seq("clicks")) @@ -44,30 +44,27 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { val sparkSession = spark hyperspace = new Hyperspace(sparkSession) - FileUtils.delete(new Path(sampleNonPartitionedParquetDataLocation)) - FileUtils.delete(new Path(samplePartitionedParquetDataLocation)) + FileUtils.delete(new Path(testDir), true) val dataColumns = Seq("Date", "RGUID", "Query", "imprs", "clicks") // save test non-partitioned. - SampleData.saveTestDataNonPartitioned( + SampleData.save( spark, sampleNonPartitionedParquetDataLocation, - dataColumns: _*) + dataColumns) nonPartitionedDataDF = spark.read.parquet(sampleNonPartitionedParquetDataLocation) // save test data partitioned. - SampleData.saveTestDataPartitioned( + SampleData.save( spark, samplePartitionedParquetDataLocation, - partitionKey1, - partitionKey2, - dataColumns: _*) + dataColumns, + Some(partitionKeys)) partitionedDataDF = spark.read.parquet(samplePartitionedParquetDataLocation) } override def afterAll(): Unit = { - FileUtils.delete(new Path(sampleNonPartitionedParquetDataLocation)) - FileUtils.delete(new Path(samplePartitionedParquetDataLocation)) + FileUtils.delete(new Path(testDir), true) super.afterAll() } @@ -191,10 +188,7 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { assert( indexRecordsDF.schema.fieldNames.sorted.corresponds( (indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ - Seq( - IndexConstants.DATA_FILE_NAME_COLUMN, - partitionKey1, - partitionKey2)).sorted)(_.equals(_))) + Seq(IndexConstants.DATA_FILE_NAME_COLUMN) ++ partitionKeys).sorted)(_.equals(_))) } } @@ -217,7 +211,7 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { val dataDF = spark.read.parquet( - s"$samplePartitionedParquetDataLocation/$partitionKey1=2017-09-03") + s"$samplePartitionedParquetDataLocation/${partitionKeys.head}=2017-09-03") hyperspace.createIndex(dataDF, indexConfig3) val indexRecordsDF = spark.read.parquet( s"$systemPath/${indexConfig3.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") @@ -227,7 +221,7 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { assert( indexRecordsDF.schema.fieldNames.sorted.corresponds( (indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN, partitionKey2)).sorted)( + Seq(IndexConstants.DATA_FILE_NAME_COLUMN, partitionKeys(1))).sorted)( _.equals(_))) } } diff --git a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala index f1dd4b796..8bbb94f00 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala @@ -41,23 +41,21 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { spark.conf.set("spark.sql.autoBroadcastJoinThreshold", -1) hyperspace = new Hyperspace(spark) - fileSystem.delete(new Path(sampleNonPartitionedParquetDataLocation), true) - fileSystem.delete(new Path(samplePartitionedParquetDataLocation), true) + fileSystem.delete(new Path(testDir), true) val dataColumns = Seq("c1", "c2", "c3", "c4", "c5") // save test non-partitioned. - SampleData.saveTestDataNonPartitioned( + SampleData.save( spark, sampleNonPartitionedParquetDataLocation, - dataColumns: _*) + dataColumns) // save test partitioned. - SampleData.saveTestDataPartitioned( + SampleData.save( spark, samplePartitionedParquetDataLocation, - "c1", - "c3", - dataColumns: _*) + dataColumns, + Some(Seq("c1", "c3"))) } before { From 261136295a5de18307535c382cb1f8aecbe08e80 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Sat, 22 Aug 2020 14:22:42 -0700 Subject: [PATCH 17/21] Clean up lineage tests --- .../hyperspace/index/CreateIndexTests.scala | 33 ++- .../index/E2EHyperspaceRulesTests.scala | 207 ++++++++---------- 2 files changed, 109 insertions(+), 131 deletions(-) diff --git a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala index 615a55dc2..84f19307f 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/CreateIndexTests.scala @@ -28,8 +28,8 @@ import com.microsoft.hyperspace.util.FileUtils class CreateIndexTests extends HyperspaceSuite with SQLHelper { override val systemPath = new Path("src/test/resources/indexLocation") private val testDir = "src/test/resources/createIndexTests/" - private val sampleNonPartitionedParquetDataLocation = testDir + "sampleparquet" - private val samplePartitionedParquetDataLocation = testDir + "samplepartitionedparquet" + private val nonPartitionedDataPath = testDir + "sampleparquet" + private val partitionedDataPath = testDir + "samplepartitionedparquet" private val partitionKeys = Seq("Date", "Query") private val indexConfig1 = IndexConfig("index1", Seq("RGUID"), Seq("Date")) private val indexConfig2 = IndexConfig("index2", Seq("Query"), Seq("imprs")) @@ -47,20 +47,20 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { FileUtils.delete(new Path(testDir), true) val dataColumns = Seq("Date", "RGUID", "Query", "imprs", "clicks") - // save test non-partitioned. + // save test data non-partitioned. SampleData.save( spark, - sampleNonPartitionedParquetDataLocation, + nonPartitionedDataPath, dataColumns) - nonPartitionedDataDF = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + nonPartitionedDataDF = spark.read.parquet(nonPartitionedDataPath) // save test data partitioned. SampleData.save( spark, - samplePartitionedParquetDataLocation, + partitionedDataPath, dataColumns, Some(partitionKeys)) - partitionedDataDF = spark.read.parquet(samplePartitionedParquetDataLocation) + partitionedDataDF = spark.read.parquet(partitionedDataPath) } override def afterAll(): Unit = { @@ -171,9 +171,9 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { // For non-partitioned data, only file name lineage column should be added to index schema. assert( - indexRecordsDF.schema.fieldNames.sorted.corresponds( + indexRecordsDF.schema.fieldNames.sorted === (indexConfig1.indexedColumns ++ indexConfig1.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN)).sorted)(_.equals(_))) + Seq(IndexConstants.DATA_FILE_NAME_COLUMN)).sorted) } } @@ -186,9 +186,9 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { // For partitioned data, beside file name lineage column all partition keys columns // should be added to index schema if they are not already among index config columns. assert( - indexRecordsDF.schema.fieldNames.sorted.corresponds( + indexRecordsDF.schema.fieldNames.sorted === (indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN) ++ partitionKeys).sorted)(_.equals(_))) + Seq(IndexConstants.DATA_FILE_NAME_COLUMN) ++ partitionKeys).sorted) } } @@ -201,9 +201,9 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { // For partitioned data, if partition keys are already in index config columns, // there should be no duplicates due to adding lineage. assert( - indexRecordsDF.schema.fieldNames.sorted.corresponds( + indexRecordsDF.schema.fieldNames.sorted === (indexConfig4.indexedColumns ++ indexConfig4.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN)).sorted)(_.equals(_))) + Seq(IndexConstants.DATA_FILE_NAME_COLUMN)).sorted) } } @@ -211,7 +211,7 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { val dataDF = spark.read.parquet( - s"$samplePartitionedParquetDataLocation/${partitionKeys.head}=2017-09-03") + s"$partitionedDataPath/${partitionKeys.head}=2017-09-03") hyperspace.createIndex(dataDF, indexConfig3) val indexRecordsDF = spark.read.parquet( s"$systemPath/${indexConfig3.indexName}/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") @@ -219,10 +219,9 @@ class CreateIndexTests extends HyperspaceSuite with SQLHelper { // As data load path includes first partition key, index schema should only contain // file name column and second partition key column as lineage columns. assert( - indexRecordsDF.schema.fieldNames.sorted.corresponds( + indexRecordsDF.schema.fieldNames.sorted === (indexConfig3.indexedColumns ++ indexConfig3.includedColumns ++ - Seq(IndexConstants.DATA_FILE_NAME_COLUMN, partitionKeys(1))).sorted)( - _.equals(_))) + Seq(IndexConstants.DATA_FILE_NAME_COLUMN, partitionKeys(1))).sorted) } } } diff --git a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala index 8bbb94f00..1e38724c6 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala @@ -29,11 +29,11 @@ import com.microsoft.hyperspace.util.PathUtils class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { private val testDir = "src/test/resources/e2eTests/" - private val sampleNonPartitionedParquetDataLocation = testDir + "sampleparquet" - private val samplePartitionedParquetDataLocation = testDir + "samplepartitionedparquet" + private val nonPartitionedDataPath = testDir + "sampleparquet" + private val partitionedDataPath = testDir + "samplepartitionedparquet" override val systemPath = PathUtils.makeAbsolute("src/test/resources/indexLocation") private val fileSystem = - new Path(sampleNonPartitionedParquetDataLocation).getFileSystem(new Configuration) + new Path(nonPartitionedDataPath).getFileSystem(new Configuration) private var hyperspace: Hyperspace = _ override def beforeAll(): Unit = { @@ -44,16 +44,13 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { fileSystem.delete(new Path(testDir), true) val dataColumns = Seq("c1", "c2", "c3", "c4", "c5") - // save test non-partitioned. - SampleData.save( - spark, - sampleNonPartitionedParquetDataLocation, - dataColumns) + // save test data non-partitioned. + SampleData.save(spark, nonPartitionedDataPath, dataColumns) - // save test partitioned. + // save test data partitioned. SampleData.save( spark, - samplePartitionedParquetDataLocation, + partitionedDataPath, dataColumns, Some(Seq("c1", "c3"))) } @@ -91,32 +88,31 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { .containsSlice(expectedOptimizationRuleBatch)) } - test("E2E test for filter query on non-partitioned data.") { - val df = spark.read.parquet(sampleNonPartitionedParquetDataLocation) - val indexConfig = IndexConfig("filterIndex", Seq("c3"), Seq("c1")) - - hyperspace.createIndex(df, indexConfig) - - def query(): DataFrame = df.filter("c3 == 'facebook'").select("c3", "c1") - - verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) - } + test( + "E2E test for filter query on partitioned and non-partitioned data with and without lineage.") { + Seq(nonPartitionedDataPath, partitionedDataPath).foreach { + loc => + Seq(true, false).foreach { enableLineage => + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { + val df = spark.read.parquet(loc) + val indexConfig = IndexConfig("filterIndex", Seq("c3"), Seq("c1")) - test("E2E test for filter query on partitioned data with lineage.") { - withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { - val df = spark.read.parquet(samplePartitionedParquetDataLocation) - val indexConfig = IndexConfig("filterIndex", Seq("c3"), Seq("c1")) + hyperspace.createIndex(df, indexConfig) - hyperspace.createIndex(df, indexConfig) + def query(): DataFrame = df.filter("c3 == 'facebook'").select("c3", "c1") - def query(): DataFrame = df.filter("c3 == 'facebook'").select("c3", "c1") + verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) - verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) + clearCache() + fileSystem.delete(systemPath, true) + spark.disableHyperspace() + } + } } } test("E2E test for case insensitive filter query utilizing indexes.") { - val df = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val df = spark.read.parquet(nonPartitionedDataPath) val indexConfig = IndexConfig("filterIndex", Seq("C3"), Seq("C1")) hyperspace.createIndex(df, indexConfig) @@ -128,7 +124,7 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } test("E2E test for case sensitive filter query where changing conf changes behavior.") { - val df = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val df = spark.read.parquet(nonPartitionedDataPath) val indexConfig = IndexConfig("filterIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(df, indexConfig) @@ -145,92 +141,75 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } } - test("E2E test for filter query when all columns are selected on non-partitioned data.") { - val df = spark.read.parquet(sampleNonPartitionedParquetDataLocation) - val indexConfig = IndexConfig("filterIndex", Seq("c4", "c3"), Seq("c1", "c2", "c5")) - - hyperspace.createIndex(df, indexConfig) - df.createOrReplaceTempView("t") - - def query(): DataFrame = spark.sql("SELECT * from t where c4 = 1") - - // Verify no Project node is present in the query plan, as a result of using SELECT * - assert(query().queryExecution.optimizedPlan.collect { case p: Project => p }.isEmpty) - - verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) - } - test( - "E2E test for filter query when all columns are selected on partitioned data with lineage.") { - withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { - val df = spark.read.parquet(samplePartitionedParquetDataLocation) - val indexConfig = - IndexConfig("filterIndex", Seq("c4", "c3"), Seq("c1", "c2", "c5")) - - hyperspace.createIndex(df, indexConfig) - df.createOrReplaceTempView("t") - - def query(): DataFrame = spark.sql("SELECT * from t where c4 = 1") - - // Verify no Project node is present in the query plan, as a result of using SELECT * - assert(query().queryExecution.optimizedPlan.collect { case p: Project => p }.isEmpty) - - verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) - } - } - - test("E2E test for join query on non-partitioned data.") { - val leftDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) - val leftDfIndexConfig = IndexConfig("leftIndex", Seq("c3"), Seq("c1")) - - hyperspace.createIndex(leftDf, leftDfIndexConfig) - - val rightDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) - val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("c4")) - hyperspace.createIndex(rightDf, rightDfIndexConfig) - - def query(): DataFrame = { - leftDf.join(rightDf, leftDf("c3") === rightDf("c3")).select(leftDf("c1"), rightDf("c4")) + "E2E test for filter query when all columns are selected on partitioned and " + + "non-partitioned data with and without lineage.") { + Seq(nonPartitionedDataPath, partitionedDataPath).foreach { + loc => + Seq(true, false).foreach { enableLineage => + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { + val df = spark.read.parquet(loc) + val indexConfig = IndexConfig("filterIndex", Seq("c4", "c3"), Seq("c1", "c2", "c5")) + + hyperspace.createIndex(df, indexConfig) + df.createOrReplaceTempView("t") + + def query(): DataFrame = spark.sql("SELECT * from t where c4 = 1") + + // Verify no Project node is present in the query plan, as a result of using SELECT * + assert(query().queryExecution.optimizedPlan.collect { case p: Project => p }.isEmpty) + + verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) + + clearCache() + fileSystem.delete(systemPath, true) + spark.disableHyperspace() + } + } } - - verifyIndexUsage( - query, - Seq( - getIndexFilesPath(leftDfIndexConfig.indexName), - getIndexFilesPath(rightDfIndexConfig.indexName))) } - test("E2E test for join query on partitioned data with lineage.") { - withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") { - val leftDf = spark.read.parquet(samplePartitionedParquetDataLocation) - val leftDfIndexConfig = IndexConfig("leftIndex", Seq("c3"), Seq("c1")) - - hyperspace.createIndex(leftDf, leftDfIndexConfig) - - val rightDf = spark.read.parquet(samplePartitionedParquetDataLocation) - val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("c4")) - hyperspace.createIndex(rightDf, rightDfIndexConfig) - - def query(): DataFrame = { - leftDf - .join(rightDf, leftDf("c3") === rightDf("c3")) - .select(leftDf("c1"), rightDf("c4")) - } - - verifyIndexUsage( - query, - Seq( - getIndexFilesPath(leftDfIndexConfig.indexName), - getIndexFilesPath(rightDfIndexConfig.indexName))) + test( + "E2E test for join query on partitioned and non-partitioned data with and without lineage.") { + Seq(nonPartitionedDataPath, partitionedDataPath).foreach { + loc => + Seq(true, false).foreach { enableLineage => + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { + val leftDf = spark.read.parquet(loc) + val leftDfIndexConfig = IndexConfig("leftIndex", Seq("c3"), Seq("c1")) + + hyperspace.createIndex(leftDf, leftDfIndexConfig) + + val rightDf = spark.read.parquet(loc) + val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("c4")) + hyperspace.createIndex(rightDf, rightDfIndexConfig) + + def query(): DataFrame = { + leftDf + .join(rightDf, leftDf("c3") === rightDf("c3")) + .select(leftDf("c1"), rightDf("c4")) + } + + verifyIndexUsage( + query, + Seq( + getIndexFilesPath(leftDfIndexConfig.indexName), + getIndexFilesPath(rightDfIndexConfig.indexName))) + + clearCache() + fileSystem.delete(systemPath, true) + spark.disableHyperspace() + } + } } } test("E2E test for join query with case-insensitive column names.") { - val leftDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val leftDf = spark.read.parquet(nonPartitionedDataPath) val leftDfIndexConfig = IndexConfig("leftIndex", Seq("C3"), Seq("c1")) hyperspace.createIndex(leftDf, leftDfIndexConfig) - val rightDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val rightDf = spark.read.parquet(nonPartitionedDataPath) val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("C4")) hyperspace.createIndex(rightDf, rightDfIndexConfig) @@ -254,11 +233,11 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } withView("t1", "t2") { - val leftDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val leftDf = spark.read.parquet(nonPartitionedDataPath) val leftDfIndexConfig = IndexConfig("leftIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(leftDf, leftDfIndexConfig) - val rightDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val rightDf = spark.read.parquet(nonPartitionedDataPath) val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("c4")) hyperspace.createIndex(rightDf, rightDfIndexConfig) @@ -281,11 +260,11 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { test("E2E test for join query on catalog temp tables/views") { withView("t1", "t2") { - val leftDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val leftDf = spark.read.parquet(nonPartitionedDataPath) val leftDfIndexConfig = IndexConfig("leftIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(leftDf, leftDfIndexConfig) - val rightDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val rightDf = spark.read.parquet(nonPartitionedDataPath) val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("c4")) hyperspace.createIndex(rightDf, rightDfIndexConfig) @@ -311,13 +290,13 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { |CREATE EXTERNAL TABLE t1 |(c1 string, c3 string) |STORED AS PARQUET - |LOCATION '$sampleNonPartitionedParquetDataLocation' + |LOCATION '$nonPartitionedDataPath' """.stripMargin) spark.sql(s""" |CREATE EXTERNAL TABLE t2 |(c3 string, c4 int) |STORED AS PARQUET - |LOCATION '$sampleNonPartitionedParquetDataLocation' + |LOCATION '$nonPartitionedDataPath' """.stripMargin) val leftDf = spark.table("t1") @@ -343,7 +322,7 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { withTable("t1", "t2") { val table1Location = testDir + "tables/t1" val table2Location = testDir + "tables/t2" - val originalDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val originalDf = spark.read.parquet(nonPartitionedDataPath) originalDf.select("c1", "c3").write.option("path", table1Location).saveAsTable("t1") originalDf.select("c3", "c4").write.option("path", table2Location).saveAsTable("t2") @@ -365,14 +344,14 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } test("E2E test for join query with two child sub-query as both filter query.") { - val leftDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val leftDf = spark.read.parquet(nonPartitionedDataPath) val leftDfJoinIndexConfig = IndexConfig("leftJoinIndex", Seq("c3"), Seq("c4")) hyperspace.createIndex(leftDf, leftDfJoinIndexConfig) val leftDfFilterIndexConfig = IndexConfig("leftDfFilterIndex", Seq("c4"), Seq("c3")) hyperspace.createIndex(leftDf, leftDfFilterIndexConfig) - val rightDf = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val rightDf = spark.read.parquet(nonPartitionedDataPath) val rightDfJoinIndexConfig = IndexConfig("rightDfJoinIndex", Seq("c3"), Seq("c5")) hyperspace.createIndex(rightDf, rightDfJoinIndexConfig) @@ -397,7 +376,7 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { } test("E2E test for first enableHyperspace() followed by disableHyperspace().") { - val df = spark.read.parquet(sampleNonPartitionedParquetDataLocation) + val df = spark.read.parquet(nonPartitionedDataPath) val indexConfig = IndexConfig("filterIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(df, indexConfig) From 7bd60de77370531aa00621b359b79d216028c2c6 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Sat, 22 Aug 2020 19:21:35 -0700 Subject: [PATCH 18/21] clean up lineage tests --- .../index/E2EHyperspaceRulesTests.scala | 54 ++++++++----------- .../hyperspace/index/HyperspaceSuite.scala | 14 +++++ .../hyperspace/index/IndexManagerTests.scala | 1 + 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala index 1e38724c6..d65cf9b45 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala @@ -48,11 +48,7 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { SampleData.save(spark, nonPartitionedDataPath, dataColumns) // save test data partitioned. - SampleData.save( - spark, - partitionedDataPath, - dataColumns, - Some(Seq("c1", "c3"))) + SampleData.save(spark, partitionedDataPath, dataColumns, Some(Seq("c1", "c3"))) } before { @@ -90,24 +86,22 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { test( "E2E test for filter query on partitioned and non-partitioned data with and without lineage.") { - Seq(nonPartitionedDataPath, partitionedDataPath).foreach { - loc => - Seq(true, false).foreach { enableLineage => - withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { + Seq(nonPartitionedDataPath, partitionedDataPath).foreach { loc => + Seq(true, false).foreach { enableLineage => + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { + val indexName = "filterIndex" + withIndex(indexName) { val df = spark.read.parquet(loc) - val indexConfig = IndexConfig("filterIndex", Seq("c3"), Seq("c1")) + val indexConfig = IndexConfig(indexName, Seq("c3"), Seq("c1")) hyperspace.createIndex(df, indexConfig) def query(): DataFrame = df.filter("c3 == 'facebook'").select("c3", "c1") verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) - - clearCache() - fileSystem.delete(systemPath, true) - spark.disableHyperspace() } } + } } } @@ -144,10 +138,11 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { test( "E2E test for filter query when all columns are selected on partitioned and " + "non-partitioned data with and without lineage.") { - Seq(nonPartitionedDataPath, partitionedDataPath).foreach { - loc => - Seq(true, false).foreach { enableLineage => - withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { + Seq(nonPartitionedDataPath, partitionedDataPath).foreach { loc => + Seq(true, false).foreach { enableLineage => + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { + val indexName = "filterIndex" + withIndex(indexName) { val df = spark.read.parquet(loc) val indexConfig = IndexConfig("filterIndex", Seq("c4", "c3"), Seq("c1", "c2", "c5")) @@ -160,28 +155,26 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { assert(query().queryExecution.optimizedPlan.collect { case p: Project => p }.isEmpty) verifyIndexUsage(query, Seq(getIndexFilesPath(indexConfig.indexName))) - - clearCache() - fileSystem.delete(systemPath, true) - spark.disableHyperspace() } } + } } } test( "E2E test for join query on partitioned and non-partitioned data with and without lineage.") { - Seq(nonPartitionedDataPath, partitionedDataPath).foreach { - loc => - Seq(true, false).foreach { enableLineage => - withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { + Seq(nonPartitionedDataPath, partitionedDataPath).foreach { loc => + Seq(true, false).foreach { enableLineage => + withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { + val indexNames = Seq("leftIndex", "rightIndex") + withIndex(indexNames: _*) { val leftDf = spark.read.parquet(loc) - val leftDfIndexConfig = IndexConfig("leftIndex", Seq("c3"), Seq("c1")) + val leftDfIndexConfig = IndexConfig(indexNames.head, Seq("c3"), Seq("c1")) hyperspace.createIndex(leftDf, leftDfIndexConfig) val rightDf = spark.read.parquet(loc) - val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("c4")) + val rightDfIndexConfig = IndexConfig(indexNames(1), Seq("c3"), Seq("c4")) hyperspace.createIndex(rightDf, rightDfIndexConfig) def query(): DataFrame = { @@ -195,12 +188,9 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { Seq( getIndexFilesPath(leftDfIndexConfig.indexName), getIndexFilesPath(rightDfIndexConfig.indexName))) - - clearCache() - fileSystem.delete(systemPath, true) - spark.disableHyperspace() } } + } } } diff --git a/src/test/scala/com/microsoft/hyperspace/index/HyperspaceSuite.scala b/src/test/scala/com/microsoft/hyperspace/index/HyperspaceSuite.scala index bc2b53e8f..5caca93f7 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/HyperspaceSuite.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/HyperspaceSuite.scala @@ -72,4 +72,18 @@ trait HyperspaceSuite extends SparkFunSuite with SparkInvolvedSuite { } } } + + /** + * Vacuum indexes with the given names after calling `f`. + */ + protected def withIndex(indexNames: String*)(f: => Unit): Unit = { + try f + finally { + val hs = new Hyperspace(spark) + indexNames.foreach { name => + hs.deleteIndex(name) + hs.vacuumIndex(name) + } + } + } } diff --git a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala index 2507f5077..ab73d28a0 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala @@ -86,6 +86,7 @@ class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite with SQLHe Constants.States.ACTIVE) assert(actual.equals(expected)) } + hyperspace.deleteIndex(indexConfig1.indexName) FileUtils.delete(new Path(indexStorageLocation)) } } From 4b6b657135ca11e2aedf5ec05fd343a3a4650cf5 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Sat, 22 Aug 2020 19:24:05 -0700 Subject: [PATCH 19/21] format fix in HyperspaceSuite --- .../com/microsoft/hyperspace/index/HyperspaceSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/scala/com/microsoft/hyperspace/index/HyperspaceSuite.scala b/src/test/scala/com/microsoft/hyperspace/index/HyperspaceSuite.scala index 5caca93f7..8fb37e7da 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/HyperspaceSuite.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/HyperspaceSuite.scala @@ -74,8 +74,8 @@ trait HyperspaceSuite extends SparkFunSuite with SparkInvolvedSuite { } /** - * Vacuum indexes with the given names after calling `f`. - */ + * Vacuum indexes with the given names after calling `f`. + */ protected def withIndex(indexNames: String*)(f: => Unit): Unit = { try f finally { From 6b531124fa94ef24b2568ef1b4790bf455033026 Mon Sep 17 00:00:00 2001 From: Pouria Pirzadeh Date: Sun, 23 Aug 2020 11:23:58 -0700 Subject: [PATCH 20/21] Clean up lineage tests --- .../index/E2EHyperspaceRulesTests.scala | 15 ++-- .../hyperspace/index/IndexManagerTests.scala | 68 +++++++++---------- 2 files changed, 37 insertions(+), 46 deletions(-) diff --git a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala index d65cf9b45..d0a578d2c 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTests.scala @@ -89,10 +89,9 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { Seq(nonPartitionedDataPath, partitionedDataPath).foreach { loc => Seq(true, false).foreach { enableLineage => withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { - val indexName = "filterIndex" - withIndex(indexName) { + withIndex("filterIndex") { val df = spark.read.parquet(loc) - val indexConfig = IndexConfig(indexName, Seq("c3"), Seq("c1")) + val indexConfig = IndexConfig("filterIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(df, indexConfig) @@ -141,8 +140,7 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { Seq(nonPartitionedDataPath, partitionedDataPath).foreach { loc => Seq(true, false).foreach { enableLineage => withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { - val indexName = "filterIndex" - withIndex(indexName) { + withIndex("filterIndex") { val df = spark.read.parquet(loc) val indexConfig = IndexConfig("filterIndex", Seq("c4", "c3"), Seq("c1", "c2", "c5")) @@ -166,15 +164,14 @@ class E2EHyperspaceRulesTests extends HyperspaceSuite with SQLHelper { Seq(nonPartitionedDataPath, partitionedDataPath).foreach { loc => Seq(true, false).foreach { enableLineage => withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { - val indexNames = Seq("leftIndex", "rightIndex") - withIndex(indexNames: _*) { + withIndex("leftIndex", "rightIndex") { val leftDf = spark.read.parquet(loc) - val leftDfIndexConfig = IndexConfig(indexNames.head, Seq("c3"), Seq("c1")) + val leftDfIndexConfig = IndexConfig("leftIndex", Seq("c3"), Seq("c1")) hyperspace.createIndex(leftDf, leftDfIndexConfig) val rightDf = spark.read.parquet(loc) - val rightDfIndexConfig = IndexConfig(indexNames(1), Seq("c3"), Seq("c4")) + val rightDfIndexConfig = IndexConfig("rightIndex", Seq("c3"), Seq("c4")) hyperspace.createIndex(rightDf, rightDfIndexConfig) def query(): DataFrame = { diff --git a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala index ab73d28a0..f89aad646 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala @@ -17,23 +17,22 @@ package com.microsoft.hyperspace.index import org.apache.hadoop.fs.Path -import org.apache.spark.SparkFunSuite import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, PartitioningAwareFileIndex} import org.apache.spark.sql.sources.DataSourceRegister import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} -import com.microsoft.hyperspace.{Hyperspace, HyperspaceException, SampleData, SparkInvolvedSuite} +import com.microsoft.hyperspace.{Hyperspace, HyperspaceException, SampleData} import com.microsoft.hyperspace.TestUtils.copyWithState import com.microsoft.hyperspace.actions.Constants import com.microsoft.hyperspace.index.Content.Directory.FileInfo import com.microsoft.hyperspace.util.{FileUtils, PathUtils} -class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite with SQLHelper { +class IndexManagerTests extends HyperspaceSuite with SQLHelper { private val sampleParquetDataLocation = "src/test/resources/sampleparquet" - private val indexStorageLocation = - PathUtils.makeAbsolute("src/test/resources/indexLocation").toString + override val systemPath = + PathUtils.makeAbsolute("src/test/resources/indexLocation") private val indexConfig1 = IndexConfig("index1", Seq("RGUID"), Seq("Date")) private val indexConfig2 = IndexConfig("index2", Seq("Query"), Seq("imprs")) private lazy val hyperspace: Hyperspace = new Hyperspace(spark) @@ -41,22 +40,17 @@ class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite with SQLHe override def beforeAll(): Unit = { super.beforeAll() - spark.conf.set(IndexConstants.INDEX_SYSTEM_PATH, indexStorageLocation) - FileUtils.delete(new Path(indexStorageLocation)) FileUtils.delete(new Path(sampleParquetDataLocation)) - - import spark.implicits._ - SampleData.testData - .toDF("Date", "RGUID", "Query", "imprs", "clicks") - .write - .parquet(sampleParquetDataLocation) - + SampleData.save( + spark, + sampleParquetDataLocation, + Seq("Date", "RGUID", "Query", "imprs", "clicks")) df = spark.read.parquet(sampleParquetDataLocation) } after { - FileUtils.delete(new Path(indexStorageLocation)) + FileUtils.delete(systemPath, true) } override def afterAll(): Unit = { @@ -67,27 +61,27 @@ class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite with SQLHe test("Verify that indexes() returns the correct dataframe with and without lineage.") { Seq(true, false).foreach { enableLineage => withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> enableLineage.toString) { - import spark.implicits._ - hyperspace.createIndex(df, indexConfig1) - val actual = hyperspace.indexes.as[IndexSummary].collect()(0) - var expectedSchema = - StructType(Seq(StructField("RGUID", StringType), StructField("Date", StringType))) - if (enableLineage) { - expectedSchema = - expectedSchema.add(StructField(IndexConstants.DATA_FILE_NAME_COLUMN, StringType)) + withIndex(indexConfig1.indexName) { + import spark.implicits._ + hyperspace.createIndex(df, indexConfig1) + val actual = hyperspace.indexes.as[IndexSummary].collect().head + var expectedSchema = + StructType(Seq(StructField("RGUID", StringType), StructField("Date", StringType))) + if (enableLineage) { + expectedSchema = + expectedSchema.add(StructField(IndexConstants.DATA_FILE_NAME_COLUMN, StringType)) + } + val expected = new IndexSummary( + indexConfig1.indexName, + indexConfig1.indexedColumns, + indexConfig1.includedColumns, + 200, + expectedSchema.json, + s"${systemPath.toUri.toString}/${indexConfig1.indexName}/v__=0", + Constants.States.ACTIVE) + assert(actual.equals(expected)) } - val expected = new IndexSummary( - indexConfig1.indexName, - indexConfig1.indexedColumns, - indexConfig1.includedColumns, - 200, - expectedSchema.json, - s"$indexStorageLocation/index1/v__=0", - Constants.States.ACTIVE) - assert(actual.equals(expected)) } - hyperspace.deleteIndex(indexConfig1.indexName) - FileUtils.delete(new Path(indexStorageLocation)) } } @@ -218,7 +212,7 @@ class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite with SQLHe hyperspace.createIndex(df, indexConfig) var indexCount = spark.read - .parquet(s"$indexStorageLocation/index_$format" + + .parquet(s"${systemPath.toUri.toString}/index_$format" + s"/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") .count() assert(indexCount == 10) @@ -234,7 +228,7 @@ class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite with SQLHe .save(refreshTestLocation) hyperspace.refreshIndex(indexConfig.indexName) indexCount = spark.read - .parquet(s"$indexStorageLocation/index_$format" + + .parquet(s"${systemPath.toUri.toString}/index_$format" + s"/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=1") .count() @@ -297,7 +291,7 @@ class IndexManagerTests extends SparkFunSuite with SparkInvolvedSuite with SQLHe IndexLogEntry.schemaString(schema), IndexConstants.INDEX_NUM_BUCKETS_DEFAULT)), Content( - s"$indexStorageLocation/${indexConfig.indexName}" + + s"${systemPath.toUri.toString}/${indexConfig.indexName}" + s"/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0", Seq()), Source(SparkPlan(sourcePlanProperties)), From 7cdd529b1f8157c5241f57e360535b6b9467604d Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sun, 23 Aug 2020 13:59:42 -0700 Subject: [PATCH 21/21] nit --- .../microsoft/hyperspace/index/IndexManagerTests.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala index f89aad646..7cc016e69 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/IndexManagerTests.scala @@ -77,7 +77,8 @@ class IndexManagerTests extends HyperspaceSuite with SQLHelper { indexConfig1.includedColumns, 200, expectedSchema.json, - s"${systemPath.toUri.toString}/${indexConfig1.indexName}/v__=0", + s"$systemPath/${indexConfig1.indexName}" + + s"/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0", Constants.States.ACTIVE) assert(actual.equals(expected)) } @@ -212,7 +213,7 @@ class IndexManagerTests extends HyperspaceSuite with SQLHelper { hyperspace.createIndex(df, indexConfig) var indexCount = spark.read - .parquet(s"${systemPath.toUri.toString}/index_$format" + + .parquet(s"$systemPath/index_$format" + s"/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0") .count() assert(indexCount == 10) @@ -228,7 +229,7 @@ class IndexManagerTests extends HyperspaceSuite with SQLHelper { .save(refreshTestLocation) hyperspace.refreshIndex(indexConfig.indexName) indexCount = spark.read - .parquet(s"${systemPath.toUri.toString}/index_$format" + + .parquet(s"$systemPath/index_$format" + s"/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=1") .count() @@ -291,7 +292,7 @@ class IndexManagerTests extends HyperspaceSuite with SQLHelper { IndexLogEntry.schemaString(schema), IndexConstants.INDEX_NUM_BUCKETS_DEFAULT)), Content( - s"${systemPath.toUri.toString}/${indexConfig.indexName}" + + s"$systemPath/${indexConfig.indexName}" + s"/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0", Seq()), Source(SparkPlan(sourcePlanProperties)),