From 8e5fc96bc75b4b238aebb814591759a74ca782fd Mon Sep 17 00:00:00 2001 From: Fanaen Date: Thu, 24 Jul 2025 18:21:58 +0200 Subject: [PATCH 1/7] chore(prqlc): deterministic except field in `LineageColumn` --- prqlc/prqlc/src/ir/pl/lineage.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/prqlc/prqlc/src/ir/pl/lineage.rs b/prqlc/prqlc/src/ir/pl/lineage.rs index 21a64ebfb3d4..15a82c2545ae 100644 --- a/prqlc/prqlc/src/ir/pl/lineage.rs +++ b/prqlc/prqlc/src/ir/pl/lineage.rs @@ -4,7 +4,7 @@ use std::fmt::{Debug, Display, Formatter}; use enum_as_inner::EnumAsInner; use itertools::{Itertools, Position}; use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize, Serializer}; use super::Ident; @@ -48,10 +48,23 @@ pub enum LineageColumn { /// All columns (including unknown ones) from an input (i.e. `foo_table.*`) All { input_id: usize, + + #[serde(serialize_with = "sorted_set")] except: HashSet, }, } +pub fn sorted_set( + value: &HashSet, + serializer: S, +) -> Result { + value + .iter() + .sorted() + .collect::>() + .serialize(serializer) +} + impl Display for Lineage { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { display_lineage(self, f, false) From ad9cee9edc9f1749cc6c83d3dd6b6d9e99ce0aee Mon Sep 17 00:00:00 2001 From: Fanaen Date: Mon, 21 Jul 2025 18:29:30 +0200 Subject: [PATCH 2/7] fix(prqlc): remove unnecessary select to avoid missing columns Forcing selection in requirements meant compute columns with dependencies sometimes appeared in SELECT statement where those dependencies were not available (not selected in subsequent CTE). Selection status is likely to be properly set before-hand anyway. --- prqlc/prqlc/src/sql/pq/anchor.rs | 2 - .../queries/distinct_on_sort_on_compute.prql | 8 + ..._compile__distinct_on_sort_on_compute.snap | 44 ++ ...mpileall__distinct_on_sort_on_compute.snap | 183 +++++++ ..._lineage__distinct_on_sort_on_compute.snap | 490 ++++++++++++++++++ ...ies__fmt__distinct_on_sort_on_compute.snap | 13 + ...ies__lex__distinct_on_sort_on_compute.snap | 82 +++ ..._results__distinct_on_sort_on_compute.snap | 6 + prqlc/prqlc/tests/integration/sql.rs | 21 +- 9 files changed, 830 insertions(+), 19 deletions(-) create mode 100644 prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__distinct_on_sort_on_compute.snap create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__distinct_on_sort_on_compute.snap create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on_sort_on_compute.snap create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__results__distinct_on_sort_on_compute.snap diff --git a/prqlc/prqlc/src/sql/pq/anchor.rs b/prqlc/prqlc/src/sql/pq/anchor.rs index 0e424b7389e7..8deb5bc49495 100644 --- a/prqlc/prqlc/src/sql/pq/anchor.rs +++ b/prqlc/prqlc/src/sql/pq/anchor.rs @@ -553,8 +553,6 @@ pub(super) fn get_requirements( } SqlTransform::DistinctOn(partition) => Requirements::from_cids(partition.iter()) - // Partition columns must be selected in order to push compute columns down CTE. - .should_select(true) // Since there is aggregation anyway, columns can have any complexity .allow_up_to(Complexity::highest()), diff --git a/prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql b/prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql new file mode 100644 index 000000000000..5ffdbc8e2da0 --- /dev/null +++ b/prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql @@ -0,0 +1,8 @@ +from invoices +derive code = case [customer_id < 10 => billing_postal_code, true => null] +group {customer_id, billing_city, billing_country} ( + sort {-this.code} + take 1 +) +filter (customer_id | in [4]) +group {billing_country} (aggregate {total = math.round 2 (sum total)}) diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__distinct_on_sort_on_compute.snap new file mode 100644 index 000000000000..1f7822326316 --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__distinct_on_sort_on_compute.snap @@ -0,0 +1,44 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: "from invoices\nderive code = case [customer_id < 10 => billing_postal_code, true => null]\ngroup {customer_id, billing_city, billing_country} (\n sort {-this.code}\n take 1\n)\nfilter (customer_id | in [4])\ngroup {billing_country} (aggregate {total = math.round 2 (sum total)})\n" +input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql +--- +WITH table_1 AS ( + SELECT + billing_country, + total, + customer_id, + billing_city, + CASE + WHEN customer_id < 10 THEN billing_postal_code + ELSE NULL + END AS _expr_1, + billing_postal_code + FROM + invoices +), +table_0 AS ( + SELECT + billing_country, + total, + customer_id, + ROW_NUMBER() OVER ( + PARTITION BY customer_id, + billing_city, + billing_country + ORDER BY + _expr_1 DESC + ) AS _expr_0 + FROM + table_1 +) +SELECT + billing_country, + ROUND(COALESCE(SUM(total), 0), 2) AS total +FROM + table_0 +WHERE + _expr_0 <= 1 + AND customer_id IN (4) +GROUP BY + billing_country diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap new file mode 100644 index 000000000000..6a7f9d6674c7 --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap @@ -0,0 +1,183 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: "from invoices\nderive code = case [customer_id < 10 => billing_postal_code, true => null]\ngroup {customer_id, billing_city, billing_country} (\n sort {-this.code}\n take 1\n)\nfilter (customer_id | in [4])\ngroup {billing_country} (aggregate {total = math.round 2 (sum total)})\n" +input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql +--- +--- generic ++++ clickhouse +@@ -1,39 +1,27 @@ +-WITH table_1 AS ( ++WITH table_0 AS ( + SELECT +- billing_country, ++ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, + total, + customer_id, + billing_city, + CASE + WHEN customer_id < 10 THEN billing_postal_code + ELSE NULL +- END AS _expr_1, +- billing_postal_code ++ END AS _expr_0 + FROM + invoices +-), +-table_0 AS ( +- SELECT ++ ORDER BY ++ customer_id, ++ billing_city, + billing_country, +- total, +- customer_id, +- ROW_NUMBER() OVER ( +- PARTITION BY customer_id, +- billing_city, +- billing_country +- ORDER BY +- _expr_1 DESC +- ) AS _expr_0 +- FROM +- table_1 ++ _expr_0 DESC + ) + SELECT + billing_country, + ROUND(COALESCE(SUM(total), 0), 2) AS total + FROM + table_0 + WHERE +- _expr_0 <= 1 +- AND customer_id IN (4) ++ customer_id IN (4) + GROUP BY + billing_country + +--- generic ++++ duckdb +@@ -1,39 +1,27 @@ +-WITH table_1 AS ( ++WITH table_0 AS ( + SELECT +- billing_country, ++ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, + total, + customer_id, + billing_city, + CASE + WHEN customer_id < 10 THEN billing_postal_code + ELSE NULL +- END AS _expr_1, +- billing_postal_code ++ END AS _expr_0 + FROM + invoices +-), +-table_0 AS ( +- SELECT ++ ORDER BY ++ customer_id, ++ billing_city, + billing_country, +- total, +- customer_id, +- ROW_NUMBER() OVER ( +- PARTITION BY customer_id, +- billing_city, +- billing_country +- ORDER BY +- _expr_1 DESC +- ) AS _expr_0 +- FROM +- table_1 ++ _expr_0 DESC + ) + SELECT + billing_country, + ROUND(COALESCE(SUM(total), 0), 2) AS total + FROM + table_0 + WHERE +- _expr_0 <= 1 +- AND customer_id IN (4) ++ customer_id IN (4) + GROUP BY + billing_country + + +--- generic ++++ glaredb +@@ -22,18 +22,18 @@ + billing_city, + billing_country + ORDER BY + _expr_1 DESC + ) AS _expr_0 + FROM + table_1 + ) + SELECT + billing_country, +- ROUND(COALESCE(SUM(total), 0), 2) AS total ++ ROUND((COALESCE(SUM(total), 0))::numeric, 2) AS total + FROM + table_0 + WHERE + _expr_0 <= 1 + AND customer_id IN (4) + GROUP BY + billing_country + + +--- generic ++++ postgres +@@ -1,39 +1,27 @@ +-WITH table_1 AS ( ++WITH table_0 AS ( + SELECT +- billing_country, ++ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, + total, + customer_id, + billing_city, + CASE + WHEN customer_id < 10 THEN billing_postal_code + ELSE NULL +- END AS _expr_1, +- billing_postal_code ++ END AS _expr_0 + FROM + invoices +-), +-table_0 AS ( +- SELECT ++ ORDER BY ++ customer_id, ++ billing_city, + billing_country, +- total, +- customer_id, +- ROW_NUMBER() OVER ( +- PARTITION BY customer_id, +- billing_city, +- billing_country +- ORDER BY +- _expr_1 DESC +- ) AS _expr_0 +- FROM +- table_1 ++ _expr_0 DESC + ) + SELECT + billing_country, +- ROUND(COALESCE(SUM(total), 0), 2) AS total ++ ROUND((COALESCE(SUM(total), 0))::numeric, 2) AS total + FROM + table_0 + WHERE +- _expr_0 <= 1 +- AND customer_id IN (4) ++ customer_id IN (4) + GROUP BY + billing_country diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap new file mode 100644 index 000000000000..e86e7c0f66c3 --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap @@ -0,0 +1,490 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: "from invoices\nderive code = case [customer_id < 10 => billing_postal_code, true => null]\ngroup {customer_id, billing_city, billing_country} (\n sort {-this.code}\n take 1\n)\nfilter (customer_id | in [4])\ngroup {billing_country} (aggregate {total = math.round 2 (sum total)})\n" +input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql +--- +frames: +- - 1:14-88 + - columns: + - !All + input_id: 124 + except: [] + - !Single + name: + - code + target_id: 126 + target_name: null + inputs: + - id: 124 + name: invoices + table: + - default_db + - invoices +- - 1:164-170 + - columns: + - !Single + name: + - invoices + - customer_id + target_id: 136 + target_name: null + - !Single + name: + - invoices + - billing_city + target_id: 137 + target_name: null + - !Single + name: + - invoices + - billing_country + target_id: 138 + target_name: null + - !All + input_id: 124 + except: + - billing_city + - billing_country + - customer_id + - !Single + name: + - code + target_id: 126 + target_name: null + inputs: + - id: 124 + name: invoices + table: + - default_db + - invoices +- - 1:173-202 + - columns: + - !Single + name: + - invoices + - customer_id + target_id: 136 + target_name: null + - !Single + name: + - invoices + - billing_city + target_id: 137 + target_name: null + - !Single + name: + - invoices + - billing_country + target_id: 138 + target_name: null + - !All + input_id: 124 + except: + - billing_city + - billing_country + - customer_id + - !Single + name: + - code + target_id: 126 + target_name: null + inputs: + - id: 124 + name: invoices + table: + - default_db + - invoices +- - 1:228-272 + - columns: + - !Single + name: + - invoices + - billing_country + target_id: 179 + target_name: null + - !Single + name: + - total + target_id: 196 + target_name: null + inputs: + - id: 124 + name: invoices + table: + - default_db + - invoices +nodes: +- id: 124 + kind: Ident + span: 1:0-13 + ident: !Ident + - default_db + - invoices + parent: 135 +- id: 126 + kind: Case + span: 1:28-88 + alias: code + targets: + - 127 + - 131 + - 132 + - 133 + parent: 134 +- id: 127 + kind: RqOperator + span: 1:34-50 + targets: + - 129 + - 130 +- id: 129 + kind: Ident + span: 1:34-45 + ident: !Ident + - this + - invoices + - customer_id + targets: + - 124 +- id: 130 + kind: Literal + span: 1:48-50 +- id: 131 + kind: Ident + span: 1:54-73 + ident: !Ident + - this + - invoices + - billing_postal_code + targets: + - 124 +- id: 132 + kind: Literal + span: 1:75-79 +- id: 133 + kind: Literal + span: 1:83-87 +- id: 134 + kind: Tuple + span: 1:28-88 + children: + - 126 + parent: 135 +- id: 135 + kind: 'TransformCall: Derive' + span: 1:14-88 + children: + - 124 + - 134 + parent: 167 +- id: 136 + kind: Ident + span: 1:96-107 + ident: !Ident + - this + - invoices + - customer_id + targets: + - 124 + parent: 139 +- id: 137 + kind: Ident + span: 1:109-121 + ident: !Ident + - this + - invoices + - billing_city + targets: + - 124 + parent: 139 +- id: 138 + kind: Ident + span: 1:123-138 + ident: !Ident + - this + - invoices + - billing_country + targets: + - 124 + parent: 139 +- id: 139 + kind: Tuple + span: 1:95-139 + children: + - 136 + - 137 + - 138 +- id: 163 + kind: Ident + span: 1:151-160 + ident: !Ident + - this + - code + targets: + - 126 +- id: 167 + kind: 'TransformCall: Take' + span: 1:164-170 + children: + - 135 + - 168 + parent: 178 +- id: 168 + kind: Literal + parent: 167 +- id: 174 + kind: Array + span: 1:198-201 + children: + - 175 +- id: 175 + kind: Literal + span: 1:199-200 + parent: 174 +- id: 176 + kind: Ident + span: 1:181-192 + ident: !Ident + - this + - invoices + - customer_id + targets: + - 136 +- id: 177 + kind: RqOperator + span: 1:195-201 + targets: + - 176 + - 174 + parent: 178 +- id: 178 + kind: 'TransformCall: Filter' + span: 1:173-202 + children: + - 167 + - 177 + parent: 203 +- id: 179 + kind: Ident + span: 1:210-225 + ident: !Ident + - this + - invoices + - billing_country + targets: + - 138 + parent: 180 +- id: 180 + kind: Tuple + span: 1:209-226 + children: + - 179 + parent: 203 +- id: 196 + kind: RqOperator + span: 1:247-271 + alias: total + targets: + - 198 + - 199 + parent: 202 +- id: 198 + kind: Literal + span: 1:258-259 +- id: 199 + kind: RqOperator + span: 1:261-270 + targets: + - 201 +- id: 201 + kind: Ident + span: 1:265-270 + ident: !Ident + - this + - invoices + - total + targets: + - 124 +- id: 202 + kind: Tuple + span: 1:238-272 + children: + - 196 + parent: 203 +- id: 203 + kind: 'TransformCall: Aggregate' + span: 1:228-272 + children: + - 178 + - 202 + - 180 +ast: + name: Project + stmts: + - VarDef: + kind: Main + name: main + value: + Pipeline: + exprs: + - FuncCall: + name: + Ident: + - from + span: 1:0-4 + args: + - Ident: + - invoices + span: 1:5-13 + span: 1:0-13 + - FuncCall: + name: + Ident: + - derive + span: 1:14-20 + args: + - Case: + - condition: + Binary: + left: + Ident: + - customer_id + span: 1:34-45 + op: Lt + right: + Literal: + Integer: 10 + span: 1:48-50 + span: 1:34-50 + value: + Ident: + - billing_postal_code + span: 1:54-73 + - condition: + Literal: + Boolean: true + span: 1:75-79 + value: + Literal: 'Null' + span: 1:83-87 + span: 1:28-88 + alias: code + span: 1:14-88 + - FuncCall: + name: + Ident: + - group + span: 1:89-94 + args: + - Tuple: + - Ident: + - customer_id + span: 1:96-107 + - Ident: + - billing_city + span: 1:109-121 + - Ident: + - billing_country + span: 1:123-138 + span: 1:95-139 + - Pipeline: + exprs: + - FuncCall: + name: + Ident: + - sort + span: 1:144-148 + args: + - Tuple: + - Unary: + op: Neg + expr: + Ident: + - this + - code + span: 1:151-160 + span: 1:150-160 + span: 1:149-161 + span: 1:144-161 + - FuncCall: + name: + Ident: + - take + span: 1:164-168 + args: + - Literal: + Integer: 1 + span: 1:169-170 + span: 1:164-170 + span: 1:144-170 + span: 1:89-172 + - FuncCall: + name: + Ident: + - filter + span: 1:173-179 + args: + - Pipeline: + exprs: + - Ident: + - customer_id + span: 1:181-192 + - FuncCall: + name: + Ident: + - in + span: 1:195-197 + args: + - Array: + - Literal: + Integer: 4 + span: 1:199-200 + span: 1:198-201 + span: 1:195-201 + span: 1:181-201 + span: 1:173-202 + - FuncCall: + name: + Ident: + - group + span: 1:203-208 + args: + - Tuple: + - Ident: + - billing_country + span: 1:210-225 + span: 1:209-226 + - FuncCall: + name: + Ident: + - aggregate + span: 1:228-237 + args: + - Tuple: + - FuncCall: + name: + Ident: + - math + - round + span: 1:247-257 + args: + - Literal: + Integer: 2 + span: 1:258-259 + - FuncCall: + name: + Ident: + - sum + span: 1:261-264 + args: + - Ident: + - total + span: 1:265-270 + span: 1:261-270 + span: 1:247-271 + alias: total + span: 1:238-272 + span: 1:228-272 + span: 1:203-273 + span: 1:0-273 + span: 1:0-273 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__distinct_on_sort_on_compute.snap new file mode 100644 index 000000000000..c1b47dc3ce29 --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__distinct_on_sort_on_compute.snap @@ -0,0 +1,13 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: "from invoices\nderive code = case [customer_id < 10 => billing_postal_code, true => null]\ngroup {customer_id, billing_city, billing_country} (\n sort {-this.code}\n take 1\n)\nfilter (customer_id | in [4])\ngroup {billing_country} (aggregate {total = math.round 2 (sum total)})\n" +input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql +--- +from invoices +derive code = case [customer_id < 10 => billing_postal_code, true => null] +group {customer_id, billing_city, billing_country} ( + sort {-this.code} + take 1 +) +filter (customer_id | in [4]) +group {billing_country} (aggregate {total = math.round 2 (sum total)}) diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on_sort_on_compute.snap new file mode 100644 index 000000000000..2a3679afc0a7 --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on_sort_on_compute.snap @@ -0,0 +1,82 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: tokens +input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql +--- +Tokens( + [ + 0..0: Start, + 0..4: Ident("from"), + 5..13: Ident("invoices"), + 13..14: NewLine, + 14..20: Ident("derive"), + 21..25: Ident("code"), + 26..27: Control('='), + 28..32: Keyword("case"), + 33..34: Control('['), + 34..45: Ident("customer_id"), + 46..47: Control('<'), + 48..50: Literal(Integer(10)), + 51..53: ArrowFat, + 54..73: Ident("billing_postal_code"), + 73..74: Control(','), + 75..79: Literal(Boolean(true)), + 80..82: ArrowFat, + 83..87: Literal(Null), + 87..88: Control(']'), + 88..89: NewLine, + 89..94: Ident("group"), + 95..96: Control('{'), + 96..107: Ident("customer_id"), + 107..108: Control(','), + 109..121: Ident("billing_city"), + 121..122: Control(','), + 123..138: Ident("billing_country"), + 138..139: Control('}'), + 140..141: Control('('), + 141..142: NewLine, + 144..148: Ident("sort"), + 149..150: Control('{'), + 150..151: Control('-'), + 151..155: Ident("this"), + 155..156: Control('.'), + 156..160: Ident("code"), + 160..161: Control('}'), + 161..162: NewLine, + 164..168: Ident("take"), + 169..170: Literal(Integer(1)), + 170..171: NewLine, + 171..172: Control(')'), + 172..173: NewLine, + 173..179: Ident("filter"), + 180..181: Control('('), + 181..192: Ident("customer_id"), + 193..194: Control('|'), + 195..197: Ident("in"), + 198..199: Control('['), + 199..200: Literal(Integer(4)), + 200..201: Control(']'), + 201..202: Control(')'), + 202..203: NewLine, + 203..208: Ident("group"), + 209..210: Control('{'), + 210..225: Ident("billing_country"), + 225..226: Control('}'), + 227..228: Control('('), + 228..237: Ident("aggregate"), + 238..239: Control('{'), + 239..244: Ident("total"), + 245..246: Control('='), + 247..251: Ident("math"), + 251..252: Control('.'), + 252..257: Ident("round"), + 258..259: Literal(Integer(2)), + 260..261: Control('('), + 261..264: Ident("sum"), + 265..270: Ident("total"), + 270..271: Control(')'), + 271..272: Control('}'), + 272..273: Control(')'), + 273..274: NewLine, + ], +) diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__distinct_on_sort_on_compute.snap new file mode 100644 index 000000000000..3f98eec01f67 --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__distinct_on_sort_on_compute.snap @@ -0,0 +1,6 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: "from invoices\nderive code = case [customer_id < 10 => billing_postal_code, true => null ]\ngroup {customer_id, billing_city, billing_country} (\n sort {-this.code}\n take 1\n)\nfilter (customer_id | in [4])\ngroup {billing_country} (aggregate {total = sum total})\n" +input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql +--- +Norway,3.96 diff --git a/prqlc/prqlc/tests/integration/sql.rs b/prqlc/prqlc/tests/integration/sql.rs index 0964a69fc7d8..7ca1890832d1 100644 --- a/prqlc/prqlc/tests/integration/sql.rs +++ b/prqlc/prqlc/tests/integration/sql.rs @@ -2518,17 +2518,11 @@ fn test_distinct_on_03() { derive foo = 1 select foo "###).unwrap()), @r" - WITH table_1 AS ( + WITH table_0 AS ( SELECT - DISTINCT ON (col1) col1 + DISTINCT ON (col1) NULL FROM tab1 - ), - table_0 AS ( - SELECT - NULL - FROM - table_1 ) SELECT 1 AS foo @@ -5715,17 +5709,10 @@ fn test_missing_columns_group_complex_compute() { city FROM employees - ), - table_1 AS ( - SELECT - DISTINCT ON (_expr_0, year_label) year_label, - _expr_0 - FROM - table_0 ) SELECT - year_label + DISTINCT ON (_expr_0, year_label) year_label FROM - table_1 + table_0 "); } From 151358658b966d848c36bd3c29678c7b53c61d0a Mon Sep 17 00:00:00 2001 From: Fanaen Date: Tue, 22 Jul 2025 11:59:28 +0200 Subject: [PATCH 3/7] fix(prqlc): also require columns for super-less sorts --- prqlc/prqlc/src/sql/pq/anchor.rs | 4 + ...mpileall__distinct_on_sort_on_compute.snap | 114 +++++++++--------- ..._lineage__distinct_on_sort_on_compute.snap | 2 + 3 files changed, 66 insertions(+), 54 deletions(-) diff --git a/prqlc/prqlc/src/sql/pq/anchor.rs b/prqlc/prqlc/src/sql/pq/anchor.rs index 8deb5bc49495..7718d6018abb 100644 --- a/prqlc/prqlc/src/sql/pq/anchor.rs +++ b/prqlc/prqlc/src/sql/pq/anchor.rs @@ -552,6 +552,10 @@ pub(super) fn get_requirements( .should_select(true) } + SqlTransform::Sort(sorts) if !following.contains("Aggregate") => { + Requirements::from_cids(sorts.iter().map(|s| &s.column)) + } + SqlTransform::DistinctOn(partition) => Requirements::from_cids(partition.iter()) // Since there is aggregation anyway, columns can have any complexity .allow_up_to(Complexity::highest()), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap index 6a7f9d6674c7..592d2a8e4cae 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap @@ -5,12 +5,10 @@ input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.pr --- --- generic +++ clickhouse -@@ -1,39 +1,27 @@ --WITH table_1 AS ( -+WITH table_0 AS ( +@@ -1,39 +1,38 @@ + WITH table_1 AS ( SELECT -- billing_country, -+ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, + billing_country, total, customer_id, billing_city, @@ -18,19 +16,17 @@ input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.pr WHEN customer_id < 10 THEN billing_postal_code ELSE NULL - END AS _expr_1, -- billing_postal_code -+ END AS _expr_0 ++ END AS _expr_0, + billing_postal_code FROM invoices --), --table_0 AS ( -- SELECT -+ ORDER BY -+ customer_id, -+ billing_city, - billing_country, -- total, -- customer_id, + ), + table_0 AS ( + SELECT +- billing_country, ++ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, + total, + customer_id, - ROW_NUMBER() OVER ( - PARTITION BY customer_id, - billing_city, @@ -38,8 +34,14 @@ input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.pr - ORDER BY - _expr_1 DESC - ) AS _expr_0 -- FROM -- table_1 ++ billing_city, ++ _expr_0 + FROM + table_1 ++ ORDER BY ++ customer_id, ++ billing_city, ++ billing_country, + _expr_0 DESC ) SELECT @@ -56,12 +58,10 @@ input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.pr --- generic +++ duckdb -@@ -1,39 +1,27 @@ --WITH table_1 AS ( -+WITH table_0 AS ( +@@ -1,39 +1,38 @@ + WITH table_1 AS ( SELECT -- billing_country, -+ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, + billing_country, total, customer_id, billing_city, @@ -69,19 +69,17 @@ input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.pr WHEN customer_id < 10 THEN billing_postal_code ELSE NULL - END AS _expr_1, -- billing_postal_code -+ END AS _expr_0 ++ END AS _expr_0, + billing_postal_code FROM invoices --), --table_0 AS ( -- SELECT -+ ORDER BY -+ customer_id, -+ billing_city, - billing_country, -- total, -- customer_id, + ), + table_0 AS ( + SELECT +- billing_country, ++ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, + total, + customer_id, - ROW_NUMBER() OVER ( - PARTITION BY customer_id, - billing_city, @@ -89,8 +87,14 @@ input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.pr - ORDER BY - _expr_1 DESC - ) AS _expr_0 -- FROM -- table_1 ++ billing_city, ++ _expr_0 + FROM + table_1 ++ ORDER BY ++ customer_id, ++ billing_city, ++ billing_country, + _expr_0 DESC ) SELECT @@ -132,12 +136,10 @@ input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.pr --- generic +++ postgres -@@ -1,39 +1,27 @@ --WITH table_1 AS ( -+WITH table_0 AS ( +@@ -1,39 +1,38 @@ + WITH table_1 AS ( SELECT -- billing_country, -+ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, + billing_country, total, customer_id, billing_city, @@ -145,19 +147,17 @@ input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.pr WHEN customer_id < 10 THEN billing_postal_code ELSE NULL - END AS _expr_1, -- billing_postal_code -+ END AS _expr_0 ++ END AS _expr_0, + billing_postal_code FROM invoices --), --table_0 AS ( -- SELECT -+ ORDER BY -+ customer_id, -+ billing_city, - billing_country, -- total, -- customer_id, + ), + table_0 AS ( + SELECT +- billing_country, ++ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, + total, + customer_id, - ROW_NUMBER() OVER ( - PARTITION BY customer_id, - billing_city, @@ -165,8 +165,14 @@ input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.pr - ORDER BY - _expr_1 DESC - ) AS _expr_0 -- FROM -- table_1 ++ billing_city, ++ _expr_0 + FROM + table_1 ++ ORDER BY ++ customer_id, ++ billing_city, ++ billing_country, + _expr_0 DESC ) SELECT diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap index e86e7c0f66c3..3fc7113398b7 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap @@ -44,6 +44,7 @@ frames: input_id: 124 except: - billing_city + - customer_id - billing_country - customer_id - !Single @@ -81,6 +82,7 @@ frames: input_id: 124 except: - billing_city + - customer_id - billing_country - customer_id - !Single From 2c134b83a3e437d4e3d1ce660350ae57f652e0d1 Mon Sep 17 00:00:00 2001 From: Fanaen Date: Thu, 24 Jul 2025 17:44:47 +0200 Subject: [PATCH 4/7] feat(prqlc, requirements): add helpers --- prqlc/prqlc/src/sql/pq/anchor.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/prqlc/prqlc/src/sql/pq/anchor.rs b/prqlc/prqlc/src/sql/pq/anchor.rs index 7718d6018abb..7fb042a049e2 100644 --- a/prqlc/prqlc/src/sql/pq/anchor.rs +++ b/prqlc/prqlc/src/sql/pq/anchor.rs @@ -483,6 +483,14 @@ impl Requirements { } self } + + pub fn is_selected(&self, id: &CId) -> bool { + self.0.iter().any(|r| r.selected && &r.col == id) + } + + pub fn is_required(&self, id: &CId) -> bool { + self.0.iter().any(|r| &r.col == id) + } } impl std::fmt::Debug for Requirement { From 381ca2b6694d378b678fccfb6c64194b4991d1e4 Mon Sep 17 00:00:00 2001 From: Fanaen Date: Thu, 24 Jul 2025 17:39:27 +0200 Subject: [PATCH 5/7] fix(prqlc): ignore requirements for unselected columns Chain of unselected compute columns added a lot of useless requirements so skipping them improve results and avoid edge cases in positional mapping. --- prqlc/prqlc/src/sql/pq/anchor.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/prqlc/prqlc/src/sql/pq/anchor.rs b/prqlc/prqlc/src/sql/pq/anchor.rs index 7fb042a049e2..c7241e8d007c 100644 --- a/prqlc/prqlc/src/sql/pq/anchor.rs +++ b/prqlc/prqlc/src/sql/pq/anchor.rs @@ -100,7 +100,7 @@ pub(super) fn split_off_back( } // anchor and record all requirements - let required = get_requirements(&transform, &following_transforms); + let required = get_requirements(&transform, &following_transforms, &inputs_required); log::debug!(".. transform {} requires {required:?}", transform.as_str(),); inputs_required = inputs_required.append(required.clone()); @@ -504,19 +504,14 @@ impl std::fmt::Debug for Requirement { pub(super) fn get_requirements( transform: &SqlTransform, following: &HashSet, + previous_requirements: &Requirements, ) -> Requirements { use SqlTransform::Super; match transform { - Super(Transform::Aggregate { partition, compute }) => { - let partition_requirements = Requirements::from_cids(partition.iter()); - let compute_requirements = - Requirements::from_cids(compute.iter()).allow_up_to(Complexity::Aggregation); + Super(Transform::Aggregate { partition, .. }) => Requirements::from_cids(partition.iter()), - partition_requirements.append(compute_requirements) - } - - Super(Transform::Compute(compute)) => { + Super(Transform::Compute(compute)) if previous_requirements.is_required(&compute.id) => { let requirements = Requirements::from_expr(&compute.expr).allow_up_to( match infer_complexity(compute) { // plain expressions can be included in anything less complex than Aggregation From 7857394858d62aadebe859cab8c1fde0a9c4f44b Mon Sep 17 00:00:00 2001 From: Fanaen Date: Thu, 24 Jul 2025 17:46:49 +0200 Subject: [PATCH 6/7] fix(prqlc): ignore unselected columns in positional mapping and fix matching Previous version of positional mapping worked on all columns. Now, we stick to selected ones to make sure we don't have extra columns. Also, there was a mistake in positional mapping matching, which meant e.g. RIId 1 was associated with RIId 2, and therefore absurd mappings. --- prqlc/prqlc/src/sql/pq/anchor.rs | 12 +- prqlc/prqlc/src/sql/pq/positional_mapping.rs | 56 +- .../queries/append_select_compute.prql | 2 + .../queries/append_select_multiple.prql | 18 + ...eries__compile__append_select_compute.snap | 14 +- ...ries__compile__append_select_multiple.snap | 66 ++ ...es__compileall__append_select_compute.snap | 36 +- ...s__compileall__append_select_multiple.snap | 130 ++ ..._debug_lineage__append_select_compute.snap | 614 +++++++--- ...debug_lineage__append_select_multiple.snap | 1053 +++++++++++++++++ ..._lineage__distinct_on_sort_on_compute.snap | 2 - ...__queries__fmt__append_select_compute.snap | 10 +- ..._queries__fmt__append_select_multiple.snap | 25 + ...__queries__lex__append_select_compute.snap | 142 ++- ..._queries__lex__append_select_multiple.snap | 132 +++ ...eries__results__append_select_compute.snap | 20 +- ...ries__results__append_select_multiple.snap | 18 + prqlc/prqlc/tests/integration/sql.rs | 28 +- 18 files changed, 2073 insertions(+), 305 deletions(-) create mode 100644 prqlc/prqlc/tests/integration/queries/append_select_multiple.prql create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_multiple.snap create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_multiple.snap create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_multiple.snap create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_multiple.snap create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_multiple.snap create mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_multiple.snap diff --git a/prqlc/prqlc/src/sql/pq/anchor.rs b/prqlc/prqlc/src/sql/pq/anchor.rs index c7241e8d007c..5b41a636d212 100644 --- a/prqlc/prqlc/src/sql/pq/anchor.rs +++ b/prqlc/prqlc/src/sql/pq/anchor.rs @@ -75,7 +75,7 @@ pub(super) fn split_off_back( return (None, Vec::new()); } - let mapping_before = compute_positional_mappings(&pipeline); + let mapping_before = compute_positional_mappings(&pipeline, None); log::debug!("traversing pipeline to obtain columns: {output:?}"); @@ -188,10 +188,12 @@ pub(super) fn split_off_back( curr_pipeline_rev.reverse(); // This will compare columns for order sensitive transform and correct it in subsequent relation. - let mapping_after = compute_positional_mappings(&curr_pipeline_rev); - for (before, after) in mapping_before.iter().zip(mapping_after.iter()) { - ctx.positional_mapping - .compute_and_store_mapping(before, after); + let mapping_after = compute_positional_mappings(&curr_pipeline_rev, Some(&inputs_required)); + for (riid, after) in mapping_after { + if let Some((_, before)) = mapping_before.iter().find(|(r, _)| &riid == r) { + ctx.positional_mapping + .compute_and_store_mapping(before, &after, &riid); + } } (remaining_pipeline, curr_pipeline_rev) diff --git a/prqlc/prqlc/src/sql/pq/positional_mapping.rs b/prqlc/prqlc/src/sql/pq/positional_mapping.rs index e58ada0eee8f..b1e087531f41 100644 --- a/prqlc/prqlc/src/sql/pq/positional_mapping.rs +++ b/prqlc/prqlc/src/sql/pq/positional_mapping.rs @@ -1,8 +1,11 @@ use std::collections::HashMap; use crate::{ - ir::rq::{CId, Transform}, - sql::{pq::context::RIId, pq_ast::SqlTransform}, + ir::rq::{CId, Compute, Transform}, + sql::{ + pq::{anchor::Requirements, context::RIId}, + pq_ast::SqlTransform, + }, }; /// State required to properly handle the transforms that are order sensitive like `Union`. @@ -16,29 +19,24 @@ impl PositionalMapper { /// Remember the mapping for this `RIId` to know what to apply for `apply_positional_mapping`. pub(crate) fn activate_mapping(&mut self, riid: &RIId) { self.active_positional_mapping = self.relation_positional_mapping.remove(riid); + log::trace!( + "loading remapping for {riid:?}: {:?}", + self.active_positional_mapping + ); } /// Reorder or remove columns to make `Union` happy. pub(crate) fn apply_active_mapping(&mut self, output: Vec) -> Vec { if let Some(mapping) = &self.active_positional_mapping { let new_output = mapping.iter().map(|idx| output[*idx]).collect(); - log::debug!("remapping {output:?} to {new_output:?}"); + log::debug!("remapping {output:?} to {new_output:?} via {mapping:?}"); new_output } else { output } } - pub fn compute_and_store_mapping( - &mut self, - (_, before): &(RIId, Vec), - (riid, after): &(RIId, Vec), - ) { - if after == before { - log::trace!(".. relation {riid:?} is already correctly mapped: {after:?}"); - return; - } - + pub fn compute_and_store_mapping(&mut self, before: &[CId], after: &[CId], riid: &RIId) { let mapping: Vec<_> = after .iter() .flat_map(|a| match before.iter().position(|b| b == a) { @@ -60,28 +58,37 @@ impl PositionalMapper { /// Outputs the columns required for position sensitive transforms in the pipeline. pub fn compute_positional_mappings( pipeline: &[SqlTransform], + requirements: Option<&Requirements>, ) -> Vec<(RIId, Vec)> { let mut constraints = vec![]; let mut columns = vec![]; log::trace!("traversing pipeline to obtain positional mapping:"); + // Only process selected columns to avoid surnumerary one + let add_columns = |columns: &mut Vec, cids: &[CId]| { + if let Some(requirements) = requirements { + columns.extend(cids.iter().filter(|cid| requirements.is_selected(cid))); + } else { + columns.extend_from_slice(cids); + } + }; + for transform in pipeline { match transform { SqlTransform::Super(s) => match s { - Transform::Compute(compute) => { - if !columns.contains(&compute.id) { - columns.push(compute.id); + Transform::Compute(Compute { id, .. }) => { + if !columns.contains(id) { + add_columns(&mut columns, &[*id]); } } Transform::Select(cids) => { columns.clear(); - columns.extend_from_slice(cids.as_slice()); + add_columns(&mut columns, cids); } - Transform::Aggregate { partition, compute } => { + Transform::Aggregate { compute, .. } => { columns.clear(); - columns.extend_from_slice(partition.as_slice()); - columns.extend_from_slice(compute.as_slice()); + add_columns(&mut columns, compute); } _ => (), }, @@ -89,10 +96,17 @@ pub fn compute_positional_mappings( | SqlTransform::Intersect { bottom, .. } | SqlTransform::Union { bottom, .. } => { constraints.push((*bottom, columns.clone())); + log::trace!( + ".. mapping for {}/{bottom:?}: {columns:?}", + transform.as_str() + ); } _ => (), } - log::trace!(".. columns after {}: {columns:?}", transform.as_str()); + log::trace!( + ".. selected columns after {}: {columns:?}", + transform.as_str() + ); } constraints diff --git a/prqlc/prqlc/tests/integration/queries/append_select_compute.prql b/prqlc/prqlc/tests/integration/queries/append_select_compute.prql index ee64ba181c32..9ea6a89f4862 100644 --- a/prqlc/prqlc/tests/integration/queries/append_select_compute.prql +++ b/prqlc/prqlc/tests/integration/queries/append_select_compute.prql @@ -1,8 +1,10 @@ from invoices +derive total = case [total < 10 => total * 2, true => total] select { customer_id, invoice_id, total } take 5 append ( from invoice_items + derive unit_price = case [unit_price < 1 => unit_price * 2, true => unit_price] select { invoice_line_id, invoice_id, unit_price } take 5 ) diff --git a/prqlc/prqlc/tests/integration/queries/append_select_multiple.prql b/prqlc/prqlc/tests/integration/queries/append_select_multiple.prql new file mode 100644 index 000000000000..ff2e35882442 --- /dev/null +++ b/prqlc/prqlc/tests/integration/queries/append_select_multiple.prql @@ -0,0 +1,18 @@ +from invoices +select { customer_id, invoice_id, total, useless1, useless2 } +take 5 +append ( + from employees + select { employee_id, employee_id + 1, reports_to, useless3, useless4 } + take 5 +) +group { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 }) +append ( + from invoice_items + select { invoice_id, invoice_line_id, 0, useless5 } + take 5 +) +sort { +invoice_id, +total } +select { total, invoice_id } + +# sqlite:skip diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_compute.snap index 2e94e3fc0458..63ecf95c65c9 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_compute.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_compute.snap @@ -1,6 +1,6 @@ --- source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nselect { customer_id, invoice_id, total }\ntake 5\nappend (\n from invoice_items\n select { invoice_line_id, invoice_id, unit_price }\n take 5\n)\nselect { a = customer_id * 2, b = math.round 1 (invoice_id * total) }\n" +expression: "from invoices\nderive total = case [total < 10 => total * 2, true => total]\nselect { customer_id, invoice_id, total }\ntake 5\nappend (\n from invoice_items\n derive unit_price = case [unit_price < 1 => unit_price * 2, true => unit_price]\n select { invoice_line_id, invoice_id, unit_price }\n take 5\n)\nselect { a = customer_id * 2, b = math.round 1 (invoice_id * total) }\n" input_file: prqlc/prqlc/tests/integration/queries/append_select_compute.prql --- WITH table_1 AS ( @@ -10,7 +10,10 @@ WITH table_1 AS ( ( SELECT invoice_id, - total, + CASE + WHEN total < 10 THEN total * 2 + ELSE total + END AS _expr_0, customer_id FROM invoices @@ -25,7 +28,10 @@ WITH table_1 AS ( ( SELECT invoice_id, - unit_price, + CASE + WHEN unit_price < 1 THEN unit_price * 2 + ELSE unit_price + END AS unit_price, invoice_line_id FROM invoice_items @@ -35,6 +41,6 @@ WITH table_1 AS ( ) SELECT customer_id * 2 AS a, - ROUND(invoice_id * total, 1) AS b + ROUND(invoice_id * _expr_0, 1) AS b FROM table_1 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_multiple.snap new file mode 100644 index 000000000000..c06e7f10f1f3 --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_multiple.snap @@ -0,0 +1,66 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: "from invoices\nselect { customer_id, invoice_id, total, useless1, useless2 }\ntake 5\nappend (\n from employees\n select { employee_id, employee_id + 1, reports_to, useless3, useless4 }\n take 5\n)\ngroup { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 })\nappend (\n from invoice_items\n select { invoice_id, invoice_line_id, 0, useless5 }\n take 5\n)\nsort { +invoice_id, +total }\nselect { total, invoice_id }\n\n# sqlite:skip\n" +input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql +--- +WITH table_3 AS ( + SELECT + * + FROM + ( + SELECT + customer_id, + total, + invoice_id + FROM + invoices + LIMIT + 5 + ) AS table_6 + UNION + ALL + SELECT + * + FROM + ( + SELECT + employee_id, + reports_to, + employee_id + 1 + FROM + employees + LIMIT + 5 + ) AS table_7 +), +table_2 AS ( + SELECT + ROUND(COALESCE(SUM(total), 0), 1) AS total, + ROUND(COALESCE(SUM(invoice_id), 0), 1) AS invoice_id + FROM + table_3 + GROUP BY + customer_id + UNION + ALL + SELECT + * + FROM + ( + SELECT + invoice_id, + invoice_line_id + FROM + invoice_items + LIMIT + 5 + ) AS table_8 +) +SELECT + total, + invoice_id +FROM + table_2 +ORDER BY + invoice_id, + total diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_compute.snap index a49972b431b3..7fe1d3938ac8 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_compute.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_compute.snap @@ -1,12 +1,12 @@ --- source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nselect { customer_id, invoice_id, total }\ntake 5\nappend (\n from invoice_items\n select { invoice_line_id, invoice_id, unit_price }\n take 5\n)\nselect { a = customer_id * 2, b = math.round 1 (invoice_id * total) }\n" +expression: "from invoices\nderive total = case [total < 10 => total * 2, true => total]\nselect { customer_id, invoice_id, total }\ntake 5\nappend (\n from invoice_items\n derive unit_price = case [unit_price < 1 => unit_price * 2, true => unit_price]\n select { invoice_line_id, invoice_id, unit_price }\n take 5\n)\nselect { a = customer_id * 2, b = math.round 1 (invoice_id * total) }\n" input_file: prqlc/prqlc/tests/integration/queries/append_select_compute.prql --- --- generic +++ glaredb -@@ -23,13 +23,13 @@ - unit_price, +@@ -29,13 +29,13 @@ + END AS unit_price, invoice_line_id FROM invoice_items @@ -16,15 +16,15 @@ input_file: prqlc/prqlc/tests/integration/queries/append_select_compute.prql ) SELECT customer_id * 2 AS a, -- ROUND(invoice_id * total, 1) AS b -+ ROUND((invoice_id * total)::numeric, 1) AS b +- ROUND(invoice_id * _expr_0, 1) AS b ++ ROUND((invoice_id * _expr_0)::numeric, 1) AS b FROM table_1 --- generic +++ postgres -@@ -1,35 +1,28 @@ +@@ -1,41 +1,34 @@ WITH table_1 AS ( - SELECT - * @@ -32,7 +32,10 @@ input_file: prqlc/prqlc/tests/integration/queries/append_select_compute.prql - ( - SELECT - invoice_id, -- total, +- CASE +- WHEN total < 10 THEN total * 2 +- ELSE total +- END AS _expr_0, - customer_id - FROM - invoices @@ -42,7 +45,10 @@ input_file: prqlc/prqlc/tests/integration/queries/append_select_compute.prql + ( + SELECT + invoice_id, -+ total, ++ CASE ++ WHEN total < 10 THEN total * 2 ++ ELSE total ++ END AS _expr_0, + customer_id + FROM + invoices @@ -57,7 +63,10 @@ input_file: prqlc/prqlc/tests/integration/queries/append_select_compute.prql - ( - SELECT - invoice_id, -- unit_price, +- CASE +- WHEN unit_price < 1 THEN unit_price * 2 +- ELSE unit_price +- END AS unit_price, - invoice_line_id - FROM - invoice_items @@ -67,7 +76,10 @@ input_file: prqlc/prqlc/tests/integration/queries/append_select_compute.prql + ALL ( + SELECT + invoice_id, -+ unit_price, ++ CASE ++ WHEN unit_price < 1 THEN unit_price * 2 ++ ELSE unit_price ++ END AS unit_price, + invoice_line_id + FROM + invoice_items @@ -77,7 +89,7 @@ input_file: prqlc/prqlc/tests/integration/queries/append_select_compute.prql ) SELECT customer_id * 2 AS a, -- ROUND(invoice_id * total, 1) AS b -+ ROUND((invoice_id * total)::numeric, 1) AS b +- ROUND(invoice_id * _expr_0, 1) AS b ++ ROUND((invoice_id * _expr_0)::numeric, 1) AS b FROM table_1 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_multiple.snap new file mode 100644 index 000000000000..dd4bcb643cbc --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_multiple.snap @@ -0,0 +1,130 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: "from invoices\nselect { customer_id, invoice_id, total, useless1, useless2 }\ntake 5\nappend (\n from employees\n select { employee_id, employee_id + 1, reports_to, useless3, useless4 }\n take 5\n)\ngroup { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 })\nappend (\n from invoice_items\n select { invoice_id, invoice_line_id, 0, useless5 }\n take 5\n)\nsort { +invoice_id, +total }\nselect { total, invoice_id }\n\n# sqlite:skip\n" +input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql +--- +--- generic ++++ glaredb +@@ -23,22 +23,22 @@ + reports_to, + employee_id + 1 + FROM + employees + LIMIT + 5 + ) AS table_7 + ), + table_2 AS ( + SELECT +- ROUND(COALESCE(SUM(total), 0), 1) AS total, +- ROUND(COALESCE(SUM(invoice_id), 0), 1) AS invoice_id ++ ROUND((COALESCE(SUM(total), 0))::numeric, 1) AS total, ++ ROUND((COALESCE(SUM(invoice_id), 0))::numeric, 1) AS invoice_id + FROM + table_3 + GROUP BY + customer_id + UNION + ALL + SELECT + * + FROM + ( + + +--- generic ++++ postgres +@@ -1,61 +1,49 @@ + WITH table_3 AS ( +- SELECT +- * +- FROM +- ( +- SELECT +- customer_id, +- total, +- invoice_id +- FROM +- invoices +- LIMIT +- 5 +- ) AS table_6 ++ ( ++ SELECT ++ customer_id, ++ total, ++ invoice_id ++ FROM ++ invoices ++ LIMIT ++ 5 ++ ) + UNION +- ALL ++ ALL ( ++ SELECT ++ employee_id, ++ reports_to, ++ employee_id + 1 ++ FROM ++ employees ++ LIMIT ++ 5 ++ ) ++), table_2 AS ( + SELECT +- * ++ ROUND((COALESCE(SUM(total), 0))::numeric, 1) AS total, ++ ROUND((COALESCE(SUM(invoice_id), 0))::numeric, 1) AS invoice_id + FROM +- ( +- SELECT +- employee_id, +- reports_to, +- employee_id + 1 +- FROM +- employees +- LIMIT +- 5 +- ) AS table_7 +-), +-table_2 AS ( +- SELECT +- ROUND(COALESCE(SUM(total), 0), 1) AS total, +- ROUND(COALESCE(SUM(invoice_id), 0), 1) AS invoice_id +- FROM + table_3 + GROUP BY + customer_id + UNION +- ALL +- SELECT +- * +- FROM +- ( +- SELECT +- invoice_id, +- invoice_line_id +- FROM +- invoice_items +- LIMIT +- 5 +- ) AS table_8 ++ ALL ( ++ SELECT ++ invoice_id, ++ invoice_line_id ++ FROM ++ invoice_items ++ LIMIT ++ 5 ++ ) + ) + SELECT + total, + invoice_id + FROM + table_2 + ORDER BY + invoice_id, + total diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_compute.snap index e78d7c60000f..2453eedbd9f9 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_compute.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_compute.snap @@ -1,357 +1,523 @@ --- source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nselect { customer_id, invoice_id, total }\ntake 5\nappend (\n from invoice_items\n select { invoice_line_id, invoice_id, unit_price }\n take 5\n)\nselect { a = customer_id * 2, b = math.round 1 (invoice_id * total) }\n" +expression: "from invoices\nderive total = case [total < 10 => total * 2, true => total]\nselect { customer_id, invoice_id, total }\ntake 5\nappend (\n from invoice_items\n derive unit_price = case [unit_price < 1 => unit_price * 2, true => unit_price]\n select { invoice_line_id, invoice_id, unit_price }\n take 5\n)\nselect { a = customer_id * 2, b = math.round 1 (invoice_id * total) }\n" input_file: prqlc/prqlc/tests/integration/queries/append_select_compute.prql --- frames: -- - 1:14-55 +- - 1:14-74 + - columns: + - !All + input_id: 161 + except: [] + - !Single + name: + - total + target_id: 163 + target_name: null + inputs: + - id: 161 + name: invoices + table: + - default_db + - invoices +- - 1:75-116 - columns: - !Single name: - invoices - customer_id - target_id: 144 + target_id: 176 target_name: null - !Single name: - invoices - invoice_id - target_id: 145 + target_id: 177 target_name: null - !Single name: - - invoices - total - target_id: 146 + target_id: 178 target_name: null inputs: - - id: 142 + - id: 161 name: invoices table: - default_db - invoices -- - 1:56-62 +- - 1:117-123 - columns: - !Single name: - invoices - customer_id - target_id: 144 + target_id: 176 target_name: null - !Single name: - invoices - invoice_id - target_id: 145 + target_id: 177 target_name: null - !Single name: - - invoices - total - target_id: 146 + target_id: 178 target_name: null inputs: - - id: 142 + - id: 161 name: invoices table: - default_db - invoices -- - 1:95-145 +- - 1:156-235 + - columns: + - !All + input_id: 127 + except: [] + - !Single + name: + - unit_price + target_id: 129 + target_name: null + inputs: + - id: 127 + name: invoice_items + table: + - default_db + - invoice_items +- - 1:238-288 - columns: - !Single name: - invoice_items - invoice_line_id - target_id: 126 + target_id: 142 target_name: null - !Single name: - invoice_items - invoice_id - target_id: 127 + target_id: 143 target_name: null - !Single name: - - invoice_items - unit_price - target_id: 128 + target_id: 144 target_name: null inputs: - - id: 124 + - id: 127 name: invoice_items table: - default_db - invoice_items -- - 1:148-154 +- - 1:291-297 - columns: - !Single name: - invoice_items - invoice_line_id - target_id: 126 + target_id: 142 target_name: null - !Single name: - invoice_items - invoice_id - target_id: 127 + target_id: 143 target_name: null - !Single name: - - invoice_items - unit_price - target_id: 128 + target_id: 144 target_name: null inputs: - - id: 124 + - id: 127 name: invoice_items table: - default_db - invoice_items -- - 1:63-156 +- - 1:124-299 - columns: - !Single name: - invoices - customer_id - target_id: 144 + target_id: 176 target_name: null - !Single name: - invoices - invoice_id - target_id: 145 + target_id: 177 target_name: null - !Single name: - - invoices - total - target_id: 146 + target_id: 178 target_name: null inputs: - - id: 142 + - id: 161 name: invoices table: - default_db - invoices -- - 1:157-226 +- - 1:300-369 - columns: - !Single name: - a - target_id: 153 + target_id: 185 target_name: null - !Single name: - b - target_id: 157 + target_id: 189 target_name: null inputs: - - id: 142 + - id: 161 name: invoices table: - default_db - invoices nodes: -- id: 124 +- id: 127 kind: Ident - span: 1:74-92 + span: 1:135-153 ident: !Ident - default_db - invoice_items - parent: 130 -- id: 126 + parent: 141 +- id: 129 + kind: Case + span: 1:176-235 + alias: unit_price + targets: + - 130 + - 134 + - 138 + - 139 + parent: 140 +- id: 130 + kind: RqOperator + span: 1:182-196 + targets: + - 132 + - 133 +- id: 132 kind: Ident - span: 1:104-119 + span: 1:182-192 ident: !Ident - this - invoice_items - - invoice_line_id + - unit_price targets: - - 124 - parent: 129 -- id: 127 + - 127 +- id: 133 + kind: Literal + span: 1:195-196 +- id: 134 + kind: RqOperator + span: 1:200-214 + targets: + - 136 + - 137 +- id: 136 kind: Ident - span: 1:121-131 + span: 1:200-210 ident: !Ident - this - invoice_items - - invoice_id + - unit_price targets: - - 124 - parent: 129 -- id: 128 + - 127 +- id: 137 + kind: Literal + span: 1:213-214 +- id: 138 + kind: Literal + span: 1:216-220 +- id: 139 kind: Ident - span: 1:133-143 + span: 1:224-234 ident: !Ident - this - invoice_items - unit_price targets: - - 124 - parent: 129 -- id: 129 + - 127 +- id: 140 kind: Tuple - span: 1:102-145 + span: 1:176-235 + children: + - 129 + parent: 141 +- id: 141 + kind: 'TransformCall: Derive' + span: 1:156-235 children: - - 126 - 127 - - 128 - parent: 130 -- id: 130 + - 140 + parent: 146 +- id: 142 + kind: Ident + span: 1:247-262 + ident: !Ident + - this + - invoice_items + - invoice_line_id + targets: + - 127 + parent: 145 +- id: 143 + kind: Ident + span: 1:264-274 + ident: !Ident + - this + - invoice_items + - invoice_id + targets: + - 127 + parent: 145 +- id: 144 + kind: Ident + span: 1:276-286 + ident: !Ident + - this + - unit_price + targets: + - 129 + parent: 145 +- id: 145 + kind: Tuple + span: 1:245-288 + children: + - 142 + - 143 + - 144 + parent: 146 +- id: 146 kind: 'TransformCall: Select' - span: 1:95-145 + span: 1:238-288 children: - - 124 - - 129 - parent: 132 -- id: 132 + - 141 + - 145 + parent: 148 +- id: 148 kind: 'TransformCall: Take' - span: 1:148-154 + span: 1:291-297 children: - - 130 - - 133 - parent: 152 -- id: 133 + - 146 + - 149 + parent: 184 +- id: 149 kind: Literal - parent: 132 -- id: 142 + parent: 148 +- id: 161 kind: Ident span: 1:0-13 ident: !Ident - default_db - invoices - parent: 148 -- id: 144 + parent: 175 +- id: 163 + kind: Case + span: 1:29-74 + alias: total + targets: + - 164 + - 168 + - 172 + - 173 + parent: 174 +- id: 164 + kind: RqOperator + span: 1:35-45 + targets: + - 166 + - 167 +- id: 166 + kind: Ident + span: 1:35-40 + ident: !Ident + - this + - invoices + - total + targets: + - 161 +- id: 167 + kind: Literal + span: 1:43-45 +- id: 168 + kind: RqOperator + span: 1:49-58 + targets: + - 170 + - 171 +- id: 170 + kind: Ident + span: 1:49-54 + ident: !Ident + - this + - invoices + - total + targets: + - 161 +- id: 171 + kind: Literal + span: 1:57-58 +- id: 172 + kind: Literal + span: 1:60-64 +- id: 173 + kind: Ident + span: 1:68-73 + ident: !Ident + - this + - invoices + - total + targets: + - 161 +- id: 174 + kind: Tuple + span: 1:29-74 + children: + - 163 + parent: 175 +- id: 175 + kind: 'TransformCall: Derive' + span: 1:14-74 + children: + - 161 + - 174 + parent: 180 +- id: 176 kind: Ident - span: 1:23-34 + span: 1:84-95 ident: !Ident - this - invoices - customer_id targets: - - 142 - parent: 147 -- id: 145 + - 161 + parent: 179 +- id: 177 kind: Ident - span: 1:36-46 + span: 1:97-107 ident: !Ident - this - invoices - invoice_id targets: - - 142 - parent: 147 -- id: 146 + - 161 + parent: 179 +- id: 178 kind: Ident - span: 1:48-53 + span: 1:109-114 ident: !Ident - this - - invoices - total targets: - - 142 - parent: 147 -- id: 147 + - 163 + parent: 179 +- id: 179 kind: Tuple - span: 1:21-55 + span: 1:82-116 children: - - 144 - - 145 - - 146 - parent: 148 -- id: 148 + - 176 + - 177 + - 178 + parent: 180 +- id: 180 kind: 'TransformCall: Select' - span: 1:14-55 + span: 1:75-116 children: - - 142 - - 147 - parent: 150 -- id: 150 + - 175 + - 179 + parent: 182 +- id: 182 kind: 'TransformCall: Take' - span: 1:56-62 + span: 1:117-123 children: - - 148 - - 151 - parent: 152 -- id: 151 + - 180 + - 183 + parent: 184 +- id: 183 kind: Literal - parent: 150 -- id: 152 + parent: 182 +- id: 184 kind: 'TransformCall: Append' - span: 1:63-156 + span: 1:124-299 children: - - 150 - - 132 - parent: 165 -- id: 153 + - 182 + - 148 + parent: 197 +- id: 185 kind: RqOperator - span: 1:170-185 + span: 1:313-328 alias: a targets: - - 155 - - 156 - parent: 164 -- id: 155 + - 187 + - 188 + parent: 196 +- id: 187 kind: Ident - span: 1:170-181 + span: 1:313-324 ident: !Ident - this - invoices - customer_id targets: - - 144 -- id: 156 + - 176 +- id: 188 kind: Literal - span: 1:184-185 -- id: 157 + span: 1:327-328 +- id: 189 kind: RqOperator - span: 1:191-224 + span: 1:334-367 alias: b targets: - - 159 - - 160 - parent: 164 -- id: 159 + - 191 + - 192 + parent: 196 +- id: 191 kind: Literal - span: 1:202-203 -- id: 160 + span: 1:345-346 +- id: 192 kind: RqOperator - span: 1:205-223 + span: 1:348-366 targets: - - 162 - - 163 -- id: 162 + - 194 + - 195 +- id: 194 kind: Ident - span: 1:205-215 + span: 1:348-358 ident: !Ident - this - invoices - invoice_id targets: - - 145 -- id: 163 + - 177 +- id: 195 kind: Ident - span: 1:218-223 + span: 1:361-366 ident: !Ident - this - - invoices - total targets: - - 146 -- id: 164 + - 178 +- id: 196 kind: Tuple - span: 1:164-226 + span: 1:307-369 children: - - 153 - - 157 - parent: 165 -- id: 165 + - 185 + - 189 + parent: 197 +- id: 197 kind: 'TransformCall: Select' - span: 1:157-226 + span: 1:300-369 children: - - 152 - - 164 + - 184 + - 196 ast: name: Project stmts: @@ -374,36 +540,78 @@ ast: - FuncCall: name: Ident: - - select + - derive span: 1:14-20 args: + - Case: + - condition: + Binary: + left: + Ident: + - total + span: 1:35-40 + op: Lt + right: + Literal: + Integer: 10 + span: 1:43-45 + span: 1:35-45 + value: + Binary: + left: + Ident: + - total + span: 1:49-54 + op: Mul + right: + Literal: + Integer: 2 + span: 1:57-58 + span: 1:49-58 + - condition: + Literal: + Boolean: true + span: 1:60-64 + value: + Ident: + - total + span: 1:68-73 + span: 1:29-74 + alias: total + span: 1:14-74 + - FuncCall: + name: + Ident: + - select + span: 1:75-81 + args: - Tuple: - Ident: - customer_id - span: 1:23-34 + span: 1:84-95 - Ident: - invoice_id - span: 1:36-46 + span: 1:97-107 - Ident: - total - span: 1:48-53 - span: 1:21-55 - span: 1:14-55 + span: 1:109-114 + span: 1:82-116 + span: 1:75-116 - FuncCall: name: Ident: - take - span: 1:56-60 + span: 1:117-121 args: - Literal: Integer: 5 - span: 1:61-62 - span: 1:56-62 + span: 1:122-123 + span: 1:117-123 - FuncCall: name: Ident: - append - span: 1:63-69 + span: 1:124-130 args: - Pipeline: exprs: @@ -411,85 +619,127 @@ ast: name: Ident: - from - span: 1:74-78 + span: 1:135-139 args: - Ident: - invoice_items - span: 1:79-92 - span: 1:74-92 + span: 1:140-153 + span: 1:135-153 + - FuncCall: + name: + Ident: + - derive + span: 1:156-162 + args: + - Case: + - condition: + Binary: + left: + Ident: + - unit_price + span: 1:182-192 + op: Lt + right: + Literal: + Integer: 1 + span: 1:195-196 + span: 1:182-196 + value: + Binary: + left: + Ident: + - unit_price + span: 1:200-210 + op: Mul + right: + Literal: + Integer: 2 + span: 1:213-214 + span: 1:200-214 + - condition: + Literal: + Boolean: true + span: 1:216-220 + value: + Ident: + - unit_price + span: 1:224-234 + span: 1:176-235 + alias: unit_price + span: 1:156-235 - FuncCall: name: Ident: - select - span: 1:95-101 + span: 1:238-244 args: - Tuple: - Ident: - invoice_line_id - span: 1:104-119 + span: 1:247-262 - Ident: - invoice_id - span: 1:121-131 + span: 1:264-274 - Ident: - unit_price - span: 1:133-143 - span: 1:102-145 - span: 1:95-145 + span: 1:276-286 + span: 1:245-288 + span: 1:238-288 - FuncCall: name: Ident: - take - span: 1:148-152 + span: 1:291-295 args: - Literal: Integer: 5 - span: 1:153-154 - span: 1:148-154 - span: 1:74-154 - span: 1:63-156 + span: 1:296-297 + span: 1:291-297 + span: 1:135-297 + span: 1:124-299 - FuncCall: name: Ident: - select - span: 1:157-163 + span: 1:300-306 args: - Tuple: - Binary: left: Ident: - customer_id - span: 1:170-181 + span: 1:313-324 op: Mul right: Literal: Integer: 2 - span: 1:184-185 - span: 1:170-185 + span: 1:327-328 + span: 1:313-328 alias: a - FuncCall: name: Ident: - math - round - span: 1:191-201 + span: 1:334-344 args: - Literal: Integer: 1 - span: 1:202-203 + span: 1:345-346 - Binary: left: Ident: - invoice_id - span: 1:205-215 + span: 1:348-358 op: Mul right: Ident: - total - span: 1:218-223 - span: 1:205-223 - span: 1:191-224 + span: 1:361-366 + span: 1:348-366 + span: 1:334-367 alias: b - span: 1:164-226 - span: 1:157-226 - span: 1:0-226 - span: 1:0-226 + span: 1:307-369 + span: 1:300-369 + span: 1:0-369 + span: 1:0-369 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_multiple.snap new file mode 100644 index 000000000000..47a733b9737b --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_multiple.snap @@ -0,0 +1,1053 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: "from invoices\nselect { customer_id, invoice_id, total, useless1, useless2 }\ntake 5\nappend (\n from employees\n select { employee_id, employee_id + 1, reports_to, useless3, useless4 }\n take 5\n)\ngroup { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 })\nappend (\n from invoice_items\n select { invoice_id, invoice_line_id, 0, useless5 }\n take 5\n)\nsort { +invoice_id, +total }\nselect { total, invoice_id }\n\n# sqlite:skip\n" +input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql +--- +frames: +- - 1:14-75 + - columns: + - !Single + name: + - invoices + - customer_id + target_id: 177 + target_name: null + - !Single + name: + - invoices + - invoice_id + target_id: 178 + target_name: null + - !Single + name: + - invoices + - total + target_id: 179 + target_name: null + - !Single + name: + - invoices + - useless1 + target_id: 180 + target_name: null + - !Single + name: + - invoices + - useless2 + target_id: 181 + target_name: null + inputs: + - id: 175 + name: invoices + table: + - default_db + - invoices +- - 1:76-82 + - columns: + - !Single + name: + - invoices + - customer_id + target_id: 177 + target_name: null + - !Single + name: + - invoices + - invoice_id + target_id: 178 + target_name: null + - !Single + name: + - invoices + - total + target_id: 179 + target_name: null + - !Single + name: + - invoices + - useless1 + target_id: 180 + target_name: null + - !Single + name: + - invoices + - useless2 + target_id: 181 + target_name: null + inputs: + - id: 175 + name: invoices + table: + - default_db + - invoices +- - 1:111-182 + - columns: + - !Single + name: + - employees + - employee_id + target_id: 154 + target_name: null + - !Single + name: null + target_id: 155 + target_name: null + - !Single + name: + - employees + - reports_to + target_id: 159 + target_name: null + - !Single + name: + - employees + - useless3 + target_id: 160 + target_name: null + - !Single + name: + - employees + - useless4 + target_id: 161 + target_name: null + inputs: + - id: 152 + name: employees + table: + - default_db + - employees +- - 1:185-191 + - columns: + - !Single + name: + - employees + - employee_id + target_id: 154 + target_name: null + - !Single + name: null + target_id: 155 + target_name: null + - !Single + name: + - employees + - reports_to + target_id: 159 + target_name: null + - !Single + name: + - employees + - useless3 + target_id: 160 + target_name: null + - !Single + name: + - employees + - useless4 + target_id: 161 + target_name: null + inputs: + - id: 152 + name: employees + table: + - default_db + - employees +- - 1:83-193 + - columns: + - !Single + name: + - invoices + - customer_id + target_id: 177 + target_name: null + - !Single + name: + - invoices + - invoice_id + target_id: 178 + target_name: null + - !Single + name: + - invoices + - total + target_id: 179 + target_name: null + - !Single + name: + - invoices + - useless1 + target_id: 180 + target_name: null + - !Single + name: + - invoices + - useless2 + target_id: 181 + target_name: null + inputs: + - id: 175 + name: invoices + table: + - default_db + - invoices +- - 1:217-332 + - columns: + - !Single + name: + - invoices + - customer_id + target_id: 188 + target_name: null + - !Single + name: + - invoice_id + target_id: 208 + target_name: null + - !Single + name: + - total + target_id: 214 + target_name: null + - !Single + name: + - useless1 + target_id: 220 + target_name: null + inputs: + - id: 175 + name: invoices + table: + - default_db + - invoices +- - 1:366-417 + - columns: + - !Single + name: + - invoice_items + - invoice_id + target_id: 129 + target_name: null + - !Single + name: + - invoice_items + - invoice_line_id + target_id: 130 + target_name: null + - !Single + name: null + target_id: 131 + target_name: null + - !Single + name: + - invoice_items + - useless5 + target_id: 132 + target_name: null + inputs: + - id: 127 + name: invoice_items + table: + - default_db + - invoice_items +- - 1:420-426 + - columns: + - !Single + name: + - invoice_items + - invoice_id + target_id: 129 + target_name: null + - !Single + name: + - invoice_items + - invoice_line_id + target_id: 130 + target_name: null + - !Single + name: null + target_id: 131 + target_name: null + - !Single + name: + - invoice_items + - useless5 + target_id: 132 + target_name: null + inputs: + - id: 127 + name: invoice_items + table: + - default_db + - invoice_items +- - 1:334-428 + - columns: + - !Single + name: + - invoices + - customer_id + target_id: 188 + target_name: null + - !Single + name: + - invoice_id + target_id: 208 + target_name: null + - !Single + name: + - total + target_id: 214 + target_name: null + - !Single + name: + - useless1 + target_id: 220 + target_name: null + inputs: + - id: 175 + name: invoices + table: + - default_db + - invoices +- - 1:429-457 + - columns: + - !Single + name: + - invoices + - customer_id + target_id: 188 + target_name: null + - !Single + name: + - invoice_id + target_id: 208 + target_name: null + - !Single + name: + - total + target_id: 214 + target_name: null + - !Single + name: + - useless1 + target_id: 220 + target_name: null + inputs: + - id: 175 + name: invoices + table: + - default_db + - invoices +- - 1:458-486 + - columns: + - !Single + name: + - total + target_id: 232 + target_name: null + - !Single + name: + - invoice_id + target_id: 233 + target_name: null + inputs: + - id: 175 + name: invoices + table: + - default_db + - invoices +nodes: +- id: 127 + kind: Ident + span: 1:345-363 + ident: !Ident + - default_db + - invoice_items + parent: 134 +- id: 129 + kind: Ident + span: 1:375-385 + ident: !Ident + - this + - invoice_items + - invoice_id + targets: + - 127 + parent: 133 +- id: 130 + kind: Ident + span: 1:387-402 + ident: !Ident + - this + - invoice_items + - invoice_line_id + targets: + - 127 + parent: 133 +- id: 131 + kind: Literal + span: 1:404-405 + parent: 133 +- id: 132 + kind: Ident + span: 1:407-415 + ident: !Ident + - this + - invoice_items + - useless5 + targets: + - 127 + parent: 133 +- id: 133 + kind: Tuple + span: 1:373-417 + children: + - 129 + - 130 + - 131 + - 132 + parent: 134 +- id: 134 + kind: 'TransformCall: Select' + span: 1:366-417 + children: + - 127 + - 133 + parent: 136 +- id: 136 + kind: 'TransformCall: Take' + span: 1:420-426 + children: + - 134 + - 137 + parent: 227 +- id: 137 + kind: Literal + parent: 136 +- id: 152 + kind: Ident + span: 1:94-108 + ident: !Ident + - default_db + - employees + parent: 163 +- id: 154 + kind: Ident + span: 1:120-131 + ident: !Ident + - this + - employees + - employee_id + targets: + - 152 + parent: 162 +- id: 155 + kind: RqOperator + span: 1:133-148 + targets: + - 157 + - 158 + parent: 162 +- id: 157 + kind: Ident + span: 1:133-144 + ident: !Ident + - this + - employees + - employee_id + targets: + - 152 +- id: 158 + kind: Literal + span: 1:147-148 +- id: 159 + kind: Ident + span: 1:150-160 + ident: !Ident + - this + - employees + - reports_to + targets: + - 152 + parent: 162 +- id: 160 + kind: Ident + span: 1:162-170 + ident: !Ident + - this + - employees + - useless3 + targets: + - 152 + parent: 162 +- id: 161 + kind: Ident + span: 1:172-180 + ident: !Ident + - this + - employees + - useless4 + targets: + - 152 + parent: 162 +- id: 162 + kind: Tuple + span: 1:118-182 + children: + - 154 + - 155 + - 159 + - 160 + - 161 + parent: 163 +- id: 163 + kind: 'TransformCall: Select' + span: 1:111-182 + children: + - 152 + - 162 + parent: 165 +- id: 165 + kind: 'TransformCall: Take' + span: 1:185-191 + children: + - 163 + - 166 + parent: 187 +- id: 166 + kind: Literal + parent: 165 +- id: 175 + kind: Ident + span: 1:0-13 + ident: !Ident + - default_db + - invoices + parent: 183 +- id: 177 + kind: Ident + span: 1:23-34 + ident: !Ident + - this + - invoices + - customer_id + targets: + - 175 + parent: 182 +- id: 178 + kind: Ident + span: 1:36-46 + ident: !Ident + - this + - invoices + - invoice_id + targets: + - 175 + parent: 182 +- id: 179 + kind: Ident + span: 1:48-53 + ident: !Ident + - this + - invoices + - total + targets: + - 175 + parent: 182 +- id: 180 + kind: Ident + span: 1:55-63 + ident: !Ident + - this + - invoices + - useless1 + targets: + - 175 + parent: 182 +- id: 181 + kind: Ident + span: 1:65-73 + ident: !Ident + - this + - invoices + - useless2 + targets: + - 175 + parent: 182 +- id: 182 + kind: Tuple + span: 1:21-75 + children: + - 177 + - 178 + - 179 + - 180 + - 181 + parent: 183 +- id: 183 + kind: 'TransformCall: Select' + span: 1:14-75 + children: + - 175 + - 182 + parent: 185 +- id: 185 + kind: 'TransformCall: Take' + span: 1:76-82 + children: + - 183 + - 186 + parent: 187 +- id: 186 + kind: Literal + parent: 185 +- id: 187 + kind: 'TransformCall: Append' + span: 1:83-193 + children: + - 185 + - 165 + parent: 224 +- id: 188 + kind: Ident + span: 1:202-213 + ident: !Ident + - this + - invoices + - customer_id + targets: + - 177 + parent: 189 +- id: 189 + kind: Tuple + span: 1:200-215 + children: + - 188 + parent: 224 +- id: 208 + kind: RqOperator + span: 1:242-271 + alias: invoice_id + targets: + - 210 + - 211 + parent: 223 +- id: 210 + kind: Literal + span: 1:253-254 +- id: 211 + kind: RqOperator + span: 1:256-270 + targets: + - 213 +- id: 213 + kind: Ident + span: 1:260-270 + ident: !Ident + - this + - invoices + - invoice_id + targets: + - 178 +- id: 214 + kind: RqOperator + span: 1:281-305 + alias: total + targets: + - 216 + - 217 + parent: 223 +- id: 216 + kind: Literal + span: 1:292-293 +- id: 217 + kind: RqOperator + span: 1:295-304 + targets: + - 219 +- id: 219 + kind: Ident + span: 1:299-304 + ident: !Ident + - this + - invoices + - total + targets: + - 179 +- id: 220 + kind: RqOperator + span: 1:318-330 + alias: useless1 + targets: + - 222 + parent: 223 +- id: 222 + kind: Ident + span: 1:322-330 + ident: !Ident + - this + - invoices + - useless1 + targets: + - 180 +- id: 223 + kind: Tuple + span: 1:227-332 + children: + - 208 + - 214 + - 220 + parent: 224 +- id: 224 + kind: 'TransformCall: Aggregate' + span: 1:217-332 + children: + - 187 + - 223 + - 189 + parent: 227 +- id: 227 + kind: 'TransformCall: Append' + span: 1:334-428 + children: + - 224 + - 136 + parent: 231 +- id: 228 + kind: Ident + span: 1:436-447 + ident: !Ident + - this + - invoice_id + targets: + - 208 + parent: 231 +- id: 229 + kind: Ident + span: 1:449-455 + ident: !Ident + - this + - total + targets: + - 214 + parent: 231 +- id: 231 + kind: 'TransformCall: Sort' + span: 1:429-457 + children: + - 227 + - 228 + - 229 + parent: 235 +- id: 232 + kind: Ident + span: 1:467-472 + ident: !Ident + - this + - total + targets: + - 214 + parent: 234 +- id: 233 + kind: Ident + span: 1:474-484 + ident: !Ident + - this + - invoice_id + targets: + - 208 + parent: 234 +- id: 234 + kind: Tuple + span: 1:465-486 + children: + - 232 + - 233 + parent: 235 +- id: 235 + kind: 'TransformCall: Select' + span: 1:458-486 + children: + - 231 + - 234 +ast: + name: Project + stmts: + - VarDef: + kind: Main + name: main + value: + Pipeline: + exprs: + - FuncCall: + name: + Ident: + - from + span: 1:0-4 + args: + - Ident: + - invoices + span: 1:5-13 + span: 1:0-13 + - FuncCall: + name: + Ident: + - select + span: 1:14-20 + args: + - Tuple: + - Ident: + - customer_id + span: 1:23-34 + - Ident: + - invoice_id + span: 1:36-46 + - Ident: + - total + span: 1:48-53 + - Ident: + - useless1 + span: 1:55-63 + - Ident: + - useless2 + span: 1:65-73 + span: 1:21-75 + span: 1:14-75 + - FuncCall: + name: + Ident: + - take + span: 1:76-80 + args: + - Literal: + Integer: 5 + span: 1:81-82 + span: 1:76-82 + - FuncCall: + name: + Ident: + - append + span: 1:83-89 + args: + - Pipeline: + exprs: + - FuncCall: + name: + Ident: + - from + span: 1:94-98 + args: + - Ident: + - employees + span: 1:99-108 + span: 1:94-108 + - FuncCall: + name: + Ident: + - select + span: 1:111-117 + args: + - Tuple: + - Ident: + - employee_id + span: 1:120-131 + - Binary: + left: + Ident: + - employee_id + span: 1:133-144 + op: Add + right: + Literal: + Integer: 1 + span: 1:147-148 + span: 1:133-148 + - Ident: + - reports_to + span: 1:150-160 + - Ident: + - useless3 + span: 1:162-170 + - Ident: + - useless4 + span: 1:172-180 + span: 1:118-182 + span: 1:111-182 + - FuncCall: + name: + Ident: + - take + span: 1:185-189 + args: + - Literal: + Integer: 5 + span: 1:190-191 + span: 1:185-191 + span: 1:94-191 + span: 1:83-193 + - FuncCall: + name: + Ident: + - group + span: 1:194-199 + args: + - Tuple: + - Ident: + - customer_id + span: 1:202-213 + span: 1:200-215 + - FuncCall: + name: + Ident: + - aggregate + span: 1:217-226 + args: + - Tuple: + - FuncCall: + name: + Ident: + - math + - round + span: 1:242-252 + args: + - Literal: + Integer: 1 + span: 1:253-254 + - FuncCall: + name: + Ident: + - sum + span: 1:256-259 + args: + - Ident: + - invoice_id + span: 1:260-270 + span: 1:256-270 + span: 1:242-271 + alias: invoice_id + - FuncCall: + name: + Ident: + - math + - round + span: 1:281-291 + args: + - Literal: + Integer: 1 + span: 1:292-293 + - FuncCall: + name: + Ident: + - sum + span: 1:295-298 + args: + - Ident: + - total + span: 1:299-304 + span: 1:295-304 + span: 1:281-305 + alias: total + - FuncCall: + name: + Ident: + - sum + span: 1:318-321 + args: + - Ident: + - useless1 + span: 1:322-330 + span: 1:318-330 + alias: useless1 + span: 1:227-332 + span: 1:217-332 + span: 1:194-333 + - FuncCall: + name: + Ident: + - append + span: 1:334-340 + args: + - Pipeline: + exprs: + - FuncCall: + name: + Ident: + - from + span: 1:345-349 + args: + - Ident: + - invoice_items + span: 1:350-363 + span: 1:345-363 + - FuncCall: + name: + Ident: + - select + span: 1:366-372 + args: + - Tuple: + - Ident: + - invoice_id + span: 1:375-385 + - Ident: + - invoice_line_id + span: 1:387-402 + - Literal: + Integer: 0 + span: 1:404-405 + - Ident: + - useless5 + span: 1:407-415 + span: 1:373-417 + span: 1:366-417 + - FuncCall: + name: + Ident: + - take + span: 1:420-424 + args: + - Literal: + Integer: 5 + span: 1:425-426 + span: 1:420-426 + span: 1:345-426 + span: 1:334-428 + - FuncCall: + name: + Ident: + - sort + span: 1:429-433 + args: + - Tuple: + - Unary: + op: Add + expr: + Ident: + - invoice_id + span: 1:437-447 + span: 1:436-447 + - Unary: + op: Add + expr: + Ident: + - total + span: 1:450-455 + span: 1:449-455 + span: 1:434-457 + span: 1:429-457 + - FuncCall: + name: + Ident: + - select + span: 1:458-464 + args: + - Tuple: + - Ident: + - total + span: 1:467-472 + - Ident: + - invoice_id + span: 1:474-484 + span: 1:465-486 + span: 1:458-486 + span: 1:0-486 + span: 1:0-486 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap index 3fc7113398b7..e86e7c0f66c3 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap @@ -44,7 +44,6 @@ frames: input_id: 124 except: - billing_city - - customer_id - billing_country - customer_id - !Single @@ -82,7 +81,6 @@ frames: input_id: 124 except: - billing_city - - customer_id - billing_country - customer_id - !Single diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_compute.snap index 9caef69eb576..1870538f72c8 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_compute.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_compute.snap @@ -1,13 +1,21 @@ --- source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nselect { customer_id, invoice_id, total }\ntake 5\nappend (\n from invoice_items\n select { invoice_line_id, invoice_id, unit_price }\n take 5\n)\nselect { a = customer_id * 2, b = math.round 1 (invoice_id * total) }\n" +expression: "from invoices\nderive total = case [total < 10 => total * 2, true => total]\nselect { customer_id, invoice_id, total }\ntake 5\nappend (\n from invoice_items\n derive unit_price = case [unit_price < 1 => unit_price * 2, true => unit_price]\n select { invoice_line_id, invoice_id, unit_price }\n take 5\n)\nselect { a = customer_id * 2, b = math.round 1 (invoice_id * total) }\n" input_file: prqlc/prqlc/tests/integration/queries/append_select_compute.prql --- from invoices +derive total = case [ + total < 10 => total * 2, + true => total, +] select {customer_id, invoice_id, total} take 5 append ( from invoice_items + derive unit_price = case [ + unit_price < 1 => unit_price * 2, + true => unit_price, + ] select {invoice_line_id, invoice_id, unit_price} take 5 ) diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_multiple.snap new file mode 100644 index 000000000000..b50a6e4cfe08 --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_multiple.snap @@ -0,0 +1,25 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: "from invoices\nselect { customer_id, invoice_id, total, useless1, useless2 }\ntake 5\nappend (\n from employees\n select { employee_id, employee_id + 1, reports_to, useless3, useless4 }\n take 5\n)\ngroup { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 })\nappend (\n from invoice_items\n select { invoice_id, invoice_line_id, 0, useless5 }\n take 5\n)\nsort { +invoice_id, +total }\nselect { total, invoice_id }\n\n# sqlite:skip\n" +input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql +--- +from invoices +select {customer_id, invoice_id, total, useless1, useless2} +take 5 +append ( + from employees + select {employee_id, employee_id + 1, reports_to, useless3, useless4} + take 5 +) +group {customer_id} (aggregate { + invoice_id = math.round 1 (sum invoice_id), + total = math.round 1 (sum total), + useless1 = sum useless1, +}) +append ( + from invoice_items + select {invoice_id, invoice_line_id, 0, useless5} + take 5 +) +sort {+invoice_id, +total} +select {total, invoice_id} diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_compute.snap index ee9ab54bf862..739d9d6b00c3 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_compute.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_compute.snap @@ -9,58 +9,94 @@ Tokens( 0..4: Ident("from"), 5..13: Ident("invoices"), 13..14: NewLine, - 14..20: Ident("select"), - 21..22: Control('{'), - 23..34: Ident("customer_id"), - 34..35: Control(','), - 36..46: Ident("invoice_id"), - 46..47: Control(','), - 48..53: Ident("total"), - 54..55: Control('}'), - 55..56: NewLine, - 56..60: Ident("take"), - 61..62: Literal(Integer(5)), - 62..63: NewLine, - 63..69: Ident("append"), - 70..71: Control('('), - 71..72: NewLine, - 74..78: Ident("from"), - 79..92: Ident("invoice_items"), - 92..93: NewLine, - 95..101: Ident("select"), - 102..103: Control('{'), - 104..119: Ident("invoice_line_id"), - 119..120: Control(','), - 121..131: Ident("invoice_id"), - 131..132: Control(','), - 133..143: Ident("unit_price"), - 144..145: Control('}'), - 145..146: NewLine, - 148..152: Ident("take"), - 153..154: Literal(Integer(5)), - 154..155: NewLine, - 155..156: Control(')'), - 156..157: NewLine, - 157..163: Ident("select"), - 164..165: Control('{'), - 166..167: Ident("a"), - 168..169: Control('='), - 170..181: Ident("customer_id"), - 182..183: Control('*'), - 184..185: Literal(Integer(2)), - 185..186: Control(','), - 187..188: Ident("b"), - 189..190: Control('='), - 191..195: Ident("math"), - 195..196: Control('.'), - 196..201: Ident("round"), - 202..203: Literal(Integer(1)), - 204..205: Control('('), - 205..215: Ident("invoice_id"), - 216..217: Control('*'), - 218..223: Ident("total"), - 223..224: Control(')'), - 225..226: Control('}'), - 226..227: NewLine, + 14..20: Ident("derive"), + 21..26: Ident("total"), + 27..28: Control('='), + 29..33: Keyword("case"), + 34..35: Control('['), + 35..40: Ident("total"), + 41..42: Control('<'), + 43..45: Literal(Integer(10)), + 46..48: ArrowFat, + 49..54: Ident("total"), + 55..56: Control('*'), + 57..58: Literal(Integer(2)), + 58..59: Control(','), + 60..64: Literal(Boolean(true)), + 65..67: ArrowFat, + 68..73: Ident("total"), + 73..74: Control(']'), + 74..75: NewLine, + 75..81: Ident("select"), + 82..83: Control('{'), + 84..95: Ident("customer_id"), + 95..96: Control(','), + 97..107: Ident("invoice_id"), + 107..108: Control(','), + 109..114: Ident("total"), + 115..116: Control('}'), + 116..117: NewLine, + 117..121: Ident("take"), + 122..123: Literal(Integer(5)), + 123..124: NewLine, + 124..130: Ident("append"), + 131..132: Control('('), + 132..133: NewLine, + 135..139: Ident("from"), + 140..153: Ident("invoice_items"), + 153..154: NewLine, + 156..162: Ident("derive"), + 163..173: Ident("unit_price"), + 174..175: Control('='), + 176..180: Keyword("case"), + 181..182: Control('['), + 182..192: Ident("unit_price"), + 193..194: Control('<'), + 195..196: Literal(Integer(1)), + 197..199: ArrowFat, + 200..210: Ident("unit_price"), + 211..212: Control('*'), + 213..214: Literal(Integer(2)), + 214..215: Control(','), + 216..220: Literal(Boolean(true)), + 221..223: ArrowFat, + 224..234: Ident("unit_price"), + 234..235: Control(']'), + 235..236: NewLine, + 238..244: Ident("select"), + 245..246: Control('{'), + 247..262: Ident("invoice_line_id"), + 262..263: Control(','), + 264..274: Ident("invoice_id"), + 274..275: Control(','), + 276..286: Ident("unit_price"), + 287..288: Control('}'), + 288..289: NewLine, + 291..295: Ident("take"), + 296..297: Literal(Integer(5)), + 297..298: NewLine, + 298..299: Control(')'), + 299..300: NewLine, + 300..306: Ident("select"), + 307..308: Control('{'), + 309..310: Ident("a"), + 311..312: Control('='), + 313..324: Ident("customer_id"), + 325..326: Control('*'), + 327..328: Literal(Integer(2)), + 328..329: Control(','), + 330..331: Ident("b"), + 332..333: Control('='), + 334..338: Ident("math"), + 338..339: Control('.'), + 339..344: Ident("round"), + 345..346: Literal(Integer(1)), + 347..348: Control('('), + 348..358: Ident("invoice_id"), + 359..360: Control('*'), + 361..366: Ident("total"), + 366..367: Control(')'), + 368..369: Control('}'), + 369..370: NewLine, ], ) diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_multiple.snap new file mode 100644 index 000000000000..1c1cc15ceec6 --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_multiple.snap @@ -0,0 +1,132 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: tokens +input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql +--- +Tokens( + [ + 0..0: Start, + 0..4: Ident("from"), + 5..13: Ident("invoices"), + 13..14: NewLine, + 14..20: Ident("select"), + 21..22: Control('{'), + 23..34: Ident("customer_id"), + 34..35: Control(','), + 36..46: Ident("invoice_id"), + 46..47: Control(','), + 48..53: Ident("total"), + 53..54: Control(','), + 55..63: Ident("useless1"), + 63..64: Control(','), + 65..73: Ident("useless2"), + 74..75: Control('}'), + 75..76: NewLine, + 76..80: Ident("take"), + 81..82: Literal(Integer(5)), + 82..83: NewLine, + 83..89: Ident("append"), + 90..91: Control('('), + 91..92: NewLine, + 94..98: Ident("from"), + 99..108: Ident("employees"), + 108..109: NewLine, + 111..117: Ident("select"), + 118..119: Control('{'), + 120..131: Ident("employee_id"), + 131..132: Control(','), + 133..144: Ident("employee_id"), + 145..146: Control('+'), + 147..148: Literal(Integer(1)), + 148..149: Control(','), + 150..160: Ident("reports_to"), + 160..161: Control(','), + 162..170: Ident("useless3"), + 170..171: Control(','), + 172..180: Ident("useless4"), + 181..182: Control('}'), + 182..183: NewLine, + 185..189: Ident("take"), + 190..191: Literal(Integer(5)), + 191..192: NewLine, + 192..193: Control(')'), + 193..194: NewLine, + 194..199: Ident("group"), + 200..201: Control('{'), + 202..213: Ident("customer_id"), + 214..215: Control('}'), + 216..217: Control('('), + 217..226: Ident("aggregate"), + 227..228: Control('{'), + 229..239: Ident("invoice_id"), + 240..241: Control('='), + 242..246: Ident("math"), + 246..247: Control('.'), + 247..252: Ident("round"), + 253..254: Literal(Integer(1)), + 255..256: Control('('), + 256..259: Ident("sum"), + 260..270: Ident("invoice_id"), + 270..271: Control(')'), + 271..272: Control(','), + 273..278: Ident("total"), + 279..280: Control('='), + 281..285: Ident("math"), + 285..286: Control('.'), + 286..291: Ident("round"), + 292..293: Literal(Integer(1)), + 294..295: Control('('), + 295..298: Ident("sum"), + 299..304: Ident("total"), + 304..305: Control(')'), + 305..306: Control(','), + 307..315: Ident("useless1"), + 316..317: Control('='), + 318..321: Ident("sum"), + 322..330: Ident("useless1"), + 331..332: Control('}'), + 332..333: Control(')'), + 333..334: NewLine, + 334..340: Ident("append"), + 341..342: Control('('), + 342..343: NewLine, + 345..349: Ident("from"), + 350..363: Ident("invoice_items"), + 363..364: NewLine, + 366..372: Ident("select"), + 373..374: Control('{'), + 375..385: Ident("invoice_id"), + 385..386: Control(','), + 387..402: Ident("invoice_line_id"), + 402..403: Control(','), + 404..405: Literal(Integer(0)), + 405..406: Control(','), + 407..415: Ident("useless5"), + 416..417: Control('}'), + 417..418: NewLine, + 420..424: Ident("take"), + 425..426: Literal(Integer(5)), + 426..427: NewLine, + 427..428: Control(')'), + 428..429: NewLine, + 429..433: Ident("sort"), + 434..435: Control('{'), + 436..437: Control('+'), + 437..447: Ident("invoice_id"), + 447..448: Control(','), + 449..450: Control('+'), + 450..455: Ident("total"), + 456..457: Control('}'), + 457..458: NewLine, + 458..464: Ident("select"), + 465..466: Control('{'), + 467..472: Ident("total"), + 472..473: Control(','), + 474..484: Ident("invoice_id"), + 485..486: Control('}'), + 486..487: NewLine, + 487..488: NewLine, + 488..501: Comment(" sqlite:skip"), + 501..502: NewLine, + ], +) diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_compute.snap index 07016729aac5..9e8c92958086 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_compute.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_compute.snap @@ -1,15 +1,15 @@ --- source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nselect { customer_id, invoice_id, total }\ntake 5\nappend (\n from invoice_items\n select { invoice_line_id, invoice_id, unit_price }\n take 5\n)\nselect { a = customer_id * 2, b = math.round 1 (invoice_id * total) }\n" +expression: "from invoices\nderive total = case [total < 10 => total * 2, true => total]\nselect { customer_id, invoice_id, total }\ntake 5\nappend (\n from invoice_items\n derive unit_price = case [unit_price < 1 => unit_price * 2, true => unit_price]\n select { invoice_line_id, invoice_id, unit_price }\n take 5\n)\nselect { a = customer_id * 2, b = math.round 1 (invoice_id * total) }\n" input_file: prqlc/prqlc/tests/integration/queries/append_select_compute.prql --- -4,2 -8,7.9 -16,17.8 -28,35.6 +4,4 +8,15.8 +16,35.6 +28,71.3 46,69.3 -2,1 -4,1 -6,2 -8,2 -10,2 +2,2 +4,2 +6,4 +8,4 +10,4 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_multiple.snap new file mode 100644 index 000000000000..ae5f0981065f --- /dev/null +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_multiple.snap @@ -0,0 +1,18 @@ +--- +source: prqlc/prqlc/tests/integration/queries.rs +expression: "from invoices\nselect { customer_id, invoice_id, total, useless1, useless2 }\ntake 5\nappend (\n from employees\n select { employee_id, employee_id + 1, reports_to, useless3, useless4 }\n take 5\n)\ngroup { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 })\nappend (\n from invoice_items\n select { invoice_id, invoice_line_id, 0, useless5 }\n take 5\n)\nsort { +invoice_id, +total }\nselect { total, invoice_id }\n\n# sqlite:skip\n" +input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql +--- +1,1 +1,2 +6,2 +2,3 +5.9,3 +2,4 +2,4 +3,4 +8.9,4 +2,5 +13.9,5 +2,6 +6,7 diff --git a/prqlc/prqlc/tests/integration/sql.rs b/prqlc/prqlc/tests/integration/sql.rs index 7ca1890832d1..34aab6caf77f 100644 --- a/prqlc/prqlc/tests/integration/sql.rs +++ b/prqlc/prqlc/tests/integration/sql.rs @@ -5687,8 +5687,13 @@ fn test_missing_columns_group_complex_compute() { "#, ) .unwrap(), @r" - WITH table_0 AS ( - SELECT + SELECT + DISTINCT ON ( + EXTRACT( + year + from + hire_date + ), CONCAT( 'Year ', EXTRACT( @@ -5696,23 +5701,16 @@ fn test_missing_columns_group_complex_compute() { from hire_date ) - ) AS year_label, + ) + ) CONCAT( + 'Year ', EXTRACT( year from hire_date - ) AS _expr_0, - CASE - WHEN city = 'Calgary' THEN 'A city' - ELSE city - END AS _expr_1, - city - FROM - employees - ) - SELECT - DISTINCT ON (_expr_0, year_label) year_label + ) + ) AS year_label FROM - table_0 + employees "); } From 06548a17a13c3fa4326c83e21b907f01cc1c8b3e Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 13 Aug 2025 19:04:35 -0700 Subject: [PATCH 7/7] use inline rather than integration tests --- .../queries/append_select_multiple.prql | 18 - .../queries/distinct_on_sort_on_compute.prql | 8 - ...ries__compile__append_select_multiple.snap | 66 -- ..._compile__distinct_on_sort_on_compute.snap | 44 - ...s__compileall__append_select_multiple.snap | 130 -- ...mpileall__distinct_on_sort_on_compute.snap | 189 --- ...debug_lineage__append_select_multiple.snap | 1053 ----------------- ..._lineage__distinct_on_sort_on_compute.snap | 490 -------- ..._queries__fmt__append_select_multiple.snap | 25 - ...ies__fmt__distinct_on_sort_on_compute.snap | 13 - ..._queries__lex__append_select_multiple.snap | 132 --- ...ies__lex__distinct_on_sort_on_compute.snap | 82 -- ...ries__results__append_select_multiple.snap | 18 - ..._results__distinct_on_sort_on_compute.snap | 6 - prqlc/prqlc/tests/integration/sql.rs | 200 ++++ 15 files changed, 200 insertions(+), 2274 deletions(-) delete mode 100644 prqlc/prqlc/tests/integration/queries/append_select_multiple.prql delete mode 100644 prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_multiple.snap delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__distinct_on_sort_on_compute.snap delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_multiple.snap delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_multiple.snap delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_multiple.snap delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__distinct_on_sort_on_compute.snap delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_multiple.snap delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on_sort_on_compute.snap delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_multiple.snap delete mode 100644 prqlc/prqlc/tests/integration/snapshots/integration__queries__results__distinct_on_sort_on_compute.snap diff --git a/prqlc/prqlc/tests/integration/queries/append_select_multiple.prql b/prqlc/prqlc/tests/integration/queries/append_select_multiple.prql deleted file mode 100644 index ff2e35882442..000000000000 --- a/prqlc/prqlc/tests/integration/queries/append_select_multiple.prql +++ /dev/null @@ -1,18 +0,0 @@ -from invoices -select { customer_id, invoice_id, total, useless1, useless2 } -take 5 -append ( - from employees - select { employee_id, employee_id + 1, reports_to, useless3, useless4 } - take 5 -) -group { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 }) -append ( - from invoice_items - select { invoice_id, invoice_line_id, 0, useless5 } - take 5 -) -sort { +invoice_id, +total } -select { total, invoice_id } - -# sqlite:skip diff --git a/prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql b/prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql deleted file mode 100644 index 5ffdbc8e2da0..000000000000 --- a/prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql +++ /dev/null @@ -1,8 +0,0 @@ -from invoices -derive code = case [customer_id < 10 => billing_postal_code, true => null] -group {customer_id, billing_city, billing_country} ( - sort {-this.code} - take 1 -) -filter (customer_id | in [4]) -group {billing_country} (aggregate {total = math.round 2 (sum total)}) diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_multiple.snap deleted file mode 100644 index c06e7f10f1f3..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__append_select_multiple.snap +++ /dev/null @@ -1,66 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nselect { customer_id, invoice_id, total, useless1, useless2 }\ntake 5\nappend (\n from employees\n select { employee_id, employee_id + 1, reports_to, useless3, useless4 }\n take 5\n)\ngroup { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 })\nappend (\n from invoice_items\n select { invoice_id, invoice_line_id, 0, useless5 }\n take 5\n)\nsort { +invoice_id, +total }\nselect { total, invoice_id }\n\n# sqlite:skip\n" -input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql ---- -WITH table_3 AS ( - SELECT - * - FROM - ( - SELECT - customer_id, - total, - invoice_id - FROM - invoices - LIMIT - 5 - ) AS table_6 - UNION - ALL - SELECT - * - FROM - ( - SELECT - employee_id, - reports_to, - employee_id + 1 - FROM - employees - LIMIT - 5 - ) AS table_7 -), -table_2 AS ( - SELECT - ROUND(COALESCE(SUM(total), 0), 1) AS total, - ROUND(COALESCE(SUM(invoice_id), 0), 1) AS invoice_id - FROM - table_3 - GROUP BY - customer_id - UNION - ALL - SELECT - * - FROM - ( - SELECT - invoice_id, - invoice_line_id - FROM - invoice_items - LIMIT - 5 - ) AS table_8 -) -SELECT - total, - invoice_id -FROM - table_2 -ORDER BY - invoice_id, - total diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__distinct_on_sort_on_compute.snap deleted file mode 100644 index 1f7822326316..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compile__distinct_on_sort_on_compute.snap +++ /dev/null @@ -1,44 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nderive code = case [customer_id < 10 => billing_postal_code, true => null]\ngroup {customer_id, billing_city, billing_country} (\n sort {-this.code}\n take 1\n)\nfilter (customer_id | in [4])\ngroup {billing_country} (aggregate {total = math.round 2 (sum total)})\n" -input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql ---- -WITH table_1 AS ( - SELECT - billing_country, - total, - customer_id, - billing_city, - CASE - WHEN customer_id < 10 THEN billing_postal_code - ELSE NULL - END AS _expr_1, - billing_postal_code - FROM - invoices -), -table_0 AS ( - SELECT - billing_country, - total, - customer_id, - ROW_NUMBER() OVER ( - PARTITION BY customer_id, - billing_city, - billing_country - ORDER BY - _expr_1 DESC - ) AS _expr_0 - FROM - table_1 -) -SELECT - billing_country, - ROUND(COALESCE(SUM(total), 0), 2) AS total -FROM - table_0 -WHERE - _expr_0 <= 1 - AND customer_id IN (4) -GROUP BY - billing_country diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_multiple.snap deleted file mode 100644 index dd4bcb643cbc..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__append_select_multiple.snap +++ /dev/null @@ -1,130 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nselect { customer_id, invoice_id, total, useless1, useless2 }\ntake 5\nappend (\n from employees\n select { employee_id, employee_id + 1, reports_to, useless3, useless4 }\n take 5\n)\ngroup { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 })\nappend (\n from invoice_items\n select { invoice_id, invoice_line_id, 0, useless5 }\n take 5\n)\nsort { +invoice_id, +total }\nselect { total, invoice_id }\n\n# sqlite:skip\n" -input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql ---- ---- generic -+++ glaredb -@@ -23,22 +23,22 @@ - reports_to, - employee_id + 1 - FROM - employees - LIMIT - 5 - ) AS table_7 - ), - table_2 AS ( - SELECT -- ROUND(COALESCE(SUM(total), 0), 1) AS total, -- ROUND(COALESCE(SUM(invoice_id), 0), 1) AS invoice_id -+ ROUND((COALESCE(SUM(total), 0))::numeric, 1) AS total, -+ ROUND((COALESCE(SUM(invoice_id), 0))::numeric, 1) AS invoice_id - FROM - table_3 - GROUP BY - customer_id - UNION - ALL - SELECT - * - FROM - ( - - ---- generic -+++ postgres -@@ -1,61 +1,49 @@ - WITH table_3 AS ( -- SELECT -- * -- FROM -- ( -- SELECT -- customer_id, -- total, -- invoice_id -- FROM -- invoices -- LIMIT -- 5 -- ) AS table_6 -+ ( -+ SELECT -+ customer_id, -+ total, -+ invoice_id -+ FROM -+ invoices -+ LIMIT -+ 5 -+ ) - UNION -- ALL -+ ALL ( -+ SELECT -+ employee_id, -+ reports_to, -+ employee_id + 1 -+ FROM -+ employees -+ LIMIT -+ 5 -+ ) -+), table_2 AS ( - SELECT -- * -+ ROUND((COALESCE(SUM(total), 0))::numeric, 1) AS total, -+ ROUND((COALESCE(SUM(invoice_id), 0))::numeric, 1) AS invoice_id - FROM -- ( -- SELECT -- employee_id, -- reports_to, -- employee_id + 1 -- FROM -- employees -- LIMIT -- 5 -- ) AS table_7 --), --table_2 AS ( -- SELECT -- ROUND(COALESCE(SUM(total), 0), 1) AS total, -- ROUND(COALESCE(SUM(invoice_id), 0), 1) AS invoice_id -- FROM - table_3 - GROUP BY - customer_id - UNION -- ALL -- SELECT -- * -- FROM -- ( -- SELECT -- invoice_id, -- invoice_line_id -- FROM -- invoice_items -- LIMIT -- 5 -- ) AS table_8 -+ ALL ( -+ SELECT -+ invoice_id, -+ invoice_line_id -+ FROM -+ invoice_items -+ LIMIT -+ 5 -+ ) - ) - SELECT - total, - invoice_id - FROM - table_2 - ORDER BY - invoice_id, - total diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap deleted file mode 100644 index 592d2a8e4cae..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__compileall__distinct_on_sort_on_compute.snap +++ /dev/null @@ -1,189 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nderive code = case [customer_id < 10 => billing_postal_code, true => null]\ngroup {customer_id, billing_city, billing_country} (\n sort {-this.code}\n take 1\n)\nfilter (customer_id | in [4])\ngroup {billing_country} (aggregate {total = math.round 2 (sum total)})\n" -input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql ---- ---- generic -+++ clickhouse -@@ -1,39 +1,38 @@ - WITH table_1 AS ( - SELECT - billing_country, - total, - customer_id, - billing_city, - CASE - WHEN customer_id < 10 THEN billing_postal_code - ELSE NULL -- END AS _expr_1, -+ END AS _expr_0, - billing_postal_code - FROM - invoices - ), - table_0 AS ( - SELECT -- billing_country, -+ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, - total, - customer_id, -- ROW_NUMBER() OVER ( -- PARTITION BY customer_id, -- billing_city, -- billing_country -- ORDER BY -- _expr_1 DESC -- ) AS _expr_0 -+ billing_city, -+ _expr_0 - FROM - table_1 -+ ORDER BY -+ customer_id, -+ billing_city, -+ billing_country, -+ _expr_0 DESC - ) - SELECT - billing_country, - ROUND(COALESCE(SUM(total), 0), 2) AS total - FROM - table_0 - WHERE -- _expr_0 <= 1 -- AND customer_id IN (4) -+ customer_id IN (4) - GROUP BY - billing_country - ---- generic -+++ duckdb -@@ -1,39 +1,38 @@ - WITH table_1 AS ( - SELECT - billing_country, - total, - customer_id, - billing_city, - CASE - WHEN customer_id < 10 THEN billing_postal_code - ELSE NULL -- END AS _expr_1, -+ END AS _expr_0, - billing_postal_code - FROM - invoices - ), - table_0 AS ( - SELECT -- billing_country, -+ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, - total, - customer_id, -- ROW_NUMBER() OVER ( -- PARTITION BY customer_id, -- billing_city, -- billing_country -- ORDER BY -- _expr_1 DESC -- ) AS _expr_0 -+ billing_city, -+ _expr_0 - FROM - table_1 -+ ORDER BY -+ customer_id, -+ billing_city, -+ billing_country, -+ _expr_0 DESC - ) - SELECT - billing_country, - ROUND(COALESCE(SUM(total), 0), 2) AS total - FROM - table_0 - WHERE -- _expr_0 <= 1 -- AND customer_id IN (4) -+ customer_id IN (4) - GROUP BY - billing_country - - ---- generic -+++ glaredb -@@ -22,18 +22,18 @@ - billing_city, - billing_country - ORDER BY - _expr_1 DESC - ) AS _expr_0 - FROM - table_1 - ) - SELECT - billing_country, -- ROUND(COALESCE(SUM(total), 0), 2) AS total -+ ROUND((COALESCE(SUM(total), 0))::numeric, 2) AS total - FROM - table_0 - WHERE - _expr_0 <= 1 - AND customer_id IN (4) - GROUP BY - billing_country - - ---- generic -+++ postgres -@@ -1,39 +1,38 @@ - WITH table_1 AS ( - SELECT - billing_country, - total, - customer_id, - billing_city, - CASE - WHEN customer_id < 10 THEN billing_postal_code - ELSE NULL -- END AS _expr_1, -+ END AS _expr_0, - billing_postal_code - FROM - invoices - ), - table_0 AS ( - SELECT -- billing_country, -+ DISTINCT ON (customer_id, billing_city, billing_country) billing_country, - total, - customer_id, -- ROW_NUMBER() OVER ( -- PARTITION BY customer_id, -- billing_city, -- billing_country -- ORDER BY -- _expr_1 DESC -- ) AS _expr_0 -+ billing_city, -+ _expr_0 - FROM - table_1 -+ ORDER BY -+ customer_id, -+ billing_city, -+ billing_country, -+ _expr_0 DESC - ) - SELECT - billing_country, -- ROUND(COALESCE(SUM(total), 0), 2) AS total -+ ROUND((COALESCE(SUM(total), 0))::numeric, 2) AS total - FROM - table_0 - WHERE -- _expr_0 <= 1 -- AND customer_id IN (4) -+ customer_id IN (4) - GROUP BY - billing_country diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_multiple.snap deleted file mode 100644 index 47a733b9737b..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__append_select_multiple.snap +++ /dev/null @@ -1,1053 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nselect { customer_id, invoice_id, total, useless1, useless2 }\ntake 5\nappend (\n from employees\n select { employee_id, employee_id + 1, reports_to, useless3, useless4 }\n take 5\n)\ngroup { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 })\nappend (\n from invoice_items\n select { invoice_id, invoice_line_id, 0, useless5 }\n take 5\n)\nsort { +invoice_id, +total }\nselect { total, invoice_id }\n\n# sqlite:skip\n" -input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql ---- -frames: -- - 1:14-75 - - columns: - - !Single - name: - - invoices - - customer_id - target_id: 177 - target_name: null - - !Single - name: - - invoices - - invoice_id - target_id: 178 - target_name: null - - !Single - name: - - invoices - - total - target_id: 179 - target_name: null - - !Single - name: - - invoices - - useless1 - target_id: 180 - target_name: null - - !Single - name: - - invoices - - useless2 - target_id: 181 - target_name: null - inputs: - - id: 175 - name: invoices - table: - - default_db - - invoices -- - 1:76-82 - - columns: - - !Single - name: - - invoices - - customer_id - target_id: 177 - target_name: null - - !Single - name: - - invoices - - invoice_id - target_id: 178 - target_name: null - - !Single - name: - - invoices - - total - target_id: 179 - target_name: null - - !Single - name: - - invoices - - useless1 - target_id: 180 - target_name: null - - !Single - name: - - invoices - - useless2 - target_id: 181 - target_name: null - inputs: - - id: 175 - name: invoices - table: - - default_db - - invoices -- - 1:111-182 - - columns: - - !Single - name: - - employees - - employee_id - target_id: 154 - target_name: null - - !Single - name: null - target_id: 155 - target_name: null - - !Single - name: - - employees - - reports_to - target_id: 159 - target_name: null - - !Single - name: - - employees - - useless3 - target_id: 160 - target_name: null - - !Single - name: - - employees - - useless4 - target_id: 161 - target_name: null - inputs: - - id: 152 - name: employees - table: - - default_db - - employees -- - 1:185-191 - - columns: - - !Single - name: - - employees - - employee_id - target_id: 154 - target_name: null - - !Single - name: null - target_id: 155 - target_name: null - - !Single - name: - - employees - - reports_to - target_id: 159 - target_name: null - - !Single - name: - - employees - - useless3 - target_id: 160 - target_name: null - - !Single - name: - - employees - - useless4 - target_id: 161 - target_name: null - inputs: - - id: 152 - name: employees - table: - - default_db - - employees -- - 1:83-193 - - columns: - - !Single - name: - - invoices - - customer_id - target_id: 177 - target_name: null - - !Single - name: - - invoices - - invoice_id - target_id: 178 - target_name: null - - !Single - name: - - invoices - - total - target_id: 179 - target_name: null - - !Single - name: - - invoices - - useless1 - target_id: 180 - target_name: null - - !Single - name: - - invoices - - useless2 - target_id: 181 - target_name: null - inputs: - - id: 175 - name: invoices - table: - - default_db - - invoices -- - 1:217-332 - - columns: - - !Single - name: - - invoices - - customer_id - target_id: 188 - target_name: null - - !Single - name: - - invoice_id - target_id: 208 - target_name: null - - !Single - name: - - total - target_id: 214 - target_name: null - - !Single - name: - - useless1 - target_id: 220 - target_name: null - inputs: - - id: 175 - name: invoices - table: - - default_db - - invoices -- - 1:366-417 - - columns: - - !Single - name: - - invoice_items - - invoice_id - target_id: 129 - target_name: null - - !Single - name: - - invoice_items - - invoice_line_id - target_id: 130 - target_name: null - - !Single - name: null - target_id: 131 - target_name: null - - !Single - name: - - invoice_items - - useless5 - target_id: 132 - target_name: null - inputs: - - id: 127 - name: invoice_items - table: - - default_db - - invoice_items -- - 1:420-426 - - columns: - - !Single - name: - - invoice_items - - invoice_id - target_id: 129 - target_name: null - - !Single - name: - - invoice_items - - invoice_line_id - target_id: 130 - target_name: null - - !Single - name: null - target_id: 131 - target_name: null - - !Single - name: - - invoice_items - - useless5 - target_id: 132 - target_name: null - inputs: - - id: 127 - name: invoice_items - table: - - default_db - - invoice_items -- - 1:334-428 - - columns: - - !Single - name: - - invoices - - customer_id - target_id: 188 - target_name: null - - !Single - name: - - invoice_id - target_id: 208 - target_name: null - - !Single - name: - - total - target_id: 214 - target_name: null - - !Single - name: - - useless1 - target_id: 220 - target_name: null - inputs: - - id: 175 - name: invoices - table: - - default_db - - invoices -- - 1:429-457 - - columns: - - !Single - name: - - invoices - - customer_id - target_id: 188 - target_name: null - - !Single - name: - - invoice_id - target_id: 208 - target_name: null - - !Single - name: - - total - target_id: 214 - target_name: null - - !Single - name: - - useless1 - target_id: 220 - target_name: null - inputs: - - id: 175 - name: invoices - table: - - default_db - - invoices -- - 1:458-486 - - columns: - - !Single - name: - - total - target_id: 232 - target_name: null - - !Single - name: - - invoice_id - target_id: 233 - target_name: null - inputs: - - id: 175 - name: invoices - table: - - default_db - - invoices -nodes: -- id: 127 - kind: Ident - span: 1:345-363 - ident: !Ident - - default_db - - invoice_items - parent: 134 -- id: 129 - kind: Ident - span: 1:375-385 - ident: !Ident - - this - - invoice_items - - invoice_id - targets: - - 127 - parent: 133 -- id: 130 - kind: Ident - span: 1:387-402 - ident: !Ident - - this - - invoice_items - - invoice_line_id - targets: - - 127 - parent: 133 -- id: 131 - kind: Literal - span: 1:404-405 - parent: 133 -- id: 132 - kind: Ident - span: 1:407-415 - ident: !Ident - - this - - invoice_items - - useless5 - targets: - - 127 - parent: 133 -- id: 133 - kind: Tuple - span: 1:373-417 - children: - - 129 - - 130 - - 131 - - 132 - parent: 134 -- id: 134 - kind: 'TransformCall: Select' - span: 1:366-417 - children: - - 127 - - 133 - parent: 136 -- id: 136 - kind: 'TransformCall: Take' - span: 1:420-426 - children: - - 134 - - 137 - parent: 227 -- id: 137 - kind: Literal - parent: 136 -- id: 152 - kind: Ident - span: 1:94-108 - ident: !Ident - - default_db - - employees - parent: 163 -- id: 154 - kind: Ident - span: 1:120-131 - ident: !Ident - - this - - employees - - employee_id - targets: - - 152 - parent: 162 -- id: 155 - kind: RqOperator - span: 1:133-148 - targets: - - 157 - - 158 - parent: 162 -- id: 157 - kind: Ident - span: 1:133-144 - ident: !Ident - - this - - employees - - employee_id - targets: - - 152 -- id: 158 - kind: Literal - span: 1:147-148 -- id: 159 - kind: Ident - span: 1:150-160 - ident: !Ident - - this - - employees - - reports_to - targets: - - 152 - parent: 162 -- id: 160 - kind: Ident - span: 1:162-170 - ident: !Ident - - this - - employees - - useless3 - targets: - - 152 - parent: 162 -- id: 161 - kind: Ident - span: 1:172-180 - ident: !Ident - - this - - employees - - useless4 - targets: - - 152 - parent: 162 -- id: 162 - kind: Tuple - span: 1:118-182 - children: - - 154 - - 155 - - 159 - - 160 - - 161 - parent: 163 -- id: 163 - kind: 'TransformCall: Select' - span: 1:111-182 - children: - - 152 - - 162 - parent: 165 -- id: 165 - kind: 'TransformCall: Take' - span: 1:185-191 - children: - - 163 - - 166 - parent: 187 -- id: 166 - kind: Literal - parent: 165 -- id: 175 - kind: Ident - span: 1:0-13 - ident: !Ident - - default_db - - invoices - parent: 183 -- id: 177 - kind: Ident - span: 1:23-34 - ident: !Ident - - this - - invoices - - customer_id - targets: - - 175 - parent: 182 -- id: 178 - kind: Ident - span: 1:36-46 - ident: !Ident - - this - - invoices - - invoice_id - targets: - - 175 - parent: 182 -- id: 179 - kind: Ident - span: 1:48-53 - ident: !Ident - - this - - invoices - - total - targets: - - 175 - parent: 182 -- id: 180 - kind: Ident - span: 1:55-63 - ident: !Ident - - this - - invoices - - useless1 - targets: - - 175 - parent: 182 -- id: 181 - kind: Ident - span: 1:65-73 - ident: !Ident - - this - - invoices - - useless2 - targets: - - 175 - parent: 182 -- id: 182 - kind: Tuple - span: 1:21-75 - children: - - 177 - - 178 - - 179 - - 180 - - 181 - parent: 183 -- id: 183 - kind: 'TransformCall: Select' - span: 1:14-75 - children: - - 175 - - 182 - parent: 185 -- id: 185 - kind: 'TransformCall: Take' - span: 1:76-82 - children: - - 183 - - 186 - parent: 187 -- id: 186 - kind: Literal - parent: 185 -- id: 187 - kind: 'TransformCall: Append' - span: 1:83-193 - children: - - 185 - - 165 - parent: 224 -- id: 188 - kind: Ident - span: 1:202-213 - ident: !Ident - - this - - invoices - - customer_id - targets: - - 177 - parent: 189 -- id: 189 - kind: Tuple - span: 1:200-215 - children: - - 188 - parent: 224 -- id: 208 - kind: RqOperator - span: 1:242-271 - alias: invoice_id - targets: - - 210 - - 211 - parent: 223 -- id: 210 - kind: Literal - span: 1:253-254 -- id: 211 - kind: RqOperator - span: 1:256-270 - targets: - - 213 -- id: 213 - kind: Ident - span: 1:260-270 - ident: !Ident - - this - - invoices - - invoice_id - targets: - - 178 -- id: 214 - kind: RqOperator - span: 1:281-305 - alias: total - targets: - - 216 - - 217 - parent: 223 -- id: 216 - kind: Literal - span: 1:292-293 -- id: 217 - kind: RqOperator - span: 1:295-304 - targets: - - 219 -- id: 219 - kind: Ident - span: 1:299-304 - ident: !Ident - - this - - invoices - - total - targets: - - 179 -- id: 220 - kind: RqOperator - span: 1:318-330 - alias: useless1 - targets: - - 222 - parent: 223 -- id: 222 - kind: Ident - span: 1:322-330 - ident: !Ident - - this - - invoices - - useless1 - targets: - - 180 -- id: 223 - kind: Tuple - span: 1:227-332 - children: - - 208 - - 214 - - 220 - parent: 224 -- id: 224 - kind: 'TransformCall: Aggregate' - span: 1:217-332 - children: - - 187 - - 223 - - 189 - parent: 227 -- id: 227 - kind: 'TransformCall: Append' - span: 1:334-428 - children: - - 224 - - 136 - parent: 231 -- id: 228 - kind: Ident - span: 1:436-447 - ident: !Ident - - this - - invoice_id - targets: - - 208 - parent: 231 -- id: 229 - kind: Ident - span: 1:449-455 - ident: !Ident - - this - - total - targets: - - 214 - parent: 231 -- id: 231 - kind: 'TransformCall: Sort' - span: 1:429-457 - children: - - 227 - - 228 - - 229 - parent: 235 -- id: 232 - kind: Ident - span: 1:467-472 - ident: !Ident - - this - - total - targets: - - 214 - parent: 234 -- id: 233 - kind: Ident - span: 1:474-484 - ident: !Ident - - this - - invoice_id - targets: - - 208 - parent: 234 -- id: 234 - kind: Tuple - span: 1:465-486 - children: - - 232 - - 233 - parent: 235 -- id: 235 - kind: 'TransformCall: Select' - span: 1:458-486 - children: - - 231 - - 234 -ast: - name: Project - stmts: - - VarDef: - kind: Main - name: main - value: - Pipeline: - exprs: - - FuncCall: - name: - Ident: - - from - span: 1:0-4 - args: - - Ident: - - invoices - span: 1:5-13 - span: 1:0-13 - - FuncCall: - name: - Ident: - - select - span: 1:14-20 - args: - - Tuple: - - Ident: - - customer_id - span: 1:23-34 - - Ident: - - invoice_id - span: 1:36-46 - - Ident: - - total - span: 1:48-53 - - Ident: - - useless1 - span: 1:55-63 - - Ident: - - useless2 - span: 1:65-73 - span: 1:21-75 - span: 1:14-75 - - FuncCall: - name: - Ident: - - take - span: 1:76-80 - args: - - Literal: - Integer: 5 - span: 1:81-82 - span: 1:76-82 - - FuncCall: - name: - Ident: - - append - span: 1:83-89 - args: - - Pipeline: - exprs: - - FuncCall: - name: - Ident: - - from - span: 1:94-98 - args: - - Ident: - - employees - span: 1:99-108 - span: 1:94-108 - - FuncCall: - name: - Ident: - - select - span: 1:111-117 - args: - - Tuple: - - Ident: - - employee_id - span: 1:120-131 - - Binary: - left: - Ident: - - employee_id - span: 1:133-144 - op: Add - right: - Literal: - Integer: 1 - span: 1:147-148 - span: 1:133-148 - - Ident: - - reports_to - span: 1:150-160 - - Ident: - - useless3 - span: 1:162-170 - - Ident: - - useless4 - span: 1:172-180 - span: 1:118-182 - span: 1:111-182 - - FuncCall: - name: - Ident: - - take - span: 1:185-189 - args: - - Literal: - Integer: 5 - span: 1:190-191 - span: 1:185-191 - span: 1:94-191 - span: 1:83-193 - - FuncCall: - name: - Ident: - - group - span: 1:194-199 - args: - - Tuple: - - Ident: - - customer_id - span: 1:202-213 - span: 1:200-215 - - FuncCall: - name: - Ident: - - aggregate - span: 1:217-226 - args: - - Tuple: - - FuncCall: - name: - Ident: - - math - - round - span: 1:242-252 - args: - - Literal: - Integer: 1 - span: 1:253-254 - - FuncCall: - name: - Ident: - - sum - span: 1:256-259 - args: - - Ident: - - invoice_id - span: 1:260-270 - span: 1:256-270 - span: 1:242-271 - alias: invoice_id - - FuncCall: - name: - Ident: - - math - - round - span: 1:281-291 - args: - - Literal: - Integer: 1 - span: 1:292-293 - - FuncCall: - name: - Ident: - - sum - span: 1:295-298 - args: - - Ident: - - total - span: 1:299-304 - span: 1:295-304 - span: 1:281-305 - alias: total - - FuncCall: - name: - Ident: - - sum - span: 1:318-321 - args: - - Ident: - - useless1 - span: 1:322-330 - span: 1:318-330 - alias: useless1 - span: 1:227-332 - span: 1:217-332 - span: 1:194-333 - - FuncCall: - name: - Ident: - - append - span: 1:334-340 - args: - - Pipeline: - exprs: - - FuncCall: - name: - Ident: - - from - span: 1:345-349 - args: - - Ident: - - invoice_items - span: 1:350-363 - span: 1:345-363 - - FuncCall: - name: - Ident: - - select - span: 1:366-372 - args: - - Tuple: - - Ident: - - invoice_id - span: 1:375-385 - - Ident: - - invoice_line_id - span: 1:387-402 - - Literal: - Integer: 0 - span: 1:404-405 - - Ident: - - useless5 - span: 1:407-415 - span: 1:373-417 - span: 1:366-417 - - FuncCall: - name: - Ident: - - take - span: 1:420-424 - args: - - Literal: - Integer: 5 - span: 1:425-426 - span: 1:420-426 - span: 1:345-426 - span: 1:334-428 - - FuncCall: - name: - Ident: - - sort - span: 1:429-433 - args: - - Tuple: - - Unary: - op: Add - expr: - Ident: - - invoice_id - span: 1:437-447 - span: 1:436-447 - - Unary: - op: Add - expr: - Ident: - - total - span: 1:450-455 - span: 1:449-455 - span: 1:434-457 - span: 1:429-457 - - FuncCall: - name: - Ident: - - select - span: 1:458-464 - args: - - Tuple: - - Ident: - - total - span: 1:467-472 - - Ident: - - invoice_id - span: 1:474-484 - span: 1:465-486 - span: 1:458-486 - span: 1:0-486 - span: 1:0-486 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap deleted file mode 100644 index e86e7c0f66c3..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on_sort_on_compute.snap +++ /dev/null @@ -1,490 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nderive code = case [customer_id < 10 => billing_postal_code, true => null]\ngroup {customer_id, billing_city, billing_country} (\n sort {-this.code}\n take 1\n)\nfilter (customer_id | in [4])\ngroup {billing_country} (aggregate {total = math.round 2 (sum total)})\n" -input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql ---- -frames: -- - 1:14-88 - - columns: - - !All - input_id: 124 - except: [] - - !Single - name: - - code - target_id: 126 - target_name: null - inputs: - - id: 124 - name: invoices - table: - - default_db - - invoices -- - 1:164-170 - - columns: - - !Single - name: - - invoices - - customer_id - target_id: 136 - target_name: null - - !Single - name: - - invoices - - billing_city - target_id: 137 - target_name: null - - !Single - name: - - invoices - - billing_country - target_id: 138 - target_name: null - - !All - input_id: 124 - except: - - billing_city - - billing_country - - customer_id - - !Single - name: - - code - target_id: 126 - target_name: null - inputs: - - id: 124 - name: invoices - table: - - default_db - - invoices -- - 1:173-202 - - columns: - - !Single - name: - - invoices - - customer_id - target_id: 136 - target_name: null - - !Single - name: - - invoices - - billing_city - target_id: 137 - target_name: null - - !Single - name: - - invoices - - billing_country - target_id: 138 - target_name: null - - !All - input_id: 124 - except: - - billing_city - - billing_country - - customer_id - - !Single - name: - - code - target_id: 126 - target_name: null - inputs: - - id: 124 - name: invoices - table: - - default_db - - invoices -- - 1:228-272 - - columns: - - !Single - name: - - invoices - - billing_country - target_id: 179 - target_name: null - - !Single - name: - - total - target_id: 196 - target_name: null - inputs: - - id: 124 - name: invoices - table: - - default_db - - invoices -nodes: -- id: 124 - kind: Ident - span: 1:0-13 - ident: !Ident - - default_db - - invoices - parent: 135 -- id: 126 - kind: Case - span: 1:28-88 - alias: code - targets: - - 127 - - 131 - - 132 - - 133 - parent: 134 -- id: 127 - kind: RqOperator - span: 1:34-50 - targets: - - 129 - - 130 -- id: 129 - kind: Ident - span: 1:34-45 - ident: !Ident - - this - - invoices - - customer_id - targets: - - 124 -- id: 130 - kind: Literal - span: 1:48-50 -- id: 131 - kind: Ident - span: 1:54-73 - ident: !Ident - - this - - invoices - - billing_postal_code - targets: - - 124 -- id: 132 - kind: Literal - span: 1:75-79 -- id: 133 - kind: Literal - span: 1:83-87 -- id: 134 - kind: Tuple - span: 1:28-88 - children: - - 126 - parent: 135 -- id: 135 - kind: 'TransformCall: Derive' - span: 1:14-88 - children: - - 124 - - 134 - parent: 167 -- id: 136 - kind: Ident - span: 1:96-107 - ident: !Ident - - this - - invoices - - customer_id - targets: - - 124 - parent: 139 -- id: 137 - kind: Ident - span: 1:109-121 - ident: !Ident - - this - - invoices - - billing_city - targets: - - 124 - parent: 139 -- id: 138 - kind: Ident - span: 1:123-138 - ident: !Ident - - this - - invoices - - billing_country - targets: - - 124 - parent: 139 -- id: 139 - kind: Tuple - span: 1:95-139 - children: - - 136 - - 137 - - 138 -- id: 163 - kind: Ident - span: 1:151-160 - ident: !Ident - - this - - code - targets: - - 126 -- id: 167 - kind: 'TransformCall: Take' - span: 1:164-170 - children: - - 135 - - 168 - parent: 178 -- id: 168 - kind: Literal - parent: 167 -- id: 174 - kind: Array - span: 1:198-201 - children: - - 175 -- id: 175 - kind: Literal - span: 1:199-200 - parent: 174 -- id: 176 - kind: Ident - span: 1:181-192 - ident: !Ident - - this - - invoices - - customer_id - targets: - - 136 -- id: 177 - kind: RqOperator - span: 1:195-201 - targets: - - 176 - - 174 - parent: 178 -- id: 178 - kind: 'TransformCall: Filter' - span: 1:173-202 - children: - - 167 - - 177 - parent: 203 -- id: 179 - kind: Ident - span: 1:210-225 - ident: !Ident - - this - - invoices - - billing_country - targets: - - 138 - parent: 180 -- id: 180 - kind: Tuple - span: 1:209-226 - children: - - 179 - parent: 203 -- id: 196 - kind: RqOperator - span: 1:247-271 - alias: total - targets: - - 198 - - 199 - parent: 202 -- id: 198 - kind: Literal - span: 1:258-259 -- id: 199 - kind: RqOperator - span: 1:261-270 - targets: - - 201 -- id: 201 - kind: Ident - span: 1:265-270 - ident: !Ident - - this - - invoices - - total - targets: - - 124 -- id: 202 - kind: Tuple - span: 1:238-272 - children: - - 196 - parent: 203 -- id: 203 - kind: 'TransformCall: Aggregate' - span: 1:228-272 - children: - - 178 - - 202 - - 180 -ast: - name: Project - stmts: - - VarDef: - kind: Main - name: main - value: - Pipeline: - exprs: - - FuncCall: - name: - Ident: - - from - span: 1:0-4 - args: - - Ident: - - invoices - span: 1:5-13 - span: 1:0-13 - - FuncCall: - name: - Ident: - - derive - span: 1:14-20 - args: - - Case: - - condition: - Binary: - left: - Ident: - - customer_id - span: 1:34-45 - op: Lt - right: - Literal: - Integer: 10 - span: 1:48-50 - span: 1:34-50 - value: - Ident: - - billing_postal_code - span: 1:54-73 - - condition: - Literal: - Boolean: true - span: 1:75-79 - value: - Literal: 'Null' - span: 1:83-87 - span: 1:28-88 - alias: code - span: 1:14-88 - - FuncCall: - name: - Ident: - - group - span: 1:89-94 - args: - - Tuple: - - Ident: - - customer_id - span: 1:96-107 - - Ident: - - billing_city - span: 1:109-121 - - Ident: - - billing_country - span: 1:123-138 - span: 1:95-139 - - Pipeline: - exprs: - - FuncCall: - name: - Ident: - - sort - span: 1:144-148 - args: - - Tuple: - - Unary: - op: Neg - expr: - Ident: - - this - - code - span: 1:151-160 - span: 1:150-160 - span: 1:149-161 - span: 1:144-161 - - FuncCall: - name: - Ident: - - take - span: 1:164-168 - args: - - Literal: - Integer: 1 - span: 1:169-170 - span: 1:164-170 - span: 1:144-170 - span: 1:89-172 - - FuncCall: - name: - Ident: - - filter - span: 1:173-179 - args: - - Pipeline: - exprs: - - Ident: - - customer_id - span: 1:181-192 - - FuncCall: - name: - Ident: - - in - span: 1:195-197 - args: - - Array: - - Literal: - Integer: 4 - span: 1:199-200 - span: 1:198-201 - span: 1:195-201 - span: 1:181-201 - span: 1:173-202 - - FuncCall: - name: - Ident: - - group - span: 1:203-208 - args: - - Tuple: - - Ident: - - billing_country - span: 1:210-225 - span: 1:209-226 - - FuncCall: - name: - Ident: - - aggregate - span: 1:228-237 - args: - - Tuple: - - FuncCall: - name: - Ident: - - math - - round - span: 1:247-257 - args: - - Literal: - Integer: 2 - span: 1:258-259 - - FuncCall: - name: - Ident: - - sum - span: 1:261-264 - args: - - Ident: - - total - span: 1:265-270 - span: 1:261-270 - span: 1:247-271 - alias: total - span: 1:238-272 - span: 1:228-272 - span: 1:203-273 - span: 1:0-273 - span: 1:0-273 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_multiple.snap deleted file mode 100644 index b50a6e4cfe08..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__append_select_multiple.snap +++ /dev/null @@ -1,25 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nselect { customer_id, invoice_id, total, useless1, useless2 }\ntake 5\nappend (\n from employees\n select { employee_id, employee_id + 1, reports_to, useless3, useless4 }\n take 5\n)\ngroup { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 })\nappend (\n from invoice_items\n select { invoice_id, invoice_line_id, 0, useless5 }\n take 5\n)\nsort { +invoice_id, +total }\nselect { total, invoice_id }\n\n# sqlite:skip\n" -input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql ---- -from invoices -select {customer_id, invoice_id, total, useless1, useless2} -take 5 -append ( - from employees - select {employee_id, employee_id + 1, reports_to, useless3, useless4} - take 5 -) -group {customer_id} (aggregate { - invoice_id = math.round 1 (sum invoice_id), - total = math.round 1 (sum total), - useless1 = sum useless1, -}) -append ( - from invoice_items - select {invoice_id, invoice_line_id, 0, useless5} - take 5 -) -sort {+invoice_id, +total} -select {total, invoice_id} diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__distinct_on_sort_on_compute.snap deleted file mode 100644 index c1b47dc3ce29..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__fmt__distinct_on_sort_on_compute.snap +++ /dev/null @@ -1,13 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nderive code = case [customer_id < 10 => billing_postal_code, true => null]\ngroup {customer_id, billing_city, billing_country} (\n sort {-this.code}\n take 1\n)\nfilter (customer_id | in [4])\ngroup {billing_country} (aggregate {total = math.round 2 (sum total)})\n" -input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql ---- -from invoices -derive code = case [customer_id < 10 => billing_postal_code, true => null] -group {customer_id, billing_city, billing_country} ( - sort {-this.code} - take 1 -) -filter (customer_id | in [4]) -group {billing_country} (aggregate {total = math.round 2 (sum total)}) diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_multiple.snap deleted file mode 100644 index 1c1cc15ceec6..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__append_select_multiple.snap +++ /dev/null @@ -1,132 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: tokens -input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql ---- -Tokens( - [ - 0..0: Start, - 0..4: Ident("from"), - 5..13: Ident("invoices"), - 13..14: NewLine, - 14..20: Ident("select"), - 21..22: Control('{'), - 23..34: Ident("customer_id"), - 34..35: Control(','), - 36..46: Ident("invoice_id"), - 46..47: Control(','), - 48..53: Ident("total"), - 53..54: Control(','), - 55..63: Ident("useless1"), - 63..64: Control(','), - 65..73: Ident("useless2"), - 74..75: Control('}'), - 75..76: NewLine, - 76..80: Ident("take"), - 81..82: Literal(Integer(5)), - 82..83: NewLine, - 83..89: Ident("append"), - 90..91: Control('('), - 91..92: NewLine, - 94..98: Ident("from"), - 99..108: Ident("employees"), - 108..109: NewLine, - 111..117: Ident("select"), - 118..119: Control('{'), - 120..131: Ident("employee_id"), - 131..132: Control(','), - 133..144: Ident("employee_id"), - 145..146: Control('+'), - 147..148: Literal(Integer(1)), - 148..149: Control(','), - 150..160: Ident("reports_to"), - 160..161: Control(','), - 162..170: Ident("useless3"), - 170..171: Control(','), - 172..180: Ident("useless4"), - 181..182: Control('}'), - 182..183: NewLine, - 185..189: Ident("take"), - 190..191: Literal(Integer(5)), - 191..192: NewLine, - 192..193: Control(')'), - 193..194: NewLine, - 194..199: Ident("group"), - 200..201: Control('{'), - 202..213: Ident("customer_id"), - 214..215: Control('}'), - 216..217: Control('('), - 217..226: Ident("aggregate"), - 227..228: Control('{'), - 229..239: Ident("invoice_id"), - 240..241: Control('='), - 242..246: Ident("math"), - 246..247: Control('.'), - 247..252: Ident("round"), - 253..254: Literal(Integer(1)), - 255..256: Control('('), - 256..259: Ident("sum"), - 260..270: Ident("invoice_id"), - 270..271: Control(')'), - 271..272: Control(','), - 273..278: Ident("total"), - 279..280: Control('='), - 281..285: Ident("math"), - 285..286: Control('.'), - 286..291: Ident("round"), - 292..293: Literal(Integer(1)), - 294..295: Control('('), - 295..298: Ident("sum"), - 299..304: Ident("total"), - 304..305: Control(')'), - 305..306: Control(','), - 307..315: Ident("useless1"), - 316..317: Control('='), - 318..321: Ident("sum"), - 322..330: Ident("useless1"), - 331..332: Control('}'), - 332..333: Control(')'), - 333..334: NewLine, - 334..340: Ident("append"), - 341..342: Control('('), - 342..343: NewLine, - 345..349: Ident("from"), - 350..363: Ident("invoice_items"), - 363..364: NewLine, - 366..372: Ident("select"), - 373..374: Control('{'), - 375..385: Ident("invoice_id"), - 385..386: Control(','), - 387..402: Ident("invoice_line_id"), - 402..403: Control(','), - 404..405: Literal(Integer(0)), - 405..406: Control(','), - 407..415: Ident("useless5"), - 416..417: Control('}'), - 417..418: NewLine, - 420..424: Ident("take"), - 425..426: Literal(Integer(5)), - 426..427: NewLine, - 427..428: Control(')'), - 428..429: NewLine, - 429..433: Ident("sort"), - 434..435: Control('{'), - 436..437: Control('+'), - 437..447: Ident("invoice_id"), - 447..448: Control(','), - 449..450: Control('+'), - 450..455: Ident("total"), - 456..457: Control('}'), - 457..458: NewLine, - 458..464: Ident("select"), - 465..466: Control('{'), - 467..472: Ident("total"), - 472..473: Control(','), - 474..484: Ident("invoice_id"), - 485..486: Control('}'), - 486..487: NewLine, - 487..488: NewLine, - 488..501: Comment(" sqlite:skip"), - 501..502: NewLine, - ], -) diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on_sort_on_compute.snap deleted file mode 100644 index 2a3679afc0a7..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on_sort_on_compute.snap +++ /dev/null @@ -1,82 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: tokens -input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql ---- -Tokens( - [ - 0..0: Start, - 0..4: Ident("from"), - 5..13: Ident("invoices"), - 13..14: NewLine, - 14..20: Ident("derive"), - 21..25: Ident("code"), - 26..27: Control('='), - 28..32: Keyword("case"), - 33..34: Control('['), - 34..45: Ident("customer_id"), - 46..47: Control('<'), - 48..50: Literal(Integer(10)), - 51..53: ArrowFat, - 54..73: Ident("billing_postal_code"), - 73..74: Control(','), - 75..79: Literal(Boolean(true)), - 80..82: ArrowFat, - 83..87: Literal(Null), - 87..88: Control(']'), - 88..89: NewLine, - 89..94: Ident("group"), - 95..96: Control('{'), - 96..107: Ident("customer_id"), - 107..108: Control(','), - 109..121: Ident("billing_city"), - 121..122: Control(','), - 123..138: Ident("billing_country"), - 138..139: Control('}'), - 140..141: Control('('), - 141..142: NewLine, - 144..148: Ident("sort"), - 149..150: Control('{'), - 150..151: Control('-'), - 151..155: Ident("this"), - 155..156: Control('.'), - 156..160: Ident("code"), - 160..161: Control('}'), - 161..162: NewLine, - 164..168: Ident("take"), - 169..170: Literal(Integer(1)), - 170..171: NewLine, - 171..172: Control(')'), - 172..173: NewLine, - 173..179: Ident("filter"), - 180..181: Control('('), - 181..192: Ident("customer_id"), - 193..194: Control('|'), - 195..197: Ident("in"), - 198..199: Control('['), - 199..200: Literal(Integer(4)), - 200..201: Control(']'), - 201..202: Control(')'), - 202..203: NewLine, - 203..208: Ident("group"), - 209..210: Control('{'), - 210..225: Ident("billing_country"), - 225..226: Control('}'), - 227..228: Control('('), - 228..237: Ident("aggregate"), - 238..239: Control('{'), - 239..244: Ident("total"), - 245..246: Control('='), - 247..251: Ident("math"), - 251..252: Control('.'), - 252..257: Ident("round"), - 258..259: Literal(Integer(2)), - 260..261: Control('('), - 261..264: Ident("sum"), - 265..270: Ident("total"), - 270..271: Control(')'), - 271..272: Control('}'), - 272..273: Control(')'), - 273..274: NewLine, - ], -) diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_multiple.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_multiple.snap deleted file mode 100644 index ae5f0981065f..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__append_select_multiple.snap +++ /dev/null @@ -1,18 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nselect { customer_id, invoice_id, total, useless1, useless2 }\ntake 5\nappend (\n from employees\n select { employee_id, employee_id + 1, reports_to, useless3, useless4 }\n take 5\n)\ngroup { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 })\nappend (\n from invoice_items\n select { invoice_id, invoice_line_id, 0, useless5 }\n take 5\n)\nsort { +invoice_id, +total }\nselect { total, invoice_id }\n\n# sqlite:skip\n" -input_file: prqlc/prqlc/tests/integration/queries/append_select_multiple.prql ---- -1,1 -1,2 -6,2 -2,3 -5.9,3 -2,4 -2,4 -3,4 -8.9,4 -2,5 -13.9,5 -2,6 -6,7 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__distinct_on_sort_on_compute.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__distinct_on_sort_on_compute.snap deleted file mode 100644 index 3f98eec01f67..000000000000 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__results__distinct_on_sort_on_compute.snap +++ /dev/null @@ -1,6 +0,0 @@ ---- -source: prqlc/prqlc/tests/integration/queries.rs -expression: "from invoices\nderive code = case [customer_id < 10 => billing_postal_code, true => null ]\ngroup {customer_id, billing_city, billing_country} (\n sort {-this.code}\n take 1\n)\nfilter (customer_id | in [4])\ngroup {billing_country} (aggregate {total = sum total})\n" -input_file: prqlc/prqlc/tests/integration/queries/distinct_on_sort_on_compute.prql ---- -Norway,3.96 diff --git a/prqlc/prqlc/tests/integration/sql.rs b/prqlc/prqlc/tests/integration/sql.rs index 34aab6caf77f..dd3767856fb0 100644 --- a/prqlc/prqlc/tests/integration/sql.rs +++ b/prqlc/prqlc/tests/integration/sql.rs @@ -5714,3 +5714,203 @@ fn test_missing_columns_group_complex_compute() { employees "); } + +#[test] +fn test_append_select_compute() { + // Test for handling complex append with select and compute operations + assert_snapshot!(compile(r###" + from invoices + derive total = case [total < 10 => total * 2, true => total] + select { customer_id, invoice_id, total } + take 5 + append ( + from invoice_items + derive unit_price = case [unit_price < 1 => unit_price * 2, true => unit_price] + select { invoice_line_id, invoice_id, unit_price } + take 5 + ) + select { a = customer_id * 2, b = math.round 1 (invoice_id * total) } + "###).unwrap(), @r" + WITH table_1 AS ( + SELECT + * + FROM + ( + SELECT + invoice_id, + CASE + WHEN total < 10 THEN total * 2 + ELSE total + END AS _expr_0, + customer_id + FROM + invoices + LIMIT + 5 + ) AS table_3 + UNION + ALL + SELECT + * + FROM + ( + SELECT + invoice_id, + CASE + WHEN unit_price < 1 THEN unit_price * 2 + ELSE unit_price + END AS unit_price, + invoice_line_id + FROM + invoice_items + LIMIT + 5 + ) AS table_4 + ) + SELECT + customer_id * 2 AS a, + ROUND(invoice_id * _expr_0, 1) AS b + FROM + table_1 + "); +} + +#[test] +fn test_append_select_multiple() { + // Test for handling multiple append operations with grouping and aggregation + assert_snapshot!(compile(r###" + from invoices + select { customer_id, invoice_id, total, useless1, useless2 } + take 5 + append ( + from employees + select { employee_id, employee_id + 1, reports_to, useless3, useless4 } + take 5 + ) + group { customer_id } (aggregate { invoice_id = math.round 1 (sum invoice_id), total = math.round 1 (sum total), useless1 = sum useless1 }) + append ( + from invoice_items + select { invoice_id, invoice_line_id, 0, useless5 } + take 5 + ) + sort { +invoice_id, +total } + select { total, invoice_id } + "###).unwrap(), @r" + WITH table_3 AS ( + SELECT + * + FROM + ( + SELECT + customer_id, + total, + invoice_id + FROM + invoices + LIMIT + 5 + ) AS table_6 + UNION + ALL + SELECT + * + FROM + ( + SELECT + employee_id, + reports_to, + employee_id + 1 + FROM + employees + LIMIT + 5 + ) AS table_7 + ), + table_2 AS ( + SELECT + ROUND(COALESCE(SUM(total), 0), 1) AS total, + ROUND(COALESCE(SUM(invoice_id), 0), 1) AS invoice_id + FROM + table_3 + GROUP BY + customer_id + UNION + ALL + SELECT + * + FROM + ( + SELECT + invoice_id, + invoice_line_id + FROM + invoice_items + LIMIT + 5 + ) AS table_8 + ) + SELECT + total, + invoice_id + FROM + table_2 + ORDER BY + invoice_id, + total + "); +} + +#[test] +fn test_distinct_on_sort_on_compute() { + // Test for handling distinct on with sorting on computed columns + assert_snapshot!(compile(r###" + from invoices + derive code = case [customer_id < 10 => billing_postal_code, true => null] + group {customer_id, billing_city, billing_country} ( + sort {-this.code} + take 1 + ) + filter (customer_id | in [4]) + group {billing_country} (aggregate {total = math.round 2 (sum total)}) + "###).unwrap(), @r" + WITH table_1 AS ( + SELECT + billing_country, + total, + customer_id, + billing_city, + CASE + WHEN customer_id < 10 THEN billing_postal_code + ELSE NULL + END AS _expr_1, + billing_postal_code + FROM + invoices + ), + table_0 AS ( + SELECT + billing_country, + total, + customer_id, + ROW_NUMBER() OVER ( + PARTITION BY customer_id, + billing_city, + billing_country + ORDER BY + _expr_1 DESC + ) AS _expr_0 + FROM + table_1 + ) + SELECT + billing_country, + ROUND(COALESCE(SUM(total), 0), 2) AS total + FROM + table_0 + WHERE + _expr_0 <= 1 + AND customer_id IN (4) + GROUP BY + billing_country + "); +}