diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c581a54..fd0627c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- Added support for the [PostgreSQL connector](https://trino.io/docs/current/connector/postgresql.html) using the new generic database connection mechanism. + Previously, users had to use the `generic` connector ([#883]). + ### Changed - BREAKING: `configOverrides` now only accepts the supported config file names @@ -26,6 +31,7 @@ All notable changes to this project will be documented in this file. [#869]: https://github.com/stackabletech/trino-operator/pull/869 [#876]: https://github.com/stackabletech/trino-operator/pull/876 [#878]: https://github.com/stackabletech/trino-operator/pull/878 +[#883]: https://github.com/stackabletech/trino-operator/pull/883 ## [26.3.0] - 2026-03-16 diff --git a/Cargo.nix b/Cargo.nix index 856fbaaa..95127645 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4863,7 +4863,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "k8s_version"; authors = [ @@ -9492,7 +9492,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_certs"; authors = [ @@ -9595,7 +9595,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_operator"; authors = [ @@ -9775,7 +9775,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -9810,7 +9810,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_shared"; authors = [ @@ -9891,7 +9891,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_telemetry"; authors = [ @@ -10102,7 +10102,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_versioned"; authors = [ @@ -10146,7 +10146,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10214,7 +10214,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "95490f2d703d20cf1895e5976fdc5fb8f02aa293"; - sha256 = "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2"; + sha256 = "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 33e75762..1f3488f2 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#k8s-version@0.1.3": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-certs@0.4.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-operator-derive@0.3.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-operator@0.110.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-shared@0.1.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-telemetry@0.6.3": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-versioned-macros@0.9.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-versioned@0.9.0": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-webhook@0.9.1": "1fgc7i8rhq1nl9m4s69sbfiywy2jx4narpynvm3g54vd5yd4c6m2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#k8s-version@0.1.3": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-certs@0.4.0": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-operator-derive@0.3.1": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-operator@0.110.0": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-shared@0.1.0": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-telemetry@0.6.3": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-versioned-macros@0.9.0": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-versioned@0.9.0": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.0#stackable-webhook@0.9.1": "0svjfddsbx72mscsmg1nh581pkdb4ay8nia5gmly0wbfzj7wgq44", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/docs/modules/trino/pages/usage-guide/catalogs/black-hole.adoc b/docs/modules/trino/pages/usage-guide/catalogs/black-hole.adoc index 55688f7c..8b48ec05 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/black-hole.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/black-hole.adoc @@ -4,6 +4,7 @@ Primarily the https://trino.io/docs/current/connector/blackhole.html[Black Hole It works like the `/dev/null` device on Unix-like operating systems for data writing and like `/dev/null` or `/dev/zero` for data reading. == Example Black Hole catalog configuration + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 diff --git a/docs/modules/trino/pages/usage-guide/catalogs/delta-lake.adoc b/docs/modules/trino/pages/usage-guide/catalogs/delta-lake.adoc index 91c7d9a6..ba166114 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/delta-lake.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/delta-lake.adoc @@ -30,5 +30,6 @@ spec: ---- == Connect to S3 store or HDFS + The Delta Lake connector connects to S3 or HDFS in the same way the xref:usage-guide/catalogs/hive.adoc[] connector does. Refer to that documentation for access configuration. diff --git a/docs/modules/trino/pages/usage-guide/catalogs/generic.adoc b/docs/modules/trino/pages/usage-guide/catalogs/generic.adoc index 420acab8..427d0b3a 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/generic.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/generic.adoc @@ -8,6 +8,7 @@ In case the Stackable operator for Trino does not support a specific connector y This is how you can e.g. use the {trino-psql-connector}[PostgreSQL connector]: == Example generic catalog properties + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 diff --git a/docs/modules/trino/pages/usage-guide/catalogs/google-sheets.adoc b/docs/modules/trino/pages/usage-guide/catalogs/google-sheets.adoc index a7a9d266..cb77ca63 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/google-sheets.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/google-sheets.adoc @@ -6,6 +6,7 @@ It needs a service-user to access the Google APIs. Consult the {trino-google-sheets-connector}[official documentation] on how to use the Google Sheets connector. == Example Google sheets catalog configuration + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 diff --git a/docs/modules/trino/pages/usage-guide/catalogs/hive.adoc b/docs/modules/trino/pages/usage-guide/catalogs/hive.adoc index 556636dd..db3de6ac 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/hive.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/hive.adoc @@ -5,6 +5,7 @@ The Apache Hive connector allows Trino to connect to a Hive metastore and query Deploy a Hive Stacklet with the xref:hive:index.adoc[]. == Example Hive catalog configuration + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 @@ -34,6 +35,7 @@ spec: <4> Use `configOverrides` to add arbitrary properties to the Trino catalog configuration == Connect to S3 store + The Hive connector can connect to an S3 store as follows: [source,yaml] @@ -58,6 +60,7 @@ See xref:concepts:s3.adoc[] for details about S3 connections. IMPORTANT: Make sure that the underlying Hive metastore also has access to the S3 store, because it will e.g. check if the directory exists when creating tables. == Connect to HDFS cluster + The hive connector can connect to an HDFS operated by Stackable as follows: [source,yaml] @@ -72,10 +75,12 @@ spec: IMPORTANT: Make sure that the underlying Hive metastore also has access to the HDFS, because it will e.g. check if the directory exists when creating tables. == Adding unmanaged Hive clusters + You can add connect Trino to Hive catalogs from systems that are not managed by Stackable, including Hive running on existing Hadoop clusters. Unmanaged Hive instances can be defined by creating a ConfigMap containing the configuration for the remote Hive Metastore and HDFS or S3 storage services. === Create a Hive Metastore configMap + The Hive metastore ConfigMap contains the URL for the metastore's thrift endpoint. [source,yaml] ---- @@ -88,6 +93,7 @@ data: ---- === Create a HDFS configMap + When the Hive data is stored on HDFS you will need to provide a ConfigMap containing the HDFS configuration. To do this take the `core-site.xml` and `hdfs-site.xml` from your Hadoop cluster and create a ConfigMap with the keys `core-site.xml` and `hdfs-site.xml`. @@ -119,6 +125,7 @@ data: ---- === Create the Trino Hive catalog + To use the unmanaged Hive metastore we define a TrinoCatalog object in the same way we would for a managed cluster, referencing the custom ConfigMap we created for Hive and HDFS. [source,yaml] diff --git a/docs/modules/trino/pages/usage-guide/catalogs/index.adoc b/docs/modules/trino/pages/usage-guide/catalogs/index.adoc index ae6b560a..3c62ede9 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/index.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/index.adoc @@ -4,6 +4,7 @@ Trino does not have a built-in catalog and instead provides connectors to extern This allows Trino to connect to, read from and join a wide variety of data sources. == Using catalogs + Catalogs are defined in their own resources and referenced from cluster objects. See the xref:concepts.adoc[] page for more details. diff --git a/docs/modules/trino/pages/usage-guide/catalogs/postgresql.adoc b/docs/modules/trino/pages/usage-guide/catalogs/postgresql.adoc new file mode 100644 index 00000000..18564917 --- /dev/null +++ b/docs/modules/trino/pages/usage-guide/catalogs/postgresql.adoc @@ -0,0 +1,32 @@ += PostgreSQL + +The https://trino.io/docs/current/connector/postgresql.html[PostgreSQL connector] provides connectivity to a PostgreSQL databases. + +== Example PostgreSQL catalog configuration + +[source,yaml] +---- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + # The name of the catalog as it will appear in Trino + name: postgresql + # TrinoCluster can use these labels to select which catalogs to include + labels: + trino: simple-trino +spec: + connector: + postgresql: + host: my-postgresql + port: 5432 # defaults to 5432 + database: mydatabase + credentialsSecretName: my-postgresql-credentials # Must provide the username and password keys +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-postgresql-credentials +stringData: + username: myusername + password: mypassword +---- diff --git a/docs/modules/trino/pages/usage-guide/catalogs/tpcds.adoc b/docs/modules/trino/pages/usage-guide/catalogs/tpcds.adoc index 454aebfb..b7623011 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/tpcds.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/tpcds.adoc @@ -4,6 +4,7 @@ The https://trino.io/docs/current/connector/tpcds.html[TPC-DS connector] provide This is not a stored data set and is instead a virtual data set generated at query time using a deterministic algorithm. == Example TPC-DS catalog configuration + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 diff --git a/docs/modules/trino/pages/usage-guide/catalogs/tpch.adoc b/docs/modules/trino/pages/usage-guide/catalogs/tpch.adoc index 332abc65..963c9bfd 100644 --- a/docs/modules/trino/pages/usage-guide/catalogs/tpch.adoc +++ b/docs/modules/trino/pages/usage-guide/catalogs/tpch.adoc @@ -4,6 +4,7 @@ The https://trino.io/docs/current/connector/tpch.html[TPC-H connector] provides This is not a stored data set and is instead a virtual data set generated at query time using a deterministic algorithm. == Example TPC-H catalog configuration + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 diff --git a/docs/modules/trino/partials/nav.adoc b/docs/modules/trino/partials/nav.adoc index 453c0fa0..d90607af 100644 --- a/docs/modules/trino/partials/nav.adoc +++ b/docs/modules/trino/partials/nav.adoc @@ -22,6 +22,7 @@ *** xref:trino:usage-guide/catalogs/google-sheets.adoc[] *** xref:trino:usage-guide/catalogs/hive.adoc[] *** xref:trino:usage-guide/catalogs/iceberg.adoc[] +*** xref:trino:usage-guide/catalogs/postgresql.adoc[] *** xref:trino:usage-guide/catalogs/tpcds.adoc[] *** xref:trino:usage-guide/catalogs/tpch.adoc[] ** xref:trino:usage-guide/operations/index.adoc[] diff --git a/extra/crds.yaml b/extra/crds.yaml index 4729af1d..f762ef6b 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -3577,6 +3577,8 @@ spec: - hive - required: - iceberg + - required: + - postgresql - required: - tpcds - required: @@ -4216,6 +4218,41 @@ spec: type: string type: object type: object + postgresql: + description: An [PostgreSQL](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/postgresql) connector. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the PostgreSQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the PostgreSQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of JDBC connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 5432 + description: Port the PostgreSQL server is listening on. Defaults to `5432`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object tpcds: description: A [TPC-DS](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/tpcds) connector. type: object diff --git a/rust/operator-binary/src/catalog/config.rs b/rust/operator-binary/src/catalog/config.rs index de6a18ad..8d4ef553 100644 --- a/rust/operator-binary/src/catalog/config.rs +++ b/rust/operator-binary/src/catalog/config.rs @@ -114,48 +114,20 @@ impl CatalogConfig { .ok_or(FromTrinoCatalogError::InvalidCatalogSpec)?; let catalog_namespace = catalog.namespace(); - let mut catalog_config = match &catalog.spec.connector { - TrinoCatalogConnector::BlackHole(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::DeltaLake(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::GoogleSheet(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::Generic(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::Hive(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::Iceberg(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::Tpcds(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - TrinoCatalogConnector::Tpch(connector) => { - connector - .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) - .await - } - }?; + let to_catalog_config: &dyn ToCatalogConfig = match &catalog.spec.connector { + TrinoCatalogConnector::BlackHole(black_hole_connector) => black_hole_connector, + TrinoCatalogConnector::DeltaLake(delta_lake_connector) => delta_lake_connector, + TrinoCatalogConnector::GoogleSheet(google_sheet_connector) => google_sheet_connector, + TrinoCatalogConnector::Generic(generic_connector) => generic_connector, + TrinoCatalogConnector::Hive(hive_connector) => hive_connector, + TrinoCatalogConnector::Iceberg(iceberg_connector) => iceberg_connector, + TrinoCatalogConnector::Postgresql(postgresql_connector) => postgresql_connector, + TrinoCatalogConnector::Tpcds(tpcds_connector) => tpcds_connector, + TrinoCatalogConnector::Tpch(tpch_connector) => tpch_connector, + }; + let mut catalog_config = to_catalog_config + .to_catalog_config(&catalog_name, catalog_namespace, client, trino_version) + .await?; catalog_config .properties diff --git a/rust/operator-binary/src/catalog/mod.rs b/rust/operator-binary/src/catalog/mod.rs index 63c670a6..9abc549a 100644 --- a/rust/operator-binary/src/catalog/mod.rs +++ b/rust/operator-binary/src/catalog/mod.rs @@ -6,6 +6,7 @@ pub mod generic; pub mod google_sheet; pub mod hive; pub mod iceberg; +pub mod postgresql; pub mod tpcds; pub mod tpch; @@ -61,6 +62,11 @@ pub enum FromTrinoCatalogError { CreateS3CredentialsSecretOperatorVolume { source: stackable_operator::builder::pod::volume::SecretOperatorVolumeSourceBuilderError, }, + + #[snafu(display("failed to get PostgreSQL connection details"))] + GetPostgresConnectionDetails { + source: stackable_operator::database_connections::Error, + }, } #[async_trait] diff --git a/rust/operator-binary/src/catalog/postgresql.rs b/rust/operator-binary/src/catalog/postgresql.rs new file mode 100644 index 00000000..e5731641 --- /dev/null +++ b/rust/operator-binary/src/catalog/postgresql.rs @@ -0,0 +1,50 @@ +use async_trait::async_trait; +use snafu::ResultExt; +use stackable_operator::{ + client::Client, database_connections::drivers::jdbc::JdbcDatabaseConnection, +}; + +use super::{FromTrinoCatalogError, ToCatalogConfig, config::CatalogConfig}; +use crate::{ + catalog::from_trino_catalog_error::GetPostgresConnectionDetailsSnafu, + crd::catalog::postgresql::PostgresqlConnector, +}; + +pub const CONNECTOR_NAME: &str = "postgresql"; + +#[async_trait] +impl ToCatalogConfig for PostgresqlConnector { + async fn to_catalog_config( + &self, + catalog_name: &str, + _catalog_namespace: Option, + _client: &Client, + _trino_version: u16, + ) -> Result { + let mut config = CatalogConfig::new(catalog_name.to_string(), CONNECTOR_NAME); + // SAFETY: `unique_database_name` must only contain uppercase ASCII letters and underscores. + let unique_database_name = format!( + "POSTGRESQL_{}", + catalog_name.replace('-', "_").to_uppercase() + ); + let jdbc_connection_details = self + .inner + .jdbc_connection_details(&unique_database_name) + .context(GetPostgresConnectionDetailsSnafu)?; + + config.add_property("connection-url", jdbc_connection_details.connection_url); + if let Some(username_env) = jdbc_connection_details.username_env { + config.add_property("connection-user", format!("${{ENV:{}}}", username_env.name)); + config.env_bindings.push(username_env); + }; + if let Some(password_env) = jdbc_connection_details.password_env { + config.add_property( + "connection-password", + format!("${{ENV:{}}}", password_env.name), + ); + config.env_bindings.push(password_env); + }; + + Ok(config) + } +} diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index cac09f42..bac46c9a 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -32,6 +32,7 @@ pub fn get_affinity( TrinoCatalogConnector::BlackHole(_) | TrinoCatalogConnector::Generic(_) | TrinoCatalogConnector::GoogleSheet(_) + | TrinoCatalogConnector::Postgresql(_) | TrinoCatalogConnector::Tpcds(_) | TrinoCatalogConnector::Tpch(_) => None, }) @@ -59,6 +60,7 @@ pub fn get_affinity( TrinoCatalogConnector::BlackHole(_) | TrinoCatalogConnector::Generic(_) | TrinoCatalogConnector::GoogleSheet(_) + | TrinoCatalogConnector::Postgresql(_) | TrinoCatalogConnector::Tpcds(_) | TrinoCatalogConnector::Tpch(_) => None, }) diff --git a/rust/operator-binary/src/crd/catalog/mod.rs b/rust/operator-binary/src/crd/catalog/mod.rs index 2c81c98d..c10dd80c 100644 --- a/rust/operator-binary/src/crd/catalog/mod.rs +++ b/rust/operator-binary/src/crd/catalog/mod.rs @@ -5,6 +5,7 @@ pub mod generic; pub mod google_sheet; pub mod hive; pub mod iceberg; +pub mod postgresql; pub mod tpcds; pub mod tpch; @@ -25,6 +26,7 @@ use tpcds::TpcdsConnector; use tpch::TpchConnector; use self::delta_lake::DeltaLakeConnector; +use crate::crd::catalog::postgresql::PostgresqlConnector; #[versioned( version(name = "v1alpha1"), @@ -86,6 +88,9 @@ pub enum TrinoCatalogConnector { /// An [Apache Iceberg](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/iceberg) connector. Iceberg(IcebergConnector), + /// An [PostgreSQL](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/postgresql) connector. + Postgresql(PostgresqlConnector), + /// A [TPC-DS](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/tpcds) connector. Tpcds(TpcdsConnector), diff --git a/rust/operator-binary/src/crd/catalog/postgresql.rs b/rust/operator-binary/src/crd/catalog/postgresql.rs new file mode 100644 index 00000000..8d13c0b0 --- /dev/null +++ b/rust/operator-binary/src/crd/catalog/postgresql.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + database_connections::databases::postgresql::PostgresqlConnection, + schemars::{self, JsonSchema}, +}; + +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PostgresqlConnector { + // Docs are on the struct fields + #[serde(flatten)] + pub inner: PostgresqlConnection, +} diff --git a/tests/templates/kuttl/commons/check-s3.py b/tests/templates/kuttl/commons/check-s3.py index 9c2474f6..5749214b 100755 --- a/tests/templates/kuttl/commons/check-s3.py +++ b/tests/templates/kuttl/commons/check-s3.py @@ -279,16 +279,17 @@ def run_query(connection, query): # Test could be improved by also testing update and deletes - # # Test postgres connection - # run_query(connection, "SHOW SCHEMAS IN postgresgeneric") - # run_query(connection, "CREATE SCHEMA IF NOT EXISTS postgresgeneric.tpch") - # run_query( - # connection, - # "CREATE TABLE IF NOT EXISTS postgresgeneric.tpch.nation AS SELECT * FROM tpch.tiny.nation", - # ) - # assert ( - # run_query(connection, "SELECT COUNT(*) FROM postgresgeneric.tpch.nation")[0][0] - # == 25 - # ) + # Test postgres connection(s) + for catalog in ["postgresql", "postgresqlgeneric"]: + run_query(connection, f"SHOW SCHEMAS IN {catalog}") + run_query(connection, f"CREATE SCHEMA IF NOT EXISTS {catalog}.tpch") + run_query( + connection, + f"CREATE TABLE IF NOT EXISTS {catalog}.tpch.nation AS SELECT * FROM tpch.tiny.nation", + ) + assert ( + run_query(connection, f"SELECT COUNT(*) FROM {catalog}.tpch.nation")[0][0] + == 25 + ) print("[SUCCESS] All tests in check-s3.py succeeded!") diff --git a/tests/templates/kuttl/delta/08-install-hive.yaml.j2 b/tests/templates/kuttl/delta/08-install-hive.yaml.j2 index 9eff5093..d145f800 100644 --- a/tests/templates/kuttl/delta/08-install-hive.yaml.j2 +++ b/tests/templates/kuttl/delta/08-install-hive.yaml.j2 @@ -8,10 +8,11 @@ spec: productVersion: "{{ test_scenario['values']['hive-latest'] }}" pullPolicy: IfNotPresent clusterConfig: - database: - connString: jdbc:postgresql://postgresql:5432/hive - credentialsSecret: postgres-credentials - dbType: postgres + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: postgres-credentials s3: reference: minio {% if lookup('env', 'VECTOR_AGGREGATOR') %} diff --git a/tests/templates/kuttl/opa-authorization/03-install-hive.yaml.j2 b/tests/templates/kuttl/opa-authorization/03-install-hive.yaml.j2 index 9eff5093..d145f800 100644 --- a/tests/templates/kuttl/opa-authorization/03-install-hive.yaml.j2 +++ b/tests/templates/kuttl/opa-authorization/03-install-hive.yaml.j2 @@ -8,10 +8,11 @@ spec: productVersion: "{{ test_scenario['values']['hive-latest'] }}" pullPolicy: IfNotPresent clusterConfig: - database: - connString: jdbc:postgresql://postgresql:5432/hive - credentialsSecret: postgres-credentials - dbType: postgres + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: postgres-credentials s3: reference: minio {% if lookup('env', 'VECTOR_AGGREGATOR') %} diff --git a/tests/templates/kuttl/smoke/08-install-hive.yaml.j2 b/tests/templates/kuttl/smoke/08-install-hive.yaml.j2 index 446ce72a..647d6ea8 100644 --- a/tests/templates/kuttl/smoke/08-install-hive.yaml.j2 +++ b/tests/templates/kuttl/smoke/08-install-hive.yaml.j2 @@ -8,10 +8,11 @@ spec: productVersion: "{{ test_scenario['values']['hive'] }}" pullPolicy: IfNotPresent clusterConfig: - database: - connString: jdbc:postgresql://postgresql:5432/hive - credentialsSecret: postgres-credentials - dbType: postgres + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: postgres-credentials hdfs: configMap: hdfs s3: diff --git a/tests/templates/kuttl/smoke/10-install-trino.yaml.j2 b/tests/templates/kuttl/smoke/10-install-trino.yaml.j2 index d94cfb81..3b085475 100644 --- a/tests/templates/kuttl/smoke/10-install-trino.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-trino.yaml.j2 @@ -64,7 +64,20 @@ spec: apiVersion: trino.stackable.tech/v1alpha1 kind: TrinoCatalog metadata: - name: postgresgeneric + name: postgresql + labels: + trino: trino +spec: + connector: + postgresql: + host: postgresql + database: hive + credentialsSecretName: my-postgresql-credentials-secret +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + name: postgresqlgeneric labels: trino: trino spec: @@ -78,7 +91,7 @@ spec: connection-user: valueFromSecret: name: my-postgresql-credentials-secret - key: user + key: username connection-password: valueFromSecret: name: my-postgresql-credentials-secret @@ -89,7 +102,7 @@ kind: Secret metadata: name: my-postgresql-credentials-secret stringData: - user: hive + username: hive password: hive --- # We need to create the TrinoCluster last, so that the ConfigMaps/Secrets it mounts are already diff --git a/tests/templates/kuttl/smoke_aws/08-install-hive.yaml.j2 b/tests/templates/kuttl/smoke_aws/08-install-hive.yaml.j2 index 41d811e7..a875f348 100644 --- a/tests/templates/kuttl/smoke_aws/08-install-hive.yaml.j2 +++ b/tests/templates/kuttl/smoke_aws/08-install-hive.yaml.j2 @@ -8,10 +8,11 @@ spec: productVersion: "{{ test_scenario['values']['hive'] }}" pullPolicy: IfNotPresent clusterConfig: - database: - connString: jdbc:postgresql://postgresql:5432/hive - credentialsSecret: postgres-credentials - dbType: postgres + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: postgres-credentials hdfs: configMap: hdfs s3: