diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java index 786821514822e..e71fb87c7041a 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java @@ -111,6 +111,12 @@ public Table createTable( return asTableCatalog().createTable(ident, columns, partitions, properties); } + @Override + public Table createTable(Identifier ident, TableInfo tableInfo) + throws TableAlreadyExistsException, NoSuchNamespaceException { + return asTableCatalog().createTable(ident, tableInfo); + } + @Override public Table alterTable( Identifier ident, diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Dependency.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Dependency.java new file mode 100644 index 0000000000000..b681b78f1033f --- /dev/null +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Dependency.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connector.catalog; + +import org.apache.spark.annotation.Evolving; + +/** + * Represents a dependency of a SQL object such as a view or metric view. + *

+ * A dependency is one of: {@link TableDependency} or {@link FunctionDependency}. + * + * @since 4.2.0 + */ +@Evolving +public interface Dependency { + + static TableDependency table(String tableFullName) { + return new TableDependency(tableFullName); + } + + static FunctionDependency function(String functionFullName) { + return new FunctionDependency(functionFullName); + } +} diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DependencyList.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DependencyList.java new file mode 100644 index 0000000000000..6a1092bd94d73 --- /dev/null +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DependencyList.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connector.catalog; + +import java.util.Objects; + +import org.apache.spark.annotation.Evolving; + +/** + * A list of dependencies for a SQL object such as a view or metric view. + *

+ *

+ * + * @param dependencies array of dependencies + * @since 4.2.0 + */ +@Evolving +public record DependencyList(Dependency[] dependencies) { + + public DependencyList { + Objects.requireNonNull(dependencies, "dependencies must not be null"); + } + + public static DependencyList of(Dependency... dependencies) { + return new DependencyList(dependencies); + } +} diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/FunctionDependency.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/FunctionDependency.java new file mode 100644 index 0000000000000..dd76190788c83 --- /dev/null +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/FunctionDependency.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connector.catalog; + +import java.util.Objects; + +import org.apache.spark.annotation.Evolving; + +/** + * A function dependency of a SQL object. + *

+ * The dependent function is identified by its fully-qualified three-part name + * in the form {@code catalog_name.schema_name.function_name}. + * + * @param functionFullName fully-qualified three-part function name + * @since 4.2.0 + */ +@Evolving +public record FunctionDependency(String functionFullName) implements Dependency { + public FunctionDependency { + Objects.requireNonNull(functionFullName, "functionFullName must not be null"); + } +} diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableDependency.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableDependency.java new file mode 100644 index 0000000000000..5e44397139f46 --- /dev/null +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableDependency.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connector.catalog; + +import java.util.Objects; + +import org.apache.spark.annotation.Evolving; + +/** + * A table dependency of a SQL object. + *

+ * The dependent table is identified by its fully-qualified three-part name + * in the form {@code catalog_name.schema_name.table_name}. + * + * @param tableFullName fully-qualified three-part table name + * @since 4.2.0 + */ +@Evolving +public record TableDependency(String tableFullName) implements Dependency { + public TableDependency { + Objects.requireNonNull(tableFullName, "tableFullName must not be null"); + } +} diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableInfo.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableInfo.java index 9870a3b0fa45d..f27e219283788 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableInfo.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableInfo.java @@ -30,6 +30,9 @@ public class TableInfo { private final Map properties; private final Transform[] partitions; private final Constraint[] constraints; + private final String tableType; + private final String viewDefinition; + private final DependencyList viewDependencies; /** * Constructor for TableInfo used by the builder. @@ -40,6 +43,9 @@ private TableInfo(Builder builder) { this.properties = builder.properties; this.partitions = builder.partitions; this.constraints = builder.constraints; + this.tableType = builder.tableType; + this.viewDefinition = builder.viewDefinition; + this.viewDependencies = builder.viewDependencies; } public Column[] columns() { @@ -60,11 +66,39 @@ public Transform[] partitions() { public Constraint[] constraints() { return constraints; } + /** + * The table type (e.g. "MANAGED", "EXTERNAL", "VIEW", "METRIC_VIEW"). + * May be null if the connector should infer the type from properties. + * + * @see TableSummary for table type constants + * @since 4.2.0 + */ + public String tableType() { return tableType; } + + /** + * The view definition text. For metric views this is the YAML body; + * for regular views this would be the SQL text. May be null for non-view tables. + * + * @since 4.2.0 + */ + public String viewDefinition() { return viewDefinition; } + + /** + * The list of dependencies for this view or metric view. May be null + * if dependency information is not provided. + * + * @since 4.2.0 + */ + public DependencyList viewDependencies() { return viewDependencies; } + public static class Builder { private Column[] columns = new Column[0]; private Map properties = new HashMap<>(); private Transform[] partitions = new Transform[0]; private Constraint[] constraints = new Constraint[0]; + private String tableType; + private String viewDefinition; + private DependencyList viewDependencies; public Builder withColumns(Column[] columns) { this.columns = columns; @@ -86,6 +120,24 @@ public Builder withConstraints(Constraint[] constraints) { return this; } + /** @since 4.2.0 */ + public Builder withTableType(String tableType) { + this.tableType = tableType; + return this; + } + + /** @since 4.2.0 */ + public Builder withViewDefinition(String viewDefinition) { + this.viewDefinition = viewDefinition; + return this; + } + + /** @since 4.2.0 */ + public Builder withViewDependencies(DependencyList viewDependencies) { + this.viewDependencies = viewDependencies; + return this; + } + public TableInfo build() { Objects.requireNonNull(columns, "columns should not be null"); return new TableInfo(this); diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableSummary.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableSummary.java index 8f46a372342a8..17a4f23bdd1f2 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableSummary.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableSummary.java @@ -27,6 +27,7 @@ public interface TableSummary { String EXTERNAL_TABLE_TYPE = "EXTERNAL"; String VIEW_TABLE_TYPE = "VIEW"; String FOREIGN_TABLE_TYPE = "FOREIGN"; + String METRIC_VIEW_TABLE_TYPE = "METRIC_VIEW"; Identifier identifier(); String tableType();