Merge branch 'alamb/less_cfg3' into extend-ci

Weijun-H · Weijun-H · commit 4ec839908a11 · 2023-10-26T22:22:39.000+02:00
diff --git a/datafusion/common/src/file_options/file_type.rs b/datafusion/common/src/file_options/file_type.rs
@@ -48,7 +48,6 @@ pub enum FileType {
     /// Apache Avro file
     AVRO,
     /// Apache Parquet file
-    #[cfg(feature = "parquet")]
     PARQUET,
     /// CSV file
     CSV,
@@ -61,7 +60,6 @@ impl GetExt for FileType {
         match self {
             FileType::ARROW => DEFAULT_ARROW_EXTENSION.to_owned(),
             FileType::AVRO => DEFAULT_AVRO_EXTENSION.to_owned(),
-            #[cfg(feature = "parquet")]
             FileType::PARQUET => DEFAULT_PARQUET_EXTENSION.to_owned(),
             FileType::CSV => DEFAULT_CSV_EXTENSION.to_owned(),
             FileType::JSON => DEFAULT_JSON_EXTENSION.to_owned(),
@@ -74,7 +72,6 @@ impl Display for FileType {
         let out = match self {
             FileType::CSV => "csv",
             FileType::JSON => "json",
-            #[cfg(feature = "parquet")]
             FileType::PARQUET => "parquet",
             FileType::AVRO => "avro",
             FileType::ARROW => "arrow",
@@ -91,7 +88,6 @@ impl FromStr for FileType {
         match s.as_str() {
             "ARROW" => Ok(FileType::ARROW),
             "AVRO" => Ok(FileType::AVRO),
-            #[cfg(feature = "parquet")]
             "PARQUET" => Ok(FileType::PARQUET),
             "CSV" => Ok(FileType::CSV),
             "JSON" | "NDJSON" => Ok(FileType::JSON),
diff --git a/datafusion/common/src/file_options/mod.rs b/datafusion/common/src/file_options/mod.rs
@@ -22,7 +22,6 @@ pub mod avro_writer;
 pub mod csv_writer;
 pub mod file_type;
 pub mod json_writer;
-#[cfg(feature = "parquet")]
 pub mod parquet_writer;
 pub(crate) mod parse_utils;
 
@@ -38,14 +37,13 @@ use crate::{
     DataFusionError, FileType, Result,
 };
 
-#[cfg(feature = "parquet")]
-use self::parquet_writer::ParquetWriterOptions;
-
 use self::{
     arrow_writer::ArrowWriterOptions, avro_writer::AvroWriterOptions,
     csv_writer::CsvWriterOptions, json_writer::JsonWriterOptions,
 };
 
+use self::parquet_writer::ParquetWriterOptions;
+
 /// Represents a single arbitrary setting in a
 /// [StatementOptions] where OptionTuple.0 determines
 /// the specific setting to be modified and OptionTuple.1
@@ -148,7 +146,6 @@ impl StatementOptions {
 /// plus any DataFusion specific writing options (e.g. CSV compression)
 #[derive(Clone, Debug)]
 pub enum FileTypeWriterOptions {
-    #[cfg(feature = "parquet")]
     Parquet(ParquetWriterOptions),
     CSV(CsvWriterOptions),
     JSON(JsonWriterOptions),
@@ -168,7 +165,6 @@ impl FileTypeWriterOptions {
         let options = (config_defaults, statement_options);
 
         let file_type_write_options = match file_type {
-            #[cfg(feature = "parquet")]
             FileType::PARQUET => {
                 FileTypeWriterOptions::Parquet(ParquetWriterOptions::try_from(options)?)
             }
@@ -198,7 +194,6 @@ impl FileTypeWriterOptions {
         let options = (config_defaults, &empty_statement);
 
         let file_type_write_options = match file_type {
-            #[cfg(feature = "parquet")]
             FileType::PARQUET => {
                 FileTypeWriterOptions::Parquet(ParquetWriterOptions::try_from(options)?)
             }
@@ -221,7 +216,6 @@ impl FileTypeWriterOptions {
 
     /// Tries to extract ParquetWriterOptions from this FileTypeWriterOptions enum.
     /// Returns an error if a different type from parquet is set.
-    #[cfg(feature = "parquet")]
     pub fn try_into_parquet(&self) -> Result<&ParquetWriterOptions> {
         match self {
             FileTypeWriterOptions::Parquet(opt) => Ok(opt),
@@ -288,7 +282,6 @@ impl Display for FileTypeWriterOptions {
             FileTypeWriterOptions::Avro(_) => "AvroWriterOptions",
             FileTypeWriterOptions::CSV(_) => "CsvWriterOptions",
             FileTypeWriterOptions::JSON(_) => "JsonWriterOptions",
-            #[cfg(feature = "parquet")]
             FileTypeWriterOptions::Parquet(_) => "ParquetWriterOptions",
         };
         write!(f, "{}", name)
diff --git a/datafusion/common/src/file_options/parquet_writer/mod.rs b/datafusion/common/src/file_options/parquet_writer/mod.rs
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "parquet")]
+mod parquet;
+
+#[cfg(feature = "parquet")]
+pub use self::parquet::*;
+
+#[cfg(not(feature = "parquet"))]
+mod parquet_stub;
+
+#[cfg(not(feature = "parquet"))]
+pub use parquet_stub::*;
diff --git a/datafusion/common/src/file_options/parquet_writer/parquet.rs b/datafusion/common/src/file_options/parquet_writer/parquet.rs
@@ -21,8 +21,7 @@ use parquet::file::properties::{WriterProperties, WriterPropertiesBuilder};
 
 use crate::{config::ConfigOptions, DataFusionError, Result};
 
-use super::StatementOptions;
-
+use crate::file_options::StatementOptions;
 use parquet::{
     basic::{BrotliLevel, GzipLevel, ZstdLevel},
     file::properties::{EnabledStatistics, WriterVersion},
diff --git a/datafusion/common/src/file_options/parquet_writer/parquet_stub.rs b/datafusion/common/src/file_options/parquet_writer/parquet_stub.rs
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::config::ConfigOptions;
+use crate::error::_not_impl_err;
+use crate::file_options::StatementOptions;
+use crate::{DataFusionError, Result};
+
+/// Stub implementation of `ParquetWriterOptions` that always returns a
+/// NotYetImplemented error used when parquet feature is not activated.
+#[derive(Clone, Debug)]
+pub struct ParquetWriterOptions {}
+
+impl TryFrom<(&ConfigOptions, &StatementOptions)> for ParquetWriterOptions {
+    type Error = DataFusionError;
+
+    fn try_from(_: (&ConfigOptions, &StatementOptions)) -> Result<Self> {
+        _not_impl_err!(
+            "Parquet support is not enabled. Hint enable the `parquet` feature flag"
+        )
+    }
+}
diff --git a/datafusion/common/src/test_util.rs b/datafusion/common/src/test_util.rs
@@ -180,7 +180,6 @@ pub fn arrow_test_data() -> String {
 /// let filename = format!("{}/binary.parquet", testdata);
 /// assert!(std::path::PathBuf::from(filename).exists());
 /// ```
-#[cfg(feature = "parquet")]
 pub fn parquet_test_data() -> String {
     match get_data_dir("PARQUET_TEST_DATA", "../../parquet-testing/data") {
         Ok(pb) => pb.display().to_string(),
diff --git a/datafusion/core/src/datasource/file_format/file_compression_type.rs b/datafusion/core/src/datasource/file_format/file_compression_type.rs
@@ -243,7 +243,6 @@ impl FileTypeExt for FileType {
                     "FileCompressionType can be specified for CSV/JSON FileType.".into(),
                 )),
             },
-            #[cfg(feature = "parquet")]
             FileType::PARQUET => match c.variant {
                 UNCOMPRESSED => Ok(ext),
                 _ => Err(DataFusionError::Internal(
@@ -285,7 +284,6 @@ mod tests {
 
         let mut ty_ext_tuple = vec![];
         ty_ext_tuple.push((FileType::AVRO, ".avro"));
-        #[cfg(feature = "parquet")]
         ty_ext_tuple.push((FileType::PARQUET, ".parquet"));
 
         // Cannot specify compression for these file types
diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs
@@ -27,8 +27,8 @@ pub mod csv;
 pub mod file_compression_type;
 pub mod json;
 pub mod options;
-#[cfg(feature = "parquet")]
 pub mod parquet;
+
 pub mod write;
 
 use std::any::Any;
diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs
@@ -25,7 +25,6 @@ use datafusion_common::{plan_err, DataFusionError};
 
 use crate::datasource::file_format::arrow::ArrowFormat;
 use crate::datasource::file_format::file_compression_type::FileCompressionType;
-#[cfg(feature = "parquet")]
 use crate::datasource::file_format::parquet::ParquetFormat;
 use crate::datasource::file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD;
 use crate::datasource::listing::{ListingTableInsertMode, ListingTableUrl};
diff --git a/datafusion/core/src/datasource/file_format/parquet/mod.rs b/datafusion/core/src/datasource/file_format/parquet/mod.rs
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! FileFormat for parquet
+
+/// If parquet is enabled, use actual implementation
+#[cfg(feature = "parquet")]
+mod parquet_impl;
+#[cfg(feature = "parquet")]
+pub use parquet_impl::*;
+
+/// If parquet is not enabled, use dummy implementation
+#[cfg(not(feature = "parquet"))]
+mod parquet_stub;
+#[cfg(not(feature = "parquet"))]
+pub use parquet_stub::ParquetFormat;
diff --git a/datafusion/core/src/datasource/file_format/parquet/parquet_impl.rs b/datafusion/core/src/datasource/file_format/parquet/parquet_impl.rs
@@ -54,17 +54,20 @@ use parquet::file::metadata::ParquetMetaData;
 use parquet::file::properties::WriterProperties;
 use parquet::file::statistics::Statistics as ParquetStatistics;
 
-use super::write::demux::start_demuxer_task;
-use super::write::{create_writer, AbortableWrite, FileWriterMode};
-use super::{FileFormat, FileScanConfig};
 use crate::arrow::array::{
     BooleanArray, Float32Array, Float64Array, Int32Array, Int64Array,
 };
 use crate::arrow::datatypes::DataType;
 use crate::config::ConfigOptions;
+use crate::datasource::file_format::write::demux::start_demuxer_task;
+use crate::datasource::file_format::write::{
+    create_writer, AbortableWrite, FileWriterMode,
+};
+use crate::datasource::file_format::FileFormat;
 
 use crate::datasource::physical_plan::{
-    FileGroupDisplay, FileMeta, FileSinkConfig, ParquetExec, SchemaAdapter,
+    FileGroupDisplay, FileMeta, FileScanConfig, FileSinkConfig, ParquetExec,
+    SchemaAdapter,
 };
 use crate::error::Result;
 use crate::execution::context::SessionState;
@@ -1204,14 +1207,14 @@ pub(crate) mod test_util {
 
 #[cfg(test)]
 mod tests {
-    use super::super::test_util::scan_format;
     use crate::physical_plan::collect;
     use std::fmt::{Display, Formatter};
     use std::sync::atomic::{AtomicUsize, Ordering};
 
     use super::*;
 
     use crate::datasource::file_format::parquet::test_util::store_parquet;
+    use crate::datasource::file_format::test_util::scan_format;
     use crate::physical_plan::metrics::MetricValue;
     use crate::prelude::{SessionConfig, SessionContext};
     use arrow::array::{Array, ArrayRef, StringArray};
diff --git a/datafusion/core/src/datasource/file_format/parquet/parquet_stub.rs b/datafusion/core/src/datasource/file_format/parquet/parquet_stub.rs
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::datasource::file_format::FileFormat;
+use crate::datasource::physical_plan::FileScanConfig;
+use crate::execution::context::SessionState;
+use arrow_schema::SchemaRef;
+use async_trait::async_trait;
+use datafusion_common::{
+    not_impl_datafusion_err, DataFusionError, FileType, Result, Statistics,
+};
+use datafusion_physical_expr::PhysicalExpr;
+use datafusion_physical_plan::ExecutionPlan;
+use object_store::{ObjectMeta, ObjectStore};
+use std::any::Any;
+use std::sync::Arc;
+
+/// Stub implementation of `ParquetFormat` that always returns a NotYetImplemented error.
+#[derive(Debug, Default)]
+pub struct ParquetFormat;
+
+impl ParquetFormat {
+    /// Create a new instance of the Parquet format
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+fn nyi_error() -> DataFusionError {
+    not_impl_datafusion_err!(
+        "Parquet support is not enabled. Hint enable the `parquet` feature flag"
+    )
+}
+
+#[async_trait]
+impl FileFormat for ParquetFormat {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    async fn infer_schema(
+        &self,
+        _: &SessionState,
+        _: &Arc<dyn ObjectStore>,
+        _: &[ObjectMeta],
+    ) -> Result<SchemaRef> {
+        Err(nyi_error())
+    }
+
+    async fn infer_stats(
+        &self,
+        _: &SessionState,
+        _: &Arc<dyn ObjectStore>,
+        _: SchemaRef,
+        _: &ObjectMeta,
+    ) -> Result<Statistics> {
+        Err(nyi_error())
+    }
+
+    async fn create_physical_plan(
+        &self,
+        _: &SessionState,
+        _: FileScanConfig,
+        _: Option<&Arc<dyn PhysicalExpr>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        Err(nyi_error())
+    }
+
+    fn file_type(&self) -> FileType {
+        FileType::PARQUET
+    }
+}
diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs
@@ -23,7 +23,6 @@ use std::{any::Any, sync::Arc};
 use super::helpers::{expr_applicable_for_cols, pruned_partition_list, split_files};
 use super::PartitionedFile;
 
-#[cfg(feature = "parquet")]
 use crate::datasource::file_format::parquet::ParquetFormat;
 use crate::datasource::{
     file_format::{
@@ -151,7 +150,6 @@ impl ListingTableConfig {
             FileType::JSON => Arc::new(
                 JsonFormat::default().with_file_compression_type(file_compression_type),
             ),
-            #[cfg(feature = "parquet")]
             FileType::PARQUET => Arc::new(ParquetFormat::default()),
         };
 
@@ -1021,7 +1019,6 @@ mod tests {
     use std::fs::File;
 
     use super::*;
-    #[cfg(feature = "parquet")]
     use crate::datasource::file_format::parquet::ParquetFormat;
     use crate::datasource::{provider_as_source, MemTable};
     use crate::execution::options::ArrowReadOptions;
diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rs
diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rs
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
