diff --git a/crates/polars-lazy/src/scan/ndjson.rs b/crates/polars-lazy/src/scan/ndjson.rs index e38270ec3e09..e53171d28f51 100644 --- a/crates/polars-lazy/src/scan/ndjson.rs +++ b/crates/polars-lazy/src/scan/ndjson.rs @@ -157,7 +157,6 @@ impl LazyFileListReader for LazyJsonLineReader { Ok(LazyFrame::from(DslPlan::Scan { sources: Arc::new(Mutex::new(self.sources.to_dsl(false))), file_info: Arc::new(RwLock::new(None)), - hive_parts: None, predicate: None, file_options, scan_type, diff --git a/crates/polars-plan/src/plans/builder_dsl.rs b/crates/polars-plan/src/plans/builder_dsl.rs index 7efa55417509..bbbe12ef95de 100644 --- a/crates/polars-plan/src/plans/builder_dsl.rs +++ b/crates/polars-plan/src/plans/builder_dsl.rs @@ -63,7 +63,6 @@ impl DslBuilder { is_expanded: true, })), file_info: Arc::new(RwLock::new(Some(file_info))), - hive_parts: None, predicate: None, file_options, scan_type: FileScan::Anonymous { @@ -107,7 +106,6 @@ impl DslBuilder { Ok(DslPlan::Scan { sources: Arc::new(Mutex::new(sources)), file_info: Arc::new(RwLock::new(None)), - hive_parts: None, predicate: None, file_options: options, scan_type: FileScan::Parquet { @@ -139,7 +137,6 @@ impl DslBuilder { Ok(DslPlan::Scan { sources: Arc::new(Mutex::new(sources)), file_info: Arc::new(RwLock::new(None)), - hive_parts: None, file_options: FileScanOptions { with_columns: None, cache, @@ -192,7 +189,6 @@ impl DslBuilder { Ok(DslPlan::Scan { sources: Arc::new(Mutex::new(sources)), file_info: Arc::new(RwLock::new(None)), - hive_parts: None, file_options: options, predicate: None, scan_type: FileScan::Csv { diff --git a/crates/polars-plan/src/plans/conversion/dsl_to_ir.rs b/crates/polars-plan/src/plans/conversion/dsl_to_ir.rs index a908378e6f5c..23f2050dec4c 100644 --- a/crates/polars-plan/src/plans/conversion/dsl_to_ir.rs +++ b/crates/polars-plan/src/plans/conversion/dsl_to_ir.rs @@ -107,7 +107,6 @@ pub fn to_alp_impl(lp: DslPlan, ctxt: &mut DslConversionContext) -> PolarsResult DslPlan::Scan { sources, file_info, - hive_parts, predicate, mut file_options, mut scan_type, @@ -188,9 +187,7 @@ pub fn to_alp_impl(lp: DslPlan, ctxt: &mut DslConversionContext) -> PolarsResult } }; - let hive_parts = if hive_parts.is_some() { - hive_parts - } else if file_options.hive_options.enabled.unwrap_or(false) + let hive_parts = if file_options.hive_options.enabled.unwrap_or(false) && resolved_file_info.reader_schema.is_some() { let paths = sources diff --git a/crates/polars-plan/src/plans/conversion/mod.rs b/crates/polars-plan/src/plans/conversion/mod.rs index b9ed8711a438..c79b6444effe 100644 --- a/crates/polars-plan/src/plans/conversion/mod.rs +++ b/crates/polars-plan/src/plans/conversion/mod.rs @@ -52,7 +52,7 @@ impl IR { IR::Scan { sources, file_info, - hive_parts, + hive_parts: _, predicate, scan_type, output_schema: _, @@ -63,7 +63,6 @@ impl IR { is_expanded: true, })), file_info: Arc::new(RwLock::new(Some(file_info))), - hive_parts, predicate: predicate.map(|e| e.to_expr(expr_arena)), scan_type, file_options: options, diff --git a/crates/polars-plan/src/plans/mod.rs b/crates/polars-plan/src/plans/mod.rs index 92eeb783bf76..5f06f240b25f 100644 --- a/crates/polars-plan/src/plans/mod.rs +++ b/crates/polars-plan/src/plans/mod.rs @@ -2,7 +2,6 @@ use std::fmt; use std::fmt::Debug; use std::sync::{Arc, Mutex, RwLock}; -use hive::HivePartitions; use polars_core::prelude::*; use recursive::recursive; @@ -89,7 +88,6 @@ pub enum DslPlan { // are used as base of different queries in a loop. That way // the expensive schema resolving is cached. file_info: Arc>>, - hive_parts: Option>>, predicate: Option, file_options: FileScanOptions, scan_type: FileScan, @@ -199,7 +197,7 @@ impl Clone for DslPlan { Self::PythonScan { options } => Self::PythonScan { options: options.clone() }, Self::Filter { input, predicate } => Self::Filter { input: input.clone(), predicate: predicate.clone() }, Self::Cache { input, id, cache_hits } => Self::Cache { input: input.clone(), id: id.clone(), cache_hits: cache_hits.clone() }, - Self::Scan { sources, file_info, hive_parts, predicate, file_options, scan_type } => Self::Scan { sources: sources.clone(), file_info: file_info.clone(), hive_parts: hive_parts.clone(), predicate: predicate.clone(), file_options: file_options.clone(), scan_type: scan_type.clone() }, + Self::Scan { sources, file_info, predicate, file_options, scan_type } => Self::Scan { sources: sources.clone(), file_info: file_info.clone(), predicate: predicate.clone(), file_options: file_options.clone(), scan_type: scan_type.clone() }, Self::DataFrameScan { df, schema, output_schema, filter: selection } => Self::DataFrameScan { df: df.clone(), schema: schema.clone(), output_schema: output_schema.clone(), filter: selection.clone() }, Self::Select { expr, input, options } => Self::Select { expr: expr.clone(), input: input.clone(), options: options.clone() }, Self::GroupBy { input, keys, aggs, apply, maintain_order, options } => Self::GroupBy { input: input.clone(), keys: keys.clone(), aggs: aggs.clone(), apply: apply.clone(), maintain_order: maintain_order.clone(), options: options.clone() },