@@ -76,7 +76,84 @@ use crate::datasource::schema_adapter::{
7676pub use metrics:: ParquetFileMetrics ;
7777pub use statistics:: { RequestedStatistics , StatisticsConverter } ;
7878
79- /// Execution plan for scanning one or more Parquet partitions
79+ /// Execution plan for reading one or more Parquet files.
80+ ///
81+ /// ```text
82+ /// ▲
83+ /// │
84+ /// │ Produce a stream of
85+ /// │ RecordBatches
86+ /// │
87+ /// ┌───────────────────────┐
88+ /// │ │
89+ /// │ ParquetExec │
90+ /// │ │
91+ /// └───────────────────────┘
92+ /// ▲
93+ /// │ Asynchronously read from one
94+ /// │ or more parquet files via
95+ /// │ ObjectStore interface
96+ /// │
97+ /// │
98+ /// .───────────────────.
99+ /// │ )
100+ /// │`───────────────────'│
101+ /// │ ObjectStore │
102+ /// │.───────────────────.│
103+ /// │ )
104+ /// `───────────────────'
105+ ///
106+ /// ```
107+ /// # Features
108+ ///
109+ /// Supports the following optimizations:
110+ ///
111+ /// * Concurrent reads: Can read from one or more files in parallel as multiple
112+ /// partitions, including concurrently reading multiple row groups from a single
113+ /// file.
114+ ///
115+ /// * Predicate push down: skips row groups and pages based on
116+ /// min/max/null_counts in the row group metadata, the page index and bloom
117+ /// filters.
118+ ///
119+ /// * Projection pushdown: reads and decodes only the columns required.
120+ ///
121+ /// * Limit pushdown: stop execution early after some number of rows are read.
122+ ///
123+ /// * Custom readers: customize reading parquet files, e.g. to cache metadata,
124+ /// coalesce I/O operations, etc. See [`ParquetFileReaderFactory`] for more
125+ /// details.
126+ ///
127+ /// * Schema adapters: read parquet files with different schemas into a unified
128+ /// table schema. This can be used to implement "schema evolution". See
129+ /// [`SchemaAdapterFactory`] for more details.
130+ ///
131+ /// * metadata_size_hint: controls the number of bytes read from the end of the
132+ /// file in the initial I/O when the default [`ParquetFileReaderFactory`]. If a
133+ /// custom reader is used, it supplies the metadata directly and this parameter
134+ /// is ignored. See [`Self::with_parquet_file_reader_factory`] for more details.
135+ ///
136+ /// # Execution Overview
137+ ///
138+ /// * Step 1: [`ParquetExec::execute`] is called, returning a [`FileStream`]
139+ /// configured to open parquet files with a [`ParquetOpener`].
140+ ///
141+ /// * Step 2: When the stream is polled, the [`ParquetOpener`] is called to open
142+ /// the file.
143+ ///
144+ /// * Step 3: The `ParquetOpener` gets the file metadata via
145+ /// [`ParquetFileReaderFactory`] and applies any predicates
146+ /// and projections to determine what pages must be read.
147+ ///
148+ /// * Step 4: The stream begins reading data, fetching the required pages
149+ /// and incrementally decoding them.
150+ ///
151+ /// * Step 5: As each [`RecordBatch]` is read, it may be adapted by a
152+ /// [`SchemaAdapter`] to match the table schema. By default missing columns are
153+ /// filled with nulls, but this can be customized via [`SchemaAdapterFactory`].
154+ ///
155+ /// [`RecordBatch`]: arrow::record_batch::RecordBatch
156+ /// [`SchemaAdapter`]: crate::datasource::schema_adapter::SchemaAdapter
80157#[ derive( Debug , Clone ) ]
81158pub struct ParquetExec {
82159 /// Base configuration for this scan
@@ -86,9 +163,9 @@ pub struct ParquetExec {
86163metrics : ExecutionPlanMetricsSet ,
87164 /// Optional predicate for row filtering during parquet scan
88165predicate : Option < Arc < dyn PhysicalExpr > > ,
89- /// Optional predicate for pruning row groups
166+ /// Optional predicate for pruning row groups (derived from `predicate`)
90167pruning_predicate : Option < Arc < PruningPredicate > > ,
91- /// Optional predicate for pruning pages
168+ /// Optional predicate for pruning pages (derived from `predicate`)
92169page_pruning_predicate : Option < Arc < PagePruningPredicate > > ,
93170 /// Optional hint for the size of the parquet metadata
94171metadata_size_hint : Option < usize > ,
@@ -190,11 +267,13 @@ impl ParquetExec {
190267
191268 /// Optional user defined parquet file reader factory.
192269///
193- /// `ParquetFileReaderFactory` complements `TableProvider`, It enables users to provide custom
194- /// implementation for data access operations.
270+ /// You can use [`ParquetFileReaderFactory`] to more precisely control how
271+ /// data is read from parquet files (e.g. skip re-reading metadata, coalesce
272+ /// I/O operations, etc).
195273///
196- /// If custom `ParquetFileReaderFactory` is provided, then data access operations will be routed
197- /// to this factory instead of `ObjectStore`.
274+ /// The default reader factory reads directly from an [`ObjectStore`]
275+ /// instance using individual I/O operations for the footer and then for
276+ /// each page.
198277pub fn with_parquet_file_reader_factory (
199278 mut self ,
200279 parquet_file_reader_factory : Arc < dyn ParquetFileReaderFactory > ,
@@ -643,11 +722,21 @@ fn should_enable_page_index(
643722 . unwrap_or ( false )
644723}
645724
646- /// Factory of parquet file readers .
725+ /// Interface for reading parquet files .
647726///
648- /// Provides means to implement custom data access interface.
727+ /// The combined implementations of [`ParquetFileReaderFactory`] and
728+ /// [`AsyncFileReader`] can be used to provide custom data access operations
729+ /// such as pre-cached data, I/O coalescing, etc.
730+ ///
731+ /// See [`DefaultParquetFileReaderFactory`] for a simple implementation.
649732pub trait ParquetFileReaderFactory : Debug + Send + Sync + ' static {
650- /// Provides `AsyncFileReader` over parquet file specified in `FileMeta`
733+ /// Provides an `AsyncFileReader` for reading data from a parquet file specified
734+ ///
735+ /// # Arguments
736+ /// * partition_index - Index of the partition (for reporting metrics)
737+ /// * file_meta - The file to be read
738+ /// * metadata_size_hint - If specified, the first IO reads this many bytes from the footer
739+ /// * metrics - Execution metrics
651740fn create_reader (
652741 & self ,
653742 partition_index : usize ,
@@ -657,20 +746,32 @@ pub trait ParquetFileReaderFactory: Debug + Send + Sync + 'static {
657746 ) -> Result < Box < dyn AsyncFileReader + Send > > ;
658747}
659748
660- /// Default parquet reader factory.
749+ /// Default implementation of [`ParquetFileReaderFactory`]
750+ ///
751+ /// This implementation:
752+ /// 1. Reads parquet directly from an underlying [`ObjectStore`] instance.
753+ /// 2. Reads the footer and page metadata on demand.
754+ /// 3. Does not cache metadata or coalesce I/O operations.
661755#[ derive( Debug ) ]
662756pub struct DefaultParquetFileReaderFactory {
663757 store : Arc < dyn ObjectStore > ,
664758}
665759
666760impl DefaultParquetFileReaderFactory {
667- /// Create a factory .
761+ /// Create a new `DefaultParquetFileReaderFactory` .
668762pub fn new ( store : Arc < dyn ObjectStore > ) -> Self {
669763 Self { store }
670764 }
671765}
672766
673- /// Implements [`AsyncFileReader`] for a parquet file in object storage
767+ /// Implements [`AsyncFileReader`] for a parquet file in object storage.
768+ ///
769+ /// This implementation uses the [`ParquetObjectReader`] to read data from the
770+ /// object store on demand, as required, tracking the number of bytes read.
771+ ///
772+ /// This implementation does not coalesce I/O operations or cache bytes. Such
773+ /// optimizations can be done either at the object store level or by providing a
774+ /// custom implementation of [`ParquetFileReaderFactory`].
674775pub ( crate ) struct ParquetFileReader {
675776 file_metrics : ParquetFileMetrics ,
676777 inner : ParquetObjectReader ,
0 commit comments