Comments

Author: alamb

datafusion/core/src/datasource/physical_plan/parquet/page_filter.rs

@@ -109,9 +109,15 @@ use super::metrics::ParquetFileMetrics;
 /// can not contain rows that match the predicate.
 ///
 /// # Implementation notes
+///
+/// Single column predicates are evaluated using the PageIndex information
+/// for that column to determine which row ranges can be skipped based.
+///
+/// The resulting [`RowSelection`]'s are combined into a final
+/// row selection that is added to the [`ParquetAccessPlan`].
 #[derive(Debug)]
 pub struct PagePruningAccessPlanFilter {
-    /// single column predicates (e.g. (`col = 5`) extracted from the  overall
+    /// single column predicates (e.g. (`col = 5`) extracted from the overall
     /// predicate. Must all be true for a row to be included in the result.
     predicates: Vec<PruningPredicate>,
 }
@@ -175,7 +181,7 @@ impl PagePruningAccessPlanFilter {
             (file_metadata.offset_index(), file_metadata.column_index())
         else {
             trace!(
-                    "skip page pruning due to lack of indexes. Have offset: {}, column index: {}",
+                    "Can not prune Parquet pages due to lack of indexes. Have offset: {}, column index: {}",
                     file_metadata.offset_index().is_some(), file_metadata.column_index().is_some()
                 );
             return access_plan;
@@ -184,11 +190,13 @@ impl PagePruningAccessPlanFilter {
         // track the total number of rows that should be skipped
         let mut total_skip = 0;
 
+        // for each row group specified in the access plan
         let row_group_indexes = access_plan.row_group_indexes();
         for r in row_group_indexes {
             // The selection for this particular row group
             let mut overall_selection = None;
             for predicate in page_index_predicates {
+
                 // find column index in the parquet schema
                 let col_idx = find_column_index(predicate, arrow_schema, parquet_schema);
                 let row_group_metadata = &groups[r];