move notes

Author: xudong963

datafusion/datasource/benches/split_groups_by_statistics.rs

@@ -41,34 +41,6 @@ pub fn compare_split_groups_by_statistics_algorithms(c: &mut Criterion) {
 
     // Small, medium, large number of files
     let file_counts = [10, 100, 1000];
-    // overlap_factors controls how much the value ranges in generated test files overlap:
-    // - 0.0: No overlap between files (completely disjoint ranges)
-    // - 0.2: Low overlap (20% of the range size overlaps with adjacent files)
-    // - 0.5: Medium overlap (50% of ranges overlap)
-    // - 0.8: High overlap (80% of ranges overlap between files)
-    //
-    // Example with 5 files and different overlap factors:
-    // [min, max]
-    // overlap_factor = 0.0 (no overlap):
-    //   File 0: [0, 20]
-    //   File 1: [20, 40]
-    //   File 2: [40, 60]
-    //   File 3: [60, 80]
-    //   File 4: [80, 100]
-    //
-    // overlap_factor = 0.5 (50% overlap):
-    //   File 0: [0, 40]
-    //   File 1: [20, 60]
-    //   File 2: [40, 80]
-    //   File 3: [60, 100]
-    //   File 4: [80, 120]
-    //
-    // overlap_factor = 0.8 (80% overlap):
-    //   File 0: [0, 100]
-    //   File 1: [20, 120]
-    //   File 2: [40, 140]
-    //   File 3: [60, 160]
-    //   File 4: [80, 180]
     let overlap_factors = [0.0, 0.2, 0.5, 0.8]; // No, low, medium, high overlap
 
     let target_partitions: [usize; 4] = [4, 8, 16, 32];

datafusion/datasource/src/mod.rs

@@ -314,8 +314,45 @@ async fn find_first_newline(
     Ok(index)
 }
 
-/// Generates test files with min-max statistics in different overlap patterns
-/// Used by tests and benchmarks
+/// Generates test files with min-max statistics in different overlap patterns.
+///
+/// Used by tests and benchmarks.
+///
+/// # Overlap Factors
+///
+/// The `overlap_factor` parameter controls how much the value ranges in generated test files overlap:
+/// - `0.0`: No overlap between files (completely disjoint ranges)
+/// - `0.2`: Low overlap (20% of the range size overlaps with adjacent files)
+/// - `0.5`: Medium overlap (50% of ranges overlap)
+/// - `0.8`: High overlap (80% of ranges overlap between files)
+///
+/// # Examples
+///
+/// With 5 files and different overlap factors showing `[min, max]` ranges:
+///
+/// overlap_factor = 0.0 (no overlap):
+///
+/// File 0: [0, 20]
+/// File 1: [20, 40]
+/// File 2: [40, 60]
+/// File 3: [60, 80]
+/// File 4: [80, 100]
+///
+/// overlap_factor = 0.5 (50% overlap):
+///
+/// File 0: [0, 40]
+/// File 1: [20, 60]
+/// File 2: [40, 80]
+/// File 3: [60, 100]
+/// File 4: [80, 120]
+///
+/// overlap_factor = 0.8 (80% overlap):
+///
+/// File 0: [0, 100]
+/// File 1: [20, 120]
+/// File 2: [40, 140]
+/// File 3: [60, 160]
+/// File 4: [80, 180]
 pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGroup> {
     let mut files = Vec::with_capacity(num_files);
     if num_files == 0 {