Skip to content

Commit ec0fb31

Browse files
committed
PERF : modify SMJ shuffle file reader to skip validation
1 parent 6ba4152 commit ec0fb31

File tree

2 files changed

+2
-2
lines changed

2 files changed

+2
-2
lines changed

datafusion/physical-plan/src/joins/sort_merge_join.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -2313,7 +2313,7 @@ fn fetch_right_columns_from_batch_by_idxs(
23132313
Vec::with_capacity(buffered_indices.len());
23142314

23152315
let file = BufReader::new(File::open(spill_file.path())?);
2316-
let reader = StreamReader::try_new(file, None)?;
2316+
let reader = unsafe {StreamReader::try_new(file, None)?.with_skip_validation(true)};
23172317

23182318
for batch in reader {
23192319
batch?.columns().iter().for_each(|column| {

parquet-testing

Submodule parquet-testing updated 67 files

0 commit comments

Comments
 (0)