Skip to content

Commit b0a921d

Browse files
committed
Fix allocation failures during sorting with spill-to-disk
This change replaces `try_resize` with `resize` in three sites, allowing memory to overshoot the configured pool size. These are sites where we don't fall back to spilling to disk when the allocation fails. Fixes: apache#12136
1 parent 8fd129f commit b0a921d

File tree

2 files changed

+4
-2
lines changed

2 files changed

+4
-2
lines changed

datafusion/physical-plan/src/sorts/builder.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ impl BatchBuilder {
6969

7070
/// Append a new batch in `stream_idx`
7171
pub fn push_batch(&mut self, stream_idx: usize, batch: RecordBatch) -> Result<()> {
72-
self.reservation.try_grow(batch.get_array_memory_size())?;
72+
// Allow memory to exceed the limit
73+
self.reservation.grow(batch.get_array_memory_size());
7374
let batch_idx = self.batches.len();
7475
self.batches.push((stream_idx, batch));
7576
self.cursors[stream_idx] = BatchCursor {

datafusion/physical-plan/src/sorts/sort.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -580,7 +580,8 @@ impl ExternalSorter {
580580
if self.runtime.disk_manager.tmp_files_enabled() {
581581
let size = self.sort_spill_reservation_bytes;
582582
if self.merge_reservation.size() != size {
583-
self.merge_reservation.try_resize(size)?;
583+
// Allow memory to exceed the limit
584+
self.merge_reservation.resize(size);
584585
}
585586
}
586587

0 commit comments

Comments
 (0)