Skip to content

Commit 9a084a2

Browse files
korowawiedld
authored andcommitted
minor: split repartition time and send time metrics (apache#11440)
1 parent 98258ae commit 9a084a2

File tree

1 file changed

+9
-3
lines changed
  • datafusion/physical-plan/src/repartition

1 file changed

+9
-3
lines changed

datafusion/physical-plan/src/repartition/mod.rs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@ impl BatchPartitioner {
261261
num_partitions: partitions,
262262
hash_buffer,
263263
} => {
264+
// Tracking time required for distributing indexes across output partitions
264265
let timer = self.timer.timer();
265266

266267
let arrays = exprs
@@ -282,6 +283,11 @@ impl BatchPartitioner {
282283
.append_value(index as u64);
283284
}
284285

286+
// Finished building index-arrays for output partitions
287+
timer.done();
288+
289+
// Borrowing partitioner timer to prevent moving `self` to closure
290+
let partitioner_timer = &self.timer;
285291
let it = indices
286292
.into_iter()
287293
.enumerate()
@@ -290,6 +296,9 @@ impl BatchPartitioner {
290296
(!indices.is_empty()).then_some((partition, indices))
291297
})
292298
.map(move |(partition, indices)| {
299+
// Tracking time required for repartitioned batches construction
300+
let _timer = partitioner_timer.timer();
301+
293302
// Produce batches based on indices
294303
let columns = batch
295304
.columns()
@@ -303,9 +312,6 @@ impl BatchPartitioner {
303312
let batch =
304313
RecordBatch::try_new(batch.schema(), columns).unwrap();
305314

306-
// bind timer so it drops w/ this iterator
307-
let _ = &timer;
308-
309315
Ok((partition, batch))
310316
});
311317

0 commit comments

Comments
 (0)