File tree 1 file changed +16
-4
lines changed
datafusion/core/src/datasource/listing
1 file changed +16
-4
lines changed Original file line number Diff line number Diff line change @@ -139,10 +139,22 @@ pub fn split_files(
139
139
140
140
// effectively this is div with rounding up instead of truncating
141
141
let chunk_size = ( partitioned_files. len ( ) + n - 1 ) / n;
142
- partitioned_files
143
- . chunks_mut ( chunk_size)
144
- . map ( |c| c. iter_mut ( ) . map ( mem:: take) . collect ( ) )
145
- . collect ( )
142
+ let mut chunks = Vec :: with_capacity ( n) ;
143
+ let mut current_chunk = Vec :: with_capacity ( chunk_size) ;
144
+ for file in partitioned_files. drain ( ..) {
145
+ current_chunk. push ( file) ;
146
+ if current_chunk. len ( ) == chunk_size {
147
+ let full_chunk =
148
+ mem:: replace ( & mut current_chunk, Vec :: with_capacity ( chunk_size) ) ;
149
+ chunks. push ( full_chunk) ;
150
+ }
151
+ }
152
+
153
+ if !current_chunk. is_empty ( ) {
154
+ chunks. push ( current_chunk)
155
+ }
156
+
157
+ chunks
146
158
}
147
159
148
160
struct Partition {
You can’t perform that action at this time.
0 commit comments