Skip to content

Commit 920a944

Browse files
XiangpengHaoXiangpengHao
authored
Move avoid using copy-based buffer creation (#6039)
1 parent e70c16d commit 920a944

File tree

25 files changed

+82
-79
lines changed

25 files changed

+82
-79
lines changed

‎arrow-array/src/array/dictionary_array.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,13 +1025,13 @@ mod tests {
10251025
let value_data = ArrayData::builder(DataType::Int8)
10261026
.len(8)
10271027
.add_buffer(Buffer::from(
1028-
&[10_i8, 11, 12, 13, 14, 15, 16, 17].to_byte_slice(),
1028+
[10_i8, 11, 12, 13, 14, 15, 16, 17].to_byte_slice(),
10291029
))
10301030
.build()
10311031
.unwrap();
10321032

10331033
// Construct a buffer for value offsets, for the nested array:
1034-
let keys = Buffer::from(&[2_i16, 3, 4].to_byte_slice());
1034+
let keys = Buffer::from([2_i16, 3, 4].to_byte_slice());
10351035

10361036
// Construct a dictionary array from the above two
10371037
let key_type = DataType::Int16;

‎arrow-array/src/array/fixed_size_list_array.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -674,7 +674,7 @@ mod tests {
674674
assert_eq!(err.to_string(), "Invalid argument error: Found unmasked nulls for non-nullable FixedSizeListArray field \"item\"");
675675

676676
// Valid as nulls in child masked by parent
677-
let nulls = NullBuffer::new(BooleanBuffer::new(vec![0b0000101].into(), 0, 3));
677+
let nulls = NullBuffer::new(BooleanBuffer::new(Buffer::from([0b0000101]), 0, 3));
678678
FixedSizeListArray::new(field, 2, values.clone(), Some(nulls));
679679

680680
let field = Arc::new(Field::new("item", DataType::Int64, true));

‎arrow-array/src/array/map_array.rs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -448,20 +448,20 @@ mod tests {
448448
// Construct key and values
449449
let keys_data = ArrayData::builder(DataType::Int32)
450450
.len(8)
451-
.add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
451+
.add_buffer(Buffer::from([0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
452452
.build()
453453
.unwrap();
454454
let values_data = ArrayData::builder(DataType::UInt32)
455455
.len(8)
456456
.add_buffer(Buffer::from(
457-
&[0u32, 10, 20, 30, 40, 50, 60, 70].to_byte_slice(),
457+
[0u32, 10, 20, 30, 40, 50, 60, 70].to_byte_slice(),
458458
))
459459
.build()
460460
.unwrap();
461461

462462
// Construct a buffer for value offsets, for the nested array:
463463
// [[0, 1, 2], [3, 4, 5], [6, 7]]
464-
let entry_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
464+
let entry_offsets = Buffer::from([0, 3, 6, 8].to_byte_slice());
465465

466466
let keys = Arc::new(Field::new("keys", DataType::Int32, false));
467467
let values = Arc::new(Field::new("values", DataType::UInt32, false));
@@ -493,21 +493,21 @@ mod tests {
493493
// Construct key and values
494494
let key_data = ArrayData::builder(DataType::Int32)
495495
.len(8)
496-
.add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
496+
.add_buffer(Buffer::from([0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
497497
.build()
498498
.unwrap();
499499
let value_data = ArrayData::builder(DataType::UInt32)
500500
.len(8)
501501
.add_buffer(Buffer::from(
502-
&[0u32, 10, 20, 0, 40, 0, 60, 70].to_byte_slice(),
502+
[0u32, 10, 20, 0, 40, 0, 60, 70].to_byte_slice(),
503503
))
504504
.null_bit_buffer(Some(Buffer::from(&[0b11010110])))
505505
.build()
506506
.unwrap();
507507

508508
// Construct a buffer for value offsets, for the nested array:
509509
// [[0, 1, 2], [3, 4, 5], [6, 7]]
510-
let entry_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
510+
let entry_offsets = Buffer::from([0, 3, 6, 8].to_byte_slice());
511511

512512
let keys_field = Arc::new(Field::new("keys", DataType::Int32, false));
513513
let values_field = Arc::new(Field::new("values", DataType::UInt32, true));
@@ -617,18 +617,18 @@ mod tests {
617617
// Construct key and values
618618
let keys_data = ArrayData::builder(DataType::Int32)
619619
.len(5)
620-
.add_buffer(Buffer::from(&[3, 4, 5, 6, 7].to_byte_slice()))
620+
.add_buffer(Buffer::from([3, 4, 5, 6, 7].to_byte_slice()))
621621
.build()
622622
.unwrap();
623623
let values_data = ArrayData::builder(DataType::UInt32)
624624
.len(5)
625-
.add_buffer(Buffer::from(&[30u32, 40, 50, 60, 70].to_byte_slice()))
625+
.add_buffer(Buffer::from([30u32, 40, 50, 60, 70].to_byte_slice()))
626626
.build()
627627
.unwrap();
628628

629629
// Construct a buffer for value offsets, for the nested array:
630630
// [[3, 4, 5], [6, 7]]
631-
let entry_offsets = Buffer::from(&[0, 3, 5].to_byte_slice());
631+
let entry_offsets = Buffer::from([0, 3, 5].to_byte_slice());
632632

633633
let keys = Arc::new(Field::new("keys", DataType::Int32, false));
634634
let values = Arc::new(Field::new("values", DataType::UInt32, false));

‎arrow-array/src/array/struct_array.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -549,15 +549,15 @@ mod tests {
549549
let expected_string_data = ArrayData::builder(DataType::Utf8)
550550
.len(4)
551551
.null_bit_buffer(Some(Buffer::from(&[9_u8])))
552-
.add_buffer(Buffer::from(&[0, 3, 3, 3, 7].to_byte_slice()))
552+
.add_buffer(Buffer::from([0, 3, 3, 3, 7].to_byte_slice()))
553553
.add_buffer(Buffer::from(b"joemark"))
554554
.build()
555555
.unwrap();
556556

557557
let expected_int_data = ArrayData::builder(DataType::Int32)
558558
.len(4)
559559
.null_bit_buffer(Some(Buffer::from(&[11_u8])))
560-
.add_buffer(Buffer::from(&[1, 2, 0, 4].to_byte_slice()))
560+
.add_buffer(Buffer::from([1, 2, 0, 4].to_byte_slice()))
561561
.build()
562562
.unwrap();
563563

‎arrow-buffer/src/buffer/immutable.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -543,7 +543,7 @@ mod tests {
543543

544544
#[test]
545545
fn test_access_concurrently() {
546-
let buffer = Buffer::from(vec![1, 2, 3, 4, 5]);
546+
let buffer = Buffer::from([1, 2, 3, 4, 5]);
547547
let buffer2 = buffer.clone();
548548
assert_eq!([1, 2, 3, 4, 5], buffer.as_slice());
549549

‎arrow-buffer/src/util/bit_chunk_iterator.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,7 @@ mod tests {
456456
const ALLOC_SIZE: usize = 4 * 1024;
457457
let input = vec![0xFF_u8; ALLOC_SIZE];
458458

459-
let buffer: Buffer = Buffer::from(input);
459+
let buffer: Buffer = Buffer::from_vec(input);
460460

461461
let bitchunks = buffer.bit_chunks(57, ALLOC_SIZE * 8 - 57);
462462

‎arrow-cast/src/base64.rs

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
//! [`StringArray`]: arrow_array::StringArray
2121
2222
use arrow_array::{Array, GenericBinaryArray, GenericStringArray, OffsetSizeTrait};
23-
use arrow_buffer::OffsetBuffer;
23+
use arrow_buffer::{Buffer, OffsetBuffer};
2424
use arrow_schema::ArrowError;
2525
use base64::encoded_len;
2626
use base64::engine::Config;
@@ -50,7 +50,9 @@ pub fn b64_encode<E: Engine, O: OffsetSizeTrait>(
5050
assert_eq!(offset, buffer_len);
5151

5252
// Safety: Base64 is valid UTF-8
53-
unsafe { GenericStringArray::new_unchecked(offsets, buffer.into(), array.nulls().cloned()) }
53+
unsafe {
54+
GenericStringArray::new_unchecked(offsets, Buffer::from_vec(buffer), array.nulls().cloned())
55+
}
5456
}
5557

5658
/// Base64 decode each element of `array` with the provided [`Engine`]
@@ -79,7 +81,7 @@ pub fn b64_decode<E: Engine, O: OffsetSizeTrait>(
7981

8082
Ok(GenericBinaryArray::new(
8183
offsets,
82-
buffer.into(),
84+
Buffer::from_vec(buffer),
8385
array.nulls().cloned(),
8486
))
8587
}

‎arrow-data/src/data.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1959,7 +1959,7 @@ mod tests {
19591959
.len(20)
19601960
.offset(5)
19611961
.add_buffer(b1)
1962-
.null_bit_buffer(Some(Buffer::from(vec![
1962+
.null_bit_buffer(Some(Buffer::from([
19631963
0b01011111, 0b10110101, 0b01100011, 0b00011110,
19641964
])))
19651965
.build()
@@ -2164,7 +2164,7 @@ mod tests {
21642164

21652165
#[test]
21662166
fn test_count_nulls() {
2167-
let buffer = Buffer::from(vec![0b00010110, 0b10011111]);
2167+
let buffer = Buffer::from([0b00010110, 0b10011111]);
21682168
let buffer = NullBuffer::new(BooleanBuffer::new(buffer, 0, 16));
21692169
let count = count_nulls(Some(&buffer), 0, 16);
21702170
assert_eq!(count, 7);

‎arrow-flight/src/utils.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ pub fn flight_data_to_arrow_batch(
9494
})
9595
.map(|batch| {
9696
reader::read_record_batch(
97-
&Buffer::from(&data.data_body),
97+
&Buffer::from(data.data_body.as_ref()),
9898
batch,
9999
schema,
100100
dictionaries_by_id,

‎arrow-integration-test/src/lib.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -696,7 +696,7 @@ pub fn array_from_json(
696696
let list_data = ArrayData::builder(field.data_type().clone())
697697
.len(json_col.count)
698698
.offset(0)
699-
.add_buffer(Buffer::from(&offsets.to_byte_slice()))
699+
.add_buffer(Buffer::from(offsets.to_byte_slice()))
700700
.add_child_data(child_array.into_data())
701701
.null_bit_buffer(Some(null_buf))
702702
.build()
@@ -720,7 +720,7 @@ pub fn array_from_json(
720720
let list_data = ArrayData::builder(field.data_type().clone())
721721
.len(json_col.count)
722722
.offset(0)
723-
.add_buffer(Buffer::from(&offsets.to_byte_slice()))
723+
.add_buffer(Buffer::from(offsets.to_byte_slice()))
724724
.add_child_data(child_array.into_data())
725725
.null_bit_buffer(Some(null_buf))
726726
.build()
@@ -839,7 +839,7 @@ pub fn array_from_json(
839839
.collect();
840840
let array_data = ArrayData::builder(field.data_type().clone())
841841
.len(json_col.count)
842-
.add_buffer(Buffer::from(&offsets.to_byte_slice()))
842+
.add_buffer(Buffer::from(offsets.to_byte_slice()))
843843
.add_child_data(child_array.into_data())
844844
.null_bit_buffer(Some(null_buf))
845845
.build()

‎arrow-integration-testing/src/flight_client_scenarios/integration_test.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ async fn receive_batch_flight_data(
262262

263263
while message.header_type() == ipc::MessageHeader::DictionaryBatch {
264264
reader::read_dictionary(
265-
&Buffer::from(&data.data_body),
265+
&Buffer::from(data.data_body.as_ref()),
266266
message
267267
.header_as_dictionary_batch()
268268
.expect("Error parsing dictionary"),

‎arrow-integration-testing/src/flight_server_scenarios/integration_test.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ async fn save_uploaded_chunks(
364364

365365
let batch = record_batch_from_message(
366366
message,
367-
&Buffer::from(data.data_body),
367+
&Buffer::from(data.data_body.as_ref()),
368368
schema_ref.clone(),
369369
&dictionaries_by_id,
370370
)
@@ -375,7 +375,7 @@ async fn save_uploaded_chunks(
375375
ipc::MessageHeader::DictionaryBatch => {
376376
dictionary_from_message(
377377
message,
378-
&Buffer::from(data.data_body),
378+
&Buffer::from(data.data_body.as_ref()),
379379
schema_ref.clone(),
380380
&mut dictionaries_by_id,
381381
)

‎arrow-ipc/src/compression.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,8 +103,8 @@ impl CompressionCodec {
103103
} else if let Ok(decompressed_length) = usize::try_from(decompressed_length) {
104104
// decompress data using the codec
105105
let input_data = &input[(LENGTH_OF_PREFIX_DATA as usize)..];
106-
self.decompress(input_data, decompressed_length as _)?
107-
.into()
106+
let v = self.decompress(input_data, decompressed_length as _)?;
107+
Buffer::from_vec(v)
108108
} else {
109109
return Err(ArrowError::IpcError(format!(
110110
"Invalid uncompressed length: {decompressed_length}"

‎arrow-json/src/reader/mod.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1850,7 +1850,7 @@ mod tests {
18501850
let c = ArrayDataBuilder::new(c_field.data_type().clone())
18511851
.len(7)
18521852
.add_child_data(d.to_data())
1853-
.null_bit_buffer(Some(Buffer::from(vec![0b00111011])))
1853+
.null_bit_buffer(Some(Buffer::from([0b00111011])))
18541854
.build()
18551855
.unwrap();
18561856
let b = BooleanArray::from(vec![
@@ -1866,14 +1866,14 @@ mod tests {
18661866
.len(7)
18671867
.add_child_data(b.to_data())
18681868
.add_child_data(c.clone())
1869-
.null_bit_buffer(Some(Buffer::from(vec![0b00111111])))
1869+
.null_bit_buffer(Some(Buffer::from([0b00111111])))
18701870
.build()
18711871
.unwrap();
18721872
let a_list = ArrayDataBuilder::new(a_field.data_type().clone())
18731873
.len(6)
18741874
.add_buffer(Buffer::from_slice_ref([0i32, 2, 3, 6, 6, 6, 7]))
18751875
.add_child_data(a)
1876-
.null_bit_buffer(Some(Buffer::from(vec![0b00110111])))
1876+
.null_bit_buffer(Some(Buffer::from([0b00110111])))
18771877
.build()
18781878
.unwrap();
18791879
let expected = make_array(a_list);

‎arrow-json/src/writer.rs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -927,12 +927,12 @@ mod tests {
927927

928928
let a_values = StringArray::from(vec!["a", "a1", "b", "c", "d", "e"]);
929929
// list column rows: ["a", "a1"], ["b"], ["c"], ["d"], ["e"]
930-
let a_value_offsets = Buffer::from(&[0, 2, 3, 4, 5, 6].to_byte_slice());
930+
let a_value_offsets = Buffer::from([0, 2, 3, 4, 5, 6].to_byte_slice());
931931
let a_list_data = ArrayData::builder(field_c1.data_type().clone())
932932
.len(5)
933933
.add_buffer(a_value_offsets)
934934
.add_child_data(a_values.into_data())
935-
.null_bit_buffer(Some(Buffer::from(vec![0b00011111])))
935+
.null_bit_buffer(Some(Buffer::from([0b00011111])))
936936
.build()
937937
.unwrap();
938938
let a = ListArray::from(a_list_data);
@@ -976,17 +976,17 @@ mod tests {
976976
// list column rows: [[1, 2], [3]], [], [[4, 5, 6]]
977977
let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
978978

979-
let a_value_offsets = Buffer::from(&[0, 2, 3, 6].to_byte_slice());
979+
let a_value_offsets = Buffer::from([0, 2, 3, 6].to_byte_slice());
980980
// Construct a list array from the above two
981981
let a_list_data = ArrayData::builder(list_inner_type.data_type().clone())
982982
.len(3)
983983
.add_buffer(a_value_offsets)
984-
.null_bit_buffer(Some(Buffer::from(vec![0b00000111])))
984+
.null_bit_buffer(Some(Buffer::from([0b00000111])))
985985
.add_child_data(a_values.into_data())
986986
.build()
987987
.unwrap();
988988

989-
let c1_value_offsets = Buffer::from(&[0, 2, 2, 3].to_byte_slice());
989+
let c1_value_offsets = Buffer::from([0, 2, 2, 3].to_byte_slice());
990990
let c1_list_data = ArrayData::builder(field_c1.data_type().clone())
991991
.len(3)
992992
.add_buffer(c1_value_offsets)
@@ -1058,12 +1058,12 @@ mod tests {
10581058
// [{"c11": 1, "c12": {"c121": "e"}}, {"c12": {"c121": "f"}}],
10591059
// null,
10601060
// [{"c11": 5, "c12": {"c121": "g"}}]
1061-
let c1_value_offsets = Buffer::from(&[0, 2, 2, 3].to_byte_slice());
1061+
let c1_value_offsets = Buffer::from([0, 2, 2, 3].to_byte_slice());
10621062
let c1_list_data = ArrayData::builder(field_c1.data_type().clone())
10631063
.len(3)
10641064
.add_buffer(c1_value_offsets)
10651065
.add_child_data(struct_values.into_data())
1066-
.null_bit_buffer(Some(Buffer::from(vec![0b00000101])))
1066+
.null_bit_buffer(Some(Buffer::from([0b00000101])))
10671067
.build()
10681068
.unwrap();
10691069
let c1 = ListArray::from(c1_list_data);
@@ -1225,7 +1225,7 @@ mod tests {
12251225
);
12261226

12271227
// [{"foo": 10}, null, {}, {"bar": 20, "baz": 30, "qux": 40}, {"quux": 50}, {}]
1228-
let entry_offsets = Buffer::from(&[0, 1, 1, 1, 4, 5, 5].to_byte_slice());
1228+
let entry_offsets = Buffer::from([0, 1, 1, 1, 4, 5, 5].to_byte_slice());
12291229
let valid_buffer = Buffer::from([0b00111101]);
12301230

12311231
let map_data = ArrayData::builder(map_data_type.clone())
@@ -1408,7 +1408,7 @@ mod tests {
14081408
);
14091409

14101410
// [{"list":[{"int32":1,"utf8":"a"},{"int32":null,"utf8":"b"}]},{"list":null},{"list":[{int32":5,"utf8":null}]},{"list":null}]
1411-
let entry_offsets = Buffer::from(&[0, 2, 2, 3, 3].to_byte_slice());
1411+
let entry_offsets = Buffer::from([0, 2, 2, 3, 3].to_byte_slice());
14121412
let data = ArrayData::builder(field.data_type().clone())
14131413
.len(4)
14141414
.add_buffer(entry_offsets)

‎arrow-select/src/dictionary.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,7 @@ mod tests {
297297

298298
#[test]
299299
fn test_merge_nulls() {
300-
let buffer = Buffer::from("helloworldbingohelloworld");
300+
let buffer = Buffer::from(b"helloworldbingohelloworld");
301301
let offsets = OffsetBuffer::from_lengths([5, 5, 5, 5, 5]);
302302
let nulls = NullBuffer::from(vec![true, false, true, true, true]);
303303
let values = StringArray::new(offsets, buffer, Some(nulls));

‎arrow-string/src/substring.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -732,7 +732,7 @@ mod tests {
732732
}
733733

734734
fn generic_string_with_non_zero_offset<O: OffsetSizeTrait>() {
735-
let values = "hellotherearrow";
735+
let values = b"hellotherearrow";
736736
let offsets = &[
737737
O::zero(),
738738
O::from_usize(5).unwrap(),
@@ -867,7 +867,7 @@ mod tests {
867867
let data = ArrayData::builder(GenericStringArray::<O>::DATA_TYPE)
868868
.len(2)
869869
.add_buffer(Buffer::from_slice_ref(offsets))
870-
.add_buffer(Buffer::from(values))
870+
.add_buffer(Buffer::from(values.as_bytes()))
871871
.null_bit_buffer(Some(Buffer::from(bitmap)))
872872
.offset(1)
873873
.build()

‎arrow/examples/builders.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,13 +88,13 @@ fn main() {
8888
// buffer.
8989
let value_data = ArrayData::builder(DataType::Int32)
9090
.len(8)
91-
.add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
91+
.add_buffer(Buffer::from([0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
9292
.build()
9393
.unwrap();
9494

9595
// Construct a buffer for value offsets, for the nested array:
9696
// [[0, 1, 2], [3, 4, 5], [6, 7]]
97-
let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
97+
let value_offsets = Buffer::from([0, 3, 6, 8].to_byte_slice());
9898

9999
// Construct a list array from the above two
100100
let list_data_type = DataType::List(Arc::new(Field::new("item", DataType::Int32, false)));

‎arrow/examples/tensor_builder.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ fn main() -> Result<()> {
5757

5858
// In order to build a tensor from an array the function to_byte_slice add the
5959
// required padding to the elements in the array.
60-
let buf = Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7, 9, 10].to_byte_slice());
60+
let buf = Buffer::from([0, 1, 2, 3, 4, 5, 6, 7, 9, 10].to_byte_slice());
6161
let tensor = Int32Tensor::try_new(buf, Some(vec![2, 5]), None, None)?;
6262
println!("\nInt32 Tensor");
6363
println!("{tensor:?}");

0 commit comments

Comments
 (0)