From 5665d6f5a92bd655cb0d5c95b4aeb32e527ff4af Mon Sep 17 00:00:00 2001 From: Tyler Dunn Date: Tue, 14 May 2024 11:12:37 -0400 Subject: [PATCH 1/4] dart: Return dart:typed_data equivalent where possible * Remove _FbUint8List, _FbInt8List and _FbUint16List to return typed_data types * Add test for Int8List * Test Int8List, Uint8List and Uint16List for type equivalence * Remove unnecessary cast in _writeUTFString to silence linting warning --- dart/lib/flat_buffers.dart | 59 +++++++++++++------------------- dart/test/flat_buffers_test.dart | 32 +++++++++++++++-- 2 files changed, 53 insertions(+), 38 deletions(-) diff --git a/dart/lib/flat_buffers.dart b/dart/lib/flat_buffers.dart index 6f307872e10..8ccea8cd28c 100644 --- a/dart/lib/flat_buffers.dart +++ b/dart/lib/flat_buffers.dart @@ -41,6 +41,14 @@ class BufferContext { @pragma('vm:prefer-inline') Uint8List _asUint8List(int offset, int length) => _buffer.buffer.asUint8List(_buffer.offsetInBytes + offset, length); + + @pragma('vm:prefer-inline') + Int8List _asInt8List(int offset, int length) => + _buffer.buffer.asInt8List(_buffer.offsetInBytes + offset, length); + + @pragma('vm:prefer-inline') + Uint16List _asUint16List(int offset, int length) => + _buffer.buffer.asUint16List(_buffer.offsetInBytes + offset, length); @pragma('vm:prefer-inline') double _getFloat64(int offset) => _buffer.getFloat64(offset, Endian.little); @@ -729,7 +737,7 @@ class Builder { @pragma('vm:prefer-inline') void _writeUTFString(String value) { - final bytes = utf8.encode(value) as Uint8List; + final bytes = utf8.encode(value); final length = bytes.length; _prepare(4, 1, additionalBytes: length + 1); _setUint32AtTail(_tail, length); @@ -953,7 +961,7 @@ class Int32Reader extends Reader { int read(BufferContext bc, int offset) => bc._getInt32(offset); } -/// The reader of signed 32-bit integers. +/// The reader of signed 16-bit integers. class Int16Reader extends Reader { const Int16Reader() : super(); @@ -966,7 +974,7 @@ class Int16Reader extends Reader { int read(BufferContext bc, int offset) => bc._getInt16(offset); } -/// The reader of 8-bit signed integers. +/// The reader of signed 8-bit integers. class Int8Reader extends Reader { const Int8Reader() : super(); @@ -1165,8 +1173,12 @@ class Uint16ListReader extends Reader> { @override @pragma('vm:prefer-inline') - List read(BufferContext bc, int offset) => - _FbUint16List(bc, bc.derefObject(offset)); + List read(BufferContext bc, int offset) { + final listOffset = bc.derefObject(offset); + final length = bc._getUint32(listOffset); + + return bc._asUint16List(listOffset + _sizeofUint32, length); + } } /// The reader of unsigned 32-bit integers. @@ -1202,9 +1214,10 @@ class Uint8ListReader extends Reader> { @pragma('vm:prefer-inline') List read(BufferContext bc, int offset) { final listOffset = bc.derefObject(offset); - if (lazy) return _FbUint8List(bc, listOffset); - final length = bc._getUint32(listOffset); + + if (lazy) return bc._asUint8List(listOffset + _sizeofUint32, length); + final result = Uint8List(length); var pos = listOffset + _sizeofUint32; for (var i = 0; i < length; i++, pos++) { @@ -1247,9 +1260,10 @@ class Int8ListReader extends Reader> { @pragma('vm:prefer-inline') List read(BufferContext bc, int offset) { final listOffset = bc.derefObject(offset); - if (lazy) return _FbUint8List(bc, listOffset); - final length = bc._getUint32(listOffset); + + if (lazy) return bc._asInt8List(listOffset + _sizeofUint32, length); + final result = Int8List(length); var pos = listOffset + _sizeofUint32; for (var i = 0; i < length; i++, pos++) { @@ -1328,33 +1342,6 @@ class _FbUint32List extends _FbList { int operator [](int i) => bc._getUint32(offset + 4 + 4 * i); } -/// List backed by 16-bit unsigned integers. -class _FbUint16List extends _FbList { - _FbUint16List(BufferContext bc, int offset) : super(bc, offset); - - @override - @pragma('vm:prefer-inline') - int operator [](int i) => bc._getUint16(offset + 4 + 2 * i); -} - -/// List backed by 8-bit unsigned integers. -class _FbUint8List extends _FbList { - _FbUint8List(BufferContext bc, int offset) : super(bc, offset); - - @override - @pragma('vm:prefer-inline') - int operator [](int i) => bc._getUint8(offset + 4 + i); -} - -/// List backed by 8-bit signed integers. -class _FbInt8List extends _FbList { - _FbInt8List(BufferContext bc, int offset) : super(bc, offset); - - @override - @pragma('vm:prefer-inline') - int operator [](int i) => bc._getInt8(offset + 4 + i); -} - /// List backed by 8-bit unsigned integers. class _FbBoolList extends _FbList { _FbBoolList(BufferContext bc, int offset) : super(bc, offset); diff --git a/dart/test/flat_buffers_test.dart b/dart/test/flat_buffers_test.dart index caf2fc79a9e..c7c60623fef 100644 --- a/dart/test/flat_buffers_test.dart +++ b/dart/test/flat_buffers_test.dart @@ -666,6 +666,7 @@ class BuilderTest { // read and verify BufferContext buf = BufferContext.fromBytes(byteList); List items = const Uint16ListReader().read(buf, 0); + expect(items is Uint16List, true); expect(items, hasLength(3)); expect(items, orderedEquals([1, 2, 60000])); } @@ -683,18 +684,45 @@ class BuilderTest { const buffOffset = 8; // 32-bit offset to the list, + 32-bit length for (final lazy in [true, false]) { List items = Uint8ListReader(lazy: lazy).read(buf, 0); + expect(items is Uint8List, true); expect(items, hasLength(6)); - expect(items, orderedEquals([1, 2, 3, 4, 0x9A, 0xFA])); + expect(items, orderedEquals([1, 2, 3, 4, 154, 250])); // overwrite the buffer to verify the laziness buf.buffer.setUint8(buffOffset + 1, 99); - expect(items, orderedEquals([1, lazy ? 99 : 2, 3, 4, 0x9A, 0xFA])); + expect(items, orderedEquals([1, lazy ? 99 : 2, 3, 4, 154, 250])); // restore the previous value for the next loop buf.buffer.setUint8(buffOffset + 1, 2); } } + void test_writeList_ofInt8() { + List byteList; + { + Builder builder = Builder(initialSize: 0); + int offset = builder.writeListInt8([1, 2, 3, 4, 0x9A, 0xFA]); + builder.finish(offset); + byteList = builder.buffer; + } + // read and verify + BufferContext buf = BufferContext.fromBytes(byteList); + const buffOffset = 8; // 32-bit offset to the list, + 32-bit length + for (final lazy in [true, false]) { + List items = Int8ListReader(lazy: lazy).read(buf, 0); + expect(items is Int8List, true); + expect(items, hasLength(6)); + expect(items, orderedEquals([1, 2, 3, 4, -102, -6])); + + // overwrite the buffer to verify the laziness + buf.buffer.setInt8(buffOffset + 1, 99); + expect(items, orderedEquals([1, lazy ? 99 : 2, 3, 4, -102, -6])); + + // restore the previous value for the next loop + buf.buffer.setInt8(buffOffset + 1, 2); + } + } + void test_reset() { // We'll run a selection of tests , reusing the builder between them. final testCases = [ From ccdb632f0190e1fd39814fa1563caf149d21e590 Mon Sep 17 00:00:00 2001 From: Tyler Dunn Date: Thu, 30 May 2024 13:12:26 -0400 Subject: [PATCH 2/4] Retain compatibility with big-endian --- dart/lib/flat_buffers.dart | 26 ++++++++++++++++++++------ dart/test/flat_buffers_test.dart | 2 +- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/dart/lib/flat_buffers.dart b/dart/lib/flat_buffers.dart index 8ccea8cd28c..b3ca4fdd161 100644 --- a/dart/lib/flat_buffers.dart +++ b/dart/lib/flat_buffers.dart @@ -47,8 +47,10 @@ class BufferContext { _buffer.buffer.asInt8List(_buffer.offsetInBytes + offset, length); @pragma('vm:prefer-inline') - Uint16List _asUint16List(int offset, int length) => - _buffer.buffer.asUint16List(_buffer.offsetInBytes + offset, length); + Uint16List _asUint16List(int offset, int length) { + assert(Endian.host == Endian.little); + return _buffer.buffer.asUint16List(_buffer.offsetInBytes + offset, length); + } @pragma('vm:prefer-inline') double _getFloat64(int offset) => _buffer.getFloat64(offset, Endian.little); @@ -1174,10 +1176,13 @@ class Uint16ListReader extends Reader> { @override @pragma('vm:prefer-inline') List read(BufferContext bc, int offset) { - final listOffset = bc.derefObject(offset); - final length = bc._getUint32(listOffset); - - return bc._asUint16List(listOffset + _sizeofUint32, length); + if (Endian.host == Endian.little) { + final listOffset = bc.derefObject(offset); + final length = bc._getUint32(listOffset); + return bc._asUint16List(listOffset + _sizeofUint32, length); + } else { + return _FbUint16List(bc, bc.derefObject(offset)); + } } } @@ -1342,6 +1347,15 @@ class _FbUint32List extends _FbList { int operator [](int i) => bc._getUint32(offset + 4 + 4 * i); } +/// List backed by 16-bit unsigned integers. +class _FbUint16List extends _FbList { + _FbUint16List(BufferContext bc, int offset) : super(bc, offset); + + @override + @pragma('vm:prefer-inline') + int operator [](int i) => bc._getUint16(offset + 4 + 2 * i); +} + /// List backed by 8-bit unsigned integers. class _FbBoolList extends _FbList { _FbBoolList(BufferContext bc, int offset) : super(bc, offset); diff --git a/dart/test/flat_buffers_test.dart b/dart/test/flat_buffers_test.dart index c7c60623fef..a36a3f1ab58 100644 --- a/dart/test/flat_buffers_test.dart +++ b/dart/test/flat_buffers_test.dart @@ -666,7 +666,7 @@ class BuilderTest { // read and verify BufferContext buf = BufferContext.fromBytes(byteList); List items = const Uint16ListReader().read(buf, 0); - expect(items is Uint16List, true); + expect(items is Uint16List, true, skip: Endian.host == Endian.big); expect(items, hasLength(3)); expect(items, orderedEquals([1, 2, 60000])); } From 9956d9829d27d3e5f6f01470bdd6e13149774bf9 Mon Sep 17 00:00:00 2001 From: Tyler Dunn Date: Fri, 31 May 2024 15:52:55 -0400 Subject: [PATCH 3/4] Extend dart:typed_data support to other currently supported List types --- dart/lib/flat_buffers.dart | 52 ++++++++++++++++++++++++++++---- dart/test/flat_buffers_test.dart | 10 ++++-- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/dart/lib/flat_buffers.dart b/dart/lib/flat_buffers.dart index b3ca4fdd161..dcf4560337b 100644 --- a/dart/lib/flat_buffers.dart +++ b/dart/lib/flat_buffers.dart @@ -52,6 +52,24 @@ class BufferContext { return _buffer.buffer.asUint16List(_buffer.offsetInBytes + offset, length); } + @pragma('vm:prefer-inline') + Uint32List _asUint32List(int offset, int length) { + assert(Endian.host == Endian.little); + return _buffer.buffer.asUint32List(_buffer.offsetInBytes + offset, length); + } + + @pragma('vm:prefer-inline') + Float32List _asFloat32List(int offset, int length) { + assert(Endian.host == Endian.little); + return _buffer.buffer.asFloat32List(_buffer.offsetInBytes + offset, length); + } + + @pragma('vm:prefer-inline') + Float64List _asFloat64List(int offset, int length) { + assert(Endian.host == Endian.little); + return _buffer.buffer.asFloat64List(_buffer.offsetInBytes + offset, length); + } + @pragma('vm:prefer-inline') double _getFloat64(int offset) => _buffer.getFloat64(offset, Endian.little); @@ -897,8 +915,15 @@ class Float64ListReader extends Reader> { @override @pragma('vm:prefer-inline') - List read(BufferContext bc, int offset) => - _FbFloat64List(bc, bc.derefObject(offset)); + List read(BufferContext bc, int offset) { + if (Endian.host == Endian.little) { + final listOffset = bc.derefObject(offset); + final length = bc._getUint32(listOffset); + return bc._asFloat64List(listOffset + _sizeofUint32, length); + } else { + return _FbFloat64List(bc, bc.derefObject(offset)); + } + } } class Float32ListReader extends Reader> { @@ -910,8 +935,15 @@ class Float32ListReader extends Reader> { @override @pragma('vm:prefer-inline') - List read(BufferContext bc, int offset) => - _FbFloat32List(bc, bc.derefObject(offset)); + List read(BufferContext bc, int offset) { + if (Endian.host == Endian.little) { + final listOffset = bc.derefObject(offset); + final length = bc._getUint32(listOffset); + return bc._asFloat32List(listOffset + _sizeofUint32, length); + } else { + return _FbFloat32List(bc, bc.derefObject(offset)); + } + } } class Float64Reader extends Reader { @@ -1131,8 +1163,16 @@ class Uint32ListReader extends Reader> { @override @pragma('vm:prefer-inline') - List read(BufferContext bc, int offset) => - _FbUint32List(bc, bc.derefObject(offset)); + List read(BufferContext bc, int offset) { + if (Endian.host == Endian.little) { + final listOffset = bc.derefObject(offset); + final length = bc._getUint32(listOffset); + return bc._asUint32List(listOffset + _sizeofUint32, length); + } else { + return _FbUint32List(bc, bc.derefObject(offset)); + } + } + } /// The reader of unsigned 64-bit integers. diff --git a/dart/test/flat_buffers_test.dart b/dart/test/flat_buffers_test.dart index a36a3f1ab58..c86622ced76 100644 --- a/dart/test/flat_buffers_test.dart +++ b/dart/test/flat_buffers_test.dart @@ -1,3 +1,5 @@ +// ignore_for_file: non_constant_identifier_names + import 'dart:typed_data'; import 'dart:io' as io; @@ -539,7 +541,7 @@ class BuilderTest { // read and verify BufferContext buf = BufferContext.fromBytes(byteList); List items = const Float64ListReader().read(buf, 0); - + expect(items is Float64List, Endian.host == Endian.little); expect(items, hasLength(values.length)); for (int i = 0; i < values.length; i++) { expect(values[i], closeTo(items[i], .001)); @@ -559,6 +561,7 @@ class BuilderTest { // read and verify BufferContext buf = BufferContext.fromBytes(byteList); List items = const Float32ListReader().read(buf, 0); + expect(items is Float32List, Endian.host == Endian.little); expect(items, hasLength(5)); for (int i = 0; i < values.length; i++) { expect(values[i], closeTo(items[i], .001)); @@ -651,6 +654,7 @@ class BuilderTest { // read and verify BufferContext buf = BufferContext.fromBytes(byteList); List items = const Uint32ListReader().read(buf, 0); + expect(items is Uint32List, Endian.host == Endian.little); expect(items, hasLength(3)); expect(items, orderedEquals([1, 2, 0x9ABCDEF0])); } @@ -666,7 +670,7 @@ class BuilderTest { // read and verify BufferContext buf = BufferContext.fromBytes(byteList); List items = const Uint16ListReader().read(buf, 0); - expect(items is Uint16List, true, skip: Endian.host == Endian.big); + expect(items is Uint16List, Endian.host == Endian.little); expect(items, hasLength(3)); expect(items, orderedEquals([1, 2, 60000])); } @@ -841,9 +845,9 @@ class ObjectAPITest { final offset = monster.pack(fbBuilder); expect(offset, isNonZero); fbBuilder.finish(offset); - final data = fbBuilder.buffer; // TODO currently broken because of struct builder issue, see #6688 + // final data = fbBuilder.buffer; // final monster2 = example.Monster(data); // Monster (reader) // expect( // // map Monster => MonsterT, Vec3 => Vec3T, ... From cd5592085ff084ba589d023d15db9762f68b1a75 Mon Sep 17 00:00:00 2001 From: Tyler Dunn Date: Thu, 13 Jun 2024 15:32:09 -0400 Subject: [PATCH 4/4] Return original type for Float32List/Float64List --- dart/lib/flat_buffers.dart | 41 +++++++++----------------------- dart/test/flat_buffers_test.dart | 2 -- 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/dart/lib/flat_buffers.dart b/dart/lib/flat_buffers.dart index 170ac241d6b..4956286e5c4 100644 --- a/dart/lib/flat_buffers.dart +++ b/dart/lib/flat_buffers.dart @@ -58,18 +58,6 @@ class BufferContext { return _buffer.buffer.asUint32List(_buffer.offsetInBytes + offset, length); } - @pragma('vm:prefer-inline') - Float32List _asFloat32List(int offset, int length) { - assert(Endian.host == Endian.little); - return _buffer.buffer.asFloat32List(_buffer.offsetInBytes + offset, length); - } - - @pragma('vm:prefer-inline') - Float64List _asFloat64List(int offset, int length) { - assert(Endian.host == Endian.little); - return _buffer.buffer.asFloat64List(_buffer.offsetInBytes + offset, length); - } - @pragma('vm:prefer-inline') double _getFloat64(int offset) => _buffer.getFloat64(offset, Endian.little); @@ -906,6 +894,8 @@ class BoolReader extends Reader { /// The reader of lists of 64-bit float values. /// /// The returned unmodifiable lists lazily read values on access. +/// +/// TODO: Return dart:typed_data Float64List type on LE systems class Float64ListReader extends Reader> { const Float64ListReader(); @@ -915,17 +905,15 @@ class Float64ListReader extends Reader> { @override @pragma('vm:prefer-inline') - List read(BufferContext bc, int offset) { - if (Endian.host == Endian.little) { - final listOffset = bc.derefObject(offset); - final length = bc._getUint32(listOffset); - return bc._asFloat64List(listOffset + _sizeofUint32, length); - } else { - return _FbFloat64List(bc, bc.derefObject(offset)); - } - } + List read(BufferContext bc, int offset) => + _FbFloat64List(bc, bc.derefObject(offset)); } +/// The reader of lists of 32-bit float values. +/// +/// The returned unmodifiable lists lazily read values on access. +/// +/// TODO: Return dart:typed_data Float32List type on LE systems class Float32ListReader extends Reader> { const Float32ListReader(); @@ -935,15 +923,8 @@ class Float32ListReader extends Reader> { @override @pragma('vm:prefer-inline') - List read(BufferContext bc, int offset) { - if (Endian.host == Endian.little) { - final listOffset = bc.derefObject(offset); - final length = bc._getUint32(listOffset); - return bc._asFloat32List(listOffset + _sizeofUint32, length); - } else { - return _FbFloat32List(bc, bc.derefObject(offset)); - } - } + List read(BufferContext bc, int offset) => + _FbFloat32List(bc, bc.derefObject(offset)); } class Float64Reader extends Reader { diff --git a/dart/test/flat_buffers_test.dart b/dart/test/flat_buffers_test.dart index 2f36fa5480a..23df82f2acc 100644 --- a/dart/test/flat_buffers_test.dart +++ b/dart/test/flat_buffers_test.dart @@ -545,7 +545,6 @@ class BuilderTest { // read and verify BufferContext buf = BufferContext.fromBytes(byteList); List items = const Float64ListReader().read(buf, 0); - expect(items is Float64List, Endian.host == Endian.little); expect(items, hasLength(values.length)); for (int i = 0; i < values.length; i++) { expect(values[i], closeTo(items[i], .001)); @@ -565,7 +564,6 @@ class BuilderTest { // read and verify BufferContext buf = BufferContext.fromBytes(byteList); List items = const Float32ListReader().read(buf, 0); - expect(items is Float32List, Endian.host == Endian.little); expect(items, hasLength(5)); for (int i = 0; i < values.length; i++) { expect(values[i], closeTo(items[i], .001));