diff --git a/.github/workflows/ffi.yaml b/.github/workflows/ffi.yaml new file mode 100644 index 0000000000..9020f5975b --- /dev/null +++ b/.github/workflows/ffi.yaml @@ -0,0 +1,61 @@ +name: Dart CI + +on: + # Run on PRs and pushes to the default branch. + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + - cron: "0 0 * * 0" + +env: + PUB_ENVIRONMENT: bot.github + +jobs: + # Check code formatting and static analysis on a single OS (linux) + # against Dart dev. + analyze: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + sdk: [dev] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: dart-lang/setup-dart@b64355ae6ca0b5d484f0106a033dd1388965d06d + with: + sdk: ${{ matrix.sdk }} + - id: install + name: Install dependencies + run: dart pub get + - name: Check formatting + run: dart format --output=none --set-exit-if-changed . + if: always() && steps.install.outcome == 'success' + - name: Analyze code + run: dart analyze --fatal-infos + if: always() && steps.install.outcome == 'success' + + # Run tests on a matrix consisting of two dimensions: + # 1. OS: ubuntu-latest, (macos-latest, windows-latest) + # 2. release channel: dev + test: + needs: analyze + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # Add macos-latest and/or windows-latest if relevant for this package. + os: [ubuntu-latest] + sdk: [beta, dev] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: dart-lang/setup-dart@b64355ae6ca0b5d484f0106a033dd1388965d06d + with: + sdk: ${{ matrix.sdk }} + - id: install + name: Install dependencies + run: dart pub get + - name: Run VM tests + run: dart test --platform vm + if: always() && steps.install.outcome == 'success' diff --git a/pkgs/ffi/.gitignore b/pkgs/ffi/.gitignore new file mode 100644 index 0000000000..79f51c3d50 --- /dev/null +++ b/pkgs/ffi/.gitignore @@ -0,0 +1,3 @@ +.dart_tool +.packages +pubspec.lock diff --git a/pkgs/ffi/AUTHORS b/pkgs/ffi/AUTHORS new file mode 100644 index 0000000000..846e4a1568 --- /dev/null +++ b/pkgs/ffi/AUTHORS @@ -0,0 +1,6 @@ +# Below is a list of people and organizations that have contributed +# to the Dart project. Names should be added to the list like so: +# +# Name/Organization + +Google LLC diff --git a/pkgs/ffi/CHANGELOG.md b/pkgs/ffi/CHANGELOG.md new file mode 100644 index 0000000000..02410cccf5 --- /dev/null +++ b/pkgs/ffi/CHANGELOG.md @@ -0,0 +1,146 @@ +## 2.1.1 + +- Require Dart 3.3.0 or greater. +- Migrate `elementAt` use to `operator +`. + +## 2.1.0 + +- Require Dart 3.0.0 or greater. +- Expose native equivalent to `free` (`nativeFree`) from `malloc` and + `calloc` allocators. + +## 2.0.2 + +- Fixed a typo in a doc comment. +- Added package topics to the pubspec file. + +## 2.0.1 + +- Only zero out memory on successful allocation on Windows. +- Upgrade test dev dependency. + +## 2.0.0 + +- Switch Windows memory allocation to use `CoTaskMemAlloc` and `CoTaskMemFree`, + which will enable support for `NativeFinalizer`. +- Require Dart 2.17.0 or greater. + +## 1.2.1 + +Revert added common C integer types as ABI-specific integers. +Instead, these are available in Dart 2.17. + +## 1.2.0 (retracted) + +This release requires Dart `2.16.0` or greater. + +## 1.2.0-dev.0 + +Added common C integer types as ABI-specific integers. These common C integer +types will make their way into `dart:ffi` in 2.17 and be deprecated from this +package. Having them in this package enables using them in Dart 2.16. + +This pre-release requires Dart `2.16.0-118.0.dev` or greater. + +## 1.1.2 + +Fixed unhandled exception in `withZoneArena` (#107). + +## 1.1.1 + +Adds a sanity check to `Pointer` and `Pointer` extension methods +that receiver is not `nullptr`. + +## 1.1.0 + +Adds the `arena` allocator. + +Moves from static analysis with lints in package:pedantic to package:lints. + +## 1.0.0 + +Bumping the version of this package to `1.0.0`. + +Removes all deprecated methods, use `0.3.0-nullsafety.3` for migration. + +## 0.3.1-nullsafety.0 + +Deprecates the static methods on `Utf8` and `Utf16` and introduces +extension methods to replace them. + +## 0.3.0-nullsafety.3 + +Adds back in deprecated `allocate` and `free` to ease migration. +These will be removed in the next release. + +This pre-release requires Dart `2.12.0-259.9.beta` or greater. + +## 0.3.0-nullsafety.1 + +This pre-release requires Dart `2.12.0-259.8.beta` or greater. + +Note that this pre-release does _not_ work in Flutter versions containing Dart +`2.12.0-260.0.dev` - `2.12.0-264.0.dev`. +Using `Allocator.call` throws a `NoSuchMethodError` in these versions. +See [Flutter Engine #23954](https://github.com/flutter/engine/pull/23954) for more info. + +## 0.3.0-nullsafety.0 + +Changes `Utf8` and `Utf16` to extend `Opaque` instead of `Struct`. +This means `.ref` is no longer available and `Pointer` should be used. +See [breaking change #44622](https://github.com/dart-lang/sdk/issues/44622) for more info. + +Removes `allocate` and `free`. +Instead, introduces `calloc` which implements the new `Allocator` interface. +See [breaking change #44621](https://github.com/dart-lang/sdk/issues/44621) for more info. + +This pre-release requires Dart `2.12.0-265.0.dev` or greater. + +## 0.2.0-nullsafety.1 + +Adds an optional named `length` argument to `Utf8.fromUtf8()`. + +## 0.2.0-nullsafety.0 + +Pre-release (non-stable) release supporting null safety. +Requires Dart 2.12.0 or greater. + +## 0.1.3 + +Stable release incorporating all the previous dev release changes. + +Bump SDK constraint to `>= 2.6.0`. + +## 0.1.3-dev.4 + +Bump SDK constraint to `>= 2.6.0-dev.8.2` which contains the new API of `dart:ffi`. + +## 0.1.3-dev.3 + +Replace use of deprecated `asExternalTypedData` with `asTypedList`. + +## 0.1.3-dev.2 + +Incorporate struct API changes, drop type argument of structs. + +## 0.1.3-dev.1 + +* Adds top-level `allocate()` and `free()` methods which can be used as a + replacement for the deprecated `Pointer.allocate()` and `Pointer.free()` + members in `dart:ffi`. + +## 0.1.1+2 + +* Expand readme + +## 0.1.1+1 + +* Fix documentation link + +## 0.1.1 + +* Add basic Utf16 support + +## 0.1.0 + +* Initial release supporting Utf8 diff --git a/pkgs/ffi/LICENSE b/pkgs/ffi/LICENSE new file mode 100644 index 0000000000..767000764b --- /dev/null +++ b/pkgs/ffi/LICENSE @@ -0,0 +1,27 @@ +Copyright 2019, the Dart project authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google LLC nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pkgs/ffi/README.md b/pkgs/ffi/README.md new file mode 100644 index 0000000000..84a55c311e --- /dev/null +++ b/pkgs/ffi/README.md @@ -0,0 +1,11 @@ +[![Build Status](https://github.com/dart-lang/ffi/workflows/Dart%20CI/badge.svg)](https://github.com/dart-lang/ffi/actions?query=workflow%3A"Dart+CI") +[![pub package](https://img.shields.io/pub/v/ffi.svg)](https://pub.dev/packages/ffi) +[![package publisher](https://img.shields.io/pub/publisher/ffi.svg)](https://pub.dev/packages/ffi/publisher) + +Utilities for working with Foreign Function Interface (FFI) code, incl. +converting between Dart strings and C strings encoded with UTF-8 and UTF-16. + +Please see the [API reference](https://pub.dev/documentation/ffi/latest/ffi/ffi-library.html) for more documentation and the [tests](https://github.com/dart-lang/ffi/tree/main/test) for example usage. + +For additional details about Dart FFI (`dart:ffi`), see +https://dart.dev/guides/libraries/c-interop. diff --git a/pkgs/ffi/analysis_options.yaml b/pkgs/ffi/analysis_options.yaml new file mode 100644 index 0000000000..1498fff989 --- /dev/null +++ b/pkgs/ffi/analysis_options.yaml @@ -0,0 +1,6 @@ +include: package:lints/recommended.yaml + +analyzer: + language: + strict-casts: true + strict-inference: true diff --git a/pkgs/ffi/example/main.dart b/pkgs/ffi/example/main.dart new file mode 100644 index 0000000000..27623f0e91 --- /dev/null +++ b/pkgs/ffi/example/main.dart @@ -0,0 +1,18 @@ +import 'dart:ffi'; + +import 'package:ffi/ffi.dart'; + +void main() { + // Allocate and free some native memory with calloc and free. + final pointer = calloc(); + pointer.value = 3; + print(pointer.value); + calloc.free(pointer); + + // Use the Utf8 helper to encode zero-terminated UTF-8 strings in native memory. + final String myString = 'πŸ˜ŽπŸ‘ΏπŸ’¬'; + final Pointer charPointer = myString.toNativeUtf8(); + print('First byte is: ${charPointer.cast().value}'); + print(charPointer.toDartString()); + calloc.free(charPointer); +} diff --git a/pkgs/ffi/lib/ffi.dart b/pkgs/ffi/lib/ffi.dart new file mode 100644 index 0000000000..774ee9e86f --- /dev/null +++ b/pkgs/ffi/lib/ffi.dart @@ -0,0 +1,8 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +export 'src/allocation.dart' show calloc, malloc; +export 'src/arena.dart'; +export 'src/utf8.dart'; +export 'src/utf16.dart'; diff --git a/pkgs/ffi/lib/src/allocation.dart b/pkgs/ffi/lib/src/allocation.dart new file mode 100644 index 0000000000..42021458ad --- /dev/null +++ b/pkgs/ffi/lib/src/allocation.dart @@ -0,0 +1,229 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:io'; + +// Note that ole32.dll is the correct name in both 32-bit and 64-bit. +final DynamicLibrary stdlib = Platform.isWindows + ? DynamicLibrary.open('ole32.dll') + : DynamicLibrary.process(); + +typedef PosixMallocNative = Pointer Function(IntPtr); +typedef PosixMalloc = Pointer Function(int); +final PosixMalloc posixMalloc = + stdlib.lookupFunction('malloc'); + +typedef PosixCallocNative = Pointer Function(IntPtr num, IntPtr size); +typedef PosixCalloc = Pointer Function(int num, int size); +final PosixCalloc posixCalloc = + stdlib.lookupFunction('calloc'); + +typedef PosixFreeNative = Void Function(Pointer); +typedef PosixFree = void Function(Pointer); +final Pointer> posixFreePointer = + stdlib.lookup('free'); +final PosixFree posixFree = posixFreePointer.asFunction(); + +typedef WinCoTaskMemAllocNative = Pointer Function(Size); +typedef WinCoTaskMemAlloc = Pointer Function(int); +final WinCoTaskMemAlloc winCoTaskMemAlloc = + stdlib.lookupFunction( + 'CoTaskMemAlloc'); + +typedef WinCoTaskMemFreeNative = Void Function(Pointer); +typedef WinCoTaskMemFree = void Function(Pointer); +final Pointer> winCoTaskMemFreePointer = + stdlib.lookup('CoTaskMemFree'); +final WinCoTaskMemFree winCoTaskMemFree = winCoTaskMemFreePointer.asFunction(); + +/// Manages memory on the native heap. +/// +/// Does not initialize newly allocated memory to zero. Use [_CallocAllocator] +/// for zero-initialized memory on allocation. +/// +/// For POSIX-based systems, this uses `malloc` and `free`. On Windows, it uses +/// `CoTaskMemAlloc`. +final class MallocAllocator implements Allocator { + const MallocAllocator._(); + + /// Allocates [byteCount] bytes of of unitialized memory on the native heap. + /// + /// For POSIX-based systems, this uses `malloc`. On Windows, it uses + /// `CoTaskMemAlloc`. + /// + /// Throws an [ArgumentError] if the number of bytes or alignment cannot be + /// satisfied. + // TODO: Stop ignoring alignment if it's large, for example for SSE data. + @override + Pointer allocate(int byteCount, {int? alignment}) { + Pointer result; + if (Platform.isWindows) { + result = winCoTaskMemAlloc(byteCount).cast(); + } else { + result = posixMalloc(byteCount).cast(); + } + if (result.address == 0) { + throw ArgumentError('Could not allocate $byteCount bytes.'); + } + return result; + } + + /// Releases memory allocated on the native heap. + /// + /// For POSIX-based systems, this uses `free`. On Windows, it uses + /// `CoTaskMemFree`. It may only be used against pointers allocated in a + /// manner equivalent to [allocate]. + @override + void free(Pointer pointer) { + if (Platform.isWindows) { + winCoTaskMemFree(pointer); + } else { + posixFree(pointer); + } + } + + /// Returns a pointer to a native free function. + /// + /// This function can be used to release memory allocated by [allocated] + /// from the native side. It can also be used as a finalization callback + /// passed to `NativeFinalizer` constructor or `Pointer.atTypedList` + /// method. + /// + /// For example to automatically free native memory when the Dart object + /// wrapping it is reclaimed by GC: + /// + /// ```dart + /// class Wrapper implements Finalizable { + /// static final finalizer = NativeFinalizer(malloc.nativeFree); + /// + /// final Pointer data; + /// + /// Wrapper() : data = malloc.allocate(length) { + /// finalizer.attach(this, data); + /// } + /// } + /// ``` + /// + /// or to free native memory that is owned by a typed list: + /// + /// ```dart + /// malloc.allocate(n).asTypedList(n, finalizer: malloc.nativeFree) + /// ``` + /// + Pointer get nativeFree => + Platform.isWindows ? winCoTaskMemFreePointer : posixFreePointer; +} + +/// Manages memory on the native heap. +/// +/// Does not initialize newly allocated memory to zero. Use [calloc] for +/// zero-initialized memory allocation. +/// +/// For POSIX-based systems, this uses `malloc` and `free`. On Windows, it uses +/// `CoTaskMemAlloc` and `CoTaskMemFree`. +const MallocAllocator malloc = MallocAllocator._(); + +/// Manages memory on the native heap. +/// +/// Initializes newly allocated memory to zero. +/// +/// For POSIX-based systems, this uses `calloc` and `free`. On Windows, it uses +/// `CoTaskMemAlloc` and `CoTaskMemFree`. +final class CallocAllocator implements Allocator { + const CallocAllocator._(); + + /// Fills a block of memory with a specified value. + void _fillMemory(Pointer destination, int length, int fill) { + final ptr = destination.cast(); + for (var i = 0; i < length; i++) { + ptr[i] = fill; + } + } + + /// Fills a block of memory with zeros. + /// + void _zeroMemory(Pointer destination, int length) => + _fillMemory(destination, length, 0); + + /// Allocates [byteCount] bytes of zero-initialized of memory on the native + /// heap. + /// + /// For POSIX-based systems, this uses `malloc`. On Windows, it uses + /// `CoTaskMemAlloc`. + /// + /// Throws an [ArgumentError] if the number of bytes or alignment cannot be + /// satisfied. + // TODO: Stop ignoring alignment if it's large, for example for SSE data. + @override + Pointer allocate(int byteCount, {int? alignment}) { + Pointer result; + if (Platform.isWindows) { + result = winCoTaskMemAlloc(byteCount).cast(); + } else { + result = posixCalloc(byteCount, 1).cast(); + } + if (result.address == 0) { + throw ArgumentError('Could not allocate $byteCount bytes.'); + } + if (Platform.isWindows) { + _zeroMemory(result, byteCount); + } + return result; + } + + /// Releases memory allocated on the native heap. + /// + /// For POSIX-based systems, this uses `free`. On Windows, it uses + /// `CoTaskMemFree`. It may only be used against pointers allocated in a + /// manner equivalent to [allocate]. + @override + void free(Pointer pointer) { + if (Platform.isWindows) { + winCoTaskMemFree(pointer); + } else { + posixFree(pointer); + } + } + + /// Returns a pointer to a native free function. + /// + /// This function can be used to release memory allocated by [allocated] + /// from the native side. It can also be used as a finalization callback + /// passed to `NativeFinalizer` constructor or `Pointer.atTypedList` + /// method. + /// + /// For example to automatically free native memory when the Dart object + /// wrapping it is reclaimed by GC: + /// + /// ```dart + /// class Wrapper implements Finalizable { + /// static final finalizer = NativeFinalizer(calloc.nativeFree); + /// + /// final Pointer data; + /// + /// Wrapper() : data = calloc.allocate(length) { + /// finalizer.attach(this, data); + /// } + /// } + /// ``` + /// + /// or to free native memory that is owned by a typed list: + /// + /// ```dart + /// calloc.allocate(n).asTypedList(n, finalizer: calloc.nativeFree) + /// ``` + /// + Pointer get nativeFree => + Platform.isWindows ? winCoTaskMemFreePointer : posixFreePointer; +} + +/// Manages memory on the native heap. +/// +/// Initializes newly allocated memory to zero. Use [malloc] for uninitialized +/// memory allocation. +/// +/// For POSIX-based systems, this uses `calloc` and `free`. On Windows, it uses +/// `CoTaskMemAlloc` and `CoTaskMemFree`. +const CallocAllocator calloc = CallocAllocator._(); diff --git a/pkgs/ffi/lib/src/arena.dart b/pkgs/ffi/lib/src/arena.dart new file mode 100644 index 0000000000..747d120408 --- /dev/null +++ b/pkgs/ffi/lib/src/arena.dart @@ -0,0 +1,185 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +// +// Explicit arena used for managing resources. + +import 'dart:async'; +import 'dart:ffi'; + +import 'package:ffi/ffi.dart'; + +/// An [Allocator] which frees all allocations at the same time. +/// +/// The arena allows you to allocate heap memory, but ignores calls to [free]. +/// Instead you call [releaseAll] to release all the allocations at the same +/// time. +/// +/// Also allows other resources to be associated with the arena, through the +/// [using] method, to have a release function called for them when the arena +/// is released. +/// +/// An [Allocator] can be provided to do the actual allocation and freeing. +/// Defaults to using [calloc]. +class Arena implements Allocator { + /// The [Allocator] used for allocation and freeing. + final Allocator _wrappedAllocator; + + /// Native memory under management by this [Arena]. + final List> _managedMemoryPointers = []; + + /// Callbacks for releasing native resources under management by this [Arena]. + final List _managedResourceReleaseCallbacks = []; + + bool _inUse = true; + + /// Creates a arena of allocations. + /// + /// The [allocator] is used to do the actual allocation and freeing of + /// memory. It defaults to using [calloc]. + Arena([Allocator allocator = calloc]) : _wrappedAllocator = allocator; + + /// Allocates memory and includes it in the arena. + /// + /// Uses the allocator provided to the [Arena] constructor to do the + /// allocation. + /// + /// Throws an [ArgumentError] if the number of bytes or alignment cannot be + /// satisfied. + @override + Pointer allocate(int byteCount, {int? alignment}) { + _ensureInUse(); + final p = _wrappedAllocator.allocate(byteCount, alignment: alignment); + _managedMemoryPointers.add(p); + return p; + } + + /// Registers [resource] in this arena. + /// + /// Executes [releaseCallback] on [releaseAll]. + /// + /// Returns [resource] again, to allow for easily inserting + /// `arena.using(resource, ...)` where the resource is allocated. + T using(T resource, void Function(T) releaseCallback) { + _ensureInUse(); + releaseCallback = Zone.current.bindUnaryCallback(releaseCallback); + _managedResourceReleaseCallbacks.add(() => releaseCallback(resource)); + return resource; + } + + /// Registers [releaseResourceCallback] to be executed on [releaseAll]. + void onReleaseAll(void Function() releaseResourceCallback) { + _managedResourceReleaseCallbacks.add(releaseResourceCallback); + } + + /// Releases all resources that this [Arena] manages. + /// + /// If [reuse] is `true`, the arena can be used again after resources + /// have been released. If not, the default, then the [allocate] + /// and [using] methods must not be called after a call to `releaseAll`. + /// + /// If any of the callbacks throw, [releaseAll] is interrupted, and should + /// be started again. + void releaseAll({bool reuse = false}) { + if (!reuse) { + _inUse = false; + } + // The code below is deliberately wirtten to allow allocations to happen + // during `releaseAll(reuse:true)`. The arena will still be guaranteed + // empty when the `releaseAll` call returns. + while (_managedResourceReleaseCallbacks.isNotEmpty) { + _managedResourceReleaseCallbacks.removeLast()(); + } + for (final p in _managedMemoryPointers) { + _wrappedAllocator.free(p); + } + _managedMemoryPointers.clear(); + } + + /// Does nothing, invoke [releaseAll] instead. + @override + void free(Pointer pointer) {} + + void _ensureInUse() { + if (!_inUse) { + throw StateError( + 'Arena no longer in use, `releaseAll(reuse: false)` was called.'); + } + } +} + +/// Runs [computation] with a new [Arena], and releases all allocations at the +/// end. +/// +/// If the return value of [computation] is a [Future], all allocations are +/// released when the future completes. +/// +/// If the isolate is shut down, through `Isolate.kill()`, resources are _not_ +/// cleaned up. +R using(R Function(Arena) computation, + [Allocator wrappedAllocator = calloc]) { + final arena = Arena(wrappedAllocator); + bool isAsync = false; + try { + final result = computation(arena); + if (result is Future) { + isAsync = true; + return (result.whenComplete(arena.releaseAll) as R); + } + return result; + } finally { + if (!isAsync) { + arena.releaseAll(); + } + } +} + +/// Creates a zoned [Arena] to manage native resources. +/// +/// The arena is available through [zoneArena]. +/// +/// If the isolate is shut down, through `Isolate.kill()`, resources are _not_ +/// cleaned up. +R withZoneArena(R Function() computation, + [Allocator wrappedAllocator = calloc]) { + final arena = Arena(wrappedAllocator); + var arenaHolder = [arena]; + bool isAsync = false; + try { + return runZoned(() { + final result = computation(); + if (result is Future) { + isAsync = true; + return result.whenComplete(() { + arena.releaseAll(); + }) as R; + } + return result; + }, zoneValues: {#_arena: arenaHolder}); + } finally { + if (!isAsync) { + arena.releaseAll(); + arenaHolder.clear(); + } + } +} + +/// A zone-specific [Arena]. +/// +/// Asynchronous computations can share a [Arena]. Use [withZoneArena] to create +/// a new zone with a fresh [Arena], and that arena will then be released +/// automatically when the function passed to [withZoneArena] completes. +/// All code inside that zone can use `zoneArena` to access the arena. +/// +/// The current arena must not be accessed by code which is not running inside +/// a zone created by [withZoneArena]. +Arena get zoneArena { + final arenaHolder = Zone.current[#_arena] as List?; + if (arenaHolder == null) { + throw StateError('Not inside a zone created by `useArena`'); + } + if (arenaHolder.isNotEmpty) { + return arenaHolder.single; + } + throw StateError('Arena has already been cleared with releaseAll.'); +} diff --git a/pkgs/ffi/lib/src/utf16.dart b/pkgs/ffi/lib/src/utf16.dart new file mode 100644 index 0000000000..73450ad7b5 --- /dev/null +++ b/pkgs/ffi/lib/src/utf16.dart @@ -0,0 +1,101 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'package:ffi/ffi.dart'; + +/// The contents of a native zero-terminated array of UTF-16 code units. +/// +/// The Utf16 type itself has no functionality, it's only intended to be used +/// through a `Pointer` representing the entire array. This pointer is +/// the equivalent of a char pointer (`const wchar_t*`) in C code. The +/// individual UTF-16 code units are stored in native byte order. +final class Utf16 extends Opaque {} + +/// Extension method for converting a`Pointer` to a [String]. +extension Utf16Pointer on Pointer { + /// The number of UTF-16 code units in this zero-terminated UTF-16 string. + /// + /// The UTF-16 code units of the strings are the non-zero code units up to + /// the first zero code unit. + int get length { + _ensureNotNullptr('length'); + final codeUnits = cast(); + return _length(codeUnits); + } + + /// Converts this UTF-16 encoded string to a Dart string. + /// + /// Decodes the UTF-16 code units of this zero-terminated code unit array as + /// Unicode code points and creates a Dart string containing those code + /// points. + /// + /// If [length] is provided, zero-termination is ignored and the result can + /// contain NUL characters. + /// + /// If [length] is not provided, the returned string is the string up til + /// but not including the first NUL character. + String toDartString({int? length}) { + _ensureNotNullptr('toDartString'); + final codeUnits = cast(); + if (length == null) { + return _toUnknownLengthString(codeUnits); + } else { + RangeError.checkNotNegative(length, 'length'); + return _toKnownLengthString(codeUnits, length); + } + } + + static String _toKnownLengthString(Pointer codeUnits, int length) => + String.fromCharCodes(codeUnits.asTypedList(length)); + + static String _toUnknownLengthString(Pointer codeUnits) { + final buffer = StringBuffer(); + var i = 0; + while (true) { + final char = (codeUnits + i).value; + if (char == 0) { + return buffer.toString(); + } + buffer.writeCharCode(char); + i++; + } + } + + static int _length(Pointer codeUnits) { + var length = 0; + while (codeUnits[length] != 0) { + length++; + } + return length; + } + + void _ensureNotNullptr(String operation) { + if (this == nullptr) { + throw UnsupportedError( + "Operation '$operation' not allowed on a 'nullptr'."); + } + } +} + +/// Extension method for converting a [String] to a `Pointer`. +extension StringUtf16Pointer on String { + /// Creates a zero-terminated [Utf16] code-unit array from this String. + /// + /// If this [String] contains NUL characters, converting it back to a string + /// using [Utf16Pointer.toDartString] will truncate the result if a length is + /// not passed. + /// + /// Returns an [allocator]-allocated pointer to the result. + Pointer toNativeUtf16({Allocator allocator = malloc}) { + final units = codeUnits; + final Pointer result = allocator(units.length + 1); + final Uint16List nativeString = result.asTypedList(units.length + 1); + nativeString.setRange(0, units.length, units); + nativeString[units.length] = 0; + return result.cast(); + } +} diff --git a/pkgs/ffi/lib/src/utf8.dart b/pkgs/ffi/lib/src/utf8.dart new file mode 100644 index 0000000000..cdf0e7e420 --- /dev/null +++ b/pkgs/ffi/lib/src/utf8.dart @@ -0,0 +1,89 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'package:ffi/ffi.dart'; + +/// The contents of a native zero-terminated array of UTF-8 code units. +/// +/// The Utf8 type itself has no functionality, it's only intended to be used +/// through a `Pointer` representing the entire array. This pointer is +/// the equivalent of a char pointer (`const char*`) in C code. +final class Utf8 extends Opaque {} + +/// Extension method for converting a`Pointer` to a [String]. +extension Utf8Pointer on Pointer { + /// The number of UTF-8 code units in this zero-terminated UTF-8 string. + /// + /// The UTF-8 code units of the strings are the non-zero code units up to the + /// first zero code unit. + int get length { + _ensureNotNullptr('length'); + final codeUnits = cast(); + return _length(codeUnits); + } + + /// Converts this UTF-8 encoded string to a Dart string. + /// + /// Decodes the UTF-8 code units of this zero-terminated byte array as + /// Unicode code points and creates a Dart string containing those code + /// points. + /// + /// If [length] is provided, zero-termination is ignored and the result can + /// contain NUL characters. + /// + /// If [length] is not provided, the returned string is the string up til + /// but not including the first NUL character. + String toDartString({int? length}) { + _ensureNotNullptr('toDartString'); + final codeUnits = cast(); + if (length != null) { + RangeError.checkNotNegative(length, 'length'); + } else { + length = _length(codeUnits); + } + return utf8.decode(codeUnits.asTypedList(length)); + } + + static int _length(Pointer codeUnits) { + var length = 0; + while (codeUnits[length] != 0) { + length++; + } + return length; + } + + void _ensureNotNullptr(String operation) { + if (this == nullptr) { + throw UnsupportedError( + "Operation '$operation' not allowed on a 'nullptr'."); + } + } +} + +/// Extension method for converting a [String] to a `Pointer`. +extension StringUtf8Pointer on String { + /// Creates a zero-terminated [Utf8] code-unit array from this String. + /// + /// If this [String] contains NUL characters, converting it back to a string + /// using [Utf8Pointer.toDartString] will truncate the result if a length is + /// not passed. + /// + /// Unpaired surrogate code points in this [String] will be encoded as + /// replacement characters (U+FFFD, encoded as the bytes 0xEF 0xBF 0xBD) in + /// the UTF-8 encoded result. See [Utf8Encoder] for details on encoding. + /// + /// Returns an [allocator]-allocated pointer to the result. + Pointer toNativeUtf8({Allocator allocator = malloc}) { + final units = utf8.encode(this); + final Pointer result = allocator(units.length + 1); + final Uint8List nativeString = result.asTypedList(units.length + 1); + nativeString.setAll(0, units); + nativeString[units.length] = 0; + return result.cast(); + } +} diff --git a/pkgs/ffi/pubspec.yaml b/pkgs/ffi/pubspec.yaml new file mode 100644 index 0000000000..dde8122760 --- /dev/null +++ b/pkgs/ffi/pubspec.yaml @@ -0,0 +1,16 @@ +name: ffi +version: 2.1.1 +description: Utilities for working with Foreign Function Interface (FFI) code. +repository: https://github.com/dart-lang/ffi + +topics: + - interop + - ffi + - codegen + +environment: + sdk: '>=3.3.0-279.1.beta <4.0.0' + +dev_dependencies: + test: ^1.21.2 + lints: ^2.0.0 diff --git a/pkgs/ffi/test/allocation_test.dart b/pkgs/ffi/test/allocation_test.dart new file mode 100644 index 0000000000..899b801a89 --- /dev/null +++ b/pkgs/ffi/test/allocation_test.dart @@ -0,0 +1,50 @@ +// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:math' show Random; + +import 'package:ffi/ffi.dart'; +import 'package:test/test.dart'; + +const testRuns = 1000; + +void main() async { + test('calloc', () { + // Tests that calloc successfully zeroes out memory. + for (var i = 0; i < testRuns; i++) { + final allocBytes = Random().nextInt(1000); + final mem = calloc(allocBytes); + expect(mem.asTypedList(allocBytes).where(((element) => element != 0)), + isEmpty); + calloc.free(mem); + } + }); + + test('testPointerAllocateTooLarge', () { + // Try to allocate something that doesn't fit in 64 bit address space. + int maxInt = 9223372036854775807; // 2^63 - 1 + expect(() => calloc(maxInt), throwsA(isA())); + + // Try to allocate almost the full 64 bit address space. + int maxInt1_8 = 1152921504606846975; // 2^60 -1 + expect(() => calloc(maxInt1_8), throwsA(isA())); + }); + + test('testPointerAllocateNegative', () { + // Passing in -1 will be converted into an unsigned integer. So, it will try + // to allocate SIZE_MAX - 1 + 1 bytes. This will fail as it is the max + // amount of addressable memory on the system. + expect(() => calloc(-1), throwsA(isA())); + }); + + test('nativeFree', () { + // malloc.nativeFree should be able to free memory allocated by malloc. + final ptr1 = malloc.allocate(1024); + malloc.nativeFree.asFunction)>()(ptr1.cast()); + // calloc.nativeFree should be able to free memory allocated by calloc. + final ptr2 = calloc.allocate(1024); + calloc.nativeFree.asFunction)>()(ptr2.cast()); + }); +} diff --git a/pkgs/ffi/test/arena_test.dart b/pkgs/ffi/test/arena_test.dart new file mode 100644 index 0000000000..51cb9bfff6 --- /dev/null +++ b/pkgs/ffi/test/arena_test.dart @@ -0,0 +1,228 @@ +// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:ffi'; + +import 'package:ffi/ffi.dart'; +import 'package:test/test.dart'; + +void main() async { + test('sync', () async { + List freed = []; + void freeInt(int i) { + freed.add(i); + } + + using((Arena arena) { + arena.using(1234, freeInt); + expect(freed.isEmpty, true); + }); + expect(freed, [1234]); + }); + + test('async', () async { + /// Calling [using] waits with releasing its resources until after + /// [Future]s complete. + List freed = []; + void freeInt(int i) { + freed.add(i); + } + + Future myFutureInt = using((Arena arena) { + return Future.microtask(() { + arena.using(1234, freeInt); + return 1; + }); + }); + + expect(freed.isEmpty, true); + await myFutureInt; + expect(freed, [1234]); + }); + + test('throw', () { + /// [using] waits with releasing its resources until after [Future]s + /// complete. + List freed = []; + void freeInt(int i) { + freed.add(i); + } + + // Resources are freed also when abnormal control flow occurs. + var didThrow = false; + try { + using((Arena arena) { + arena.using(1234, freeInt); + expect(freed.isEmpty, true); + throw Exception('Exception 1'); + }); + } on Exception { + expect(freed.single, 1234); + didThrow = true; + } + expect(didThrow, true); + }); + + test( + 'allocate', + () { + final countingAllocator = CountingAllocator(); + // To ensure resources are freed, wrap them in a [using] call. + using((Arena arena) { + final p = arena(2); + p[1] = p[0]; + }, countingAllocator); + expect(countingAllocator.freeCount, 1); + }, + ); + + test('allocate throw', () { + // Resources are freed also when abnormal control flow occurs. + bool didThrow = false; + final countingAllocator = CountingAllocator(); + try { + using((Arena arena) { + final p = arena(2); + p[0] = 25; + throw Exception('Exception 2'); + }, countingAllocator); + } on Exception { + expect(countingAllocator.freeCount, 1); + didThrow = true; + } + expect(didThrow, true); + }); + + test('toNativeUtf8', () { + final countingAllocator = CountingAllocator(); + using((Arena arena) { + final p = 'Hello world!'.toNativeUtf8(allocator: arena); + expect(p.toDartString(), 'Hello world!'); + }, countingAllocator); + expect(countingAllocator.freeCount, 1); + }); + + test('zone', () async { + List freed = []; + void freeInt(int i) { + freed.add(i); + } + + withZoneArena(() { + zoneArena.using(1234, freeInt); + expect(freed.isEmpty, true); + }); + expect(freed.length, 1); + expect(freed.single, 1234); + }); + + test('zone async', () async { + /// [using] waits with releasing its resources until after [Future]s + /// complete. + List freed = []; + void freeInt(int i) { + freed.add(i); + } + + Future myFutureInt = withZoneArena(() { + return Future.microtask(() { + zoneArena.using(1234, freeInt); + return 1; + }); + }); + + expect(freed.isEmpty, true); + await myFutureInt; + expect(freed.length, 1); + expect(freed.single, 1234); + }); + + test('zone throw', () { + /// [using] waits with releasing its resources until after [Future]s + /// complete. + List freed = []; + void freeInt(int i) { + freed.add(i); + } + + // Resources are freed also when abnormal control flow occurs. + bool didThrow = false; + try { + withZoneArena(() { + zoneArena.using(1234, freeInt); + expect(freed.isEmpty, true); + throw Exception('Exception 3'); + }); + } on Exception { + expect(freed.single, 1234); + didThrow = true; + } + expect(didThrow, true); + expect(freed.single, 1234); + }); + + test('zone future error', () async { + bool caughtError = false; + bool uncaughtError = false; + + Future asyncFunction() async { + throw Exception('Exception 4'); + } + + final future = runZonedGuarded(() { + return withZoneArena(asyncFunction).catchError((error) { + caughtError = true; + return 5; + }); + }, (error, stackTrace) { + uncaughtError = true; + }); + + final result = (await Future.wait([future!])).single; + + expect(result, 5); + expect(caughtError, true); + expect(uncaughtError, false); + }); + + test('allocate during releaseAll', () { + final countingAllocator = CountingAllocator(); + final arena = Arena(countingAllocator); + + arena.using(arena(), (Pointer discard) { + arena(); + }); + + expect(countingAllocator.allocationCount, 1); + expect(countingAllocator.freeCount, 0); + + arena.releaseAll(reuse: true); + + expect(countingAllocator.allocationCount, 2); + expect(countingAllocator.freeCount, 2); + }); +} + +/// Keeps track of the number of allocates and frees for testing purposes. +class CountingAllocator implements Allocator { + final Allocator wrappedAllocator; + + int allocationCount = 0; + int freeCount = 0; + + CountingAllocator([this.wrappedAllocator = calloc]); + + @override + Pointer allocate(int byteCount, {int? alignment}) { + allocationCount++; + return wrappedAllocator.allocate(byteCount, alignment: alignment); + } + + @override + void free(Pointer pointer) { + freeCount++; + return wrappedAllocator.free(pointer); + } +} diff --git a/pkgs/ffi/test/utf16_test.dart b/pkgs/ffi/test/utf16_test.dart new file mode 100644 index 0000000000..c962b46097 --- /dev/null +++ b/pkgs/ffi/test/utf16_test.dart @@ -0,0 +1,88 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'package:ffi/ffi.dart'; +import 'package:test/test.dart'; + +void main() { + test('toUtf16 ASCII', () { + final String start = 'Hello World!\n'; + final Pointer converted = start.toNativeUtf16().cast(); + final Uint16List end = converted.asTypedList(start.codeUnits.length + 1); + final matcher = equals(start.codeUnits.toList()..add(0)); + expect(end, matcher); + calloc.free(converted); + }); + + test('toUtf16 emoji', () { + final String start = '😎'; + final Pointer converted = start.toNativeUtf16().cast(); + final int length = start.codeUnits.length; + final Uint16List end = converted.cast().asTypedList(length + 1); + final matcher = equals(start.codeUnits.toList()..add(0)); + expect(end, matcher); + calloc.free(converted); + }); + + test('from Utf16 ASCII', () { + final string = 'Hello World!\n'; + final utf16Pointer = string.toNativeUtf16(); + final stringAgain = utf16Pointer.toDartString(); + expect(stringAgain, string); + calloc.free(utf16Pointer); + }); + + test('from Utf16 emoji', () { + final string = '😎'; + final utf16Pointer = string.toNativeUtf16(); + final stringAgain = utf16Pointer.toDartString(); + expect(stringAgain, string); + calloc.free(utf16Pointer); + }); + + test('zero bytes', () { + final string = 'Hello\x00World!\n'; + final utf16Pointer = string.toNativeUtf16(); + final stringAgain = utf16Pointer.toDartString(length: 13); + expect(stringAgain, string); + calloc.free(utf16Pointer); + }); + + test('length', () { + final string = 'Hello'; + final utf16Pointer = string.toNativeUtf16(); + expect(utf16Pointer.length, 5); + calloc.free(utf16Pointer); + }); + + test('fromUtf8 with negative length', () { + final string = 'Hello'; + final utf16 = string.toNativeUtf16(); + expect(() => utf16.toDartString(length: -1), throwsRangeError); + calloc.free(utf16); + }); + + test('nullptr.toDartString()', () { + final Pointer utf16 = nullptr; + try { + utf16.toDartString(); + } on UnsupportedError { + return; + } + fail('Expected an error.'); + }); + + test('nullptr.length', () { + final Pointer utf16 = nullptr; + try { + utf16.length; + } on UnsupportedError { + return; + } + fail('Expected an error.'); + }); +} diff --git a/pkgs/ffi/test/utf8_test.dart b/pkgs/ffi/test/utf8_test.dart new file mode 100644 index 0000000000..d23afdee15 --- /dev/null +++ b/pkgs/ffi/test/utf8_test.dart @@ -0,0 +1,127 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'package:ffi/ffi.dart'; +import 'package:test/test.dart'; + +Pointer _bytesFromList(List ints) { + final Pointer ptr = calloc(ints.length); + final Uint8List list = ptr.asTypedList(ints.length); + list.setAll(0, ints); + return ptr; +} + +void main() { + test('toUtf8 ASCII', () { + final String start = 'Hello World!\n'; + final Pointer converted = start.toNativeUtf8().cast(); + final Uint8List end = converted.asTypedList(start.length + 1); + final matcher = + equals([72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 10, 0]); + expect(end, matcher); + calloc.free(converted); + }); + + test('fromUtf8 ASCII', () { + final Pointer utf8 = _bytesFromList( + [72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 10, 0]).cast(); + final String end = utf8.toDartString(); + expect(end, 'Hello World!\n'); + }); + + test('toUtf8 emoji', () { + final String start = 'πŸ˜ŽπŸ‘ΏπŸ’¬'; + final Pointer converted = start.toNativeUtf8().cast(); + final int length = converted.length; + final Uint8List end = converted.cast().asTypedList(length + 1); + final matcher = + equals([240, 159, 152, 142, 240, 159, 145, 191, 240, 159, 146, 172, 0]); + expect(end, matcher); + calloc.free(converted); + }); + + test('formUtf8 emoji', () { + final Pointer utf8 = _bytesFromList( + [240, 159, 152, 142, 240, 159, 145, 191, 240, 159, 146, 172, 0]).cast(); + final String end = utf8.toDartString(); + expect(end, 'πŸ˜ŽπŸ‘ΏπŸ’¬'); + }); + + test('fromUtf8 invalid', () { + final Pointer utf8 = _bytesFromList([0x80, 0x00]).cast(); + expect(() => utf8.toDartString(), throwsA(isFormatException)); + }); + + test('fromUtf8 ASCII with length', () { + final Pointer utf8 = _bytesFromList( + [72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 10, 0]).cast(); + final String end = utf8.toDartString(length: 5); + expect(end, 'Hello'); + }); + + test('fromUtf8 emoji with length', () { + final Pointer utf8 = _bytesFromList( + [240, 159, 152, 142, 240, 159, 145, 191, 240, 159, 146, 172, 0]).cast(); + final String end = utf8.toDartString(length: 4); + expect(end, '😎'); + }); + + test('fromUtf8 with zero length', () { + final Pointer utf8 = _bytesFromList( + [72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 10, 0]).cast(); + final String end = utf8.toDartString(length: 0); + expect(end, ''); + }); + + test('fromUtf8 with negative length', () { + final Pointer utf8 = _bytesFromList( + [72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 10, 0]).cast(); + expect(() => utf8.toDartString(length: -1), throwsRangeError); + }); + + test('fromUtf8 with length and containing a zero byte', () { + final Pointer utf8 = _bytesFromList( + [72, 101, 108, 108, 111, 0, 87, 111, 114, 108, 100, 33, 10]).cast(); + final String end = utf8.toDartString(length: 13); + expect(end, 'Hello\x00World!\n'); + }); + + test('length', () { + final string = 'Hello'; + final utf8Pointer = string.toNativeUtf8(); + expect(utf8Pointer.length, 5); + calloc.free(utf8Pointer); + }); + + test('nullptr.toDartString()', () { + final Pointer utf8 = nullptr; + try { + utf8.toDartString(); + } on UnsupportedError { + return; + } + fail('Expected an error.'); + }); + + test('nullptr.length', () { + final Pointer utf8 = nullptr; + try { + utf8.length; + } on UnsupportedError { + return; + } + fail('Expected an error.'); + }); + + test('zero terminated', () { + final string = 'Hello'; + final utf8Pointer = string.toNativeUtf8(); + final charPointer = utf8Pointer.cast(); + expect(charPointer[utf8Pointer.length], 0); + calloc.free(utf8Pointer); + }); +}