Verify binaries using hashes (#813)

* Squashed commit: Verify using hashes * Set version to newest tag * Reset hashes * Remove checking from intl4x build * Fix path * Add pub get * Do not fail fast * Remove checks for local hashes * Switch order of execution * Fix setting envs * Cleaner separation * Remove unused * Fix paths * Try different env setter * Compile to arm * Add nicer check for LOCAL_ICU4X_BINARY * Try different env for win * Point to correct macos * Path seperators * replace slashes * try something different * non relative * More tryes * Copy local file * Add newline * Add wildcard
dart-lang · Jul 30, 2024 · deeda2f · deeda2f
1 parent 8d63e87
commit deeda2f
Show file tree

Hide file tree

Showing 27 changed files with 1,078 additions and 73 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1 +1 @@
-pkgs/intl4x/lib/src/bindings linguist-generated=true
+pkgs/intl4x/lib/src/bindings/* linguist-generated=true
diff --git a/.github/workflows/intl4x.yml b/.github/workflows/intl4x.yml
@@ -16,8 +16,8 @@ on:
     - cron: '0 0 * * 0' # weekly
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
+  build_checkout:
+    runs-on:  ${{ matrix.os }}
 
     env:
       ICU4X_BUILD_MODE: checkout
@@ -29,6 +29,7 @@ jobs:
     strategy:
       matrix:
         sdk: [stable, dev] # {pkgs.versions}
+        os: [ubuntu-latest, windows-latest, macos-latest]
         include:
           - sdk: dev
             run-tests: true
@@ -53,3 +54,114 @@ jobs:
 
       - run: dart --enable-experiment=native-assets test -p chrome
         if: ${{matrix.run-tests}}
+
+  build_fetch:
+    runs-on:  ${{ matrix.os }}
+
+    env:
+      ICU4X_BUILD_MODE: fetch
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+
+    defaults:
+      run:
+        working-directory: pkgs/intl4x
+
+    steps:
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
+        with:
+          submodules: true
+
+      - uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672
+        with:
+          sdk: dev
+
+      - run: dart --enable-experiment=native-assets pub get
+
+      - run: dart --enable-experiment=native-assets test
+
+  build_local:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+    runs-on: ${{ matrix.os }}
+
+    env:
+      ICU4X_BUILD_MODE: local
+
+    steps:
+    - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
+      with:
+        submodules: true
+
+    - name: Install Rust toolchains
+      run: |
+        rustup toolchain install stable
+    
+    - name: Show the selected Rust toolchain
+      run: rustup show
+
+    - uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672
+      with:
+        sdk: dev
+
+    - name: Build Linux
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        cd submodules/icu4x
+
+        mkdir bin
+
+        cd ffi/dart
+        dart pub get
+        cd ../..
+        dart run ffi/dart/tool/build_libs.dart bin/linux_x64 linux_x64 dynamic default_components,experimental_components
+
+    - name: Build Mac
+      if: matrix.os == 'macos-latest'
+      run: |
+        cd submodules/icu4x
+
+        mkdir bin
+
+        cd ffi/dart
+        dart pub get
+        cd ../..
+        dart run ffi/dart/tool/build_libs.dart bin/macos_arm64 macos_arm64 dynamic default_components,experimental_components
+
+    - name: Build Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        cd submodules/icu4x
+      
+        mkdir bin
+
+        cd ffi/dart
+        dart pub get
+        cd ../..
+        dart run ffi/dart/tool/build_libs.dart bin/windows_x64 windows_x64 dynamic default_components,experimental_components
+
+    - run: echo "LOCAL_ICU4X_BINARY=$(realpath submodules/icu4x/bin/linux_x64)" >> $GITHUB_ENV
+      if: matrix.os == 'ubuntu-latest'
+
+    - run: echo "LOCAL_ICU4X_BINARY=$(realpath submodules/icu4x/bin/macos_arm64)" >> $GITHUB_ENV
+      if: matrix.os == 'macos-latest'
+
+    - run: echo ("LOCAL_ICU4X_BINARY=" + (Get-Item submodules\icu4x\bin\windows_x64).FullName -replace '/', '\') >> $env:GITHUB_ENV
+      if: matrix.os == 'windows-latest'
+
+    - run: echo $LOCAL_ICU4X_BINARY
+
+    - name: Display structure of downloaded files
+      run: ls -R
+
+    - run: |
+        cd pkgs/intl4x
+        dart pub get
+
+    - run: |
+        cd pkgs/intl4x
+        dart --enable-experiment=native-assets test
diff --git a/.github/workflows/intl4x_artifacts.yml b/.github/workflows/intl4x_artifacts.yml
@@ -4,6 +4,10 @@ permissions:
   contents: write
 
 on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - pkgs/intl4x/hook/hashes.dart
   push:
     tags:
       - 'intl4x-icu*'
@@ -140,7 +144,38 @@ jobs:
       with:
         name: dart-${{matrix.os}}-libs
         path: submodules/icu4x/bin
-
+
+  check_hashes:
+    needs: dart-libs
+    runs-on: ubuntu-latest
+
+    env:
+      ICU4X_BUILD_MODE: local
+
+    steps:
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
+        with:
+          submodules: true
+
+      - name: Download binaries
+        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
+        with:
+          merge-multiple: true
+
+      - name: Display structure of downloaded files
+        run: ls -R
+
+      - uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672
+        with:
+          sdk: dev
+
+      - name: Check hashes of released artifacts
+        run: |
+          cd pkgs/intl4x
+          dart pub get
+          dart --enable-experiment=native-assets tool/generate_hashes.dart
+          git diff --exit-code
+
   release:
     needs: dart-libs
     runs-on: ubuntu-latest

diff --git a/pkgs/intl4x/CHANGELOG.md b/pkgs/intl4x/CHANGELOG.md
@@ -1,10 +1,11 @@
 ## 0.9.2-wip
 
 - Copy files instead of symlinking, for easier upgrading.
+- Get binaries from Github and check their hashes.
 
 ## 0.9.1
 
-- Small fixes in imports
+- Small fixes in imports.
 
 ## 0.9.0
 

diff --git a/pkgs/intl4x/hook/README.md b/pkgs/intl4x/hook/README.md
@@ -0,0 +1,14 @@
+### How to update the ICU4X version used in package:intl4x.
+
+#### First PR
+1. Create PR.
+2. Update `submodules/icu4x` to whatever branch/hash you want.
+3. Land PR.
+4. Tag with `intl4x-icu*`, push. This creates new release with the new binaries.
+
+#### Second PR
+1. Create PR.
+2. Run `bash tools/regenerate_bindings.sh`, and fix resulting errors.
+3. Update `const version` in `hook/version.dart` to tag.
+4. Regenerate hashes using `dart --enable-experiment=native-assets run tool/generate_hashes.dart`.
+5. Land PR.
diff --git a/pkgs/intl4x/hook/build.dart b/pkgs/intl4x/hook/build.dart
@@ -4,18 +4,24 @@
 
 import 'dart:io';
 
-import 'package:archive/archive_io.dart';
+import 'package:crypto/crypto.dart' show sha256;
 import 'package:native_assets_cli/native_assets_cli.dart';
 import 'package:path/path.dart' as path;
 
+import 'hashes.dart';
+import 'version.dart';
+
 const crateName = 'icu_capi';
 const package = 'intl4x';
 const assetId = 'src/bindings/lib.g.dart';
 
+final env = 'ICU4X_BUILD_MODE';
+
 void main(List<String> args) async {
   await build(args, (config, output) async {
-    final buildMode = switch (Platform.environment['ICU4X_BUILD_MODE']) {
-      'local' => LocalMode(),
+    final environmentBuildMode = Platform.environment[env];
+    final buildMode = switch (environmentBuildMode) {
+      'local' => LocalMode(config),
       'checkout' => CheckoutMode(config),
       'fetch' || null => FetchMode(config),
       String() => throw ArgumentError('''
@@ -30,6 +36,8 @@ Unknown build mode for icu4x. Set the `ICU4X_BUILD_MODE` environment variable wi
     };
 
     final builtLibrary = await buildMode.build();
+    // For debugging purposes
+    output.addMetadatum(env, environmentBuildMode ?? 'fetch');
 
     output.addAsset(NativeCodeAsset(
       package: package,
@@ -44,86 +52,91 @@ Unknown build mode for icu4x. Set the `ICU4X_BUILD_MODE` environment variable wi
       [
         ...buildMode.dependencies,
         config.packageRoot.resolve('hook/build.dart'),
-        //TODO: Fix this, currently causes a rebuild for checkout mode
-        //builtLibrary,
       ],
     );
   });
 }
 
-void unzipFirstFile({required File input, required File output}) {
-  final inputStream = InputFileStream(input.path);
-  final archive = ZipDecoder().decodeBuffer(inputStream);
-  final file = archive.files.firstOrNull;
-  // If it's a file and not a directory
-  if (file?.isFile ?? false) {
-    final outputStream = OutputFileStream(output.path);
-    file!.writeContent(outputStream);
-    outputStream.close();
-  }
-}
-
 sealed class BuildMode {
+  final BuildConfig config;
+
+  const BuildMode(this.config);
+
   List<Uri> get dependencies;
 
   Future<Uri> build();
 }
 
-final class FetchMode implements BuildMode {
-  final BuildConfig config;
-
-  FetchMode(this.config);
+final class FetchMode extends BuildMode {
+  FetchMode(super.config);
 
   @override
   Future<Uri> build() async {
-    // TODO: Get a nicer CDN than a generated link to a privately owned repo.
+    final target = '${config.targetOS}_${config.targetArchitecture}';
     final uri = Uri.parse(
-        'https://nightly.link/mosuem/i18n/workflows/intl4x_artifacts/main/lib-$platformName-latest.zip');
+        'https://github.com/dart-lang/i18n/releases/download/$version/$target');
     final request = await HttpClient().getUrl(uri);
     final response = await request.close();
     if (response.statusCode != 200) {
       throw ArgumentError('The request to $uri failed');
     }
-    final zippedDynamicLibrary =
-        File(path.join(Directory.systemTemp.path, 'tmp.zip'));
-    zippedDynamicLibrary.createSync();
-    await response.pipe(zippedDynamicLibrary.openWrite());
-
-    final dynamicLibrary =
-        File.fromUri(config.outputDirectory.resolve('icu4xlib'));
+    final dynamicLibrary = File.fromUri(
+        config.outputDirectory.resolve(config.targetOS.dylibFileName('icu4x')));
     await dynamicLibrary.create();
-    unzipFirstFile(input: zippedDynamicLibrary, output: dynamicLibrary);
-    return dynamicLibrary.uri;
-  }
+    await response.pipe(dynamicLibrary.openWrite());
 
-  String get platformName {
-    if (Platform.isMacOS) {
-      return 'macos';
-    } else if (Platform.isWindows) {
-      return 'windows';
+    final bytes = await dynamicLibrary.readAsBytes();
+    final fileHash = sha256.convert(bytes).toString();
+    final expectedFileHash = fileHashes[(
+      config.targetOS,
+      config.targetArchitecture,
+    )];
+    if (fileHash == expectedFileHash) {
+      return dynamicLibrary.uri;
     } else {
-      return 'ubuntu';
+      throw Exception(
+          'The pre-built binary for the target $target at $uri has a hash of '
+          '$fileHash, which does not match $expectedFileHash fixed in the '
+          'build hook of package:intl4x.');
     }
   }
 
   @override
   List<Uri> get dependencies => [];
 }
 
-final class LocalMode implements BuildMode {
-  String get _localBinaryPath => Platform.environment['LOCAL_ICU4X_BINARY']!;
+final class LocalMode extends BuildMode {
+  LocalMode(super.config);
+
+  String get _localBinaryPath {
+    final localPath = Platform.environment['LOCAL_ICU4X_BINARY'];
+    if (localPath != null) {
+      return localPath;
+    }
+    throw ArgumentError('`LOCAL_ICU4X_BINARY` is empty. '
+        'If the `ICU4X_BUILD_MODE` is set to `local`, the '
+        '`LOCAL_ICU4X_BINARY` environment variable must contain the path to '
+        'the binary.');
+  }
 
   @override
-  Future<Uri> build() async => Uri.file(_localBinaryPath);
+  Future<Uri> build() async {
+    final dylibFileName = config.targetOS.dylibFileName('icu4x');
+    final dylibFileUri = config.outputDirectory.resolve(dylibFileName);
+    final file = File(_localBinaryPath);
+    if (!(await file.exists())) {
+      throw FileSystemException('Could not find binary.', _localBinaryPath);
+    }
+    await file.copy(dylibFileUri.toFilePath(windows: Platform.isWindows));
+    return dylibFileUri;
+  }
 
   @override
   List<Uri> get dependencies => [Uri.file(_localBinaryPath)];
 }
 
-final class CheckoutMode implements BuildMode {
-  final BuildConfig config;
-
-  CheckoutMode(this.config);
+final class CheckoutMode extends BuildMode {
+  CheckoutMode(super.config);
 
   String? get workingDirectory => Platform.environment['LOCAL_ICU4X_CHECKOUT'];
 
@@ -235,7 +248,7 @@ Future<Uri> buildLib(BuildConfig config, String workingDirectory) async {
     if (!(await file.exists())) {
       throw FileSystemException('Building the dylib failed', builtPath);
     }
-    await file.copy(dylibFileUri.path);
+    await file.copy(dylibFileUri.toFilePath(windows: Platform.isWindows));
   }
   return dylibFileUri;
 }
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		pkgs/intl4x/lib/src/bindings linguist-generated=true
		pkgs/intl4x/lib/src/bindings/* linguist-generated=true