Skip to content

Commit deeda2f

Browse files
authored
Verify binaries using hashes (#813)
* Squashed commit: Verify using hashes * Set version to newest tag * Reset hashes * Remove checking from intl4x build * Fix path * Add pub get * Do not fail fast * Remove checks for local hashes * Switch order of execution * Fix setting envs * Cleaner separation * Remove unused * Fix paths * Try different env setter * Compile to arm * Add nicer check for LOCAL_ICU4X_BINARY * Try different env for win * Point to correct macos * Path seperators * replace slashes * try something different * non relative * More tryes * Copy local file * Add newline * Add wildcard
1 parent 8d63e87 commit deeda2f

27 files changed

+1078
-73
lines changed

.gitattributes

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
pkgs/intl4x/lib/src/bindings linguist-generated=true
1+
pkgs/intl4x/lib/src/bindings/* linguist-generated=true

.github/workflows/intl4x.yml

Lines changed: 114 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ on:
1616
- cron: '0 0 * * 0' # weekly
1717

1818
jobs:
19-
build:
20-
runs-on: ubuntu-latest
19+
build_checkout:
20+
runs-on: ${{ matrix.os }}
2121

2222
env:
2323
ICU4X_BUILD_MODE: checkout
@@ -29,6 +29,7 @@ jobs:
2929
strategy:
3030
matrix:
3131
sdk: [stable, dev] # {pkgs.versions}
32+
os: [ubuntu-latest, windows-latest, macos-latest]
3233
include:
3334
- sdk: dev
3435
run-tests: true
@@ -53,3 +54,114 @@ jobs:
5354

5455
- run: dart --enable-experiment=native-assets test -p chrome
5556
if: ${{matrix.run-tests}}
57+
58+
build_fetch:
59+
runs-on: ${{ matrix.os }}
60+
61+
env:
62+
ICU4X_BUILD_MODE: fetch
63+
64+
strategy:
65+
matrix:
66+
os: [ubuntu-latest, windows-latest, macos-latest]
67+
68+
defaults:
69+
run:
70+
working-directory: pkgs/intl4x
71+
72+
steps:
73+
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
74+
with:
75+
submodules: true
76+
77+
- uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672
78+
with:
79+
sdk: dev
80+
81+
- run: dart --enable-experiment=native-assets pub get
82+
83+
- run: dart --enable-experiment=native-assets test
84+
85+
build_local:
86+
strategy:
87+
fail-fast: false
88+
matrix:
89+
os: [ ubuntu-latest, macos-latest, windows-latest ]
90+
runs-on: ${{ matrix.os }}
91+
92+
env:
93+
ICU4X_BUILD_MODE: local
94+
95+
steps:
96+
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
97+
with:
98+
submodules: true
99+
100+
- name: Install Rust toolchains
101+
run: |
102+
rustup toolchain install stable
103+
104+
- name: Show the selected Rust toolchain
105+
run: rustup show
106+
107+
- uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672
108+
with:
109+
sdk: dev
110+
111+
- name: Build Linux
112+
if: matrix.os == 'ubuntu-latest'
113+
run: |
114+
cd submodules/icu4x
115+
116+
mkdir bin
117+
118+
cd ffi/dart
119+
dart pub get
120+
cd ../..
121+
dart run ffi/dart/tool/build_libs.dart bin/linux_x64 linux_x64 dynamic default_components,experimental_components
122+
123+
- name: Build Mac
124+
if: matrix.os == 'macos-latest'
125+
run: |
126+
cd submodules/icu4x
127+
128+
mkdir bin
129+
130+
cd ffi/dart
131+
dart pub get
132+
cd ../..
133+
dart run ffi/dart/tool/build_libs.dart bin/macos_arm64 macos_arm64 dynamic default_components,experimental_components
134+
135+
- name: Build Windows
136+
if: matrix.os == 'windows-latest'
137+
run: |
138+
cd submodules/icu4x
139+
140+
mkdir bin
141+
142+
cd ffi/dart
143+
dart pub get
144+
cd ../..
145+
dart run ffi/dart/tool/build_libs.dart bin/windows_x64 windows_x64 dynamic default_components,experimental_components
146+
147+
- run: echo "LOCAL_ICU4X_BINARY=$(realpath submodules/icu4x/bin/linux_x64)" >> $GITHUB_ENV
148+
if: matrix.os == 'ubuntu-latest'
149+
150+
- run: echo "LOCAL_ICU4X_BINARY=$(realpath submodules/icu4x/bin/macos_arm64)" >> $GITHUB_ENV
151+
if: matrix.os == 'macos-latest'
152+
153+
- run: echo ("LOCAL_ICU4X_BINARY=" + (Get-Item submodules\icu4x\bin\windows_x64).FullName -replace '/', '\') >> $env:GITHUB_ENV
154+
if: matrix.os == 'windows-latest'
155+
156+
- run: echo $LOCAL_ICU4X_BINARY
157+
158+
- name: Display structure of downloaded files
159+
run: ls -R
160+
161+
- run: |
162+
cd pkgs/intl4x
163+
dart pub get
164+
165+
- run: |
166+
cd pkgs/intl4x
167+
dart --enable-experiment=native-assets test

.github/workflows/intl4x_artifacts.yml

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ permissions:
44
contents: write
55

66
on:
7+
pull_request:
8+
branches: [ main ]
9+
paths:
10+
- pkgs/intl4x/hook/hashes.dart
711
push:
812
tags:
913
- 'intl4x-icu*'
@@ -140,7 +144,38 @@ jobs:
140144
with:
141145
name: dart-${{matrix.os}}-libs
142146
path: submodules/icu4x/bin
143-
147+
148+
check_hashes:
149+
needs: dart-libs
150+
runs-on: ubuntu-latest
151+
152+
env:
153+
ICU4X_BUILD_MODE: local
154+
155+
steps:
156+
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
157+
with:
158+
submodules: true
159+
160+
- name: Download binaries
161+
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
162+
with:
163+
merge-multiple: true
164+
165+
- name: Display structure of downloaded files
166+
run: ls -R
167+
168+
- uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672
169+
with:
170+
sdk: dev
171+
172+
- name: Check hashes of released artifacts
173+
run: |
174+
cd pkgs/intl4x
175+
dart pub get
176+
dart --enable-experiment=native-assets tool/generate_hashes.dart
177+
git diff --exit-code
178+
144179
release:
145180
needs: dart-libs
146181
runs-on: ubuntu-latest

pkgs/intl4x/CHANGELOG.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
## 0.9.2-wip
22

33
- Copy files instead of symlinking, for easier upgrading.
4+
- Get binaries from Github and check their hashes.
45

56
## 0.9.1
67

7-
- Small fixes in imports
8+
- Small fixes in imports.
89

910
## 0.9.0
1011

pkgs/intl4x/hook/README.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
### How to update the ICU4X version used in package:intl4x.
2+
3+
#### First PR
4+
1. Create PR.
5+
2. Update `submodules/icu4x` to whatever branch/hash you want.
6+
3. Land PR.
7+
4. Tag with `intl4x-icu*`, push. This creates new release with the new binaries.
8+
9+
#### Second PR
10+
1. Create PR.
11+
2. Run `bash tools/regenerate_bindings.sh`, and fix resulting errors.
12+
3. Update `const version` in `hook/version.dart` to tag.
13+
4. Regenerate hashes using `dart --enable-experiment=native-assets run tool/generate_hashes.dart`.
14+
5. Land PR.

pkgs/intl4x/hook/build.dart

Lines changed: 60 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,24 @@
44

55
import 'dart:io';
66

7-
import 'package:archive/archive_io.dart';
7+
import 'package:crypto/crypto.dart' show sha256;
88
import 'package:native_assets_cli/native_assets_cli.dart';
99
import 'package:path/path.dart' as path;
1010

11+
import 'hashes.dart';
12+
import 'version.dart';
13+
1114
const crateName = 'icu_capi';
1215
const package = 'intl4x';
1316
const assetId = 'src/bindings/lib.g.dart';
1417

18+
final env = 'ICU4X_BUILD_MODE';
19+
1520
void main(List<String> args) async {
1621
await build(args, (config, output) async {
17-
final buildMode = switch (Platform.environment['ICU4X_BUILD_MODE']) {
18-
'local' => LocalMode(),
22+
final environmentBuildMode = Platform.environment[env];
23+
final buildMode = switch (environmentBuildMode) {
24+
'local' => LocalMode(config),
1925
'checkout' => CheckoutMode(config),
2026
'fetch' || null => FetchMode(config),
2127
String() => throw ArgumentError('''
@@ -30,6 +36,8 @@ Unknown build mode for icu4x. Set the `ICU4X_BUILD_MODE` environment variable wi
3036
};
3137

3238
final builtLibrary = await buildMode.build();
39+
// For debugging purposes
40+
output.addMetadatum(env, environmentBuildMode ?? 'fetch');
3341

3442
output.addAsset(NativeCodeAsset(
3543
package: package,
@@ -44,86 +52,91 @@ Unknown build mode for icu4x. Set the `ICU4X_BUILD_MODE` environment variable wi
4452
[
4553
...buildMode.dependencies,
4654
config.packageRoot.resolve('hook/build.dart'),
47-
//TODO: Fix this, currently causes a rebuild for checkout mode
48-
//builtLibrary,
4955
],
5056
);
5157
});
5258
}
5359

54-
void unzipFirstFile({required File input, required File output}) {
55-
final inputStream = InputFileStream(input.path);
56-
final archive = ZipDecoder().decodeBuffer(inputStream);
57-
final file = archive.files.firstOrNull;
58-
// If it's a file and not a directory
59-
if (file?.isFile ?? false) {
60-
final outputStream = OutputFileStream(output.path);
61-
file!.writeContent(outputStream);
62-
outputStream.close();
63-
}
64-
}
65-
6660
sealed class BuildMode {
61+
final BuildConfig config;
62+
63+
const BuildMode(this.config);
64+
6765
List<Uri> get dependencies;
6866

6967
Future<Uri> build();
7068
}
7169

72-
final class FetchMode implements BuildMode {
73-
final BuildConfig config;
74-
75-
FetchMode(this.config);
70+
final class FetchMode extends BuildMode {
71+
FetchMode(super.config);
7672

7773
@override
7874
Future<Uri> build() async {
79-
// TODO: Get a nicer CDN than a generated link to a privately owned repo.
75+
final target = '${config.targetOS}_${config.targetArchitecture}';
8076
final uri = Uri.parse(
81-
'https://nightly.link/mosuem/i18n/workflows/intl4x_artifacts/main/lib-$platformName-latest.zip');
77+
'https://github.com/dart-lang/i18n/releases/download/$version/$target');
8278
final request = await HttpClient().getUrl(uri);
8379
final response = await request.close();
8480
if (response.statusCode != 200) {
8581
throw ArgumentError('The request to $uri failed');
8682
}
87-
final zippedDynamicLibrary =
88-
File(path.join(Directory.systemTemp.path, 'tmp.zip'));
89-
zippedDynamicLibrary.createSync();
90-
await response.pipe(zippedDynamicLibrary.openWrite());
91-
92-
final dynamicLibrary =
93-
File.fromUri(config.outputDirectory.resolve('icu4xlib'));
83+
final dynamicLibrary = File.fromUri(
84+
config.outputDirectory.resolve(config.targetOS.dylibFileName('icu4x')));
9485
await dynamicLibrary.create();
95-
unzipFirstFile(input: zippedDynamicLibrary, output: dynamicLibrary);
96-
return dynamicLibrary.uri;
97-
}
86+
await response.pipe(dynamicLibrary.openWrite());
9887

99-
String get platformName {
100-
if (Platform.isMacOS) {
101-
return 'macos';
102-
} else if (Platform.isWindows) {
103-
return 'windows';
88+
final bytes = await dynamicLibrary.readAsBytes();
89+
final fileHash = sha256.convert(bytes).toString();
90+
final expectedFileHash = fileHashes[(
91+
config.targetOS,
92+
config.targetArchitecture,
93+
)];
94+
if (fileHash == expectedFileHash) {
95+
return dynamicLibrary.uri;
10496
} else {
105-
return 'ubuntu';
97+
throw Exception(
98+
'The pre-built binary for the target $target at $uri has a hash of '
99+
'$fileHash, which does not match $expectedFileHash fixed in the '
100+
'build hook of package:intl4x.');
106101
}
107102
}
108103

109104
@override
110105
List<Uri> get dependencies => [];
111106
}
112107

113-
final class LocalMode implements BuildMode {
114-
String get _localBinaryPath => Platform.environment['LOCAL_ICU4X_BINARY']!;
108+
final class LocalMode extends BuildMode {
109+
LocalMode(super.config);
110+
111+
String get _localBinaryPath {
112+
final localPath = Platform.environment['LOCAL_ICU4X_BINARY'];
113+
if (localPath != null) {
114+
return localPath;
115+
}
116+
throw ArgumentError('`LOCAL_ICU4X_BINARY` is empty. '
117+
'If the `ICU4X_BUILD_MODE` is set to `local`, the '
118+
'`LOCAL_ICU4X_BINARY` environment variable must contain the path to '
119+
'the binary.');
120+
}
115121

116122
@override
117-
Future<Uri> build() async => Uri.file(_localBinaryPath);
123+
Future<Uri> build() async {
124+
final dylibFileName = config.targetOS.dylibFileName('icu4x');
125+
final dylibFileUri = config.outputDirectory.resolve(dylibFileName);
126+
final file = File(_localBinaryPath);
127+
if (!(await file.exists())) {
128+
throw FileSystemException('Could not find binary.', _localBinaryPath);
129+
}
130+
await file.copy(dylibFileUri.toFilePath(windows: Platform.isWindows));
131+
return dylibFileUri;
132+
}
118133

119134
@override
120135
List<Uri> get dependencies => [Uri.file(_localBinaryPath)];
121136
}
122137

123-
final class CheckoutMode implements BuildMode {
124-
final BuildConfig config;
125-
126-
CheckoutMode(this.config);
138+
final class CheckoutMode extends BuildMode {
139+
CheckoutMode(super.config);
127140

128141
String? get workingDirectory => Platform.environment['LOCAL_ICU4X_CHECKOUT'];
129142

@@ -235,7 +248,7 @@ Future<Uri> buildLib(BuildConfig config, String workingDirectory) async {
235248
if (!(await file.exists())) {
236249
throw FileSystemException('Building the dylib failed', builtPath);
237250
}
238-
await file.copy(dylibFileUri.path);
251+
await file.copy(dylibFileUri.toFilePath(windows: Platform.isWindows));
239252
}
240253
return dylibFileUri;
241254
}

0 commit comments

Comments
 (0)