Skip to content

Commit

Permalink
Update to Swift 5.7.1, save SDK on CI, and disable new NIO test that'…
Browse files Browse the repository at this point in the history
…s failing
  • Loading branch information
finagolfin committed Nov 11, 2022
1 parent f2e8a01 commit 64bcef4
Show file tree
Hide file tree
Showing 6 changed files with 80 additions and 57 deletions.
65 changes: 34 additions & 31 deletions .github/workflows/sdks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ jobs:
uses: actions/cache@v2
with:
path: ~/swift-${{ matrix.version }}-android-${{ matrix.arch }}-*-sdk.tar.xz
key: ${{ matrix.version }}-${{ steps.version.outputs.latest }}-${{ matrix.arch }}-release-sdk
key: ${{ matrix.version }}-${{ steps.version.outputs.latest }}-${{ matrix.arch }}-upload-sdk
- name: Clone
uses: actions/checkout@v2
with:
Expand Down Expand Up @@ -120,6 +120,7 @@ jobs:
./swift/utils/build-script -RA --skip-build-cmark --build-llvm=0 --android --android-ndk $ANDROID_NDK --android-arch ${{ matrix.arch }} --android-api-level $ANDROID_API_LEVEL --build-swift-tools=0 --native-swift-tools-path=`pwd`/$SWIFT_TAG-ubuntu20.04/usr/bin --native-clang-tools-path=`pwd`/$SWIFT_TAG-ubuntu20.04/usr/bin --cross-compile-hosts=android-${{ matrix.arch }} --cross-compile-deps-path=$SDK --skip-local-build --build-swift-static-stdlib --xctest --skip-early-swift-driver --install-swift --install-libdispatch --install-foundation --install-xctest --install-destdir=$SDK --swift-install-components='clang-resource-dir-symlink;license;stdlib;sdk-overlay' --cross-compile-append-host-target-to-destdir=False -b -p --install-llbuild --sourcekit-lsp
cp $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/$(echo ${{ matrix.arch }} | sed "s/v7//")-linux-android*/libc++_shared.so $SDK/usr/lib
patchelf --set-rpath \$ORIGIN $SDK/usr/lib/swift/android/libdispatch.so
patchelf --set-rpath \$ORIGIN/../..:\$ORIGIN $SDK/usr/lib/swift/android/lib[FXs]*.so
Expand Down Expand Up @@ -255,44 +256,45 @@ jobs:
with:
path: ~/termux-debug.apk
key: termux-app
- name: Put tests in one directory to push to Android x86_64 emulator
if: ${{ matrix.arch == 'x86_64' }}
- name: Put SDK and x86_64 tests in one directory to push to Android x86_64 emulator
run: |
mkdir -p pack/lib/swift/android
TARGET="x86_64-unknown-linux-android$ANDROID_API_LEVEL"
cp swift-argument-parser/.build/$TARGET/debug/{generate-manual,math,repeat,roll,swift-argument-parserPackageTests.xctest} pack
mkdir pack
cp ~/swift-${{ matrix.version }}-android-${{ matrix.arch }}*-${ANDROID_API_LEVEL}-sdk.tar.xz pack
wget -q https://raw.githubusercontent.com/termux/termux-elf-cleaner/v1.10/termux-elf-cleaner.cpp
wget -q https://raw.githubusercontent.com/termux/termux-elf-cleaner/v1.10/elf.h
clang -D__ANDROID_API__=$ANDROID_API_LEVEL -o elf-cleaner termux-elf-cleaner.cpp
if [ ${{ matrix.arch }} = 'x86_64' ]; then
mkdir -p pack/lib/swift/android
TARGET="x86_64-unknown-linux-android$ANDROID_API_LEVEL"
cp swift-argument-parser/.build/$TARGET/debug/{generate-manual,math,repeat,roll,swift-argument-parserPackageTests.xctest} pack
./elf-cleaner pack/{generate-manual,math,repeat,roll} || true
wget -q https://raw.githubusercontent.com/termux/termux-elf-cleaner/v1.10/termux-elf-cleaner.cpp
wget -q https://raw.githubusercontent.com/termux/termux-elf-cleaner/v1.10/elf.h
clang -D__ANDROID_API__=$ANDROID_API_LEVEL -o elf-cleaner termux-elf-cleaner.cpp
./elf-cleaner pack/{generate-manual,math,repeat,roll} || true
cp swift-crypto/.build/$TARGET/debug/swift-cryptoPackageTests.xctest pack
cp swift-nio/.build/$TARGET/debug/swift-nioPackageTests.xctest pack
cp sn/.build/$TARGET/debug/swift-numericsPackageTests.xctest pack
cp ss/.build/$TARGET/debug/swift-systemPackageTests.xctest pack
cp sco/.build/$TARGET/debug/swift-collectionsPackageTests.xctest pack
cp sns/.build/$TARGET/debug/swift-nio-sshPackageTests.xctest pack
cp snl/.build/$TARGET/debug/swift-nio-sslPackageTests.xctest pack
cp yams/.build/$TARGET/debug/YamsPackageTests.xctest pack
cp snh/.build/$TARGET/debug/swift-nio-http2PackageTests.xctest pack
cp sa/.build/$TARGET/debug/swift-algorithmsPackageTests.xctest pack
cp swift-crypto/.build/$TARGET/debug/swift-cryptoPackageTests.xctest pack
cp swift-nio/.build/$TARGET/debug/swift-nioPackageTests.xctest pack
cp sn/.build/$TARGET/debug/swift-numericsPackageTests.xctest pack
cp ss/.build/$TARGET/debug/swift-systemPackageTests.xctest pack
cp sco/.build/$TARGET/debug/swift-collectionsPackageTests.xctest pack
cp sns/.build/$TARGET/debug/swift-nio-sshPackageTests.xctest pack
cp snl/.build/$TARGET/debug/swift-nio-sslPackageTests.xctest pack
cp yams/.build/$TARGET/debug/YamsPackageTests.xctest pack
cp snh/.build/$TARGET/debug/swift-nio-http2PackageTests.xctest pack
cp sa/.build/$TARGET/debug/swift-algorithmsPackageTests.xctest pack
mkdir pack/crypto-vectors
cp swift-crypto/Tests/Test\ Vectors/* swift-crypto/Tests/_CryptoExtrasVectors/* pack/crypto-vectors
cp yams/Tests/YamsTests/Fixtures/SourceKitten#289/debug.yaml pack
cp sdk-config/swift-${{ matrix.version }}*-android-x86_64-*${ANDROID_API_LEVEL}-sdk/usr/lib/lib*so pack/lib
cp sdk-config/swift-${{ matrix.version }}*-android-x86_64-*${ANDROID_API_LEVEL}-sdk/usr/lib/swift/android/lib*so pack/lib/swift/android
cp $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/x86_64-linux-android/libc++_shared.so pack/lib/swift/android
cp ~/termux-debug.apk pack
mkdir pack/crypto-vectors
cp swift-crypto/Tests/Test\ Vectors/* swift-crypto/Tests/_CryptoExtrasVectors/* pack/crypto-vectors
cp yams/Tests/YamsTests/Fixtures/SourceKitten#289/debug.yaml pack
cp sdk-config/swift-${{ matrix.version }}*-android-x86_64-*${ANDROID_API_LEVEL}-sdk/usr/lib/lib*so pack/lib
cp sdk-config/swift-${{ matrix.version }}*-android-x86_64-*${ANDROID_API_LEVEL}-sdk/usr/lib/swift/android/lib*so pack/lib/swift/android
mv pack/lib/libc++_shared.so pack/lib/swift/android
cp ~/termux-debug.apk pack
fi
- name: Upload test directory
if: ${{ matrix.arch == 'x86_64' }}
uses: actions/upload-artifact@v2
with:
name: tests-${{ matrix.version }}
name: sdk-tests-${{ matrix.version }}-${{ matrix.arch }}
path: pack/
retention-days: 3
run-tests-on-android-x86_64-emulator:
runs-on: macos-latest
needs: build-sdk-and-tests
Expand All @@ -304,14 +306,15 @@ jobs:
- name: Download test directory
uses: actions/download-artifact@v2
with:
name: tests-${{ matrix.version }}
name: sdk-tests-${{ matrix.version }}-x86_64
path: pack
- name: Run tests on Android x86_64 emulator
uses: reactivecircus/android-emulator-runner@v2
with:
api-level: 24
arch: x86_64
script: |
rm pack/swift-${{ matrix.version }}-android-x86_64*-sdk.tar.xz
adb install pack/termux-debug.apk
chmod 755 pack/{generate-manual,math,repeat,roll,*.xctest}
adb push pack /data/local/tmp
Expand Down
40 changes: 20 additions & 20 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,21 @@ toolchain for AArch64, armv7, and x86_64, builds several Swift packages against
those SDKs, and then runs their tests in the Android x86_64
emulator](https://github.com/buttaface/swift-android-sdk/blob/main/.github/workflows/sdks.yml).

To build with a Swift 5.7 SDK, first download [the latest Android LTS NDK
25b](https://developer.android.com/ndk/downloads) and [Swift 5.7
To build with a Swift 5.7.1 SDK, first download [the latest Android LTS NDK
25b](https://developer.android.com/ndk/downloads) and [Swift 5.7.1
compiler](https://swift.org/download/#releases) (make sure to install the Swift
compiler's dependencies listed there). Unpack these archives and the SDK.

I will write up a Swift script to do this SDK configuration one day, but you
will need to do it manually for now (I'll show aarch64 below, the same will
need to be done separately for the armv7 or x86_64 SDKs).

Change the symbolic link at `swift-5.7-android-aarch64-24-sdk/usr/lib/swift/clang`
Change the symbolic link at `swift-5.7.1-android-aarch64-24-sdk/usr/lib/swift/clang`
to point to the clang headers that come with your swift compiler, eg

```
ln -sf /home/yourname/swift-5.7-RELEASE-ubuntu20.04/usr/lib/clang/13.0.0
swift-5.7-android-aarch64-24-sdk/usr/lib/swift/clang
ln -sf /home/yourname/swift-5.7.1-RELEASE-ubuntu20.04/usr/lib/clang/13.0.0
swift-5.7.1-android-aarch64-24-sdk/usr/lib/swift/clang
```

Next, modify the cross-compilation JSON file `android-aarch64.json` in this repo
Expand All @@ -30,11 +30,11 @@ similarly:
1. All paths to the NDK should change from `/home/butta/android-ndk-r25b`
to the path to your NDK, `/home/yourname/android-ndk-r25b`.

2. The path to the compiler should change from `/home/butta/swift-5.7-RELEASE-ubuntu20.04`
to the path to your Swift compiler, `/home/yourname/swift-5.7-RELEASE-centos8`.
2. The path to the compiler should change from `/home/butta/swift-5.7.1-RELEASE-ubuntu20.04`
to the path to your Swift compiler, `/home/yourname/swift-5.7.1-RELEASE-centos8`.

3. The path to the Android SDK should change from `/home/butta/swift-5.7-android-aarch64-24-sdk`
to the path where you unpacked the Android SDK, `/home/yourname/swift-5.7-android-aarch64-24-sdk`.
3. The path to the Android SDK should change from `/home/butta/swift-5.7.1-android-aarch64-24-sdk`
to the path where you unpacked the Android SDK, `/home/yourname/swift-5.7.1-android-aarch64-24-sdk`.

Now you're ready to cross-compile a Swift package with the cross-compilation
configuration JSON file, `android-aarch64.json`, and run its tests on Android.
Expand All @@ -44,9 +44,9 @@ git clone --depth 1 https://github.com/apple/swift-argument-parser.git
cd swift-argument-parser/
/home/yourname/swift-5.7-RELEASE-ubuntu20.04/usr/bin/swift build --build-tests
/home/yourname/swift-5.7.1-RELEASE-ubuntu20.04/usr/bin/swift build --build-tests
--enable-test-discovery --destination ~/swift-android-sdk/android-aarch64.json
-Xlinker -rpath -Xlinker \$ORIGIN/swift-5.7-android-aarch64-24-sdk/usr/lib/swift/android
-Xlinker -rpath -Xlinker \$ORIGIN/swift-5.7.1-android-aarch64-24-sdk/usr/lib/swift/android
```
This will cross-compile the package for Android aarch64 and produce a test
runner executable with the `.xctest` extension, in this case at
Expand All @@ -60,13 +60,13 @@ one depends on the example executables `generate-manual`, `math`, `repeat`, and
data in the repo: I've had success moving this data with the test runner, after
modifying the test source so it has the path to this test data in the Android
test environment. See the example of [swift-crypto on the
CI](https://github.com/buttaface/swift-android-sdk/blob/5.7/.github/workflows/sdks.yml#L282).
CI](https://github.com/buttaface/swift-android-sdk/blob/5.7.1/.github/workflows/sdks.yml#L286).

You can copy these executables and the SDK to [an emulator or a USB
debugging-enabled device with adb](https://github.com/apple/swift/blob/release/5.7/docs/Android.md#3-deploying-the-build-products-to-the-device),
or put them on an Android device with [a terminal emulator app like Termux](https://termux.dev/en/).
I test aarch64 with Termux so I'll show how to run the test runner there, but
the process is similar with adb, [as can be seen on the CI](https://github.com/buttaface/swift-android-sdk/blob/5.7/.github/workflows/sdks.yml#L316).
the process is similar with adb, [as can be seen on the CI](https://github.com/buttaface/swift-android-sdk/blob/5.7.1/.github/workflows/sdks.yml#L320).

Copy the test executables to the same directory as the SDK:
```
Expand All @@ -79,10 +79,10 @@ uname -m # check if you're running on the right architecture, should say `aarch6
cd # move to the Termux app's home directory
pkg install openssh
scp [email protected]:{swift-5.7-android-aarch64-24-sdk.tar.xz,
scp [email protected]:{swift-5.7.1-android-aarch64-24-sdk.tar.xz,
swift-argument-parserPackageTests.xctest,generate-manual,math,repeat,roll} .
tar xf swift-5.7-android-aarch64-24-sdk.tar.xz
tar xf swift-5.7.1-android-aarch64-24-sdk.tar.xz
./swift-argument-parserPackageTests.xctest
```
Expand Down Expand Up @@ -129,9 +129,9 @@ packagingOptions {

# Building the Android SDKs

Download the Swift 5.7 compiler and Android NDK 25b as above. Check out this
Download the Swift 5.7.1 compiler and Android NDK 25b as above. Check out this
repo and run
`SWIFT_TAG=swift-5.7-RELEASE ANDROID_ARCH=aarch64 swift get-packages-and-swift-source.swift`
`SWIFT_TAG=swift-5.7.1-RELEASE ANDROID_ARCH=aarch64 swift get-packages-and-swift-source.swift`
to get some prebuilt Android libraries and the Swift source to build the SDK. If
you pass in a different tag like `swift-DEVELOPMENT-SNAPSHOT-2022-10-18-a`
for the latest Swift trunk snapshot and pass in the path to the corresponding
Expand All @@ -152,8 +152,8 @@ substituted instead:
```
./swift/utils/build-script -RA --skip-build-cmark --build-llvm=0 --android
--android-ndk /home/butta/android-ndk-r25b/ --android-arch aarch64 --android-api-level 24
--build-swift-tools=0 --native-swift-tools-path=/home/butta/swift-5.7-RELEASE-ubuntu20.04/usr/bin/
--native-clang-tools-path=/home/butta/swift-5.7-RELEASE-ubuntu20.04/usr/bin/
--build-swift-tools=0 --native-swift-tools-path=/home/butta/swift-5.7.1-RELEASE-ubuntu20.04/usr/bin/
--native-clang-tools-path=/home/butta/swift-5.7.1-RELEASE-ubuntu20.04/usr/bin/
--host-cc=/usr/bin/clang-13 --host-cxx=/usr/bin/clang++-13
--cross-compile-hosts=android-aarch64 --cross-compile-deps-path=/home/butta/swift-release-android-aarch64-24-sdk
--skip-local-build --xctest --swift-install-components='clang-resource-dir-symlink;license;stdlib;sdk-overlay'
Expand Down Expand Up @@ -226,7 +226,7 @@ instead, so this Swift SDK for Android could be built without using
any prebuilt Termux packages, if you're willing to put in the effort to
cross-compile them yourself, for example, against a different Android API.

Finally, it gets [the 5.7 source](https://github.com/apple/swift/releases/tag/swift-5.7-RELEASE)
Finally, it gets [the 5.7.1 source](https://github.com/apple/swift/releases/tag/swift-5.7.1-RELEASE)
tarballs for five Swift repos and renames them to `llvm-project/`, `swift/`,
`swift-corelibs-libdispatch`, `swift-corelibs-foundation`, and
`swift-corelibs-xctest`, as required by the Swift `build-script`, and creates
Expand Down
4 changes: 2 additions & 2 deletions android-aarch64.json
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
{
"version": 1,
"target": "aarch64-unknown-linux-android24",
"toolchain-bin-dir": "/home/butta/swift-5.7-RELEASE-ubuntu20.04/usr/bin",
"toolchain-bin-dir": "/home/butta/swift-5.7.1-RELEASE-ubuntu20.04/usr/bin",
"sdk": "/home/butta/android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/sysroot",
"extra-cc-flags": [
"-fPIC"
],
"extra-swiftc-flags": [
"-resource-dir", "/home/butta/swift-5.7-android-aarch64-24-sdk/usr/lib/swift",
"-resource-dir", "/home/butta/swift-5.7.1-android-aarch64-24-sdk/usr/lib/swift",
"-tools-directory", "/home/butta/android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/bin",
],
"extra-cpp-flags": [
Expand Down
4 changes: 2 additions & 2 deletions android-armv7.json
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
{
"version": 1,
"target": "armv7-unknown-linux-androideabi24",
"toolchain-bin-dir": "/home/butta/swift-5.7-RELEASE-ubuntu20.04/usr/bin",
"toolchain-bin-dir": "/home/butta/swift-5.7.1-RELEASE-ubuntu20.04/usr/bin",
"sdk": "/home/butta/android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/sysroot",
"extra-cc-flags": [
"-fPIC"
],
"extra-swiftc-flags": [
"-resource-dir", "/home/butta/swift-5.7-android-armv7-24-sdk/usr/lib/swift",
"-resource-dir", "/home/butta/swift-5.7.1-android-armv7-24-sdk/usr/lib/swift",
"-tools-directory", "/home/butta/android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/bin",
],
"extra-cpp-flags": [
Expand Down
4 changes: 2 additions & 2 deletions android-x86_64.json
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
{
"version": 1,
"target": "x86_64-unknown-linux-android24",
"toolchain-bin-dir": "/home/butta/swift-5.7-RELEASE-ubuntu20.04/usr/bin",
"toolchain-bin-dir": "/home/butta/swift-5.7.1-RELEASE-ubuntu20.04/usr/bin",
"sdk": "/home/butta/android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/sysroot",
"extra-cc-flags": [
"-fPIC"
],
"extra-swiftc-flags": [
"-resource-dir", "/home/butta/swift-5.7-android-x86_64-24-sdk/usr/lib/swift",
"-resource-dir", "/home/butta/swift-5.7.1-android-x86_64-24-sdk/usr/lib/swift",
"-tools-directory", "/home/butta/android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/bin",
],
"extra-cpp-flags": [
Expand Down
20 changes: 20 additions & 0 deletions package-patches/swift-nio-change-tmp-and-disable-tests.patch
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,26 @@ index f3e452dc..a2db1282 100644
("testReceiveEcnAndPacketInfoIPV6VectorReadVectorWrite", testReceiveEcnAndPacketInfoIPV6VectorReadVectorWrite),
("testSendingAddressedEnvelopeOnUnconnectedSocketSucceeds", testSendingAddressedEnvelopeOnUnconnectedSocketSucceeds),
("testSendingByteBufferOnUnconnectedSocketFails", testSendingByteBufferOnUnconnectedSocketFails),
diff --git a/Tests/NIOPosixTests/SocketChannelTest.swift b/Tests/NIOPosixTests/SocketChannelTest.swift
index 082cb70b..609c8aff 100644
--- a/Tests/NIOPosixTests/SocketChannelTest.swift
+++ b/Tests/NIOPosixTests/SocketChannelTest.swift
@@ -907,6 +907,7 @@ public final class SocketChannelTest : XCTestCase {
}

func testSimpleMPTCP() throws {
+ #if os(Linux)
let group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
defer { XCTAssertNoThrow(try group.syncShutdownGracefully()) }

@@ -942,6 +943,7 @@ public final class SocketChannelTest : XCTestCase {
XCTAssertEqual(error.errnoCode, EOPNOTSUPP, "Unexpected error: \(error)")
return
}
+ #endif
}
}

diff --git a/Tests/NIOPosixTests/TestUtils.swift b/Tests/NIOPosixTests/TestUtils.swift
index 603b4141..f0937993 100644
--- a/Tests/NIOPosixTests/TestUtils.swift
Expand Down

0 comments on commit 64bcef4

Please sign in to comment.