Skip to content

Commit 53d1535

Browse files
FuPeiJiangwjones127
FuPeiJiang
and
wjones127
authored
ci: musl x64,arm64 (lancedb#1853)
untested 4 artifacts at: https://github.com/FuPeiJiang/lancedb/actions/runs/11926579058 node-native-linux-aarch64-musl 22.6 MB node-native-linux-x86_64-musl 23.6 MB nodejs-native-linux-aarch64-musl 26.7 MB nodejs-native-linux-x86_64-musl 27 MB this follows the same process as: lancedb#1816 (comment) Closes lancedb#1388 Closes lancedb#1107 --------- Co-authored-by: Will Jones <[email protected]>
1 parent b2f88f0 commit 53d1535

File tree

11 files changed

+172
-13
lines changed

11 files changed

+172
-13
lines changed

‎.bumpversion.toml

+10
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,16 @@ glob = "node/package.json"
8787
replace = "\"@lancedb/vectordb-linux-x64-gnu\": \"{new_version}\""
8888
search = "\"@lancedb/vectordb-linux-x64-gnu\": \"{current_version}\""
8989

90+
[[tool.bumpversion.files]]
91+
glob = "node/package.json"
92+
replace = "\"@lancedb/vectordb-linux-arm64-musl\": \"{new_version}\""
93+
search = "\"@lancedb/vectordb-linux-arm64-musl\": \"{current_version}\""
94+
95+
[[tool.bumpversion.files]]
96+
glob = "node/package.json"
97+
replace = "\"@lancedb/vectordb-linux-x64-musl\": \"{new_version}\""
98+
search = "\"@lancedb/vectordb-linux-x64-musl\": \"{current_version}\""
99+
90100
[[tool.bumpversion.files]]
91101
glob = "node/package.json"
92102
replace = "\"@lancedb/vectordb-win32-x64-msvc\": \"{new_version}\""

‎.cargo/config.toml

+3
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ rustflags = [
3131
[target.x86_64-unknown-linux-gnu]
3232
rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
3333

34+
[target.x86_64-unknown-linux-musl]
35+
rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=-crt-static,+avx2,+fma,+f16c"]
36+
3437
[target.aarch64-apple-darwin]
3538
rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]
3639

‎.github/workflows/npm-publish.yml

+114-6
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ jobs:
101101
path: |
102102
nodejs/dist/*.node
103103
104-
node-linux:
104+
node-linux-gnu:
105105
name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu)
106106
runs-on: ${{ matrix.config.runner }}
107107
# Only runs on tags that matches the make-release action
@@ -137,11 +137,63 @@ jobs:
137137
- name: Upload Linux Artifacts
138138
uses: actions/upload-artifact@v4
139139
with:
140-
name: node-native-linux-${{ matrix.config.arch }}
140+
name: node-native-linux-${{ matrix.config.arch }}-gnu
141141
path: |
142142
node/dist/lancedb-vectordb-linux*.tgz
143143
144-
nodejs-linux:
144+
node-linux-musl:
145+
name: vectordb (${{ matrix.config.arch}}-unknown-linux-musl)
146+
runs-on: ubuntu-latest
147+
container: alpine:edge
148+
# Only runs on tags that matches the make-release action
149+
if: startsWith(github.ref, 'refs/tags/v')
150+
strategy:
151+
fail-fast: false
152+
matrix:
153+
config:
154+
- arch: x86_64
155+
- arch: aarch64
156+
steps:
157+
- name: Checkout
158+
uses: actions/checkout@v4
159+
- name: Install common dependencies
160+
run: |
161+
apk add protobuf-dev curl clang mold grep npm bash
162+
curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
163+
echo "source $HOME/.cargo/env" >> saved_env
164+
echo "export CC=clang" >> saved_env
165+
echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env
166+
- name: Configure aarch64 build
167+
if: ${{ matrix.config.arch == 'aarch64' }}
168+
run: |
169+
source "$HOME/.cargo/env"
170+
rustup target add aarch64-unknown-linux-musl --toolchain 1.80.0
171+
crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained)
172+
sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib
173+
apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/
174+
curl -sSf $apk_url > apk_list
175+
for pkg in gcc libgcc musl; do curl -sSf $apk_url$(cat apk_list | grep -oP '(?<=")'$pkg'-\d.*?(?=")') | tar zxf -; done
176+
mkdir -p $sysroot_lib
177+
echo 'GROUP ( libgcc_s.so.1 -lgcc )' > $sysroot_lib/libgcc_s.so
178+
cp usr/lib/libgcc_s.so.1 $sysroot_lib
179+
cp usr/lib/gcc/aarch64-alpine-linux-musl/*/libgcc.a $sysroot_lib
180+
cp lib/ld-musl-aarch64.so.1 $sysroot_lib/libc.so
181+
echo '!<arch>' > $sysroot_lib/libdl.a
182+
(cd $crt && cp crti.o crtbeginS.o crtendS.o crtn.o -t $sysroot_lib)
183+
echo "export CARGO_BUILD_TARGET=aarch64-unknown-linux-musl" >> saved_env
184+
echo "export RUSTFLAGS='-Ctarget-cpu=apple-m1 -Ctarget-feature=-crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=--target=aarch64-unknown-linux-musl -Clink-arg=--sysroot=/usr/aarch64-unknown-linux-musl -Clink-arg=-lc'" >> saved_env
185+
- name: Build Linux Artifacts
186+
run: |
187+
source ./saved_env
188+
bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }}
189+
- name: Upload Linux Artifacts
190+
uses: actions/upload-artifact@v4
191+
with:
192+
name: node-native-linux-${{ matrix.config.arch }}-musl
193+
path: |
194+
node/dist/lancedb-vectordb-linux*.tgz
195+
196+
nodejs-linux-gnu:
145197
name: lancedb (${{ matrix.config.arch}}-unknown-linux-gnu
146198
runs-on: ${{ matrix.config.runner }}
147199
# Only runs on tags that matches the make-release action
@@ -178,7 +230,7 @@ jobs:
178230
- name: Upload Linux Artifacts
179231
uses: actions/upload-artifact@v4
180232
with:
181-
name: nodejs-native-linux-${{ matrix.config.arch }}
233+
name: nodejs-native-linux-${{ matrix.config.arch }}-gnu
182234
path: |
183235
nodejs/dist/*.node
184236
# The generic files are the same in all distros so we just pick
@@ -192,6 +244,62 @@ jobs:
192244
nodejs/dist/*
193245
!nodejs/dist/*.node
194246
247+
nodejs-linux-musl:
248+
name: lancedb (${{ matrix.config.arch}}-unknown-linux-musl
249+
runs-on: ubuntu-latest
250+
container: alpine:edge
251+
# Only runs on tags that matches the make-release action
252+
if: startsWith(github.ref, 'refs/tags/v')
253+
strategy:
254+
fail-fast: false
255+
matrix:
256+
config:
257+
- arch: x86_64
258+
- arch: aarch64
259+
steps:
260+
- name: Checkout
261+
uses: actions/checkout@v4
262+
- name: Install common dependencies
263+
run: |
264+
apk add protobuf-dev curl clang mold grep npm bash openssl-dev openssl-libs-static
265+
curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
266+
echo "source $HOME/.cargo/env" >> saved_env
267+
echo "export CC=clang" >> saved_env
268+
echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env
269+
echo "export X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_INCLUDE_DIR=/usr/include" >> saved_env
270+
echo "export X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_LIB_DIR=/usr/lib" >> saved_env
271+
- name: Configure aarch64 build
272+
if: ${{ matrix.config.arch == 'aarch64' }}
273+
run: |
274+
source "$HOME/.cargo/env"
275+
rustup target add aarch64-unknown-linux-musl --toolchain 1.80.0
276+
crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained)
277+
sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib
278+
apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/
279+
curl -sSf $apk_url > apk_list
280+
for pkg in gcc libgcc musl openssl-dev openssl-libs-static; do curl -sSf $apk_url$(cat apk_list | grep -oP '(?<=")'$pkg'-\d.*?(?=")') | tar zxf -; done
281+
mkdir -p $sysroot_lib
282+
echo 'GROUP ( libgcc_s.so.1 -lgcc )' > $sysroot_lib/libgcc_s.so
283+
cp usr/lib/libgcc_s.so.1 $sysroot_lib
284+
cp usr/lib/gcc/aarch64-alpine-linux-musl/*/libgcc.a $sysroot_lib
285+
cp lib/ld-musl-aarch64.so.1 $sysroot_lib/libc.so
286+
echo '!<arch>' > $sysroot_lib/libdl.a
287+
(cd $crt && cp crti.o crtbeginS.o crtendS.o crtn.o -t $sysroot_lib)
288+
echo "export CARGO_BUILD_TARGET=aarch64-unknown-linux-musl" >> saved_env
289+
echo "export RUSTFLAGS='-Ctarget-feature=-crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=--target=aarch64-unknown-linux-musl -Clink-arg=--sysroot=/usr/aarch64-unknown-linux-musl -Clink-arg=-lc'" >> saved_env
290+
echo "export AARCH64_UNKNOWN_LINUX_MUSL_OPENSSL_INCLUDE_DIR=$(realpath usr/include)" >> saved_env
291+
echo "export AARCH64_UNKNOWN_LINUX_MUSL_OPENSSL_LIB_DIR=$(realpath usr/lib)" >> saved_env
292+
- name: Build Linux Artifacts
293+
run: |
294+
source ./saved_env
295+
bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }}
296+
- name: Upload Linux Artifacts
297+
uses: actions/upload-artifact@v4
298+
with:
299+
name: nodejs-native-linux-${{ matrix.config.arch }}-musl
300+
path: |
301+
nodejs/dist/*.node
302+
195303
node-windows:
196304
name: vectordb ${{ matrix.target }}
197305
runs-on: windows-2022
@@ -460,7 +568,7 @@ jobs:
460568

461569
release:
462570
name: vectordb NPM Publish
463-
needs: [node, node-macos, node-linux, node-windows]
571+
needs: [node, node-macos, node-linux-gnu, node-linux-musl, node-windows]
464572
runs-on: ubuntu-latest
465573
# Only runs on tags that matches the make-release action
466574
if: startsWith(github.ref, 'refs/tags/v')
@@ -500,7 +608,7 @@ jobs:
500608

501609
release-nodejs:
502610
name: lancedb NPM Publish
503-
needs: [nodejs-macos, nodejs-linux, nodejs-windows]
611+
needs: [nodejs-macos, nodejs-linux-gnu, nodejs-linux-musl, nodejs-windows]
504612
runs-on: ubuntu-latest
505613
# Only runs on tags that matches the make-release action
506614
if: startsWith(github.ref, 'refs/tags/v')

‎ci/manylinux_node/build_lancedb.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ fi
1111
export OPENSSL_STATIC=1
1212
export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
1313

14-
source $HOME/.bashrc
14+
#Alpine doesn't have .bashrc
15+
FILE=$HOME/.bashrc && test -f $FILE && source $FILE
1516

1617
cd nodejs
1718
npm ci

‎ci/manylinux_node/build_vectordb.sh

+3-2
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,14 @@ ARCH=${1:-x86_64}
55

66
if [ "$ARCH" = "x86_64" ]; then
77
export OPENSSL_LIB_DIR=/usr/local/lib64/
8-
else
8+
else
99
export OPENSSL_LIB_DIR=/usr/local/lib/
1010
fi
1111
export OPENSSL_STATIC=1
1212
export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
1313

14-
source $HOME/.bashrc
14+
#Alpine doesn't have .bashrc
15+
FILE=$HOME/.bashrc && test -f $FILE && source $FILE
1516

1617
cd node
1718
npm ci

‎node/package.json

+4-2
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,12 @@
8989
}
9090
},
9191
"optionalDependencies": {
92-
"@lancedb/vectordb-darwin-arm64": "0.13.0",
9392
"@lancedb/vectordb-darwin-x64": "0.13.0",
94-
"@lancedb/vectordb-linux-arm64-gnu": "0.13.0",
93+
"@lancedb/vectordb-darwin-arm64": "0.13.0",
9594
"@lancedb/vectordb-linux-x64-gnu": "0.13.0",
95+
"@lancedb/vectordb-linux-arm64-gnu": "0.13.0",
96+
"@lancedb/vectordb-linux-x64-musl": "0.13.0",
97+
"@lancedb/vectordb-linux-arm64-musl": "0.13.0",
9698
"@lancedb/vectordb-win32-x64-msvc": "0.13.0",
9799
"@lancedb/vectordb-win32-arm64-msvc": "0.13.0"
98100
}

‎nodejs/npm/linux-arm64-musl/README.md

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# `@lancedb/lancedb-linux-arm64-musl`
2+
3+
This is the **aarch64-unknown-linux-musl** binary for `@lancedb/lancedb`

‎nodejs/npm/linux-arm64-musl/package.json

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"name": "@lancedb/lancedb-linux-arm64-musl",
3+
"version": "0.13.0",
4+
"os": ["linux"],
5+
"cpu": ["arm64"],
6+
"main": "lancedb.linux-arm64-musl.node",
7+
"files": ["lancedb.linux-arm64-musl.node"],
8+
"license": "Apache 2.0",
9+
"engines": {
10+
"node": ">= 18"
11+
},
12+
"libc": ["musl"]
13+
}

‎nodejs/npm/linux-x64-musl/README.md

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# `@lancedb/lancedb-linux-x64-musl`
2+
3+
This is the **x86_64-unknown-linux-musl** binary for `@lancedb/lancedb`

‎nodejs/npm/linux-x64-musl/package.json

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"name": "@lancedb/lancedb-linux-x64-musl",
3+
"version": "0.13.0",
4+
"os": ["linux"],
5+
"cpu": ["x64"],
6+
"main": "lancedb.linux-x64-musl.node",
7+
"files": ["lancedb.linux-x64-musl.node"],
8+
"license": "Apache 2.0",
9+
"engines": {
10+
"node": ">= 18"
11+
},
12+
"libc": ["musl"]
13+
}

‎nodejs/package.json

+4-2
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,12 @@
2424
"triples": {
2525
"defaults": false,
2626
"additional": [
27-
"aarch64-apple-darwin",
28-
"aarch64-unknown-linux-gnu",
2927
"x86_64-apple-darwin",
28+
"aarch64-apple-darwin",
3029
"x86_64-unknown-linux-gnu",
30+
"aarch64-unknown-linux-gnu",
31+
"x86_64-unknown-linux-musl",
32+
"aarch64-unknown-linux-musl",
3133
"x86_64-pc-windows-msvc"
3234
]
3335
}

0 commit comments

Comments
 (0)