From b1a94401f9e3eed226f3a010849301ee722a8ec9 Mon Sep 17 00:00:00 2001 From: xxchan Date: Thu, 30 Nov 2023 14:53:18 +0800 Subject: [PATCH 01/13] build: try upgrade openssl --- src/utils/workspace-config/Cargo.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/utils/workspace-config/Cargo.toml b/src/utils/workspace-config/Cargo.toml index df70a2c6d0054..dc6b17e866914 100644 --- a/src/utils/workspace-config/Cargo.toml +++ b/src/utils/workspace-config/Cargo.toml @@ -16,9 +16,7 @@ rw-dynamic-link = ["zstd-sys"] [dependencies] log = { version = "0.4", features = ["release_max_level_debug"] } -# FIXME: 0.9.93 upgrades openssl-src to openssl@3, but we failed to build it. -# fix it later https://github.com/risingwavelabs/risingwave/pull/12198 -openssl-sys = { version = "=0.9.92", optional = true, features = ["vendored"] } +openssl-sys = { version = "0.9.96", optional = true, features = ["vendored"] } sasl2-sys = { version = "0.1", optional = true, features = ["gssapi-vendored"] } tracing = { version = "0.1", features = ["release_max_level_debug"] } zstd-sys = { version = "2", optional = true, default-features = false, features = ["pkg-config"] } From dcf3acdfe0dfaea14f87dffbe9b2947dec47884d Mon Sep 17 00:00:00 2001 From: xxchan Date: Thu, 30 Nov 2023 14:59:39 +0800 Subject: [PATCH 02/13] fix --- Cargo.lock | 8 ++++---- src/utils/workspace-config/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00550fede73ab..fdd1ff2b121a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5612,18 +5612,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.25.3+1.1.1t" +version = "300.1.6+3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924757a6a226bf60da5f7dd0311a34d2b52283dd82ddeb103208ddc66362f80c" +checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.92" +version = "0.9.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" dependencies = [ "cc", "libc", diff --git a/src/utils/workspace-config/Cargo.toml b/src/utils/workspace-config/Cargo.toml index dc6b17e866914..a83978a9acba7 100644 --- a/src/utils/workspace-config/Cargo.toml +++ b/src/utils/workspace-config/Cargo.toml @@ -26,7 +26,7 @@ zstd-sys = { version = "2", optional = true, default-features = false, features # FIXME(xxchan): This is a temporary fix due to how cargo and hakari works. See related PR for more details. # We will revisit how to handle workspace-hack and build-dependency issues later. [build-dependencies] -openssl-sys = { version = "=0.9.92", optional = true, features = ["vendored"] } +openssl-sys = { version = "0.9.96", optional = true, features = ["vendored"] } [lints] workspace = true From 77e35fe72df13aa5d7cbe4d41132a89ca004f65a Mon Sep 17 00:00:00 2001 From: xxchan Date: Thu, 30 Nov 2023 15:16:02 +0800 Subject: [PATCH 03/13] fix --- ci/scripts/release.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index eb05e4aa63871..8e8826639b9b9 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -39,6 +39,9 @@ echo "--- Install aws cli" curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip -q awscliv2.zip && ./aws/install && mv /usr/local/bin/aws /bin/aws +echo "--- Install dependencies for openssl" +yum install -y perl-core + echo "--- Check risingwave release version" if [[ -n "${BUILDKITE_TAG}" ]]; then CARGO_PKG_VERSION="$(toml get --toml-path Cargo.toml workspace.package.version)" From 8811800436b5d2b3d2adb6dfac62279bca8f4fd0 Mon Sep 17 00:00:00 2001 From: xxchan Date: Thu, 30 Nov 2023 16:35:37 +0800 Subject: [PATCH 04/13] try install lld from source --- ci/docker-compose.yml | 10 ++++++++++ ci/scripts/release.sh | 18 ++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index c6289789d5e67..8bf3a2a02aa83 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -117,6 +117,16 @@ services: - ..:/risingwave release-env: + # build binaries on a earlier Linux distribution (therefore with earlier version GLIBC) + # See https://github.com/risingwavelabs/risingwave/issues/4556 for more details. + # + # GLIBC versions on some systems: + # Amazon Linux 2: 2.26 (EOL 2025-06-30) (We will definitely want to support this) + # AL2023: 2.34 + # Ubuntu 18.04: 2.27 (Already EOL 2023-05-31) + # Ubuntu 20.04: 2.31 + # + # manylinux2014: CentOS 7 (EOL 2024-06-30), GLIBC 2.17 image: quay.io/pypa/manylinux2014_x86_64 working_dir: /mnt volumes: diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index 8e8826639b9b9..21eb0696fd18f 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -10,6 +10,16 @@ if [ "${BUILDKITE_SOURCE}" != "schedule" ] && [ "${BUILDKITE_SOURCE}" != "webhoo exit 0 fi +echo "--- Install lld" +curl -LO https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-17.0.6.tar.gz +tar -xf llvmorg-17.0.6.tar.gz +mkdir build +cd build +cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=lld -DCMAKE_INSTALL_PREFIX=/usr/local ../llvm-project-llvmorg-17.0.6/llvm +make install + +ld.lld --version + echo "--- Install java and maven" yum install -y java-11-openjdk java-11-openjdk-devel wget python3 cyrus-sasl-devel pip3 install toml-cli @@ -30,18 +40,10 @@ unzip -o protoc-3.15.8-linux-x86_64.zip -d protoc mv ./protoc/bin/protoc /usr/local/bin/ mv ./protoc/include/* /usr/local/include/ -echo "--- Install lld" -yum install -y centos-release-scl-rh -yum install -y llvm-toolset-7.0-lld -source /opt/rh/llvm-toolset-7.0/enable - echo "--- Install aws cli" curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip -q awscliv2.zip && ./aws/install && mv /usr/local/bin/aws /bin/aws -echo "--- Install dependencies for openssl" -yum install -y perl-core - echo "--- Check risingwave release version" if [[ -n "${BUILDKITE_TAG}" ]]; then CARGO_PKG_VERSION="$(toml get --toml-path Cargo.toml workspace.package.version)" From 4f1558f6c02cb9430366818f6175136c069346a4 Mon Sep 17 00:00:00 2001 From: xxchan Date: Thu, 30 Nov 2023 16:43:18 +0800 Subject: [PATCH 05/13] make -j --- ci/scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index 21eb0696fd18f..7e69681ad7615 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -16,7 +16,7 @@ tar -xf llvmorg-17.0.6.tar.gz mkdir build cd build cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=lld -DCMAKE_INSTALL_PREFIX=/usr/local ../llvm-project-llvmorg-17.0.6/llvm -make install +make install -j ld.lld --version From c741ef8a1ff2316bec61db9dfc454ab84a674458 Mon Sep 17 00:00:00 2001 From: xxchan Date: Fri, 1 Dec 2023 00:53:07 +0800 Subject: [PATCH 06/13] install pre-built lld --- ci/scripts/release.sh | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index 7e69681ad7615..cb3002b551361 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -10,14 +10,13 @@ if [ "${BUILDKITE_SOURCE}" != "schedule" ] && [ "${BUILDKITE_SOURCE}" != "webhoo exit 0 fi -echo "--- Install lld" -curl -LO https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-17.0.6.tar.gz -tar -xf llvmorg-17.0.6.tar.gz -mkdir build -cd build -cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=lld -DCMAKE_INSTALL_PREFIX=/usr/local ../llvm-project-llvmorg-17.0.6/llvm -make install -j +echo "--- Install aws cli" +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip -q awscliv2.zip && ./aws/install && mv /usr/local/bin/aws /bin/aws +echo "--- Install lld" +aws s3 cp s3://ci-deps-dist/llvm-lld-manylinux2014_x86_64.tar.gz . +tar -zxvf llvm-lld-manylinux2014_x86_64.tar.gz /usr/local ld.lld --version echo "--- Install java and maven" @@ -40,10 +39,6 @@ unzip -o protoc-3.15.8-linux-x86_64.zip -d protoc mv ./protoc/bin/protoc /usr/local/bin/ mv ./protoc/include/* /usr/local/include/ -echo "--- Install aws cli" -curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -unzip -q awscliv2.zip && ./aws/install && mv /usr/local/bin/aws /bin/aws - echo "--- Check risingwave release version" if [[ -n "${BUILDKITE_TAG}" ]]; then CARGO_PKG_VERSION="$(toml get --toml-path Cargo.toml workspace.package.version)" From 116fb03f0816efe2e75d17b017d4a7ecdebd0056 Mon Sep 17 00:00:00 2001 From: xxchan Date: Fri, 1 Dec 2023 00:56:30 +0800 Subject: [PATCH 07/13] fix --- ci/scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index cb3002b551361..280f353cf29d1 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -16,7 +16,7 @@ unzip -q awscliv2.zip && ./aws/install && mv /usr/local/bin/aws /bin/aws echo "--- Install lld" aws s3 cp s3://ci-deps-dist/llvm-lld-manylinux2014_x86_64.tar.gz . -tar -zxvf llvm-lld-manylinux2014_x86_64.tar.gz /usr/local +tar -zxvf llvm-lld-manylinux2014_x86_64.tar.gz --directory=/usr/local ld.lld --version echo "--- Install java and maven" From f39edab738864f5216c5a6c50b15d01f4401aebb Mon Sep 17 00:00:00 2001 From: xxchan Date: Fri, 1 Dec 2023 01:01:34 +0800 Subject: [PATCH 08/13] fix --- ci/scripts/release.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index 280f353cf29d1..7db0f5ffe43ae 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -19,6 +19,9 @@ aws s3 cp s3://ci-deps-dist/llvm-lld-manylinux2014_x86_64.tar.gz . tar -zxvf llvm-lld-manylinux2014_x86_64.tar.gz --directory=/usr/local ld.lld --version +echo "--- Install dependencies for openssl" +yum install -y perl-core + echo "--- Install java and maven" yum install -y java-11-openjdk java-11-openjdk-devel wget python3 cyrus-sasl-devel pip3 install toml-cli From efebb49126b3b4d040d74eabeed453d06f0c2bcc Mon Sep 17 00:00:00 2001 From: xxchan Date: Fri, 1 Dec 2023 01:02:11 +0800 Subject: [PATCH 09/13] try debug build --- ci/scripts/release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index 7db0f5ffe43ae..b1d166e11ebd0 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -54,8 +54,8 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then fi echo "--- Build risingwave release binary" -cargo build -p risingwave_cmd_all --features "rw-static-link" --profile release -cargo build --bin risectl --features "rw-static-link" --profile release +cargo build -p risingwave_cmd_all --features "rw-static-link" --profile dev +cargo build --bin risectl --features "rw-static-link" --profile dev cd target/release && chmod +x risingwave risectl echo "--- Upload nightly binary to s3" From 8f3992da6834199e88a8f1b8a8a7e1057a8d83fd Mon Sep 17 00:00:00 2001 From: xxchan Date: Fri, 1 Dec 2023 01:25:03 +0800 Subject: [PATCH 10/13] Revert "try debug build" This reverts commit efebb49126b3b4d040d74eabeed453d06f0c2bcc. --- ci/scripts/release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index b1d166e11ebd0..7db0f5ffe43ae 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -54,8 +54,8 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then fi echo "--- Build risingwave release binary" -cargo build -p risingwave_cmd_all --features "rw-static-link" --profile dev -cargo build --bin risectl --features "rw-static-link" --profile dev +cargo build -p risingwave_cmd_all --features "rw-static-link" --profile release +cargo build --bin risectl --features "rw-static-link" --profile release cd target/release && chmod +x risingwave risectl echo "--- Upload nightly binary to s3" From e5beed6b8027ec8aeb1ae04884fcb07861321229 Mon Sep 17 00:00:00 2001 From: xxchan Date: Sun, 3 Dec 2023 22:28:14 +0800 Subject: [PATCH 11/13] add a note --- ci/scripts/release.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index 7db0f5ffe43ae..32d9b801a7e26 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -15,6 +15,9 @@ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip unzip -q awscliv2.zip && ./aws/install && mv /usr/local/bin/aws /bin/aws echo "--- Install lld" +# The lld in the CentOS 7 repository is too old and contains a bug that causes a linker error. +# So we install a newer version here. (17.0.6, latest version at the time of writing) +# It is manually built in the same environent and uploaded to S3. aws s3 cp s3://ci-deps-dist/llvm-lld-manylinux2014_x86_64.tar.gz . tar -zxvf llvm-lld-manylinux2014_x86_64.tar.gz --directory=/usr/local ld.lld --version From 925b3a338f328c6933fd8b2b722a08499a1d6af3 Mon Sep 17 00:00:00 2001 From: xxchan Date: Mon, 4 Dec 2023 14:10:24 +0800 Subject: [PATCH 12/13] also upgrade openssl --- Cargo.lock | 4 ++-- src/utils/pgwire/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba75baf49175a..a63f9a8c8309e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5582,9 +5582,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" dependencies = [ "bitflags 2.4.0", "cfg-if", diff --git a/src/utils/pgwire/Cargo.toml b/src/utils/pgwire/Cargo.toml index 73ed5a4200b1c..51588721fb7de 100644 --- a/src/utils/pgwire/Cargo.toml +++ b/src/utils/pgwire/Cargo.toml @@ -21,7 +21,7 @@ byteorder = "1.5" bytes = "1" futures = { version = "0.3", default-features = false, features = ["alloc"] } itertools = "0.12" -openssl = "0.10.57" +openssl = "0.10.60" panic-message = "0.3" risingwave_common = { workspace = true } risingwave_sqlparser = { workspace = true } From ad144d77f48c3364f6c218d72c0c0eab510f6778 Mon Sep 17 00:00:00 2001 From: xxchan Date: Mon, 4 Dec 2023 14:13:24 +0800 Subject: [PATCH 13/13] fix trailing space --- ci/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index 8bf3a2a02aa83..9bde343335b55 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -119,13 +119,13 @@ services: release-env: # build binaries on a earlier Linux distribution (therefore with earlier version GLIBC) # See https://github.com/risingwavelabs/risingwave/issues/4556 for more details. - # + # # GLIBC versions on some systems: # Amazon Linux 2: 2.26 (EOL 2025-06-30) (We will definitely want to support this) # AL2023: 2.34 # Ubuntu 18.04: 2.27 (Already EOL 2023-05-31) # Ubuntu 20.04: 2.31 - # + # # manylinux2014: CentOS 7 (EOL 2024-06-30), GLIBC 2.17 image: quay.io/pypa/manylinux2014_x86_64 working_dir: /mnt