Skip to content

Commit

Permalink
[2024.1] Separate FIPS version from other OpenSSL version, bump OpenS…
Browse files Browse the repository at this point in the history
…SL to 3.0.15 (#289)

Bumping OpenSSL to 3.0.15. FIPS must stay on 3.0.8, so separating the
two. Same as #285

Also moved aarch64 builds from CircleCI (which no longer works) to
GitHub Actions and removed EOL CentOS 7 builds.
  • Loading branch information
es1024 authored Oct 23, 2024
1 parent 749c75f commit 06f6a13
Show file tree
Hide file tree
Showing 6 changed files with 137 additions and 131 deletions.
79 changes: 0 additions & 79 deletions .circleci/config.yml

This file was deleted.

114 changes: 65 additions & 49 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,49 +46,6 @@ jobs:
fail-fast: false
matrix:
include:
# ---------------------------------------------------------------------------------------
# CentOS 7
# ---------------------------------------------------------------------------------------
# GCC 11
- name: centos7-x86_64-gcc11
os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
build_thirdparty_args: >-
--devtoolset=11
--expected-major-compiler-version=11
# Clang 16
- name: centos7-x86_64-clang16
os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
build_thirdparty_args: >-
--toolchain=llvm16
--expected-major-compiler-version=16
- name: centos7-x86_64-clang16-full-lto
os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
build_thirdparty_args: >-
--toolchain=llvm16
--expected-major-compiler-version=16
--lto=full
# Clang 17
- name: centos7-x86_64-clang17
os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
build_thirdparty_args: >-
--toolchain=llvm17
--expected-major-compiler-version=17
- name: centos7-x86_64-clang17-full-lto
os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
build_thirdparty_args: >-
--toolchain=llvm17
--expected-major-compiler-version=17
--lto=full
# ---------------------------------------------------------------------------------------
# Ubuntu 20.04
# ---------------------------------------------------------------------------------------
Expand Down Expand Up @@ -145,6 +102,29 @@ jobs:
--toolchain=llvm16
--expected-major-compiler-version=16
- name: almalinux8-x86_64-clang16-full-lto
os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
build_thirdparty_args: >-
--toolchain=llvm16
--expected-major-compiler-version=16
--lto=full
- name: almalinux8-aarch64-clang16
os: ubuntu-24.04-aarch64-4core-16gb
docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06
build_thirdparty_args: >-
--toolchain=llvm16
--expected-major-compiler-version=16
- name: almalinux8-aarch64-clang16-full-lto
os: ubuntu-24.04-aarch64-4core-16gb
docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06
build_thirdparty_args: >-
--toolchain=llvm16
--expected-major-compiler-version=16
--lto=full
# Clang/LLVM 17
- name: almalinux8-x86_64-clang17
os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
Expand All @@ -153,6 +133,30 @@ jobs:
--toolchain=llvm17
--expected-major-compiler-version=17
- name: almalinux8-x86_64-clang17-full-lto
os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
build_thirdparty_args: >-
--toolchain=llvm17
--expected-major-compiler-version=17
--lto=full
- name: almalinux8-aarch64-clang17
os: ubuntu-24.04-aarch64-4core-16gb
docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06
build_thirdparty_args: >-
--toolchain=llvm17
--expected-major-compiler-version=17
- name: almalinux8-aarch64-clang17
os: ubuntu-24.04-aarch64-4core-16gb
docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06
build_thirdparty_args: >-
--toolchain=llvm17
--expected-major-compiler-version=17
--lto=full
# ---------------------------------------------------------------------------------------
# Ubuntu 23.04
# ---------------------------------------------------------------------------------------
Expand Down Expand Up @@ -203,10 +207,22 @@ jobs:
steps:
- name: Increase disk space
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo rm -rf \
"$AGENT_TOOLSDIRECTORY" \
/opt/google/chrome \
/opt/microsoft/msedge \
/opt/microsoft/powershell \
/opt/pipx \
/usr/lib/mono \
/usr/local/julia* \
/usr/local/lib/android \
/usr/local/lib/node_modules \
/usr/local/share/chromium \
/usr/local/share/powershell \
/usr/share/dotnet \
/usr/share/swift \
/opt/ghc \
/usr/local/share/boost
- name: Clone repository
uses: actions/checkout@v2
Expand All @@ -224,12 +240,12 @@ jobs:
GITHUB_TOKEN: ${{ secrets.CUSTOM_GITHUB_TOKEN }}
SNYK_TOKEN: ${{ secrets.SNYK_AUTH_TOKEN }}

- uses: actions/upload-artifact@v2
- uses: actions/upload-artifact@v3
with:
name: yugabyte-db-thirdparty-${{ matrix.name }}.tar.gz
path: archive.tar.gz

- uses: actions/upload-artifact@v2
- uses: actions/upload-artifact@v3
with:
name: yugabyte-db-thirdparty-${{ matrix.name }}.tar.gz.sha256
path: archive.tar.gz.sha256
6 changes: 3 additions & 3 deletions python/build_definitions/openssl.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class OpenSSLDependency(Dependency):
def __init__(self) -> None:
super(OpenSSLDependency, self).__init__(
name='openssl',
version='3.0.8',
version='3.0.15',
url_pattern='https://www.openssl.org/source/openssl-{0}.tar.gz',
build_group=BuildGroup.COMMON)
self.copy_sources = True
Expand All @@ -46,7 +46,7 @@ def __init__(self) -> None:
self.patches = ['openssl-fix-afalg-link-on-centos7.patch']

def build(self, builder: BuilderInterface) -> None:
common_configure_options = ['shared', 'no-tests', 'enable-fips']
common_configure_options = ['shared', 'no-tests']
install_path = os.path.join(
builder.fs_layout.tp_installed_common_dir, "lib")
if is_macos_arm64_build():
Expand All @@ -59,7 +59,7 @@ def build(self, builder: BuilderInterface) -> None:
dep=self,
configure_cmd=configure_cmd,
# https://bit.ly/openssl_install_without_manpages
install=['install_sw', 'install_fips']
install=['install_sw']
)

def use_cppflags_env_var(self) -> bool:
Expand Down
66 changes: 66 additions & 0 deletions python/build_definitions/openssl_fips.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#
# Copyright (c) YugaByte, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations
# under the License.
#

import os
from typing import List

from yugabyte_db_thirdparty.build_definition_helpers import * # noqa


def use_arm64_bash_in_script(script_path: str) -> None:
with open(script_path) as script_file:
lines = [line.rstrip() for line in script_file]

if not lines:
return
if not lines[0].startswith('#!') and lines[0].endswith('bash'):
return
lines[0] = '#!/opt/homebrew/bin/bash'
with open(script_path, 'w') as output_file:
output_file.write('\n'.join(lines) + '\n')


class OpenSSLFIPSDependency(Dependency):
def __init__(self) -> None:
super(OpenSSLFIPSDependency, self).__init__(
name='openssl_fips',
version='3.0.8',
url_pattern='https://www.openssl.org/source/openssl-{0}.tar.gz',
build_group=BuildGroup.COMMON)
self.copy_sources = True
# Patch fixes the following error on kernel versions < 4.1.0:
# ld.lld: error: version script assignment of 'global' to symbol 'bind_engine' failed:
# symbol not defined
# ld.lld: error: version script assignment of 'global' to symbol 'v_check' failed:
# symbol not defined
self.patches = ['openssl-fix-afalg-link-on-centos7.patch']

def build(self, builder: BuilderInterface) -> None:
common_configure_options = ['shared', 'no-tests', 'enable-fips']
install_path = os.path.join(
builder.fs_layout.tp_installed_common_dir, "lib")
if is_macos_arm64_build():
use_arm64_bash_in_script('config')
configure_cmd = ['./config'] + common_configure_options
if not is_macos():
configure_cmd += ['-Wl,-rpath=' + install_path]

builder.build_with_configure(
dep=self,
configure_cmd=configure_cmd,
install=['install_fips']
)

def use_cppflags_env_var(self) -> bool:
return True
1 change: 1 addition & 0 deletions python/yugabyte_db_thirdparty/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,7 @@ def populate_dependencies(self) -> None:
'zlib_dependency',
'lz4',
'openssl',
'openssl_fips',
'libev',
'rapidjson',
'squeasel',
Expand Down
2 changes: 2 additions & 0 deletions thirdparty_src_checksums.txt
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,9 @@ c48450d27524c2e5856997133e059e3cf9909241110a6e21ad278890ac425afc lz4-r130.tar.g
8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b openssl-1.1.1t.tar.gz
cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 openssl-1.1.1w.tar.gz
6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e openssl-3.0.8.tar.gz
23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533 openssl-3.0.15.tar.gz
f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6 openssl-3.1.3.tar.gz
6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e openssl_fips-3.0.8.tar.gz
0fdbefbdc2c154634728097e26de52a8210ed95cb032beb5f35da0a493cd5066 opentelemetry-cpp-1.9.0.tar.gz
464bc2b348e674a1a03142e403cbccb01be8655b6de0f8bfe733ea31fcd421be opentelemetry-proto-0.19.0.tar.gz
05e28e5141c1962b1c9d8793cc9cfee8cd11bc24cea13fb9689ac3fc0a379bd3 protobuf-3.5.1-yb-1.tar.gz
Expand Down

0 comments on commit 06f6a13

Please sign in to comment.