diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 195dfbf4..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) Yugabyte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -version: 2.1 - -jobs: - aarch64: - parameters: - archive_name_suffix: - type: string - docker_image: - type: string - build_thirdparty_args: - type: string - default: "" - - machine: - image: ubuntu-2004:current - resource_class: arm.large - - steps: - - checkout - - run: - name: Build YugabyteDB third-party dependencies - command: | - export YB_DOCKER_IMAGE="<< parameters.docker_image >>" - export YB_BUILD_THIRDPARTY_ARGS="<< parameters.build_thirdparty_args >>" - export YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX="<< parameters.archive_name_suffix >>" - .github/workflows/github_actions_build.sh - -workflows: - build-release: - jobs: - # TODO: find a way to assign different names to each build. - # As specified below, they are named build-1, build-2, etc. - - - aarch64: - archive_name_suffix: centos7-aarch64-clang16 - docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2023-10-25T03_32_55 - build_thirdparty_args: >- - --toolchain=llvm16 - --expected-major-compiler-version=16 - --skip-sanitizers - - - aarch64: - archive_name_suffix: centos7-aarch64-clang16-full-lto - docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2023-10-25T03_32_55 - build_thirdparty_args: >- - --toolchain=llvm16 - --expected-major-compiler-version=16 - --skip-sanitizers - --lto=full - - - aarch64: - archive_name_suffix: centos7-aarch64-clang17 - docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2023-10-25T03_32_55 - build_thirdparty_args: >- - --toolchain=llvm17 - --expected-major-compiler-version=17 - --skip-sanitizers - - - aarch64: - archive_name_suffix: centos7-aarch64-clang17-full-lto - docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2023-10-25T03_32_55 - build_thirdparty_args: >- - --toolchain=llvm17 - --expected-major-compiler-version=17 - --skip-sanitizers - --lto=full diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f1486b3..c89bbd57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,49 +46,6 @@ jobs: fail-fast: false matrix: include: - # --------------------------------------------------------------------------------------- - # CentOS 7 - # --------------------------------------------------------------------------------------- - # GCC 11 - - name: centos7-x86_64-gcc11 - os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. - docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02 - build_thirdparty_args: >- - --devtoolset=11 - --expected-major-compiler-version=11 - - # Clang 16 - - name: centos7-x86_64-clang16 - os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. - docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02 - build_thirdparty_args: >- - --toolchain=llvm16 - --expected-major-compiler-version=16 - - - name: centos7-x86_64-clang16-full-lto - os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. - docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02 - build_thirdparty_args: >- - --toolchain=llvm16 - --expected-major-compiler-version=16 - --lto=full - - # Clang 17 - - name: centos7-x86_64-clang17 - os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. - docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02 - build_thirdparty_args: >- - --toolchain=llvm17 - --expected-major-compiler-version=17 - - - name: centos7-x86_64-clang17-full-lto - os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. - docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02 - build_thirdparty_args: >- - --toolchain=llvm17 - --expected-major-compiler-version=17 - --lto=full - # --------------------------------------------------------------------------------------- # Ubuntu 20.04 # --------------------------------------------------------------------------------------- @@ -145,6 +102,29 @@ jobs: --toolchain=llvm16 --expected-major-compiler-version=16 + - name: almalinux8-x86_64-clang16-full-lto + os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. + docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01 + build_thirdparty_args: >- + --toolchain=llvm16 + --expected-major-compiler-version=16 + --lto=full + + - name: almalinux8-aarch64-clang16 + os: ubuntu-24.04-aarch64-4core-16gb + docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06 + build_thirdparty_args: >- + --toolchain=llvm16 + --expected-major-compiler-version=16 + + - name: almalinux8-aarch64-clang16-full-lto + os: ubuntu-24.04-aarch64-4core-16gb + docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06 + build_thirdparty_args: >- + --toolchain=llvm16 + --expected-major-compiler-version=16 + --lto=full + # Clang/LLVM 17 - name: almalinux8-x86_64-clang17 os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. @@ -153,6 +133,30 @@ jobs: --toolchain=llvm17 --expected-major-compiler-version=17 + - name: almalinux8-x86_64-clang17-full-lto + os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. + docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01 + build_thirdparty_args: >- + --toolchain=llvm17 + --expected-major-compiler-version=17 + --lto=full + + - name: almalinux8-aarch64-clang17 + os: ubuntu-24.04-aarch64-4core-16gb + docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06 + build_thirdparty_args: >- + --toolchain=llvm17 + --expected-major-compiler-version=17 + + - name: almalinux8-aarch64-clang17 + os: ubuntu-24.04-aarch64-4core-16gb + docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06 + build_thirdparty_args: >- + --toolchain=llvm17 + --expected-major-compiler-version=17 + --lto=full + + # --------------------------------------------------------------------------------------- # Ubuntu 23.04 # --------------------------------------------------------------------------------------- @@ -203,10 +207,22 @@ jobs: steps: - name: Increase disk space run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf \ + "$AGENT_TOOLSDIRECTORY" \ + /opt/google/chrome \ + /opt/microsoft/msedge \ + /opt/microsoft/powershell \ + /opt/pipx \ + /usr/lib/mono \ + /usr/local/julia* \ + /usr/local/lib/android \ + /usr/local/lib/node_modules \ + /usr/local/share/chromium \ + /usr/local/share/powershell \ + /usr/share/dotnet \ + /usr/share/swift \ + /opt/ghc \ + /usr/local/share/boost - name: Clone repository uses: actions/checkout@v2 @@ -224,12 +240,12 @@ jobs: GITHUB_TOKEN: ${{ secrets.CUSTOM_GITHUB_TOKEN }} SNYK_TOKEN: ${{ secrets.SNYK_AUTH_TOKEN }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: yugabyte-db-thirdparty-${{ matrix.name }}.tar.gz path: archive.tar.gz - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: yugabyte-db-thirdparty-${{ matrix.name }}.tar.gz.sha256 path: archive.tar.gz.sha256 diff --git a/python/build_definitions/openssl.py b/python/build_definitions/openssl.py index 4d0bea3e..89af898f 100644 --- a/python/build_definitions/openssl.py +++ b/python/build_definitions/openssl.py @@ -34,7 +34,7 @@ class OpenSSLDependency(Dependency): def __init__(self) -> None: super(OpenSSLDependency, self).__init__( name='openssl', - version='3.0.8', + version='3.0.15', url_pattern='https://www.openssl.org/source/openssl-{0}.tar.gz', build_group=BuildGroup.COMMON) self.copy_sources = True @@ -46,7 +46,7 @@ def __init__(self) -> None: self.patches = ['openssl-fix-afalg-link-on-centos7.patch'] def build(self, builder: BuilderInterface) -> None: - common_configure_options = ['shared', 'no-tests', 'enable-fips'] + common_configure_options = ['shared', 'no-tests'] install_path = os.path.join( builder.fs_layout.tp_installed_common_dir, "lib") if is_macos_arm64_build(): @@ -59,7 +59,7 @@ def build(self, builder: BuilderInterface) -> None: dep=self, configure_cmd=configure_cmd, # https://bit.ly/openssl_install_without_manpages - install=['install_sw', 'install_fips'] + install=['install_sw'] ) def use_cppflags_env_var(self) -> bool: diff --git a/python/build_definitions/openssl_fips.py b/python/build_definitions/openssl_fips.py new file mode 100644 index 00000000..652a3830 --- /dev/null +++ b/python/build_definitions/openssl_fips.py @@ -0,0 +1,66 @@ +# +# Copyright (c) YugaByte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. +# + +import os +from typing import List + +from yugabyte_db_thirdparty.build_definition_helpers import * # noqa + + +def use_arm64_bash_in_script(script_path: str) -> None: + with open(script_path) as script_file: + lines = [line.rstrip() for line in script_file] + + if not lines: + return + if not lines[0].startswith('#!') and lines[0].endswith('bash'): + return + lines[0] = '#!/opt/homebrew/bin/bash' + with open(script_path, 'w') as output_file: + output_file.write('\n'.join(lines) + '\n') + + +class OpenSSLFIPSDependency(Dependency): + def __init__(self) -> None: + super(OpenSSLFIPSDependency, self).__init__( + name='openssl_fips', + version='3.0.8', + url_pattern='https://www.openssl.org/source/openssl-{0}.tar.gz', + build_group=BuildGroup.COMMON) + self.copy_sources = True + # Patch fixes the following error on kernel versions < 4.1.0: + # ld.lld: error: version script assignment of 'global' to symbol 'bind_engine' failed: + # symbol not defined + # ld.lld: error: version script assignment of 'global' to symbol 'v_check' failed: + # symbol not defined + self.patches = ['openssl-fix-afalg-link-on-centos7.patch'] + + def build(self, builder: BuilderInterface) -> None: + common_configure_options = ['shared', 'no-tests', 'enable-fips'] + install_path = os.path.join( + builder.fs_layout.tp_installed_common_dir, "lib") + if is_macos_arm64_build(): + use_arm64_bash_in_script('config') + configure_cmd = ['./config'] + common_configure_options + if not is_macos(): + configure_cmd += ['-Wl,-rpath=' + install_path] + + builder.build_with_configure( + dep=self, + configure_cmd=configure_cmd, + install=['install_fips'] + ) + + def use_cppflags_env_var(self) -> bool: + return True diff --git a/python/yugabyte_db_thirdparty/builder.py b/python/yugabyte_db_thirdparty/builder.py index e539e7b9..8306be21 100644 --- a/python/yugabyte_db_thirdparty/builder.py +++ b/python/yugabyte_db_thirdparty/builder.py @@ -322,6 +322,7 @@ def populate_dependencies(self) -> None: 'zlib_dependency', 'lz4', 'openssl', + 'openssl_fips', 'libev', 'rapidjson', 'squeasel', diff --git a/thirdparty_src_checksums.txt b/thirdparty_src_checksums.txt index 78b745f7..f55410c3 100644 --- a/thirdparty_src_checksums.txt +++ b/thirdparty_src_checksums.txt @@ -114,7 +114,9 @@ c48450d27524c2e5856997133e059e3cf9909241110a6e21ad278890ac425afc lz4-r130.tar.g 8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b openssl-1.1.1t.tar.gz cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 openssl-1.1.1w.tar.gz 6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e openssl-3.0.8.tar.gz +23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533 openssl-3.0.15.tar.gz f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6 openssl-3.1.3.tar.gz +6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e openssl_fips-3.0.8.tar.gz 0fdbefbdc2c154634728097e26de52a8210ed95cb032beb5f35da0a493cd5066 opentelemetry-cpp-1.9.0.tar.gz 464bc2b348e674a1a03142e403cbccb01be8655b6de0f8bfe733ea31fcd421be opentelemetry-proto-0.19.0.tar.gz 05e28e5141c1962b1c9d8793cc9cfee8cd11bc24cea13fb9689ac3fc0a379bd3 protobuf-3.5.1-yb-1.tar.gz