diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 7984410c..bf8226f9 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -55,9 +55,6 @@ jobs: - name: Package Pulsar source run: build-support/generate-source-archive.sh - - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - - name: Build dependencies Docker image uses: docker/build-push-action@v3 with: @@ -66,8 +63,6 @@ jobs: tags: build:latest platforms: linux/${{matrix.cpu.platform}} build-args: PLATFORM=${{matrix.cpu.arch}} - cache-from: type=gha - cache-to: type=gha,mode=max - name: Build packages run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest @@ -208,17 +203,19 @@ jobs: steps: - name: checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive - name: Install dependencies - run: | - export ARCH=${{ matrix.arch }} - ./pkg/mac/build-static-library.sh + run: ARCH=${{ matrix.arch }} ./pkg/mac/build-static-library.sh - name: Zip artifact run: | - cd ./pkg/mac/.install - zip -r macos-${{ matrix.arch }}.zip ./include/pulsar/* ./lib/* - cp macos-${{ matrix.arch }}.zip ../../../ + mv build/include/pulsar/Version.h include/pulsar/ + mv build/lib/*.a lib/ + mv build/lib/*.dylib lib/ + zip -r macos-${{ matrix.arch }}.zip ./include/pulsar/* ./lib/*.a ./lib/*.dylib - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 9f113b03..a42ebae9 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -28,256 +28,15 @@ concurrency: cancel-in-progress: true jobs: - - formatting-check: - name: Formatting Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Run clang-format style check for C/C++/Protobuf programs. - uses: jidicula/clang-format-action@v4.11.0 - with: - clang-format-version: '11' - exclude-regex: '.*\.(proto|hpp)' - - wireshark-dissector-build: - name: Build the Wireshark dissector - needs: formatting-check - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - strategy: - matrix: - os: [ubuntu-20.04, macos-12] - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Install deps (Ubuntu) - if: ${{ startsWith(matrix.os, 'ubuntu') }} - run: | - sudo apt-get update -y - sudo apt-get install -y protobuf-compiler libprotobuf-dev wireshark-dev - - - name: Install deps (macOS) - if: ${{ startsWith(matrix.os, 'macos') }} - run: | - # See https://github.com/Homebrew/homebrew-core/issues/157142 - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - cd /usr/local/bin - rm -f 2to3* idle3* pydoc* python3* - rm -f /usr/local/share/man/man1/python3.1 /usr/local/lib/pkgconfig/python3* - cd /usr/local/Frameworks/Python.framework - rm -rf Headers Python Resources Versions/Current - brew update - brew install pkg-config wireshark protobuf - - name: Build wireshark plugin - run: | - cmake -S wireshark -B build-wireshark - cmake --build build-wireshark - - unit-tests: - name: Run unit tests - needs: formatting-check - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - - name: Build core libraries - run: | - cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=OFF - cmake --build . -j8 - - - name: Check formatting - run: | - ./vcpkg/vcpkg format-manifest vcpkg.json - if [[ $(git diff | wc -l) -gt 0 ]]; then - echo "Please run `./vcpkg/vcpkg format-manifest vcpkg.json` to reformat vcpkg.json" - exit 1 - fi - - - name: Build tests - run: | - cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON - cmake --build . -j8 - - - name: Install gtest-parallel - run: | - sudo curl -o /gtest-parallel https://raw.githubusercontent.com/google/gtest-parallel/master/gtest_parallel.py - - - name: Run unit tests - run: RETRY_FAILED=3 ./run-unit-tests.sh - - - name: Build perf tools - run: | - cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON -DBUILD_PERF_TOOLS=ON - cmake --build . -j8 - - - name: Verify custom vcpkg installation - run: | - mv vcpkg /tmp/ - cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_TOOLCHAIN_FILE="/tmp/vcpkg/scripts/buildsystems/vcpkg.cmake" - - cpp20-build: - name: Build with the C++20 standard - needs: formatting-check - runs-on: ubuntu-22.04 - timeout-minutes: 60 - - steps: - - name: checkout - uses: actions/checkout@v3 - - name: Install deps - run: | - sudo apt-get update -y - sudo apt-get install -y libcurl4-openssl-dev libssl-dev \ - protobuf-compiler libprotobuf-dev libboost-dev \ - libboost-dev libboost-program-options-dev \ - libzstd-dev libsnappy-dev libgmock-dev libgtest-dev - - name: CMake - run: cmake -B build -DBUILD_PERF_TOOLS=ON -DCMAKE_CXX_STANDARD=20 - - name: Build - run: | - cmake --build build -j8 --target pulsarShared pulsarStatic - cmake --build build -j8 - - cpp-build-windows: - timeout-minutes: 120 - name: Build CPP Client on ${{ matrix.name }} - needs: unit-tests - runs-on: ${{ matrix.os }} - env: - VCPKG_ROOT: '${{ github.workspace }}/vcpkg' - INSTALL_DIR: 'C:\\pulsar-cpp' - strategy: - fail-fast: false - matrix: - include: - - name: 'Windows x64' - os: windows-2019 - triplet: x64-windows-static - suffix: 'windows-win64' - generator: 'Visual Studio 16 2019' - arch: '-A x64' - - name: 'Windows x86' - os: windows-2019 - triplet: x86-windows-static - suffix: 'windows-win32' - generator: 'Visual Studio 16 2019' - arch: '-A Win32' - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Restore vcpkg and its artifacts. - uses: actions/cache@v3 - id: vcpkg-cache - with: - path: | - ${{ env.VCPKG_ROOT }} - vcpkg_installed - !${{ env.VCPKG_ROOT }}/.git - !${{ env.VCPKG_ROOT }}/buildtrees - !${{ env.VCPKG_ROOT }}/packages - !${{ env.VCPKG_ROOT }}/downloads - key: | - ${{ runner.os }}-${{ matrix.triplet}}-${{ hashFiles( 'vcpkg.json' ) }} - - - name: Get vcpkg(windows) - if: ${{ runner.os == 'Windows' && steps.vcpkg-cache.outputs.cache-hit != 'true' }} - run: | - cd ${{ github.workspace }} - mkdir build -force - git clone https://github.com/Microsoft/vcpkg.git - cd vcpkg - .\bootstrap-vcpkg.bat - - - name: remove system vcpkg(windows) - if: runner.os == 'Windows' - run: rm -rf "$VCPKG_INSTALLATION_ROOT" - shell: bash - - - name: Install vcpkg packages - run: | - ${{ env.VCPKG_ROOT }}\vcpkg.exe install --triplet ${{ matrix.triplet }} - - - name: Configure - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cmake \ - -B ./build-1 \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DUSE_ASIO=ON \ - -DBUILD_TESTS=OFF \ - -DVCPKG_TRIPLET="${{ matrix.triplet }}" \ - -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \ - -S . - fi - - - name: Install - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cmake --build ./build-1 --parallel --config Release - cmake --install ./build-1 - fi - - - name: Test examples - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cd win-examples - cmake \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DLINK_STATIC=OFF \ - -DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \ - -B build-dynamic - cmake --build build-dynamic --config Release - cmake \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DLINK_STATIC=ON \ - -DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \ - -B build-static - cmake --build build-static --config Release - ./build-static/Release/win-example.exe - fi - - - name: Build (Debug) - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cmake \ - -B ./build-2 \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DUSE_ASIO=ON \ - -DBUILD_TESTS=OFF \ - -DVCPKG_TRIPLET="${{ matrix.triplet }}" \ - -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \ - -DCMAKE_BUILD_TYPE=Debug \ - -S . - cmake --build ./build-2 --parallel --config Debug - fi - package: name: Build ${{matrix.pkg.name}} ${{matrix.cpu.platform}} runs-on: ubuntu-22.04 - needs: unit-tests timeout-minutes: 500 strategy: fail-fast: true matrix: pkg: - - { name: 'RPM', type: 'rpm', path: 'pkg/rpm/RPMS' } - - { name: 'Deb', type: 'deb', path: 'pkg/deb/BUILD/DEB' } - { name: 'Alpine', type: 'apk', path: 'pkg/apk/build' } cpu: - { arch: 'x86_64', platform: 'x86_64' } @@ -292,9 +51,6 @@ jobs: - name: Package Pulsar source run: build-support/generate-source-archive.sh - - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - - name: Build dependencies Docker image uses: docker/build-push-action@v3 with: @@ -303,66 +59,6 @@ jobs: tags: build:latest platforms: linux/${{matrix.cpu.platform}} build-args: PLATFORM=${{matrix.cpu.arch}} - cache-from: type=gha - cache-to: type=gha,mode=max - name: Build packages run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest - - cpp-build-macos: - timeout-minutes: 120 - name: Build CPP Client on macOS - needs: formatting-check - runs-on: macos-12 - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Install dependencies - run: brew install openssl protobuf boost zstd snappy googletest - - - name: Configure (default) - shell: bash - run: cmake -B ./build-macos -S . - - - name: Compile - shell: bash - run: | - cmake --build ./build-macos --parallel --config Release - - - name: Build with C++20 - shell: bash - run: | - cmake -B build-macos-cpp20 -DCMAKE_CXX_STANDARD=20 - cmake --build build-macos-cpp20 -j8 - - cpp-build-macos-static: - timeout-minutes: 120 - name: Build CPP Client on macOS with static dependencies - runs-on: macos-12 - needs: unit-tests - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Build libraries - run: ./pkg/mac/build-static-library.sh - - - name: Test static libraries - run: | - export PULSAR_DIR=$PWD/pkg/mac/.install - echo "Build with static library" - clang++ win-examples/example.cc -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a - ./static.out - echo "Build with dynamic library" - clang++ win-examples/example.cc -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar - ./dynamic.out - - # Job that will be required to complete and depends on all the other jobs - check-completion: - name: Check Completion - runs-on: ubuntu-latest - needs: [formatting-check, wireshark-dissector-build, unit-tests, cpp20-build, cpp-build-windows, package, cpp-build-macos] - - steps: - - run: true diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index c877c642..00000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,73 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "CodeQL" - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: '16 21 * * 2' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-22.04 - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'cpp' ] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - - - name: Install deps - run: | - sudo apt-get update -y - sudo apt-get install -y libcurl4-openssl-dev libssl-dev \ - protobuf-compiler libprotobuf-dev libboost-dev \ - libboost-dev libboost-program-options-dev \ - libzstd-dev libsnappy-dev - - - name: Build - run: | - cmake . -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DBUILD_PERF_TOOLS=ON - cmake --build . -j8 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - with: - category: "/language:${{matrix.language}}" diff --git a/build-support/copy-deps-versionfile.sh b/build-support/copy-deps-versionfile.sh deleted file mode 100755 index 8ecc8c00..00000000 --- a/build-support/copy-deps-versionfile.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e -x - -ROOT_DIR=$(git rev-parse --show-toplevel) - -for dir in apk deb rpm; do - mkdir -p pkg/$dir/.build - cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build - cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build -done diff --git a/build-support/dep-version.py b/build-support/dep-version.py deleted file mode 100755 index 2421d428..00000000 --- a/build-support/dep-version.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import yaml, sys - -deps = yaml.safe_load(open('dependencies.yaml')) -print(deps[sys.argv[1]]) diff --git a/build-support/merge_archives_vcpkg.sh b/build-support/merge_archives_vcpkg.sh index de1756e5..23c439d6 100755 --- a/build-support/merge_archives_vcpkg.sh +++ b/build-support/merge_archives_vcpkg.sh @@ -29,4 +29,4 @@ fi CMAKE_BUILD_DIRECTORY=$1 ./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \ $CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \ - $(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug) + $(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug | grep "$TRIPLET") diff --git a/dependencies.yaml b/dependencies.yaml deleted file mode 100644 index 8d338e4d..00000000 --- a/dependencies.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Note: GCC 4.8 is incompatible with Boost >= 1.84 for the missed std::align -boost: 1.83.0 -cmake: 3.28.3 -protobuf: 3.20.0 -zlib: 1.3.1 -zstd: 1.5.5 -snappy: 1.1.10 -openssl: 1.1.1w -curl: 8.6.0 diff --git a/pkg/apk/APKBUILD b/pkg/apk/APKBUILD index 3c3cdc70..3220f0f1 100644 --- a/pkg/apk/APKBUILD +++ b/pkg/apk/APKBUILD @@ -38,19 +38,27 @@ build() { if [ "$CBUILD" != "$CHOST" ]; then CMAKE_CROSSOPTS="-DCMAKE_SYSTEM_NAME=Linux -DCMAKE_HOST_SYSTEM_NAME=Linux" fi - cmake \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DBUILD_SHARED_LIBS=True \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ - -DCMAKE_C_FLAGS="$CFLAGS" \ - -DBUILD_TESTS=OFF \ - -DLINK_STATIC=ON \ - ${CMAKE_CROSSOPTS} ${ROOT_DIR} - make -j6 + git clone https://github.com/microsoft/vcpkg.git + mv vcpkg $ROOT_DIR/ + if [[ $PLATFORM == "aarch64" ]]; then + export VCPKG_FORCE_SYSTEM_BINARIES=1 + fi + set +e + cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_MAKE_PROGRAM=make \ + -DCMAKE_INSTALL_PREFIX="$pkgdir" \ + -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON \ + $ROOT_DIR + if [[ $? -ne 0 ]]; then + cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-CMakeCache.txt.log + cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-out.log + cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-err.log + exit 1 + fi + set -e + cmake --build build -j8 } package() { - make DESTDIR="$pkgdir" install + cmake --build build --target install } diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index d7d87181..ccf70e39 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -21,91 +21,23 @@ FROM alpine:3.12 ARG PLATFORM -# perl is required to install OpenSSL -RUN apk add \ - build-base \ +RUN apk add build-base \ linux-headers \ abuild \ - bash \ - curl \ - g++ \ make \ + curl \ cmake \ - python3 \ - py3-pip \ + git \ + ninja \ + zip \ + unzip \ + openssl-dev \ perl \ - sudo - -RUN pip3 install pyyaml - -ADD .build/dependencies.yaml / -ADD .build/dep-version.py /usr/local/bin - -# Download and compile boost -RUN BOOST_VERSION=$(dep-version.py boost) && \ - curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \ - tar zxf boost-${BOOST_VERSION}.tar.gz && \ - cd boost-${BOOST_VERSION} && \ - ./bootstrap.sh --with-libraries=regex && \ - ./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ - rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION} - -# Download and compile protobuf -RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ - curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ - tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ - cd protobuf-${PROTOBUF_VERSION}/ && \ - CXXFLAGS=-fPIC ./configure && \ - make -j8 && make install && \ - rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} - -# ZLib -RUN ZLIB_VERSION=$(dep-version.py zlib) && \ - curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \ - tar xfz v${ZLIB_VERSION}.tar.gz && \ - cd zlib-${ZLIB_VERSION} && \ - CFLAGS="-fPIC -O3" ./configure && \ - make -j8 && make install && \ - rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} - -# Zstandard -RUN ZSTD_VERSION=$(dep-version.py zstd) && \ - curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \ - tar xfz zstd-${ZSTD_VERSION}.tar.gz && \ - cd zstd-${ZSTD_VERSION} && \ - CFLAGS="-fPIC -O3" make -j8 && \ - make install && \ - rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz - -# Snappy -RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ - curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \ - tar xfz ${SNAPPY_VERSION}.tar.gz && \ - cd snappy-${SNAPPY_VERSION} && \ - CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ - make -j8 && make install && \ - rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz - -RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ - OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ - curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ - tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ - cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ - ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \ - make -j8 && make install && \ - rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} - -ENV LD_LIBRARY_PATH /usr/local/ssl/lib/: -ENV OPENSSL_ROOT_DIR /usr/local/ssl/ - -# LibCurl -RUN CURL_VERSION=$(dep-version.py curl) && \ - CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \ - curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \ - tar xfz curl-${CURL_VERSION}.tar.gz && \ - cd curl-${CURL_VERSION} && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd --without-libpsl && \ - make -j8 && make install && \ - rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} - - + bash + +# Build latest CMake for vcpkg +RUN curl -O -L https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0.tar.gz \ + && tar zxf cmake-3.30.0.tar.gz \ + && cd cmake-3.30.0 \ + && ./configure && make -j8 && make install \ + && cd .. && rm -rf cmake-3.30.0 cmake-3.30.0.tar.gz diff --git a/pkg/apk/docker-build-apk-arm64.sh b/pkg/apk/docker-build-apk-arm64.sh index 75631523..8480ee5f 100755 --- a/pkg/apk/docker-build-apk-arm64.sh +++ b/pkg/apk/docker-build-apk-arm64.sh @@ -25,7 +25,7 @@ ROOT_DIR=$(git rev-parse --show-toplevel) IMAGE_NAME=${1:-apachepulsar/pulsar-build:alpine-3.16-arm64} docker run -v $ROOT_DIR:/pulsar-client-cpp \ - --env PLATFORM=arm64 \ + --env PLATFORM=aarch64 \ $IMAGE_NAME \ /pulsar-client-cpp/pkg/apk/build-apk.sh diff --git a/pkg/deb/Dockerfile b/pkg/deb/Dockerfile index 502b0934..8cc986b8 100644 --- a/pkg/deb/Dockerfile +++ b/pkg/deb/Dockerfile @@ -29,88 +29,10 @@ RUN apt-get update -y && \ curl \ g++ \ make \ - perl \ dpkg-dev \ - python3 \ - python3-pip - -RUN pip3 install pyyaml - -ADD .build/dependencies.yaml / -ADD .build/dep-version.py /usr/local/bin - -# Download and compile boost -RUN BOOST_VERSION=$(dep-version.py boost) && \ - curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \ - tar zxf boost-${BOOST_VERSION}.tar.gz && \ - cd boost-${BOOST_VERSION} && \ - ./bootstrap.sh --with-libraries=regex && \ - ./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ - rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION} - -RUN CMAKE_VERSION=$(dep-version.py cmake) && \ - curl -O -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \ - tar xfz cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \ - cp cmake-${CMAKE_VERSION}-linux-${PLATFORM}/bin/* /usr/bin/ && \ - cp -r cmake-${CMAKE_VERSION}-linux-${PLATFORM}/share/cmake-* /usr/share/ && \ - rm -rf cmake-${CMAKE_VERSION}-linux-${PLATFORM} cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz - -# Download and compile protobuf -RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ - curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ - tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ - cd protobuf-${PROTOBUF_VERSION}/ && \ - CXXFLAGS=-fPIC ./configure && \ - make -j8 && make install && ldconfig && \ - rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} - -# ZLib -RUN ZLIB_VERSION=$(dep-version.py zlib) && \ - curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \ - tar xfz v${ZLIB_VERSION}.tar.gz && \ - cd zlib-${ZLIB_VERSION} && \ - CFLAGS="-fPIC -O3" ./configure && \ - make -j8 && make install && \ - rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} - -# Zstandard -RUN ZSTD_VERSION=$(dep-version.py zstd) && \ - curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \ - tar xfz zstd-${ZSTD_VERSION}.tar.gz && \ - cd zstd-${ZSTD_VERSION} && \ - CFLAGS="-fPIC -O3" make -j8 && \ - make install && \ - rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz - -# Snappy -RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ - curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \ - tar xfz ${SNAPPY_VERSION}.tar.gz && \ - cd snappy-${SNAPPY_VERSION} && \ - CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ - make -j8 && make install && \ - rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz - -RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ - OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ - curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ - tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ - cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ - ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \ - make -j8 && make install && \ - rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} - -ENV LD_LIBRARY_PATH /usr/local/ssl/lib/: -ENV OPENSSL_ROOT_DIR /usr/local/ssl/ - -# LibCurl -RUN CURL_VERSION=$(dep-version.py curl) && \ - CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \ - curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \ - tar xfz curl-${CURL_VERSION}.tar.gz && \ - cd curl-${CURL_VERSION} && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd --without-libpsl && \ - make -j8 && make install && \ - rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} - - + zip \ + unzip \ + tar \ + git \ + pkg-config \ + ninja-build diff --git a/pkg/deb/build-deb.sh b/pkg/deb/build-deb.sh index 8415056c..d0a33797 100755 --- a/pkg/deb/build-deb.sh +++ b/pkg/deb/build-deb.sh @@ -20,6 +20,10 @@ set -e -x +if [[ $# -gt 0 ]]; then + CMAKE_ARCH=$1 +fi + cd /pulsar-client-cpp SRC_ROOT_DIR=$(pwd) cd pkg/deb @@ -36,13 +40,24 @@ mkdir BUILD cd BUILD tar xfz $SRC_ROOT_DIR/apache-pulsar-client-cpp-$POM_VERSION.tar.gz pushd $CPP_DIR - -# link libraries for protoc -export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - +git clone https://github.com/microsoft/vcpkg.git +if [[ $CMAKE_ARCH ]]; then + curl -O -L https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-$CMAKE_ARCH.tar.gz + export PATH=$PWD/cmake-3.30.0-linux-$CMAKE_ARCH/bin/:$PATH + tar zxf cmake-*.tar.gz +fi chmod +x $(find . -name "*.sh") -cmake . -DBUILD_TESTS=OFF -DBUILD_PERF_TOOLS=OFF -DLINK_STATIC=ON -make -j 3 +if [[ $CMAKE_ARCH == "aarch64" ]]; then + export VCPKG_FORCE_SYSTEM_BINARIES=1 +fi +cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON +cmake --build build -j8 +./build-support/merge_archives_vcpkg.sh $PWD/build + +cp build/lib/libpulsar.a lib/libpulsar.a +cp build/lib/libpulsar.so lib/libpulsar.so +cp build/libpulsarwithdeps.a lib/libpulsarwithdeps.a popd DEST_DIR=apache-pulsar-client diff --git a/pkg/deb/docker-build-deb-arm64.sh b/pkg/deb/docker-build-deb-arm64.sh index f8c397c3..918ef9a8 100755 --- a/pkg/deb/docker-build-deb-arm64.sh +++ b/pkg/deb/docker-build-deb-arm64.sh @@ -27,4 +27,6 @@ IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-arm64} docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=arm64 \ $IMAGE_NAME \ - /pulsar-client-cpp/pkg/deb/build-deb.sh + /pulsar-client-cpp/pkg/deb/build-deb.sh \ + aarch64 + https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-aarch64.tar.gz diff --git a/pkg/deb/docker-build-deb-x86_64.sh b/pkg/deb/docker-build-deb-x86_64.sh index fecf6a1e..75376965 100755 --- a/pkg/deb/docker-build-deb-x86_64.sh +++ b/pkg/deb/docker-build-deb-x86_64.sh @@ -27,4 +27,5 @@ IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-x86_64} docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=amd64 \ $IMAGE_NAME \ - /pulsar-client-cpp/pkg/deb/build-deb.sh + /pulsar-client-cpp/pkg/deb/build-deb.sh \ + x86_64 diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index 4b97ac73..e7475263 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -19,179 +19,21 @@ # set -ex -cd `dirname $0` +cd `dirname $0`/../.. -pip3 install pyyaml - -MACOSX_DEPLOYMENT_TARGET=10.15 -if [[ -z ${ARCH} ]]; then - ARCH=`uname -m` -fi - -BUILD_DIR=$PWD/.build -INSTALL_DIR=$PWD/.install -PREFIX=$BUILD_DIR/install -mkdir -p $BUILD_DIR -cp -f ../../build-support/dep-version.py $BUILD_DIR/ -cp -f ../../dependencies.yaml $BUILD_DIR/ - -pushd $BUILD_DIR - -BOOST_VERSION=$(./dep-version.py boost) -ZLIB_VERSION=$(./dep-version.py zlib) -OPENSSL_VERSION=$(./dep-version.py openssl) -PROTOBUF_VERSION=$(./dep-version.py protobuf) -ZSTD_VERSION=$(./dep-version.py zstd) -SNAPPY_VERSION=$(./dep-version.py snappy) -CURL_VERSION=$(./dep-version.py curl) - -if [ ! -f boost/.done ]; then - echo "Building Boost $BOOST_VERSION" - curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz - tar zxf boost-${BOOST_VERSION}.tar.gz - mkdir -p $PREFIX/include - pushd boost-${BOOST_VERSION} - ./bootstrap.sh - ./b2 headers - cp -rf boost $PREFIX/include/ - popd - mkdir -p boost - touch boost/.done -else - echo "Using cached Boost $BOOST_VERSION" -fi - -if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then - echo "Building ZLib $ZLIB_VERSION" - curl -O -L https://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz - tar zxf zlib-${ZLIB_VERSION}.tar.gz - pushd zlib-$ZLIB_VERSION - CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX - make -j16 - make install - touch .done - popd -else - echo "Using cached ZLib $ZLIB_VERSION" -fi - -OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') -if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then - echo "Building OpenSSL $OPENSSL_VERSION" - curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz - tar zxf OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz - - pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} - if [[ $ARCH = 'arm64' ]]; then - PLATFORM=darwin64-arm64-cc - else - PLATFORM=darwin64-x86_64-cc - fi - CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./Configure --prefix=$PREFIX no-shared no-unit-test $PLATFORM - make -j8 >/dev/null - make install_sw >/dev/null - popd - - touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done -else - echo "Using cached OpenSSL $OPENSSL_VERSION" -fi - -if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then - echo "Building Protobuf $PROTOBUF_VERSION" - curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz - tar zxf protobuf-cpp-${PROTOBUF_VERSION}.tar.gz - pushd protobuf-${PROTOBUF_VERSION} - pushd cmake/ - # Build protoc that can run on both x86 and arm architectures - cmake -B build -DCMAKE_CXX_FLAGS="-fPIC -arch x86_64 -arch arm64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - -Dprotobuf_BUILD_TESTS=OFF \ - -DCMAKE_INSTALL_PREFIX=$PREFIX - cmake --build build -j16 --target install - popd - - # Retain the library for one architecture so that `ar` can work on the library - pushd $PREFIX/lib - mv libprotobuf.a libprotobuf_universal.a - lipo libprotobuf_universal.a -thin ${ARCH} -output libprotobuf.a - popd - touch .done - popd -else - echo "Using cached Protobuf $PROTOBUF_VERSION" -fi - -if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then - echo "Building ZStd $ZSTD_VERSION" - curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz - tar zxf zstd-${ZSTD_VERSION}.tar.gz - pushd zstd-${ZSTD_VERSION} - CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \ - make -j16 -C lib install-static install-includes - touch .done - popd +if [[ $ARCH == "arm64" ]]; then + VCPKG_TARGET_TRIPLET=arm64-osx + CMAKE_OSX_ARCHITECTURES=arm64 else - echo "Using cached ZStd $ZSTD_VERSION" -fi - -if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then - echo "Building Snappy $SNAPPY_VERSION" - curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz - tar zxf ${SNAPPY_VERSION}.tar.gz - pushd snappy-${SNAPPY_VERSION} - # Without this patch, snappy 1.10 will report a sign-compare error, which cannot be suppressed with the -Wno-sign-compare option in CI - curl -O -L https://raw.githubusercontent.com/microsoft/vcpkg/2024.02.14/ports/snappy/no-werror.patch - patch