Refactor CI

Set libraries download in alphabetical order. Bump actions/checkout to v4. Change run_tests matrix to more readable. Add env field to avoid code duplications inside job. resolves Desbordante#512
p-senichenkov · Jan 30, 2025 · 8c9450c · 8c9450c
1 parent c0b8efc
commit 8c9450c
Show file tree

Hide file tree

Showing 5 changed files with 104 additions and 102 deletions.
diff --git a/.github/composite-actions/install-dependencies/action.yml b/.github/composite-actions/install-dependencies/action.yml
@@ -1,18 +1,9 @@
 name: 'Install dependencies'
 description: 'Download and install build system and libraries'
 inputs:
-  download-pybind:
-    type: boolean
-    description: 'Download pybind11'
-    default: false
-
-  download-googletest:
-    type: boolean
-    description: 'Download googletest'
-    default: true
-
   os:
     type: string
+    description: 'Any default runner name in Github Actions'
     required: true
 
   toolset:
@@ -31,10 +22,20 @@ inputs:
     type: boolean
     default: true
 
+  download-googletest:
+    type: boolean
+    description: 'Download googletest'
+    default: true
+
+  download-pybind:
+    type: boolean
+    description: 'Download pybind11'
+    default: false
+
 runs:
   using: 'composite'
   steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Get short OS name
       shell: bash
@@ -88,51 +89,54 @@ runs:
         mkdir -p lib
       shell: bash
 
-    - name: Download googletest
-      uses: ./.github/composite-actions/download-library
-      with:
-        directory: googletest
-        download-command: git clone https://github.com/google/googletest/ --branch v1.14.0 --depth 1
-      if: inputs.download-googletest != 'false'
-
-    - name: Download easyloggingpp
+    - name: Download atomicbitvector
       uses: ./.github/composite-actions/download-library
       with:
-        directory: easyloggingpp
-        download-command: git clone https://github.com/amrayn/easyloggingpp/ --branch v9.97.0 --depth 1
+        directory: atomicbitvector
+        download-command: git clone https://github.com/ekg/atomicbitvector.git --depth 1
 
     - name: Download better-enums
       uses: ./.github/composite-actions/download-library
       with:
         directory: better-enums
         download-command: git clone https://github.com/aantron/better-enums.git --branch 0.11.3 --depth 1
 
-    - name: Download pybind11
+    - name: Download boost
       uses: ./.github/composite-actions/download-library
       with:
-        directory: pybind11
-        download-command: git clone https://github.com/pybind/pybind11.git --branch v2.13.4 --depth 1
-      if: inputs.download-pybind != 'false'
+        directory: boost
+        download-command: wget -O boost_1_85_0.tar.gz https://sourceforge.net/projects/boost/files/boost/1.85.0/boost_1_85_0.tar.gz/download && tar xzvf boost_1_85_0.tar.gz && mv boost_1_85_0 boost
+
+    - name: Download easyloggingpp
+      uses: ./.github/composite-actions/download-library
+      with:
+        directory: easyloggingpp
+        download-command: git clone https://github.com/amrayn/easyloggingpp/ --branch v9.97.0 --depth 1
+
     - name: Download emhash
       uses: ./.github/composite-actions/download-library
       with:
         directory: emhash
         download-command: git clone https://github.com/ktprime/emhash.git --depth 1
-    - name: Download atomicbitvector
+
+    - name: Download googletest
       uses: ./.github/composite-actions/download-library
       with:
-        directory: atomicbitvector
-        download-command: git clone https://github.com/ekg/atomicbitvector.git --depth 1
-    - name: Download boost
+        directory: googletest
+        download-command: git clone https://github.com/google/googletest/ --branch v1.14.0 --depth 1
+      if: inputs.download-googletest != 'false'
+
+    - name: Download pybind11
       uses: ./.github/composite-actions/download-library
       with:
-        directory: boost
-        download-command: wget -O boost_1_85_0.tar.gz https://sourceforge.net/projects/boost/files/boost/1.85.0/boost_1_85_0.tar.gz/download && tar xzvf boost_1_85_0.tar.gz && mv boost_1_85_0 boost
+        directory: pybind11
+        download-command: git clone https://github.com/pybind/pybind11.git --branch v2.13.4 --depth 1
+      if: inputs.download-pybind != 'false'
 
-    - name: Install Boost built with GCC
+    - name: Install Boost built with GCC (on Ubuntu)
       run: |
         cd lib/boost
-        ./bootstrap.sh --prefix=/usr
+        ./bootstrap.sh --with-libraries=container,thread,graph
         sudo ./b2 install --prefix=/usr
       shell: bash
       if: inputs.install-boost == 'true' && inputs.toolset == 'gcc'

diff --git a/.github/workflows/bindings-tests.yml b/.github/workflows/bindings-tests.yml
@@ -30,33 +30,44 @@ on:
   workflow_dispatch:
 jobs:
   test-python-bindings:
+    name: Run python-bindings tests on ${{ matrix.os }} with ${{ matrix.toolset }}
     strategy:
       matrix:
         include:
           - os: ubuntu-latest
-            compiler: gcc
-            env: CC=gcc-10 CXX=g++-10
+            toolset: gcc
+            env: CXX=g++-10
+
           - os: ubuntu-latest
-            compiler: llvm-clang
-            env: CC=clang-17 CXX=clang++-17 CXXFLAGS="-stdlib=libc++" LDFLAGS="-lc++abi"
-          # Uncomment this to enable macOS-llvm-clang tests:
+            toolset: llvm-clang
+            env: CXX=clang++-17 CXXFLAGS="-stdlib=libc++" LDFLAGS="-lc++abi"
+
+          # Uncomment this to enable macOS gcc tests:
           # - os: macos-latest
-          #   compiler: llvm-clang
-          #   env: CC=$(brew --prefix llvm@17)/bin/clang CXX=$(brew --prefix llvm@17)/bin/clang++ BOOST_ROOT=/usr/local
+          #   toolset: gcc 
+          #   env: CXX=g++-14 BOOST_ROOT=/usr/local
           #   runtime-env: DYLD_LIBRARY_PATH=/usr/local/lib:${DYLD_LIBRARY_PATH}
+
+          # Uncomment this to enable macOS llvm-clang tests:
+          # - os: macos-latest
+          #   toolset: llvm-clang
+          #   env: CXX=$(brew --prefix llvm@17)/bin/clang++ BOOST_ROOT=/usr/local
+          #   runtime-env: DYLD_LIBRARY_PATH=/usr/local/lib:${DYLD_LIBRARY_PATH}
+
           - os: macos-latest
-            compiler: apple-clang
-            env: CC=clang CXX=clang++ BOOST_ROOT=$(brew --prefix boost)
+            toolset: apple-clang
+            env: CXX=clang++ BOOST_ROOT=$(brew --prefix boost)
+
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install dependencies
         uses: ./.github/composite-actions/install-dependencies
         with:
           os: ${{ matrix.os }}
-          toolset: ${{ matrix.compiler }}
-          download-pybind: true
+          toolset: ${{ matrix.toolset }}
           download-googletest: false
+          download-pybind: true
       - name: Build pip package
         shell: bash
         run: |

diff --git a/.github/workflows/check-codestyle.yml b/.github/workflows/check-codestyle.yml
@@ -17,7 +17,7 @@ jobs:
   clang-format-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Pull clang-format

diff --git a/.github/workflows/core-tests.yml b/.github/workflows/core-tests.yml
@@ -28,71 +28,60 @@ on:
   workflow_dispatch:
 jobs:
   run_tests:
+    name: Run core-tests on ${{ matrix.system.os }} with ${{ matrix.system.toolset }}, ${{matrix.cfg.BUILD_TYPE}} ${{ matrix.cfg.SANITIZER }}
     strategy:
       matrix:
-        os:
-          - ubuntu-latest
-          - macos-latest
-        compiler:
-          - gcc
-          - llvm-clang
-          - apple-clang
+        system:
+          - { 
+              os: ubuntu-latest, 
+              toolset: gcc,
+              env: CXX=g++-10,
+            }
+          - { 
+              os: ubuntu-latest, 
+              toolset: llvm-clang,
+              env: CXX=clang++-17 CXXFLAGS="-stdlib=libc++" LDFLAGS="-lc++abi"
+            }
+          # Uncomment this to enable macOS GCC tests:
+          # - { 
+          #     os: macos-latest, 
+          #     toolset: gcc,
+          #     env: CXX=g++-14 BOOST_ROOT=/usr/local,
+          #     runtime-env: DYLD_LIBRARY_PATH=/usr/local/lib:${DYLD_LIBRARY_PATH}
+          #   }
+          # Uncomment this to enable macOS LLVM Clang tests:
+          # - { 
+          #     os: macos-latest, 
+          #     toolset: llvm-clang,
+          #     env: CXX=$(brew --prefix llvm@17)/bin/clang++ CXXFLAGS="-I$(brew --prefix llvm@17)/include" \
+          #       LDFLAGS="-L$(brew --prefix llvm@17)/lib/c++" BOOST_ROOT=/usr/local,
+          #     runtime-env: DYLD_LIBRARY_PATH=/usr/local/lib:${DYLD_LIBRARY_PATH}
+          #   }
+          - { 
+              os: macos-latest, 
+              toolset: apple-clang,
+              env: CXX=clang++,
+            }
         cfg:
           - { BUILD_TYPE: Release }
           - { BUILD_TYPE: Debug }
           - { BUILD_TYPE: Debug, SANITIZER : ADDRESS }
           - { BUILD_TYPE: Debug, SANITIZER : UB }
-        exclude:
-          - os: ubuntu-latest
-            compiler: apple-clang
-          - os: macos-latest
-            compiler: gcc
-          # Comment this to enable macOS-llvm-clang tests:
-          - os: macos-latest
-            compiler: llvm-clang
-    runs-on: ${{ matrix.os }}
+
+    runs-on: ${{ matrix.system.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install dependencies
         uses: ./.github/composite-actions/install-dependencies
         with:
-          os: ${{ matrix.os }}
-          toolset: ${{ matrix.compiler }}
+          os: ${{ matrix.system.os }}
+          toolset: ${{ matrix.system.toolset }}
       - name: Download datasets
         uses: ./.github/composite-actions/download-datasets
       - name: Build
         shell: bash
-        # Maybe separate composite action will be more readable?
         run: |
-          if [[ ${{ matrix.os }} == 'ubuntu-latest' && ${{ matrix.compiler }} == 'gcc' ]]; then
-            echo "Ubuntu, GCC"
-
-            export CC=gcc-10
-            export CXX=g++-10
-          elif [[ ${{ matrix.os }} == 'ubuntu-latest' && ${{ matrix.compiler }} == 'llvm-clang' ]]; then
-            echo "Ubuntu, LLVM Clang"
-
-            export CC=clang-17
-            export CXX=clang++-17
-            export CXXFLAGS="-stdlib=libc++"
-            export LDFLAGS="-lc++abi"
-          elif [[ ${{ matrix.os }} == 'macos-latest' && ${{ matrix.compiler }} == 'llvm-clang' ]]; then
-            echo "macOS, LLVM Clang"
-
-            export CC=$(brew --prefix llvm@17)/bin/clang
-            export CXX=$(brew --prefix llvm@17)/bin/clang++
-            export CXXFLAGS="-I$(brew --prefix llvm@17)/include"
-            export LDFLAGS="-L$(brew --prefix llvm@17)/lib/c++"
-            export BOOST_ROOT=/usr/local
-          elif [[ ${{ matrix.os }} == 'macos-latest' && ${{ matrix.compiler }} == 'apple-clang' ]]; then
-            echo "macOS, Apple Clang"
-
-            export CC=clang
-            export CXX=clang++
-          else
-            echo 'Invalid matrix configuration'
-            exit 1
-          fi
+          export ${{ matrix.system.env }}
 
           if [[ "${{matrix.cfg.BUILD_TYPE}}" == "Debug" ]]; then
             ./build.sh --debug --sanitizer=${{ matrix.cfg.SANITIZER }}
@@ -103,8 +92,6 @@ jobs:
         working-directory: ${{github.workspace}}/build/target
         shell: bash
         run: |
-          if [[ ${{ matrix.os }} == 'macos-latest' ]]; then
-            export DYLD_LIBRARY_PATH=/usr/local/lib:${DYLD_LIBRARY_PATH}
-          fi
+          export ${{ matrix.system.runtime-env }}
 
           ./Desbordante_test --gtest_filter='*:-*HeavyDatasets*'
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
@@ -39,7 +39,7 @@ jobs:
     outputs:
       include: ${{ steps.set-matrix.outputs.include }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install cibuildwheel
         run: pipx install cibuildwheel==2.16.2
       - id: set-matrix
@@ -68,7 +68,7 @@ jobs:
         include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }}
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install dependencies
         uses: ./.github/composite-actions/install-dependencies
@@ -108,7 +108,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-wheels
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Download all artifacts from previous step
         uses: actions/download-artifact@v4
         with:
@@ -142,7 +142,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'release' && github.event.action == 'published'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Download wheels
         uses: actions/download-artifact@v4
         with: