fix ci

spack · May 6, 2024 · dac4e1d · dac4e1d
1 parent eeb903a
commit dac4e1d
Show file tree

Hide file tree

Showing 14 changed files with 121 additions and 283 deletions.
diff --git a/.github/workflows/clingo.yml b/.github/workflows/clingo.yml
@@ -15,90 +15,44 @@ concurrency:
   group: clingo-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  SPACK_COLOR: always
+  SPACK_BACKTRACE: please
+
 jobs:
-  macos_clingo_x86_64:
-    runs-on: macos-latest
+  macos_clingo:
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+        os: ["macos-12", "macos-14"]
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+        with:
+          repository: spack/spack
+          ref: 9b4ca0be40edb02115be94bbc428ac546d63030e
+          path: spack
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Prepare Spack
         run: |
-          pip install --upgrade pip
           brew install bison
           export PATH="/usr/local/opt/bison/bin:$PATH"
-          git clone https://github.com/spack/spack.git spack-src
-          cd spack-src && git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f
-          if [ ${{ matrix.python-version }} = "3.12" ] ; then
-            # distutils -> setuptools patch
-            wget -q -O - https://github.com/spack/spack/commit/1452e927719ddbbf07118e88f9fd8d18fabf0fae.patch | git apply -v             
-          fi
-          cd ..
-          . spack-src/share/spack/setup-env.sh
-          spack external find --not-buildable cmake bison
-          spack config add "config:install_tree:padded_length:256"
-      - name: Install clingo
-        run: |
-          . spack-src/share/spack/setup-env.sh
-          spack python clingo/scripts/install_clingo.py
-          mkdir -p binary-mirror
-          spack buildcache push --unsigned --force $PWD/binary-mirror clingo-bootstrap
-      - uses: actions/upload-artifact@v3
-        with:
-          name: clingo_binary_mirror
-          path: binary-mirror
-
-  macos_clingo_aarch64:
-    runs-on: ["self-hosted", "macOS", "ARM64"]
-    strategy:
-      matrix:
-        python-version: ["3.6.15", "3.7.14", "3.8.14", "3.9.14", "3.10.7", "3.11.5", "3.12.0"]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Prepare Spack
-        env:
-          PYENV_VERSION: ${{ matrix.python-version }}
-        run: |
-          export PATH=/Users/spack-bootstrap-runner/homebrew/bin:/Users/spack-bootstrap-runner/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
-          export PATH="$(pyenv prefix)/bin:$PATH"
-          python --version
-          
-          brew install cmake bison
-          export PATH="/Users/spack-bootstrap-runner/homebrew/opt/bison/bin:$PATH"
-          
-          pyenv exec pip install --upgrade pip
-          
-          git clone https://github.com/spack/spack.git spack-src
-          cd spack-src && git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f
-          if [ ${{ matrix.python-version }} = "3.12.0" ] ; then
-            # distutils -> setuptools patch
-            wget -q -O - https://github.com/spack/spack/commit/1452e927719ddbbf07118e88f9fd8d18fabf0fae.patch | git apply -v             
-          fi
-          cd ..
-          . spack-src/share/spack/setup-env.sh
+          . spack/share/spack/setup-env.sh
           spack external find --not-buildable cmake bison
-          spack config add "config:install_tree:padded_length:256"
+          spack config add config:install_tree:padded_length:256
       - name: Install clingo
-        env:
-          PYENV_VERSION: ${{ matrix.python-version }}
         run: |
-          export PATH=/Users/spack-bootstrap-runner/homebrew/bin:/Users/spack-bootstrap-runner/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
-          export PATH="$(pyenv prefix)/bin:$PATH"
-          python --version
-          . spack-src/share/spack/setup-env.sh
+          . spack/share/spack/setup-env.sh
           spack python clingo/scripts/install_clingo.py
-          mkdir -p binary-mirror
-          spack buildcache push --unsigned --force $PWD/binary-mirror clingo-bootstrap
+          spack buildcache push --unsigned ./binary-mirror clingo-bootstrap
       - uses: actions/upload-artifact@v3
         with:
           name: clingo_binary_mirror
           path: binary-mirror
 
-
   manylinux2014:
     runs-on: ["self-hosted", "Linux"]
     outputs:
@@ -150,30 +104,22 @@ jobs:
       SPACK_MANYLINUX2014_TAG: ${{ needs.manylinux2014.outputs.spack_manylinux_tag }}
     steps:
       - uses: actions/checkout@v4
-      - run: |
-          . ./clingo/scripts/copy_mirror_manylinux2014.sh
+      - run: ./contrib/copy_mirror_manylinux2014.sh
       - uses: actions/upload-artifact@v3
         with:
           name: clingo_binary_mirror
-          path: |
-            binary-mirror
+          path: binary-mirror
 
   clingo_json:
     runs-on: ubuntu-latest
     needs: [ upload-manylinux2014, macos_clingo_x86_64, macos_clingo_aarch64 ]
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
       - uses: actions/download-artifact@v3
         with:
           name: clingo_binary_mirror
-      - run: |
-          pip install ruamel.yaml
-          python clingo/scripts/clingo_json.py
+      - run: python3 clingo/scripts/clingo_json.py
       - uses: actions/upload-artifact@v3
         with:
           name: clingo_manifest
-          path: |
-            clingo.json
+          path: clingo.json
diff --git a/.github/workflows/gnupg.yml b/.github/workflows/gnupg.yml
@@ -15,42 +15,40 @@ concurrency:
   group: gnupg-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  SPACK_COLOR: always
+  SPACK_BACKTRACE: please
+
 jobs:
   macos_gnupg:
-    runs-on: ${{ matrix.runner[0] }}
+    runs-on: ${{ matrix.os }}
     strategy:
-      # List of:
-      # 1. Runners
-      # 2. Target architectures
       matrix:
-        runner: [
-          ["macos-latest", "x86_64"], 
-          [["self-hosted", "macOS", "ARM64"], "aarch64"]
-        ]
+        os: ["macos-12", "macos-14"]
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+        with:
+          repository: spack/spack
+          ref: 9b4ca0be40edb02115be94bbc428ac546d63030e
+          path: spack
       - run: |
-          if [[ ${{ matrix.runner[1] }} == "aarch64" ]] ; then
-              export PATH=/Users/spack-bootstrap-runner/homebrew/bin:/Users/spack-bootstrap-runner/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
-          fi
           brew install gawk perl
-          git clone https://github.com/spack/spack.git spack-src
-          cd spack-src && git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f && cd ..
 
           # Disables internationalization to avoid linking to
           # libintl on MacOS, since that will make the binary
           # non portable
-          cd spack-src && git apply ../gnupg/patches/gnupg_macos.patch && cd ..    
+          git -C spack apply ../gnupg/patches/gnupg_macos.patch
 
-          . spack-src/share/spack/setup-env.sh
+          . spack/share/spack/setup-env.sh
           spack external find --not-buildable gawk perl
-          spack config add "config:concretizer:original"
-          spack config add "config:install_tree:padded_length:256"
-          mkdir -p binary-mirror
+          spack config add config:concretizer:original
+          spack config add config:install_tree:padded_length:256
+          spack python -c "import archspec.cpu;print(archspec.cpu.host().family)" > target.txt
 
           # Inject flags to have compatibility with MacOSX version 10.13 or higher
-          spack install gnupg cflags="-mmacosx-version-min=10.13" target=${{ matrix.runner[1] }}
-          spack buildcache push --unsigned --force $PWD/binary-mirror gnupg
+          spack install gnupg cflags="-mmacosx-version-min=10.13" "target=$(cat target.txt)"
+          spack buildcache push --unsigned ./binary-mirror gnupg
       - uses: actions/upload-artifact@v3
         with:
           name: gnupg_binary_mirror
@@ -106,31 +104,27 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - run: |
-          . ./gnupg/scripts/copy_mirror_manylinux2014.sh
+      - run: ./contrib/copy_mirror_manylinux2014.sh
       - uses: actions/upload-artifact@v3
         with:
           name: gnupg_binary_mirror
-          path: |
-            binary-mirror
+          path: binary-mirror
 
   gnupg_json:
     runs-on: ubuntu-latest
     needs: [ upload-manylinux2014, macos_gnupg ]
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
         with:
-          python-version: 3.11
+          repository: spack/spack
+          ref: 9b4ca0be40edb02115be94bbc428ac546d63030e
+          path: spack
       - uses: actions/download-artifact@v3
         with:
           name: gnupg_binary_mirror
-      - run: |
-          git clone https://github.com/spack/spack.git spack-src
-          . spack-src/share/spack/setup-env.sh
-          spack python gnupg/scripts/gnupg_json.py
+      - run: ./spack/bin/spack python gnupg/scripts/gnupg_json.py
       - uses: actions/upload-artifact@v3
         with:
           name: gnupg_manifest
-          path: |
-            gnupg.json
+          path: gnupg.json
diff --git a/.github/workflows/patchelf.yml b/.github/workflows/patchelf.yml
@@ -15,8 +15,11 @@ concurrency:
   group: patchelf-${{ github.ref }}
   cancel-in-progress: true
 
-jobs:
+env:
+  SPACK_COLOR: always
+  SPACK_BACKTRACE: please
 
+jobs:
   manylinux2014:
     runs-on: ubuntu-20.04
     outputs:
@@ -67,30 +70,22 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - run: |
-          . ./patchelf/scripts/copy_mirror_manylinux2014.sh
+      - run: ./contrib/copy_mirror_manylinux2014.sh
       - uses: actions/upload-artifact@v3
         with:
           name: patchelf_binary_mirror
-          path: |
-            binary-mirror
+          path: binary-mirror
 
   patchelf_json:
     runs-on: ubuntu-latest
     needs: [ upload-manylinux2014 ]
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
       - uses: actions/download-artifact@v3
         with:
           name: patchelf_binary_mirror
-      - run: |
-          pip install ruamel.yaml
-          python patchelf/scripts/patchelf_json.py
+      - run: python3 patchelf/scripts/patchelf_json.py
       - uses: actions/upload-artifact@v3
         with:
           name: patchelf_manifest
-          path: |
-            patchelf.json
+          path: patchelf.json
diff --git a/clingo/Dockerfile.manylinux2014 b/clingo/Dockerfile.manylinux2014
@@ -1,37 +1,28 @@
 FROM ghcr.io/spack/manylinux2014:v2023-10-03
 
-RUN cd /opt/_internal && tar xvf static-libs-for-embedding-only.tar.xz && cd -
+RUN tar -C /opt/_internal -xvf static-libs-for-embedding-only.tar.xz
 
-RUN adduser -m spack
+ENV SPACK_PYTHON=/opt/python/cp311-cp311/bin/python
+ENV PATH="/root/spack/bin:${PATH}"
+WORKDIR /root
 
-# Switch to a non-root user
-USER spack
-ENV SPACK_CMD="/opt/python/cp311-cp311/bin/python /home/spack/spack/bin/spack"
-WORKDIR /home/spack
-
-# Clone the repo and install Spack
-RUN git clone https://www.github.com/spack/spack.git && \
-    cd spack && \
-    git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f && \
-    cd ..
-
-# Set externals, locate compilers
-RUN ${SPACK_CMD} external find -j 1 --not-buildable bison cmake
-RUN ${SPACK_CMD} compiler find
-RUN ${SPACK_CMD} config add "config:concretizer:original"
-RUN ${SPACK_CMD} config add "config:install_tree:padded_length:256"
+RUN git clone -q https://github.com/spack/spack.git && \
+    git -C spack checkout -b docker-reference 9b4ca0be40edb02115be94bbc428ac546d63030e
+RUN spack external find -j 1 --not-buildable bison cmake
+RUN spack compiler find
+RUN spack config add config:concretizer:original
+RUN spack config add config:install_tree:padded_length:256
 
 # Run a script to build all the versions of clingo we could
-COPY --chown=spack:spack clingo/scripts/bootstrap_clingo_manylinux2014.sh /home/spack/bootstrap_clingo2014.sh
-COPY --chown=spack:spack clingo/scripts/install_clingo.py /home/spack/install_clingo.py
-
-RUN ${HOME}/bootstrap_clingo2014.sh 36
-RUN ${HOME}/bootstrap_clingo2014.sh 37
-RUN ${HOME}/bootstrap_clingo2014.sh 38
-RUN ${HOME}/bootstrap_clingo2014.sh 39
-RUN ${HOME}/bootstrap_clingo2014.sh 310
-RUN ${HOME}/bootstrap_clingo2014.sh 311
-RUN ${HOME}/bootstrap_clingo2014.sh 312
-
-COPY --chown=spack:spack clingo/scripts/create_binary_mirror.sh /home/spack/create_binary_mirror.sh
-RUN ${HOME}/create_binary_mirror.sh
+COPY clingo/scripts/bootstrap_clingo_manylinux2014.sh /root/bootstrap_clingo2014.sh
+COPY clingo/scripts/install_clingo.py /root/install_clingo.py
+
+RUN ./bootstrap_clingo2014.sh 36
+RUN ./bootstrap_clingo2014.sh 37
+RUN ./bootstrap_clingo2014.sh 38
+RUN ./bootstrap_clingo2014.sh 39
+RUN ./bootstrap_clingo2014.sh 310
+RUN ./bootstrap_clingo2014.sh 311
+RUN ./bootstrap_clingo2014.sh 312
+
+RUN spack buildcache push --unsigned ./binary-mirror $(spack find --format="/{hash}" clingo-bootstrap)
diff --git a/clingo/scripts/bootstrap_clingo_manylinux2014.sh b/clingo/scripts/bootstrap_clingo_manylinux2014.sh
@@ -1,14 +1,9 @@
-#!/bin/bash
+#!/bin/sh
 
 # Cycle over all the Python interpreters in manylinux2014
 # and install the corresponding clingo-bootstrap binary
-for PYTHON in /opt/python/cp${1}*/bin/python; do
-
+for PYTHON in /opt/python/cp"$1"*/bin/python; do
   # Install clingo using the current Python as an external
   # The Python version will be output to stdout
   ${PYTHON} spack/bin/spack python install_clingo.py
-
 done
-
-# Create a binary mirror
-#/bin/bash create_binary_mirror.sh
diff --git a/clingo/scripts/copy_mirror_manylinux2014.sh b/clingo/scripts/copy_mirror_manylinux2014.sh
diff --git a/clingo/scripts/create_binary_mirror.sh b/clingo/scripts/create_binary_mirror.sh