Skip to content

Commit

Permalink
Update bootstrap buildcache for SC release (#24)
Browse files Browse the repository at this point in the history
  • Loading branch information
alalazo authored Oct 9, 2023
1 parent 3c25050 commit 16caa84
Show file tree
Hide file tree
Showing 12 changed files with 118 additions and 115 deletions.
28 changes: 19 additions & 9 deletions .github/workflows/clingo.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
runs-on: macos-latest
strategy:
matrix:
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
Expand All @@ -32,6 +32,12 @@ jobs:
brew install bison
export PATH="/usr/local/opt/bison/bin:$PATH"
git clone https://github.com/spack/spack.git spack-src
cd spack-src && git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f
if [ ${{ matrix.python-version }} = "3.12" ] ; then
# distutils -> setuptools patch
wget -q -O - https://github.com/spack/spack/commit/1452e927719ddbbf07118e88f9fd8d18fabf0fae.patch | git apply -v
fi
cd ..
. spack-src/share/spack/setup-env.sh
spack external find --not-buildable cmake bison
spack config add "config:install_tree:padded_length:256"
Expand All @@ -40,7 +46,7 @@ jobs:
. spack-src/share/spack/setup-env.sh
spack python clingo/scripts/install_clingo.py
mkdir -p binary-mirror
spack buildcache create -d $PWD/binary-mirror -a -u -f clingo-bootstrap
spack buildcache push --unsigned --force $PWD/binary-mirror clingo-bootstrap
- uses: actions/upload-artifact@v3
with:
name: clingo_binary_mirror
Expand All @@ -50,7 +56,7 @@ jobs:
runs-on: ["self-hosted", "macOS", "ARM64"]
strategy:
matrix:
python-version: ["3.6.15", "3.7.14", "3.8.14", "3.9.14", "3.10.7", "3.11.0"]
python-version: ["3.6.15", "3.7.14", "3.8.14", "3.9.14", "3.10.7", "3.11.5", "3.12.0"]
steps:
- uses: actions/checkout@v3
- name: Prepare Spack
Expand All @@ -67,6 +73,12 @@ jobs:
pyenv exec pip install --upgrade pip
git clone https://github.com/spack/spack.git spack-src
cd spack-src && git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f
if [ ${{ matrix.python-version }} = "3.12.0" ] ; then
# distutils -> setuptools patch
wget -q -O - https://github.com/spack/spack/commit/1452e927719ddbbf07118e88f9fd8d18fabf0fae.patch | git apply -v
fi
cd ..
. spack-src/share/spack/setup-env.sh
spack external find --not-buildable cmake bison
spack config add "config:install_tree:padded_length:256"
Expand All @@ -80,7 +92,7 @@ jobs:
. spack-src/share/spack/setup-env.sh
spack python clingo/scripts/install_clingo.py
mkdir -p binary-mirror
spack buildcache create -d $PWD/binary-mirror -a -u -f clingo-bootstrap
spack buildcache push --unsigned --force $PWD/binary-mirror clingo-bootstrap
- uses: actions/upload-artifact@v3
with:
name: clingo_binary_mirror
Expand Down Expand Up @@ -122,11 +134,9 @@ jobs:
with:
file: ./clingo/Dockerfile.manylinux2014
platforms: linux/arm64,linux/ppc64le,linux/amd64
# Self-hosted runners use the local cache
# cache-from: type=gha
# cache-to: type=gha,mode=max
cache-from: type=local,src=/tmp/.buildx-cache-clingo-manylinux2014
cache-to: type=local,dest=/tmp/.buildx-cache-clingo-manylinux2014,mode=max
cache-from: |
${{ steps.docker_meta.outputs.tags }}
cache-to: type=inline
pull: ${{ github.event_name == 'pull_request' }}
push: true
tags: ${{ steps.docker_meta.outputs.tags }}
Expand Down
18 changes: 9 additions & 9 deletions .github/workflows/gnupg.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ jobs:
fi
brew install gawk perl
git clone https://github.com/spack/spack.git spack-src
cd spack-src && git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f && cd ..
# Disables internationalization to avoid linking to
# libintl on MacOS, since that will make the binary
Expand All @@ -49,7 +50,7 @@ jobs:
# Inject flags to have compatibility with MacOSX version 10.13 or higher
spack install gnupg cflags="-mmacosx-version-min=10.13" target=${{ matrix.runner[1] }}
spack buildcache create -d $PWD/binary-mirror -a -u -f gnupg
spack buildcache push --unsigned --force $PWD/binary-mirror gnupg
- uses: actions/upload-artifact@v3
with:
name: gnupg_binary_mirror
Expand Down Expand Up @@ -89,11 +90,9 @@ jobs:
file: ./gnupg/Dockerfile.manylinux2014
platforms: linux/arm64,linux/ppc64le,linux/amd64
pull: ${{ github.event_name == 'pull_request' }}
# Self-hosted runners use the local cache
# cache-from: type=gha
# cache-to: type=gha,mode=max
cache-from: type=local,src=/tmp/.buildx-cache-gnupg-manylinux2014
cache-to: type=local,dest=/tmp/.buildx-cache-gnupg-manylinux2014,mode=max
cache-from: |
${{ steps.docker_meta.outputs.tags }}
cache-to: type=inline
push: true
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
Expand Down Expand Up @@ -121,13 +120,14 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: 3.9
python-version: 3.11
- uses: actions/download-artifact@v3
with:
name: gnupg_binary_mirror
- run: |
pip install ruamel.yaml
python gnupg/scripts/gnupg_json.py
git clone https://github.com/spack/spack.git spack-src
. spack-src/share/spack/setup-env.sh
spack python gnupg/scripts/gnupg_json.py
- uses: actions/upload-artifact@v3
with:
name: gnupg_manifest
Expand Down
8 changes: 3 additions & 5 deletions .github/workflows/patchelf.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,9 @@ jobs:
file: ./patchelf/Dockerfile.manylinux2014
platforms: linux/arm64,linux/ppc64le,linux/amd64
pull: ${{ github.event_name == 'pull_request' }}
# Self-hosted runners use the local cache
# cache-from: type=gha
# cache-to: type=gha,mode=max
cache-from: type=local,src=/tmp/.buildx-cache-patchelf-manylinux2014
cache-to: type=local,dest=/tmp/.buildx-cache-patchelf-manylinux2014,mode=max
cache-from: |
${{ steps.docker_meta.outputs.tags }}
cache-to: type=inline
push: true
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
Expand Down
14 changes: 7 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ For completeness we report a summary of Spack requirements below:

Name | Supported Versions | System Requirement | Requirement Reason
--- | -------------------|---------------------|--------------------
Python | 3.6-3.11 | Yes | Interpreter for Spack
Python | 3.6-3.12 | Yes | Interpreter for Spack
C/C++ compilers | - | Yes | Building software
GNU make | - | Yes | Building software
patch | - | Yes | Building software
Expand All @@ -39,7 +39,7 @@ xz | - | Yes | Archive compression
zstd | - | Yes | Archive compression
file | - | Yes | Binary packages
patchelf | 0.13 or later | No | Binary packages
GnuPG | 2.1 or later | No | Binary packages
GnuPG | 2.3 or later | No | Binary packages
clingo | 5.5 | No | Concretization
git | - | Yes | Software repositories
hg | - | Yes | Software repositories
Expand All @@ -53,11 +53,11 @@ by compatibility with the [manylinux](https://github.com/pypa/manylinux) project

Platform | OS | Compiler Toolchain | Architecture | Python
---------|----|--------------------|--------------|-------
`linux` | `centos7` | `GCC 10.2.1`| `x86_64` | 3.6-3.11
`linux` | `centos7` | `GCC 10.2.1`| `aarch64` | 3.6-3.11
`linux` | `centos7` | `GCC 10.2.1`| `ppc64le` | 3.6-3.11
`darwin`| `MacOS 10.13` or later | `Apple Clang 13.0.0` | `x86_64` | 3.6-3.11
`darwin`| `MacOS 10.13` or later | `Apple Clang 13.1.6` | `aarch64` | 3.6-3.11
`linux` | `centos7` | `GCC 10.2.1`| `x86_64` | 3.6-3.12
`linux` | `centos7` | `GCC 10.2.1`| `aarch64` | 3.6-3.12
`linux` | `centos7` | `GCC 10.2.1`| `ppc64le` | 3.6-3.12
`darwin`| `MacOS 10.13` or later | `Apple Clang 14.0.0` | `x86_64` | 3.6-3.12
`darwin`| `MacOS 10.13` or later | `Apple Clang 14.0.3` | `aarch64` | 3.6-3.12

## Github Actions Workflows

Expand Down
22 changes: 13 additions & 9 deletions clingo/Dockerfile.manylinux2014
Original file line number Diff line number Diff line change
@@ -1,33 +1,37 @@
FROM ghcr.io/spack/manylinux2014:v2022-10-25
FROM ghcr.io/spack/manylinux2014:v2023-10-03

RUN cd /opt/_internal && tar xvf static-libs-for-embedding-only.tar.xz && cd -

RUN adduser -m spack

# Switch to a non-root user
USER spack
ENV SPACK_CMD="/opt/python/cp39-cp39/bin/python /home/spack/spack/bin/spack"
ENV SPACK_CMD="/opt/python/cp311-cp311/bin/python /home/spack/spack/bin/spack"
WORKDIR /home/spack

# Clone the repo and install Spack
RUN git clone https://www.github.com/spack/spack.git

# Run a script to build all the versions of clingo we could
COPY --chown=spack:spack clingo/scripts/bootstrap_clingo_manylinux2014.sh /home/spack/bootstrap_clingo2014.sh
COPY --chown=spack:spack clingo/scripts/install_clingo.py /home/spack/install_clingo.py
COPY --chown=spack:spack clingo/scripts/create_binary_mirror.sh /home/spack/create_binary_mirror.sh
RUN git clone https://www.github.com/spack/spack.git && \
cd spack && \
git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f && \
cd ..

# Set externals, locate compilers
RUN ${SPACK_CMD} external find --not-buildable bison
RUN ${SPACK_CMD} external find -j 1 --not-buildable bison cmake
RUN ${SPACK_CMD} compiler find
RUN ${SPACK_CMD} config add "config:concretizer:original"
RUN ${SPACK_CMD} config add "config:install_tree:padded_length:256"

# Run a script to build all the versions of clingo we could
COPY --chown=spack:spack clingo/scripts/bootstrap_clingo_manylinux2014.sh /home/spack/bootstrap_clingo2014.sh
COPY --chown=spack:spack clingo/scripts/install_clingo.py /home/spack/install_clingo.py

RUN ${HOME}/bootstrap_clingo2014.sh 36
RUN ${HOME}/bootstrap_clingo2014.sh 37
RUN ${HOME}/bootstrap_clingo2014.sh 38
RUN ${HOME}/bootstrap_clingo2014.sh 39
RUN ${HOME}/bootstrap_clingo2014.sh 310
RUN ${HOME}/bootstrap_clingo2014.sh 311
RUN ${HOME}/bootstrap_clingo2014.sh 312

COPY --chown=spack:spack clingo/scripts/create_binary_mirror.sh /home/spack/create_binary_mirror.sh
RUN ${HOME}/create_binary_mirror.sh
11 changes: 7 additions & 4 deletions clingo/scripts/clingo_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@
('centos7', 'ppc64le'): {
'spec': 'clingo-bootstrap%gcc platform=linux target=ppc64le',
},
('bigsur', 'x86_64'): {
('monterey', 'x86_64'): {
'spec': 'clingo-bootstrap%apple-clang platform=darwin target=x86_64',
},
('monterey', 'aarch64'): {
('ventura', 'aarch64'): {
'spec': 'clingo-bootstrap%apple-clang platform=darwin target=aarch64',
}
}
Expand Down Expand Up @@ -70,9 +70,12 @@ def tarball_hash(path):
spec_yaml_files = glob.glob(glob_expr)

mirror_info = []
for spec_yaml in spec_yaml_files:
for spec_json in spec_yaml_files:
if "clingo" not in spec_json:
continue

# Get the raw data from spec.json
with open(spec_yaml) as f:
with open(spec_json) as f:
spec_yaml_data = json.load(f)['spec']['nodes']

# Cycle through the specs in raw data. We are only interested
Expand Down
3 changes: 1 addition & 2 deletions clingo/scripts/create_binary_mirror.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,4 @@

specs=$(${SPACK_CMD} find --format="/{hash}" clingo-bootstrap)
mkdir -p /home/spack/binary-mirror
${SPACK_CMD} buildcache create -d /home/spack/binary-mirror -a -u -f ${specs}

${SPACK_CMD} buildcache push --unsigned --force /home/spack/binary-mirror ${specs}
19 changes: 13 additions & 6 deletions clingo/scripts/install_clingo.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,21 @@
import sys

import archspec.cpu

import spack.bootstrap
import spack.bootstrap.config
import spack.main


if sys.platform == "linux":
CLINGO_BASE_SPEC = "clingo-bootstrap@spack +static_libstdcpp +optimized +ipo ~docs"
else:
CLINGO_BASE_SPEC = "clingo-bootstrap@spack +optimized +ipo ~docs"

install = spack.main.SpackCommand('install')

with spack.bootstrap.spack_python_interpreter():

with spack.bootstrap.config.spack_python_interpreter():
msg = 'Installing clingo-bootstrap with Python: {0}'
print(msg.format(spack.bootstrap.spec_for_current_python()))
clingo_str = 'clingo-bootstrap@spack +static_libstdcpp target={0}'.format(
str(archspec.cpu.host().family)
)
print(msg.format(spack.bootstrap.config.spec_for_current_python()))
clingo_str = f'{CLINGO_BASE_SPEC} target={str(archspec.cpu.host().family)}'
install(clingo_str)
13 changes: 8 additions & 5 deletions gnupg/Dockerfile.manylinux2014
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
FROM ghcr.io/spack/manylinux2014:v2022-10-25
FROM ghcr.io/spack/manylinux2014:v2023-10-03

RUN adduser -m spack

# Switch to a non-root user
USER spack
ENV SPACK_CMD="/opt/python/cp39-cp39/bin/python /home/spack/spack/bin/spack"
ENV SPACK_CMD="/opt/python/cp311-cp311/bin/python /home/spack/spack/bin/spack"
WORKDIR /home/spack

# Clone the repo and install Spack
RUN git clone https://www.github.com/spack/spack.git
RUN git clone https://www.github.com/spack/spack.git && \
cd spack && \
git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f && \
cd ..

# Set externals, locate compilers
RUN ${SPACK_CMD} external find --not-buildable gawk perl
RUN ${SPACK_CMD} compiler find

RUN ${SPACK_CMD} python -c "import archspec.cpu;print(archspec.cpu.host().family)" > target.txt
RUN ${SPACK_CMD} -c "config:install_tree:padded_length:256" install gnupg target=$(cat target.txt)
RUN ${SPACK_CMD} -c "config:install_tree:padded_length:256" install gnupg "target=$(cat target.txt)"

RUN mkdir -p /home/spack/binary-mirror && \
${SPACK_CMD} -c "config:install_tree:padded_length:256" buildcache create -d /home/spack/binary-mirror -a -u -f gnupg target=$(cat target.txt)
${SPACK_CMD} -c "config:install_tree:padded_length:256" buildcache push --unsigned --force /home/spack/binary-mirror gnupg "target=$(cat target.txt)"
12 changes: 6 additions & 6 deletions gnupg/patches/gnupg_macos.patch
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
diff --git a/var/spack/repos/builtin/packages/gnupg/package.py b/var/spack/repos/builtin/packages/gnupg/package.py
index 6d9b89b72a..d80d042c88 100644
index 6e512a105e..5d4a96beaf 100644
--- a/var/spack/repos/builtin/packages/gnupg/package.py
+++ b/var/spack/repos/builtin/packages/gnupg/package.py
@@ -142,6 +142,7 @@ def configure_args(self):
"--with-zlib=" + self.spec["zlib"].prefix,
@@ -146,6 +146,7 @@ def configure_args(self):
"--disable-regex",
"--with-zlib=" + self.spec["zlib-api"].prefix,
"--without-tar",
"--without-libiconv-prefix",
+ "--without-libintl-prefix",
"--without-readline",
]

diff --git a/var/spack/repos/builtin/packages/libgpg-error/package.py b/var/spack/repos/builtin/packages/libgpg-error/package.py
index 8b6924492f..42c9276516 100644
index e2eeb228b4..23282a2883 100644
--- a/var/spack/repos/builtin/packages/libgpg-error/package.py
+++ b/var/spack/repos/builtin/packages/libgpg-error/package.py
@@ -35,6 +35,8 @@ def configure_args(self):
@@ -36,6 +36,8 @@ def configure_args(self):
args = [
"--enable-static",
"--enable-shared",
Expand Down
Loading

0 comments on commit 16caa84

Please sign in to comment.