Skip to content

Update manylinux build scripts #10467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

Update manylinux build scripts #10467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
36f7b44
update
snnn Jan 28, 2022
6d5c27f
update
snnn Jan 28, 2022
7cd1597
Merge remote-tracking branch 'origin/master' into snnn/manylinux_update
snnn Jan 28, 2022
3e8044a
Merge remote-tracking branch 'origin/master' into snnn/manylinux_update
snnn Feb 3, 2022
3f8c5d7
update
snnn Feb 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions cgmanifests/cgmanifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "56195b3a67d988fa9c6eafdc459d92f0909f4782",
"commitHash": "72ab18b4f3d83469c4e497cdb5a3877044808930",
"repositoryUrl": "https://github.com/pypa/manylinux"
},
"comments": "For building our CI build docker image"
Expand All @@ -448,30 +448,30 @@
"component": {
"type": "git",
"git": {
"commitHash": "1e5d33e9b9b8631b36f061103a30208b206fd03a",
"commitHash": "f2f3f537829ab0ef6948be5ee7f46b8ce8213ff2",
"repositoryUrl": "https://github.com/python/cpython"
},
"comments": "Python 3.9.1"
"comments": "Python 3.9.10"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "6503f05dd59e26a9986bdea097b3da9b3546f45b",
"commitHash": "07119dd38c9a6e5da84ca8a0a46acdf8a3e60ecf",
"repositoryUrl": "https://github.com/python/cpython"
},
"comments": "Python 3.8.7"
"comments": "Python 3.8.12"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "13c94747c74437e594b7fc242ff7da668e81887c",
"commitHash": "1f97973f630fda109039b2a8c8024a70eb92932f",
"repositoryUrl": "https://github.com/python/cpython"
},
"comments": "Python 3.7.9"
"comments": "Python 3.7.12"
}
},
{
Expand Down
27 changes: 8 additions & 19 deletions cgmanifests/generated/cgmanifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,24 +34,13 @@
"comments": "manylinux dependency"
}
},
{
"Component": {
"Type": "other",
"other": {
"Name": "patchelf",
"Version": "0.13",
"DownloadUrl": "https://github.com/NixOS/patchelf/archive/0.13.tar.gz"
},
"comments": "manylinux dependency"
}
},
{
"Component": {
"Type": "other",
"other": {
"Name": "libxcrypt",
"Version": "4.4.26",
"DownloadUrl": "https://github.com/besser82/libxcrypt/archive/v4.4.26.tar.gz"
"Version": "4.4.27",
"DownloadUrl": "https://github.com/besser82/libxcrypt/archive/v4.4.27.tar.gz"
},
"comments": "manylinux dependency"
}
Expand All @@ -61,8 +50,8 @@
"Type": "other",
"other": {
"Name": "git",
"Version": "2.33.1",
"DownloadUrl": "https://www.kernel.org/pub/software/scm/git/git-2.33.1.tar.gz"
"Version": "2.34.1",
"DownloadUrl": "https://www.kernel.org/pub/software/scm/git/git-2.34.1.tar.gz"
},
"comments": "manylinux dependency"
}
Expand All @@ -83,8 +72,8 @@
"Type": "other",
"other": {
"Name": "sqlite_autoconf",
"Version": "3360000",
"DownloadUrl": "https://www.sqlite.org/2021/sqlite-autoconf-3360000.tar.gz"
"Version": "3370200",
"DownloadUrl": "https://www.sqlite.org/2022/sqlite-autoconf-3370200.tar.gz"
},
"comments": "manylinux dependency"
}
Expand All @@ -94,8 +83,8 @@
"Type": "other",
"other": {
"Name": "openssl",
"Version": "1.1.1l",
"DownloadUrl": "https://www.openssl.org/source/openssl-1.1.1l.tar.gz"
"Version": "1.1.1m",
"DownloadUrl": "https://www.openssl.org/source/openssl-1.1.1m.tar.gz"
},
"comments": "manylinux dependency"
}
Expand Down
36 changes: 16 additions & 20 deletions tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,9 @@ RUN export LIBTOOL_ROOT=libtool-2.4.6 && \
export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \
manylinux-entrypoint /build_scripts/install-libtool.sh

COPY build_scripts/install-patchelf.sh /build_scripts/
RUN export PATCHELF_VERSION=0.13 && \
export PATCHELF_HASH=60c6aeadb673de9cc1838b630c81f61e31c501de324ef7f1e8094a2431197d09 && \
export PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive && \
manylinux-entrypoint /build_scripts/install-patchelf.sh

COPY build_scripts/install-libxcrypt.sh /build_scripts/
RUN export LIBXCRYPT_VERSION=4.4.26 && \
export LIBXCRYPT_HASH=e8a544dd19171c1e6191a6044c96cc31496d781ba08b5a00f53310d001d58114 && \
RUN export LIBXCRYPT_VERSION=4.4.27 && \
export LIBXCRYPT_HASH=e4ff47b7fa5370fa08e27258f8a0de5cb7e9b09c297ec7ab56a8a82e229a1c84 && \
export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \
export PERL_ROOT=perl-5.34.0 && \
export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \
Expand All @@ -87,8 +81,8 @@ RUN manylinux-entrypoint /build_scripts/install-build-packages.sh

FROM build_base AS build_git
COPY build_scripts/build-git.sh /build_scripts/
RUN export GIT_ROOT=git-2.33.1 && \
export GIT_HASH=02047f8dc8934d57ff5e02aadd8a2fe8e0bcf94a7158da375e48086cc46fce1d && \
RUN export GIT_ROOT=git-2.34.1 && \
export GIT_HASH=fc4eb5ecb9299db91cdd156c06cdeb41833f53adc5631ddf8c0cb13eaa2911c1 && \
export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \
manylinux-entrypoint /build_scripts/build-git.sh

Expand All @@ -106,14 +100,14 @@ RUN export SWIG_ROOT=swig-4.0.2 && \

FROM build_base AS build_cpython
COPY build_scripts/build-sqlite3.sh /build_scripts/
RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3360000 && \
export SQLITE_AUTOCONF_HASH=bd90c3eb96bee996206b83be7065c9ce19aef38c3f4fb53073ada0d0b69bbce3 && \
export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2021 && \
RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3370200 && \
export SQLITE_AUTOCONF_HASH=4089a8d9b467537b3f246f217b84cd76e00b1d1a971fe5aca1e30e230e46b2d8 && \
export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \
manylinux-entrypoint /build_scripts/build-sqlite3.sh

COPY build_scripts/build-openssl.sh /build_scripts/
RUN export OPENSSL_ROOT=openssl-1.1.1l && \
export OPENSSL_HASH=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 && \
RUN export OPENSSL_ROOT=openssl-1.1.1m && \
export OPENSSL_HASH=f89199be8b23ca45fc7cb9f1d8d3ee67312318286ad030f5316aca6462db6c96 && \
export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \
manylinux-entrypoint /build_scripts/build-openssl.sh

Expand All @@ -137,25 +131,27 @@ RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.12

FROM build_cpython AS build_cpython39
COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.8
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.10


FROM build_cpython AS build_cpython310
COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.0
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.2


FROM build_cpython AS all_python
COPY build_scripts/install-pypy.sh /build_scripts/install-pypy.sh
COPY build_scripts/pypy.sha256 /build_scripts/pypy.sha256
COPY build_scripts/install-pypy.sh \
build_scripts/pypy.sha256 \
build_scripts/finalize-python.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.7
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.7
COPY --from=build_cpython36 /opt/_internal /opt/_internal/
COPY --from=build_cpython37 /opt/_internal /opt/_internal/
COPY --from=build_cpython38 /opt/_internal /opt/_internal/
COPY --from=build_cpython39 /opt/_internal /opt/_internal/
COPY --from=build_cpython310 /opt/_internal /opt/_internal/
RUN hardlink -cv /opt/_internal
RUN manylinux-entrypoint /build_scripts/finalize-python.sh


FROM runtime_base
Expand Down
36 changes: 16 additions & 20 deletions tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_1_tensorrt7_2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,9 @@ RUN export LIBTOOL_ROOT=libtool-2.4.6 && \
export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \
manylinux-entrypoint /build_scripts/install-libtool.sh

COPY build_scripts/install-patchelf.sh /build_scripts/
RUN export PATCHELF_VERSION=0.13 && \
export PATCHELF_HASH=60c6aeadb673de9cc1838b630c81f61e31c501de324ef7f1e8094a2431197d09 && \
export PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive && \
manylinux-entrypoint /build_scripts/install-patchelf.sh

COPY build_scripts/install-libxcrypt.sh /build_scripts/
RUN export LIBXCRYPT_VERSION=4.4.26 && \
export LIBXCRYPT_HASH=e8a544dd19171c1e6191a6044c96cc31496d781ba08b5a00f53310d001d58114 && \
RUN export LIBXCRYPT_VERSION=4.4.27 && \
export LIBXCRYPT_HASH=e4ff47b7fa5370fa08e27258f8a0de5cb7e9b09c297ec7ab56a8a82e229a1c84 && \
export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \
export PERL_ROOT=perl-5.34.0 && \
export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \
Expand All @@ -87,8 +81,8 @@ RUN manylinux-entrypoint /build_scripts/install-build-packages.sh

FROM build_base AS build_git
COPY build_scripts/build-git.sh /build_scripts/
RUN export GIT_ROOT=git-2.33.1 && \
export GIT_HASH=02047f8dc8934d57ff5e02aadd8a2fe8e0bcf94a7158da375e48086cc46fce1d && \
RUN export GIT_ROOT=git-2.34.1 && \
export GIT_HASH=fc4eb5ecb9299db91cdd156c06cdeb41833f53adc5631ddf8c0cb13eaa2911c1 && \
export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \
manylinux-entrypoint /build_scripts/build-git.sh

Expand All @@ -106,14 +100,14 @@ RUN export SWIG_ROOT=swig-4.0.2 && \

FROM build_base AS build_cpython
COPY build_scripts/build-sqlite3.sh /build_scripts/
RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3360000 && \
export SQLITE_AUTOCONF_HASH=bd90c3eb96bee996206b83be7065c9ce19aef38c3f4fb53073ada0d0b69bbce3 && \
export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2021 && \
RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3370200 && \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we put these variable definitions into a common place so that we don't need to update different files when the version changes?

e.g., RUN source export_sqlite_vars.sh && manylinux-entrypoint /build_scripts/build-sqlite3.sh

export_sqlite_vars.sh:

export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3370200
export SQLITE_AUTOCONF_HASH=4089a8d9b467537b3f246f217b84cd76e00b1d1a971fe5aca1e30e230e46b2d8 && \
export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022

we'd also need to update the cgmanifest file generation

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just tried it. Not very good. Currently I just copy and paste the code from manylinux repo to ours. If there are too many Dockerfiles to modify, I can make a script. I already put makers in the docker files.

#Build manylinux2014 docker image begin
...
#Build manylinux2014 docker image end

If I extract the export commands to a separated file, I think it will need more human work to maintain.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can docker file include another docker file?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No. moby/moby#735

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you elaborate on how it's more work to maintain?
I counted 11 instances of SQLITE_AUTOCONF_ROOT in the changes (different Dockerfiles). Changing it in one place seems less error-prone and easier to review.
Updating them with a script is a good idea too.

export SQLITE_AUTOCONF_HASH=4089a8d9b467537b3f246f217b84cd76e00b1d1a971fe5aca1e30e230e46b2d8 && \
export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \
manylinux-entrypoint /build_scripts/build-sqlite3.sh

COPY build_scripts/build-openssl.sh /build_scripts/
RUN export OPENSSL_ROOT=openssl-1.1.1l && \
export OPENSSL_HASH=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 && \
RUN export OPENSSL_ROOT=openssl-1.1.1m && \
export OPENSSL_HASH=f89199be8b23ca45fc7cb9f1d8d3ee67312318286ad030f5316aca6462db6c96 && \
export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \
manylinux-entrypoint /build_scripts/build-openssl.sh

Expand All @@ -137,25 +131,27 @@ RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.12

FROM build_cpython AS build_cpython39
COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.8
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.10


FROM build_cpython AS build_cpython310
COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.0
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.2


FROM build_cpython AS all_python
COPY build_scripts/install-pypy.sh /build_scripts/install-pypy.sh
COPY build_scripts/pypy.sha256 /build_scripts/pypy.sha256
COPY build_scripts/install-pypy.sh \
build_scripts/pypy.sha256 \
build_scripts/finalize-python.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.7
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.7
COPY --from=build_cpython36 /opt/_internal /opt/_internal/
COPY --from=build_cpython37 /opt/_internal /opt/_internal/
COPY --from=build_cpython38 /opt/_internal /opt/_internal/
COPY --from=build_cpython39 /opt/_internal /opt/_internal/
COPY --from=build_cpython310 /opt/_internal /opt/_internal/
RUN hardlink -cv /opt/_internal
RUN manylinux-entrypoint /build_scripts/finalize-python.sh


FROM runtime_base
Expand Down
36 changes: 16 additions & 20 deletions tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_4_tensorrt8_0
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,9 @@ RUN export LIBTOOL_ROOT=libtool-2.4.6 && \
export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \
manylinux-entrypoint /build_scripts/install-libtool.sh

COPY build_scripts/install-patchelf.sh /build_scripts/
RUN export PATCHELF_VERSION=0.13 && \
export PATCHELF_HASH=60c6aeadb673de9cc1838b630c81f61e31c501de324ef7f1e8094a2431197d09 && \
export PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive && \
manylinux-entrypoint /build_scripts/install-patchelf.sh

COPY build_scripts/install-libxcrypt.sh /build_scripts/
RUN export LIBXCRYPT_VERSION=4.4.26 && \
export LIBXCRYPT_HASH=e8a544dd19171c1e6191a6044c96cc31496d781ba08b5a00f53310d001d58114 && \
RUN export LIBXCRYPT_VERSION=4.4.27 && \
export LIBXCRYPT_HASH=e4ff47b7fa5370fa08e27258f8a0de5cb7e9b09c297ec7ab56a8a82e229a1c84 && \
export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \
export PERL_ROOT=perl-5.34.0 && \
export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \
Expand All @@ -87,8 +81,8 @@ RUN manylinux-entrypoint /build_scripts/install-build-packages.sh

FROM build_base AS build_git
COPY build_scripts/build-git.sh /build_scripts/
RUN export GIT_ROOT=git-2.33.1 && \
export GIT_HASH=02047f8dc8934d57ff5e02aadd8a2fe8e0bcf94a7158da375e48086cc46fce1d && \
RUN export GIT_ROOT=git-2.34.1 && \
export GIT_HASH=fc4eb5ecb9299db91cdd156c06cdeb41833f53adc5631ddf8c0cb13eaa2911c1 && \
export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \
manylinux-entrypoint /build_scripts/build-git.sh

Expand All @@ -106,14 +100,14 @@ RUN export SWIG_ROOT=swig-4.0.2 && \

FROM build_base AS build_cpython
COPY build_scripts/build-sqlite3.sh /build_scripts/
RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3360000 && \
export SQLITE_AUTOCONF_HASH=bd90c3eb96bee996206b83be7065c9ce19aef38c3f4fb53073ada0d0b69bbce3 && \
export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2021 && \
RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3370200 && \
export SQLITE_AUTOCONF_HASH=4089a8d9b467537b3f246f217b84cd76e00b1d1a971fe5aca1e30e230e46b2d8 && \
export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \
manylinux-entrypoint /build_scripts/build-sqlite3.sh

COPY build_scripts/build-openssl.sh /build_scripts/
RUN export OPENSSL_ROOT=openssl-1.1.1l && \
export OPENSSL_HASH=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 && \
RUN export OPENSSL_ROOT=openssl-1.1.1m && \
export OPENSSL_HASH=f89199be8b23ca45fc7cb9f1d8d3ee67312318286ad030f5316aca6462db6c96 && \
export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \
manylinux-entrypoint /build_scripts/build-openssl.sh

Expand All @@ -137,25 +131,27 @@ RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.12

FROM build_cpython AS build_cpython39
COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.8
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.10


FROM build_cpython AS build_cpython310
COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.0
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.2


FROM build_cpython AS all_python
COPY build_scripts/install-pypy.sh /build_scripts/install-pypy.sh
COPY build_scripts/pypy.sha256 /build_scripts/pypy.sha256
COPY build_scripts/install-pypy.sh \
build_scripts/pypy.sha256 \
build_scripts/finalize-python.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.7
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.7
COPY --from=build_cpython36 /opt/_internal /opt/_internal/
COPY --from=build_cpython37 /opt/_internal /opt/_internal/
COPY --from=build_cpython38 /opt/_internal /opt/_internal/
COPY --from=build_cpython39 /opt/_internal /opt/_internal/
COPY --from=build_cpython310 /opt/_internal /opt/_internal/
RUN hardlink -cv /opt/_internal
RUN manylinux-entrypoint /build_scripts/finalize-python.sh


FROM runtime_base
Expand Down
Loading