From 831a1333bbb21171aecde54031058e5baba9d69a Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 19 Dec 2023 06:40:17 +0000 Subject: [PATCH 1/2] Update to hpc python, now using python 3.11 and fixed security holes with older python packages --- python/hpc-python-hdf5mpi/Dockerfile | 77 +------ .../buildhpcpythonhdf5mpi.dockerfile | 44 ++++ .../requirements-24Mar2022.txt | 200 ---------------- python/hpc-python-hdf5mpi/requirements.in | 18 -- python/hpc-python/Dockerfile | 50 +--- python/hpc-python/buildhpcpython.dockerfile | 44 ++++ python/hpc-python/requirements-12-2023.txt | 218 ++++++++++++++++++ python/hpc-python/requirements-24Mar2022.txt | 206 ----------------- 8 files changed, 308 insertions(+), 549 deletions(-) mode change 100644 => 120000 python/hpc-python-hdf5mpi/Dockerfile create mode 100644 python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile delete mode 100644 python/hpc-python-hdf5mpi/requirements-24Mar2022.txt delete mode 100644 python/hpc-python-hdf5mpi/requirements.in mode change 100644 => 120000 python/hpc-python/Dockerfile create mode 100644 python/hpc-python/buildhpcpython.dockerfile create mode 100644 python/hpc-python/requirements-12-2023.txt delete mode 100644 python/hpc-python/requirements-24Mar2022.txt diff --git a/python/hpc-python-hdf5mpi/Dockerfile b/python/hpc-python-hdf5mpi/Dockerfile deleted file mode 100644 index 212c6eb..0000000 --- a/python/hpc-python-hdf5mpi/Dockerfile +++ /dev/null @@ -1,76 +0,0 @@ -ARG PY_VERSION="3.9" -FROM python:${PY_VERSION}-slim - -LABEL maintainer="marco.delapierre@pawsey.org.au" - - -# Install package dependencies -RUN apt-get update -qq \ - && apt-get -y --no-install-recommends install \ - build-essential \ - ca-certificates \ - gdb \ - gfortran \ - wget \ - && apt-get clean all \ - && rm -r /var/lib/apt/lists/* - - -# Build MPICH for mpi4py - -ARG MPICH_VERSION="3.4.3" -#ARG MPICH_CONFIGURE_OPTIONS="--enable-fast=all,O3 --prefix=/usr FFLAGS=-fallow-argument-mismatch" -ARG MPICH_CONFIGURE_OPTIONS="--enable-fast=all,O3 --prefix=/usr FFLAGS=-fallow-argument-mismatch --with-device=ch4:ofi" -ARG MPICH_MAKE_OPTIONS="-j4" - -RUN mkdir -p /tmp/mpich-build \ - && cd /tmp/mpich-build \ - && wget http://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz \ - && tar xvzf mpich-${MPICH_VERSION}.tar.gz \ - && cd mpich-${MPICH_VERSION} \ - && ./configure ${MPICH_CONFIGURE_OPTIONS} \ - && make ${MPICH_MAKE_OPTIONS} \ - && make install \ - && ldconfig \ - && cp -p /tmp/mpich-build/mpich-${MPICH_VERSION}/examples/cpi /usr/bin/ \ - && cd / \ - && rm -rf /tmp/mpich-build - - -# Install Python packages (apart from h5py) - -ARG DATE_FILE="24Mar2022" - -ADD requirements.in requirements-${DATE_FILE}.txt / -RUN pip3 --no-cache-dir install -r /requirements-${DATE_FILE}.txt --no-deps - - -# Install HDF5-parallel - -ARG HDF5_VERSION="1.12.1" -ARG HDF5_CONFIGURE_OPTIONS="--prefix=/usr/local --enable-parallel CC=mpicc" -ARG HDF5_MAKE_OPTIONS="-j4" - -RUN mkdir -p /tmp/hdf5-build \ - && cd /tmp/hdf5-build \ - && HDF5_VER_MM="${HDF5_VERSION%.*}" \ - && wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${HDF5_VER_MM}/hdf5-${HDF5_VERSION}/src/hdf5-${HDF5_VERSION}.tar.gz \ - && tar xzf hdf5-${HDF5_VERSION}.tar.gz \ - && cd hdf5-${HDF5_VERSION} \ - && ./configure ${HDF5_CONFIGURE_OPTIONS} \ - && make ${HDF5_MAKE_OPTIONS} \ - && make install \ - && ldconfig \ - && cd / \ - && rm -rf /tmp/hdf5-build - - -# Install h5py and h5netcdf - -RUN H5PY_VERSION=$( grep '#h5py' /requirements-${DATE_FILE}.txt ) && H5PY_VERSION=${H5PY_VERSION#*==} && H5PY_VERSION=${H5PY_VERSION%% *} \ - && H5NETCDF_VERSION=$( grep '#h5netcdf' /requirements-${DATE_FILE}.txt ) && H5NETCDF_VERSION=${H5NETCDF_VERSION#*==} && H5NETCDF_VERSION=${H5NETCDF_VERSION%% *} \ - && CC="mpicc" HDF5_MPI="ON" HDF5_DIR="/usr/local" pip3 --no-cache-dir install --no-deps --no-binary=h5py h5py==${H5PY_VERSION} h5netcdf==${H5NETCDF_VERSION} - - -# Final -CMD ["/bin/bash"] diff --git a/python/hpc-python-hdf5mpi/Dockerfile b/python/hpc-python-hdf5mpi/Dockerfile new file mode 120000 index 0000000..54642c7 --- /dev/null +++ b/python/hpc-python-hdf5mpi/Dockerfile @@ -0,0 +1 @@ +buildhpcpythonhdf5mpi.dockerfile \ No newline at end of file diff --git a/python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile b/python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile new file mode 100644 index 0000000..d20ef5e --- /dev/null +++ b/python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile @@ -0,0 +1,44 @@ +ARG HPCPYTHON_VERSION="3.11" +FROM quay.io/pawsey/hpc-python:${HPCPYTHON_VERSION} + +LABEL org.opencontainers.image.created="2023-12" +LABEL org.opencontainers.image.authors="Pascal Jahan Elahi " +LABEL org.opencontainers.image.documentation="https://github.com/PawseySC/pawsey-containers/" +LABEL org.opencontainers.image.source="https://github.com/PawseySC/pawsey-containers/python/hpcpython/buildhpcpython.dockerfile" +LABEL org.opencontainers.image.vendor="Pawsey Supercomputing Research Centre" +LABEL org.opencontainers.image.licenses="GNU GPL3.0" +LABEL org.opencontainers.image.title="Setonix compatible python container with parallel h5py" +LABEL org.opencontainers.image.description="Common base image providing python on Setonix built on the lustre aware mpich" +LABEL org.opencontainers.image.base.name="pawsey/hpcpython:3.11-hdf5-mpi" + +# add HDF5 +ARG HDF5_VERSION="1.14.3" +ARG HDF5_CONFIGURE_OPTIONS="--prefix=/usr/local --enable-parallel CC=mpicc" +ARG HDF5_MAKE_OPTIONS="-j4" +RUN mkdir -p /tmp/hdf5-build \ + && cd /tmp/hdf5-build \ + && HDF5_VER_MM="${HDF5_VERSION%.*}" \ + && wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${HDF5_VER_MM}/hdf5-${HDF5_VERSION}/src/hdf5-${HDF5_VERSION}.tar.gz \ + && tar xzf hdf5-${HDF5_VERSION}.tar.gz \ + && cd hdf5-${HDF5_VERSION} \ + && ./configure ${HDF5_CONFIGURE_OPTIONS} \ + && make ${HDF5_MAKE_OPTIONS} \ + && make install \ + && ldconfig \ + && cd / \ + && rm -rf /tmp/hdf5-build + + +# # Install h5py and h5netcdf +ARG DATE_FILE="12-2023" +RUN echo "Install h5py " \ + && H5PY_VERSION=$(grep 'h5py' /requirements-${DATE_FILE}.txt | sed 's/==/ /g' | sed 's/ # / /'g | awk '{print $2}') \ + && H5NETCDF_VERSION=$(grep 'h5netcdf' /requirements-${DATE_FILE}.txt | sed 's/==/ /g' | sed 's/ # / /'g | awk '{print $2}') \ + && CC="mpicc" CXX="mpic++" HDF5_MPI="ON" HDF5_DIR="/usr/local" pip3 --no-cache-dir install --no-deps --no-binary=h5py h5py==${H5PY_VERSION} h5netcdf==${H5NETCDF_VERSION} + +# and copy the recipe into the docker recipes directory +RUN mkdir -p /opt/docker-recipes/ +COPY buildhpcpythonhdf5mpi.dockerfile /opt/docker-recipes/ + +# Final +CMD ["/bin/bash"] diff --git a/python/hpc-python-hdf5mpi/requirements-24Mar2022.txt b/python/hpc-python-hdf5mpi/requirements-24Mar2022.txt deleted file mode 100644 index fcd3349..0000000 --- a/python/hpc-python-hdf5mpi/requirements-24Mar2022.txt +++ /dev/null @@ -1,200 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile --output-file=requirements-24Mar2022.txt requirements.in -# -astropy==5.0.2 - # via -r requirements.in -asttokens==2.0.5 - # via stack-data -attrs==21.4.0 - # via pytest -backcall==0.2.0 - # via ipython -bokeh==2.4.2 - # via dask -click==8.0.4 - # via distributed -cloudpickle==2.0.0 - # via - # dask - # distributed -cycler==0.11.0 - # via matplotlib -cython==0.29.28 - # via -r requirements.in -dask[complete]==2022.03.0 - # via - # -r requirements.in - # distributed -decorator==5.1.1 - # via ipython -distributed==2022.03.0 - # via dask -executing==0.8.3 - # via stack-data -fonttools==4.31.2 - # via matplotlib -fsspec==2022.2.0 - # via dask -#h5netcdf==0.15.0 - # via -r requirements.in -#h5py==3.6.0 - # via - # -r requirements.in - # h5netcdf -heapdict==1.0.1 - # via zict -iniconfig==1.1.1 - # via pytest -ipython==8.1.1 - # via -r requirements.in -jedi==0.18.1 - # via ipython -jinja2==3.0.3 - # via - # bokeh - # dask - # distributed -joblib==1.1.0 - # via scikit-learn -kiwisolver==1.4.0 - # via matplotlib -llvmlite==0.38.0 - # via numba -locket==0.2.1 - # via partd -markupsafe==2.1.1 - # via jinja2 -matplotlib==3.5.1 - # via -r requirements.in -matplotlib-inline==0.1.3 - # via ipython -mpi4py==3.1.3 - # via -r requirements.in -msgpack==1.0.3 - # via distributed -nose==1.3.7 - # via -r requirements.in -numba==0.55.1 - # via -r requirements.in -numpy==1.21.5 - # via - # -r requirements.in - # astropy - # bokeh - # dask - # h5py - # matplotlib - # numba - # pandas - # pyerfa - # scikit-learn - # scipy -packaging==21.3 - # via - # astropy - # bokeh - # dask - # distributed - # h5netcdf - # matplotlib - # pytest -pandas==1.4.1 - # via - # -r requirements.in - # dask -parso==0.8.3 - # via jedi -partd==1.2.0 - # via dask -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -pillow==9.0.1 - # via - # bokeh - # matplotlib -plotly==5.6.0 - # via -r requirements.in -pluggy==1.0.0 - # via pytest -prompt-toolkit==3.0.28 - # via ipython -psutil==5.9.0 - # via distributed -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via pytest -pyerfa==2.0.0.1 - # via astropy -pygments==2.11.2 - # via ipython -pyparsing==3.0.7 - # via - # matplotlib - # packaging -pytest==7.1.1 - # via -r requirements.in -python-dateutil==2.8.2 - # via - # matplotlib - # pandas -pytz==2022.1 - # via pandas -pyyaml==6.0 - # via - # astropy - # bokeh - # dask - # distributed -scikit-learn==1.0.2 - # via -r requirements.in -scipy==1.8.0 - # via - # -r requirements.in - # scikit-learn -six==1.16.0 - # via - # asttokens - # plotly - # python-dateutil -sortedcontainers==2.4.0 - # via distributed -stack-data==0.2.0 - # via ipython -tblib==1.7.0 - # via distributed -tenacity==8.0.1 - # via plotly -threadpoolctl==3.1.0 - # via scikit-learn -tomli==2.0.1 - # via pytest -toolz==0.11.2 - # via - # dask - # distributed - # partd -tornado==6.1 - # via - # bokeh - # distributed -traitlets==5.1.1 - # via - # ipython - # matplotlib-inline -typing-extensions==4.1.1 - # via bokeh -wcwidth==0.2.5 - # via prompt-toolkit -zict==2.1.0 - # via distributed - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/python/hpc-python-hdf5mpi/requirements.in b/python/hpc-python-hdf5mpi/requirements.in deleted file mode 100644 index 5fe7c5c..0000000 --- a/python/hpc-python-hdf5mpi/requirements.in +++ /dev/null @@ -1,18 +0,0 @@ -# Pure requirements (final packages, no version) -# Requires pip3 install pip-tools in the host building machine -astropy -cython -dask[complete] -h5netcdf -h5py -ipython -matplotlib -mpi4py -nose -numba -numpy -pandas -plotly -pytest -scikit-learn -scipy diff --git a/python/hpc-python/Dockerfile b/python/hpc-python/Dockerfile deleted file mode 100644 index 559d118..0000000 --- a/python/hpc-python/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -ARG PY_VERSION="3.9" -FROM python:${PY_VERSION}-slim - -LABEL maintainer="marco.delapierre@pawsey.org.au" - - -# Install package dependencies -RUN apt-get update -qq \ - && apt-get -y --no-install-recommends install \ - build-essential \ - ca-certificates \ - gdb \ - gfortran \ - wget \ - && apt-get clean all \ - && rm -r /var/lib/apt/lists/* - - -# Build MPICH for mpi4py - -ARG MPICH_VERSION="3.4.3" -#ARG MPICH_CONFIGURE_OPTIONS="--enable-fast=all,O3 --prefix=/usr FFLAGS=-fallow-argument-mismatch" -ARG MPICH_CONFIGURE_OPTIONS="--enable-fast=all,O3 --prefix=/usr FFLAGS=-fallow-argument-mismatch --with-device=ch4:ofi" -ARG MPICH_MAKE_OPTIONS="-j4" - -RUN mkdir -p /tmp/mpich-build \ - && cd /tmp/mpich-build \ - && wget http://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz \ - && tar xvzf mpich-${MPICH_VERSION}.tar.gz \ - && cd mpich-${MPICH_VERSION} \ - && ./configure ${MPICH_CONFIGURE_OPTIONS} \ - && make ${MPICH_MAKE_OPTIONS} \ - && make install \ - && ldconfig \ - && cp -p /tmp/mpich-build/mpich-${MPICH_VERSION}/examples/cpi /usr/bin/ \ - && cd / \ - && rm -rf /tmp/mpich-build - - -# Install Python packages - -ARG DATE_FILE="24Mar2022" - -ADD requirements.in requirements-${DATE_FILE}.txt / -RUN pip3 --no-cache-dir install -r /requirements-${DATE_FILE}.txt --no-deps - - -# Final -CMD ["/bin/bash"] diff --git a/python/hpc-python/Dockerfile b/python/hpc-python/Dockerfile new file mode 120000 index 0000000..051a301 --- /dev/null +++ b/python/hpc-python/Dockerfile @@ -0,0 +1 @@ +buildhpcpython.dockerfile \ No newline at end of file diff --git a/python/hpc-python/buildhpcpython.dockerfile b/python/hpc-python/buildhpcpython.dockerfile new file mode 100644 index 0000000..08f032e --- /dev/null +++ b/python/hpc-python/buildhpcpython.dockerfile @@ -0,0 +1,44 @@ +ARG IMAGE_VERSION="3.4.3_ubuntu20.04_lustrerelease" +FROM quay.io/pawsey/mpich-lustre-base:${IMAGE_VERSION} +ARG PY_VERSION=3.11 + +LABEL org.opencontainers.image.created="2023-12" +LABEL org.opencontainers.image.authors="Pascal Jahan Elahi " +LABEL org.opencontainers.image.documentation="https://github.com/PawseySC/pawsey-containers/" +LABEL org.opencontainers.image.source="https://github.com/PawseySC/pawsey-containers/python/hpcpython/buildhpcpython.dockerfile" +LABEL org.opencontainers.image.vendor="Pawsey Supercomputing Research Centre" +LABEL org.opencontainers.image.licenses="GNU GPL3.0" +LABEL org.opencontainers.image.title="Setonix compatible python container" +LABEL org.opencontainers.image.description="Common base image providing python on Setonix built on the lustre aware mpich" +LABEL org.opencontainers.image.base.name="pawsey/hpcpython:${PY_VERSION}" + +# add python +ENV DEBIAN_FRONTEND="noninteractive" +RUN echo "Updating python to ${PY_VERSION}" \ + && apt-get update -qq \ + && apt install -y --no-install-recommends software-properties-common \ + && add-apt-repository ppa:deadsnakes/ppa \ + && apt-get update -qq \ + && apt-get -y --no-install-recommends install \ + python${PY_VERSION}-dev python${PY_VERSION}-distutils python${PY_VERSION}-full \ + && apt-get clean all \ + && rm -r /var/lib/apt/lists/* \ + && update-alternatives --install /usr/bin/python python /usr/bin/python${PY_VERSION} 1 \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PY_VERSION} 1 \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | sudo python3 \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python3 \ + && python -m pip install pip --upgrade \ + && pip install pip-tools \ + && echo "Done " + +# Install Python packages +ARG DATE_FILE="12-2023" +ADD requirements.in requirements-${DATE_FILE}.txt / +RUN pip3 --no-cache-dir install -r /requirements-${DATE_FILE}.txt --no-deps + +# and copy the recipe into the docker recipes directory +RUN mkdir -p /opt/docker-recipes/ +COPY buildhpcpython.dockerfile /opt/docker-recipes/ + +# Final +CMD ["/bin/bash"] diff --git a/python/hpc-python/requirements-12-2023.txt b/python/hpc-python/requirements-12-2023.txt new file mode 100644 index 0000000..2dd3795 --- /dev/null +++ b/python/hpc-python/requirements-12-2023.txt @@ -0,0 +1,218 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --output-file=/opt/tmp/requirements-12-2023.txt /opt/tmp/requirements.in +# +astropy==6.0.0 + # via -r /opt/tmp/requirements.in +astropy-iers-data==0.2023.12.18.0.30.18 + # via astropy +asttokens==2.4.1 + # via stack-data +bokeh==3.3.2 + # via dask +certifi==2023.11.17 + # via netcdf4 +cftime==1.6.3 + # via netcdf4 +click==8.1.7 + # via + # dask + # distributed +cloudpickle==3.0.0 + # via + # dask + # distributed +contourpy==1.2.0 + # via + # bokeh + # matplotlib +cycler==0.12.1 + # via matplotlib +cython==3.0.6 + # via -r /opt/tmp/requirements.in +dask[array,complete,dataframe,diagnostics,distributed]==2023.12.1 + # via + # -r /opt/tmp/requirements.in + # distributed +decorator==5.1.1 + # via ipython +distributed==2023.12.1 + # via dask +executing==2.0.1 + # via stack-data +fonttools==4.47.0 + # via matplotlib +fsspec==2023.12.2 + # via dask +h5netcdf==1.3.0 + # via -r /opt/tmp/requirements.in +h5py==3.10.0 + # via + # -r /opt/tmp/requirements.in + # h5netcdf +importlib-metadata==7.0.0 + # via dask +iniconfig==2.0.0 + # via pytest +ipython==8.18.1 + # via -r /opt/tmp/requirements.in +jedi==0.19.1 + # via ipython +jinja2==3.1.2 + # via + # bokeh + # dask + # distributed +joblib==1.3.2 + # via scikit-learn +kiwisolver==1.4.5 + # via matplotlib +llvmlite==0.41.1 + # via numba +locket==1.0.0 + # via + # distributed + # partd +lz4==4.3.2 + # via dask +markupsafe==2.1.3 + # via jinja2 +matplotlib==3.8.2 + # via -r /opt/tmp/requirements.in +matplotlib-inline==0.1.6 + # via ipython +mpi4py==3.1.5 + # via -r /opt/tmp/requirements.in +msgpack==1.0.7 + # via distributed +netcdf4==1.6.5 + # via -r /opt/tmp/requirements.in +nose==1.3.7 + # via -r /opt/tmp/requirements.in +numba==0.58.1 + # via -r /opt/tmp/requirements.in +numpy==1.26.2 + # via + # -r /opt/tmp/requirements.in + # astropy + # bokeh + # cftime + # contourpy + # dask + # h5py + # matplotlib + # netcdf4 + # numba + # pandas + # pyarrow + # pyerfa + # scikit-learn + # scipy +packaging==23.2 + # via + # astropy + # bokeh + # dask + # distributed + # h5netcdf + # matplotlib + # plotly + # pytest +pandas==2.1.4 + # via + # -r /opt/tmp/requirements.in + # bokeh + # dask +parso==0.8.3 + # via jedi +partd==1.4.1 + # via dask +pexpect==4.9.0 + # via ipython +pillow==10.1.0 + # via + # bokeh + # matplotlib +plotly==5.18.0 + # via -r /opt/tmp/requirements.in +pluggy==1.3.0 + # via pytest +prompt-toolkit==3.0.43 + # via ipython +psutil==5.9.7 + # via distributed +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +pyarrow==14.0.2 + # via dask +pyarrow-hotfix==0.6 + # via dask +pyerfa==2.0.1.1 + # via astropy +pygments==2.17.2 + # via ipython +pyparsing==3.1.1 + # via matplotlib +pytest==7.4.3 + # via -r /opt/tmp/requirements.in +python-dateutil==2.8.2 + # via + # matplotlib + # pandas +pytz==2023.3.post1 + # via pandas +pyyaml==6.0.1 + # via + # astropy + # bokeh + # dask + # distributed +scikit-learn==1.3.2 + # via -r /opt/tmp/requirements.in +scipy==1.11.4 + # via + # -r /opt/tmp/requirements.in + # scikit-learn +six==1.16.0 + # via + # asttokens + # python-dateutil +sortedcontainers==2.4.0 + # via distributed +stack-data==0.6.3 + # via ipython +tblib==3.0.0 + # via distributed +tenacity==8.2.3 + # via plotly +threadpoolctl==3.2.0 + # via scikit-learn +toolz==0.12.0 + # via + # dask + # distributed + # partd +tornado==6.4 + # via + # bokeh + # distributed +traitlets==5.14.0 + # via + # ipython + # matplotlib-inline +tzdata==2023.3 + # via pandas +urllib3==2.1.0 + # via distributed +wcwidth==0.2.12 + # via prompt-toolkit +xyzservices==2023.10.1 + # via bokeh +zict==3.0.0 + # via distributed +zipp==3.17.0 + # via importlib-metadata diff --git a/python/hpc-python/requirements-24Mar2022.txt b/python/hpc-python/requirements-24Mar2022.txt deleted file mode 100644 index 46f50c8..0000000 --- a/python/hpc-python/requirements-24Mar2022.txt +++ /dev/null @@ -1,206 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile --output-file=requirements-24Mar2022.txt requirements.in -# -astropy==5.0.2 - # via -r requirements.in -asttokens==2.0.5 - # via stack-data -attrs==21.4.0 - # via pytest -backcall==0.2.0 - # via ipython -bokeh==2.4.2 - # via dask -cftime==1.6.0 - # via netcdf4 -click==8.0.4 - # via distributed -cloudpickle==2.0.0 - # via - # dask - # distributed -cycler==0.11.0 - # via matplotlib -cython==0.29.28 - # via -r requirements.in -dask[complete]==2022.03.0 - # via - # -r requirements.in - # distributed -decorator==5.1.1 - # via ipython -distributed==2022.03.0 - # via dask -executing==0.8.3 - # via stack-data -fonttools==4.31.2 - # via matplotlib -fsspec==2022.2.0 - # via dask -h5netcdf==0.15.0 - # via -r requirements.in -h5py==3.6.0 - # via - # -r requirements.in - # h5netcdf -heapdict==1.0.1 - # via zict -iniconfig==1.1.1 - # via pytest -ipython==8.1.1 - # via -r requirements.in -jedi==0.18.1 - # via ipython -jinja2==3.0.3 - # via - # bokeh - # dask - # distributed -joblib==1.1.0 - # via scikit-learn -kiwisolver==1.4.0 - # via matplotlib -llvmlite==0.38.0 - # via numba -locket==0.2.1 - # via partd -markupsafe==2.1.1 - # via jinja2 -matplotlib==3.5.1 - # via -r requirements.in -matplotlib-inline==0.1.3 - # via ipython -mpi4py==3.1.3 - # via -r requirements.in -msgpack==1.0.3 - # via distributed -netcdf4==1.5.8 - # via -r requirements.in -nose==1.3.7 - # via -r requirements.in -numba==0.55.1 - # via -r requirements.in -numpy==1.21.5 - # via - # -r requirements.in - # astropy - # bokeh - # cftime - # dask - # h5py - # matplotlib - # netcdf4 - # numba - # pandas - # pyerfa - # scikit-learn - # scipy -packaging==21.3 - # via - # astropy - # bokeh - # dask - # distributed - # h5netcdf - # matplotlib - # pytest -pandas==1.4.1 - # via - # -r requirements.in - # dask -parso==0.8.3 - # via jedi -partd==1.2.0 - # via dask -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -pillow==9.0.1 - # via - # bokeh - # matplotlib -plotly==5.6.0 - # via -r requirements.in -pluggy==1.0.0 - # via pytest -prompt-toolkit==3.0.28 - # via ipython -psutil==5.9.0 - # via distributed -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via pytest -pyerfa==2.0.0.1 - # via astropy -pygments==2.11.2 - # via ipython -pyparsing==3.0.7 - # via - # matplotlib - # packaging -pytest==7.1.1 - # via -r requirements.in -python-dateutil==2.8.2 - # via - # matplotlib - # pandas -pytz==2022.1 - # via pandas -pyyaml==6.0 - # via - # astropy - # bokeh - # dask - # distributed -scikit-learn==1.0.2 - # via -r requirements.in -scipy==1.8.0 - # via - # -r requirements.in - # scikit-learn -six==1.16.0 - # via - # asttokens - # plotly - # python-dateutil -sortedcontainers==2.4.0 - # via distributed -stack-data==0.2.0 - # via ipython -tblib==1.7.0 - # via distributed -tenacity==8.0.1 - # via plotly -threadpoolctl==3.1.0 - # via scikit-learn -tomli==2.0.1 - # via pytest -toolz==0.11.2 - # via - # dask - # distributed - # partd -tornado==6.1 - # via - # bokeh - # distributed -traitlets==5.1.1 - # via - # ipython - # matplotlib-inline -typing-extensions==4.1.1 - # via bokeh -wcwidth==0.2.5 - # via prompt-toolkit -zict==2.1.0 - # via distributed - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From 0c59dcfd7dd704cfcfbf88633e023117f5c49fa7 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 3 Jan 2024 03:59:41 +0000 Subject: [PATCH 2/2] Updated python recipes - removed some older intel based python recipes to deprecated - updated the requirements files of some python recipes - updated recipes to be able to handle newer os base images --- python/cuda-hpc-python/Dockerfile | 1 + .../buildhpcpythoncuda.dockerfile | 82 +++++++++++++++++++ python/cuda-hpc-python_cuda10.2/Dockerfile | 67 --------------- python/{ => deprecated}/build_python.sh | 0 .../cuda-intel-hpc-python/Dockerfile | 0 .../intel-hpc-python-hdf5mpi/Dockerfile | 0 .../environment-24Mar2022.yaml | 0 .../requirements-24Mar2022.yaml | 0 .../intel-hpc-python-hdf5mpi/requirements.in | 0 python/deprecated/intel-hpc-python/Dockerfile | 1 + .../buildhpcpythonintel.dockerfile} | 9 +- .../environment-24Mar2022.yaml | 0 .../requirements-24Mar2022.yaml | 0 .../intel-hpc-python/requirements.in | 0 .../buildhpcpythonhdf5mpi.dockerfile | 2 +- python/hpc-python/buildhpcpython.dockerfile | 27 +++--- 16 files changed, 106 insertions(+), 83 deletions(-) create mode 120000 python/cuda-hpc-python/Dockerfile create mode 100644 python/cuda-hpc-python/buildhpcpythoncuda.dockerfile delete mode 100644 python/cuda-hpc-python_cuda10.2/Dockerfile rename python/{ => deprecated}/build_python.sh (100%) rename python/{ => deprecated}/cuda-intel-hpc-python/Dockerfile (100%) rename python/{ => deprecated}/intel-hpc-python-hdf5mpi/Dockerfile (100%) rename python/{ => deprecated}/intel-hpc-python-hdf5mpi/environment-24Mar2022.yaml (100%) rename python/{ => deprecated}/intel-hpc-python-hdf5mpi/requirements-24Mar2022.yaml (100%) rename python/{ => deprecated}/intel-hpc-python-hdf5mpi/requirements.in (100%) create mode 120000 python/deprecated/intel-hpc-python/Dockerfile rename python/{intel-hpc-python/Dockerfile => deprecated/intel-hpc-python/buildhpcpythonintel.dockerfile} (89%) rename python/{ => deprecated}/intel-hpc-python/environment-24Mar2022.yaml (100%) rename python/{ => deprecated}/intel-hpc-python/requirements-24Mar2022.yaml (100%) rename python/{ => deprecated}/intel-hpc-python/requirements.in (100%) diff --git a/python/cuda-hpc-python/Dockerfile b/python/cuda-hpc-python/Dockerfile new file mode 120000 index 0000000..504dccc --- /dev/null +++ b/python/cuda-hpc-python/Dockerfile @@ -0,0 +1 @@ +buildhpcpythoncuda.dockerfile \ No newline at end of file diff --git a/python/cuda-hpc-python/buildhpcpythoncuda.dockerfile b/python/cuda-hpc-python/buildhpcpythoncuda.dockerfile new file mode 100644 index 0000000..857a4d7 --- /dev/null +++ b/python/cuda-hpc-python/buildhpcpythoncuda.dockerfile @@ -0,0 +1,82 @@ +ARG PAWSEY_BASE="3.11" +FROM quay.io/pawsey/hpc-python:${PAWSEY_BASE} + +ARG CUDA_VERSION=12.3 +ARG CUDA_PKG_VERSION=12-3 + +RUN echo "Installing CUDA" \ + && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb \ + && dpkg -i cuda-keyring_1.1-1_all.deb \ + && apt-get update \ + && apt-get -y install cuda-toolkit-${CUDA_PKG_VERSION} \ + && apt-get install -y nvidia-kernel-open-545 \ + && apt-get install -y cuda-drivers-545 \ + && echo "Done" + +RUN echo "Installing NVHPC SDK" \ + && curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg \ + && echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list \ + && apt-get update -y \ + && apt-get install -y nvhpc-23-11 \ + && echo "Done" + +# Update to libmozjs which is at this point version 68. Upgrade to 102 +#RUN echo "Updating libmozjs" \ +# && sed -i "s/focal/jammy/g" /etc/apt/sources.list \ +# && apt-get update \ +# && apt-get install -y libmozjs-102-0 \ +# && apt-get remove -y libmozjs-68-0 \ +# && echo "Done" + +ENV CUDA_VERSION 12.3 +ENV CUDA_PKG_VERSION 12-3=$CUDA_VERSION-1 + +# # For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# cuda-cudart-${CUDA_PKG_VERSION} \ +# cuda-compat-${CUDA_PKG_VERSION} \ +# && ln -s cuda-${CUDA_VERSION}} /usr/local/cuda && \ +# rm -rf /var/lib/apt/lists/* + +# # Required for nvidia-docker v1 +# RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \ +# echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf + +# ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +# ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 + +# nvidia-container-runtime +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility +ENV NVIDIA_REQUIRE_CUDA "cuda>=12.3" +#brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411 brand=tesla,driver>=418,driver<419 brand=tesla,driver>=440,driver<441" + +# ENV NCCL_VERSION 2.10.3 + + +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# cuda-libraries-$CUDA_PKG_VERSION \ +# cuda-npp-$CUDA_PKG_VERSION \ +# cuda-nvtx-$CUDA_PKG_VERSION \ +# cuda-cusparse-$CUDA_PKG_VERSION \ +# libcublas10=10.2.2.89-1 \ +# libnccl2=$NCCL_VERSION-1+cuda10.2 \ +# && apt-mark hold libcublas10 libnccl2 \ +# && rm -rf /var/lib/apt/lists/* + +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# cuda-nvml-dev-$CUDA_PKG_VERSION \ +# cuda-command-line-tools-$CUDA_PKG_VERSION \ +# cuda-nvprof-$CUDA_PKG_VERSION \ +# cuda-npp-dev-$CUDA_PKG_VERSION \ +# cuda-libraries-dev-$CUDA_PKG_VERSION \ +# cuda-minimal-build-$CUDA_PKG_VERSION \ +# libcublas-dev=10.2.2.89-1 \ +# libnccl-dev=$NCCL_VERSION-1+cuda10.2 \ +# && apt-mark hold libcublas-dev libnccl-dev \ +# && rm -rf /var/lib/apt/lists/* + +# ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs + +##### END NVIDIA CUDA DOCKERFILES #### + diff --git a/python/cuda-hpc-python_cuda10.2/Dockerfile b/python/cuda-hpc-python_cuda10.2/Dockerfile deleted file mode 100644 index aa0cef6..0000000 --- a/python/cuda-hpc-python_cuda10.2/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -ARG PAWSEY_BASE="2022.03" -FROM quay.io/pawsey/hpc-python:${PAWSEY_BASE} - - -##### START NVIDIA CUDA DOCKERFILES #### -# CUDA VERSION 10.2 -- composition of -# https://gitlab.com/nvidia/container-images/cuda/-/blob/master/dist/10.2/ubuntu18.04-x86_64/base/Dockerfile -# https://gitlab.com/nvidia/container-images/cuda/-/blob/master/dist/10.2/ubuntu18.04-x86_64/runtime/Dockerfile -# https://gitlab.com/nvidia/container-images/cuda/-/blob/master/dist/10.2/ubuntu18.04-x86_64/devel/Dockerfile -RUN apt-get update && apt-get install -y --no-install-recommends \ - gnupg2 curl ca-certificates && \ - curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub | apt-key add - && \ - echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/cuda.list && \ - echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list && \ - apt-get purge --autoremove -y curl \ - && rm -rf /var/lib/apt/lists/* - -ENV CUDA_VERSION 10.2.89 -ENV CUDA_PKG_VERSION 10-2=$CUDA_VERSION-1 - -# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a -RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-cudart-$CUDA_PKG_VERSION \ - cuda-compat-10-2 \ - && ln -s cuda-10.2 /usr/local/cuda && \ - rm -rf /var/lib/apt/lists/* - -# Required for nvidia-docker v1 -RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \ - echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf - -ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} -ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 - -# nvidia-container-runtime -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES compute,utility -ENV NVIDIA_REQUIRE_CUDA "cuda>=10.2 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411 brand=tesla,driver>=418,driver<419 brand=tesla,driver>=440,driver<441" - -ENV NCCL_VERSION 2.10.3 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-libraries-$CUDA_PKG_VERSION \ - cuda-npp-$CUDA_PKG_VERSION \ - cuda-nvtx-$CUDA_PKG_VERSION \ - cuda-cusparse-$CUDA_PKG_VERSION \ - libcublas10=10.2.2.89-1 \ - libnccl2=$NCCL_VERSION-1+cuda10.2 \ - && apt-mark hold libcublas10 libnccl2 \ - && rm -rf /var/lib/apt/lists/* - -RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-nvml-dev-$CUDA_PKG_VERSION \ - cuda-command-line-tools-$CUDA_PKG_VERSION \ - cuda-nvprof-$CUDA_PKG_VERSION \ - cuda-npp-dev-$CUDA_PKG_VERSION \ - cuda-libraries-dev-$CUDA_PKG_VERSION \ - cuda-minimal-build-$CUDA_PKG_VERSION \ - libcublas-dev=10.2.2.89-1 \ - libnccl-dev=$NCCL_VERSION-1+cuda10.2 \ - && apt-mark hold libcublas-dev libnccl-dev \ - && rm -rf /var/lib/apt/lists/* - -ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs - -##### END NVIDIA CUDA DOCKERFILES #### - diff --git a/python/build_python.sh b/python/deprecated/build_python.sh similarity index 100% rename from python/build_python.sh rename to python/deprecated/build_python.sh diff --git a/python/cuda-intel-hpc-python/Dockerfile b/python/deprecated/cuda-intel-hpc-python/Dockerfile similarity index 100% rename from python/cuda-intel-hpc-python/Dockerfile rename to python/deprecated/cuda-intel-hpc-python/Dockerfile diff --git a/python/intel-hpc-python-hdf5mpi/Dockerfile b/python/deprecated/intel-hpc-python-hdf5mpi/Dockerfile similarity index 100% rename from python/intel-hpc-python-hdf5mpi/Dockerfile rename to python/deprecated/intel-hpc-python-hdf5mpi/Dockerfile diff --git a/python/intel-hpc-python-hdf5mpi/environment-24Mar2022.yaml b/python/deprecated/intel-hpc-python-hdf5mpi/environment-24Mar2022.yaml similarity index 100% rename from python/intel-hpc-python-hdf5mpi/environment-24Mar2022.yaml rename to python/deprecated/intel-hpc-python-hdf5mpi/environment-24Mar2022.yaml diff --git a/python/intel-hpc-python-hdf5mpi/requirements-24Mar2022.yaml b/python/deprecated/intel-hpc-python-hdf5mpi/requirements-24Mar2022.yaml similarity index 100% rename from python/intel-hpc-python-hdf5mpi/requirements-24Mar2022.yaml rename to python/deprecated/intel-hpc-python-hdf5mpi/requirements-24Mar2022.yaml diff --git a/python/intel-hpc-python-hdf5mpi/requirements.in b/python/deprecated/intel-hpc-python-hdf5mpi/requirements.in similarity index 100% rename from python/intel-hpc-python-hdf5mpi/requirements.in rename to python/deprecated/intel-hpc-python-hdf5mpi/requirements.in diff --git a/python/deprecated/intel-hpc-python/Dockerfile b/python/deprecated/intel-hpc-python/Dockerfile new file mode 120000 index 0000000..95c3e1d --- /dev/null +++ b/python/deprecated/intel-hpc-python/Dockerfile @@ -0,0 +1 @@ +buildhpcpythonintel.dockerfile \ No newline at end of file diff --git a/python/intel-hpc-python/Dockerfile b/python/deprecated/intel-hpc-python/buildhpcpythonintel.dockerfile similarity index 89% rename from python/intel-hpc-python/Dockerfile rename to python/deprecated/intel-hpc-python/buildhpcpythonintel.dockerfile index 707ef9a..f594c97 100644 --- a/python/intel-hpc-python/Dockerfile +++ b/python/deprecated/intel-hpc-python/buildhpcpythonintel.dockerfile @@ -1,13 +1,11 @@ -ARG IPY_VERSION="2020.2" +ARG IPY_VERSION="2023.1.0-0" FROM intelpython/intelpython3_core:${IPY_VERSION} -LABEL maintainer="marco.delapierre@pawsey.org.au" - +LABEL maintainer="pascal.elahi@pawsey.org.au" # Just one extra conda variable ENV CONDA_PREFIX="/opt/conda" - # Install package dependencies RUN apt-get update -qq --allow-releaseinfo-change \ && apt-get -y --no-install-recommends install \ @@ -68,6 +66,9 @@ ARG MPI4PY_VERSION="3.1.3" RUN pip --no-cache-dir install --no-deps ${DEPENDENCIES} h5netcdf==${H5NETCDF_VERSION} h5py==${H5PY_VERSION} mpi4py==${MPI4PY_VERSION} +# and copy the recipe into the docker recipes directory +RUN mkdir -p /opt/docker-recipes/ +COPY buildhpcpythonintel.dockerfile /opt/docker-recipes/ # Final diff --git a/python/intel-hpc-python/environment-24Mar2022.yaml b/python/deprecated/intel-hpc-python/environment-24Mar2022.yaml similarity index 100% rename from python/intel-hpc-python/environment-24Mar2022.yaml rename to python/deprecated/intel-hpc-python/environment-24Mar2022.yaml diff --git a/python/intel-hpc-python/requirements-24Mar2022.yaml b/python/deprecated/intel-hpc-python/requirements-24Mar2022.yaml similarity index 100% rename from python/intel-hpc-python/requirements-24Mar2022.yaml rename to python/deprecated/intel-hpc-python/requirements-24Mar2022.yaml diff --git a/python/intel-hpc-python/requirements.in b/python/deprecated/intel-hpc-python/requirements.in similarity index 100% rename from python/intel-hpc-python/requirements.in rename to python/deprecated/intel-hpc-python/requirements.in diff --git a/python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile b/python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile index d20ef5e..1f1344f 100644 --- a/python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile +++ b/python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile @@ -34,7 +34,7 @@ ARG DATE_FILE="12-2023" RUN echo "Install h5py " \ && H5PY_VERSION=$(grep 'h5py' /requirements-${DATE_FILE}.txt | sed 's/==/ /g' | sed 's/ # / /'g | awk '{print $2}') \ && H5NETCDF_VERSION=$(grep 'h5netcdf' /requirements-${DATE_FILE}.txt | sed 's/==/ /g' | sed 's/ # / /'g | awk '{print $2}') \ - && CC="mpicc" CXX="mpic++" HDF5_MPI="ON" HDF5_DIR="/usr/local" pip3 --no-cache-dir install --no-deps --no-binary=h5py h5py==${H5PY_VERSION} h5netcdf==${H5NETCDF_VERSION} + && CC="mpicc" CXX="mpic++" HDF5_MPI="ON" HDF5_DIR="/usr/local" pip3 --no-cache-dir install --break-system-packages --no-deps --no-binary=h5py h5py==${H5PY_VERSION} h5netcdf==${H5NETCDF_VERSION} # and copy the recipe into the docker recipes directory RUN mkdir -p /opt/docker-recipes/ diff --git a/python/hpc-python/buildhpcpython.dockerfile b/python/hpc-python/buildhpcpython.dockerfile index 08f032e..0accb70 100644 --- a/python/hpc-python/buildhpcpython.dockerfile +++ b/python/hpc-python/buildhpcpython.dockerfile @@ -1,8 +1,10 @@ -ARG IMAGE_VERSION="3.4.3_ubuntu20.04_lustrerelease" -FROM quay.io/pawsey/mpich-lustre-base:${IMAGE_VERSION} +ARG MPICH_VERSION=3.4.3 +ARG OS_VERSION=ubuntu23.04 +#ARG IMAGE_VERSION="3.4.3_ubuntu23.04_lustrerelease" +FROM quay.io/pawsey/mpich-lustre-base:${MPICH_VERSION}_${OS_VERSION}_lustrerelease ARG PY_VERSION=3.11 -LABEL org.opencontainers.image.created="2023-12" +LABEL org.opencontainers.image.created="2024-01" LABEL org.opencontainers.image.authors="Pascal Jahan Elahi " LABEL org.opencontainers.image.documentation="https://github.com/PawseySC/pawsey-containers/" LABEL org.opencontainers.image.source="https://github.com/PawseySC/pawsey-containers/python/hpcpython/buildhpcpython.dockerfile" @@ -10,31 +12,34 @@ LABEL org.opencontainers.image.vendor="Pawsey Supercomputing Research Centre" LABEL org.opencontainers.image.licenses="GNU GPL3.0" LABEL org.opencontainers.image.title="Setonix compatible python container" LABEL org.opencontainers.image.description="Common base image providing python on Setonix built on the lustre aware mpich" -LABEL org.opencontainers.image.base.name="pawsey/hpcpython:${PY_VERSION}" +LABEL org.opencontainers.image.base.name="pawsey/hpcpython:${PY_VERSION}-ubuntu23.04" # add python ENV DEBIAN_FRONTEND="noninteractive" RUN echo "Updating python to ${PY_VERSION}" \ && apt-get update -qq \ && apt install -y --no-install-recommends software-properties-common \ - && add-apt-repository ppa:deadsnakes/ppa \ - && apt-get update -qq \ + && if [ ${OS_VERSION} != "ubuntu23.04" ]; then add-apt-repository ppa:deadsnakes/ppa; apt-get update -qq; fi \ && apt-get -y --no-install-recommends install \ python${PY_VERSION}-dev python${PY_VERSION}-distutils python${PY_VERSION}-full \ && apt-get clean all \ && rm -r /var/lib/apt/lists/* \ + && echo "Done" +RUN echo "Setting default python to desired version" \ && update-alternatives --install /usr/bin/python python /usr/bin/python${PY_VERSION} 1 \ && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PY_VERSION} 1 \ - && curl -sS https://bootstrap.pypa.io/get-pip.py | sudo python3 \ - && curl -sS https://bootstrap.pypa.io/get-pip.py | python3 \ - && python -m pip install pip --upgrade \ - && pip install pip-tools \ + && if [ ${OS_VERSION} != "ubuntu23.04" ]; then \ + curl -sS https://bootstrap.pypa.io/get-pip.py | sudo python3; \ + curl -sS https://bootstrap.pypa.io/get-pip.py | python3; \ + python -m pip install --upgrade --break-system-packages pip; \ + fi \ + && pip install --break-system-packages pip-tools \ && echo "Done " # Install Python packages ARG DATE_FILE="12-2023" ADD requirements.in requirements-${DATE_FILE}.txt / -RUN pip3 --no-cache-dir install -r /requirements-${DATE_FILE}.txt --no-deps +RUN pip3 --no-cache-dir install --break-system-packages -r /requirements-${DATE_FILE}.txt --no-deps # and copy the recipe into the docker recipes directory RUN mkdir -p /opt/docker-recipes/