From 0c59dcfd7dd704cfcfbf88633e023117f5c49fa7 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 3 Jan 2024 03:59:41 +0000 Subject: [PATCH] Updated python recipes - removed some older intel based python recipes to deprecated - updated the requirements files of some python recipes - updated recipes to be able to handle newer os base images --- python/cuda-hpc-python/Dockerfile | 1 + .../buildhpcpythoncuda.dockerfile | 82 +++++++++++++++++++ python/cuda-hpc-python_cuda10.2/Dockerfile | 67 --------------- python/{ => deprecated}/build_python.sh | 0 .../cuda-intel-hpc-python/Dockerfile | 0 .../intel-hpc-python-hdf5mpi/Dockerfile | 0 .../environment-24Mar2022.yaml | 0 .../requirements-24Mar2022.yaml | 0 .../intel-hpc-python-hdf5mpi/requirements.in | 0 python/deprecated/intel-hpc-python/Dockerfile | 1 + .../buildhpcpythonintel.dockerfile} | 9 +- .../environment-24Mar2022.yaml | 0 .../requirements-24Mar2022.yaml | 0 .../intel-hpc-python/requirements.in | 0 .../buildhpcpythonhdf5mpi.dockerfile | 2 +- python/hpc-python/buildhpcpython.dockerfile | 27 +++--- 16 files changed, 106 insertions(+), 83 deletions(-) create mode 120000 python/cuda-hpc-python/Dockerfile create mode 100644 python/cuda-hpc-python/buildhpcpythoncuda.dockerfile delete mode 100644 python/cuda-hpc-python_cuda10.2/Dockerfile rename python/{ => deprecated}/build_python.sh (100%) rename python/{ => deprecated}/cuda-intel-hpc-python/Dockerfile (100%) rename python/{ => deprecated}/intel-hpc-python-hdf5mpi/Dockerfile (100%) rename python/{ => deprecated}/intel-hpc-python-hdf5mpi/environment-24Mar2022.yaml (100%) rename python/{ => deprecated}/intel-hpc-python-hdf5mpi/requirements-24Mar2022.yaml (100%) rename python/{ => deprecated}/intel-hpc-python-hdf5mpi/requirements.in (100%) create mode 120000 python/deprecated/intel-hpc-python/Dockerfile rename python/{intel-hpc-python/Dockerfile => deprecated/intel-hpc-python/buildhpcpythonintel.dockerfile} (89%) rename python/{ => deprecated}/intel-hpc-python/environment-24Mar2022.yaml (100%) rename python/{ => deprecated}/intel-hpc-python/requirements-24Mar2022.yaml (100%) rename python/{ => deprecated}/intel-hpc-python/requirements.in (100%) diff --git a/python/cuda-hpc-python/Dockerfile b/python/cuda-hpc-python/Dockerfile new file mode 120000 index 0000000..504dccc --- /dev/null +++ b/python/cuda-hpc-python/Dockerfile @@ -0,0 +1 @@ +buildhpcpythoncuda.dockerfile \ No newline at end of file diff --git a/python/cuda-hpc-python/buildhpcpythoncuda.dockerfile b/python/cuda-hpc-python/buildhpcpythoncuda.dockerfile new file mode 100644 index 0000000..857a4d7 --- /dev/null +++ b/python/cuda-hpc-python/buildhpcpythoncuda.dockerfile @@ -0,0 +1,82 @@ +ARG PAWSEY_BASE="3.11" +FROM quay.io/pawsey/hpc-python:${PAWSEY_BASE} + +ARG CUDA_VERSION=12.3 +ARG CUDA_PKG_VERSION=12-3 + +RUN echo "Installing CUDA" \ + && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb \ + && dpkg -i cuda-keyring_1.1-1_all.deb \ + && apt-get update \ + && apt-get -y install cuda-toolkit-${CUDA_PKG_VERSION} \ + && apt-get install -y nvidia-kernel-open-545 \ + && apt-get install -y cuda-drivers-545 \ + && echo "Done" + +RUN echo "Installing NVHPC SDK" \ + && curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg \ + && echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list \ + && apt-get update -y \ + && apt-get install -y nvhpc-23-11 \ + && echo "Done" + +# Update to libmozjs which is at this point version 68. Upgrade to 102 +#RUN echo "Updating libmozjs" \ +# && sed -i "s/focal/jammy/g" /etc/apt/sources.list \ +# && apt-get update \ +# && apt-get install -y libmozjs-102-0 \ +# && apt-get remove -y libmozjs-68-0 \ +# && echo "Done" + +ENV CUDA_VERSION 12.3 +ENV CUDA_PKG_VERSION 12-3=$CUDA_VERSION-1 + +# # For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# cuda-cudart-${CUDA_PKG_VERSION} \ +# cuda-compat-${CUDA_PKG_VERSION} \ +# && ln -s cuda-${CUDA_VERSION}} /usr/local/cuda && \ +# rm -rf /var/lib/apt/lists/* + +# # Required for nvidia-docker v1 +# RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \ +# echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf + +# ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +# ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 + +# nvidia-container-runtime +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility +ENV NVIDIA_REQUIRE_CUDA "cuda>=12.3" +#brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411 brand=tesla,driver>=418,driver<419 brand=tesla,driver>=440,driver<441" + +# ENV NCCL_VERSION 2.10.3 + + +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# cuda-libraries-$CUDA_PKG_VERSION \ +# cuda-npp-$CUDA_PKG_VERSION \ +# cuda-nvtx-$CUDA_PKG_VERSION \ +# cuda-cusparse-$CUDA_PKG_VERSION \ +# libcublas10=10.2.2.89-1 \ +# libnccl2=$NCCL_VERSION-1+cuda10.2 \ +# && apt-mark hold libcublas10 libnccl2 \ +# && rm -rf /var/lib/apt/lists/* + +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# cuda-nvml-dev-$CUDA_PKG_VERSION \ +# cuda-command-line-tools-$CUDA_PKG_VERSION \ +# cuda-nvprof-$CUDA_PKG_VERSION \ +# cuda-npp-dev-$CUDA_PKG_VERSION \ +# cuda-libraries-dev-$CUDA_PKG_VERSION \ +# cuda-minimal-build-$CUDA_PKG_VERSION \ +# libcublas-dev=10.2.2.89-1 \ +# libnccl-dev=$NCCL_VERSION-1+cuda10.2 \ +# && apt-mark hold libcublas-dev libnccl-dev \ +# && rm -rf /var/lib/apt/lists/* + +# ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs + +##### END NVIDIA CUDA DOCKERFILES #### + diff --git a/python/cuda-hpc-python_cuda10.2/Dockerfile b/python/cuda-hpc-python_cuda10.2/Dockerfile deleted file mode 100644 index aa0cef6..0000000 --- a/python/cuda-hpc-python_cuda10.2/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -ARG PAWSEY_BASE="2022.03" -FROM quay.io/pawsey/hpc-python:${PAWSEY_BASE} - - -##### START NVIDIA CUDA DOCKERFILES #### -# CUDA VERSION 10.2 -- composition of -# https://gitlab.com/nvidia/container-images/cuda/-/blob/master/dist/10.2/ubuntu18.04-x86_64/base/Dockerfile -# https://gitlab.com/nvidia/container-images/cuda/-/blob/master/dist/10.2/ubuntu18.04-x86_64/runtime/Dockerfile -# https://gitlab.com/nvidia/container-images/cuda/-/blob/master/dist/10.2/ubuntu18.04-x86_64/devel/Dockerfile -RUN apt-get update && apt-get install -y --no-install-recommends \ - gnupg2 curl ca-certificates && \ - curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub | apt-key add - && \ - echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/cuda.list && \ - echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list && \ - apt-get purge --autoremove -y curl \ - && rm -rf /var/lib/apt/lists/* - -ENV CUDA_VERSION 10.2.89 -ENV CUDA_PKG_VERSION 10-2=$CUDA_VERSION-1 - -# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a -RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-cudart-$CUDA_PKG_VERSION \ - cuda-compat-10-2 \ - && ln -s cuda-10.2 /usr/local/cuda && \ - rm -rf /var/lib/apt/lists/* - -# Required for nvidia-docker v1 -RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \ - echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf - -ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} -ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 - -# nvidia-container-runtime -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES compute,utility -ENV NVIDIA_REQUIRE_CUDA "cuda>=10.2 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411 brand=tesla,driver>=418,driver<419 brand=tesla,driver>=440,driver<441" - -ENV NCCL_VERSION 2.10.3 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-libraries-$CUDA_PKG_VERSION \ - cuda-npp-$CUDA_PKG_VERSION \ - cuda-nvtx-$CUDA_PKG_VERSION \ - cuda-cusparse-$CUDA_PKG_VERSION \ - libcublas10=10.2.2.89-1 \ - libnccl2=$NCCL_VERSION-1+cuda10.2 \ - && apt-mark hold libcublas10 libnccl2 \ - && rm -rf /var/lib/apt/lists/* - -RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-nvml-dev-$CUDA_PKG_VERSION \ - cuda-command-line-tools-$CUDA_PKG_VERSION \ - cuda-nvprof-$CUDA_PKG_VERSION \ - cuda-npp-dev-$CUDA_PKG_VERSION \ - cuda-libraries-dev-$CUDA_PKG_VERSION \ - cuda-minimal-build-$CUDA_PKG_VERSION \ - libcublas-dev=10.2.2.89-1 \ - libnccl-dev=$NCCL_VERSION-1+cuda10.2 \ - && apt-mark hold libcublas-dev libnccl-dev \ - && rm -rf /var/lib/apt/lists/* - -ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs - -##### END NVIDIA CUDA DOCKERFILES #### - diff --git a/python/build_python.sh b/python/deprecated/build_python.sh similarity index 100% rename from python/build_python.sh rename to python/deprecated/build_python.sh diff --git a/python/cuda-intel-hpc-python/Dockerfile b/python/deprecated/cuda-intel-hpc-python/Dockerfile similarity index 100% rename from python/cuda-intel-hpc-python/Dockerfile rename to python/deprecated/cuda-intel-hpc-python/Dockerfile diff --git a/python/intel-hpc-python-hdf5mpi/Dockerfile b/python/deprecated/intel-hpc-python-hdf5mpi/Dockerfile similarity index 100% rename from python/intel-hpc-python-hdf5mpi/Dockerfile rename to python/deprecated/intel-hpc-python-hdf5mpi/Dockerfile diff --git a/python/intel-hpc-python-hdf5mpi/environment-24Mar2022.yaml b/python/deprecated/intel-hpc-python-hdf5mpi/environment-24Mar2022.yaml similarity index 100% rename from python/intel-hpc-python-hdf5mpi/environment-24Mar2022.yaml rename to python/deprecated/intel-hpc-python-hdf5mpi/environment-24Mar2022.yaml diff --git a/python/intel-hpc-python-hdf5mpi/requirements-24Mar2022.yaml b/python/deprecated/intel-hpc-python-hdf5mpi/requirements-24Mar2022.yaml similarity index 100% rename from python/intel-hpc-python-hdf5mpi/requirements-24Mar2022.yaml rename to python/deprecated/intel-hpc-python-hdf5mpi/requirements-24Mar2022.yaml diff --git a/python/intel-hpc-python-hdf5mpi/requirements.in b/python/deprecated/intel-hpc-python-hdf5mpi/requirements.in similarity index 100% rename from python/intel-hpc-python-hdf5mpi/requirements.in rename to python/deprecated/intel-hpc-python-hdf5mpi/requirements.in diff --git a/python/deprecated/intel-hpc-python/Dockerfile b/python/deprecated/intel-hpc-python/Dockerfile new file mode 120000 index 0000000..95c3e1d --- /dev/null +++ b/python/deprecated/intel-hpc-python/Dockerfile @@ -0,0 +1 @@ +buildhpcpythonintel.dockerfile \ No newline at end of file diff --git a/python/intel-hpc-python/Dockerfile b/python/deprecated/intel-hpc-python/buildhpcpythonintel.dockerfile similarity index 89% rename from python/intel-hpc-python/Dockerfile rename to python/deprecated/intel-hpc-python/buildhpcpythonintel.dockerfile index 707ef9a..f594c97 100644 --- a/python/intel-hpc-python/Dockerfile +++ b/python/deprecated/intel-hpc-python/buildhpcpythonintel.dockerfile @@ -1,13 +1,11 @@ -ARG IPY_VERSION="2020.2" +ARG IPY_VERSION="2023.1.0-0" FROM intelpython/intelpython3_core:${IPY_VERSION} -LABEL maintainer="marco.delapierre@pawsey.org.au" - +LABEL maintainer="pascal.elahi@pawsey.org.au" # Just one extra conda variable ENV CONDA_PREFIX="/opt/conda" - # Install package dependencies RUN apt-get update -qq --allow-releaseinfo-change \ && apt-get -y --no-install-recommends install \ @@ -68,6 +66,9 @@ ARG MPI4PY_VERSION="3.1.3" RUN pip --no-cache-dir install --no-deps ${DEPENDENCIES} h5netcdf==${H5NETCDF_VERSION} h5py==${H5PY_VERSION} mpi4py==${MPI4PY_VERSION} +# and copy the recipe into the docker recipes directory +RUN mkdir -p /opt/docker-recipes/ +COPY buildhpcpythonintel.dockerfile /opt/docker-recipes/ # Final diff --git a/python/intel-hpc-python/environment-24Mar2022.yaml b/python/deprecated/intel-hpc-python/environment-24Mar2022.yaml similarity index 100% rename from python/intel-hpc-python/environment-24Mar2022.yaml rename to python/deprecated/intel-hpc-python/environment-24Mar2022.yaml diff --git a/python/intel-hpc-python/requirements-24Mar2022.yaml b/python/deprecated/intel-hpc-python/requirements-24Mar2022.yaml similarity index 100% rename from python/intel-hpc-python/requirements-24Mar2022.yaml rename to python/deprecated/intel-hpc-python/requirements-24Mar2022.yaml diff --git a/python/intel-hpc-python/requirements.in b/python/deprecated/intel-hpc-python/requirements.in similarity index 100% rename from python/intel-hpc-python/requirements.in rename to python/deprecated/intel-hpc-python/requirements.in diff --git a/python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile b/python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile index d20ef5e..1f1344f 100644 --- a/python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile +++ b/python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile @@ -34,7 +34,7 @@ ARG DATE_FILE="12-2023" RUN echo "Install h5py " \ && H5PY_VERSION=$(grep 'h5py' /requirements-${DATE_FILE}.txt | sed 's/==/ /g' | sed 's/ # / /'g | awk '{print $2}') \ && H5NETCDF_VERSION=$(grep 'h5netcdf' /requirements-${DATE_FILE}.txt | sed 's/==/ /g' | sed 's/ # / /'g | awk '{print $2}') \ - && CC="mpicc" CXX="mpic++" HDF5_MPI="ON" HDF5_DIR="/usr/local" pip3 --no-cache-dir install --no-deps --no-binary=h5py h5py==${H5PY_VERSION} h5netcdf==${H5NETCDF_VERSION} + && CC="mpicc" CXX="mpic++" HDF5_MPI="ON" HDF5_DIR="/usr/local" pip3 --no-cache-dir install --break-system-packages --no-deps --no-binary=h5py h5py==${H5PY_VERSION} h5netcdf==${H5NETCDF_VERSION} # and copy the recipe into the docker recipes directory RUN mkdir -p /opt/docker-recipes/ diff --git a/python/hpc-python/buildhpcpython.dockerfile b/python/hpc-python/buildhpcpython.dockerfile index 08f032e..0accb70 100644 --- a/python/hpc-python/buildhpcpython.dockerfile +++ b/python/hpc-python/buildhpcpython.dockerfile @@ -1,8 +1,10 @@ -ARG IMAGE_VERSION="3.4.3_ubuntu20.04_lustrerelease" -FROM quay.io/pawsey/mpich-lustre-base:${IMAGE_VERSION} +ARG MPICH_VERSION=3.4.3 +ARG OS_VERSION=ubuntu23.04 +#ARG IMAGE_VERSION="3.4.3_ubuntu23.04_lustrerelease" +FROM quay.io/pawsey/mpich-lustre-base:${MPICH_VERSION}_${OS_VERSION}_lustrerelease ARG PY_VERSION=3.11 -LABEL org.opencontainers.image.created="2023-12" +LABEL org.opencontainers.image.created="2024-01" LABEL org.opencontainers.image.authors="Pascal Jahan Elahi " LABEL org.opencontainers.image.documentation="https://github.com/PawseySC/pawsey-containers/" LABEL org.opencontainers.image.source="https://github.com/PawseySC/pawsey-containers/python/hpcpython/buildhpcpython.dockerfile" @@ -10,31 +12,34 @@ LABEL org.opencontainers.image.vendor="Pawsey Supercomputing Research Centre" LABEL org.opencontainers.image.licenses="GNU GPL3.0" LABEL org.opencontainers.image.title="Setonix compatible python container" LABEL org.opencontainers.image.description="Common base image providing python on Setonix built on the lustre aware mpich" -LABEL org.opencontainers.image.base.name="pawsey/hpcpython:${PY_VERSION}" +LABEL org.opencontainers.image.base.name="pawsey/hpcpython:${PY_VERSION}-ubuntu23.04" # add python ENV DEBIAN_FRONTEND="noninteractive" RUN echo "Updating python to ${PY_VERSION}" \ && apt-get update -qq \ && apt install -y --no-install-recommends software-properties-common \ - && add-apt-repository ppa:deadsnakes/ppa \ - && apt-get update -qq \ + && if [ ${OS_VERSION} != "ubuntu23.04" ]; then add-apt-repository ppa:deadsnakes/ppa; apt-get update -qq; fi \ && apt-get -y --no-install-recommends install \ python${PY_VERSION}-dev python${PY_VERSION}-distutils python${PY_VERSION}-full \ && apt-get clean all \ && rm -r /var/lib/apt/lists/* \ + && echo "Done" +RUN echo "Setting default python to desired version" \ && update-alternatives --install /usr/bin/python python /usr/bin/python${PY_VERSION} 1 \ && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PY_VERSION} 1 \ - && curl -sS https://bootstrap.pypa.io/get-pip.py | sudo python3 \ - && curl -sS https://bootstrap.pypa.io/get-pip.py | python3 \ - && python -m pip install pip --upgrade \ - && pip install pip-tools \ + && if [ ${OS_VERSION} != "ubuntu23.04" ]; then \ + curl -sS https://bootstrap.pypa.io/get-pip.py | sudo python3; \ + curl -sS https://bootstrap.pypa.io/get-pip.py | python3; \ + python -m pip install --upgrade --break-system-packages pip; \ + fi \ + && pip install --break-system-packages pip-tools \ && echo "Done " # Install Python packages ARG DATE_FILE="12-2023" ADD requirements.in requirements-${DATE_FILE}.txt / -RUN pip3 --no-cache-dir install -r /requirements-${DATE_FILE}.txt --no-deps +RUN pip3 --no-cache-dir install --break-system-packages -r /requirements-${DATE_FILE}.txt --no-deps # and copy the recipe into the docker recipes directory RUN mkdir -p /opt/docker-recipes/