Merge pull request #31 from PawseySC/python-update

Python update

python/cuda-hpc-python/Dockerfile

@@ -0,0 +1 @@
+buildhpcpythoncuda.dockerfile

python/cuda-hpc-python/buildhpcpythoncuda.dockerfile

@@ -0,0 +1,82 @@
+ARG PAWSEY_BASE="3.11"
+FROM quay.io/pawsey/hpc-python:${PAWSEY_BASE}
+
+ARG CUDA_VERSION=12.3
+ARG CUDA_PKG_VERSION=12-3
+
+RUN echo "Installing CUDA" \
+    && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb \
+    && dpkg -i cuda-keyring_1.1-1_all.deb \
+    && apt-get update \
+    && apt-get -y install cuda-toolkit-${CUDA_PKG_VERSION} \
+    && apt-get install -y nvidia-kernel-open-545 \
+    && apt-get install -y cuda-drivers-545 \
+    && echo "Done"
+
+RUN echo "Installing NVHPC SDK" \
+    && curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg \
+    && echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list \
+    && apt-get update -y \
+    && apt-get install -y nvhpc-23-11 \
+    && echo "Done"
+
+# Update to libmozjs which is at this point version 68. Upgrade to 102 
+#RUN echo "Updating libmozjs" \
+#    && sed -i "s/focal/jammy/g"  /etc/apt/sources.list \
+#    && apt-get update \ 
+#    && apt-get install -y libmozjs-102-0 \
+#    && apt-get remove -y libmozjs-68-0 \
+#    && echo "Done"
+
+ENV CUDA_VERSION 12.3
+ENV CUDA_PKG_VERSION 12-3=$CUDA_VERSION-1
+
+# # For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+#     cuda-cudart-${CUDA_PKG_VERSION} \
+#     cuda-compat-${CUDA_PKG_VERSION} \
+#     && ln -s cuda-${CUDA_VERSION}} /usr/local/cuda && \
+#     rm -rf /var/lib/apt/lists/*
+
+# # Required for nvidia-docker v1
+# RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
+#     echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
+
+# ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
+# ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
+
+# nvidia-container-runtime
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+ENV NVIDIA_REQUIRE_CUDA "cuda>=12.3" 
+#brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411 brand=tesla,driver>=418,driver<419 brand=tesla,driver>=440,driver<441"
+
+# ENV NCCL_VERSION 2.10.3
+
+
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+#     cuda-libraries-$CUDA_PKG_VERSION \
+#     cuda-npp-$CUDA_PKG_VERSION \
+#     cuda-nvtx-$CUDA_PKG_VERSION \
+#     cuda-cusparse-$CUDA_PKG_VERSION \
+#     libcublas10=10.2.2.89-1 \
+#     libnccl2=$NCCL_VERSION-1+cuda10.2 \
+#     && apt-mark hold libcublas10 libnccl2 \
+#     && rm -rf /var/lib/apt/lists/*
+
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+#     cuda-nvml-dev-$CUDA_PKG_VERSION \
+#     cuda-command-line-tools-$CUDA_PKG_VERSION \
+#     cuda-nvprof-$CUDA_PKG_VERSION \
+#     cuda-npp-dev-$CUDA_PKG_VERSION \
+#     cuda-libraries-dev-$CUDA_PKG_VERSION \
+#     cuda-minimal-build-$CUDA_PKG_VERSION \
+#     libcublas-dev=10.2.2.89-1 \
+#     libnccl-dev=$NCCL_VERSION-1+cuda10.2 \
+#     && apt-mark hold libcublas-dev libnccl-dev \
+#     && rm -rf /var/lib/apt/lists/*
+
+# ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs
+
+##### END   NVIDIA CUDA DOCKERFILES ####
+

python/deprecated/intel-hpc-python/Dockerfile

@@ -0,0 +1 @@
+buildhpcpythonintel.dockerfile

python/intel-hpc-python/Dockerfile → python/deprecated/intel-hpc-python/buildhpcpythonintel.dockerfile

@@ -1,13 +1,11 @@
-ARG IPY_VERSION="2020.2"
+ARG IPY_VERSION="2023.1.0-0"
 FROM intelpython/intelpython3_core:${IPY_VERSION}
 
-LABEL maintainer="[email protected]"
-
+LABEL maintainer="[email protected]"
 
 # Just one extra conda variable
 ENV CONDA_PREFIX="/opt/conda"
 
-
 # Install package dependencies
 RUN apt-get update -qq --allow-releaseinfo-change \
       && apt-get -y --no-install-recommends install \
@@ -68,6 +66,9 @@ ARG MPI4PY_VERSION="3.1.3"
 
 RUN pip --no-cache-dir install --no-deps ${DEPENDENCIES} h5netcdf==${H5NETCDF_VERSION} h5py==${H5PY_VERSION} mpi4py==${MPI4PY_VERSION}
 
+# and copy the recipe into the docker recipes directory
+RUN mkdir -p /opt/docker-recipes/
+COPY buildhpcpythonintel.dockerfile /opt/docker-recipes/
 
 
 # Final

python/hpc-python-hdf5mpi/Dockerfile

@@ -0,0 +1 @@
+buildhpcpythonhdf5mpi.dockerfile

python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile

@@ -0,0 +1,44 @@
+ARG HPCPYTHON_VERSION="3.11"
+FROM quay.io/pawsey/hpc-python:${HPCPYTHON_VERSION}
+
+LABEL org.opencontainers.image.created="2023-12"
+LABEL org.opencontainers.image.authors="Pascal Jahan Elahi <[email protected]>"
+LABEL org.opencontainers.image.documentation="https://github.com/PawseySC/pawsey-containers/"
+LABEL org.opencontainers.image.source="https://github.com/PawseySC/pawsey-containers/python/hpcpython/buildhpcpython.dockerfile"
+LABEL org.opencontainers.image.vendor="Pawsey Supercomputing Research Centre"
+LABEL org.opencontainers.image.licenses="GNU GPL3.0"
+LABEL org.opencontainers.image.title="Setonix compatible python container with parallel h5py"
+LABEL org.opencontainers.image.description="Common base image providing python on Setonix built on the lustre aware mpich"
+LABEL org.opencontainers.image.base.name="pawsey/hpcpython:3.11-hdf5-mpi"
+
+# add HDF5
+ARG HDF5_VERSION="1.14.3"
+ARG HDF5_CONFIGURE_OPTIONS="--prefix=/usr/local --enable-parallel CC=mpicc"
+ARG HDF5_MAKE_OPTIONS="-j4"
+RUN mkdir -p /tmp/hdf5-build \
+    && cd /tmp/hdf5-build \
+    && HDF5_VER_MM="${HDF5_VERSION%.*}" \
+    && wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${HDF5_VER_MM}/hdf5-${HDF5_VERSION}/src/hdf5-${HDF5_VERSION}.tar.gz \
+    && tar xzf hdf5-${HDF5_VERSION}.tar.gz \
+    && cd hdf5-${HDF5_VERSION} \
+    && ./configure ${HDF5_CONFIGURE_OPTIONS} \
+    && make ${HDF5_MAKE_OPTIONS} \
+    && make install \
+    && ldconfig \
+    && cd / \
+    && rm -rf /tmp/hdf5-build
+
+
+# # Install h5py and h5netcdf
+ARG DATE_FILE="12-2023"
+RUN echo "Install h5py " \
+    && H5PY_VERSION=$(grep 'h5py' /requirements-${DATE_FILE}.txt | sed 's/==/ /g' | sed 's/ # / /'g | awk '{print $2}') \
+    && H5NETCDF_VERSION=$(grep 'h5netcdf' /requirements-${DATE_FILE}.txt | sed 's/==/ /g' | sed 's/ # / /'g | awk '{print $2}') \
+    && CC="mpicc" CXX="mpic++" HDF5_MPI="ON" HDF5_DIR="/usr/local" pip3 --no-cache-dir install --break-system-packages --no-deps --no-binary=h5py h5py==${H5PY_VERSION} h5netcdf==${H5NETCDF_VERSION}
+
+# and copy the recipe into the docker recipes directory
+RUN mkdir -p /opt/docker-recipes/
+COPY buildhpcpythonhdf5mpi.dockerfile /opt/docker-recipes/
+
+# Final
+CMD ["/bin/bash"]