fix cmake version issue for manylinux image (#354)

.github/workflows/build_wheel.yml

@@ -21,7 +21,6 @@ env:
 jobs:
   build_wheel:
     strategy:
-      fail-fast: false
       matrix:
         python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         cuda: ["11.8", "12.1", "12.4"]

.github/workflows/publish_devel_image.yml

@@ -17,12 +17,16 @@ env:
   CI_CACHE_DIR: ${{ github.workspace }}/../../ci_cache
 
 jobs:
-  publish_base:
+  publish_devel_image:
+    strategy:
+      fail-fast: false
+      matrix:
+        cuda: ["11.8", "12.1", "12.4", "12.6"]
     runs-on: [self-hosted, linux, build]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -38,40 +42,7 @@ jobs:
       - name: Create cache directory
         run: mkdir -p $CI_CACHE_DIR/.buildx-cache
 
-      - name: Build devel image for cuda 12.6 (experimental)
-        uses: docker/build-push-action@v5
-        with:
-          context: ./docker
-          file: ./docker/Dockerfile.devel
-          push: ${{ inputs.push || 'true' }}
-          cache-from: type=local,src=$CI_CACHE_DIR/.buildx-cache
-          cache-to: type=local,dest=$CI_CACHE_DIR/.buildx-cache
-          build-args: |
-            UBUNTU_VERSION=22.04
-            CUDA_VERSION=12.6
-            GCC_VERSION=12
-          tags: |
-            vectorchai/scalellm_devel:cuda12.6-ubuntu22.04
-            vectorchai/scalellm_devel:cuda12.6
-
-      - name: Build devel image for cuda 12.4
-        uses: docker/build-push-action@v5
-        with:
-          context: ./docker
-          file: ./docker/Dockerfile.devel
-          push: ${{ inputs.push || 'true' }}
-          cache-from: type=local,src=$CI_CACHE_DIR/.buildx-cache
-          cache-to: type=local,dest=$CI_CACHE_DIR/.buildx-cache
-          build-args: |
-            UBUNTU_VERSION=22.04
-            CUDA_VERSION=12.4
-            GCC_VERSION=12
-          tags: |
-            vectorchai/scalellm_devel:cuda12.4-ubuntu22.04
-            vectorchai/scalellm_devel:cuda12.4
-            vectorchai/scalellm_devel:latest
-
-      - name: Build devel image for cuda 12.1
+      - name: Build devel image for cuda ${{ matrix.cuda }}
         uses: docker/build-push-action@v5
         with:
           context: ./docker
@@ -81,25 +52,9 @@ jobs:
           cache-to: type=local,dest=$CI_CACHE_DIR/.buildx-cache
           build-args: |
             UBUNTU_VERSION=22.04
-            CUDA_VERSION=12.1
+            CUDA_VERSION=${{ matrix.cuda }}
             GCC_VERSION=12
           tags: |
-            vectorchai/scalellm_devel:cuda12.1-ubuntu22.04
-            vectorchai/scalellm_devel:cuda12.1
-
-      - name: Build devel image for cuda 11.8
-        uses: docker/build-push-action@v5
-        with:
-          context: ./docker
-          file: ./docker/Dockerfile.devel
-          push: ${{ inputs.push || 'true' }}
-          cache-from: type=local,src=$CI_CACHE_DIR/.buildx-cache
-          cache-to: type=local,dest=$CI_CACHE_DIR/.buildx-cache
-          build-args: |
-            UBUNTU_VERSION=22.04
-            CUDA_VERSION=11.8
-            GCC_VERSION=11
-          tags: |
-            vectorchai/scalellm_devel:cuda11.8-ubuntu22.04
-            vectorchai/scalellm_devel:cuda11.8
+            vectorchai/scalellm_devel:cuda${{ matrix.cuda }}-ubuntu22.04
+            vectorchai/scalellm_devel:cuda${{ matrix.cuda }}
 

.github/workflows/publish_manylinux_image.yml

@@ -17,7 +17,7 @@ env:
   CI_CACHE_DIR: ${{ github.workspace }}/../../ci_cache
 
 jobs:
-  publish_base:
+  publish_manylinux_image:
     strategy:
       fail-fast: false
       matrix:

.github/workflows/publish_wheel.yml

@@ -20,7 +20,6 @@ env:
 jobs:
   build_wheel:
     strategy:
-      fail-fast: false
       matrix:
         python: ["3.9", "3.10", "3.11", "3.12"]
         cuda: ["12.4"]

docker/Dockerfile.devel

@@ -19,29 +19,34 @@ RUN apt-get update \
     software-properties-common gpg-agent
 COPY ./common/install_gcc.sh install_gcc.sh
 RUN bash ./install_gcc.sh && rm install_gcc.sh
+RUN gcc --version; g++ --version
 
 ARG CMAKE_VERSION=3.29.3
 COPY ./common/install_cmake.sh install_cmake.sh
 RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
 RUN rm install_cmake.sh
+RUN cmake --version
 
 ARG NINJA_VERSION=1.11.1
 COPY ./common/install_ninja.sh install_ninja.sh
 RUN if [ -n "${NINJA_VERSION}" ]; then bash ./install_ninja.sh; fi
 RUN rm install_ninja.sh
+RUN ninja --version
 
 # Install ccache
 ARG CCACHE_VERSION=4.8.3
 COPY ./common/install_ccache.sh install_ccache.sh
 RUN if [ -n "${CCACHE_VERSION}" ]; then bash ./install_ccache.sh; fi
 RUN rm install_ccache.sh
+RUN ccache --version
 
 # Install cuda, cudnn and nccl
 ARG CUDA_VERSION=12.1
 COPY ./common/install_cuda.sh install_cuda.sh
 RUN bash ./install_cuda.sh ${CUDA_VERSION} && rm install_cuda.sh
 ENV DESIRED_CUDA=${CUDA_VERSION}
 ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH
+RUN nvcc --version
 
 # Install rust
 ENV RUSTUP_HOME=/usr/local/rustup

docker/Dockerfile.manylinux

@@ -19,16 +19,22 @@ RUN yum -y update && yum -y install \
     perl-IPC-Cmd \
     sudo 
 
+# Show versions of gcc, g++, nvcc
+RUN gcc --version; g++ --version; nvcc --version
+
+# Install cmake
 ARG CMAKE_VERSION=3.29.3
 COPY ./common/install_cmake.sh install_cmake.sh
 RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
 RUN rm install_cmake.sh
+RUN cmake --version
 
 # Install ccache
 ARG CCACHE_VERSION=4.8.3
 COPY ./common/install_ccache.sh install_ccache.sh
 RUN if [ -n "${CCACHE_VERSION}" ]; then bash ./install_ccache.sh; fi
 RUN rm install_ccache.sh
+RUN ccache --version
 
 # Install rust
 ENV RUSTUP_HOME=/usr/local/rustup

docker/common/install_cmake.sh

@@ -4,6 +4,9 @@ set -ex
 
 [ -n "$CMAKE_VERSION" ]
 
+# Uninstall cmake package if it exists
+command -v pip3 >/dev/null && pip3 uninstall -y cmake
+
 # Remove existing CMake installation
 rm -f /usr/local/bin/cmake
 

tools/run_in_docker.sh

@@ -37,7 +37,8 @@ function usage() {
 
 (( $# < 1 )) && usage
 
-IMAGE="vectorchai/scalellm_devel:latest"
+# Default image
+IMAGE="vectorchai/scalellm_devel:cuda12.4"
 RUN_OPTS=()
 
 INTERACTIVE=1