From 7a1a6907cd2e99256dbe9ff9b8b3bda32ba9d5d9 Mon Sep 17 00:00:00 2001 From: Yong Hoon Shin Date: Mon, 6 Jan 2025 15:36:27 -0800 Subject: [PATCH] Update gcc version for FBGEMM install in CI (#2654) Summary: TorchRec CI currently is failing with issues on incompatible GLIBCXX version. The cause is that FBGEMM now requires g++ 11.1+ for building binaries that reference GLIBCXX_3.4.29 (as of https://github.com/pytorch/pytorch/pull/141035) As recommended in https://github.com/pytorch/FBGEMM/blob/main/.github/scripts/utils_build.bash and https://github.com/pytorch/FBGEMM/issues/3423, install GCC using conda to control glibcxx version being used. Differential Revision: D67607624 --- .github/scripts/install_fbgemm.sh | 4 +-- .github/workflows/docs.yml | 34 +++++++++++++++++++----- .github/workflows/pyre.yml | 6 ++--- .github/workflows/release_build.yml | 26 ++++++++++++++++--- .github/workflows/unittest_ci.yml | 6 ++--- .github/workflows/unittest_ci_cpu.yml | 37 +++++++++++++++++++++------ 6 files changed, 87 insertions(+), 26 deletions(-) diff --git a/.github/scripts/install_fbgemm.sh b/.github/scripts/install_fbgemm.sh index 08765d62f..16be50099 100644 --- a/.github/scripts/install_fbgemm.sh +++ b/.github/scripts/install_fbgemm.sh @@ -12,7 +12,7 @@ echo "CHANNEL" echo "$CHANNEL" if [ "$CHANNEL" = "nightly" ]; then - ${CONDA_RUN} pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/"$CU_VERSION" + ${CONDA_RUN} pip install --pre fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/"$CU_VERSION" elif [ "$CHANNEL" = "test" ]; then - ${CONDA_RUN} pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/test/"$CU_VERSION" + ${CONDA_RUN} pip install --pre fbgemm-gpu --index-url https://download.pytorch.org/whl/test/"$CU_VERSION" fi diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 51217790e..562622a7f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -33,7 +33,8 @@ jobs: - name: Setup conda run: | wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh - bash ~/miniconda.sh -b -p $HOME/miniconda + bash ~/miniconda.sh -b -p $HOME/miniconda -u + conda update -n base -c defaults -y conda - name: setup Path run: | echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH @@ -51,23 +52,42 @@ jobs: - name: Install gcc shell: bash run: | - sudo apt-get install build-essential + conda install -n build_binary -c conda-forge -y gxx_linux-64=11.4.0 sysroot_linux-64=2.17 + echo "[INSTALL] Setting the C/C++ compiler symlinks ..." + cc_path=$(conda run -n build_binary printenv CC) + cxx_path=$(conda run -n build_binary printenv CXX) + ln -sf "${cc_path}" "$(dirname "$cc_path")/cc" + ln -sf "${cc_path}" "$(dirname "$cc_path")/gcc" + ln -sf "${cxx_path}" "$(dirname "$cxx_path")/c++" + ln -sf "${cxx_path}" "$(dirname "$cxx_path")/g++" + + conda_prefix=$(conda run -n build_binary printenv CONDA_PREFIX) + echo "[TEST] Enumerating libstdc++.so files ..." + all_libcxx_libs=$(find "${conda_prefix}/lib" -type f -name 'libstdc++.so*' -print | sort) + for f in $all_libcxx_libs; do + echo "$f"; + objdump -TC "$f" | grep GLIBCXX_ | sed 's/.*GLIBCXX_\([.0-9]*\).*/GLIBCXX_\1/g' | sort -Vu | cat + echo "" + done + + echo "[TEST] Appending the Conda-installed libstdc++ to LD_PRELOAD ..." + conda env config vars set -n build_binary LD_PRELOAD="${all_libcxx_libs[0]}" - name: setup Path run: | echo /usr/local/bin >> $GITHUB_PATH - name: Install PyTorch shell: bash run: | - conda install -n build_binary --yes pytorch cpuonly -c pytorch-nightly + conda run -n build_binary pip install --pre pytorch --index-url https://download.pytorch.org/whl/nightly/cpu/ - name: Install fbgemm run: | - conda run -n build_binary pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu + conda run -n build_binary pip install --pre fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu/ - name: Install torchmetrics run: | - conda run -n build_binary pip install torchmetrics==1.0.3 + conda run -n build_binary pip install --pre torchmetrics - name: Install TorchRec run: | - conda run -n build_binary pip install -r requirements.txt + conda run -n build_binary pip install --pre -r requirements.txt conda run -n build_binary python setup.py bdist_wheel --python-tag=${{ matrix.python-tag }} - name: Test fbgemm_gpu and torchrec installation shell: bash @@ -78,7 +98,7 @@ jobs: python -c "import torchrec" - name: Build the docset run: | - conda run -n build_binary python -m pip install -r docs/requirements.txt + conda run -n build_binary python -m pip install --pre -r docs/requirements.txt cd ./docs conda run -n build_binary make html cd .. diff --git a/.github/workflows/pyre.yml b/.github/workflows/pyre.yml index ed25404e7..ef3726b16 100644 --- a/.github/workflows/pyre.yml +++ b/.github/workflows/pyre.yml @@ -19,9 +19,9 @@ jobs: uses: actions/checkout@v2 - name: Install dependencies run: > - pip install torch --index-url https://download.pytorch.org/whl/nightly/cpu && - pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu && - pip install -r requirements.txt && + pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu/ && + pip install --pre fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu/ && + pip install --pre -r requirements.txt && pip install pyre-check-nightly==$(cat .pyre_configuration | grep version | awk '{print $2}' | sed 's/\"//g') - name: Pyre check run: pyre check diff --git a/.github/workflows/release_build.yml b/.github/workflows/release_build.yml index 1ea837d4b..f988b3462 100644 --- a/.github/workflows/release_build.yml +++ b/.github/workflows/release_build.yml @@ -48,6 +48,7 @@ jobs: run: | wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh bash ~/miniconda.sh -b -p $HOME/miniconda -u + conda update -n base -c defaults -y conda - name: setup Path run: | echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH @@ -62,9 +63,28 @@ jobs: - name: check python version run: | conda run -n build_binary python --version - - name: Install C/C++ compilers - run: | - sudo yum install -y gcc gcc-c++ + - name: Install gcc + run: | + conda install -n build_binary -c conda-forge -y gxx_linux-64=11.4.0 sysroot_linux-64=2.17 + echo "[INSTALL] Setting the C/C++ compiler symlinks ..." + cc_path=$(conda run -n build_binary printenv CC) + cxx_path=$(conda run -n build_binary printenv CXX) + ln -sf "${cc_path}" "$(dirname "$cc_path")/cc" + ln -sf "${cc_path}" "$(dirname "$cc_path")/gcc" + ln -sf "${cxx_path}" "$(dirname "$cxx_path")/c++" + ln -sf "${cxx_path}" "$(dirname "$cxx_path")/g++" + + conda_prefix=$(conda run -n build_binary printenv CONDA_PREFIX) + echo "[TEST] Enumerating libstdc++.so files ..." + all_libcxx_libs=$(find "${conda_prefix}/lib" -type f -name 'libstdc++.so*' -print | sort) + for f in $all_libcxx_libs; do + echo "$f"; + objdump -TC "$f" | grep GLIBCXX_ | sed 's/.*GLIBCXX_\([.0-9]*\).*/GLIBCXX_\1/g' | sort -Vu | cat + echo "" + done + + echo "[TEST] Appending the Conda-installed libstdc++ to LD_PRELOAD ..." + conda env config vars set -n build_binary LD_PRELOAD="${all_libcxx_libs[0]}" - name: Install PyTorch and CUDA shell: bash run: | diff --git a/.github/workflows/unittest_ci.yml b/.github/workflows/unittest_ci.yml index 0b3fe0638..d334439bd 100644 --- a/.github/workflows/unittest_ci.yml +++ b/.github/workflows/unittest_ci.yml @@ -74,19 +74,19 @@ jobs: python --version conda run -n build_binary python --version conda run -n build_binary \ - pip install torch --index-url https://download.pytorch.org/whl/nightly/${{ matrix.cuda-tag }} + pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/${{ matrix.cuda-tag }} conda run -n build_binary \ python -c "import torch" echo "torch succeeded" conda run -n build_binary \ python -c "import torch.distributed" conda run -n build_binary \ - pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/${{ matrix.cuda-tag }} + pip install --pre fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/${{ matrix.cuda-tag }} conda run -n build_binary \ python -c "import fbgemm_gpu" echo "fbgemm_gpu succeeded" conda run -n build_binary \ - pip install -r requirements.txt + pip install --pre -r requirements.txt conda run -n build_binary \ python setup.py bdist_wheel \ --python-tag=${{ matrix.python-tag }} diff --git a/.github/workflows/unittest_ci_cpu.yml b/.github/workflows/unittest_ci_cpu.yml index 1efe64178..afc34cad8 100644 --- a/.github/workflows/unittest_ci_cpu.yml +++ b/.github/workflows/unittest_ci_cpu.yml @@ -45,20 +45,44 @@ jobs: conda info python --version conda run -n build_binary python --version + + echo "[INSTALL] Installing gcc..." + conda install -n build_binary -c conda-forge -y gxx_linux-64=11.4.0 sysroot_linux-64=2.17 + + echo "[INSTALL] Setting the C/C++ compiler symlinks ..." + cc_path=$(conda run -n build_binary printenv CC) + cxx_path=$(conda run -n build_binary printenv CXX) + ln -sf "${cc_path}" "$(dirname "$cc_path")/cc" + ln -sf "${cc_path}" "$(dirname "$cc_path")/gcc" + ln -sf "${cxx_path}" "$(dirname "$cxx_path")/c++" + ln -sf "${cxx_path}" "$(dirname "$cxx_path")/g++" + + conda_prefix=$(conda run -n build_binary printenv CONDA_PREFIX) + echo "[INSTALL] Enumerating libstdc++.so files ..." + all_libcxx_libs=$(find "${conda_prefix}/lib" -type f -name 'libstdc++.so*' -print | sort) + for f in $all_libcxx_libs; do + echo "$f"; + objdump -TC "$f" | grep GLIBCXX_ | sed 's/.*GLIBCXX_\([.0-9]*\).*/GLIBCXX_\1/g' | sort -Vu | cat + echo "" + done + + echo "[INSTALL] Appending the Conda-installed libstdc++ to LD_PRELOAD ..." + conda env config vars set -n build_binary LD_PRELOAD="${all_libcxx_libs[0]}" + conda run -n build_binary \ - pip install torch --index-url https://download.pytorch.org/whl/nightly/cpu + pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu/ conda run -n build_binary \ - python -c "import torch" + python -c "import torch; print(torch.__version__, torch.version.cuda); " echo "torch succeeded" conda run -n build_binary \ python -c "import torch.distributed" conda run -n build_binary \ - pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu + pip install --pre fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu/ conda run -n build_binary \ - python -c "import fbgemm_gpu" + python -c "import torch; import fbgemm_gpu; print(fbgemm_gpu.__version__)" echo "fbgemm_gpu succeeded" conda run -n build_binary \ - pip install -r requirements.txt + pip install --pre -r requirements.txt conda run -n build_binary \ python setup.py bdist_wheel \ --python-tag=${{ matrix.python-tag }} @@ -73,9 +97,6 @@ jobs: python -m pytest torchrec -v -s -W ignore::pytest.PytestCollectionWarning --continue-on-collection-errors \ --ignore-glob=**/test_utils/ echo "Starting C++ Tests" - conda install -n build_binary -y gxx_linux-64 - conda run -n build_binary \ - x86_64-conda-linux-gnu-g++ --version conda install -n build_binary -c anaconda redis -y conda run -n build_binary redis-server --daemonize yes mkdir cpp-build