From 8566ca92bd9778b9a54d1a5fe497e2ef90eccd05 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 29 Mar 2024 00:38:46 -0500 Subject: [PATCH 01/19] [ci] [python-package] add arm64 macOS wheels --- .appveyor.yml | 43 --- .ci/setup.sh | 4 + .ci/test.sh | 9 +- .github/workflows/cuda.yml | 115 -------- .github/workflows/r_package.yml | 323 ---------------------- .vsts-ci.yml | 460 +++++++++++++++++--------------- 6 files changed, 257 insertions(+), 697 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 .github/workflows/cuda.yml delete mode 100644 .github/workflows/r_package.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index bb15958136c3..000000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: 4.3.0.99.{build} - -image: Visual Studio 2015 -platform: x64 -configuration: # a trick to construct a build matrix with multiple Python versions - - '3.8' - -# only build pull requests and -# commits to 'master' or any branch starting with 'release' -branches: - only: - - master - - /^release/ - -environment: - matrix: - - COMPILER: MSVC - TASK: python - - COMPILER: MINGW - TASK: python - -clone_depth: 5 - -install: - - git submodule update --init --recursive # get `external_libs` folder - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - set CONDA_ENV="test-env" - - ps: | - $env:MINICONDA = "C:\Miniconda3-x64" - $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" - $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() - -build: false - -test_script: - - conda config --remove channels defaults - - conda config --add channels nodefaults - - conda config --add channels conda-forge - - conda config --set channel_priority strict - - conda init powershell - - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.ci/setup.sh b/.ci/setup.sh index ab47ad70745a..52dfec006ed4 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -1,5 +1,9 @@ #!/bin/bash +echo -n "--- detected arch: " +echo $(uname -m) +echo "---" + if [[ $OS_NAME == "macos" ]]; then if [[ $COMPILER == "clang" ]]; then brew install libomp diff --git a/.ci/test.sh b/.ci/test.sh index 15934aa7e9de..b725fb711470 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -163,20 +163,25 @@ if [[ $TASK == "sdist" ]]; then pytest $BUILD_DIRECTORY/tests/python_package_test || exit 1 exit 0 elif [[ $TASK == "bdist" ]]; then + ARCH=$(uname -m) if [[ $OS_NAME == "macos" ]]; then cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel || exit 1 sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1 mv \ ./dist/*.whl \ ./dist/tmp.whl || exit 1 + if [[ $ARCH == "x86_64" ]]; then + PLATFORM="macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64" + else + PLATFORM="macosx_14_0_arm64" + fi mv \ ./dist/tmp.whl \ - dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl || exit 1 + dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1 fi else - ARCH=$(uname -m) if [[ $ARCH == "x86_64" ]]; then PLATFORM="manylinux_2_28_x86_64" else diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml deleted file mode 100644 index b7f825b245b8..000000000000 --- a/.github/workflows/cuda.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: CUDA Version - -on: - push: - branches: - - master - pull_request: - branches: - - master - - release/* - -# automatically cancel in-progress builds if another commit is pushed -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - github_actions: 'true' - os_name: linux - conda_env: test-env - -jobs: - test: - name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - method: wheel - compiler: gcc - python_version: "3.11" - cuda_version: "11.8.0" - task: cuda - - method: source - compiler: gcc - python_version: "3.9" - cuda_version: "12.2.0" - task: cuda - - method: pip - compiler: clang - python_version: "3.10" - cuda_version: "11.8.0" - task: cuda - steps: - - name: Setup or update software on host machine - run: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - git \ - gnupg-agent \ - lsb-release \ - software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: Remove old folder with repository - run: sudo rm -rf $GITHUB_WORKSPACE - - name: Checkout repository - uses: actions/checkout@v1 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh <=2.7.1 is uses a too-new glibc for it. - # ref: https://github.com/microsoft/LightGBM/issues/6298 - - name: Install pandoc - uses: r-lib/actions/setup-pandoc@v2.6.0 - if: matrix.container == 'ubuntu:18.04' - - name: install tinytex - if: startsWith(matrix.os, 'windows') - uses: r-lib/actions/setup-tinytex@v2 - env: - CTAN_MIRROR: https://ctan.math.illinois.edu/systems/win32/miktex - TINYTEX_INSTALLER: TinyTeX - - name: Setup and run tests on Linux and macOS - if: matrix.os == 'macOS-latest' || matrix.os == 'ubuntu-latest' - shell: bash - run: | - export TASK="${{ matrix.task }}" - export COMPILER="${{ matrix.compiler }}" - export GITHUB_ACTIONS="true" - if [[ "${{ matrix.os }}" == "macOS-latest" ]]; then - export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then - export OS_NAME="linux" - export IN_UBUNTU_BASE_CONTAINER="true" - # the default version of cmake provided on Ubuntu 18.04 (v3.10.2), is not supported by LightGBM - # see https://github.com/microsoft/LightGBM/issues/5642 - if [[ "${{ matrix.container }}" == "ubuntu:18.04" ]]; then - export INSTALL_CMAKE_FROM_RELEASES="true" - fi - fi - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export R_VERSION="${{ matrix.r_version }}" - export R_BUILD_TYPE="${{ matrix.build_type }}" - $GITHUB_WORKSPACE/.ci/setup.sh - $GITHUB_WORKSPACE/.ci/test.sh - - name: Setup and run tests on Windows - if: startsWith(matrix.os, 'windows') - shell: pwsh -command ". {0}" - run: | - $env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE - $env:LGB_VER = (Get-Content -TotalCount 1 $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim().replace('rc', '-') - $env:TOOLCHAIN = "${{ matrix.toolchain }}" - $env:R_VERSION = "${{ matrix.r_version }}" - $env:R_BUILD_TYPE = "${{ matrix.build_type }}" - $env:COMPILER = "${{ matrix.compiler }}" - $env:GITHUB_ACTIONS = "true" - $env:TASK = "${{ matrix.task }}" - & "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1" - test-r-sanitizers: - name: r-sanitizers (ubuntu-latest, R-devel, ${{ matrix.compiler }} ASAN/UBSAN) - timeout-minutes: 60 - runs-on: ubuntu-latest - container: wch1/r-debug - strategy: - fail-fast: false - matrix: - include: - - r_customization: san - compiler: gcc - - r_customization: csan - compiler: clang - steps: - - name: Trust git cloning LightGBM - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 5 - submodules: true - - name: Install packages - shell: bash - run: | - RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }} - RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit 1 - - name: Run tests with sanitizers - shell: bash - run: | - cd R-package/tests - exit_code=0 - RDscript${{ matrix.r_customization }} testthat.R >> tests.log 2>&1 || exit_code=-1 - cat ./tests.log - exit ${exit_code} - test-r-debian-clang: - name: r-package (debian, R-devel, clang-${{ matrix.clang-version }}) - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - # list of versions tested in CRAN "Additional Checks": - # https://cran.r-project.org/web/checks/check_issue_kinds.html - clang-version: - - 16 - - 17 - runs-on: ubuntu-latest - container: rhub/debian-clang-devel - env: - DEBIAN_FRONTEND: noninteractive - steps: - - name: Install Git before checkout - shell: bash - run: | - apt-get update --allow-releaseinfo-change - apt-get install --no-install-recommends -y git - - name: Trust git cloning LightGBM - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 5 - submodules: true - - name: install clang - run: | - ./.ci/install-clang-devel.sh ${{ matrix.clang-version }} - - name: Install packages and run tests - shell: bash - run: | - export PATH=/opt/R-devel/bin/:${PATH} - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - sh build-cran-package.sh - R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit 1 - echo "" - echo "install logs:" - echo "" - cat lightgbm.Rcheck/00install.out - echo "" - if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then - echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" - exit 1 - fi - all-r-package-jobs-successful: - if: always() - runs-on: ubuntu-latest - needs: [test, test-r-sanitizers, test-r-debian-clang] - steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.vsts-ci.yml b/.vsts-ci.yml index bfb17ddeedcb..eef6fde7e2f3 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -26,193 +26,224 @@ resources: - container: rbase image: wch1/r-debug jobs: +# ########################################### +# - job: Linux +# ########################################### +# variables: +# COMPILER: gcc +# SETUP_CONDA: 'false' +# OS_NAME: 'linux' +# PRODUCES_ARTIFACTS: 'true' +# pool: sh-mariner +# container: linux-artifact-builder +# strategy: +# matrix: +# regular: +# TASK: regular +# PYTHON_VERSION: '3.9' +# sdist: +# TASK: sdist +# PYTHON_VERSION: '3.7' +# bdist: +# TASK: bdist +# PYTHON_VERSION: '3.8' +# inference: +# TASK: if-else +# mpi_source: +# TASK: mpi +# METHOD: source +# PYTHON_VERSION: '3.8' +# gpu_source: +# TASK: gpu +# METHOD: source +# swig: +# TASK: swig +# steps: +# - script: | +# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" +# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" +# echo "##vso[task.prependpath]/usr/lib64/openmpi/bin" +# echo "##vso[task.prependpath]$CONDA/bin" +# displayName: 'Set variables' +# - script: | +# echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt' +# displayName: 'Add commit hash to artifacts archive' +# - bash: $(Build.SourcesDirectory)/.ci/setup.sh +# displayName: Setup +# - bash: $(Build.SourcesDirectory)/.ci/test.sh +# displayName: Test +# - task: PublishBuildArtifacts@1 +# condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) +# inputs: +# pathtoPublish: '$(Build.ArtifactStagingDirectory)' +# artifactName: PackageAssets +# artifactType: container +# ########################################### +# - job: Linux_latest +# ########################################### +# variables: +# COMPILER: clang-17 +# DEBIAN_FRONTEND: 'noninteractive' +# IN_UBUNTU_BASE_CONTAINER: 'true' +# OS_NAME: 'linux' +# SETUP_CONDA: 'true' +# pool: sh-mariner +# container: ubuntu-latest +# strategy: +# matrix: +# regular: +# TASK: regular +# sdist: +# TASK: sdist +# bdist: +# TASK: bdist +# PYTHON_VERSION: '3.9' +# inference: +# TASK: if-else +# mpi_source: +# TASK: mpi +# METHOD: source +# mpi_pip: +# TASK: mpi +# METHOD: pip +# PYTHON_VERSION: '3.10' +# mpi_wheel: +# TASK: mpi +# METHOD: wheel +# PYTHON_VERSION: '3.8' +# gpu_source: +# TASK: gpu +# METHOD: source +# PYTHON_VERSION: '3.10' +# gpu_pip: +# TASK: gpu +# METHOD: pip +# PYTHON_VERSION: '3.9' +# gpu_wheel: +# TASK: gpu +# METHOD: wheel +# PYTHON_VERSION: '3.8' +# cpp_tests: +# TASK: cpp-tests +# METHOD: with-sanitizers +# steps: +# - script: | +# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" +# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" +# CONDA=$HOME/miniforge +# echo "##vso[task.setvariable variable=CONDA]$CONDA" +# echo "##vso[task.prependpath]$CONDA/bin" +# displayName: 'Set variables' +# # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 +# - script: | +# /tmp/docker exec -t -u 0 ci-container \ +# sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" +# displayName: 'Install sudo' +# - bash: $(Build.SourcesDirectory)/.ci/setup.sh +# displayName: Setup +# - bash: $(Build.SourcesDirectory)/.ci/test.sh +# displayName: Test +# ########################################### +# - job: QEMU_multiarch +# ########################################### +# variables: +# COMPILER: gcc +# OS_NAME: 'linux' +# PRODUCES_ARTIFACTS: 'true' +# pool: +# vmImage: ubuntu-22.04 +# timeoutInMinutes: 180 +# strategy: +# matrix: +# bdist: +# TASK: bdist +# ARCH: aarch64 +# steps: +# - script: | +# sudo apt-get update +# sudo apt-get install --no-install-recommends -y \ +# binfmt-support \ +# qemu \ +# qemu-user \ +# qemu-user-static +# displayName: 'Install QEMU' +# - script: | +# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# displayName: 'Enable Docker multi-architecture support' +# - script: | +# export ROOT_DOCKER_FOLDER=/LightGBM +# cat > docker.env < docker-script.sh < '$(Build.ArtifactStagingDirectory)/commit.txt' - displayName: 'Add commit hash to artifacts archive' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container -########################################### -- job: Linux_latest -########################################### - variables: - COMPILER: clang-17 - DEBIAN_FRONTEND: 'noninteractive' - IN_UBUNTU_BASE_CONTAINER: 'true' - OS_NAME: 'linux' - SETUP_CONDA: 'true' - pool: sh-mariner - container: ubuntu-latest + pool: + vmImage: 'macOS-14' strategy: matrix: - regular: - TASK: regular - sdist: - TASK: sdist bdist: TASK: bdist - PYTHON_VERSION: '3.9' - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - mpi_pip: - TASK: mpi - METHOD: pip - PYTHON_VERSION: '3.10' - mpi_wheel: - TASK: mpi - METHOD: wheel - PYTHON_VERSION: '3.8' - gpu_source: - TASK: gpu - METHOD: source - PYTHON_VERSION: '3.10' - gpu_pip: - TASK: gpu - METHOD: pip - PYTHON_VERSION: '3.9' - gpu_wheel: - TASK: gpu - METHOD: wheel - PYTHON_VERSION: '3.8' - cpp_tests: - TASK: cpp-tests - METHOD: with-sanitizers steps: - script: | echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - CONDA=$HOME/miniforge + CONDA=$AGENT_HOMEDIRECTORY/miniforge echo "##vso[task.setvariable variable=CONDA]$CONDA" echo "##vso[task.prependpath]$CONDA/bin" displayName: 'Set variables' - # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 - - script: | - /tmp/docker exec -t -u 0 ci-container \ - sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - displayName: 'Install sudo' - bash: $(Build.SourcesDirectory)/.ci/setup.sh displayName: Setup - bash: $(Build.SourcesDirectory)/.ci/test.sh displayName: Test -########################################### -- job: QEMU_multiarch -########################################### - variables: - COMPILER: gcc - OS_NAME: 'linux' - PRODUCES_ARTIFACTS: 'true' - pool: - vmImage: ubuntu-22.04 - timeoutInMinutes: 180 - strategy: - matrix: - bdist: - TASK: bdist - ARCH: aarch64 - steps: - - script: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - binfmt-support \ - qemu \ - qemu-user \ - qemu-user-static - displayName: 'Install QEMU' - - script: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - displayName: 'Enable Docker multi-architecture support' - - script: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < Date: Fri, 29 Mar 2024 00:57:11 -0500 Subject: [PATCH 02/19] focus on GitHub Actions --- .github/workflows/python_package.yml | 31 +- .vsts-ci.yml | 432 --------------------------- 2 files changed, 17 insertions(+), 446 deletions(-) delete mode 100644 .vsts-ci.yml diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index 27aecf82b419..8d1e1456889c 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -36,21 +36,24 @@ jobs: - os: macOS-latest task: bdist python_version: '3.7' - - os: macOS-latest - task: if-else - python_version: '3.9' - - os: macOS-latest - task: mpi - method: source + # - os: macOS-latest + # task: if-else + # python_version: '3.9' + # - os: macOS-latest + # task: mpi + # method: source + # python_version: '3.10' + # - os: macOS-latest + # task: mpi + # method: pip + # python_version: '3.11' + # - os: macOS-latest + # task: mpi + # method: wheel + # python_version: '3.8' + - os: macOS-14 + method: bdist python_version: '3.10' - - os: macOS-latest - task: mpi - method: pip - python_version: '3.11' - - os: macOS-latest - task: mpi - method: wheel - python_version: '3.8' steps: - name: Checkout repository uses: actions/checkout@v3 diff --git a/.vsts-ci.yml b/.vsts-ci.yml deleted file mode 100644 index eef6fde7e2f3..000000000000 --- a/.vsts-ci.yml +++ /dev/null @@ -1,432 +0,0 @@ -trigger: - branches: - include: - - master - tags: - include: - - v* -pr: -- master -- release/* -variables: - AZURE: 'true' - PYTHON_VERSION: '3.11' - CONDA_ENV: test-env - runCodesignValidationInjection: false - skipComponentGovernanceDetection: true - DOTNET_CLI_TELEMETRY_OPTOUT: true - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true -resources: - containers: - - container: linux-artifact-builder - image: lightgbm/vsts-agent:manylinux_2_28_x86_64 - - container: ubuntu-latest - image: 'ubuntu:22.04' - options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" - - container: rbase - image: wch1/r-debug -jobs: -# ########################################### -# - job: Linux -# ########################################### -# variables: -# COMPILER: gcc -# SETUP_CONDA: 'false' -# OS_NAME: 'linux' -# PRODUCES_ARTIFACTS: 'true' -# pool: sh-mariner -# container: linux-artifact-builder -# strategy: -# matrix: -# regular: -# TASK: regular -# PYTHON_VERSION: '3.9' -# sdist: -# TASK: sdist -# PYTHON_VERSION: '3.7' -# bdist: -# TASK: bdist -# PYTHON_VERSION: '3.8' -# inference: -# TASK: if-else -# mpi_source: -# TASK: mpi -# METHOD: source -# PYTHON_VERSION: '3.8' -# gpu_source: -# TASK: gpu -# METHOD: source -# swig: -# TASK: swig -# steps: -# - script: | -# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" -# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" -# echo "##vso[task.prependpath]/usr/lib64/openmpi/bin" -# echo "##vso[task.prependpath]$CONDA/bin" -# displayName: 'Set variables' -# - script: | -# echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt' -# displayName: 'Add commit hash to artifacts archive' -# - bash: $(Build.SourcesDirectory)/.ci/setup.sh -# displayName: Setup -# - bash: $(Build.SourcesDirectory)/.ci/test.sh -# displayName: Test -# - task: PublishBuildArtifacts@1 -# condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) -# inputs: -# pathtoPublish: '$(Build.ArtifactStagingDirectory)' -# artifactName: PackageAssets -# artifactType: container -# ########################################### -# - job: Linux_latest -# ########################################### -# variables: -# COMPILER: clang-17 -# DEBIAN_FRONTEND: 'noninteractive' -# IN_UBUNTU_BASE_CONTAINER: 'true' -# OS_NAME: 'linux' -# SETUP_CONDA: 'true' -# pool: sh-mariner -# container: ubuntu-latest -# strategy: -# matrix: -# regular: -# TASK: regular -# sdist: -# TASK: sdist -# bdist: -# TASK: bdist -# PYTHON_VERSION: '3.9' -# inference: -# TASK: if-else -# mpi_source: -# TASK: mpi -# METHOD: source -# mpi_pip: -# TASK: mpi -# METHOD: pip -# PYTHON_VERSION: '3.10' -# mpi_wheel: -# TASK: mpi -# METHOD: wheel -# PYTHON_VERSION: '3.8' -# gpu_source: -# TASK: gpu -# METHOD: source -# PYTHON_VERSION: '3.10' -# gpu_pip: -# TASK: gpu -# METHOD: pip -# PYTHON_VERSION: '3.9' -# gpu_wheel: -# TASK: gpu -# METHOD: wheel -# PYTHON_VERSION: '3.8' -# cpp_tests: -# TASK: cpp-tests -# METHOD: with-sanitizers -# steps: -# - script: | -# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" -# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" -# CONDA=$HOME/miniforge -# echo "##vso[task.setvariable variable=CONDA]$CONDA" -# echo "##vso[task.prependpath]$CONDA/bin" -# displayName: 'Set variables' -# # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 -# - script: | -# /tmp/docker exec -t -u 0 ci-container \ -# sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" -# displayName: 'Install sudo' -# - bash: $(Build.SourcesDirectory)/.ci/setup.sh -# displayName: Setup -# - bash: $(Build.SourcesDirectory)/.ci/test.sh -# displayName: Test -# ########################################### -# - job: QEMU_multiarch -# ########################################### -# variables: -# COMPILER: gcc -# OS_NAME: 'linux' -# PRODUCES_ARTIFACTS: 'true' -# pool: -# vmImage: ubuntu-22.04 -# timeoutInMinutes: 180 -# strategy: -# matrix: -# bdist: -# TASK: bdist -# ARCH: aarch64 -# steps: -# - script: | -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# binfmt-support \ -# qemu \ -# qemu-user \ -# qemu-user-static -# displayName: 'Install QEMU' -# - script: | -# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# displayName: 'Enable Docker multi-architecture support' -# - script: | -# export ROOT_DOCKER_FOLDER=/LightGBM -# cat > docker.env < docker-script.sh < Date: Fri, 29 Mar 2024 01:04:59 -0500 Subject: [PATCH 03/19] fix architecture --- .ci/setup.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index 52dfec006ed4..feb293a9f913 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -1,8 +1,8 @@ #!/bin/bash -echo -n "--- detected arch: " -echo $(uname -m) -echo "---" +ARCH=$(uname -m) +echo "--- detected arch: ${ARCH}" +echo "" if [[ $OS_NAME == "macos" ]]; then if [[ $COMPILER == "clang" ]]; then @@ -25,7 +25,7 @@ if [[ $OS_NAME == "macos" ]]; then curl \ -sL \ -o miniforge.sh \ - https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh + https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-${ARCH}.sh else # Linux if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then # fixes error "unable to initialize frontend: Dialog" @@ -137,7 +137,6 @@ else # Linux cmake fi if [[ $SETUP_CONDA != "false" ]]; then - ARCH=$(uname -m) curl \ -sL \ -o miniforge.sh \ From 6869586add2d4be62c0f34ea7a597e5f536b1322 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 29 Apr 2024 20:30:03 -0500 Subject: [PATCH 04/19] [R-package] [ci] Manually install 'Matrix' (fixes #6433) --- .ci/test_r_package.sh | 8 ++++++-- docs/FAQ.rst | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index c1f933bfc8a0..ce102fbc3d8f 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -112,13 +112,17 @@ if [[ "${R_MAJOR_VERSION}" == "3" ]]; then Rscript --vanilla -e "install.packages('https://cran.r-project.org/src/contrib/Archive/lattice/lattice_0.20-41.tar.gz', repos = NULL, lib = '${R_LIB_PATH}')" fi +# manually install {Matrix}, as {Matrix}=1.7-0 raised its R floor all the way to R 4.4.0 +# ref: https://github.com/microsoft/LightGBM/issues/6433 +Rscript --vanilla -e "install.packages('https://cran.r-project.org/src/contrib/Archive/Matrix/Matrix_1.6-5.tar.gz', repos = NULL, lib = '${R_LIB_PATH}')" + # Manually install Depends and Imports libraries + 'knitr', 'markdown', 'RhpcBLASctl', 'testthat' # to avoid a CI-time dependency on devtools (for devtools::install_deps()) # NOTE: testthat is not required when running rchk if [[ "${TASK}" == "r-rchk" ]]; then - packages="c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'R6', 'RhpcBLASctl')" + packages="c('data.table', 'jsonlite', 'knitr', 'markdown', 'R6', 'RhpcBLASctl')" else - packages="c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'R6', 'RhpcBLASctl', 'testthat')" + packages="c('data.table', 'jsonlite', 'knitr', 'markdown', 'R6', 'RhpcBLASctl', 'testthat')" fi compile_from_source="both" if [[ $OS_NAME == "macos" ]]; then diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 43999931ca07..8f0024b45730 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -236,6 +236,22 @@ As of LightGBM v4.0.0, ``setinfo()`` has been replaced by a new method, ``set_fi If you are experiencing this error when running ``lightgbm``, you may be facing the same issue reported in `#2715 `_ and later in `#2989 `_. We have seen that in some situations, using ``data.table`` 1.11.x results in this error. To get around this, you can upgrade your version of ``data.table`` to at least version 1.12.0. +4. package ‘Matrix’ is not available + +In April 2024, ``Matrix==1.7-0`` was published to CRAN. +That version had a floor of ``R (>=4.4.0)``. +``{Matrix}`` is a hard runtime dependency of ``{lightgbm}``, so on any version of R older than ``4.4.0``, running ``install.packages("lightgbm")`` results in something like the following. + +.. code-block:: text + + package ‘Matrix’ is not available for this version of R + +To fix that without upgrading to R 4.4.0 or greater, manually install an older version of ``{Matrix}``. + +.. code-block:: R + + install.packages('https://cran.r-project.org/src/contrib/Archive/Matrix/Matrix_1.6-5.tar.gz', repos = NULL) + ------ Python-package From c5595b59ad067061b86022116d19beccc5df6ad8 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 29 Apr 2024 20:55:23 -0500 Subject: [PATCH 05/19] fix R 4.3 jobs --- .ci/test_r_package.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index ce102fbc3d8f..66a3ecaa663d 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -110,6 +110,11 @@ fi # "Warning: dependency ‘lattice’ is not available" if [[ "${R_MAJOR_VERSION}" == "3" ]]; then Rscript --vanilla -e "install.packages('https://cran.r-project.org/src/contrib/Archive/lattice/lattice_0.20-41.tar.gz', repos = NULL, lib = '${R_LIB_PATH}')" +else + # {Matrix} needs {lattice}, so this needs to run before manually installing {Matrix}. + # This should be unnecessary on R >=4.4.0 + # ref: https://github.com/microsoft/LightGBM/issues/6433 + Rscript --vanilla -e "install.packages('lattice', repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}')" fi # manually install {Matrix}, as {Matrix}=1.7-0 raised its R floor all the way to R 4.4.0 From b9e905516c0850b92836da40f7618a7a77503950 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 29 Apr 2024 23:19:05 -0500 Subject: [PATCH 06/19] upload wheels to GitHub artifacts --- .github/workflows/python_package.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index 5a0e9a5e3da0..f0563aadb606 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -81,6 +81,12 @@ jobs: export PATH=${CONDA}/bin:${PATH} $GITHUB_WORKSPACE/.ci/setup.sh || exit 1 $GITHUB_WORKSPACE/.ci/test.sh || exit 1 + - name: upload wheels + if: ${{ matrix.task == 'bdist' && matrix.os == "macOS-14" }} + uses: actions/upload-artifact@v4 + with: + name: macosx-arm64-wheel + path: dist/*.whl test-oldest-versions: name: Python - oldest supported versions (ubuntu-latest) runs-on: ubuntu-latest From 5900a90bea49ac87358fabc942d6c7696ef48c59 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 30 Apr 2024 21:47:56 -0500 Subject: [PATCH 07/19] fix config --- .github/workflows/python_package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index f0563aadb606..d2cba1d133c1 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -39,7 +39,7 @@ jobs: - os: macos-13 task: if-else python_version: '3.9' - - os: macOS-14 + - os: macos-14 method: bdist python_version: '3.10' # We're currently skipping MPI jobs on macOS, see https://github.com/microsoft/LightGBM/pull/6425 @@ -82,7 +82,7 @@ jobs: $GITHUB_WORKSPACE/.ci/setup.sh || exit 1 $GITHUB_WORKSPACE/.ci/test.sh || exit 1 - name: upload wheels - if: ${{ matrix.task == 'bdist' && matrix.os == "macOS-14" }} + if: ${{ matrix.task == 'bdist' && matrix.os == 'macos-14' }} uses: actions/upload-artifact@v4 with: name: macosx-arm64-wheel From 607045074349b7f5f269564e9cec15934ad45476 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 30 Apr 2024 22:28:31 -0500 Subject: [PATCH 08/19] fix environment specifiers --- .github/workflows/python_package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index d2cba1d133c1..20ae3f322aae 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -68,10 +68,10 @@ jobs: export TASK="${{ matrix.task }}" export METHOD="${{ matrix.method }}" export PYTHON_VERSION="${{ matrix.python_version }}" - if [[ "${{ matrix.os }}" == "macos-13" ]]; then + if [[ "${{ matrix.os }}" == "macos*" ]]; then export COMPILER="gcc" export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then + elif [[ "${{ matrix.os }}" == "ubuntu*" ]]; then export COMPILER="clang" export OS_NAME="linux" fi From 9cd47d4ab702c4f46ca3257ff79c898a1d10c76d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 30 Apr 2024 22:30:33 -0500 Subject: [PATCH 09/19] use clang for release artifacts --- .github/workflows/python_package.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index 20ae3f322aae..35b7b165f710 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -68,7 +68,11 @@ jobs: export TASK="${{ matrix.task }}" export METHOD="${{ matrix.method }}" export PYTHON_VERSION="${{ matrix.python_version }}" - if [[ "${{ matrix.os }}" == "macos*" ]]; then + if [[ "${{ matrix.os }}" == "macos-14" ]]; then + # use clang when creating macOS release artifacts + export COMPILER="clang" + export OS_NAME="macos" + elif [[ "${{ matrix.os }}" == "macos*" ]]; then export COMPILER="gcc" export OS_NAME="macos" elif [[ "${{ matrix.os }}" == "ubuntu*" ]]; then From 1f9b811da25715975a354e92e15b158721be9783 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 30 Apr 2024 22:50:41 -0500 Subject: [PATCH 10/19] fix conditions --- .github/workflows/python_package.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index 35b7b165f710..a6ef5b8f3791 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -40,7 +40,8 @@ jobs: task: if-else python_version: '3.9' - os: macos-14 - method: bdist + task: bdist + method: wheel python_version: '3.10' # We're currently skipping MPI jobs on macOS, see https://github.com/microsoft/LightGBM/pull/6425 # for further details. @@ -72,10 +73,10 @@ jobs: # use clang when creating macOS release artifacts export COMPILER="clang" export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "macos*" ]]; then + elif [[ "${{ matrix.os }}" == "macos-13" ]]; then export COMPILER="gcc" export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "ubuntu*" ]]; then + elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then export COMPILER="clang" export OS_NAME="linux" fi @@ -86,7 +87,7 @@ jobs: $GITHUB_WORKSPACE/.ci/setup.sh || exit 1 $GITHUB_WORKSPACE/.ci/test.sh || exit 1 - name: upload wheels - if: ${{ matrix.task == 'bdist' && matrix.os == 'macos-14' }} + if: ${{ matrix.method == 'bdist' && matrix.os == 'macos-14' }} uses: actions/upload-artifact@v4 with: name: macosx-arm64-wheel From 11c025fd070bcb901b5764e54e074ee921b351ca Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 5 May 2024 21:35:35 -0500 Subject: [PATCH 11/19] fix artifact upload --- .github/workflows/python_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index a6ef5b8f3791..9a5c79e2aa0f 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -87,7 +87,7 @@ jobs: $GITHUB_WORKSPACE/.ci/setup.sh || exit 1 $GITHUB_WORKSPACE/.ci/test.sh || exit 1 - name: upload wheels - if: ${{ matrix.method == 'bdist' && matrix.os == 'macos-14' }} + if: ${{ matrix.method == 'wheel' && matrix.os == 'macos-14' }} uses: actions/upload-artifact@v4 with: name: macosx-arm64-wheel From 99b9a2d45cfd78f22c5c6d6e05409a503342ae79 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 7 May 2024 23:24:10 -0500 Subject: [PATCH 12/19] revert unnecessary changes --- .ci/test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.ci/test.sh b/.ci/test.sh index 0f1a28e71aaf..1590f5c64274 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -160,7 +160,6 @@ if [[ $TASK == "sdist" ]]; then pytest $BUILD_DIRECTORY/tests/python_package_test || exit 1 exit 0 elif [[ $TASK == "bdist" ]]; then - ARCH=$(uname -m) if [[ $OS_NAME == "macos" ]]; then cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel || exit 1 sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1 From 84fc7df2c93a877a8f6853fafa517ebd30de11a1 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 11 Jun 2024 00:18:06 -0500 Subject: [PATCH 13/19] add some comments --- CMakeLists.txt | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index d4eefbf3d840..b92965ab25d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,12 @@ endif() project(lightgbm LANGUAGES C CXX) +# if(APPLE) +# set(CMAKE_BUILD_RPATH_USE_ORIGIN TRUE) +# set(MACOSX_RPATH TRUE) +# set(CMAKE_INSTALL_RPATH "") +# endif() + if(BUILD_CPP_TEST) set(CMAKE_CXX_STANDARD 14) else() @@ -180,6 +186,54 @@ if(USE_OPENMP) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") endif() +# Get all properties that cmake supports +execute_process(COMMAND cmake --help-property-list OUTPUT_VARIABLE CMAKE_PROPERTY_LIST) + +# Convert command output into a CMake list +STRING(REGEX REPLACE ";" "\\\\;" CMAKE_PROPERTY_LIST "${CMAKE_PROPERTY_LIST}") +STRING(REGEX REPLACE "\n" ";" CMAKE_PROPERTY_LIST "${CMAKE_PROPERTY_LIST}") +# Fix https://stackoverflow.com/questions/32197663/how-can-i-remove-the-the-location-property-may-not-be-read-from-target-error-i +list(FILTER CMAKE_PROPERTY_LIST EXCLUDE REGEX "^LOCATION$|^LOCATION_|_LOCATION$") +# For some reason, "TYPE" shows up twice - others might too? +list(REMOVE_DUPLICATES CMAKE_PROPERTY_LIST) + +# build whitelist by filtering down from CMAKE_PROPERTY_LIST in case cmake is +# a different version, and one of our hardcoded whitelisted properties +# doesn't exist! +unset(CMAKE_WHITELISTED_PROPERTY_LIST) + +foreach(prop ${CMAKE_PROPERTY_LIST}) + if(prop MATCHES "^(INTERFACE|[_a-z]|IMPORTED_LIBNAME_|MAP_IMPORTED_CONFIG_)|^(COMPATIBLE_INTERFACE_(BOOL|NUMBER_MAX|NUMBER_MIN|STRING)|EXPORT_NAME|IMPORTED(_GLOBAL|_CONFIGURATIONS|_LIBNAME)?|NAME|TYPE|NO_SYSTEM_FROM_IMPORTED)$") + list(APPEND CMAKE_WHITELISTED_PROPERTY_LIST ${prop}) + endif() +endforeach(prop) + +function(print_target_properties tgt) + if(NOT TARGET ${tgt}) + message("There is no target named '${tgt}'") + return() + endif() + + get_target_property(target_type ${tgt} TYPE) + if(target_type STREQUAL "INTERFACE_LIBRARY") + set(PROP_LIST ${CMAKE_WHITELISTED_PROPERTY_LIST}) + else() + set(PROP_LIST ${CMAKE_PROPERTY_LIST}) + endif() + + foreach (prop ${PROP_LIST}) + string(REPLACE "" "${CMAKE_BUILD_TYPE}" prop ${prop}) + # message ("Checking ${prop}") + get_property(propval TARGET ${tgt} PROPERTY ${prop} SET) + if (propval) + get_target_property(propval ${tgt} ${prop}) + message ("${tgt} ${prop} = ${propval}") + endif() + endforeach(prop) +endfunction(print_target_properties) + +print_target_properties(OpenMP::OpenMP_CXX) + if(USE_GPU) set(BOOST_COMPUTE_HEADER_DIR ${PROJECT_SOURCE_DIR}/external_libs/compute/include) include_directories(${BOOST_COMPUTE_HEADER_DIR}) @@ -705,6 +759,48 @@ endif() if(__BUILD_FOR_PYTHON) set(CMAKE_INSTALL_PREFIX "lightgbm") + + if(APPLE AND USE_OPENMP) + get_target_property( + OpenMP_LIBRARY_LOCATION + OpenMP::OpenMP_CXX + INTERFACE_LINK_LIBRARIES + ) + endif() +endif() + + +# on macOS +if(APPLE AND USE_OPENMP) + # store path to libomp in a variable + get_target_property( + OpenMP_LIBRARY_LOCATION + OpenMP::OpenMP_CXX + INTERFACE_LINK_LIBRARIES + ) + # get just the filename of that path + # (to deal with the possibility that it might be 'libomp.dylib' or 'libgomp.dylib' or 'libiomp.dylib') + get_filename_component( + OpenMP_LIBRARY_NAME + ${OpenMP_LIBRARY_LOCATION} + NAME + ) + message(STATUS "hello: ${OpenMP_LIBRARY_LOCATION}") + message(STATUS "yo: ${OpenMP_LIBRARY_NAME}") + set_target_properties( + _lightgbm + PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "@rpath/${OpenMP_LIBRARY_NAME};${OpenMP_LIBRARY_LOCATION}" + INSTALL_RPATH_USE_LINK_PATH FALSE + ) + add_custom_command( + TARGET _lightgbm + POST_BUILD + COMMAND install_name_tool -change ${OpenMP_LIBRARY_LOCATION} "@rpath/${OpenMP_LIBRARY_NAME}" "${PROJECT_SOURCE_DIR}/lib_lightgbm.dylib" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMENT "Replacing hard-coded OpenMP install_name with '@rpath/${OpenMP_LIBRARY_NAME}'..." + ) endif() install( From cd4b7e6220f3cd7093b1dc137b6808b37426ceaa Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 11 Jun 2024 22:00:36 -0500 Subject: [PATCH 14/19] cmake changes --- .ci/setup.sh | 1 + .github/workflows/cuda.yml | 134 ++++++++++ .github/workflows/r_package.yml | 312 ++++++++++++++++++++++ .vsts-ci.yml | 449 ++++++++++++++++++++++++++++++++ CMakeLists.txt | 89 ++----- 5 files changed, 918 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/cuda.yml create mode 100644 .github/workflows/r_package.yml create mode 100644 .vsts-ci.yml diff --git a/.ci/setup.sh b/.ci/setup.sh index b29821cf4f8c..7fe54db9c2fc 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -9,6 +9,7 @@ SETUP_CONDA=${SETUP_CONDA:-"true"} ARCH=$(uname -m) + if [[ $OS_NAME == "macos" ]]; then if [[ $COMPILER == "clang" ]]; then brew install libomp diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml new file mode 100644 index 000000000000..a6005ec0344b --- /dev/null +++ b/.github/workflows/cuda.yml @@ -0,0 +1,134 @@ +name: CUDA Version + +on: + push: + branches: + - master + pull_request: + branches: + - master + # Run manually by clicking a button in the UI + workflow_dispatch: + inputs: + restart_docker: + description: 'Restart nvidia-docker on the runner before building?' + required: true + type: boolean + default: false + +# automatically cancel in-progress builds if another commit is pushed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Optionally reinstall + restart docker on the runner before building. + # This is safe as long as only 1 of these jobs runs at a time. + restart-docker: + name: set up docker + runs-on: [self-hosted, linux] + timeout-minutes: 30 + steps: + - name: Setup or update software on host machine + if: ${{ inputs.restart_docker }} + run: | + # install core packages + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + lsb-release \ + software-properties-common + # set up nvidia-docker + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y + curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - + curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + containerd.io \ + docker-ce \ + docker-ce-cli \ + nvidia-docker2 + sudo chmod a+rw /var/run/docker.sock + sudo systemctl restart docker + - name: mark job successful + run: | + exit 0 + test: + name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (${{ matrix.linux_version }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) + runs-on: [self-hosted, linux] + needs: [restart-docker] + container: + image: nvcr.io/nvidia/cuda:${{ matrix.cuda_version }}-devel-${{ matrix.linux_version }} + env: + CMAKE_BUILD_PARALLEL_LEVEL: 4 + COMPILER: ${{ matrix.compiler }} + CONDA: /tmp/miniforge + DEBIAN_FRONTEND: noninteractive + METHOD: ${{ matrix.method }} + OS_NAME: linux + PYTHON_VERSION: ${{ matrix.python_version }} + TASK: ${{ matrix.task }} + options: --gpus all + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - method: wheel + compiler: gcc + python_version: "3.10" + cuda_version: "11.8.0" + linux_version: "ubuntu20.04" + task: cuda + - method: source + compiler: gcc + python_version: "3.12" + cuda_version: "12.2.0" + linux_version: "ubuntu22.04" + task: cuda + - method: pip + compiler: clang + python_version: "3.11" + cuda_version: "11.8.0" + linux_version: "ubuntu20.04" + task: cuda + steps: + - name: Install latest git + run: | + apt-get update + apt-get install --no-install-recommends -y \ + ca-certificates \ + software-properties-common + add-apt-repository ppa:git-core/ppa -y + apt-get update + apt-get install --no-install-recommends -y \ + git + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: true + - name: Setup and run tests + run: | + export BUILD_DIRECTORY="$GITHUB_WORKSPACE" + export PATH=$CONDA/bin:$PATH + + # check GPU usage + nvidia-smi + + # build and test + $GITHUB_WORKSPACE/.ci/setup.sh + $GITHUB_WORKSPACE/.ci/test.sh + all-cuda-jobs-successful: + if: always() + runs-on: ubuntu-latest + needs: [test] + steps: + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml new file mode 100644 index 000000000000..c72ec7fae112 --- /dev/null +++ b/.github/workflows/r_package.yml @@ -0,0 +1,312 @@ +name: R-package + +on: + push: + branches: + - master + pull_request: + branches: + - master + +# automatically cancel in-progress builds if another commit is pushed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CMAKE_BUILD_PARALLEL_LEVEL: 4 + # hack to get around this: + # https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html + _R_CHECK_SYSTEM_CLOCK_: 0 + # ignore R CMD CHECK NOTE checking how long it has + # been since the last submission + _R_CHECK_CRAN_INCOMING_REMOTE_: 0 + # CRAN ignores the "installed size is too large" NOTE, + # so our CI can too. Setting to a large value here just + # to catch extreme problems + _R_CHECK_PKG_SIZES_THRESHOLD_: 100 + +jobs: + test: + name: ${{ matrix.task }} (${{ matrix.os }}, ${{ matrix.compiler }}, R ${{ matrix.r_version }}, ${{ matrix.build_type }}) + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + ################ + # CMake builds # + ################ + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 3.6 + build_type: cmake + container: 'ubuntu:18.04' + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cmake + container: 'ubuntu:22.04' + - os: ubuntu-latest + task: r-package + compiler: clang + r_version: 3.6 + build_type: cmake + container: 'ubuntu:18.04' + - os: ubuntu-latest + task: r-package + compiler: clang + r_version: 4.3 + build_type: cmake + container: 'ubuntu:22.04' + - os: macos-13 + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cmake + container: null + - os: macos-13 + task: r-package + compiler: clang + r_version: 4.3 + build_type: cmake + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cmake + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.3 + build_type: cmake + container: null + # Visual Studio 2019 + - os: windows-2019 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 3.6 + build_type: cmake + container: null + # Visual Studio 2022 + - os: windows-2022 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 4.3 + build_type: cmake + container: null + ############### + # CRAN builds # + ############### + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cran + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.3 + build_type: cran + container: null + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cran + container: 'ubuntu:22.04' + - os: macos-13 + task: r-package + compiler: clang + r_version: 4.3 + build_type: cran + container: null + steps: + - name: Prevent conversion of line endings on Windows + if: startsWith(matrix.os, 'windows') + shell: pwsh + run: git config --global core.autocrlf false + - name: Install packages used by third-party actions + if: startsWith(matrix.os, 'ubuntu') + shell: bash + run: | + apt-get update -y + apt-get install --no-install-recommends -y \ + ca-certificates \ + dirmngr \ + gpg \ + gpg-agent \ + software-properties-common \ + sudo + # install newest version of git + # ref: + # - https://unix.stackexchange.com/a/170831/550004 + # - https://git-scm.com/download/linux + add-apt-repository ppa:git-core/ppa -y + apt-get update -y + apt-get install --no-install-recommends -y \ + git + - name: Trust git cloning LightGBM + if: startsWith(matrix.os, 'ubuntu') + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 5 + submodules: true + - name: Install pandoc + uses: r-lib/actions/setup-pandoc@v2 + if: matrix.container != 'ubuntu:18.04' + # R 3.6 binary isn't easily available on buntu 18.04, + # but setup-pandoc>=2.7.1 is uses a too-new glibc for it. + # ref: https://github.com/microsoft/LightGBM/issues/6298 + - name: Install pandoc + uses: r-lib/actions/setup-pandoc@v2.6.0 + if: matrix.container == 'ubuntu:18.04' + - name: install tinytex + if: startsWith(matrix.os, 'windows') + uses: r-lib/actions/setup-tinytex@v2 + env: + CTAN_MIRROR: https://ctan.math.illinois.edu/systems/win32/miktex + TINYTEX_INSTALLER: TinyTeX + - name: Setup and run tests on Linux and macOS + if: matrix.os == 'macos-13' || matrix.os == 'ubuntu-latest' + shell: bash + run: | + export TASK="${{ matrix.task }}" + export COMPILER="${{ matrix.compiler }}" + if [[ "${{ matrix.os }}" == "macos-13" ]]; then + export OS_NAME="macos" + elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then + export OS_NAME="linux" + export IN_UBUNTU_BASE_CONTAINER="true" + # the default version of cmake provided on Ubuntu 18.04 (v3.10.2), is not supported by LightGBM + # see https://github.com/microsoft/LightGBM/issues/5642 + if [[ "${{ matrix.container }}" == "ubuntu:18.04" ]]; then + export INSTALL_CMAKE_FROM_RELEASES="true" + fi + fi + export BUILD_DIRECTORY="$GITHUB_WORKSPACE" + export R_VERSION="${{ matrix.r_version }}" + export R_BUILD_TYPE="${{ matrix.build_type }}" + $GITHUB_WORKSPACE/.ci/setup.sh + $GITHUB_WORKSPACE/.ci/test.sh + - name: Setup and run tests on Windows + if: startsWith(matrix.os, 'windows') + shell: pwsh -command ". {0}" + run: | + $env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE + $env:LGB_VER = (Get-Content -TotalCount 1 $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim().replace('rc', '-') + $env:TOOLCHAIN = "${{ matrix.toolchain }}" + $env:R_VERSION = "${{ matrix.r_version }}" + $env:R_BUILD_TYPE = "${{ matrix.build_type }}" + $env:COMPILER = "${{ matrix.compiler }}" + $env:TASK = "${{ matrix.task }}" + & "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1" + test-r-sanitizers: + name: r-sanitizers (ubuntu-latest, R-devel, ${{ matrix.compiler }} ASAN/UBSAN) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: wch1/r-debug + strategy: + fail-fast: false + matrix: + include: + - r_customization: san + compiler: gcc + - r_customization: csan + compiler: clang + steps: + - name: Trust git cloning LightGBM + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: true + - name: Install packages + shell: bash + run: | + RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }} + RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit 1 + - name: Run tests with sanitizers + shell: bash + run: | + cd R-package/tests + exit_code=0 + RDscript${{ matrix.r_customization }} testthat.R >> tests.log 2>&1 || exit_code=-1 + cat ./tests.log + exit ${exit_code} + test-r-debian-clang: + name: r-package (debian, R-devel, clang-${{ matrix.clang-version }}) + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # list of versions tested in CRAN "Additional Checks": + # https://cran.r-project.org/web/checks/check_issue_kinds.html + clang-version: + - 16 + - 17 + runs-on: ubuntu-latest + container: rhub/debian-clang-devel + env: + DEBIAN_FRONTEND: noninteractive + steps: + - name: Install Git before checkout + shell: bash + run: | + apt-get update --allow-releaseinfo-change + apt-get install --no-install-recommends -y git + - name: Trust git cloning LightGBM + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: true + - name: install clang + run: | + ./.ci/install-clang-devel.sh ${{ matrix.clang-version }} + - name: Install packages and run tests + shell: bash + run: | + export PATH=/opt/R-devel/bin/:${PATH} + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + sh build-cran-package.sh + R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit 1 + echo "" + echo "install logs:" + echo "" + cat lightgbm.Rcheck/00install.out + echo "" + if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then + echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" + exit 1 + fi + all-r-package-jobs-successful: + if: always() + runs-on: ubuntu-latest + needs: [test, test-r-sanitizers, test-r-debian-clang] + steps: + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.vsts-ci.yml b/.vsts-ci.yml new file mode 100644 index 000000000000..9b440cb9fac1 --- /dev/null +++ b/.vsts-ci.yml @@ -0,0 +1,449 @@ +trigger: + branches: + include: + - master + tags: + include: + - v* +pr: +- master +variables: + AZURE: 'true' + CMAKE_BUILD_PARALLEL_LEVEL: 4 + PYTHON_VERSION: '3.12' + runCodesignValidationInjection: false + skipComponentGovernanceDetection: true + DOTNET_CLI_TELEMETRY_OPTOUT: true + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true +resources: + # The __work/ directory, where Azure DevOps writes the source files, needs to be read-write because + # LightGBM's CI jobs write files in the source directory. + # + # For all the containers included here, all other directories that Azure mounts in are mounted as read-only + # to minimize the risk of side effects from one run affecting future runs. + # ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container + containers: + - container: linux-artifact-builder + image: lightgbm/vsts-agent:manylinux_2_28_x86_64 + mountReadOnly: + work: false + externals: true + tools: true + tasks: true + - container: ubuntu-latest + image: 'ubuntu:22.04' + options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" + mountReadOnly: + work: false + externals: true + tools: true + tasks: true + - container: rbase + image: wch1/r-debug + mountReadOnly: + work: false + externals: true + tools: true + tasks: true +jobs: +########################################### +- job: Linux +########################################### + variables: + COMPILER: gcc + SETUP_CONDA: 'false' + OS_NAME: 'linux' + PRODUCES_ARTIFACTS: 'true' + pool: mariner-20240410-0 + container: linux-artifact-builder + strategy: + matrix: + regular: + TASK: regular + PYTHON_VERSION: '3.10' + sdist: + TASK: sdist + PYTHON_VERSION: '3.8' + bdist: + TASK: bdist + PYTHON_VERSION: '3.9' + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + PYTHON_VERSION: '3.9' + gpu_source: + TASK: gpu + METHOD: source + swig: + TASK: swig + steps: + - script: | + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + echo "##vso[task.prependpath]/usr/lib64/openmpi/bin" + echo "##vso[task.prependpath]$CONDA/bin" + displayName: 'Set variables' + - script: | + git clean -d -f -x + displayName: 'Clean source directory' + - script: | + echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt' + displayName: 'Add commit hash to artifacts archive' + - task: Bash@3 + displayName: Setup + inputs: + filePath: $(Build.SourcesDirectory)/.ci/setup.sh + targetType: filePath + - task: Bash@3 + displayName: Test + inputs: + filePath: $(Build.SourcesDirectory)/.ci/test.sh + targetType: filePath + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container +########################################### +- job: Linux_latest +########################################### + variables: + COMPILER: clang-17 + DEBIAN_FRONTEND: 'noninteractive' + IN_UBUNTU_BASE_CONTAINER: 'true' + OS_NAME: 'linux' + SETUP_CONDA: 'true' + pool: mariner-20240410-0 + container: ubuntu-latest + strategy: + matrix: + regular: + TASK: regular + sdist: + TASK: sdist + bdist: + TASK: bdist + PYTHON_VERSION: '3.10' + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + mpi_pip: + TASK: mpi + METHOD: pip + PYTHON_VERSION: '3.11' + mpi_wheel: + TASK: mpi + METHOD: wheel + PYTHON_VERSION: '3.9' + gpu_source: + TASK: gpu + METHOD: source + PYTHON_VERSION: '3.11' + gpu_pip: + TASK: gpu + METHOD: pip + PYTHON_VERSION: '3.10' + gpu_wheel: + TASK: gpu + METHOD: wheel + PYTHON_VERSION: '3.9' + cpp_tests: + TASK: cpp-tests + METHOD: with-sanitizers + steps: + - script: | + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + CONDA=$HOME/miniforge + echo "##vso[task.setvariable variable=CONDA]$CONDA" + echo "##vso[task.prependpath]$CONDA/bin" + displayName: 'Set variables' + # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 + - script: | + /tmp/docker exec -t -u 0 ci-container \ + sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" + displayName: 'Install sudo' + - script: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends git + git clean -d -f -x + displayName: 'Clean source directory' + - task: Bash@3 + displayName: Setup + inputs: + filePath: $(Build.SourcesDirectory)/.ci/setup.sh + targetType: 'filePath' + - task: Bash@3 + displayName: Test + inputs: + filePath: $(Build.SourcesDirectory)/.ci/test.sh + targetType: 'filePath' +########################################### +- job: QEMU_multiarch +########################################### + variables: + BUILD_DIRECTORY: /LightGBM + COMPILER: gcc + PRODUCES_ARTIFACTS: 'true' + pool: + vmImage: ubuntu-22.04 + timeoutInMinutes: 180 + strategy: + matrix: + bdist: + TASK: bdist + ARCH: aarch64 + steps: + - script: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + binfmt-support \ + qemu \ + qemu-user \ + qemu-user-static + displayName: 'Install QEMU' + - script: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + displayName: 'Enable Docker multi-architecture support' + - script: | + git clean -d -f -x + displayName: 'Clean source directory' + - script: | + cat > docker-script.sh <" "${CMAKE_BUILD_TYPE}" prop ${prop}) - # message ("Checking ${prop}") - get_property(propval TARGET ${tgt} PROPERTY ${prop} SET) - if (propval) - get_target_property(propval ${tgt} ${prop}) - message ("${tgt} ${prop} = ${propval}") - endif() - endforeach(prop) -endfunction(print_target_properties) - -print_target_properties(OpenMP::OpenMP_CXX) - if(USE_GPU) set(BOOST_COMPUTE_HEADER_DIR ${PROJECT_SOURCE_DIR}/external_libs/compute/include) include_directories(${BOOST_COMPUTE_HEADER_DIR}) @@ -759,18 +705,24 @@ endif() if(__BUILD_FOR_PYTHON) set(CMAKE_INSTALL_PREFIX "lightgbm") - - if(APPLE AND USE_OPENMP) - get_target_property( - OpenMP_LIBRARY_LOCATION - OpenMP::OpenMP_CXX - INTERFACE_LINK_LIBRARIES - ) - endif() endif() - -# on macOS +# The macOS linker puts an absolute path to linked libraries in lib_lightgb.dylib. +# That can cause problems at runtime... for example, using the Python package in a +# conda environment (but not compiling with conda's compilers), can result in segfaults +# because lib_lightgbm uses a hard-coded llvm-openmp frmo Homebrew like +# '/opt/homebrew/Cellar/libomp/17.0.6/lib/libomp.dylib' and that's loaded alongside +# the libomp.dylib vendored with sklearn, or the one from the llvm-openmp conda package. +# +# This block modifies lib_lightgbm.dylib, such that it has the following properties for OpenMP +# +# - install name: '@rpath/libomp.dylib' +# +# RPATHs (in this order): +# +# - '@rpath/libomp.dylib' (look for libomp.dylib right next to where lib_lightgbm.dylib is) +# - '/opt/homebrew/Cellar/libomp/{version}/lib/libomp.dylib' (libomp.dylib from Homebrew) +# if(APPLE AND USE_OPENMP) # store path to libomp in a variable get_target_property( @@ -785,15 +737,18 @@ if(APPLE AND USE_OPENMP) ${OpenMP_LIBRARY_LOCATION} NAME ) - message(STATUS "hello: ${OpenMP_LIBRARY_LOCATION}") - message(STATUS "yo: ${OpenMP_LIBRARY_NAME}") + # add RPATH entries too ensure the loader looks in the following, in the following order: + # + # - @rpath/{openmp}.dylib (whatever runpaths the linker is configured to look in) + # - set_target_properties( _lightgbm PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH "@rpath/${OpenMP_LIBRARY_NAME};${OpenMP_LIBRARY_LOCATION}" + INSTALL_RPATH "@loader_path/;${OpenMP_LIBRARY_LOCATION}" INSTALL_RPATH_USE_LINK_PATH FALSE ) + # override the absolute path to libomp.dylib with a relative one using @rpath add_custom_command( TARGET _lightgbm POST_BUILD From d7e3cb56a4e80467b60df008cff6301c01c50818 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 11 Jun 2024 22:52:57 -0500 Subject: [PATCH 15/19] clean up --- CMakeLists.txt | 56 +++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d39b12ba4c7..8e215eb5a729 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -708,23 +708,13 @@ if(__BUILD_FOR_PYTHON) endif() # The macOS linker puts an absolute path to linked libraries in lib_lightgb.dylib. -# That can cause problems at runtime... for example, using the Python package in a -# conda environment (but not compiling with conda's compilers), can result in segfaults -# because lib_lightgbm uses a hard-coded llvm-openmp frmo Homebrew like -# '/opt/homebrew/Cellar/libomp/17.0.6/lib/libomp.dylib' and that's loaded alongside -# the libomp.dylib vendored with sklearn, or the one from the llvm-openmp conda package. +# This block overrides that information for LightGBM's OpenMP dependency, to allow +# finding that library in more places. # -# This block modifies lib_lightgbm.dylib, such that it has the following properties for OpenMP -# -# - install name: '@rpath/libomp.dylib' -# -# RPATHs (in this order): -# -# - '@rpath/libomp.dylib' (look for libomp.dylib right next to where lib_lightgbm.dylib is) -# - '/opt/homebrew/Cellar/libomp/{version}/lib/libomp.dylib' (libomp.dylib from Homebrew) +# This reduces the risk of runtime issues resulting from multiple libomp.dylib being loaded. # if(APPLE AND USE_OPENMP) - # store path to libomp in a variable + # store path to libomp found at build time in a variable get_target_property( OpenMP_LIBRARY_LOCATION OpenMP::OpenMP_CXX @@ -737,25 +727,39 @@ if(APPLE AND USE_OPENMP) ${OpenMP_LIBRARY_LOCATION} NAME ) - # add RPATH entries too ensure the loader looks in the following, in the following order: - # - # - @rpath/{openmp}.dylib (whatever runpaths the linker is configured to look in) - # - - set_target_properties( - _lightgbm - PROPERTIES - BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH "@loader_path/;${OpenMP_LIBRARY_LOCATION}" - INSTALL_RPATH_USE_LINK_PATH FALSE + # get directory of that path + get_filename_component( + OpenMP_LIBRARY_DIR + ${OpenMP_LIBRARY_LOCATION} + DIRECTORY ) - # override the absolute path to libomp.dylib with a relative one using @rpath + # Override the absolute path to OpenMP with a relative one using @rpath. + # + # This also ensures that if a libomp.dylib has already been loaded, it'll just use that. add_custom_command( TARGET _lightgbm POST_BUILD - COMMAND install_name_tool -change ${OpenMP_LIBRARY_LOCATION} "@rpath/${OpenMP_LIBRARY_NAME}" "${PROJECT_SOURCE_DIR}/lib_lightgbm.dylib" + COMMAND + install_name_tool + -change + ${OpenMP_LIBRARY_LOCATION} + "@rpath/${OpenMP_LIBRARY_NAME}" + "${PROJECT_SOURCE_DIR}/lib_lightgbm.dylib" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Replacing hard-coded OpenMP install_name with '@rpath/${OpenMP_LIBRARY_NAME}'..." ) + # add RPATH entries to ensure the loader looks in the following, in the following order: + # + # - /opt/homebrew/opt/libomp/lib (where 'brew install' / 'brew link' puts libomp.dylib) + # - ${OpenMP_LIBRARY_DIR} (wherever find_package(OpenMP) found OpenMP at build time) + # + set_target_properties( + _lightgbm + PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "/opt/homebrew/opt/libomp/lib;${OpenMP_LIBRARY_DIR}" + INSTALL_RPATH_USE_LINK_PATH FALSE + ) endif() install( From 1366c48576a82a806fadd8fb472420d63a42257d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 11 Jun 2024 23:01:37 -0500 Subject: [PATCH 16/19] appveyor --- .appveyor.yml | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000000..3c12ebaa36f8 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,42 @@ +version: 4.3.0.99.{build} + +image: Visual Studio 2015 +platform: x64 +configuration: + - '3.8' + +# only build on 'master' and pull requests targeting it +branches: + only: + - master + +environment: + matrix: + - COMPILER: MSVC + TASK: python + - COMPILER: MINGW + TASK: python + +clone_depth: 5 + +install: + - git submodule update --init --recursive # get `external_libs` folder + - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + - set PYTHON_VERSION=%CONFIGURATION% + - ps: | + $env:ALLOW_SKIP_ARROW_TESTS = "1" + $env:APPVEYOR = "true" + $env:CMAKE_BUILD_PARALLEL_LEVEL = 4 + $env:MINICONDA = "C:\Miniconda3-x64" + $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" + +build: false + +test_script: + - conda config --remove channels defaults + - conda config --add channels nodefaults + - conda config --add channels conda-forge + - conda config --set channel_priority strict + - conda init powershell + - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 From ca65937496184afd600ac055b7149ca590166596 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 13 Jun 2024 15:07:51 -0500 Subject: [PATCH 17/19] fix R CMake installation, try to re-enable pre-commit --- .ci/lint-python.sh | 8 +++----- CMakeLists.txt | 8 +++++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.ci/lint-python.sh b/.ci/lint-python.sh index edab8993a799..e1e9e306c883 100755 --- a/.ci/lint-python.sh +++ b/.ci/lint-python.sh @@ -2,11 +2,9 @@ set -e -E -u -o pipefail -# this can be re-enabled when this is fixed: -# https://github.com/tox-dev/filelock/issues/337 -# echo "running pre-commit checks" -# pre-commit run --all-files || exit 1 -# echo "done running pre-commit checks" +echo "running pre-commit checks" +pre-commit run --all-files || exit 1 +echo "done running pre-commit checks" echo "running mypy" mypy \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e215eb5a729..a26acd48e651 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -733,6 +733,12 @@ if(APPLE AND USE_OPENMP) ${OpenMP_LIBRARY_LOCATION} DIRECTORY ) + # get exact name of the library in a variable + get_target_property( + __LIB_LIGHTGBM_OUTPUT_NAME + _lightgbm + OUTPUT_NAME + ) # Override the absolute path to OpenMP with a relative one using @rpath. # # This also ensures that if a libomp.dylib has already been loaded, it'll just use that. @@ -744,7 +750,7 @@ if(APPLE AND USE_OPENMP) -change ${OpenMP_LIBRARY_LOCATION} "@rpath/${OpenMP_LIBRARY_NAME}" - "${PROJECT_SOURCE_DIR}/lib_lightgbm.dylib" + "${PROJECT_SOURCE_DIR}/${__LIB_LIGHTGBM_OUTPUT_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX_CXX}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Replacing hard-coded OpenMP install_name with '@rpath/${OpenMP_LIBRARY_NAME}'..." ) From 2184d010b96bc4ba0bc5f0f0c0364e396a88cc97 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 13 Jun 2024 16:13:19 -0500 Subject: [PATCH 18/19] more fixes to support both R and Python builds --- CMakeLists.txt | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a26acd48e651..cf46e414f368 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -739,6 +739,22 @@ if(APPLE AND USE_OPENMP) _lightgbm OUTPUT_NAME ) + if(NOT __LIB_LIGHTGBM_OUTPUT_NAME) + set(__LIB_LIGHTGBM_OUTPUT_NAME "lib_lightgbm") + endif() + + if(CMAKE_SHARED_LIBRARY_SUFFIX_CXX) + set( + __LIB_LIGHTGBM_FILENAME "${__LIB_LIGHTGBM_OUTPUT_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX_CXX}" + CACHE INTERNAL "lightgbm shared library filename" + ) + else() + set( + __LIB_LIGHTGBM_FILENAME "${__LIB_LIGHTGBM_OUTPUT_NAME}.dylib" + CACHE INTERNAL "lightgbm shared library filename" + ) + endif() + # Override the absolute path to OpenMP with a relative one using @rpath. # # This also ensures that if a libomp.dylib has already been loaded, it'll just use that. @@ -750,9 +766,9 @@ if(APPLE AND USE_OPENMP) -change ${OpenMP_LIBRARY_LOCATION} "@rpath/${OpenMP_LIBRARY_NAME}" - "${PROJECT_SOURCE_DIR}/${__LIB_LIGHTGBM_OUTPUT_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX_CXX}" + "${__LIB_LIGHTGBM_FILENAME}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMENT "Replacing hard-coded OpenMP install_name with '@rpath/${OpenMP_LIBRARY_NAME}'..." + COMMENT "Replacing hard-coded OpenMP install_name in '${__LIB_LIGHTGBM_FILENAME}' with '@rpath/${OpenMP_LIBRARY_NAME}'..." ) # add RPATH entries to ensure the loader looks in the following, in the following order: # From 1ca306d7cacfec6b540f04bd00f5075bfb02298e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 13 Jun 2024 16:28:17 -0500 Subject: [PATCH 19/19] cmakelint --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cf46e414f368..3492289be078 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -768,7 +768,7 @@ if(APPLE AND USE_OPENMP) "@rpath/${OpenMP_LIBRARY_NAME}" "${__LIB_LIGHTGBM_FILENAME}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMENT "Replacing hard-coded OpenMP install_name in '${__LIB_LIGHTGBM_FILENAME}' with '@rpath/${OpenMP_LIBRARY_NAME}'..." + COMMENT "Replacing hard-coded OpenMP install_name with '@rpath/${OpenMP_LIBRARY_NAME}'..." ) # add RPATH entries to ensure the loader looks in the following, in the following order: #