Merge branch 'master' into ci/clang-18-jobs

.appveyor.yml

@@ -1,16 +1,14 @@
-version: 4.3.0.99.{build}
+version: 4.4.0.99.{build}
 
 image: Visual Studio 2015
 platform: x64
-configuration:  # a trick to construct a build matrix with multiple Python versions
+configuration:
   - '3.8'
 
-# only build pull requests and
-# commits to 'master' or any branch starting with 'release'
+# only build on 'master' and pull requests targeting it
 branches:
   only:
     - master
-    - /^release/
 
 environment:
   matrix:
@@ -25,12 +23,13 @@ install:
   - git submodule update --init --recursive  # get `external_libs` folder
   - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
   - set PYTHON_VERSION=%CONFIGURATION%
-  - set CONDA_ENV="test-env"
   - ps: |
+      $env:ALLOW_SKIP_ARROW_TESTS = "1"
+      $env:APPVEYOR = "true"
+      $env:CMAKE_BUILD_PARALLEL_LEVEL = 4
       $env:MINICONDA = "C:\Miniconda3-x64"
       $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH"
       $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER"
-      $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim()
 
 build: false
 

.ci/append_comment.sh

@@ -10,7 +10,7 @@
 #
 # BODY: Text that will be appended to the original comment body.
 
-set -e
+set -e -E -u -o pipefail
 
 if [ -z "$GITHUB_ACTIONS" ]; then
   echo "Must be run inside GitHub Actions CI"

.ci/check_python_dists.sh

@@ -1,7 +1,13 @@
 #!/bin/sh
 
+set -e -E -u
+
 DIST_DIR=${1}
 
+# defaults
+METHOD=${METHOD:-""}
+TASK=${TASK:-""}
+
 echo "checking Python package distributions in '${DIST_DIR}'"
 
 pip install \
@@ -20,11 +26,12 @@ fi
 PY_MINOR_VER=$(python -c "import sys; print(sys.version_info.minor)")
 if [ $PY_MINOR_VER -gt 7 ]; then
     echo "pydistcheck..."
-    pip install pydistcheck
+    pip install 'pydistcheck>=0.7.0'
     if { test "${TASK}" = "cuda" || test "${METHOD}" = "wheel"; }; then
         pydistcheck \
             --inspect \
-            --ignore 'compiled-objects-have-debug-symbols,distro-too-large-compressed' \
+            --ignore 'compiled-objects-have-debug-symbols'\
+            --ignore 'distro-too-large-compressed' \
             --max-allowed-size-uncompressed '100M' \
             --max-allowed-files 800 \
             ${DIST_DIR}/* || exit 1

.ci/conda-envs/README.md

@@ -0,0 +1,11 @@
+# conda-envs
+
+This directory contains files used to create `conda` environments for development
+and testing of LightGBM.
+
+The `.txt` files here are intended to be used with `conda create --file`.
+
+For details on that, see the `conda` docs:
+
+* `conda create` docs ([link](https://conda.io/projects/conda/en/latest/commands/create.html))
+* "Managing Environments" ([link](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html))

.ci/conda-envs/ci-core-py37.txt

@@ -0,0 +1,56 @@
+# [description]
+#
+#   Similar to ci-core.txt, but specific to Python 3.7.
+#
+#   Unlike ci-core.txt, this includes a Python version and uses
+#   `=` and `<=` pins to make solves faster and prevent against
+#   issues like https://github.com/microsoft/LightGBM/pull/6370.
+#
+# [usage]
+#
+#   conda create \
+#     --name test-env \
+#     --file ./.ci/conda-envs/ci-core-py37.txt
+#
+
+# python
+python=3.7.*
+
+# direct imports
+cffi=1.15.*
+# older versions of Dask are incompatible with pandas>=2.0, but not all conda packages' metadata accurately reflects that
+#
+# ref: https://github.com/microsoft/LightGBM/issues/6030
+dask=2022.2.*
+distributed=2022.2.*
+joblib=1.3.*
+matplotlib-base=3.5.*
+numpy=1.21.*
+pandas=1.3.*
+pyarrow=9.0.*
+# python-graphviz 0.20.2 is not compatible with Python 3.7
+# ref: https://github.com/microsoft/LightGBM/pull/6370
+python-graphviz=0.20.1
+scikit-learn=1.0.*
+scipy=1.7.*
+
+# testing-only dependencies
+cloudpickle=2.2.*
+pluggy=1.0.*
+psutil=5.9.3
+pytest=7.4.*
+
+# other recursive dependencies, just
+# pinned here to help speed up solves
+bokeh=2.4.*
+fsspec=2023.1.*
+msgpack-python=1.0.*
+pluggy=1.0.*
+pytz=2024.1
+setuptools=59.8.*
+snappy=1.1.*
+tomli=2.0.*
+tornado=6.1.*
+wheel=0.42.*
+zict=2.2.*
+zipp=3.15.*

.ci/conda-envs/ci-core-py38.txt

@@ -0,0 +1,51 @@
+# [description]
+#
+#   Similar to ci-core.txt, but specific to Python 3.8.
+#
+#   Unlike ci-core.txt, this includes a Python version and uses
+#   `=` and `<=` pins to make solves faster and prevent against
+#   issues like https://github.com/microsoft/LightGBM/pull/6370.
+#
+# [usage]
+#
+#   conda create \
+#     --name test-env \
+#     --file ./.ci/conda-envs/ci-core-py38.txt
+#
+
+# python
+python=3.8.*
+
+# direct imports
+cffi=1.15.*
+dask=2023.5.*
+distributed=2023.5.*
+joblib=1.4.*
+matplotlib-base=3.7.*
+numpy=1.24.*
+pandas=1.5.*
+pyarrow-core=16.1.*
+python-graphviz=0.20.*
+scikit-learn=1.3.*
+scipy=1.10.*
+
+# testing-only dependencies
+cloudpickle=3.0.*
+pluggy=1.5.*
+psutil=5.9.8
+pytest=8.2.*
+
+# other recursive dependencies, just
+# pinned here to help speed up solves
+bokeh=3.1.*
+fsspec=2024.5.*
+msgpack-python=1.0.*
+pluggy=1.5.*
+pytz=2024.1
+setuptools=69.5.*
+snappy=1.2.*
+tomli=2.0.*
+tornado=6.4.*
+wheel=0.43.*
+zict=3.0.*
+zipp=3.17.*

.ci/conda-envs/ci-core.txt

@@ -0,0 +1,40 @@
+# [description]
+#
+#   Core dependencies used across most LightGBM continuous integration (CI) jobs.
+#
+#   'python' constraint is intentionally omitted, so this file can be reused across
+#   Python versions.
+#
+#   These floors are not the oldest versions LightGBM supports... they're here just to make conda
+#   solves faster, and should generally be the latest versions that work for all CI jobs using this.
+#
+# [usage]
+#
+#   conda create \
+#     --name test-env \
+#     --file ./.ci/conda-envs/ci-core.txt \
+#     python=3.10
+#
+
+# direct imports
+cffi>=1.16
+dask>=2023.5.0
+joblib>=1.3.2
+matplotlib-base>=3.7.3
+numpy>=1.24.4
+pandas>2.0
+pyarrow>=6.0
+python-graphviz>=0.20.3
+scikit-learn>=1.3.2
+scipy>=1.1
+
+# testing-only dependencies
+cloudpickle>=3.0.0
+psutil>=5.9.8
+pytest>=8.1.1
+
+# other recursive dependencies, just
+# pinned here to help speed up solves
+pluggy>=1.4.0
+setuptools>=69.2
+wheel>=0.43

.ci/get_workflow_status.py

@@ -6,15 +6,12 @@
 
 TRIGGER_PHRASE: Code phrase that triggers workflow.
 """
+
 import json
 from os import environ
 from sys import argv, exit
 from time import sleep
-
-try:
-    from urllib import request
-except ImportError:
-    import urllib2 as request
+from urllib import request
 
 
 def get_runs(trigger_phrase):

.ci/lint-cpp.sh

@@ -1,4 +1,6 @@
-#!/bin/sh
+#!/bin/bash
+
+set -e -E -u -o pipefail
 
 echo "running cpplint"
 cpplint \
@@ -32,9 +34,17 @@ get_omp_pragmas_without_num_threads() {
         'pragma omp parallel' \
     | grep -v ' num_threads'
 }
+
+# 'grep' returns a non-0 exit code if 0 lines were found.
+# Turning off '-e -o pipefail' options here so that bash doesn't
+# consider this a failure and stop execution of the script.
+#
+# ref: https://www.gnu.org/software/grep/manual/html_node/Exit-Status.html
+set +e +o pipefail
 PROBLEMATIC_LINES=$(
     get_omp_pragmas_without_num_threads
 )
+set -e -o pipefail
 if test "${PROBLEMATIC_LINES}" != ""; then
     get_omp_pragmas_without_num_threads
     echo "Found '#pragma omp parallel' not using explicit num_threads() configuration. Fix those."

.ci/lint-python.sh

@@ -1,4 +1,6 @@
-#!/bin/sh
+#!/bin/bash
+
+set -e -E -u -o pipefail
 
 echo "running pre-commit checks"
 pre-commit run --all-files || exit 1

.ci/lint_r_code.R

@@ -52,6 +52,8 @@ LINTERS_TO_USE <- list(
     , "inner_combine"        = lintr::inner_combine_linter()
     , "is_numeric"           = lintr::is_numeric_linter()
     , "lengths"              = lintr::lengths_linter()
+    , "length_levels"        = lintr::length_levels_linter()
+    , "length_test"          = lintr::length_test_linter()
     , "line_length"          = lintr::line_length_linter(length = 120L)
     , "literal_coercion"     = lintr::literal_coercion_linter()
     , "matrix"               = lintr::matrix_apply_linter()
@@ -66,6 +68,7 @@ LINTERS_TO_USE <- list(
     , "redundant_equals"     = lintr::redundant_equals_linter()
     , "regex_subset"         = lintr::regex_subset_linter()
     , "routine_registration" = lintr::routine_registration_linter()
+    , "scalar_in"            = lintr::scalar_in_linter()
     , "semicolon"            = lintr::semicolon_linter()
     , "seq"                  = lintr::seq_linter()
     , "spaces_inside"        = lintr::spaces_inside_linter()

.ci/rerun_workflow.sh

@@ -12,7 +12,7 @@
 #
 # PR_BRANCH: Name of pull request's branch.
 
-set -e
+set -e -E -u -o pipefail
 
 if [ -z "$GITHUB_ACTIONS" ]; then
   echo "Must be run inside GitHub Actions CI"

.ci/set_commit_status.sh

@@ -14,7 +14,7 @@
 #
 # SHA: SHA of a commit to set a status on.
 
-set -e
+set -e -E -u -o pipefail
 
 if [ -z "$GITHUB_ACTIONS" ]; then
   echo "Must be run inside GitHub Actions CI"