Merge pull request #349 from IntelPython/feature/organize_package_meta

Organize package meta
IntelPython · May 2, 2024 · 71c1061 · 71c1061
2 parents 4501644 + 6542b52
commit 71c1061
Show file tree

Hide file tree

Showing 6 changed files with 123 additions and 85 deletions.
diff --git a/.github/workflows/build_and_run.yml b/.github/workflows/build_and_run.yml
@@ -77,7 +77,6 @@ jobs:
         uses: conda-incubator/setup-miniconda@v2
         with:
           python-version: ${{ matrix.python }}
-          miniforge-variant: Mambaforge
           miniforge-version: latest
           activate-environment: "build"
           channel-priority: "disabled"

diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml
@@ -55,15 +55,13 @@ jobs:
           fetch-depth: 0
 
       - name: Setup miniconda
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
-          python-version: ${{ matrix.python }}
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
-          activate-environment: "build"
+          auto-activate-base: true
+          activate-environment: ""
           channels: ${{ env.CHANNELS }}
           channel-priority: "disabled"
-          run-post: false
+          miniforge-version: latest
 
       - name: Disable defautls
         run: conda config --remove channels defaults
@@ -74,18 +72,21 @@ jobs:
           echo "CONDA_BLD=$CONDA_PREFIX/conda-bld/${{ runner.os == 'Linux' && 'linux' || 'win' }}-64/" | tr "\\" '/' >> $GITHUB_ENV
           echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE${{ runner.os == 'Linux' && '/' || '\\' }}" >> $GITHUB_ENV
 
-      # boa is an extention to conda so we can use mamba resolver in conda build
       - name: Install conda-build
-        run: mamba install boa
+        run: conda install conda-build
 
       - name: Configure MSBuild
         if: runner.os == 'Windows'
         uses: microsoft/[email protected]
         with:
           vs-version: '14.35'
 
+      - name: Show conda-rc
+        shell: bash -el {0}
+        run: cat ~/.condarc
+
       - name: Build conda package
-        run: conda mambabuild --no-test --python ${{ matrix.python }} conda-recipe
+        run: conda build --python ${{ matrix.python }} conda-recipe
 
       - name: Upload artifact
         uses: actions/[email protected]
@@ -122,15 +123,13 @@ jobs:
 
     steps:
       - name: Setup miniconda
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python }}
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
           activate-environment: "build"
           channels: ${{ env.CHANNELS }}
           channel-priority: "disabled"
-          run-post: false
+          miniforge-version: latest
 
       - name: Disable defautls
         run: conda config --remove channels defaults
@@ -159,7 +158,7 @@ jobs:
 
       # Needed to be able to run conda index
       - name: Install conda-build
-        run: mamba install conda-build conda-index
+        run: conda install conda-build conda-index
 
       - name: Create conda channel
         run: python -m conda_index ${{ env.CHANNEL_PATH }}
@@ -170,7 +169,7 @@ jobs:
           cat ${{ env.VER_JSON_PATH }}
 
       - name: Install dpbench
-        run: mamba install ${{ env.PACKAGE_NAME }}=${{ env.PACKAGE_VERSION }} pytest intel::intel-opencl-rt python=${{ matrix.python }} -c ${{ env.CHANNEL_PATH }}
+        run: conda install ${{ env.PACKAGE_NAME }}=${{ env.PACKAGE_VERSION }} pytest intel::intel-opencl-rt python=${{ matrix.python }} -c ${{ env.CHANNEL_PATH }}
 
       - name: Setup OpenCL CPU device
         if: runner.os == 'Windows'

diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat
@@ -17,8 +17,8 @@ set "DPBENCH_SYCL=1"
 set "CMAKE_GENERATOR=Ninja"
 set "CC=icx"
 set "CXX=icx"
-
-"%PYTHON%" setup.py clean --all
+:: Make CMake verbose
+set "VERBOSE=1"
 
 FOR %%V IN (14.0.0 14 15.0.0 15 16.0.0 16 17.0.0 17) DO @(
   REM set DIR_HINT if directory exists
@@ -41,18 +41,41 @@ if EXIST "%PLATFORM_DIR%" (
   if errorlevel 1 exit 1
 )
 
-@REM TODO: switch to pip build. Currently results in broken binary
-@REM %PYTHON% -m pip install --no-index --no-deps --no-build-isolation . -v
+:: -wnx flags mean: --wheel --no-isolation --skip-dependency-check
+%PYTHON% -m build -w -n -x
+if %ERRORLEVEL% neq 0 exit 1
+
+:: `pip install dist\dpbench*.whl` does not work on windows,
+:: so use a loop; there's only one wheel in dist/ anyway
+for /f %%f in ('dir /b /S .\dist') do (
+    %PYTHON% -m wheel tags --remove --build %GIT_DESCRIBE_NUMBER% %%f
+    if %ERRORLEVEL% neq 0 exit 1
+)
+
+:: wheel file was renamed
+for /f %%f in ('dir /b /S .\dist') do (
+    %PYTHON% -m pip install %%f ^
+      --no-build-isolation ^
+      --no-deps ^
+      --only-binary :all: ^
+      --no-index ^
+      --prefix %PREFIX% ^
+      -vv
+    if %ERRORLEVEL% neq 0 exit 1
+)
+
+:: Must be consistent with pyproject.toml project.scritps. Currently pip does
+:: not allow to ignore scripts installation, so we have to remove them manually.
+:: https://github.com/pypa/pip/issues/3980
+:: We have to let conda-build manage it for use in order to set proper python
+:: path.
+:: https://docs.conda.io/projects/conda-build/en/stable/resources/define-metadata.html#python-entry-points
+rm %PREFIX%\Scripts\dpbench.exe
+
+:: Copy wheel package
 if NOT "%WHEELS_OUTPUT_FOLDER%"=="" (
-    rem Install and assemble wheel package from the build bits
-    "%PYTHON%" setup.py install --single-version-externally-managed --record=record.txt bdist_wheel --build-number %GIT_DESCRIBE_NUMBER%
-    if errorlevel 1 exit 1
     copy dist\dpbench*.whl %WHEELS_OUTPUT_FOLDER%
     if errorlevel 1 exit 1
-) ELSE (
-    rem Only install
-    "%PYTHON%" setup.py install --single-version-externally-managed --record=record.txt
-    if errorlevel 1 exit 1
 )
 
 rem copy back

diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh
@@ -16,18 +16,30 @@ export CMAKE_GENERATOR="Ninja"
 export CC=icx
 export CXX=icpx
 
-if [ -e "_skbuild" ]; then
-    ${PYTHON} setup.py clean --all
-fi
-
 # TODO: switch to pip build. Currently results in broken binary on Windows
 # $PYTHON -m pip install --no-index --no-deps --no-build-isolation . -v
 
-# Build wheel package
-if [ -n "${WHEELS_OUTPUT_FOLDER}" ]; then
-    $PYTHON setup.py install --single-version-externally-managed --record=record.txt bdist_wheel -p manylinux2014_x86_64 --build-number $GIT_DESCRIBE_NUMBER
-    mkdir -p ${WHEELS_OUTPUT_FOLDER}
-    cp dist/dpbench*.whl ${WHEELS_OUTPUT_FOLDER}
-else
-    $PYTHON setup.py install --single-version-externally-managed --record=record.txt
+# -wnx flags mean: --wheel --no-isolation --skip-dependency-check
+${PYTHON} -m build -w -n -x
+${PYTHON} -m wheel tags --remove --build "$GIT_DESCRIBE_NUMBER" \
+    --platform-tag manylinux2014_x86_64 dist/dpbench*.whl
+${PYTHON} -m pip install dist/dpbench*.whl \
+    --no-build-isolation \
+    --no-deps \
+    --only-binary :all: \
+    --no-index \
+    --prefix ${PREFIX} \
+    -vv
+
+# Must be consistent with pyproject.toml project.scritps. Currently pip does
+# not allow to ignore scripts installation, so we have to remove them manually.
+# https://github.com/pypa/pip/issues/3980
+# We have to let conda-build manage it for use in order to set proper python
+# path.
+# https://docs.conda.io/projects/conda-build/en/stable/resources/define-metadata.html#python-entry-points
+rm ${PREFIX}/bin/dpbench
+
+# Copy wheel package
+if [[ -v WHEELS_OUTPUT_FOLDER ]]; then
+    cp dist/dpbench*.whl "${WHEELS_OUTPUT_FOLDER[@]}"
 fi
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
@@ -2,6 +2,11 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+{% set pyproject = load_file_data('pyproject.toml') %}
+{% set py_deps = pyproject.get('project', {}).get('dependencies', []) %}
+{% set py_build_deps = pyproject.get('build-system', {}).get('requires', []) %}
+{% set project_scripts = pyproject.get('project', {}).get('scripts', {}) %}
+
 package:
   name: dpbench
   version: {{ GIT_DESCRIBE_TAG }}
@@ -13,49 +18,40 @@ build:
     number: {{ GIT_DESCRIBE_NUMBER }}
     script_env:
         - WHEELS_OUTPUT_FOLDER
+    entry_points:
+        {% for script, module in project_scripts | dictsort %}
+        - {{ script ~ " = " ~ module }}
+        {% endfor %}
 
 requirements:
   build:
     - {{ compiler('cxx') }}
     - {{ compiler('dpcpp') }}
+    # This is required to get compatible headers with the system installed glibc
     - sysroot_linux-64 >=2.28  # [linux]
   host:
     - python
-    - setuptools
-    - cmake==3.26* # [win]
-    - cmake # [not win]
-    - ninja # [not win]
-    - scikit-build
-    - cython
-    - pybind11
-    - versioneer
-    - intel::numpy
-    - numba
-    - dpctl
-    - dpnp
-    - numba-dpex
+    - pip
+    {% for dep in py_build_deps %}
+      {% if dep.startswith('ninja') %}
+    - {{ dep.split(';')[0] }} # [not win]
+      {% elif dep.startswith('cmake') %}
+    - cmake=3.26 # [win]
+    - {{ dep }} # [not win]
+      {% elif dep.startswith('build>=') %}
+    - {{ 'python-' ~ dep }}
+      {% else %}
+    - {{ dep|replace('_','-') }}
+      {% endif %}
+    {% endfor %}
   run:
-    - python
-    - tomli
-    - alembic
-    - sqlalchemy
-    - py-cpuinfo
-    - scipy
-    - scikit-learn
-    - pandas
-    - intel::numpy
-    - numba
-    - dpctl
-    - dpnp
-    - numba-dpex
+    {% for dep in py_deps %}
+    - {{ dep|replace('_','-') }}
+    {% endfor %}
 
 test:
-  requires:
-    - dpctl
-    - dpnp
-    - numba-dpex
-    - numba
-    - numpy
+  commands:
+    - dpbench --help
 
 about:
   home: https://github.com/IntelPython/dpbench

diff --git a/pyproject.toml b/pyproject.toml
@@ -28,26 +28,25 @@ classifiers = [
 license = { text = "Apache-2.0" }
 requires-python = ">=3.9"
 dependencies = [
-    "tomli",
+    "tomli>=2.0.0",
     "alembic>=1.10.0",
     "sqlalchemy>=2.0.0",
-    "py-cpuinfo",
-    "scipy",
-    "scikit_learn",
-    "pandas",
-    "numpy",
-    "dpctl",
-    "dpnp",
-    "numba",
-    "numba_dpex",
+    "py-cpuinfo>=9.0.0",
+    "scipy>=1.13.0",
+    "scikit_learn>=1.4.0",
+    "pandas>=2.2.0",
+    "numpy>=1.24.0",
+    "dpctl>=0.16.1",
+    "dpnp>=0.14.0",
+    "numba>=0.59.0",
+    "numba_dpex>=0.23.0a0",
 ]
-dynamic =["version"]
+dynamic = ["version"]
 
 [project.optional-dependencies]
 numba-mlir = ["numba_mlir"]
 color-config = ["pygments"]
-sycl-build = ["scikit-build>=0.13", "cmake>=3.18", "ninja"]
-npbench = ["dace","dask","legate"]
+npbench = ["dace", "dask", "legate"]
 json-to-toml = ["tomli_w"]
 expected-failure = ["tomlkit"]
 
@@ -56,14 +55,24 @@ expected-failure = ["tomlkit"]
 "Homepage" = "https://https://github.com/IntelPython/dpbench"
 
 [project.scripts]
+# Keep consistent with conda build scripts
 dpbench = "dpbench.console.entry:main"
 
 [build-system]
 # TODO: make it optional for no sycl build. Workaround: `--no-deps`.
 # https://github.com/scikit-build/scikit-build/issues/981
 requires = [
-    "setuptools>=42,<64",
-    "versioneer[toml]",
+    "wheel>=0.43",
+    "build>=1.1",
+    "setuptools>=63.0.0",
+    "scikit-build>=0.17.0",                      # sycl build dep
+    "ninja>=1.11.1; platform_system!='Windows'", # sycl build dep
+    "cmake>=3.29.0",                             # sycl build dep
+    # We need dpctl for UsmNdArray integration for dpcpp code
+    "dpctl>=0.16.1",
+    "pybind11>=2.12.0",
+    # WARNING: check with doc how to upgrade
+    "versioneer[toml]==0.29",
 ]
 build-backend = "setuptools.build_meta"