From a1bf3bfc4166e97cf392d891cd226483794c2416 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sun, 27 Oct 2024 21:16:26 +0100 Subject: [PATCH 1/4] MAINT: drop Python 3.7 support --- .cirrus.yml | 4 ++-- .github/workflows/tests.yml | 2 +- pyproject.toml | 2 +- tests/conftest.py | 9 +++------ 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index fddf4c380..48336992c 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -55,12 +55,12 @@ manylinux-python3.13t_task: PATH: "/opt/python/cp313-cp313t/bin/:${PATH}" << : *test -manylinux-python3.7_task: +manylinux-python3.8_task: container: dockerfile: ci/manylinux.docker cpu: 1 env: - PATH: "/opt/python/cp37-cp37m/bin/:${PATH}" + PATH: "/opt/python/cp38-cp38/bin/:${PATH}" << : *test miniconda_task: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d1dc8917a..0bfca7e2d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -50,7 +50,7 @@ jobs: - macos-13 - windows-latest python: - - '3.7' + - '3.8' - '3.13' meson: - diff --git a/pyproject.toml b/pyproject.toml index 09394db7d..2247a46df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ name = 'meson-python' version = '0.18.0.dev0' description = 'Meson Python build backend (PEP 517)' readme = 'README.rst' -requires-python = '>= 3.7' +requires-python = '>= 3.8' license = { file = 'LICENSES/MIT.txt' } keywords = ['meson', 'build', 'backend', 'pep517', 'package'] maintainers = [ diff --git a/tests/conftest.py b/tests/conftest.py index 54964be9d..1280a5d32 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT import contextlib +import importlib.metadata import os import os.path import pathlib @@ -58,8 +59,8 @@ def adjust_packaging_platform_tag(platform: str) -> str: @contextlib.contextmanager def in_git_repo_context(path=os.path.curdir): - # Resist the tentation of using pathlib.Path here: it is not - # supporded by subprocess in Python 3.7. + # Resist the temptation of using pathlib.Path here: it is not + # supported by subprocess in Python 3.7. path = os.path.abspath(path) shutil.rmtree(os.path.join(path, '.git'), ignore_errors=True) try: @@ -96,10 +97,6 @@ def __init__(self, env_dir): # Free-threaded Python 3.13 requires pip 24.1b1 or later. if sysconfig.get_config_var('Py_GIL_DISABLED'): - # importlib.metadata is not available on Python 3.7 and - # earlier, however no-gil builds are available only for - # Python 3.13 and later. - import importlib.metadata if packaging.version.Version(importlib.metadata.version('pip')) < packaging.version.Version('24.1b1'): self.pip('install', '--upgrade', 'pip >= 24.1b1') From 75bea0e6b73c1adaaca8a069db221e035ec2f670 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sun, 27 Oct 2024 18:40:55 +0100 Subject: [PATCH 2/4] TST: add test package with internal shared libraries, installed in site-packages --- .../packages/sharedlib-in-package/meson.build | 9 ++++ .../sharedlib-in-package/mypkg/__init__.py | 51 +++++++++++++++++++ .../sharedlib-in-package/mypkg/_examplemod.c | 51 +++++++++++++++++++ .../sharedlib-in-package/mypkg/examplelib.c | 9 ++++ .../sharedlib-in-package/mypkg/examplelib.h | 7 +++ .../sharedlib-in-package/mypkg/meson.build | 41 +++++++++++++++ .../mypkg/sub/examplelib2.c | 9 ++++ .../mypkg/sub/examplelib2.h | 7 +++ .../mypkg/sub/meson.build | 16 ++++++ .../mypkg/sub/mypkg_dll.h | 23 +++++++++ .../sharedlib-in-package/pyproject.toml | 7 +++ tests/test_wheel.py | 8 +++ 12 files changed, 238 insertions(+) create mode 100644 tests/packages/sharedlib-in-package/meson.build create mode 100644 tests/packages/sharedlib-in-package/mypkg/__init__.py create mode 100644 tests/packages/sharedlib-in-package/mypkg/_examplemod.c create mode 100644 tests/packages/sharedlib-in-package/mypkg/examplelib.c create mode 100644 tests/packages/sharedlib-in-package/mypkg/examplelib.h create mode 100644 tests/packages/sharedlib-in-package/mypkg/meson.build create mode 100644 tests/packages/sharedlib-in-package/mypkg/sub/examplelib2.c create mode 100644 tests/packages/sharedlib-in-package/mypkg/sub/examplelib2.h create mode 100644 tests/packages/sharedlib-in-package/mypkg/sub/meson.build create mode 100644 tests/packages/sharedlib-in-package/mypkg/sub/mypkg_dll.h create mode 100644 tests/packages/sharedlib-in-package/pyproject.toml diff --git a/tests/packages/sharedlib-in-package/meson.build b/tests/packages/sharedlib-in-package/meson.build new file mode 100644 index 000000000..71921cfea --- /dev/null +++ b/tests/packages/sharedlib-in-package/meson.build @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2022 The meson-python developers +# +# SPDX-License-Identifier: MIT + +project('sharedlib-in-package', 'c', version: '1.0.0') + +py = import('python').find_installation(pure: false) + +subdir('mypkg') diff --git a/tests/packages/sharedlib-in-package/mypkg/__init__.py b/tests/packages/sharedlib-in-package/mypkg/__init__.py new file mode 100644 index 000000000..088c1dbb8 --- /dev/null +++ b/tests/packages/sharedlib-in-package/mypkg/__init__.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: 2024 The meson-python developers +# +# SPDX-License-Identifier: MIT + +import os +import sys + + +# start-literalinclude +def _enable_sharedlib_loading(): + """ + Ensure the shared libraries in this dir and the ``sub`` subdir can be + loaded on Windows. + + One shared library is installed alongside this ``__init__.py`` file. + Windows can load it because it searches for DLLs in the directory where a + ``.pyd`` (Python extension module) is located in. Cygwin does not though. + For a shared library in another directory inside the package, Windows also + needs a hint. + + This function is Windows-specific due to lack of RPATH support on Windows. + It cannot find shared libraries installed within wheels unless we either + amend the DLL search path or pre-load the DLL. + + Note that ``delvewheel`` inserts a similar snippet into the main + ``__init__.py`` of a package when it vendors external shared libraries. + + .. note:: + + `os.add_dll_directory` is only available for Python >=3.8, and with + the Conda ``python`` packages only works as advertised for >=3.10. + If you require support for older versions, pre-loading the DLL + with `ctypes.WinDLL` may be preferred (the SciPy code base has an + example of this). + """ + basedir = os.path.dirname(__file__) + subdir = os.path.join(basedir, 'sub') + if os.name == 'nt': + os.add_dll_directory(subdir) + elif sys.platform == 'cygwin': + os.environ['PATH'] = f'os.environ["PATH"]:{basedir}:{subdir}' + + +_enable_sharedlib_loading() +# end-literalinclude + + +from ._example import example_prod, example_sum #noqa: E402 + + +__all__ = ['example_prod', 'example_sum'] diff --git a/tests/packages/sharedlib-in-package/mypkg/_examplemod.c b/tests/packages/sharedlib-in-package/mypkg/_examplemod.c new file mode 100644 index 000000000..080e03c18 --- /dev/null +++ b/tests/packages/sharedlib-in-package/mypkg/_examplemod.c @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2022 The meson-python developers +// +// SPDX-License-Identifier: MIT + +#include + +#include "examplelib.h" +#include "examplelib2.h" + +static PyObject* example_sum(PyObject* self, PyObject *args) +{ + int a, b; + if (!PyArg_ParseTuple(args, "ii", &a, &b)) { + return NULL; + } + + long result = sum(a, b); + + return PyLong_FromLong(result); +} + +static PyObject* example_prod(PyObject* self, PyObject *args) +{ + int a, b; + if (!PyArg_ParseTuple(args, "ii", &a, &b)) { + return NULL; + } + + long result = prod(a, b); + + return PyLong_FromLong(result); +} + +static PyMethodDef methods[] = { + {"example_prod", (PyCFunction)example_prod, METH_VARARGS, NULL}, + {"example_sum", (PyCFunction)example_sum, METH_VARARGS, NULL}, + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, + "_example", + NULL, + -1, + methods, +}; + +PyMODINIT_FUNC PyInit__example(void) +{ + return PyModule_Create(&module); +} diff --git a/tests/packages/sharedlib-in-package/mypkg/examplelib.c b/tests/packages/sharedlib-in-package/mypkg/examplelib.c new file mode 100644 index 000000000..f486bd7fb --- /dev/null +++ b/tests/packages/sharedlib-in-package/mypkg/examplelib.c @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2022 The meson-python developers +// +// SPDX-License-Identifier: MIT + +#include "sub/mypkg_dll.h" + +MYPKG_DLL int sum(int a, int b) { + return a + b; +} diff --git a/tests/packages/sharedlib-in-package/mypkg/examplelib.h b/tests/packages/sharedlib-in-package/mypkg/examplelib.h new file mode 100644 index 000000000..c09f4f785 --- /dev/null +++ b/tests/packages/sharedlib-in-package/mypkg/examplelib.h @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: 2022 The meson-python developers +// +// SPDX-License-Identifier: MIT + +#include "sub/mypkg_dll.h" + +MYPKG_DLL int sum(int a, int b); diff --git a/tests/packages/sharedlib-in-package/mypkg/meson.build b/tests/packages/sharedlib-in-package/mypkg/meson.build new file mode 100644 index 000000000..75904bed6 --- /dev/null +++ b/tests/packages/sharedlib-in-package/mypkg/meson.build @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2022 The meson-python developers +# +# SPDX-License-Identifier: MIT + +if meson.get_compiler('c').get_id() in ['msvc', 'clang-cl', 'intel-cl'] + export_dll_args = ['-DMYPKG_DLL_EXPORTS'] + import_dll_args = ['-DMYPKG_DLL_IMPORTS'] +else + export_dll_args = [] + import_dll_args = [] +endif + +example_lib = shared_library( + 'examplelib', + 'examplelib.c', + c_args: export_dll_args, + install: true, + install_dir: py.get_install_dir() / 'mypkg', +) + +example_lib_dep = declare_dependency( + compile_args: import_dll_args, + link_with: example_lib, +) + +subdir('sub') + +py.extension_module( + '_example', + '_examplemod.c', + dependencies: [example_lib_dep, example_lib2_dep], + include_directories: 'sub', + install: true, + subdir: 'mypkg', + install_rpath: '$ORIGIN', +) + +py.install_sources( + ['__init__.py'], + subdir: 'mypkg', +) diff --git a/tests/packages/sharedlib-in-package/mypkg/sub/examplelib2.c b/tests/packages/sharedlib-in-package/mypkg/sub/examplelib2.c new file mode 100644 index 000000000..12f5b87a7 --- /dev/null +++ b/tests/packages/sharedlib-in-package/mypkg/sub/examplelib2.c @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2022 The meson-python developers +// +// SPDX-License-Identifier: MIT + +#include "mypkg_dll.h" + +MYPKG_DLL int prod(int a, int b) { + return a * b; +} diff --git a/tests/packages/sharedlib-in-package/mypkg/sub/examplelib2.h b/tests/packages/sharedlib-in-package/mypkg/sub/examplelib2.h new file mode 100644 index 000000000..64b6a907e --- /dev/null +++ b/tests/packages/sharedlib-in-package/mypkg/sub/examplelib2.h @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: 2022 The meson-python developers +// +// SPDX-License-Identifier: MIT + +#include "mypkg_dll.h" + +MYPKG_DLL int prod(int a, int b); diff --git a/tests/packages/sharedlib-in-package/mypkg/sub/meson.build b/tests/packages/sharedlib-in-package/mypkg/sub/meson.build new file mode 100644 index 000000000..7a1978d4a --- /dev/null +++ b/tests/packages/sharedlib-in-package/mypkg/sub/meson.build @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2022 The meson-python developers +# +# SPDX-License-Identifier: MIT + +example_lib2 = shared_library( + 'examplelib2', + 'examplelib2.c', + c_args: export_dll_args, + install: true, + install_dir: py.get_install_dir() / 'mypkg/sub', +) + +example_lib2_dep = declare_dependency( + compile_args: import_dll_args, + link_with: example_lib2, +) diff --git a/tests/packages/sharedlib-in-package/mypkg/sub/mypkg_dll.h b/tests/packages/sharedlib-in-package/mypkg/sub/mypkg_dll.h new file mode 100644 index 000000000..8460e6c7d --- /dev/null +++ b/tests/packages/sharedlib-in-package/mypkg/sub/mypkg_dll.h @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2022 The meson-python developers +// +// SPDX-License-Identifier: MIT + +#pragma once + +// MYPKG_DLL +// inspired by https://github.com/abseil/abseil-cpp/blob/20240116.2/absl/base/config.h#L736-L753 +// and https://github.com/scipy/scipy/blob/9ded83b51099eee745418ccbb30196db96c81f3f/scipy/_build_utils/src/scipy_dll.h +// +// When building the `examplelib` DLL, this macro expands to `__declspec(dllexport)` +// so we can annotate symbols appropriately as being exported. When used in +// headers consuming a DLL, this macro expands to `__declspec(dllimport)` so +// that consumers know the symbol is defined inside the DLL. In all other cases, +// the macro expands to nothing. +// Note: MYPKG_DLL_{EX,IM}PORTS are set in mypkg/meson.build +#if defined(MYPKG_DLL_EXPORTS) + #define MYPKG_DLL __declspec(dllexport) +#elif defined(MYPKG_DLL_IMPORTS) + #define MYPKG_DLL __declspec(dllimport) +#else + #define MYPKG_DLL +#endif diff --git a/tests/packages/sharedlib-in-package/pyproject.toml b/tests/packages/sharedlib-in-package/pyproject.toml new file mode 100644 index 000000000..2542e4395 --- /dev/null +++ b/tests/packages/sharedlib-in-package/pyproject.toml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2022 The meson-python developers +# +# SPDX-License-Identifier: MIT + +[build-system] +build-backend = 'mesonpy' +requires = ['meson-python'] diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 1b8198235..a663d15bc 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -161,6 +161,14 @@ def test_local_lib(venv, wheel_link_against_local_lib): assert int(output) == 3 +def test_sharedlib_in_package(venv, wheel_sharedlib_in_package): + venv.pip('install', wheel_sharedlib_in_package) + output = venv.python('-c', 'import mypkg; print(mypkg.example_sum(2, 5))') + assert int(output) == 7 + output = venv.python('-c', 'import mypkg; print(mypkg.example_prod(6, 7))') + assert int(output) == 42 + + @pytest.mark.skipif(sys.platform not in {'linux', 'darwin', 'sunos5'}, reason='Not supported on this platform') def test_rpath(wheel_link_against_local_lib, tmp_path): artifact = wheel.wheelfile.WheelFile(wheel_link_against_local_lib) From 30e9656bd505e9c6a26198555ad75d81b68d625c Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Mon, 28 Oct 2024 18:02:11 +0100 Subject: [PATCH 3/4] TST: add package linking a library from a subproject Note that for Meson versions older than 1.2.0, CI failed with: ``` mesonpy.BuildError: Could not map installation path to an equivalent wheel directory: '{libdir_static}/libexamplelib.a' ``` because the `--skip-subprojects` install option isn't honored. Hence the test skip on older versions. In addition, the `c_shared_libs` usage requires Meson 1.3.0 --- .../foo/__init__.py | 8 ++++ .../foo/_examplemod.c | 37 +++++++++++++++++++ .../foo/meson.build | 16 ++++++++ .../link-library-in-subproject/meson.build | 17 +++++++++ .../link-library-in-subproject/pyproject.toml | 15 ++++++++ .../subprojects/bar/bar_dll.h | 19 ++++++++++ .../subprojects/bar/examplelib.c | 9 +++++ .../subprojects/bar/examplelib.h | 7 ++++ .../subprojects/bar/meson.build | 36 ++++++++++++++++++ tests/test_wheel.py | 7 ++++ 10 files changed, 171 insertions(+) create mode 100644 tests/packages/link-library-in-subproject/foo/__init__.py create mode 100644 tests/packages/link-library-in-subproject/foo/_examplemod.c create mode 100644 tests/packages/link-library-in-subproject/foo/meson.build create mode 100644 tests/packages/link-library-in-subproject/meson.build create mode 100644 tests/packages/link-library-in-subproject/pyproject.toml create mode 100644 tests/packages/link-library-in-subproject/subprojects/bar/bar_dll.h create mode 100644 tests/packages/link-library-in-subproject/subprojects/bar/examplelib.c create mode 100644 tests/packages/link-library-in-subproject/subprojects/bar/examplelib.h create mode 100644 tests/packages/link-library-in-subproject/subprojects/bar/meson.build diff --git a/tests/packages/link-library-in-subproject/foo/__init__.py b/tests/packages/link-library-in-subproject/foo/__init__.py new file mode 100644 index 000000000..584824912 --- /dev/null +++ b/tests/packages/link-library-in-subproject/foo/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: 2024 The meson-python developers +# +# SPDX-License-Identifier: MIT + +from ._example import example_sum + + +__all__ = ['example_sum'] diff --git a/tests/packages/link-library-in-subproject/foo/_examplemod.c b/tests/packages/link-library-in-subproject/foo/_examplemod.c new file mode 100644 index 000000000..d69556999 --- /dev/null +++ b/tests/packages/link-library-in-subproject/foo/_examplemod.c @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2022 The meson-python developers +// +// SPDX-License-Identifier: MIT + +#include + +#include "examplelib.h" + +static PyObject* example_sum(PyObject* self, PyObject *args) +{ + int a, b; + if (!PyArg_ParseTuple(args, "ii", &a, &b)) { + return NULL; + } + + long result = sum(a, b); + + return PyLong_FromLong(result); +} + +static PyMethodDef methods[] = { + {"example_sum", (PyCFunction)example_sum, METH_VARARGS, NULL}, + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, + "_example", + NULL, + -1, + methods, +}; + +PyMODINIT_FUNC PyInit__example(void) +{ + return PyModule_Create(&module); +} diff --git a/tests/packages/link-library-in-subproject/foo/meson.build b/tests/packages/link-library-in-subproject/foo/meson.build new file mode 100644 index 000000000..a171be52b --- /dev/null +++ b/tests/packages/link-library-in-subproject/foo/meson.build @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2022 The meson-python developers +# +# SPDX-License-Identifier: MIT + +py.extension_module( + '_example', + '_examplemod.c', + dependencies: bar_dep, + install: true, + subdir: 'foo', +) + +py.install_sources( + ['__init__.py'], + subdir: 'foo', +) diff --git a/tests/packages/link-library-in-subproject/meson.build b/tests/packages/link-library-in-subproject/meson.build new file mode 100644 index 000000000..eaa829510 --- /dev/null +++ b/tests/packages/link-library-in-subproject/meson.build @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2022 The meson-python developers +# +# SPDX-License-Identifier: MIT + +project( + 'link-library-in-subproject', + 'c', + version: '1.0.0', + meson_version: '>=1.2.0', +) + +py = import('python').find_installation(pure: false) + +bar_proj = subproject('bar') +bar_dep = bar_proj.get_variable('bar_dep') + +subdir('foo') diff --git a/tests/packages/link-library-in-subproject/pyproject.toml b/tests/packages/link-library-in-subproject/pyproject.toml new file mode 100644 index 000000000..212bb3d12 --- /dev/null +++ b/tests/packages/link-library-in-subproject/pyproject.toml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2022 The meson-python developers +# +# SPDX-License-Identifier: MIT + +[build-system] +build-backend = 'mesonpy' +requires = ['meson-python'] + +[project] +name = 'link-library-in-subproject' +version = '1.2.3' + +[tool.meson-python.args] +setup = ['--default-library=static'] +install = ['--skip-subprojects'] diff --git a/tests/packages/link-library-in-subproject/subprojects/bar/bar_dll.h b/tests/packages/link-library-in-subproject/subprojects/bar/bar_dll.h new file mode 100644 index 000000000..61a6d7bcc --- /dev/null +++ b/tests/packages/link-library-in-subproject/subprojects/bar/bar_dll.h @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2024 The meson-python developers +// +// SPDX-License-Identifier: MIT + +#pragma once + +// When building the `examplelib` DLL, this macro expands to `__declspec(dllexport)` +// so we can annotate symbols appropriately as being exported. When used in +// headers consuming a DLL, this macro expands to `__declspec(dllimport)` so +// that consumers know the symbol is defined inside the DLL. In all other cases, +// the macro expands to nothing. +// Note: BAR_DLL_{EX,IM}PORTS are set in meson.build +#if defined(BAR_DLL_EXPORTS) + #define BAR_DLL __declspec(dllexport) +#elif defined(BAR_DLL_IMPORTS) + #define BAR_DLL __declspec(dllimport) +#else + #define BAR_DLL +#endif diff --git a/tests/packages/link-library-in-subproject/subprojects/bar/examplelib.c b/tests/packages/link-library-in-subproject/subprojects/bar/examplelib.c new file mode 100644 index 000000000..7efb3667c --- /dev/null +++ b/tests/packages/link-library-in-subproject/subprojects/bar/examplelib.c @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2022 The meson-python developers +// +// SPDX-License-Identifier: MIT + +#include "bar_dll.h" + +BAR_DLL int sum(int a, int b) { + return a + b; +} diff --git a/tests/packages/link-library-in-subproject/subprojects/bar/examplelib.h b/tests/packages/link-library-in-subproject/subprojects/bar/examplelib.h new file mode 100644 index 000000000..c0f9e5ec4 --- /dev/null +++ b/tests/packages/link-library-in-subproject/subprojects/bar/examplelib.h @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: 2022 The meson-python developers +// +// SPDX-License-Identifier: MIT + +#include "bar_dll.h" + +BAR_DLL int sum(int a, int b); diff --git a/tests/packages/link-library-in-subproject/subprojects/bar/meson.build b/tests/packages/link-library-in-subproject/subprojects/bar/meson.build new file mode 100644 index 000000000..503257bc5 --- /dev/null +++ b/tests/packages/link-library-in-subproject/subprojects/bar/meson.build @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: 2022 The meson-python developers +# +# SPDX-License-Identifier: MIT + +project('bar', 'c', version: '1.2.3', meson_version: '>= 1.3.0') + +if get_option('default_library') == 'shared' and meson.get_compiler('c').get_id() in ['msvc', 'clang-cl', 'intel-cl'] + export_dll_args = ['-DBAR_DLL_EXPORTS'] + import_dll_args = ['-DBAR_DLL_IMPORTS'] +else + export_dll_args = [] + import_dll_args = [] +endif + +example_lib = library( + 'examplelib', + 'examplelib.c', + c_shared_args: export_dll_args, + install: true, +) + +# A second library that we don't link from `foo`. If we install the subproject, +# this second library also ends up in the wheel. To prevent that, we need to +# skip installing this `bar` subproject, and statically link `example_lib`. +unwanted_lib = library( + 'unwantedlib', + 'examplelib.c', + c_shared_args: export_dll_args, + install: true, +) + +bar_dep = declare_dependency( + compile_args: import_dll_args, + link_with: example_lib, + include_directories: '.', +) diff --git a/tests/test_wheel.py b/tests/test_wheel.py index a663d15bc..5b3f679d6 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -169,6 +169,13 @@ def test_sharedlib_in_package(venv, wheel_sharedlib_in_package): assert int(output) == 42 +@pytest.mark.skipif(MESON_VERSION < (1, 3, 0), reason='Meson version too old') +def test_link_library_in_subproject(venv, wheel_link_library_in_subproject): + venv.pip('install', wheel_link_library_in_subproject) + output = venv.python('-c', 'import foo; print(foo.example_sum(3, 6))') + assert int(output) == 9 + + @pytest.mark.skipif(sys.platform not in {'linux', 'darwin', 'sunos5'}, reason='Not supported on this platform') def test_rpath(wheel_link_against_local_lib, tmp_path): artifact = wheel.wheelfile.WheelFile(wheel_link_against_local_lib) From e6b52a90bd32362c682a2e746446bdc3863792ed Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sat, 26 Oct 2024 19:13:06 +0200 Subject: [PATCH 4/4] DOC: add documentation about using shared libraries --- docs/how-to-guides/shared-libraries.rst | 250 ++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 251 insertions(+) create mode 100644 docs/how-to-guides/shared-libraries.rst diff --git a/docs/how-to-guides/shared-libraries.rst b/docs/how-to-guides/shared-libraries.rst new file mode 100644 index 000000000..532a254fa --- /dev/null +++ b/docs/how-to-guides/shared-libraries.rst @@ -0,0 +1,250 @@ +.. SPDX-FileCopyrightText: 2024 The meson-python developers +.. +.. SPDX-License-Identifier: MIT + +.. _shared-libraries: + +********************** +Using shared libraries +********************** + +Python projects may build shared libraries as part of their project, or link +with shared libraries from a dependency. This tends to be a common source of +issues, hence this page aims to explain how to include shared libraries in +wheels, any limitations and gotchas, and how support is implemented in +``meson-python`` under the hood. + +We distinguish between *internal* shared libraries that are built as part of +the project, and *external* shared libraries that are provided by project +dependencies and that are linked with the project build artifacts. +For internal shared libraries, we also distinguish whether the shared library +is being installed to its default system location (typically +``/usr/local/lib`` on Unix-like systems, and ``C:\\lib`` on Windows - we call +this ``libdir`` in this guide) or to a location in ``site-packages`` within the +Python package install tree. All these scenarios are (or will be) supported, +with some caveats: + ++-----------------------+------------------+---------+-------+-------+ +| shared library source | install location | Windows | macOS | Linux | ++=======================+==================+=========+=======+=======+ +| internal | libdir | no (1) | ✓ | ✓ | ++-----------------------+------------------+---------+-------+-------+ +| internal | site-packages | ✓ | ✓ | ✓ | ++-----------------------+------------------+---------+-------+-------+ +| external | n/a | ✓ (2) | ✓ | ✓ | ++-----------------------+------------------+---------+-------+-------+ + +.. TODO: add subproject as a source + +1: Internal shared libraries on Windows cannot be automatically handled +correctly, and currently ``meson-python`` therefore raises an error for them. +`PR meson-python#551 `__ +may improve that situation in the near future. + +2: External shared libraries require ``delvewheel`` usage on Windows (or +some equivalent way, like amending the DLL search path to include the directory +in which the external shared library is located). Due to the lack of RPATH +support on Windows, there is no good way around this. + +.. _internal-shared-libraries: + +Internal shared libraries +========================= + +A shared library produced by ``library()`` or ``shared_library()`` built like this + +.. code-block:: meson + + example_lib = shared_library( + 'example', + 'examplelib.c', + install: true, + ) + +is installed to ``libdir`` by default. If the only reason the shared library exists +is to be used inside the Python package being built, then it is best to modify +the install location to be within the Python package itself: + +.. code-block:: python + + install_path: py.get_install_dir() / 'mypkg/subdir' + +Then an extension module in the same install directory can link against the +shared library in a portable manner by using ``install_rpath``: + +.. code-block:: meson + + py3.extension_module('_extmodule', + '_extmodule.c', + link_with: example_lib, + install: true, + subdir: 'mypkg/subdir', + install_rpath: '$ORIGIN' + ) + +The above method will work as advertised on macOS and Linux; ``meson-python`` does +nothing special for this case. Windows needs some special handling though, due to +the lack of RPATH support: + +.. literalinclude:: ../../tests/packages/sharedlib-in-package/mypkg/__init__.py + :start-after: start-literalinclude + :end-before: end-literalinclude + +If an internal shared library is not only used as part of a Python package, but +for example also as a regular shared library in a C/C++ project or as a +standalone library, then the method shown above won't work. The library is` +then marked for installation into the system default ``libdir`` location. +Actually installing into ``libdir`` isn't possible with wheels, hence +``meson-python`` will instead do the following *on platforms other than +Windows*: + +1. Install the shared library to ``.mesonpy.libs`` (i.e., a + top-level directory in the wheel, which on install will end up in + ``site-packages``). +2. Rewrite RPATH entries for install targets that depend on the shared library + to point to that new install location instead. + +This will make the shared library work automatically, with no other action needed +from the package author. *However*, currently an error is raised for this situation +on Windows. This is documented also in :ref:`reference-limitations`. + + +External shared libraries +========================= + +External shared libraries are installed somewhere on the build machine, and +usually detected by a ``dependency()`` or ``compiler.find_library()`` call in a +``meson.build`` file. When a Python extension module or executable uses the +dependency, the shared library will be linked against at build time. + +If the shared library is located in a directory on the loader search path, +the wheel created by ``meson-python`` will work locally when installed. +If it's in a non-standard location however, the shared library will go +missing at runtime. The Python extension module linked against it needs an +RPATH entry - and Meson will not automatically manage RPATH entries for you. +Hence you'll need to add the needed RPATH yourself, for example by adding +``-Wl,rpath=/path/to/dir/sharedlib/is/in`` to ``LDFLAGS`` before starting +the build. In case you run into this problem after a wheel is built and +installed, adding that same path to ``LD_LIBRARY_PATH`` is a quick way of +checking if that is indeed the problem. + +On Windows, the solution is similar - the shared library can either be +preloaded, or the directory that the library is in added to ``PATH`` or with +``os.add_dll_directory``, or vendored into the wheel with ``delvewheel`` in +order to make the built Python package usable locally. + +Publishing wheels which depend on external shared libraries +----------------------------------------------------------- + +On all platforms, wheels which depend on external shared libraries usually need +post-processing to make them usable on machines other than the one on which +they were built. This is because the RPATH entry for an external shared library +contains a path specific to the build machine. This post-processing is done by +tools like ``auditwheel`` (Linux), ``delvewheel`` (Windows), ``delocate`` +(macOS) or ``repair-wheel`` (any platform, wraps the other tools). + +Running any of those tools on a wheel produced by ``meson-python`` will vendor +the external shared library into the wheel and rewrite the RPATH entries (it +may also do some other things, like symbol mangling). + +On Windows, the package author may also have to add the preloading like shown +above with ``_enable_sharedlib_loading()`` to the main ``__init__.py`` of the +package, ``delvewheel`` may or may not take care of this (please check its +documentation if your shared library goes missing at runtime). + +Note that we don't cover using shared libraries contained in another wheel +and depending on such a wheel at runtime in this guide. This is inherently +complex and not recommended (you need to be in control of both packages, or +upgrades may be impossible/breaking). + + +Using libraries from a Meson subproject +======================================= + +It can often be useful to build a shared library in a +`Meson subproject `__, for example as +a fallback in case an external dependency isn't detected. There are two main +strategies for folding a library built in a subproject into a wheel built with +``meson-python``: + +1. Build the library as a static library instead of a shared library, and + link it into a Python extension module that needs it. +2. Build the library as a shared library, and either change its install path + to be within the Python package's tree, or rely on ``meson-python`` to fold + it into the wheel when it'd otherwise be installed to ``libdir``. + +Option (1) tends to be easier, so unless the library of interest cannot be +built as a static library or it would inflate the wheel size too much because +it's needed by multiple Python extension modules, we recommend trying option +(1) first. + +A typical C or C++ project providing a library to link against tends to provide +(a) one or more ``library()`` targets, which can be built as shared, static, or both, +and (b) headers, pkg-config files, tests and perhaps other development targets +that are needed to use the ``library()`` target(s). One of the challenges to use +such projects as a subproject is that the headers and other installable targets +are targeting system locations (e.g., ``/include/``) which isn't supported +by wheels and hence ``meson-python`` errors out when it encounters such an install +target. This is perhaps the main issue one encounters with subproject usage, +and the following two sections discuss how options (1) and (2) can work around +that. + +Static library from subproject +------------------------------ + +The major advantage of building a library target as static and folding it directly +into an extension module is that no targets from the subproject need to be installed. +To configure the subproject for this use case, add the following to the +``pyproject.toml`` file of your package: + +.. code-block:: toml + + [tool.meson-python.args] + setup = ['--default-library=static'] + install = ['--skip-subprojects'] + +This ensures that ``library`` targets are built as static, and nothing gets installed. + +To then link against the static library in the subproject, say for a subproject +named ``bar`` with the main library target contained in a ``bar_dep`` dependency, +add this to your ``meson.build`` file: + +.. code-block:: meson + + bar_proj = subproject('bar') + bar_dep = bar_proj.get_variable('bar_dep') + + py.extension_module( + '_example', + '_examplemod.c', + dependencies: bar_dep, + install: true, + ) + +That is all! + +Shared library from subproject +------------------------------ + +If we can't use the static library approach from the section above and we need +a shared library, then we must have ``install: true`` for that shared library +target. This can only work if we can pass some build option to the subproject +that tells it to *only* install the shared library and not headers or other +targets that we don't need. Install tags don't work per subproject, so +this will look something like: + +.. code-block:: meson + + foo_subproj = subproject('foo', + default_options: { + # This is a custom option - if it doesn't exist, can you add it + # upstream or in WrapDB? + 'only_install_main_lib': true, + }) + foo_dep = foo_subproj.get_variable('foo_dep') + +Now we can use ``foo_dep`` like a normal dependency, ``meson-python`` will +include it into the wheel in ``.mesonpy.libs`` just like an +internal shared library that targets ``libdir`` (see +:ref:`internal-shared-libraries`). +*Remember: this method doesn't support Windows (yet)!* diff --git a/docs/index.rst b/docs/index.rst index 1835e19e6..f48e60e06 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -82,6 +82,7 @@ the use of ``meson-python`` and Meson for Python packaging. how-to-guides/config-settings how-to-guides/meson-args how-to-guides/debug-builds + how-to-guides/shared-libraries reference/limitations projects-using-meson-python