From 938e4c46b8d66967cda41a1d2513f69d37faa2b3 Mon Sep 17 00:00:00 2001 From: Yuval Date: Tue, 8 Dec 2020 10:53:27 +0200 Subject: [PATCH 1/2] Add support for hermetic Python --- README.md | 12 ++++++++++++ python_configure.bzl | 13 +++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9ee536a..ad1a86c 100644 --- a/README.md +++ b/README.md @@ -57,3 +57,15 @@ Then, in your `BUILD` file: ```starlark load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") ``` + +## Hermetic Python + +To configure `pybind11_bazel` for hermetic Python, `python_configure` can take +the target providing the Python runtime as an argument: + +```starlark +python_configure( + name = "local_config_python", + python_interpreter_target = "@python_interpreter//:python_bin", +) +``` diff --git a/python_configure.bzl b/python_configure.bzl index ade190a..1f5bffa 100644 --- a/python_configure.bzl +++ b/python_configure.bzl @@ -87,7 +87,7 @@ def _read_dir(repository_ctx, src_dir): result = find_result.stdout return result -def _genrule(src_dir, genrule_name, command, outs): +def _genrule(src_dir, genrule_name, command, outs, local): """Returns a string with a genrule. Genrule executes the given command and produces the given outputs. @@ -95,7 +95,8 @@ def _genrule(src_dir, genrule_name, command, outs): return ( "genrule(\n" + ' name = "' + - genrule_name + '",\n' + + genrule_name + '",\n' + ( + " local = 1,\n" if local else "") + " outs = [\n" + outs + "\n ],\n" + @@ -149,11 +150,15 @@ def _symlink_genrule_for_dir( genrule_name, " && ".join(command), "\n".join(outs), + local = True, ) return genrule def _get_python_bin(repository_ctx): """Gets the python bin path.""" + if repository_ctx.attr.python_interpreter_target != None: + return str(repository_ctx.path(repository_ctx.attr.python_interpreter_target)) + python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) if python_bin != None: return python_bin @@ -391,6 +396,7 @@ python_configure = repository_rule( ], attrs = { "python_version": attr.string(default=""), + "python_interpreter_target": attr.label(), }, ) """Detects and configures the local Python. @@ -409,4 +415,7 @@ Args: If set to "2", will build for Python 2 (`which python2`). By default, will build for whatever Python version is returned by `which python`. + python_interpreter_target: If set to a target providing a Python binary, + will configure for that Python version instead of the installed + binary. This configuration takes precedence over python_version. """ From ee09c40a26c46eac7c72d07016de358c4ab1fe72 Mon Sep 17 00:00:00 2001 From: madisetti Date: Mon, 5 Apr 2021 12:32:51 -0700 Subject: [PATCH 2/2] Allow hook for setting library --- WORKSPACE | 1 + python_configure.bzl | 115 ++++++++++++++++++++++++++++--------------- 2 files changed, 77 insertions(+), 39 deletions(-) create mode 100644 WORKSPACE diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..9bfe9f1 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "pybind11_bazel") diff --git a/python_configure.bzl b/python_configure.bzl index 1f5bffa..6aa7015 100644 --- a/python_configure.bzl +++ b/python_configure.bzl @@ -2,8 +2,11 @@ `python_configure` depends on the following environment variables: - * `PYTHON_BIN_PATH`: location of python binary. + * `PYTHON_BIN_PATH`: Location of python binary. * `PYTHON_LIB_PATH`: Location of python libraries. + +These can be directly set with the `python_interpreter_target` and +`python_library_target` respectively. """ _BAZEL_SH = "BAZEL_SH" @@ -11,6 +14,45 @@ _PYTHON_BIN_PATH = "PYTHON_BIN_PATH" _PYTHON_CONFIG_BIN_PATH = "PYTHON_CONFIG_BIN_PATH" _PYTHON_LIB_PATH = "PYTHON_LIB_PATH" +# TODO: Move scripts to respective files and expose to rule. +_INCLUDE_SCRIPT = """ +from __future__ import print_function +from distutils import sysconfig +import shutil + +python_inc_dir = sysconfig.get_python_inc() +config_h_path = sysconfig.get_config_h_filename() +shutil.copyfile(config_h_path, python_inc_dir + "/pyconfig.h") +print(python_inc_dir)""" + +_LIBRARY_SCRIPT = """ +from __future__ import print_function +import site +import os + +try: + input = raw_input +except NameError: + pass + +python_paths = [] +if os.getenv('PYTHONPATH') is not None: + python_paths = os.getenv('PYTHONPATH').split(':') +try: + library_paths = site.getsitepackages() +except AttributeError: + from distutils.sysconfig import get_python_lib + library_paths = [get_python_lib()] + +all_paths = set(python_paths + library_paths) +paths = [] +for path in all_paths: + if os.path.isdir(path): + paths.append(path) +if len(paths) >=1: + print(paths[0]) +""" + def _tpl(repository_ctx, tpl, substitutions = {}, out = None): if not out: out = tpl @@ -96,7 +138,8 @@ def _genrule(src_dir, genrule_name, command, outs, local): "genrule(\n" + ' name = "' + genrule_name + '",\n' + ( - " local = 1,\n" if local else "") + + " local = 1,\n" if local else "" + ) + " outs = [\n" + outs + "\n ],\n" + @@ -194,37 +237,26 @@ def _get_bash_bin(repository_ctx): def _get_python_lib(repository_ctx, python_bin): """Gets the python lib path.""" - python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) - if python_lib != None: - return python_lib - print_lib = ("<=1:\n" + - " print(paths[0])\n" + - "END") - cmd = "%s - %s" % (python_bin, print_lib) - result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + if repository_ctx.attr.python_library_target != None: + return str(repository_ctx.path(repository_ctx.attr.python_library_target)) + + # If an interpreter is explicitly passed down, we should should not regard + # the environmental variable. + if repository_ctx.attr.python_interpreter_target == None: + python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) + if python_lib: + return python_lib + + result = _execute( + repository_ctx, + [ + python_bin, + "-c", + _LIBRARY_SCRIPT, + ], + error_msg = "Unable to detect library path.", + error_details = ("Is Python installed correctly?"), + ) return result.stdout.strip("\n") def _check_python_lib(repository_ctx, python_lib): @@ -251,9 +283,7 @@ def _get_python_include(repository_ctx, python_bin): [ python_bin, "-c", - "from __future__ import print_function;" + - "from distutils import sysconfig;" + - "print(sysconfig.get_python_inc())", + _INCLUDE_SCRIPT, ], error_msg = "Problem getting python include path.", error_details = ("Is the Python binary path set up right? " + @@ -314,6 +344,7 @@ def _get_embed_flags(repository_ctx, python_config): # See https://github.com/python/cpython/pull/13500 link_cmd = python_config + " --ldflags --embed" + # TODO: Resolve ctx.execute vs _execute. err = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", err_cmd]).stdout.strip("\n") compiler_flags = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", comp_cmd]).stdout.strip("\n") linker_flags = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", link_cmd]).stdout.strip("\n") @@ -395,8 +426,9 @@ python_configure = repository_rule( _PYTHON_LIB_PATH, ], attrs = { - "python_version": attr.string(default=""), + "python_version": attr.string(default = ""), "python_interpreter_target": attr.label(), + "python_library_target": attr.string(), }, ) """Detects and configures the local Python. @@ -416,6 +448,11 @@ Args: By default, will build for whatever Python version is returned by `which python`. python_interpreter_target: If set to a target providing a Python binary, - will configure for that Python version instead of the installed - binary. This configuration takes precedence over python_version. + this will configure for that Python version instead of the installed + binary. This configuration takes precedence over python_version. In + addition, setting this label will ignore `PYTHON_LIB_PATH`- if needed + use `python_library_target` to explicitly set the desired library. + python_library_target: If this string is set, this will override the + environmental variable `PYTHON_LIB_PATH`. Otherwise implicit discovery, + is used from the availible binary. """