diff --git a/.bazelrc b/.bazelrc index 4e6f2fa187..d2e0721526 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma test --test_output=errors diff --git a/CHANGELOG.md b/CHANGELOG.md index dbb0c03e59..719af546b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,6 +115,7 @@ Unreleased changes template. allow specifying links to create within the venv site packages (only applicable with {obj}`--bootstrap_impl=script`) ([#2156](https://github.com/bazelbuild/rules_python/issues/2156)). +* (pypi) Add support for passing different extra pip args depending on the platform via `extra_pip_args_by_platform`. {#v0-0-0-removed} ### Removed diff --git a/examples/pip_parse/WORKSPACE b/examples/pip_parse/WORKSPACE index bb4714d941..b9f47f83ef 100644 --- a/examples/pip_parse/WORKSPACE +++ b/examples/pip_parse/WORKSPACE @@ -40,6 +40,12 @@ pip_parse( # Here, make pip output verbose (this is usable with `quiet = False`). # extra_pip_args = ["-v"], + # (Optional) You can provide extra parameters to pip, keyed by the host platform. + # For example, set `--only-binary=...` on Linux platforms. + # extra_pip_args_by_platform = { + # "linux_*": ["--only-binary=grpcio"], + # }, + # (Optional) You can exclude custom elements in the data section of the generated BUILD files for pip packages. # Exclude directories with spaces in their names in this example (avoids build errors if there are such directories). #pip_data_exclude = ["**/* */**"], diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 21e05f2895..94f0d6e34a 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -90,6 +90,7 @@ bzl_library( ":hub_repository_bzl", ":parse_requirements_bzl", ":parse_whl_name_bzl", + ":pip_args_bzl", ":pip_repository_attrs_bzl", ":simpleapi_download_bzl", ":whl_config_setting_bzl", @@ -242,6 +243,11 @@ bzl_library( ], ) +bzl_library( + name = "pip_args_bzl", + srcs = ["pip_args.bzl"], +) + bzl_library( name = "pip_bzl", srcs = ["pip.bzl"], @@ -267,6 +273,7 @@ bzl_library( ":attrs_bzl", ":evaluate_markers_bzl", ":parse_requirements_bzl", + ":pip_args_bzl", ":pip_repository_attrs_bzl", ":render_pkg_aliases_bzl", ":whl_config_setting_bzl", diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 8fce47656b..81ad635aa3 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -26,6 +26,7 @@ load(":evaluate_markers.bzl", "evaluate_markers") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_requirements.bzl", "parse_requirements") load(":parse_whl_name.bzl", "parse_whl_name") +load(":pip_args.bzl", "resolve_extra_pip_args") load(":pip_repository_attrs.bzl", "ATTRS") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":simpleapi_download.bzl", "simpleapi_download") @@ -153,6 +154,7 @@ def _create_whl_repos( whl_group_mapping = {} requirement_cycles = {} + extra_pip_args = resolve_extra_pip_args(module_ctx, pip_attr.extra_pip_args, pip_attr.extra_pip_args_by_platform) requirements_by_platform = parse_requirements( module_ctx, requirements_by_platform = requirements_files_by_platform( @@ -161,11 +163,11 @@ def _create_whl_repos( requirements_lock = pip_attr.requirements_lock, requirements_osx = pip_attr.requirements_darwin, requirements_windows = pip_attr.requirements_windows, - extra_pip_args = pip_attr.extra_pip_args, + extra_pip_args = extra_pip_args, python_version = major_minor, logger = logger, ), - extra_pip_args = pip_attr.extra_pip_args, + extra_pip_args = extra_pip_args, get_index_urls = get_index_urls, # NOTE @aignas 2025-02-24: we will use the "cp3xx_os_arch" platform labels # for converting to the PEP508 environment and will evaluate them in starlark diff --git a/python/private/pypi/pip_args.bzl b/python/private/pypi/pip_args.bzl new file mode 100644 index 0000000000..d1c6452936 --- /dev/null +++ b/python/private/pypi/pip_args.bzl @@ -0,0 +1,65 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +def _get_extra_pip_args_for_platform(ctx, extra_pip_args): + """Returns the pip args for the given platform from the dict. + + Args: + ctx: Current repository context. + extra_pip_args: (string keyed string list dict): Extra pip arguments. + + Returns: + List of arguments for the current platform. + """ + + os = ctx.os.name.replace(" ", "") + arch = ctx.os.arch.replace(" ", "") + + # Check if there's an exact match. + full_match = "{}_{}".format(os, arch) + if full_match in extra_pip_args: + return extra_pip_args[full_match] + + # Match on the os. + os_match = "{}_*".format(os) + if os_match in extra_pip_args: + return extra_pip_args[os_match] + + # Match on the arch. + arch_match = "*_{}".format(arch) + if arch_match in extra_pip_args: + return extra_pip_args[arch_match] + + # Wildcard match last to allow for a more specific match. + if "*" in extra_pip_args: + return extra_pip_args["*"] + + return [] + +def resolve_extra_pip_args(ctx, extra_pip_args, extra_pip_args_by_platform): + """Resolves the given set of extra pip args to the list that should be used on the current platform. + + Args: + ctx: Current repository or module context. + extra_pip_args: (string list): Extra pip arguments. + extra_pip_args_by_platform: (string keyed string list dict): Extra pip arguments keyed by platform. + + Returns: + List of arguments for the current platform. + """ + if len(extra_pip_args_by_platform): + return _get_extra_pip_args_for_platform(ctx, extra_pip_args_by_platform) + return extra_pip_args diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 01a541cf2f..5050529963 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -20,6 +20,7 @@ load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR") load("//python/private:text_util.bzl", "render") load(":evaluate_markers.bzl", "evaluate_markers") load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") +load(":pip_args.bzl", "resolve_extra_pip_args") load(":pip_repository_attrs.bzl", "ATTRS") load(":render_pkg_aliases.bzl", "render_pkg_aliases") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") @@ -71,6 +72,8 @@ exports_files(["requirements.bzl"]) """ def _pip_repository_impl(rctx): + extra_pip_args = resolve_extra_pip_args(rctx, rctx.attr.extra_pip_args, rctx.attr.extra_pip_args_by_platform) + requirements_by_platform = parse_requirements( rctx, requirements_by_platform = requirements_files_by_platform( @@ -79,9 +82,9 @@ def _pip_repository_impl(rctx): requirements_lock = rctx.attr.requirements_lock, requirements_osx = rctx.attr.requirements_darwin, requirements_windows = rctx.attr.requirements_windows, - extra_pip_args = rctx.attr.extra_pip_args, + extra_pip_args = extra_pip_args, ), - extra_pip_args = rctx.attr.extra_pip_args, + extra_pip_args = extra_pip_args, evaluate_markers = evaluate_markers, ) selected_requirements = {} diff --git a/python/private/pypi/pip_repository_attrs.bzl b/python/private/pypi/pip_repository_attrs.bzl index 23000869e9..f394068204 100644 --- a/python/private/pypi/pip_repository_attrs.bzl +++ b/python/private/pypi/pip_repository_attrs.bzl @@ -21,6 +21,21 @@ repositories.""" load(":attrs.bzl", COMMON_ATTRS = "ATTRS") ATTRS = { + "extra_pip_args_by_platform": attr.string_list_dict( + doc = """Similar to `extra_pip_args`. + +Extra arguments to pass on to pip on the assoicated platform specifiers. Args must not contain spaces. + +The keys in the dict take the form `_`, wildcards in the form `_*` are supported. +Key values are as those returned in Bazels `repository_os` struct, with spaces removed. + +Supports environment variables using the syntax `$VARNAME` or +`${VARNAME}` (expanding to empty string if unset) or +`${VARNAME:-default}` (expanding to default if the variable is unset +or empty in the environment), if `"VARNAME"` is listed in the +`envsubst` attribute. See also `envsubst`. +""", + ), "requirements_by_platform": attr.label_keyed_string_dict( doc = """\ The requirements files and the comma delimited list of target platforms as values. diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 1652e76156..e4cf6efe8d 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -22,12 +22,12 @@ load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # bu _tests = [] -def _mock_mctx(*modules, environ = {}, read = None): +def _mock_mctx(*modules, environ = {}, os_name = "unittest", os_arch = "exotic", read = None): return struct( os = struct( environ = environ, - name = "unittest", - arch = "exotic", + name = os_name, + arch = os_arch, ), read = read or (lambda _: """\ simple==0.0.1 \ @@ -49,7 +49,7 @@ simple==0.0.1 \ ], ) -def _mod(*, name, parse = [], override = [], whl_mods = [], is_root = True): +def _mod(*, name, parse = [], override = [], whl_mods = [], is_root = True, os = {}): return struct( name = name, tags = struct( @@ -58,6 +58,7 @@ def _mod(*, name, parse = [], override = [], whl_mods = [], is_root = True): whl_mods = whl_mods, ), is_root = is_root, + os = os, ) def _parse_modules(env, **kwargs): @@ -87,6 +88,7 @@ def _parse( experimental_target_platforms = [], extra_hub_aliases = {}, extra_pip_args = [], + extra_pip_args_by_platform = {}, isolated = True, netrc = None, parse_all_requirements_files = True, @@ -115,6 +117,7 @@ def _parse( experimental_target_platforms = experimental_target_platforms, extra_hub_aliases = extra_hub_aliases, extra_pip_args = extra_pip_args, + extra_pip_args_by_platform = extra_pip_args_by_platform, hub_name = hub_name, isolated = isolated, netrc = netrc, @@ -182,6 +185,46 @@ def _test_simple(env): _tests.append(_test_simple) +def _test_simple_platform_pip_args(env): + pypi = _parse_modules( + env, + module_ctx = _mock_mctx( + _mod( + name = "rules_python", + parse = [ + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_darwin = "darwin.txt", + extra_pip_args_by_platform = { + "macosx_*": ["--only-binary=simple"], + }, + ), + ], + ), + os_name = "mac os x", + os_arch = "arch64", + read = lambda x: { + "darwin.txt": "simple==0.0.2 --hash=sha256:deadb00f", + }[x], + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + ) + + pypi.whl_libraries().contains_exactly({ + "pypi_315_simple": { + "dep_template": "@pypi//{name}:{target}", + "extra_pip_args": ["--only-binary=simple"], + "python_interpreter_target": "unit_test_interpreter_target", + "repo": "pypi_315", + "requirement": "simple==0.0.2 --hash=sha256:deadb00f", + }, + }) + +_tests.append(_test_simple_platform_pip_args) + def _test_simple_multiple_requirements(env): pypi = _parse_modules( env,