Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Implement binary_toolchain #875

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions lib/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ toolchain_type(
name = "jq_toolchain_type",
)

toolchain_type(
name = "jq_runtime_toolchain_type",
)

toolchain_type(
name = "yq_toolchain_type",
)
Expand Down
69 changes: 69 additions & 0 deletions lib/binary_toolchain.bzl
Copy link
Collaborator

@alexeagle alexeagle Jul 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems very similar to bazel-contrib/rules_oci#590 by @EdSchouten

I wonder if bazel-lib could provide the abstraction that's needed for this case globally. We should bike-shed on this name because "BinaryInfo" can also apply to a binary run as an action on the exec platform.

Copy link
Author

@BoleynSu BoleynSu Jul 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agree we should have the abstraction shared. binary_toolchain is my answer. While there are still some decision we want to make which are not clear to me yet.

I think ideally we only limit the scope of the solution to 2) in bazelbuild/bazel#19645 (comment)

Then we only need to have one toolchain_type for the binary. Using jq as an example.

We will need

toolchain_type(name = "jq_binary_toolchain_type") # User should register different binary to this toolchain
binary_toolchain(name="jq_linux_amd64", binary="@jq_linux_amd64")
toolchain(
    name = "toolchain_linux_amd64",
    toolchain = ":jq_linux_amd64",
    toolchain_type = :jq_binary_toolchain_type",
    target_compatible_with = linux_amd64,
)
resolved_binary(name = "resolved_jq_binary") # This resolves to a binary that run on the target platform

# The jq_toolchain_type will be made private regarding registering, i.e. we only register jq_toolchain to it and do not allow user to register new toolchain to jq_toolchain_type.
toolchain_type(name = "jq_toolchain_type")
jq_toolchain(name = "jq_toolchain") # jq_toolchain use resolved_jq_binary with cfg=exec
toolchain(
    name = "toolchain",
    toolchain = ":jq_toolchain",
    toolchain_type = :jq_toolchain_type",
)

Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
BinaryInfo = provider(
doc = "Provide info for binary",
fields = {
"bin": "Target for the binary",
},
)

def _toolchain_impl(ctx):
binary_info = BinaryInfo(
bin = ctx.attr.bin,
)

toolchain_info = platform_common.ToolchainInfo(
binary_info = binary_info,
)

return [toolchain_info]

binary_toolchain = rule(
implementation = _toolchain_impl,
attrs = {
"bin": attr.label(
mandatory = True,
allow_single_file = True,
executable = True,
cfg = "exec",
),
},
)

binary_runtime_toolchain = rule(
implementation = _toolchain_impl,
attrs = {
"bin": attr.label(
mandatory = True,
allow_single_file = True,
executable = True,
cfg = "target",
),
},
)

def _resolved_binary_rule_impl(ctx, toolchain_type, template_variable):
bin = ctx.toolchains[toolchain_type].binary_info.bin[DefaultInfo]

out = ctx.actions.declare_file(ctx.attr.name + ".exe")
ctx.actions.symlink(
target_file = bin.files_to_run.executable,
output = out,
is_executable = True,
)

return [
DefaultInfo(
executable = out,
files = bin.files,
runfiles = bin.default_runfiles,
),
platform_common.TemplateVariableInfo({
template_variable: out.path,
} if template_variable != None else {}),
]

def resolved_binary_rule(*, toolchain_type, template_variable = None):
return rule(
implementation = lambda ctx: _resolved_binary_rule_impl(ctx, toolchain_type, template_variable),
executable = True,
toolchains = [toolchain_type],
)
2 changes: 1 addition & 1 deletion lib/private/jq.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def _expand_locations(ctx, s):
return expand_locations(ctx, s, targets = ctx.attr.data).split(" ")

def _jq_impl(ctx):
jq_bin = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"].jqinfo.bin
jq_bin = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"].binary_info.bin[DefaultInfo].files_to_run.executable

out = ctx.outputs.out or ctx.actions.declare_file(ctx.attr.name + ".json")
if ctx.attr.expand_args:
Expand Down
78 changes: 14 additions & 64 deletions lib/private/jq_toolchain.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -57,72 +57,15 @@ JQ_VERSIONS = {
},
}

JqInfo = provider(
doc = "Provide info for executing jq",
fields = {
"bin": "Executable jq binary",
},
)

def _jq_toolchain_impl(ctx):
binary = ctx.file.bin

# Make the $(JQ_BIN) variable available in places like genrules.
# See https://docs.bazel.build/versions/main/be/make-variables.html#custom_variables
template_variables = platform_common.TemplateVariableInfo({
"JQ_BIN": binary.path,
})
default_info = DefaultInfo(
files = depset([binary]),
runfiles = ctx.runfiles(files = [binary]),
)
jq_info = JqInfo(
bin = binary,
)

# Export all the providers inside our ToolchainInfo
# so the resolved_toolchain rule can grab and re-export them.
toolchain_info = platform_common.ToolchainInfo(
jqinfo = jq_info,
template_variables = template_variables,
default = default_info,
)

return [default_info, toolchain_info, template_variables]

jq_toolchain = rule(
implementation = _jq_toolchain_impl,
attrs = {
"bin": attr.label(
mandatory = True,
allow_single_file = True,
),
},
)

def _jq_toolchains_repo_impl(rctx):
# Expose a concrete toolchain which is the result of Bazel resolving the toolchain
# for the execution or target platform.
# Workaround for https://github.com/bazelbuild/bazel/issues/14009
starlark_content = """# @generated by @aspect_bazel_lib//lib/private:jq_toolchain.bzl
load("@aspect_bazel_lib//lib:binary_toolchain.bzl", "resolved_binary_rule")

# Forward all the providers
def _resolved_toolchain_impl(ctx):
toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"]
return [
toolchain_info,
toolchain_info.default,
toolchain_info.jqinfo,
toolchain_info.template_variables,
]

# Copied from java_toolchain_alias
# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl
resolved_toolchain = rule(
implementation = _resolved_toolchain_impl,
toolchains = ["@aspect_bazel_lib//lib:jq_toolchain_type"],
incompatible_use_toolchain_transition = True,
)
resolved_toolchain = resolved_binary_rule(toolchain_type = "@aspect_bazel_lib//lib:jq_toolchain_type", template_variable = "JQ_BIN")
resolved_binary = resolved_binary_rule(toolchain_type = "@aspect_bazel_lib//lib:jq_runtime_toolchain_type", template_variable = "JQ_BIN")
"""
rctx.file("defs.bzl", starlark_content)

Expand All @@ -132,10 +75,10 @@ resolved_toolchain = rule(
# By default all these toolchains are registered by the jq_register_toolchains macro
# so you don't normally need to interact with these targets.

load(":defs.bzl", "resolved_toolchain")
load(":defs.bzl", "resolved_toolchain", "resolved_binary")

resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"])

resolved_binary(name = "resolved_binary", visibility = ["//visibility:public"])
"""

for [platform, meta] in JQ_PLATFORMS.items():
Expand All @@ -146,6 +89,12 @@ toolchain(
toolchain = "@{user_repository_name}_{platform}//:jq_toolchain",
toolchain_type = "@aspect_bazel_lib//lib:jq_toolchain_type",
)
toolchain(
name = "{platform}_runtime_toolchain",
target_compatible_with = {compatible_with},
toolchain = "@{user_repository_name}_{platform}//:jq_runtime_toolchain",
toolchain_type = "@aspect_bazel_lib//lib:jq_runtime_toolchain_type",
)
""".format(
platform = platform,
user_repository_name = rctx.attr.user_repository_name,
Expand Down Expand Up @@ -182,9 +131,10 @@ def _jq_platform_repo_impl(rctx):
integrity = JQ_VERSIONS[rctx.attr.version][release_platform],
)
build_content = """# @generated by @aspect_bazel_lib//lib/private:jq_toolchain.bzl
load("@aspect_bazel_lib//lib/private:jq_toolchain.bzl", "jq_toolchain")
load("@aspect_bazel_lib//lib:binary_toolchain.bzl", "binary_toolchain", "binary_runtime_toolchain")
exports_files(["{0}"])
jq_toolchain(name = "jq_toolchain", bin = "{0}", visibility = ["//visibility:public"])
binary_toolchain(name = "jq_toolchain", bin = "{0}", visibility = ["//visibility:public"])
binary_runtime_toolchain(name = "jq_runtime_toolchain", bin = "{0}", visibility = ["//visibility:public"])
""".format("jq.exe" if is_windows else "jq")

# Base BUILD file for this repository
Expand Down
14 changes: 14 additions & 0 deletions lib/tests/jq/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,20 @@ diff_test(
file2 = "a_pretty.json",
)

genrule(
name = "case_genrule_tool",
srcs = ["a.json"],
outs = ["genrule_tool_output.json"],
cmd = "$(execpath @jq_toolchains//:resolved_binary) '.' $(location a.json) > $@",
tools = ["@jq_toolchains//:resolved_binary"],
)

diff_test(
name = "case_genrule_tool_test",
file1 = "genrule_tool_output.json",
file2 = "a_pretty.json",
)

# Raw input (--raw-input / -R)
jq(
name = "case_raw_input",
Expand Down
Loading